BadBob / Playground
GoCV · OpenCV · ONNX · MIT
07 / Go
Go。由模型到相機。
喺本機執行 ONNX 偵測器,即時顯示標註,並限制等候處理嘅影像數量。
範例集中喺受控環境嘅物件同動物,唔涉及人員監控、身份識別或行為分析。
07.01
準備專案
安裝列出嘅相依套件,再執行測試。開啟相機之前,先試一張本機圖片。
基礎說明
go test -race- 執行自動檢查,唔會開啟鏡頭。
Terminalbash
unzip badbob-playground-go.zip
cd badbob-playground-go
sh build.sh ./build/badbob-camera
tags='gocv_specific_modules,gocv_dnn,gocv_imgproc,gocv_imgcodecs,gocv_videoio'
go test -race -tags "$tags" ./...07.02
連接模型
開啟鏡頭之前,先驗證模型同類別標籤。
基礎說明
detection.ParseLabels(text)- 按模型嘅類別次序讀取名稱;拒絕空白或重複標籤。
opencv.LoadDetector(path, classes)- 載入 ONNX 模型,並驗證輸入、輸出同標籤數量。
prepareInput(frame)- 建立 640×640 嘅 RGB 輸入,保留比例,再填補剩低嘅範圍。
opencv.Detector.Detect(image)- 預處理影像,執行模型並傳回偵測結果。
要留意
- 症狀
- 偵測框移位,或者模型載入失敗。
- 原因
- 前處理、輸出格式或者標籤次序唔配合。
- 處理方法
- 呢個介接器只支援以下約定。Core ML 檔案同其他 ONNX 輸出格式需要另一個介接器。
呢個介接器只支援以下約定。Core ML 檔案同其他 ONNX 輸出格式需要另一個介接器。
深入了解
座標表示偵測框嘅中心點同尺寸,單位係 640×640 輸入影像嘅像素。標籤必須嚴格跟返模型嘅類別次序排列。
ONNXyaml
input:
color: RGB
dtype: float32
shape: [1, 3, 640, 640]
letterbox: 114
scale: 1/255
output:
dtype: float32
shape: [1, 4+C, N]
boxes: [cx, cy, width, height]
class_scores: probabilities
objectness: false
embedded_nms: falseinternal/opencv/opencv_adapter.gogo
// Package opencv adapts native inference, capture, presentation and image output.
package opencv
import (
"errors"
"fmt"
"image"
"image/color"
"math"
"os"
"path/filepath"
"github.com/silvergrund/badbob-playground/go/internal/detection"
"gocv.io/x/gocv"
)
type Detector struct {
network gocv.Net
classes int
}
// GoCV 0.43's legacy DNN/capture exception state is used only by the worker.
// HighGUI uses our separate per-call error bridge, never that global state.
func checkCV(operation string) error {
if err := gocv.LastExceptionError(); err != nil {
return fmt.Errorf("%s: %w", operation, err)
}
return nil
}
// LoadDetector validates a trusted ONNX model with a real dry forward.
func LoadDetector(path string, classes int) (*Detector, error) {
info, err := os.Stat(path)
if err != nil || !info.Mode().IsRegular() || filepath.Ext(path) != ".onnx" || classes < 1 || classes > 1024 {
return nil, errors.New("provide an existing .onnx file and 1-1024 labels")
}
gocv.ClearLastException()
network := gocv.ReadNetFromONNX(path)
// Check before Empty: GoCV returns a null native Net when parsing fails.
if err := checkCV("load ONNX"); err != nil {
_ = network.Close()
return nil, err
}
model := &Detector{network, classes}
if err := model.initialize(); err != nil {
_ = model.Close()
return nil, err
}
return model, nil
}
func (model *Detector) initialize() error {
if model.network.Empty() {
return errors.New("ONNX network has no layers")
}
if err := model.network.SetPreferableBackend(gocv.NetBackendOpenCV); err != nil {
return err
}
if err := checkCV("set CPU backend"); err != nil {
return err
}
if err := model.network.SetPreferableTarget(gocv.NetTargetCPU); err != nil {
return err
}
if err := checkCV("set CPU target"); err != nil {
return err
}
outputs := model.network.GetUnconnectedOutLayers()
if err := checkCV("read output layers"); err != nil {
return err
}
if len(outputs) != 1 {
return errors.New("expected exactly one detector output")
}
sample := gocv.NewMatWithSizeFromScalar(gocv.NewScalar(114, 114, 114, 0), 640, 640, gocv.MatTypeCV8UC3)
defer sample.Close()
_, err := model.Detect(sample)
return err
}
func (model *Detector) Detect(image gocv.Mat) ([]detection.Detection, error) {
blob, transform, err := prepareInput(image)
if err != nil {
return nil, err
}
defer blob.Close()
model.network.SetInput(blob, "")
if err := checkCV("set RGB NCHW input"); err != nil {
return nil, err
}
output := model.network.Forward("")
defer output.Close()
if err := checkCV("forward ONNX"); err != nil {
return nil, err
}
shape := output.Size()
if output.Type() != gocv.MatTypeCV32F || len(shape) != 3 || shape[0] != 1 || shape[1] != 4+model.classes || shape[2] < 1 || shape[2] > 100000 {
return nil, errors.New("expected float32[1,4+C,N], XYWH + class probabilities; no objectness/embedded NMS")
}
contiguous := output.Clone()
defer contiguous.Close()
data, err := contiguous.DataPtrFloat32()
if err != nil {
return nil, err
}
return detection.Decode(data, model.classes, shape[2], transform)
}
func (model *Detector) Close() error {
return model.network.Close()
}
func prepareInput(frame gocv.Mat) (gocv.Mat, detection.Letterbox, error) {
if frame.Closed() || frame.Empty() || frame.Type() != gocv.MatTypeCV8UC3 {
return gocv.Mat{}, detection.Letterbox{}, errors.New("expected a nonempty 8-bit BGR image")
}
transform, err := detection.NewLetterbox(frame.Cols(), frame.Rows())
if err != nil {
return gocv.Mat{}, detection.Letterbox{}, err
}
resized := gocv.NewMat()
defer resized.Close()
if err := gocv.Resize(frame, &resized, image.Pt(transform.Width, transform.Height), 0, 0, gocv.InterpolationLinear); err != nil {
return gocv.Mat{}, detection.Letterbox{}, err
}
padded := gocv.NewMat()
defer padded.Close()
err = gocv.CopyMakeBorder(resized, &padded, transform.Top, 640-transform.Height-transform.Top, transform.Left, 640-transform.Width-transform.Left, gocv.BorderConstant, color.RGBA{114, 114, 114, 255})
if err != nil {
return gocv.Mat{}, detection.Letterbox{}, err
}
blob := gocv.BlobFromImage(padded, 1.0/255, image.Pt(640, 640), gocv.NewScalar(0, 0, 0, 0), true, false)
if err := checkCV("prepare float32 blob"); err != nil {
_ = blob.Close()
return gocv.Mat{}, detection.Letterbox{}, err
}
return blob, transform, nil
}
type Camera struct {
capture *gocv.VideoCapture
}
// OpenCamera acquires an explicitly selected device and transfers ownership to the caller.
func OpenCamera(index int) (*Camera, error) {
capture, err := gocv.VideoCaptureDevice(index)
if err != nil {
if capture != nil {
_ = capture.Close()
}
return nil, err
}
if err := checkCV("open camera"); err != nil {
_ = capture.Close()
return nil, err
}
if !capture.IsOpened() {
_ = capture.Close()
return nil, errors.New("cannot open camera; check index, permissions and backend")
}
return &Camera{capture}, nil
}
func (camera *Camera) NextFrame() (gocv.Mat, error) {
frame := gocv.NewMat()
read := camera.capture.Read(&frame)
if err := checkCV("read camera"); err != nil {
_ = frame.Close()
return gocv.Mat{}, err
}
if !read || frame.Empty() {
_ = frame.Close()
return gocv.Mat{}, errors.New("camera read failed or returned an empty frame")
}
return frame, nil
}
func (camera *Camera) Close() error {
return camera.capture.Close()
}
// Annotate draws validated detections and their class names onto a BGR image.
func Annotate(frame *gocv.Mat, detections []detection.Detection, labels []string) error {
for _, detection := range detections {
b := detection.Bounds
left := max(0, min(frame.Cols()-1, int(math.Floor(b.X))))
top := max(0, min(frame.Rows()-1, int(math.Floor(b.Y))))
right := min(frame.Cols(), int(math.Ceil(b.X+b.Width)))
bottom := min(frame.Rows(), int(math.Ceil(b.Y+b.Height)))
if err := gocv.Rectangle(frame, image.Rect(left, top, max(left+1, right), max(top+1, bottom)), color.RGBA{250, 220, 74, 255}, 2); err != nil {
return err
}
text := fmt.Sprintf("%s %.2f", labels[detection.Label], detection.Score)
if err := gocv.PutText(frame, text, image.Pt(left, max(16, top-6)), gocv.FontHersheySimplex, .5, color.RGBA{250, 220, 74, 255}, 1); err != nil {
return err
}
}
return nil
}
// EncodePNG returns owned bytes without retaining the image or native encoding buffer.
func EncodePNG(frame gocv.Mat) ([]byte, error) {
buffer, err := gocv.IMEncode(gocv.PNGFileExt, frame)
if buffer != nil {
defer buffer.Close()
}
if err != nil {
return nil, err
}
return append([]byte(nil), buffer.GetBytes()...), nil
}07.03
接入相機
畫面擷取同推論會依序執行。介面只會收到最近完成處理嘅圖片同佢嘅偵測結果。
基礎說明
pipeline.Worker.run(source, detector)- 喺同一個循序背景任務入面處理相機畫面擷取同推論。
pipeline.Latest.Take()- 取出最新處理好嘅影像同對應偵測結果,避免結果喺佇列不斷累積。
pipeline.Worker.Close()- 停止接收新結果,等進行中嘅工作完成之後釋放資源。
要留意
- 症狀
- 預覽愈來愈慢,記憶體用量不斷增加。
- 原因
- 相機產生影像快過推論處理嘅速度,令冇上限嘅佇列不斷累積。
- 處理方法
- 畫面擷取同推論會依序執行。介面只會收到最近完成處理嘅圖片同佢嘅偵測結果。
深入了解
要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。
internal/pipeline/runtime.gogo
// Package pipeline owns sequential acquisition/inference and the latest paired result.
package pipeline
import (
"errors"
"sync"
"sync/atomic"
"github.com/silvergrund/badbob-playground/go/internal/detection"
"gocv.io/x/gocv"
)
// FrameSource transfers ownership of each acquired image to its caller.
type FrameSource interface {
NextFrame() (gocv.Mat, error)
Close() error
}
// Detector borrows an image for inference; the caller retains image ownership.
type Detector interface {
Detect(image gocv.Mat) ([]detection.Detection, error)
Close() error
}
// FrameResult owns an image and its matching predictions, or a terminal error.
type FrameResult struct {
Image gocv.Mat
Detections []detection.Detection
Err error
}
func (result *FrameResult) Close() {
if !result.Image.Closed() {
_ = result.Image.Close()
}
}
// Latest retains at most one result, disposing replacements and late publications.
type Latest struct {
mutex sync.Mutex
pending *FrameResult
closed bool
}
func (slot *Latest) Publish(result *FrameResult) {
slot.mutex.Lock()
defer slot.mutex.Unlock()
if slot.closed {
result.Close()
return
}
if slot.pending != nil {
slot.pending.Close()
}
slot.pending = result
}
func (slot *Latest) Take() *FrameResult {
slot.mutex.Lock()
defer slot.mutex.Unlock()
result := slot.pending
slot.pending = nil
return result
}
func (slot *Latest) Close() {
slot.mutex.Lock()
defer slot.mutex.Unlock()
slot.closed = true
if slot.pending != nil {
slot.pending.Close()
slot.pending = nil
}
}
// Worker owns its adapters until acquisition/inference and cleanup have completed.
type Worker struct {
stopped atomic.Bool
done chan struct{}
results Latest
}
// StartWorker validates the detector before acquiring a source, then processes frames.
// Close always joins the worker; an active native call can delay shutdown.
func StartWorker(source func() (FrameSource, error), detector func() (Detector, error)) *Worker {
worker := &Worker{done: make(chan struct{})}
go func() {
defer close(worker.done)
if err := worker.run(source, detector); err != nil && !worker.stopped.Load() {
worker.results.Publish(&FrameResult{Err: err})
}
}()
return worker
}
func (worker *Worker) run(source func() (FrameSource, error), detector func() (Detector, error)) (failure error) {
model, err := detector()
if err != nil {
return err
}
if model == nil {
return errors.New("missing detector adapter")
}
defer func() { failure = errors.Join(failure, model.Close()) }()
if worker.stopped.Load() {
return nil
}
frames, err := source()
if err != nil {
return err
}
if frames == nil {
return errors.New("missing capture adapter")
}
defer func() { failure = errors.Join(failure, frames.Close()) }()
for !worker.stopped.Load() {
result, err := processFrame(frames, model, &worker.stopped)
if err != nil {
return err
}
if result == nil {
break
}
worker.results.Publish(result)
}
return nil
}
func processFrame(frames FrameSource, model Detector, stopped *atomic.Bool) (*FrameResult, error) {
image, err := frames.NextFrame()
result := &FrameResult{Image: image}
if err != nil {
result.Close()
return nil, err
}
if stopped.Load() {
result.Close()
return nil, nil
}
if image.Empty() {
result.Close()
return nil, errors.New("camera returned an empty frame")
}
detections, err := model.Detect(image)
if err != nil {
result.Close()
return nil, err
}
if stopped.Load() {
result.Close()
return nil, nil
}
result.Detections = detections
return result, nil
}
func (worker *Worker) Take() *FrameResult {
return worker.results.Take()
}
func (worker *Worker) Stop() {
worker.stopped.Store(true)
worker.results.Close()
}
func (worker *Worker) Wait() {
<-worker.done
}
func (worker *Worker) IsDone() bool {
select {
case <-worker.done:
return true
default:
return false
}
}
func (worker *Worker) Close() {
worker.Stop()
worker.Wait()
}07.04
放好預測標記
還原 letterbox 縮放同填補,再將偵測框裁切到影像範圍內先繪畫。
基礎說明
detection.NewLetterbox(...) · detection.Unproject(...)- 保持影像長寬比,將偵測框映射返原始影像。
detection.Decode(...)- 驗證預測結果,移除同一類別入面重疊嘅偵測框。
要留意
影像尺寸、裁切、方向同座標要有同一套約定。先用已知偵測框測試,再信任畫面標記。
internal/detection/geometry.gogo
// Package detection validates detector geometry and class labels without native dependencies.
package detection
import (
"errors"
"math"
"sort"
)
type Box struct {
X, Y, Width, Height float64
}
type Detection struct {
Bounds Box
Label int
Score float32
}
type Letterbox struct {
OriginalWidth, OriginalHeight int
Width, Height, Left, Top int
}
// NewLetterbox computes the rounded resize and padding for a 640-pixel square input.
func NewLetterbox(width, height int) (Letterbox, error) {
if width <= 0 || height <= 0 {
return Letterbox{}, errors.New("image dimensions must be positive")
}
ratio := math.Min(640/float64(width), 640/float64(height))
resizedWidth := max(1, int(math.Round(float64(width)*ratio)))
resizedHeight := max(1, int(math.Round(float64(height)*ratio)))
return Letterbox{width, height, resizedWidth, resizedHeight, (640 - resizedWidth) / 2, (640 - resizedHeight) / 2}, nil
}
func finite(value float64) bool {
return !math.IsNaN(value) && !math.IsInf(value, 0)
}
// Unproject maps a model-space box to the original image and clips it to the image.
func Unproject(box Box, transform Letterbox) (Box, error) {
if transform.OriginalWidth <= 0 || transform.OriginalHeight <= 0 || transform.Width <= 0 || transform.Height <= 0 || transform.Width > 640 || transform.Height > 640 || transform.Left < 0 || transform.Top < 0 || transform.Left > 640-transform.Width || transform.Top > 640-transform.Height {
return Box{}, errors.New("invalid NewLetterbox geometry")
}
for _, value := range []float64{box.X, box.Y, box.Width, box.Height, box.X + box.Width, box.Y + box.Height} {
if !finite(value) {
return Box{}, errors.New("box geometry must be finite")
}
}
if box.Width < 0 || box.Height < 0 {
return Box{}, errors.New("box dimensions must be nonnegative")
}
x := func(value float64) float64 {
return min(1, max(0, (value-float64(transform.Left))/float64(transform.Width))) * float64(transform.OriginalWidth)
}
y := func(value float64) float64 {
return min(1, max(0, (value-float64(transform.Top))/float64(transform.Height))) * float64(transform.OriginalHeight)
}
left, top := x(box.X), y(box.Y)
return Box{left, top, x(box.X+box.Width) - left, y(box.Y+box.Height) - top}, nil
}
func intersectionOverUnion(a, b Box) float64 {
width := max(0, min(a.X+a.Width, b.X+b.Width)-max(a.X, b.X))
height := max(0, min(a.Y+a.Height, b.Y+b.Height)-max(a.Y, b.Y))
intersection := width * height
area := a.Width*a.Height + b.Width*b.Height - intersection
if area <= 0 {
return 0
}
return intersection / area
}
// Decode validates XYWH probabilities and applies bounded, class-aware suppression.
func Decode(data []float32, classes, count int, transform Letterbox) ([]Detection, error) {
if classes < 1 || classes > 1024 || count < 1 || count > 100000 || len(data) != (4+classes)*count {
return nil, errors.New("expected float32[1,4+C,N], C<=1024, N<=100000")
}
candidates := make([]Detection, 0)
for i := 0; i < count; i++ {
cx, cy := float64(data[i]), float64(data[count+i])
width, height := float64(data[2*count+i]), float64(data[3*count+i])
bounds, err := Unproject(Box{cx - width/2, cy - height/2, width, height}, transform)
if err != nil {
return nil, err
}
var score float32
label := 0
for class := 0; class < classes; class++ {
probability := data[(4+class)*count+i]
if !finite(float64(probability)) || probability < 0 || probability > 1 {
return nil, errors.New("expected probabilities in [0,1], not logits/objectness")
}
if probability > score {
score, label = probability, class
}
}
if score >= .25 && bounds.Width > 0 && bounds.Height > 0 {
candidates = append(candidates, Detection{bounds, label, score})
}
}
sort.SliceStable(candidates, func(i, j int) bool { return candidates[i].Score > candidates[j].Score })
candidates = candidates[:min(len(candidates), 3000)]
selected := make([]Detection, 0)
for _, candidate := range candidates {
suppressed := false
for _, previous := range selected {
if previous.Label == candidate.Label && intersectionOverUnion(previous.Bounds, candidate.Bounds) > .45 {
suppressed = true
break
}
}
if suppressed {
continue
}
selected = append(selected, candidate)
if len(selected) == 300 {
break
}
}
return selected, nil
}07.05
建置、執行、試模型。
明確選擇鏡頭,或者離線處理影像。撳 Q 或 Escape 停止。
基礎說明
parseOptions(arguments)- 驗證參數,並且只揀一種輸入:影像或鏡頭。
run(arguments)- 驗證揀咗嘅來源,再交由圖片或相機流程處理。
要留意
- 症狀
- 停止所需時間比預期長。
- 原因
- 原生畫面擷取或推論呼叫仲執行緊。
- 處理方法
- 要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。
Terminalbash
./build/badbob-camera --model detector.onnx --labels labels.txt --image frame.png --output result.png
./build/badbob-camera --model detector.onnx --labels labels.txt --camera 007.06 / 最佳實踐
做好細節,範例更好。
- 擷取同推論唔好放喺介面執行緒。限制佇列長度,避免積埋舊影格。
- 影像尺寸、裁切、方向同座標要有同一套約定。先用已知偵測框測試,再信任畫面標記。
- 要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。