BadBob / Playground
ONNX Runtime · OpenCV · MIT
04 / Python
Python。由模型到相機。
喺本機執行 ONNX 偵測器,即時顯示標註,並限制等候處理嘅影像數量。
範例集中喺受控環境嘅物件同動物,唔涉及人員監控、身份識別或行為分析。
04.01
準備專案
安裝列出嘅相依套件,再執行測試。開啟相機之前,先試一張本機圖片。
基礎說明
python -m unittest- 執行自動檢查,唔會開啟鏡頭。
Terminalbash
unzip badbob-playground-python.zip
cd badbob-playground-python
python3.12 -m venv .venv
. .venv/bin/activate
python -m pip install -r requirements.txt
python -m unittest discover -s tests -v04.02
連接模型
開啟鏡頭之前,先驗證模型同類別標籤。
基礎說明
OnnxDetector.__init__(...)- 載入 ONNX 模型,並驗證輸入、輸出同標籤數量。
prepare_input(...)- 建立 640×640 嘅 RGB 輸入,保留比例,再填補剩低嘅範圍。
OnnxDetector.detect(...)- 預處理影像,執行模型並傳回偵測結果。
要留意
- 症狀
- 偵測框移位,或者模型載入失敗。
- 原因
- 前處理、輸出格式或者標籤次序唔配合。
- 處理方法
- 呢個介接器只支援以下約定。Core ML 檔案同其他 ONNX 輸出格式需要另一個介接器。
呢個介接器只支援以下約定。Core ML 檔案同其他 ONNX 輸出格式需要另一個介接器。
深入了解
座標表示偵測框嘅中心點同尺寸,單位係 640×640 輸入影像嘅像素。標籤必須嚴格跟返模型嘅類別次序排列。
ONNXyaml
input:
color: RGB
dtype: float32
shape: [1, 3, 640, 640]
letterbox: 114
scale: 1/255
output:
dtype: float32
shape: [1, 4+C, N]
boxes: [cx, cy, width, height]
class_scores: probabilities
objectness: false
embedded_nms: falsedetector.pypython
"""Explicit CPU ONNX adapter; another export contract needs another adapter."""
from pathlib import Path
from typing import cast
import cv2
import numpy as np
import onnxruntime as ort
from .geometry import calculate_letterbox, decode_predictions
from .image_contract import validate_image
from .labels import validate_label_metadata
from .types import Detection, Image, Letterbox, Tensor
def prepare_input(frame: Image) -> tuple[Tensor, Letterbox]:
validate_image(frame)
transform = calculate_letterbox(frame.shape[1], frame.shape[0])
resized = cv2.resize(frame, (transform.width, transform.height), interpolation=cv2.INTER_LINEAR)
canvas = np.full((640, 640, 3), 114, dtype=np.uint8)
canvas[
transform.top : transform.top + transform.height,
transform.left : transform.left + transform.width,
] = resized
rgb = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)
tensor = np.ascontiguousarray(rgb.transpose(2, 0, 1)[None], dtype=np.float32)
tensor /= np.float32(255)
return tensor, transform
class OnnxDetector:
def __init__(self, model: Path, labels: tuple[str, ...]) -> None:
if not model.is_file() or model.suffix.lower() != ".onnx":
raise ValueError("Supply an existing local .onnx model")
if not 1 <= len(labels) <= 1024:
raise ValueError("Expected 1..1024 class labels")
self._session: ort.InferenceSession | None = None
self._classes = len(labels)
ort.disable_telemetry_events()
session = ort.InferenceSession(str(model), providers=["CPUExecutionProvider"])
inputs, outputs = session.get_inputs(), session.get_outputs()
if (
len(inputs) != 1
or inputs[0].type != "tensor(float)"
or inputs[0].shape != [1, 3, 640, 640]
):
raise ValueError("Expected one float32 RGB NCHW input [1,3,640,640]")
if len(outputs) != 1 or outputs[0].type != "tensor(float)":
raise ValueError("Expected one float32 detector output")
self._validate_output_shape(outputs[0].shape)
validate_label_metadata(session.get_modelmeta().custom_metadata_map, labels)
self._input_name, self._output_name = inputs[0].name, outputs[0].name
self._session = session
# Validate actual output values before the runner may acquire a camera.
self.detect(np.full((640, 640, 3), 114, dtype=np.uint8))
def _validate_output_shape(self, shape: object) -> None:
if not isinstance(shape, list) or len(shape) != 3 or shape[:2] != [1, 4 + self._classes]:
raise ValueError("Expected output metadata [1,4+C,N] matching label count")
candidates = shape[2]
if isinstance(candidates, int) and not 1 <= candidates <= 100000:
raise ValueError("Expected 1..100000 detector candidates")
if candidates is not None and not isinstance(candidates, (str, int)):
raise ValueError("Invalid detector candidate dimension")
def detect(self, frame: Image) -> tuple[Detection, ...]:
session = self._session
if session is None:
raise RuntimeError("Detector is closed")
tensor, transform = prepare_input(frame)
results = session.run([self._output_name], {self._input_name: tensor})
output: object = results[0]
if not isinstance(output, np.ndarray) or output.dtype != np.float32:
raise ValueError("Expected a float32 NumPy detector tensor")
return decode_predictions(cast(Tensor, output), self._classes, transform)
def close(self) -> None:
# The Python CPU session has no public close(); drop its owner reference.
self._session = None04.03
接入相機
擷取同推論共用一條循序工作執行緒。介面只會接收最新處理好嘅影像同相應偵測結果。
基礎說明
Runner._run()- 喺同一條工作執行緒處理影像擷取同推論。
LatestResult.take()- 取出最新處理好嘅影像同對應偵測結果,避免結果喺佇列不斷累積。
Runner.stop()- 停止接收新結果,等進行中嘅工作完成之後釋放資源。
要留意
- 症狀
- 預覽愈來愈慢,記憶體用量不斷增加。
- 原因
- 相機產生影像快過推論處理嘅速度,令冇上限嘅佇列不斷累積。
- 處理方法
- 擷取同推論共用一條循序工作執行緒。介面只會接收最新處理好嘅影像同相應偵測結果。
深入了解
要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。
runner.pypython
"""One joined worker, no input queue, and one complete owned output pair."""
import threading
from collections.abc import Callable
from .types import Detector, FrameResult, FrameSource
class LatestResult:
def __init__(self) -> None:
self._lock = threading.Lock()
self._pending: FrameResult | None = None
self._closed = False
def publish(self, result: FrameResult) -> bool:
with self._lock:
if self._closed:
return False
self._pending = result
return True
def take(self) -> FrameResult | None:
with self._lock:
result, self._pending = self._pending, None
return result
def close(self) -> None:
with self._lock:
self._closed = True
self._pending = None
class Runner:
def __init__(
self, detector_factory: Callable[[], Detector], source_factory: Callable[[], FrameSource]
):
self.results = LatestResult()
self.finished = threading.Event()
self.stop_requested = threading.Event()
self.error: str | None = None
self._detector_factory = detector_factory
self._source_factory = source_factory
self._thread: threading.Thread | None = None
def start(self) -> None:
if self._thread is not None or self.stop_requested.is_set():
raise RuntimeError("A Runner starts once; create a new Runner for another session")
self._thread = threading.Thread(target=self._run, name="badbob-inference", daemon=False)
self._thread.start()
def stop(self) -> None:
self.stop_requested.set()
self.results.close()
if self._thread is not None:
self._thread.join()
def _capture(self, detector: Detector, source: FrameSource) -> None:
sequence = 0
while not self.stop_requested.is_set():
frame = source.read()
if frame is None:
return
if self.stop_requested.is_set():
return
# Own the pixels even when a backend or test double reuses its buffer.
owned_frame = frame.copy()
detections = detector.detect(owned_frame)
if self.stop_requested.is_set():
return
sequence += 1
self.results.publish(FrameResult(owned_frame, detections, sequence))
def _run(self) -> None:
detector: Detector | None = None
source: FrameSource | None = None
try:
detector = self._detector_factory()
if self.stop_requested.is_set():
return
source = self._source_factory()
self._capture(detector, source)
except Exception as error:
self.error = f"{type(error).__name__}: {error}"
self.results.close()
finally:
self._release(source, detector)
self.finished.set()
def _release(self, source: FrameSource | None, detector: Detector | None) -> None:
for resource in (source, detector):
if resource is None:
continue
try:
resource.close()
except Exception as error:
self.error = self.error or f"Cleanup failed: {error}"
self.results.close()04.04
放好預測標記
還原 letterbox 縮放同填補,再將偵測框裁切到影像範圍內先繪畫。
基礎說明
calculate_letterbox(...) · unproject_box(...)- 保持影像長寬比,將偵測框映射返原始影像。
decode_predictions(...)- 驗證預測結果,移除同一類別入面重疊嘅偵測框。
要留意
影像尺寸、裁切、方向同座標要有同一套約定。先用已知偵測框測試,再信任畫面標記。
geometry.pypython
"""Letterbox geometry and bounded class-aware suppression."""
import math
import numpy as np
from .types import Box, Detection, Letterbox, Tensor
def calculate_letterbox(width: int, height: int, edge: int = 640) -> Letterbox:
if width <= 0 or height <= 0 or not 0 < edge <= 4096:
raise ValueError("Positive image dimensions and input edge <=4096 required")
scale = min(edge / width, edge / height)
resized_width = max(1, math.floor(width * scale + 0.5))
resized_height = max(1, math.floor(height * scale + 0.5))
return Letterbox(
width,
height,
resized_width,
resized_height,
(edge - resized_width) // 2,
(edge - resized_height) // 2,
edge,
)
def validate_transform(transform: Letterbox) -> None:
if (
transform.original_width <= 0
or transform.original_height <= 0
or not 0 < transform.width <= transform.edge <= 4096
or not 0 < transform.height <= transform.edge
or not 0 <= transform.left <= transform.edge - transform.width
or not 0 <= transform.top <= transform.edge - transform.height
):
raise ValueError("Invalid letterbox geometry")
def unproject_box(box: Box, transform: Letterbox) -> Box:
validate_transform(transform)
values = (box.x, box.y, box.width, box.height, box.x + box.width, box.y + box.height)
if not all(math.isfinite(value) for value in values) or min(box.width, box.height) < 0:
raise ValueError("Box coordinates must be finite with nonnegative dimensions")
def x(value: float) -> float:
return (
min(1.0, max(0.0, (value - transform.left) / transform.width))
* transform.original_width
)
def y(value: float) -> float:
return (
min(1.0, max(0.0, (value - transform.top) / transform.height))
* transform.original_height
)
left, top = x(box.x), y(box.y)
return Box(left, top, x(box.x + box.width) - left, y(box.y + box.height) - top)
def intersection_over_union(first: Box, second: Box) -> float:
width = max(0.0, min(first.x + first.width, second.x + second.width) - max(first.x, second.x))
height = max(
0.0, min(first.y + first.height, second.y + second.height) - max(first.y, second.y)
)
intersection = width * height
union = first.width * first.height + second.width * second.height - intersection
return intersection / union if union > 0 else 0.0
def validate_predictions(data: Tensor, classes: int, confidence: float, overlap: float) -> None:
if (
data.dtype != np.float32
or data.ndim != 3
or not 1 <= classes <= 1024
or data.shape[:2] != (1, 4 + classes)
or not 1 <= data.shape[2] <= 100000
):
raise ValueError("Expected float32 [1,4+C,N], C<=1024 and 1<=N<=100000")
if not all(math.isfinite(value) and 0 <= value <= 1 for value in (confidence, overlap)):
raise ValueError("Confidence and overlap must be finite probabilities in [0,1]")
if not np.isfinite(data).all() or (data[0, 2:4] < 0).any():
raise ValueError("Unsupported detector coordinates: finite nonnegative sizes required")
if (data[0, 4:] < 0).any() or (data[0, 4:] > 1).any():
raise ValueError("Expected class probabilities in [0,1], not logits/objectness")
def decode_predictions(
data: Tensor,
classes: int,
transform: Letterbox,
confidence: float = 0.25,
overlap: float = 0.45,
) -> tuple[Detection, ...]:
validate_transform(transform)
validate_predictions(data, classes, confidence, overlap)
labels = np.argmax(data[0, 4:], axis=0)
scores = np.max(data[0, 4:], axis=0)
indices = np.flatnonzero(scores >= confidence)
indices = indices[np.argsort(-scores[indices], kind="stable")]
candidates: list[Detection] = []
for index in indices:
cx, cy, width, height = (float(value) for value in data[0, :4, index])
box = unproject_box(Box(cx - width / 2, cy - height / 2, width, height), transform)
if box.width <= 0 or box.height <= 0:
continue
candidates.append(Detection(box, int(labels[index]), float(scores[index])))
if len(candidates) == 3000:
break
return suppress_overlaps(candidates, overlap)
def suppress_overlaps(candidates: list[Detection], overlap: float) -> tuple[Detection, ...]:
selected: list[Detection] = []
for candidate in candidates:
if any(
previous.label == candidate.label
and intersection_over_union(previous.box, candidate.box) > overlap
for previous in selected
):
continue
selected.append(candidate)
if len(selected) == 300:
break
return tuple(selected)04.05
建置、執行、試模型。
明確選擇鏡頭,或者離線處理影像。撳 Q 或 Escape 停止。
基礎說明
parse_options(...)- 驗證參數,並且只揀一種輸入:影像或鏡頭。
main(...)- 驗證揀咗嘅來源,再交由圖片或相機流程處理。
要留意
- 症狀
- 停止所需時間比預期長。
- 原因
- 原生畫面擷取或推論呼叫仲執行緊。
- 處理方法
- 要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。
Terminalbash
python -m badbob_camera --model detector.onnx --labels labels.txt --image frame.png --output result.png
python -m badbob_camera --model detector.onnx --labels labels.txt --camera 004.06 / 最佳實踐
做好細節,範例更好。
- 擷取同推論唔好放喺介面執行緒。限制佇列長度,避免積埋舊影格。
- 影像尺寸、裁切、方向同座標要有同一套約定。先用已知偵測框測試,再信任畫面標記。
- 要求停止、捨棄遲到嘅結果,等進行中嘅工作完成之後先釋放資源。可唔可以取消底層執行,要睇平台支唔支援。