BadBob / Playground
ONNX Runtime · OpenCV · MIT
04 / Python
Python。從模型到攝影機。
在本機執行 ONNX 偵測器,即時顯示標註,並限制待處理影像的數量。
範例聚焦於受控環境中的物品與動物,不涉及人員監控、身分識別或行為分析。
04.01
準備專案
安裝列出的相依套件並執行測試。開啟攝影機前,先試用一張本機影像。
基礎說明
python -m unittest- 執行自動檢查,不開啟攝影機。
Terminalbash
unzip badbob-playground-python.zip
cd badbob-playground-python
python3.12 -m venv .venv
. .venv/bin/activate
python -m pip install -r requirements.txt
python -m unittest discover -s tests -v04.02
連接模型
在開啟攝影機之前驗證模型和類別標籤。
基礎說明
OnnxDetector.__init__(...)- 載入 ONNX 模型,並驗證輸入、輸出與標籤數量。
prepare_input(...)- 建立 640×640 的 RGB 輸入,保留原始比例,並填滿剩餘區域。
OnnxDetector.detect(...)- 前處理影像,執行模型並傳回偵測結果。
注意事項
- 症狀
- 偵測框偏移,或模型無法載入。
- 原因
- 前處理、輸出格式或標籤順序不符。
- 解決方法
- 此介接器僅支援以下約定。Core ML 檔案和其他 ONNX 輸出格式需要不同的介接器。
此介接器僅支援以下約定。Core ML 檔案和其他 ONNX 輸出格式需要不同的介接器。
深入了解
座標表示偵測框的中心點和尺寸,單位為 640×640 輸入影像中的像素。標籤必須嚴格依照模型的類別順序排列。
ONNXyaml
input:
color: RGB
dtype: float32
shape: [1, 3, 640, 640]
letterbox: 114
scale: 1/255
output:
dtype: float32
shape: [1, 4+C, N]
boxes: [cx, cy, width, height]
class_scores: probabilities
objectness: false
embedded_nms: falsedetector.pypython
"""Explicit CPU ONNX adapter; another export contract needs another adapter."""
from pathlib import Path
from typing import cast
import cv2
import numpy as np
import onnxruntime as ort
from .geometry import calculate_letterbox, decode_predictions
from .image_contract import validate_image
from .labels import validate_label_metadata
from .types import Detection, Image, Letterbox, Tensor
def prepare_input(frame: Image) -> tuple[Tensor, Letterbox]:
validate_image(frame)
transform = calculate_letterbox(frame.shape[1], frame.shape[0])
resized = cv2.resize(frame, (transform.width, transform.height), interpolation=cv2.INTER_LINEAR)
canvas = np.full((640, 640, 3), 114, dtype=np.uint8)
canvas[
transform.top : transform.top + transform.height,
transform.left : transform.left + transform.width,
] = resized
rgb = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)
tensor = np.ascontiguousarray(rgb.transpose(2, 0, 1)[None], dtype=np.float32)
tensor /= np.float32(255)
return tensor, transform
class OnnxDetector:
def __init__(self, model: Path, labels: tuple[str, ...]) -> None:
if not model.is_file() or model.suffix.lower() != ".onnx":
raise ValueError("Supply an existing local .onnx model")
if not 1 <= len(labels) <= 1024:
raise ValueError("Expected 1..1024 class labels")
self._session: ort.InferenceSession | None = None
self._classes = len(labels)
ort.disable_telemetry_events()
session = ort.InferenceSession(str(model), providers=["CPUExecutionProvider"])
inputs, outputs = session.get_inputs(), session.get_outputs()
if (
len(inputs) != 1
or inputs[0].type != "tensor(float)"
or inputs[0].shape != [1, 3, 640, 640]
):
raise ValueError("Expected one float32 RGB NCHW input [1,3,640,640]")
if len(outputs) != 1 or outputs[0].type != "tensor(float)":
raise ValueError("Expected one float32 detector output")
self._validate_output_shape(outputs[0].shape)
validate_label_metadata(session.get_modelmeta().custom_metadata_map, labels)
self._input_name, self._output_name = inputs[0].name, outputs[0].name
self._session = session
# Validate actual output values before the runner may acquire a camera.
self.detect(np.full((640, 640, 3), 114, dtype=np.uint8))
def _validate_output_shape(self, shape: object) -> None:
if not isinstance(shape, list) or len(shape) != 3 or shape[:2] != [1, 4 + self._classes]:
raise ValueError("Expected output metadata [1,4+C,N] matching label count")
candidates = shape[2]
if isinstance(candidates, int) and not 1 <= candidates <= 100000:
raise ValueError("Expected 1..100000 detector candidates")
if candidates is not None and not isinstance(candidates, (str, int)):
raise ValueError("Invalid detector candidate dimension")
def detect(self, frame: Image) -> tuple[Detection, ...]:
session = self._session
if session is None:
raise RuntimeError("Detector is closed")
tensor, transform = prepare_input(frame)
results = session.run([self._output_name], {self._input_name: tensor})
output: object = results[0]
if not isinstance(output, np.ndarray) or output.dtype != np.float32:
raise ValueError("Expected a float32 NumPy detector tensor")
return decode_predictions(cast(Tensor, output), self._classes, transform)
def close(self) -> None:
# The Python CPU session has no public close(); drop its owner reference.
self._session = None04.03
接入相機
擷取和推論共用一個循序工作執行緒。介面只接收最新處理完成的影像及其偵測結果。
基礎說明
Runner._run()- 在同一個工作執行緒中處理影像擷取與推論。
LatestResult.take()- 取出最新處理完成的影像及其偵測結果,避免結果在佇列中持續累積。
Runner.stop()- 停止接收新結果,並在目前工作結束後釋放資源。
注意事項
- 症狀
- 預覽延遲,記憶體用量持續增加。
- 原因
- 攝影機產生影像的速度快於推論處理速度,導致無界佇列持續增長。
- 解決方法
- 擷取和推論共用一個循序工作執行緒。介面只接收最新處理完成的影像及其偵測結果。
深入了解
要求停止、捨棄延遲傳回的結果,並等待目前工作結束後再釋放其資源。能否取消底層執行取決於平台支援。
runner.pypython
"""One joined worker, no input queue, and one complete owned output pair."""
import threading
from collections.abc import Callable
from .types import Detector, FrameResult, FrameSource
class LatestResult:
def __init__(self) -> None:
self._lock = threading.Lock()
self._pending: FrameResult | None = None
self._closed = False
def publish(self, result: FrameResult) -> bool:
with self._lock:
if self._closed:
return False
self._pending = result
return True
def take(self) -> FrameResult | None:
with self._lock:
result, self._pending = self._pending, None
return result
def close(self) -> None:
with self._lock:
self._closed = True
self._pending = None
class Runner:
def __init__(
self, detector_factory: Callable[[], Detector], source_factory: Callable[[], FrameSource]
):
self.results = LatestResult()
self.finished = threading.Event()
self.stop_requested = threading.Event()
self.error: str | None = None
self._detector_factory = detector_factory
self._source_factory = source_factory
self._thread: threading.Thread | None = None
def start(self) -> None:
if self._thread is not None or self.stop_requested.is_set():
raise RuntimeError("A Runner starts once; create a new Runner for another session")
self._thread = threading.Thread(target=self._run, name="badbob-inference", daemon=False)
self._thread.start()
def stop(self) -> None:
self.stop_requested.set()
self.results.close()
if self._thread is not None:
self._thread.join()
def _capture(self, detector: Detector, source: FrameSource) -> None:
sequence = 0
while not self.stop_requested.is_set():
frame = source.read()
if frame is None:
return
if self.stop_requested.is_set():
return
# Own the pixels even when a backend or test double reuses its buffer.
owned_frame = frame.copy()
detections = detector.detect(owned_frame)
if self.stop_requested.is_set():
return
sequence += 1
self.results.publish(FrameResult(owned_frame, detections, sequence))
def _run(self) -> None:
detector: Detector | None = None
source: FrameSource | None = None
try:
detector = self._detector_factory()
if self.stop_requested.is_set():
return
source = self._source_factory()
self._capture(detector, source)
except Exception as error:
self.error = f"{type(error).__name__}: {error}"
self.results.close()
finally:
self._release(source, detector)
self.finished.set()
def _release(self, source: FrameSource | None, detector: Detector | None) -> None:
for resource in (source, detector):
if resource is None:
continue
try:
resource.close()
except Exception as error:
self.error = self.error or f"Cleanup failed: {error}"
self.results.close()04.04
正確定位預測結果
還原 letterbox 縮放和填補,並在繪製前將偵測框裁切至影像邊界內。
基礎說明
calculate_letterbox(...) · unproject_box(...)- 維持影像長寬比,並將偵測框映射回原始影像。
decode_predictions(...)- 驗證預測結果,移除同一類別中重疊的偵測框。
注意事項
將影像尺寸、裁切、方向與座標視為統一約定。先用已知偵測框測試,再信任畫面標註。
geometry.pypython
"""Letterbox geometry and bounded class-aware suppression."""
import math
import numpy as np
from .types import Box, Detection, Letterbox, Tensor
def calculate_letterbox(width: int, height: int, edge: int = 640) -> Letterbox:
if width <= 0 or height <= 0 or not 0 < edge <= 4096:
raise ValueError("Positive image dimensions and input edge <=4096 required")
scale = min(edge / width, edge / height)
resized_width = max(1, math.floor(width * scale + 0.5))
resized_height = max(1, math.floor(height * scale + 0.5))
return Letterbox(
width,
height,
resized_width,
resized_height,
(edge - resized_width) // 2,
(edge - resized_height) // 2,
edge,
)
def validate_transform(transform: Letterbox) -> None:
if (
transform.original_width <= 0
or transform.original_height <= 0
or not 0 < transform.width <= transform.edge <= 4096
or not 0 < transform.height <= transform.edge
or not 0 <= transform.left <= transform.edge - transform.width
or not 0 <= transform.top <= transform.edge - transform.height
):
raise ValueError("Invalid letterbox geometry")
def unproject_box(box: Box, transform: Letterbox) -> Box:
validate_transform(transform)
values = (box.x, box.y, box.width, box.height, box.x + box.width, box.y + box.height)
if not all(math.isfinite(value) for value in values) or min(box.width, box.height) < 0:
raise ValueError("Box coordinates must be finite with nonnegative dimensions")
def x(value: float) -> float:
return (
min(1.0, max(0.0, (value - transform.left) / transform.width))
* transform.original_width
)
def y(value: float) -> float:
return (
min(1.0, max(0.0, (value - transform.top) / transform.height))
* transform.original_height
)
left, top = x(box.x), y(box.y)
return Box(left, top, x(box.x + box.width) - left, y(box.y + box.height) - top)
def intersection_over_union(first: Box, second: Box) -> float:
width = max(0.0, min(first.x + first.width, second.x + second.width) - max(first.x, second.x))
height = max(
0.0, min(first.y + first.height, second.y + second.height) - max(first.y, second.y)
)
intersection = width * height
union = first.width * first.height + second.width * second.height - intersection
return intersection / union if union > 0 else 0.0
def validate_predictions(data: Tensor, classes: int, confidence: float, overlap: float) -> None:
if (
data.dtype != np.float32
or data.ndim != 3
or not 1 <= classes <= 1024
or data.shape[:2] != (1, 4 + classes)
or not 1 <= data.shape[2] <= 100000
):
raise ValueError("Expected float32 [1,4+C,N], C<=1024 and 1<=N<=100000")
if not all(math.isfinite(value) and 0 <= value <= 1 for value in (confidence, overlap)):
raise ValueError("Confidence and overlap must be finite probabilities in [0,1]")
if not np.isfinite(data).all() or (data[0, 2:4] < 0).any():
raise ValueError("Unsupported detector coordinates: finite nonnegative sizes required")
if (data[0, 4:] < 0).any() or (data[0, 4:] > 1).any():
raise ValueError("Expected class probabilities in [0,1], not logits/objectness")
def decode_predictions(
data: Tensor,
classes: int,
transform: Letterbox,
confidence: float = 0.25,
overlap: float = 0.45,
) -> tuple[Detection, ...]:
validate_transform(transform)
validate_predictions(data, classes, confidence, overlap)
labels = np.argmax(data[0, 4:], axis=0)
scores = np.max(data[0, 4:], axis=0)
indices = np.flatnonzero(scores >= confidence)
indices = indices[np.argsort(-scores[indices], kind="stable")]
candidates: list[Detection] = []
for index in indices:
cx, cy, width, height = (float(value) for value in data[0, :4, index])
box = unproject_box(Box(cx - width / 2, cy - height / 2, width, height), transform)
if box.width <= 0 or box.height <= 0:
continue
candidates.append(Detection(box, int(labels[index]), float(scores[index])))
if len(candidates) == 3000:
break
return suppress_overlaps(candidates, overlap)
def suppress_overlaps(candidates: list[Detection], overlap: float) -> tuple[Detection, ...]:
selected: list[Detection] = []
for candidate in candidates:
if any(
previous.label == candidate.label
and intersection_over_union(previous.box, candidate.box) > overlap
for previous in selected
):
continue
selected.append(candidate)
if len(selected) == 300:
break
return tuple(selected)04.05
建置、執行、測試模型。
明確選擇攝影機,或離線處理影像。按 Q 或 Escape 停止。
基礎說明
parse_options(...)- 驗證參數,且只選擇一種輸入:影像或攝影機。
main(...)- 驗證所選來源,並將處理工作交給影像或攝影機處理流程。
注意事項
- 症狀
- 停止所需時間超出預期。
- 原因
- 原生影像擷取或推論呼叫仍在執行。
- 解決方法
- 要求停止、捨棄延遲傳回的結果,並等待目前工作結束後再釋放其資源。能否取消底層執行取決於平台支援。
Terminalbash
python -m badbob_camera --model detector.onnx --labels labels.txt --image frame.png --output result.png
python -m badbob_camera --model detector.onnx --labels labels.txt --camera 004.06 / 最佳實務
做好細節,寫好範例。
- 讓擷取與推論遠離介面執行緒。限制佇列長度,不要堆積舊影格。
- 將影像尺寸、裁切、方向與座標視為統一約定。先用已知偵測框測試,再信任畫面標註。
- 要求停止、捨棄延遲傳回的結果,並等待目前工作結束後再釋放其資源。能否取消底層執行取決於平台支援。