Swift
使用你的 Core ML 检测器构建原生 Mac 摄像头应用。
Core ML · macOSBadBob / Playground
让训练好的模型发挥作用。小巧的开源示例、清晰的步骤,以及可自由修改的代码。
开源 · MIT · 模型始终属于你

01 / 选择你的路径
使用你的 Core ML 检测器构建原生 Mac 摄像头应用。
Core ML · macOS通过脚本检查模型,构建实用工具。
ONNX Runtime · OpenCV使用受支持的 ONNX 模型构建浏览器示例。
Web · ONNX明确模型和运行环境要求的原生开发路径。
OpenCV · ONNX通过清晰的模型运行接口构建实用工具。
GoCV · ONNX使用 OpenCV 和受支持的 ONNX 模型进行摄像头采集与本地推理。
OpenCV · ONNXDeviconGo Gopher — Renee FrenchCC BY 3.0
02 / Swift Core ML + Vision
完整的 Swift 示例:选择模型、启动摄像头,并绘制与画面对应的预测结果。按步骤阅读,或浏览完整源代码。
示例聚焦于受控环境中的物品与动物,不涉及人员监控、身份识别或行为分析。
下载源代码并运行测试。你需要摄像头,以及与 Vision 兼容的 Core ML 目标检测模型。示例不包含模型权重。
swift testunzip badbob-playground-swift.zip
cd badbob-playground-swift
swift test加载模型并创建 Vision 请求。启动摄像头前检查输入,并在整个推理期间保留临时编译文件。
VisionDetector.init(modelURL:)VisionDetector.predict(_:)此示例需要能返回 Vision 对象观察结果的 Core ML 目标检测模型。其他模型类型和 ONNX 文件需要使用不同的适配器。
Vision 返回以左下角为原点的归一化坐标。将其转换为视图坐标,保持画面宽高比,并同时镜像画面与检测框。
model:
runtime: Core ML + Vision
formats: [.mlmodel, .mlpackage, .mlmodelc]
image_inputs: 1
results: VNRecognizedObjectObservation
preprocessing:
crop_and_scale: scaleFill
orientation: up
coordinates:
range: [0, 1]
origin: bottom_left
preview:
scaling: aspect_fit
mirroring: pairedimport CoreML
import Foundation
import Vision
/// Owned by one frame processor and used only on its serial capture queue.
final class VisionDetector {
private let request: VNCoreMLRequest
private let compiledURL: URL?
private let scopedURL: URL?
init(modelURL: URL) throws {
guard modelURL.isFileURL,
["mlmodel", "mlpackage", "mlmodelc"].contains(modelURL.pathExtension.lowercased())
else {
throw CaptureFailure.unsupportedModel
}
let scoped = modelURL.startAccessingSecurityScopedResource()
var ownedCompilation: URL?
do {
let loadURL: URL
if modelURL.pathExtension.lowercased() == "mlmodelc" {
loadURL = modelURL
} else {
loadURL = try MLModel.compileModel(at: modelURL)
ownedCompilation = loadURL
}
let model = try MLModel(contentsOf: loadURL)
guard
model.modelDescription.inputDescriptionsByName.values.filter({ $0.type == .image })
.count == 1
else {
throw CaptureFailure.unsupportedModel
}
request = VNCoreMLRequest(model: try VNCoreMLModel(for: model))
request.imageCropAndScaleOption = .scaleFill
compiledURL = ownedCompilation
scopedURL = scoped ? modelURL : nil
} catch {
if scoped {
modelURL.stopAccessingSecurityScopedResource()
}
if let ownedCompilation {
try? FileManager.default.removeItem(at: ownedCompilation)
}
throw error
}
}
deinit {
// Predictions have ended; remove only our own temporary compilation.
if let compiledURL {
try? FileManager.default.removeItem(at: compiledURL)
}
scopedURL?.stopAccessingSecurityScopedResource()
}
func predict(_ buffer: CVPixelBuffer) throws -> [Detection] {
try VNImageRequestHandler(cvPixelBuffer: buffer, orientation: .up).perform([request])
guard let objects = request.results as? [VNRecognizedObjectObservation] else {
throw CaptureFailure.unsupportedModel
}
return objects.enumerated().compactMap { index, observation in
guard let label = observation.labels.first, label.confidence.isFinite else {
return nil
}
return Detection(
id: index, label: label.identifier, confidence: label.confidence,
bounds: observation.boundingBox)
}
}
}采集在专用串行执行器上运行。丢弃延迟帧,只有最新处理的帧会等待界面显示。画面与预测结果始终对应。
CameraService.start(runID:modelURL:)CameraService.stop(runID:)import AVFoundation
import CoreImage
import Dispatch
import Foundation
/// A custom executor keeps blocking AVFoundation work off the main actor.
public actor CameraService: CameraCapturing {
private nonisolated let queue = DispatchSerialQueue(label: "ai.badbob.playground.camera")
public nonisolated var unownedExecutor: UnownedSerialExecutor {
queue.asUnownedSerialExecutor()
}
private var session: AVCaptureSession?
private var output: AVCaptureVideoDataOutput?
private var processor: FrameProcessor?
private var monitor: CaptureSessionMonitor?
private var activeRunID: UUID?
public init() {}
public func start(runID: UUID, modelURL: URL) async throws -> FrameStream {
stopActiveRun()
let detector = try loadDetector(modelURL: modelURL)
try Task.checkCancellation()
guard let camera = AVCaptureDevice.default(for: .video) else {
throw CaptureFailure.noCamera
}
let candidate = AVCaptureSession()
let frames = AVCaptureVideoDataOutput()
let (stream, continuation) = FrameStream.makeStream(bufferingPolicy: .bufferingNewest(1))
let delegate = FrameProcessor(detector: detector, continuation: continuation)
try configureSession(candidate, output: frames, processor: delegate, camera: camera)
try Task.checkCancellation()
session = candidate
output = frames
processor = delegate
activeRunID = runID
monitor = CaptureSessionMonitor(session: candidate, continuation: continuation)
continuation.onTermination = { [weak self] _ in
Task { await self?.stop(runID: runID) }
}
candidate.startRunning()
guard candidate.isRunning else {
stopActiveRun()
throw CaptureFailure.configuration
}
return stream
}
private func loadDetector(modelURL: URL) throws -> VisionDetector {
do {
return try VisionDetector(modelURL: modelURL)
} catch let error as CaptureFailure {
throw error
} catch {
throw CaptureFailure.model(error.localizedDescription)
}
}
private func configureSession(
_ candidate: AVCaptureSession,
output frames: AVCaptureVideoDataOutput,
processor delegate: FrameProcessor,
camera: AVCaptureDevice
) throws {
candidate.beginConfiguration()
do {
let input = try AVCaptureDeviceInput(device: camera)
guard candidate.canAddInput(input), candidate.canAddOutput(frames) else {
throw CaptureFailure.configuration
}
candidate.addInput(input)
candidate.addOutput(frames)
if candidate.canSetSessionPreset(.hd1280x720) {
candidate.sessionPreset = .hd1280x720
}
frames.alwaysDiscardsLateVideoFrames = true
frames.videoSettings = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
]
// The delegate, Vision request and frame renderer share this one serial queue.
frames.setSampleBufferDelegate(delegate, queue: queue)
if let connection = frames.connection(with: .video),
connection.isVideoMirroringSupported
{
connection.automaticallyAdjustsVideoMirroring = false
connection.isVideoMirrored = false
}
candidate.commitConfiguration()
} catch {
candidate.commitConfiguration()
frames.setSampleBufferDelegate(nil, queue: nil)
delegate.finish()
throw error
}
}
public func stop(runID: UUID) {
// A cancelled old task must not stop a more recent capture run.
guard activeRunID == runID else {
return
}
stopActiveRun()
}
private func stopActiveRun() {
// Remove observers first so our deliberate stop cannot become a failure.
monitor?.invalidate()
monitor = nil
session?.stopRunning()
output?.setSampleBufferDelegate(nil, queue: nil)
processor?.finish()
processor = nil
output = nil
session = nil
activeRunID = nil
}
}
/// AVFoundation invokes this delegate only on CameraService's serial queue.
/// No sample buffer or mutable Vision object crosses into a Task or the UI.
private final class FrameProcessor: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
private let detector: VisionDetector
private let continuation: FrameStream.Continuation
private let renderer = CIContext(options: [.cacheIntermediates: false])
private var finished = false
init(detector: VisionDetector, continuation: FrameStream.Continuation) {
self.detector = detector
self.continuation = continuation
}
func finish() {
guard !finished else {
return
}
finished = true
continuation.finish()
}
func captureOutput(
_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
from connection: AVCaptureConnection
) {
guard !finished, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
autoreleasepool {
do {
let detections = try detector.predict(buffer)
let image = CIImage(cvPixelBuffer: buffer)
guard let pixels = renderer.createCGImage(image, from: image.extent) else {
throw CaptureFailure.imageCreation
}
continuation.yield(DetectionFrame(image: pixels, detections: detections))
} catch {
finished = true
continuation.finish(throwing: error)
}
}
}
}Vision 返回以左下角为原点的归一化坐标。将其转换为视图坐标,保持画面宽高比,并同时镜像画面与检测框。
OverlayGeometry.project(...)将图像尺寸、裁剪、方向和坐标作为统一约定。先用已知检测框测试,再信任画面标注。
import CoreGraphics
/// Convert Vision's normalized, bottom-left coordinates to an aspect-fit view.
public enum OverlayGeometry {
public static func project(
_ bounds: CGRect, imageSize: CGSize, viewport: CGSize, mirrored: Bool
) -> CGRect? {
guard
[imageSize.width, imageSize.height, viewport.width, viewport.height].allSatisfy({
$0.isFinite && $0 > 0
}),
[bounds.origin.x, bounds.origin.y, bounds.width, bounds.height].allSatisfy(\.isFinite),
bounds.width > 0, bounds.height > 0
else { return nil }
let clipped = bounds.intersection(CGRect(x: 0, y: 0, width: 1, height: 1))
guard !clipped.isNull, !clipped.isEmpty else {
return nil
}
let scale = min(viewport.width / imageSize.width, viewport.height / imageSize.height)
let width = imageSize.width * scale
let height = imageSize.height * scale
return CGRect(
x: (viewport.width - width) / 2 + (mirrored ? 1 - clipped.maxX : clipped.minX) * width,
y: (viewport.height - height) / 2 + (1 - clipped.maxY) * height,
width: clipped.width * width, height: clipped.height * height
)
}
}构建本地应用,选择检测模型并点击 Start camera。调整置信度阈值和镜像设置。完成后停止采集。
CameraController.start()CameraController.stop()