Swift
Innbyggt myndavélaapp fyrir Mac með Core ML-greininum þínum.
Core ML · macOSBadBob / Playground
Gerðu þjálfað líkan að einhverju gagnlegu. Lítil dæmi með opnum kóða, skýr skref og kóði sem þú getur aðlagað þínum hugmyndum.
Opinn kóði · MIT · Líkanið er áfram þitt

01 / VELDU ÞÍNA LEIÐ
Innbyggt myndavélaapp fyrir Mac með Core ML-greininum þínum.
Core ML · macOSForskriftaleið til að skoða líkön og smíða gagnleg verkfæri.
ONNX Runtime · OpenCVVafradæmi með studdu ONNX-líkani.
Web · ONNXInnbyggð leið með skýrum kröfum um líkan og keyrsluumhverfi.
OpenCV · ONNXHagnýt verkfæri með skýru viðmóti við keyrslu líkans.
GoCV · ONNXMyndavél og staðbundin ályktun með OpenCV og studdu ONNX-líkani.
OpenCV · ONNXDeviconGo Gopher — Renee FrenchCC BY 3.0
02 / Swift Core ML + Vision
Heilstætt Swift-dæmi: veldu líkanið, ræstu myndavélina og teiknaðu samsvarandi spár. Fylgdu skrefunum eða skoðaðu allan kóðann.
Dæmin snúast um hluti og dýr í afmörkuðu umhverfi. Þau fjalla ekki um eftirlit með fólki, persónugreiningu eða hegðunargreiningu.
Sæktu frumkóðann og keyrðu prófin. Þú þarft myndavél og Core ML-hlutagreini sem vinnur með Vision. Líkansþyngdir fylgja ekki.
swift testunzip badbob-playground-swift.zip
cd badbob-playground-swift
swift testHladdu líkaninu og búðu til Vision-beiðni. Athugaðu inntakið áður en myndavélin fer í gang og haltu tímabundnum þýðingum á meðan ályktunin notar þær.
VisionDetector.init(modelURL:)VisionDetector.predict(_:)Dæmið krefst Core ML-hlutgreinis sem skilar hlutathugunum í Vision. Aðrar líkanagerðir og ONNX-skrár þurfa annan aðlaga.
Vision skilar stöðluðum hnitum frá neðra vinstra horni. Umbreyttu þeim í hnit viðmótsins, haltu hlutföllum og speglaðu mynd og ramma saman.
model:
runtime: Core ML + Vision
formats: [.mlmodel, .mlpackage, .mlmodelc]
image_inputs: 1
results: VNRecognizedObjectObservation
preprocessing:
crop_and_scale: scaleFill
orientation: up
coordinates:
range: [0, 1]
origin: bottom_left
preview:
scaling: aspect_fit
mirroring: pairedimport CoreML
import Foundation
import Vision
/// Owned by one frame processor and used only on its serial capture queue.
final class VisionDetector {
private let request: VNCoreMLRequest
private let compiledURL: URL?
private let scopedURL: URL?
init(modelURL: URL) throws {
guard modelURL.isFileURL,
["mlmodel", "mlpackage", "mlmodelc"].contains(modelURL.pathExtension.lowercased())
else {
throw CaptureFailure.unsupportedModel
}
let scoped = modelURL.startAccessingSecurityScopedResource()
var ownedCompilation: URL?
do {
let loadURL: URL
if modelURL.pathExtension.lowercased() == "mlmodelc" {
loadURL = modelURL
} else {
loadURL = try MLModel.compileModel(at: modelURL)
ownedCompilation = loadURL
}
let model = try MLModel(contentsOf: loadURL)
guard
model.modelDescription.inputDescriptionsByName.values.filter({ $0.type == .image })
.count == 1
else {
throw CaptureFailure.unsupportedModel
}
request = VNCoreMLRequest(model: try VNCoreMLModel(for: model))
request.imageCropAndScaleOption = .scaleFill
compiledURL = ownedCompilation
scopedURL = scoped ? modelURL : nil
} catch {
if scoped {
modelURL.stopAccessingSecurityScopedResource()
}
if let ownedCompilation {
try? FileManager.default.removeItem(at: ownedCompilation)
}
throw error
}
}
deinit {
// Predictions have ended; remove only our own temporary compilation.
if let compiledURL {
try? FileManager.default.removeItem(at: compiledURL)
}
scopedURL?.stopAccessingSecurityScopedResource()
}
func predict(_ buffer: CVPixelBuffer) throws -> [Detection] {
try VNImageRequestHandler(cvPixelBuffer: buffer, orientation: .up).perform([request])
guard let objects = request.results as? [VNRecognizedObjectObservation] else {
throw CaptureFailure.unsupportedModel
}
return objects.enumerated().compactMap { index, observation in
guard let label = observation.labels.first, label.confidence.isFinite else {
return nil
}
return Detection(
id: index, label: label.identifier, confidence: label.confidence,
bounds: observation.boundingBox)
}
}
}Myndatakan keyrir á eigin raðbundna keyrara. Seinum römmum er sleppt og aðeins nýjasti unninn rammi bíður eftir viðmótinu. Mynd og spár fylgjast að.
CameraService.start(runID:modelURL:)CameraService.stop(runID:)import AVFoundation
import CoreImage
import Dispatch
import Foundation
/// A custom executor keeps blocking AVFoundation work off the main actor.
public actor CameraService: CameraCapturing {
private nonisolated let queue = DispatchSerialQueue(label: "ai.badbob.playground.camera")
public nonisolated var unownedExecutor: UnownedSerialExecutor {
queue.asUnownedSerialExecutor()
}
private var session: AVCaptureSession?
private var output: AVCaptureVideoDataOutput?
private var processor: FrameProcessor?
private var monitor: CaptureSessionMonitor?
private var activeRunID: UUID?
public init() {}
public func start(runID: UUID, modelURL: URL) async throws -> FrameStream {
stopActiveRun()
let detector = try loadDetector(modelURL: modelURL)
try Task.checkCancellation()
guard let camera = AVCaptureDevice.default(for: .video) else {
throw CaptureFailure.noCamera
}
let candidate = AVCaptureSession()
let frames = AVCaptureVideoDataOutput()
let (stream, continuation) = FrameStream.makeStream(bufferingPolicy: .bufferingNewest(1))
let delegate = FrameProcessor(detector: detector, continuation: continuation)
try configureSession(candidate, output: frames, processor: delegate, camera: camera)
try Task.checkCancellation()
session = candidate
output = frames
processor = delegate
activeRunID = runID
monitor = CaptureSessionMonitor(session: candidate, continuation: continuation)
continuation.onTermination = { [weak self] _ in
Task { await self?.stop(runID: runID) }
}
candidate.startRunning()
guard candidate.isRunning else {
stopActiveRun()
throw CaptureFailure.configuration
}
return stream
}
private func loadDetector(modelURL: URL) throws -> VisionDetector {
do {
return try VisionDetector(modelURL: modelURL)
} catch let error as CaptureFailure {
throw error
} catch {
throw CaptureFailure.model(error.localizedDescription)
}
}
private func configureSession(
_ candidate: AVCaptureSession,
output frames: AVCaptureVideoDataOutput,
processor delegate: FrameProcessor,
camera: AVCaptureDevice
) throws {
candidate.beginConfiguration()
do {
let input = try AVCaptureDeviceInput(device: camera)
guard candidate.canAddInput(input), candidate.canAddOutput(frames) else {
throw CaptureFailure.configuration
}
candidate.addInput(input)
candidate.addOutput(frames)
if candidate.canSetSessionPreset(.hd1280x720) {
candidate.sessionPreset = .hd1280x720
}
frames.alwaysDiscardsLateVideoFrames = true
frames.videoSettings = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
]
// The delegate, Vision request and frame renderer share this one serial queue.
frames.setSampleBufferDelegate(delegate, queue: queue)
if let connection = frames.connection(with: .video),
connection.isVideoMirroringSupported
{
connection.automaticallyAdjustsVideoMirroring = false
connection.isVideoMirrored = false
}
candidate.commitConfiguration()
} catch {
candidate.commitConfiguration()
frames.setSampleBufferDelegate(nil, queue: nil)
delegate.finish()
throw error
}
}
public func stop(runID: UUID) {
// A cancelled old task must not stop a more recent capture run.
guard activeRunID == runID else {
return
}
stopActiveRun()
}
private func stopActiveRun() {
// Remove observers first so our deliberate stop cannot become a failure.
monitor?.invalidate()
monitor = nil
session?.stopRunning()
output?.setSampleBufferDelegate(nil, queue: nil)
processor?.finish()
processor = nil
output = nil
session = nil
activeRunID = nil
}
}
/// AVFoundation invokes this delegate only on CameraService's serial queue.
/// No sample buffer or mutable Vision object crosses into a Task or the UI.
private final class FrameProcessor: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
private let detector: VisionDetector
private let continuation: FrameStream.Continuation
private let renderer = CIContext(options: [.cacheIntermediates: false])
private var finished = false
init(detector: VisionDetector, continuation: FrameStream.Continuation) {
self.detector = detector
self.continuation = continuation
}
func finish() {
guard !finished else {
return
}
finished = true
continuation.finish()
}
func captureOutput(
_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
from connection: AVCaptureConnection
) {
guard !finished, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
autoreleasepool {
do {
let detections = try detector.predict(buffer)
let image = CIImage(cvPixelBuffer: buffer)
guard let pixels = renderer.createCGImage(image, from: image.extent) else {
throw CaptureFailure.imageCreation
}
continuation.yield(DetectionFrame(image: pixels, detections: detections))
} catch {
finished = true
continuation.finish(throwing: error)
}
}
}
}Vision skilar stöðluðum hnitum frá neðra vinstra horni. Umbreyttu þeim í hnit viðmótsins, haltu hlutföllum og speglaðu mynd og ramma saman.
OverlayGeometry.project(...)Líttu á myndstærð, skurð, stefnu og hnit sem eina heild. Prófaðu þekkta ramma áður en þú treystir merkingunum.
import CoreGraphics
/// Convert Vision's normalized, bottom-left coordinates to an aspect-fit view.
public enum OverlayGeometry {
public static func project(
_ bounds: CGRect, imageSize: CGSize, viewport: CGSize, mirrored: Bool
) -> CGRect? {
guard
[imageSize.width, imageSize.height, viewport.width, viewport.height].allSatisfy({
$0.isFinite && $0 > 0
}),
[bounds.origin.x, bounds.origin.y, bounds.width, bounds.height].allSatisfy(\.isFinite),
bounds.width > 0, bounds.height > 0
else { return nil }
let clipped = bounds.intersection(CGRect(x: 0, y: 0, width: 1, height: 1))
guard !clipped.isNull, !clipped.isEmpty else {
return nil
}
let scale = min(viewport.width / imageSize.width, viewport.height / imageSize.height)
let width = imageSize.width * scale
let height = imageSize.height * scale
return CGRect(
x: (viewport.width - width) / 2 + (mirrored ? 1 - clipped.maxX : clipped.minX) * width,
y: (viewport.height - height) / 2 + (1 - clipped.maxY) * height,
width: clipped.width * width, height: clipped.height * height
)
}
}Byggðu staðbundna appið, veldu greininn og ýttu á Start camera. Stilltu öryggismörk spár og speglun. Stöðvaðu myndatökuna þegar þú ert búinn.
CameraController.start()CameraController.stop()./build-app.sh
open ".build/BadBob Playground.app"02.06 / Góð vinnubrögð