929d77a802
engine.py is the process-wide, thread-safe core the server is built on: one YOLO instance behind a lock, thread-local matcher access, a classification pool so detection is never starved, the stock catalog under its own lock, empty-slot geometry normalized to a reference width (the detector's gap thresholds are absolute pixels), and a small LRU of recent still images for later classify/inspect calls. Detection output is normalized so every client sees the same box shape. tracking.py wraps ultralytics BYTETracker. The important part: ultralytics mints track ids from a process-global counter and resets it when a tracker is constructed, so with two live clients one client's ids would be re-issued while the other still held cached labels. SessionTracker gives each tracker its own counter and never touches the global one. Measured defaults (bench_engine.py, M5): detection p50 26-29 ms on CPU vs 112 ms on MPS with byte-identical boxes, so device defaults to cpu; hybrid classifies at ~51 ms/crop, 92 crops/s across 8 threads. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
337 lines
13 KiB
Python
337 lines
13 KiB
Python
"""
|
|
Component benchmark for the recognition engine.
|
|
|
|
Measures the two things that decide the live-mode defaults: how long detection
|
|
takes per frame (per device) and how long classification takes per product crop
|
|
(per method), including parallel throughput.
|
|
|
|
Usage (from recognization/):
|
|
.venv/bin/python code/bench_engine.py
|
|
.venv/bin/python code/bench_engine.py --devices cpu mps --image image.jpg
|
|
.venv/bin/python code/bench_engine.py --compare hybrid hybrid_fast
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import statistics
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
project_root = os.path.dirname(current_dir)
|
|
if os.path.exists(os.path.join(project_root, "stock")):
|
|
os.chdir(project_root)
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
import cv2 # noqa: E402
|
|
import numpy as np # noqa: E402
|
|
|
|
|
|
def percentile(values: List[float], pct: float) -> float:
|
|
if not values:
|
|
return 0.0
|
|
ordered = sorted(values)
|
|
index = min(len(ordered) - 1, max(0, int(round((pct / 100.0) * (len(ordered) - 1)))))
|
|
return ordered[index]
|
|
|
|
|
|
def format_table(rows: List[List[str]], headers: List[str]) -> str:
|
|
widths = [len(h) for h in headers]
|
|
for row in rows:
|
|
for i, cell in enumerate(row):
|
|
widths[i] = max(widths[i], len(str(cell)))
|
|
lines = [" | ".join(h.ljust(widths[i]) for i, h in enumerate(headers)),
|
|
"-+-".join("-" * w for w in widths)]
|
|
for row in rows:
|
|
lines.append(" | ".join(str(cell).ljust(widths[i]) for i, cell in enumerate(row)))
|
|
return "\n".join(lines)
|
|
|
|
|
|
def load_image(path: str, width: Optional[int] = None) -> Any:
|
|
image = cv2.imread(path)
|
|
if image is None:
|
|
raise SystemExit(f"cannot read {path}")
|
|
if width:
|
|
scale = width / float(image.shape[1])
|
|
image = cv2.resize(image, (int(round(image.shape[1] * scale)), int(round(image.shape[0] * scale))))
|
|
return image
|
|
|
|
|
|
def bench_detection(image_path: str, devices: List[str], runs: int, imgsz: int) -> List[Dict[str, Any]]:
|
|
from detector import ShelfProductDetector
|
|
|
|
weights = "runs/detect/train/weights/best.pt"
|
|
results: List[Dict[str, Any]] = []
|
|
native = load_image(image_path)
|
|
frame = load_image(image_path, width=1280)
|
|
for device in devices:
|
|
try:
|
|
detector = ShelfProductDetector(weights, device=None if device == "default" else device, imgsz=imgsz)
|
|
except Exception as exc:
|
|
results.append({"device": device, "error": str(exc)})
|
|
continue
|
|
for label, sample in (("native", native), ("1280x720-ish", frame)):
|
|
latencies: List[float] = []
|
|
boxes = 0
|
|
for _ in range(runs):
|
|
started = time.perf_counter()
|
|
det = detector.detect(sample)
|
|
latencies.append((time.perf_counter() - started) * 1000.0)
|
|
boxes = len(det.get("boxes", []))
|
|
results.append(
|
|
{
|
|
"device": device,
|
|
"size": label,
|
|
"shape": f"{sample.shape[1]}x{sample.shape[0]}",
|
|
"boxes": boxes,
|
|
"p50_ms": round(statistics.median(latencies), 2),
|
|
"p95_ms": round(percentile(latencies, 95), 2),
|
|
"min_ms": round(min(latencies), 2),
|
|
}
|
|
)
|
|
return results
|
|
|
|
|
|
def box_iou(a: List[int], b: List[int]) -> float:
|
|
ix1, iy1 = max(a[0], b[0]), max(a[1], b[1])
|
|
ix2, iy2 = min(a[2], b[2]), min(a[3], b[3])
|
|
iw, ih = max(0, ix2 - ix1), max(0, iy2 - iy1)
|
|
inter = iw * ih
|
|
if inter <= 0:
|
|
return 0.0
|
|
area_a = (a[2] - a[0]) * (a[3] - a[1])
|
|
area_b = (b[2] - b[0]) * (b[3] - b[1])
|
|
return inter / max(1, area_a + area_b - inter)
|
|
|
|
|
|
def detection_agreement(a: List[Dict[str, Any]], b: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
if not a or not b:
|
|
return {"matched": 0, "mean_best_iou": 0.0, "count_ratio": 0.0}
|
|
ious = []
|
|
for box_a in a:
|
|
best = max((box_iou(box_a["box"], box_b["box"]) for box_b in b), default=0.0)
|
|
ious.append(best)
|
|
return {
|
|
"matched": sum(1 for i in ious if i >= 0.5),
|
|
"mean_best_iou": round(float(np.mean(ious)), 3),
|
|
"count_ratio": round(len(a) / float(max(1, len(b))), 3),
|
|
}
|
|
|
|
|
|
def bench_crops(image_path: str, methods: List[str]) -> Dict[str, Any]:
|
|
from detector import ShelfProductDetector
|
|
from feature_matcher import ProductFeatureMatcher
|
|
|
|
detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
|
|
matcher = ProductFeatureMatcher(stock_dir="stock")
|
|
image = load_image(image_path)
|
|
det = detector.detect(image)
|
|
crops = []
|
|
for box in det.get("boxes", []):
|
|
x1, y1, x2, y2 = box["box"]
|
|
crops.append(image[y1:y2, x1:x2].copy())
|
|
report: Dict[str, Any] = {"crop_count": len(crops), "methods": {}}
|
|
for method in methods:
|
|
latencies: List[float] = []
|
|
levels: Dict[str, int] = {}
|
|
skus: List[str] = []
|
|
for crop in crops:
|
|
started = time.perf_counter()
|
|
result = matcher.classify_crop(crop, method=method)
|
|
latencies.append((time.perf_counter() - started) * 1000.0)
|
|
level = result.get("identity_level", "error")
|
|
levels[level] = levels.get(level, 0) + 1
|
|
if result.get("best_sku"):
|
|
skus.append(result["best_sku"])
|
|
report["methods"][method] = {
|
|
"p50_ms": round(statistics.median(latencies), 2) if latencies else 0.0,
|
|
"p95_ms": round(percentile(latencies, 95), 2),
|
|
"mean_ms": round(float(np.mean(latencies)), 2) if latencies else 0.0,
|
|
"levels": levels,
|
|
"accepted_skus": len(set(skus)),
|
|
}
|
|
return report
|
|
|
|
|
|
def bench_parallel(crops: List[Any], matcher: Any, method: str, pools: List[int], num_threads: List[int]) -> List[Dict[str, Any]]:
|
|
rows = []
|
|
for threads in num_threads:
|
|
previous = cv2.getNumThreads()
|
|
cv2.setNumThreads(threads)
|
|
for pool in pools:
|
|
started = time.perf_counter()
|
|
with ThreadPoolExecutor(max_workers=pool) as executor:
|
|
list(executor.map(lambda c: matcher.classify_crop(c, method=method), crops))
|
|
elapsed = time.perf_counter() - started
|
|
rows.append(
|
|
{
|
|
"cv2_threads": threads,
|
|
"pool": pool,
|
|
"crops": len(crops),
|
|
"seconds": round(elapsed, 2),
|
|
"crops_per_s": round(len(crops) / elapsed, 1),
|
|
}
|
|
)
|
|
cv2.setNumThreads(previous)
|
|
return rows
|
|
|
|
|
|
def compare_methods(image_path: str, methods: List[str], extra_images: Optional[List[str]] = None) -> Dict[str, Any]:
|
|
"""Agreement between two methods on the same crops (adoption gate)."""
|
|
from detector import ShelfProductDetector
|
|
from feature_matcher import ProductFeatureMatcher
|
|
|
|
if len(methods) != 2:
|
|
raise SystemExit("--compare needs exactly two method names")
|
|
reference, candidate = methods
|
|
detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
|
|
matcher = ProductFeatureMatcher(stock_dir="stock")
|
|
images = [image_path] + (extra_images or [])
|
|
accepted = 0
|
|
agree = 0
|
|
disagree_accepts = 0
|
|
total = 0
|
|
for path in images:
|
|
image = load_image(path)
|
|
for box in detector.detect(image).get("boxes", []):
|
|
x1, y1, x2, y2 = box["box"]
|
|
crop = image[y1:y2, x1:x2].copy()
|
|
if min(crop.shape[:2]) < 16:
|
|
continue
|
|
a = matcher.classify_crop(crop, method=reference)
|
|
b = matcher.classify_crop(crop, method=candidate)
|
|
total += 1
|
|
if a.get("identity_level") == "accepted":
|
|
accepted += 1
|
|
if a.get("best_sku") == b.get("best_sku"):
|
|
agree += 1
|
|
elif b.get("identity_level") == "accepted":
|
|
disagree_accepts += 1
|
|
return {
|
|
"reference": reference,
|
|
"candidate": candidate,
|
|
"crops": total,
|
|
"reference_accepted": accepted,
|
|
"agreement_on_accepted": round(agree / accepted, 4) if accepted else None,
|
|
"conflicting_accepts": disagree_accepts,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Dorayaki engine component benchmark")
|
|
parser.add_argument("--image", default="image.jpg")
|
|
parser.add_argument("--devices", nargs="*", default=["default", "cpu", "mps"])
|
|
parser.add_argument("--runs", type=int, default=15)
|
|
parser.add_argument("--imgsz", type=int, default=640)
|
|
parser.add_argument("--methods", nargs="*", default=["surf", "orb", "brisk", "hybrid"])
|
|
parser.add_argument("--pools", nargs="*", type=int, default=[1, 4, 8])
|
|
parser.add_argument("--cv2-threads", nargs="*", type=int, default=[0, 1])
|
|
parser.add_argument("--compare", nargs="*", default=None)
|
|
parser.add_argument("--compare-images", nargs="*", default=None)
|
|
parser.add_argument("--json", default=None)
|
|
args = parser.parse_args()
|
|
|
|
print("=" * 78)
|
|
print("DORAYAKI ENGINE BENCHMARK")
|
|
print("=" * 78)
|
|
|
|
from detector import ShelfProductDetector
|
|
|
|
started = time.perf_counter()
|
|
detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
|
|
load_ms = (time.perf_counter() - started) * 1000.0
|
|
print(f"model load: {load_ms:.0f} ms")
|
|
|
|
warm_image = load_image(args.image, width=1280)
|
|
started = time.perf_counter()
|
|
detector.detect(warm_image)
|
|
print(f"first predict (cold): {(time.perf_counter() - started) * 1000.0:.0f} ms")
|
|
started = time.perf_counter()
|
|
detector.detect(warm_image)
|
|
print(f"second predict (warm): {(time.perf_counter() - started) * 1000.0:.0f} ms")
|
|
|
|
print("\n--- detection latency ---")
|
|
detection = bench_detection(args.image, args.devices, args.runs, args.imgsz)
|
|
rows = []
|
|
for row in detection:
|
|
if "error" in row:
|
|
rows.append([row["device"], "-", "-", "-", f"ERROR: {row['error'][:40]}"])
|
|
continue
|
|
rows.append([row["device"], row["size"], row["shape"], row["boxes"],
|
|
f"{row['p50_ms']} / {row['p95_ms']}"])
|
|
print(format_table(rows, ["device", "size", "shape", "boxes", "p50/p95 ms"]))
|
|
|
|
# device agreement: do mps boxes match cpu boxes?
|
|
by_device = {}
|
|
for device in args.devices:
|
|
if device in ("default", "cpu"):
|
|
continue
|
|
try:
|
|
detector_d = ShelfProductDetector("runs/detect/train/weights/best.pt",
|
|
device=None if device == "default" else device)
|
|
except Exception:
|
|
continue
|
|
by_device[device] = detector_d.detect(warm_image).get("boxes", [])
|
|
cpu_boxes = None
|
|
try:
|
|
cpu_detector = ShelfProductDetector("runs/detect/train/weights/best.pt", device="cpu")
|
|
cpu_boxes = cpu_detector.detect(warm_image).get("boxes", [])
|
|
except Exception:
|
|
pass
|
|
if cpu_boxes is not None:
|
|
for device, boxes in by_device.items():
|
|
print(f"agreement cpu vs {device}: {json.dumps(detection_agreement(cpu_boxes, boxes))}")
|
|
|
|
print("\n--- classification latency (real crops from", args.image, ") ---")
|
|
crop_report = bench_crops(args.image, args.methods)
|
|
print(f"crops: {crop_report['crop_count']}")
|
|
rows = []
|
|
for method, stats in crop_report["methods"].items():
|
|
levels = stats["levels"]
|
|
rows.append([method, f"{stats['p50_ms']} / {stats['p95_ms']}",
|
|
f"acc={levels.get('accepted', 0)}",
|
|
f"amb={levels.get('ambiguous', 0)}",
|
|
f"unk={levels.get('unknown', 0)}",
|
|
f"rej={levels.get('rejected', 0)}",
|
|
f"skus={stats['accepted_skus']}"])
|
|
print(format_table(rows, ["method", "p50/p95 ms", "accepted", "ambiguous", "unknown", "rejected", "distinct skus"]))
|
|
|
|
print("\n--- parallel throughput (hybrid) ---")
|
|
from feature_matcher import ProductFeatureMatcher
|
|
|
|
matcher = ProductFeatureMatcher(stock_dir="stock")
|
|
image = load_image(args.image)
|
|
boxes = detector.detect(image).get("boxes", [])
|
|
crops = [image[b["box"][1]:b["box"][3], b["box"][0]:b["box"][2]].copy() for b in boxes]
|
|
rows = []
|
|
for row in bench_parallel(crops, matcher, "hybrid", args.pools, args.cv2_threads):
|
|
rows.append([row["cv2_threads"], row["pool"], row["crops"], row["seconds"], row["crops_per_s"]])
|
|
print(format_table(rows, ["cv2_threads", "pool", "crops", "seconds", "crops/s"]))
|
|
|
|
if args.compare:
|
|
print("\n--- method agreement ---")
|
|
comparison = compare_methods(args.image, args.compare, args.compare_images)
|
|
print(json.dumps(comparison, indent=2))
|
|
|
|
if args.json:
|
|
with open(args.json, "w") as handle:
|
|
json.dump(
|
|
{
|
|
"model_load_ms": round(load_ms, 1),
|
|
"detection": detection,
|
|
"classification": crop_report,
|
|
},
|
|
handle,
|
|
indent=2,
|
|
)
|
|
print(f"\nwrote {args.json}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|