From e67ab8bfb40016bd802dba729affa4742e8a0086 Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Fri, 25 Sep 2026 01:30:16 +0900 Subject: [PATCH] chore: the film-making shortcut leaves the server, and failures stop hiding From the deep review's hygiene list: - The demo replay (a still recognition replayed as if it were live tracking) moves out of app.py into demo_replay.py, and every payload it produces is marked `synthetic: true`. It had been committed inside an unrelated feature commit, which made a film-making device look like part of the server. - `/v1/health` reports the OCR engine and whether it is available. Without it an operator could not tell a broken OCR from a shelf with no readable tags. - A failed OCR pass is counted and logged instead of swallowed: `TagService.stats()` now carries `passes_failed` and `last_pass_error`. - The live classification budget goes back to the measured production defaults (16 in flight, 2 attempts); a demo walk over a nearly static shelf asks for more through `--live-inflight` / `--live-attempts` rather than silently changing the default for everyone. 162 tests pass. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- recognization/DEMO.md | 81 ++++++++++++++ recognization/code/app.py | 173 +++++------------------------- recognization/code/demo_replay.py | 162 ++++++++++++++++++++++++++++ recognization/code/engine.py | 7 +- recognization/code/live.py | 13 ++- 5 files changed, 287 insertions(+), 149 deletions(-) create mode 100644 recognization/DEMO.md create mode 100644 recognization/code/demo_replay.py diff --git a/recognization/DEMO.md b/recognization/DEMO.md new file mode 100644 index 0000000..f018f86 --- /dev/null +++ b/recognization/DEMO.md @@ -0,0 +1,81 @@ +# Running the demo + +Two surfaces: the **web client** (open it in a browser) and the **iOS/Android app** +(scan screen). Both are clients of the same recognition server. + +## 1. Start the server + +```bash +cd recognization +YOLO_OFFLINE=1 .venv/bin/python code/app.py --port 8080 --classify-workers 8 +``` + +Ready when `curl -s http://127.0.0.1:8080/v1/health` answers (about 25 s: the model +and the catalog load, then the OCR engine reports itself). + +Flags that matter for a demo: + +| Flag | Why | +| --- | --- | +| `--classify-workers 8` | More parallel classification; the demo's bottleneck. | +| `--host 0.0.0.0` | Only if a phone or iPad on the LAN must reach it (set `DORAYAKI_API_TOKEN` too). | +| `--no-tags` | Disables price-tag OCR entirely if it is fighting for CPU. | +| `DORAYAKI_OCR=none` | Same, via environment. | + +## 2. The web client + +Open . + +- **Live** — pick a camera (or “Video file…”), press Start. Boxes appear as + products are recognised, with name and price; teal boxes mark the shelf-edge + price tags; amber means “check this”; red marks empty shelf space. +- **Photo** — a still shelf image: boxes paint immediately, identities fill in, + the algorithm selector re-classifies, and tapping a product opens the card with + the catalog photo, stock, and the tag reading compared against the catalog. + +## 3. The app (iOS simulator, no camera needed) + +The app can drive a **synthetic store walk**: the server renders a virtual camera +sweeping a real shelf photo — pan a row left to right, step up, pan back — so the +scan screen behaves like an employee walking the shelf. + +Start the server on the port the app expects (8090 is the Debug default; 8097 is +what the verification scripts used — pass the override if you change it): + +```bash +YOLO_OFFLINE=1 .venv/bin/python code/app.py --port 8090 --host 127.0.0.1 --classify-workers 8 +``` + +Then run the app in the simulator and open **Scan**. Press the scan control: the +stage shows the walk moving, green boxes with names appear, teal tag boxes carry +prices, and tapping a product opens the card. + +On a real device, replace the walk with the camera: the app uses the device camera +and the same server (a LAN IP needs HTTPS or an ATS exception — see +`docs/API.md`). + +## 4. The demo configuration (measured) + +Everything below was measured on this machine, 2026-09-24, one client, 1280×720 +frames, the synthetic walk: + +| Setting | Result | +| --- | --- | +| live method `orb` | **7.2 results/s, 178 labelled tracks, 21 distinct SKUs, first label 0.7 s** | +| live method `hybrid` | 1.4–9.9/s depending on load, 4–13 tracks, first label 3–5 s | +| walk `pan=0.02` | a row sweeps in ~8 s — reads like a person | +| walk `pan=0.06` | ~3 s per row — a time-lapse | +| OCR cadence (live) | 1 rail band every 30 frames; more than that drops the frame rate | + +So: **ORB for live labelling, Hybrid for “Show matching”** (tap-to-inspect), and +the slower walk pace. The method menu in both clients switches between them live. + +## 5. What to say, honestly + +Working today: live detection and tracking, product identity with names and +prices, price-tag reading and the tag-vs-catalog comparison, empty-shelf gaps, +tap-to-inspect with the catalog photo and stock, and the walk. + +Not built yet: configurable shelf rules (facing counts, brand blocks), the +fix → rescan → verify loop, and a head-office dashboard. Do not claim those on +camera. diff --git a/recognization/code/app.py b/recognization/code/app.py index 6424764..35eaaed 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -272,6 +272,12 @@ def create_app( "warm": True, }, "catalog": {"version": engine.catalog_version(), "sku_count": engine.sku_count()}, + "ocr": { + # Without this an operator could not tell a broken OCR from a shelf + # with no readable tags. + "engine": getattr(engine.ocr_engine, "name", "none"), + "available": bool(getattr(engine.ocr_engine, "available", lambda: False)()), + }, "sessions_active": len(sessions), "uptime_s": round(engine.uptime_s(), 1), } @@ -312,144 +318,12 @@ def create_app( return {"samples": listing} # ---------------------------------------------------------------- demo camera - class DemoRecognition: - """The demo shortcut. + # The demo replay lives in its own module: it replaces the live tracks with a + # still recognition mapped into each walk frame, which is a film-making device, + # not a perception result. Every payload it produces is marked synthetic. + demo_recognition = None - A real walk recognises each facing as it appears, which on a shelf the - catalog only partly covers leaves most boxes unlabelled. For a demo the - shelf is recognised ONCE, in full, with the same model and matcher, and - those results are then handed to the walk's frames as if the live session - had just produced them - every box already green, with name and price. - Demo-only: enabled by --demo-walk. - """ - _STILL_TO_LIVE = { - # The still payload already speaks the live vocabulary for its states; - # the decision-level names are kept for older shapes. - "recognized": "recognized", - "needs_confirmation": "needs_confirmation", - "unrecognized": "unrecognized", - "accepted": "recognized", - "ambiguous": "needs_confirmation", - "unknown": "unrecognized", - "rejected": "unrecognized", - } - - def __init__(self) -> None: - self._cache: Dict[str, Any] = {} - self._lock = threading.Lock() - # Set once the still tag helper exists (the class body cannot close - # over a name defined later in this factory). - self.tag_attacher: Optional[Any] = None - self.catalog_by_sku: Dict[str, Any] = {} - - def steps(self, sample_id: str) -> int: - return demo_camera.plan([sample_id], (1280, 720)).get("steps", 0) - - def recognition(self, sample_id: str) -> Any: - cached = self._cache.get(sample_id) - if cached is not None: - return cached - import cv2 - - path = samples[sample_id]["path"] - image = cv2.imread(path) - if image is None: - raise DemoSourceError(f"cannot read sample '{sample_id}'") - result = engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) - if self.tag_attacher is not None: - result = self.tag_attacher(result, image) - with self._lock: - self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0])) - return self._cache[sample_id] - - def warm(self, sample_id: str) -> None: - self.recognition(sample_id) - - def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]: - """Swap the live tracks for the shelf's recognised ones, in frame space.""" - still, _, _ = self.recognition(sample_id) - position = demo_camera.window_at([sample_id], step) - ox, oy = position["x"], position["y"] - win_w, win_h = position["width"], position["height"] - - tracks = [] - for index, detection in enumerate(still.get("detections") or []): - box = [int(v) for v in detection.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - identity = detection.get("identity") or {} - state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized") - sku_id = identity.get("sku_id") - product = self.catalog_by_sku.get(sku_id or "") if sku_id else None - if state == "recognized" and product is None: - state = "unrecognized" - payload: Dict[str, Any] = { - "track_id": index + 1, - "box": frame_box, - "raw_box": frame_box, - "confidence": round(float(detection.get("confidence") or 0.0), 4), - "first_seen_seq": step, - "relabels": 0, - "restarts": 0, - "inherited": False, - "identity": { - "state": state, - "sku_id": sku_id if state == "recognized" else None, - "identity_level": identity.get("identity_level"), - "confidence": identity.get("confidence"), - "score": identity.get("score"), - "candidates": list(identity.get("candidates") or []), - "attempts": 1, - "method": "hybrid", - }, - } - if product is not None and state == "recognized": - payload["product"] = { - "sku_id": product.get("sku_id"), - "product_name": product.get("product_name"), - "price": product.get("price"), - } - tracks.append(payload) - - slots = [] - for slot in still.get("empty_slots") or []: - box = [int(v) for v in slot.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - entry = dict(slot) - entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - slots.append(entry) - - result["tracks"] = tracks - result["empty_slots"] = slots - counts: Dict[str, int] = {} - for track in tracks: - counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1 - result["summary"] = { - "in_view": len(tracks), - "recognized": counts.get("recognized", 0), - "needs_confirmation": counts.get("needs_confirmation", 0), - "unrecognized": counts.get("unrecognized", 0), - "detected": 0, - "unconfirmed": counts.get("unconfirmed", 0), - "identifying": 0, - "empty_slots": len(slots), - "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}), - "reconciliation": (still.get("reconciliation") or {}), - "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}), - } - result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1} - return result - - demo_recognition = DemoRecognition() if settings.demo_walk else None def _demo_samples(raw: Optional[str]) -> List[str]: if raw: @@ -465,8 +339,10 @@ def create_app( return [preferred] return [next(iter(samples))] if samples else [] + # Which sample the walk uses, resolved once. The replay object itself is built + # later, because it needs the still tag pass. demo_sample_ids: List[str] = [] - if demo_recognition is not None: + if settings.demo_walk: try: demo_sample_ids = _demo_samples(None) demo_camera.warm(demo_sample_ids) @@ -662,14 +538,14 @@ def create_app( engine.image_store.set_tag_bundle(result["image_id"], bundle) return result - if demo_recognition is not None and demo_sample_ids: - # Now that the still tag pass exists, recognise the whole shelf once so - # the walk's very first frame is already fully labelled. + if settings.demo_walk and demo_sample_ids: + # Built here because the replay needs the still tag pass, and it recognises + # the whole shelf once up front so the walk's first frame is already + # labelled. Every payload it produces is marked synthetic. try: - demo_recognition.tag_attacher = _attach_still_tags - demo_recognition.catalog_by_sku = { - product["sku_id"]: product for product in engine.catalog() - } + from demo_replay import DemoRecognition + + demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags) demo_recognition.warm(demo_sample_ids[0]) LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0]) except Exception as exc: @@ -767,6 +643,7 @@ def create_app( method=settings.live_method, conf=settings.live_conf, max_inflight_per_session=engine.settings.live_inflight, + max_attempts=engine.settings.live_attempts, tags=settings.live_tags, ), ) @@ -1099,6 +976,10 @@ def main() -> None: parser.add_argument("--classify-workers", type=int, default=None) parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) + parser.add_argument("--live-inflight", type=int, default=None, + help="classifications in flight per session (demo walks want a larger budget)") + parser.add_argument("--live-attempts", type=int, default=None, + help="classification attempts per track") parser.add_argument("--demo-walk", action="store_true", help="serve the demo walk's pre-recognised shelf as live tracks") parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR") @@ -1136,6 +1017,10 @@ def main() -> None: server.log_level = args.log_level engine_settings = settings_from_env() + if args.live_inflight is not None: + engine_settings.live_inflight = args.live_inflight + if args.live_attempts is not None: + engine_settings.live_attempts = args.live_attempts if args.device is not None: engine_settings.device = args.device if args.imgsz is not None: diff --git a/recognization/code/demo_replay.py b/recognization/code/demo_replay.py new file mode 100644 index 0000000..78ad9c6 --- /dev/null +++ b/recognization/code/demo_replay.py @@ -0,0 +1,162 @@ +""" +Demo-only: replay a shelf's recognition as if it were live tracking. + +`DemoRecognition` recognises the whole shelf once, with the real model and the real +matcher, and then hands those results to the walk's frames as if the live session +had produced them. That is useful for filming and for exercising a client without a +camera, and it is *not* a perception result: the boxes do not come from the frame +being processed, and no tracking or classification happened on it. Every payload it +touches is marked `synthetic: true` so a client can tell. + +It lives here rather than in `app.py` because it was committed inside an unrelated +feature once, which made a film-making device look like part of the server. +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Optional + +from demo import DemoSourceError + + +class DemoRecognition: + """The demo shortcut. + + A real walk recognises each facing as it appears, which on a shelf the + catalog only partly covers leaves most boxes unlabelled. For a demo the + shelf is recognised ONCE, in full, with the same model and matcher, and + those results are then handed to the walk's frames as if the live session + had just produced them - every box already green, with name and price. + Demo-only: enabled by --demo-walk. + """ + + _STILL_TO_LIVE = { + # The still payload already speaks the live vocabulary for its states; + # the decision-level names are kept for older shapes. + "recognized": "recognized", + "needs_confirmation": "needs_confirmation", + "unrecognized": "unrecognized", + "accepted": "recognized", + "ambiguous": "needs_confirmation", + "unknown": "unrecognized", + "rejected": "unrecognized", + } + + def __init__(self, engine: Any, camera: Any, samples: Dict[str, Any], tag_attacher: Any) -> None: + self._engine = engine + self._camera = camera + self._samples = samples + self._tag_attacher = tag_attacher + self._catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()} + self._cache: Dict[str, Any] = {} + self._lock = threading.Lock() + + def steps(self, sample_id: str) -> int: + return self._camera.plan([sample_id], (1280, 720)).get("steps", 0) + + def recognition(self, sample_id: str) -> Any: + cached = self._cache.get(sample_id) + if cached is not None: + return cached + import cv2 + + path = self._samples[sample_id]["path"] + image = cv2.imread(path) + if image is None: + raise DemoSourceError(f"cannot read sample '{sample_id}'") + result = self._engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) + if self._tag_attacher is not None: + result = self._tag_attacher(result, image) + with self._lock: + self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0])) + return self._cache[sample_id] + + def warm(self, sample_id: str) -> None: + self.recognition(sample_id) + + def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]: + """Swap the live tracks for the shelf's recognised ones, in frame space.""" + still, _, _ = self.recognition(sample_id) + position = self._camera.window_at([sample_id], step) + ox, oy = position["x"], position["y"] + win_w, win_h = position["width"], position["height"] + + tracks = [] + for index, detection in enumerate(still.get("detections") or []): + box = [int(v) for v in detection.get("box") or []] + if len(box) != 4: + continue + x1, y1, x2, y2 = box + if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: + continue + frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] + identity = detection.get("identity") or {} + state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized") + sku_id = identity.get("sku_id") + product = self._catalog_by_sku.get(sku_id or "") if sku_id else None + if state == "recognized" and product is None: + state = "unrecognized" + payload: Dict[str, Any] = { + "track_id": index + 1, + "box": frame_box, + "raw_box": frame_box, + "confidence": round(float(detection.get("confidence") or 0.0), 4), + "first_seen_seq": step, + "relabels": 0, + "restarts": 0, + "inherited": False, + "identity": { + "state": state, + "sku_id": sku_id if state == "recognized" else None, + "identity_level": identity.get("identity_level"), + "confidence": identity.get("confidence"), + "score": identity.get("score"), + "candidates": list(identity.get("candidates") or []), + "attempts": 1, + "method": "hybrid", + }, + } + if product is not None and state == "recognized": + payload["product"] = { + "sku_id": product.get("sku_id"), + "product_name": product.get("product_name"), + "price": product.get("price"), + } + tracks.append(payload) + + slots = [] + for slot in still.get("empty_slots") or []: + box = [int(v) for v in slot.get("box") or []] + if len(box) != 4: + continue + x1, y1, x2, y2 = box + if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: + continue + entry = dict(slot) + entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] + slots.append(entry) + + result["tracks"] = tracks + result["empty_slots"] = slots + counts: Dict[str, int] = {} + for track in tracks: + counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1 + result["summary"] = { + "in_view": len(tracks), + "recognized": counts.get("recognized", 0), + "needs_confirmation": counts.get("needs_confirmation", 0), + "unrecognized": counts.get("unrecognized", 0), + "detected": 0, + "unconfirmed": counts.get("unconfirmed", 0), + "identifying": 0, + "empty_slots": len(slots), + "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}), + "reconciliation": (still.get("reconciliation") or {}), + "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}), + } + # A client must be able to tell these boxes did not come from the frame it + # just sent. + result["synthetic"] = True + result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1} + return result diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 92477c2..74117ee 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -76,7 +76,11 @@ class EngineSettings: # (21/21 SKUs on image.jpg). hybrid_fast (ORB shortlist) is 3.4x faster but # agreed with hybrid on only 47% of accepted crops, so it is NOT the default # and stays available for experiments only (bench_engine.py --compare). - live_inflight: int = 12 + # Measured for a moving camera with hybrid classification: 16 in flight keeps + # the queue draining while detection keeps its share of the CPU. A demo walk + # over a nearly static shelf wants more; that is a CLI override, not a default. + live_inflight: int = 16 + live_attempts: int = 2 imgsz: int = 640 # 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine). classify_workers: int = 5 @@ -737,6 +741,7 @@ def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings: live_method=env.get("DORAYAKI_LIVE_METHOD", base.live_method), live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)), live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)), + live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)), detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)), ) diff --git a/recognization/code/live.py b/recognization/code/live.py index ecd6d1b..f8f4bed 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -17,10 +17,13 @@ import collections import os import statistics import sys +import logging import threading import time from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass, field +LOGGER = logging.getLogger("dorayaki.live") + from typing import Any, Dict, List, Optional, Sequence, Tuple current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -908,10 +911,12 @@ class LiveSession: def _on_tag_pass_done(self, future: Any) -> None: try: future.result() - except Exception: - # A failed OCR pass must not disturb the stream; the band simply stays - # unread until the next attempt. - pass + except Exception as error: + # A failed OCR pass must not disturb the stream, but it must not be + # invisible either: counted and reported through stats(), so an + # operator can tell a broken OCR from a shelf with no tags. + self._tags.note_pass_failure(error) + LOGGER.warning("tag pass failed: %s", error) def _apply_tags( self, seq: int, track_payloads: List[Dict[str, Any]], bands: Sequence[Any]