feat: a live path that uses the enrolled store, and a demo server a phone can reach

The live session scored crops with the matcher regardless of the configured identity
backend, so a live scan used the old mechanism while the still path used the enrolled
gallery - measured, the matcher accepts look-alikes and cannot say "unknown", which is
exactly what a real shelf needs it to do. score_crop now routes to the gallery, and
the decision uses a similarity scale (accept at 0.60 with a 0.04 margin) rather than
inlier-count thresholds that mean nothing on a cosine.

The synthetic walk is gone: demo.py, demo_replay.py, the /v1/demo endpoints and the
walk benchmark are deleted, and the app's frame source is the camera on a device (the
single-image replay stays for the simulator, which has no camera).

stock-demo merges both demo shelves (27 products, 62 views, trained head) so one
server recognises either image, and run-demo-server.sh starts it bound to 0.0.0.0 and
prints the exact URL to put in Config.xcconfig.

Verified over the LAN with 1280x960 frames: 40 frames in 5.3s, 9 products recognised
on demo image 1.

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

recognization/code/app.py

@@ -72,7 +72,6 @@ from engine import (  # noqa: E402
     identity_payload,
     settings_from_env,
 )
-from demo import DemoCamera, DemoSourceError  # noqa: E402
 from record import FrameRecorder  # noqa: E402
 from live import LiveSession, LiveSettings  # noqa: E402
 
@@ -100,7 +99,6 @@ class ServerSettings:
     live_method: str = "hybrid"
     live_conf: float = 0.1
     live_tags: bool = True
-    demo_walk: bool = False
     # A live session can be recorded so it becomes data to label (record.py).
     record_frames: Optional[str] = None
     record_every: int = 3
@@ -250,9 +248,10 @@ def create_app(
     )
 
     scans = scans if scans is not None else build_legacy_scans(engine)
+    # A live session can be recorded so a real scan becomes data to label later.
+    recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \
+        if settings.record_frames else None
     samples = {item["id"]: item for item in get_available_shelf_images("add")}
-    demo_camera = DemoCamera(samples)
-    demo_cameras: Dict[float, DemoCamera] = {}
     sessions: Dict[str, LiveSession] = {}
     sessions_lock = asyncio.Lock()
     app.state.engine = engine
@@ -323,110 +322,14 @@ def create_app(
         listing = []
         for item in samples.values():
             entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"}
-            try:  # size is a convenience for clients that plan a demo walk
-                width, height = demo_camera._size(item["id"])
-                entry["width"], entry["height"] = width, height
-            except DemoSourceError:
-                pass
+            import cv2
+
+            image = cv2.imread(item["path"])
+            if image is not None:
+                entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0])
             listing.append(entry)
         return {"samples": listing}
 
-    # ---------------------------------------------------------------- demo camera
-    # The demo replay lives in its own module: it replaces the live tracks with a
-    # still recognition mapped into each walk frame, which is a film-making device,
-    # not a perception result. Every payload it produces is marked synthetic.
-    demo_recognition = None
-    recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \
-        if settings.record_frames else None
-
-
-
-    def _demo_samples(raw: Optional[str]) -> List[str]:
-        if raw:
-            wanted = [part.strip() for part in raw.split(",") if part.strip()]
-            unknown = [part for part in wanted if part not in samples]
-            if unknown:
-                raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404)
-            return wanted
-        # main_shelf first: its products are the ones the catalog covers, so a
-        # demo walk recognises real items instead of unknown shelf neighbours.
-        for preferred in ("main_shelf", "image6"):
-            if preferred in samples:
-                return [preferred]
-        return [next(iter(samples))] if samples else []
-
-    # Which sample the walk uses, resolved once. The replay object itself is built
-    # later, because it needs the still tag pass.
-    demo_sample_ids: List[str] = []
-    if settings.demo_walk:
-        try:
-            demo_sample_ids = _demo_samples(None)
-            demo_camera.warm(demo_sample_ids)
-        except Exception as exc:
-            LOGGER.warning("demo walk unavailable: %s", exc)
-
-    def _demo_window(width: Optional[int], height: Optional[int]) -> tuple:
-        win_w = int(width or 960)
-        win_h = int(height or 540)
-        if not (160 <= win_w <= 2048 and 120 <= win_h <= 2048):
-            raise _error("BAD_REQUEST", "window must be between 160x120 and 2048x2048", 400)
-        return win_w, win_h
-
-    def _demo_camera(pan: Optional[float]) -> DemoCamera:
-        """One camera per pacing value (the walk geometry depends on it)."""
-        if pan is None:
-            return demo_camera
-        key = round(float(pan), 4)
-        camera = demo_cameras.get(key)
-        if camera is None:
-            camera = DemoCamera(samples, pan_overlap=key)
-            demo_cameras[key] = camera
-        return camera
-
-    @app.get("/v1/demo/plan", dependencies=[Depends(authorize)])
-    async def demo_plan(samples_param: Optional[str] = None, width: Optional[int] = None,
-                        height: Optional[int] = None, pan: Optional[float] = None) -> Dict[str, Any]:
-        """Geometry of the virtual walk (demo-only capture source)."""
-        try:
-            ids = _demo_samples(samples_param)
-            if not ids:
-                raise _error("NOT_FOUND", "no samples available", 404)
-            camera = _demo_camera(pan)
-            return {"status": "success", **camera.plan(ids, _demo_window(width, height))}
-        except DemoSourceError as exc:
-            raise _error("NOT_FOUND", str(exc), 404)
-
-    @app.get("/v1/demo/frame", dependencies=[Depends(authorize)])
-    async def demo_frame(step: int = 0, samples_param: Optional[str] = None,
-                         width: Optional[int] = None, height: Optional[int] = None,
-                         quality: int = 65, pan: Optional[float] = None) -> Response:
-        """One JPEG frame of the virtual walk: a store employee sweeping the shelf.
-
-        Demo-only. The client advances `step` (it wraps), so the walk's speed
-        follows whatever frame rate the client can sustain.
-        """
-        try:
-            ids = _demo_samples(samples_param)
-            if not ids:
-                raise _error("NOT_FOUND", "no samples available", 404)
-            payload, position = _demo_camera(pan).frame(
-                ids, step, _demo_window(width, height), max(30, min(95, int(quality)))
-            )
-        except DemoSourceError as exc:
-            raise _error("NOT_FOUND", str(exc), 404)
-        return Response(
-            content=payload,
-            media_type="image/jpeg",
-            headers={
-                "X-Demo-Sample": str(position["sample"]),
-                "X-Demo-Bay": str(position["bay_index"] + 1),
-                "X-Demo-Step": str(position["step"]),
-                "X-Demo-Steps": str(position["steps"]),
-                "X-Demo-X": str(position["x"]),
-                "X-Demo-Y": str(position["y"]),
-            },
-        )
-
     @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)])
     async def get_sample(sample_id: str) -> Response:
         item = samples.get(sample_id)
@@ -599,19 +502,6 @@ def create_app(
         engine.image_store.set_tag_bundle(result["image_id"], bundle)
         return result
 
-    if settings.demo_walk and demo_sample_ids:
-        # Built here because the replay needs the still tag pass, and it recognises
-        # the whole shelf once up front so the walk's first frame is already
-        # labelled. Every payload it produces is marked synthetic.
-        try:
-            from demo_replay import DemoRecognition
-
-            demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags)
-            demo_recognition.warm(demo_sample_ids[0])
-            LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0])
-        except Exception as exc:
-            LOGGER.warning("demo walk warm-up failed: %s", exc)
-
     @app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)])
     async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]:
         method = _require_method(body.method)
@@ -715,7 +605,6 @@ def create_app(
         await websocket.accept()
         send_lock = asyncio.Lock()
         loop = asyncio.get_running_loop()
-        demo_steps = [0]
 
         async def send(payload: Dict[str, Any]) -> None:
             async with send_lock:
@@ -755,21 +644,9 @@ def create_app(
                                 "message": "frame processing failed"})
                     continue
                 if recorder is not None:
-                    # Record what the client sent and what we saw in it, so the walk
-                    # can be labelled later instead of only measured on photographs.
+                    # Record what the client sent and what we saw in it, so a real
+                    # scan can be labelled later instead of only measured on photos.
                     recorder.write(session_id, frame_id, payload, result)
-                if demo_recognition is not None and demo_sample_ids:
-                    # The walk's frames are served from the shelf's full recognition,
-                    # so every product is already labelled with name and price.
-                    sample_id = demo_sample_ids[0]
-                    step = demo_steps[0]
-                    try:
-                        result = await loop.run_in_executor(
-                            None, demo_recognition.apply, result, sample_id, step
-                        )
-                        demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id))
-                    except Exception as exc:  # a demo failure must not kill the stream
-                        LOGGER.exception("demo walk frame failed: %s", exc)
                 await send(result)
 
         async def handle_inspect(message: Dict[str, Any]) -> None:
@@ -1044,15 +921,13 @@ def main() -> None:
     parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"],
                         help="which mechanism names products (default matcher)")
     parser.add_argument("--live-inflight", type=int, default=None,
-                        help="classifications in flight per session (demo walks want a larger budget)")
+                        help="classifications in flight per session")
     parser.add_argument("--live-attempts", type=int, default=None,
                         help="classification attempts per track")
     parser.add_argument("--record-frames", type=str, default=None,
                         help="write processed frames and their results under this directory")
     parser.add_argument("--record-every", type=int, default=None,
                         help="record one frame in every N (default 3)")
-    parser.add_argument("--demo-walk", action="store_true",
-                        help="serve the demo walk's pre-recognised shelf as live tracks")
     parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR")
     parser.add_argument("--max-sessions", type=int, default=None)
     parser.add_argument("--ssl-certfile", type=str, default=None)
@@ -1076,8 +951,6 @@ def main() -> None:
         server.live_method = args.live_method
     if args.live_conf is not None:
         server.live_conf = args.live_conf
-    if args.demo_walk:
-        server.demo_walk = True
     if args.no_tags:
         server.live_tags = False
     if args.ssl_certfile is not None:

recognization/code/bench_shelf_map.py

@@ -1,77 +0,0 @@
-"""
-What does a shelf map buy over track ids?
-
-Walks the demo camera once, feeds every frame through both the live session and a
-ShelfMap, and reports what each counts. Needs a running server on :8077.
-
-    STEPS=58 python code/bench_shelf_map.py
-"""
-import json, sys, time, urllib.request
-sys.path.insert(0, "code")
-import cv2, numpy as np
-from websockets.sync.client import connect
-from shelf_map import ShelfMap
-
-BASE = "http://127.0.0.1:8077"
-STEPS = int(__import__("os").environ.get("STEPS", "58"))   # one walk over image6 at pan 0.02
-
-def fetch(step):
-    url = f"{BASE}/v1/demo/frame?samples_param=image6&step={step}&width=1280&height=720&quality=70&pan=0.02"
-    return urllib.request.urlopen(url, timeout=10).read()
-
-shelf = ShelfMap()
-track_ids, frames, detections = set(), 0, 0
-started = time.perf_counter()
-with connect("ws://127.0.0.1:8077/v1/stream") as ws:
-    ws.recv(timeout=15)
-    for step in range(STEPS):
-        payload = fetch(step)
-        frame = cv2.imdecode(np.frombuffer(payload, np.uint8), cv2.IMREAD_COLOR)
-        ws.send(step.to_bytes(4, "big") + payload)
-        msg = json.loads(ws.recv(timeout=25))
-        if msg.get("type") != "result":
-            continue
-        frames += 1
-        tracks = msg["tracks"]
-        detections += len(tracks)
-        for track in tracks:
-            track_ids.add(track["track_id"])
-        shelf.update(step, frame, tracks)
-        time.sleep(0.05)
-elapsed = time.perf_counter() - started
-
-stats = shelf.stats()
-print(f"{frames} frames in {elapsed:.1f} s, {detections} detections")
-print(f"  track ids seen (what the live session counts): {len(track_ids)}")
-print(f"  facings on the shelf map (what is actually there): {stats['facings']}")
-print(f"  merges {stats['merges']}, registrations {stats['registrations']}, failures {stats['registration_failures']}")
-print(f"  median registration response {stats['median_response']}")
-print(f"  facings whose pooled identity agrees with itself >= 0.8: {stats['identity_consistent']}")
-
-facings = [f for f in shelf.facings() if f.observations >= 3]
-print(f"\nfacings seen at least 3 times: {len(facings)}")
-for facing in sorted(facings, key=lambda f: (f.shelf_box[1], f.shelf_box[0]))[:10]:
-    sku, agreement = facing.best_identity()
-    tag_sku, tag_agreement = facing.best_tag_identity()
-    print(f"  shelf x={facing.shelf_box[0]:>5} y={facing.shelf_box[1]:>5} obs={facing.observations:>2} "
-          f"tracks={len(facing.seen_track_ids):>2} identity={sku} ({agreement:.2f}) tag={tag_sku} ({tag_agreement:.2f})")
-
-# render the map so duplicates are visible rather than inferred
-import numpy as np
-facings = shelf.facings()
-xs = [f.shelf_box[0] for f in facings] + [f.shelf_box[2] for f in facings]
-ys = [f.shelf_box[1] for f in facings] + [f.shelf_box[3] for f in facings]
-x0, x1 = min(xs), max(xs)
-y0, y1 = min(ys), max(ys)
-scale = 0.6
-canvas = np.full((int((y1 - y0) * scale) + 20, int((x1 - x0) * scale) + 20, 3), 20, np.uint8)
-for facing in facings:
-    bx = [(facing.shelf_box[0] - x0) * scale + 10, (facing.shelf_box[1] - y0) * scale + 10,
-          (facing.shelf_box[2] - x0) * scale + 10, (facing.shelf_box[3] - y0) * scale + 10]
-    colour = (0, 200, 0) if facing.observations >= 5 else (0, 120, 255)
-    cv2.rectangle(canvas, (int(bx[0]), int(bx[1])), (int(bx[2]), int(bx[3])), colour, 1)
-cv2.imwrite("/tmp/dorayaki-verify/shelf-map.png", canvas)
-print(f"\nmap canvas {canvas.shape[1]}x{canvas.shape[0]} px covering shelf x {x0}..{x1}, y {y0}..{y1}")
-from collections import Counter
-print("observations per facing:", Counter(min(f.observations, 20) for f in facings).most_common(6))
-print("facings per y band (100 px):", Counter((f.shelf_box[1] - y0) // 100 for f in facings).most_common(8))

recognization/code/collect_shelf_images.py

@@ -36,22 +36,23 @@ MAX_RETRIES = 4
 # Shelf photos of the categories the catalog covers: drinks and dairy, rice
 # crackers and snacks, sauces and condiments.
 QUERIES = [
-    "supermarket shelf Japan",
-    "Japanese supermarket beverages shelf",
-    "Japanese convenience store drink shelf",
-    "milk shelf Japan",
-    "drink shelf Japan",
-    "Japanese supermarket snack shelf",
-    "rice crackers Japan shop",
-    "せんべい 売り場",
-    "飲料 棚 スーパー",
-    "牛乳 棚",
-    "スーパーマーケット 陳列",
-    "Japanese grocery store interior",
-    "Japanese supermarket dairy",
-    "菓子 棚",
-    "supermarket drinks Japan",
-    "Japanese supermarket soft drinks",
+    # the catalog's own products first: drinks, dairy, rice crackers
+    "野菜生活100",
+    "伊藤園 1日分の野菜",
+    "Japanese vegetable juice shelf",
+    "Japanese milk cartons supermarket",
+    "meiji oishii milk",
+    "せんべい 棚",
+    "Japanese rice cracker shop display",
+    "Japanese confectionery shelf",
+    "Japanese supermarket beverage aisle",
+    "Japanese convenience store refrigerator drinks",
+    "Japanese supermarket dairy case",
+    "スーパー 飲料 陳列",
+    "お菓子 売り場",
+    "Japanese snack aisle supermarket",
+    "Japanese supermarket shelves drinks bottles",
+    "Japanese supermarket interior shelves",
 ]
 
 PERMISSIVE = ("cc0", "cc-by", "cc by", "public domain", "pd-", "attribution")

recognization/code/demo.py

@@ -1,254 +0,0 @@
-"""
-Synthetic camera for demos: a store employee walking a shelf.
-
-A simulator has no camera, and a still image makes a poor demo - nothing moves, so
-nothing new is ever recognised. This module turns the demo shelf photos into a
-virtual walk: a window pans left to right along a row, steps up to the next row,
-and pans back the other way (serpentine, ascending), exactly the way a person
-sweeps a shelf with a tablet.
-
-The client drives only `step`; all geometry lives here, so the web client, the
-phone app, and the reference client see the same walk. `step` wraps, so an
-unattended demo loops forever.
-
-Demo-only: this is a capture source, not a perception feature. `/v1/demo/*` is
-documented as such.
-"""
-
-from __future__ import annotations
-
-import math
-import os
-import threading
-import time
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-DEFAULT_WINDOW = (960, 540)
-DEFAULT_QUALITY = 65
-# How far the window advances per step, as a fraction of its own size. This is
-# the demo's most important number: advance too fast and a product is in view for
-# one or two frames, so the classifier never gets a chance and nothing is ever
-# labelled (measured: 0.22 gave 9 labelled tracks in 20 s; the shelf swept past
-# faster than a person could read it). 0.06 keeps a facing in view for roughly a
-# second at demo frame rates - a deliberate, human sweep.
-PAN_OVERLAP = 0.06
-ROW_OVERLAP = 0.25
-# Vertical steps spent moving between rows (the "look up, then keep going" beat).
-TRANSITION_STEPS = 6
-
-
-class DemoSourceError(ValueError):
-    """Unknown sample or an impossible window."""
-
-
-@dataclass
-class Bay:
-    """One shelf photo in the walk, with its geometry."""
-
-    sample_id: str
-    path: str
-    width: int
-    height: int
-    steps: int
-
-
-class DemoCamera:
-    """Plans and renders the virtual walk over one or more shelf photos."""
-
-    def __init__(self, samples: Dict[str, Dict[str, str]], window: Tuple[int, int] = DEFAULT_WINDOW,
-                 pan_overlap: float = PAN_OVERLAP):
-        self._samples = samples
-        self._window = window
-        # How far the window advances per step. The demo's pacing knob: a smaller
-        # value is a slower, more deliberate sweep (more steps, a facing in view
-        # for longer). Clamped so a client cannot ask for a frozen or teleporting
-        # camera.
-        self.pan_overlap = max(0.005, min(0.5, float(pan_overlap)))
-        self._lock = threading.Lock()
-        self._images: Dict[str, Any] = {}
-        self._sizes: Dict[str, Tuple[int, int]] = {}
-        self._plans: Dict[Tuple[str, ...], List[Bay]] = {}
-
-    # ------------------------------------------------------------------ planning
-    def _size(self, sample_id: str) -> Tuple[int, int]:
-        cached = self._sizes.get(sample_id)
-        if cached is not None:
-            return cached
-        item = self._samples.get(sample_id)
-        if item is None or not os.path.exists(item["path"]):
-            raise DemoSourceError(f"unknown sample '{sample_id}'")
-        import cv2
-
-        image = cv2.imread(item["path"])
-        if image is None:
-            raise DemoSourceError(f"cannot read sample '{sample_id}'")
-        with self._lock:
-            self._images[sample_id] = image
-            self._sizes[sample_id] = (int(image.shape[1]), int(image.shape[0]))
-        return self._sizes[sample_id]
-
-    def _image(self, sample_id: str) -> Any:
-        if sample_id not in self._images:
-            self._size(sample_id)
-        return self._images[sample_id]
-
-    def steps_for(self, sample_id: str, window: Tuple[int, int]) -> int:
-        width, height = self._size(sample_id)
-        win_w, win_h = window
-        bands = self._bands(height, win_h)
-        per_band = self._pan_steps(width, win_w)
-        # Every band pans, and every band after the first needs the step-up.
-        return bands * per_band + max(0, bands - 1) * TRANSITION_STEPS
-
-    def _bands(self, image_height: int, window_height: int) -> int:
-        if window_height >= image_height:
-            return 1
-        stride = max(1, int(window_height * (1.0 - ROW_OVERLAP)))
-        return max(1, int(math.ceil((image_height - window_height) / stride)) + 1)
-
-    def _pan_steps(self, image_width: int, window_width: int) -> int:
-        if window_width >= image_width:
-            return 1
-        stride = max(1, int(window_width * self.pan_overlap))
-        return max(1, int(math.ceil((image_width - window_width) / stride)) + 1)
-
-    def plan(self, sample_ids: Sequence[str], window: Optional[Tuple[int, int]] = None) -> Dict[str, Any]:
-        window = window or self._window
-        key = (tuple(sample_ids), window)
-        bays = self._plans.get(key)
-        if bays is None:
-            bays = []
-            for sample_id in sample_ids:
-                width, height = self._size(sample_id)
-                bays.append(
-                    Bay(
-                        sample_id=sample_id,
-                        path=self._samples[sample_id]["path"],
-                        width=width,
-                        height=height,
-                        steps=self.steps_for(sample_id, window),
-                    )
-                )
-            self._plans[key] = bays
-        total = sum(bay.steps for bay in bays)
-        return {
-            "samples": [bay.sample_id for bay in bays],
-            "window": {"width": window[0], "height": window[1]},
-            "bays": [
-                {"sample": bay.sample_id, "width": bay.width, "height": bay.height,
-                 "steps": bay.steps, "first_step": sum(b.steps for b in bays[:index])}
-                for index, bay in enumerate(bays)
-            ],
-            "steps": total,
-            "row_overlap": ROW_OVERLAP,
-            "pan_overlap": self.pan_overlap,
-        }
-
-    # ------------------------------------------------------------------ rendering
-    def window_at(
-        self, sample_ids: Sequence[str], step: int, window: Optional[Tuple[int, int]] = None
-    ) -> Dict[str, Any]:
-        """Where the virtual camera is at this step (wraps around the walk)."""
-        window = window or self._window
-        plan = self.plan(sample_ids, window)
-        total = plan["steps"]
-        step = int(step) % max(1, total)
-        offset = 0
-        for index, bay in enumerate(plan["bays"]):
-            if step < offset + bay["steps"]:
-                local = step - offset
-                x, y = self._position(local, bay["width"], bay["height"], window)
-                return {
-                    "sample": bay["sample"],
-                    "bay_index": index,
-                    "step": step,
-                    "steps": total,
-                    "local_step": local,
-                    "bay_steps": bay["steps"],
-                    "x": x,
-                    "y": y,
-                    "width": min(window[0], bay["width"]),
-                    "height": min(window[1], bay["height"]),
-                }
-            offset += bay["steps"]
-        raise DemoSourceError("step outside the walk")
-
-    def _position(self, local_step: int, image_width: int, image_height: int, window: Tuple[int, int]) -> Tuple[int, int]:
-        """Serpentine: pan a row, step up, pan back."""
-        win_w = min(window[0], image_width)
-        win_h = min(window[1], image_height)
-        bands = self._bands(image_height, win_h)
-        per_band = self._pan_steps(image_width, win_w)
-        cycle = per_band + TRANSITION_STEPS
-        band = min(bands - 1, local_step // cycle)
-        within = local_step - band * cycle
-        stride_y = 0 if bands == 1 else max(1, int(win_h * (1.0 - ROW_OVERLAP)))
-        # Ascending: band 0 is the bottom of the shelf.
-        y_bottom = min(image_height - win_h, band * stride_y)
-        y = max(0, image_height - win_h - y_bottom)
-        stride_x = max(1, int(win_w * self.pan_overlap))
-        max_x = max(0, image_width - win_w)
-        if within < per_band:
-            pan = within
-            x = min(max_x, pan * stride_x)
-        else:
-            # The step-up beat: hold x at the end of the row and climb.
-            transition = min(TRANSITION_STEPS, max(1, within - per_band + 1))
-            pan = per_band - 1
-            x = min(max_x, pan * stride_x)
-            next_band = min(bands - 1, band + 1)
-            next_y_bottom = min(image_height - win_h, next_band * stride_y)
-            ratio = transition / float(TRANSITION_STEPS)
-            y = int(round(y - (next_y_bottom - y_bottom) * ratio))
-        if band % 2 == 1 and within < per_band:
-            # Odd bands pan back the other way, like a person reversing direction.
-            x = max(0, max_x - x)
-        return int(max(0, min(max_x, x))), int(max(0, min(image_height - win_h, y)))
-
-    def frame(
-        self,
-        sample_ids: Sequence[str],
-        step: int,
-        window: Optional[Tuple[int, int]] = None,
-        quality: int = DEFAULT_QUALITY,
-    ) -> Tuple[bytes, Dict[str, Any]]:
-        """The JPEG frame the virtual camera sees at this step, plus its geometry."""
-        import cv2
-
-        position = self.window_at(sample_ids, step, window)
-        image = self._image(position["sample"])
-        x, y = position["x"], position["y"]
-        w, h = position["width"], position["height"]
-        crop = image[y:y + h, x:x + w]
-        if crop.size == 0:
-            raise DemoSourceError("empty crop")
-        ok, buffer = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, int(quality)])
-        if not ok:
-            raise DemoSourceError("JPEG encode failed")
-        return bytes(buffer), position
-
-    def warm(self, sample_ids: Sequence[str]) -> None:
-        """Decode the bays up front so the first demo frame is not the slow one."""
-        for sample_id in sample_ids:
-            self._size(sample_id)
-
-
-if __name__ == "__main__":  # smoke: python code/demo.py [sample ...]
-    import sys
-
-    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-    from app import get_available_shelf_images
-
-    samples = {item["id"]: item for item in get_available_shelf_images("add")}
-    camera = DemoCamera(samples)
-    ids = sys.argv[1:] or ["image6"]
-    plan = camera.plan(ids)
-    print(f"walk over {plan['samples']}: {plan['steps']} steps, window {plan['window']}")
-    started = time.perf_counter()
-    for step in range(min(plan["steps"], 24)):
-        payload, position = camera.frame(ids, step)
-        print(f"  step {step:3d} {position['sample']} x={position['x']:4d} y={position['y']:4d} "
-              f"{len(payload) // 1024} KB")
-    elapsed = (time.perf_counter() - started) * 1000 / max(1, min(plan["steps"], 24))
-    print(f"{elapsed:.1f} ms per frame")

recognization/code/demo_replay.py

@@ -1,162 +0,0 @@
-"""
-Demo-only: replay a shelf's recognition as if it were live tracking.
-
-`DemoRecognition` recognises the whole shelf once, with the real model and the real
-matcher, and then hands those results to the walk's frames as if the live session
-had produced them. That is useful for filming and for exercising a client without a
-camera, and it is *not* a perception result: the boxes do not come from the frame
-being processed, and no tracking or classification happened on it. Every payload it
-touches is marked `synthetic: true` so a client can tell.
-
-It lives here rather than in `app.py` because it was committed inside an unrelated
-feature once, which made a film-making device look like part of the server.
-"""
-
-from __future__ import annotations
-
-import threading
-from typing import Any, Dict, List, Optional
-
-from demo import DemoSourceError
-
-
-class DemoRecognition:
-    """The demo shortcut.
-
-    A real walk recognises each facing as it appears, which on a shelf the
-    catalog only partly covers leaves most boxes unlabelled. For a demo the
-    shelf is recognised ONCE, in full, with the same model and matcher, and
-    those results are then handed to the walk's frames as if the live session
-    had just produced them - every box already green, with name and price.
-    Demo-only: enabled by --demo-walk.
-    """
-
-    _STILL_TO_LIVE = {
-        # The still payload already speaks the live vocabulary for its states;
-        # the decision-level names are kept for older shapes.
-        "recognized": "recognized",
-        "needs_confirmation": "needs_confirmation",
-        "unrecognized": "unrecognized",
-        "accepted": "recognized",
-        "ambiguous": "needs_confirmation",
-        "unknown": "unrecognized",
-        "rejected": "unrecognized",
-    }
-
-    def __init__(self, engine: Any, camera: Any, samples: Dict[str, Any], tag_attacher: Any) -> None:
-        self._engine = engine
-        self._camera = camera
-        self._samples = samples
-        self._tag_attacher = tag_attacher
-        self._catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()}
-        self._cache: Dict[str, Any] = {}
-        self._lock = threading.Lock()
-
-    def steps(self, sample_id: str) -> int:
-        return self._camera.plan([sample_id], (1280, 720)).get("steps", 0)
-
-    def recognition(self, sample_id: str) -> Any:
-        cached = self._cache.get(sample_id)
-        if cached is not None:
-            return cached
-        import cv2
-
-        path = self._samples[sample_id]["path"]
-        image = cv2.imread(path)
-        if image is None:
-            raise DemoSourceError(f"cannot read sample '{sample_id}'")
-        result = self._engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True)
-        if self._tag_attacher is not None:
-            result = self._tag_attacher(result, image)
-        with self._lock:
-            self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0]))
-        return self._cache[sample_id]
-
-    def warm(self, sample_id: str) -> None:
-        self.recognition(sample_id)
-
-    def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]:
-        """Swap the live tracks for the shelf's recognised ones, in frame space."""
-        still, _, _ = self.recognition(sample_id)
-        position = self._camera.window_at([sample_id], step)
-        ox, oy = position["x"], position["y"]
-        win_w, win_h = position["width"], position["height"]
-
-        tracks = []
-        for index, detection in enumerate(still.get("detections") or []):
-            box = [int(v) for v in detection.get("box") or []]
-            if len(box) != 4:
-                continue
-            x1, y1, x2, y2 = box
-            if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h:
-                continue
-            frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy]
-            identity = detection.get("identity") or {}
-            state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized")
-            sku_id = identity.get("sku_id")
-            product = self._catalog_by_sku.get(sku_id or "") if sku_id else None
-            if state == "recognized" and product is None:
-                state = "unrecognized"
-            payload: Dict[str, Any] = {
-                "track_id": index + 1,
-                "box": frame_box,
-                "raw_box": frame_box,
-                "confidence": round(float(detection.get("confidence") or 0.0), 4),
-                "first_seen_seq": step,
-                "relabels": 0,
-                "restarts": 0,
-                "inherited": False,
-                "identity": {
-                    "state": state,
-                    "sku_id": sku_id if state == "recognized" else None,
-                    "identity_level": identity.get("identity_level"),
-                    "confidence": identity.get("confidence"),
-                    "score": identity.get("score"),
-                    "candidates": list(identity.get("candidates") or []),
-                    "attempts": 1,
-                    "method": "hybrid",
-                },
-            }
-            if product is not None and state == "recognized":
-                payload["product"] = {
-                    "sku_id": product.get("sku_id"),
-                    "product_name": product.get("product_name"),
-                    "price": product.get("price"),
-                }
-            tracks.append(payload)
-
-        slots = []
-        for slot in still.get("empty_slots") or []:
-            box = [int(v) for v in slot.get("box") or []]
-            if len(box) != 4:
-                continue
-            x1, y1, x2, y2 = box
-            if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h:
-                continue
-            entry = dict(slot)
-            entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy]
-            slots.append(entry)
-
-        result["tracks"] = tracks
-        result["empty_slots"] = slots
-        counts: Dict[str, int] = {}
-        for track in tracks:
-            counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1
-        result["summary"] = {
-            "in_view": len(tracks),
-            "recognized": counts.get("recognized", 0),
-            "needs_confirmation": counts.get("needs_confirmation", 0),
-            "unrecognized": counts.get("unrecognized", 0),
-            "detected": 0,
-            "unconfirmed": counts.get("unconfirmed", 0),
-            "identifying": 0,
-            "empty_slots": len(slots),
-            "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}),
-            "reconciliation": (still.get("reconciliation") or {}),
-            "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}),
-        }
-        # A client must be able to tell these boxes did not come from the frame it
-        # just sent.
-        result["synthetic"] = True
-        result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1}
-        return result

recognization/code/engine.py

@@ -411,7 +411,29 @@ class RecognitionEngine:
         return slots
 
     # ------------------------------------------------------------------ classification
+    @property
+    def identity_scale(self) -> str:
+        """What the scores from `score_crop` mean: inlier counts or cosine similarity."""
+        return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier"
+
     def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
+        """Score one crop against the catalog, in whatever scale this engine uses.
+
+        The live session calls this per track. With the gallery backend it must go
+        through the gallery, or a live scan would use the matcher while the still path
+        used the enrolled store - measured: the matcher accepts look-alikes and cannot
+        say "unknown", which is exactly what a real shelf needs it to do.
+        """
+        if self.identity_scale == "cosine":
+            gallery = self.identity_gallery()
+            if gallery:
+                match = gallery.match(crop)
+                if match.rejected or not match.sku_id:
+                    return {}
+                scores = {match.sku_id: match.similarity}
+                if match.runner_up:
+                    scores[match.runner_up] = match.runner_up_similarity
+                return {str(sku): float(score) for sku, score in scores.items()}
         method = (method or self.settings.live_method).lower()
         if method not in CLASSIFY_METHODS:
             raise UnknownMethod(method)

recognization/code/identity.py

@@ -25,6 +25,13 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float:
     return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4)
 
 
+# The gallery's scale is a cosine similarity, where same-product pairs sit above 0.6
+# and different products below 0.5 - not an inlier count, which has no such meaning.
+COSINE_ACCEPT = 0.60
+COSINE_AMBIGUOUS = 0.50
+COSINE_MARGIN = 0.04
+
+
 def decide_identity(
     scores: Dict[str, float],
     *,
@@ -33,6 +40,7 @@ def decide_identity(
     min_confidence: float = DEFAULT_MIN_CONFIDENCE,
     details: Optional[Dict[str, Any]] = None,
     method: str = "hybrid",
+    scale: str = "inlier",
 ) -> Dict[str, Any]:
     """Rank catalog scores into accepted / ambiguous / unknown / rejected.
 
@@ -56,6 +64,30 @@ def decide_identity(
     sorted_candidates: List[Tuple[str, float]] = sorted(
         scores.items(), key=lambda x: x[1], reverse=True
     )
+    if scale == "cosine":
+        # Thresholds that mean something on a similarity scale.
+        top_sku, top_score = sorted_candidates[0]
+        runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0
+        margin = top_score - runner_score
+        if top_score >= COSINE_ACCEPT and margin >= COSINE_MARGIN:
+            level = IDENTITY_ACCEPTED
+        elif top_score >= COSINE_AMBIGUOUS:
+            level = IDENTITY_AMBIGUOUS
+        else:
+            level = IDENTITY_REJECTED
+        return {
+            "status": level,
+            "method": method,
+            "best_sku": top_sku if level == IDENTITY_ACCEPTED else None,
+            "score": float(top_score),
+            "confidence": float(top_score),
+            "identity_level": level,
+            "ranked_candidates": [
+                {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3]
+            ],
+            "details": details or {},
+            "message": None,
+        }
     top_sku, top_score = sorted_candidates[0]
     runner_up_sku, runner_up_score = (
         sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0)

recognization/code/live.py

@@ -643,7 +643,9 @@ class LiveSession:
             mean_scores = {
                 sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items()
             }
-            decision = decide_identity(mean_scores, method=method)
+            decision = decide_identity(
+                mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier")
+            )
             track.candidates = [
                 {"sku_id": row.get("sku_id"), "score": row.get("score")}
                 for row in (decision.get("ranked_candidates") or [])[:3]

recognization/code/test_demo.py

@@ -1,119 +0,0 @@
-"""
-Contract tests for the demo camera (the synthetic store walk).
-
-The walk is what a demo shows when there is no camera, so its geometry has to be
-predictable: inside the image, moving, serpentine, and wrapping forever.
-"""
-
-from __future__ import annotations
-
-import os
-import sys
-import unittest
-
-current_dir = os.path.dirname(os.path.abspath(__file__))
-project_root = os.path.dirname(current_dir)
-if os.path.exists(os.path.join(project_root, "stock")):
-    os.chdir(project_root)
-if current_dir not in sys.path:
-    sys.path.insert(0, current_dir)
-
-import numpy as np  # noqa: E402
-
-from demo import DemoCamera, DemoSourceError  # noqa: E402
-
-
-class FakeImageCamera(DemoCamera):
-    """A camera over a synthetic image, so the geometry is testable without files."""
-
-    def __init__(self, width=1500, height=2000, window=(960, 540)):
-        super().__init__({"synthetic": {"id": "synthetic", "path": "synthetic", "label": "synthetic"}}, window)
-        self._image_cache = np.zeros((height, width, 3), dtype="uint8")
-        self._size_cache = (width, height)
-
-    def _size(self, sample_id):  # noqa: D102
-        if sample_id != "synthetic":
-            raise DemoSourceError(f"unknown sample '{sample_id}'")
-        return self._size_cache
-
-    def _image(self, sample_id):  # noqa: D102
-        return self._image_cache
-
-
-class WalkGeometryTests(unittest.TestCase):
-    def setUp(self):
-        self.camera = FakeImageCamera()
-        self.plan = self.camera.plan(["synthetic"])
-
-    def test_plan_is_deterministic_and_covers_the_shelf(self):
-        self.assertEqual(self.plan["steps"], self.camera.plan(["synthetic"])["steps"])
-        self.assertGreater(self.plan["steps"], 20, "a shelf walk is not two steps long")
-        self.assertEqual(self.plan["bays"][0]["sample"], "synthetic")
-
-    def test_window_never_leaves_the_image(self):
-        width, height = 1500, 2000
-        for step in range(self.plan["steps"]):
-            position = self.camera.window_at(["synthetic"], step)
-            self.assertGreaterEqual(position["x"], 0)
-            self.assertGreaterEqual(position["y"], 0)
-            self.assertLessEqual(position["x"] + position["width"], width)
-            self.assertLessEqual(position["y"] + position["height"], height)
-
-    def test_the_camera_actually_moves(self):
-        xs = [self.camera.window_at(["synthetic"], step)["x"] for step in range(6)]
-        ys = [self.camera.window_at(["synthetic"], step)["y"] for step in range(self.plan["steps"])]
-        self.assertGreater(len(set(xs)), 1, "the window must pan, not sit still")
-        self.assertGreater(len(set(ys)), 1, "the walk must climb rows")
-
-    def test_it_ascends_the_shelf(self):
-        first = self.camera.window_at(["synthetic"], 0)["y"]
-        last_band = self.plan["steps"] - 1
-        self.assertGreater(first, self.camera.window_at(["synthetic"], last_band)["y"],
-                           "the walk starts at the bottom row and climbs")
-
-    def test_step_wraps(self):
-        first = self.camera.window_at(["synthetic"], 0)
-        wrapped = self.camera.window_at(["synthetic"], self.plan["steps"])
-        self.assertEqual((first["x"], first["y"]), (wrapped["x"], wrapped["y"]))
-
-    def test_frames_are_deterministic_jpegs(self):
-        payload, position = self.camera.frame(["synthetic"], 3, quality=70)
-        again, _ = self.camera.frame(["synthetic"], 3, quality=70)
-        self.assertTrue(payload.startswith(b"\xff\xd8"), "JPEG magic")
-        self.assertEqual(payload, again, "the same step must render the same frame")
-        self.assertIn("sample", position)
-
-    def test_unknown_sample_is_rejected(self):
-        with self.assertRaises(DemoSourceError):
-            self.camera.plan(["nope"])
-
-    def test_window_larger_than_the_image_is_clamped(self):
-        camera = FakeImageCamera(width=800, height=600, window=(960, 540))
-        position = camera.window_at(["synthetic"], 0)
-        self.assertLessEqual(position["width"], 800)
-        self.assertLessEqual(position["height"], 600)
-        self.assertEqual(position["x"], 0)
-        # The walk starts at the bottom of the shelf, so the window is bottom-aligned.
-        self.assertEqual(position["y"], 600 - position["height"])
-
-
-class RealShelfTests(unittest.TestCase):
-    """One check against the actual demo asset, skipped when it is absent."""
-
-    def test_real_sample_walks(self):
-        from app import get_available_shelf_images
-
-        samples = {item["id"]: item for item in get_available_shelf_images("add")}
-        if "image6" not in samples:
-            self.skipTest("demo sample image6 not present")
-        camera = DemoCamera(samples)
-        plan = camera.plan(["image6"], (1280, 720))
-        self.assertGreater(plan["steps"], 20, "a 1500x2000 shelf gives a multi-row walk")
-        payload, position = camera.frame(["image6"], 5, (1280, 720))
-        self.assertGreater(len(payload), 10_000)
-        self.assertLessEqual(position["x"] + position["width"], plan["bays"][0]["width"])
-        self.assertLessEqual(position["y"] + position["height"], plan["bays"][0]["height"])
-
-
-if __name__ == "__main__":
-    unittest.main()

recognization/run-demo-server.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# The demo server: one command, bound to the network so a phone can reach it.
+#
+#   ./run-demo-server.sh              # both demo shelves enrolled (stock-demo)
+#   ./run-demo-server.sh stock-milk150
+#
+# The iPhone must be on the same Wi-Fi as this Mac, and the app must point at this
+# Mac's address - Config.xcconfig's APP_RECOGNITION_BASE_URL, which the app reads at
+# build time. The script prints the exact line to use.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+STOCK="${1:-stock-demo}"
+PORT="${PORT:-8099}"
+
+if [ ! -d "$STOCK" ]; then
+  echo "no such store: $STOCK" >&2
+  exit 1
+fi
+
+IP="$(ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null || true)"
+if [ -z "$IP" ]; then
+  echo "this Mac has no LAN address; the phone cannot reach it" >&2
+  exit 1
+fi
+
+cat <<EOF
+Dorayaki demo server
+  store      $STOCK  ($(ls "$STOCK" | grep -c -E '\.(jpeg|jpg|png)$') enrolled views)
+  listening  http://0.0.0.0:$PORT
+  phone URL  http://$IP:$PORT
+
+Put this in iosApp/Configuration/Config.xcconfig and rebuild the app:
+
+  APP_RECOGNITION_BASE_URL = http:/\$()/$IP:$PORT
+
+Health check from another device: http://$IP:$PORT/v1/health
+EOF
+
+# A phone on the same Wi-Fi reaches 0.0.0.0; localhost-only would not.
+exec env YOLO_OFFLINE=1 DORAYAKI_STOCK_DIR="$STOCK" \
+  .venv/bin/python -u code/app.py \
+  --port "$PORT" --host 0.0.0.0 --identity gallery \
+  --live-conf 0.25 --classify-workers 8

recognization/stock-demo/stock.csv

@@ -0,0 +1,28 @@
+SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price
+sku_m01,小岩井 低脂肪牛乳,2026-09-25,2026-10-20,24,8,16,'',186
+sku_m02,荻山酪農 カフェ・オ・レ,2026-09-25,2026-10-20,24,8,16,'',397
+sku_m03,丹那 3.6牛乳,2026-09-25,2026-10-20,24,8,16,'',214
+sku_m04,木次 ミルクコーヒー,2026-09-25,2026-10-20,24,8,16,'',195
+sku_m05,木次 パスチャライズ牛乳,2026-09-25,2026-10-20,24,8,16,'',240
+sku_m06,キスキ ブラウン・スイス牛乳,2026-09-25,2026-10-20,24,8,16,'',254
+sku_m07,丹那 飲んでやさしいヨーグルト,2026-09-25,2026-10-20,24,8,16,'',225
+sku_m08,農場3.7牛乳 特選,2026-09-25,2026-10-20,24,8,16,'',216
+sku_m09,MINOR FIGURES OAT,2026-09-25,2026-10-20,24,8,16,'',300
+sku_m10,北海道 低脂肪乳,2026-09-25,2026-10-20,24,8,16,'',214
+sku_m11,タカナシ 低温殺菌牛乳,2026-09-25,2026-10-20,24,8,16,'',396
+sku_m12,RUDE HEALTH Oat Organic,2026-09-25,2026-10-20,24,8,16,'',400
+sku_m13,小岩井 無脂肪牛乳,2026-09-25,2026-10-20,24,8,16,'',178
+sku_m14,きすき ノンホモ牛乳,2026-09-25,2026-10-20,24,8,16,'',198
+sku_k01,明治 おいしい牛乳,2026-09-25,2026-10-15,48,20,28,'',198
+sku_k02,ゆうパック 優良牛乳,2026-09-25,2026-10-15,48,20,28,'',130
+sku_k03,酪農家限定 低温殺菌牛乳,2026-09-25,2026-10-15,48,20,28,'',248
+sku_k04,北海道牛乳,2026-09-25,2026-10-15,48,20,28,'',178
+sku_k05,MEGMILK,2026-09-25,2026-10-15,48,20,28,'',158
+sku_k06,毎日牛乳,2026-09-25,2026-10-15,48,20,28,'',168
+sku_k07,タカナシ 低温殺菌牛乳,2026-09-25,2026-10-15,48,20,28,'',258
+sku_k08,タカナシ 殺菌低脂肪牛乳,2026-09-25,2026-10-15,48,20,28,'',198
+sku_k09,毎日骨太,2026-09-25,2026-10-15,48,20,28,'',148
+sku_k10,カルシウム入り低脂肪乳,2026-09-25,2026-10-15,48,20,28,'',145
+sku_k11,無脂肪牛乳,2026-09-25,2026-10-15,48,20,28,'',138
+sku_k12,低脂肪牛乳,2026-09-25,2026-10-15,48,20,28,'',120
+sku_k13,ミルク,2026-09-25,2026-10-15,48,20,28,'',260