docs: describe cross-visit gallery replay

recognization/code/eval_live.py

@@ -11,10 +11,14 @@ store was enrolled; scoring them is training-set accuracy.
 
     python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH]
                               [--json PATH] [--iou 0.5]
+                              [--replay] [--identity matcher|gallery] [--device cpu|mps|cuda]
 
 Prints a human count table and writes the same report as JSON (default
 <recording>/eval_live.json). Missing input exits 2 with {"error": ...} on
-stdout. The same pinned recording, catalog, weights, and labels produce the
+stdout. --identity gallery uses the same LiveSession path as the server and
+exits 2 when that gallery cannot load; it does not score the matcher instead.
+The report names the backend, whether the head loaded, and the weight pins.
+The same pinned recording, catalog, weights, and labels produce the
 same counts; timings are omitted because they are not.
 
 No held-out labels means denominators and fingerprints only. That is not a
@@ -95,6 +99,42 @@ def load_index(recording_dir: str) -> List[Dict[str, Any]]:
     return rows
 
 
+def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]:
+    """Backend named by the recording, when every present value agrees.
+
+    FrameRecorder does not write this. A later index may, on the first line or
+    on each frame, under identity_backend or identity.backend. Silence means
+    the caller must say --identity. A mix is not a default.
+    """
+    found: List[str] = []
+    for row in rows:
+        raw = row.get("identity_backend")
+        if raw is None and isinstance(row.get("identity"), dict):
+            raw = row["identity"].get("backend")
+        if raw is None:
+            continue
+        name = str(raw).strip().lower()
+        if name not in ("matcher", "gallery"):
+            raise EvalInputError(f"recorded identity backend is not matcher or gallery: {raw}")
+        found.append(name)
+    unique = set(found)
+    if len(unique) > 1:
+        raise EvalInputError("recording names more than one identity backend")
+    return found[0] if found else None
+
+
+def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str:
+    """CLI wins. A recording may fill the gap. Neither one stays the matcher."""
+    chosen = (requested or recorded or "matcher").strip().lower()
+    if chosen not in ("matcher", "gallery"):
+        raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}")
+    if recorded and requested and recorded != chosen:
+        raise EvalInputError(
+            f"recording identity is {recorded}; pass --identity {recorded} or omit --identity"
+        )
+    return chosen
+
+
 def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
     """Hash the JPEGs the client sent, in index order. Missing files are an error."""
     digest = hashlib.sha256()
@@ -176,17 +216,46 @@ def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[st
     }
 
 
-def model_fingerprint(weights_path: str) -> Dict[str, Any]:
+def model_fingerprint(
+    weights_path: str,
+    *,
+    identity: str = "matcher",
+    gallery: Any = None,
+) -> Dict[str, Any]:
+    """YOLO pin plus, for gallery, the backbone and head that actually loaded."""
     try:
         import ultralytics
 
         library = f"ultralytics-{ultralytics.__version__}"
     except Exception:
         library = "ultralytics-unknown"
-    return {
-        "version": f"{os.path.basename(weights_path)}@{library}",
+    pin: Dict[str, Any] = {
+        "version": f"{os.path.basename(weights_path)}@{library}" if weights_path else f"none@{library}",
         "weights_sha256": file_sha256(weights_path) if weights_path else None,
+        "identity": identity,
     }
+    if identity != "gallery" or gallery is None:
+        if identity == "gallery":
+            pin["gallery_loaded"] = False
+            pin["gallery_head_loaded"] = False
+        return pin
+    from gallery import DINOV2_SHA256, dinov2_weights_path
+
+    backbone = dinov2_weights_path()
+    digest = file_sha256(backbone)
+    pin.update(
+        {
+            "gallery_loaded": True,
+            "gallery_head": getattr(gallery, "head_reason", None),
+            "gallery_head_loaded": getattr(gallery, "_head", None) is not None,
+            "gallery_views": gallery.view_count() if hasattr(gallery, "view_count") else None,
+            "dinov2_path": backbone,
+            "dinov2_sha256": digest,
+            "dinov2_sha256_expected": DINOV2_SHA256,
+            "dinov2_pinned": digest == DINOV2_SHA256,
+        }
+    )
+    return pin
 
 
 def _decode_image(payload: bytes) -> Any:
@@ -476,11 +545,15 @@ def score_recording(
     labels_path: Optional[str] = None,
     iou_min: float = 0.5,
     live_frames: Optional[Sequence[Dict[str, Any]]] = None,
+    identity: str = "matcher",
+    gallery: Any = None,
 ) -> Dict[str, Any]:
     """Score one session directory. live_frames, when given, are the replayed results.
 
     Without live_frames the recorded index is what the server already decided.
-    That still produces denominators; it is not a fresh model run.
+    That still produces denominators; it is not a fresh model run. identity is
+    the backend those frames were named with, not a claim that this process
+    loaded it. gallery, when passed, is the loaded ProductGallery.
     """
     recording_dir = os.path.abspath(recording_dir)
     rows = load_index(recording_dir)
@@ -491,12 +564,21 @@ def score_recording(
     source = list(live_frames) if live_frames is not None else rows
     tracks = [track for frame in source for track in (frame.get("tracks") or [])]
     decisions = score_tracks(tracks)
+    backend = (identity or "matcher").strip().lower()
     report = {
         "recording": recording_dir,
         "source": "live_replay" if live_frames is not None else "recorded_index",
+        "identity": {
+            "backend": backend,
+            "gallery_loaded": gallery is not None if backend == "gallery" else None,
+            "gallery_head": getattr(gallery, "head_reason", None) if gallery is not None else None,
+            "gallery_head_loaded": (
+                bool(getattr(gallery, "_head", None) is not None) if gallery is not None else None
+            ),
+        },
         "fingerprints": {
             "sample": sample,
-            "model": model_fingerprint(weights_path),
+            "model": model_fingerprint(weights_path, identity=backend, gallery=gallery),
             "catalog": catalog_fingerprint(stock_dir),
         },
         "decisions": decisions,
@@ -590,8 +672,13 @@ def format_report(report: Dict[str, Any]) -> str:
     top1 = report["labelled_top1"]
     churn = report["churn"]
     fingerprints = report["fingerprints"]
+    identity = report.get("identity") or {}
+    model = fingerprints["model"]
+    head = identity.get("gallery_head")
+    head_note = f" head {head}" if head else ""
     lines = [
         f"source: {report['source']}",
+        f"identity: {identity.get('backend', 'matcher')}{head_note}",
         (
             f"decisions: accepted {decisions['accepted']}  ambiguous {decisions['ambiguous']}  "
             f"rejected {decisions['rejected']}  unknown {decisions['unknown']}  "
@@ -617,15 +704,66 @@ def format_report(report: Dict[str, Any]) -> str:
         ),
         (
             f"fingerprints: sample {fingerprints['sample']['sha256'][:12]}  "
-            f"model {fingerprints['model']['version']}  "
+            f"model {model['version']}  "
+            f"weights {str(model.get('weights_sha256') or 'none')[:12]}  "
             f"catalog {fingerprints['catalog']['version']}"
         ),
     ]
+    if model.get("identity") == "gallery":
+        lines.append(
+            f"gallery: loaded {model.get('gallery_loaded')}  "
+            f"head {model.get('gallery_head')}  "
+            f"dinov2 {str(model.get('dinov2_sha256') or 'missing')[:12]}  "
+            f"pinned {model.get('dinov2_pinned')}"
+        )
     for gap in report["evidence"]:
         lines.append(f"gap: {gap}")
     return "\n".join(lines)
 
 
+def replay_engine(stock_dir: str, weights: str, identity: str, device: Optional[str]) -> Tuple[Any, str, Any]:
+    """Build the engine the server would, and refuse a gallery that did not load.
+
+    The backbone file is checked before YOLO loads. A missing or unpinned file
+    is the same refusal the server makes, without scoring the matcher.
+    """
+    from engine import EngineSettings, build_default_engine
+
+    if identity == "gallery":
+        from gallery import DINOV2_SHA256, dinov2_weights_path
+
+        backbone = dinov2_weights_path()
+        digest = file_sha256(backbone)
+        if digest is None:
+            raise EvalInputError(
+                f"gallery identity backend requires local DINOv2 weights at {backbone}; "
+                "refusing to score the matcher"
+            )
+        if digest != DINOV2_SHA256:
+            raise EvalInputError(
+                f"DINOv2 weights at {backbone} are sha256 {digest}, expected {DINOV2_SHA256}"
+            )
+    settings = EngineSettings(stock_dir=stock_dir, identity_backend=identity)
+    if weights:
+        settings.weights_path = weights
+    if device:
+        settings.device = device
+    engine = build_default_engine(settings)
+    chosen = (engine.settings.identity_backend or "matcher").lower()
+    if chosen != identity:
+        engine.shutdown()
+        raise EvalInputError(f"engine identity is {chosen}, requested {identity}")
+    gallery = None
+    if identity == "gallery":
+        gallery = engine.identity_gallery()
+        if gallery is None:
+            engine.shutdown()
+            raise EvalInputError(
+                "gallery identity backend did not load; refusing to score the matcher"
+            )
+    return engine, engine.settings.weights_path, gallery
+
+
 def main(argv: Optional[Sequence[str]] = None) -> int:
     parser = argparse.ArgumentParser(description="Score a recorded live walk.")
     parser.add_argument("recording", nargs="?", help="session directory with index.jsonl")
@@ -640,6 +778,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
         help="re-run frames through LiveSession (needs the recognition stack)",
     )
     parser.add_argument("--method", default="hybrid")
+    parser.add_argument(
+        "--identity",
+        default=None,
+        choices=["matcher", "gallery"],
+        help="which mechanism names products (default: recorded backend, else matcher)",
+    )
+    parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0")
     args = parser.parse_args(argv)
 
     def fail(message: str) -> int:
@@ -653,17 +798,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     if not os.path.isdir(recording):
         return fail(f"recording directory not found: {recording}")
     try:
+        rows = load_index(recording)
+        identity = resolve_identity(args.identity, recorded_identity(rows))
         live_frames = None
+        gallery = None
         weights = args.weights
         if args.replay:
-            from engine import EngineSettings, build_default_engine
             from live import LiveSettings
 
-            settings = EngineSettings(stock_dir=args.stock_dir)
-            if weights:
-                settings.weights_path = weights
-            engine = build_default_engine(settings)
-            weights = engine.settings.weights_path
+            engine, weights, gallery = replay_engine(
+                args.stock_dir, args.weights, identity, args.device
+            )
             live_frames = replay_live(
                 recording, engine, settings=LiveSettings(method=args.method, tags=True)
             )
@@ -674,6 +819,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
             labels_path=args.labels,
             iou_min=args.iou,
             live_frames=live_frames,
+            identity=identity,
+            gallery=gallery,
         )
     except EvalInputError as exc:
         return fail(str(exc))

recognization/code/test_eval_live.py

@@ -237,5 +237,41 @@ class CliTests(unittest.TestCase):
             load_index(root)
 
 
+class IdentityContractTests(unittest.TestCase):
+    def test_missing_gallery_weights_exit_without_a_matcher_score(self):
+        root = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, root, ignore_errors=True)
+        recording = os.path.join(root, "session")
+        os.makedirs(recording)
+        write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8))
+        with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
+            handle.write(json.dumps({"frame": 1, "path": "000000.jpg", "tracks": []}) + "\n")
+        missing = os.path.join(root, "no-such-dinov2.safetensors")
+        report_path = os.path.join(recording, "eval_live.json")
+        env = os.environ.copy()
+        env["DORAYAKI_DINOV2_WEIGHTS"] = missing
+        completed = subprocess.run(
+            [
+                sys.executable,
+                os.path.join(current_dir, "eval_live.py"),
+                recording,
+                "--replay",
+                "--identity",
+                "gallery",
+                "--json",
+                report_path,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+            env=env,
+        )
+        self.assertEqual(completed.returncode, 2)
+        payload = json.loads(completed.stdout)
+        self.assertIn("DINOv2", payload["error"])
+        self.assertIn("matcher", payload["error"])
+        self.assertFalse(os.path.exists(report_path))
+        self.assertNotIn("accepted_rate", completed.stdout)
+
 if __name__ == "__main__":
     unittest.main()

recognization/docs/DATA.md

@@ -9,22 +9,31 @@ the device that will do the scanning.
 
 ## 1. Record a walk
 
-Start the server with recording on. Frames are written every Nth processed frame,
-and each one is written with a line describing what the server saw in it.
+From `recognization`, run `./run-demo-server.sh stock-lopia` once to provision
+the ignored `config/local.env` token and the native apps' local LAN settings,
+then stop that server. Rebuild the app after provisioning. The token is a secret:
+keep `.demo-token`, `config/local.env`, and app-local settings out of commits and
+shared logs. For a recorded walk, start the same gallery backend on the
+launcher's port; `code/app.py` reads the token from `config/local.env`:
 
 ```bash
-cd recognization
+cd product-recognition-demo/recognization  # from the app repository root
 YOLO_OFFLINE=1 .venv/bin/python code/app.py \
-  --port 8090 --host 0.0.0.0 \
-  --classify-workers 8 \
-  --record-frames ~/dorayaki-data/2026-09-25-storeA \
+  --port 8099 --host 0.0.0.0 --stock-dir stock-lopia --identity gallery \
+  --live-conf 0.15 --classify-workers 8 \
+  --record-frames ~/dorayaki-data/storeA-visit1 \
   --record-every 3
 ```
 
+This requires the local pinned DINOv2 file described in `../README.md`
+(`weights/dinov2_vits14_lvd142m.safetensors`, or
+`DORAYAKI_DINOV2_WEIGHTS`). A LAN bind without a token is refused. Record the
+second visit to a separate directory with the same catalog and model.
+
 Then run the app and scan. Each scan session becomes its own directory:
 
 ```
-~/dorayaki-data/2026-09-25-storeA/<session>/
+~/dorayaki-data/storeA-visit1/<session>/
     000000.jpg        the frame, exactly as the client sent it
     000001.jpg
     index.jsonl       one line per recorded frame:
@@ -60,33 +69,42 @@ The tag channel seeds the labels. A facing whose tag named a product, with a pri
 the tag's own arithmetic corroborates, is a candidate label rather than a guess:
 
 ```bash
-python -c "
+python -c '
 import json, sys
-sys.path.insert(0, 'code')
+sys.path.insert(0, "code")
 from record import labelling_candidates
-seeds = labelling_candidates('~/dorayaki-data/2026-09-25-storeA/<session>/index.jsonl')
-print(len(seeds), 'candidate labels')
+seeds = labelling_candidates(sys.argv[1])
+print(len(seeds), "candidate labels")
 print(json.dumps(seeds[:3], ensure_ascii=False, indent=1))
-"
+' "$HOME/dorayaki-data/storeA-visit1/REPLACE_WITH_SESSION/index.jsonl"
 ```
 
 A human then confirms or corrects them in the app's confirmation screen — the same
 screen that closes the flywheel. Nothing here is a training set on its own; it is
 the raw material for one.
 
-## 4. Freeze a benchmark
+## 4. Replay a cross-visit benchmark
 
-Write the confirmed labels into a manifest that the evaluation reads, and keep it in
-the repository so results are reproducible:
+Enroll the gallery from visit 1 only, then freeze that catalog. Confirm labels
+for visit 2 independently; `labels.json` beside its `index.jsonl` (or supplied
+with `--labels`) is a JSON list, or `{"labels": [...]}`. Each entry has `frame`
+(the number in `index.jsonl`), `box` (pixel coordinates `[x1, y1, x2, y2]`),
+and `sku_id` (a catalog SKU). These are confirmed annotations, not the server's
+predictions. For the second visit, run from `recognization` with the same frozen
+`stock-lopia` catalog:
 
-```
-benchmarks/<store>-<date>/
-    visit1/<sku_id>/<view>.jpg      the gallery: 2-3 views per SKU
-    visit2/*.jpg                    the test frames
-    labels.json                     {"visit2/0007.jpg": [{"box": [...], "sku_id": "..."}]}
+```bash
+SESSION="$HOME/dorayaki-data/storeA-visit2/REPLACE_WITH_SESSION"
+YOLO_OFFLINE=1 .venv/bin/python code/eval_live.py "$SESSION" \
+  --stock-dir stock-lopia --labels "$SESSION/labels.json" \
+  --replay --identity gallery --device cpu
 ```
 
-Then the identity benchmark (see `code/bench_identity.py`) can be run with a
-cross-visit split, and every number it reports is a number worth acting on. Until
-that exists, treat the current figures as what they are: measurements on a
-photograph.
+Replace `REPLACE_WITH_SESSION` with the actual session name. The evaluator replays
+recorded JPEGs through `LiveSession`, writes `eval_live.json` in that directory,
+and reports backend, loaded head, weight/catalog fingerprints, decision counts,
+and held-out labelled top-1 when valid labels exist. It rejects missing or
+unpinned DINOv2 weights rather than substituting matcher scores; labels copied
+from catalog templates are excluded as leakage. Without confirmed held-out
+visit-2 labels, counts are diagnostic only, **not gallery walk accuracy**. No
+held-out cross-visit result is available here yet.