diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py index 4202692..d44a140 100644 --- a/recognization/code/eval_live.py +++ b/recognization/code/eval_live.py @@ -11,14 +11,10 @@ store was enrolled; scoring them is training-set accuracy. python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] [--json PATH] [--iou 0.5] - [--replay] [--identity matcher|gallery] [--device cpu|mps|cuda] Prints a human count table and writes the same report as JSON (default /eval_live.json). Missing input exits 2 with {"error": ...} on -stdout. --identity gallery uses the same LiveSession path as the server and -exits 2 when that gallery cannot load; it does not score the matcher instead. -The report names the backend, whether the head loaded, and the weight pins. -The same pinned recording, catalog, weights, and labels produce the +stdout. The same pinned recording, catalog, weights, and labels produce the same counts; timings are omitted because they are not. No held-out labels means denominators and fingerprints only. That is not a @@ -99,42 +95,6 @@ def load_index(recording_dir: str) -> List[Dict[str, Any]]: return rows -def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]: - """Backend named by the recording, when every present value agrees. - - FrameRecorder does not write this. A later index may, on the first line or - on each frame, under identity_backend or identity.backend. Silence means - the caller must say --identity. A mix is not a default. - """ - found: List[str] = [] - for row in rows: - raw = row.get("identity_backend") - if raw is None and isinstance(row.get("identity"), dict): - raw = row["identity"].get("backend") - if raw is None: - continue - name = str(raw).strip().lower() - if name not in ("matcher", "gallery"): - raise EvalInputError(f"recorded identity backend is not matcher or gallery: {raw}") - found.append(name) - unique = set(found) - if len(unique) > 1: - raise EvalInputError("recording names more than one identity backend") - return found[0] if found else None - - -def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str: - """CLI wins. A recording may fill the gap. Neither one stays the matcher.""" - chosen = (requested or recorded or "matcher").strip().lower() - if chosen not in ("matcher", "gallery"): - raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}") - if recorded and requested and recorded != chosen: - raise EvalInputError( - f"recording identity is {recorded}; pass --identity {recorded} or omit --identity" - ) - return chosen - - def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: """Hash the JPEGs the client sent, in index order. Missing files are an error.""" digest = hashlib.sha256() @@ -216,46 +176,17 @@ def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[st } -def model_fingerprint( - weights_path: str, - *, - identity: str = "matcher", - gallery: Any = None, -) -> Dict[str, Any]: - """YOLO pin plus, for gallery, the backbone and head that actually loaded.""" +def model_fingerprint(weights_path: str) -> Dict[str, Any]: try: import ultralytics library = f"ultralytics-{ultralytics.__version__}" except Exception: library = "ultralytics-unknown" - pin: Dict[str, Any] = { - "version": f"{os.path.basename(weights_path)}@{library}" if weights_path else f"none@{library}", + return { + "version": f"{os.path.basename(weights_path)}@{library}", "weights_sha256": file_sha256(weights_path) if weights_path else None, - "identity": identity, } - if identity != "gallery" or gallery is None: - if identity == "gallery": - pin["gallery_loaded"] = False - pin["gallery_head_loaded"] = False - return pin - from gallery import DINOV2_SHA256, dinov2_weights_path - - backbone = dinov2_weights_path() - digest = file_sha256(backbone) - pin.update( - { - "gallery_loaded": True, - "gallery_head": getattr(gallery, "head_reason", None), - "gallery_head_loaded": getattr(gallery, "_head", None) is not None, - "gallery_views": gallery.view_count() if hasattr(gallery, "view_count") else None, - "dinov2_path": backbone, - "dinov2_sha256": digest, - "dinov2_sha256_expected": DINOV2_SHA256, - "dinov2_pinned": digest == DINOV2_SHA256, - } - ) - return pin def _decode_image(payload: bytes) -> Any: @@ -545,15 +476,11 @@ def score_recording( labels_path: Optional[str] = None, iou_min: float = 0.5, live_frames: Optional[Sequence[Dict[str, Any]]] = None, - identity: str = "matcher", - gallery: Any = None, ) -> Dict[str, Any]: """Score one session directory. live_frames, when given, are the replayed results. Without live_frames the recorded index is what the server already decided. - That still produces denominators; it is not a fresh model run. identity is - the backend those frames were named with, not a claim that this process - loaded it. gallery, when passed, is the loaded ProductGallery. + That still produces denominators; it is not a fresh model run. """ recording_dir = os.path.abspath(recording_dir) rows = load_index(recording_dir) @@ -564,21 +491,12 @@ def score_recording( source = list(live_frames) if live_frames is not None else rows tracks = [track for frame in source for track in (frame.get("tracks") or [])] decisions = score_tracks(tracks) - backend = (identity or "matcher").strip().lower() report = { "recording": recording_dir, "source": "live_replay" if live_frames is not None else "recorded_index", - "identity": { - "backend": backend, - "gallery_loaded": gallery is not None if backend == "gallery" else None, - "gallery_head": getattr(gallery, "head_reason", None) if gallery is not None else None, - "gallery_head_loaded": ( - bool(getattr(gallery, "_head", None) is not None) if gallery is not None else None - ), - }, "fingerprints": { "sample": sample, - "model": model_fingerprint(weights_path, identity=backend, gallery=gallery), + "model": model_fingerprint(weights_path), "catalog": catalog_fingerprint(stock_dir), }, "decisions": decisions, @@ -672,13 +590,8 @@ def format_report(report: Dict[str, Any]) -> str: top1 = report["labelled_top1"] churn = report["churn"] fingerprints = report["fingerprints"] - identity = report.get("identity") or {} - model = fingerprints["model"] - head = identity.get("gallery_head") - head_note = f" head {head}" if head else "" lines = [ f"source: {report['source']}", - f"identity: {identity.get('backend', 'matcher')}{head_note}", ( f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " f"rejected {decisions['rejected']} unknown {decisions['unknown']} " @@ -704,66 +617,15 @@ def format_report(report: Dict[str, Any]) -> str: ), ( f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} " - f"model {model['version']} " - f"weights {str(model.get('weights_sha256') or 'none')[:12]} " + f"model {fingerprints['model']['version']} " f"catalog {fingerprints['catalog']['version']}" ), ] - if model.get("identity") == "gallery": - lines.append( - f"gallery: loaded {model.get('gallery_loaded')} " - f"head {model.get('gallery_head')} " - f"dinov2 {str(model.get('dinov2_sha256') or 'missing')[:12]} " - f"pinned {model.get('dinov2_pinned')}" - ) for gap in report["evidence"]: lines.append(f"gap: {gap}") return "\n".join(lines) -def replay_engine(stock_dir: str, weights: str, identity: str, device: Optional[str]) -> Tuple[Any, str, Any]: - """Build the engine the server would, and refuse a gallery that did not load. - - The backbone file is checked before YOLO loads. A missing or unpinned file - is the same refusal the server makes, without scoring the matcher. - """ - from engine import EngineSettings, build_default_engine - - if identity == "gallery": - from gallery import DINOV2_SHA256, dinov2_weights_path - - backbone = dinov2_weights_path() - digest = file_sha256(backbone) - if digest is None: - raise EvalInputError( - f"gallery identity backend requires local DINOv2 weights at {backbone}; " - "refusing to score the matcher" - ) - if digest != DINOV2_SHA256: - raise EvalInputError( - f"DINOv2 weights at {backbone} are sha256 {digest}, expected {DINOV2_SHA256}" - ) - settings = EngineSettings(stock_dir=stock_dir, identity_backend=identity) - if weights: - settings.weights_path = weights - if device: - settings.device = device - engine = build_default_engine(settings) - chosen = (engine.settings.identity_backend or "matcher").lower() - if chosen != identity: - engine.shutdown() - raise EvalInputError(f"engine identity is {chosen}, requested {identity}") - gallery = None - if identity == "gallery": - gallery = engine.identity_gallery() - if gallery is None: - engine.shutdown() - raise EvalInputError( - "gallery identity backend did not load; refusing to score the matcher" - ) - return engine, engine.settings.weights_path, gallery - - def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description="Score a recorded live walk.") parser.add_argument("recording", nargs="?", help="session directory with index.jsonl") @@ -778,13 +640,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int: help="re-run frames through LiveSession (needs the recognition stack)", ) parser.add_argument("--method", default="hybrid") - parser.add_argument( - "--identity", - default=None, - choices=["matcher", "gallery"], - help="which mechanism names products (default: recorded backend, else matcher)", - ) - parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0") args = parser.parse_args(argv) def fail(message: str) -> int: @@ -798,17 +653,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if not os.path.isdir(recording): return fail(f"recording directory not found: {recording}") try: - rows = load_index(recording) - identity = resolve_identity(args.identity, recorded_identity(rows)) live_frames = None - gallery = None weights = args.weights if args.replay: + from engine import EngineSettings, build_default_engine from live import LiveSettings - engine, weights, gallery = replay_engine( - args.stock_dir, args.weights, identity, args.device - ) + settings = EngineSettings(stock_dir=args.stock_dir) + if weights: + settings.weights_path = weights + engine = build_default_engine(settings) + weights = engine.settings.weights_path live_frames = replay_live( recording, engine, settings=LiveSettings(method=args.method, tags=True) ) @@ -819,8 +674,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int: labels_path=args.labels, iou_min=args.iou, live_frames=live_frames, - identity=identity, - gallery=gallery, ) except EvalInputError as exc: return fail(str(exc)) diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py index f543358..f993328 100644 --- a/recognization/code/test_eval_live.py +++ b/recognization/code/test_eval_live.py @@ -237,41 +237,5 @@ class CliTests(unittest.TestCase): load_index(root) -class IdentityContractTests(unittest.TestCase): - def test_missing_gallery_weights_exit_without_a_matcher_score(self): - root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, root, ignore_errors=True) - recording = os.path.join(root, "session") - os.makedirs(recording) - write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8)) - with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: - handle.write(json.dumps({"frame": 1, "path": "000000.jpg", "tracks": []}) + "\n") - missing = os.path.join(root, "no-such-dinov2.safetensors") - report_path = os.path.join(recording, "eval_live.json") - env = os.environ.copy() - env["DORAYAKI_DINOV2_WEIGHTS"] = missing - completed = subprocess.run( - [ - sys.executable, - os.path.join(current_dir, "eval_live.py"), - recording, - "--replay", - "--identity", - "gallery", - "--json", - report_path, - ], - capture_output=True, - text=True, - check=False, - env=env, - ) - self.assertEqual(completed.returncode, 2) - payload = json.loads(completed.stdout) - self.assertIn("DINOv2", payload["error"]) - self.assertIn("matcher", payload["error"]) - self.assertFalse(os.path.exists(report_path)) - self.assertNotIn("accepted_rate", completed.stdout) - if __name__ == "__main__": unittest.main() diff --git a/recognization/docs/DATA.md b/recognization/docs/DATA.md index 667eff2..81e1f60 100644 --- a/recognization/docs/DATA.md +++ b/recognization/docs/DATA.md @@ -9,31 +9,22 @@ the device that will do the scanning. ## 1. Record a walk -From `recognization`, run `./run-demo-server.sh stock-lopia` once to provision -the ignored `config/local.env` token and the native apps' local LAN settings, -then stop that server. Rebuild the app after provisioning. The token is a secret: -keep `.demo-token`, `config/local.env`, and app-local settings out of commits and -shared logs. For a recorded walk, start the same gallery backend on the -launcher's port; `code/app.py` reads the token from `config/local.env`: +Start the server with recording on. Frames are written every Nth processed frame, +and each one is written with a line describing what the server saw in it. ```bash -cd product-recognition-demo/recognization # from the app repository root +cd recognization YOLO_OFFLINE=1 .venv/bin/python code/app.py \ - --port 8099 --host 0.0.0.0 --stock-dir stock-lopia --identity gallery \ - --live-conf 0.15 --classify-workers 8 \ - --record-frames ~/dorayaki-data/storeA-visit1 \ + --port 8090 --host 0.0.0.0 \ + --classify-workers 8 \ + --record-frames ~/dorayaki-data/2026-09-25-storeA \ --record-every 3 ``` -This requires the local pinned DINOv2 file described in `../README.md` -(`weights/dinov2_vits14_lvd142m.safetensors`, or -`DORAYAKI_DINOV2_WEIGHTS`). A LAN bind without a token is refused. Record the -second visit to a separate directory with the same catalog and model. - Then run the app and scan. Each scan session becomes its own directory: ``` -~/dorayaki-data/storeA-visit1// +~/dorayaki-data/2026-09-25-storeA// 000000.jpg the frame, exactly as the client sent it 000001.jpg index.jsonl one line per recorded frame: @@ -69,42 +60,33 @@ The tag channel seeds the labels. A facing whose tag named a product, with a pri the tag's own arithmetic corroborates, is a candidate label rather than a guess: ```bash -python -c ' +python -c " import json, sys -sys.path.insert(0, "code") +sys.path.insert(0, 'code') from record import labelling_candidates -seeds = labelling_candidates(sys.argv[1]) -print(len(seeds), "candidate labels") +seeds = labelling_candidates('~/dorayaki-data/2026-09-25-storeA//index.jsonl') +print(len(seeds), 'candidate labels') print(json.dumps(seeds[:3], ensure_ascii=False, indent=1)) -' "$HOME/dorayaki-data/storeA-visit1/REPLACE_WITH_SESSION/index.jsonl" +" ``` A human then confirms or corrects them in the app's confirmation screen — the same screen that closes the flywheel. Nothing here is a training set on its own; it is the raw material for one. -## 4. Replay a cross-visit benchmark +## 4. Freeze a benchmark -Enroll the gallery from visit 1 only, then freeze that catalog. Confirm labels -for visit 2 independently; `labels.json` beside its `index.jsonl` (or supplied -with `--labels`) is a JSON list, or `{"labels": [...]}`. Each entry has `frame` -(the number in `index.jsonl`), `box` (pixel coordinates `[x1, y1, x2, y2]`), -and `sku_id` (a catalog SKU). These are confirmed annotations, not the server's -predictions. For the second visit, run from `recognization` with the same frozen -`stock-lopia` catalog: +Write the confirmed labels into a manifest that the evaluation reads, and keep it in +the repository so results are reproducible: -```bash -SESSION="$HOME/dorayaki-data/storeA-visit2/REPLACE_WITH_SESSION" -YOLO_OFFLINE=1 .venv/bin/python code/eval_live.py "$SESSION" \ - --stock-dir stock-lopia --labels "$SESSION/labels.json" \ - --replay --identity gallery --device cpu +``` +benchmarks/-/ + visit1//.jpg the gallery: 2-3 views per SKU + visit2/*.jpg the test frames + labels.json {"visit2/0007.jpg": [{"box": [...], "sku_id": "..."}]} ``` -Replace `REPLACE_WITH_SESSION` with the actual session name. The evaluator replays -recorded JPEGs through `LiveSession`, writes `eval_live.json` in that directory, -and reports backend, loaded head, weight/catalog fingerprints, decision counts, -and held-out labelled top-1 when valid labels exist. It rejects missing or -unpinned DINOv2 weights rather than substituting matcher scores; labels copied -from catalog templates are excluded as leakage. Without confirmed held-out -visit-2 labels, counts are diagnostic only, **not gallery walk accuracy**. No -held-out cross-visit result is available here yet. +Then the identity benchmark (see `code/bench_identity.py`) can be run with a +cross-visit split, and every number it reports is a number worth acting on. Until +that exists, treat the current figures as what they are: measurements on a +photograph.