From 44badcf1933ab1e75ec9bb6780a6d0115346a42f Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Mon, 28 Sep 2026 14:17:14 +0900 Subject: [PATCH] fix: replay recorded walks with selected identity backend --- recognization/code/eval_live.py | 173 +++++++++++++++++++++++++-- recognization/code/test_eval_live.py | 36 ++++++ 2 files changed, 196 insertions(+), 13 deletions(-) diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py index d44a140..4202692 100644 --- a/recognization/code/eval_live.py +++ b/recognization/code/eval_live.py @@ -11,10 +11,14 @@ store was enrolled; scoring them is training-set accuracy. python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] [--json PATH] [--iou 0.5] + [--replay] [--identity matcher|gallery] [--device cpu|mps|cuda] Prints a human count table and writes the same report as JSON (default /eval_live.json). Missing input exits 2 with {"error": ...} on -stdout. The same pinned recording, catalog, weights, and labels produce the +stdout. --identity gallery uses the same LiveSession path as the server and +exits 2 when that gallery cannot load; it does not score the matcher instead. +The report names the backend, whether the head loaded, and the weight pins. +The same pinned recording, catalog, weights, and labels produce the same counts; timings are omitted because they are not. No held-out labels means denominators and fingerprints only. That is not a @@ -95,6 +99,42 @@ def load_index(recording_dir: str) -> List[Dict[str, Any]]: return rows +def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]: + """Backend named by the recording, when every present value agrees. + + FrameRecorder does not write this. A later index may, on the first line or + on each frame, under identity_backend or identity.backend. Silence means + the caller must say --identity. A mix is not a default. + """ + found: List[str] = [] + for row in rows: + raw = row.get("identity_backend") + if raw is None and isinstance(row.get("identity"), dict): + raw = row["identity"].get("backend") + if raw is None: + continue + name = str(raw).strip().lower() + if name not in ("matcher", "gallery"): + raise EvalInputError(f"recorded identity backend is not matcher or gallery: {raw}") + found.append(name) + unique = set(found) + if len(unique) > 1: + raise EvalInputError("recording names more than one identity backend") + return found[0] if found else None + + +def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str: + """CLI wins. A recording may fill the gap. Neither one stays the matcher.""" + chosen = (requested or recorded or "matcher").strip().lower() + if chosen not in ("matcher", "gallery"): + raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}") + if recorded and requested and recorded != chosen: + raise EvalInputError( + f"recording identity is {recorded}; pass --identity {recorded} or omit --identity" + ) + return chosen + + def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: """Hash the JPEGs the client sent, in index order. Missing files are an error.""" digest = hashlib.sha256() @@ -176,17 +216,46 @@ def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[st } -def model_fingerprint(weights_path: str) -> Dict[str, Any]: +def model_fingerprint( + weights_path: str, + *, + identity: str = "matcher", + gallery: Any = None, +) -> Dict[str, Any]: + """YOLO pin plus, for gallery, the backbone and head that actually loaded.""" try: import ultralytics library = f"ultralytics-{ultralytics.__version__}" except Exception: library = "ultralytics-unknown" - return { - "version": f"{os.path.basename(weights_path)}@{library}", + pin: Dict[str, Any] = { + "version": f"{os.path.basename(weights_path)}@{library}" if weights_path else f"none@{library}", "weights_sha256": file_sha256(weights_path) if weights_path else None, + "identity": identity, } + if identity != "gallery" or gallery is None: + if identity == "gallery": + pin["gallery_loaded"] = False + pin["gallery_head_loaded"] = False + return pin + from gallery import DINOV2_SHA256, dinov2_weights_path + + backbone = dinov2_weights_path() + digest = file_sha256(backbone) + pin.update( + { + "gallery_loaded": True, + "gallery_head": getattr(gallery, "head_reason", None), + "gallery_head_loaded": getattr(gallery, "_head", None) is not None, + "gallery_views": gallery.view_count() if hasattr(gallery, "view_count") else None, + "dinov2_path": backbone, + "dinov2_sha256": digest, + "dinov2_sha256_expected": DINOV2_SHA256, + "dinov2_pinned": digest == DINOV2_SHA256, + } + ) + return pin def _decode_image(payload: bytes) -> Any: @@ -476,11 +545,15 @@ def score_recording( labels_path: Optional[str] = None, iou_min: float = 0.5, live_frames: Optional[Sequence[Dict[str, Any]]] = None, + identity: str = "matcher", + gallery: Any = None, ) -> Dict[str, Any]: """Score one session directory. live_frames, when given, are the replayed results. Without live_frames the recorded index is what the server already decided. - That still produces denominators; it is not a fresh model run. + That still produces denominators; it is not a fresh model run. identity is + the backend those frames were named with, not a claim that this process + loaded it. gallery, when passed, is the loaded ProductGallery. """ recording_dir = os.path.abspath(recording_dir) rows = load_index(recording_dir) @@ -491,12 +564,21 @@ def score_recording( source = list(live_frames) if live_frames is not None else rows tracks = [track for frame in source for track in (frame.get("tracks") or [])] decisions = score_tracks(tracks) + backend = (identity or "matcher").strip().lower() report = { "recording": recording_dir, "source": "live_replay" if live_frames is not None else "recorded_index", + "identity": { + "backend": backend, + "gallery_loaded": gallery is not None if backend == "gallery" else None, + "gallery_head": getattr(gallery, "head_reason", None) if gallery is not None else None, + "gallery_head_loaded": ( + bool(getattr(gallery, "_head", None) is not None) if gallery is not None else None + ), + }, "fingerprints": { "sample": sample, - "model": model_fingerprint(weights_path), + "model": model_fingerprint(weights_path, identity=backend, gallery=gallery), "catalog": catalog_fingerprint(stock_dir), }, "decisions": decisions, @@ -590,8 +672,13 @@ def format_report(report: Dict[str, Any]) -> str: top1 = report["labelled_top1"] churn = report["churn"] fingerprints = report["fingerprints"] + identity = report.get("identity") or {} + model = fingerprints["model"] + head = identity.get("gallery_head") + head_note = f" head {head}" if head else "" lines = [ f"source: {report['source']}", + f"identity: {identity.get('backend', 'matcher')}{head_note}", ( f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " f"rejected {decisions['rejected']} unknown {decisions['unknown']} " @@ -617,15 +704,66 @@ def format_report(report: Dict[str, Any]) -> str: ), ( f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} " - f"model {fingerprints['model']['version']} " + f"model {model['version']} " + f"weights {str(model.get('weights_sha256') or 'none')[:12]} " f"catalog {fingerprints['catalog']['version']}" ), ] + if model.get("identity") == "gallery": + lines.append( + f"gallery: loaded {model.get('gallery_loaded')} " + f"head {model.get('gallery_head')} " + f"dinov2 {str(model.get('dinov2_sha256') or 'missing')[:12]} " + f"pinned {model.get('dinov2_pinned')}" + ) for gap in report["evidence"]: lines.append(f"gap: {gap}") return "\n".join(lines) +def replay_engine(stock_dir: str, weights: str, identity: str, device: Optional[str]) -> Tuple[Any, str, Any]: + """Build the engine the server would, and refuse a gallery that did not load. + + The backbone file is checked before YOLO loads. A missing or unpinned file + is the same refusal the server makes, without scoring the matcher. + """ + from engine import EngineSettings, build_default_engine + + if identity == "gallery": + from gallery import DINOV2_SHA256, dinov2_weights_path + + backbone = dinov2_weights_path() + digest = file_sha256(backbone) + if digest is None: + raise EvalInputError( + f"gallery identity backend requires local DINOv2 weights at {backbone}; " + "refusing to score the matcher" + ) + if digest != DINOV2_SHA256: + raise EvalInputError( + f"DINOv2 weights at {backbone} are sha256 {digest}, expected {DINOV2_SHA256}" + ) + settings = EngineSettings(stock_dir=stock_dir, identity_backend=identity) + if weights: + settings.weights_path = weights + if device: + settings.device = device + engine = build_default_engine(settings) + chosen = (engine.settings.identity_backend or "matcher").lower() + if chosen != identity: + engine.shutdown() + raise EvalInputError(f"engine identity is {chosen}, requested {identity}") + gallery = None + if identity == "gallery": + gallery = engine.identity_gallery() + if gallery is None: + engine.shutdown() + raise EvalInputError( + "gallery identity backend did not load; refusing to score the matcher" + ) + return engine, engine.settings.weights_path, gallery + + def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description="Score a recorded live walk.") parser.add_argument("recording", nargs="?", help="session directory with index.jsonl") @@ -640,6 +778,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int: help="re-run frames through LiveSession (needs the recognition stack)", ) parser.add_argument("--method", default="hybrid") + parser.add_argument( + "--identity", + default=None, + choices=["matcher", "gallery"], + help="which mechanism names products (default: recorded backend, else matcher)", + ) + parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0") args = parser.parse_args(argv) def fail(message: str) -> int: @@ -653,17 +798,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if not os.path.isdir(recording): return fail(f"recording directory not found: {recording}") try: + rows = load_index(recording) + identity = resolve_identity(args.identity, recorded_identity(rows)) live_frames = None + gallery = None weights = args.weights if args.replay: - from engine import EngineSettings, build_default_engine from live import LiveSettings - settings = EngineSettings(stock_dir=args.stock_dir) - if weights: - settings.weights_path = weights - engine = build_default_engine(settings) - weights = engine.settings.weights_path + engine, weights, gallery = replay_engine( + args.stock_dir, args.weights, identity, args.device + ) live_frames = replay_live( recording, engine, settings=LiveSettings(method=args.method, tags=True) ) @@ -674,6 +819,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: labels_path=args.labels, iou_min=args.iou, live_frames=live_frames, + identity=identity, + gallery=gallery, ) except EvalInputError as exc: return fail(str(exc)) diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py index f993328..f543358 100644 --- a/recognization/code/test_eval_live.py +++ b/recognization/code/test_eval_live.py @@ -237,5 +237,41 @@ class CliTests(unittest.TestCase): load_index(root) +class IdentityContractTests(unittest.TestCase): + def test_missing_gallery_weights_exit_without_a_matcher_score(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + recording = os.path.join(root, "session") + os.makedirs(recording) + write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8)) + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps({"frame": 1, "path": "000000.jpg", "tracks": []}) + "\n") + missing = os.path.join(root, "no-such-dinov2.safetensors") + report_path = os.path.join(recording, "eval_live.json") + env = os.environ.copy() + env["DORAYAKI_DINOV2_WEIGHTS"] = missing + completed = subprocess.run( + [ + sys.executable, + os.path.join(current_dir, "eval_live.py"), + recording, + "--replay", + "--identity", + "gallery", + "--json", + report_path, + ], + capture_output=True, + text=True, + check=False, + env=env, + ) + self.assertEqual(completed.returncode, 2) + payload = json.loads(completed.stdout) + self.assertIn("DINOv2", payload["error"]) + self.assertIn("matcher", payload["error"]) + self.assertFalse(os.path.exists(report_path)) + self.assertNotIn("accepted_rate", completed.stdout) + if __name__ == "__main__": unittest.main()