diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py index 4202692..948ab19 100644 --- a/recognization/code/eval_live.py +++ b/recognization/code/eval_live.py @@ -15,8 +15,11 @@ store was enrolled; scoring them is training-set accuracy. Prints a human count table and writes the same report as JSON (default /eval_live.json). Missing input exits 2 with {"error": ...} on -stdout. --identity gallery uses the same LiveSession path as the server and -exits 2 when that gallery cannot load; it does not score the matcher instead. +stdout. A recording that names identity_backend is scored as that backend unless +--identity repeats it. A recording that does not name one requires --identity; +omitting it does not score the matcher. --identity gallery uses the same +LiveSession path as the server and exits 2 when that gallery cannot load; it +does not score the matcher instead. The report names the backend, whether the head loaded, and the weight pins. The same pinned recording, catalog, weights, and labels produce the same counts; timings are omitted because they are not. @@ -102,9 +105,9 @@ def load_index(recording_dir: str) -> List[Dict[str, Any]]: def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]: """Backend named by the recording, when every present value agrees. - FrameRecorder does not write this. A later index may, on the first line or - on each frame, under identity_backend or identity.backend. Silence means - the caller must say --identity. A mix is not a default. + FrameRecorder writes identity_backend on each line when the server constructed + it with the active backend. Older indexes may omit it, or name it once under + identity.backend. Silence is not a backend. A mix is not a default. """ found: List[str] = [] for row in rows: @@ -124,13 +127,17 @@ def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]: def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str: - """CLI wins. A recording may fill the gap. Neither one stays the matcher.""" - chosen = (requested or recorded or "matcher").strip().lower() + """CLI and recording must name the same backend. Silence is not matcher.""" + chosen = (requested or recorded or "").strip().lower() + if not chosen: + raise EvalInputError( + "recording does not name an identity backend; pass --identity matcher or --identity gallery" + ) if chosen not in ("matcher", "gallery"): raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}") if recorded and requested and recorded != chosen: raise EvalInputError( - f"recording identity is {recorded}; pass --identity {recorded} or omit --identity" + f"recording identity is {recorded}; pass --identity {recorded}" ) return chosen @@ -219,7 +226,7 @@ def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[st def model_fingerprint( weights_path: str, *, - identity: str = "matcher", + identity: str, gallery: Any = None, ) -> Dict[str, Any]: """YOLO pin plus, for gallery, the backbone and head that actually loaded.""" @@ -545,7 +552,7 @@ def score_recording( labels_path: Optional[str] = None, iou_min: float = 0.5, live_frames: Optional[Sequence[Dict[str, Any]]] = None, - identity: str = "matcher", + identity: str, gallery: Any = None, ) -> Dict[str, Any]: """Score one session directory. live_frames, when given, are the replayed results. @@ -564,7 +571,9 @@ def score_recording( source = list(live_frames) if live_frames is not None else rows tracks = [track for frame in source for track in (frame.get("tracks") or [])] decisions = score_tracks(tracks) - backend = (identity or "matcher").strip().lower() + backend = identity.strip().lower() + if backend not in ("matcher", "gallery"): + raise EvalInputError(f"identity must be matcher or gallery, not {identity}") report = { "recording": recording_dir, "source": "live_replay" if live_frames is not None else "recorded_index", @@ -678,7 +687,7 @@ def format_report(report: Dict[str, Any]) -> str: head_note = f" head {head}" if head else "" lines = [ f"source: {report['source']}", - f"identity: {identity.get('backend', 'matcher')}{head_note}", + f"identity: {identity.get('backend')}{head_note}", ( f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " f"rejected {decisions['rejected']} unknown {decisions['unknown']} " @@ -782,7 +791,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "--identity", default=None, choices=["matcher", "gallery"], - help="which mechanism names products (default: recorded backend, else matcher)", + help="which mechanism names products (required when the recording omits identity_backend)", ) parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0") args = parser.parse_args(argv) diff --git a/recognization/code/record.py b/recognization/code/record.py index c753459..cba02f2 100644 --- a/recognization/code/record.py +++ b/recognization/code/record.py @@ -15,7 +15,8 @@ Layout: DIR//000123.jpg the frame, exactly as the client sent it DIR//index.jsonl one line per frame: - {"frame": 123, "path": "...", "at": 1699..., "summary": {...}, + {"frame": 123, "path": "...", "at": 1699..., "identity_backend": "gallery", + "summary": {...}, "tracks": [{"box": [...], "identity": {...}, "tag": {...}}]} Frames are written at most every `every` processed frames (default 3) so a long @@ -35,10 +36,17 @@ from typing import Any, Dict, List, Optional class FrameRecorder: """Write frames and their results to disk, for labelling later.""" - def __init__(self, root: str, every: int = 3, max_frames_per_session: int = 4000) -> None: + def __init__( + self, + root: str, + every: int = 3, + max_frames_per_session: int = 4000, + identity_backend: Optional[str] = None, + ) -> None: self.root = root self.every = max(1, int(every)) self.max_frames_per_session = int(max_frames_per_session) + self.identity_backend = (identity_backend or "").strip().lower() or None self._lock = threading.Lock() self._counts: Dict[str, int] = {} self._written: Dict[str, int] = {} @@ -86,6 +94,8 @@ class FrameRecorder: "summary": result.get("summary"), "tracks": tracks, } + if self.identity_backend: + line["identity_backend"] = self.identity_backend try: with open(os.path.join(directory, "index.jsonl"), "a", encoding="utf-8") as handle: handle.write(json.dumps(line, ensure_ascii=False) + "\n") diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py index f543358..afd816f 100644 --- a/recognization/code/test_eval_live.py +++ b/recognization/code/test_eval_live.py @@ -23,6 +23,8 @@ from eval_live import ( # noqa: E402 labelled_top1, leaked_labels, load_index, + recorded_identity, + resolve_identity, score_recording, score_tracks, track_churn, @@ -176,7 +178,7 @@ class LeakageTests(unittest.TestCase): ) with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: json.dump({"labels": [{"frame": 1, "box": [0, 0, 12, 12], "sku_id": "sku_a"}]}, handle) - report = score_recording(recording, stock_dir=stock) + report = score_recording(recording, stock_dir=stock, identity="matcher") self.assertEqual(report["leakage"]["excluded"], 1) self.assertEqual(report["labelled_top1"]["labels"], 0) self.assertEqual(report["labelled_top1"]["top1"], None) @@ -220,8 +222,8 @@ class CliTests(unittest.TestCase): handle.write(json.dumps(line) + "\n") with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: json.dump({"labels": [{"frame": 4, "box": [0, 0, 8, 8], "sku_id": "sku_a"}]}, handle) - first = score_recording(recording, stock_dir=stock) - second = score_recording(recording, stock_dir=stock) + first = score_recording(recording, stock_dir=stock, identity="matcher") + second = score_recording(recording, stock_dir=stock, identity="matcher") self.assertEqual(first, second) self.assertEqual(first["decisions"]["accepted"], 1) self.assertEqual(first["decisions"]["ambiguous"], 1) @@ -273,5 +275,26 @@ class IdentityContractTests(unittest.TestCase): self.assertFalse(os.path.exists(report_path)) self.assertNotIn("accepted_rate", completed.stdout) + def test_a_silent_recording_does_not_become_the_matcher(self): + with self.assertRaises(EvalInputError): + resolve_identity(None, None) + self.assertEqual(resolve_identity("matcher", None), "matcher") + self.assertEqual(resolve_identity(None, "gallery"), "gallery") + with self.assertRaises(EvalInputError): + resolve_identity("matcher", "gallery") + + def test_recorded_backend_is_read_and_a_mix_is_refused(self): + self.assertEqual(recorded_identity([{"identity_backend": "Gallery"}]), "gallery") + self.assertIsNone(recorded_identity([{"frame": 1}])) + self.assertEqual( + recorded_identity([{"identity": {"backend": "matcher", "sku_id": "sku_a"}}]), + "matcher", + ) + with self.assertRaises(EvalInputError): + recorded_identity([ + {"identity_backend": "gallery"}, + {"identity_backend": "matcher"}, + ]) + if __name__ == "__main__": unittest.main() diff --git a/recognization/code/test_record.py b/recognization/code/test_record.py index 140d23f..ed4d6df 100644 --- a/recognization/code/test_record.py +++ b/recognization/code/test_record.py @@ -45,6 +45,21 @@ class RecorderTests(unittest.TestCase): self.assertEqual([line["frame"] for line in lines], [0, 1, 2]) self.assertEqual(lines[0]["tracks"][0]["tag_identity"]["sku_id"], "sku_l05") + def test_the_active_backend_is_written_on_each_line(self): + recorder = FrameRecorder(self.root, every=1, identity_backend="gallery") + recorder.write("session-d", 0, b"x", a_result("sku_l05")) + recorder.write("session-d", 1, b"y", a_result()) + with open(os.path.join(self.root, "session-d", "index.jsonl"), encoding="utf-8") as handle: + lines = [json.loads(line) for line in handle] + self.assertEqual([line["identity_backend"] for line in lines], ["gallery", "gallery"]) + + def test_an_unspecified_backend_is_omitted(self): + recorder = FrameRecorder(self.root, every=1) + recorder.write("session-e", 0, b"x", a_result()) + with open(os.path.join(self.root, "session-e", "index.jsonl"), encoding="utf-8") as handle: + line = json.loads(handle.readline()) + self.assertNotIn("identity_backend", line) + def test_sampling_keeps_the_frame_numbering_visible(self): recorder = FrameRecorder(self.root, every=3) for frame_id in range(9):