vnguyen/product-recognition-demo
1
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: replay recorded walks with selected identity backend

Browse source
This commit is contained in:
kiyo9w 2026-09-28 14:17:14 +09:00
parent 2798322417
commit 44badcf193
2 changed files with 196 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

173
recognization/code/eval_live.py
View file

@ -11,10 +11,14 @@ store was enrolled; scoring them is training-set accuracy.
python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH]
[--json PATH] [--iou 0.5]
[--replay] [--identity matcher|gallery] [--device cpu|mps|cuda]
Prints a human count table and writes the same report as JSON (default
<recording>/eval_live.json). Missing input exits 2 with {"error": ...} on
stdout. The same pinned recording, catalog, weights, and labels produce the
stdout. --identity gallery uses the same LiveSession path as the server and
exits 2 when that gallery cannot load; it does not score the matcher instead.
The report names the backend, whether the head loaded, and the weight pins.
The same pinned recording, catalog, weights, and labels produce the
same counts; timings are omitted because they are not.
No held-out labels means denominators and fingerprints only. That is not a
@ -95,6 +99,42 @@ def load_index(recording_dir: str) -> List[Dict[str, Any]]:
return rows
def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]:
"""Backend named by the recording, when every present value agrees.
FrameRecorder does not write this. A later index may, on the first line or
on each frame, under identity_backend or identity.backend. Silence means
the caller must say --identity. A mix is not a default.
"""
found: List[str] = []
for row in rows:
raw = row.get("identity_backend")
if raw is None and isinstance(row.get("identity"), dict):
raw = row["identity"].get("backend")
if raw is None:
continue
name = str(raw).strip().lower()
if name not in ("matcher", "gallery"):
raise EvalInputError(f"recorded identity backend is not matcher or gallery: {raw}")
found.append(name)
unique = set(found)
if len(unique) > 1:
raise EvalInputError("recording names more than one identity backend")
return found[0] if found else None
def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str:
"""CLI wins. A recording may fill the gap. Neither one stays the matcher."""
chosen = (requested or recorded or "matcher").strip().lower()
if chosen not in ("matcher", "gallery"):
raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}")
if recorded and requested and recorded != chosen:
raise EvalInputError(
f"recording identity is {recorded}; pass --identity {recorded} or omit --identity"
)
return chosen
def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
"""Hash the JPEGs the client sent, in index order. Missing files are an error."""
digest = hashlib.sha256()
@ -176,17 +216,46 @@ def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[st
}
def model_fingerprint(weights_path: str) -> Dict[str, Any]:
def model_fingerprint(
weights_path: str,
*,
identity: str = "matcher",
gallery: Any = None,
) -> Dict[str, Any]:
"""YOLO pin plus, for gallery, the backbone and head that actually loaded."""
try:
import ultralytics
library = f"ultralytics-{ultralytics.__version__}"
except Exception:
library = "ultralytics-unknown"
return {
"version": f"{os.path.basename(weights_path)}@{library}",
pin: Dict[str, Any] = {
"version": f"{os.path.basename(weights_path)}@{library}" if weights_path else f"none@{library}",
"weights_sha256": file_sha256(weights_path) if weights_path else None,
"identity": identity,
}
if identity != "gallery" or gallery is None:
if identity == "gallery":
pin["gallery_loaded"] = False
pin["gallery_head_loaded"] = False
return pin
from gallery import DINOV2_SHA256, dinov2_weights_path
backbone = dinov2_weights_path()
digest = file_sha256(backbone)
pin.update(
{
"gallery_loaded": True,
"gallery_head": getattr(gallery, "head_reason", None),
"gallery_head_loaded": getattr(gallery, "_head", None) is not None,
"gallery_views": gallery.view_count() if hasattr(gallery, "view_count") else None,
"dinov2_path": backbone,
"dinov2_sha256": digest,
"dinov2_sha256_expected": DINOV2_SHA256,
"dinov2_pinned": digest == DINOV2_SHA256,
}
)
return pin
def _decode_image(payload: bytes) -> Any:
@ -476,11 +545,15 @@ def score_recording(
labels_path: Optional[str] = None,
iou_min: float = 0.5,
live_frames: Optional[Sequence[Dict[str, Any]]] = None,
identity: str = "matcher",
gallery: Any = None,
) -> Dict[str, Any]:
"""Score one session directory. live_frames, when given, are the replayed results.
Without live_frames the recorded index is what the server already decided.
That still produces denominators; it is not a fresh model run.
That still produces denominators; it is not a fresh model run. identity is
the backend those frames were named with, not a claim that this process
loaded it. gallery, when passed, is the loaded ProductGallery.
"""
recording_dir = os.path.abspath(recording_dir)
rows = load_index(recording_dir)
@ -491,12 +564,21 @@ def score_recording(
source = list(live_frames) if live_frames is not None else rows
tracks = [track for frame in source for track in (frame.get("tracks") or [])]
decisions = score_tracks(tracks)
backend = (identity or "matcher").strip().lower()
report = {
"recording": recording_dir,
"source": "live_replay" if live_frames is not None else "recorded_index",
"identity": {
"backend": backend,
"gallery_loaded": gallery is not None if backend == "gallery" else None,
"gallery_head": getattr(gallery, "head_reason", None) if gallery is not None else None,
"gallery_head_loaded": (
bool(getattr(gallery, "_head", None) is not None) if gallery is not None else None
),
},
"fingerprints": {
"sample": sample,
"model": model_fingerprint(weights_path),
"model": model_fingerprint(weights_path, identity=backend, gallery=gallery),
"catalog": catalog_fingerprint(stock_dir),
},
"decisions": decisions,
@ -590,8 +672,13 @@ def format_report(report: Dict[str, Any]) -> str:
top1 = report["labelled_top1"]
churn = report["churn"]
fingerprints = report["fingerprints"]
identity = report.get("identity") or {}
model = fingerprints["model"]
head = identity.get("gallery_head")
head_note = f" head {head}" if head else ""
lines = [
f"source: {report['source']}",
f"identity: {identity.get('backend', 'matcher')}{head_note}",
(
f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} "
f"rejected {decisions['rejected']} unknown {decisions['unknown']} "
@ -617,15 +704,66 @@ def format_report(report: Dict[str, Any]) -> str:
),
(
f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} "
f"model {fingerprints['model']['version']} "
f"model {model['version']} "
f"weights {str(model.get('weights_sha256') or 'none')[:12]} "
f"catalog {fingerprints['catalog']['version']}"
),
]
if model.get("identity") == "gallery":
lines.append(
f"gallery: loaded {model.get('gallery_loaded')} "
f"head {model.get('gallery_head')} "
f"dinov2 {str(model.get('dinov2_sha256') or 'missing')[:12]} "
f"pinned {model.get('dinov2_pinned')}"
)
for gap in report["evidence"]:
lines.append(f"gap: {gap}")
return "\n".join(lines)
def replay_engine(stock_dir: str, weights: str, identity: str, device: Optional[str]) -> Tuple[Any, str, Any]:
"""Build the engine the server would, and refuse a gallery that did not load.
The backbone file is checked before YOLO loads. A missing or unpinned file
is the same refusal the server makes, without scoring the matcher.
"""
from engine import EngineSettings, build_default_engine
if identity == "gallery":
from gallery import DINOV2_SHA256, dinov2_weights_path
backbone = dinov2_weights_path()
digest = file_sha256(backbone)
if digest is None:
raise EvalInputError(
f"gallery identity backend requires local DINOv2 weights at {backbone}; "
"refusing to score the matcher"
)
if digest != DINOV2_SHA256:
raise EvalInputError(
f"DINOv2 weights at {backbone} are sha256 {digest}, expected {DINOV2_SHA256}"
)
settings = EngineSettings(stock_dir=stock_dir, identity_backend=identity)
if weights:
settings.weights_path = weights
if device:
settings.device = device
engine = build_default_engine(settings)
chosen = (engine.settings.identity_backend or "matcher").lower()
if chosen != identity:
engine.shutdown()
raise EvalInputError(f"engine identity is {chosen}, requested {identity}")
gallery = None
if identity == "gallery":
gallery = engine.identity_gallery()
if gallery is None:
engine.shutdown()
raise EvalInputError(
"gallery identity backend did not load; refusing to score the matcher"
)
return engine, engine.settings.weights_path, gallery
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description="Score a recorded live walk.")
parser.add_argument("recording", nargs="?", help="session directory with index.jsonl")
@ -640,6 +778,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
help="re-run frames through LiveSession (needs the recognition stack)",
)
parser.add_argument("--method", default="hybrid")
parser.add_argument(
"--identity",
default=None,
choices=["matcher", "gallery"],
help="which mechanism names products (default: recorded backend, else matcher)",
)
parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0")
args = parser.parse_args(argv)
def fail(message: str) -> int:
@ -653,17 +798,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
if not os.path.isdir(recording):
return fail(f"recording directory not found: {recording}")
try:
rows = load_index(recording)
identity = resolve_identity(args.identity, recorded_identity(rows))
live_frames = None
gallery = None
weights = args.weights
if args.replay:
from engine import EngineSettings, build_default_engine
from live import LiveSettings
settings = EngineSettings(stock_dir=args.stock_dir)
if weights:
settings.weights_path = weights
engine = build_default_engine(settings)
weights = engine.settings.weights_path
engine, weights, gallery = replay_engine(
args.stock_dir, args.weights, identity, args.device
)
live_frames = replay_live(
recording, engine, settings=LiveSettings(method=args.method, tags=True)
)
@ -674,6 +819,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
labels_path=args.labels,
iou_min=args.iou,
live_frames=live_frames,
identity=identity,
gallery=gallery,
)
except EvalInputError as exc:
return fail(str(exc))

36
recognization/code/test_eval_live.py
View file

@ -237,5 +237,41 @@ class CliTests(unittest.TestCase):
load_index(root)
class IdentityContractTests(unittest.TestCase):
def test_missing_gallery_weights_exit_without_a_matcher_score(self):
root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, root, ignore_errors=True)
recording = os.path.join(root, "session")
os.makedirs(recording)
write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8))
with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write(json.dumps({"frame": 1, "path": "000000.jpg", "tracks": []}) + "\n")
missing = os.path.join(root, "no-such-dinov2.safetensors")
report_path = os.path.join(recording, "eval_live.json")
env = os.environ.copy()
env["DORAYAKI_DINOV2_WEIGHTS"] = missing
completed = subprocess.run(
[
sys.executable,
os.path.join(current_dir, "eval_live.py"),
recording,
"--replay",
"--identity",
"gallery",
"--json",
report_path,
],
capture_output=True,
text=True,
check=False,
env=env,
)
self.assertEqual(completed.returncode, 2)
payload = json.loads(completed.stdout)
self.assertIn("DINOv2", payload["error"])
self.assertIn("matcher", payload["error"])
self.assertFalse(os.path.exists(report_path))
self.assertNotIn("accepted_rate", completed.stdout)
if __name__ == "__main__":
unittest.main()