diff --git a/recognization/code/app.py b/recognization/code/app.py index b73a5d7..274346a 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -1010,8 +1010,9 @@ def main() -> None: parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) parser.add_argument("--identity", type=str, default=None, - choices=["matcher", "local", "gallery"], - help="which mechanism names products (default local)") + choices=["matcher", "local", "gallery", "van"], + help="which mechanism names products (default local; " + "van is the always-argmax baseline)") parser.add_argument("--live-inflight", type=int, default=None, help="classifications in flight per session") parser.add_argument("--live-attempts", type=int, default=None, diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 51aa1ad..3d803df 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -89,8 +89,11 @@ class EngineSettings: # calibrated open-set gate: measured on the corrected image6 labels it shows # 36 of 61 in-catalog facings at 95% precision with the text lane, against # 35 at 35% for Van's always-argmax matcher. `matcher` keeps Van's composite - # as a selectable baseline lane (E-1). `gallery` is fenced off: raw DINOv2 - # is 12-21% top-1 on 7 SKUs and its recall@10 falls to 7/42 at 493 SKUs. + # as a selectable baseline lane (E-1). `van` is that same scorer with the + # gate removed - it answers its top SKU every time, which is Van's actual + # contract and the only way to reproduce the stream comparison in + # DECISION.md. `gallery` is fenced off: raw DINOv2 is 12-21% top-1 on 7 + # SKUs and its recall@10 falls to 7/42 at 493 SKUs. identity_backend: str = "local" # Cosine similarity below which the gallery calls a facing unknown. Measured: # same-product pairs sit above 0.6, different products below 0.5. @@ -469,6 +472,8 @@ class RecognitionEngine: return "cosine" if backend == "local": return "local" + if backend == "van": + return "van" return "inlier" def identity_text_lane(self) -> Any: diff --git a/recognization/code/eval_external.py b/recognization/code/eval_external.py new file mode 100644 index 0000000..631c1ee --- /dev/null +++ b/recognization/code/eval_external.py @@ -0,0 +1,116 @@ +"""Score external-proxy image sets through the shipped identity lane. + +These sets are not Tier 2 (see data/external-eval/README.md). They exist to test +the pipeline on data it had no part in building, so the one thing this script +must never do is score an image that has since been enrolled - that is +training-set accuracy wearing a held-out label. + +`leaked()` is the same check the shelf evaluator runs, and it is applied here +for that reason: enrolling a packshot from an external set is a normal thing to +do (it is how F-12 was closed), and it silently converts one of these rows into +a leak unless something refuses it. + + .venv/bin/python code/eval_external.py --stock stock-lopia-clean \ + data/external-eval/kewpie data/external-eval/other-mayo +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +from typing import Any, Dict, List + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 + +from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter, leaked # noqa: E402 + + +def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]: + rows: List[Dict[str, Any]] = [] + for path in sorted(glob.glob(os.path.join(directory, "*.jpg"))): + name = os.path.basename(path) + truth = "sku_" + name.split("_")[1] + image = cv2.imread(path) + if image is None: + rows.append({"file": name, "truth": truth, "verdict": "unreadable"}) + continue + # The whole image is the crop: an external packshot is not a shelf facing. + box = (0, 0, image.shape[1], image.shape[0]) + leak = leaked(image, box, stock_dir) + if leak: + rows.append({ + "file": name, "truth": truth, "verdict": "leaked", + "leaked_into": leak, + "note": "enrolled in the catalog; scoring it would be training accuracy", + }) + continue + decision = lane.decide( + image, lane.rank(image), accept=LOCAL_ACCEPT, margin=0.0, scale="full" + ) + best = decision.get("best_sku") + verdict = "correct" if best == truth else ("abstain" if not best else "wrong") + rows.append({ + "file": name, "truth": truth, "best": best, + "confidence": decision.get("confidence"), + "status": decision.get("status"), + "source": decision.get("source"), + "verdict": verdict, + }) + + scored = [r for r in rows if r["verdict"] in ("correct", "abstain", "wrong")] + counts = {v: sum(1 for r in scored if r["verdict"] == v) + for v in ("correct", "abstain", "wrong")} + return { + "dataset": os.path.basename(directory.rstrip("/")), + "catalog": stock_dir, + "note": "External proxy images, not Tier 2. Scored through the shipped " + "LocalLaneAdapter + DecisionPolicy.", + "rows": rows, + "scored": len(scored), + "leaked": sum(1 for r in rows if r["verdict"] == "leaked"), + **counts, + } + + +def main(argv: List[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directories", nargs="+", help="external set directories") + parser.add_argument("--stock", default="stock-lopia-clean") + parser.add_argument("--write", action="store_true", + help="write eval_result.json into each set directory") + args = parser.parse_args(argv) + + lane = LocalLaneAdapter(args.stock) + totals = {"correct": 0, "abstain": 0, "wrong": 0, "leaked": 0, "scored": 0} + for directory in args.directories: + report = score_set(lane, directory, args.stock) + for key in totals: + totals[key] += report[key] + print(f"{report['dataset']}: correct {report['correct']} | " + f"abstain {report['abstain']} | wrong {report['wrong']} | " + f"leaked {report['leaked']} of {report['scored']} scored") + for row in report["rows"]: + if row["verdict"] in ("wrong", "leaked"): + print(f" {row['verdict'].upper():<7} {row['file']} " + f"truth={row['truth']} best={row.get('best')} " + f"{row.get('leaked_into', '')}") + if args.write: + out = os.path.join(directory, "eval_result.json") + with open(out, "w") as handle: + json.dump(report, handle, ensure_ascii=False, indent=1) + print(f" wrote {out}") + print(f"\nTOTAL: {totals['scored']} scored, correct {totals['correct']}, " + f"abstain {totals['abstain']}, FALSE ACCEPTS {totals['wrong']}, " + f"leaked {totals['leaked']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index 2f9d5f8..f6011fb 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -78,6 +78,12 @@ LIVE_LONG_SIDE = 1280 # the enrollment protocol's session manifest (DECISION ยง8); this is the cheap # backstop that runs without one. LEAK_SIGNATURE_MAD = 20.0 +# A signature hit is confirmed by correlation before it counts as a leak. A +# genuine re-cut, re-encode or rescale of an enrolled view correlates at 0.99+; +# two different photographs of the same product sit near 0.3. Measured on the +# l09 pair that the signature alone misjudged (0.28) and on re-encoded and +# rescaled copies of the enrolled view (0.999, 0.997). +LEAK_CORRELATION = 0.90 def load_labels(path: str) -> Dict[str, Any]: @@ -219,6 +225,24 @@ def _catalog_templates(stock_dir: str) -> List[Tuple[str, str, Tuple[int, int], return cached +def _correlation(left: Any, right: Any) -> float: + """Mean-subtracted correlation of two images, resizing `right` to `left`. + + Scale- and re-encode-tolerant, unlike a pixel comparison: the enrolled view + and a re-saved or rescaled copy of it correlate at 0.99+, while two + different photographs of the same product sit near 0.3. + """ + if right.shape[:2] != left.shape[:2]: + right = cv2.resize(right, (left.shape[1], left.shape[0]), + interpolation=cv2.INTER_AREA) + a = cv2.cvtColor(left, cv2.COLOR_BGR2GRAY).astype(np.float32).ravel() + b = cv2.cvtColor(right, cv2.COLOR_BGR2GRAY).astype(np.float32).ravel() + a -= a.mean() + b -= b.mean() + denom = float(np.linalg.norm(a) * np.linalg.norm(b)) + return float((a * b).sum() / denom) if denom else 0.0 + + def leaked(image: Any, box: Sequence[int], stock_dir: str) -> Optional[str]: """Name of the catalog template this crop is, or None when it is held out. @@ -238,9 +262,21 @@ def leaked(image: Any, box: Sequence[int], stock_dir: str) -> Optional[str]: return None crop_signatures = _signatures(crop) for stem, name, shape, signatures in _catalog_templates(stock_dir): - if min(float(np.abs(signature - crop_signature).mean()) - for signature in signatures for crop_signature in crop_signatures) < LEAK_SIGNATURE_MAD: - return stem + close = min(float(np.abs(signature - crop_signature).mean()) + for signature in signatures for crop_signature in crop_signatures) + if close < LEAK_SIGNATURE_MAD: + # The signature is a 64x64 grayscale thumbnail, so it is coarse + # enough to fire on two different photos of the same product: the + # l09 rear label from a retailer page sits 19.5 from the enrolled + # front packshot, under the 20.0 line, while being a different + # photograph (resized MAD 26.3). Confirm before calling it a leak. + # A genuine re-cut or re-encode correlates at 0.99+; that pair + # correlates at 0.28, so the two signals together are decisive + # where either alone is not. + template = cv2.imread(os.path.join(stock_dir, name)) + if template is None or _correlation(crop, template) >= LEAK_CORRELATION: + return stem + continue if shape != crop.shape[:2]: continue template = cv2.imread(os.path.join(stock_dir, name)) diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 51fa3bb..5028401 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -248,6 +248,26 @@ def decide_identity( "message": message, } + if scale == "van": + # Van's contract: the top SKU, always answered. No gate, no abstention - + # this is the baseline the decision is measured against, and it exists + # so the stream comparison is reproducible rather than asserted. + if top_score <= 0.0: + level, message, best = IDENTITY_REJECTED, "No matching evidence", None + else: + level, message, best = IDENTITY_ACCEPTED, "Always-argmax baseline", top_sku + return { + "status": "success" if level == IDENTITY_ACCEPTED else level, + "method": method, + "best_sku": best, + "score": round(top_score, 2), + "confidence": confidence, + "identity_level": level, + "ranked_candidates": ranked, + "details": top_details, + "message": message, + } + if top_score <= 0.0: level = IDENTITY_REJECTED message = "No matching evidence" diff --git a/recognization/code/live.py b/recognization/code/live.py index 4fcef5c..de36458 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -1301,6 +1301,10 @@ class LiveSession: decision = DecisionPolicy( signatures=text_lane.signatures if text_lane else None ).decide(list(scores.items()), text_rank, text_read=text_read) + elif scale == "van": + # Van's contract on the stream: the aggregate's top SKU, always + # answered. Same scorer as `matcher`, no gate. + decision = decide_identity(mean_scores, method=method, scale="van") else: decision = decide_identity(mean_scores, method=method, scale=scale) track.candidates = [ diff --git a/recognization/code/test_identity.py b/recognization/code/test_identity.py index ef2904d..96849c4 100644 --- a/recognization/code/test_identity.py +++ b/recognization/code/test_identity.py @@ -50,3 +50,20 @@ class IdentityRankingTests(unittest.TestCase): self.assertEqual(ranked["identity_level"], IDENTITY_ACCEPTED) self.assertEqual(ranked["best_sku"], "sku_a") self.assertEqual(ranked["status"], "success") + + def test_van_answers_what_the_gate_would_refuse(self): + # Van's contract is the baseline the decision is measured against, so it + # must answer exactly where the shipped gate abstains. A weak score and + # a close pair are both refused above; here both are named. + weak = decide_identity({"sku_a": 0.4, "sku_b": 0.0}, scale="van") + self.assertEqual(weak["identity_level"], IDENTITY_ACCEPTED) + self.assertEqual(weak["best_sku"], "sku_a") + close = decide_identity({"sku_a": 20.0, "sku_b": 19.5}, scale="van") + self.assertEqual(close["best_sku"], "sku_a") + + def test_van_still_rejects_when_there_is_no_evidence(self): + # Always-argmax is not always-answer: with nothing to rank there is no + # argmax, and naming a SKU from an empty score set would be a fabrication. + ranked = decide_identity({}, scale="van") + self.assertEqual(ranked["identity_level"], IDENTITY_REJECTED) + self.assertIsNone(ranked["best_sku"]) diff --git a/recognization/data/external-eval/kewpie/eval_result.json b/recognization/data/external-eval/kewpie/eval_result.json index 5fdddaf..91591b5 100644 --- a/recognization/data/external-eval/kewpie/eval_result.json +++ b/recognization/data/external-eval/kewpie/eval_result.json @@ -7,112 +7,112 @@ "file": "sku_l01_1.jpg", "truth": "sku_l01", "best": "sku_l01", - "conf": 13.1618, + "confidence": 13.1618, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l01_2.jpg", "truth": "sku_l01", "best": "sku_l01", - "conf": 20.986, + "confidence": 20.986, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l02_1.jpg", "truth": "sku_l02", "best": "sku_l02", - "conf": 10.4288, + "confidence": 10.4288, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l02_2.jpg", "truth": "sku_l02", "best": "sku_l02", - "conf": 7.542, + "confidence": 7.542, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l03_1.jpg", "truth": "sku_l03", "best": "sku_l03", - "conf": 14.3581, + "confidence": 14.3581, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l03_2.jpg", "truth": "sku_l03", "best": "sku_l03", - "conf": 48.9767, + "confidence": 48.9767, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l05_1.jpg", "truth": "sku_l05", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l05_2.jpg", "truth": "sku_l05", "best": "sku_l05", - "conf": -3.9674, + "confidence": -3.9674, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l06_1.jpg", "truth": "sku_l06", - "best": "sku_l06", - "conf": 93.1283, - "status": "success", - "source": "local", - "verdict": "OK" + "verdict": "leaked", + "leaked_into": "sku_l06", + "note": "enrolled in the catalog; scoring it would be training accuracy" }, { "file": "sku_l06_2.jpg", "truth": "sku_l06", "best": "sku_l06", - "conf": 294.8985, + "confidence": 294.8985, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l11_1.jpg", "truth": "sku_l11", "best": null, - "conf": -29.9175, + "confidence": -29.9175, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l11_2.jpg", "truth": "sku_l11", "best": null, - "conf": 14.1552, + "confidence": 14.1552, "status": "ambiguous", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" } ], - "correct": 9, + "scored": 11, + "leaked": 1, + "correct": 8, "abstain": 3, "wrong": 0 } \ No newline at end of file diff --git a/recognization/data/external-eval/other-mayo/eval_result.json b/recognization/data/external-eval/other-mayo/eval_result.json index 762176c..eed3924 100644 --- a/recognization/data/external-eval/other-mayo/eval_result.json +++ b/recognization/data/external-eval/other-mayo/eval_result.json @@ -7,75 +7,77 @@ "file": "sku_l04_1.jpg", "truth": "sku_l04", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "rejected", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_2.jpg", "truth": "sku_l04", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_3.jpg", "truth": "sku_l04", "best": null, - "conf": 0.3294, + "confidence": 0.3294, "status": "ambiguous", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_4.jpg", "truth": "sku_l04", "best": null, - "conf": -1.6712, + "confidence": -1.6712, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l07_1.jpg", "truth": "sku_l07", "best": "sku_l07", - "conf": 0.0, + "confidence": 0.0, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l07_2.jpg", "truth": "sku_l07", "best": "sku_l07", - "conf": -0.5503, + "confidence": -0.5503, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l09_1.jpg", "truth": "sku_l09", "best": "sku_l09", - "conf": 27.9546, + "confidence": 27.9546, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l09_2.jpg", "truth": "sku_l09", "best": "sku_l09", - "conf": 0.975, + "confidence": 0.975, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" } ], + "scored": 8, + "leaked": 0, "correct": 4, "abstain": 4, "wrong": 0