From e25fcda67dc1a6d3a3d8afd389954da29d7f59b2 Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Tue, 29 Sep 2026 03:29:56 +0900 Subject: [PATCH] fix(eval): confirm a leak signature before calling it one, and score external sets in code Enrolling a packshot from an external set is normal - it is how F-12 was closed - and it silently turns one of those rows into training data. The external sets were scored by an ad-hoc script that never checked, so sku_l06_1.jpg was being reported as a held-out correct accept while being byte-identical to the enrolled view. code/eval_external.py now scores those sets with the same leaked() check the shelf evaluator runs, and refuses a row that has since been enrolled. That check needed fixing first. Its 64x64 signature fires on two different photographs of the same product: the l09 rear label from a retailer page sits 19.5 from the enrolled front packshot, under the 20.0 line, while being a different photograph (resized MAD 26.3). A signature hit is now confirmed by correlation - a genuine re-cut, re-encode or rescale correlates at 0.99+, that pair at 0.28 - so the two signals together are decisive where either alone is not. Verified against both directions: identical, q55 re-encode and 700px rescale are caught; the different-photo pair is not. Also adds the van backend (always-argmax, no gate) so the stream comparison in DECISION.md is reproducible rather than asserted, with two tests pinning that it answers where the shipped gate abstains and still refuses an empty score set. External result on the corrected basis: 19 scored, 12 correct, 7 abstain, 0 false accepts, 1 leak excluded. Suite 307 -> 309. --- recognization/code/app.py | 5 +- recognization/code/engine.py | 9 +- recognization/code/eval_external.py | 116 ++++++++++++++++++ recognization/code/eval_identity.py | 42 ++++++- recognization/code/identity.py | 20 +++ recognization/code/live.py | 4 + recognization/code/test_identity.py | 17 +++ .../external-eval/kewpie/eval_result.json | 56 ++++----- .../external-eval/other-mayo/eval_result.json | 34 ++--- 9 files changed, 252 insertions(+), 51 deletions(-) create mode 100644 recognization/code/eval_external.py diff --git a/recognization/code/app.py b/recognization/code/app.py index b73a5d7..274346a 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -1010,8 +1010,9 @@ def main() -> None: parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) parser.add_argument("--identity", type=str, default=None, - choices=["matcher", "local", "gallery"], - help="which mechanism names products (default local)") + choices=["matcher", "local", "gallery", "van"], + help="which mechanism names products (default local; " + "van is the always-argmax baseline)") parser.add_argument("--live-inflight", type=int, default=None, help="classifications in flight per session") parser.add_argument("--live-attempts", type=int, default=None, diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 51aa1ad..3d803df 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -89,8 +89,11 @@ class EngineSettings: # calibrated open-set gate: measured on the corrected image6 labels it shows # 36 of 61 in-catalog facings at 95% precision with the text lane, against # 35 at 35% for Van's always-argmax matcher. `matcher` keeps Van's composite - # as a selectable baseline lane (E-1). `gallery` is fenced off: raw DINOv2 - # is 12-21% top-1 on 7 SKUs and its recall@10 falls to 7/42 at 493 SKUs. + # as a selectable baseline lane (E-1). `van` is that same scorer with the + # gate removed - it answers its top SKU every time, which is Van's actual + # contract and the only way to reproduce the stream comparison in + # DECISION.md. `gallery` is fenced off: raw DINOv2 is 12-21% top-1 on 7 + # SKUs and its recall@10 falls to 7/42 at 493 SKUs. identity_backend: str = "local" # Cosine similarity below which the gallery calls a facing unknown. Measured: # same-product pairs sit above 0.6, different products below 0.5. @@ -469,6 +472,8 @@ class RecognitionEngine: return "cosine" if backend == "local": return "local" + if backend == "van": + return "van" return "inlier" def identity_text_lane(self) -> Any: diff --git a/recognization/code/eval_external.py b/recognization/code/eval_external.py new file mode 100644 index 0000000..631c1ee --- /dev/null +++ b/recognization/code/eval_external.py @@ -0,0 +1,116 @@ +"""Score external-proxy image sets through the shipped identity lane. + +These sets are not Tier 2 (see data/external-eval/README.md). They exist to test +the pipeline on data it had no part in building, so the one thing this script +must never do is score an image that has since been enrolled - that is +training-set accuracy wearing a held-out label. + +`leaked()` is the same check the shelf evaluator runs, and it is applied here +for that reason: enrolling a packshot from an external set is a normal thing to +do (it is how F-12 was closed), and it silently converts one of these rows into +a leak unless something refuses it. + + .venv/bin/python code/eval_external.py --stock stock-lopia-clean \ + data/external-eval/kewpie data/external-eval/other-mayo +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +from typing import Any, Dict, List + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 + +from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter, leaked # noqa: E402 + + +def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]: + rows: List[Dict[str, Any]] = [] + for path in sorted(glob.glob(os.path.join(directory, "*.jpg"))): + name = os.path.basename(path) + truth = "sku_" + name.split("_")[1] + image = cv2.imread(path) + if image is None: + rows.append({"file": name, "truth": truth, "verdict": "unreadable"}) + continue + # The whole image is the crop: an external packshot is not a shelf facing. + box = (0, 0, image.shape[1], image.shape[0]) + leak = leaked(image, box, stock_dir) + if leak: + rows.append({ + "file": name, "truth": truth, "verdict": "leaked", + "leaked_into": leak, + "note": "enrolled in the catalog; scoring it would be training accuracy", + }) + continue + decision = lane.decide( + image, lane.rank(image), accept=LOCAL_ACCEPT, margin=0.0, scale="full" + ) + best = decision.get("best_sku") + verdict = "correct" if best == truth else ("abstain" if not best else "wrong") + rows.append({ + "file": name, "truth": truth, "best": best, + "confidence": decision.get("confidence"), + "status": decision.get("status"), + "source": decision.get("source"), + "verdict": verdict, + }) + + scored = [r for r in rows if r["verdict"] in ("correct", "abstain", "wrong")] + counts = {v: sum(1 for r in scored if r["verdict"] == v) + for v in ("correct", "abstain", "wrong")} + return { + "dataset": os.path.basename(directory.rstrip("/")), + "catalog": stock_dir, + "note": "External proxy images, not Tier 2. Scored through the shipped " + "LocalLaneAdapter + DecisionPolicy.", + "rows": rows, + "scored": len(scored), + "leaked": sum(1 for r in rows if r["verdict"] == "leaked"), + **counts, + } + + +def main(argv: List[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directories", nargs="+", help="external set directories") + parser.add_argument("--stock", default="stock-lopia-clean") + parser.add_argument("--write", action="store_true", + help="write eval_result.json into each set directory") + args = parser.parse_args(argv) + + lane = LocalLaneAdapter(args.stock) + totals = {"correct": 0, "abstain": 0, "wrong": 0, "leaked": 0, "scored": 0} + for directory in args.directories: + report = score_set(lane, directory, args.stock) + for key in totals: + totals[key] += report[key] + print(f"{report['dataset']}: correct {report['correct']} | " + f"abstain {report['abstain']} | wrong {report['wrong']} | " + f"leaked {report['leaked']} of {report['scored']} scored") + for row in report["rows"]: + if row["verdict"] in ("wrong", "leaked"): + print(f" {row['verdict'].upper():<7} {row['file']} " + f"truth={row['truth']} best={row.get('best')} " + f"{row.get('leaked_into', '')}") + if args.write: + out = os.path.join(directory, "eval_result.json") + with open(out, "w") as handle: + json.dump(report, handle, ensure_ascii=False, indent=1) + print(f" wrote {out}") + print(f"\nTOTAL: {totals['scored']} scored, correct {totals['correct']}, " + f"abstain {totals['abstain']}, FALSE ACCEPTS {totals['wrong']}, " + f"leaked {totals['leaked']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index 2f9d5f8..f6011fb 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -78,6 +78,12 @@ LIVE_LONG_SIDE = 1280 # the enrollment protocol's session manifest (DECISION ยง8); this is the cheap # backstop that runs without one. LEAK_SIGNATURE_MAD = 20.0 +# A signature hit is confirmed by correlation before it counts as a leak. A +# genuine re-cut, re-encode or rescale of an enrolled view correlates at 0.99+; +# two different photographs of the same product sit near 0.3. Measured on the +# l09 pair that the signature alone misjudged (0.28) and on re-encoded and +# rescaled copies of the enrolled view (0.999, 0.997). +LEAK_CORRELATION = 0.90 def load_labels(path: str) -> Dict[str, Any]: @@ -219,6 +225,24 @@ def _catalog_templates(stock_dir: str) -> List[Tuple[str, str, Tuple[int, int], return cached +def _correlation(left: Any, right: Any) -> float: + """Mean-subtracted correlation of two images, resizing `right` to `left`. + + Scale- and re-encode-tolerant, unlike a pixel comparison: the enrolled view + and a re-saved or rescaled copy of it correlate at 0.99+, while two + different photographs of the same product sit near 0.3. + """ + if right.shape[:2] != left.shape[:2]: + right = cv2.resize(right, (left.shape[1], left.shape[0]), + interpolation=cv2.INTER_AREA) + a = cv2.cvtColor(left, cv2.COLOR_BGR2GRAY).astype(np.float32).ravel() + b = cv2.cvtColor(right, cv2.COLOR_BGR2GRAY).astype(np.float32).ravel() + a -= a.mean() + b -= b.mean() + denom = float(np.linalg.norm(a) * np.linalg.norm(b)) + return float((a * b).sum() / denom) if denom else 0.0 + + def leaked(image: Any, box: Sequence[int], stock_dir: str) -> Optional[str]: """Name of the catalog template this crop is, or None when it is held out. @@ -238,9 +262,21 @@ def leaked(image: Any, box: Sequence[int], stock_dir: str) -> Optional[str]: return None crop_signatures = _signatures(crop) for stem, name, shape, signatures in _catalog_templates(stock_dir): - if min(float(np.abs(signature - crop_signature).mean()) - for signature in signatures for crop_signature in crop_signatures) < LEAK_SIGNATURE_MAD: - return stem + close = min(float(np.abs(signature - crop_signature).mean()) + for signature in signatures for crop_signature in crop_signatures) + if close < LEAK_SIGNATURE_MAD: + # The signature is a 64x64 grayscale thumbnail, so it is coarse + # enough to fire on two different photos of the same product: the + # l09 rear label from a retailer page sits 19.5 from the enrolled + # front packshot, under the 20.0 line, while being a different + # photograph (resized MAD 26.3). Confirm before calling it a leak. + # A genuine re-cut or re-encode correlates at 0.99+; that pair + # correlates at 0.28, so the two signals together are decisive + # where either alone is not. + template = cv2.imread(os.path.join(stock_dir, name)) + if template is None or _correlation(crop, template) >= LEAK_CORRELATION: + return stem + continue if shape != crop.shape[:2]: continue template = cv2.imread(os.path.join(stock_dir, name)) diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 51fa3bb..5028401 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -248,6 +248,26 @@ def decide_identity( "message": message, } + if scale == "van": + # Van's contract: the top SKU, always answered. No gate, no abstention - + # this is the baseline the decision is measured against, and it exists + # so the stream comparison is reproducible rather than asserted. + if top_score <= 0.0: + level, message, best = IDENTITY_REJECTED, "No matching evidence", None + else: + level, message, best = IDENTITY_ACCEPTED, "Always-argmax baseline", top_sku + return { + "status": "success" if level == IDENTITY_ACCEPTED else level, + "method": method, + "best_sku": best, + "score": round(top_score, 2), + "confidence": confidence, + "identity_level": level, + "ranked_candidates": ranked, + "details": top_details, + "message": message, + } + if top_score <= 0.0: level = IDENTITY_REJECTED message = "No matching evidence" diff --git a/recognization/code/live.py b/recognization/code/live.py index 4fcef5c..de36458 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -1301,6 +1301,10 @@ class LiveSession: decision = DecisionPolicy( signatures=text_lane.signatures if text_lane else None ).decide(list(scores.items()), text_rank, text_read=text_read) + elif scale == "van": + # Van's contract on the stream: the aggregate's top SKU, always + # answered. Same scorer as `matcher`, no gate. + decision = decide_identity(mean_scores, method=method, scale="van") else: decision = decide_identity(mean_scores, method=method, scale=scale) track.candidates = [ diff --git a/recognization/code/test_identity.py b/recognization/code/test_identity.py index ef2904d..96849c4 100644 --- a/recognization/code/test_identity.py +++ b/recognization/code/test_identity.py @@ -50,3 +50,20 @@ class IdentityRankingTests(unittest.TestCase): self.assertEqual(ranked["identity_level"], IDENTITY_ACCEPTED) self.assertEqual(ranked["best_sku"], "sku_a") self.assertEqual(ranked["status"], "success") + + def test_van_answers_what_the_gate_would_refuse(self): + # Van's contract is the baseline the decision is measured against, so it + # must answer exactly where the shipped gate abstains. A weak score and + # a close pair are both refused above; here both are named. + weak = decide_identity({"sku_a": 0.4, "sku_b": 0.0}, scale="van") + self.assertEqual(weak["identity_level"], IDENTITY_ACCEPTED) + self.assertEqual(weak["best_sku"], "sku_a") + close = decide_identity({"sku_a": 20.0, "sku_b": 19.5}, scale="van") + self.assertEqual(close["best_sku"], "sku_a") + + def test_van_still_rejects_when_there_is_no_evidence(self): + # Always-argmax is not always-answer: with nothing to rank there is no + # argmax, and naming a SKU from an empty score set would be a fabrication. + ranked = decide_identity({}, scale="van") + self.assertEqual(ranked["identity_level"], IDENTITY_REJECTED) + self.assertIsNone(ranked["best_sku"]) diff --git a/recognization/data/external-eval/kewpie/eval_result.json b/recognization/data/external-eval/kewpie/eval_result.json index 5fdddaf..91591b5 100644 --- a/recognization/data/external-eval/kewpie/eval_result.json +++ b/recognization/data/external-eval/kewpie/eval_result.json @@ -7,112 +7,112 @@ "file": "sku_l01_1.jpg", "truth": "sku_l01", "best": "sku_l01", - "conf": 13.1618, + "confidence": 13.1618, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l01_2.jpg", "truth": "sku_l01", "best": "sku_l01", - "conf": 20.986, + "confidence": 20.986, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l02_1.jpg", "truth": "sku_l02", "best": "sku_l02", - "conf": 10.4288, + "confidence": 10.4288, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l02_2.jpg", "truth": "sku_l02", "best": "sku_l02", - "conf": 7.542, + "confidence": 7.542, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l03_1.jpg", "truth": "sku_l03", "best": "sku_l03", - "conf": 14.3581, + "confidence": 14.3581, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l03_2.jpg", "truth": "sku_l03", "best": "sku_l03", - "conf": 48.9767, + "confidence": 48.9767, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l05_1.jpg", "truth": "sku_l05", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l05_2.jpg", "truth": "sku_l05", "best": "sku_l05", - "conf": -3.9674, + "confidence": -3.9674, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l06_1.jpg", "truth": "sku_l06", - "best": "sku_l06", - "conf": 93.1283, - "status": "success", - "source": "local", - "verdict": "OK" + "verdict": "leaked", + "leaked_into": "sku_l06", + "note": "enrolled in the catalog; scoring it would be training accuracy" }, { "file": "sku_l06_2.jpg", "truth": "sku_l06", "best": "sku_l06", - "conf": 294.8985, + "confidence": 294.8985, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l11_1.jpg", "truth": "sku_l11", "best": null, - "conf": -29.9175, + "confidence": -29.9175, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l11_2.jpg", "truth": "sku_l11", "best": null, - "conf": 14.1552, + "confidence": 14.1552, "status": "ambiguous", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" } ], - "correct": 9, + "scored": 11, + "leaked": 1, + "correct": 8, "abstain": 3, "wrong": 0 } \ No newline at end of file diff --git a/recognization/data/external-eval/other-mayo/eval_result.json b/recognization/data/external-eval/other-mayo/eval_result.json index 762176c..eed3924 100644 --- a/recognization/data/external-eval/other-mayo/eval_result.json +++ b/recognization/data/external-eval/other-mayo/eval_result.json @@ -7,75 +7,77 @@ "file": "sku_l04_1.jpg", "truth": "sku_l04", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "rejected", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_2.jpg", "truth": "sku_l04", "best": null, - "conf": 0.0, + "confidence": 0.0, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_3.jpg", "truth": "sku_l04", "best": null, - "conf": 0.3294, + "confidence": 0.3294, "status": "ambiguous", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l04_4.jpg", "truth": "sku_l04", "best": null, - "conf": -1.6712, + "confidence": -1.6712, "status": "unknown", "source": null, - "verdict": "ABSTAIN" + "verdict": "abstain" }, { "file": "sku_l07_1.jpg", "truth": "sku_l07", "best": "sku_l07", - "conf": 0.0, + "confidence": 0.0, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l07_2.jpg", "truth": "sku_l07", "best": "sku_l07", - "conf": -0.5503, + "confidence": -0.5503, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l09_1.jpg", "truth": "sku_l09", "best": "sku_l09", - "conf": 27.9546, + "confidence": 27.9546, "status": "success", "source": "local", - "verdict": "OK" + "verdict": "correct" }, { "file": "sku_l09_2.jpg", "truth": "sku_l09", "best": "sku_l09", - "conf": 0.975, + "confidence": 0.975, "status": "success", "source": "text", - "verdict": "OK" + "verdict": "correct" } ], + "scored": 8, + "leaked": 0, "correct": 4, "abstain": 4, "wrong": 0