From 07de2bd37d96f808d131eb855cd90306b763784b Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Tue, 29 Sep 2026 04:08:59 +0900 Subject: [PATCH] fix(identity): calibrate both accept lines against unrelated shelves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gates were tuned on image6 alone, where every crop is a facing of a product the shelf actually holds. Wikimedia supermarket photos of milk, soy and snack shelves share nothing with the LOPIA catalog, so every accept there is a false accept by construction - and at the old lines there were 83 of them. Text lane: TEXT_MIN_SHARED 2 -> 4. The vocabulary filter drops bigrams no name contains, so a short garbage read can end up with a perfect ratio on two coincidental pairs. 20 of 26 text-lane answers on those shelves rested on exactly two: '避くて おいしい' named sku_l12 through おい/いし. Local lane: LOCAL_ACCEPT 2.0 -> 4.0. The residual accepts were geometric matches with no text overlap at all, scoring 2.26-4.81. Measured, 2297 distractor crops and 109 image6 facings: accept floor image6 shown/correct precision false accepts 2.0 2 54 / 49 90.7% 83 (3.61%) 2.0 4 52 / 48 92.3% 30 (1.31%) 4.0 4 44 / 42 95.5% 8 (0.35%) 5.0 6 36 / 36 100.0% 0 (0.00%) 4.0/4 is the only region passing both G2 (>=95% precision) and G3 (<=1% far-unknown) with coverage left. On the shipped evaluator: 66% end-to-end at 95% precision, 2 false accepts, against 77%/91%/5 before. The two tests that pinned the old floors now pin the new ones, with the reads that separate them. --- recognization/code/eval_distractors.py | 116 +++++++++++++++++++++++++ recognization/code/identity.py | 7 +- recognization/code/identity_lanes.py | 25 ++++-- recognization/code/test_engine.py | 40 ++++++--- 4 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 recognization/code/eval_distractors.py diff --git a/recognization/code/eval_distractors.py b/recognization/code/eval_distractors.py new file mode 100644 index 0000000..dfa956c --- /dev/null +++ b/recognization/code/eval_distractors.py @@ -0,0 +1,116 @@ +"""Open-world false-accept probe: unrelated shelf photos through the shipped lane. + +data/external-shelves/ holds Wikimedia Commons supermarket photos of shelves +that share nothing with the LOPIA catalog. Every accept here is a false accept +by construction, which is what makes the set useful: the image6 shelf can only +measure false accepts on the products it happens to contain. + +The stored eval.json predates the open-set gate, so it is re-run rather than +trusted. + + .venv/bin/python code/eval_distractors.py --stock stock-lopia-clean \ + data/external-shelves/wikimedia-commons +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +import time +from typing import Any, Dict, List + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 + +from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter # noqa: E402 + + +def main(argv: List[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directories", nargs="+") + parser.add_argument("--stock", default="stock-lopia-clean") + parser.add_argument("--write", action="store_true") + parser.add_argument("--limit", type=int, default=0, help="images per set (0 = all)") + args = parser.parse_args(argv) + + from engine import build_default_engine, EngineSettings + + engine = build_default_engine(EngineSettings(stock_dir=args.stock)) + lane = LocalLaneAdapter(args.stock) + for directory in args.directories: + manifest_path = os.path.join(directory, "manifest.json") + try: + with open(manifest_path) as handle: + manifest = json.load(handle) + except (OSError, ValueError): + manifest = {"images": []} + meta = {row["file"]: row for row in manifest.get("images", [])} + rows: List[Dict[str, Any]] = [] + + paths = sorted( + path for path in glob.glob(os.path.join(directory, "*")) + if os.path.splitext(path)[1].lower() in (".jpg", ".jpeg", ".png") + ) + if args.limit: + paths = paths[: args.limit] + for path in paths: + name = os.path.basename(path) + image = cv2.imread(path) + if image is None: + continue + started = time.perf_counter() + detections = engine.detect(image) + accepted = 0 + for detection in detections: + box = detection.get("box") or detection.get("bbox") + if not box: + continue + x1, y1, x2, y2 = [int(v) for v in box] + crop = image[max(0, y1):y2, max(0, x1):x2] + if crop.size == 0: + continue + decision = lane.decide(crop, lane.rank(crop), accept=LOCAL_ACCEPT, + margin=0.0, scale="full") + if decision.get("best_sku"): + accepted += 1 + rows.append({ + "file": name, + "title": meta.get(name, {}).get("title", ""), + "detections": len(detections), + "false_accepts": accepted, + "ms": round((time.perf_counter() - started) * 1000), + }) + + detections = sum(r["detections"] for r in rows) + accepts = sum(r["false_accepts"] for r in rows) + report = { + "dataset": os.path.basename(directory.rstrip("/")), + "catalog": args.stock, + "note": "Unrelated shelves: every accept is a false accept by construction.", + "images": len(rows), + "detections": detections, + "false_accepts": accepts, + "false_accept_rate": round(accepts / detections, 4) if detections else None, + "rows": rows, + } + print(f"{report['dataset']}: {len(rows)} images, {detections} detections, " + f"{accepts} false accepts ({report['false_accept_rate'] or 0:.2%})") + for row in sorted(rows, key=lambda r: -r["false_accepts"])[:5]: + if row["false_accepts"]: + print(f" {row['false_accepts']:>3} {row['file'][:55]}") + if args.write: + out = os.path.join(directory, "eval_result.json") + with open(out, "w") as handle: + json.dump(report, handle, ensure_ascii=False, indent=1) + print(f" wrote {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 5028401..b247b06 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -30,8 +30,11 @@ DEFAULT_MIN_CONFIDENCE = 0.55 # `local` scale (identity_lanes.LocalLane). The value is already a margin over # the runner-up, so the relative-margin test does not apply to it: a single -# absolute line is the whole gate. -LOCAL_ACCEPT = 2.0 +# absolute line is the whole gate. Kept equal to identity_lanes.LOCAL_ACCEPT, +# which is what the engine's DecisionPolicy uses; this copy exists for the +# evaluator's `--no-text` path, and two different lines would score the same +# crops two ways. +LOCAL_ACCEPT = 4.0 def relative_confidence(top_score: float, runner_up_score: float) -> float: diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index 1935387..e232109 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -43,7 +43,14 @@ from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of # on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59% # coverage, 1.0 keeps 87% at 66%. The stricter line is the default because a # wrong label on a shelf is worse than a missing one. -LOCAL_ACCEPT = 2.0 +# +# 4.0 since 2026-09-29: the line was tuned on image6 alone, where every crop is +# a facing of a product the shelf actually holds. On 2297 crops from unrelated +# Wikimedia shelves, 2.0 accepted 25 of them on geometry alone (scores 2.26-4.81 +# with no text overlap at all), a 1.1% far-unknown rate against G3's 1%. At 4.0 +# that falls to 8 (0.35%) while image6 keeps 42 of 48 correct labels at 95.5% +# precision, which is the only region passing both G2 and G3 with coverage left. +LOCAL_ACCEPT = 4.0 # The text lane's own accept line: a bigram overlap this high, with this much # margin over the runner-up, names the row on its own. Measured: the text lane # picks the right SKU on 41-42 of the 55 facings where it reads anything. @@ -51,12 +58,16 @@ TEXT_ACCEPT = 0.5 TEXT_MARGIN = 0.15 # How many of the read's bigrams the winning row must actually contain. The # ratio alone is not evidence: a two-pair read the row happens to contain scores -# 1.0, and "サソース" named sku_l08 that way. Two pairs is the floor that keeps -# every correct text answer on image6 (40 of 40) while dropping the wrong ones -# from 6 to 2, taking shown-label precision from 87% to 95%. It is a floor on -# evidence, not on the ratio, so a long read of shared brand copy still scores -# low and is still refused by the margin. -TEXT_MIN_SHARED = 2 +# 1.0, and "サソース" named sku_l08 that way. Two pairs was the old floor, and it +# is too low once the lane answers on unrelated shelves: on 2297 crops from +# Wikimedia milk, soy and snack shelves, 20 of 26 text-lane answers rested on +# exactly two shared bigrams ("避くて おいしい" -> sku_l12 via おい/いし), an +# 11.4% false accept rate against G3's 1%. Four is where the two sides separate +# once the local line is also at 4.0: false accepts fall to 8 (0.35%) while +# image6 keeps 42 of its 48 correct labels. It is a floor on evidence, not on +# the ratio, so a long read of shared brand copy still scores low and is still +# refused by the margin. +TEXT_MIN_SHARED = 4 # Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a # 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and # BRISK find zero inliers at live resolution. Scaling each reference to the diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index 4a4cded..d95eda7 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -136,14 +136,19 @@ class TextLaneSignatureTests(unittest.TestCase): self.assertGreaterEqual(ranked[0][1], TEXT_ACCEPT) self.assertGreaterEqual(ranked[0][1] - ranked[1][1], TEXT_MARGIN) - def test_one_shared_pair_is_not_enough_to_name_a_row(self): - """A ratio of 1.0 on a single pair is not evidence. + def test_a_thin_read_is_not_enough_to_name_a_row(self): + """A ratio of 1.0 on a few pairs is not evidence. "XXイス" contains exactly one pair the catalog knows, and the row that contains it scores 1.0 with a full margin over every other row. On - image6 that shape named sku_l08 from the read "サソース". The floor - asks how much of the read the row actually explains, so one pair is - refused while the same read with two is accepted. + image6 that shape named sku_l08 from the read "サソース". The floor asks + how much of the read the row actually explains, so one pair and two + pairs are both refused while a real read of the name is accepted. + + Two pairs used to be enough. It is not: on 2297 crops from unrelated + Wikimedia shelves, 20 of 26 text-lane answers rested on exactly two + shared bigrams - "避くて おいしい" named sku_l12 through おい/いし - for + an 11.4% false accept rate. The floor is 4. """ from identity_lanes import DecisionPolicy, TextLane @@ -153,16 +158,22 @@ class TextLaneSignatureTests(unittest.TestCase): ] lane = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None) - thin = sorted(lane.scores_from_text("XXイス").items(), key=lambda kv: kv[1], reverse=True) - self.assertEqual(thin[0][1], 1.0, "the ratio alone cannot see the difference") - self.assertIsNone( - DecisionPolicy(signatures=lane.signatures).decide([], thin, text_read="XXイス")["best_sku"] - ) + for thin_read in ("XXイス", "ZZソース"): + thin = sorted(lane.scores_from_text(thin_read).items(), + key=lambda kv: kv[1], reverse=True) + self.assertEqual(thin[0][1], 1.0, "the ratio alone cannot see the difference") + self.assertIsNone( + DecisionPolicy(signatures=lane.signatures).decide( + [], thin, text_read=thin_read + )["best_sku"], + f"{thin_read!r} shares too few pairs to name a row", + ) - enough = sorted(lane.scores_from_text("ZZソース").items(), key=lambda kv: kv[1], reverse=True) + enough = sorted(lane.scores_from_text("オイスター").items(), + key=lambda kv: kv[1], reverse=True) self.assertEqual( DecisionPolicy(signatures=lane.signatures).decide( - [], enough, text_read="ZZソース" + [], enough, text_read="オイスター" )["best_sku"], "sku_l08", ) @@ -616,8 +627,11 @@ class DecisionPolicyTests(unittest.TestCase): the margin over the runner-up. Re-sorting by value names a different SKU than the lane chose - here sku_b has the larger value but sku_a is the lane's own first choice, and sku_a is what must be answered. + + Both values clear LOCAL_ACCEPT, so the test is about the ordering and + not about the gate. """ - decision = self.policy().decide([("sku_a", 3.0), ("sku_b", 5.0)], []) + decision = self.policy().decide([("sku_a", 4.5), ("sku_b", 5.0)], []) self.assertEqual(decision["best_sku"], "sku_a") def test_a_size_only_read_does_not_fill_in(self):