From 5094b5f6ca800b95bfeaaa99e928d6fdc885b720 Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Tue, 29 Sep 2026 09:28:13 +0900 Subject: [PATCH] fix(recognition): calibrate the accept lines against unrelated shelves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every threshold was fitted on image6, where each crop is a facing of a product the shelf holds, so the set could not price a false accept. Measured on 4580 crops from 64 unrelated Wikimedia shelf photos at the shipped configuration: LOCAL_ACCEPT 2.0 -> 4.0 false accepts 60 (1.31%) -> 3 (0.07%) TEXT_MIN_SHARED 2 -> 4 the two-bigram reads that named sku_l12 via おい/いし On image6 that is 44 shown / 42 correct at full resolution - 95.5% precision, 65.6% coverage - the highest-coverage cell of the joint sweep that passes both G2 (>= 95%) and G3 (<= 1%). At live resolution 56% end-to-end at 97%. The text lane's evidence floor gates its answer only: zeroing the row's score would reorder the ranking and make the margin gate compare a different pair than the one it was measured on. code/sweep_accept_lines.py sweeps both axes jointly through the shipped DecisionPolicy; the earlier table swept one axis while pinning the other, so its baseline rows were not the old line. --- recognization/code/eval_distractors.py | 30 ++- recognization/code/eval_identity.py | 22 +- recognization/code/eval_live.py | 4 +- recognization/code/identity_lanes.py | 57 +++-- recognization/code/live.py | 13 +- recognization/code/score_stream.py | 155 ++++++++++-- recognization/code/sweep_accept_lines.py | 225 ++++++++++++++++++ recognization/code/test_eval_live.py | 38 +++ recognization/code/test_live.py | 29 +++ recognization/code/test_score_stream.py | 137 +++++++++++ .../wikimedia-commons-2/eval_result.json | 91 ++++--- .../wikimedia-commons/eval_result.json | 209 ++++++++-------- 12 files changed, 815 insertions(+), 195 deletions(-) create mode 100644 recognization/code/sweep_accept_lines.py create mode 100644 recognization/code/test_score_stream.py diff --git a/recognization/code/eval_distractors.py b/recognization/code/eval_distractors.py index dfa956c..edc4492 100644 --- a/recognization/code/eval_distractors.py +++ b/recognization/code/eval_distractors.py @@ -5,8 +5,11 @@ that share nothing with the LOPIA catalog. Every accept here is a false accept by construction, which is what makes the set useful: the image6 shelf can only measure false accepts on the products it happens to contain. -The stored eval.json predates the open-set gate, so it is re-run rather than -trusted. +Measured the way the server measures: a 1280-long-side frame detected at +`LiveSettings.conf`. The sources are 4032-4512 px photos, and detecting them at +full resolution and at the default `detect_conf` instead measures a +configuration the phone never runs, which is how a threshold change reads as a +pass on evidence the product does not produce. .venv/bin/python code/eval_distractors.py --stock stock-lopia-clean \ data/external-shelves/wikimedia-commons @@ -28,7 +31,13 @@ if current_dir not in sys.path: import cv2 # noqa: E402 -from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter # noqa: E402 +from eval_identity import ( # noqa: E402 + LIVE_LONG_SIDE, + LOCAL_ACCEPT, + LocalLaneAdapter, + at_live_resolution, +) +from identity_lanes import TEXT_MIN_SHARED # noqa: E402 def main(argv: List[str] | None = None) -> int: @@ -40,9 +49,11 @@ def main(argv: List[str] | None = None) -> int: args = parser.parse_args(argv) from engine import build_default_engine, EngineSettings + from live import LiveSettings engine = build_default_engine(EngineSettings(stock_dir=args.stock)) lane = LocalLaneAdapter(args.stock) + conf = LiveSettings().conf for directory in args.directories: manifest_path = os.path.join(directory, "manifest.json") try: @@ -65,14 +76,15 @@ def main(argv: List[str] | None = None) -> int: if image is None: continue started = time.perf_counter() - detections = engine.detect(image) + frame = at_live_resolution(image) + detections = engine.detect(frame, conf=conf) accepted = 0 for detection in detections: box = detection.get("box") or detection.get("bbox") if not box: continue x1, y1, x2, y2 = [int(v) for v in box] - crop = image[max(0, y1):y2, max(0, x1):x2] + crop = frame[max(0, y1):y2, max(0, x1):x2] if crop.size == 0: continue decision = lane.decide(crop, lane.rank(crop), accept=LOCAL_ACCEPT, @@ -93,6 +105,14 @@ def main(argv: List[str] | None = None) -> int: "dataset": os.path.basename(directory.rstrip("/")), "catalog": args.stock, "note": "Unrelated shelves: every accept is a false accept by construction.", + # The configuration the numbers above were measured under, so the + # stored evidence is checkable against the constants it claims. + "config": { + "frame_long_side": LIVE_LONG_SIDE, + "detect_conf": conf, + "local_accept": LOCAL_ACCEPT, + "text_min_shared": TEXT_MIN_SHARED, + }, "images": len(rows), "detections": detections, "false_accepts": accepts, diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index 71e4e4e..1763f8a 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -418,11 +418,16 @@ class LocalLaneAdapter(Lane): self.lane = LocalLane(stock_dir) # E-3 asks what the text lane is worth, which is only answerable by - # running the same crops without it. Building the lane anyway keeps the - # comparison to one variable: the OCR cost is paid either way. + # running the same crops without it. `--no-text` therefore drops the + # lane whole: nothing in the decision reads it, and an ablation that + # still had to build an OCR engine could not be run on a machine + # without one. self.use_text = use_text - self.text_lane = TextLane.from_catalog( - _catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine() + self.text_lane = ( + TextLane.from_catalog( + _catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine() + ) + if use_text else None ) def rank(self, crop: Any) -> List[Tuple[str, float]]: @@ -433,9 +438,10 @@ class LocalLaneAdapter(Lane): from identity_lanes import DecisionPolicy if not self.use_text: - from identity import decide_identity - - return decide_identity(dict(ranked), scale="local", min_score=accept) + # E-3's other arm: the same policy, over the same ranking, with no + # text evidence. `dict(ranked)` would not do - it drops the lane's + # order, and the value is not the order (see `LocalLane.rank`). + return DecisionPolicy(local_accept=accept).decide(ranked) read = self.text_lane.read(crop) text_rank = sorted( self.text_lane.scores_from_text(read).items(), @@ -523,8 +529,6 @@ def score_lane( the SKU is on the shelf but has no reference, so an accept there is wrong and is the failure the enrollment protocol exists to prevent. """ - from identity import decide_identity - in_catalog = [f for f in facings if f["label"] != UNKNOWN] out_catalog = [f for f in facings if f["label"] == UNKNOWN] enrolled_count = sum(1 for f in in_catalog if f["label"] in enrolled_skus) diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py index 02b4c78..55b3671 100644 --- a/recognization/code/eval_live.py +++ b/recognization/code/eval_live.py @@ -11,7 +11,7 @@ store was enrolled; scoring them is training-set accuracy. python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] [--json PATH] [--iou 0.5] - [--replay] [--identity matcher|local|gallery] [--device cpu|mps|cuda] + [--replay] [--identity matcher|local|gallery|van] [--device cpu|mps|cuda] Prints a human count table and writes the same report as JSON (default /eval_live.json). Missing input exits 2 with {"error": ...} on @@ -66,7 +66,7 @@ def content_sha256(payload: bytes) -> str: # Every backend the server can be started with, so a recording made by the # shipped `run-demo-server.sh` (--identity local) can be replayed. `local` is # the default; `matcher` is Van's scorer kept as the E-1 baseline lane. -IDENTITY_BACKENDS = ("matcher", "local", "gallery") +IDENTITY_BACKENDS = ("matcher", "local", "gallery", "van") def _iou(a: Sequence[float], b: Sequence[float]) -> float: diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index e232109..5ad3224 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -45,28 +45,38 @@ from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of # wrong label on a shelf is worse than a missing one. # # 4.0 since 2026-09-29: the line was tuned on image6 alone, where every crop is -# a facing of a product the shelf actually holds. On 2297 crops from unrelated -# Wikimedia shelves, 2.0 accepted 25 of them on geometry alone (scores 2.26-4.81 -# with no text overlap at all), a 1.1% far-unknown rate against G3's 1%. At 4.0 -# that falls to 8 (0.35%) while image6 keeps 42 of 48 correct labels at 95.5% -# precision, which is the only region passing both G2 and G3 with coverage left. +# a facing of a product the shelf actually holds, so that set could not price +# it. Re-measured with `code/sweep_accept_lines.py` on 4580 crops from 64 +# unrelated Wikimedia shelf photos - 1280-long-side frames at +# `LiveSettings.conf`, the configuration the phone runs - against +# `TEXT_MIN_SHARED = 4`: 2.0 accepts 60 of them (1.31%) and 4.0 accepts 3 +# (0.07%), against G3's 1%. On image6 at full resolution 4.0 shows 44 labels of +# which 42 are right - 95.5% precision, and 65.6% of the 64 facings that hold a +# catalog row - and it is the highest-coverage cell in the sweep that passes +# both G2 and G3. LOCAL_ACCEPT = 4.0 # The text lane's own accept line: a bigram overlap this high, with this much # margin over the runner-up, names the row on its own. Measured: the text lane # picks the right SKU on 41-42 of the 55 facings where it reads anything. TEXT_ACCEPT = 0.5 TEXT_MARGIN = 0.15 -# How many of the read's bigrams the winning row must actually contain. The -# ratio alone is not evidence: a two-pair read the row happens to contain scores -# 1.0, and "サソース" named sku_l08 that way. Two pairs was the old floor, and it -# is too low once the lane answers on unrelated shelves: on 2297 crops from -# Wikimedia milk, soy and snack shelves, 20 of 26 text-lane answers rested on -# exactly two shared bigrams ("避くて おいしい" -> sku_l12 via おい/いし), an -# 11.4% false accept rate against G3's 1%. Four is where the two sides separate -# once the local line is also at 4.0: false accepts fall to 8 (0.35%) while -# image6 keeps 42 of its 48 correct labels. It is a floor on evidence, not on -# the ratio, so a long read of shared brand copy still scores low and is still -# refused by the margin. +# How many of the read's bigrams the winning row must actually contain before +# that read may name a product. A floor on the amount of shared evidence, not +# on the ratio: a two-pair read the row happens to contain scores 1.0, and +# "サソース" named sku_l08 that way. +# +# It gates the text lane's *answer* and nothing else. Zeroing the row's score +# would be a different policy under the same name: it reorders the ranking, so +# a zeroed top row promotes a runner-up and the margin gate ends up comparing a +# different pair than the one it was measured on. It would also gate the veto, +# which is backwards - the veto only ever refuses a local accept, so a floor +# under it would let a two-bigram read rescue a match the local lane earned. +# `code/sweep_accept_lines.py` measures the line by calling the policy below +# with this value, so the sweep and the shipped lane cannot disagree about what +# it means. +# +# The ratio stays the ranking's own gate, so a long read of shared brand copy +# still scores low and is still refused by the margin. TEXT_MIN_SHARED = 4 # Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a # 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and @@ -348,8 +358,15 @@ def text_similarity( the most real text to match were punished hardest. Bounded this way the same reads score 0.86 and 1.0 against their true rows, and the garbage read ties across three rows and is refused by the margin. + + `vocabulary` defaults to the weight table's own keys, which are by + construction the bigrams some signature produces. A second copy of that set + on the lane is a second thing to keep in step, and it is the drift the + module already paid for once with the folding rules. """ read_grams = bigrams(read) + if vocabulary is None and weights is not None: + vocabulary = set(weights) if vocabulary is not None: read_grams &= vocabulary if not read_grams: @@ -404,12 +421,6 @@ class TextLane: # Bigram weights over the catalog's own names, so the lane can tell # copy every sibling prints from the words that name one product. self.weights = bigram_weights(list(self.signatures.values())) - # Every pair any name can produce. A read's pairs outside this set are - # OCR noise or word-boundary artifacts, and counting them in the - # denominator charges a row for text it could never have explained. - self.vocabulary: Set[str] = set() - for text in self.signatures.values(): - self.vocabulary |= signature_bigrams(text) self.ocr_engine = ocr_engine # The engine's OCR lock. Vision tolerates concurrency but the CPU does # not, and this lane runs one OCR per crop on the classify pool, so it @@ -472,7 +483,7 @@ class TextLane: if len(normalize_text(read)) < 4: return {} return { - sku: text_similarity(read, signature, self.weights, self.vocabulary) + sku: text_similarity(read, signature, self.weights) for sku, signature in self.signatures.items() } diff --git a/recognization/code/live.py b/recognization/code/live.py index de36458..402b055 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -306,6 +306,9 @@ class LiveSession: self._frames_dropped = 0 self._process_times: "collections.deque[float]" = collections.deque(maxlen=STATS_WINDOW) self._classify_times: "collections.deque[float]" = collections.deque(maxlen=STATS_WINDOW) + # A lane that raised is an operational failure, not "nothing matched". + # Counted so a broken run is never read as a coverage change. + self._classify_failures = 0 self._slot_history: "collections.deque[List[Dict[str, Any]]]" = collections.deque( maxlen=EMPTY_SLOT_HISTORY ) @@ -1198,7 +1201,14 @@ class LiveSession: ) -> None: try: scores, text_rank, text_read = future.result() - except Exception: + except Exception as exc: + # A lane that raised is an operational failure, not a crop that + # matched nothing. Swallowing it made a broken run (tracks, no + # labels, clean log) indistinguishable from a coverage change, so + # it is counted and logged before the empty answer is used. + with self._lock: + self._classify_failures += 1 + LOGGER.exception("classification failed for track %s: %s", track_id, exc) scores, text_rank, text_read = {}, [], "" finished_at = time.perf_counter() # Resolved before the session lock is taken. `identity_text_lane` builds @@ -1457,6 +1467,7 @@ class LiveSession: "frames_dropped": self._frames_dropped, "server_fps": fps, "classify_pending": sum(track.in_flight for track in self._tracks.values()), + "classify_failures": self._classify_failures, "classify_ms_p50": classify_p50, } diff --git a/recognization/code/score_stream.py b/recognization/code/score_stream.py index e5bbbb6..09bc3d7 100644 --- a/recognization/code/score_stream.py +++ b/recognization/code/score_stream.py @@ -15,6 +15,13 @@ Reported per run: correct / in-catalog facings, precision, unknown accepts, and tracks that matched no labelled facing. A track whose box overlaps a facing the catalog does not enrol is an unenrolled false accept - the dangerous kind, and the one the evaluator's own precision denominator used to hide. + +G1 has two halves - new covers at least as much as Van, and the lower bound of +the paired-bootstrap difference clears -2 pp - and the second half needs the +per-facing pairing, so every row carries the `facing` index it matched. Two +reports plus the labels then answer the gate on the same facings: + + python code/score_stream.py NEW.json --van VAN.json """ from __future__ import annotations @@ -22,6 +29,7 @@ from __future__ import annotations import argparse import json import os +import random import sys from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -33,6 +41,14 @@ if current_dir not in sys.path: # so the denominator keeps only facings at least this far inside the window. VISIBLE_MIN = 0.5 +# G1's second half: the lower bound of the paired difference must clear -2 pp. +G1_BOUND_MIN = -0.02 + +# Resampling draws for the paired bootstrap, and the seed that makes the bound +# a number this repo can print twice. +BOOTSTRAP_DRAWS = 10000 +BOOTSTRAP_SEED = 0 + def iou(a: Sequence[int], b: Sequence[int]) -> float: ax1, ay1, ax2, ay2 = a @@ -72,6 +88,22 @@ def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float: return (inter_w * inter_h) / area if area > 0 else 0.0 +def reachable(facings: Sequence[Dict[str, Any]], + windows: Optional[Sequence[Sequence[int]]] + ) -> List[Dict[str, Any]]: + """Facings a run's windows showed well enough for the run to have named.""" + if not windows: + return list(facings) + return [f for f in facings + if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN] + + +def correct_facings(rows: Sequence[Dict[str, Any]]) -> set: + """Facing indices at least one track got right - one hit each, not one per track.""" + return {row["facing"] for row in rows + if row["outcome"] == "correct" and row["facing"] is not None} + + def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], stock_dir: str, iou_min: float, windows: Optional[Sequence[Sequence[int]]] = None @@ -90,11 +122,7 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], text lane names SKUs the image lane has no view for, which is the point of it, and scoring that as a false accept punishes the lane for working. """ - if windows: - facings = [ - f for f in facings - if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN - ] + facings = reachable(facings, windows) in_catalog = [f for f in facings if f["label"] != "unknown"] enrolled = enrolled_skus(stock_dir) @@ -122,7 +150,8 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], if best is None or best_iou < iou_min: unmatched += 1 rows.append({"track_id": track["track_id"], "predicted": track["sku_id"], - "truth": None, "iou": round(best_iou, 3), "outcome": "unmatched"}) + "facing": None, "truth": None, "iou": round(best_iou, 3), + "outcome": "unmatched"}) continue truth = best["label"] predicted = track["sku_id"] @@ -140,7 +169,8 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], wrong += 1 outcome = "wrong" rows.append({"track_id": track["track_id"], "predicted": predicted, - "truth": truth, "iou": round(best_iou, 3), "outcome": outcome}) + "facing": best["index"], "truth": truth, + "iou": round(best_iou, 3), "outcome": outcome}) shown = correct + wrong + unenrolled_false + unknown_false return { @@ -155,10 +185,81 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], "shown": shown, "coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0, "precision": round(correct / shown, 4) if shown else 0.0, + # The window set travels with the report: a paired comparison has to + # know what each run was shown, not just what it named. + "windows": [list(w) for w in windows] if windows else None, "rows": rows, } +def paired_bootstrap(van: Dict[str, Any], new: Dict[str, Any], + facings: Sequence[Dict[str, Any]], + draws: int = BOOTSTRAP_DRAWS, + seed: int = BOOTSTRAP_SEED) -> Dict[str, Any]: + """G1's second half: new minus Van, per facing, with the 95 % lower bound. + + Both reports are graded on one facing set - the in-catalog facings either + run's windows showed - so a facing one run panned past cannot flatter the + other. Resampling draws facings with replacement rather than comparing two + independent rates, because the two runs saw the same photo: pairing removes + the facing-to-facing variance that otherwise swamps a 27-facing sample. + + Percentile method on the sorted draw differences; the seed is fixed so the + number in a report is one this repo reproduces. + """ + windows = list(van.get("windows") or []) + list(new.get("windows") or []) + shared = sorted(f["index"] for f in reachable(facings, windows) + if f["label"] != "unknown") + if not shared: + return {"facings": 0, "difference": 0.0, "lower_95": 0.0, "upper_95": 0.0, + "van": 0, "new": 0, "draws": draws, "seed": seed} + + van_hits = correct_facings(van["rows"]) + new_hits = correct_facings(new["rows"]) + van_vector = [1 if i in van_hits else 0 for i in shared] + new_vector = [1 if i in new_hits else 0 for i in shared] + n = len(shared) + + rng = random.Random(seed) + spread = [] + for _ in range(draws): + van_sum = new_sum = 0 + for _ in range(n): + # The same draw index into both vectors: that is the pairing. + drawn = rng.randrange(n) + van_sum += van_vector[drawn] + new_sum += new_vector[drawn] + spread.append((new_sum - van_sum) / n) + spread.sort() + tail = max(1, int(0.025 * draws)) + return { + "facings": n, + "van": sum(van_vector), + "new": sum(new_vector), + "van_rate": round(sum(van_vector) / n, 4), + "new_rate": round(sum(new_vector) / n, 4), + "difference": round((sum(new_vector) - sum(van_vector)) / n, 4), + "lower_95": round(spread[tail - 1], 4), + "upper_95": round(spread[min(draws - 1, int(0.975 * draws))], 4), + "draws": draws, + "seed": seed, + } + + +def load_run(path: str, window: Optional[str] = None + ) -> Tuple[Dict[str, Any], Optional[List[Tuple[int, ...]]]]: + """A stream_client --tracks dump and the source-image windows it showed.""" + with open(path) as handle: + payload = json.load(handle) + if window: + windows = [tuple(int(part) for part in window.split(","))] + elif payload.get("windows"): + windows = [tuple(w) for w in payload["windows"]] + else: + windows = None + return payload, windows + + def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description="Score streamed labels against ground truth") parser.add_argument("tracks", help="JSON written by stream_client.py --tracks") @@ -169,22 +270,20 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument("--window", default=None, help="x0,y0,x1,y1 the client showed; the denominator is " "restricted to facings at least half inside it") + parser.add_argument("--van", default=None, + help="Van baseline track dump on the same clip; scores G1's " + "paired-bootstrap half against it") + parser.add_argument("--draws", type=int, default=BOOTSTRAP_DRAWS) + parser.add_argument("--seed", type=int, default=BOOTSTRAP_SEED) parser.add_argument("--verbose", action="store_true", help="print every scored track") args = parser.parse_args(argv) - with open(args.tracks) as handle: - payload = json.load(handle) - windows = None - if args.window: - windows = [tuple(int(part) for part in args.window.split(","))] - elif payload.get("windows"): - windows = [tuple(w) for w in payload["windows"]] - report = score(payload["tracks"], load_facings(args.labels), args.stock_dir, - args.iou, windows) + facings = load_facings(args.labels) + payload, windows = load_run(args.tracks, args.window) + report = score(payload["tracks"], facings, args.stock_dir, args.iou, windows) report["source"] = payload.get("source") report["pace"] = payload.get("pace") report["seconds"] = payload.get("seconds") - report["windows"] = [list(w) for w in windows] if windows else None print(f"{os.path.basename(args.tracks)} pace={report['pace']} " f"{report['seconds']}s") @@ -197,7 +296,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if args.verbose: for row in report["rows"]: print(f" t{row['track_id']:4d} {row['outcome']:24s} " - f"pred={row['predicted']} truth={row['truth']} iou={row['iou']}") + f"facing={row['facing']} pred={row['predicted']} " + f"truth={row['truth']} iou={row['iou']}") + + if args.van: + van_payload, van_windows = load_run(args.van, args.window) + van_report = score(van_payload["tracks"], facings, args.stock_dir, + args.iou, van_windows) + paired = paired_bootstrap(van_report, report, facings, args.draws, args.seed) + report["van"] = {k: van_report[k] for k in + ("in_catalog_facings", "facings_named", "coverage", + "correct", "wrong", "precision", "shown")} + report["g1_paired"] = paired + print(f" G1 vs {os.path.basename(args.van)}: new {paired['new']}/" + f"{paired['facings']} ({paired['new_rate']:.0%}) - van {paired['van']}/" + f"{paired['facings']} ({paired['van_rate']:.0%}) = " + f"{paired['difference'] * 100:+.1f} pp") + print(f" paired bootstrap {paired['draws']} draws seed {paired['seed']}: " + f"95% bounds {paired['lower_95'] * 100:+.1f} pp to " + f"{paired['upper_95'] * 100:+.1f} pp; lower bound clears " + f"{G1_BOUND_MIN * 100:+.0f} pp: " + f"{'yes' if paired['lower_95'] >= G1_BOUND_MIN else 'NO'}") if args.json_path: with open(args.json_path, "w") as handle: diff --git a/recognization/code/sweep_accept_lines.py b/recognization/code/sweep_accept_lines.py new file mode 100644 index 0000000..1af7cb1 --- /dev/null +++ b/recognization/code/sweep_accept_lines.py @@ -0,0 +1,225 @@ +"""Joint sweep of the two open-set gates: LOCAL_ACCEPT x TEXT_MIN_SHARED. + +Both lines were fitted on image6, where every crop is a facing of a product the +shelf actually holds. That set can only measure false accepts on the products it +happens to contain, so it cannot price a line on its own - the far-unknown rate +comes from `data/external-shelves/*`: Wikimedia Commons shelf photos that share +nothing with the catalog, where every accept is a false accept by construction. + +Each cell is produced by calling the shipped `DecisionPolicy` with that cell's +values and the catalog signatures, never by re-implementing the gate in the +sweep: a sweep that scores its own copy of the policy reports on a machine the +server does not run. `TEXT_MIN_SHARED` is a floor on the evidence behind the +text lane's *answer*. It is deliberately not a floor on the score - zeroing a +row reorders the ranking, so the margin gate would compare a different pair - +and deliberately not a floor on the veto, which only ever refuses. The constant +in `identity_lanes.py` says the same thing next to the code that does it. + +The distractor side is measured the way the phone measures it - a 1280-long-side +frame at `LiveSettings.conf`, the path `code/eval_distractors.py` runs. The +shelf is scored at both resolutions: the full-resolution column is the one the +published table quotes, the live column is what the phone sees. + +RootSIFT and OCR dominate the runtime and neither depends on the gates, so the +per-crop evidence is cached. Without `--cache` the sweep rebuilds it every run +and prints the same numbers, only slower. + + .venv/bin/python code/sweep_accept_lines.py \\ + --cache data/evaluations/2026-09-29-session/scratch/accept_line_cache.pkl +""" + +from __future__ import annotations + +import argparse +import glob +import os +import pickle +import sys +from typing import Any, Dict, List, Optional, Sequence, Tuple + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 + +from eval_identity import ( # noqa: E402 + EXCLUDE, + LIVE_LONG_SIDE, + UNKNOWN, + LocalLaneAdapter, + at_live_resolution, + crop_of, + load_labels, +) +from identity_lanes import LOCAL_ACCEPT, TEXT_MIN_SHARED, DecisionPolicy # noqa: E402 + +SHELF_LABELS = "eval/labels/image6.json" +SHELF_IMAGE = "add/image6.jpg" +DISTRACTOR_DIRS = ( + "data/external-shelves/wikimedia-commons", + "data/external-shelves/wikimedia-commons-2", +) +LOCAL_ACCEPTS = (2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0) +TEXT_MIN_SHAREDS = (2, 3, 4, 5, 6) +CACHE_VERSION = 3 + +# One crop's evidence: the local lane's ranking and the text lane's raw read. +Evidence = Tuple[Any, str] +# One facing: its ground-truth label plus that evidence. +Facing = Tuple[str, Any, str] + + +def _evidence(lane: LocalLaneAdapter, frame: Any, box: Sequence[int]) -> Optional[Evidence]: + """Rank and read one crop, or None when the box is off the frame.""" + crop = crop_of(frame, box) + if crop.size == 0: + return None + return lane.rank(crop), lane.text_lane.read(crop) + + +def build_cache(stock_dir: str) -> Dict[str, Any]: + """Every crop's evidence, gathered once and swept many times.""" + from engine import EngineSettings, build_default_engine + from live import LiveSettings + + lane = LocalLaneAdapter(stock_dir) + image = cv2.imread(SHELF_IMAGE) + live = at_live_resolution(image) + factor = live.shape[1] / float(image.shape[1]) + + shelf_full: List[Facing] = [] + shelf_live: List[Facing] = [] + for facing in load_labels(SHELF_LABELS)["facings"]: + label = facing.get("label") + if label in (None, EXCLUDE): + continue + full = _evidence(lane, image, facing["box"]) + if full is None: + continue + shelf_full.append((label, full[0], full[1])) + # A facing the live frame cuts to nothing is dropped rather than + # scored at full resolution: the two columns have to be the same crops + # at two sizes, or the resolution is not the only variable. + small = _evidence(lane, live, [int(round(v * factor)) for v in facing["box"]]) + if small is not None: + shelf_live.append((label, small[0], small[1])) + + engine = build_default_engine(EngineSettings(stock_dir=stock_dir)) + conf = LiveSettings().conf + distractors: List[Evidence] = [] + photos = 0 + for directory in DISTRACTOR_DIRS: + paths = sorted( + path for path in glob.glob(os.path.join(directory, "*")) + if os.path.splitext(path)[1].lower() in (".jpg", ".jpeg", ".png") + ) + for path in paths: + shelf_photo = cv2.imread(path) + if shelf_photo is None: + continue + photos += 1 + frame = at_live_resolution(shelf_photo) + for detection in engine.detect(frame, conf=conf): + box = detection.get("box") or detection.get("bbox") + if not box: + continue + item = _evidence(lane, frame, [int(v) for v in box]) + if item is not None: + distractors.append(item) + return { + "version": CACHE_VERSION, + "conf": conf, + "photos": photos, + "shelf_full": shelf_full, + "shelf_live": shelf_live, + "distractors": distractors, + } + + +def _score(lane: LocalLaneAdapter, policy: DecisionPolicy, facings: Sequence[Facing]) -> Tuple[int, int]: + """Shown and correct labels for one shelf column under one policy.""" + shown = correct = 0 + for label, ranked, read in facings: + text = sorted( + lane.text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True + ) + answer = policy.decide(ranked, text, text_read=read)["best_sku"] + if answer: + shown += 1 + correct += answer == label + return shown, correct + + +def _false_accepts(lane: LocalLaneAdapter, policy: DecisionPolicy, + distractors: Sequence[Evidence]) -> int: + """Every accept on an unrelated shelf is a false accept by construction.""" + accepts = 0 + for ranked, read in distractors: + text = sorted( + lane.text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True + ) + if policy.decide(ranked, text, text_read=read)["best_sku"]: + accepts += 1 + return accepts + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stock", default="stock-lopia-clean") + parser.add_argument("--cache", default="", help="build once, reuse on later runs") + parser.add_argument("--rebuild", action="store_true", help="ignore an existing cache") + args = parser.parse_args(argv) + + cache: Optional[Dict[str, Any]] = None + if args.cache and os.path.exists(args.cache) and not args.rebuild: + with open(args.cache, "rb") as handle: + cache = pickle.load(handle) + if cache.get("version") != CACHE_VERSION: + cache = None + if cache is None: + cache = build_cache(args.stock) + if args.cache: + os.makedirs(os.path.dirname(os.path.abspath(args.cache)), exist_ok=True) + with open(args.cache, "wb") as handle: + pickle.dump(cache, handle) + + lane = LocalLaneAdapter(args.stock) + shelf_full: List[Facing] = cache["shelf_full"] + shelf_live: List[Facing] = cache["shelf_live"] + distractors: List[Evidence] = cache["distractors"] + known = sum(1 for label, _, _ in shelf_full if label != UNKNOWN) + + print(f"catalog {args.stock}, {LIVE_LONG_SIDE}-long-side frames at detect_conf {cache['conf']}") + print(f"image6: {len(shelf_full)} facings, {known} holding a catalog row, " + f"{len(shelf_live)} of them cut from the live frame") + print(f"distractors: {len(distractors)} crops from {cache['photos']} unrelated shelf photos") + print() + print(f"{'accept':>6} {'floor':>5} | {'shown':>5} {'corr':>4} {'prec':>6} {'cover':>6} " + f"| {'shown':>5} {'corr':>4} {'prec':>6} | {'false':>5} {'rate':>6}") + print(f"{'':>6} {'':>5} | {'--- full resolution ---':^25} " + f"| {'---- live resolution ----':^25} | {'-- unknown shelves --':^14}") + for accept in LOCAL_ACCEPTS: + for floor in TEXT_MIN_SHAREDS: + policy = DecisionPolicy( + local_accept=accept, text_min_shared=floor, + signatures=lane.text_lane.signatures, + ) + full_shown, full_correct = _score(lane, policy, shelf_full) + live_shown, live_correct = _score(lane, policy, shelf_live) + false = _false_accepts(lane, policy, distractors) + shown, correct = full_shown, full_correct + rate = false / len(distractors) * 100 + shipped = " <- shipped" if (accept, floor) == (LOCAL_ACCEPT, TEXT_MIN_SHARED) else "" + print( + f"{accept:>6.1f} {floor:>5} | {shown:>5} {correct:>4} " + f"{correct / shown * 100 if shown else 0:>5.1f}% {correct / known * 100:>5.1f}% " + f"| {live_shown:>5} {live_correct:>4} " + f"{live_correct / live_shown * 100 if live_shown else 0:>5.1f}% " + f"| {false:>5} {rate:>5.2f}%{shipped}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py index afd816f..9fc8a49 100644 --- a/recognization/code/test_eval_live.py +++ b/recognization/code/test_eval_live.py @@ -18,6 +18,7 @@ import cv2 # noqa: E402 import numpy as np # noqa: E402 from eval_live import ( # noqa: E402 + IDENTITY_BACKENDS, EvalInputError, identity_level, labelled_top1, @@ -296,5 +297,42 @@ class IdentityContractTests(unittest.TestCase): {"identity_backend": "matcher"}, ]) + def test_van_is_accepted_by_the_cli_and_by_a_recording_that_names_it(self): + self.assertIn("van", IDENTITY_BACKENDS) + self.assertEqual(resolve_identity("van", None), "van") + self.assertEqual(recorded_identity([{"identity_backend": "van"}]), "van") + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + stock = os.path.join(root, "stock") + os.makedirs(stock) + recording = os.path.join(root, "session") + os.makedirs(recording) + write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8)) + line = {"frame": 1, "path": "000000.jpg", "tracks": [], "identity_backend": "van"} + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps(line) + "\n") + for extra in ([], ["--identity", "van"]): + report_path = os.path.join(root, "eval_live.json") + completed = subprocess.run( + [ + sys.executable, + os.path.join(current_dir, "eval_live.py"), + recording, + "--stock-dir", + stock, + "--json", + report_path, + *extra, + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + with open(report_path, "r", encoding="utf-8") as handle: + report = json.load(handle) + self.assertEqual(report["identity"]["backend"], "van") + self.assertEqual(report["fingerprints"]["model"]["identity"], "van") + if __name__ == "__main__": unittest.main() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index 4357776..f7b337e 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -455,6 +455,35 @@ class LocalScaleTests(unittest.TestCase): self.assertGreater(engine.ocr_calls, 1, "the size-only read must not be cached") engine.shutdown() + def test_a_failing_lane_is_counted_and_logged(self): + """A lane that raises is a failure, not a crop that matched nothing. + + `_on_classified` substituted an empty answer for the exception, so a + run in which `score_crop_lanes` failed on every crop produced tracks, + zero labels and a clean log - indistinguishable from a real coverage + drop. The counter and the log line are what tell the two apart, and + the session has to keep serving frames either way. + """ + session, engine, _, _ = make_session(engine_cls=LocalScaleEngine) + + def boom(crop, method=None, skip_ocr=False): + raise RuntimeError("lane unavailable") + + engine.score_crop_lanes = boom + named = None + with self.assertLogs("dorayaki.live", level="ERROR") as logs: + for frame_id in range(1, 14): + tracks = process(session, frame_id)["tracks"] + self.assertTrue(tracks, "a failing lane must not take the session down") + named = named or next( + (t["identity"]["sku_id"] for t in tracks if t["identity"]["sku_id"]), None) + self.assertTrue(wait_for(lambda: bool(logs.output)), + "the exception must be logged, not swallowed") + self.assertGreaterEqual(session.snapshot()["stats"]["classify_failures"], 1, + "a lane failure must be counted in the session stats") + self.assertIsNone(named, "a failed lane must not name a product") + engine.shutdown() + class TagPassIntegrationTests(unittest.TestCase): """The tag pass must reach the live result without blocking frames.""" diff --git a/recognization/code/test_score_stream.py b/recognization/code/test_score_stream.py new file mode 100644 index 0000000..5294657 --- /dev/null +++ b/recognization/code/test_score_stream.py @@ -0,0 +1,137 @@ +""" +Tests for the G1 scorer. + +The gate has two halves - the new lane covers at least as much as Van, and the +lower bound of the paired-bootstrap difference clears -2 pp - and the second +half is only computable if a scored row still says which facing it matched. +These tests pin that pairing and the bound it produces. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import unittest + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from score_stream import ( # noqa: E402 + G1_BOUND_MIN, + correct_facings, + paired_bootstrap, + score, +) + + +def facing(index, box, label="sku_a"): + return {"index": index, "box": list(box), "label": label} + + +def track(track_id, box, sku="sku_a"): + return {"track_id": track_id, "sku_id": sku, "label_box": list(box)} + + +def report(rows, windows=None): + """A score report shaped the way `score` returns one, without the boxes.""" + return {"rows": rows, "windows": windows} + + +def correct(track_id, matched): + return {"track_id": track_id, "predicted": "sku_a", "facing": matched, + "truth": "sku_a", "iou": 1.0, "outcome": "correct"} + + +def grid(count): + return [facing(i, [i * 20, 0, i * 20 + 10, 10]) for i in range(count)] + + +class ScoreRowTests(unittest.TestCase): + def setUp(self): + self.stock = tempfile.TemporaryDirectory() + open(os.path.join(self.stock.name, "sku_a.jpg"), "wb").close() + self.addCleanup(self.stock.cleanup) + + def test_rows_carry_the_matched_facing(self): + facings = [facing(7, [0, 0, 100, 100]), facing(8, [900, 0, 1000, 100])] + rows = score([track(1, [0, 0, 100, 100]), track(2, [400, 400, 500, 500])], + facings, self.stock.name, 0.5)["rows"] + by_track = {row["track_id"]: row for row in rows} + self.assertEqual(by_track[1]["facing"], 7) + self.assertEqual(by_track[1]["outcome"], "correct") + # A track that hit nothing is unpaired, and says so instead of guessing. + self.assertIsNone(by_track[2]["facing"]) + self.assertEqual(by_track[2]["outcome"], "unmatched") + + def test_correct_facings_counts_a_facing_once(self): + rows = [correct(1, 7), correct(2, 7), correct(3, 9)] + self.assertEqual(correct_facings(rows), {7, 9}) + + +class PairedBootstrapTests(unittest.TestCase): + """27 facings is a small sample, so the pairing is what carries the gate.""" + + def test_identical_systems_sit_at_zero_everywhere(self): + rows = [correct(1, i) for i in range(10)] + result = paired_bootstrap(report(rows), report(rows), []) + self.assertEqual(result["difference"], 0.0) + self.assertEqual(result["lower_95"], 0.0) + self.assertEqual(result["upper_95"], 0.0) + self.assertEqual(result["van"], result["new"]) + + def test_a_superset_of_vans_hits_can_never_bound_below_zero(self): + # The image6 shape: the new lane gets everything Van got, plus a facing. + van = [correct(i, i) for i in range(17)] + new = van + [correct(99, 17)] + result = paired_bootstrap(report(van), report(new), grid(27)) + self.assertEqual(result["facings"], 27) + self.assertEqual(result["van"], 17) + self.assertEqual(result["new"], 18) + self.assertAlmostEqual(result["difference"], 1 / 27, places=4) + # No resample of these 27 can put a facing in the new run that Van's run + # did not already have, so the 2.5th percentile is a hard zero. A bound + # printed as +0.0 pp is the floor of this pairing, not a margin. + self.assertEqual(result["lower_95"], 0.0) + self.assertGreaterEqual(result["lower_95"], G1_BOUND_MIN) + + def test_a_swapped_facing_spreads_across_zero(self): + van = [correct(i, i) for i in range(17)] + new = van[:16] + [correct(99, 17)] + result = paired_bootstrap(report(van), report(new), grid(27)) + self.assertEqual(result["difference"], 0.0) + self.assertLess(result["lower_95"], G1_BOUND_MIN) + self.assertGreater(result["upper_95"], 0.0) + + def test_both_runs_are_graded_on_what_either_window_reached(self): + # Two facings at opposite edges; each run panned to one of them. Grading + # on the union keeps both on the same facings; the intersection would + # score each run on its own pan and the pairing would be empty. + facings = [facing(0, [0, 0, 100, 100]), facing(1, [900, 0, 1000, 100])] + left, right = [0, 0, 500, 1000], [500, 0, 1000, 1000] + result = paired_bootstrap(report([correct(1, 0)], [left]), + report([correct(2, 1)], [right]), facings) + self.assertEqual(result["facings"], 2) + self.assertEqual((result["van"], result["new"]), (1, 1)) + self.assertEqual(result["difference"], 0.0) + + def test_the_bound_repeats_for_the_same_seed(self): + van = [correct(i, i) for i in range(17)] + new = van[:5] + [correct(90 + i, 20 + i) for i in range(9)] + facings = grid(27) + first = paired_bootstrap(report(van), report(new), facings) + self.assertEqual(first, paired_bootstrap(report(van), report(new), facings)) + + def test_no_reachable_facing_scores_zero_rather_than_dividing_by_zero(self): + # Both runs panned to a corner the only facing is not in: the pairing + # set is empty, and a rate is not a number there. + elsewhere = [facing(0, [0, 0, 10, 10])] + corner = [[500, 500, 600, 600]] + result = paired_bootstrap(report([], corner), report([], corner), elsewhere) + self.assertEqual(result["facings"], 0) + self.assertEqual(result["lower_95"], 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/data/external-shelves/wikimedia-commons-2/eval_result.json b/recognization/data/external-shelves/wikimedia-commons-2/eval_result.json index e784242..f4e17fa 100644 --- a/recognization/data/external-shelves/wikimedia-commons-2/eval_result.json +++ b/recognization/data/external-shelves/wikimedia-commons-2/eval_result.json @@ -2,129 +2,142 @@ "dataset": "wikimedia-commons-2", "catalog": "stock-lopia-clean", "note": "Unrelated shelves: every accept is a false accept by construction.", - "images": 17, - "detections": 462, - "false_accepts": 1, - "false_accept_rate": 0.0022, + "config": { + "frame_long_side": 1280, + "detect_conf": 0.1, + "local_accept": 4.0, + "text_min_shared": 4 + }, + "images": 18, + "detections": 1076, + "false_accepts": 0, + "false_accept_rate": 0.0, "rows": [ { "file": "000_1920px-2011年4月の牛乳_(5588310773).jpg", "title": "File:2011年4月の牛乳 (5588310773).jpg", - "detections": 78, - "false_accepts": 1, - "ms": 3504 + "detections": 105, + "false_accepts": 0, + "ms": 3108 }, { "file": "001_Candy_in_Japan_2005_(5256472).jpg", "title": "File:Candy in Japan 2005 (5256472).jpg", - "detections": 12, + "detections": 41, "false_accepts": 0, - "ms": 408 + "ms": 1077 }, { "file": "002_1920px-Empty_shelves_in_Tokyo_area_as_people_get_ready_for_t.jpg", "title": "File:Empty shelves in Tokyo area as people get ready for typhoon Hagibis - Oct 11 2019 330pm.jpeg", - "detections": 8, + "detections": 55, "false_accepts": 0, - "ms": 101 + "ms": 637 }, { "file": "003_1920px-HK_SW_上環_Sheung_Wan_德輔道西_88_Des_Voeux_Road_West_shop_.jpg", "title": "File:HK SW 上環 Sheung Wan 德輔道西 88 Des Voeux Road West shop 惠康超級市場 Wellcome Supermarket goods August 2020 SS2 27.jpg", - "detections": 10, + "detections": 38, "false_accepts": 0, - "ms": 1560 + "ms": 952 }, { "file": "005_1920px-Illuminated_wooden_shelf_with_many_glass_jars_contain.jpg", "title": "File:Illuminated wooden shelf with many glass jars containing cookies for sale in Tokyo.jpg", - "detections": 41, + "detections": 47, "false_accepts": 0, - "ms": 2946 + "ms": 862 + }, + { + "file": "006_1920px-Interior_of_JUSCO_Shingu_Shopping_Center_20100213c.jpg", + "title": "File:Interior of JUSCO Shingu Shopping Center 20100213c.JPG", + "detections": 105, + "false_accepts": 0, + "ms": 1087 }, { "file": "007_Japan_earthquake_store_shelves.jpg", "title": "File:Japan earthquake store shelves.jpg", - "detections": 4, + "detections": 11, "false_accepts": 0, - "ms": 298 + "ms": 277 }, { "file": "009_Milk_(2655574568).jpg", "title": "File:Milk (2655574568).jpg", - "detections": 58, + "detections": 153, "false_accepts": 0, - "ms": 2802 + "ms": 2720 }, { "file": "010_Natsuge-mikado_okashi.jpg", "title": "File:Natsuge-mikado okashi.jpg", - "detections": 7, + "detections": 30, "false_accepts": 0, - "ms": 704 + "ms": 724 }, { "file": "012_1920px-Supermarket_in_2006_(646332846).jpg", "title": "File:Supermarket in 2006 (646332846).jpg", - "detections": 129, + "detections": 174, "false_accepts": 0, - "ms": 6707 + "ms": 1950 }, { "file": "014_1920px-「ペヤング」予想通りの大売れ！_(18634213682).jpg", "title": "File:「ペヤング」予想通りの大売れ！ (18634213682).jpg", - "detections": 27, + "detections": 30, "false_accepts": 0, - "ms": 762 + "ms": 707 }, { "file": "015_1920px-お客様各位_2008_国内産バター_(2429997079).jpg", "title": "File:お客様各位 2008 国内産バター (2429997079).jpg", - "detections": 14, + "detections": 28, "false_accepts": 0, - "ms": 1499 + "ms": 1036 }, { "file": "016_1920px-パン売り場_手動ドア式_2016_(28574754416).jpg", "title": "File:パン売り場 手動ドア式 2016 (28574754416).jpg", - "detections": 21, + "detections": 63, "false_accepts": 0, - "ms": 1842 + "ms": 1248 }, { "file": "017_1920px-プラレール_2015_(22339689873).jpg", "title": "File:プラレール 2015 (22339689873).jpg", - "detections": 12, + "detections": 73, "false_accepts": 0, - "ms": 864 + "ms": 1324 }, { "file": "018_一家族様一本限り_(5588902836).jpg", "title": "File:一家族様一本限り (5588902836).jpg", - "detections": 27, + "detections": 41, "false_accepts": 0, - "ms": 3880 + "ms": 1208 }, { "file": "019_1920px-洗濯洗剤_(10852819193).jpg", "title": "File:洗濯洗剤 (10852819193).jpg", - "detections": 8, + "detections": 44, "false_accepts": 0, - "ms": 704 + "ms": 606 }, { "file": "020_1920px-申し訳ございませんが_ごめんなさい_お魚本日の調理は終了いたしました。_どうぞよろしくお願いします_(11.jpg", "title": "File:申し訳ございませんが ごめんなさい お魚本日の調理は終了いたしました。 どうぞよろしくお願いします (11331761984).jpg", - "detections": 6, + "detections": 25, "false_accepts": 0, - "ms": 957 + "ms": 571 }, { "file": "021_1920px-要冷蔵_(4513315790).jpg", "title": "File:要冷蔵 (4513315790).jpg", - "detections": 0, + "detections": 13, "false_accepts": 0, - "ms": 190 + "ms": 259 } ] } \ No newline at end of file diff --git a/recognization/data/external-shelves/wikimedia-commons/eval_result.json b/recognization/data/external-shelves/wikimedia-commons/eval_result.json index b21976e..d5eb659 100644 --- a/recognization/data/external-shelves/wikimedia-commons/eval_result.json +++ b/recognization/data/external-shelves/wikimedia-commons/eval_result.json @@ -2,325 +2,338 @@ "dataset": "wikimedia-commons", "catalog": "stock-lopia-clean", "note": "Unrelated shelves: every accept is a false accept by construction.", - "images": 45, - "detections": 2133, - "false_accepts": 7, - "false_accept_rate": 0.0033, + "config": { + "frame_long_side": 1280, + "detect_conf": 0.1, + "local_accept": 4.0, + "text_min_shared": 4 + }, + "images": 46, + "detections": 3504, + "false_accepts": 3, + "false_accept_rate": 0.0009, "rows": [ { "file": "000_1920px-198円_中華弁当_2016_(25800617695).jpg", "title": "File:198円 中華弁当 2016 (25800617695).jpg", - "detections": 2, + "detections": 14, "false_accepts": 0, - "ms": 1774 + "ms": 1669 }, { "file": "001_1920px-2011年4月の牛乳_(5588310773).jpg", "title": "File:2011年4月の牛乳 (5588310773).jpg", - "detections": 78, - "false_accepts": 1, - "ms": 4553 + "detections": 105, + "false_accepts": 0, + "ms": 3225 }, { "file": "002_1920px-20140909-0252_Costa_Mesa_Mitsuwa.JPG", "title": "File:20140909-0252 Costa Mesa Mitsuwa.JPG", - "detections": 95, + "detections": 121, "false_accepts": 0, - "ms": 3025 + "ms": 2674 }, { "file": "003_1920px-Aomori_Home_(4721758999).jpg", "title": "File:Aomori Home (4721758999).jpg", - "detections": 76, + "detections": 97, "false_accepts": 0, - "ms": 3634 + "ms": 3333 }, { "file": "004_1920px-Asakusa_15_-_making_rice_crackers_the_old_way_(155797.jpg", "title": "File:Asakusa 15 - making rice crackers the old way (15579727827).jpg", - "detections": 14, + "detections": 40, "false_accepts": 0, - "ms": 162 + "ms": 456 }, { "file": "005_1920px-Asakusa_16_-_making_rice_crackers_the_old_way_(155794.jpg", "title": "File:Asakusa 16 - making rice crackers the old way (15579472088).jpg", - "detections": 3, + "detections": 30, "false_accepts": 0, - "ms": 131 + "ms": 537 }, { "file": "006_1920px-Asakusa_17_-_making_rice_crackers_the_old_way_(155794.jpg", "title": "File:Asakusa 17 - making rice crackers the old way (15579471588).jpg", - "detections": 4, + "detections": 30, "false_accepts": 0, - "ms": 132 + "ms": 485 }, { "file": "007_1920px-Asakusa_19_-_the_woman_selling_the_rice_crackers_(157.jpg", "title": "File:Asakusa 19 - the woman selling the rice crackers (15764889785).jpg", - "detections": 3, + "detections": 33, "false_accepts": 0, - "ms": 67 + "ms": 502 }, { "file": "008_1920px-Azabudai-Hills---2025-04-05_015.jpg", "title": "File:Azabudai-Hills---2025-04-05 015.jpg", - "detections": 37, + "detections": 54, "false_accepts": 2, - "ms": 2113 + "ms": 2525 }, { "file": "010_1920px-Breathalyzers_in_a_MaxValu_supermarket.jpg", "title": "File:Breathalyzers in a MaxValu supermarket.jpg", - "detections": 5, + "detections": 24, "false_accepts": 0, - "ms": 368 + "ms": 1508 }, { "file": "011_1920px-Chocolate_in_Japan_(5137097169).jpg", "title": "File:Chocolate in Japan (5137097169).jpg", - "detections": 65, - "false_accepts": 1, - "ms": 3829 + "detections": 78, + "false_accepts": 0, + "ms": 1961 }, { "file": "013_1920px-Confectionery_at_WCTC_Shopping_Center,_Koror_City,_Pa.jpg", "title": "File:Confectionery at WCTC Shopping Center, Koror City, Palau.jpg", - "detections": 96, + "detections": 154, "false_accepts": 0, - "ms": 6114 + "ms": 3424 }, { "file": "014_1920px-Deer_cookies_at_Nara,_Japan.jpg", "title": "File:Deer cookies at Nara, Japan.jpg", - "detections": 10, + "detections": 24, "false_accepts": 0, - "ms": 733 + "ms": 465 }, { "file": "015_1920px-ESL_Price_Tag.JPG", "title": "File:ESL Price Tag.JPG", - "detections": 32, + "detections": 56, "false_accepts": 0, - "ms": 1506 + "ms": 1383 }, { "file": "017_1920px-Food-with-physiological-energy-value-example.jpg", "title": "File:Food-with-physiological-energy-value-example.jpg", - "detections": 10, + "detections": 32, "false_accepts": 0, - "ms": 454 + "ms": 1070 }, { "file": "019_1920px-HK_CWB_Wellcome_Shop_Supermarket_goods_Carbonated_sof.jpg", "title": "File:HK CWB Wellcome Shop Supermarket goods Carbonated soft drink codd-neck bottles 波子汽水 Apple Ramune Japanese products July 2017 IX1.jpg", - "detections": 18, + "detections": 42, "false_accepts": 0, - "ms": 712 + "ms": 1094 }, { "file": "021_1920px-Interior_of_Supermarket_in_Japan_02.jpg", "title": "File:Interior of Supermarket in Japan 02.jpg", - "detections": 5, + "detections": 18, "false_accepts": 0, - "ms": 286 + "ms": 491 }, { "file": "022_1920px-JP_日本_Japan_沖繩_OKINAWA_浦添市_Urasoe-city_AEON_Supermark.jpg", "title": "File:JP 日本 Japan 沖繩 OKINAWA 浦添市 Urasoe-city AEON Supermarket food drinks February 2026 N13p 02.jpg", - "detections": 119, + "detections": 165, "false_accepts": 0, - "ms": 4603 + "ms": 3522 }, { "file": "023_1920px-thumbnail.jpg", "title": "File:JP 日本 Japan 沖繩 OKINAWA 那霸 Naha Omoromachi Shintoshin 新都心 Naha Main Place 商場 Shopping Mall shop Aeon Supermarket drink 益力多 January 2025 R12S 01.jpg", - "detections": 117, + "detections": 197, "false_accepts": 0, - "ms": 2581 + "ms": 3389 }, { "file": "024_1920px-JaSupermarket_Instant_noodles.JPG", "title": "File:JaSupermarket Instant noodles.JPG", - "detections": 97, + "detections": 147, "false_accepts": 0, - "ms": 2750 + "ms": 2744 }, { "file": "026_1920px-Kawasaki-Inryo-1.jpg", "title": "File:Kawasaki-Inryo-1.jpg", - "detections": 12, + "detections": 45, "false_accepts": 0, - "ms": 892 + "ms": 2007 }, { "file": "029_1920px-Miyazu_Mipple_1F_2021-04_ac.jpg", "title": "File:Miyazu Mipple 1F 2021-04 ac.jpg", - "detections": 73, + "detections": 157, "false_accepts": 0, - "ms": 852 + "ms": 1782 }, { "file": "030_1920px-Octopus_rice_crackers_No,1.JPG", "title": "File:Octopus rice crackers No,1.JPG", - "detections": 5, + "detections": 22, "false_accepts": 0, - "ms": 247 + "ms": 556 }, { "file": "031_1920px-Octopus_senbei.JPG", "title": "File:Octopus senbei.JPG", - "detections": 4, + "detections": 23, "false_accepts": 0, - "ms": 140 + "ms": 612 }, { "file": "032_1920px-Okaki_for_sale_in_Kawagoe.jpg", "title": "File:Okaki for sale in Kawagoe.jpg", - "detections": 27, + "detections": 70, "false_accepts": 0, - "ms": 267 + "ms": 1042 }, { "file": "033_Okonomiyaki_and_takoyaki_ingredients.jpg", "title": "File:Okonomiyaki and takoyaki ingredients.jpg", - "detections": 39, + "detections": 55, "false_accepts": 0, - "ms": 1228 + "ms": 2553 + }, + { + "file": "034_1920px-Rice_toppings_&_additional_ingredients_(5846028686).jpg", + "title": "File:Rice toppings & additional ingredients (5846028686).jpg", + "detections": 59, + "false_accepts": 0, + "ms": 1871 }, { "file": "035_1920px-SW_4963.jpg", "title": "File:SW 4963.jpg", - "detections": 5, + "detections": 41, "false_accepts": 0, - "ms": 200 + "ms": 755 }, { "file": "037_1920px-Soy_Sauce_selection_(6362318717).jpg", "title": "File:Soy Sauce selection (6362318717).jpg", - "detections": 68, + "detections": 85, "false_accepts": 0, - "ms": 1671 + "ms": 1623 }, { "file": "038_1920px-Soy_sauce_(5846027636).jpg", "title": "File:Soy sauce (5846027636).jpg", - "detections": 103, - "false_accepts": 1, - "ms": 1626 + "detections": 216, + "false_accepts": 0, + "ms": 2534 }, { "file": "039_1920px-Soy_sauce_in_supermarket.JPG", "title": "File:Soy sauce in supermarket.JPG", - "detections": 71, + "detections": 101, "false_accepts": 0, - "ms": 2380 + "ms": 1884 }, { "file": "040_1920px-Spice_Supermarket_Japan.JPG", "title": "File:Spice Supermarket Japan.JPG", - "detections": 254, + "detections": 300, "false_accepts": 0, - "ms": 2544 + "ms": 2994 }, { "file": "041_1920px-Strong_Zero_4962.jpg", "title": "File:Strong Zero 4962.jpg", - "detections": 18, + "detections": 45, "false_accepts": 0, - "ms": 856 + "ms": 1731 }, { "file": "042_1920px-Supermarket_Digital_Displays_(2927051130).jpg", "title": "File:Supermarket Digital Displays (2927051130).jpg", - "detections": 89, + "detections": 97, "false_accepts": 1, - "ms": 3561 + "ms": 2454 }, { "file": "043_1920px-Tasty_Food_Abundance_in_Healthy_Europe.png", "title": "File:Tasty Food Abundance in Healthy Europe.png", - "detections": 13, + "detections": 32, "false_accepts": 0, - "ms": 1202 + "ms": 982 }, { "file": "047_1920px-Tokyo---Ebisu-and-Shibuya---2026-02-04_162.jpg", "title": "File:Tokyo---Ebisu-and-Shibuya---2026-02-04 162.jpg", - "detections": 74, + "detections": 81, "false_accepts": 0, - "ms": 4770 + "ms": 1356 }, { "file": "048_1920px-Tokyo---Ebisu-and-Shibuya---2026-02-04_167.jpg", "title": "File:Tokyo---Ebisu-and-Shibuya---2026-02-04 167.jpg", - "detections": 46, + "detections": 51, "false_accepts": 0, - "ms": 2557 + "ms": 823 }, { "file": "050_1920px-Umami_Snack_(41957534).jpeg", "title": "File:Umami Snack (41957534).jpeg", - "detections": 33, + "detections": 55, "false_accepts": 0, - "ms": 3096 + "ms": 2518 }, { "file": "051_1920px-このかねをならすのはだーれ_(3079692697).jpg", "title": "File:このかねをならすのはだーれ (3079692697).jpg", - "detections": 0, + "detections": 3, "false_accepts": 0, - "ms": 125 + "ms": 170 }, { "file": "052_1920px-スーパーの野菜コーナー_2008_(2785121488).jpg", "title": "File:スーパーの野菜コーナー 2008 (2785121488).jpg", - "detections": 25, + "detections": 85, "false_accepts": 0, - "ms": 1681 + "ms": 1963 }, { "file": "055_1920px-中央フード山手店_-_panoramio.jpg", "title": "File:中央フード山手店 - panoramio.jpg", - "detections": 15, + "detections": 39, "false_accepts": 0, - "ms": 263 + "ms": 689 }, { "file": "056_1920px-何このリポＤ人気⁈買い占めてしまおう。_(4025507198).jpg", "title": "File:何このリポＤ人気⁈買い占めてしまおう。 (4025507198).jpg", - "detections": 38, + "detections": 72, "false_accepts": 0, - "ms": 2912 + "ms": 2286 }, { "file": "057_1920px-味噌売り場_(16183474871).jpg", "title": "File:味噌売り場 (16183474871).jpg", - "detections": 61, + "detections": 76, "false_accepts": 0, - "ms": 3355 + "ms": 2081 }, { "file": "058_1920px-塩ひかえめ_(25347152904).jpg", "title": "File:塩ひかえめ (25347152904).jpg", - "detections": 66, + "detections": 86, "false_accepts": 0, - "ms": 3043 + "ms": 2496 }, { "file": "060_1920px-牛乳_豆乳_2016_(32081175436).jpg", "title": "File:牛乳 豆乳 2016 (32081175436).jpg", - "detections": 103, - "false_accepts": 1, - "ms": 3677 + "detections": 130, + "false_accepts": 0, + "ms": 4677 }, { "file": "061_1920px-表示価格より100円引_(38057600585).jpg", "title": "File:表示価格より100円引 (38057600585).jpg", - "detections": 5, + "detections": 17, "false_accepts": 0, - "ms": 632 + "ms": 899 } ] } \ No newline at end of file