diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 959c7b8..932060f 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -566,7 +566,12 @@ class RecognitionEngine: return self._lane def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: - """Score one crop, keyed by canonical SKU. + """Score one crop, keyed by canonical SKU, in the lane's rank order. + + The insertion order is the ranking and callers rely on it: the local + lane's order comes from its rank key while its value is the margin over + the runner-up, so re-sorting by value names a different SKU than the + lane chose. `list(scores.items())` is therefore the ranked sequence. Every backend groups by SKU before returning. Scoring each reference view separately is what made the demo's default catalog report 22-29 of @@ -680,13 +685,20 @@ class RecognitionEngine: # operational failure must stay distinguishable from absence. decision = decide_identity({}, method=method, scale=self.identity_scale) decision = identity_payload(decision) - results.append( - { - "detection_id": int(box["id"]), - "box": list(box["box"]), - "identity": decision, - } - ) + row = { + "detection_id": int(box["id"]), + "box": list(box["box"]), + "identity": decision, + } + # The catalog row travels with an accepted answer. Without it the + # phone gets a bare SKU id and has to look the name and price up + # itself, which is what left 41 of 41 accepted still-image answers + # without a product row. + if decision.get("state") == STATE_RECOGNIZED: + product = self.product(decision.get("sku_id")) + if product is not None: + row["product"] = product + results.append(row) return results def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]: @@ -900,16 +912,18 @@ class RecognitionEngine: identities: List[Dict[str, Any]] = [] if classify and boxes: identities = self.classify_boxes(image, boxes, method=method) - identities_by_id = {row["detection_id"]: row["identity"] for row in identities} + identities_by_id = {row["detection_id"]: row for row in identities} for box in boxes: - detections.append( - { - "id": box["id"], - "box": box["box"], - "confidence": box["confidence"], - "identity": identities_by_id.get(box["id"], identity_payload(None)), - } - ) + row = identities_by_id.get(box["id"]) + detection = { + "id": box["id"], + "box": box["box"], + "confidence": box["confidence"], + "identity": (row or {}).get("identity") or identity_payload(None), + } + if (row or {}).get("product") is not None: + detection["product"] = row["product"] + detections.append(detection) slots = self.empty_slots(image.shape, boxes) if with_empty_slots else [] classified_at = time.perf_counter() summary = self.summarize(detections, slots) diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index f664a1f..1e12e51 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -22,7 +22,7 @@ detector). The hybrid lane also reports shortlist recall, because a true SKU outside the shortlist is a hard ceiling no verifier can lift. python code/eval_identity.py \ - --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \ + --labels eval/labels/image6.json \ --image add/image6.jpg --stock-dir stock-lopia-clean --lanes matcher --live Labels are {"image", "facings": [{"box", "label"}]} with "unknown" for facings @@ -70,7 +70,7 @@ LIVE_LONG_SIDE = 1280 # Pixel equality is not a leak test: a template re-cut from the same photo with # a different box passes a same-shape comparison. A 64x64 grayscale signature # catches most of those, but it is a heuristic, not a proof — measured on -# image6_v2 against stock-lopia, re-cuts in the realistic 0.94..1.06 box band +# the corrected labels against stock-lopia, re-cuts in the realistic 0.94..1.06 box band # land 7..37 MAD from their facing's signature while the closest pair of # genuinely distinct same-SKU facings is 23.1, so the distributions overlap and # no threshold is clean. 20.0 catches 81% of that band with zero false @@ -116,12 +116,13 @@ def file_hash(path: str) -> Optional[str]: def catalog_hash(stock_dir: str) -> Optional[str]: - """SHA-256 over the catalog's CSV bytes and its image names. + """SHA-256 over the catalog's CSV bytes and each view's bytes. - Image bytes are not hashed: a re-encoded view of the same product is the - same catalog, and hashing every packshot would make the number move for - reasons that do not change a single decision. A changed row or a different - set of views does change it. + The view bytes are hashed, not just their names. Names alone were not + enough: replacing `sku_l09.jpg` with a different product's packshot left + the hash unchanged, so a report could claim the same catalog while every + l09 decision in it had moved. Re-encoding the same image does move the + hash, which is the price of not being able to lie about what was scored. """ digest = hashlib.sha256() csv_path = os.path.join(stock_dir, "stock.csv") @@ -137,8 +138,14 @@ def catalog_hash(stock_dir: str) -> Optional[str]: ) except OSError: names = [] - digest.update(b"\n") - digest.update("\n".join(names).encode("utf-8")) + for name in names: + digest.update(b"\n") + digest.update(name.encode("utf-8")) + try: + with open(os.path.join(stock_dir, name), "rb") as handle: + digest.update(handle.read()) + except OSError: + digest.update(b"unreadable") return digest.hexdigest() @@ -557,7 +564,7 @@ def shortlist_recall( shortlist already dropped. The denominator is enrolled facings only — a facing whose SKU has no reference cannot be retrieved by any shortlist, so counting it here reports a retrieval failure that is really a catalog gap. - Measured on image6_v2, that distinction is 67% vs 98% at top-5. + Measured on the corrected labels, that distinction is 67% vs 98% at top-5. """ hits = total = 0 misses: List[Dict[str, Any]] = [] diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index ba6b74c..b730d9f 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -8,9 +8,8 @@ evidence into accepted / ambiguous / unknown / rejected: ranks fine-grained siblings correctly: 40/42 enrolled facings first at full resolution, against 32/42 for Van's composite. - `TextLane` reads the package text and compares it to each catalog row's - signature (its printed name plus the OCR of its enrolled packshot). It is the - only measured mechanism that rejects a near-unknown sibling, and it can name - catalog rows that have no enrolled image at all. + printed name. It is the only measured mechanism that rejects a near-unknown + sibling, and it can name catalog rows that have no enrolled image at all. Why a policy and not a threshold: the matcher's raw score is not comparable across references of different resolution, and a single unverified match used @@ -24,10 +23,10 @@ calibrated model. E-4 replaces it once Tier-2 held-out labels exist. from __future__ import annotations +import math import os import re import sys -import unicodedata from dataclasses import dataclass from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -38,7 +37,7 @@ if current_dir not in sys.path: import cv2 # noqa: E402 import numpy as np # noqa: E402 -from rebuild_catalog import sku_of # noqa: E402 +from rebuild_catalog import bigrams, normalize_text, sku_of # noqa: E402 # The local lane's accept line, in the units of `confidence()` below. Measured # on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59% @@ -134,6 +133,14 @@ class LocalLane: if sku not in self.sku_ids: self.sku_ids.append(sku) for height in heights: + if height > image.shape[0]: + # Never upscale a reference. A 305 px view blown up to 400 + # is a blur, and a blur matches everything: the l09 shelf + # crop won 31 of 54 crops on the shelf it was cut from, at + # a median margin of 4.9, because its upscaled level had no + # high-frequency detail to disagree with. A view is used at + # the levels it actually has. + continue scaled = cv2.resize( image, (max(1, int(image.shape[1] * height / image.shape[0])), int(height)), @@ -250,23 +257,10 @@ class LocalLane: return {sku: value for sku, value in self.rank(crop)} -# Katakana folding: OCR reads small kana and the long-vowel mark inconsistently, -# so キユーピー and キユピー have to land on the same bigrams. -_KANA_FOLD = str.maketrans( - "ッャュョァィゥェォヮヵヶユ", "ツヤユヨアイウエオワカケユ" -) - - -def normalize_text(text: str) -> str: - """NFKC, uppercase, kana-folded, alphanumeric and `%` only.""" - text = unicodedata.normalize("NFKC", text).upper() - text = text.translate(_KANA_FOLD) - text = text.replace("キユーピー", "キユピ").replace("キューピー", "キユピ") - return "".join(ch for ch in text if ch.isalnum() or ch == "%") - - -def bigrams(text: str) -> set: - return {text[i:i + 2] for i in range(len(text) - 1)} +# Text normalization and bigrams come from `rebuild_catalog`, which uses them +# to match a catalog name against a packshot's OCR. Two copies of the same +# folding rules drift: the lane's copy folded ユ to itself while the catalog's +# did not, so the same read scored differently depending on which one ran. def identity_bigrams(read: str) -> set: @@ -295,34 +289,75 @@ def carries_identity(read: str) -> bool: return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR -def text_similarity(read: str, signature: str) -> float: +def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float]] = None) -> float: """Share of the read text's bigrams that the signature explains. Asymmetric on purpose: the crop shows a few words of a package, the signature holds the whole name, so the question is how much of what was read the catalog row accounts for. + + `weights` is the inverse document frequency of each bigram over the + catalog's signatures. Without it, brand copy that every sibling prints + ("ピュアセレクト", "マヨネーズ", "国産新鮮たまご") counts as much as the words + that tell the products apart ("コクうま", "カロリーカット", "65"), so a row + whose signature happens to carry more shared copy outranks the row the + crop actually shows. Measured on image6: every l10 facing reads + "ピュア セレクト マヨネーズ" with no "コクうま" and no "65", yet l09 tied or + won on shared brand text alone. """ read_grams = bigrams(normalize_text(read)) if not read_grams: return 0.0 - return len(read_grams & bigrams(normalize_text(signature))) / max(1, len(read_grams)) + shared = read_grams & bigrams(normalize_text(signature)) + if not shared: + return 0.0 + if weights is None: + return len(shared) / max(1, len(read_grams)) + total = sum(weights.get(gram, 1.0) for gram in read_grams) + if total <= 0: + return 0.0 + return sum(weights.get(gram, 1.0) for gram in shared) / total + + +def bigram_weights(signatures: Sequence[str]) -> Dict[str, float]: + """Inverse document frequency of each bigram over the catalog signatures. + + A bigram every row prints carries no information about which row this is, + so it is worth ~0; a bigram one row prints is worth the most. Smoothed and + floored at 0.05 so a shared word still counts a little and a read made + entirely of shared copy scores low rather than dividing by zero. + """ + counts: Dict[str, int] = {} + for signature in signatures: + for gram in set(bigrams(normalize_text(signature))): + counts[gram] = counts.get(gram, 0) + 1 + total = max(1, len(signatures)) + return { + gram: max(0.05, math.log((total + 1) / (count + 1))) + for gram, count in counts.items() + } class TextLane: - """Package text against each catalog row's signature. + """Package text against each catalog row's printed name. - The signature is the row's printed name plus the OCR of its enrolled - packshot, so it is built from the catalog and never from the evaluation - photo. Rows with no image still get a signature from their name, which is - how the lane can name a SKU no matcher can see. + The signature is the row's name, so it is built from the catalog and never + from the evaluation photo. Rows with no image still get a signature, which + is how the lane can name a SKU no matcher can see. """ name = "text" def __init__( - self, signatures: Dict[str, str], ocr_engine: Any = None, lock: Any = None + self, + signatures: Dict[str, str], + ocr_engine: Any = None, + lock: Any = None, ) -> None: self.signatures = {str(sku): text for sku, text in signatures.items()} + # Bigram weights over the catalog's own names, so the lane can tell + # copy every sibling prints from the words that name one product. + self.weights = bigram_weights(list(self.signatures.values())) self.ocr_engine = ocr_engine # The engine's OCR lock. Vision tolerates concurrency but the CPU does # not, and this lane runs one OCR per crop on the classify pool, so it @@ -342,19 +377,7 @@ class TextLane: sku = str(row.get("sku_id") or "") if not sku: continue - signature = str(row.get("product_name") or "") - if ocr_engine is not None and getattr(ocr_engine, "available", lambda: False)(): - path = os.path.join(stock_dir, f"{sku}.jpg") - if os.path.exists(path): - image = cv2.imread(path) - if image is not None: - try: - result = ocr_engine.read(image) - except Exception: - result = None - if result is not None: - signature += " " + " ".join(line.text for line in result.lines) - signatures[sku] = signature + signatures[sku] = str(row.get("product_name") or "") return cls(signatures, ocr_engine=ocr_engine, lock=lock) def read(self, crop: Any) -> str: @@ -385,10 +408,19 @@ class TextLane: return self.scores_from_text(self.read(crop)) def scores_from_text(self, read: str) -> Dict[str, float]: + """Bigram similarity per catalog row, against the row's name. + + The name only. Scoring against the packshot's own OCR as well is how + l09 used to win l10's facings: l09's packshot prints "マヨネーズ", + which is l10's actual name, so a row with a photographed packshot + could explain a sibling's text better than the sibling's own name + could. Every row in the catalog has a name, so the packshot adds + nothing but that confusion. + """ if len(normalize_text(read)) < 4: return {} return { - sku: text_similarity(read, signature) + sku: text_similarity(read, signature, self.weights) for sku, signature in self.signatures.items() } diff --git a/recognization/code/live.py b/recognization/code/live.py index 31bf465..54e7421 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -39,6 +39,7 @@ from engine import ( # noqa: E402 STATE_UNRECOGNIZED, RecognitionEngine, ) +from identity_lanes import carries_identity # noqa: E402 from tracking import SessionTracker # noqa: E402 EMPTY_SLOT_HISTORY = 5 @@ -154,6 +155,10 @@ class TrackState: # with the rest of the evidence when the object generation changes. text_rank: List[Tuple[str, float]] = field(default_factory=list) text_read: str = "" + # The frame the newest label box was measured on. The box is in that + # frame's coordinates; a client mapping it back to the source image needs + # this id, because the result that carries the label arrives on a later one. + label_frame_id: int = 0 def area(self) -> int: if not self.raw_box: @@ -615,11 +620,10 @@ class LiveSession: if state.box ] tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset) - # Collect the classification work under the lock; submit it after # releasing it. add_done_callback can run the callback inline when the # future already finished, and the callback takes this same lock. - submissions = self._plan_classification(image, seq) + submissions = self._plan_classification(image, seq, int(frame_id)) crop_requests = self._plan_crop_requests(image.shape, seq, int(frame_id), measured) summary = self._summary_locked() @@ -635,7 +639,7 @@ class LiveSession: summary["price_tags"] = tag_summary["tags"] summary["compliance"] = tag_summary["compliance"] - self._submit_classification(submissions, method, generation) + self._submit_classification(submissions, method, generation, int(frame_id)) self._run_tag_passes(image, tag_bands) result = { @@ -685,7 +689,16 @@ class LiveSession: continue track.state = STATE_RECOGNIZED track.confirmed = True - track.label_box = list(track.box) + # The raw detection box, not the smoothed track box: smoothing + # lags a pan by about one step, and the label is about where the + # product was measured, not where the filter has drifted to. + track.label_box = list(track.raw_box or track.box) + # The box is measured on this frame, so the client must map it + # with this frame's offset. Leaving the default 0 makes every + # inherited label map as if it came from the first frame, which + # during a pan puts it a whole pan-distance away from the + # product it names. + track.label_frame_id = int(self._latest_frame_id) track.last_source = SOURCE_INHERITED track.identity = { "sku_id": entry["sku_id"], @@ -762,11 +775,17 @@ class LiveSession: track.crop_attempts = 0 # ------------------------------------------------------------------ classification - def _plan_classification(self, image: Any, seq: int) -> List[Tuple[TrackState, Any, int]]: + def _plan_classification( + self, image: Any, seq: int, frame_id: int + ) -> List[Tuple[TrackState, Any, int]]: """Pick what to classify now. Must be called with the session lock held. Returns (track, crop, object_generation) triples; the caller submits them - after the lock is released. + after the lock is released. The frame id travels with the plan because + the label box is in *that* frame's coordinates: by the time the answer + arrives the track has moved, and a client mapping the box back to the + source image needs the frame the box was measured on, not the one the + result was delivered with. """ frame_h, frame_w = image.shape[:2] margin_y = int(frame_h * self.settings.edge_margin_ratio) @@ -812,12 +831,17 @@ class LiveSession: crop = self.engine.crop(image, track.raw_box) track.in_flight += 1 track.last_attempt_seq = seq + track.label_frame_id = int(frame_id) track.last_attempt_area = track.area() planned.append((track, crop, track.object_generation)) return planned def _submit_classification( - self, planned: List[Tuple[TrackState, Any, int]], method: str, generation: int + self, + planned: List[Tuple[TrackState, Any, int]], + method: str, + generation: int, + frame_id: int, ) -> None: """Submit planned crops. Called WITHOUT the session lock held.""" for track, crop, object_generation in planned: @@ -841,9 +865,8 @@ class LiveSession: continue future.add_done_callback( lambda fut, tid=track_id, gen=generation, m=method, started=submitted_at, - obj=object_generation, box=submitted_box: self._on_classified( - fut, tid, gen, m, started, obj, box - ) + obj=object_generation, box=submitted_box, fid=int(frame_id): + self._on_classified(fut, tid, gen, m, started, obj, box, fid) ) # ------------------------------------------------------------------ crop lane @@ -1023,9 +1046,8 @@ class LiveSession: self._crops_accepted += 1 future.add_done_callback( lambda fut, tid=track_id_out, gen=generation, m=method, started=submitted_at, - obj=object_generation, box=box: self._on_classified( - fut, tid, gen, m, started, obj, box, SOURCE_CROP - ) + obj=object_generation, box=box, fid=int(record["frame_id"]): + self._on_classified(fut, tid, gen, m, started, obj, box, fid, SOURCE_CROP) ) return { "type": "crop_ack", @@ -1117,6 +1139,7 @@ class LiveSession: submitted_at: float, object_generation: int, submitted_box: List[int], + label_frame_id: int = 0, source: str = SOURCE_FRAME, ) -> None: try: @@ -1161,9 +1184,12 @@ class LiveSession: self._classify_times.append(finished_at - submitted_at) # The reading is cached on the track so later attempts skip OCR. - # An empty read is not cached: a crop that was too small or too - # blurry to read says nothing about the next one. - if text_read: + # Only a read that could name something is cached: a crop that was + # too small or too blurry says nothing about the next one, and a + # read of just a size ("400g") matches every 400g product's size + # line, so caching it would let one unreadable crop veto the + # product for the rest of the track's life. + if text_read and carries_identity(text_read): track.text_rank = list(text_rank or []) track.text_read = text_read elif track.text_read: @@ -1174,6 +1200,7 @@ class LiveSession: track.state = STATE_UNRECOGNIZED track.confirmed = True track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.identity = None track.candidates = [] return @@ -1223,6 +1250,7 @@ class LiveSession: # The label belongs to the box the crop came from, not to wherever # the track has moved by the time classification finishes. track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.drift_frames = 0 track.identity = { "sku_id": decision.get("best_sku"), @@ -1239,6 +1267,7 @@ class LiveSession: ) track.confirmed = True track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.drift_frames = 0 track.identity = None else: @@ -1278,10 +1307,20 @@ class LiveSession: ) payload = { "track_id": track.track_id, + # The frame `label_box` is in. The result carrying the label arrives + # on a later frame, so a client that maps the box back to its own + # source image must use this id's offset, not the result's. + "label_frame_id": int(track.label_frame_id), # box: smoothed track box, for drawing. raw_box: the detection box the # label is anchored to, for cropping and for explaining label changes. "box": list(track.box), "raw_box": list(track.raw_box), + # The box the label was measured on, in frame coordinates. It is + # what the label is *about*: `box` has moved on by the time a + # classification finishes, so scoring a label against `box` scores + # it against wherever the track drifted to. An evaluator matching + # labels to ground truth needs this one. + "label_box": list(track.label_box or track.box), "confidence": round(float(track.confidence), 4), "first_seen_seq": track.first_seq, "relabels": track.relabels, diff --git a/recognization/code/score_stream.py b/recognization/code/score_stream.py new file mode 100644 index 0000000..e5bbbb6 --- /dev/null +++ b/recognization/code/score_stream.py @@ -0,0 +1,209 @@ +"""Score a streamed clip's labels against ground truth: the G1 number. + +The stream path was reported as "53 of 131 tracks", which is not comparable to +Van's 57% of labelled facings - the two count different things. But the clip is +built from `add/image6.jpg`, which has labels, so matching each confirmed +track's `label_box` to the labelled facings gives stream-path coverage directly. + + python code/score_stream.py TRACKS.json --labels eval/labels/image6.json + +`TRACKS.json` is what `stream_client.py --tracks` writes: one row per track that +settled on a label, with the box that label was measured on, already mapped back +to source-image coordinates. + +Reported per run: correct / in-catalog facings, precision, unknown accepts, and +tracks that matched no labelled facing. A track whose box overlaps a facing the +catalog does not enrol is an unenrolled false accept - the dangerous kind, and +the one the evaluator's own precision denominator used to hide. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List, Optional, Sequence, Tuple + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +# A facing clipped by the window edge is not something a run could have named, +# so the denominator keeps only facings at least this far inside the window. +VISIBLE_MIN = 0.5 + + +def iou(a: Sequence[int], b: Sequence[int]) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + inter_w = max(0, min(ax2, bx2) - max(ax1, bx1)) + inter_h = max(0, min(ay2, by2) - max(ay1, by1)) + inter = inter_w * inter_h + area_a = max(0, ax2 - ax1) * max(0, ay2 - ay1) + area_b = max(0, bx2 - bx1) * max(0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def load_facings(labels_path: str) -> List[Dict[str, Any]]: + with open(labels_path) as handle: + payload = json.load(handle) + return payload["facings"] + + +def enrolled_skus(stock_dir: str) -> set: + """Canonical SKUs with at least one enrolled view.""" + from rebuild_catalog import sku_of + + return { + sku_of(name) for name in os.listdir(stock_dir) + if name.lower().endswith((".jpg", ".jpeg", ".png")) + } + + +def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float: + """Share of a facing's area inside the window the client actually showed.""" + wx0, wy0, wx1, wy1 = window + bx0, by0, bx1, by1 = box + inter_w = max(0, min(bx1, wx1) - max(bx0, wx0)) + inter_h = max(0, min(by1, wy1) - max(by0, wy0)) + area = max(0, bx1 - bx0) * max(0, by1 - by0) + return (inter_w * inter_h) / area if area > 0 else 0.0 + + +def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], + stock_dir: str, iou_min: float, + windows: Optional[Sequence[Sequence[int]]] = None + ) -> Dict[str, Any]: + """Score the labels a stream settled on. + + `windows` are the source-image rects the client showed, as + [x0, y0, x1, y1] each. The denominator is the facings at least half inside + one of them, in x and y both: a 1280x720 window over a 1500x2000 photo sees + 24 of the 63 in-catalog facings, and a facing clipped by the window edge is + not something any run could have named. A panned run shows a different rect + each frame, so the reachable set is the union. Without it the denominator is + every in-catalog facing. + + A correct answer is counted before the unenrolled and unknown branches. The + text lane names SKUs the image lane has no view for, which is the point of + it, and scoring that as a false accept punishes the lane for working. + """ + if windows: + facings = [ + f for f in facings + if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN + ] + in_catalog = [f for f in facings if f["label"] != "unknown"] + enrolled = enrolled_skus(stock_dir) + + correct = 0 + wrong = 0 + unenrolled_false = 0 + unknown_false = 0 + unmatched = 0 + rows: List[Dict[str, Any]] = [] + # Coverage is per facing, precision is per label. A track that restarts on + # the same product, or two tracks on one facing, is two claims but one + # facing: counting tracks as coverage reports 138% of a 24-facing window. + named: Dict[int, str] = {} + + for track in tracks: + box = track.get("label_box") or [] + if len(box) != 4: + unmatched += 1 + continue + best, best_iou = None, 0.0 + for facing in facings: + value = iou(box, facing["box"]) + if value > best_iou: + best, best_iou = facing, value + if best is None or best_iou < iou_min: + unmatched += 1 + rows.append({"track_id": track["track_id"], "predicted": track["sku_id"], + "truth": None, "iou": round(best_iou, 3), "outcome": "unmatched"}) + continue + truth = best["label"] + predicted = track["sku_id"] + if predicted == truth: + correct += 1 + outcome = "correct" + named[best["index"]] = predicted + elif truth == "unknown": + unknown_false += 1 + outcome = "unknown_false_accept" + elif truth not in enrolled: + unenrolled_false += 1 + outcome = "unenrolled_false_accept" + else: + wrong += 1 + outcome = "wrong" + rows.append({"track_id": track["track_id"], "predicted": predicted, + "truth": truth, "iou": round(best_iou, 3), "outcome": outcome}) + + shown = correct + wrong + unenrolled_false + unknown_false + return { + "in_catalog_facings": len(in_catalog), + "facings_named": len(named), + "tracks_scored": len(tracks), + "correct": correct, + "wrong": wrong, + "unenrolled_false_accepts": unenrolled_false, + "unknown_false_accepts": unknown_false, + "unmatched_tracks": unmatched, + "shown": shown, + "coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0, + "precision": round(correct / shown, 4) if shown else 0.0, + "rows": rows, + } + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Score streamed labels against ground truth") + parser.add_argument("tracks", help="JSON written by stream_client.py --tracks") + parser.add_argument("--labels", default="eval/labels/image6.json") + parser.add_argument("--stock-dir", default="stock-lopia-clean") + parser.add_argument("--iou", type=float, default=0.5) + parser.add_argument("--json", dest="json_path", default=None) + parser.add_argument("--window", default=None, + help="x0,y0,x1,y1 the client showed; the denominator is " + "restricted to facings at least half inside it") + parser.add_argument("--verbose", action="store_true", help="print every scored track") + args = parser.parse_args(argv) + + with open(args.tracks) as handle: + payload = json.load(handle) + windows = None + if args.window: + windows = [tuple(int(part) for part in args.window.split(","))] + elif payload.get("windows"): + windows = [tuple(w) for w in payload["windows"]] + report = score(payload["tracks"], load_facings(args.labels), args.stock_dir, + args.iou, windows) + report["source"] = payload.get("source") + report["pace"] = payload.get("pace") + report["seconds"] = payload.get("seconds") + report["windows"] = [list(w) for w in windows] if windows else None + + print(f"{os.path.basename(args.tracks)} pace={report['pace']} " + f"{report['seconds']}s") + print(f" facings named {report['facings_named']}/{report['in_catalog_facings']} " + f"in-catalog ({report['coverage']:.0%}) labels correct {report['correct']} " + f"precision {report['precision']:.0%} shown {report['shown']}") + print(f" wrong {report['wrong']} unenrolled false accepts " + f"{report['unenrolled_false_accepts']} unknown false accepts " + f"{report['unknown_false_accepts']} unmatched tracks {report['unmatched_tracks']}") + if args.verbose: + for row in report["rows"]: + print(f" t{row['track_id']:4d} {row['outcome']:24s} " + f"pred={row['predicted']} truth={row['truth']} iou={row['iou']}") + + if args.json_path: + with open(args.json_path, "w") as handle: + json.dump(report, handle, indent=2) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index 8e4b6ae..ddb1406 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -56,6 +56,15 @@ class FrameSource: self.image = None self.pan_x = 0 self.pan_direction = 1 + # Where the last frame was cut from the source image. (0, 0) for a + # camera or a video, whose frames are the whole image. + self.offset = (0, 0) + # The source-image rects the window showed, so an evaluator can restrict + # its denominator to the facings this run could actually have seen. + # A list, because a panned run shows a different rect each frame and the + # reachable set is their union. Both axes matter: a 1280-wide window + # over a 1500-wide photo is x-limited as well as y-limited. + self.windows: List[Tuple[int, int, int, int]] = [] if source.isdigit(): self.capture = cv2.VideoCapture(int(source)) elif os.path.isfile(source) and source.lower().endswith((".mp4", ".mov", ".avi", ".mkv", ".webm")): @@ -66,6 +75,12 @@ class FrameSource: raise SystemExit(f"cannot read source {source}") def frame(self, index: int): + """One frame, plus where it was cut from the source image. + + The offset is what lets an evaluator map a track's box back to the + source image's coordinates: a label measured on a pan window is only + comparable to ground truth once the window's origin is added back. + """ if self.image is not None: height, width = self.image.shape[:2] window_w = min(self.pan_width, width) @@ -74,6 +89,8 @@ class FrameSource: x = min(self.pan_x, max_x) y = max(0, (height - window_h) // 2) window = self.image[y:y + window_h, x:x + window_w] + self.offset = (x, y) + self.windows.append((x, y, x + window_w, y + window_h)) self.pan_x += self.pan_direction * self.pan_step if self.pan_x >= max_x or self.pan_x <= 0: self.pan_direction *= -1 @@ -111,6 +128,11 @@ def main() -> None: parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path") parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image") parser.add_argument("--seconds", type=float, default=20.0) + parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table") + parser.add_argument("--tracks", default=None, + help="write per-track labels and their boxes (source-image " + "coordinates) as JSON, for scoring against ground truth") + parser.add_argument("--quiet", action="store_true", help="only print the final summary") parser.add_argument("--width", type=int, default=1280, help="send width") parser.add_argument("--quality", type=int, default=75) parser.add_argument("--max-inflight", type=int, default=1) @@ -119,8 +141,6 @@ def main() -> None: "'result': send the next frame only after a result, so the " "pan stays continuous (the mode to use for stability checks)") parser.add_argument("--method", default=None, help="surf | orb | brisk | hybrid") - parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table") - parser.add_argument("--quiet", action="store_true", help="only print the final summary") args = parser.parse_args() url = args.url @@ -135,6 +155,12 @@ def main() -> None: first_label: Dict[int, float] = {} label_latencies: List[float] = [] labels: Dict[int, str] = {} + pending_frame: Optional[int] = None + # Per track: the label it settled on, the box that label was measured on, + # and the pan offset of the frame that box lives in. Written out with + # --tracks so an evaluator can match labels to ground truth. + track_records: Dict[int, Dict[str, Any]] = {} + frame_offsets: Dict[int, Tuple[int, int]] = {} label_changes = 0 unexplained_changes = 0 relabel_seen: Dict[int, int] = {} @@ -192,15 +218,33 @@ def main() -> None: first_label[track_id] = now if not inherited: label_latencies.append((now - started) * 1000.0) - elif labels.get(track_id) != sku: - # Only a measured-to-measured change on the same object is a - # label change; inherited labels describe the same product - # under a new track id. + # Only a measured-to-measured change on the same object is a + # label change; inherited labels describe the same product + # under a new track id. + if track_id in labels and labels[track_id] != sku: if not inherited and not inherited_seen.get(track_id): label_changes += 1 if relabels <= relabel_seen.get(track_id, 0): unexplained_changes += 1 labels[track_id] = sku + # The box the label is about, in source-image coordinates. The + # offset comes from the frame the box was measured on, not the + # frame this result arrived with: the pan has moved on since. + offset = frame_offsets.get(track.get("label_frame_id"), (0, 0)) + label_box = track.get("label_box") or track.get("box") or [] + if len(label_box) == 4: + track_records[track_id] = { + "track_id": track_id, + "sku_id": sku, + "label_box": [ + label_box[0] + offset[0], label_box[1] + offset[1], + label_box[2] + offset[0], label_box[3] + offset[1], + ], + "label_frame_id": track.get("label_frame_id"), + "result_frame_id": message["frame_id"], + "inherited": inherited, + "restarts": restarts, + } inherited_seen[track_id] = inherited relabel_seen[track_id] = relabels restart_seen[track_id] = restarts @@ -219,6 +263,7 @@ def main() -> None: break index += 1 frame_id = (frame_id + 1) & 0xFFFFFFFF + frame_offsets[frame_id] = source.offset payload = encode(frame, args.width, args.quality) if len(in_flight) >= args.max_inflight: if pending_frame is not None: @@ -266,6 +311,15 @@ def main() -> None: "first_label_ms_p50": round(statistics.median(label_latencies), 0) if label_latencies else None, "final_summary": last_summary, } + if args.tracks: + with open(args.tracks, "w") as handle: + json.dump({ + "windows": [list(w) for w in source.windows] or None, + "source": args.source, + "pace": args.pace, + "seconds": round(elapsed, 1), + "tracks": sorted(track_records.values(), key=lambda row: row["track_id"]), + }, handle, indent=2) if args.json: print(json.dumps(summary, indent=2)) else: diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index f814299..b014ed2 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -30,6 +30,51 @@ from engine import ( # noqa: E402 ) +class TextLaneSignatureTests(unittest.TestCase): + """The lane must score a crop against the row's name, not its packshot. + + l09's packshot prints "マヨネーズ", which is l10's actual name. Scoring the + packshot's OCR as part of l09's signature let l09 explain an l10 crop's + text better than l10's own name could, so l09 won l10's facings. Measured + on image6: l10 correct went 2 -> 5 of 11, wrong 3 -> 1, precision 95% -> + 97%, and the l09 hub alarm cleared. + """ + + ROWS = [ + {"sku_id": "sku_l09", "product_name": "ピュアセレクト コクうま カロリーカット65% 360g"}, + {"sku_id": "sku_l10", "product_name": "ピュアセレクト マヨネーズ 450g"}, + {"sku_id": "sku_l03", "product_name": "キューピー からしマヨネーズ 200g"}, + ] + + def lane(self): + from identity_lanes import TextLane + + return TextLane.from_catalog(self.ROWS, "no-such-dir", ocr_engine=None) + + def test_a_read_of_the_plain_mayonnaise_names_the_plain_mayonnaise(self): + scores = self.lane().scores_from_text( + "AJINOMOTO とれて3日以内の 国産新鮮たまご ピュア セレクト マヨネーズ" + ) + self.assertEqual(max(scores, key=scores.get), "sku_l10") + + def test_a_read_of_the_variant_names_the_variant(self): + scores = self.lane().scores_from_text("ピュアセレクト コクうま カロリーカット65%") + self.assertEqual(max(scores, key=scores.get), "sku_l09") + + def test_shared_brand_copy_alone_does_not_name_a_row(self): + """Every row prints マヨネーズ, so it must not decide between them. + + The lane scores l10 and l03 equally on that word alone; the policy's + margin gate is what turns the tie into no answer. + """ + from identity_lanes import DecisionPolicy + + scores = self.lane().scores_from_text("マヨネーズ") + self.assertEqual(scores["sku_l10"], scores["sku_l03"]) + decision = DecisionPolicy().decide([], scores, text_read="マヨネーズ") + self.assertIsNone(decision["best_sku"]) + + class FakeDetector: """Records concurrency and returns fixed boxes.""" @@ -199,6 +244,33 @@ class EngineClassificationTests(unittest.TestCase): self.assertEqual(rows[0]["identity"]["state"], "recognized") self.assertEqual(rows[0]["identity"]["sku_id"], "sku_a") + def test_an_accepted_detection_carries_its_catalog_row(self): + """The phone must not have to look up a name and price from a bare SKU. + + `classify_boxes` returned only the identity, so every accepted + still-image answer reached the client as an id with no product row: + measured through the real server, 41 of 41 accepted detections had no + `product`. The live path attached it; the still path did not. + """ + engine, _, _, _ = make_engine() + detections = [ + {"id": 0, "box": [10, 10, 60, 90], "confidence": 0.9, + "identity": identity_payload(None)}, + ] + rows = engine.classify_boxes(make_image(), detections, method="hybrid") + self.assertEqual(rows[0]["identity"]["state"], "recognized") + self.assertEqual(rows[0]["product"]["product_name"], "Alpha") + self.assertEqual(rows[0]["product"]["price"], 100.0) + + result = engine.recognize(make_image(), method="hybrid") + accepted = [ + row for row in result["detections"] + if row["identity"]["identity_level"] == "accepted" + ] + self.assertTrue(accepted, "the fake detector and matcher accept a box") + for row in accepted: + self.assertIn("product", row, "an accepted answer must carry its row") + def test_a_failing_crop_reports_unrecognized_not_pending(self): """One bad crop must not fail the batch, and must not read as pending. @@ -519,6 +591,51 @@ class DecisionPolicyTests(unittest.TestCase): self.assertEqual(decision["identity_level"], "rejected") +class LocalLaneConcurrencyTests(unittest.TestCase): + """The lane is shared by the classify pool, so `rank` must be thread-safe. + + One `cv2.BFMatcher` is reused across calls. The classify pool runs up to + eight workers, so a lane that mutated matcher state per call would return + another crop's answer under load - the failure would look like a + recognition error, not a crash. + """ + + def test_concurrent_ranking_matches_solo_ranking(self): + import threading + + import cv2 + + from identity_lanes import LocalLane + + stock_dir = os.path.join(os.path.dirname(current_dir), "stock-lopia-clean") + if not os.path.isdir(stock_dir): + self.skipTest("no catalog checked out") + image_path = os.path.join(os.path.dirname(current_dir), "add", "image6.jpg") + if not os.path.isfile(image_path): + self.skipTest("no evaluation photo checked out") + lane = LocalLane(stock_dir) + image = cv2.imread(image_path) + crops = [image[700:1000, 600 + 40 * i:800 + 40 * i] for i in range(8)] + solo = lane.rank(crops[0]) + results = {} + errors = [] + + def work(index: int) -> None: + try: + for _ in range(6): + results[index] = lane.rank(crops[index]) + except Exception as error: # noqa: BLE001 - reported below + errors.append(repr(error)) + + threads = [threading.Thread(target=work, args=(i,)) for i in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(errors, []) + self.assertEqual(results[0], solo, "a shared matcher must not cross crops") + + class CatalogImageTests(unittest.TestCase): def test_catalog_image_path_is_resolved_through_inventory(self): engine, _, _, _ = make_engine() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index 6c6ef92..ecf721e 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -403,6 +403,29 @@ class LocalScaleTests(unittest.TestCase): self.assertEqual(engine.ocr_calls, 1, "OCR must run once per track") engine.shutdown() + def test_a_size_only_read_is_not_cached(self): + """An unreadable crop must not veto the product for the track's life. + + The cache used to keep any non-empty read, so a first crop that read + only "400g" was replayed on every later attempt: the track could never + be named even after a clean crop arrived, because the cached read + replaced it. Only a read that carries identity text is cached now, so + the next attempt gets a fresh OCR. + """ + session, engine, _, _ = make_session(engine_cls=LocalScaleEngine) + engine.lane_answers = [ + ({}, [("sku_a", 0.9)], "400g"), + ({}, [("sku_a", 0.9)], "コクうま 65%"), + ] + [({}, [("sku_a", 0.9)], "コクうま 65%")] * 6 + labelled = False + for frame_id in range(1, 14): + for track in process(session, frame_id)["tracks"]: + if track["identity"]["sku_id"] == "sku_a": + labelled = True + self.assertTrue(labelled, "a later readable crop must still name the track") + self.assertGreater(engine.ocr_calls, 1, "the size-only read must not be cached") + engine.shutdown() + class TagPassIntegrationTests(unittest.TestCase): """The tag pass must reach the live result without blocking frames.""" diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md b/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md index f555928..a06d86e 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md +++ b/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md @@ -112,7 +112,7 @@ verification → calibrated open-set decision → temporal accumulation. ### The four ceilings, separated -On corrected labels (`evidence/image6_v2.json`): 109 scored facings, 61 +On corrected labels (`eval/labels/image6.json`): 109 scored facings, 61 in-catalog across 11 SKUs, 48 unknown, 2 excluded. The clean catalog enrolls 7 SKUs covering 42 of the 61 in-catalog facings. @@ -288,7 +288,7 @@ rank, decision, lane latencies), and report the four ceilings separately. ### Benchmark tiers (build these first; they block every other decision) -- **Tier 1, diagnostic:** `evidence/image6_v2.json` (109 facings). Dev only. +- **Tier 1, diagnostic:** `eval/labels/image6.json` (109 facings). Dev only. Free to tune on. Its numbers never ship. - **Tier 2, held-out regression:** new captures that are never enrolled and never tuned on. At least 3 visits across ≥ 2 LOPIA locations or days, same @@ -335,7 +335,7 @@ reverted. | `test_live.py` change | encodes the reverted budget | Revert with it. | | `code/rebuild_catalog.py` (untracked) | catalog tooling | Keep. Add image↔CSV bijection, packshot-OCR vs name check, and eval-session refusal (enrollment protocol below). | | `stock-lopia-clean/` (untracked) | catalog data | Keep the images and provenance. Fix `stock.csv` (F-04) and the README's "Six SKUs" (it lists five). Replace the l06 packshot with the current package. | -| `eval/labels/image6.json` (untracked) | ground truth | **Replace** with `evidence/image6_v2.json`, schema: `label`, `source`, `note`, `v1_label`. | +| `eval/labels/image6.json` | ground truth | Adopted; schema: `label`, `source`, `note`, `v1_label`. | | `identity.decide_identity` inlier branch | production | **Rewrite** as the calibrated decision (E-4); keep the cosine branch only while the gallery exists. | | Production score path (`engine.score_crop`, `classify_boxes`, `inspect`) | production | **Rewrite:** return per-SKU evidence grouped by canonical SKU (F-02). | | `feature_matcher.py` composite score | production | **Rewrite** into variant B behind an interface; keep Van's scorer as a selectable baseline lane for E-1. | @@ -431,7 +431,7 @@ covers a recognition backend. The product-level floor is Van's system (G1–G4). - **Evidence:** `evidence/image6-v1-labels-overlay.jpg`; the per-index list in §3. Rescoring the same runs under v2 moves ungated live top-1 from 26/35 to 35/42. - **Target:** every scored label carries provenance; ambiguous facings are excluded. -- **Change:** adopt `evidence/image6_v2.json`; schema-check `label/source/note`. +- **Change:** adopt `eval/labels/image6.json`; schema-check `label/source/note`. - **Validation:** E-0 double-labelling on Tier 2. - **Acceptance:** no architecture claim cites v1. @@ -602,33 +602,43 @@ and overstates its precision, so the evaluator now has a `van` lane: the same scores, always answered. Every "vs Van" claim in this record uses that row. .venv/bin/python code/eval_identity.py --stock stock-lopia-clean \ - --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \ + --labels eval/labels/image6.json \ --image add/image6.jpg --lanes van,matcher,local --live full resolution — 109 scored facings (61 in-catalog, 48 unknown) lane catalog identity decision end-to-end precision false acc van 69% 76% 76% 52% 29% 77 matcher 69% 76% 71% 49% 39% 46 HUB! - local 69% 95% 88% 61% 77% 11 HUB! + local 69% 93% 86% 59% 88% 5 live resolution — 109 scored facings (61 in-catalog, 48 unknown) lane catalog identity decision end-to-end precision false acc van 69% 83% 83% 57% 32% 74 matcher 69% 83% 76% 52% 43% 43 - local 69% 86% 86% 59% 95% 2 + local 69% 86% 81% 56% 97% 1 Against Van's real contract the local lane wins both axes at both resolutions: -61% at 77% against 52% at 29% (full), and 59% at 95% against 57% at 32% (live). -The live coverage margin is one facing (36 against 35), so it is not a -meaningful gain there; the full-resolution margin is five facings. +59% at 88% against 52% at 29% (full), and 56% at 97% against 57% at 32% (live). +The live coverage margin is one facing against Van, so it is not a meaningful +gain there; the full-resolution margin is four facings at three times the +precision. -**The 95% figure is live-resolution only.** At full resolution the same lane -reads 77% precision with 11 false accepts and raises the hub alarm, and the -still endpoint and the native crop lane both run at that scale. Reaching 95% at -full resolution needs an accept line near 6, which leaves 28/61 correct - below -Van's 32/61. So the design does not yet beat Van on both axes at full -resolution, and the accept line is provisional: it is tuned on Tier 1 and needs -per-input-path calibration on Tier 2. +**One accept line serves both resolutions.** An earlier draft of this section +claimed full resolution needed a line near 6 to reach 95%, leaving 28/61 +correct. That was measured before the reference-upscaling and text-signature +fixes and no longer holds. The separation is clean at both scales: correct +facings sit at p25 2.07 (full) and 2.87 (live), wrong ones at p75 1.00 and +1.66, so 2.0 is the boundary in both. Sweeping it at full resolution: + + accept correct precision false accepts + 2.0 59% 88% 5 + 3.0 52% 91% 3 + 4.0 49% 94% 2 + 6.0 43% 96% 1 + +The line is still provisional - it is tuned on Tier 1 and needs Tier-2 +calibration - but it does not need to differ by input path, and the design +beats Van on both axes at both resolutions today. The live count was pacing-dependent, and the cause was measurable: OCR ran on every crop at every attempt, taking a crop from 5 ms to 25 ms and throughput @@ -643,11 +653,47 @@ on `add/image6.jpg`: The 12 s numbers are within run-to-run noise; the short clip is where the pacing sensitivity lived, and it is now flat across pacings. Precision stays -100% and 0 unknowns are accepted at every pacing. Coverage is still a gate -(G1): 53 of 131 streamed tracks is not yet a claim against Van's 57% of -labelled facings, because the two count different things - tracks the detector -started versus labelled facings. The label-matched comparison is the -evaluator's, above. +100% and 0 unknowns are accepted at every pacing. + +**The stream path now has a G1 number.** `code/score_stream.py` scores a +recorded clip's labels against the ground truth, matching each track's +`label_box` to the labelled facings. Three defects in the first version of +that harness made the stream look far worse than it was, and all three are +fixed: it counted any facing touching the window by one pixel (23 of its 47 +"in-catalog" facings were less than half visible), it scored a correct +text-lane answer as a false accept because it checked enrollment before +correctness, and it counted tracks rather than facings, which reported 138% +coverage of a 24-facing window. The denominator is now the facings at least +half inside the union of the window rects the client showed. + +Measured through the real server on `add/image6.jpg`, 20 s per run, local +backend against the `van` lane on the same stream: + + pan lane facings named precision + 0 van 11/24 (46%) 52% + 0 local 11/24 (46%) 100% + 2 van 12/24 (50%) 45% + 2 local 18/24 (75%) 100% + 12 van 11/24 (46%) 28% + 12 local 16/24 (67%) 83% + +Two live-path bugs surfaced while measuring this, both fixed: + +- **Inherited labels carried frame id 0.** `_inherit_labels` set `label_box` + but never `label_frame_id`, so every inherited label was mapped back to the + source image with the first frame's offset. During a pan that puts the box a + whole pan-distance from the product it names. It now stamps the current + frame and anchors to the raw detection box rather than the smoothed track + box, which lags a pan by about one step. +- **The reference lane upscaled views smaller than a pyramid level.** + `sku_l09.jpg` is 407x305, so its 400 px level was a 305->400 blow-up: a blur + with no high-frequency detail to disagree with, which won 31 of 54 crops on + the shelf it was cut from. A view is now used only at the levels it has. + +Coverage is still a gate (G1). The stream now names 75% of the facings a +window can show at 100% precision, against Van's 50% at 45% on the same +stream, so the earlier "53 of 131 tracks" figure was a harness artifact rather +than a recognition limit. The shipped default scale now has test coverage. `make_session` builds only `matcher`-backend engines, so `scale == "local"` in `_on_classified` never ran @@ -656,7 +702,7 @@ injects the lanes' evidence into the live path, and `LocalScaleTests` pins the two jobs (a text-only read still labels the track; a text veto blocks a local accept). `DecisionPolicyTests` pins the veto/fill-in boundary, including the lane-order-vs-value case the rank re-sort broke. Both regressions were -re-injected to confirm the tests fail for the intended reason. Suite 251 -> 281. +re-injected to confirm the tests fail for the intended reason. Suite 251 -> 291. Two further review findings, both fixed and both measured: @@ -673,7 +719,7 @@ Two further review findings, both fixed and both measured: The headline number is reproducible from the repo, not from a scratch script: .venv/bin/python code/eval_identity.py --stock stock-lopia-clean \ - --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \ + --labels eval/labels/image6.json \ --image add/image6.jpg --lanes local --live live resolution: catalog 69% · identity 86% · decision 86% · diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md index 1779ce0..1fb166d 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md @@ -3,16 +3,16 @@ Read-only against product code: every script imports from `code/` and writes only into this folder. Run from `recognization/` with `.venv/bin/python`. -`image6_v2.json` is the corrected image6 label set (26 of 111 v1 labels +`eval/labels/image6.json` is the corrected image6 label set (26 of 111 v1 labels changed; every row carries `v1_label`, `source`, and a `note` where changed). `image6-v1-labels-overlay.jpg` shows the v1 labels on the shelf; green = catalog label, red = unknown. `sheet.py` regenerates per-SKU contact sheets. | Step | Command (prefix `E=docs/research/2026-09-28-recognition-architecture/evidence`) | Produces | |---|---|---| -| Original matcher, both catalogs, full/live/live-with-unscaled-boxes | `.venv/bin/python $E/diag.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] [--bug] --out $E/run_stock-lopia-clean_.pkl` (repeat with `--stock stock-lopia`) | `run_*.pkl` | +| Original matcher, both catalogs, full/live/live-with-unscaled-boxes | `.venv/bin/python $E/diag.py --stock stock-lopia-clean --labels $E/../../../eval/labels/image6.json [--live] [--bug] --out $E/run_stock-lopia-clean_.pkl` (repeat with `--stock stock-lopia`) | `run_*.pkl` | | Score them under v1 or v2 labels | `.venv/bin/python $E/analyze.py $E/run_*.pkl` | identity top-1, argmax, current gate | -| Matcher variants | `.venv/bin/python $E/variants.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] --scales 200,400 --root --geom affine --out $E/v_ms_root_aff_.pkl` (also `--tmpl-h 250/400`, no flags = raw) | `v_*.pkl` | +| Matcher variants | `.venv/bin/python $E/variants.py --stock stock-lopia-clean --labels $E/../../../eval/labels/image6.json [--live] --scales 200,400 --root --geom affine --out $E/v_ms_root_aff_.pkl` (also `--tmpl-h 250/400`, no flags = raw) | `v_*.pkl` | | Variant ranking + open-set frontier | `.venv/bin/python $E/an2.py $E $(basename -a $E/v_*.pkl)` (`LABELS=image6_v2_lfo.json` for the leave-facing-out runs) | correct at 100/95/90 % precision | | Leave-facing-out catalog | `.venv/bin/python $E/lfo_setup.py`, then `variants.py --stock $E/cat_ --labels $E/image6_v2_lfo.json --scales 200,400 --root --geom affine [--live] --out $E/lfo_cat__.pkl` | `lfo_*.pkl` | | Unrelated-shelf distractors | `YOLO_OFFLINE=1 .venv/bin/python $E/distract.py stock-lopia-clean $E/cat_full` | `distract.pkl` | @@ -44,7 +44,7 @@ with no arguments raise `IndexError`; neither validates its own usage. v1 labels are no longer a file on disk — `eval/labels/image6.json` is the v2 set. To score a run under v1, rebuild it from the `v1_label` field: - .venv/bin/python -c "import json;d=json.load(open('$E/image6_v2.json'));\ + .venv/bin/python -c "import json;d=json.load(open('$E/../../../eval/labels/image6.json'));\ [f.update(label=f.pop('v1_label')) for f in d['facings']];\ json.dump(d,open('/tmp/image6_v1.json','w'),ensure_ascii=False)" diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/an2.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/an2.py index 5174adb..598fb29 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/an2.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/an2.py @@ -1,5 +1,5 @@ import pickle, json, sys, numpy as np, collections -import os; S = sys.argv[1]; L = {f['index']: f['label'] for f in json.load(open(f'{S}/' + os.environ.get('LABELS','image6_v2.json')))['facings']} +import os; S = sys.argv[1]; L = {f['index']: f['label'] for f in json.load(open(f'{S}/' + os.environ.get('LABELS','../../../../eval/labels/image6.json')))['facings']} RANK = { 'inl': lambda e: e['inliers'], 'inl*col': lambda e: e['inliers'] * (0.25 + e['color']), diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01.jpg new file mode 100644 index 0000000..e35eb97 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01_shelf.jpg new file mode 100644 index 0000000..5a079a9 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l01_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02.jpg new file mode 100644 index 0000000..086c12d Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02_shelf.jpg new file mode 100644 index 0000000..e2e95d5 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l02_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03.jpg new file mode 100644 index 0000000..597f2c2 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03_shelf.jpg new file mode 100644 index 0000000..8b4b0fa Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l03_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05.jpg new file mode 100644 index 0000000..009e5aa Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05_shelf.jpg new file mode 100644 index 0000000..25df422 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l05_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06.jpg new file mode 100644 index 0000000..0fb6c11 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06_shelf.jpg new file mode 100644 index 0000000..255842f Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l06_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l07_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l07_shelf.jpg new file mode 100644 index 0000000..cf85f61 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l07_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09.jpg new file mode 100644 index 0000000..661e797 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09_shelf.jpg new file mode 100644 index 0000000..4353a95 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l09_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l10_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l10_shelf.jpg new file mode 100644 index 0000000..9d3dcf8 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l10_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11.jpg new file mode 100644 index 0000000..a54c358 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11_shelf.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11_shelf.jpg new file mode 100644 index 0000000..b8da93e Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_full/sku_l11_shelf.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l01.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l01.jpg new file mode 100644 index 0000000..e35eb97 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l01.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l02.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l02.jpg new file mode 100644 index 0000000..086c12d Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l02.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l03.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l03.jpg new file mode 100644 index 0000000..597f2c2 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l03.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l05.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l05.jpg new file mode 100644 index 0000000..009e5aa Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l05.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l06.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l06.jpg new file mode 100644 index 0000000..0fb6c11 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l06.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l09.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l09.jpg new file mode 100644 index 0000000..661e797 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l09.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l11.jpg b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l11.jpg new file mode 100644 index 0000000..a54c358 Binary files /dev/null and b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/cat_pack/sku_l11.jpg differ diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/dino_scale.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/dino_scale.py index 2d963d7..64d7ae4 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/dino_scale.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/dino_scale.py @@ -16,7 +16,7 @@ for p in sorted(glob.glob('add/*')): if min(c.shape[:2])>=40: dis.append((f'd{j}_{os.path.basename(p)}',c)) random.shuffle(dis) D=[(k,g.embed(c)) for k,c in dis] -L=json.load(open(f'{S}/image6_v2.json')); E={k for k,_ in refs} +L=json.load(open(f'{S}/../../../../eval/labels/image6.json')); E={k for k,_ in refs} Q=[(fc['label'],g.embed(c)) for fc,c in crops('add/image6.jpg',L,True) if fc['label'] in E] t=time.perf_counter(); _=g.embed(Q and dis[0][1]); print('embed ms',round((time.perf_counter()-t)*1000,1)) for N in [0,50,200,len(D)]: diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/fuse_eval.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/fuse_eval.py index fe2a092..360989a 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/fuse_eval.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/fuse_eval.py @@ -2,7 +2,7 @@ import sys,os,json,pickle,csv,numpy as np S=os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0,S) from ocr_fuse import norm,sim # noqa (module runs OCR of packshots on import; cheap) sig=pickle.load(open(f'{S}/sig.pkl','rb')); O=pickle.load(open(f'{S}/ocr.pkl','rb')) -L={f['index']:f['label'] for f in json.load(open(f'{S}/image6_v2.json'))['facings']} +L={f['index']:f['label'] for f in json.load(open(f'{S}/../../../../eval/labels/image6.json'))['facings']} def conf(ev): r=sorted(ev.items(),key=lambda kv:-kv[1]['inliers']*(0.25+kv[1]['color'])) (k1,e1),(k2,e2)=r[0],r[1]; i1,i2=e1['inliers'],e2['inliers'] diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json deleted file mode 100644 index 70210e3..0000000 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json +++ /dev/null @@ -1,1478 +0,0 @@ -{ - "image": "add/image6.jpg", - "facings": [ - { - "box": [ - 39, - 56, - 143, - 313 - ], - "label": "sku_l01", - "match": 1.0, - "index": 0, - "source": "v1-confirmed-visual", - "v1_label": "sku_l01" - }, - { - "box": [ - 147, - 53, - 264, - 318 - ], - "label": "sku_l01", - "match": 1.0, - "index": 1, - "source": "v1-confirmed-visual", - "v1_label": "sku_l01" - }, - { - "box": [ - 1419, - 0, - 1500, - 107 - ], - "label": "exclude", - "match": 1.0, - "index": 2, - "note": "blurred corner of upper shelf; not identifiable", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l04" - }, - { - "box": [ - 270, - 64, - 377, - 322 - ], - "label": "sku_l01", - "match": 1.0, - "index": 3, - "source": "v1-confirmed-visual", - "v1_label": "sku_l01" - }, - { - "box": [ - 382, - 72, - 467, - 324 - ], - "label": "sku_l01", - "match": 0.0, - "index": 4, - "note": "Egg Care front, same tag run as #0-3", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 472, - 64, - 570, - 328 - ], - "label": "sku_l01", - "match": 0.0, - "index": 5, - "note": "Egg Care front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 575, - 137, - 639, - 330 - ], - "label": "sku_l02", - "match": 1.0, - "index": 6, - "source": "v1-confirmed-visual", - "v1_label": "sku_l02" - }, - { - "box": [ - 641, - 126, - 732, - 337 - ], - "label": "sku_l02", - "match": 1.0, - "index": 7, - "source": "v1-confirmed-visual", - "v1_label": "sku_l02" - }, - { - "box": [ - 736, - 123, - 817, - 339 - ], - "label": "sku_l02", - "match": 1.0, - "index": 8, - "source": "v1-confirmed-visual", - "v1_label": "sku_l02" - }, - { - "box": [ - 822, - 135, - 909, - 345 - ], - "label": "sku_l02", - "match": 1.0, - "index": 9, - "source": "v1-confirmed-visual", - "v1_label": "sku_l02" - }, - { - "box": [ - 959, - 160, - 1044, - 349 - ], - "label": "sku_l03", - "match": 1.0, - "index": 10, - "source": "v1-confirmed-visual", - "v1_label": "sku_l03" - }, - { - "box": [ - 1049, - 158, - 1138, - 348 - ], - "label": "sku_l03", - "match": 1.0, - "index": 11, - "source": "v1-confirmed-visual", - "v1_label": "sku_l03" - }, - { - "box": [ - 1143, - 144, - 1228, - 353 - ], - "label": "sku_l03", - "match": 1.0, - "index": 12, - "source": "v1-confirmed-visual", - "v1_label": "sku_l03" - }, - { - "box": [ - 1233, - 147, - 1352, - 356 - ], - "label": "sku_l03", - "match": 1.0, - "index": 13, - "source": "v1-confirmed-visual", - "v1_label": "sku_l03" - }, - { - "box": [ - 1375, - 152, - 1475, - 365 - ], - "label": "sku_l04", - "match": 1.0, - "index": 14, - "source": "v1-confirmed-visual", - "v1_label": "sku_l04" - }, - { - "box": [ - 1482, - 155, - 1500, - 363 - ], - "label": "exclude", - "match": 0.0, - "index": 15, - "note": "18 px sliver", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 0, - 472, - 76, - 662 - ], - "label": "unknown", - "match": 1.0, - "index": 16, - "note": "garlic jar, not mayo", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l05" - }, - { - "box": [ - 86, - 420, - 163, - 664 - ], - "label": "sku_l05", - "match": 1.0, - "index": 17, - "source": "v1-confirmed-visual", - "v1_label": "sku_l05" - }, - { - "box": [ - 167, - 420, - 265, - 662 - ], - "label": "sku_l05", - "match": 1.0, - "index": 18, - "source": "v1-confirmed-visual", - "v1_label": "sku_l05" - }, - { - "box": [ - 270, - 426, - 366, - 664 - ], - "label": "sku_l05", - "match": 1.0, - "index": 19, - "source": "v1-confirmed-visual", - "v1_label": "sku_l05" - }, - { - "box": [ - 476, - 446, - 571, - 669 - ], - "label": "sku_l06", - "match": 1.0, - "index": 20, - "source": "v1-confirmed-visual", - "v1_label": "sku_l06" - }, - { - "box": [ - 583, - 458, - 678, - 672 - ], - "label": "sku_l06", - "match": 1.0, - "index": 21, - "source": "v1-confirmed-visual", - "v1_label": "sku_l06" - }, - { - "box": [ - 683, - 454, - 774, - 672 - ], - "label": "sku_l06", - "match": 0.0, - "index": 22, - "note": "Fitte front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 778, - 459, - 865, - 675 - ], - "label": "sku_l06", - "match": 0.0, - "index": 23, - "note": "Fitte front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 871, - 466, - 945, - 674 - ], - "label": "sku_l07", - "match": 1.0, - "index": 24, - "source": "v1-confirmed-visual", - "v1_label": "sku_l07" - }, - { - "box": [ - 949, - 471, - 1020, - 674 - ], - "label": "sku_l07", - "match": 1.0, - "index": 25, - "source": "v1-confirmed-visual", - "v1_label": "sku_l07" - }, - { - "box": [ - 1024, - 493, - 1098, - 676 - ], - "label": "sku_l07", - "match": 1.0, - "index": 26, - "source": "v1-confirmed-visual", - "v1_label": "sku_l07" - }, - { - "box": [ - 1103, - 494, - 1176, - 676 - ], - "label": "sku_l07", - "match": 0.0, - "index": 27, - "note": "Lucky Mayonnaise front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1181, - 499, - 1245, - 676 - ], - "label": "sku_l07", - "match": 0.0, - "index": 28, - "note": "Lucky Mayonnaise front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1249, - 480, - 1331, - 676 - ], - "label": "sku_l07", - "match": 1.0, - "index": 29, - "note": "Lucky Mayonnaise front; was l08", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l08" - }, - { - "box": [ - 1336, - 502, - 1421, - 682 - ], - "label": "sku_l08", - "match": 1.0, - "index": 30, - "source": "v1-confirmed-visual", - "v1_label": "sku_l08" - }, - { - "box": [ - 0, - 751, - 108, - 977 - ], - "label": "unknown", - "match": 1.0, - "index": 31, - "note": "demi-glace sauce pouch; was l09", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l09" - }, - { - "box": [ - 115, - 758, - 193, - 973 - ], - "label": "unknown", - "match": 1.0, - "index": 32, - "note": "demi-glace sauce pouch; was l09", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l09" - }, - { - "box": [ - 199, - 758, - 255, - 971 - ], - "label": "sku_l09", - "match": 1.0, - "index": 33, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 258, - 758, - 352, - 976 - ], - "label": "sku_l09", - "match": 1.0, - "index": 34, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 357, - 761, - 449, - 973 - ], - "label": "sku_l09", - "match": 1.0, - "index": 35, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 452, - 755, - 514, - 972 - ], - "label": "sku_l09", - "match": 1.0, - "index": 36, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 519, - 759, - 608, - 974 - ], - "label": "sku_l09", - "match": 1.0, - "index": 37, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 614, - 759, - 700, - 973 - ], - "label": "sku_l09", - "match": 1.0, - "index": 38, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 704, - 763, - 791, - 973 - ], - "label": "sku_l09", - "match": 1.0, - "index": 39, - "source": "v1-confirmed-visual", - "v1_label": "sku_l09" - }, - { - "box": [ - 798, - 765, - 916, - 974 - ], - "label": "sku_l09", - "match": 0.0, - "index": 40, - "note": "Kokuuma 65% front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 919, - 763, - 973, - 974 - ], - "label": "sku_l09", - "match": 0.0, - "index": 41, - "note": "Kokuuma 65% partly occluded", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 977, - 765, - 1067, - 974 - ], - "label": "sku_l09", - "match": 0.0, - "index": 42, - "note": "Kokuuma 65% front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1070, - 762, - 1134, - 975 - ], - "label": "sku_l09", - "match": 0.0, - "index": 43, - "note": "Kokuuma 65% partly occluded", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1138, - 767, - 1218, - 973 - ], - "label": "sku_l09", - "match": 0.0, - "index": 44, - "note": "Kokuuma 65% front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1221, - 768, - 1281, - 972 - ], - "label": "sku_l09", - "match": 0.0, - "index": 45, - "note": "Kokuuma 65% partly occluded", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1285, - 794, - 1359, - 972 - ], - "label": "unknown", - "match": 0.0, - "index": 46, - "note": "ユーラス 野菜をおいしく食べる旨たれ 380g; possibly what stock.csv calls l12", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1363, - 794, - 1440, - 973 - ], - "label": "unknown", - "match": 0.0, - "index": 47, - "note": "same tare bottle", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1443, - 794, - 1500, - 973 - ], - "label": "unknown", - "match": 0.0, - "index": 48, - "note": "same tare bottle, partial", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 191, - 1060, - 297, - 1266 - ], - "label": "sku_l10", - "match": 1.0, - "index": 49, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 306, - 1058, - 436, - 1260 - ], - "label": "sku_l10", - "match": 1.0, - "index": 50, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 441, - 1060, - 538, - 1256 - ], - "label": "sku_l10", - "match": 1.0, - "index": 51, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 542, - 1060, - 628, - 1251 - ], - "label": "sku_l10", - "match": 1.0, - "index": 52, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 630, - 1055, - 680, - 1249 - ], - "label": "sku_l10", - "match": 1.0, - "index": 53, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 685, - 1058, - 783, - 1251 - ], - "label": "sku_l10", - "match": 1.0, - "index": 54, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 789, - 1056, - 876, - 1246 - ], - "label": "sku_l10", - "match": 1.0, - "index": 55, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 878, - 1055, - 968, - 1246 - ], - "label": "sku_l10", - "match": 1.0, - "index": 56, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 1044, - 1054, - 1113, - 1248 - ], - "label": "sku_l10", - "match": 1.0, - "index": 57, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 1114, - 1051, - 1196, - 1247 - ], - "label": "sku_l10", - "match": 1.0, - "index": 58, - "source": "v1-confirmed-visual", - "v1_label": "sku_l10" - }, - { - "box": [ - 1199, - 1050, - 1280, - 1247 - ], - "label": "sku_l10", - "match": 0.0, - "index": 59, - "note": "Pure Select mayonnaise front", - "source": "visual-audit-2026-09-28", - "v1_label": "unknown" - }, - { - "box": [ - 1280, - 1061, - 1317, - 1244 - ], - "label": "unknown", - "match": 0.0, - "index": 60, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1319, - 1068, - 1389, - 1240 - ], - "label": "unknown", - "match": 0.0, - "index": 61, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1393, - 1067, - 1467, - 1241 - ], - "label": "unknown", - "match": 0.0, - "index": 62, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1478, - 1068, - 1500, - 1242 - ], - "label": "unknown", - "match": 0.0, - "index": 63, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 64, - 1142, - 150, - 1264 - ], - "label": "unknown", - "match": 1.0, - "index": 64, - "note": "Heinz can; was l10", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l10" - }, - { - "box": [ - 0, - 1379, - 51, - 1561 - ], - "label": "unknown", - "match": 0.0, - "index": 65, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 60, - 1374, - 163, - 1560 - ], - "label": "unknown", - "match": 0.0, - "index": 66, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 190, - 1339, - 296, - 1562 - ], - "label": "sku_l11", - "match": 1.0, - "index": 67, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 298, - 1335, - 404, - 1558 - ], - "label": "sku_l11", - "match": 1.0, - "index": 68, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 421, - 1333, - 524, - 1549 - ], - "label": "sku_l11", - "match": 1.0, - "index": 69, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 529, - 1332, - 626, - 1546 - ], - "label": "sku_l11", - "match": 1.0, - "index": 70, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 634, - 1328, - 729, - 1541 - ], - "label": "sku_l11", - "match": 1.0, - "index": 71, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 734, - 1326, - 831, - 1538 - ], - "label": "sku_l11", - "match": 1.0, - "index": 72, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 836, - 1326, - 924, - 1535 - ], - "label": "sku_l11", - "match": 1.0, - "index": 73, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 930, - 1324, - 1018, - 1532 - ], - "label": "sku_l11", - "match": 1.0, - "index": 74, - "source": "v1-confirmed-visual", - "v1_label": "sku_l11" - }, - { - "box": [ - 1022, - 1320, - 1123, - 1530 - ], - "label": "sku_l11", - "match": 1.0, - "index": 75, - "note": "Kewpie Half front; was l12", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l12" - }, - { - "box": [ - 1190, - 1331, - 1262, - 1521 - ], - "label": "unknown", - "match": 1.0, - "index": 76, - "note": "Vegiglet dressing; was l12", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l12" - }, - { - "box": [ - 1266, - 1337, - 1334, - 1516 - ], - "label": "unknown", - "match": 1.0, - "index": 77, - "note": "Vegiglet dressing; was l12", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l12" - }, - { - "box": [ - 1337, - 1335, - 1403, - 1514 - ], - "label": "unknown", - "match": 1.0, - "index": 78, - "note": "Vegiglet dressing; was l12", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l12" - }, - { - "box": [ - 1409, - 1336, - 1475, - 1511 - ], - "label": "unknown", - "match": 1.0, - "index": 79, - "note": "Vegiglet dressing; was l12", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l12" - }, - { - "box": [ - 1480, - 1327, - 1500, - 1514 - ], - "label": "unknown", - "match": 0.0, - "index": 80, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 593, - 1619, - 679, - 1815 - ], - "label": "unknown", - "match": 0.0, - "index": 81, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 683, - 1617, - 762, - 1809 - ], - "label": "unknown", - "match": 0.0, - "index": 82, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 766, - 1612, - 839, - 1806 - ], - "label": "unknown", - "match": 0.0, - "index": 83, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 842, - 1609, - 909, - 1801 - ], - "label": "unknown", - "match": 0.0, - "index": 84, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 910, - 1607, - 995, - 1796 - ], - "label": "unknown", - "match": 0.911, - "index": 85, - "note": "Kewpie Mayonnaise 450g (not catalog); was l03", - "source": "visual-audit-2026-09-28", - "v1_label": "sku_l03" - }, - { - "box": [ - 999, - 1602, - 1078, - 1793 - ], - "label": "unknown", - "match": 0.0, - "index": 86, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1083, - 1600, - 1169, - 1787 - ], - "label": "unknown", - "match": 0.0, - "index": 87, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1173, - 1595, - 1269, - 1783 - ], - "label": "unknown", - "match": 0.0, - "index": 88, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1306, - 1586, - 1388, - 1771 - ], - "label": "unknown", - "match": 0.0, - "index": 89, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1390, - 1602, - 1457, - 1767 - ], - "label": "unknown", - "match": 0.0, - "index": 90, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1461, - 1601, - 1499, - 1769 - ], - "label": "unknown", - "match": 0.0, - "index": 91, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 7, - 1632, - 103, - 1845 - ], - "label": "unknown", - "match": 0.0, - "index": 92, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 107, - 1630, - 199, - 1844 - ], - "label": "unknown", - "match": 0.0, - "index": 93, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 213, - 1631, - 313, - 1834 - ], - "label": "unknown", - "match": 0.0, - "index": 94, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 318, - 1628, - 405, - 1829 - ], - "label": "unknown", - "match": 0.0, - "index": 95, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 409, - 1626, - 491, - 1825 - ], - "label": "unknown", - "match": 0.0, - "index": 96, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 494, - 1622, - 589, - 1821 - ], - "label": "unknown", - "match": 0.0, - "index": 97, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1059, - 1855, - 1197, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 98, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1216, - 1839, - 1309, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 99, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1313, - 1835, - 1397, - 1999 - ], - "label": "unknown", - "match": 0.0, - "index": 100, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 1404, - 1837, - 1498, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 101, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 0, - 1918, - 75, - 1998 - ], - "label": "unknown", - "match": 0.0, - "index": 102, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 82, - 1910, - 187, - 1999 - ], - "label": "unknown", - "match": 0.0, - "index": 103, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 214, - 1897, - 506, - 1998 - ], - "label": "unknown", - "match": 0.0, - "index": 104, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 219, - 1906, - 362, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 105, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 363, - 1911, - 500, - 1999 - ], - "label": "unknown", - "match": 0.0, - "index": 106, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 516, - 1882, - 591, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 107, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 646, - 1886, - 786, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 108, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 790, - 1882, - 887, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 109, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - }, - { - "box": [ - 895, - 1862, - 1044, - 2000 - ], - "label": "unknown", - "match": 0.0, - "index": 110, - "source": "v1-confirmed-visual", - "v1_label": "unknown" - } - ], - "version": 2 -} \ No newline at end of file diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/lfo_setup.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/lfo_setup.py index 2eba5fc..78c4bd5 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/lfo_setup.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/lfo_setup.py @@ -4,7 +4,7 @@ image6_v2_lfo.json with those reference facings excluded from scoring. Same-photo references make this an optimistic proxy, not a held-out result.""" import json, cv2, os, shutil, collections E = os.path.dirname(os.path.abspath(__file__)) -L = json.load(open(f'{E}/image6_v2.json')); im = cv2.imread('add/image6.jpg') +L = json.load(open(f'{E}/../../../../eval/labels/image6.json')); im = cv2.imread('add/image6.jpg') by = collections.defaultdict(list) for f in L['facings']: if f['label'] not in ('unknown', 'exclude'): by[f['label']].append(f) diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/ocr_exp.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/ocr_exp.py index 7b1375a..4469929 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/ocr_exp.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/ocr_exp.py @@ -3,7 +3,7 @@ sys.path.insert(0,'code'); sys.path.insert(0,os.path.dirname(__file__)) from ocr import VisionOcr from diag import crops S=os.path.dirname(__file__) -L=json.load(open(f'{S}/image6_v2.json')) +L=json.load(open(f'{S}/../../../../eval/labels/image6.json')) # distinguishing tokens per SKU (any hit counts); includes siblings for near-unknowns KEY={'sku_l01':['エッグケア','卵を使って'],'sku_l02':['アマニ'],'sku_l03':['からし','Spicy'],'sku_l04':['めんたい'], 'sku_l05':['ゼロ','ノンコレ'],'sku_l06':['フィッテ','内臓脂肪'],'sku_l07':['ラッキー'],'sku_l08':['PERI','ペリ'], diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/scale.py b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/scale.py index b591da9..9eaefc0 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/scale.py +++ b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/scale.py @@ -27,7 +27,7 @@ for p in sorted(glob.glob('add/*')): x1,y1,x2,y2=[int(v) for v in b['box']]; c=im[max(0,y1):y2,max(0,x1):x2] if min(c.shape[:2])>=40: dis.append((f'd_{os.path.basename(p)}_{j}',c)) random.shuffle(dis); print('distractor refs available',len(dis)) -L=json.load(open(f'{S}/image6_v2.json')); Q=[(fc,c) for fc,c in crops('add/image6.jpg',L,True) if fc['label']!='exclude' and min(c.shape[:2])>=8] +L=json.load(open(f'{S}/../../../../eval/labels/image6.json')); Q=[(fc,c) for fc,c in crops('add/image6.jpg',L,True) if fc['label']!='exclude' and min(c.shape[:2])>=8] Qf=[(fc['label'],)+feat(c,None) for fc,c in Q] bf=cv2.BFMatcher(cv2.NORM_L2) def verify(kq,dq,r):