diff --git a/recognization/README.md b/recognization/README.md index 2a2be37..7acdfdd 100644 --- a/recognization/README.md +++ b/recognization/README.md @@ -126,34 +126,10 @@ python code/stream_client.py --source 0 # webcam Bất kỳ client nào gửi được ảnh JPEG đều dùng được: điện thoại, iPad, camera gắn xe đẩy, camera đeo cổ, robot. Giao thức không phụ thuộc trình duyệt. -For the native app demo, `./run-demo-server.sh` uses `stock-lopia-clean` by -default: `stock-lopia` is shelf crops cut from the `image6` evaluation photo, so -scoring against it is memorization. Pass another stock directory explicitly to +For the native app demo, `./run-demo-server.sh` uses `stock-lopia` by default, +matching its `image6` replay frame. Pass another stock directory explicitly to change stores; the script writes ignored local app endpoints and a local token. -### Identity backends - -`--identity` picks what names a product. The default is `local`: - -- `local` — RootSIFT over a per-SKU reference pyramid, similarity-constrained - RANSAC, a hue check, then a text lane (OCR of the crop against each catalog - row's name plus its packshot OCR) that vetoes a match whose package text names - a different row and answers on its own when geometry finds nothing. Measured - on the corrected `image6` labels at live resolution: 59% of in-catalog facings - named at 95% precision, 0 of 48 unknowns accepted. Rows with no enrolled image - are still answerable, because the text lane builds their signature from the - catalog name. -- `matcher` — Van's original SURF/ORB/BRISK composite, kept as the E-1 baseline - lane for comparison. Always answers, so it accepts near-unknown siblings. -- `gallery` — the DINOv2 embedding gallery. Fenced off: it is not the default - and it refuses to start without the pinned weights. Its recall collapses as - the catalog grows (7/42 recall@10 at 493 SKUs), so it is not the recognition - path. - -The text lane needs OCR. On macOS that is Vision, which is already wired in -`code/ocr.py`; `DORAYAKI_OCR=none` disables it and the local lane then stands -alone (43% coverage at 79% precision on the same labels). - --- diff --git a/recognization/code/app.py b/recognization/code/app.py index b73a5d7..f6be125 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -1009,9 +1009,8 @@ def main() -> None: parser.add_argument("--classify-workers", type=int, default=None) parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) - parser.add_argument("--identity", type=str, default=None, - choices=["matcher", "local", "gallery"], - help="which mechanism names products (default local)") + parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"], + help="which mechanism names products (default matcher)") parser.add_argument("--live-inflight", type=int, default=None, help="classifications in flight per session") parser.add_argument("--live-attempts", type=int, default=None, diff --git a/recognization/code/engine.py b/recognization/code/engine.py index f655b7b..6a84c92 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -85,13 +85,11 @@ class EngineSettings: # over a nearly static shelf wants more; that is a CLI override, not a default. live_inflight: int = 16 live_attempts: int = 2 - # Which mechanism names products. `local` is the RootSIFT lane behind a - # calibrated open-set gate: measured on the corrected image6 labels it shows - # 36 of 61 in-catalog facings at 95% precision with the text lane, against - # 35 at 35% for Van's always-argmax matcher. `matcher` keeps Van's composite - # as a selectable baseline lane (E-1). `gallery` is fenced off: raw DINOv2 - # is 12-21% top-1 on 7 SKUs and its recall@10 falls to 7/42 at 493 SKUs. - identity_backend: str = "local" + # Which mechanism names products: the hand-crafted local-feature matcher, or the + # embedding gallery with a calibrated rejection. Measured on 54 unseen store + # photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its + # raw inlier score cannot be thresholded; the gallery can say "unknown". + identity_backend: str = "matcher" # Cosine similarity below which the gallery calls a facing unknown. Measured: # same-product pairs sit above 0.6, different products below 0.5. gallery_threshold: float = 0.55 @@ -123,24 +121,6 @@ class StoredImage: tag_bundle: Optional[Dict[str, Any]] = None -def group_by_sku(scores: Dict[str, float]) -> Dict[str, float]: - """Collapse per-view template names onto their canonical SKU. - - A SKU's score is its best view, which is also how the gallery ranks several - views. `sku_l01_1` and `sku_l01_2` become one `sku_l01`, so two views of one - product can no longer compete with each other and no view-suffixed id can - reach the wire. - """ - from rebuild_catalog import sku_of - - grouped: Dict[str, float] = {} - for name, value in (scores or {}).items(): - number = float(value) - sku = sku_of(str(name)) - grouped[sku] = max(grouped.get(sku, 0.0), number) - return grouped - - def identity_state_from_level(level: Optional[str]) -> str: """Map a decide_identity() level to the wire state vocabulary.""" if level == "accepted": @@ -272,13 +252,6 @@ class RecognitionEngine: # enrolled view), and cached as False when it cannot be built at all. self._gallery: Any = None self._gallery_lock = threading.Lock() - # The local identity lane, built on first use. It loads every reference - # view and its descriptors, which is why it is lazy like the gallery. - self._lane: Any = None - self._lane_lock = threading.Lock() - # The text lane, built on first use from the catalog and the OCR engine. - self._text_lane: Any = None - self._text_lane_lock = threading.Lock() workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2) self.classify_workers = int(workers) self._executor = classify_executor or ThreadPoolExecutor( @@ -328,25 +301,12 @@ class RecognitionEngine: return self.ocr_engine.read(image) def warm_up(self) -> float: - """One dummy predict so the first real frame is not the slow one. - - The identity lanes are built here too. `TextLane.from_catalog` OCRs - every enrolled packshot, which is seconds of work; left lazy, the first - live frame would pay it inside a classify worker while the phone waits. - """ + """One dummy predict so the first real frame is not the slow one.""" import numpy as np blank = np.zeros((self.settings.imgsz, self.settings.imgsz, 3), dtype="uint8") started = time.perf_counter() self.detect(blank, conf=self.settings.detect_conf) - # A lane that cannot build here is not fatal: the lazy builders retry it - # on first use, so a bad stock dir or a Vision failure degrades the - # backend instead of refusing to start the server. - for name, build in (("local", self.identity_lane), ("text", self.identity_text_lane)): - try: - build() - except Exception as exc: - LOGGER.warning("identity %s lane not warmed: %s", name, exc) return (time.perf_counter() - started) * 1000.0 def shutdown(self) -> None: @@ -458,113 +418,14 @@ class RecognitionEngine: # ------------------------------------------------------------------ classification @property def identity_scale(self) -> str: - """What the scores from `score_crop` mean. - - `local` is the RootSIFT lane's margin-over-runner-up, `cosine` the - gallery's similarity, `inlier` Van's composite. The live path reads this - to pick the matching gate, so the three must never be mixed. - """ - backend = (self.settings.identity_backend or "matcher").lower() - if backend == "gallery": - return "cosine" - if backend == "local": - return "local" - return "inlier" - - def identity_text_lane(self) -> Any: - """The text lane, built once from the catalog. None without OCR. - - Signatures come from the catalog rows (their printed names plus the OCR - of their enrolled packshots), never from an evaluation photo, so a row - with no image can still be named. That is what lets the lane reject the - near-unknown siblings no geometric threshold can separate. - """ - if self.identity_scale != "local": - return None - if self._text_lane is None: - with self._text_lane_lock: - if self._text_lane is None: - from identity_lanes import TextLane - - self._text_lane = TextLane.from_catalog( - self.catalog(), self.settings.stock_dir, self.ocr_engine, - lock=self.ocr_lock, - ) - LOGGER.info("identity text lane: %d signatures", len(self._text_lane.signatures)) - return self._text_lane - - def decide_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, Any]: - """One identity decision for one crop, fusing every lane that is on. - - The local lane answers first and the text lane vetoes or fills in, which - is the measured design: geometry ranks fine-grained siblings correctly - and text is the only thing that rejects a sibling the catalog does not - know about. Without OCR the local lane stands alone. - """ - from identity import decide_identity - from identity_lanes import DecisionPolicy - - if self.identity_scale != "local": - return decide_identity( - self.score_crop(crop, method), method=method or self.settings.live_method, - scale=self.identity_scale, - ) - lane = self.identity_lane() - if lane is None: - return decide_identity({}, method="local", scale="local") - text_lane = self.identity_text_lane() - text_rank: Any = None - if text_lane is not None: - read = text_lane.read(crop) - text_rank = sorted( - text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True - ) - return DecisionPolicy().decide(lane.rank(crop), text_rank) - - def score_crop_lanes( - self, crop: Any, method: Optional[str] = None - ) -> Tuple[Dict[str, float], List[Tuple[str, float]]]: - """Both lanes' evidence for one crop: local scores and text ranking. - - The live path accumulates evidence across looks, so it needs the lanes - separately rather than one fused decision: it averages the local scores - over the track's window and keeps the newest text reading, then decides - once. The still path calls `decide_crop`, which fuses a single crop. - """ - scores = self.score_crop(crop, method) - text_lane = self.identity_text_lane() - if text_lane is None: - return scores, [] - read = text_lane.read(crop) - return scores, sorted( - text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True - ) - - def identity_lane(self) -> Any: - """The local lane, built once. None when the backend is not `local`.""" - if self.identity_scale != "local": - return None - if self._lane is None: - with self._lane_lock: - if self._lane is None: - from identity_lanes import LocalLane - - self._lane = LocalLane(self.settings.stock_dir) - LOGGER.info( - "identity lane: %d reference views over %d SKUs", - self._lane.view_count(), - len(self._lane.sku_ids), - ) - return self._lane + """What the scores from `score_crop` mean: inlier counts or cosine similarity.""" + return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier" def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: - """Score one crop, keyed by canonical SKU. + """Score one crop. Gallery never falls back to the matcher. - Every backend groups by SKU before returning. Scoring each reference - view separately is what made the demo's default catalog report 22-29 of - 109 facings as "ambiguous" when the top two candidates were two views of - the same product, and what put view-suffixed ids like `sku_l01_1` on the - wire, where no catalog row exists to supply a name or price. + A missing gallery returns no scores (unknown), not inlier counts. Mixing + those scales is how a live scan accepted a look-alike the still path rejected. """ if self.identity_scale == "cosine": gallery = self.identity_gallery() @@ -577,11 +438,6 @@ class RecognitionEngine: if match.runner_up: scores[match.runner_up] = match.runner_up_similarity return {str(sku): float(score) for sku, score in scores.items()} - if self.identity_scale == "local": - lane = self.identity_lane() - if lane is None: - return {} - return {sku: float(value) for sku, value in lane.rank(crop)} method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) @@ -589,7 +445,7 @@ class RecognitionEngine: scores, _ = self.matcher.score_crop_fast(crop) else: scores, _ = self.matcher.score_crop(crop, method=method) - return group_by_sku(scores) + return {str(sku): float(score) for sku, score in scores.items()} def identity_gallery(self) -> Any: """The embedding gallery, loaded once (None when unavailable).""" @@ -638,6 +494,8 @@ class RecognitionEngine: method: Optional[str] = None, ) -> List[Dict[str, Any]]: """Classifies every box in parallel; returns one entry per box, ordered by area.""" + from identity import decide_identity + method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) @@ -653,37 +511,33 @@ class RecognitionEngine: ordered = sorted(boxes, key=area_of, reverse=True) crops = [self.crop(image, box["box"]) for box in ordered] - if self.identity_scale == "cosine": + if (self.settings.identity_backend or "matcher").lower() == "gallery": gallery = self.identity_gallery() # No matcher fallback: an unavailable gallery is unrecognized, not a different model. return self._classify_with_gallery(gallery, ordered, crops) - from identity import decide_identity - - # Every backend goes through decide_crop, so the scores are keyed by - # canonical SKU and the text lane is fused in whichever lane produced them. - futures = [self._executor.submit(self.decide_crop, crop, method) for crop in crops] + if method == "hybrid_fast": + futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops] + else: + futures = [self._executor.submit(self.matcher.score_crop, crop, method) for crop in crops] results: List[Dict[str, Any]] = [] for box, future in zip(ordered, futures): try: - decision = future.result() + scores, details = future.result() except Exception as exc: # a single bad crop must not fail the batch - LOGGER.debug("classification failed for box %s: %s", box.get("id"), exc) - # A rejected decision, not the "not yet classified" payload: an - # operational failure must stay distinguishable from absence. - decision = decide_identity({}, method=method, scale=self.identity_scale) - decision = identity_payload(decision) + scores, details = {}, {"error": str(exc)} + decision = decide_identity(scores, details=details, method=method) results.append( { "detection_id": int(box["id"]), "box": list(box["box"]), - "identity": decision, + "identity": identity_payload(decision), } ) return results def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]: """Full single-crop result, using the same backend as recognize/live.""" - from identity import decision_for_match + from identity import decide_identity, decision_for_match crop = self.crop(image, box) if self.identity_scale == "cosine": @@ -693,11 +547,17 @@ class RecognitionEngine: # with the published cosine gates must not invent a second line. decision = decision_for_match(match) return self._inspect_payload(box, "gallery", decision, crop, visualize=False) + method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) - decision = self.decide_crop(crop, method) - visual_method = "hybrid" if method == "hybrid_fast" else method + if method == "hybrid_fast": + scores, details = self.matcher.score_crop_fast(crop) + visual_method = "hybrid" + else: + scores, details = self.matcher.score_crop(crop, method=method) + visual_method = method + decision = decide_identity(scores, details=details, method=method) return self._inspect_payload(box, method, decision, crop, visual_method=visual_method) def _inspect_payload( diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py deleted file mode 100644 index 01e3dfe..0000000 --- a/recognization/code/eval_identity.py +++ /dev/null @@ -1,607 +0,0 @@ -"""Score identity lanes on labelled shelf facings. Architecture-neutral. - -The original matcher is a candidate final architecture, not just a baseline. -Every lane is scored the same way, on the same crops, at the same resolutions: - - matcher full-catalog SIFT/ORB/BRISK + RANSAC (the original demo) - gallery DINOv2 embedding, raw or with the trained head - hybrid-verify DINO shortlist, then the matcher re-ranks inside it - -Reported per lane, as four separate ceilings rather than one conflated number: - - catalog ceiling in-catalog facings with an enrolled view / all in-catalog - identity ceiling ungated top-1 rank correct on enrolled / enrolled - decision ceiling accepted-and-correct on enrolled / enrolled - end-to-end coverage accepted-and-correct / all in-catalog facings - -plus precision over every accept — including accepts of unenrolled in-catalog -facings, which are false accepts and the most dangerous kind — the prediction -histogram, and the largest single-SKU share of accepts (the L09-apocalypse -detector). The hybrid lane also reports shortlist recall, because a true SKU -outside the shortlist is a hard ceiling no verifier can lift. - - python code/eval_identity.py \ - --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \ - --image add/image6.jpg --stock-dir stock-lopia-clean --lanes matcher --live - -Labels are {"image", "facings": [{"box", "label"}]} with "unknown" for facings -not in the catalog and "exclude" for rows two label passes disagreed on. A -facing whose crop is a catalog template is refused as leakage, not scored: -those crops are how the store was enrolled. -""" - -from __future__ import annotations - -import argparse -import csv -import json -import os -import sys -import time -from collections import Counter -from typing import Any, Collection, Dict, List, Optional, Sequence, Tuple - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import cv2 # noqa: E402 -import numpy as np # noqa: E402 - -from rebuild_catalog import sku_of # noqa: E402 - -from identity import LOCAL_ACCEPT # noqa: E402 - -UNKNOWN = "unknown" -# Two independent label passes disagreed here, so the row is not ground truth. -# Scoring it would be scoring a coin flip. -EXCLUDE = "exclude" -LANES = ("matcher", "gallery", "hybrid-verify") -# A single SKU above this share of accepts is the failure the last audit found: -# every dark package collapsing onto one enrolled product. -HUB_SHARE = 0.30 -# Live frames arrive at this long side; the original demo classified a 2048 px -# still. Scoring both is how resolution stops being an invisible variable. -LIVE_LONG_SIDE = 1280 - -# Pixel equality is not a leak test: a template re-cut from the same photo with -# a different box passes a same-shape comparison. A 64x64 grayscale signature -# catches most of those, but it is a heuristic, not a proof — measured on -# image6_v2 against stock-lopia, re-cuts in the realistic 0.94..1.06 box band -# land 7..37 MAD from their facing's signature while the closest pair of -# genuinely distinct same-SKU facings is 23.1, so the distributions overlap and -# no threshold is clean. 20.0 catches 81% of that band with zero false -# positives on 5,886 genuine facing pairs and 7 clean packshots. The real fix is -# the enrollment protocol's session manifest (DECISION §8); this is the cheap -# backstop that runs without one. -LEAK_SIGNATURE_MAD = 20.0 - - -def load_labels(path: str) -> Dict[str, Any]: - with open(path) as handle: - payload = json.load(handle) - if not isinstance(payload, dict) or "facings" not in payload: - raise SystemExit(f"{path}: expected {{'image', 'facings': [...]}}") - return payload - - -def crop_of(image: Any, box: Sequence[int]) -> Any: - x1, y1, x2, y2 = [int(v) for v in box] - return image[max(0, y1):y2, max(0, x1):x2] - - -def at_live_resolution(image: Any, long_side: int = LIVE_LONG_SIDE) -> Any: - height, width = image.shape[:2] - if max(height, width) <= long_side: - return image - scale = long_side / float(max(height, width)) - return cv2.resize(image, (int(width * scale), int(height * scale)), interpolation=cv2.INTER_AREA) - - -def _signature(image: Any) -> Any: - """64x64 grayscale signature. Cheap, and tolerant of a re-cut.""" - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if image.ndim == 3 else image - return cv2.resize(gray, (64, 64), interpolation=cv2.INTER_AREA).astype(np.float32) - - -def _signatures(image: Any) -> List[Any]: - """The crop's signature and a centre 90% crop's. - - A re-cut box shifts the content inside the frame. Comparing a second, - tighter crop against the template absorbs that shift: measured on the - realistic re-cut band it catches 81% of re-cuts against 68% for the single - signature, at no cost in false positives. - """ - height, width = image.shape[:2] - dy, dx = int(height * 0.05), int(width * 0.05) - centre = image[dy:height - dy, dx:width - dx] - return [_signature(image), _signature(centre if centre.size else image)] - - -_TEMPLATES: Dict[str, List[Tuple[str, str, Tuple[int, int], List[Any]]]] = {} - - -def _catalog_templates(stock_dir: str) -> List[Tuple[str, str, Tuple[int, int], List[Any]]]: - """(stem, name, shape, signatures) per catalog image, read once per stock dir. - - Cached because the signature is what catches a re-cut crop, and re-reading - every template for every facing is what made the old check expensive. - """ - cached = _TEMPLATES.get(stock_dir) - if cached is None: - cached = [] - for name in sorted(os.listdir(stock_dir)): - stem, ext = os.path.splitext(name) - if ext.lower() not in (".jpg", ".jpeg", ".png"): - continue - template = cv2.imread(os.path.join(stock_dir, name)) - if template is None: - continue - cached.append((stem, name, template.shape[:2], _signatures(template))) - _TEMPLATES[stock_dir] = cached - return cached - - -def leaked(image: Any, box: Sequence[int], stock_dir: str) -> Optional[str]: - """Name of the catalog template this crop is, or None when it is held out. - - Byte-identical crops and near-identical pastes both count: enrolment crops - scored as test data are training-set accuracy, not recognition. - - Pixel equality is not a leak test. A template re-cut from the same photo - with a slightly different box is the same leak and passes a same-shape - comparison, so grayscale signatures are compared as well. That check is a - heuristic with overlapping distributions (see LEAK_SIGNATURE_MAD): it - catches most re-cuts and no genuine facing in the measured set, but a - determined re-cut can still pass. The enrollment protocol's session manifest - is the real refusal; this is the backstop for catalogs that lack one. - """ - crop = crop_of(image, box) - if crop.size == 0: - return None - crop_signatures = _signatures(crop) - for stem, name, shape, signatures in _catalog_templates(stock_dir): - if min(float(np.abs(signature - crop_signature).mean()) - for signature in signatures for crop_signature in crop_signatures) < LEAK_SIGNATURE_MAD: - return stem - if shape != crop.shape[:2]: - continue - template = cv2.imread(os.path.join(stock_dir, name)) - if template is None: - continue - if np.array_equal(template, crop): - return stem - left = crop.astype(np.float32).ravel() - right = template.astype(np.float32).ravel() - denom = float(np.linalg.norm(left) * np.linalg.norm(right)) - if denom and float((left * right).sum() / denom) >= 0.98: - return stem - return None - - -def catalog_sku_ids(stock_dir: str) -> List[str]: - """SKU ids from the store's catalog CSV, falling back to enrolled images. - - Candidate sets must come from the catalog, never from the labels: a lane - told which SKUs the shelf holds is being scored against the answer key, and - a retrieval failure can never show up. - """ - path = os.path.join(stock_dir, "stock.csv") - if os.path.isfile(path): - with open(path, newline="") as handle: - ids = [row["SKU_ID"].strip() for row in csv.DictReader(handle) - if row.get("SKU_ID", "").strip()] - if ids: - return sorted(set(ids)) - return sorted({sku_of(name) for name in os.listdir(stock_dir) - if os.path.splitext(name)[1].lower() in (".jpg", ".jpeg", ".png")}) - - -def group_by_sku(scores: Dict[str, float]) -> Dict[str, float]: - """Collapse per-view template names onto their SKU. - - The matcher scores `sku_l01_1` and `sku_l01_2` separately; a SKU's score is - its best view, which is also how the gallery ranks several views. `sku_of` - is the canonical collapse, so a view outside the `sku_l` prefix groups with - its SKU instead of ranking as a product of its own. - """ - grouped: Dict[str, float] = {} - for name, value in (scores or {}).items(): - sku = sku_of(name) - grouped[sku] = max(grouped.get(sku, 0.0), float(value)) - return grouped - - -class Lane: - """One identity architecture, scored identically to every other.""" - - name = "lane" - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - """Catalog candidates, best first. Empty means nothing to answer.""" - raise NotImplementedError - - def shortlist(self, crop: Any, size: int) -> List[str]: - """Candidate SKUs the verifier is allowed to consider.""" - return [sku for sku, _ in self.rank(crop)[:size]] - - def decide(self, crop: Any, ranked: List[Tuple[str, float]], *, accept: float, - margin: float, scale: str) -> Dict[str, Any]: - """The lane's own decision. Default: the shared gate over its ranking. - - A lane whose decision is not `decide_identity` over one ranking - the - fused local+text lane is one - overrides this, so every lane is still - scored through the same four ceilings. - """ - from identity import decide_identity - - return decide_identity(dict(ranked), method=self.name, scale=scale, - min_score=accept, min_margin=margin) - - -class MatcherLane(Lane): - """The original demo: local features and RANSAC over the whole catalog.""" - - name = "matcher" - - def __init__(self, stock_dir: str, method: str = "hybrid") -> None: - from feature_matcher import ProductFeatureMatcher - - self.matcher = ProductFeatureMatcher(stock_dir=stock_dir) - self.method = method - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - scores, _ = self.matcher.score_crop(crop, method=self.method) - return sorted(group_by_sku(scores).items(), key=lambda kv: kv[1], reverse=True) - - -class LocalLaneAdapter(Lane): - """The shipped default: RootSIFT pyramid plus the OCR text lane. - - Wraps `identity_lanes` so the production decision is what gets scored, - rather than a re-implementation of it. `rank` returns the lane's own order - (its rank key, not its gate value), and `decide` runs the same - `DecisionPolicy` the server runs. - """ - - name = "local" - - def __init__(self, stock_dir: str, ocr: Any = None) -> None: - from identity_lanes import LocalLane, TextLane - - self.lane = LocalLane(stock_dir) - self.text_lane = TextLane.from_catalog( - _catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine() - ) - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - return self.lane.rank(crop) - - def decide(self, crop: Any, ranked: List[Tuple[str, float]], *, accept: float, - margin: float, scale: str) -> Dict[str, Any]: - from identity_lanes import DecisionPolicy - - text_rank = sorted( - self.text_lane.scores_from_text(self.text_lane.read(crop)).items(), - key=lambda kv: kv[1], reverse=True, - ) - return DecisionPolicy(local_accept=accept).decide(ranked, text_rank) - - -def _catalog_rows(stock_dir: str) -> List[Dict[str, Any]]: - """The catalog's rows, in the shape TextLane.from_catalog expects.""" - from inventory import InventoryManager - - return InventoryManager(stock_dir).products.values() - - -def _ocr_engine() -> Any: - from ocr import build_ocr_engine - - return build_ocr_engine() - - -class GalleryLane(Lane): - """DINOv2 embedding, with the store's trained head when it has one.""" - - name = "gallery" - - def __init__(self, stock_dir: str, sku_ids: Sequence[str], image_size: int = 518) -> None: - from gallery import ProductGallery - - self.gallery = ProductGallery(stock_dir, list(sku_ids), image_size=image_size) - if not self.gallery.load(): - raise SystemExit(f"gallery unavailable for {stock_dir}") - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - # Ungated: the lane's own accept floor is applied once, by score_lane. - # Gating here too would reject before the floor is even consulted. - return self.gallery.rank(crop) - - -class HybridLane(Lane): - """DINO shortlist, then the original matcher re-ranks inside it.""" - - name = "hybrid-verify" - - def __init__(self, shortlist_lane: Lane, verifier: MatcherLane, size: int = 5) -> None: - self.shortlist_lane = shortlist_lane - self.verifier = verifier - self.size = int(size) - self.recall_hits = 0 - self.recall_total = 0 - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - allowed = self.shortlist_lane.shortlist(crop, self.size) - if not allowed: - return [] - scores, _ = self.verifier.matcher.score_crop(crop, method=self.verifier.method) - grouped = group_by_sku(scores) - ranked = [(sku, float(grouped.get(sku, 0.0))) for sku in allowed] - return sorted(ranked, key=lambda kv: kv[1], reverse=True) - - -def score_lane( - lane: Lane, - facings: Sequence[Dict[str, Any]], - image: Any, - *, - accept: float, - margin: float, - scale: str, - enrolled_skus: Collection[str], -) -> Dict[str, Any]: - """One lane over one shelf. Accepts only unique, above-floor answers. - - `top1` and `coverage` were both accepted-correct over enrolled facings, so - ranking and deciding were never separated. They are now four quantities: - - catalog_ceiling in-catalog facings with an enrolled view / all in-catalog - identity_ceiling ungated top-1 rank correct on enrolled / enrolled - decision_ceiling accepted-and-correct on enrolled / enrolled - end_to_end_coverage accepted-and-correct / all in-catalog facings - - Precision divides by every accept, unenrolled in-catalog facings included: - the SKU is on the shelf but has no reference, so an accept there is wrong - and is the failure the enrollment protocol exists to prevent. - """ - from identity import decide_identity - - in_catalog = [f for f in facings if f["label"] != UNKNOWN] - out_catalog = [f for f in facings if f["label"] == UNKNOWN] - enrolled_count = sum(1 for f in in_catalog if f["label"] in enrolled_skus) - unenrolled_count = len(in_catalog) - enrolled_count - - correct = wrong = rejected = top1_correct = 0 - unknown_false = unenrolled_false = 0 - accepts: Counter = Counter() - per_sku: Dict[str, List[int]] = {} - started = time.perf_counter() - - for facing in facings: - crop = crop_of(image, facing["box"]) - if crop.size == 0: - continue - ranked = lane.rank(crop) - decision = lane.decide(crop, ranked, accept=accept, margin=margin, scale=scale) - # decide_identity reports an accept as status "success"; identity_level - # is the stable field across both the inlier and cosine scales. - answered = decision.get("best_sku") if decision.get("identity_level") == "accepted" else None - truth = facing["label"] - if truth != UNKNOWN: - bucket = per_sku.setdefault(truth, [0, 0]) - bucket[1] += 1 - if truth in enrolled_skus and ranked and ranked[0][0] == truth: - top1_correct += 1 - if answered == truth: - correct += 1 - bucket[0] += 1 - elif answered: - wrong += 1 - if truth not in enrolled_skus: - unenrolled_false += 1 - elif answered: - wrong += 1 - unknown_false += 1 - if answered: - accepts[answered] += 1 - else: - rejected += 1 - - elapsed = time.perf_counter() - started - total_accepts = sum(accepts.values()) - hub = max(accepts.values()) / total_accepts if total_accepts else 0.0 - return { - "lane": lane.name, - "accept": accept, - "margin": margin, - "in_catalog": len(in_catalog), - "out_catalog": len(out_catalog), - "enrolled": enrolled_count, - "correct": correct, - "wrong": wrong, - "rejected": rejected, - "catalog_ceiling": round(enrolled_count / len(in_catalog), 4) if in_catalog else None, - "identity_ceiling": round(top1_correct / enrolled_count, 4) if enrolled_count else None, - "decision_ceiling": round(correct / enrolled_count, 4) if enrolled_count else None, - "end_to_end_coverage": round(correct / len(in_catalog), 4) if in_catalog else None, - "false_accepts": wrong, - "unknown_false_accepts": unknown_false, - "unenrolled_false_accepts": unenrolled_false, - "false_accept_rate": round(unknown_false / len(out_catalog), 4) if out_catalog else None, - "near_unknown_false_accept_rate": round(unenrolled_false / unenrolled_count, 4) if unenrolled_count else None, - "precision": round(correct / total_accepts, 4) if total_accepts else None, - "max_sku_share": round(hub, 4), - "hub_sku": accepts.most_common(1)[0][0] if accepts else None, - "hub_alarm": hub > HUB_SHARE, - "histogram": dict(accepts.most_common()), - "per_sku": {sku: {"correct": hit, "total": seen} for sku, (hit, seen) in sorted(per_sku.items())}, - "ms_per_crop": round(elapsed * 1000 / max(1, len(facings)), 1), - } - - -def shortlist_recall( - lane: HybridLane, - facings: Sequence[Dict[str, Any]], - image: Any, - enrolled_skus: Collection[str], -) -> Dict[str, Any]: - """Share of enrolled facings whose true SKU survives the shortlist. - - This is the hybrid's hard ceiling: a verifier cannot pick a SKU the - shortlist already dropped. The denominator is enrolled facings only — a - facing whose SKU has no reference cannot be retrieved by any shortlist, so - counting it here reports a retrieval failure that is really a catalog gap. - Measured on image6_v2, that distinction is 67% vs 98% at top-5. - """ - hits = total = 0 - misses: List[Dict[str, Any]] = [] - for facing in facings: - if facing["label"] == UNKNOWN or facing["label"] not in enrolled_skus: - continue - crop = crop_of(image, facing["box"]) - if crop.size == 0: - continue - total += 1 - allowed = lane.shortlist_lane.shortlist(crop, lane.size) - if facing["label"] in allowed: - hits += 1 - else: - misses.append({"index": facing.get("index"), "truth": facing["label"], "shortlist": allowed}) - return { - "size": lane.size, - "hits": hits, - "total": total, - "recall": round(hits / total, 4) if total else None, - "misses": misses, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--labels", required=True, help="labelled facings JSON") - parser.add_argument("--image", help="shelf photo; defaults to the labels' own image") - parser.add_argument("--stock-dir", default="stock-lopia") - parser.add_argument("--lanes", default="local,matcher,gallery,hybrid-verify") - parser.add_argument("--shortlist", type=int, default=5) - parser.add_argument("--accept", type=float, default=None, help="override the lane's own floor") - parser.add_argument("--margin", type=float, default=None) - parser.add_argument("--live", action="store_true", help="also score at live resolution") - parser.add_argument("--json", help="write the report here") - args = parser.parse_args() - - payload = load_labels(args.labels) - image_path = args.image or payload["image"] - image = cv2.imread(image_path) - if image is None: - raise SystemExit(f"cannot read {image_path}") - - facings = payload["facings"] - held: List[Dict[str, Any]] = [] - refused: List[Dict[str, Any]] = [] - for facing in facings: - if facing["label"] == EXCLUDE: - continue - template = leaked(image, facing["box"], args.stock_dir) - if template: - refused.append({**facing, "template": template}) - else: - held.append(facing) - - wanted = [name.strip() for name in args.lanes.split(",") if name.strip()] - # Candidate sets come from the catalog, never from the labels: a lane told - # which SKUs the shelf holds is being scored against the answer key. - catalog_skus = catalog_sku_ids(args.stock_dir) - # A SKU with no enrolled view cannot be answered by any lane. Counting those - # facings as lane failures hides the real ceiling: catalog coverage. - enrolled_skus = {sku_of(name) for name in os.listdir(args.stock_dir) - if os.path.splitext(name)[1].lower() in (".jpg", ".jpeg", ".png")} - unenrolled = Counter(f["label"] for f in held if f["label"] != UNKNOWN and f["label"] not in enrolled_skus) - - lanes: Dict[str, Lane] = {} - if "matcher" in wanted or "hybrid-verify" in wanted: - lanes["matcher"] = MatcherLane(args.stock_dir) - if "local" in wanted: - lanes["local"] = LocalLaneAdapter(args.stock_dir) - if "gallery" in wanted or "hybrid-verify" in wanted: - lanes["gallery"] = GalleryLane(args.stock_dir, catalog_skus) - if "hybrid-verify" in wanted: - lanes["hybrid-verify"] = HybridLane(lanes["gallery"], lanes["matcher"], size=args.shortlist) - # Boxes are in the original image's coordinates. Scoring a downscaled copy - # needs the boxes scaled with it, or every crop is taken from the wrong place. - resolutions = [("full", image, 1.0)] - if args.live: - live = at_live_resolution(image) - resolutions.append(("live", live, live.shape[1] / float(image.shape[1]))) - - report: Dict[str, Any] = { - "image": image_path, - "stock_dir": args.stock_dir, - "labels": args.labels, - "facings": len(facings), - "leakage": {"excluded": len(refused), "labels": refused}, - "hub_share_limit": HUB_SHARE, - "enrollment": { - "catalog_skus": catalog_skus, - "enrolled_skus": sorted(enrolled_skus), - "unenrolled_skus": dict(unenrolled.most_common()), - "facings_without_enrollment": sum(unenrolled.values()), - "scored_facings": len(held), - }, - "results": {}, - } - - for resolution, picture, factor in resolutions: - scaled = held if factor == 1.0 else [ - {**f, "box": [int(round(v * factor)) for v in f["box"]]} for f in held - ] - rows = [] - for name in wanted: - lane = lanes[name] - scale = {"gallery": "cosine", "local": "local"}.get(name, "inlier") - default_accept = {"cosine": 0.60, "local": LOCAL_ACCEPT}.get(scale, 1.0) - default_margin = 0.04 if scale == "cosine" else 0.15 - accept = args.accept if args.accept is not None else default_accept - margin = args.margin if args.margin is not None else default_margin - row = score_lane(lane, scaled, picture, accept=accept, margin=margin, scale=scale, - enrolled_skus=enrolled_skus) - if isinstance(lane, HybridLane): - row["shortlist_recall"] = shortlist_recall(lane, scaled, picture, enrolled_skus) - rows.append(row) - report["results"][resolution] = rows - - if args.json: - with open(args.json, "w") as handle: - json.dump(report, handle, indent=1) - - enrollment = report["enrollment"] - if enrollment["unenrolled_skus"]: - print(f"\nenrollment ceiling: {enrollment['facings_without_enrollment']} facing(s) belong to " - f"{len(enrollment['unenrolled_skus'])} SKU(s) with no enrolled view — " - f"unanswerable by any lane: {enrollment['unenrolled_skus']}") - for resolution, rows in report["results"].items(): - print(f"\n{resolution} resolution — {enrollment['scored_facings']} scored facings " - f"({sum(1 for f in held if f['label'] != UNKNOWN)} in-catalog, " - f"{sum(1 for f in held if f['label'] == UNKNOWN)} unknown)") - print(f" {'lane':<14} {'catalog':>8} {'identity':>9} {'decision':>9} {'end-to-end':>11} " - f"{'precision':>10} {'false acc':>10} {'max SKU':>8} {'ms/crop':>8}") - for row in rows: - cells = [f"{row[key]:.0%}" if row[key] is not None else "-" for key in - ("catalog_ceiling", "identity_ceiling", "decision_ceiling", - "end_to_end_coverage", "precision")] - flag = " HUB!" if row["hub_alarm"] else "" - print(f" {row['lane']:<14} {cells[0]:>8} {cells[1]:>9} {cells[2]:>9} {cells[3]:>11} " - f"{cells[4]:>10} {row['false_accepts']:>10} {row['max_sku_share']:>7.0%} " - f"{row['ms_per_crop']:>8.1f}{flag}") - if "shortlist_recall" in row: - recall = row["shortlist_recall"] - share = f"{recall['recall']:.0%}" if recall["recall"] is not None else "-" - print(f" shortlist top-{recall['size']} recall: {share} " - f"({recall['hits']}/{recall['total']}) — hard ceiling for this lane") - if refused: - print(f"\nleakage: {len(refused)} facing(s) refused as catalog templates") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py index 56eeeca..948ab19 100644 --- a/recognization/code/eval_live.py +++ b/recognization/code/eval_live.py @@ -63,11 +63,6 @@ def file_sha256(path: str) -> Optional[str]: def content_sha256(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() -# Every backend the server can be started with, so a recording made by the -# shipped `run-demo-server.sh` (--identity local) can be replayed. `local` is -# the default; `matcher` is Van's scorer kept as the E-1 baseline lane. -IDENTITY_BACKENDS = ("matcher", "local", "gallery") - def _iou(a: Sequence[float], b: Sequence[float]) -> float: if len(a) != 4 or len(b) != 4: @@ -122,10 +117,8 @@ def recorded_identity(rows: Sequence[Dict[str, Any]]) -> Optional[str]: if raw is None: continue name = str(raw).strip().lower() - if name not in IDENTITY_BACKENDS: - raise EvalInputError( - f"recorded identity backend is not one of {', '.join(IDENTITY_BACKENDS)}: {raw}" - ) + if name not in ("matcher", "gallery"): + raise EvalInputError(f"recorded identity backend is not matcher or gallery: {raw}") found.append(name) unique = set(found) if len(unique) > 1: @@ -138,13 +131,10 @@ def resolve_identity(requested: Optional[str], recorded: Optional[str]) -> str: chosen = (requested or recorded or "").strip().lower() if not chosen: raise EvalInputError( - "recording does not name an identity backend; pass --identity " - + " or --identity ".join(IDENTITY_BACKENDS) - ) - if chosen not in IDENTITY_BACKENDS: - raise EvalInputError( - f"identity must be one of {', '.join(IDENTITY_BACKENDS)}, not {requested or recorded}" + "recording does not name an identity backend; pass --identity matcher or --identity gallery" ) + if chosen not in ("matcher", "gallery"): + raise EvalInputError(f"identity must be matcher or gallery, not {requested or recorded}") if recorded and requested and recorded != chosen: raise EvalInputError( f"recording identity is {recorded}; pass --identity {recorded}" @@ -582,10 +572,8 @@ def score_recording( tracks = [track for frame in source for track in (frame.get("tracks") or [])] decisions = score_tracks(tracks) backend = identity.strip().lower() - if backend not in IDENTITY_BACKENDS: - raise EvalInputError( - f"identity must be one of {', '.join(IDENTITY_BACKENDS)}, not {identity}" - ) + if backend not in ("matcher", "gallery"): + raise EvalInputError(f"identity must be matcher or gallery, not {identity}") report = { "recording": recording_dir, "source": "live_replay" if live_frames is not None else "recorded_index", @@ -802,7 +790,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument( "--identity", default=None, - choices=list(IDENTITY_BACKENDS), + choices=["matcher", "gallery"], help="which mechanism names products (required when the recording omits identity_backend)", ) parser.add_argument("--device", default=None, help="cpu | mps | cuda | cuda:0") diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py index 9c1bd32..c952ed7 100644 --- a/recognization/code/gallery.py +++ b/recognization/code/gallery.py @@ -650,38 +650,6 @@ class ProductGallery: return GalleryMatch(sku_id=top_sku, similarity=top_score, runner_up=runner_sku, runner_up_similarity=runner_score) - def rank(self, crop: Any, limit: Optional[int] = None) -> List[Tuple[str, float]]: - """Every enrolled SKU by similarity, best first, with no rejection. - - `match()` answers "is anything close enough"; a shortlist needs "what are - the closest candidates", including when none of them is close. Gating a - shortlist at the raw threshold empties it for exactly the crops the - verifier exists to rescue. - """ - if not self._available or self._matrix is None: - return [] - import numpy as np - - embedding = self.embed(crop) - if self._head is not None: - import torch - with torch.no_grad(): - projected = torch.nn.functional.normalize( - self._head["projection"](torch.from_numpy(embedding).float()), dim=0 - ) - similarity = (self._head["prototypes"] @ projected).numpy() - ranked = sorted(zip(self._head["class_ids"], similarity.tolist()), - key=lambda kv: kv[1], reverse=True) - else: - similarity = self._matrix @ embedding - order, grouped, starts = self._rank_index() - if not order: - return [] - best_by_sku = np.maximum.reduceat(similarity[grouped], starts) - ranked = sorted(zip(order, best_by_sku.tolist()), key=lambda item: item[1], reverse=True) - return [(str(sku), float(score)) for sku, score in ranked[:limit]] if limit else \ - [(str(sku), float(score)) for sku, score in ranked] - def head_loaded(self) -> bool: return self._head is not None diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 51fa3bb..6fab9c0 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -1,16 +1,8 @@ """Identity ranking for catalog scores. Separates raw matcher scores from accept/unknown/ambiguous/reject so a -zero-score crop cannot become a SKU. Three score scales are recognized: - -- `inlier`: Van's composite, kept as the selectable baseline lane (E-1). -- `local`: `identity_lanes.LocalLane`'s margin-over-runner-up, in inlier units. -- `cosine`: the embedding gallery, fenced off and not in the live path. - -The `local` line is measured on the corrected image6 labels: 2.0 keeps 95% of -shown labels correct at 59% coverage with the text lane, 1.0 keeps 87% at 66%. -It is tuned on one photo, so it is a direction, not a calibration; E-4 replaces -it once Tier-2 held-out labels exist. +zero-score crop cannot become a SKU. Thresholds are conservative defaults +for the 21-SKU closed set; they are not calibrated accuracy claims. """ from __future__ import annotations @@ -22,17 +14,11 @@ IDENTITY_AMBIGUOUS = "ambiguous" IDENTITY_UNKNOWN = "unknown" IDENTITY_REJECTED = "rejected" -# ponytail: closed-set defaults for the baseline inlier scale. Recalibrate when -# held-out scores exist. +# ponytail: closed-set defaults. Recalibrate when held-out scores exist. DEFAULT_MIN_SCORE = 1.0 DEFAULT_MIN_MARGIN = 0.15 DEFAULT_MIN_CONFIDENCE = 0.55 -# `local` scale (identity_lanes.LocalLane). The value is already a margin over -# the runner-up, so the relative-margin test does not apply to it: a single -# absolute line is the whole gate. -LOCAL_ACCEPT = 2.0 - def relative_confidence(top_score: float, runner_up_score: float) -> float: """Existing matcher formula: top / (top + 0.5 * runner).""" @@ -184,8 +170,6 @@ def decide_identity( min_margin = COSINE_MARGIN if min_confidence == DEFAULT_MIN_CONFIDENCE: min_confidence = COSINE_AMBIGUOUS - elif scale == "local" and min_score == DEFAULT_MIN_SCORE: - min_score = LOCAL_ACCEPT cleaned: Dict[str, float] = {} for sku_id, score in (scores or {}).items(): number = _finite(score) @@ -225,29 +209,6 @@ def decide_identity( ranked = [{"sku_id": s, "score": round(sc, 2)} for s, sc in sorted_candidates[:5]] top_details = (details or {}).get(top_sku, {}) - if scale == "local": - # The score is already a margin over the runner-up, so the relative - # margin test below would apply the same correction twice. One absolute - # line is the whole gate; a candidate under it is unknown, not ambiguous, - # because the lane has already said which one it prefers. - if top_score <= 0.0: - level, message, best = IDENTITY_REJECTED, "No matching evidence", None - elif top_score < min_score: - level, message, best = IDENTITY_UNKNOWN, "Top score below accept threshold", None - else: - level, message, best = IDENTITY_ACCEPTED, "Accepted unique catalog match", top_sku - return { - "status": "success" if level == IDENTITY_ACCEPTED else level, - "method": method, - "best_sku": best, - "score": round(top_score, 2), - "confidence": round(top_score, 4), - "identity_level": level, - "ranked_candidates": ranked, - "details": top_details, - "message": message, - } - if top_score <= 0.0: level = IDENTITY_REJECTED message = "No matching evidence" diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py deleted file mode 100644 index a27a2ba..0000000 --- a/recognization/code/identity_lanes.py +++ /dev/null @@ -1,495 +0,0 @@ -"""Identity lanes and the decision that fuses them. - -Two lanes answer "which catalog row is this crop", and one policy turns their -evidence into accepted / ambiguous / unknown / rejected: - -- `LocalLane` reads printed geometry (RootSIFT over a reference pyramid, - similarity-constrained RANSAC, a hue check). It is the only primitive that - ranks fine-grained siblings correctly: 40/42 enrolled facings first at full - resolution, against 32/42 for Van's composite. -- `TextLane` reads the package text and compares it to each catalog row's - signature (its printed name plus the OCR of its enrolled packshot). It is the - only measured mechanism that rejects a near-unknown sibling, and it can name - catalog rows that have no enrolled image at all. - -Why a policy and not a threshold: the matcher's raw score is not comparable -across references of different resolution, and a single unverified match used -to clear the old floor. Measured on the corrected image6 labels, the old gate -accepts 22-28 of 48 unknowns; the local gate alone reaches 79% precision at 43% -coverage, and adding the text lane reaches 95% precision at 59% coverage. - -`ponytail:` the policy is a hand-set pair of lines tuned on one photo, not a -calibrated model. E-4 replaces it once Tier-2 held-out labels exist. -""" - -from __future__ import annotations - -import os -import sys -import unicodedata -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import cv2 # noqa: E402 -import numpy as np # noqa: E402 - -from rebuild_catalog import sku_of # noqa: E402 - -# The local lane's accept line, in the units of `confidence()` below. Measured -# on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59% -# coverage, 1.0 keeps 87% at 66%. The stricter line is the default because a -# wrong label on a shelf is worse than a missing one. -LOCAL_ACCEPT = 2.0 -# The text lane's own accept line: a bigram overlap this high, with this much -# margin over the runner-up, names the row on its own. Measured: the text lane -# picks the right SKU on 41-42 of the 55 facings where it reads anything. -TEXT_ACCEPT = 0.5 -TEXT_MARGIN = 0.15 -# Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a -# 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and -# BRISK find zero inliers at live resolution. Scaling each reference to the -# crop's own size removes the ratio instead of widening the pyramid. -REFERENCE_HEIGHTS = (200, 400) -# Lowe's ratio test. -RATIO_THRESH = 0.75 -# RANSAC reprojection tolerance, as a fraction of the reference's long side. -REPROJ_FRACTION = 0.02 -# Hue histogram: chromatic pixels only, so a white background cannot vote. -HUE_BINS = 18 -HUE_SATURATION_MIN = 60 -HUE_VALUE_MIN = 50 - - -@dataclass -class Evidence: - """What one reference view says about one crop.""" - - inliers: int = 0 - cover: float = 0.0 # inlier hull area / reference area - qcover: float = 0.0 # inlier hull area / crop area - color: float = 0.0 # hue histogram intersection - - -def _hue_histogram(image: Any) -> np.ndarray: - """Hue distribution over the chromatic pixels, L1-normalized.""" - hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) - chromatic = ((hsv[..., 1] > HUE_SATURATION_MIN) & (hsv[..., 2] > HUE_VALUE_MIN)) - hist = cv2.calcHist([hsv], [0], chromatic.astype(np.uint8), [HUE_BINS], [0, 180]).ravel() - return hist / (hist.sum() + 1e-6) - - -def _root_sift(descriptors: Optional[np.ndarray]) -> Optional[np.ndarray]: - """RootSIFT: L1-normalize then square-root. Cheap, and it beats raw SIFT.""" - if descriptors is None: - return None - normalized = descriptors / (np.abs(descriptors).sum(1, keepdims=True) + 1e-7) - return np.sqrt(normalized).astype(np.float32) - - -class LocalLane: - """RootSIFT over a reference pyramid, similarity-RANSAC, hue re-rank. - - Scores every enrolled reference view and reports the best view per SKU, so - a multi-view catalog cannot make one product compete with itself. - """ - - name = "local" - - def __init__( - self, - stock_dir: str, - *, - heights: Sequence[int] = REFERENCE_HEIGHTS, - nfeatures: int = 2000, - ) -> None: - self.stock_dir = stock_dir - self.sift = cv2.SIFT_create(nfeatures=nfeatures) - self.matcher = cv2.BFMatcher(cv2.NORM_L2) - self.views: List[Dict[str, Any]] = [] - self.sku_ids: List[str] = [] - self._load(heights) - - def _load(self, heights: Sequence[int]) -> None: - if not os.path.exists(self.stock_dir): - return - names = sorted( - name for name in os.listdir(self.stock_dir) - if name.lower().endswith((".jpg", ".jpeg", ".png")) - ) - for name in names: - image = cv2.imread(os.path.join(self.stock_dir, name)) - if image is None: - continue - sku = sku_of(name) - if sku not in self.sku_ids: - self.sku_ids.append(sku) - for height in heights: - scaled = cv2.resize( - image, - (max(1, int(image.shape[1] * height / image.shape[0])), int(height)), - interpolation=cv2.INTER_AREA, - ) - gray = cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY) - keypoints, descriptors = self.sift.detectAndCompute(gray, None) - self.views.append( - { - "sku": sku, - "keypoints": keypoints, - "descriptors": _root_sift(descriptors), - "shape": gray.shape, - "hist": _hue_histogram(scaled), - } - ) - - def view_count(self) -> int: - return len(self.views) - - def evidence(self, crop: Any) -> Dict[str, Evidence]: - """Best evidence per canonical SKU. Empty when the crop has no features.""" - if crop is None or getattr(crop, "size", 0) == 0: - return {} - gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY) - keypoints, descriptors = self.sift.detectAndCompute(gray, None) - descriptors = _root_sift(descriptors) - if descriptors is None or len(descriptors) < 2: - return {} - query_hist = _hue_histogram(crop) - crop_area = float(gray.shape[0] * gray.shape[1]) - - best: Dict[str, Evidence] = {} - for view in self.views: - reference = view["descriptors"] - if reference is None or len(reference) < 2: - continue - pairs = self.matcher.knnMatch(descriptors, reference, k=2) - good = [ - first for first, second in (pair for pair in pairs if len(pair) == 2) - if first.distance < RATIO_THRESH * second.distance - ] - if len(good) < 4: - continue - source = np.float32([keypoints[m.queryIdx].pt for m in good]) - target = np.float32([view["keypoints"][m.trainIdx].pt for m in good]) - _, mask = cv2.estimateAffinePartial2D( - source, target, method=cv2.RANSAC, - ransacReprojThreshold=REPROJ_FRACTION * max(view["shape"]), - ) - if mask is None: - continue - inliers = mask.ravel().astype(bool) - count = int(inliers.sum()) - if count < 3: - continue - reference_area = float(view["shape"][0] * view["shape"][1]) - evidence = Evidence( - inliers=count, - cover=float(cv2.contourArea(cv2.convexHull(target[inliers])) / reference_area), - qcover=float(cv2.contourArea(cv2.convexHull(source[inliers])) / crop_area), - color=float(np.minimum(query_hist, view["hist"]).sum()), - ) - current = best.get(view["sku"]) - if current is None or evidence.inliers > current.inliers: - best[view["sku"]] = evidence - return best - - @staticmethod - def rank_key(evidence: Evidence) -> float: - """Order candidates. The hue term is what separates colour-coded variants - (pink ゼロ from red ハーフ), which grayscale SIFT cannot see at all.""" - return evidence.inliers * (0.25 + evidence.color) - - @staticmethod - def confidence(evidence: Evidence, runner_up: Evidence) -> float: - """How much this crop's best candidate stands out, in inlier units. - - The runner-up is subtracted in inlier space, so a sibling that matches - the same artwork cannot be separated by a high score alone; the hue - agreement then scales the result, because a match that disagrees on - colour is not the same package. - """ - margin = 1.0 - runner_up.inliers / max(evidence.inliers, 1) - return evidence.inliers * margin * evidence.color - - def rank(self, crop: Any) -> List[Tuple[str, float]]: - """Candidates best-first, each with the confidence the gate reads. - - Order and value are different quantities on purpose: the order comes - from `rank_key` (inliers weighted by hue), the value from `confidence` - (the margin over the runner-up). Sorting by the value would reorder the - candidates, which is how a lane that ranks 40/42 correctly ends up - gating on the wrong one. - """ - evidence = self.evidence(crop) - if not evidence: - return [] - ranked = sorted(evidence.items(), key=lambda kv: self.rank_key(kv[1]), reverse=True) - return [ - ( - sku, - self.confidence( - item, - ranked[index + 1][1] if index + 1 < len(ranked) else Evidence(), - ), - ) - for index, (sku, item) in enumerate(ranked) - ] - - def scores(self, crop: Any) -> Dict[str, float]: - """`rank()` as a dict, in rank order. Convenience for callers that only - need the values; the policy uses `rank()` so the order survives.""" - return {sku: value for sku, value in self.rank(crop)} - - -# Katakana folding: OCR reads small kana and the long-vowel mark inconsistently, -# so キユーピー and キユピー have to land on the same bigrams. -_KANA_FOLD = str.maketrans( - "ッャュョァィゥェォヮヵヶユ", "ツヤユヨアイウエオワカケユ" -) - - -def normalize_text(text: str) -> str: - """NFKC, uppercase, kana-folded, alphanumeric and `%` only.""" - text = unicodedata.normalize("NFKC", text).upper() - text = text.translate(_KANA_FOLD) - text = text.replace("キユーピー", "キユピ").replace("キューピー", "キユピ") - return "".join(ch for ch in text if ch.isalnum() or ch == "%") - - -def bigrams(text: str) -> set: - return {text[i:i + 2] for i in range(len(text) - 1)} - - -def text_similarity(read: str, signature: str) -> float: - """Share of the read text's bigrams that the signature explains. - - Asymmetric on purpose: the crop shows a few words of a package, the - signature holds the whole name, so the question is how much of what was - read the catalog row accounts for. - """ - read_grams = bigrams(normalize_text(read)) - if not read_grams: - return 0.0 - return len(read_grams & bigrams(normalize_text(signature))) / max(1, len(read_grams)) - - -class TextLane: - """Package text against each catalog row's signature. - - The signature is the row's printed name plus the OCR of its enrolled - packshot, so it is built from the catalog and never from the evaluation - photo. Rows with no image still get a signature from their name, which is - how the lane can name a SKU no matcher can see. - """ - - name = "text" - - def __init__( - self, signatures: Dict[str, str], ocr_engine: Any = None, lock: Any = None - ) -> None: - self.signatures = {str(sku): text for sku, text in signatures.items()} - self.ocr_engine = ocr_engine - # The engine's OCR lock. Vision tolerates concurrency but the CPU does - # not, and this lane runs one OCR per crop on the classify pool, so it - # shares the same lock as tag reading instead of adding a second one. - self.lock = lock - - @classmethod - def from_catalog( - cls, - catalog_rows: Sequence[Dict[str, Any]], - stock_dir: str, - ocr_engine: Any = None, - lock: Any = None, - ) -> "TextLane": - signatures: Dict[str, str] = {} - for row in catalog_rows: - sku = str(row.get("sku_id") or "") - if not sku: - continue - signature = str(row.get("product_name") or "") - if ocr_engine is not None and getattr(ocr_engine, "available", lambda: False)(): - path = os.path.join(stock_dir, f"{sku}.jpg") - if os.path.exists(path): - image = cv2.imread(path) - if image is not None: - try: - result = ocr_engine.read(image) - except Exception: - result = None - if result is not None: - signature += " " + " ".join(line.text for line in result.lines) - signatures[sku] = signature - return cls(signatures, ocr_engine=ocr_engine, lock=lock) - - def read(self, crop: Any) -> str: - """The text on one crop, upsampled when the crop is small. - - Below ~200 px tall the recognizer misses the small variant words that - are the whole point of this lane, so the crop is doubled first. - """ - if self.ocr_engine is None or crop is None or getattr(crop, "size", 0) == 0: - return "" - if not getattr(self.ocr_engine, "available", lambda: False)(): - return "" - image = crop - if image.shape[0] < 200: - image = cv2.resize(image, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_CUBIC) - try: - if self.lock is not None: - with self.lock: - result = self.ocr_engine.read(image) - else: - result = self.ocr_engine.read(image) - except Exception: - return "" - return " ".join(line.text for line in result.lines) - - def scores(self, crop: Any) -> Dict[str, float]: - """Bigram similarity per catalog row. Empty when nothing was read.""" - return self.scores_from_text(self.read(crop)) - - def scores_from_text(self, read: str) -> Dict[str, float]: - if len(normalize_text(read)) < 4: - return {} - return { - sku: text_similarity(read, signature) - for sku, signature in self.signatures.items() - } - - -def _ranked(scores: Any) -> List[Tuple[str, float]]: - """Candidates best-first. - - A lane hands over an already-ordered sequence and it is used as given: the - order is the lane's ranking, and re-sorting by the confidence value would - silently replace it. A plain dict is sorted by value, which is the only - order a dict has. - """ - if scores is None: - return [] - if isinstance(scores, dict): - items = scores.items() - return sorted( - ((str(sku), float(value)) for sku, value in items), - key=lambda kv: kv[1], - reverse=True, - ) - return [(str(sku), float(value)) for sku, value in scores] - - -class DecisionPolicy: - """Turn two lanes' evidence into one identity decision. - - The local lane answers first, because it is the only lane that ranks - siblings correctly. The text lane then does one of two jobs: it vetoes a - local answer whose package text names a different catalog row, and it - answers on its own when the local lane has nothing. Both jobs are measured: - the veto takes precision from 62% to 82% for one lost correct label, and - the lane adds ten correct labels at live resolution, including SKUs with no - enrolled image. - """ - - def __init__( - self, - *, - local_accept: float = LOCAL_ACCEPT, - text_accept: float = TEXT_ACCEPT, - text_margin: float = TEXT_MARGIN, - method: str = "rootsift", - ) -> None: - self.local_accept = float(local_accept) - self.text_accept = float(text_accept) - self.text_margin = float(text_margin) - self.method = method - - def decide( - self, - local_rank: Any, - text_rank: Any = None, - *, - details: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - """One decision, in the shape `decide_identity` already returns.""" - from identity import ( - IDENTITY_ACCEPTED, - IDENTITY_AMBIGUOUS, - IDENTITY_REJECTED, - IDENTITY_UNKNOWN, - ) - - local = _ranked(local_rank) - text = _ranked(text_rank) - text_top = text[0] if text else None - text_runner = text[1][1] if len(text) > 1 else 0.0 - text_names = ( - text_top[0] - if text_top - and text_top[1] >= self.text_accept - and (text_top[1] - text_runner) >= self.text_margin - else None - ) - - answer: Optional[str] = None - source: Optional[str] = None - if local and local[0][1] >= self.local_accept: - candidate = local[0][0] - # A local match whose package text names a different row is the - # near-unknown case no threshold can reject: the sibling shares the - # artwork, so the geometry is real and only the words differ. - if text_top and text_top[0] != candidate and text_top[1] > local_text_floor(text, candidate): - answer = None - else: - answer, source = candidate, "local" - if answer is None and text_names: - answer, source = text_names, "text" - - ranked = [ - {"sku_id": sku, "score": round(value, 2)} for sku, value in local[:5] - ] - if not ranked and text: - ranked = [{"sku_id": sku, "score": round(value, 3)} for sku, value in text[:5]] - - if answer is not None: - level = IDENTITY_ACCEPTED - message = "Accepted unique catalog match" - elif local and local[0][1] > 0.0: - level = IDENTITY_AMBIGUOUS if len(local) > 1 else IDENTITY_UNKNOWN - message = "No candidate cleared the accept line" - elif text: - level = IDENTITY_UNKNOWN - message = "Text read, but it does not name one catalog row" - else: - level = IDENTITY_REJECTED - message = "No matching evidence" - - top_score = local[0][1] if local else (text_top[1] if text_top else 0.0) - return { - "status": "success" if level == IDENTITY_ACCEPTED else level, - "method": self.method, - "best_sku": answer, - "score": round(float(top_score), 2), - "confidence": round(float(top_score), 4), - "identity_level": level, - "ranked_candidates": ranked, - "details": (details or {}).get(answer, {}) if answer else {}, - "message": message, - "source": source, - } - - -def local_text_floor(text: List[Tuple[str, float]], candidate: str) -> float: - """The text score the local candidate itself explains. - - A veto needs the text to *prefer* another row, not merely to mention it: a - crop that reads "マヨネーズ" scores every mayonnaise row, and the local - candidate is usually one of them. Only a row the text explains better than - the local answer's own row is a contradiction. - """ - for sku, value in text: - if sku == candidate: - return value - return 0.0 diff --git a/recognization/code/live.py b/recognization/code/live.py index b338514..b216adb 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -814,7 +814,7 @@ class LiveSession: submitted_box = list(track.box) track_id = track.track_id try: - future = self._executor.submit(self.engine.score_crop_lanes, crop, method) + future = self._executor.submit(self.engine.score_crop, crop, method) except RuntimeError: # executor already shut down with self._lock: track.in_flight = max(0, track.in_flight - 1) @@ -989,7 +989,7 @@ class LiveSession: self._crops_rejected += 1 return _crop_error("CROP_STALE", stale) try: - future = self._executor.submit(self.engine.score_crop_lanes, crop, method) + future = self._executor.submit(self.engine.score_crop, crop, method) except RuntimeError: # executor already shut down self._release_crop_slot(record) self._crops_rejected += 1 @@ -1094,9 +1094,9 @@ class LiveSession: source: str = SOURCE_FRAME, ) -> None: try: - scores, text_rank = future.result() + scores = future.result() except Exception: - scores, text_rank = {}, [] + scores = {} finished_at = time.perf_counter() with self._lock: track = self._tracks.get(track_id) @@ -1134,7 +1134,7 @@ class LiveSession: track.score_n += 1 self._classify_times.append(finished_at - submitted_at) - if not any(track.observations) and not text_rank: + if not any(track.observations): if track.attempts >= self.settings.max_attempts: track.state = STATE_UNRECOGNIZED track.confirmed = True @@ -1143,13 +1143,10 @@ class LiveSession: track.candidates = [] return - from identity import aggregate_votes, decide_identity - from identity_lanes import DecisionPolicy mean_scores = aggregate_votes(track.observations) - scale = getattr(self.engine, "identity_scale", "inlier") - if scale == "cosine": + if getattr(self.engine, "identity_scale", "inlier") == "cosine": # score_crop only returns scores the gallery already accepted, so # the aggregate must not be re-gated at COSINE_ACCEPT: that line is # above the head's own 0.40 and refused matches the gallery had @@ -1165,17 +1162,8 @@ class LiveSession: mean_scores, method=method, scale="cosine", min_score=line, min_confidence=line, ) - elif scale == "local": - # The local lane's order and value are different quantities: the - # order comes from its rank key, the value is the margin over the - # runner-up. `score_crop` returns that order, so it is used as-is; - # re-sorting by value would name a different SKU than the lane - # chose. The newest crop is the evidence, not the mean: averaging - # in an empty observation halves a good confidence below the - # accept line, which is the dilution the cosine branch guards too. - decision = DecisionPolicy().decide(list(scores.items()), text_rank) else: - decision = decide_identity(mean_scores, method=method, scale=scale) + decision = decide_identity(mean_scores, method=method) track.candidates = [ {"sku_id": row.get("sku_id"), "score": row.get("score")} for row in (decision.get("ranked_candidates") or [])[:3] diff --git a/recognization/code/rebuild_catalog.py b/recognization/code/rebuild_catalog.py deleted file mode 100644 index f8c9787..0000000 --- a/recognization/code/rebuild_catalog.py +++ /dev/null @@ -1,412 +0,0 @@ -"""Rebuild a store catalog from packshots, refusing unsafe enrollment. - -The LOPIA catalog shipped with views cut from the test shelf itself: 71-119 px -wide, one 18 px sliver, duplicate crops, and five SKUs with no view at all. That -is why every dark package collapsed onto one SKU and why the head was trained on -its own exam. - -This rebuilds a catalog directory from named packshots and refuses, per view: - - * a view shorter than MIN_HEIGHT or narrower than MIN_WIDTH, - * a view whose capture session appears in --eval-manifest, - * a packshot whose catalog name is not supported by its OCR text, - * a view that is a near-duplicate of another view of the same SKU, - * a SKU left with fewer than --min-views distinct views. - -Every enrolled image must have a CSV row; a SKU with an image and no row refuses -the build. CSV rows with no image are kept and reported as text-only coverage -gaps, because the text lane names and vetoes those products from the CSV alone. -Capture session ids come from the source manifest (``manifest.json`` beside the -images by default), whose image entries use ``file`` plus -``capture_session_id`` or ``session_id``. Pixel equality is not a leak test: -re-cut crops pass it, so leakage is refused by capture session instead. - - python code/rebuild_catalog.py --source /tmp/lopia-clean \ - --output stock-lopia-clean --csv stock-lopia/stock.csv - python code/rebuild_catalog.py --source /tmp/lopia-clean \ - --output stock-lopia-clean --csv stock-lopia/stock.csv --replace - -Views are named .jpg, _2.jpg, ... The CSV is copied byte-for-byte, so -text-only products stay in the catalog and its content hash does not move. -""" - -from __future__ import annotations - -import argparse -import csv -import json -import os -import shutil -import sys -import tempfile -import unicodedata -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import cv2 # noqa: E402 -import numpy as np # noqa: E402 - -from ocr import OcrUnavailable, build_ocr_engine # noqa: E402 - -# A view narrower than this cannot carry the logo and text geometry that tells -# two fine-grained products apart. The shipped LOPIA views were 71-119 px. -MIN_WIDTH = 120 -# The enrollment protocol's floor: a reference view under 160 px tall carries no -# text geometry (facings under 60 px wide failed every measured lane). -MIN_HEIGHT = 160 -# Correct packshots measured 0.79-1.00 of the catalog name's bigrams in their -# OCR text; the wrong-size Kewpie 450g packshot tops out at 0.64 against every -# name in the catalog. 0.70 sits between them with room for OCR noise. -MIN_NAME_OCR = 0.70 -# Vision drops small print at native scale: the l11 packshot reads ハーフ only -# when upscaled (0.57 -> 0.79). Reading at 2x is what makes the check about the -# package rather than about Vision's recall. -OCR_SCALE = 2.0 -# Two views of one SKU this similar are the same photograph counted twice. -DUPLICATE_COSINE = 0.98 -# One view cannot be checked against anything; two is the floor for a vote. -# Official pack-shot pages publish a single front view per JAN, so a catalog -# built from them runs at 1 and says so; the floor exists to catch accidental -# single-view enrollment from shelf crops, where several views are available. -MIN_VIEWS = 2 -IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png") -SESSION_KEYS = ("capture_session_id", "session_id") -SESSION_LIST_KEYS = ("sessions", "session_ids", "evaluation_sessions") - - -class RebuildError(Exception): - """Refused input. The destination, if it existed, is unchanged.""" - - -def sku_of(filename: str) -> str: - """`sku_l01_2.jpg` -> `sku_l01`; `sku_l01.jpg` -> `sku_l01`.""" - stem = os.path.splitext(filename)[0] - parts = stem.split("_") - if len(parts) >= 2 and parts[0] == "sku": - return "_".join(parts[:2]) - return stem - - -def view_rank(filename: str) -> int: - """Order views so the primary view sorts first: `sku_l01.jpg` before `_2`.""" - stem = os.path.splitext(filename)[0] - parts = stem.split("_") - return int(parts[2]) if len(parts) > 2 and parts[2].isdigit() else 1 - - -def cosine(left: np.ndarray, right: np.ndarray) -> float: - a = left.astype(np.float32).ravel() - b = right.astype(np.float32).ravel() - denom = float(np.linalg.norm(a) * np.linalg.norm(b)) - return float((a * b).sum() / denom) if denom else 0.0 - - -def upscale(image: np.ndarray) -> np.ndarray: - """Enlarge for OCR only; the stored view keeps its own pixels.""" - return cv2.resize(image, None, fx=OCR_SCALE, fy=OCR_SCALE, interpolation=cv2.INTER_CUBIC) - - -def normalize_text(text: str) -> str: - """NFKC, uppercase, small kana folded, alnum and `%` kept.""" - value = unicodedata.normalize("NFKC", text).upper() - # Vision varies between small and full-size kana, and Kewpie prints - # キユーピー with a small ユ; folding both keeps the comparison about the - # product name rather than the typesetter. - value = value.translate(str.maketrans("ッャュョァィゥェォヮヵヶ", "ツヤユヨアイウエオワカケ")) - value = value.replace("キユーピー", "キユピ").replace("キューピー", "キユピ") - return "".join(char for char in value if char.isalnum() or char == "%") - - -def bigrams(text: str) -> Set[str]: - value = normalize_text(text) - return {value[index:index + 2] for index in range(len(value) - 1)} - - -def name_ocr_similarity(name: str, ocr_text: str) -> float: - """Share of the catalog name's bigrams that the OCR text explains.""" - name_bigrams = bigrams(name) - return len(name_bigrams & bigrams(ocr_text)) / max(1, len(name_bigrams)) - - -def collect(source: str) -> Dict[str, List[str]]: - by_sku: Dict[str, List[str]] = {} - for name in sorted(os.listdir(source)): - if os.path.splitext(name)[1].lower() not in IMAGE_EXTENSIONS: - continue - by_sku.setdefault(sku_of(name), []).append(name) - for views in by_sku.values(): - views.sort(key=view_rank) - return by_sku - - -def read_catalog(csv_path: str) -> Tuple[List[str], List[Dict[str, str]], Dict[str, str]]: - """Columns, rows, and SKU -> product_name. Refuses a malformed catalog.""" - with open(csv_path, newline="", encoding="utf-8-sig") as handle: - reader = csv.DictReader(handle) - fieldnames = list(reader.fieldnames or []) - rows = list(reader) - if "SKU_ID" not in fieldnames or "product_name" not in fieldnames: - raise RebuildError(f"{csv_path}: needs SKU_ID and product_name columns") - names: Dict[str, str] = {} - duplicates: Set[str] = set() - for row in rows: - sku = (row.get("SKU_ID") or "").strip() - if not sku: - raise RebuildError(f"{csv_path}: a row has an empty SKU_ID") - if sku in names: - duplicates.add(sku) - names[sku] = (row.get("product_name") or "").strip() - if duplicates: - raise RebuildError("duplicate CSV row(s): " + ", ".join(sorted(duplicates))) - return fieldnames, rows, names - - -def _session_values(payload: Any) -> Set[str]: - """Every session id anywhere in a manifest, whatever its shape.""" - found: Set[str] = set() - if isinstance(payload, dict): - for key, value in payload.items(): - if key in SESSION_KEYS and isinstance(value, (str, int)): - found.add(str(value)) - elif key in SESSION_LIST_KEYS and isinstance(value, list): - for item in value: - found.update([str(item)] if isinstance(item, (str, int)) else _session_values(item)) - elif isinstance(value, (dict, list)): - found.update(_session_values(value)) - elif isinstance(payload, list): - for item in payload: - found.update(_session_values(item)) - return found - - -def load_eval_sessions(path: str) -> Set[str]: - """Sessions an evaluation used. A manifest naming none is refused, not trusted.""" - with open(path, encoding="utf-8") as handle: - sessions = _session_values(json.load(handle)) - if not sessions: - raise RebuildError(f"{path}: evaluation manifest names no capture session") - return sessions - - -def load_capture_sessions(path: Optional[str]) -> Dict[str, str]: - """Image basename -> capture session, from an enrollment manifest.""" - if not path: - return {} - with open(path, encoding="utf-8") as handle: - payload = json.load(handle) - if isinstance(payload, dict): - entries = payload.get("images", []) - default = payload.get("capture_session_id") or payload.get("session_id") - else: - entries, default = payload, None - if not isinstance(entries, list): - raise RebuildError(f"{path}: 'images' must be a list") - sessions: Dict[str, str] = {} - for entry in entries: - if not isinstance(entry, dict): - continue - filename = entry.get("file") or entry.get("filename") or entry.get("image") - session = entry.get("capture_session_id") or entry.get("session_id") or default - if filename and session is not None: - sessions[os.path.basename(str(filename))] = str(session) - return sessions - - -def audit( - source: str, - min_views: int = MIN_VIEWS, - names: Optional[Dict[str, str]] = None, - ocr_engine: Any = None, - capture_sessions: Optional[Dict[str, str]] = None, - eval_sessions: Optional[Set[str]] = None, -) -> Tuple[Dict[str, List[str]], List[Dict[str, str]]]: - """Keep the views that pass, and report every refusal with its reason.""" - kept: Dict[str, List[str]] = {} - refused: List[Dict[str, str]] = [] - capture_sessions = capture_sessions or {} - eval_sessions = eval_sessions or set() - for sku, views in sorted(collect(source).items()): - accepted: List[Tuple[str, np.ndarray]] = [] - for name in views: - image = cv2.imread(os.path.join(source, name)) - if image is None: - refused.append({"sku": sku, "view": name, "reason": "unreadable"}) - continue - # Leakage first: a view from an evaluation session is refused even if - # it would fail another check, so the leak is never hidden by one. - session = capture_sessions.get(name) - if session and session in eval_sessions: - refused.append({"sku": sku, "view": name, - "reason": f"capture session {session} is in the evaluation manifest"}) - continue - if image.shape[0] < MIN_HEIGHT: - refused.append({"sku": sku, "view": name, - "reason": f"{image.shape[0]}px tall, below the {MIN_HEIGHT}px floor"}) - continue - if image.shape[1] < MIN_WIDTH: - refused.append({"sku": sku, "view": name, - "reason": f"{image.shape[1]}px wide, below the {MIN_WIDTH}px floor"}) - continue - if ocr_engine is not None and names is not None: - try: - text = " ".join(line.text for line in ocr_engine.read(upscale(image)).lines) - except OcrUnavailable as error: - print(f"OCR skipped for {name}: {error}", file=sys.stderr) - else: - similarity = name_ocr_similarity(names.get(sku, ""), text) - if similarity < MIN_NAME_OCR: - refused.append({"sku": sku, "view": name, - "reason": f"name/OCR {similarity:.2f} below {MIN_NAME_OCR:.2f}: " - f"{text[:100] or ''}"}) - continue - duplicate = next((other for other, pixels in accepted - if image.shape == pixels.shape and cosine(image, pixels) >= DUPLICATE_COSINE), None) - if duplicate: - refused.append({"sku": sku, "view": name, - "reason": f"near-duplicate of {duplicate}"}) - continue - accepted.append((name, image)) - if len(accepted) < min_views: - for name, _ in accepted: - refused.append({"sku": sku, "view": name, - "reason": f"only {len(accepted)} usable view(s), {min_views} required"}) - continue - kept[sku] = [name for name, _ in accepted] - return kept, refused - - -def write_catalog(kept: Dict[str, List[str]], source: str, output: str, csv_path: Optional[str]) -> None: - """Copy the kept views and the whole CSV into a staging dir, then swap it in.""" - staging = tempfile.mkdtemp(prefix=".catalog-", dir=os.path.dirname(os.path.abspath(output)) or ".") - try: - for views in kept.values(): - for name in views: - shutil.copy2(os.path.join(source, name), os.path.join(staging, name)) - if csv_path: - # Byte-for-byte, not re-serialized: gallery.catalog_digest hashes the - # raw file, and csv.DictWriter's default CRLF would move the digest - # and invalidate a trained identity head over a newline. - shutil.copy2(csv_path, os.path.join(staging, "stock.csv")) - if os.path.isdir(output): - # The documented rebuild runs with --source == --output, and the - # catalog directory also holds provenance this tool does not manage - # (README, references/rejected/, label files). Carry those over so a - # rebuild replaces the views and the CSV, not the record of where - # they came from. - for entry in os.listdir(output): - if entry == "stock.csv" or os.path.splitext(entry)[1].lower() in IMAGE_EXTENSIONS: - continue - origin, target = os.path.join(output, entry), os.path.join(staging, entry) - if os.path.isdir(origin): - shutil.copytree(origin, target) - else: - shutil.copy2(origin, target) - if os.path.exists(output): - shutil.rmtree(output) - os.replace(staging, output) - except BaseException: - shutil.rmtree(staging, ignore_errors=True) - raise - - -def main(argv: Optional[Sequence[str]] = None) -> int: - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--source", required=True, help="directory of pack shots") - parser.add_argument("--output", required=True, help="catalog directory to write") - parser.add_argument("--csv", help="stock.csv to validate and carry over whole") - parser.add_argument("--replace", action="store_true", help="overwrite an existing output") - parser.add_argument("--min-views", type=int, default=MIN_VIEWS, - help=f"views required per SKU (default {MIN_VIEWS}; " - "official pack-shot pages publish one front view per JAN)") - parser.add_argument("--ocr", choices=("auto", "vision", "none"), default=None, - help="packshot/name OCR backend (default: DORAYAKI_OCR, else vision " - "on macOS; an unavailable backend is skipped, not fatal)") - parser.add_argument("--source-manifest", - help="image capture metadata (default: SOURCE/manifest.json when present)") - parser.add_argument("--eval-manifest", - help="manifest of evaluation sessions; images captured in them are refused") - parser.add_argument("--json", help="write the audit here") - args = parser.parse_args(argv) - - if os.path.exists(args.output) and not args.replace: - raise SystemExit(f"{args.output} exists; pass --replace to overwrite") - - try: - if args.csv: - _, _, names = read_catalog(args.csv) - missing_rows = set(collect(args.source)) - set(names) - if missing_rows: - raise RebuildError("enrolled image SKU(s) with no CSV row: " + ", ".join(sorted(missing_rows))) - else: - names = {} - print("no --csv: image/CSV match and name/OCR checks skipped", file=sys.stderr) - - source_manifest = args.source_manifest - default_manifest = os.path.join(args.source, "manifest.json") - if source_manifest is None and os.path.isfile(default_manifest): - source_manifest = default_manifest - capture_sessions = load_capture_sessions(source_manifest) - eval_sessions = load_eval_sessions(args.eval_manifest) if args.eval_manifest else set() - if eval_sessions: - unlabelled = sorted(name for views in collect(args.source).values() for name in views - if name not in capture_sessions) - if unlabelled: - print("eval-session check cannot cover image(s) with no capture_session_id: " - + ", ".join(unlabelled), file=sys.stderr) - - engine = build_ocr_engine(args.ocr) - ocr_engine = engine if engine.available() else None - if ocr_engine is None: - print("name/OCR check skipped: no OCR backend available", file=sys.stderr) - kept, refused = audit(args.source, min_views=args.min_views, names=names, - ocr_engine=ocr_engine, capture_sessions=capture_sessions, - eval_sessions=eval_sessions) - if refused: - print(f"refused {len(refused)} view(s):", file=sys.stderr) - for row in refused: - print(f" {row['sku']:<10} {row['view']:<20} {row['reason']}", file=sys.stderr) - if not kept: - raise RebuildError("no SKU has usable views; nothing written") - write_catalog(kept, args.source, args.output, args.csv) - except (OSError, ValueError, json.JSONDecodeError, RebuildError, OcrUnavailable) as error: - raise SystemExit(str(error)) from error - - # A row with no kept image is a text-only entry: the text lane still names - # and vetoes it from the CSV alone, so it is a coverage gap, not a refusal. - text_only = sorted(set(names) - set(kept)) - - report = { - "source": args.source, - "output": args.output, - "min_width": MIN_WIDTH, - "min_height": MIN_HEIGHT, - "min_views": args.min_views, - "duplicate_cosine": DUPLICATE_COSINE, - "min_name_ocr": MIN_NAME_OCR, - "ocr": engine.name if ocr_engine is not None else "skipped", - "text_only_skus": text_only, - "skus": {sku: views for sku, views in sorted(kept.items())}, - "refused": refused, - } - if args.json: - with open(args.json, "w", encoding="utf-8") as handle: - json.dump(report, handle, indent=1, ensure_ascii=False) - - print(f"{len(kept)} SKU(s), {sum(len(v) for v in kept.values())} view(s) -> {args.output}") - for sku, views in sorted(kept.items()): - print(f" {sku:<10} {len(views)} view(s): {', '.join(views)}") - if text_only: - print(f"\ntext-only CSV row(s), no enrolled image ({len(text_only)}): {', '.join(text_only)}") - if refused: - print(f"\nrefused {len(refused)} view(s):") - for row in refused: - print(f" {row['sku']:<10} {row['view']:<20} {row['reason']}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index 49cc070..fc70c08 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -32,12 +32,9 @@ from fastapi.testclient import TestClient # noqa: E402 def make_app(*, token=None, max_sessions=4, detector=None): - # The fake matcher is what these tests exercise, so the backend is pinned to - # `matcher`; the shipped default is the `local` lane, which reads real - # reference images and would ignore the fake. engine = RecognitionEngine( detector or FakeDetector(), FakeMatcher(), FakeInventory(), - settings=EngineSettings(classify_workers=2, identity_backend="matcher"), + settings=EngineSettings(classify_workers=2), ) settings = ServerSettings(token=token, max_sessions=max_sessions) app = create_app(engine, settings, web_dir="/nonexistent") diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index 9832991..7eb686f 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -131,10 +131,6 @@ class FakeInventory: def make_engine(**overrides): - # The fake matcher is what these tests exercise, so the backend is pinned to - # `matcher`; the shipped default is the `local` lane, which reads real - # reference images and would ignore the fake. - overrides.setdefault("identity_backend", "matcher") settings = EngineSettings(device="cpu", classify_workers=4, **overrides) detector = FakeDetector() matcher = FakeMatcher() @@ -199,25 +195,6 @@ class EngineClassificationTests(unittest.TestCase): self.assertEqual(rows[0]["identity"]["state"], "recognized") self.assertEqual(rows[0]["identity"]["sku_id"], "sku_a") - def test_a_failing_crop_reports_unrecognized_not_pending(self): - """One bad crop must not fail the batch, and must not read as pending. - - `identity_payload(None)` is the "not yet classified" shape; an - operational failure has to stay distinguishable from absence. - """ - engine, _, matcher, _ = make_engine() - - def explode(crop, method=None): - raise RuntimeError("classifier blew up") - - matcher.score_crop = explode - rows = engine.classify_boxes(make_image(), [ - {"id": 0, "box": [10, 10, 60, 90], "confidence": 0.9}, - ], method="hybrid") - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0]["identity"]["state"], "unrecognized") - self.assertIsNone(rows[0]["identity"]["sku_id"]) - def test_parallel_classification_matches_sequential(self): engine, _, matcher, _ = make_engine() boxes = [ @@ -322,90 +299,6 @@ class GalleryLoadTests(unittest.TestCase): self.assertIs(engine._gallery, False) -class LocalLaneTests(unittest.TestCase): - """The shipped default backend, which the fake matcher cannot exercise. - - A two-SKU catalog of real images is built in a temp dir, so the lane reads - genuine reference files and the decision is the one production makes. - """ - - def make_catalog(self): - import tempfile - - import cv2 - import numpy as np - - root = tempfile.mkdtemp(prefix="lane-catalog-") - rng = np.random.default_rng(7) - for sku, base in (("sku_a", (200, 60, 60)), ("sku_b", (60, 60, 200))): - image = np.zeros((400, 300, 3), dtype="uint8") - image[:, :] = base - # Structured, high-contrast marks: SIFT needs corners, not noise. - for index in range(12): - x = int(rng.integers(10, 280)) - y = int(rng.integers(10, 380)) - cv2.rectangle(image, (x, y), (x + 40, y + 40), (255, 255, 255), 3) - cv2.putText(image, sku[-1].upper(), (x + 5, y + 30), - cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2) - cv2.imwrite(os.path.join(root, f"{sku}.jpg"), image) - with open(os.path.join(root, "stock.csv"), "w", encoding="utf-8") as handle: - handle.write("SKU_ID,Product_Name,Price,Number_In,Number_Out,Number_On\n") - handle.write("sku_a,Alpha,100,1,0,1\n") - handle.write("sku_b,Beta,200,1,0,1\n") - return root - - def test_decide_crop_names_the_matching_sku(self): - import cv2 - - root = self.make_catalog() - engine, _, _, _ = make_engine(identity_backend="local", stock_dir=root) - try: - self.assertEqual(engine.identity_scale, "local") - self.assertIsNotNone(engine.identity_lane()) - # The reference itself is the easiest possible query for its own lane. - crop = cv2.imread(os.path.join(root, "sku_a.jpg")) - decision = engine.decide_crop(crop) - self.assertEqual(decision.get("best_sku"), "sku_a") - self.assertEqual(decision.get("identity_level"), "accepted") - finally: - engine.shutdown() - - def test_score_crop_is_keyed_by_canonical_sku(self): - import cv2 - - root = self.make_catalog() - engine, _, _, _ = make_engine(identity_backend="local", stock_dir=root) - try: - scores = engine.score_crop(cv2.imread(os.path.join(root, "sku_b.jpg"))) - self.assertTrue(scores) - # No view-suffixed ids: every key is a catalog SKU. - self.assertTrue(all("_" not in sku[4:] for sku in scores), scores) - finally: - engine.shutdown() - - def test_a_featureless_crop_is_not_named(self): - import numpy as np - - root = self.make_catalog() - engine, _, _, _ = make_engine(identity_backend="local", stock_dir=root) - try: - blank = np.zeros((200, 150, 3), dtype="uint8") - decision = engine.decide_crop(blank) - self.assertIsNone(decision.get("best_sku")) - finally: - engine.shutdown() - - def test_warm_up_builds_both_lanes(self): - root = self.make_catalog() - engine, _, _, _ = make_engine(identity_backend="local", stock_dir=root) - try: - engine.warm_up() - # Built eagerly, so the first live frame never pays for them. - self.assertIsNotNone(engine._lane) - self.assertIsNotNone(engine._text_lane) - finally: - engine.shutdown() - class CatalogImageTests(unittest.TestCase): def test_catalog_image_path_is_resolved_through_inventory(self): diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index b8ea38f..a4dbf7f 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -129,12 +129,8 @@ class FakeInventory: def make_session(matcher=None, settings=None, engine_cls=RecognitionEngine): detector = FakeDetector() matcher = matcher or FakeMatcher() - # These fixtures inject a fake matcher, so they exercise the `matcher` - # backend explicitly: the shipped default is the `local` lane, which reads - # real reference images and would ignore the fake. engine = engine_cls(detector, matcher, FakeInventory(), - settings=__import__("engine").EngineSettings( - classify_workers=2, identity_backend="matcher")) + settings=__import__("engine").EngineSettings(classify_workers=2)) session = LiveSession("test", engine, settings=settings or LiveSettings(method="hybrid")) session.refresh_catalog() return session, engine, detector, matcher @@ -260,10 +256,7 @@ class CosineScaleTests(unittest.TestCase): def test_a_close_cosine_pair_stays_ambiguous(self): """The margin gate still applies: two near-equal scores must not name a SKU.""" session, engine, _, _ = make_session(FakeMatcher(score=0.45), engine_cls=CosineEngine) - # The matcher contract is (scores, details); a bare dict unpacks into two - # strings and raises inside the future callback, where the failure is - # swallowed and the test passes for the wrong reason. - engine.matcher.score_crop = lambda crop, method=None: ({"sku_a": 0.45, "other": 0.44}, {}) + engine.matcher.score_crop = lambda crop, method=None: {"sku_a": 0.45, "other": 0.44} for frame_id in range(1, 8): process(session, frame_id) result = process(session, 9) @@ -615,8 +608,7 @@ def make_crop_session(matcher=None, settings=None, detector=None): detector = detector or SmallBoxDetector() engine = RecognitionEngine( detector, matcher or FakeMatcher(), FakeInventory(), - settings=__import__("engine").EngineSettings( - classify_workers=2, identity_backend="matcher"), + settings=__import__("engine").EngineSettings(classify_workers=2), ) session = LiveSession("crop", engine, settings=settings or LiveSettings(method="hybrid")) session.refresh_catalog() diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md b/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md deleted file mode 100644 index 4fc9d4f..0000000 --- a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md +++ /dev/null @@ -1,610 +0,0 @@ -# Dorayaki product recognition: architecture decision - -Audited 2026-09-28 · submodule `25dbf13` on `fix/live-recognition-quality` plus -its uncommitted diff (snapshot: `git stash create` → `2785c07`) · auditor: -Claude Opus 5.5 -Scope: identity pipeline (matcher, gallery, decision, evaluator, catalogs, -labels), live track/label logic, the uncommitted diff. Not audited: detector -quality, tag/planogram pipeline, iOS/Android rendering, on-device runtime. -Evidence scripts and the corrected labels: [`evidence/`](evidence/README.md). - -## Verdict - -Build a **local-feature instance matcher plus a text lane, behind a calibrated -open-set decision, with no global embedding in the critical path**. At the -current scale the matcher runs exhaustively; past ~150 SKUs, a -RootSIFT descriptor index replaces exhaustive matching. Generic DINOv2 is -rejected for both classification and retrieval. - -The biggest risk is none of the architectures. It is that **there is no -trustworthy benchmark**. The only labelled data is one photo, `add/image6.jpg`, -and 26 of its 111 labels (23 %) were wrong. Every number in the handoff was -computed against those labels. Label and catalog errors are larger than any -architecture difference measured so far. They also hide the real product -failure: displayed-label precision is 29–43 % because same-brand siblings the -catalog doesn't know about get labelled with confidence. - -On corrected labels, with thresholds tuned on the same single photo (so read -this as direction, not proof), the recommended design beats Van's contract on -coverage and on precision together: - -| Live resolution, clean 7-SKU catalog, 61 in-catalog + 48 unknown facings | Correct shown | Coverage (of 61) | Precision of shown labels | Unknowns labelled | -|---|---|---|---|---| -| Van's matcher, always argmax (the floor) | 35 | 57 % | 35 % | 48 / 48 | -| Current server gate (`decide_identity` defaults) | 32 | 52 % | 43 % | 22 / 48 | -| Improved matcher, geometric-confidence gate only (τ=2) | 26 | 43 % | 79 % | — | -| **Improved matcher + OCR veto + OCR lane (τ=2)** | **36** | **59 %** | **95 %** | **0 / 48** | -| Same, looser (τ=1) | 40 | 66 % | 87 % | 2 / 48 | - -The one experiment that would most change this recommendation is a second, -independently captured and labelled shelf visit. §7 defines it. - ---- - -## 1. Problem formulation - -Dorayaki's identity task is **open-set instance recognition over a small, -store-scoped, enrolled catalog, with live-video evidence accumulation**. - -- **Instance, not category.** The classes are individual printed artworks - (JAN-level SKUs). Siblings share maker, palette, lattice pattern, mascot and - silhouette, and differ by a word ("コクうま 65%"), a colour (pink vs red - lattice), or a small logo. That is the regime of planar instance matching - (logo and text geometry), not of semantic embeddings. Measured: raw DINOv2 is - 12–21 % top-1 on 7 SKUs; the local matcher is 76–98 %. -- **Retrieval-shaped at scale, closed-set-shaped today.** With tens of SKUs, - every enrolled reference can be verified exhaustively. With thousands, it - becomes retrieval plus verification (§6). -- **Open set is the dominant difficulty, and it comes in two kinds.** *Far* - unknowns (other categories) are rejected easily: 0.2–0.7 % false accepts on - 602 unrelated crops at a threshold that keeps most correct labels. *Near* - unknowns (siblings of enrolled products: Kewpie 450 g next to Kewpie からし, - Pure Select plain next to Pure Select コクうま) produce genuine, - geometrically consistent evidence and can't be thresholded away. Near - unknowns get fixed by **catalog completeness** and by **text**, not by a - better score. -- **Multimodal by nature.** Text is the designed discriminator between - siblings; manufacturers print the variant name precisely so shoppers can - tell them apart. OCR identifies the right SKU among all 12 catalog rows on - 41–42 of the 55 facings where it reads anything, including SKUs with no - enrolled image, using only catalog names and packshot text. -- **Temporal.** Live video gives many cheap, noisy looks at one physical - facing. The right model is accumulated evidence per track, with a - provisional → confirmed label contract. The current model is one accepted - observation that locks the label forever. - -So the hierarchy is: store/category scope → candidate generation (trivial -now, descriptor index later) → local geometric verification + text -verification → calibrated open-set decision → temporal accumulation. - -## 2. Evidence audit - -`R` reliable · `C` directionally useful but confounded · `X` invalidated · -`U` still unknown. "Reproduced" means re-run in this audit with the scripts in -`evidence/`. - -| # | Handoff claim | Verdict | What this audit found | -|---|---|---|---| -| 2 | Van's matcher is SIFT/ORB/BRISK + RANSAC, always argmax, no rejection | R | Scoring core in `code/feature_matcher.py` is byte-for-byte Van's (`git diff b443e3c`); only `decide_identity` was added. "SURF" is SIFT (SURF isn't in this OpenCV build). | -| 2 | "170/170 named ≠ 170/170 correct" on image.jpg | R (logic) / U (numbers) | No labels exist for `image.jpg`; its accuracy was never measured. | -| 3 | Current server collapsed onto hub SKUs (sku_003, sku_l09) | C | Hubness is real for DINO (50–60 % max-SKU share on the clean catalog, reproduced). The l09 collapse was mostly a wrong l09 reference plus shelf-cut templates. | -| 3 | Live resolution degrades identity | X | Controlled rerun: with correctly scaled boxes, live is not worse than full (original matcher 35/42 live vs 32/42 full; improved 36 vs 40). | -| 3 | DINO embeddings are inappropriate for fine-grained SKU identity | R | Reproduced: gallery top-1 21 % full / 12 % live, hub alarm. | -| 4 | `stock-lopia` was corrupt, leaked, and incomplete | R | Worse than reported: its crops come from the eval photo, so scoring against it is memorization. It scores 94–97 % "identity", and the evaluator's leak check misses it because the re-cut crops aren't the same size. | -| 7 | `eval_identity.py` is the right research direction | C | Right skeleton, four more defects (F-07). | -| 8 | Bug #1, template IDs vs SKU IDs | R, and **still live in production** | Fixed only in the evaluator. The live path accepts `sku_l01_1`, then finds no catalog row for it (F-02). | -| 9 | Hybrid 80 % > matcher 70 % | X | About one facing, on wrong labels. | -| 10 | Bug #2, unscaled boxes caused the live collapse | R | Reproduced on *both* catalogs: unscaled boxes give 11–17 %, scaled boxes give 76–97 %. The confound is resolved. The coordinate fix, not the catalog cleanup, removed the collapse. | -| 11 | Clean packshots are a dramatic improvement | R, with a caveat | True against the leak-free baseline. The l06 packshot shows a superseded package design (F-12); the l09 packshot is only 305 px tall. | -| 12 | Bug #3, the reject gate inside the shortlist | R | Hybrid-49 %-vs-20 % result is invalid. | -| 13 | DINO top-K sweep: recall 71 %/100 %, K doesn't hurt the matcher | C | On a 7-SKU catalog, top-5 is 71 % of the catalog; that sweep can't speak to retrieval. Reproduced on v2: top-3 recall 79–81 %, top-5 95–98 %. | -| 14 | Matcher ~8 ms vs hybrid ~45 ms | R, with a correction | The hybrid lane scores the **full** catalog and then filters (`eval_identity.py` `HybridLane.rank`), so its latency is DINO + full matcher by construction. The measurement is honest; it doesn't describe a real retrieval design. | -| 15 | 69 % full / 66 % live after the l09 fix | R as arithmetic, X as a measurement | Reproduced exactly (24/35, 23/35) **on v1 labels**. On corrected labels the same matcher and catalog give 71 %/76 % after the gate and 76 %/83 % ungated top-1 ranking. The 69/66 gap that justified `crop_first_attempt` doesn't exist. | -| 16 | Local matcher is the strongest identity primitive | R | And improvable: multi-scale RootSIFT + colour re-rank 40/42 vs 32/42 at full (sign test p = 0.04); at live 36 vs 35 (noise). | -| 16 | DINO as coarse retriever at scale | X (this setting) | Recall@10 falls 42 → 32 → 17 → 7 of 42 as the catalog grows 7 → 57 → 207 → 493 SKUs. A RootSIFT descriptor index holds 37–38/42 recall@5 over the same range. | -| 17 | Threshold alone can't create correct identities | R | And the missing identities are mostly catalog gaps (§3). | -| 17 | Argmax precision ~20 % | C | 29 % full / 35 % live on corrected labels; the old number counted mislabelled facings. | -| 21 | Live tests: 49 tests, 1 failure, 5 errors | X (stale) | `python -m unittest test_live` now passes 49/49 on both the dirty tree and the committed baseline. One test feeds a `str` result that raises inside a future callback and is swallowed, on both trees (F-13). | -| 22 | "High confidence" list items 1–8 | R | All hold. Item 4 is now measured with corrected labels. | -| 22 | Medium confidence 1 (69 %/66 %) | X | See row 15. | -| 22 | Medium confidence 3 (more views help) | R direction | Leave-facing-out proxy (§3, E-5) supports it strongly, with a same-photo caveat. | -| 22 | Not established 1–9 | U | 1 and 4 are now partly answered (§6); 2 partly (RootSIFT/multi-scale beat Van's composite); 3, 5, 6, 7, 8, 9 remain open. Item 9 (ground truth) was the most consequential open item: it was wrong. | - -### The four ceilings, separated - -On corrected labels (`evidence/image6_v2.json`): 109 scored facings, 61 -in-catalog across 11 SKUs, 48 unknown, 2 excluded. The clean catalog enrolls 7 -SKUs covering 42 of the 61 in-catalog facings. - -| Ceiling | Value (live, improved matcher) | What limits it | -|---|---|---| -| A. Catalog | 42 / 61 = 69 % of in-catalog facings are answerable by image | l10 (11 facings), l07 (6), l04, l08 unenrolled; l12 has **zero** real facings on the shelf | -| B. Retrieval | n/a at 7 SKUs; RootSIFT index recall@5 = 38/42 at 493 SKUs | the 4 misses are catalog-quality cases (l06 package version) | -| C. Identity (ungated top-1 on enrolled) | 36 / 42 (86 %, Wilson 72–93 %) | l06 at live (1/4), two l09 facings that are mostly occluded | -| D. Decision | 26 correct at 79 % precision (geometry only); 36 at 95 % (+ text) | near-unknown siblings: 9 of 11 unenrolled l10 facings → l09, Kewpie 450 g → l03 | - -## 3. Root-cause model - -Ordered by how much of the gap to a good product each class explains. - -**Evaluator and ground-truth problems** (they decided every conclusion so far) -- 23 % of v1 labels were wrong: 14 in-catalog facings marked unknown (Egg Care - #4–5, Fitte #22–23, Lucky #27–28, Kokuuma #40–45, Pure Select #59); 9 - non-catalog items labelled as catalog SKUs (demi-glace pouches #31–32 as l09, - a garlic jar #16 as l05, a Heinz can #64 as l10, dressing bottles #76–79 as - l12, Kewpie 450 g #85 as l03); 2 neighbour shifts (#29 l08→l07, #75 - l12→l11). The pattern (runs shifted along a shelf row, a `match` IoU field on - every row) looks like labels propagated from shelf-tag or box matching rather - than read from the packages. -- The evaluator reports `top1` and `coverage` as the same quantity (accepted - correct / enrolled), so the identity ceiling and the decision ceiling were - never separated. -- Precision excludes unenrolled in-catalog facings, which are the most - dangerous false accepts. -- The gallery's candidate set comes from ground-truth labels. -- Leak detection only catches same-size crops. - -**Catalog and data problems** (the largest recognition lever) -- Incomplete category enrollment is what creates near unknowns. In the - leave-facing-out proxy (E-5), adding one shelf view for each of the 11 SKUs - on the shelf takes argmax from 34/52 to 47/52. The number that survives at - 95 % precision goes from 14–16 to 32–39. -- Wrong or stale references: the l09 history; the l06 package is a superseded - design (the match holds at full resolution, fails at live: 1/4); the l09 - packshot is 305 px, so its scores dwarf others under Van's formula. -- Metadata integrity: `stock-lopia-clean/stock.csv` has no `sku_l09` row for - `sku_l09.jpg`. Its `sku_l10` row carries l09's stock counts with l10's name - and price. The catalog name for l12 ("焼肉のたれ 中辛", ¥299) probably - names the ユーラス 旨たれ bottles on the shelf (tag ¥299). Several catalog - sizes disagree with the shelf tags (l10 tag says 400 g). - -**Identity-model problems** (real but second-order on this data) -- Scale mismatch: ~1000 px packshots vs 110–250 px crops is a 4–8× ratio. - ORB's 8×1.2 pyramid tops out at 3.6×, so ORB contributes **zero** inliers at - live resolution, and BRISK nearly none. Van's weighted composite is - effectively SIFT-only there, carrying dead weight. -- Van's score, `inliers·1.5 + inliers/min(kp)·80`, isn't comparable across - templates of different resolution. With fewer than 4 matches it counts raw - ratio-test matches as "inliers" without any geometric check (`compute_geometric_inliers`). -- Grayscale-only: pink (ゼロ) vs red (ハーフ) lattice is invisible to SIFT. The - hue re-rank fixes 2–3 facings in every configuration. - -**Open-set decision problems** -- `decide_identity` for the inlier scale accepts a score ≥ 1.0 with a relative - margin ≥ 0.15. A single unverified match clears the floor, and the margin - can't reject a sibling with a strong partial match. It accepts 22–28 of 48 - unknowns. - -**Live and tracking problems** (code-path evidence, unmeasured on device) -- A confirmed track is never re-evaluated (`_on_classified` returns early when - `track.confirmed`), and with `vote_agree = 1` the first accepted observation - locks it. A wrong first label is permanent. A later, sharper, correct view - can't correct it. -- Aggregation is the mean over the last three observations, with empty - observations counted as zeros. That's a heuristic, not evidence - accumulation. -- Uncommitted diff: the crop blur gate returns without consuming an attempt, - so a permanently soft crop (a distant facing) is retried every `retry_gap` - frames for as long as it stays in view. - -**UX latency problems** -- No provisional label exists on the wire or in the clients, so any multi-look - confirmation becomes dead time on screen. That's why `vote_agree = 2` was - (correctly) backed out. - -## 4. Architecture candidates - -"Recognition" = does it improve correct shown labels at fixed precision? -"Scalability" = does it keep latency and recall flat as the catalog grows? - -| ID | Candidate | Tested here? | Recognition | Scalability | Verdict | -|---|---|---|---|---|---| -| A | Van's matcher, full reference set, argmax | yes | 32–35 / 42 identity; 29–35 % precision | linear, ~0.1–0.25 ms per reference | Floor. Not shippable (no open set). | -| B | Matcher + engineering: RootSIFT, per-SKU reference pyramid (200/400 px tall), SKU grouping, similarity-constrained RANSAC, hue feature, calibrated gate | yes | 40/42 full (p = 0.04 vs A), 36/42 live; gate alone reaches 79–85 % precision at 36–43 % coverage | same as A | **Adopt** as identity core. | -| C | Learned local features: ALIKED/DISK/SuperPoint + LightGlue; dense LoFTR/RoMa | no (weights not installed; downloading needs your approval) | Unknown. Headroom on enrolled facings is ≤ 6/42 at live, mostly catalog-caused. Likely strongest on small or blurred crops and package-version drift. | LightGlue is per-pair; tens of ms per pair on GPU and much more on CPU, so only viable as a top-K verifier | Postpone to experiment E-C. Adopt only if it clears the gate. | -| D | Retrieval → Van's matcher | yes (DINO) | DINO recall collapses with scale (7/42 @10 at 493 SKUs) | DINO adds ~41 ms per crop | **Reject DINO retrieval.** | -| D′ | RootSIFT descriptor index (FLANN) → vote → verify top-10 | yes | 35–37/42, equal to or slightly better than exhaustive | 1.7–3.9 ms per crop flat from 7 to 493 SKUs; build 171 ms for 64k descriptors | **Adopt past ~150 SKUs.** | -| E | Retrieval → learned verifier | no | — | — | Only if C wins. | -| F | OCR text lane against catalog text signatures | yes (macOS Vision) | Veto: precision 62 → 82 % at τ=0.5 for −1 correct. As a lane: +10 correct at live, including unenrolled SKUs. | ~18–26 ms per crop on Mac; on-device on iOS (Vision) and Android (ML Kit) | **Adopt.** | -| G | Fusion (geometry + colour + text + temporal + tag prior) | partly (geometry + colour + text) | Largest measured gain | — | **Adopt**; the text and geometry errors are complementary (text reads siblings apart; geometry covers text-poor faces). | -| H | Store-trained identity head on DINO | no (it was trained on the eval shelf) | Leaked; no valid measurement exists | — | Retire with DINO. | -| I | Barcode/JAN | — | Front faces rarely show JANs; shelf tags carry product text, not scannable codes, at scan distance | — | Only for the enrollment tool (scan a JAN, pull the packshot). | -| J | Shelf-tag / planogram prior | no (`tags.py` exists) | Every facing run on image6 sits above a readable tag with name and price | — | Strong hypothesis, H-02. | - -## 5. Recommended architecture (current scale: tens of SKUs) - -``` -frame / native crop - │ - ├─ local lane: RootSIFT on crop ─► verify against EVERY reference view - │ (exhaustive while refs ≲ 300), similarity-RANSAC, - │ per-SKU evidence = max over views: - │ {inliers, 2nd-best ratio, inlier hull cover, hue agreement} - │ - ├─ text lane: OCR (upsampled ×2 when crop height < 200 px) - │ ─► bigram similarity to every catalog row's text signature - │ (name + OCR of its packshot), including rows with no image - │ - ▼ -decision: P(correct | local, text) from a small calibrated model - accepted if P ≥ p_confirm (target precision ≥ 95 %) - provisional if P ≥ p_provisional (shown as tentative) - veto if text names a different catalog row by margin - unknown otherwise - │ - ▼ -track: log-odds accumulation per track across looks; confirm on - threshold or text agreement; re-open on strong contradiction - (hysteresis), never on a single weak frame -``` - -Why each piece earns its place: -- **Local lane.** It's the only primitive that ranks fine-grained siblings - correctly: 36–40 of 42 enrolled facings, against 5–9 for DINO. It costs - ~4–10 ms per crop at 7 SKUs. -- **Reference pyramid (200/400 px) and RootSIFT.** Together they fix the scale - mismatch that kills ORB/BRISK and makes scores comparable across references. - Drop ORB/BRISK unless ablation H-08 finds a case they win. -- **Hue feature.** It's the cheapest fix for colour-coded variants (+2–3 - facings everywhere). -- **Text lane.** It's the only measured mechanism that rejects near-unknown - siblings, and it can name catalog SKUs that have no image yet. It - contributed +10 correct labels and 0 unknown accepts at live, τ=2. -- **Calibrated decision.** It replaces a hand-set floor that accepts one - unverified match. It's calibrated on Tier-2 data, never on the photo it's - evaluated on. -- **No DINO, no retrieval stage.** Neither adds recognition at this scale, and - DINO actively fails at larger scale. Exhaustive verification is cheaper than - one DINO forward pass until ~300–400 references. - -Placement: keep the matcher server-side for now; it's small and deterministic. -Put OCR on the phone when tag and label OCR move there. Apple Vision and ML -Kit are the official platform paths, matching the repo's `platform/` rule. -Send the text with the crop so the server fuses both. Until then, run OCR on -the server (macOS Vision is already wired in `code/ocr.py`). - -## 6. Scaling architecture - -Measured cost and recall to ~500 SKUs; beyond that, extrapolated and marked. - -| Catalog | Candidate generation | Verification | Evidence | -|---|---|---|---| -| 10–150 SKUs | none (exhaustive) | local + text | 3.7 ms at 7 SKUs, 7.7 ms at 57, 21 ms at 207 (small refs). Packshot pyramids cost ~2–3× per reference. | -| 150–5,000 | RootSIFT descriptor index (FLANN now; FAISS IVF-PQ when memory matters), vote per SKU, top-10 | local + text on the shortlist | Measured flat 2–4 ms and recall@5 38/42 up to 493 SKUs. Descriptor memory at 5k SKUs × 2 views × ~500 descriptors ≈ 5 M × 128 B (uint8) ≈ 640 MB before PQ. **Extrapolated.** | -| 5,000–100,000 | Scope first: store assortment → aisle/bay (tag or planogram prior) cuts candidates to hundreds. Then an aggregated-local-feature index (ASMK/VLAD-style) or a product-instance embedding fine-tuned on packshot↔shelf pairs. | local + text on ≤ 20 | **Hypothesis.** No store stocks 100k SKUs at once. Scoping by store and bay is the real scaling lever. Generic DINO is ruled out; a fine-tuned embedding needs paired training data that doesn't exist yet. | - -Exhaustive matching stops being sensible when its p95 exceeds the per-crop -budget. At ~0.1–0.25 ms per reference and a 30 ms budget, that's roughly -150–300 references (≈ 75–150 SKUs at 2 views). **Descriptor indexing removes -the need for a global DINO pass**: at 493 SKUs it retrieves better than DINO -at a tenth of the cost. The open question is same-category growth: hundreds of -mayonnaises instead of other-category distractors. That's experiment E-6. - -## 7. Experimental plan - -Rule for every experiment: change one variable, run on fixed crops and a fixed -catalog hash, log per facing (truth, enrolled?, per-SKU evidence vector, truth -rank, decision, lane latencies), and report the four ceilings separately. - -### Benchmark tiers (build these first; they block every other decision) - -- **Tier 1, diagnostic:** `evidence/image6_v2.json` (109 facings). Dev only. - Free to tune on. Its numbers never ship. -- **Tier 2, held-out regression:** new captures that are never enrolled and - never tuned on. At least 3 visits across ≥ 2 LOPIA locations or days, same - aisle, both still photos and the phone's own 1920×1080 frames. Target ≥ 300 - in-catalog facings over ≥ 20 SKUs, ≥ 300 near unknowns from the same aisle, - and the 602-crop far-unknown set from `add/`. - - Labels need provenance: shelf-tag correspondence plus readable package - text, two independent passes, disagreements marked `exclude`. - - Every label row carries `source`, `confidence`, and `labeler`. A label - file without them fails a schema check. -- **Tier 3, live replay:** FrameRecorder walks (`code/record.py`) of the same - aisles with Tier-2-style labels, replayed through `code/eval_live.py`. - -### Experiments - -| ID | Question | Variable | Metric | Acceptance / falsifier | -|---|---|---|---|---| -| E-0 | Are the labels right? | none | Double-labelling agreement on Tier 2 | κ ≥ 0.9 on SKU identity. Otherwise fix labels before anything else. | -| E-1 | Does B beat A on held-out data? | matcher variant | Paired identity top-1 on enrolled facings, McNemar | B ≥ A with p < 0.05 at native-crop resolution. If A ≥ B, keep Van's scoring and add only SKU grouping and the decision layer. | -| E-2 | Is ORB/BRISK dead weight? | drop ORB/BRISK from B | top-1, latency | Drop them if top-1 changes by ≤ 1 facing per 100. | -| E-3 | Does text move the frontier? | ± text lane | Correct shown at precision ≥ 95 %; near-unknown FAR | Text lane adds ≥ 5 pp coverage at 95 % precision. Otherwise text becomes veto only, or is removed. | -| E-4 | Calibrated decision vs hand threshold | decision policy | Coverage at 95 % precision, calibration (ECE) on Tier 2 | Model ECE ≤ 0.05 and coverage ≥ hand threshold's. | -| E-5 | How much is catalog completeness worth? | enroll the aisle's remaining SKUs with packshot + one shelf view from a *different* visit | Coverage at 95 % precision; near-unknown FAR | ≥ 70 % of the same-photo proxy gain (14→32 full, 16→39 live) survives cross-visit. If not, shelf views are overfitting to capture conditions. | -| E-6 | Same-category scaling | grow the catalog with other sauce/mayo SKUs (packshots) | top-1, recall@K of descriptor index vs exhaustive; latency p50/p95 | Index recall@10 ≥ 98 % of exhaustive top-1 at ≥ 300 SKUs. Otherwise stay exhaustive and parallelize. | -| E-C | Learned matcher | ALIKED+LightGlue (or DISK+LightGlue) as verifier over B's top-5 | top-1, coverage at 95 % precision, latency | Adopt only if +3 pp coverage at 95 % precision **and** p95 identity latency ≤ 80 ms server-side. | -| E-7 | Live temporal model | log-odds accumulation + provisional labels vs current single-shot lock | Tier 3: time to provisional, time to correct confirmed, flips, correct labels lost | Time-to-first-correct p50 ≤ current single-shot; flips ≤ 2 %; correct-then-lost ≤ 1 %. | -| E-8 | Does first-attempt native crop help? | `crop_first_attempt` on/off | Tier 3 correct confirmed labels, crop traffic | Keep only if +3 pp correct at ≤ 1.5× crop traffic. On Tier 1 it has no basis: live ≥ full. | -| E-9 | Tag prior | ± shelf-tag evidence from `tags.py` | Coverage at 95 % precision | Adopt if +5 pp; also use it for catalog-gap discovery. | - -## 8. Migration strategy - -Snapshot first: the uncommitted work is saved as stash object `2785c07` (and as -`tracked.diff` / `untracked.tgz` in the audit scratchpad). Nothing has been -reverted. - -| Item | Class | Action | -|---|---|---| -| `code/eval_identity.py` (untracked) | evaluation infrastructure | **Keep and fix** (F-07). | -| `gallery.py` `rank()` (diff) | evaluation infrastructure | Keep, eval-only; no production caller. | -| `run-demo-server.sh` `--identity matcher` (diff) | clearly correct production fix | **Keep.** Also change the default `STOCK` away from `stock-lopia` (F-03). | -| `live.py` `crop_first_attempt`, `max_attempts 2→4` | experimental | **Revert.** Its stated rationale (69 vs 66) is invalidated. Re-test as E-8. | -| `live.py` blur gate early return, `sharpen_ratio`, `best_sharpness` | experimental, with a retry hazard | **Revert.** Re-introduce inside the E-7 evidence model, where a soft crop is a low-weight observation, not a free retry. | -| `live.py` `vote_agree` + `_agreed_observations` | experimental | **Revert** (default 1 makes it dead code). E-7 replaces it. | -| `test_live.py` change | encodes the reverted budget | Revert with it. | -| `code/rebuild_catalog.py` (untracked) | catalog tooling | Keep. Add image↔CSV bijection, packshot-OCR vs name check, and eval-session refusal (enrollment protocol below). | -| `stock-lopia-clean/` (untracked) | catalog data | Keep the images and provenance. Fix `stock.csv` (F-04) and the README's "Six SKUs" (it lists five). Replace the l06 packshot with the current package. | -| `eval/labels/image6.json` (untracked) | ground truth | **Replace** with `evidence/image6_v2.json`, schema: `label`, `source`, `note`, `v1_label`. | -| `identity.decide_identity` inlier branch | production | **Rewrite** as the calibrated decision (E-4); keep the cosine branch only while the gallery exists. | -| Production score path (`engine.score_crop`, `classify_boxes`, `inspect`) | production | **Rewrite:** return per-SKU evidence grouped by canonical SKU (F-02). | -| `feature_matcher.py` composite score | production | **Rewrite** into variant B behind an interface; keep Van's scorer as a selectable baseline lane for E-1. | -| Identity interface | — | **Isolate:** `IdentityLane.evidence(crop) -> {sku: Evidence}` with `LocalLane`, `TextLane`, and `GalleryLane` (postponed); `DecisionPolicy` separate; `TrackEvidence` in live. | -| DINO gallery, identity head, `finetune_identity.py` | production code | **Postpone and fence off:** not in the live path, not the default; delete after E-6 if nothing depends on it. | -| Descriptor index | — | **Postpone** until the catalog passes ~150 SKUs (E-6). | -| Provisional label on the wire + clients | protocol + native UI | **Postponed dependency** of E-7. Shared model gets a `provisional` identity state; Compose and SwiftUI render it as tentative. | - -### Enrollment protocol - -- **Views per SKU:** at least one current-package front packshot, stored as a - 200/400/800 px-tall pyramid (source ≥ 800 px tall), plus at least one real - shelf capture at scan distance from a store visit that is **not** an - evaluation session. Add side or back views only for SKUs displayed rotated. -- **Minimum usable resolution:** reject any reference view under 160 px tall - at its largest pyramid level. Crops that small carry no text geometry - (observed: facings under 60 px wide fail every lane). -- **Metadata:** JAN, maker, product name as printed, size, package version or - date, source URL, capture session id, SHA-256. - - The enrollment tool validates a one-to-one match between images and CSV - rows. - - It OCRs each packshot and requires the catalog name's distinctive tokens - to appear (this check would have caught the l09/l10 mix-up and flagged the - l06 package version). - - It flags cross-SKU near-duplicates as "text-discriminated siblings". -- **Leakage:** enrollment refuses any image whose capture session id appears - in an evaluation manifest. Pixel equality is not a leak test; re-cut crops - pass it. -- **Versioning:** the catalog manifest has a content hash; every eval report - records catalog hash, label-file hash, and code SHA. -- **Completeness rule:** a bay is "live-ready" when every SKU on its shelf tags - is enrolled or explicitly listed as text-only. Near-unknown precision - depends on this more than on any threshold. - -## 9. Completion gate - -Measured on **Tier 2 held-out** (and Tier 3 for time-to-label). Same crops, -same catalog hash, same detector for both systems. "Van" means Van's scorer -with always-argmax, run on that same catalog. - -| Gate | Definition | Pass | -|---|---|---| -| G1 coverage | correct shown labels ÷ all in-catalog facings (enrolled or not) | new ≥ Van; lower 95 % bound of the paired-bootstrap difference ≥ −2 pp | -| G2 correctness | correct shown ÷ all shown | ≥ 95 % (Van: 29–35 % on Tier 1) | -| G3 unknown handling | near-unknown false-accept rate; far-unknown false-accept rate | ≤ 5 %; ≤ 1 % (Van: 100 %) | -| G4 time to label | Tier 3: p50 / p95 seconds from a facing entering view to a correct label shown (provisional counts only if later confirmed as the same SKU) | p50 ≤ 1.0 s and p95 ≤ 2.5 s, and p50 no worse than Van's click-to-label on a still (tap + ~50–100 ms) | -| G5 stability | confirmed labels that later change; correct labels that disappear | ≤ 2 %; ≤ 1 % | -| G6 per-SKU floor | recall of every enrolled SKU with ≥ 5 Tier-2 facings | ≥ 60 % (catches l06-type catalog faults) | -| G7 hubness | max predicted-SKU share ÷ that SKU's true share | ≤ 2.0 | -| G8 runtime | server identity latency p95 per crop; server memory | ≤ 60 ms; within the demo host's budget | - -Not complete until G1–G7 pass on the device path (Tier 3), not just offline. - ---- - -## Findings - -| ID | Finding | Severity | Evidence | Priority | Source | -|---|---|---|---|---|---| -| F-01 | image6 ground truth 23 % wrong | blocking | VERIFIED | P0 | open-world | -| F-11 | No held-out benchmark exists (one photo) | blocking | VERIFIED | P0 | open-world | -| F-02 | Production scores are per template, not per SKU → self-ambiguity and nameless labels | blocking | VERIFIED | P0 | open-world | -| F-03 | Demo server defaults to the leaked, corrupt `stock-lopia` | blocking | VERIFIED | P0 | open-world | -| F-04 | Clean catalog CSV ↔ image mismatch (l09/l10) | blocking | VERIFIED | P0 | open-world | -| F-05 | Near-unknown siblings accepted; shown-label precision 29–43 % | blocking | VERIFIED | P0 | open-world | -| F-07 | Evaluator defects beyond bugs #1–#3 | major | VERIFIED | P1 | open-world | -| F-06 | Matcher scoring defects (scale, dead ORB/BRISK, unverified "inliers", grayscale) | major | VERIFIED | P1 | open-world | -| F-08 | DINO unfit as classifier and retriever here | major | VERIFIED | P1 | open-world | -| F-09 | Confirmed labels never re-evaluated; first accept locks | major | HIGH CONFIDENCE | P1 | open-world | -| F-10 | Uncommitted live diff: unbounded blur retries; rationale invalidated | major | HIGH CONFIDENCE | P1 | open-world | -| F-12 | l06 packshot is a superseded package design | major | VERIFIED | P2 | open-world | -| F-13 | Stale handoff test status; swallowed callback exception in a test fixture | polish | VERIFIED | P3 | open-world | - -Counts: 11 VERIFIED / 2 HIGH CONFIDENCE / 0 HYPOTHESIS (hypotheses listed -separately) · 13 open-world. No reference baseline was used: this audit -covers a recognition backend. The product-level floor is Van's system (G1–G4). - -### Workstreams - -1. **Evidence (single owner, first):** F-01, F-11, F-07. Blocks every - architecture change. -2. **Catalog (parallel with 1):** F-03, F-04, F-12, enrollment tool checks. -3. **Identity core (single owner, after 1):** F-02, F-06, text lane, decision - layer, E-1–E-4. -4. **Live (after 3's interface exists):** F-10 revert now; F-09 and the - provisional label (E-7) later. The protocol plus native-UI part touches the - KMP app and needs its own owner. -5. **Scaling (after 3):** E-6, and the descriptor index only when it's needed. - -### F-01 — image6 labels are 23 % wrong — P0 · blocking · VERIFIED -- **Root cause:** labels appear propagated from tag/box matching (every row has - a `match` field) rather than read from packages. -- **Evidence:** `evidence/image6-v1-labels-overlay.jpg`; the per-index list in §3. - Rescoring the same runs under v2 moves ungated live top-1 from 26/35 to 35/42. -- **Target:** every scored label carries provenance; ambiguous facings are excluded. -- **Change:** adopt `evidence/image6_v2.json`; schema-check `label/source/note`. -- **Validation:** E-0 double-labelling on Tier 2. -- **Acceptance:** no architecture claim cites v1. - -### F-02 — template-level scores in production — P0 · blocking · VERIFIED -- **Root cause:** `ProductFeatureMatcher.score_crop` keys scores by file stem; - `engine.score_crop` (`code/engine.py:424-448`), `classify_boxes`, and `inspect` pass - them straight to `decide_identity`. The live payload looks up - `identity.sku_id` in the catalog, which has only canonical ids. -- **Evidence:** on `stock-lopia` (the demo default), 22/109 facings at full - and 29/109 at live are "ambiguous" only because the top two are views of the - same SKU. 68 of 69 (full) and 56 of 58 (live) accepted ids are - view-suffixed (`sku_l01_1`), which have no catalog row, so the payload - carries no `product` block. -- **Target:** scores grouped by canonical SKU (max over views) before any - decision; the id shown is always a catalog row. -- **Change:** group at the lane boundary (the future `IdentityLane.evidence`). - Share one canonicalizer with `rebuild_catalog.sku_of`. -- **Validation:** unit test: two strong views of one SKU → accepted, canonical id. -- **Acceptance:** zero view-suffixed ids on the wire. - -### F-03 — demo default catalog is the leaked one — P0 · blocking · VERIFIED -- **Evidence:** `run-demo-server.sh:11` `STOCK="${1:-stock-lopia}"`. That - catalog is shelf crops cut from the eval photo, and it scores 94–97 % - "identity" on its own exam. -- **Change:** default to the clean catalog once F-04 is fixed; refuse catalogs - without a provenance manifest. - -### F-04 — clean catalog metadata mismatch — P0 · blocking · VERIFIED -- **Evidence:** `stock-lopia-clean/stock.csv` has no `sku_l09` row for - `sku_l09.jpg`; its `sku_l10` row has l09's counts (30/13/17) with l10's name - and price. Every l09 answer ships without product info; l10 has a row but no - image. The README says "Six SKUs … have no official pack shot" and lists five. -- **Change:** restore the l09 row from `stock-lopia/stock.csv`, drop the - mangled l10 counts, fix the README, and add the bijection check to - `rebuild_catalog.py`. - -### F-05 — near-unknown siblings accepted — P0 · blocking · VERIFIED -- **Evidence:** the current gate accepts 22–28/48 unknowns; 9/11 unenrolled - l10 facings go to l09 with margin 0.67–0.93 (higher than many correct l11 - matches); Kewpie 450 g → l03 with 17–28 SIFT inliers. -- **Target / change:** the decision layer + text veto (§5) + enrollment - completeness (E-5). -- **Acceptance:** G2, G3. - -### F-06 — matcher scoring defects — P1 · major · VERIFIED -- **Evidence:** ORB inliers are 0 on nearly every live crop (per-facing log - from `diag.py`). `compute_geometric_inliers` returns `len(good_matches)` - unverified when there are fewer than 4. The l09 reference (305 px) scores - 85–129 while others score 10–40. -- **Change:** variant B (§5), behind the lane interface, with Van's scorer kept - as a baseline lane. -- **Acceptance:** E-1, E-2. - -### F-07 — evaluator defects — P1 · major · VERIFIED -- `top1` ≡ `coverage` (both are accepted-correct ÷ enrolled), so the identity - and decision ceilings were never separated. -- Precision's denominator excludes unenrolled in-catalog facings - (`covered` in `main`): it reports 50–53 % where the honest figure is 39–43 %. -- The gallery's SKU set comes from ground-truth labels (`sku_ids` from - `held`). -- Leak check requires the same shape. -- `HybridLane` scores the full catalog, so its latency isn't a retrieval - design. -- `group_by_sku` only handles the `sku_l` prefix. -- **Change:** report the four ceilings explicitly (the per-facing log in - `diag.py`/`variants.py` is the template). Use the catalog, not labels, for - candidate sets. Refuse leakage by session manifest. - -### F-08 — DINO — P1 · major · VERIFIED -- **Evidence:** gallery top-1 21 %/12 % with 50–60 % hub share; recall@10 of - 7/42 at 493 SKUs (`dino_scale.py`); 41 ms per embedding. -- **Change:** fence off (§8). The uncommitted `--identity matcher` default is - the right call. - -### F-09 — no identity re-evaluation — P1 · major · HIGH CONFIDENCE -- **Evidence:** `code/live.py` `_on_classified`: `if track.state != - STATE_DETECTED or track.confirmed: return`. With `vote_agree = 1` the first - accepted observation confirms. -- **Target:** E-7's evidence model with provisional labels and hysteresis. -- **Validation:** Tier 3 flips and correct-then-lost metrics. - -### F-10 — uncommitted live experiments — P1 · major · HIGH CONFIDENCE -- **Evidence:** the blur gate `return`s before `track.attempts += 1`, so - eligibility never runs out for a soft crop. `crop_first_attempt` cites the - invalidated 69/66 numbers. -- **Change:** revert per §8. - -### F-12 — l06 package version — P2 · major · VERIFIED -- **Evidence:** the packshot shows "内臓脂肪ケア フィッテ"; the shelf shows - "内臓脂肪を減らす キユーピーフィッテ". Improved matcher 4/4 at full, 1/4 at - live; 3/3 with a shelf reference. -- **Change:** enroll the current package; adopt the packshot-OCR check. - -### F-13 — stale test status — P3 · polish · VERIFIED -- **Evidence:** 49/49 pass on both trees. An `AttributeError: 'str' object has - no attribute 'items'` is raised and swallowed inside a future callback in one - test, on both trees. -- **Change:** make that fixture return a dict, or assert the callback path. - -## Hypotheses - -| ID | Suspected | Why it matters | The check that settles it | -|---|---|---|---| -| H-01 | Learned matchers (ALIKED/DISK + LightGlue) beat B on small, blurred, or re-designed packages | Could lift the identity ceiling | E-C (needs weight download approval) | -| H-02 | Shelf-tag OCR prior raises coverage and precision | Tags are readable above every run on image6 | E-9 using `code/tags.py` | -| H-03 | Cross-visit shelf references retain most of the leave-facing-out gain | Decides the enrollment protocol | E-5 | -| H-04 | Evidence accumulation shortens time-to-correct-label vs single-shot lock | G4/G5 | E-7 on Tier 3 | -| H-05 | On-device OCR is fast enough (< 30 ms per crop) and as accurate at native crop resolution | Where the text lane runs | Benchmark Vision on iPhone and ML Kit on Android over Tier-2 crops | -| H-06 | Stream JPEG quality costs SIFT inliers at live | Cheap fix if true | Rerun `variants.py` on re-encoded crops at the client's quality | -| H-07 | The hue feature is robust to store lighting | The colour gain could be lighting-specific | Tier 2 across stores and times | -| H-08 | ORB/BRISK removal costs nothing | Simplifies and speeds the matcher | E-2 at native-crop resolution, where ORB helped the small l09 reference | -| H-09 | Full-resolution OCR with ×2 upsampling matches live's text-lane gain | Full-res fusion was weaker (75 % precision) with ×1 OCR | Rerun `ocr_exp.py` + `fuse_eval.py` with full ×2 | - -## Preserved decisions - -- The matcher kept as a selectable baseline lane. The shipped default is now - `local` (variant B plus the text lane), which is what the measurements below - support; `--identity matcher` reproduces Van's scorer for E-1. -- Bounded, drift-checked crop lane protocol (`aa25f99`, `13a4e0f`); its tests - pass. -- `eval_identity.py`'s architecture-neutral lane framing and explicit - enrollment ceiling. -- Clean-catalog provenance table and the `rebuild_catalog.py` refusal rules. -- `vote_agree = 1` without provisional labels. Backing out `2` was right. - -## Implementation record (2026-09-28) - -What §8 asked for, and where it landed. Verified by the suite (251 tests) and by -running the real server against `add/image6.jpg`. - -| §8 item | Status | -|---|---| -| F-02 per-SKU scores in `score_crop`, `classify_boxes`, `inspect` | Done. `group_by_sku` on every backend; no view-suffixed id reaches the wire. | -| F-03 demo default catalog | Done. `run-demo-server.sh` defaults to `stock-lopia-clean`. | -| F-04 clean catalog CSV | Done. All 12 rows; `sku_l09` added, `sku_l10` corrected. | -| F-05 near-unknown siblings | Done via the text lane. 0 of 48 unknowns accepted at live resolution. | -| F-06 variant B behind a lane interface | Done. `identity_lanes.LocalLane`; Van's scorer stays selectable as `--identity matcher`. | -| F-07 evaluator defects | Done. Four separate ceilings, honest precision denominator, catalog-derived candidate set, `sku_of` grouping, leak signature. | -| F-08 DINO fenced off | Done. Not the default, not in the live path, refuses to start without weights. | -| F-10 uncommitted live experiments | Reverted. `crop_first_attempt`, the blur early return, `vote_agree` and `max_attempts 4` are gone. | -| F-12 l06 current package | **Blocked on a capture.** Kewpie serves the superseded design for JAN 4901577083600, so the current package cannot be enrolled from the official URL. The text lane ranks l06 first on all 4 shelf facings but abstains below the accept line, so no wrong label is shown. | -| F-13 stale status / swallowed exception | Done. The fixture returns the matcher's `(scores, details)` shape; zero swallowed callback exceptions. | -| F-09 re-evaluation with provisional labels | **Postponed as §8 specifies** — it is a dependency of E-7, which needs Tier-3 recordings that do not exist yet. | - -Two defects found while wiring the lanes into the live path, both fixed: -`live` re-sorted the local lane's scores by value, which discards the lane's own -rank order (order comes from `rank_key`, the value is only the gate), and the -early return on an empty local observation made the text lane's "answer on its -own" job unreachable. Live now decides per crop on the lane's own order. - -Measured after the change, through the real server: `/v1/recognize` names 41 of -112 detections at 90% precision with no view-suffixed ids; the live WebSocket -names 12 at 100% precision with 0 unknown accepts, first label on frame 3. -Offline on the corrected labels: 36/61 (59%) at 95% precision, 0/48 unknowns, -11 of those labels sourced from the text lane alone. - -The headline number is reproducible from the repo, not from a scratch script: - - .venv/bin/python code/eval_identity.py --stock stock-lopia-clean \ - --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \ - --image add/image6.jpg --lanes local --live - - live resolution: catalog 69% · identity 86% · decision 86% · - end-to-end 59% · precision 95% · 2 false accepts of 48 unknowns · 20.6 ms/crop - -Still open, and not implementable from here: Tier 2 held-out labels (F-11), the -E-1..E-9 experiments, and the completion gate G1-G7, all of which need a second -independent shelf visit. - -## Out of scope - -Detector recall and box quality (all experiments use the same boxes). Tag and -planogram pipeline internals. Native UI. On-device performance. Learned local -features (not installed; downloading them needs your approval). `image.jpg` / -`stock/` (no labels exist). Timings from parallel runs are relative; -`scale.py` timings are single-process on this Mac. diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md b/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md deleted file mode 100644 index 1779ce0..0000000 --- a/recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Evidence scripts (2026-09-28 audit) - -Read-only against product code: every script imports from `code/` and writes -only into this folder. Run from `recognization/` with `.venv/bin/python`. - -`image6_v2.json` is the corrected image6 label set (26 of 111 v1 labels -changed; every row carries `v1_label`, `source`, and a `note` where changed). -`image6-v1-labels-overlay.jpg` shows the v1 labels on the shelf; green = catalog -label, red = unknown. `sheet.py` regenerates per-SKU contact sheets. - -| Step | Command (prefix `E=docs/research/2026-09-28-recognition-architecture/evidence`) | Produces | -|---|---|---| -| Original matcher, both catalogs, full/live/live-with-unscaled-boxes | `.venv/bin/python $E/diag.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] [--bug] --out $E/run_stock-lopia-clean_.pkl` (repeat with `--stock stock-lopia`) | `run_*.pkl` | -| Score them under v1 or v2 labels | `.venv/bin/python $E/analyze.py $E/run_*.pkl` | identity top-1, argmax, current gate | -| Matcher variants | `.venv/bin/python $E/variants.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] --scales 200,400 --root --geom affine --out $E/v_ms_root_aff_.pkl` (also `--tmpl-h 250/400`, no flags = raw) | `v_*.pkl` | -| Variant ranking + open-set frontier | `.venv/bin/python $E/an2.py $E $(basename -a $E/v_*.pkl)` (`LABELS=image6_v2_lfo.json` for the leave-facing-out runs) | correct at 100/95/90 % precision | -| Leave-facing-out catalog | `.venv/bin/python $E/lfo_setup.py`, then `variants.py --stock $E/cat_ --labels $E/image6_v2_lfo.json --scales 200,400 --root --geom affine [--live] --out $E/lfo_cat__.pkl` | `lfo_*.pkl` | -| Unrelated-shelf distractors | `YOLO_OFFLINE=1 .venv/bin/python $E/distract.py stock-lopia-clean $E/cat_full` | `distract.pkl` | -| Catalog growth, exhaustive vs descriptor index | `YOLO_OFFLINE=1 .venv/bin/python $E/scale.py` | stdout | -| DINO retrieval under growth | `YOLO_OFFLINE=1 .venv/bin/python $E/dino_scale.py` | stdout | -| OCR read-rate | `.venv/bin/python $E/ocr_exp.py` | `ocr.pkl` | -| OCR signatures, then fusion | `.venv/bin/python $E/ocr_fuse.py && .venv/bin/python $E/fuse_eval.py` | `sig.pkl`, stdout | -| Per-SKU contact sheets + the labelled shelf | `.venv/bin/python $E/sheet.py ` | `gt_