Make the catalog hash honest, share the text helpers, drop the duplicate labels

`catalog_hash` hashed the CSV and the view *names*, so replacing
`sku_l09.jpg` with a different product's packshot left the hash unchanged and
a report could claim the same catalog while every l09 decision in it had
moved. It hashes each view's bytes now.

`identity_lanes` carried its own `normalize_text` and `bigrams`, near-copies
of the ones in `rebuild_catalog` that had already drifted (the lane's table
folded ユ to itself, the catalog's did not). The lane imports the catalog's
now, so one read scores the same either way.

`evidence/image6_v2.json` was byte-identical to `eval/labels/image6.json`; the
evidence scripts and the decision document point at the canonical file and the
copy is gone. `score_crop`'s docstring now states the contract its callers
already depend on: the dict's insertion order is the lane's ranking, and
re-sorting by value names a different SKU than the lane chose.

Tests: the lane is shared by the classify pool, so a concurrent-ranking test
pins that one BFMatcher does not cross crops under eight threads.

recognization/code/engine.py

@@ -566,7 +566,12 @@ class RecognitionEngine:
         return self._lane
 
     def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
-        """Score one crop, keyed by canonical SKU.
+        """Score one crop, keyed by canonical SKU, in the lane's rank order.
+
+        The insertion order is the ranking and callers rely on it: the local
+        lane's order comes from its rank key while its value is the margin over
+        the runner-up, so re-sorting by value names a different SKU than the
+        lane chose. `list(scores.items())` is therefore the ranked sequence.
 
         Every backend groups by SKU before returning. Scoring each reference
         view separately is what made the demo's default catalog report 22-29 of

recognization/code/eval_identity.py

@@ -22,7 +22,7 @@ detector). The hybrid lane also reports shortlist recall, because a true SKU
 outside the shortlist is a hard ceiling no verifier can lift.
 
     python code/eval_identity.py \
-        --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \
+        --labels eval/labels/image6.json \
         --image add/image6.jpg --stock-dir stock-lopia-clean --lanes matcher --live
 
 Labels are {"image", "facings": [{"box", "label"}]} with "unknown" for facings
@@ -70,7 +70,7 @@ LIVE_LONG_SIDE = 1280
 # Pixel equality is not a leak test: a template re-cut from the same photo with
 # a different box passes a same-shape comparison. A 64x64 grayscale signature
 # catches most of those, but it is a heuristic, not a proof — measured on
-# image6_v2 against stock-lopia, re-cuts in the realistic 0.94..1.06 box band
+# the corrected labels against stock-lopia, re-cuts in the realistic 0.94..1.06 box band
 # land 7..37 MAD from their facing's signature while the closest pair of
 # genuinely distinct same-SKU facings is 23.1, so the distributions overlap and
 # no threshold is clean. 20.0 catches 81% of that band with zero false
@@ -116,12 +116,13 @@ def file_hash(path: str) -> Optional[str]:
 
 
 def catalog_hash(stock_dir: str) -> Optional[str]:
-    """SHA-256 over the catalog's CSV bytes and its image names.
+    """SHA-256 over the catalog's CSV bytes and each view's bytes.
 
-    Image bytes are not hashed: a re-encoded view of the same product is the
-    same catalog, and hashing every packshot would make the number move for
-    reasons that do not change a single decision. A changed row or a different
-    set of views does change it.
+    The view bytes are hashed, not just their names. Names alone were not
+    enough: replacing `sku_l09.jpg` with a different product's packshot left
+    the hash unchanged, so a report could claim the same catalog while every
+    l09 decision in it had moved. Re-encoding the same image does move the
+    hash, which is the price of not being able to lie about what was scored.
     """
     digest = hashlib.sha256()
     csv_path = os.path.join(stock_dir, "stock.csv")
@@ -137,8 +138,14 @@ def catalog_hash(stock_dir: str) -> Optional[str]:
         )
     except OSError:
         names = []
-    digest.update(b"\n")
-    digest.update("\n".join(names).encode("utf-8"))
+    for name in names:
+        digest.update(b"\n")
+        digest.update(name.encode("utf-8"))
+        try:
+            with open(os.path.join(stock_dir, name), "rb") as handle:
+                digest.update(handle.read())
+        except OSError:
+            digest.update(b"unreadable")
     return digest.hexdigest()
 
 
@@ -557,7 +564,7 @@ def shortlist_recall(
     shortlist already dropped. The denominator is enrolled facings only — a
     facing whose SKU has no reference cannot be retrieved by any shortlist, so
     counting it here reports a retrieval failure that is really a catalog gap.
-    Measured on image6_v2, that distinction is 67% vs 98% at top-5.
+    Measured on the corrected labels, that distinction is 67% vs 98% at top-5.
     """
     hits = total = 0
     misses: List[Dict[str, Any]] = []

recognization/code/identity_lanes.py

@@ -27,7 +27,6 @@ import math
 import os
 import re
 import sys
-import unicodedata
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
@@ -38,7 +37,7 @@ if current_dir not in sys.path:
 import cv2  # noqa: E402
 import numpy as np  # noqa: E402
 
-from rebuild_catalog import sku_of  # noqa: E402
+from rebuild_catalog import bigrams, normalize_text, sku_of  # noqa: E402
 
 # The local lane's accept line, in the units of `confidence()` below. Measured
 # on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59%
@@ -258,23 +257,10 @@ class LocalLane:
         return {sku: value for sku, value in self.rank(crop)}
 
 
-# Katakana folding: OCR reads small kana and the long-vowel mark inconsistently,
-# so キユーピー and キユピー have to land on the same bigrams.
-_KANA_FOLD = str.maketrans(
-    "ッャュョァィゥェォヮヵヶユ", "ツヤユヨアイウエオワカケユ"
-)
-
-
-def normalize_text(text: str) -> str:
-    """NFKC, uppercase, kana-folded, alphanumeric and `%` only."""
-    text = unicodedata.normalize("NFKC", text).upper()
-    text = text.translate(_KANA_FOLD)
-    text = text.replace("キユーピー", "キユピ").replace("キューピー", "キユピ")
-    return "".join(ch for ch in text if ch.isalnum() or ch == "%")
-
-
-def bigrams(text: str) -> set:
-    return {text[i:i + 2] for i in range(len(text) - 1)}
+# Text normalization and bigrams come from `rebuild_catalog`, which uses them
+# to match a catalog name against a packshot's OCR. Two copies of the same
+# folding rules drift: the lane's copy folded ユ to itself while the catalog's
+# did not, so the same read scored differently depending on which one ran.
 
 
 def identity_bigrams(read: str) -> set:

recognization/code/test_engine.py

@@ -564,6 +564,51 @@ class DecisionPolicyTests(unittest.TestCase):
         self.assertEqual(decision["identity_level"], "rejected")
 
 
+class LocalLaneConcurrencyTests(unittest.TestCase):
+    """The lane is shared by the classify pool, so `rank` must be thread-safe.
+
+    One `cv2.BFMatcher` is reused across calls. The classify pool runs up to
+    eight workers, so a lane that mutated matcher state per call would return
+    another crop's answer under load - the failure would look like a
+    recognition error, not a crash.
+    """
+
+    def test_concurrent_ranking_matches_solo_ranking(self):
+        import threading
+
+        import cv2
+
+        from identity_lanes import LocalLane
+
+        stock_dir = os.path.join(os.path.dirname(current_dir), "stock-lopia-clean")
+        if not os.path.isdir(stock_dir):
+            self.skipTest("no catalog checked out")
+        image_path = os.path.join(os.path.dirname(current_dir), "add", "image6.jpg")
+        if not os.path.isfile(image_path):
+            self.skipTest("no evaluation photo checked out")
+        lane = LocalLane(stock_dir)
+        image = cv2.imread(image_path)
+        crops = [image[700:1000, 600 + 40 * i:800 + 40 * i] for i in range(8)]
+        solo = lane.rank(crops[0])
+        results = {}
+        errors = []
+
+        def work(index: int) -> None:
+            try:
+                for _ in range(6):
+                    results[index] = lane.rank(crops[index])
+            except Exception as error:  # noqa: BLE001 - reported below
+                errors.append(repr(error))
+
+        threads = [threading.Thread(target=work, args=(i,)) for i in range(8)]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        self.assertEqual(errors, [])
+        self.assertEqual(results[0], solo, "a shared matcher must not cross crops")
+
+
 class CatalogImageTests(unittest.TestCase):
     def test_catalog_image_path_is_resolved_through_inventory(self):
         engine, _, _, _ = make_engine()

recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md

@@ -112,7 +112,7 @@ verification → calibrated open-set decision → temporal accumulation.
 
 ### The four ceilings, separated
 
-On corrected labels (`evidence/image6_v2.json`): 109 scored facings, 61
+On corrected labels (`eval/labels/image6.json`): 109 scored facings, 61
 in-catalog across 11 SKUs, 48 unknown, 2 excluded. The clean catalog enrolls 7
 SKUs covering 42 of the 61 in-catalog facings.
 
@@ -288,7 +288,7 @@ rank, decision, lane latencies), and report the four ceilings separately.
 
 ### Benchmark tiers (build these first; they block every other decision)
 
-- **Tier 1, diagnostic:** `evidence/image6_v2.json` (109 facings). Dev only.
+- **Tier 1, diagnostic:** `eval/labels/image6.json` (109 facings). Dev only.
   Free to tune on. Its numbers never ship.
 - **Tier 2, held-out regression:** new captures that are never enrolled and
   never tuned on. At least 3 visits across ≥ 2 LOPIA locations or days, same
@@ -335,7 +335,7 @@ reverted.
 | `test_live.py` change | encodes the reverted budget | Revert with it. |
 | `code/rebuild_catalog.py` (untracked) | catalog tooling | Keep. Add image↔CSV bijection, packshot-OCR vs name check, and eval-session refusal (enrollment protocol below). |
 | `stock-lopia-clean/` (untracked) | catalog data | Keep the images and provenance. Fix `stock.csv` (F-04) and the README's "Six SKUs" (it lists five). Replace the l06 packshot with the current package. |
-| `eval/labels/image6.json` (untracked) | ground truth | **Replace** with `evidence/image6_v2.json`, schema: `label`, `source`, `note`, `v1_label`. |
+| `eval/labels/image6.json` | ground truth | Adopted; schema: `label`, `source`, `note`, `v1_label`. |
 | `identity.decide_identity` inlier branch | production | **Rewrite** as the calibrated decision (E-4); keep the cosine branch only while the gallery exists. |
 | Production score path (`engine.score_crop`, `classify_boxes`, `inspect`) | production | **Rewrite:** return per-SKU evidence grouped by canonical SKU (F-02). |
 | `feature_matcher.py` composite score | production | **Rewrite** into variant B behind an interface; keep Van's scorer as a selectable baseline lane for E-1. |
@@ -431,7 +431,7 @@ covers a recognition backend. The product-level floor is Van's system (G1–G4).
 - **Evidence:** `evidence/image6-v1-labels-overlay.jpg`; the per-index list in §3.
   Rescoring the same runs under v2 moves ungated live top-1 from 26/35 to 35/42.
 - **Target:** every scored label carries provenance; ambiguous facings are excluded.
-- **Change:** adopt `evidence/image6_v2.json`; schema-check `label/source/note`.
+- **Change:** adopt `eval/labels/image6.json`; schema-check `label/source/note`.
 - **Validation:** E-0 double-labelling on Tier 2.
 - **Acceptance:** no architecture claim cites v1.
 
@@ -602,7 +602,7 @@ and overstates its precision, so the evaluator now has a `van` lane: the same
 scores, always answered. Every "vs Van" claim in this record uses that row.
 
     .venv/bin/python code/eval_identity.py --stock stock-lopia-clean \
-      --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \
+      --labels eval/labels/image6.json \
       --image add/image6.jpg --lanes van,matcher,local --live
 
     full resolution — 109 scored facings (61 in-catalog, 48 unknown)
@@ -673,7 +673,7 @@ Two further review findings, both fixed and both measured:
 The headline number is reproducible from the repo, not from a scratch script:
 
     .venv/bin/python code/eval_identity.py --stock stock-lopia-clean \
-      --labels docs/research/2026-09-28-recognition-architecture/evidence/image6_v2.json \
+      --labels eval/labels/image6.json \
       --image add/image6.jpg --lanes local --live
 
     live resolution: catalog 69% · identity 86% · decision 86% ·

recognization/docs/research/2026-09-28-recognition-architecture/evidence/README.md

@@ -3,16 +3,16 @@
 Read-only against product code: every script imports from `code/` and writes
 only into this folder. Run from `recognization/` with `.venv/bin/python`.
 
-`image6_v2.json` is the corrected image6 label set (26 of 111 v1 labels
+`eval/labels/image6.json` is the corrected image6 label set (26 of 111 v1 labels
 changed; every row carries `v1_label`, `source`, and a `note` where changed).
 `image6-v1-labels-overlay.jpg` shows the v1 labels on the shelf; green = catalog
 label, red = unknown. `sheet.py` regenerates per-SKU contact sheets.
 
 | Step | Command (prefix `E=docs/research/2026-09-28-recognition-architecture/evidence`) | Produces |
 |---|---|---|
-| Original matcher, both catalogs, full/live/live-with-unscaled-boxes | `.venv/bin/python $E/diag.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] [--bug] --out $E/run_stock-lopia-clean_<full\|live\|livebug>.pkl` (repeat with `--stock stock-lopia`) | `run_*.pkl` |
+| Original matcher, both catalogs, full/live/live-with-unscaled-boxes | `.venv/bin/python $E/diag.py --stock stock-lopia-clean --labels $E/../../../eval/labels/image6.json [--live] [--bug] --out $E/run_stock-lopia-clean_<full\|live\|livebug>.pkl` (repeat with `--stock stock-lopia`) | `run_*.pkl` |
 | Score them under v1 or v2 labels | `.venv/bin/python $E/analyze.py <labels.json> $E/run_*.pkl` | identity top-1, argmax, current gate |
-| Matcher variants | `.venv/bin/python $E/variants.py --stock stock-lopia-clean --labels $E/image6_v2.json [--live] --scales 200,400 --root --geom affine --out $E/v_ms_root_aff_<full\|live>.pkl` (also `--tmpl-h 250/400`, no flags = raw) | `v_*.pkl` |
+| Matcher variants | `.venv/bin/python $E/variants.py --stock stock-lopia-clean --labels $E/../../../eval/labels/image6.json [--live] --scales 200,400 --root --geom affine --out $E/v_ms_root_aff_<full\|live>.pkl` (also `--tmpl-h 250/400`, no flags = raw) | `v_*.pkl` |
 | Variant ranking + open-set frontier | `.venv/bin/python $E/an2.py $E $(basename -a $E/v_*.pkl)` (`LABELS=image6_v2_lfo.json` for the leave-facing-out runs) | correct at 100/95/90 % precision |
 | Leave-facing-out catalog | `.venv/bin/python $E/lfo_setup.py`, then `variants.py --stock $E/cat_<pack\|full> --labels $E/image6_v2_lfo.json --scales 200,400 --root --geom affine [--live] --out $E/lfo_cat_<pack\|full>_<full\|live>.pkl` | `lfo_*.pkl` |
 | Unrelated-shelf distractors | `YOLO_OFFLINE=1 .venv/bin/python $E/distract.py stock-lopia-clean $E/cat_full` | `distract.pkl` |
@@ -44,7 +44,7 @@ with no arguments raise `IndexError`; neither validates its own usage.
 v1 labels are no longer a file on disk — `eval/labels/image6.json` is the v2
 set. To score a run under v1, rebuild it from the `v1_label` field:
 
-    .venv/bin/python -c "import json;d=json.load(open('$E/image6_v2.json'));\
+    .venv/bin/python -c "import json;d=json.load(open('$E/../../../eval/labels/image6.json'));\
     [f.update(label=f.pop('v1_label')) for f in d['facings']];\
     json.dump(d,open('/tmp/image6_v1.json','w'),ensure_ascii=False)"
 

recognization/docs/research/2026-09-28-recognition-architecture/evidence/an2.py

@@ -1,5 +1,5 @@
 import pickle, json, sys, numpy as np, collections
-import os; S = sys.argv[1]; L = {f['index']: f['label'] for f in json.load(open(f'{S}/' + os.environ.get('LABELS','image6_v2.json')))['facings']}
+import os; S = sys.argv[1]; L = {f['index']: f['label'] for f in json.load(open(f'{S}/' + os.environ.get('LABELS','../../../../eval/labels/image6.json')))['facings']}
 RANK = {
  'inl': lambda e: e['inliers'],
  'inl*col': lambda e: e['inliers'] * (0.25 + e['color']),

recognization/docs/research/2026-09-28-recognition-architecture/evidence/dino_scale.py

@@ -16,7 +16,7 @@ for p in sorted(glob.glob('add/*')):
         if min(c.shape[:2])>=40: dis.append((f'd{j}_{os.path.basename(p)}',c))
 random.shuffle(dis)
 D=[(k,g.embed(c)) for k,c in dis]
-L=json.load(open(f'{S}/image6_v2.json')); E={k for k,_ in refs}
+L=json.load(open(f'{S}/../../../../eval/labels/image6.json')); E={k for k,_ in refs}
 Q=[(fc['label'],g.embed(c)) for fc,c in crops('add/image6.jpg',L,True) if fc['label'] in E]
 t=time.perf_counter(); _=g.embed(Q and dis[0][1]); print('embed ms',round((time.perf_counter()-t)*1000,1))
 for N in [0,50,200,len(D)]:

recognization/docs/research/2026-09-28-recognition-architecture/evidence/fuse_eval.py

@@ -2,7 +2,7 @@ import sys,os,json,pickle,csv,numpy as np
 S=os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0,S)
 from ocr_fuse import norm,sim  # noqa (module runs OCR of packshots on import; cheap)
 sig=pickle.load(open(f'{S}/sig.pkl','rb')); O=pickle.load(open(f'{S}/ocr.pkl','rb'))
-L={f['index']:f['label'] for f in json.load(open(f'{S}/image6_v2.json'))['facings']}
+L={f['index']:f['label'] for f in json.load(open(f'{S}/../../../../eval/labels/image6.json'))['facings']}
 def conf(ev):
     r=sorted(ev.items(),key=lambda kv:-kv[1]['inliers']*(0.25+kv[1]['color']))
     (k1,e1),(k2,e2)=r[0],r[1]; i1,i2=e1['inliers'],e2['inliers']

recognization/docs/research/2026-09-28-recognition-architecture/evidence/lfo_setup.py

@@ -4,7 +4,7 @@ image6_v2_lfo.json with those reference facings excluded from scoring.
 Same-photo references make this an optimistic proxy, not a held-out result."""
 import json, cv2, os, shutil, collections
 E = os.path.dirname(os.path.abspath(__file__))
-L = json.load(open(f'{E}/image6_v2.json')); im = cv2.imread('add/image6.jpg')
+L = json.load(open(f'{E}/../../../../eval/labels/image6.json')); im = cv2.imread('add/image6.jpg')
 by = collections.defaultdict(list)
 for f in L['facings']:
     if f['label'] not in ('unknown', 'exclude'): by[f['label']].append(f)

recognization/docs/research/2026-09-28-recognition-architecture/evidence/ocr_exp.py

@@ -3,7 +3,7 @@ sys.path.insert(0,'code'); sys.path.insert(0,os.path.dirname(__file__))
 from ocr import VisionOcr
 from diag import crops
 S=os.path.dirname(__file__)
-L=json.load(open(f'{S}/image6_v2.json'))
+L=json.load(open(f'{S}/../../../../eval/labels/image6.json'))
 # distinguishing tokens per SKU (any hit counts); includes siblings for near-unknowns
 KEY={'sku_l01':['エッグケア','卵を使って'],'sku_l02':['アマニ'],'sku_l03':['からし','Spicy'],'sku_l04':['めんたい'],
 'sku_l05':['ゼロ','ノンコレ'],'sku_l06':['フィッテ','内臓脂肪'],'sku_l07':['ラッキー'],'sku_l08':['PERI','ペリ'],

recognization/docs/research/2026-09-28-recognition-architecture/evidence/scale.py

@@ -27,7 +27,7 @@ for p in sorted(glob.glob('add/*')):
         x1,y1,x2,y2=[int(v) for v in b['box']]; c=im[max(0,y1):y2,max(0,x1):x2]
         if min(c.shape[:2])>=40: dis.append((f'd_{os.path.basename(p)}_{j}',c))
 random.shuffle(dis); print('distractor refs available',len(dis))
-L=json.load(open(f'{S}/image6_v2.json')); Q=[(fc,c) for fc,c in crops('add/image6.jpg',L,True) if fc['label']!='exclude' and min(c.shape[:2])>=8]
+L=json.load(open(f'{S}/../../../../eval/labels/image6.json')); Q=[(fc,c) for fc,c in crops('add/image6.jpg',L,True) if fc['label']!='exclude' and min(c.shape[:2])>=8]
 Qf=[(fc['label'],)+feat(c,None) for fc,c in Q]
 bf=cv2.BFMatcher(cv2.NORM_L2)
 def verify(kq,dq,r):