From 7736465beebed869d775d862f2fe80f819406933 Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Tue, 29 Sep 2026 09:28:19 +0900 Subject: [PATCH] docs(decision): correct the record against the re-measured evidence The audit's own numbers were right; several claims built on them were not, and the advisors found each one: - 'wins both axes at both resolutions' - false at live, where the lane trades 59% coverage for 93% precision against Van's 66% at 39%. Stated as the trade it is. - G1's bootstrap now exists: the local lane's correct set is a strict superset of Van's on the stream, so the 95% lower bound is a hard +0.0 pp - a floor of that pairing, not a margin. tracks_final.json is level with Van and does not clear it. - The far-unknown rate is 0.066% on 4580 crops at the shipped configuration, not 0.30% on 2650 at the old one. - The E-3 ablation compared text-presence *and* a different argmax; both arms now go through the same DecisionPolicy ordering. Text adds 9-10 pp of coverage at full resolution while raising precision. - The sweep table swept one axis while pinning the other; replaced with the joint sweep. - The two Re-measured tables are dated snapshots, labeled as such. --- recognization/code/test_engine.py | 57 +++++++++ .../DECISION.md | 118 +++++++++++------- 2 files changed, 132 insertions(+), 43 deletions(-) diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index d95eda7..96e45de 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -178,6 +178,63 @@ class TextLaneSignatureTests(unittest.TestCase): "sku_l08", ) + def test_one_shared_pair_is_refused_by_the_floor_alone(self): + """Everything but the evidence floor says answer, and it says no. + + "アンース" holds one pair the catalog knows - "ース" - so the row scores + a perfect 1.0 with the whole margin over an empty runner-up, and the + read carries three pairs, so `carries_identity` passes as well. Only + `TEXT_MIN_SHARED` stands between a coincidental pair and a named + product, which is why the test asserts the score is 1.0: it pins the + floor as the only thing doing the refusing. + """ + from identity_lanes import DecisionPolicy, TextLane + + lane = TextLane.from_catalog( + [{"sku_id": "sku_x", "product_name": "オイスターソース"}], + "no-such-dir", + ocr_engine=None, + ) + read = "アンース" + self.assertEqual(lane.scores_from_text(read)["sku_x"], 1.0) + self.assertIsNone( + DecisionPolicy(signatures=lane.signatures).decide( + [], [("sku_x", 1.0)], text_read=read + )["best_sku"], + "one shared pair is not enough to name a row", + ) + + def test_a_thin_read_still_vetoes_even_though_it_cannot_name(self): + """The floor is a floor on naming, not on contradicting. + + "北海道塩" shares three bigrams with sku_b - one short of the floor - + so it cannot name that row. It is still the row the text prefers over + the local answer, and the local lane clears its own line at 5.0 against + a line of 4.0, so if the floor also sat under the veto this would come + back as sku_a: a three-bigram read would have rescued a match the + geometry earned. + """ + from identity_lanes import DecisionPolicy, TextLane + + lane = TextLane.from_catalog( + [ + {"sku_id": "sku_a", "product_name": "キューピー ハーフマヨネーズ 400g"}, + {"sku_id": "sku_b", "product_name": "北海道塩"}, + ], + "no-such-dir", + ocr_engine=None, + ) + read = "北海道塩" + ranked = sorted(lane.scores_from_text(read).items(), + key=lambda kv: kv[1], reverse=True) + self.assertEqual((ranked[0][0], ranked[0][1]), ("sku_b", 1.0)) + self.assertIsNone( + DecisionPolicy(signatures=lane.signatures).decide( + [("sku_a", 5.0)], ranked, text_read=read + )["best_sku"], + "a read too thin to name a row still refuses the local answer", + ) + class FakeDetector: """Records concurrency and returns fixed boxes.""" diff --git a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md b/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md index 618d048..7ae8363 100644 --- a/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md +++ b/recognization/docs/research/2026-09-28-recognition-architecture/DECISION.md @@ -59,8 +59,8 @@ store-scoped, enrolled catalog, with live-video evidence accumulation**. becomes retrieval plus verification (§6). - **Open set is the dominant difficulty, and it comes in two kinds.** *Far* unknowns (other categories) are rejected easily **once the accept lines are - calibrated against them**: 3.09 % at the old lines, 0.30 % at the shipped - ones, on 2650 crops from 60 unrelated Wikimedia supermarket photos, at a + calibrated against them**: 3.09 % at the old lines, 0.066 % at the shipped + ones, on 4580 crops from 64 unrelated Wikimedia supermarket photos, at a threshold that keeps 42 of 48 correct image6 labels. The audit's 0.2-0.7 % was measured on 602 crops from the `add/` photos with the audit's own variant matcher; the shipped lane at the shipped lines was 3.09 % until 2026-09-29 @@ -399,14 +399,17 @@ with always-argmax, run on that same catalog. | G8 runtime | server identity latency p95 per crop; server memory | ≤ 60 ms; within the demo host's budget | Status 2026-09-29, offline on image6 plus the Wikimedia distractor sets: -**G1 passes on the stream** (70-74 % coverage against Van's 63 % on the same -stream and catalog, 0 wrong against Van's 7). **G2 passes** (95 % precision -full, 97 % live). **G3's far-unknown half passes** (0.30 % on 2650 unrelated -crops); its near-unknown half is measured on image6 only, where the size gate -rejects the documented 450 g case. **G8 passes** (identity p95 well inside -60 ms). **G4, G5, G6, G7 are not yet measured on held-out data** - Tier 2 does -not exist, and G4/G5 need a recorded walk. Nothing here is a device-path -result. +**G1's point estimate passes on the stream** (70-74 % coverage against Van's +63 % on the same stream and catalog, 0 wrong against Van's 7), and the paired +bootstrap is now computed: the shipped lane's correct set is a strict superset +of Van's on 27 facings, so the 95 % lower bound is a hard +0.0 pp - a floor of +this pairing, not a margin, and it clears the -2 pp half. **G2 passes** (95 % +precision full, 97 % live). **G3's far-unknown half passes** (0.066 % on 4580 +unrelated crops at the shipped configuration); its near-unknown half is +measured on image6 only, where the size gate rejects the documented 450 g case. +**G8 passes** (identity p95 well inside 60 ms). **G4, G5, G6, G7 are not yet +measured on held-out data** - Tier 2 does not exist, and G4/G5 need a recorded +walk. Nothing here is a device-path result. Not complete until G1–G7 pass on the device path (Tier 3), not just offline. @@ -647,12 +650,12 @@ scores, always answered. Every "vs Van" claim in this record uses that row. matcher 83% 79% 74% 61% 54% 33 local 83% 85% 60% 59% 93% 3 -Against Van's real contract the local lane wins both axes at both resolutions: -75% at 91% against 64% at 38% (full), and 59% at 93% against 66% at 39% -(live). At live resolution Van answers more facings correctly - 66% against -59% - and is wrong on 67 of them to the local lane's 3. That is the trade the -decision makes deliberately: coverage is a gate (G1) and precision is not -negotiable, so the lane abstains where Van guesses. +Against Van's real contract the local lane wins both axes at full resolution: +75% at 91% against 64% at 38%. At live resolution it trades coverage for +precision - 59% at 93% against Van's 66% at 39%. Van answers more facings +correctly there and is wrong on 67 of them to the local lane's 3. That is the +trade the decision makes deliberately: coverage is a gate (G1) and precision is +not negotiable, so the lane abstains where Van guesses. **One accept line serves both resolutions.** An earlier draft of this section claimed full resolution needed a line near 6 to reach 95%, leaving 28/61 @@ -674,8 +677,8 @@ audit measured, the 4.0 row is the shipped line: 6.0 59% 97% 1 The line is still provisional - it is tuned on Tier 1 and needs Tier-2 -calibration - but it does not need to differ by input path, and the design -beats Van on both axes at both resolutions today. +calibration - but it does not need to differ by input path, and at full +resolution the design beats Van on both axes today. The live count was pacing-dependent, and the cause was measurable: OCR ran on every crop at every attempt, taking a crop from 5 ms to 25 ms and throughput @@ -738,6 +741,14 @@ Van names fewer facings than the local lane and is wrong on 7 of the 53 it does name, plus 6 accepts of products the catalog does not hold. G1 asks the new system to be no worse than Van on coverage; it is better on both axes. +The paired bootstrap (`code/score_stream.py --van ...`) grades both runs on the +same 27 facings: the local lane's correct set is a strict superset of Van's, so +the difference is +7.4 to +11.1 pp and the 95 % lower bound is a hard +0.0 pp. +That clears G1's -2 pp half, but read it as a floor of this pairing rather than +a margin - no resample of a superset can go negative. One of the five recorded +runs (`tracks_final.json`) is 17/27, level with Van, and does not clear the +bound; the other four do. + Two live-path bugs surfaced while measuring this, both fixed: - **Inherited labels carried frame id 0.** `_inherit_labels` set `label_box` @@ -807,6 +818,11 @@ unanswerable before and after. ### Re-measured 2026-09-29 +This is the state after the text lane's evidence floor landed and before the +accept lines were calibrated against unrelated shelves; the current numbers are +in § The baseline is Van's contract above (local 75% at 91% full, 59% at 93% +live). Kept as the dated snapshot that shows what the floor alone changed. + The text lane's evidence floor (shared bigrams between the read and the winning row's name) and the local lane's signature pass-through landed, and the decision ceiling was corrected: it divided a numerator that counted unenrolled @@ -823,16 +839,20 @@ facings by the enrolled count, so it read 117%. local 83% 91% 64% 62% 93% 3 **E-3 answered.** `--no-text` runs the same crops and catalog without the text -lane, which is the only way this question is answerable. Full resolution: -end-to-end 77% at 91% precision with text, 61% at 66% without (5 false accepts -against 20). Live: 62% at 93% against 56% at 84% (3 against 7). The lane adds -16 pp and 6 pp of coverage while *raising* precision, against E-3's +5 pp bar. -It is not a veto-only lane. +lane, which is the only way this question is answerable. Re-measured 2026-09-29 +after the ablation was corrected: the first version compared text-presence +*and* a different argmax, because it re-sorted candidates by confidence value +where the lane ranks by its own key. Both arms now go through the same +`DecisionPolicy` ordering, so text is the only variable. -Re-measured 2026-09-29 with the calibrated accept lines: 66% at 95% precision -with text, 55% at 81% without (2 false accepts against 8). The lane still adds -11 pp of coverage while raising precision by 14 pp, so the answer is unchanged -and the margin is wider than the audit's. +Full resolution: end-to-end 66% at 95% precision with text, 56% at 86% without +(2 false accepts against 6). Live: 56% at 97% against 48% at 100% (1 against +0). The lane adds 9-10 pp of coverage at full resolution while raising precision +by 10 pp, against E-3's +5 pp bar. At live resolution it adds 8 pp of coverage +and costs 3 pp of precision - one false accept - which is the trade the accept +lines already price. It is not a veto-only lane. + +Reproduce: `code/eval_identity.py --lanes local --live [--no-text]`. **Name-only signatures, and why.** The architecture sketch above originally said the text signature was "name + OCR of its packshot". That was tried and @@ -907,7 +927,9 @@ Every threshold in this document was fitted on image6, where each crop is a facing of a product the shelf actually holds. That is a closed-set tuning set, and it hid the far-unknown rate completely: the shipped lane accepted 82 of 2650 crops from 60 Wikimedia Commons supermarket photos of milk, soy and snack -shelves - 3.09 %, against G3's 1 %. +shelves - 3.09 %, against G3's 1 %. (The crop count is the old full-resolution, +`detect_conf=0.25` measurement; at the shipped configuration the same photos +yield 4580 crops and the shipped lines accept 3 of them - 0.066 %.) Those photos share nothing with the LOPIA catalog, so every accept is a false accept by construction. Two mechanisms produced them: @@ -919,23 +941,33 @@ accept by construction. Two mechanisms produced them: - **The local lane accepted pure geometry.** The residual accepts were matches with no text overlap at all, scoring 2.26-4.81 against an accept line of 2.0. -Both lines moved: `TEXT_MIN_SHARED` 2 -> 4, `LOCAL_ACCEPT` 2.0 -> 4.0. Measured -on 2650 distractor crops and 109 image6 facings: +Both lines moved: `TEXT_MIN_SHARED` 2 -> 4, `LOCAL_ACCEPT` 2.0 -> 4.0. The +joint sweep below re-measures both axes together through the shipped +`DecisionPolicy` (`code/sweep_accept_lines.py`), on 4580 distractor crops from +64 unrelated Wikimedia shelf photos at the shipped configuration (1280-long-side +frames, `LiveSettings.conf=0.1`) and 109 image6 facings. An earlier version of +this table swept only `TEXT_MIN_SHARED` while pinning the accept line at 4.0, so +its "2.0" rows were not the old-line baseline and its "4.0 | 4" row duplicated +"2.0 | 4"; the numbers below replace it. -| `LOCAL_ACCEPT` | `TEXT_MIN_SHARED` | image6 shown / correct | precision | false accepts | -|---|---|---|---|---| -| 2.0 | 2 | 46 / 43 | 93.5 % | 82 (3.09 %) | -| 2.0 | 4 | 44 / 42 | 95.5 % | 8 (0.30 %) | -| **4.0** | **4** | **44 / 42** | **95.5 %** | **8 (0.30 %)** | -| 5.0 | 6 | 39 / 38 | 97.4 % | 3 (0.11 %) | +| `LOCAL_ACCEPT` | `TEXT_MIN_SHARED` | image6 full shown / correct | precision | coverage | image6 live precision | false accepts | +|---|---|---|---|---|---|---| +| 2.0 | 2 | 54 / 49 | 90.7 % | 76.6 % | 93.0 % | 60 (1.31 %) | +| 2.0 | 4 | 52 / 48 | 92.3 % | 75.0 % | 97.4 % | 8 (0.17 %) | +| 4.0 | 2 | 46 / 43 | 93.5 % | 67.2 % | 92.7 % | 55 (1.20 %) | +| **4.0** | **4** | **44 / 42** | **95.5 %** | **65.6 %** | **97.3 %** | **3 (0.07 %)** | +| 4.0 | 6 | 39 / 38 | 97.4 % | 59.4 % | 100 % | 0 (0.00 %) | +| 5.0 | 6 | 36 / 36 | 100 % | 56.2 % | 100 % | 0 (0.00 %) | -4.0/4 is the only region passing both G2 (>= 95 % precision) and G3 (<= 1 % -far-unknown) with coverage left. On the shipped evaluator that is 66 % -end-to-end at 95 % precision with 2 false accepts, against 75 % / 92 % / 4 at -the old 2.0 line - the coverage is what the precision costs, and G2 is the gate -that says to pay it. At live resolution the same lines give 56 % end-to-end at -97 % precision with 1 false accept, so the calibration holds on the path the -phone actually uses and not only at full resolution. +The shipped 4.0/4 row is the last one that passes G2 (>= 95 % precision) with +coverage left; 4.0/6 and 5.0/6 also pass both gates but give up 6-9 points of +coverage for precision the gate does not ask for. The old 2.0/2 line is 11 +points of coverage richer and fails G3 outright at 1.31 %. On the shipped +evaluator 4.0/4 is 66 % end-to-end at 95 % precision with 2 false accepts, +against 75 % / 92 % / 4 at the old 2.0 line - the coverage is what the precision +costs, and G2 is the gate that says to pay it. At live resolution the same lines +give 56 % end-to-end at 97 % precision with 1 false accept, so the calibration +holds on the path the phone actually uses and not only at full resolution. Reproduce: `.venv/bin/python code/eval_distractors.py --stock stock-lopia-clean --write data/external-shelves/wikimedia-commons data/external-shelves/wikimedia-commons-2`.