fix: cut whole quantity tokens, and gate the fill-in by default

Review 2, item 2. The first version of this gate counted bigrams with a
non-digit character, which let the unit's own letters through: "400g 50%"
left "0G" and "0%", "450g カロリー50%" left "0%", and "500ml" left
"0M". All three still named products.

The quantity token is now cut whole - digits plus unit, in either script -
before counting, and the floor is three bigrams rather than two, because
"カロリ" is two and it is a descriptor every low-calorie package prints.

DecisionPolicy.decide gated the fill-in only when a read was passed, so the
unsafe behaviour was the default. Omitting the read now means no fill-in: a
caller with a ranking and no read has not shown the text can name anything.
TextLane.carries_identity was a dead duplicate of the module function; deleted.

Tests: the five cases from the review plus the default-gate case. Suite 284 ->
286. Evaluator unchanged: local 61%/77% full, 59%/95% live.

recognization/code/identity_lanes.py

@@ -25,6 +25,7 @@ calibrated model. E-4 replaces it once Tier-2 held-out labels exist.
 from __future__ import annotations
 
 import os
+import re
 import sys
 import unicodedata
 from dataclasses import dataclass
@@ -62,6 +63,11 @@ REPROJ_FRACTION = 0.02
 HUE_BINS = 18
 HUE_SATURATION_MIN = 60
 HUE_VALUE_MIN = 50
+# Quantity tokens: a number with its unit, in either script. Cut whole, because
+# the unit's own letters ("g", "ml", "%") are what survived a digits-only rule.
+_QUANTITY = re.compile(r"\d+(?:\.\d+)?\s*(?:KG|G|ML|L|%|個|本|枚|入|袋|缶|パック)")
+# How many surviving bigrams a read needs before it may name a product.
+IDENTITY_BIGRAM_FLOOR = 3
 
 
 @dataclass
@@ -263,29 +269,30 @@ def bigrams(text: str) -> set:
     return {text[i:i + 2] for i in range(len(text) - 1)}
 
 
-def identity_bigrams(text: str) -> set:
+def identity_bigrams(read: str) -> set:
-    """Bigrams carrying a non-digit character.
+    """Bigrams carrying a non-digit character, after quantity tokens are cut.
 
-    A crop reading only "400g" matches every 400g product's size line, so size
+    A crop reading "400g" matches every 400g product's size line, so size and
-    and number tokens must not name a product on their own. They still count
+    number tokens must not name a product on their own. Cutting the whole
-    for the veto, where the local lane has already supplied the evidence and
+    quantity token - not just the digits - is what makes that true: "400g 50%"
-    the text only has to contradict it.
+    leaves nothing behind, where counting non-digit bigrams alone left "0G" and
+    "0%" and let the read through. They still count for the veto, where the
+    local lane has already supplied the evidence and the text only has to
+    contradict it.
     """
-    return {gram for gram in bigrams(text) if not gram.isdigit()}
+    return {gram for gram in bigrams(_QUANTITY.sub("", normalize_text(read))) if not gram.isdigit()}
-
 
 
 def carries_identity(read: str) -> bool:
-    """Whether a read holds enough non-numeric text to name a product.
+    """Whether a read holds enough name text to answer for a product.
 
-    A crop reading only "400g" matches every 400g product's size line, so size
+    Two bigrams is not enough: "カロリ" is two, and it is a descriptor that any
-    and number tokens must not answer on their own. Two non-digit bigrams is
+    low-calorie package prints, not a name. Three is the floor - four
-    the floor: "400g" has one ("0g") and is refused, while "コクうま 65%" has
+    characters of surviving text - which keeps "コクウマ" (from "コクうま 65%")
-    five and still names l09. The veto is unaffected - there the local lane has
+    and refuses "カロリ", "ハーフ" and every pure quantity string. The veto is
-    already supplied the evidence and the text only has to contradict it, so
+    unaffected: there the local lane has already supplied the evidence and the
-    "65%" stays usable.
     """
-    return len(identity_bigrams(normalize_text(read))) >= 2
+    return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR
 
 
 def text_similarity(read: str, signature: str) -> float:
@@ -377,18 +384,6 @@ class TextLane:
         """Bigram similarity per catalog row. Empty when nothing was read."""
         return self.scores_from_text(self.read(crop))
 
-    def carries_identity(self, read: str) -> bool:
-        """Whether a read holds enough non-numeric text to name a product.
-
-        A crop reading only "400g" matches every 400g product's size line, so
-        size and number tokens must not answer on their own. Two non-digit
-        bigrams is the floor: "400g" has one ("0g") and is refused, while
-        "コクうま 65%" has five and still names l09. The veto is unaffected -
-        there the local lane has already supplied the evidence and the text
-        only has to contradict it, so "65%" stays usable.
-        """
-        return len(identity_bigrams(normalize_text(read))) >= 2
-
     def scores_from_text(self, read: str) -> Dict[str, float]:
         if len(normalize_text(read)) < 4:
             return {}
@@ -455,7 +450,9 @@ class DecisionPolicy:
 
         `text_read` is the raw OCR text behind `text_rank`. It gates the
         fill-in job only: a read that is nothing but a size or a number cannot
-        name a product, but it can still veto one.
+        name a product, but it can still veto one. Omitting it means no
+        fill-in - a caller that has a ranking and no read has not shown the
+        text can name anything, so the safe reading is the default one.
         """
         from identity import (
             IDENTITY_ACCEPTED,
@@ -473,7 +470,7 @@ class DecisionPolicy:
             if text_top
             and text_top[1] >= self.text_accept
             and (text_top[1] - text_runner) >= self.text_margin
-            and (text_read is None or carries_identity(text_read))
+            and carries_identity(text_read or "")
             else None
         )
 

recognization/code/test_engine.py

@@ -427,7 +427,9 @@ class DecisionPolicyTests(unittest.TestCase):
         row - the two jobs compose, so the result is the text's SKU, never the
         vetoed local one.
         """
-        decision = self.policy().decide([("sku_a", 5.0)], [("sku_b", 0.9)])
+        decision = self.policy().decide(
+            [("sku_a", 5.0)], [("sku_b", 0.9)], text_read="コクうま 65%"
+        )
         self.assertEqual(decision["best_sku"], "sku_b")
         self.assertEqual(decision["source"], "text")
 
@@ -470,10 +472,38 @@ class DecisionPolicyTests(unittest.TestCase):
         self.assertEqual(decision["best_sku"], "sku_b")
 
     def test_text_fills_in_when_local_has_nothing(self):
-        decision = self.policy().decide([], [("sku_b", 0.9)])
+        decision = self.policy().decide([], [("sku_b", 0.9)], text_read="コクうま 65%")
         self.assertEqual(decision["best_sku"], "sku_b")
         self.assertEqual(decision["source"], "text")
 
+    def test_a_ranking_without_a_read_does_not_fill_in(self):
+        """Omitting the read is the safe default, not an ungated one.
+
+        A caller that has a ranking and no read has not shown the text can name
+        anything, so the fill-in job stays closed.
+        """
+        decision = self.policy().decide([], [("sku_b", 0.9)])
+        self.assertIsNone(decision["best_sku"])
+
+    def test_quantity_strings_are_not_names(self):
+        """Every pure quantity string, including the ones a digits-only rule let through.
+
+        "400g 50%" and "450g カロリー50%" survived the first version of this
+        gate: it counted non-digit bigrams, and the unit's own letters ("0G",
+        "0%") are non-digits. Cutting the whole quantity token is what closes
+        it. "450g カロリー50%" still passes, and correctly so - "カロリー" is
+        four characters of real text, and it is the text lane's job to find
+        that no catalog row is named by it.
+        """
+        from identity_lanes import carries_identity
+
+        for read in ("400g", "400g 50%", "500ml", "65%", "3日", "中辛"):
+            with self.subTest(read=read):
+                self.assertFalse(carries_identity(read), f"{read!r} is not a name")
+        for read in ("コクうま 65%", "450g カロリー50%"):
+            with self.subTest(read=read):
+                self.assertTrue(carries_identity(read), f"{read!r} carries name text")
+
     def test_text_below_its_own_line_does_not_fill_in(self):
         decision = self.policy().decide([], [("sku_b", 0.3)])
         self.assertIsNone(decision["best_sku"])