diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index caedb3a..ba6b74c 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -25,6 +25,7 @@ calibrated model. E-4 replaces it once Tier-2 held-out labels exist. from __future__ import annotations import os +import re import sys import unicodedata from dataclasses import dataclass @@ -62,6 +63,11 @@ REPROJ_FRACTION = 0.02 HUE_BINS = 18 HUE_SATURATION_MIN = 60 HUE_VALUE_MIN = 50 +# Quantity tokens: a number with its unit, in either script. Cut whole, because +# the unit's own letters ("g", "ml", "%") are what survived a digits-only rule. +_QUANTITY = re.compile(r"\d+(?:\.\d+)?\s*(?:KG|G|ML|L|%|個|本|枚|入|袋|缶|パック)") +# How many surviving bigrams a read needs before it may name a product. +IDENTITY_BIGRAM_FLOOR = 3 @dataclass @@ -263,29 +269,30 @@ def bigrams(text: str) -> set: return {text[i:i + 2] for i in range(len(text) - 1)} -def identity_bigrams(text: str) -> set: - """Bigrams carrying a non-digit character. +def identity_bigrams(read: str) -> set: + """Bigrams carrying a non-digit character, after quantity tokens are cut. - A crop reading only "400g" matches every 400g product's size line, so size - and number tokens must not name a product on their own. They still count - for the veto, where the local lane has already supplied the evidence and - the text only has to contradict it. + A crop reading "400g" matches every 400g product's size line, so size and + number tokens must not name a product on their own. Cutting the whole + quantity token - not just the digits - is what makes that true: "400g 50%" + leaves nothing behind, where counting non-digit bigrams alone left "0G" and + "0%" and let the read through. They still count for the veto, where the + local lane has already supplied the evidence and the text only has to + contradict it. """ - return {gram for gram in bigrams(text) if not gram.isdigit()} - + return {gram for gram in bigrams(_QUANTITY.sub("", normalize_text(read))) if not gram.isdigit()} def carries_identity(read: str) -> bool: - """Whether a read holds enough non-numeric text to name a product. + """Whether a read holds enough name text to answer for a product. - A crop reading only "400g" matches every 400g product's size line, so size - and number tokens must not answer on their own. Two non-digit bigrams is - the floor: "400g" has one ("0g") and is refused, while "コクうま 65%" has - five and still names l09. The veto is unaffected - there the local lane has - already supplied the evidence and the text only has to contradict it, so - "65%" stays usable. + Two bigrams is not enough: "カロリ" is two, and it is a descriptor that any + low-calorie package prints, not a name. Three is the floor - four + characters of surviving text - which keeps "コクウマ" (from "コクうま 65%") + and refuses "カロリ", "ハーフ" and every pure quantity string. The veto is + unaffected: there the local lane has already supplied the evidence and the """ - return len(identity_bigrams(normalize_text(read))) >= 2 + return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR def text_similarity(read: str, signature: str) -> float: @@ -377,18 +384,6 @@ class TextLane: """Bigram similarity per catalog row. Empty when nothing was read.""" return self.scores_from_text(self.read(crop)) - def carries_identity(self, read: str) -> bool: - """Whether a read holds enough non-numeric text to name a product. - - A crop reading only "400g" matches every 400g product's size line, so - size and number tokens must not answer on their own. Two non-digit - bigrams is the floor: "400g" has one ("0g") and is refused, while - "コクうま 65%" has five and still names l09. The veto is unaffected - - there the local lane has already supplied the evidence and the text - only has to contradict it, so "65%" stays usable. - """ - return len(identity_bigrams(normalize_text(read))) >= 2 - def scores_from_text(self, read: str) -> Dict[str, float]: if len(normalize_text(read)) < 4: return {} @@ -455,7 +450,9 @@ class DecisionPolicy: `text_read` is the raw OCR text behind `text_rank`. It gates the fill-in job only: a read that is nothing but a size or a number cannot - name a product, but it can still veto one. + name a product, but it can still veto one. Omitting it means no + fill-in - a caller that has a ranking and no read has not shown the + text can name anything, so the safe reading is the default one. """ from identity import ( IDENTITY_ACCEPTED, @@ -473,7 +470,7 @@ class DecisionPolicy: if text_top and text_top[1] >= self.text_accept and (text_top[1] - text_runner) >= self.text_margin - and (text_read is None or carries_identity(text_read)) + and carries_identity(text_read or "") else None ) diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index 7806fd0..f814299 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -427,7 +427,9 @@ class DecisionPolicyTests(unittest.TestCase): row - the two jobs compose, so the result is the text's SKU, never the vetoed local one. """ - decision = self.policy().decide([("sku_a", 5.0)], [("sku_b", 0.9)]) + decision = self.policy().decide( + [("sku_a", 5.0)], [("sku_b", 0.9)], text_read="コクうま 65%" + ) self.assertEqual(decision["best_sku"], "sku_b") self.assertEqual(decision["source"], "text") @@ -470,10 +472,38 @@ class DecisionPolicyTests(unittest.TestCase): self.assertEqual(decision["best_sku"], "sku_b") def test_text_fills_in_when_local_has_nothing(self): - decision = self.policy().decide([], [("sku_b", 0.9)]) + decision = self.policy().decide([], [("sku_b", 0.9)], text_read="コクうま 65%") self.assertEqual(decision["best_sku"], "sku_b") self.assertEqual(decision["source"], "text") + def test_a_ranking_without_a_read_does_not_fill_in(self): + """Omitting the read is the safe default, not an ungated one. + + A caller that has a ranking and no read has not shown the text can name + anything, so the fill-in job stays closed. + """ + decision = self.policy().decide([], [("sku_b", 0.9)]) + self.assertIsNone(decision["best_sku"]) + + def test_quantity_strings_are_not_names(self): + """Every pure quantity string, including the ones a digits-only rule let through. + + "400g 50%" and "450g カロリー50%" survived the first version of this + gate: it counted non-digit bigrams, and the unit's own letters ("0G", + "0%") are non-digits. Cutting the whole quantity token is what closes + it. "450g カロリー50%" still passes, and correctly so - "カロリー" is + four characters of real text, and it is the text lane's job to find + that no catalog row is named by it. + """ + from identity_lanes import carries_identity + + for read in ("400g", "400g 50%", "500ml", "65%", "3日", "中辛"): + with self.subTest(read=read): + self.assertFalse(carries_identity(read), f"{read!r} is not a name") + for read in ("コクうま 65%", "450g カロリー50%"): + with self.subTest(read=read): + self.assertTrue(carries_identity(read), f"{read!r} carries name text") + def test_text_below_its_own_line_does_not_fill_in(self): decision = self.policy().decide([], [("sku_b", 0.3)]) self.assertIsNone(decision["best_sku"])