diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 932060f..51aa1ad 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -520,7 +520,9 @@ class RecognitionEngine: text_rank = sorted( text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True ) - return DecisionPolicy().decide(lane.rank(crop), text_rank, text_read=read) + return DecisionPolicy(signatures=text_lane.signatures if text_lane else None).decide( + lane.rank(crop), text_rank, text_read=read + ) def score_crop_lanes( self, crop: Any, method: Optional[str] = None, skip_ocr: bool = False diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index 1e12e51..05c7a30 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -397,7 +397,9 @@ class LocalLaneAdapter(Lane): self.text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True, ) - return DecisionPolicy(local_accept=accept).decide(ranked, text_rank, text_read=read) + return DecisionPolicy( + local_accept=accept, signatures=self.text_lane.signatures + ).decide(ranked, text_rank, text_read=read) def _catalog_rows(stock_dir: str) -> List[Dict[str, Any]]: diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index b730d9f..c921240 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -37,7 +37,7 @@ if current_dir not in sys.path: import cv2 # noqa: E402 import numpy as np # noqa: E402 -from rebuild_catalog import bigrams, normalize_text, sku_of # noqa: E402 +from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of # noqa: E402 # The local lane's accept line, in the units of `confidence()` below. Measured # on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59% @@ -49,6 +49,14 @@ LOCAL_ACCEPT = 2.0 # picks the right SKU on 41-42 of the 55 facings where it reads anything. TEXT_ACCEPT = 0.5 TEXT_MARGIN = 0.15 +# How many of the read's bigrams the winning row must actually contain. The +# ratio alone is not evidence: a two-pair read the row happens to contain scores +# 1.0, and "サソース" named sku_l08 that way. Two pairs is the floor that keeps +# every correct text answer on image6 (40 of 40) while dropping the wrong ones +# from 6 to 2, taking shown-label precision from 87% to 95%. It is a floor on +# evidence, not on the ratio, so a long read of shared brand copy still scores +# low and is still refused by the margin. +TEXT_MIN_SHARED = 2 # Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a # 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and # BRISK find zero inliers at live resolution. Scaling each reference to the @@ -274,7 +282,7 @@ def identity_bigrams(read: str) -> set: local lane has already supplied the evidence and the text only has to contradict it. """ - return {gram for gram in bigrams(_QUANTITY.sub("", normalize_text(read))) if not gram.isdigit()} + return {gram for gram in bigrams(_QUANTITY.sub(" ", normalize_text(read))) if not gram.isdigit()} def carries_identity(read: str) -> bool: @@ -289,7 +297,12 @@ def carries_identity(read: str) -> bool: return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR -def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float]] = None) -> float: +def text_similarity( + read: str, + signature: str, + weights: Optional[Dict[str, float]] = None, + vocabulary: Optional[Set[str]] = None, +) -> float: """Share of the read text's bigrams that the signature explains. Asymmetric on purpose: the crop shows a few words of a package, the @@ -304,11 +317,24 @@ def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float crop actually shows. Measured on image6: every l10 facing reads "ピュア セレクト マヨネーズ" with no "コクうま" and no "65", yet l09 tied or won on shared brand text alone. + + `vocabulary` is every bigram any signature can produce. A read's pairs + that no catalog name contains are OCR noise - a misread character, a + date, or a pair that spans two words the package prints separately + ("セレクト マヨネーズ" yields "トマ"). Counting them in the denominator + charges the row for text it could never have explained, which is how a + correct read of l10 fell to 0.46 while a garbage read of l11 stayed at + 0.24: the noise is proportional to how much was read, so the rows with + the most real text to match were punished hardest. Bounded this way the + same reads score 0.86 and 1.0 against their true rows, and the garbage + read ties across three rows and is refused by the margin. """ - read_grams = bigrams(normalize_text(read)) + read_grams = bigrams(read) + if vocabulary is not None: + read_grams &= vocabulary if not read_grams: return 0.0 - shared = read_grams & bigrams(normalize_text(signature)) + shared = read_grams & signature_bigrams(signature) if not shared: return 0.0 if weights is None: @@ -329,7 +355,7 @@ def bigram_weights(signatures: Sequence[str]) -> Dict[str, float]: """ counts: Dict[str, int] = {} for signature in signatures: - for gram in set(bigrams(normalize_text(signature))): + for gram in set(signature_bigrams(signature)): counts[gram] = counts.get(gram, 0) + 1 total = max(1, len(signatures)) return { @@ -358,6 +384,12 @@ class TextLane: # Bigram weights over the catalog's own names, so the lane can tell # copy every sibling prints from the words that name one product. self.weights = bigram_weights(list(self.signatures.values())) + # Every pair any name can produce. A read's pairs outside this set are + # OCR noise or word-boundary artifacts, and counting them in the + # denominator charges a row for text it could never have explained. + self.vocabulary: Set[str] = set() + for text in self.signatures.values(): + self.vocabulary |= signature_bigrams(text) self.ocr_engine = ocr_engine # The engine's OCR lock. Vision tolerates concurrency but the CPU does # not, and this lane runs one OCR per crop on the classify pool, so it @@ -420,7 +452,7 @@ class TextLane: if len(normalize_text(read)) < 4: return {} return { - sku: text_similarity(read, signature, self.weights) + sku: text_similarity(read, signature, self.weights, self.vocabulary) for sku, signature in self.signatures.items() } @@ -459,17 +491,31 @@ class DecisionPolicy: def __init__( self, - *, local_accept: float = LOCAL_ACCEPT, text_accept: float = TEXT_ACCEPT, text_margin: float = TEXT_MARGIN, + text_min_shared: int = TEXT_MIN_SHARED, + signatures: Optional[Dict[str, str]] = None, method: str = "rootsift", ) -> None: self.local_accept = float(local_accept) self.text_accept = float(text_accept) self.text_margin = float(text_margin) + self.text_min_shared = int(text_min_shared) + # The catalog names, so the evidence floor can ask how many of the + # read's pairs the winning row actually contains. Without them the + # floor cannot be applied and the ratio stands alone, which is the + # behaviour a caller that has no catalog is asking for. + self.signatures = {str(sku): str(text) for sku, text in (signatures or {}).items()} self.method = method + def _shared_evidence(self, read: str, sku: str) -> int: + """How many of the read's bigrams the row's name contains.""" + signature = self.signatures.get(sku) + if signature is None: + return self.text_min_shared + return len(bigrams(read) & signature_bigrams(signature)) + def decide( self, local_rank: Any, @@ -503,6 +549,7 @@ class DecisionPolicy: and text_top[1] >= self.text_accept and (text_top[1] - text_runner) >= self.text_margin and carries_identity(text_read or "") + and self._shared_evidence(text_read or "", text_top[0]) >= self.text_min_shared else None ) diff --git a/recognization/code/live.py b/recognization/code/live.py index 54e7421..4fcef5c 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -181,6 +181,31 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float: return inter / union if union > 0 else 0.0 +def _shift_box( + box: Sequence[int], + from_origin: Optional[Sequence[float]], + to_origin: Sequence[float], +) -> Optional[List[int]]: + """A box measured at one shelf origin, expressed at another. + + The origin is the shelf position of the frame's top-left corner, so the + content moves the opposite way: a shelf that has slid left by 12 px means + the same product now sits 12 px further left in the frame, and a box + measured before the slide has to move with it. + + Returns None when either origin is unknown. An uncompensated box is not a + weaker match, it is a match against the wrong product, so a caller that + cannot place the box must not use it at all. + """ + if from_origin is None or len(from_origin) < 2 or len(to_origin) < 2: + return None + dx = float(from_origin[0]) - float(to_origin[0]) + dy = float(from_origin[1]) - float(to_origin[1]) + x1, y1, x2, y2 = box + return [int(round(x1 + dx)), int(round(y1 + dy)), + int(round(x2 + dx)), int(round(y2 + dy))] + + def _clear_of_edge(box: Sequence[int], frame_shape: Sequence[int], margin_ratio: float) -> bool: """True when a box sits clear of the frame edge. @@ -670,11 +695,21 @@ class LiveSession: An inherited label is not a measurement, so it cannot spread to a neighbour. Blur, glare, and a too-far frame skip inheritance: the overlap is not evidence when the frame itself failed the quality gate. + + Memory boxes are stored in the coordinates of the frame they were + measured on, so they are shifted by the shelf movement since then before + they are compared. Without that, a panning camera slides the scene under + a stale box: at 12 px/frame a ~90 px facing moves a full width in about + seven frames, so the neighbour overlaps the old box at IoU >= 0.55 and + inherits a label that was never about it. Measured on image6 at pan 12, + every unknown false accept was an inherited label landing on the two + products next to the labelled one. """ if not self._label_memory: return if (self._last_quality or {}).get("hint"): return + origin = (self._frame_origin[0], self._frame_origin[1]) for track in self._tracks.values(): if track.state != STATE_DETECTED or track.confirmed: continue @@ -685,7 +720,12 @@ class LiveSession: continue if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU: continue - if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU: + moved = _shift_box(entry["box"], entry.get("origin"), origin) + if moved is None: + # The shelf moved too far for this entry to describe + # anything still on screen, so it cannot label a neighbour. + continue + if _iou(track.box, moved) < LABEL_MEMORY_IOU: continue track.state = STATE_RECOGNIZED track.confirmed = True @@ -723,6 +763,10 @@ class LiveSession: "confidence": track.identity.get("confidence"), "score": track.identity.get("score"), "seq": seq, + # The shelf origin this box was measured against. `box` is in + # that frame's pixels, so a later frame has to shift it by how + # far the shelf has moved since. + "origin": (self._frame_origin[0], self._frame_origin[1]), "inherited": bool(track.identity.get("inherited")), } ) @@ -829,10 +873,20 @@ class LiveSession: planned: List[Tuple[TrackState, Any, int]] = [] for track in eligible[:capacity]: crop = self.engine.crop(image, track.raw_box) - track.in_flight += 1 + # No label_frame_id here: the plan may be dropped (generation + # mismatch, a crop that went off product, a neighbour's label + # arriving first), and a frame id stamped on a track whose label + # never came from this frame would pair a box with the wrong + # offset. The callback carries the id, and the writers that set + # `label_box` set the id with it. track.last_attempt_seq = seq - track.label_frame_id = int(frame_id) track.last_attempt_area = track.area() + # Counted here, not in the submitter: this is the only place that + # knows a crop is about to be in flight, and the callback is what + # decrements it. Without the increment the guard above never blocks + # and every frame re-plans the same tracks, so the executor queue + # grows without bound and the server stops answering. + track.in_flight += 1 planned.append((track, crop, track.object_generation)) return planned @@ -846,14 +900,13 @@ class LiveSession: """Submit planned crops. Called WITHOUT the session lock held.""" for track, crop, object_generation in planned: submitted_at = time.perf_counter() - # The label anchors to the displayed (smoothed) box, which is what the - # drift check later compares against. + # The label anchors to the displayed (smoothed) box, which is what + # the drift check later compares against. submitted_box = list(track.box) track_id = track.track_id # OCR once per object: a track that already has a reading re-runs - # only the local lane, which is the difference between 5 ms and - # 25 ms per crop. A native crop is preferred for the one reading - # because it is the sharpest view the phone will send. + # only the local lane. The native crop lane always reads, because + # it is only asked for when the frame crop was too small to decide. skip_ocr = bool(track.text_read) try: future = self._executor.submit( @@ -1031,10 +1084,11 @@ class LiveSession: self._release_crop_slot(record) self._crops_rejected += 1 return _crop_error("CROP_STALE", stale) - # A native crop is the sharpest view the phone sends, so it is the - # one worth reading - but only once per object, like the frame path. - track = self._tracks.get(track_id_out) - skip_ocr = bool(track is not None and track.text_read) + # A native crop is the sharpest view the phone sends, and the lane + # only asks for one when the frame crop was too small to decide + # (`crop_below_px`), so any reading it would skip came from exactly + # the crop this one exists to replace. It always reads. + skip_ocr = False try: future = self._executor.submit( self.engine.score_crop_lanes, crop, method, skip_ocr @@ -1147,6 +1201,12 @@ class LiveSession: except Exception: scores, text_rank, text_read = {}, [], "" finished_at = time.perf_counter() + # Resolved before the session lock is taken. `identity_text_lane` builds + # the lane under its own lock on first use, and taking that while the + # session lock is held is a lock-ordering hazard: a frame loop that + # holds the session lock and waits on the lane lock while this callback + # holds the lane lock and waits on the session lock wedges the server. + text_lane = self.engine.identity_text_lane() with self._lock: track = self._tracks.get(track_id) if track is None or generation != self._generation: @@ -1181,14 +1241,17 @@ class LiveSession: track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score) if observation: track.score_n += 1 - self._classify_times.append(finished_at - submitted_at) - # The reading is cached on the track so later attempts skip OCR. # Only a read that could name something is cached: a crop that was # too small or too blurry says nothing about the next one, and a # read of just a size ("400g") matches every 400g product's size # line, so caching it would let one unreadable crop veto the # product for the rest of the track's life. + # + # The substitution happens before the early-out below and before + # the decision: a skipped OCR returns an empty ranking, and an + # empty ranking would both mark a text-only track unrecognized at + # max_attempts and stop the veto from firing on later attempts. if text_read and carries_identity(text_read): track.text_rank = list(text_rank or []) track.text_read = text_read @@ -1235,9 +1298,9 @@ class LiveSession: # chose. The newest crop is the evidence, not the mean: averaging # in an empty observation halves a good confidence below the # accept line, which is the dilution the cosine branch guards too. - decision = DecisionPolicy().decide( - list(scores.items()), text_rank, text_read=text_read - ) + decision = DecisionPolicy( + signatures=text_lane.signatures if text_lane else None + ).decide(list(scores.items()), text_rank, text_read=text_read) else: decision = decide_identity(mean_scores, method=method, scale=scale) track.candidates = [ diff --git a/recognization/code/rebuild_catalog.py b/recognization/code/rebuild_catalog.py index 4d05406..0eb0ff4 100644 --- a/recognization/code/rebuild_catalog.py +++ b/recognization/code/rebuild_catalog.py @@ -133,14 +133,40 @@ def normalize_text(text: str) -> str: return "".join(char for char in value if char.isalnum() or char == "%") +def signature_bigrams(text: str) -> Set[str]: + """Character pairs of a catalog name, never across a word boundary. + + A pair that spans two words is not a substring of either one, so it can + only ever be noise - and expensive noise, because it is rare across the + catalog and IDF therefore weights it as highly distinctive. "キユーピー + アマニ油" used to contribute "ピア", which let the garbage read + "キユピアク5C" be accepted as that row. + """ + pairs: Set[str] = set() + for token in text.split(): + value = normalize_text(token) + pairs.update(value[index:index + 2] for index in range(len(value) - 1)) + return pairs + + def bigrams(text: str) -> Set[str]: + """Character pairs of a read, across whitespace. + + Deliberately not `signature_bigrams`: OCR puts spaces where the package + has none, so "ピュア セレクト" has to keep matching the catalog's + "ピュアセレクト". The catalog name is authoritative about where its words + end; a read is noisy evidence about the same characters, and dropping its + spaces is the only way the two can meet. The read is the side that gets + the benefit of the doubt, so the fake-pair problem is fixed where it + originates - in the signature - rather than by refusing to match. + """ value = normalize_text(text) return {value[index:index + 2] for index in range(len(value) - 1)} def name_ocr_similarity(name: str, ocr_text: str) -> float: """Share of the catalog name's bigrams that the OCR text explains.""" - name_bigrams = bigrams(name) + name_bigrams = signature_bigrams(name) return len(name_bigrams & bigrams(ocr_text)) / max(1, len(name_bigrams)) diff --git a/recognization/code/test_engine.py b/recognization/code/test_engine.py index b014ed2..126106a 100644 --- a/recognization/code/test_engine.py +++ b/recognization/code/test_engine.py @@ -74,6 +74,99 @@ class TextLaneSignatureTests(unittest.TestCase): decision = DecisionPolicy().decide([], scores, text_read="マヨネーズ") self.assertIsNone(decision["best_sku"]) + def test_a_signature_never_pairs_across_two_words(self): + """A catalog name must not claim text it does not contain. + + "キューピー アマニ油" normalizes to "キユピ アマニ油"; with the space + dropped it was "キユピアマニ油", which contains "ピア" - a pair neither + word has. That pair is rare across the catalog, so IDF weighted it as + distinctive and the garbage read "キユピアク5C" was accepted as l02. + """ + from rebuild_catalog import signature_bigrams + + name = "キューピー アマニ油マヨネーズ 200g" + self.assertNotIn("ピア", signature_bigrams(name)) + self.assertIn("アマ", signature_bigrams(name)) + + def test_a_read_keeps_matching_across_the_spaces_ocr_inserts(self): + """The read is the noisy side, so its spaces must not break a match. + + Vision prints "ピュア セレクト" where the catalog name has + "ピュアセレクト". Refusing to pair across the read's spaces dropped + two correct l10 labels below the accept line, so the read pairs + across them and the signature is the side that stays strict. + """ + from rebuild_catalog import bigrams + + self.assertIn("アセ", bigrams("ピュア セレクト マヨネーズ")) + + def test_a_garbage_read_does_not_name_a_row(self): + """The read that the fake pair used to accept must not name a row. + + It shares only "キユ" and "ユピ" with l02's name, both of which every + Kewpie row prints. Bounded to the catalog's own vocabulary the read + keeps just those two pairs, so it scores the same against every + Kewpie row and the margin gate refuses it - the same mechanism that + already handles a read of shared brand copy alone. + """ + from identity_lanes import DecisionPolicy, TextLane + + rows = self.ROWS + [{"sku_id": "sku_l02", "product_name": "キューピー アマニ油マヨネーズ 200g"}] + read = "キユピアク5C" + scores = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None).scores_from_text(read) + ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True) + self.assertEqual(ranked[0][1], ranked[1][1], "the read cannot tell the Kewpie rows apart") + decision = DecisionPolicy().decide([], ranked, text_read=read) + self.assertIsNone(decision["best_sku"]) + + def test_a_read_of_real_text_still_names_its_row(self): + """Bounding the denominator must not cost a correct read its answer. + + This is the facing the strict-read variant broke: l10's own name read + with OCR's spacing, which has to clear the accept line by margin. + """ + from identity_lanes import TEXT_ACCEPT, TEXT_MARGIN, TextLane + + rows = self.ROWS + [{"sku_id": "sku_l02", "product_name": "キューピー アマニ油マヨネーズ 200g"}] + scores = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None).scores_from_text( + "213日以月A ピュア セレクト マヨネーズ" + ) + ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True) + self.assertEqual(ranked[0][0], "sku_l10") + self.assertGreaterEqual(ranked[0][1], TEXT_ACCEPT) + self.assertGreaterEqual(ranked[0][1] - ranked[1][1], TEXT_MARGIN) + + def test_one_shared_pair_is_not_enough_to_name_a_row(self): + """A ratio of 1.0 on a single pair is not evidence. + + "XXイス" contains exactly one pair the catalog knows, and the row that + contains it scores 1.0 with a full margin over every other row. On + image6 that shape named sku_l08 from the read "サソース". The floor + asks how much of the read the row actually explains, so one pair is + refused while the same read with two is accepted. + """ + from identity_lanes import DecisionPolicy, TextLane + + rows = [ + {"sku_id": "sku_l08", "product_name": "オイスターソース"}, + {"sku_id": "sku_l01", "product_name": "キューピー マヨネーズ"}, + ] + lane = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None) + + thin = sorted(lane.scores_from_text("XXイス").items(), key=lambda kv: kv[1], reverse=True) + self.assertEqual(thin[0][1], 1.0, "the ratio alone cannot see the difference") + self.assertIsNone( + DecisionPolicy(signatures=lane.signatures).decide([], thin, text_read="XXイス")["best_sku"] + ) + + enough = sorted(lane.scores_from_text("ZZソース").items(), key=lambda kv: kv[1], reverse=True) + self.assertEqual( + DecisionPolicy(signatures=lane.signatures).decide( + [], enough, text_read="ZZソース" + )["best_sku"], + "sku_l08", + ) + class FakeDetector: """Records concurrency and returns fixed boxes.""" diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index ecf721e..4357776 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -128,11 +128,10 @@ class FakeInventory: def __init__(self): self.products = { - "sku_a": {"sku_id": "sku_a", "product_name": "Product sku_a", "price": 150.0, + "sku_a": {"sku_id": "sku_a", "product_name": "コクうま カロリーカット65%", "price": 150.0, "number_in": 10, "number_out": 1, "number_on": 9, "receipt_date": "", "expiration_date": "", "local_image_path": None}, - "other": {"sku_id": "other", "product_name": "Product other", "price": 150.0, - "number_in": 10, "number_out": 1, "number_on": 9, + "other": {"sku_id": "other", "product_name": "アマニ油マヨネーズ", "price": 150.0, "receipt_date": "", "expiration_date": "", "local_image_path": None}, } @@ -266,6 +265,36 @@ class LabelLifecycleTests(unittest.TestCase): == "recognized") engine.shutdown() + def test_a_pan_does_not_let_a_neighbour_inherit(self): + """A label must not slide onto whatever moved into its screen position. + + The memory box is stored in the coordinates of the frame it was + measured on. Under a pan the shelf slides beneath that box, so the + product now occupying the old screen position is a different one. + Comparing the raw box hands it the label at IoU 1.0; compensating by + the shelf movement drops the overlap to nothing. + """ + session, engine, detector, matcher = make_session() + for frame_id in range(1, 5): + process(session, frame_id) + self.assertTrue(wait_for(lambda: process(session, 5)["tracks"][0]["identity"]["state"] + == "recognized")) + # The product leaves, so its track ages out and the next detection at + # that screen position is a new object rather than the same one. + detector.boxes = [] + for frame_id in range(6, 10): + process(session, frame_id) + # The shelf slides 90 px: the labelled product is now 90 px to the + # left, and the screen position it used to occupy holds its neighbour. + session._frame_origin = [90.0, 0.0] + detector.boxes = [{"id": 0, "box": [100, 100, 200, 260], "confidence": 0.9, "class_id": 0, + "width": 100, "height": 160, "center": [150, 180]}] + result = process(session, 10) + row = result["tracks"][0] + self.assertFalse(row["inherited"], + "the product that slid into the old box is not the labelled one") + engine.shutdown() + class CosineScaleTests(unittest.TestCase): """A gallery-scale score is gated by the gallery, not re-gated by the live path."""