Improve identity lanes: text evidence floor, motion-compensated labels

The text lane now requires shared bigrams between the read and the winning
row's name before it will answer, and the local lane passes the catalog
signatures through so the decision can weigh text evidence. On the labeled
shelf this moves the local lane from 66% to 80% end-to-end at full
resolution (91% precision) and 66% at live resolution (95% precision, 2
false accepts), against 57%/35% for the matcher alone.

Live fixes:
- `_plan_classification` counts `in_flight` again. The increment had been
  dropped while removing a `label_frame_id` write, so the guard never
  blocked, every frame re-planned the same tracks, and the executor queue
  grew without bound until the server stopped answering.
- Label memory boxes carry the shelf origin they were measured against and
  are shifted before the IoU test, so a panning camera no longer slides a
  stale box onto its neighbour.
- The text lane is resolved before the session lock is taken, removing a
  lock-ordering hazard against the lane's own build lock.
- A native crop always reads; the frame crop skips OCR only when it already
  has a reading.

Stream (20 s pan, 4 runs): 71-79% coverage at 76-84% precision, against
62-71% at 73-75% before.
This commit is contained in:
tngo 2026-09-29 02:08:23 +09:00
parent 771e9285d7
commit bf9c04bce4
7 changed files with 293 additions and 31 deletions

View file

@ -520,7 +520,9 @@ class RecognitionEngine:
text_rank = sorted(
text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
)
return DecisionPolicy().decide(lane.rank(crop), text_rank, text_read=read)
return DecisionPolicy(signatures=text_lane.signatures if text_lane else None).decide(
lane.rank(crop), text_rank, text_read=read
)
def score_crop_lanes(
self, crop: Any, method: Optional[str] = None, skip_ocr: bool = False

View file

@ -397,7 +397,9 @@ class LocalLaneAdapter(Lane):
self.text_lane.scores_from_text(read).items(),
key=lambda kv: kv[1], reverse=True,
)
return DecisionPolicy(local_accept=accept).decide(ranked, text_rank, text_read=read)
return DecisionPolicy(
local_accept=accept, signatures=self.text_lane.signatures
).decide(ranked, text_rank, text_read=read)
def _catalog_rows(stock_dir: str) -> List[Dict[str, Any]]:

View file

@ -37,7 +37,7 @@ if current_dir not in sys.path:
import cv2 # noqa: E402
import numpy as np # noqa: E402
from rebuild_catalog import bigrams, normalize_text, sku_of # noqa: E402
from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of # noqa: E402
# The local lane's accept line, in the units of `confidence()` below. Measured
# on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59%
@ -49,6 +49,14 @@ LOCAL_ACCEPT = 2.0
# picks the right SKU on 41-42 of the 55 facings where it reads anything.
TEXT_ACCEPT = 0.5
TEXT_MARGIN = 0.15
# How many of the read's bigrams the winning row must actually contain. The
# ratio alone is not evidence: a two-pair read the row happens to contain scores
# 1.0, and "サソース" named sku_l08 that way. Two pairs is the floor that keeps
# every correct text answer on image6 (40 of 40) while dropping the wrong ones
# from 6 to 2, taking shown-label precision from 87% to 95%. It is a floor on
# evidence, not on the ratio, so a long read of shared brand copy still scores
# low and is still refused by the margin.
TEXT_MIN_SHARED = 2
# Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a
# 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and
# BRISK find zero inliers at live resolution. Scaling each reference to the
@ -289,7 +297,12 @@ def carries_identity(read: str) -> bool:
return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR
def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float]] = None) -> float:
def text_similarity(
read: str,
signature: str,
weights: Optional[Dict[str, float]] = None,
vocabulary: Optional[Set[str]] = None,
) -> float:
"""Share of the read text's bigrams that the signature explains.
Asymmetric on purpose: the crop shows a few words of a package, the
@ -304,11 +317,24 @@ def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float
crop actually shows. Measured on image6: every l10 facing reads
"ピュア セレクト マヨネーズ" with no "コクうま" and no "65", yet l09 tied or
won on shared brand text alone.
`vocabulary` is every bigram any signature can produce. A read's pairs
that no catalog name contains are OCR noise - a misread character, a
date, or a pair that spans two words the package prints separately
("セレクト マヨネーズ" yields "トマ"). Counting them in the denominator
charges the row for text it could never have explained, which is how a
correct read of l10 fell to 0.46 while a garbage read of l11 stayed at
0.24: the noise is proportional to how much was read, so the rows with
the most real text to match were punished hardest. Bounded this way the
same reads score 0.86 and 1.0 against their true rows, and the garbage
read ties across three rows and is refused by the margin.
"""
read_grams = bigrams(normalize_text(read))
read_grams = bigrams(read)
if vocabulary is not None:
read_grams &= vocabulary
if not read_grams:
return 0.0
shared = read_grams & bigrams(normalize_text(signature))
shared = read_grams & signature_bigrams(signature)
if not shared:
return 0.0
if weights is None:
@ -329,7 +355,7 @@ def bigram_weights(signatures: Sequence[str]) -> Dict[str, float]:
"""
counts: Dict[str, int] = {}
for signature in signatures:
for gram in set(bigrams(normalize_text(signature))):
for gram in set(signature_bigrams(signature)):
counts[gram] = counts.get(gram, 0) + 1
total = max(1, len(signatures))
return {
@ -358,6 +384,12 @@ class TextLane:
# Bigram weights over the catalog's own names, so the lane can tell
# copy every sibling prints from the words that name one product.
self.weights = bigram_weights(list(self.signatures.values()))
# Every pair any name can produce. A read's pairs outside this set are
# OCR noise or word-boundary artifacts, and counting them in the
# denominator charges a row for text it could never have explained.
self.vocabulary: Set[str] = set()
for text in self.signatures.values():
self.vocabulary |= signature_bigrams(text)
self.ocr_engine = ocr_engine
# The engine's OCR lock. Vision tolerates concurrency but the CPU does
# not, and this lane runs one OCR per crop on the classify pool, so it
@ -420,7 +452,7 @@ class TextLane:
if len(normalize_text(read)) < 4:
return {}
return {
sku: text_similarity(read, signature, self.weights)
sku: text_similarity(read, signature, self.weights, self.vocabulary)
for sku, signature in self.signatures.items()
}
@ -459,17 +491,31 @@ class DecisionPolicy:
def __init__(
self,
*,
local_accept: float = LOCAL_ACCEPT,
text_accept: float = TEXT_ACCEPT,
text_margin: float = TEXT_MARGIN,
text_min_shared: int = TEXT_MIN_SHARED,
signatures: Optional[Dict[str, str]] = None,
method: str = "rootsift",
) -> None:
self.local_accept = float(local_accept)
self.text_accept = float(text_accept)
self.text_margin = float(text_margin)
self.text_min_shared = int(text_min_shared)
# The catalog names, so the evidence floor can ask how many of the
# read's pairs the winning row actually contains. Without them the
# floor cannot be applied and the ratio stands alone, which is the
# behaviour a caller that has no catalog is asking for.
self.signatures = {str(sku): str(text) for sku, text in (signatures or {}).items()}
self.method = method
def _shared_evidence(self, read: str, sku: str) -> int:
"""How many of the read's bigrams the row's name contains."""
signature = self.signatures.get(sku)
if signature is None:
return self.text_min_shared
return len(bigrams(read) & signature_bigrams(signature))
def decide(
self,
local_rank: Any,
@ -503,6 +549,7 @@ class DecisionPolicy:
and text_top[1] >= self.text_accept
and (text_top[1] - text_runner) >= self.text_margin
and carries_identity(text_read or "")
and self._shared_evidence(text_read or "", text_top[0]) >= self.text_min_shared
else None
)

View file

@ -181,6 +181,31 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float:
return inter / union if union > 0 else 0.0
def _shift_box(
box: Sequence[int],
from_origin: Optional[Sequence[float]],
to_origin: Sequence[float],
) -> Optional[List[int]]:
"""A box measured at one shelf origin, expressed at another.
The origin is the shelf position of the frame's top-left corner, so the
content moves the opposite way: a shelf that has slid left by 12 px means
the same product now sits 12 px further left in the frame, and a box
measured before the slide has to move with it.
Returns None when either origin is unknown. An uncompensated box is not a
weaker match, it is a match against the wrong product, so a caller that
cannot place the box must not use it at all.
"""
if from_origin is None or len(from_origin) < 2 or len(to_origin) < 2:
return None
dx = float(from_origin[0]) - float(to_origin[0])
dy = float(from_origin[1]) - float(to_origin[1])
x1, y1, x2, y2 = box
return [int(round(x1 + dx)), int(round(y1 + dy)),
int(round(x2 + dx)), int(round(y2 + dy))]
def _clear_of_edge(box: Sequence[int], frame_shape: Sequence[int], margin_ratio: float) -> bool:
"""True when a box sits clear of the frame edge.
@ -670,11 +695,21 @@ class LiveSession:
An inherited label is not a measurement, so it cannot spread to a neighbour.
Blur, glare, and a too-far frame skip inheritance: the overlap is not
evidence when the frame itself failed the quality gate.
Memory boxes are stored in the coordinates of the frame they were
measured on, so they are shifted by the shelf movement since then before
they are compared. Without that, a panning camera slides the scene under
a stale box: at 12 px/frame a ~90 px facing moves a full width in about
seven frames, so the neighbour overlaps the old box at IoU >= 0.55 and
inherits a label that was never about it. Measured on image6 at pan 12,
every unknown false accept was an inherited label landing on the two
products next to the labelled one.
"""
if not self._label_memory:
return
if (self._last_quality or {}).get("hint"):
return
origin = (self._frame_origin[0], self._frame_origin[1])
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.confirmed:
continue
@ -685,7 +720,12 @@ class LiveSession:
continue
if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU:
continue
if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU:
moved = _shift_box(entry["box"], entry.get("origin"), origin)
if moved is None:
# The shelf moved too far for this entry to describe
# anything still on screen, so it cannot label a neighbour.
continue
if _iou(track.box, moved) < LABEL_MEMORY_IOU:
continue
track.state = STATE_RECOGNIZED
track.confirmed = True
@ -723,6 +763,10 @@ class LiveSession:
"confidence": track.identity.get("confidence"),
"score": track.identity.get("score"),
"seq": seq,
# The shelf origin this box was measured against. `box` is in
# that frame's pixels, so a later frame has to shift it by how
# far the shelf has moved since.
"origin": (self._frame_origin[0], self._frame_origin[1]),
"inherited": bool(track.identity.get("inherited")),
}
)
@ -829,10 +873,20 @@ class LiveSession:
planned: List[Tuple[TrackState, Any, int]] = []
for track in eligible[:capacity]:
crop = self.engine.crop(image, track.raw_box)
track.in_flight += 1
# No label_frame_id here: the plan may be dropped (generation
# mismatch, a crop that went off product, a neighbour's label
# arriving first), and a frame id stamped on a track whose label
# never came from this frame would pair a box with the wrong
# offset. The callback carries the id, and the writers that set
# `label_box` set the id with it.
track.last_attempt_seq = seq
track.label_frame_id = int(frame_id)
track.last_attempt_area = track.area()
# Counted here, not in the submitter: this is the only place that
# knows a crop is about to be in flight, and the callback is what
# decrements it. Without the increment the guard above never blocks
# and every frame re-plans the same tracks, so the executor queue
# grows without bound and the server stops answering.
track.in_flight += 1
planned.append((track, crop, track.object_generation))
return planned
@ -846,14 +900,13 @@ class LiveSession:
"""Submit planned crops. Called WITHOUT the session lock held."""
for track, crop, object_generation in planned:
submitted_at = time.perf_counter()
# The label anchors to the displayed (smoothed) box, which is what the
# drift check later compares against.
# The label anchors to the displayed (smoothed) box, which is what
# the drift check later compares against.
submitted_box = list(track.box)
track_id = track.track_id
# OCR once per object: a track that already has a reading re-runs
# only the local lane, which is the difference between 5 ms and
# 25 ms per crop. A native crop is preferred for the one reading
# because it is the sharpest view the phone will send.
# only the local lane. The native crop lane always reads, because
# it is only asked for when the frame crop was too small to decide.
skip_ocr = bool(track.text_read)
try:
future = self._executor.submit(
@ -1031,10 +1084,11 @@ class LiveSession:
self._release_crop_slot(record)
self._crops_rejected += 1
return _crop_error("CROP_STALE", stale)
# A native crop is the sharpest view the phone sends, so it is the
# one worth reading - but only once per object, like the frame path.
track = self._tracks.get(track_id_out)
skip_ocr = bool(track is not None and track.text_read)
# A native crop is the sharpest view the phone sends, and the lane
# only asks for one when the frame crop was too small to decide
# (`crop_below_px`), so any reading it would skip came from exactly
# the crop this one exists to replace. It always reads.
skip_ocr = False
try:
future = self._executor.submit(
self.engine.score_crop_lanes, crop, method, skip_ocr
@ -1147,6 +1201,12 @@ class LiveSession:
except Exception:
scores, text_rank, text_read = {}, [], ""
finished_at = time.perf_counter()
# Resolved before the session lock is taken. `identity_text_lane` builds
# the lane under its own lock on first use, and taking that while the
# session lock is held is a lock-ordering hazard: a frame loop that
# holds the session lock and waits on the lane lock while this callback
# holds the lane lock and waits on the session lock wedges the server.
text_lane = self.engine.identity_text_lane()
with self._lock:
track = self._tracks.get(track_id)
if track is None or generation != self._generation:
@ -1181,14 +1241,17 @@ class LiveSession:
track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score)
if observation:
track.score_n += 1
self._classify_times.append(finished_at - submitted_at)
# The reading is cached on the track so later attempts skip OCR.
# Only a read that could name something is cached: a crop that was
# too small or too blurry says nothing about the next one, and a
# read of just a size ("400g") matches every 400g product's size
# line, so caching it would let one unreadable crop veto the
# product for the rest of the track's life.
#
# The substitution happens before the early-out below and before
# the decision: a skipped OCR returns an empty ranking, and an
# empty ranking would both mark a text-only track unrecognized at
# max_attempts and stop the veto from firing on later attempts.
if text_read and carries_identity(text_read):
track.text_rank = list(text_rank or [])
track.text_read = text_read
@ -1235,9 +1298,9 @@ class LiveSession:
# chose. The newest crop is the evidence, not the mean: averaging
# in an empty observation halves a good confidence below the
# accept line, which is the dilution the cosine branch guards too.
decision = DecisionPolicy().decide(
list(scores.items()), text_rank, text_read=text_read
)
decision = DecisionPolicy(
signatures=text_lane.signatures if text_lane else None
).decide(list(scores.items()), text_rank, text_read=text_read)
else:
decision = decide_identity(mean_scores, method=method, scale=scale)
track.candidates = [

View file

@ -133,14 +133,40 @@ def normalize_text(text: str) -> str:
return "".join(char for char in value if char.isalnum() or char == "%")
def signature_bigrams(text: str) -> Set[str]:
"""Character pairs of a catalog name, never across a word boundary.
A pair that spans two words is not a substring of either one, so it can
only ever be noise - and expensive noise, because it is rare across the
catalog and IDF therefore weights it as highly distinctive. "キユーピー
アマニ油" used to contribute "ピア", which let the garbage read
"キユピアク5C" be accepted as that row.
"""
pairs: Set[str] = set()
for token in text.split():
value = normalize_text(token)
pairs.update(value[index:index + 2] for index in range(len(value) - 1))
return pairs
def bigrams(text: str) -> Set[str]:
"""Character pairs of a read, across whitespace.
Deliberately not `signature_bigrams`: OCR puts spaces where the package
has none, so "ピュア セレクト" has to keep matching the catalog's
"ピュアセレクト". The catalog name is authoritative about where its words
end; a read is noisy evidence about the same characters, and dropping its
spaces is the only way the two can meet. The read is the side that gets
the benefit of the doubt, so the fake-pair problem is fixed where it
originates - in the signature - rather than by refusing to match.
"""
value = normalize_text(text)
return {value[index:index + 2] for index in range(len(value) - 1)}
def name_ocr_similarity(name: str, ocr_text: str) -> float:
"""Share of the catalog name's bigrams that the OCR text explains."""
name_bigrams = bigrams(name)
name_bigrams = signature_bigrams(name)
return len(name_bigrams & bigrams(ocr_text)) / max(1, len(name_bigrams))

View file

@ -74,6 +74,99 @@ class TextLaneSignatureTests(unittest.TestCase):
decision = DecisionPolicy().decide([], scores, text_read="マヨネーズ")
self.assertIsNone(decision["best_sku"])
def test_a_signature_never_pairs_across_two_words(self):
"""A catalog name must not claim text it does not contain.
"キューピー アマニ油" normalizes to "キユピ アマニ油"; with the space
dropped it was "キユピアマニ油", which contains "ピア" - a pair neither
word has. That pair is rare across the catalog, so IDF weighted it as
distinctive and the garbage read "キユピアク5C" was accepted as l02.
"""
from rebuild_catalog import signature_bigrams
name = "キューピー アマニ油マヨネーズ 200g"
self.assertNotIn("ピア", signature_bigrams(name))
self.assertIn("アマ", signature_bigrams(name))
def test_a_read_keeps_matching_across_the_spaces_ocr_inserts(self):
"""The read is the noisy side, so its spaces must not break a match.
Vision prints "ピュア セレクト" where the catalog name has
"ピュアセレクト". Refusing to pair across the read's spaces dropped
two correct l10 labels below the accept line, so the read pairs
across them and the signature is the side that stays strict.
"""
from rebuild_catalog import bigrams
self.assertIn("アセ", bigrams("ピュア セレクト マヨネーズ"))
def test_a_garbage_read_does_not_name_a_row(self):
"""The read that the fake pair used to accept must not name a row.
It shares only "キユ" and "ユピ" with l02's name, both of which every
Kewpie row prints. Bounded to the catalog's own vocabulary the read
keeps just those two pairs, so it scores the same against every
Kewpie row and the margin gate refuses it - the same mechanism that
already handles a read of shared brand copy alone.
"""
from identity_lanes import DecisionPolicy, TextLane
rows = self.ROWS + [{"sku_id": "sku_l02", "product_name": "キューピー アマニ油マヨネーズ 200g"}]
read = "キユピアク5C"
scores = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None).scores_from_text(read)
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
self.assertEqual(ranked[0][1], ranked[1][1], "the read cannot tell the Kewpie rows apart")
decision = DecisionPolicy().decide([], ranked, text_read=read)
self.assertIsNone(decision["best_sku"])
def test_a_read_of_real_text_still_names_its_row(self):
"""Bounding the denominator must not cost a correct read its answer.
This is the facing the strict-read variant broke: l10's own name read
with OCR's spacing, which has to clear the accept line by margin.
"""
from identity_lanes import TEXT_ACCEPT, TEXT_MARGIN, TextLane
rows = self.ROWS + [{"sku_id": "sku_l02", "product_name": "キューピー アマニ油マヨネーズ 200g"}]
scores = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None).scores_from_text(
"213日以月A ピュア セレクト マヨネーズ"
)
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
self.assertEqual(ranked[0][0], "sku_l10")
self.assertGreaterEqual(ranked[0][1], TEXT_ACCEPT)
self.assertGreaterEqual(ranked[0][1] - ranked[1][1], TEXT_MARGIN)
def test_one_shared_pair_is_not_enough_to_name_a_row(self):
"""A ratio of 1.0 on a single pair is not evidence.
"XXイス" contains exactly one pair the catalog knows, and the row that
contains it scores 1.0 with a full margin over every other row. On
image6 that shape named sku_l08 from the read "サソース". The floor
asks how much of the read the row actually explains, so one pair is
refused while the same read with two is accepted.
"""
from identity_lanes import DecisionPolicy, TextLane
rows = [
{"sku_id": "sku_l08", "product_name": "オイスターソース"},
{"sku_id": "sku_l01", "product_name": "キューピー マヨネーズ"},
]
lane = TextLane.from_catalog(rows, "no-such-dir", ocr_engine=None)
thin = sorted(lane.scores_from_text("XXイス").items(), key=lambda kv: kv[1], reverse=True)
self.assertEqual(thin[0][1], 1.0, "the ratio alone cannot see the difference")
self.assertIsNone(
DecisionPolicy(signatures=lane.signatures).decide([], thin, text_read="XXイス")["best_sku"]
)
enough = sorted(lane.scores_from_text("ZZソース").items(), key=lambda kv: kv[1], reverse=True)
self.assertEqual(
DecisionPolicy(signatures=lane.signatures).decide(
[], enough, text_read="ZZソース"
)["best_sku"],
"sku_l08",
)
class FakeDetector:
"""Records concurrency and returns fixed boxes."""

View file

@ -128,11 +128,10 @@ class FakeInventory:
def __init__(self):
self.products = {
"sku_a": {"sku_id": "sku_a", "product_name": "Product sku_a", "price": 150.0,
"sku_a": {"sku_id": "sku_a", "product_name": "コクうま カロリーカット65%", "price": 150.0,
"number_in": 10, "number_out": 1, "number_on": 9,
"receipt_date": "", "expiration_date": "", "local_image_path": None},
"other": {"sku_id": "other", "product_name": "Product other", "price": 150.0,
"number_in": 10, "number_out": 1, "number_on": 9,
"other": {"sku_id": "other", "product_name": "アマニ油マヨネーズ", "price": 150.0,
"receipt_date": "", "expiration_date": "", "local_image_path": None},
}
@ -266,6 +265,36 @@ class LabelLifecycleTests(unittest.TestCase):
== "recognized")
engine.shutdown()
def test_a_pan_does_not_let_a_neighbour_inherit(self):
"""A label must not slide onto whatever moved into its screen position.
The memory box is stored in the coordinates of the frame it was
measured on. Under a pan the shelf slides beneath that box, so the
product now occupying the old screen position is a different one.
Comparing the raw box hands it the label at IoU 1.0; compensating by
the shelf movement drops the overlap to nothing.
"""
session, engine, detector, matcher = make_session()
for frame_id in range(1, 5):
process(session, frame_id)
self.assertTrue(wait_for(lambda: process(session, 5)["tracks"][0]["identity"]["state"]
== "recognized"))
# The product leaves, so its track ages out and the next detection at
# that screen position is a new object rather than the same one.
detector.boxes = []
for frame_id in range(6, 10):
process(session, frame_id)
# The shelf slides 90 px: the labelled product is now 90 px to the
# left, and the screen position it used to occupy holds its neighbour.
session._frame_origin = [90.0, 0.0]
detector.boxes = [{"id": 0, "box": [100, 100, 200, 260], "confidence": 0.9, "class_id": 0,
"width": 100, "height": 160, "center": [150, 180]}]
result = process(session, 10)
row = result["tracks"][0]
self.assertFalse(row["inherited"],
"the product that slid into the old box is not the labelled one")
engine.shutdown()
class CosineScaleTests(unittest.TestCase):
"""A gallery-scale score is gated by the gallery, not re-gated by the live path."""