tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

Improve identity lanes: text evidence floor, motion-compensated labels

Browse source 
The text lane now requires shared bigrams between the read and the winning
row's name before it will answer, and the local lane passes the catalog
signatures through so the decision can weigh text evidence. On the labeled
shelf this moves the local lane from 66% to 80% end-to-end at full
resolution (91% precision) and 66% at live resolution (95% precision, 2
false accepts), against 57%/35% for the matcher alone.

Live fixes:
- `_plan_classification` counts `in_flight` again. The increment had been
  dropped while removing a `label_frame_id` write, so the guard never
  blocked, every frame re-planned the same tracks, and the executor queue
  grew without bound until the server stopped answering.
- Label memory boxes carry the shelf origin they were measured against and
  are shifted before the IoU test, so a panning camera no longer slides a
  stale box onto its neighbour.
- The text lane is resolved before the session lock is taken, removing a
  lock-ordering hazard against the lane's own build lock.
- A native crop always reads; the frame crop skips OCR only when it already
  has a reading.

Stream (20 s pan, 4 runs): 71-79% coverage at 76-84% precision, against
62-71% at 73-75% before.
This commit is contained in:
tngo 2026-09-29 02:08:23 +09:00
parent 771e9285d7
commit bf9c04bce4
7 changed files with 293 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

63
recognization/code/identity_lanes.py
View file

@ -37,7 +37,7 @@ if current_dir not in sys.path:
import cv2 # noqa: E402
import numpy as np # noqa: E402
from rebuild_catalog import bigrams, normalize_text, sku_of # noqa: E402
from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of # noqa: E402
# The local lane's accept line, in the units of `confidence()` below. Measured
# on the corrected image6 labels: 2.0 keeps 95% of shown labels correct at 59%
@ -49,6 +49,14 @@ LOCAL_ACCEPT = 2.0
# picks the right SKU on 41-42 of the 55 facings where it reads anything.
TEXT_ACCEPT = 0.5
TEXT_MARGIN = 0.15
# How many of the read's bigrams the winning row must actually contain. The
# ratio alone is not evidence: a two-pair read the row happens to contain scores
# 1.0, and "サソース" named sku_l08 that way. Two pairs is the floor that keeps
# every correct text answer on image6 (40 of 40) while dropping the wrong ones
# from 6 to 2, taking shown-label precision from 87% to 95%. It is a floor on
# evidence, not on the ratio, so a long read of shared brand copy still scores
# low and is still refused by the margin.
TEXT_MIN_SHARED = 2
# Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a
# 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and
# BRISK find zero inliers at live resolution. Scaling each reference to the
@ -274,7 +282,7 @@ def identity_bigrams(read: str) -> set:
local lane has already supplied the evidence and the text only has to
contradict it.
"""
return {gram for gram in bigrams(_QUANTITY.sub("", normalize_text(read))) if not gram.isdigit()}
return {gram for gram in bigrams(_QUANTITY.sub(" ", normalize_text(read))) if not gram.isdigit()}
def carries_identity(read: str) -> bool:
@ -289,7 +297,12 @@ def carries_identity(read: str) -> bool:
return len(identity_bigrams(read)) >= IDENTITY_BIGRAM_FLOOR
def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float]] = None) -> float:
def text_similarity(
read: str,
signature: str,
weights: Optional[Dict[str, float]] = None,
vocabulary: Optional[Set[str]] = None,
) -> float:
"""Share of the read text's bigrams that the signature explains.
Asymmetric on purpose: the crop shows a few words of a package, the
@ -304,11 +317,24 @@ def text_similarity(read: str, signature: str, weights: Optional[Dict[str, float
crop actually shows. Measured on image6: every l10 facing reads
"ピュア セレクト マヨネーズ" with no "コクうま" and no "65", yet l09 tied or
won on shared brand text alone.
`vocabulary` is every bigram any signature can produce. A read's pairs
that no catalog name contains are OCR noise - a misread character, a
date, or a pair that spans two words the package prints separately
("セレクト マヨネーズ" yields "トマ"). Counting them in the denominator
charges the row for text it could never have explained, which is how a
correct read of l10 fell to 0.46 while a garbage read of l11 stayed at
0.24: the noise is proportional to how much was read, so the rows with
the most real text to match were punished hardest. Bounded this way the
same reads score 0.86 and 1.0 against their true rows, and the garbage
read ties across three rows and is refused by the margin.
"""
read_grams = bigrams(normalize_text(read))
read_grams = bigrams(read)
if vocabulary is not None:
read_grams &= vocabulary
if not read_grams:
return 0.0
shared = read_grams & bigrams(normalize_text(signature))
shared = read_grams & signature_bigrams(signature)
if not shared:
return 0.0
if weights is None:
@ -329,7 +355,7 @@ def bigram_weights(signatures: Sequence[str]) -> Dict[str, float]:
"""
counts: Dict[str, int] = {}
for signature in signatures:
for gram in set(bigrams(normalize_text(signature))):
for gram in set(signature_bigrams(signature)):
counts[gram] = counts.get(gram, 0) + 1
total = max(1, len(signatures))
return {
@ -358,6 +384,12 @@ class TextLane:
# Bigram weights over the catalog's own names, so the lane can tell
# copy every sibling prints from the words that name one product.
self.weights = bigram_weights(list(self.signatures.values()))
# Every pair any name can produce. A read's pairs outside this set are
# OCR noise or word-boundary artifacts, and counting them in the
# denominator charges a row for text it could never have explained.
self.vocabulary: Set[str] = set()
for text in self.signatures.values():
self.vocabulary |= signature_bigrams(text)
self.ocr_engine = ocr_engine
# The engine's OCR lock. Vision tolerates concurrency but the CPU does
# not, and this lane runs one OCR per crop on the classify pool, so it
@ -420,7 +452,7 @@ class TextLane:
if len(normalize_text(read)) < 4:
return {}
return {
sku: text_similarity(read, signature, self.weights)
sku: text_similarity(read, signature, self.weights, self.vocabulary)
for sku, signature in self.signatures.items()
}
@ -459,17 +491,31 @@ class DecisionPolicy:
def __init__(
self,
*,
local_accept: float = LOCAL_ACCEPT,
text_accept: float = TEXT_ACCEPT,
text_margin: float = TEXT_MARGIN,
text_min_shared: int = TEXT_MIN_SHARED,
signatures: Optional[Dict[str, str]] = None,
method: str = "rootsift",
) -> None:
self.local_accept = float(local_accept)
self.text_accept = float(text_accept)
self.text_margin = float(text_margin)
self.text_min_shared = int(text_min_shared)
# The catalog names, so the evidence floor can ask how many of the
# read's pairs the winning row actually contains. Without them the
# floor cannot be applied and the ratio stands alone, which is the
# behaviour a caller that has no catalog is asking for.
self.signatures = {str(sku): str(text) for sku, text in (signatures or {}).items()}
self.method = method
def _shared_evidence(self, read: str, sku: str) -> int:
"""How many of the read's bigrams the row's name contains."""
signature = self.signatures.get(sku)
if signature is None:
return self.text_min_shared
return len(bigrams(read) & signature_bigrams(signature))
def decide(
self,
local_rank: Any,
@ -503,6 +549,7 @@ class DecisionPolicy:
and text_top[1] >= self.text_accept
and (text_top[1] - text_runner) >= self.text_margin
and carries_identity(text_read or "")
and self._shared_evidence(text_read or "", text_top[0]) >= self.text_min_shared
else None
)