tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

Improve identity lanes: text evidence floor, motion-compensated labels

Browse source 
The text lane now requires shared bigrams between the read and the winning
row's name before it will answer, and the local lane passes the catalog
signatures through so the decision can weigh text evidence. On the labeled
shelf this moves the local lane from 66% to 80% end-to-end at full
resolution (91% precision) and 66% at live resolution (95% precision, 2
false accepts), against 57%/35% for the matcher alone.

Live fixes:
- `_plan_classification` counts `in_flight` again. The increment had been
  dropped while removing a `label_frame_id` write, so the guard never
  blocked, every frame re-planned the same tracks, and the executor queue
  grew without bound until the server stopped answering.
- Label memory boxes carry the shelf origin they were measured against and
  are shifted before the IoU test, so a panning camera no longer slides a
  stale box onto its neighbour.
- The text lane is resolved before the session lock is taken, removing a
  lock-ordering hazard against the lane's own build lock.
- A native crop always reads; the frame crop skips OCR only when it already
  has a reading.

Stream (20 s pan, 4 runs): 71-79% coverage at 76-84% precision, against
62-71% at 73-75% before.
This commit is contained in:
tngo 2026-09-29 02:08:23 +09:00
parent 771e9285d7
commit bf9c04bce4
7 changed files with 293 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

97
recognization/code/live.py
View file

@ -181,6 +181,31 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float:
return inter / union if union > 0 else 0.0
def _shift_box(
box: Sequence[int],
from_origin: Optional[Sequence[float]],
to_origin: Sequence[float],
) -> Optional[List[int]]:
"""A box measured at one shelf origin, expressed at another.
The origin is the shelf position of the frame's top-left corner, so the
content moves the opposite way: a shelf that has slid left by 12 px means
the same product now sits 12 px further left in the frame, and a box
measured before the slide has to move with it.
Returns None when either origin is unknown. An uncompensated box is not a
weaker match, it is a match against the wrong product, so a caller that
cannot place the box must not use it at all.
"""
if from_origin is None or len(from_origin) < 2 or len(to_origin) < 2:
return None
dx = float(from_origin[0]) - float(to_origin[0])
dy = float(from_origin[1]) - float(to_origin[1])
x1, y1, x2, y2 = box
return [int(round(x1 + dx)), int(round(y1 + dy)),
int(round(x2 + dx)), int(round(y2 + dy))]
def _clear_of_edge(box: Sequence[int], frame_shape: Sequence[int], margin_ratio: float) -> bool:
"""True when a box sits clear of the frame edge.
@ -670,11 +695,21 @@ class LiveSession:
An inherited label is not a measurement, so it cannot spread to a neighbour.
Blur, glare, and a too-far frame skip inheritance: the overlap is not
evidence when the frame itself failed the quality gate.
Memory boxes are stored in the coordinates of the frame they were
measured on, so they are shifted by the shelf movement since then before
they are compared. Without that, a panning camera slides the scene under
a stale box: at 12 px/frame a ~90 px facing moves a full width in about
seven frames, so the neighbour overlaps the old box at IoU >= 0.55 and
inherits a label that was never about it. Measured on image6 at pan 12,
every unknown false accept was an inherited label landing on the two
products next to the labelled one.
"""
if not self._label_memory:
return
if (self._last_quality or {}).get("hint"):
return
origin = (self._frame_origin[0], self._frame_origin[1])
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.confirmed:
continue
@ -685,7 +720,12 @@ class LiveSession:
continue
if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU:
continue
if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU:
moved = _shift_box(entry["box"], entry.get("origin"), origin)
if moved is None:
# The shelf moved too far for this entry to describe
# anything still on screen, so it cannot label a neighbour.
continue
if _iou(track.box, moved) < LABEL_MEMORY_IOU:
continue
track.state = STATE_RECOGNIZED
track.confirmed = True
@ -723,6 +763,10 @@ class LiveSession:
"confidence": track.identity.get("confidence"),
"score": track.identity.get("score"),
"seq": seq,
# The shelf origin this box was measured against. `box` is in
# that frame's pixels, so a later frame has to shift it by how
# far the shelf has moved since.
"origin": (self._frame_origin[0], self._frame_origin[1]),
"inherited": bool(track.identity.get("inherited")),
}
)
@ -829,10 +873,20 @@ class LiveSession:
planned: List[Tuple[TrackState, Any, int]] = []
for track in eligible[:capacity]:
crop = self.engine.crop(image, track.raw_box)
track.in_flight += 1
# No label_frame_id here: the plan may be dropped (generation
# mismatch, a crop that went off product, a neighbour's label
# arriving first), and a frame id stamped on a track whose label
# never came from this frame would pair a box with the wrong
# offset. The callback carries the id, and the writers that set
# `label_box` set the id with it.
track.last_attempt_seq = seq
track.label_frame_id = int(frame_id)
track.last_attempt_area = track.area()
# Counted here, not in the submitter: this is the only place that
# knows a crop is about to be in flight, and the callback is what
# decrements it. Without the increment the guard above never blocks
# and every frame re-plans the same tracks, so the executor queue
# grows without bound and the server stops answering.
track.in_flight += 1
planned.append((track, crop, track.object_generation))
return planned
@ -846,14 +900,13 @@ class LiveSession:
"""Submit planned crops. Called WITHOUT the session lock held."""
for track, crop, object_generation in planned:
submitted_at = time.perf_counter()
# The label anchors to the displayed (smoothed) box, which is what the
# drift check later compares against.
# The label anchors to the displayed (smoothed) box, which is what
# the drift check later compares against.
submitted_box = list(track.box)
track_id = track.track_id
# OCR once per object: a track that already has a reading re-runs
# only the local lane, which is the difference between 5 ms and
# 25 ms per crop. A native crop is preferred for the one reading
# because it is the sharpest view the phone will send.
# only the local lane. The native crop lane always reads, because
# it is only asked for when the frame crop was too small to decide.
skip_ocr = bool(track.text_read)
try:
future = self._executor.submit(
@ -1031,10 +1084,11 @@ class LiveSession:
self._release_crop_slot(record)
self._crops_rejected += 1
return _crop_error("CROP_STALE", stale)
# A native crop is the sharpest view the phone sends, so it is the
# one worth reading - but only once per object, like the frame path.
track = self._tracks.get(track_id_out)
skip_ocr = bool(track is not None and track.text_read)
# A native crop is the sharpest view the phone sends, and the lane
# only asks for one when the frame crop was too small to decide
# (`crop_below_px`), so any reading it would skip came from exactly
# the crop this one exists to replace. It always reads.
skip_ocr = False
try:
future = self._executor.submit(
self.engine.score_crop_lanes, crop, method, skip_ocr
@ -1147,6 +1201,12 @@ class LiveSession:
except Exception:
scores, text_rank, text_read = {}, [], ""
finished_at = time.perf_counter()
# Resolved before the session lock is taken. `identity_text_lane` builds
# the lane under its own lock on first use, and taking that while the
# session lock is held is a lock-ordering hazard: a frame loop that
# holds the session lock and waits on the lane lock while this callback
# holds the lane lock and waits on the session lock wedges the server.
text_lane = self.engine.identity_text_lane()
with self._lock:
track = self._tracks.get(track_id)
if track is None or generation != self._generation:
@ -1181,14 +1241,17 @@ class LiveSession:
track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score)
if observation:
track.score_n += 1
self._classify_times.append(finished_at - submitted_at)
# The reading is cached on the track so later attempts skip OCR.
# Only a read that could name something is cached: a crop that was
# too small or too blurry says nothing about the next one, and a
# read of just a size ("400g") matches every 400g product's size
# line, so caching it would let one unreadable crop veto the
# product for the rest of the track's life.
#
# The substitution happens before the early-out below and before
# the decision: a skipped OCR returns an empty ranking, and an
# empty ranking would both mark a text-only track unrecognized at
# max_attempts and stop the veto from firing on later attempts.
if text_read and carries_identity(text_read):
track.text_rank = list(text_rank or [])
track.text_read = text_read
@ -1235,9 +1298,9 @@ class LiveSession:
# chose. The newest crop is the evidence, not the mean: averaging
# in an empty observation halves a good confidence below the
# accept line, which is the dilution the cosine branch guards too.
decision = DecisionPolicy().decide(
list(scores.items()), text_rank, text_read=text_read
)
decision = DecisionPolicy(
signatures=text_lane.signatures if text_lane else None
).decide(list(scores.items()), text_rank, text_read=text_read)
else:
decision = decide_identity(mean_scores, method=method, scale=scale)
track.candidates = [