Coverage-first live labelling, merged catalog, shelf-intelligence decision
The live path abstained too much: a detected product whose best candidate missed the accept line was left blank, so a shelf where 59% of in-catalog facings were shown read as broken. Show the best candidate immediately as `provisional`, keep the track re-classifiable so a later attempt upgrades or corrects it, and never blank a label already on screen. - identity.py / identity_lanes.py: every decision now carries `provisional_sku` - the row the evidence supports well enough to show, separate from `best_sku`, which still means "cleared the gate". The accept line does not move: far-unknown false accepts stay at 0.066%. - live.py: a below-gate candidate labels the track with `identity_level: provisional` and `confirmed = False`, so the planner and the crop lane keep offering it. Accepted labels stay frozen. Inherited provisional labels stay re-classifiable. - Clients render provisional distinctly (amber chip, dashed box) via `LiveScanOverlayPolicy.isProvisional(identity)`, which now reads the wire level instead of only the state. - build_stock_all.py merges stock, stock-lopia, stock-mediya and stock-milk150 into stock-all (111 images, 60 SKUs). The server could only read one folder, so anything outside it came back blank. stock-demo is byte-identical to mediya+milk150 and is not a source. Measured on add/image6.jpg at live resolution: in-catalog facings shown 59.4% -> 86.0%, correct shown 38 -> 55, precision 97.4% -> 87.3%. On the wire at 8fps 0.5x: 65 of 79 tracks labelled, 742 accepted + 309 provisional observations, first label 1.27s. docs/research/2026-09-29-shelf-intelligence/ carries the experiments and the decision: group signals (combined AUC 0.991 sibling / 0.990 unrelated), OCR scheduling (group representatives cut calls 4.4x and wall 3.3x), and the finding that group propagation damages more labels than it rescues on this shelf, so groups schedule compute rather than overwrite labels. 331 tests pass.
This commit is contained in:
parent
fbf31d74fe
commit
0cf64a18bf
171 changed files with 554214 additions and 35 deletions
|
|
@ -80,6 +80,10 @@ class LiveSettings:
|
|||
conf: float = 0.1
|
||||
empty_slots: bool = True
|
||||
tags: bool = True
|
||||
# The text lane: OCR every classified crop and let the reading veto or name
|
||||
# a row. It is the slowest lane (5-25 ms per crop, serialized behind one
|
||||
# lock), so a client may turn it off and run on geometry alone.
|
||||
ocr: bool = True
|
||||
min_hits: int = 2
|
||||
min_crop_px: int = 40
|
||||
max_attempts: int = 2
|
||||
|
|
@ -438,6 +442,7 @@ class LiveSession:
|
|||
conf: Optional[float] = None,
|
||||
empty_slots: Optional[bool] = None,
|
||||
tags: Optional[bool] = None,
|
||||
ocr: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
method_changed = method is not None and method != self.settings.method
|
||||
|
|
@ -449,6 +454,8 @@ class LiveSession:
|
|||
self.settings.empty_slots = bool(empty_slots)
|
||||
if tags is not None:
|
||||
self.settings.tags = bool(tags)
|
||||
if ocr is not None:
|
||||
self.settings.ocr = bool(ocr)
|
||||
if method_changed:
|
||||
self._generation += 1
|
||||
for track in self._tracks.values():
|
||||
|
|
@ -482,6 +489,7 @@ class LiveSession:
|
|||
"method": self.settings.method,
|
||||
"conf": self.settings.conf,
|
||||
"empty_slots": self.settings.empty_slots,
|
||||
"ocr": self.settings.ocr,
|
||||
"tags": self.settings.tags,
|
||||
"tags_available": self._tags is not None,
|
||||
"ocr_engine": getattr(self._tags.engine, "name", "none") if self._tags else "none",
|
||||
|
|
@ -714,7 +722,7 @@ class LiveSession:
|
|||
return
|
||||
origin = (self._frame_origin[0], self._frame_origin[1])
|
||||
for track in self._tracks.values():
|
||||
if track.state != STATE_DETECTED or track.confirmed:
|
||||
if track.confirmed or track.state != STATE_DETECTED:
|
||||
continue
|
||||
for entry in reversed(self._label_memory):
|
||||
if seq - entry["seq"] > LABEL_MEMORY_FRAMES:
|
||||
|
|
@ -743,13 +751,18 @@ class LiveSession:
|
|||
# product it names.
|
||||
track.label_frame_id = int(self._latest_frame_id)
|
||||
track.last_source = SOURCE_INHERITED
|
||||
provisional = bool(entry.get("provisional"))
|
||||
track.identity = {
|
||||
"sku_id": entry["sku_id"],
|
||||
"confidence": entry["confidence"],
|
||||
"score": entry["score"],
|
||||
"attempts": 0,
|
||||
"inherited": True,
|
||||
"provisional": provisional,
|
||||
}
|
||||
# An inherited provisional label is still a guess, so it stays
|
||||
# re-classifiable instead of freezing onto the whole run.
|
||||
track.confirmed = not provisional
|
||||
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
|
||||
self._relabel_count[track.track_id] = track.relabels
|
||||
break
|
||||
|
|
@ -771,6 +784,7 @@ class LiveSession:
|
|||
# far the shelf has moved since.
|
||||
"origin": (self._frame_origin[0], self._frame_origin[1]),
|
||||
"inherited": bool(track.identity.get("inherited")),
|
||||
"provisional": bool(track.identity.get("provisional")),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -844,7 +858,9 @@ class LiveSession:
|
|||
eligible: List[TrackState] = []
|
||||
poor_frame = bool((self._last_quality or {}).get("hint"))
|
||||
for track in self._tracks.values():
|
||||
if track.state != STATE_DETECTED or track.in_flight > 0:
|
||||
if track.in_flight > 0 or track.confirmed:
|
||||
continue
|
||||
if track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
|
||||
continue
|
||||
if poor_frame:
|
||||
continue
|
||||
|
|
@ -910,7 +926,7 @@ class LiveSession:
|
|||
# OCR once per object: a track that already has a reading re-runs
|
||||
# only the local lane. The native crop lane always reads, because
|
||||
# it is only asked for when the frame crop was too small to decide.
|
||||
skip_ocr = bool(track.text_read)
|
||||
skip_ocr = bool(track.text_read) or not self.settings.ocr
|
||||
try:
|
||||
future = self._executor.submit(
|
||||
self.engine.score_crop_lanes, crop, method, skip_ocr
|
||||
|
|
@ -978,7 +994,7 @@ class LiveSession:
|
|||
break
|
||||
if track.crop_attempts >= self.settings.max_attempts or track.in_flight > 0:
|
||||
continue
|
||||
if track.state != STATE_DETECTED or track.confirmed:
|
||||
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
|
||||
continue
|
||||
if track.hits < self.settings.min_hits or len(track.raw_box) != 4:
|
||||
continue
|
||||
|
|
@ -1090,8 +1106,9 @@ class LiveSession:
|
|||
# A native crop is the sharpest view the phone sends, and the lane
|
||||
# only asks for one when the frame crop was too small to decide
|
||||
# (`crop_below_px`), so any reading it would skip came from exactly
|
||||
# the crop this one exists to replace. It always reads.
|
||||
skip_ocr = False
|
||||
# the crop this one exists to replace. It always reads, unless the
|
||||
# session turned the text lane off.
|
||||
skip_ocr = not self.settings.ocr
|
||||
try:
|
||||
future = self._executor.submit(
|
||||
self.engine.score_crop_lanes, crop, method, skip_ocr
|
||||
|
|
@ -1183,7 +1200,7 @@ class LiveSession:
|
|||
return "the track id was recycled while the crop was decoded"
|
||||
if _crop_off_product(record["box"], track.raw_box):
|
||||
return "the track moved off the product while the crop was decoded"
|
||||
if track.state != STATE_DETECTED or track.confirmed:
|
||||
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
|
||||
return "the track is no longer waiting to be decided"
|
||||
return None
|
||||
|
||||
|
|
@ -1226,11 +1243,13 @@ class LiveSession:
|
|||
track.in_flight = max(0, track.in_flight - 1)
|
||||
return
|
||||
track.in_flight = max(0, track.in_flight - 1)
|
||||
if track.state != STATE_DETECTED or track.confirmed:
|
||||
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
|
||||
# Other evidence settled this track while the answer sat in the
|
||||
# classifier - a label inherited from a labelled neighbour, say.
|
||||
# A late answer must not repaint a label the client is already
|
||||
# showing, so the track keeps the one it settled on.
|
||||
# showing, so the track keeps the one it settled on. A track
|
||||
# showing a *provisional* label is not settled: it is here so a
|
||||
# later attempt can upgrade or correct it.
|
||||
return
|
||||
if source == SOURCE_CROP and _crop_off_product(submitted_box, track.raw_box):
|
||||
# The answer was measured on the box the track has since left.
|
||||
|
|
@ -1338,17 +1357,42 @@ class LiveSession:
|
|||
"attempts": track.attempts,
|
||||
}
|
||||
self._remember_label(track, submitted_box, self._seq)
|
||||
elif track.attempts >= self.settings.max_attempts:
|
||||
track.state = (
|
||||
STATE_NEEDS_CONFIRMATION
|
||||
if decision.get("identity_level") in ("ambiguous", "unknown")
|
||||
else STATE_UNRECOGNIZED
|
||||
)
|
||||
track.confirmed = True
|
||||
elif decision.get("provisional_sku"):
|
||||
# Coverage-first: a detected product is shown its best candidate
|
||||
# immediately, marked provisional, and stays re-classifiable so a
|
||||
# later attempt can upgrade it to accepted or correct it. Van's
|
||||
# always-answer behaviour, on the stronger stack. A blank box
|
||||
# costs more than a guess the next frame may fix.
|
||||
best = decision.get("provisional_sku")
|
||||
track.state = STATE_RECOGNIZED
|
||||
track.confirmed = False
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
track.drift_frames = 0
|
||||
track.identity = None
|
||||
track.identity = {
|
||||
"sku_id": best,
|
||||
"confidence": decision.get("confidence"),
|
||||
"score": decision.get("score"),
|
||||
"attempts": track.attempts,
|
||||
"provisional": True,
|
||||
}
|
||||
self._remember_label(track, submitted_box, self._seq)
|
||||
elif track.attempts >= self.settings.max_attempts:
|
||||
if track.identity and track.identity.get("provisional"):
|
||||
# The budget is spent and a provisional label is already on
|
||||
# screen. Blanking it now would take back the only identity
|
||||
# the operator has seen, so the guess stays and the track
|
||||
# simply stops being re-classified.
|
||||
track.confirmed = True
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
else:
|
||||
track.state = STATE_UNRECOGNIZED
|
||||
track.confirmed = True
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
track.drift_frames = 0
|
||||
track.identity = None
|
||||
else:
|
||||
track.state = STATE_DETECTED
|
||||
track.confirmed = False
|
||||
|
|
@ -1378,7 +1422,9 @@ class LiveSession:
|
|||
identity["sku_id"] = track.identity.get("sku_id")
|
||||
identity["confidence"] = track.identity.get("confidence")
|
||||
identity["score"] = track.identity.get("score")
|
||||
identity["identity_level"] = "accepted"
|
||||
identity["identity_level"] = (
|
||||
"provisional" if track.identity.get("provisional") else "accepted"
|
||||
)
|
||||
product = self._catalog_by_sku.get(identity["sku_id"] or "")
|
||||
elif track.state in (STATE_NEEDS_CONFIRMATION, STATE_UNRECOGNIZED):
|
||||
identity["identity_level"] = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue