Coverage-first live labelling, merged catalog, shelf-intelligence decision

The live path abstained too much: a detected product whose best candidate
missed the accept line was left blank, so a shelf where 59% of in-catalog
facings were shown read as broken. Show the best candidate immediately as
`provisional`, keep the track re-classifiable so a later attempt upgrades or
corrects it, and never blank a label already on screen.

- identity.py / identity_lanes.py: every decision now carries
  `provisional_sku` - the row the evidence supports well enough to show,
  separate from `best_sku`, which still means "cleared the gate". The accept
  line does not move: far-unknown false accepts stay at 0.066%.
- live.py: a below-gate candidate labels the track with
  `identity_level: provisional` and `confirmed = False`, so the planner and
  the crop lane keep offering it. Accepted labels stay frozen. Inherited
  provisional labels stay re-classifiable.
- Clients render provisional distinctly (amber chip, dashed box) via
  `LiveScanOverlayPolicy.isProvisional(identity)`, which now reads the wire
  level instead of only the state.
- build_stock_all.py merges stock, stock-lopia, stock-mediya and
  stock-milk150 into stock-all (111 images, 60 SKUs). The server could only
  read one folder, so anything outside it came back blank. stock-demo is
  byte-identical to mediya+milk150 and is not a source.

Measured on add/image6.jpg at live resolution: in-catalog facings shown
59.4% -> 86.0%, correct shown 38 -> 55, precision 97.4% -> 87.3%. On the
wire at 8fps 0.5x: 65 of 79 tracks labelled, 742 accepted + 309 provisional
observations, first label 1.27s.

docs/research/2026-09-29-shelf-intelligence/ carries the experiments and the
decision: group signals (combined AUC 0.991 sibling / 0.990 unrelated),
OCR scheduling (group representatives cut calls 4.4x and wall 3.3x), and the
finding that group propagation damages more labels than it rescues on this
shelf, so groups schedule compute rather than overwrite labels.

331 tests pass.
This commit is contained in:
tngo 2026-09-29 13:45:14 +09:00
parent fbf31d74fe
commit 0cf64a18bf
171 changed files with 554214 additions and 35 deletions

View file

@ -989,6 +989,7 @@ def create_app(
conf=message.get("conf"),
empty_slots=message.get("empty_slots"),
tags=message.get("tags"),
ocr=message.get("ocr"),
)
except (TypeError, ValueError):
await send({"type": "error", "code": "BAD_MESSAGE", "message": "bad config values"})

View file

@ -0,0 +1,79 @@
"""Merge every stock folder into one catalog the server can match against.
The demo had five stock folders and the server could only read one, so a shelf
holding anything outside the active folder came back blank. SKU ids are unique
across the folders (`sku_0xx`, `sku_lxx`, `sku_mxx`, `sku_kxx`), so the union is
a plain copy plus a CSV concat - no id rewriting, no dedup heuristics.
`stock-demo` is byte-identical to `stock-mediya` + `stock-milk150`, so it is not
a source: including it would only duplicate 62 images.
Run: .venv/bin/python code/build_stock_all.py
"""
from __future__ import annotations
import csv
import os
import shutil
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
SOURCES = ("stock", "stock-lopia", "stock-mediya", "stock-milk150")
DEST = os.path.join(ROOT, "stock-all")
IMAGE_SUFFIXES = (".jpg", ".jpeg", ".png")
FIELDS = ("SKU_ID", "product_name", "receipt_date", "expiration_date",
"number_in", "number_out", "number_on", "image_link", "price")
def main() -> int:
if os.path.isdir(DEST):
shutil.rmtree(DEST)
os.makedirs(DEST)
rows: list[dict] = []
seen_sku: dict[str, str] = {}
copied = 0
for source in SOURCES:
directory = os.path.join(ROOT, source)
if not os.path.isdir(directory):
print(f"skip {source}: not a directory", file=sys.stderr)
continue
for name in sorted(os.listdir(directory)):
if not name.lower().endswith(IMAGE_SUFFIXES):
continue
target = os.path.join(DEST, name)
if os.path.exists(target):
# Same filename in two folders is the same product photographed
# once; the first copy wins and the duplicate is reported.
print(f"duplicate image {name} from {source}", file=sys.stderr)
continue
shutil.copy2(os.path.join(directory, name), target)
copied += 1
csv_path = os.path.join(directory, "stock.csv")
if not os.path.exists(csv_path):
continue
with open(csv_path, newline="", encoding="utf-8") as handle:
for row in csv.DictReader(handle):
sku = (row.get("SKU_ID") or "").strip()
if not sku:
continue
if sku in seen_sku:
print(f"duplicate SKU {sku} in {source} (kept {seen_sku[sku]})",
file=sys.stderr)
continue
seen_sku[sku] = source
rows.append({field: (row.get(field) or "").strip() for field in FIELDS})
with open(os.path.join(DEST, "stock.csv"), "w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS)
writer.writeheader()
writer.writerows(rows)
print(f"{DEST}: {copied} images, {len(rows)} SKUs")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -107,6 +107,7 @@ def _cosine_decision(
"status": level,
"method": method,
"best_sku": top_sku if level == IDENTITY_ACCEPTED else None,
"provisional_sku": top_sku,
"score": float(top_score),
"confidence": float(top_score),
"identity_level": level,
@ -200,6 +201,7 @@ def decide_identity(
"status": IDENTITY_REJECTED,
"method": method,
"best_sku": None,
"provisional_sku": None,
"score": 0.0,
"confidence": 0.0,
"identity_level": IDENTITY_REJECTED,
@ -243,6 +245,7 @@ def decide_identity(
"status": "success" if level == IDENTITY_ACCEPTED else level,
"method": method,
"best_sku": best,
"provisional_sku": top_sku if top_score > 0.0 else None,
"score": round(top_score, 2),
"confidence": round(top_score, 4),
"identity_level": level,
@ -263,6 +266,7 @@ def decide_identity(
"status": "success" if level == IDENTITY_ACCEPTED else level,
"method": method,
"best_sku": best,
"provisional_sku": top_sku if top_score > 0.0 else None,
"score": round(top_score, 2),
"confidence": confidence,
"identity_level": level,
@ -296,6 +300,7 @@ def decide_identity(
"status": "success" if level == IDENTITY_ACCEPTED else level,
"method": method,
"best_sku": best,
"provisional_sku": top_sku if top_score > 0.0 else None,
"score": round(top_score, 2),
"confidence": confidence,
"identity_level": level,

View file

@ -658,11 +658,36 @@ class DecisionPolicy:
level = IDENTITY_REJECTED
message = "No matching evidence"
# Coverage-first: the row the evidence supports well enough to *show*,
# even when it does not clear the accept line. `None` means the crop
# carried no usable evidence at all, which is the only case that still
# ends blank - a size-only read has no identity text and stays None.
if answer is not None:
provisional = answer
elif local and local[0][1] > 0.0:
# A vetoed local answer is still the best geometric guess, unless
# the text named a different row - then that row is the better one.
vetoed = bool(
text_top
and text_top[0] != local[0][0]
and text_top[1] > local_text_floor(text, local[0][0])
)
provisional = text_top[0] if vetoed else local[0][0]
elif (
text
and carries_identity(text_read or "")
and self._shared_evidence(text_read or "", text[0][0]) >= self.text_min_shared
):
provisional = text[0][0]
else:
provisional = None
top_score = local[0][1] if local else (text_top[1] if text_top else 0.0)
return {
"status": "success" if level == IDENTITY_ACCEPTED else level,
"method": self.method,
"best_sku": answer,
"provisional_sku": provisional,
"score": round(float(top_score), 2),
"confidence": round(float(top_score), 4),
"identity_level": level,

View file

@ -80,6 +80,10 @@ class LiveSettings:
conf: float = 0.1
empty_slots: bool = True
tags: bool = True
# The text lane: OCR every classified crop and let the reading veto or name
# a row. It is the slowest lane (5-25 ms per crop, serialized behind one
# lock), so a client may turn it off and run on geometry alone.
ocr: bool = True
min_hits: int = 2
min_crop_px: int = 40
max_attempts: int = 2
@ -438,6 +442,7 @@ class LiveSession:
conf: Optional[float] = None,
empty_slots: Optional[bool] = None,
tags: Optional[bool] = None,
ocr: Optional[bool] = None,
) -> Dict[str, Any]:
with self._lock:
method_changed = method is not None and method != self.settings.method
@ -449,6 +454,8 @@ class LiveSession:
self.settings.empty_slots = bool(empty_slots)
if tags is not None:
self.settings.tags = bool(tags)
if ocr is not None:
self.settings.ocr = bool(ocr)
if method_changed:
self._generation += 1
for track in self._tracks.values():
@ -482,6 +489,7 @@ class LiveSession:
"method": self.settings.method,
"conf": self.settings.conf,
"empty_slots": self.settings.empty_slots,
"ocr": self.settings.ocr,
"tags": self.settings.tags,
"tags_available": self._tags is not None,
"ocr_engine": getattr(self._tags.engine, "name", "none") if self._tags else "none",
@ -714,7 +722,7 @@ class LiveSession:
return
origin = (self._frame_origin[0], self._frame_origin[1])
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.confirmed:
if track.confirmed or track.state != STATE_DETECTED:
continue
for entry in reversed(self._label_memory):
if seq - entry["seq"] > LABEL_MEMORY_FRAMES:
@ -743,13 +751,18 @@ class LiveSession:
# product it names.
track.label_frame_id = int(self._latest_frame_id)
track.last_source = SOURCE_INHERITED
provisional = bool(entry.get("provisional"))
track.identity = {
"sku_id": entry["sku_id"],
"confidence": entry["confidence"],
"score": entry["score"],
"attempts": 0,
"inherited": True,
"provisional": provisional,
}
# An inherited provisional label is still a guess, so it stays
# re-classifiable instead of freezing onto the whole run.
track.confirmed = not provisional
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
self._relabel_count[track.track_id] = track.relabels
break
@ -771,6 +784,7 @@ class LiveSession:
# far the shelf has moved since.
"origin": (self._frame_origin[0], self._frame_origin[1]),
"inherited": bool(track.identity.get("inherited")),
"provisional": bool(track.identity.get("provisional")),
}
)
@ -844,7 +858,9 @@ class LiveSession:
eligible: List[TrackState] = []
poor_frame = bool((self._last_quality or {}).get("hint"))
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.in_flight > 0:
if track.in_flight > 0 or track.confirmed:
continue
if track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
continue
if poor_frame:
continue
@ -910,7 +926,7 @@ class LiveSession:
# OCR once per object: a track that already has a reading re-runs
# only the local lane. The native crop lane always reads, because
# it is only asked for when the frame crop was too small to decide.
skip_ocr = bool(track.text_read)
skip_ocr = bool(track.text_read) or not self.settings.ocr
try:
future = self._executor.submit(
self.engine.score_crop_lanes, crop, method, skip_ocr
@ -978,7 +994,7 @@ class LiveSession:
break
if track.crop_attempts >= self.settings.max_attempts or track.in_flight > 0:
continue
if track.state != STATE_DETECTED or track.confirmed:
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
continue
if track.hits < self.settings.min_hits or len(track.raw_box) != 4:
continue
@ -1090,8 +1106,9 @@ class LiveSession:
# A native crop is the sharpest view the phone sends, and the lane
# only asks for one when the frame crop was too small to decide
# (`crop_below_px`), so any reading it would skip came from exactly
# the crop this one exists to replace. It always reads.
skip_ocr = False
# the crop this one exists to replace. It always reads, unless the
# session turned the text lane off.
skip_ocr = not self.settings.ocr
try:
future = self._executor.submit(
self.engine.score_crop_lanes, crop, method, skip_ocr
@ -1183,7 +1200,7 @@ class LiveSession:
return "the track id was recycled while the crop was decoded"
if _crop_off_product(record["box"], track.raw_box):
return "the track moved off the product while the crop was decoded"
if track.state != STATE_DETECTED or track.confirmed:
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
return "the track is no longer waiting to be decided"
return None
@ -1226,11 +1243,13 @@ class LiveSession:
track.in_flight = max(0, track.in_flight - 1)
return
track.in_flight = max(0, track.in_flight - 1)
if track.state != STATE_DETECTED or track.confirmed:
if track.confirmed or track.state not in (STATE_DETECTED, STATE_RECOGNIZED):
# Other evidence settled this track while the answer sat in the
# classifier - a label inherited from a labelled neighbour, say.
# A late answer must not repaint a label the client is already
# showing, so the track keeps the one it settled on.
# showing, so the track keeps the one it settled on. A track
# showing a *provisional* label is not settled: it is here so a
# later attempt can upgrade or correct it.
return
if source == SOURCE_CROP and _crop_off_product(submitted_box, track.raw_box):
# The answer was measured on the box the track has since left.
@ -1338,17 +1357,42 @@ class LiveSession:
"attempts": track.attempts,
}
self._remember_label(track, submitted_box, self._seq)
elif track.attempts >= self.settings.max_attempts:
track.state = (
STATE_NEEDS_CONFIRMATION
if decision.get("identity_level") in ("ambiguous", "unknown")
else STATE_UNRECOGNIZED
)
track.confirmed = True
elif decision.get("provisional_sku"):
# Coverage-first: a detected product is shown its best candidate
# immediately, marked provisional, and stays re-classifiable so a
# later attempt can upgrade it to accepted or correct it. Van's
# always-answer behaviour, on the stronger stack. A blank box
# costs more than a guess the next frame may fix.
best = decision.get("provisional_sku")
track.state = STATE_RECOGNIZED
track.confirmed = False
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
track.drift_frames = 0
track.identity = None
track.identity = {
"sku_id": best,
"confidence": decision.get("confidence"),
"score": decision.get("score"),
"attempts": track.attempts,
"provisional": True,
}
self._remember_label(track, submitted_box, self._seq)
elif track.attempts >= self.settings.max_attempts:
if track.identity and track.identity.get("provisional"):
# The budget is spent and a provisional label is already on
# screen. Blanking it now would take back the only identity
# the operator has seen, so the guess stays and the track
# simply stops being re-classified.
track.confirmed = True
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
else:
track.state = STATE_UNRECOGNIZED
track.confirmed = True
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
track.drift_frames = 0
track.identity = None
else:
track.state = STATE_DETECTED
track.confirmed = False
@ -1378,7 +1422,9 @@ class LiveSession:
identity["sku_id"] = track.identity.get("sku_id")
identity["confidence"] = track.identity.get("confidence")
identity["score"] = track.identity.get("score")
identity["identity_level"] = "accepted"
identity["identity_level"] = (
"provisional" if track.identity.get("provisional") else "accepted"
)
product = self._catalog_by_sku.get(identity["sku_id"] or "")
elif track.state in (STATE_NEEDS_CONFIRMATION, STATE_UNRECOGNIZED):
identity["identity_level"] = (

View file

@ -212,22 +212,33 @@ class LabelLifecycleTests(unittest.TestCase):
)), "the product should be labelled once classified")
engine.shutdown()
def test_label_is_never_shown_without_acceptance(self):
def test_a_weak_match_is_shown_provisional_not_accepted(self):
"""Coverage-first: a below-gate candidate is shown, but marked provisional.
The old contract left this box blank. Blank is now the worse outcome:
the operator sees a guess that the next attempt may upgrade, and the
wire says which it is, so a client can draw it differently.
"""
session, engine, _, _ = make_session(FakeMatcher(score=0.4)) # below the accept threshold
for frame_id in range(1, 8):
result = process(session, frame_id)
for track in result["tracks"]:
self.assertIsNone(track["identity"]["sku_id"],
"a weak match must never name a SKU")
process(session, frame_id)
result = process(session, 9)
track = result["tracks"][0]
self.assertEqual(track["identity"]["sku_id"], "sku_a",
"a weak match must still be shown")
self.assertEqual(track["identity"]["identity_level"], "provisional",
"a weak match must not claim to be accepted")
engine.shutdown()
def test_weak_match_ends_as_needs_confirmation(self):
def test_a_provisional_label_is_never_blanked(self):
"""Once shown, a guess stays: the budget running out must not take it back."""
session, engine, _, _ = make_session(FakeMatcher(score=0.4))
for frame_id in range(1, 8):
for frame_id in range(1, 12):
process(session, frame_id)
result = process(session, 8)
self.assertTrue(wait_for(lambda: process(session, 9)["tracks"][0]["identity"]["state"]
== "needs_confirmation", timeout=3.0))
result = process(session, 13)
track = result["tracks"][0]
self.assertEqual(track["identity"]["sku_id"], "sku_a",
"a spent budget must not blank a label already on screen")
engine.shutdown()
def test_restarted_id_is_re_verified_not_inherited(self):
@ -314,8 +325,10 @@ class CosineScaleTests(unittest.TestCase):
)), "a gallery-accepted head match must be labelled")
engine.shutdown()
def test_a_close_cosine_pair_stays_ambiguous(self):
"""The margin gate still applies: two near-equal scores must not name a SKU."""
def test_a_close_cosine_pair_is_shown_provisional(self):
"""The margin gate still applies to *acceptance*: two near-equal scores
must not be named as settled. Coverage-first shows the top one anyway,
marked provisional, because a blank box is the worse outcome."""
session, engine, _, _ = make_session(FakeMatcher(score=0.45), engine_cls=CosineEngine)
# The matcher contract is (scores, details); a bare dict unpacks into two
# strings and raises inside the future callback, where the failure is
@ -325,15 +338,17 @@ class CosineScaleTests(unittest.TestCase):
process(session, frame_id)
result = process(session, 9)
for track in result["tracks"]:
self.assertIsNone(track["identity"]["sku_id"],
"a cosine pair inside the margin must not be named")
self.assertEqual(track["identity"]["identity_level"], "provisional",
"a cosine pair inside the margin must not be accepted")
engine.shutdown()
def test_a_rejected_crop_cannot_dilute_an_accepted_one(self):
"""aggregate_votes divides by the whole window, so a miss must not lower the line.
A rejected crop first, then an accepted 0.45: the mean is 0.225. Gating the
aggregate at zero would label that, at a score the gallery calls unknown.
aggregate at zero would *accept* that, at a score the gallery calls unknown.
Coverage-first still shows it, but the level must stay provisional: the
dilution must not be promoted to a settled label.
"""
session, engine, _, _ = make_session(FakeMatcher(score=0.45), engine_cls=CosineEngine)
answers = iter([{}, {"sku_a": 0.45}])
@ -343,8 +358,8 @@ class CosineScaleTests(unittest.TestCase):
process(session, frame_id)
result = process(session, 13)
for track in result["tracks"]:
self.assertIsNone(track["identity"]["sku_id"],
"a diluted mean must not be labelled below the gallery's line")
self.assertEqual(track["identity"]["identity_level"], "provisional",
"a diluted mean must not be accepted below the gallery's line")
engine.shutdown()
@ -432,6 +447,38 @@ class LocalScaleTests(unittest.TestCase):
self.assertEqual(engine.ocr_calls, 1, "OCR must run once per track")
engine.shutdown()
def test_no_ocr_never_reads_a_crop(self):
"""The session's `ocr` switch turns the text lane off for every crop.
The text lane is the slowest thing the server does per crop, so the
phone can ask for a session that runs on geometry alone. Only the text
lane goes away: the local lane still names the track, because the whole
point of the setting is a faster session, not a quieter one.
"""
session, engine, _, _ = make_session(
engine_cls=LocalScaleEngine,
settings=LiveSettings(method="hybrid", ocr=False),
)
engine.lane_answers = [({"sku_a": 5.0}, [("sku_a", 0.9)], "コクうま 65%")] * 6
labelled = False
for frame_id in range(1, 14):
for track in process(session, frame_id)["tracks"]:
if track["identity"]["sku_id"] == "sku_a":
labelled = True
self.assertEqual(engine.ocr_calls, 0, "no crop may be read with the text lane off")
self.assertTrue(labelled, "the local lane must still name the track")
engine.shutdown()
def test_the_ocr_switch_is_reported_back_to_the_client(self):
"""The client reads the session's own value, not the one it asked for."""
session, engine, _, _ = make_session()
self.assertTrue(session.config()["ocr"], "the text lane is on by default")
session.configure(ocr=False)
self.assertFalse(session.config()["ocr"], "the switch must reach the config")
session.configure(ocr=True)
self.assertTrue(session.config()["ocr"], "and back on again")
engine.shutdown()
def test_a_size_only_read_is_not_cached(self):
"""An unreadable crop must not veto the product for the track's life.