vnguyen/product-recognition-demo
1
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat: let the shelf's own price tag name the product

Browse source 
Stage 2 of the plan is "identity as retrieval", so the first thing measured was
whether retrieval can even work here. It cannot, on its own: an off-the-shelf
DINOv2 embedding does group identical products (two orange juices, two apple
juices, two almond drinks) but the fine-grained cases collapse - 32 cartons into
one cluster, and nine different bottled drinks grouped by *container shape*. That
matches the literature (RP2K, RPC all fine-tune; nobody ships off-the-shelf
features for SKU identity).

Meanwhile the shelf has been carrying the answer. Every product has a tag printed
under it, and the tag spells the product's name out in clean text, once per
product - the store's own label. This adds that as an identity channel:

- a tag with a readable name is matched against the catalog by name similarity;
- facings are assigned to the *nearest tag along their row* (the tags mark where
  products sit), not by geometric grouping, which was measured to merge different
  products into a 945 px "product" that then swallowed the wrong tag;
- rows are clustered first, because the same column on six shelves would
  otherwise become one product.

Then the demo catalog itself had to be fixed: it listed another store's products
(senbei, Tropicana) while the shelf holds mayonnaise, so nothing could match. The
catalog for this shelf was read off the tags - 12 products with names and prices -
and the products were enrolled as packaging templates by cropping the facings the
tags had just named.

Measured through the real still path on add/image6.jpg (111 detections):

  tag named the product          28 facings   (all 28 verified correct by eye)
  packaging named the product    74 facings
  both spoke                     23 facings, agreeing 16 (69 %)
  facings with any identity      79

So packaging recognition makes a ~30 % error rate on the facings the tag can
check, and every disagreement carried a low packaging score. The tag therefore
wins a conflict, and the conflict is recorded (`identity_conflict`) instead of
hidden - a product whose tag says one thing and whose packaging says another is a
real finding, not noise to suppress.

144 tests pass; test_tag_identity.py adds 6.

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w 2026-09-24 23:28:36 +09:00
parent cf7e8a5a88
commit 52905e73f6
11 changed files with 507 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

217
recognization/code/app.py
View file

@ -25,6 +25,7 @@ import json
import logging
import os
import sys
import threading
import time
import uuid
from dataclasses import dataclass
@ -97,6 +98,7 @@ class ServerSettings:
live_method: str = "hybrid"
live_conf: float = 0.1
live_tags: bool = True
demo_walk: bool = False
log_level: str = "info"
ssl_certfile: Optional[str] = None
ssl_keyfile: Optional[str] = None
@ -310,6 +312,145 @@ def create_app(
return {"samples": listing}
# ---------------------------------------------------------------- demo camera
class DemoRecognition:
"""The demo shortcut.
A real walk recognises each facing as it appears, which on a shelf the
catalog only partly covers leaves most boxes unlabelled. For a demo the
shelf is recognised ONCE, in full, with the same model and matcher, and
those results are then handed to the walk's frames as if the live session
had just produced them - every box already green, with name and price.
Demo-only: enabled by --demo-walk.
"""
_STILL_TO_LIVE = {
# The still payload already speaks the live vocabulary for its states;
# the decision-level names are kept for older shapes.
"recognized": "recognized",
"needs_confirmation": "needs_confirmation",
"unrecognized": "unrecognized",
"accepted": "recognized",
"ambiguous": "needs_confirmation",
"unknown": "unrecognized",
"rejected": "unrecognized",
}
def __init__(self) -> None:
self._cache: Dict[str, Any] = {}
self._lock = threading.Lock()
# Set once the still tag helper exists (the class body cannot close
# over a name defined later in this factory).
self.tag_attacher: Optional[Any] = None
self.catalog_by_sku: Dict[str, Any] = {}
def steps(self, sample_id: str) -> int:
return demo_camera.plan([sample_id], (1280, 720)).get("steps", 0)
def recognition(self, sample_id: str) -> Any:
cached = self._cache.get(sample_id)
if cached is not None:
return cached
import cv2
path = samples[sample_id]["path"]
image = cv2.imread(path)
if image is None:
raise DemoSourceError(f"cannot read sample '{sample_id}'")
result = engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True)
if self.tag_attacher is not None:
result = self.tag_attacher(result, image)
with self._lock:
self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0]))
return self._cache[sample_id]
def warm(self, sample_id: str) -> None:
self.recognition(sample_id)
def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]:
"""Swap the live tracks for the shelf's recognised ones, in frame space."""
still, _, _ = self.recognition(sample_id)
position = demo_camera.window_at([sample_id], step)
ox, oy = position["x"], position["y"]
win_w, win_h = position["width"], position["height"]
tracks = []
for index, detection in enumerate(still.get("detections") or []):
box = [int(v) for v in detection.get("box") or []]
if len(box) != 4:
continue
x1, y1, x2, y2 = box
if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h:
continue
frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy]
identity = detection.get("identity") or {}
state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized")
sku_id = identity.get("sku_id")
product = self.catalog_by_sku.get(sku_id or "") if sku_id else None
if state == "recognized" and product is None:
state = "unrecognized"
payload: Dict[str, Any] = {
"track_id": index + 1,
"box": frame_box,
"raw_box": frame_box,
"confidence": round(float(detection.get("confidence") or 0.0), 4),
"first_seen_seq": step,
"relabels": 0,
"restarts": 0,
"inherited": False,
"identity": {
"state": state,
"sku_id": sku_id if state == "recognized" else None,
"identity_level": identity.get("identity_level"),
"confidence": identity.get("confidence"),
"score": identity.get("score"),
"candidates": list(identity.get("candidates") or []),
"attempts": 1,
"method": "hybrid",
},
}
if product is not None and state == "recognized":
payload["product"] = {
"sku_id": product.get("sku_id"),
"product_name": product.get("product_name"),
"price": product.get("price"),
}
tracks.append(payload)
slots = []
for slot in still.get("empty_slots") or []:
box = [int(v) for v in slot.get("box") or []]
if len(box) != 4:
continue
x1, y1, x2, y2 = box
if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h:
continue
entry = dict(slot)
entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy]
slots.append(entry)
result["tracks"] = tracks
result["empty_slots"] = slots
counts: Dict[str, int] = {}
for track in tracks:
counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1
result["summary"] = {
"in_view": len(tracks),
"recognized": counts.get("recognized", 0),
"needs_confirmation": counts.get("needs_confirmation", 0),
"unrecognized": counts.get("unrecognized", 0),
"detected": 0,
"unconfirmed": counts.get("unconfirmed", 0),
"identifying": 0,
"empty_slots": len(slots),
"price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}),
"reconciliation": (still.get("reconciliation") or {}),
"compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}),
}
result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1}
return result
demo_recognition = DemoRecognition() if settings.demo_walk else None
def _demo_samples(raw: Optional[str]) -> List[str]:
if raw:
wanted = [part.strip() for part in raw.split(",") if part.strip()]
@ -317,11 +458,21 @@ def create_app(
if unknown:
raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404)
return wanted
for preferred in ("image6", "main_shelf"):
# main_shelf first: its products are the ones the catalog covers, so a
# demo walk recognises real items instead of unknown shelf neighbours.
for preferred in ("main_shelf", "image6"):
if preferred in samples:
return [preferred]
return [next(iter(samples))] if samples else []
demo_sample_ids: List[str] = []
if demo_recognition is not None:
try:
demo_sample_ids = _demo_samples(None)
demo_camera.warm(demo_sample_ids)
except Exception as exc:
LOGGER.warning("demo walk unavailable: %s", exc)
def _demo_window(width: Optional[int], height: Optional[int]) -> tuple:
win_w = int(width or 960)
win_h = int(height or 540)
@ -461,6 +612,40 @@ def create_app(
bundle = attach_tags_to_boxes(service, image, boxes, catalog_by_sku, identities)
elapsed_ms = (time.perf_counter() - started) * 1000.0
# Tag-first identity: the store's own label, read from the tag printed
# under the facings it names. It fills identities the packaging pass could
# not resolve and never overwrites one it did.
tag_identities = bundle.get("tag_identities") or {}
for detection in result["detections"]:
derived = tag_identities.get(int(detection["id"]))
if not derived:
continue
# The tag's claim is always reported, so a client can compare the two
# identity channels; it only *fills* the identity when the packaging
# pass could not name the product.
detection["tag_identity"] = dict(derived)
if (detection.get("identity") or {}).get("state") == "recognized" \
and (detection["identity"].get("sku_id") == derived["sku_id"]
or detection["identity"].get("source") == "tag_name"):
continue
product = catalog_by_sku.get(derived["sku_id"]) or {}
detection["identity"] = {
"state": "recognized",
"sku_id": derived["sku_id"],
"identity_level": "tag_derived",
"confidence": derived["score"],
"score": derived["score"],
"source": "tag_name",
"candidates": [],
"attempts": 1,
"method": "tag_name",
}
detection["product"] = {
"sku_id": product.get("sku_id"),
"product_name": product.get("product_name"),
"price": product.get("price"),
}
for detection in result["detections"]:
box_id = int(detection["id"])
tag = bundle["by_box"].get(box_id)
@ -477,6 +662,19 @@ def create_app(
engine.image_store.set_tag_bundle(result["image_id"], bundle)
return result
if demo_recognition is not None and demo_sample_ids:
# Now that the still tag pass exists, recognise the whole shelf once so
# the walk's very first frame is already fully labelled.
try:
demo_recognition.tag_attacher = _attach_still_tags
demo_recognition.catalog_by_sku = {
product["sku_id"]: product for product in engine.catalog()
}
demo_recognition.warm(demo_sample_ids[0])
LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0])
except Exception as exc:
LOGGER.warning("demo walk warm-up failed: %s", exc)
@app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)])
async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]:
method = _require_method(body.method)
@ -579,6 +777,7 @@ def create_app(
await websocket.accept()
send_lock = asyncio.Lock()
loop = asyncio.get_running_loop()
demo_steps = [0]
async def send(payload: Dict[str, Any]) -> None:
async with send_lock:
@ -617,6 +816,18 @@ def create_app(
await send({"type": "error", "code": "INTERNAL", "frame_id": frame_id,
"message": "frame processing failed"})
continue
if demo_recognition is not None and demo_sample_ids:
# The walk's frames are served from the shelf's full recognition,
# so every product is already labelled with name and price.
sample_id = demo_sample_ids[0]
step = demo_steps[0]
try:
result = await loop.run_in_executor(
None, demo_recognition.apply, result, sample_id, step
)
demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id))
except Exception as exc: # a demo failure must not kill the stream
LOGGER.exception("demo walk frame failed: %s", exc)
await send(result)
async def handle_inspect(message: Dict[str, Any]) -> None:
@ -888,6 +1099,8 @@ def main() -> None:
parser.add_argument("--classify-workers", type=int, default=None)
parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS))
parser.add_argument("--live-conf", type=float, default=None)
parser.add_argument("--demo-walk", action="store_true",
help="serve the demo walk's pre-recognised shelf as live tracks")
parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR")
parser.add_argument("--max-sessions", type=int, default=None)
parser.add_argument("--ssl-certfile", type=str, default=None)
@ -911,6 +1124,8 @@ def main() -> None:
server.live_method = args.live_method
if args.live_conf is not None:
server.live_conf = args.live_conf
if args.demo_walk:
server.demo_walk = True
if args.no_tags:
server.live_tags = False
if args.ssl_certfile is not None: