tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

feat: a live path that uses the enrolled store, and a demo server a phone can reach

Browse source 
The live session scored crops with the matcher regardless of the configured identity
backend, so a live scan used the old mechanism while the still path used the enrolled
gallery - measured, the matcher accepts look-alikes and cannot say "unknown", which is
exactly what a real shelf needs it to do. score_crop now routes to the gallery, and
the decision uses a similarity scale (accept at 0.60 with a 0.04 margin) rather than
inlier-count thresholds that mean nothing on a cosine.

The synthetic walk is gone: demo.py, demo_replay.py, the /v1/demo endpoints and the
walk benchmark are deleted, and the app's frame source is the camera on a device (the
single-image replay stays for the simulator, which has no camera).

stock-demo merges both demo shelves (27 products, 62 views, trained head) so one
server recognises either image, and run-demo-server.sh starts it bound to 0.0.0.0 and
prints the exact URL to put in Config.xcconfig.

Verified over the LAN with 1280x960 frames: 40 frames in 5.3s, 9 products recognised
on demo image 1.

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w 2026-09-25 10:51:57 +09:00
parent 383ff9cf2d
commit 0e23425ef0
74 changed files with 158 additions and 767 deletions
Download patch file Download diff file Expand all files Collapse all files

149
recognization/code/app.py
View file

@ -72,7 +72,6 @@ from engine import ( # noqa: E402
identity_payload,
settings_from_env,
)
from demo import DemoCamera, DemoSourceError # noqa: E402
from record import FrameRecorder # noqa: E402
from live import LiveSession, LiveSettings # noqa: E402
@ -100,7 +99,6 @@ class ServerSettings:
live_method: str = "hybrid"
live_conf: float = 0.1
live_tags: bool = True
demo_walk: bool = False
# A live session can be recorded so it becomes data to label (record.py).
record_frames: Optional[str] = None
record_every: int = 3
@ -250,9 +248,10 @@ def create_app(
)
scans = scans if scans is not None else build_legacy_scans(engine)
# A live session can be recorded so a real scan becomes data to label later.
recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \
if settings.record_frames else None
samples = {item["id"]: item for item in get_available_shelf_images("add")}
demo_camera = DemoCamera(samples)
demo_cameras: Dict[float, DemoCamera] = {}
sessions: Dict[str, LiveSession] = {}
sessions_lock = asyncio.Lock()
app.state.engine = engine
@ -323,110 +322,14 @@ def create_app(
listing = []
for item in samples.values():
entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"}
try: # size is a convenience for clients that plan a demo walk
width, height = demo_camera._size(item["id"])
entry["width"], entry["height"] = width, height
except DemoSourceError:
pass
import cv2
image = cv2.imread(item["path"])
if image is not None:
entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0])
listing.append(entry)
return {"samples": listing}
# ---------------------------------------------------------------- demo camera
# The demo replay lives in its own module: it replaces the live tracks with a
# still recognition mapped into each walk frame, which is a film-making device,
# not a perception result. Every payload it produces is marked synthetic.
demo_recognition = None
recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \
if settings.record_frames else None
def _demo_samples(raw: Optional[str]) -> List[str]:
if raw:
wanted = [part.strip() for part in raw.split(",") if part.strip()]
unknown = [part for part in wanted if part not in samples]
if unknown:
raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404)
return wanted
# main_shelf first: its products are the ones the catalog covers, so a
# demo walk recognises real items instead of unknown shelf neighbours.
for preferred in ("main_shelf", "image6"):
if preferred in samples:
return [preferred]
return [next(iter(samples))] if samples else []
# Which sample the walk uses, resolved once. The replay object itself is built
# later, because it needs the still tag pass.
demo_sample_ids: List[str] = []
if settings.demo_walk:
try:
demo_sample_ids = _demo_samples(None)
demo_camera.warm(demo_sample_ids)
except Exception as exc:
LOGGER.warning("demo walk unavailable: %s", exc)
def _demo_window(width: Optional[int], height: Optional[int]) -> tuple:
win_w = int(width or 960)
win_h = int(height or 540)
if not (160 <= win_w <= 2048 and 120 <= win_h <= 2048):
raise _error("BAD_REQUEST", "window must be between 160x120 and 2048x2048", 400)
return win_w, win_h
def _demo_camera(pan: Optional[float]) -> DemoCamera:
"""One camera per pacing value (the walk geometry depends on it)."""
if pan is None:
return demo_camera
key = round(float(pan), 4)
camera = demo_cameras.get(key)
if camera is None:
camera = DemoCamera(samples, pan_overlap=key)
demo_cameras[key] = camera
return camera
@app.get("/v1/demo/plan", dependencies=[Depends(authorize)])
async def demo_plan(samples_param: Optional[str] = None, width: Optional[int] = None,
height: Optional[int] = None, pan: Optional[float] = None) -> Dict[str, Any]:
"""Geometry of the virtual walk (demo-only capture source)."""
try:
ids = _demo_samples(samples_param)
if not ids:
raise _error("NOT_FOUND", "no samples available", 404)
camera = _demo_camera(pan)
return {"status": "success", **camera.plan(ids, _demo_window(width, height))}
except DemoSourceError as exc:
raise _error("NOT_FOUND", str(exc), 404)
@app.get("/v1/demo/frame", dependencies=[Depends(authorize)])
async def demo_frame(step: int = 0, samples_param: Optional[str] = None,
width: Optional[int] = None, height: Optional[int] = None,
quality: int = 65, pan: Optional[float] = None) -> Response:
"""One JPEG frame of the virtual walk: a store employee sweeping the shelf.
Demo-only. The client advances `step` (it wraps), so the walk's speed
follows whatever frame rate the client can sustain.
"""
try:
ids = _demo_samples(samples_param)
if not ids:
raise _error("NOT_FOUND", "no samples available", 404)
payload, position = _demo_camera(pan).frame(
ids, step, _demo_window(width, height), max(30, min(95, int(quality)))
)
except DemoSourceError as exc:
raise _error("NOT_FOUND", str(exc), 404)
return Response(
content=payload,
media_type="image/jpeg",
headers={
"X-Demo-Sample": str(position["sample"]),
"X-Demo-Bay": str(position["bay_index"] + 1),
"X-Demo-Step": str(position["step"]),
"X-Demo-Steps": str(position["steps"]),
"X-Demo-X": str(position["x"]),
"X-Demo-Y": str(position["y"]),
},
)
@app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)])
async def get_sample(sample_id: str) -> Response:
item = samples.get(sample_id)
@ -599,19 +502,6 @@ def create_app(
engine.image_store.set_tag_bundle(result["image_id"], bundle)
return result
if settings.demo_walk and demo_sample_ids:
# Built here because the replay needs the still tag pass, and it recognises
# the whole shelf once up front so the walk's first frame is already
# labelled. Every payload it produces is marked synthetic.
try:
from demo_replay import DemoRecognition
demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags)
demo_recognition.warm(demo_sample_ids[0])
LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0])
except Exception as exc:
LOGGER.warning("demo walk warm-up failed: %s", exc)
@app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)])
async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]:
method = _require_method(body.method)
@ -715,7 +605,6 @@ def create_app(
await websocket.accept()
send_lock = asyncio.Lock()
loop = asyncio.get_running_loop()
demo_steps = [0]
async def send(payload: Dict[str, Any]) -> None:
async with send_lock:
@ -755,21 +644,9 @@ def create_app(
"message": "frame processing failed"})
continue
if recorder is not None:
# Record what the client sent and what we saw in it, so the walk
# can be labelled later instead of only measured on photographs.
# Record what the client sent and what we saw in it, so a real
# scan can be labelled later instead of only measured on photos.
recorder.write(session_id, frame_id, payload, result)
if demo_recognition is not None and demo_sample_ids:
# The walk's frames are served from the shelf's full recognition,
# so every product is already labelled with name and price.
sample_id = demo_sample_ids[0]
step = demo_steps[0]
try:
result = await loop.run_in_executor(
None, demo_recognition.apply, result, sample_id, step
)
demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id))
except Exception as exc: # a demo failure must not kill the stream
LOGGER.exception("demo walk frame failed: %s", exc)
await send(result)
async def handle_inspect(message: Dict[str, Any]) -> None:
@ -1044,15 +921,13 @@ def main() -> None:
parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"],
help="which mechanism names products (default matcher)")
parser.add_argument("--live-inflight", type=int, default=None,
help="classifications in flight per session (demo walks want a larger budget)")
help="classifications in flight per session")
parser.add_argument("--live-attempts", type=int, default=None,
help="classification attempts per track")
parser.add_argument("--record-frames", type=str, default=None,
help="write processed frames and their results under this directory")
parser.add_argument("--record-every", type=int, default=None,
help="record one frame in every N (default 3)")
parser.add_argument("--demo-walk", action="store_true",
help="serve the demo walk's pre-recognised shelf as live tracks")
parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR")
parser.add_argument("--max-sessions", type=int, default=None)
parser.add_argument("--ssl-certfile", type=str, default=None)
@ -1076,8 +951,6 @@ def main() -> None:
server.live_method = args.live_method
if args.live_conf is not None:
server.live_conf = args.live_conf
if args.demo_walk:
server.demo_walk = True
if args.no_tags:
server.live_tags = False
if args.ssl_certfile is not None: