diff --git a/recognization/.github/workflows/test.yml b/recognization/.github/workflows/test.yml new file mode 100644 index 0000000..48517a0 --- /dev/null +++ b/recognization/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: tests + +on: + push: + pull_request: + +jobs: + model-free: + runs-on: ubuntu-latest + defaults: + run: + working-directory: recognization + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install pinned dependencies + run: python -m pip install -r requirements.lock + - name: Model-free suite + env: + YOLO_OFFLINE: "1" + YOLO_AUTOINSTALL: "false" + run: python -m unittest discover -s code -p 'test_*.py' -v diff --git a/recognization/.gitignore b/recognization/.gitignore index 789e4d1..a0bb35a 100644 --- a/recognization/.gitignore +++ b/recognization/.gitignore @@ -20,3 +20,6 @@ ENV/ # Temporary weights runs/detect/train/weights/last.pt *.log +stock-*/identity_head.pt +.demo-token +config/local.env diff --git a/recognization/README.md b/recognization/README.md index 6dbf1ab..dff70a2 100644 --- a/recognization/README.md +++ b/recognization/README.md @@ -101,19 +101,17 @@ Bất kỳ client nào gửi được ảnh JPEG đều dùng được: điện ## 🛠️ Kiểm tra hệ thống (Tests) +From `recognization/`, after `pip install -r requirements.lock` (or `pip install -r requirements-dev.txt`): + ```bash -cd code -YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_observe test_engine test_tracking test_live test_api -v - -# Test với model thật (tải trọng số, chạy ~15s) -DORAYAKI_REAL_MODEL=1 YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_real_model -v - -cd .. -YOLO_OFFLINE=1 .venv/bin/python code/test_suite.py # bộ test gốc -.venv/bin/python code/bench_engine.py # đo tốc độ detect/classify +YOLO_OFFLINE=1 YOLO_AUTOINSTALL=false python -m unittest discover -s code -p 'test_*.py' -v ``` -Lưu ý: chạy test bằng danh sách module cụ thể, **không dùng** `unittest discover` vì sẽ import `test.py`/`fast_test.py` (script scratch với đường dẫn máy khác và cửa sổ OpenCV chặn). +That discovers every `code/test_*.py` module. `test_real_model` is included and skipped unless `DORAYAKI_REAL_MODEL=1` (it loads the YOLO weights). CI runs the same command from `.github/workflows/test.yml`. + +```bash +YOLO_OFFLINE=1 python code/bench_engine.py # speed measurement, not part of the suite +``` --- @@ -145,6 +143,7 @@ code/ empty_shelf_detector.py, inventory.py, pipeline.py, observe.py stream_client.py # client mẫu (camera / video / pan ảnh tĩnh) bench_engine.py # đo tốc độ + test_*.py # unittest modules (discover -s code -p 'test_*.py') web/ # web client (index.html, styles.css, app.js) docs/API.md # tài liệu API đầy đủ docs/design-price-tags-and-compliance.md # thiết kế OCR giá, đối chiếu, compliance, mobile diff --git a/recognization/code/app.py b/recognization/code/app.py index d4a3cc7..d3f5882 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -20,10 +20,11 @@ from __future__ import annotations import argparse import asyncio import base64 -import hmac +from contextlib import asynccontextmanager import json import logging import os +import re import sys import threading import time @@ -71,9 +72,19 @@ from engine import ( # noqa: E402 identity_payload, settings_from_env, ) -from demo import DemoCamera, DemoSourceError # noqa: E402 from record import FrameRecorder # noqa: E402 from live import LiveSession, LiveSettings # noqa: E402 +from transport import ( # noqa: E402 + BodyLimit, + ServiceAdvertiser, + advertise_host, + bearer_matches, + declared_length, + is_loopback_host, + read_bounded, + require_token_for_bind, + token_matches, +) LOGGER = logging.getLogger("dorayaki") @@ -94,12 +105,12 @@ class ServerSettings: host: str = "127.0.0.1" port: int = 8080 token: Optional[str] = None - cors_origins: str = "*" + # Empty means same-origin only. "*" reflects every caller and is not a default. + cors_origins: str = "" max_sessions: int = DEFAULT_MAX_SESSIONS live_method: str = "hybrid" live_conf: float = 0.1 live_tags: bool = True - demo_walk: bool = False # A live session can be recorded so it becomes data to label (record.py). record_frames: Optional[str] = None record_every: int = 3 @@ -125,6 +136,26 @@ def server_settings_from_env() -> ServerSettings: ) +def load_local_token(path: str = "config/local.env") -> None: + """Fill DORAYAKI_API_TOKEN from an ignored env file. Never prints the value. + + ponytail: one KEY=VALUE file, no quoting. Upgrade path is a real env loader + once a second secret needs the same file. + """ + if os.environ.get("DORAYAKI_API_TOKEN"): + return + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.readlines() + except OSError: + return + for line in lines: + key, sep, value = line.strip().partition("=") + if key == "DORAYAKI_API_TOKEN" and sep and value: + os.environ["DORAYAKI_API_TOKEN"] = value + return + + # --------------------------------------------------------------------------- # request models # --------------------------------------------------------------------------- @@ -235,23 +266,35 @@ def create_app( scans: Any = None, ) -> FastAPI: settings = settings or server_settings_from_env() + @asynccontextmanager + async def lifespan(_app: FastAPI): + yield + for session in list(sessions.values()): + session.close() + engine.shutdown() + app = FastAPI( title="Dorayaki Recognition Server", version="1.0", description="Shelf product detection, tracking and classification for capture clients.", + lifespan=lifespan, ) - app.add_middleware( - CORSMiddleware, - allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()] or ["*"], - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], - ) + # Same-origin is the default. A configured list is echoed back; "*" is not. + cors = [o.strip() for o in settings.cors_origins.split(",") if o.strip() and o.strip() != "*"] + if cors: + app.add_middleware( + CORSMiddleware, + allow_origins=cors, + allow_credentials=False, + allow_methods=["GET", "POST"], + allow_headers=["Authorization", "Content-Type"], + ) scans = scans if scans is not None else build_legacy_scans(engine) + # A live session can be recorded so a real scan becomes data to label later. + recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ + if settings.record_frames else None samples = {item["id"]: item for item in get_available_shelf_images("add")} - demo_camera = DemoCamera(samples) - demo_cameras: Dict[float, DemoCamera] = {} sessions: Dict[str, LiveSession] = {} sessions_lock = asyncio.Lock() app.state.engine = engine @@ -260,8 +303,7 @@ def create_app( async def authorize(authorization: Optional[str] = Header(default=None)) -> None: if not settings.token: return - expected = f"Bearer {settings.token}" - if not authorization or not hmac.compare_digest(authorization, expected): + if not bearer_matches(authorization, settings.token): raise _error("UNAUTHORIZED", "missing or invalid bearer token", 401) # ---------------------------------------------------------------- health @@ -276,6 +318,14 @@ def create_app( "warm": True, }, "catalog": {"version": engine.catalog_version(), "sku_count": engine.sku_count()}, + "identity": { + "backend": getattr(engine.settings, "identity_backend", "matcher"), + "gallery_views": ( + engine.identity_gallery().view_count() + if getattr(engine.settings, "identity_backend", "") == "gallery" and engine.identity_gallery() + else None + ), + }, "ocr": { # Without this an operator could not tell a broken OCR from a shelf # with no readable tags. @@ -308,115 +358,22 @@ def create_app( raise _error("NOT_FOUND", f"unknown sku '{sku_id}'", 404) return {"status": "success", "product": product} - # ---------------------------------------------------------------- samples - @app.get("/v1/samples", dependencies=[Depends(authorize)]) - async def list_samples() -> Dict[str, Any]: + def _sample_listing() -> List[Dict[str, Any]]: listing = [] + import cv2 + for item in samples.values(): entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"} - try: # size is a convenience for clients that plan a demo walk - width, height = demo_camera._size(item["id"]) - entry["width"], entry["height"] = width, height - except DemoSourceError: - pass + image = cv2.imread(item["path"]) + if image is not None: + entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0]) listing.append(entry) - return {"samples": listing} + return listing - # ---------------------------------------------------------------- demo camera - # The demo replay lives in its own module: it replaces the live tracks with a - # still recognition mapped into each walk frame, which is a film-making device, - # not a perception result. Every payload it produces is marked synthetic. - demo_recognition = None - recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ - if settings.record_frames else None - - - - def _demo_samples(raw: Optional[str]) -> List[str]: - if raw: - wanted = [part.strip() for part in raw.split(",") if part.strip()] - unknown = [part for part in wanted if part not in samples] - if unknown: - raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404) - return wanted - # main_shelf first: its products are the ones the catalog covers, so a - # demo walk recognises real items instead of unknown shelf neighbours. - for preferred in ("main_shelf", "image6"): - if preferred in samples: - return [preferred] - return [next(iter(samples))] if samples else [] - - # Which sample the walk uses, resolved once. The replay object itself is built - # later, because it needs the still tag pass. - demo_sample_ids: List[str] = [] - if settings.demo_walk: - try: - demo_sample_ids = _demo_samples(None) - demo_camera.warm(demo_sample_ids) - except Exception as exc: - LOGGER.warning("demo walk unavailable: %s", exc) - - def _demo_window(width: Optional[int], height: Optional[int]) -> tuple: - win_w = int(width or 960) - win_h = int(height or 540) - if not (160 <= win_w <= 2048 and 120 <= win_h <= 2048): - raise _error("BAD_REQUEST", "window must be between 160x120 and 2048x2048", 400) - return win_w, win_h - - def _demo_camera(pan: Optional[float]) -> DemoCamera: - """One camera per pacing value (the walk geometry depends on it).""" - if pan is None: - return demo_camera - key = round(float(pan), 4) - camera = demo_cameras.get(key) - if camera is None: - camera = DemoCamera(samples, pan_overlap=key) - demo_cameras[key] = camera - return camera - - @app.get("/v1/demo/plan", dependencies=[Depends(authorize)]) - async def demo_plan(samples_param: Optional[str] = None, width: Optional[int] = None, - height: Optional[int] = None, pan: Optional[float] = None) -> Dict[str, Any]: - """Geometry of the virtual walk (demo-only capture source).""" - try: - ids = _demo_samples(samples_param) - if not ids: - raise _error("NOT_FOUND", "no samples available", 404) - camera = _demo_camera(pan) - return {"status": "success", **camera.plan(ids, _demo_window(width, height))} - except DemoSourceError as exc: - raise _error("NOT_FOUND", str(exc), 404) - - @app.get("/v1/demo/frame", dependencies=[Depends(authorize)]) - async def demo_frame(step: int = 0, samples_param: Optional[str] = None, - width: Optional[int] = None, height: Optional[int] = None, - quality: int = 65, pan: Optional[float] = None) -> Response: - """One JPEG frame of the virtual walk: a store employee sweeping the shelf. - - Demo-only. The client advances `step` (it wraps), so the walk's speed - follows whatever frame rate the client can sustain. - """ - try: - ids = _demo_samples(samples_param) - if not ids: - raise _error("NOT_FOUND", "no samples available", 404) - payload, position = _demo_camera(pan).frame( - ids, step, _demo_window(width, height), max(30, min(95, int(quality))) - ) - except DemoSourceError as exc: - raise _error("NOT_FOUND", str(exc), 404) - return Response( - content=payload, - media_type="image/jpeg", - headers={ - "X-Demo-Sample": str(position["sample"]), - "X-Demo-Bay": str(position["bay_index"] + 1), - "X-Demo-Step": str(position["step"]), - "X-Demo-Steps": str(position["steps"]), - "X-Demo-X": str(position["x"]), - "X-Demo-Y": str(position["y"]), - }, - ) + @app.get("/v1/samples", dependencies=[Depends(authorize)]) + async def list_samples() -> Dict[str, Any]: + loop = asyncio.get_running_loop() + return {"samples": await loop.run_in_executor(None, _sample_listing)} @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)]) async def get_sample(sample_id: str) -> Response: @@ -438,39 +395,88 @@ def create_app( if method is not None: _require_method(method) content_type = (request.headers.get("content-type") or "").split(";")[0].strip().lower() + announced = declared_length(request.headers.get("content-length")) + if announced is not None and announced > MAX_UPLOAD_BYTES: + raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) if content_type == "multipart/form-data": - form = await request.form() + form = await request.form(max_part_size=MAX_UPLOAD_BYTES) upload = form.get("file") if upload is None or not hasattr(upload, "read"): raise _error("BAD_REQUEST", "multipart body needs a 'file' field", 400) - payload = await upload.read() + payload = await read_bounded(upload, MAX_UPLOAD_BYTES) else: if content_type and not content_type.startswith("image/") and content_type != "application/octet-stream": raise _error("UNSUPPORTED_MEDIA_TYPE", f"unsupported content type '{content_type}'", 415) - payload = await request.body() + payload = await read_bounded(request.stream(), MAX_UPLOAD_BYTES) if not payload: raise _error("BAD_REQUEST", "empty request body", 400) if len(payload) > MAX_UPLOAD_BYTES: raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) + loop = asyncio.get_running_loop() try: - image = engine.decode_image(payload) + image = await loop.run_in_executor(None, engine.decode_image, payload) except UndecodableImage as exc: raise _error("UNDECODABLE_IMAGE", str(exc), 422) try: - result = engine.recognize( - image, - classify=classify, - method=method, - conf=conf, - with_empty_slots=empty_slots, + result = await loop.run_in_executor( + None, + lambda: engine.recognize( + image, + classify=classify, + method=method, + conf=conf, + with_empty_slots=empty_slots, + ), ) except UnknownMethod as exc: raise _error("BAD_REQUEST", str(exc), 400) if tags: - result = _attach_still_tags(result, image) + result = await loop.run_in_executor(None, _attach_still_tags, result, image) return result + def _is_price_card(crop: Any) -> bool: + """Is this detection a shelf price card rather than a product? + + Measured on a supermarket photo whose shelf edge carries big price cards: + the detector boxes them as products, because a card is roughly the shape of a + pack. Their text gives them away - a price card's characters are 30-44 % + digits and it prints the tax-excluded price (本体), while a product's are 3-19 % + digits. Reporting a price card as a product would put a box labelled with a + product name on a price tag, which is worse than not detecting it. + """ + ocr_engine = getattr(engine, "ocr_engine", None) + if ocr_engine is None or not getattr(ocr_engine, "available", lambda: False)(): + return False + try: + import cv2 + + big = cv2.resize(crop, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_CUBIC) + lines = ocr_engine.read(big).lines + except Exception: + return False + text = " ".join(line.text for line in lines) + compact = re.sub(r"\s", "", text) + if not compact: + return False + digits = sum(1 for char in compact if char.isdigit() or char in "¥￥./") / len(compact) + return ("本体" in text and digits >= 0.20) or digits >= 0.30 + + def _drop_price_cards(result: Dict[str, Any], image: Any) -> int: + """Remove detections that are price cards, and say how many were removed.""" + kept, dropped = [], 0 + for detection in result["detections"]: + crop = engine.crop(image, detection["box"]) + if crop is not None and crop.size and _is_price_card(crop): + dropped += 1 + continue + kept.append(detection) + if dropped: + result["detections"] = kept + result["summary"] = engine.summarize(kept, result.get("empty_slots") or []) + result["price_cards_dropped"] = dropped + return dropped + def _attach_still_tags(result: Dict[str, Any], image: Any) -> Dict[str, Any]: """Run one synchronous tag pass over a still and merge its evidence.""" ocr_engine = getattr(engine, "ocr_engine", None) @@ -483,6 +489,9 @@ def create_app( from reconcile import attach_tags_to_boxes from tags import TagService, TagSettings + # Before anything else: a price card is not a product. + _drop_price_cards(result, image) + boxes = [ {"id": detection["id"], "box": detection["box"]} for detection in result["detections"] ] @@ -545,19 +554,6 @@ def create_app( engine.image_store.set_tag_bundle(result["image_id"], bundle) return result - if settings.demo_walk and demo_sample_ids: - # Built here because the replay needs the still tag pass, and it recognises - # the whole shelf once up front so the walk's first frame is already - # labelled. Every payload it produces is marked synthetic. - try: - from demo_replay import DemoRecognition - - demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags) - demo_recognition.warm(demo_sample_ids[0]) - LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0]) - except Exception as exc: - LOGGER.warning("demo walk warm-up failed: %s", exc) - @app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)]) async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]: method = _require_method(body.method) @@ -570,7 +566,10 @@ def create_app( wanted = set(int(i) for i in body.detection_ids) detections = [d for d in detections if int(d["id"]) in wanted] started = time.perf_counter() - rows = engine.classify_boxes(stored.image, detections, method=method) + loop = asyncio.get_running_loop() + rows = await loop.run_in_executor( + None, lambda: engine.classify_boxes(stored.image, detections, method=method) + ) rows_by_id = {row["detection_id"]: row for row in rows} for detection in stored.detections: row = rows_by_id.get(int(detection["id"])) @@ -623,21 +622,24 @@ def create_app( stored = engine.image_store.get(image_id) except ImageExpired: raise _error("IMAGE_EXPIRED", f"image '{image_id}' is no longer available", 404) - return engine.inspect(stored.image, body.box, method=method) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, lambda: engine.inspect(stored.image, body.box, method=method) + ) # ---------------------------------------------------------------- live stream @app.websocket("/v1/stream") async def stream(websocket: WebSocket) -> None: - if settings.token: - supplied = websocket.query_params.get("token") or "" - header = websocket.headers.get("authorization") or "" - expected_header = f"Bearer {settings.token}" - if not ( - hmac.compare_digest(supplied, settings.token) - or hmac.compare_digest(header, expected_header) - ): - await websocket.close(code=1008) - return + # Query tokens are rejected. A wrong header is rejected before hello. + # A missing header is the browser case: hello goes out, then the first + # client text must be {"type":"auth","token":"..."} or the socket closes. + if websocket.query_params.get("token"): + await websocket.close(code=1008) + return + header = websocket.headers.get("authorization") + if settings.token and header and not bearer_matches(header, settings.token): + await websocket.close(code=1008) + return async with sessions_lock: if len(sessions) >= settings.max_sessions: await websocket.close(code=1013) @@ -661,7 +663,6 @@ def create_app( await websocket.accept() send_lock = asyncio.Lock() loop = asyncio.get_running_loop() - demo_steps = [0] async def send(payload: Dict[str, Any]) -> None: async with send_lock: @@ -701,21 +702,10 @@ def create_app( "message": "frame processing failed"}) continue if recorder is not None: - # Record what the client sent and what we saw in it, so the walk - # can be labelled later instead of only measured on photographs. - recorder.write(session_id, frame_id, payload, result) - if demo_recognition is not None and demo_sample_ids: - # The walk's frames are served from the shelf's full recognition, - # so every product is already labelled with name and price. - sample_id = demo_sample_ids[0] - step = demo_steps[0] - try: - result = await loop.run_in_executor( - None, demo_recognition.apply, result, sample_id, step - ) - demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id)) - except Exception as exc: # a demo failure must not kill the stream - LOGGER.exception("demo walk frame failed: %s", exc) + # Disk writes must not sit on the event loop between results. + await loop.run_in_executor( + None, recorder.write, session_id, frame_id, payload, result + ) await send(result) async def handle_inspect(message: Dict[str, Any]) -> None: @@ -751,8 +741,23 @@ def create_app( processing = asyncio.create_task(processing_loop()) - async def handle_control(message: Dict[str, Any]) -> None: + authed = not settings.token or bearer_matches( + websocket.headers.get("authorization"), settings.token + ) + + async def handle_control(message: Dict[str, Any]) -> bool: + """Apply one control message. False closes the socket (auth failed).""" + nonlocal authed kind = message.get("type") + if not authed: + presented = message.get("token") if kind == "auth" else None + if isinstance(presented, str) and token_matches(presented, settings.token): + authed = True + return True + await websocket.close(code=1008) + return False + if kind == "auth": + return True if kind == "config": try: config = session.configure( @@ -763,7 +768,7 @@ def create_app( ) except (TypeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "bad config values"}) - return + return True await send({"type": "config_ack", "config": config}) elif kind == "reset": session.reset() @@ -774,6 +779,7 @@ def create_app( await handle_inspect(message) else: await send({"type": "error", "code": "BAD_MESSAGE", "message": f"unknown type '{kind}'"}) + return True try: while True: @@ -786,6 +792,9 @@ def create_app( text = message.get("text") data = message.get("bytes") if data is not None: + if not authed: + await websocket.close(code=1008) + break if len(data) < 5: await send({"type": "error", "code": "BAD_FRAME", "message": "frame shorter than 5 bytes"}) continue @@ -802,7 +811,8 @@ def create_app( except (json.JSONDecodeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "invalid JSON"}) continue - await handle_control(parsed) + if not await handle_control(parsed): + break except WebSocketDisconnect: pass finally: @@ -816,7 +826,7 @@ def create_app( sessions.pop(session_id, None) # ---------------------------------------------------------------- legacy session routes - @app.post("/api/sessions") + @app.post("/api/sessions", dependencies=[Depends(authorize)]) async def legacy_start_session(payload: Dict[str, Any] = None) -> Dict[str, Any]: payload = payload or {} session = scans.start_session( @@ -826,7 +836,7 @@ def create_app( ) return {"status": "success", "session": session.snapshot()} - @app.post("/api/sessions/{session_id}/frames") + @app.post("/api/sessions/{session_id}/frames", dependencies=[Depends(authorize)]) async def legacy_observe_frame(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import FrameRequest, ObserveError @@ -834,11 +844,19 @@ def create_app( try: session = scans.get_session(session_id) raw = payload.get("frame_bytes_b64") or payload.get("image_base64") or "" + if not isinstance(raw, str): + raw = "" if "," in raw: raw = raw.split(",", 1)[1] + # Base64 expands ~4/3. Reject the text before decoding it. + if len(raw) > (MAX_FRAME_BYTES * 4) // 3 + 8: + raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) + frame_bytes = base64.b64decode(raw) if raw else b"" + if len(frame_bytes) > MAX_FRAME_BYTES: + raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) request = FrameRequest( frame_id=str(payload.get("frame_id") or ""), - frame_bytes=base64.b64decode(raw) if raw else b"", + frame_bytes=frame_bytes, sequence=int(payload.get("sequence", 0)), captured_at=float(payload.get("captured_at") or 0), session_id=session.session_id, @@ -850,12 +868,13 @@ def create_app( ) if not request.frame_id: raise _error("BAD_REQUEST", "frame_id required", 400) - observation = scans.observe_frame(request) + loop = asyncio.get_running_loop() + observation = await loop.run_in_executor(None, scans.observe_frame, request) return {"status": "success", "observation": observation.as_dict(), "session": session.snapshot()} except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.get("/api/sessions/{session_id}") + @app.get("/api/sessions/{session_id}", dependencies=[Depends(authorize)]) async def legacy_get_session(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -864,7 +883,7 @@ def create_app( except ObserveError as exc: raise _error("NOT_FOUND", str(exc), 404) - @app.post("/api/sessions/{session_id}/review") + @app.post("/api/sessions/{session_id}/review", dependencies=[Depends(authorize)]) async def legacy_review(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -881,7 +900,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/finalize") + @app.post("/api/sessions/{session_id}/finalize", dependencies=[Depends(authorize)]) async def legacy_finalize(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -890,7 +909,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/cancel") + @app.post("/api/sessions/{session_id}/cancel", dependencies=[Depends(authorize)]) async def legacy_cancel(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -899,7 +918,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/rescan") + @app.post("/api/sessions/{session_id}/rescan", dependencies=[Depends(authorize)]) async def legacy_rescan(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -908,7 +927,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/compare_live_vs_still") + @app.post("/api/compare_live_vs_still", dependencies=[Depends(authorize)]) async def legacy_compare(payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -950,14 +969,11 @@ def create_app( # ---------------------------------------------------------------- static web client resolved_web_dir = web_dir or os.path.join(current_dir, "web") + app.add_middleware(BodyLimit, max_bytes=MAX_UPLOAD_BYTES) if os.path.isdir(resolved_web_dir): app.mount("/", StaticFiles(directory=resolved_web_dir, html=True), name="web") - @app.on_event("shutdown") - async def _shutdown() -> None: - for session in list(sessions.values()): - session.close() - engine.shutdown() + return app @@ -987,16 +1003,16 @@ def main() -> None: parser.add_argument("--classify-workers", type=int, default=None) parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) + parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"], + help="which mechanism names products (default matcher)") parser.add_argument("--live-inflight", type=int, default=None, - help="classifications in flight per session (demo walks want a larger budget)") + help="classifications in flight per session") parser.add_argument("--live-attempts", type=int, default=None, help="classification attempts per track") parser.add_argument("--record-frames", type=str, default=None, help="write processed frames and their results under this directory") parser.add_argument("--record-every", type=int, default=None, help="record one frame in every N (default 3)") - parser.add_argument("--demo-walk", action="store_true", - help="serve the demo walk's pre-recognised shelf as live tracks") parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR") parser.add_argument("--max-sessions", type=int, default=None) parser.add_argument("--ssl-certfile", type=str, default=None) @@ -1020,8 +1036,6 @@ def main() -> None: server.live_method = args.live_method if args.live_conf is not None: server.live_conf = args.live_conf - if args.demo_walk: - server.demo_walk = True if args.no_tags: server.live_tags = False if args.ssl_certfile is not None: @@ -1031,11 +1045,18 @@ def main() -> None: if args.log_level is not None: server.log_level = args.log_level + load_local_token() + if not server.token: + server.token = os.environ.get("DORAYAKI_API_TOKEN") or None + require_token_for_bind(server.host, server.token) + engine_settings = settings_from_env() if args.record_frames is not None: server.record_frames = args.record_frames if args.record_every is not None: server.record_every = args.record_every + if args.identity is not None: + engine_settings.identity_backend = args.identity if args.live_inflight is not None: engine_settings.live_inflight = args.live_inflight if args.live_attempts is not None: @@ -1065,23 +1086,34 @@ def main() -> None: scheme = "https" if server.ssl_certfile else "http" LOGGER.info("Dorayaki recognition server on %s://%s:%d (token %s)", scheme, server.host, server.port, "required" if server.token else "off") - if server.host in ("127.0.0.1", "localhost"): + if is_loopback_host(server.host): LOGGER.info("Open %s://127.0.0.1:%d in a browser for the web client", scheme, server.port) else: LOGGER.info("LAN mode: live camera needs https (secure context); use --ssl-certfile or a tunnel") import uvicorn - uvicorn.run( - app, - host=server.host, - port=server.port, - log_level=(server.log_level or "info").lower(), - access_log=False, # access log would print ?token=... query strings - ws="websockets-sansio", - ssl_certfile=server.ssl_certfile, - ssl_keyfile=server.ssl_keyfile, - ) + advertised = advertise_host(server.host) + advertiser = ServiceAdvertiser(advertised, server.port) if advertised else None + if advertiser is not None and advertiser.start(): + LOGGER.info("Advertising _dorayaki._tcp on %s:%d", advertised, server.port) + elif advertised: + LOGGER.info("No dns-sd on this host; _dorayaki._tcp was not advertised") + + try: + uvicorn.run( + app, + host=server.host, + port=server.port, + log_level=(server.log_level or "info").lower(), + access_log=False, # request lines can carry Authorization; keep them off + ws="websockets-sansio", + ssl_certfile=server.ssl_certfile, + ssl_keyfile=server.ssl_keyfile, + ) + finally: + if advertiser is not None: + advertiser.close() if __name__ == "__main__": diff --git a/recognization/code/benchmark.py b/recognization/code/benchmark.py deleted file mode 100644 index b54f6ca..0000000 --- a/recognization/code/benchmark.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Benchmark Suite for Supermarket Product Classification -Compares SURF, ORB, BRISK, and Hybrid methods following -MDPI Electronics 2023, 12(17), 3640. -""" - -import os -import sys -import time -import cv2 -import numpy as np -from typing import Dict, List, Any - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from feature_matcher import ProductFeatureMatcher - - -def run_benchmark(stock_dir: str = "stock", num_runs: int = 3): - print("=" * 70) - print("RETAIL SHELF PRODUCT RECOGNITION - ALGORITHM BENCHMARK") - print("Methods Evaluated: SURF | ORB | BRISK | HYBRID (Paper Method)") - print("=" * 70) - - matcher = ProductFeatureMatcher(stock_dir=stock_dir) - templates = matcher.db.templates - - if not templates: - print("[Error] No templates found in stock directory.") - return - - methods = ["surf", "orb", "brisk", "hybrid"] - results = {m: {"latencies": [], "inliers": [], "keypoints_query": []} for m in methods} - - # Test images: We will evaluate using crops synthesized with slight transformations - # (scale, rotation, noise) as well as the template images themselves - print(f"\nEvaluating across {len(templates)} catalog SKUs (repeat count: {num_runs})...\n") - - for sku_id, tmpl in templates.items(): - original_img = tmpl["image"] - h, w = original_img.shape[:2] - - # Apply synthetic shelf condition (rotation by 10 deg, resize by 0.85) - M = cv2.getRotationMatrix2D((w / 2, h / 2), 10, 0.85) - query_img = cv2.warpAffine(original_img, M, (w, h)) - - for method in methods: - times = [] - for _ in range(num_runs): - t0 = time.perf_counter() - res = matcher.classify_crop(query_img, method=method) - t1 = time.perf_counter() - times.append((t1 - t0) * 1000.0) # ms - - avg_latency = np.mean(times) - results[method]["latencies"].append(avg_latency) - - # Record inliers/score - if res.get("status") == "success": - top_sku = res["best_sku"] - score = res["score"] - results[method]["inliers"].append(score) - - # Summarize metrics - summary_rows = [] - for method in methods: - latencies = results[method]["latencies"] - scores = results[method]["inliers"] - - avg_lat = np.mean(latencies) if latencies else 0.0 - p95_lat = np.percentile(latencies, 95) if latencies else 0.0 - avg_score = np.mean(scores) if scores else 0.0 - - # Paper characteristics notes - notes = { - "surf": "Scale & rotation robust; Hessian gradient features", - "orb": "Ultra fast binary descriptor; FAST + BRIEF", - "brisk": "Scale-space AGAST; circular pattern descriptor", - "hybrid": "Paper fusion (SURF+ORB+BRISK); optimal accuracy/speed" - }[method] - - summary_rows.append([ - method.upper(), - f"{avg_lat:.2f} ms", - f"{p95_lat:.2f} ms", - f"{avg_score:.1f}", - notes - ]) - - headers = ["Method", "Avg Latency (ms)", "P95 Latency (ms)", "Avg Match Score", "Characteristics / Paper Findings"] - print("\n" + "-" * 105) - print(f"{headers[0]:<10} | {headers[1]:<18} | {headers[2]:<18} | {headers[3]:<16} | {headers[4]}") - print("-" * 105) - for row in summary_rows: - print(f"{row[0]:<10} | {row[1]:<18} | {row[2]:<18} | {row[3]:<16} | {row[4]}") - print("-" * 105) - print("\n[Benchmark Complete] All methods operational.") - - -if __name__ == "__main__": - run_benchmark() diff --git a/recognization/code/build_store_from_shelf.py b/recognization/code/build_store_from_shelf.py new file mode 100644 index 0000000..589328d --- /dev/null +++ b/recognization/code/build_store_from_shelf.py @@ -0,0 +1,66 @@ +""" +Enroll a shelf: the products read off its crops become the store's catalog. + +This is the onboarding flow a real store would go through once - detect the facings, +name the products, and keep their crops as the gallery that recognition matches +against. The two demo shelves were built with it; the product lists live in this file +because reading a contact sheet is a human step. + + python code/build_store_from_shelf.py +""" +import json, os, shutil, sys +sys.path.insert(0, "code") +import cv2 + +ROOT = os.path.expanduser("~/dorayaki-data/shelf-images") +manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) +entry = next(e for e in manifest["images"] if "Azabudai-Hills" in e["file"]) +image = cv2.imread(os.path.join(ROOT, entry["file"])) +height, width = image.shape[:2] +scale = min(1.0, 1600.0 / max(height, width)) +image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) +boxes = json.load(open("/tmp/dorayaki-verify/mediya-boxes.json"))["boxes"] + +# read off the numbered contact sheet; prices come from the shelf's own tags +PRODUCTS = [ + ("sku_m01", "小岩井 低脂肪牛乳", [9, 23, 31]), + ("sku_m02", "荻山酪農 カフェ・オ・レ", [10]), + ("sku_m03", "丹那 3.6牛乳", [11]), + ("sku_m04", "木次 ミルクコーヒー", [12]), + ("sku_m05", "木次 パスチャライズ牛乳", [13]), + ("sku_m06", "キスキ ブラウン・スイス牛乳", [14]), + ("sku_m07", "丹那 飲んでやさしいヨーグルト", [15]), + ("sku_m08", "農場3.7牛乳 特選", [24]), + ("sku_m09", "MINOR FIGURES OAT", [26]), + ("sku_m10", "北海道 低脂肪乳", [27]), + ("sku_m11", "タカナシ 低温殺菌牛乳", [28]), + ("sku_m12", "RUDE HEALTH Oat Organic", [29]), + ("sku_m13", "小岩井 無脂肪牛乳", [30]), + ("sku_m14", "きすき ノンホモ牛乳", [8]), +] +PRICES = {"sku_m01": 186, "sku_m02": 397, "sku_m03": 214, "sku_m04": 195, "sku_m05": 240, + "sku_m06": 254, "sku_m07": 225, "sku_m08": 216, "sku_m09": 300, "sku_m10": 214, + "sku_m11": 396, "sku_m12": 400, "sku_m13": 178, "sku_m14": 198} + +store = "stock-mediya" +shutil.rmtree(store, ignore_errors=True) +os.makedirs(store, exist_ok=True) +rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"] +written = [] +for sku, name, indices in PRODUCTS: + saved = 0 + for position, index in enumerate(indices): + box = boxes[index] + x1, y1, x2, y2 = [int(v) for v in box] + crop = image[max(0, y1):y2, max(0, x1):x2] + if crop.size == 0: + continue + suffix = "" if saved == 0 else f"_{saved}" + cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop) + saved += 1 + if saved == 0: + raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view") + written.append(sku) + rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}") +open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n") +print(f"{store}: {len(written)} products, csv written") diff --git a/recognization/code/collect_shelf_images.py b/recognization/code/collect_shelf_images.py new file mode 100644 index 0000000..56e20a2 --- /dev/null +++ b/recognization/code/collect_shelf_images.py @@ -0,0 +1,190 @@ +""" +Collect openly licensed store-shelf photographs for evaluation. + +Everything measured so far comes from two photographs. This pulls real store photos +from Wikimedia Commons - scriptable, licensed, and no authentication - so the +pipeline can be measured on images it has never seen. + +Only permissively licensed files are kept (CC0, CC BY, CC BY-SA, public domain), and +each one is written to a manifest with its author, licence and source URL, so an +attribution page can be produced from it. + + python code/collect_shelf_images.py [output_dir] +""" + +from __future__ import annotations + +import json +import os +import ssl +import sys +import time +import urllib.parse +import urllib.request + +import certifi + +CONTEXT = ssl.create_default_context(cafile=certifi.where()) +API = "https://commons.wikimedia.org/w/api.php" +USER_AGENT = "dorayaki-eval/0.1 (shelf recognition research; contact: repo owner)" +# Commons asks for a polite pace and offers thumbnails instead of originals. Both +# matter: a burst of original downloads got rate limited (HTTP 429) on the first run. +THUMBNAIL_WIDTH = 1600 +REQUEST_DELAY = 1.2 +MAX_RETRIES = 4 + +# Shelf photos of the categories the catalog covers: drinks and dairy, rice +# crackers and snacks, sauces and condiments. +QUERIES = [ + # the catalog's own products first: drinks, dairy, rice crackers + "野菜生活100", + "伊藤園 1日分の野菜", + "Japanese vegetable juice shelf", + "Japanese milk cartons supermarket", + "meiji oishii milk", + "せんべい 棚", + "Japanese rice cracker shop display", + "Japanese confectionery shelf", + "Japanese supermarket beverage aisle", + "Japanese convenience store refrigerator drinks", + "Japanese supermarket dairy case", + "スーパー 飲料 陳列", + "お菓子 売り場", + "Japanese snack aisle supermarket", + "Japanese supermarket shelves drinks bottles", + "Japanese supermarket interior shelves", +] + +PERMISSIVE = ("cc0", "cc-by", "cc by", "public domain", "pd-", "attribution") + + +def fetch(url: str, timeout: int = 60) -> bytes: + """GET with backoff: Commons answers 429 when a client is too eager.""" + for attempt in range(MAX_RETRIES): + try: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout, context=CONTEXT) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code != 429 or attempt == MAX_RETRIES - 1: + raise + time.sleep(REQUEST_DELAY * (2 ** attempt)) + raise RuntimeError("unreachable") + + +def api(params: dict) -> dict: + url = API + "?" + urllib.parse.urlencode(params) + time.sleep(REQUEST_DELAY) + return json.loads(fetch(url, timeout=30).decode("utf-8")) + + +def search_files() -> dict: + """Candidate files, with how many queries found each one.""" + found: dict = {} + for query in QUERIES: + try: + result = api({"action": "query", "list": "search", "srsearch": query, + "srnamespace": "6", "srlimit": "25", "format": "json"}) + except Exception as error: + print(f" search failed for {query!r}: {error}") + continue + for hit in result.get("query", {}).get("search", []): + title = hit["title"] + if title.lower().endswith((".svg", ".pdf", ".ogv", ".webm")): + continue + found[title] = found.get(title, 0) + 1 + time.sleep(0.2) + return found + + +def image_info(titles: list) -> dict: + """URL, size and licence for each file, in batches of 50.""" + info: dict = {} + for start in range(0, len(titles), 50): + batch = titles[start:start + 50] + try: + result = api({"action": "query", "titles": "|".join(batch), "prop": "imageinfo", + "iiprop": "url|size|extmetadata", "iiurlwidth": str(THUMBNAIL_WIDTH), + "format": "json"}) + except Exception as error: + print(f" info failed for batch {start}: {error}") + continue + for page in (result.get("query", {}).get("pages") or {}).values(): + entries = page.get("imageinfo") or [] + if not entries: + continue + entry = entries[0] + meta = entry.get("extmetadata") or {} + info[page["title"]] = { + "url": entry.get("thumburl") or entry.get("url"), + "width": entry.get("width"), + "height": entry.get("height"), + "license": (meta.get("LicenseShortName") or {}).get("value", ""), + "artist": (meta.get("Artist") or {}).get("value", ""), + "description": (meta.get("ImageDescription") or {}).get("value", "")[:200], + "page": f"https://commons.wikimedia.org/wiki/{urllib.parse.quote(page['title'].replace(' ', '_'))}", + } + time.sleep(0.2) + return info + + +def main() -> None: + root = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/dorayaki-data/shelf-images") + os.makedirs(root, exist_ok=True) + + print("searching Commons...") + found = search_files() + print(f" {len(found)} distinct candidate files") + + info = image_info(sorted(found)) + print(f" metadata for {len(info)} files") + + kept, skipped = [], {"license": 0, "size": 0, "missing": 0} + for title, entry in sorted(info.items()): + if not entry.get("url"): + skipped["missing"] += 1 + continue + licence = (entry.get("license") or "").lower() + if not any(token in licence for token in PERMISSIVE): + skipped["license"] += 1 + continue + if (entry.get("width") or 0) < 1000 or (entry.get("height") or 0) < 700: + skipped["size"] += 1 + continue + kept.append((title, entry)) + + print(f" keeping {len(kept)} (skipped: {skipped})") + manifest = [] + for index, (title, entry) in enumerate(kept): + stem = os.path.basename(urllib.parse.urlparse(entry["url"]).path).replace(" ", "_") + stem = urllib.parse.unquote(stem)[:60] + name = f"{index:03d}_{stem}" + path = os.path.join(root, name) + if not os.path.exists(path): + try: + payload = fetch(entry["url"]) + with open(path, "wb") as handle: + handle.write(payload) + except Exception as error: + print(f" download failed for {title}: {error}") + continue + manifest.append({ + "file": name, + "title": title, + "width": entry["width"], + "height": entry["height"], + "license": entry["license"], + "artist": entry["artist"], + "source": entry["page"], + "queries": found.get(title, 0), + }) + print(f" [{index:>3}] {entry['width']}x{entry['height']} {entry['license']:<14} {title}") + + manifest_path = os.path.join(root, "manifest.json") + with open(manifest_path, "w", encoding="utf-8") as handle: + json.dump({"source": "Wikimedia Commons", "images": manifest}, handle, ensure_ascii=False, indent=1) + print(f"\n{len(manifest)} images in {root}\nmanifest: {manifest_path}") + + +if __name__ == "__main__": + main() diff --git a/recognization/code/demo.py b/recognization/code/demo.py deleted file mode 100644 index 8613b21..0000000 --- a/recognization/code/demo.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -Synthetic camera for demos: a store employee walking a shelf. - -A simulator has no camera, and a still image makes a poor demo - nothing moves, so -nothing new is ever recognised. This module turns the demo shelf photos into a -virtual walk: a window pans left to right along a row, steps up to the next row, -and pans back the other way (serpentine, ascending), exactly the way a person -sweeps a shelf with a tablet. - -The client drives only `step`; all geometry lives here, so the web client, the -phone app, and the reference client see the same walk. `step` wraps, so an -unattended demo loops forever. - -Demo-only: this is a capture source, not a perception feature. `/v1/demo/*` is -documented as such. -""" - -from __future__ import annotations - -import math -import os -import threading -import time -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple - -DEFAULT_WINDOW = (960, 540) -DEFAULT_QUALITY = 65 -# How far the window advances per step, as a fraction of its own size. This is -# the demo's most important number: advance too fast and a product is in view for -# one or two frames, so the classifier never gets a chance and nothing is ever -# labelled (measured: 0.22 gave 9 labelled tracks in 20 s; the shelf swept past -# faster than a person could read it). 0.06 keeps a facing in view for roughly a -# second at demo frame rates - a deliberate, human sweep. -PAN_OVERLAP = 0.06 -ROW_OVERLAP = 0.25 -# Vertical steps spent moving between rows (the "look up, then keep going" beat). -TRANSITION_STEPS = 6 - - -class DemoSourceError(ValueError): - """Unknown sample or an impossible window.""" - - -@dataclass -class Bay: - """One shelf photo in the walk, with its geometry.""" - - sample_id: str - path: str - width: int - height: int - steps: int - - -class DemoCamera: - """Plans and renders the virtual walk over one or more shelf photos.""" - - def __init__(self, samples: Dict[str, Dict[str, str]], window: Tuple[int, int] = DEFAULT_WINDOW, - pan_overlap: float = PAN_OVERLAP): - self._samples = samples - self._window = window - # How far the window advances per step. The demo's pacing knob: a smaller - # value is a slower, more deliberate sweep (more steps, a facing in view - # for longer). Clamped so a client cannot ask for a frozen or teleporting - # camera. - self.pan_overlap = max(0.005, min(0.5, float(pan_overlap))) - self._lock = threading.Lock() - self._images: Dict[str, Any] = {} - self._sizes: Dict[str, Tuple[int, int]] = {} - self._plans: Dict[Tuple[str, ...], List[Bay]] = {} - - # ------------------------------------------------------------------ planning - def _size(self, sample_id: str) -> Tuple[int, int]: - cached = self._sizes.get(sample_id) - if cached is not None: - return cached - item = self._samples.get(sample_id) - if item is None or not os.path.exists(item["path"]): - raise DemoSourceError(f"unknown sample '{sample_id}'") - import cv2 - - image = cv2.imread(item["path"]) - if image is None: - raise DemoSourceError(f"cannot read sample '{sample_id}'") - with self._lock: - self._images[sample_id] = image - self._sizes[sample_id] = (int(image.shape[1]), int(image.shape[0])) - return self._sizes[sample_id] - - def _image(self, sample_id: str) -> Any: - if sample_id not in self._images: - self._size(sample_id) - return self._images[sample_id] - - def steps_for(self, sample_id: str, window: Tuple[int, int]) -> int: - width, height = self._size(sample_id) - win_w, win_h = window - bands = self._bands(height, win_h) - per_band = self._pan_steps(width, win_w) - # Every band pans, and every band after the first needs the step-up. - return bands * per_band + max(0, bands - 1) * TRANSITION_STEPS - - def _bands(self, image_height: int, window_height: int) -> int: - if window_height >= image_height: - return 1 - stride = max(1, int(window_height * (1.0 - ROW_OVERLAP))) - return max(1, int(math.ceil((image_height - window_height) / stride)) + 1) - - def _pan_steps(self, image_width: int, window_width: int) -> int: - if window_width >= image_width: - return 1 - stride = max(1, int(window_width * self.pan_overlap)) - return max(1, int(math.ceil((image_width - window_width) / stride)) + 1) - - def plan(self, sample_ids: Sequence[str], window: Optional[Tuple[int, int]] = None) -> Dict[str, Any]: - window = window or self._window - key = (tuple(sample_ids), window) - bays = self._plans.get(key) - if bays is None: - bays = [] - for sample_id in sample_ids: - width, height = self._size(sample_id) - bays.append( - Bay( - sample_id=sample_id, - path=self._samples[sample_id]["path"], - width=width, - height=height, - steps=self.steps_for(sample_id, window), - ) - ) - self._plans[key] = bays - total = sum(bay.steps for bay in bays) - return { - "samples": [bay.sample_id for bay in bays], - "window": {"width": window[0], "height": window[1]}, - "bays": [ - {"sample": bay.sample_id, "width": bay.width, "height": bay.height, - "steps": bay.steps, "first_step": sum(b.steps for b in bays[:index])} - for index, bay in enumerate(bays) - ], - "steps": total, - "row_overlap": ROW_OVERLAP, - "pan_overlap": self.pan_overlap, - } - - # ------------------------------------------------------------------ rendering - def window_at( - self, sample_ids: Sequence[str], step: int, window: Optional[Tuple[int, int]] = None - ) -> Dict[str, Any]: - """Where the virtual camera is at this step (wraps around the walk).""" - window = window or self._window - plan = self.plan(sample_ids, window) - total = plan["steps"] - step = int(step) % max(1, total) - offset = 0 - for index, bay in enumerate(plan["bays"]): - if step < offset + bay["steps"]: - local = step - offset - x, y = self._position(local, bay["width"], bay["height"], window) - return { - "sample": bay["sample"], - "bay_index": index, - "step": step, - "steps": total, - "local_step": local, - "bay_steps": bay["steps"], - "x": x, - "y": y, - "width": min(window[0], bay["width"]), - "height": min(window[1], bay["height"]), - } - offset += bay["steps"] - raise DemoSourceError("step outside the walk") - - def _position(self, local_step: int, image_width: int, image_height: int, window: Tuple[int, int]) -> Tuple[int, int]: - """Serpentine: pan a row, step up, pan back.""" - win_w = min(window[0], image_width) - win_h = min(window[1], image_height) - bands = self._bands(image_height, win_h) - per_band = self._pan_steps(image_width, win_w) - cycle = per_band + TRANSITION_STEPS - band = min(bands - 1, local_step // cycle) - within = local_step - band * cycle - stride_y = 0 if bands == 1 else max(1, int(win_h * (1.0 - ROW_OVERLAP))) - # Ascending: band 0 is the bottom of the shelf. - y_bottom = min(image_height - win_h, band * stride_y) - y = max(0, image_height - win_h - y_bottom) - stride_x = max(1, int(win_w * self.pan_overlap)) - max_x = max(0, image_width - win_w) - if within < per_band: - pan = within - x = min(max_x, pan * stride_x) - else: - # The step-up beat: hold x at the end of the row and climb. - transition = min(TRANSITION_STEPS, max(1, within - per_band + 1)) - pan = per_band - 1 - x = min(max_x, pan * stride_x) - next_band = min(bands - 1, band + 1) - next_y_bottom = min(image_height - win_h, next_band * stride_y) - ratio = transition / float(TRANSITION_STEPS) - y = int(round(y - (next_y_bottom - y_bottom) * ratio)) - if band % 2 == 1 and within < per_band: - # Odd bands pan back the other way, like a person reversing direction. - x = max(0, max_x - x) - return int(max(0, min(max_x, x))), int(max(0, min(image_height - win_h, y))) - - def frame( - self, - sample_ids: Sequence[str], - step: int, - window: Optional[Tuple[int, int]] = None, - quality: int = DEFAULT_QUALITY, - ) -> Tuple[bytes, Dict[str, Any]]: - """The JPEG frame the virtual camera sees at this step, plus its geometry.""" - import cv2 - - position = self.window_at(sample_ids, step, window) - image = self._image(position["sample"]) - x, y = position["x"], position["y"] - w, h = position["width"], position["height"] - crop = image[y:y + h, x:x + w] - if crop.size == 0: - raise DemoSourceError("empty crop") - ok, buffer = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, int(quality)]) - if not ok: - raise DemoSourceError("JPEG encode failed") - return bytes(buffer), position - - def warm(self, sample_ids: Sequence[str]) -> None: - """Decode the bays up front so the first demo frame is not the slow one.""" - for sample_id in sample_ids: - self._size(sample_id) - - -if __name__ == "__main__": # smoke: python code/demo.py [sample ...] - import sys - - sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - from app import get_available_shelf_images - - samples = {item["id"]: item for item in get_available_shelf_images("add")} - camera = DemoCamera(samples) - ids = sys.argv[1:] or ["image6"] - plan = camera.plan(ids) - print(f"walk over {plan['samples']}: {plan['steps']} steps, window {plan['window']}") - started = time.perf_counter() - for step in range(min(plan["steps"], 24)): - payload, position = camera.frame(ids, step) - print(f" step {step:3d} {position['sample']} x={position['x']:4d} y={position['y']:4d} " - f"{len(payload) // 1024} KB") - elapsed = (time.perf_counter() - started) * 1000 / max(1, min(plan["steps"], 24)) - print(f"{elapsed:.1f} ms per frame") diff --git a/recognization/code/demo_replay.py b/recognization/code/demo_replay.py deleted file mode 100644 index 78ad9c6..0000000 --- a/recognization/code/demo_replay.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Demo-only: replay a shelf's recognition as if it were live tracking. - -`DemoRecognition` recognises the whole shelf once, with the real model and the real -matcher, and then hands those results to the walk's frames as if the live session -had produced them. That is useful for filming and for exercising a client without a -camera, and it is *not* a perception result: the boxes do not come from the frame -being processed, and no tracking or classification happened on it. Every payload it -touches is marked `synthetic: true` so a client can tell. - -It lives here rather than in `app.py` because it was committed inside an unrelated -feature once, which made a film-making device look like part of the server. -""" - -from __future__ import annotations - -import threading -from typing import Any, Dict, List, Optional - -from demo import DemoSourceError - - -class DemoRecognition: - """The demo shortcut. - - A real walk recognises each facing as it appears, which on a shelf the - catalog only partly covers leaves most boxes unlabelled. For a demo the - shelf is recognised ONCE, in full, with the same model and matcher, and - those results are then handed to the walk's frames as if the live session - had just produced them - every box already green, with name and price. - Demo-only: enabled by --demo-walk. - """ - - _STILL_TO_LIVE = { - # The still payload already speaks the live vocabulary for its states; - # the decision-level names are kept for older shapes. - "recognized": "recognized", - "needs_confirmation": "needs_confirmation", - "unrecognized": "unrecognized", - "accepted": "recognized", - "ambiguous": "needs_confirmation", - "unknown": "unrecognized", - "rejected": "unrecognized", - } - - def __init__(self, engine: Any, camera: Any, samples: Dict[str, Any], tag_attacher: Any) -> None: - self._engine = engine - self._camera = camera - self._samples = samples - self._tag_attacher = tag_attacher - self._catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()} - self._cache: Dict[str, Any] = {} - self._lock = threading.Lock() - - def steps(self, sample_id: str) -> int: - return self._camera.plan([sample_id], (1280, 720)).get("steps", 0) - - def recognition(self, sample_id: str) -> Any: - cached = self._cache.get(sample_id) - if cached is not None: - return cached - import cv2 - - path = self._samples[sample_id]["path"] - image = cv2.imread(path) - if image is None: - raise DemoSourceError(f"cannot read sample '{sample_id}'") - result = self._engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) - if self._tag_attacher is not None: - result = self._tag_attacher(result, image) - with self._lock: - self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0])) - return self._cache[sample_id] - - def warm(self, sample_id: str) -> None: - self.recognition(sample_id) - - def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]: - """Swap the live tracks for the shelf's recognised ones, in frame space.""" - still, _, _ = self.recognition(sample_id) - position = self._camera.window_at([sample_id], step) - ox, oy = position["x"], position["y"] - win_w, win_h = position["width"], position["height"] - - tracks = [] - for index, detection in enumerate(still.get("detections") or []): - box = [int(v) for v in detection.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - identity = detection.get("identity") or {} - state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized") - sku_id = identity.get("sku_id") - product = self._catalog_by_sku.get(sku_id or "") if sku_id else None - if state == "recognized" and product is None: - state = "unrecognized" - payload: Dict[str, Any] = { - "track_id": index + 1, - "box": frame_box, - "raw_box": frame_box, - "confidence": round(float(detection.get("confidence") or 0.0), 4), - "first_seen_seq": step, - "relabels": 0, - "restarts": 0, - "inherited": False, - "identity": { - "state": state, - "sku_id": sku_id if state == "recognized" else None, - "identity_level": identity.get("identity_level"), - "confidence": identity.get("confidence"), - "score": identity.get("score"), - "candidates": list(identity.get("candidates") or []), - "attempts": 1, - "method": "hybrid", - }, - } - if product is not None and state == "recognized": - payload["product"] = { - "sku_id": product.get("sku_id"), - "product_name": product.get("product_name"), - "price": product.get("price"), - } - tracks.append(payload) - - slots = [] - for slot in still.get("empty_slots") or []: - box = [int(v) for v in slot.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - entry = dict(slot) - entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - slots.append(entry) - - result["tracks"] = tracks - result["empty_slots"] = slots - counts: Dict[str, int] = {} - for track in tracks: - counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1 - result["summary"] = { - "in_view": len(tracks), - "recognized": counts.get("recognized", 0), - "needs_confirmation": counts.get("needs_confirmation", 0), - "unrecognized": counts.get("unrecognized", 0), - "detected": 0, - "unconfirmed": counts.get("unconfirmed", 0), - "identifying": 0, - "empty_slots": len(slots), - "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}), - "reconciliation": (still.get("reconciliation") or {}), - "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}), - } - # A client must be able to tell these boxes did not come from the frame it - # just sent. - result["synthetic"] = True - result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1} - return result diff --git a/recognization/code/download_stock.py b/recognization/code/download_stock.py deleted file mode 100644 index 889e2f5..0000000 --- a/recognization/code/download_stock.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import sys -import io -import urllib.request -from PIL import Image - -# Ensure stdout handles utf-8 cleanly -if sys.platform.startswith('win'): - sys.stdout.reconfigure(encoding='utf-8') - -urls = { - 'sku_011': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_apple@2x_v2.png', '野菜生活100 アップルサラダ'), - 'sku_012': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_lemon@2x_v2.png', '野菜生活100 レモンサラダ'), - 'sku_013': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/green-product-pic_2.png', '野菜生活100 Smoothie グリーンスムージー'), - 'sku_014': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/orange-product-pic_2.png', '野菜生活100 Smoothie ビタミンスムージー'), - 'sku_015': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/yellow-product-pic_2.png', '野菜生活100 Smoothie バナナ＆アーモンドmix'), - 'sku_016': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/red-product-pic_2.png', '野菜生活100 Smoothie アサイー＆ベリーmix') -} - -os.makedirs('stock', exist_ok=True) - -for sku, (url, name) in urls.items(): - print(f'Fetching {sku}: {name} from {url}...') - req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}) - try: - with urllib.request.urlopen(req, timeout=15) as resp: - data = resp.read() - img = Image.open(io.BytesIO(data)) - if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info): - bg = Image.new('RGB', img.size, (255, 255, 255)) - if img.mode != 'RGBA': - img = img.convert('RGBA') - bg.paste(img, mask=img.split()[3]) - img = bg - else: - img = img.convert('RGB') - out_path = f'stock/{sku}.jpeg' - img.save(out_path, 'JPEG', quality=95) - print(f'Successfully saved {sku}: {name} -> {out_path} ({img.size})') - except Exception as e: - print(f'Error downloading {sku} ({url}): {e}') diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 74117ee..f63e714 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -21,6 +21,7 @@ from __future__ import annotations import base64 import collections +import logging import hashlib import json import os @@ -50,6 +51,9 @@ STATE_UNRECOGNIZED = "unrecognized" CLASSIFY_METHODS = ("surf", "orb", "brisk", "hybrid", "hybrid_fast") +LOGGER = logging.getLogger("dorayaki.engine") + + class UndecodableImage(ValueError): """Raised when an uploaded payload is not a readable image.""" @@ -81,6 +85,16 @@ class EngineSettings: # over a nearly static shelf wants more; that is a CLI override, not a default. live_inflight: int = 16 live_attempts: int = 2 + # Which mechanism names products: the hand-crafted local-feature matcher, or the + # embedding gallery with a calibrated rejection. Measured on 54 unseen store + # photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its + # raw inlier score cannot be thresholded; the gallery can say "unknown". + identity_backend: str = "matcher" + # Cosine similarity below which the gallery calls a facing unknown. Measured: + # same-product pairs sit above 0.6, different products below 0.5. + gallery_threshold: float = 0.55 + gallery_head_threshold: float = 0.40 + gallery_image_size: int = 518 imgsz: int = 640 # 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine). classify_workers: int = 5 @@ -234,6 +248,10 @@ class RecognitionEngine: self.matcher = matcher self.inventory = inventory self.settings = settings or EngineSettings() + # The embedding gallery is built on first use (it loads a model and every + # enrolled view), and cached as False when it cannot be built at all. + self._gallery: Any = None + self._gallery_lock = threading.Lock() workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2) self.classify_workers = int(workers) self._executor = classify_executor or ThreadPoolExecutor( @@ -394,7 +412,28 @@ class RecognitionEngine: return slots # ------------------------------------------------------------------ classification + @property + def identity_scale(self) -> str: + """What the scores from `score_crop` mean: inlier counts or cosine similarity.""" + return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier" + def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: + """Score one crop. Gallery never falls back to the matcher. + + A missing gallery returns no scores (unknown), not inlier counts. Mixing + those scales is how a live scan accepted a look-alike the still path rejected. + """ + if self.identity_scale == "cosine": + gallery = self.identity_gallery() + if not gallery: + return {} + match = gallery.match(crop) + if match.rejected or not match.sku_id: + return {} + scores = {match.sku_id: match.similarity} + if match.runner_up: + scores[match.runner_up] = match.runner_up_similarity + return {str(sku): float(score) for sku, score in scores.items()} method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) @@ -404,6 +443,44 @@ class RecognitionEngine: scores, _ = self.matcher.score_crop(crop, method=method) return {str(sku): float(score) for sku, score in scores.items()} + def identity_gallery(self) -> Any: + """The embedding gallery, loaded once (None when unavailable).""" + if self._gallery is False: + return None + if self._gallery is not None: + return self._gallery + with self._gallery_lock: + if self._gallery is False: + return None + if self._gallery is not None: + return self._gallery + from gallery import ProductGallery + + gallery = ProductGallery( + self.settings.stock_dir, + [product["sku_id"] for product in self.catalog()], + threshold=self.settings.gallery_threshold, + head_threshold=self.settings.gallery_head_threshold, + image_size=self.settings.gallery_image_size, + ) + try: + loaded = gallery.load() + except Exception as exc: # torch/timm missing, no weights, no network + LOGGER.warning("identity gallery unavailable: %s", exc) + loaded = False + # False means tried and failed. Do not retry and do not use the matcher. + self._gallery = gallery if loaded else False + if loaded: + LOGGER.info( + "identity gallery: %d views over %d SKUs on %s", + gallery.view_count(), + len(gallery.sku_ids), + gallery.device, + ) + else: + LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized") + return gallery if loaded else None + def classify_boxes( self, image: Any, @@ -427,6 +504,11 @@ class RecognitionEngine: ordered = sorted(boxes, key=area_of, reverse=True) crops = [self.crop(image, box["box"]) for box in ordered] + + if (self.settings.identity_backend or "matcher").lower() == "gallery": + gallery = self.identity_gallery() + # No matcher fallback: an unavailable gallery is unrecognized, not a different model. + return self._classify_with_gallery(gallery, ordered, crops) if method == "hybrid_fast": futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops] else: @@ -448,26 +530,47 @@ class RecognitionEngine: return results def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]: - """Full single-crop result: decision, per-method details, crop, visualization.""" - from identity import decide_identity + """Full single-crop result, using the same backend as recognize/live.""" + from identity import decide_identity, decision_for_match + + crop = self.crop(image, box) + if self.identity_scale == "cosine": + gallery = self.identity_gallery() + match = gallery.match(crop) if gallery else None + # match() already applied the gallery's own threshold. Ranking again + # with the published cosine gates must not invent a second line. + decision = decision_for_match(match) + return self._inspect_payload(box, "gallery", decision, crop, visualize=False) method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) - crop = self.crop(image, box) if method == "hybrid_fast": scores, details = self.matcher.score_crop_fast(crop) + visual_method = "hybrid" else: scores, details = self.matcher.score_crop(crop, method=method) + visual_method = method decision = decide_identity(scores, details=details, method=method) - identity = identity_payload(decision) + return self._inspect_payload(box, method, decision, crop, visual_method=visual_method) + def _inspect_payload( + self, + box: Sequence[int], + method: str, + decision: Dict[str, Any], + crop: Any, + *, + visual_method: Optional[str] = None, + visualize: bool = True, + ) -> Dict[str, Any]: + identity = identity_payload(decision) top_details = decision.get("details") or {} sub_details = None if isinstance(top_details, dict): raw = top_details.get("sub_details") if raw: - sub_details = {k: dict(v) for k, v in raw.items()} + sub_details = {key: dict(value) for key, value in raw.items()} product = None visualization = None @@ -479,16 +582,16 @@ class RecognitionEngine: elif identity["candidates"]: visualized_sku = identity["candidates"][0]["sku_id"] is_candidate = True - if visualized_sku: + if visualize and visualized_sku and visual_method: try: visualization = self.matcher.generate_match_visualization( - crop, visualized_sku, method=method if method != "hybrid_fast" else "hybrid" + crop, visualized_sku, method=visual_method ) except Exception: visualization = None return { - "box": [int(v) for v in box], + "box": [int(value) for value in box], "method": method, "identity": identity, "sub_details": sub_details, @@ -643,6 +746,38 @@ class RecognitionEngine: }, } + def _classify_with_gallery( + self, + gallery: Any, + ordered: Sequence[Dict[str, Any]], + crops: Sequence[Any], + ) -> List[Dict[str, Any]]: + """Identity by retrieval: nearest enrolled view, or an honest unknown. + + The gallery scores a SKU by its best view and rejects when nothing is close + enough, which is the behaviour the matcher cannot provide - measured on 54 + unseen store photos, the matcher labelled milk cartons as mayonnaise and its + score could not be thresholded to stop it. + """ + from identity import decision_for_match + + results: List[Dict[str, Any]] = [] + for box, crop in zip(ordered, crops): + match = None + if gallery is not None: + try: + match = gallery.match(crop) + except Exception as exc: # a bad crop must not fail the batch + match = None + LOGGER.debug("gallery match failed: %s", exc) + decision = decision_for_match(match) + results.append({ + "detection_id": int(box["id"]), + "box": list(box["box"]), + "identity": identity_payload(decision), + }) + return results + @staticmethod def summarize( detections: Sequence[Dict[str, Any]], @@ -742,6 +877,7 @@ def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings: live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)), live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)), live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)), + identity_backend=env.get("DORAYAKI_IDENTITY", base.identity_backend), detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)), ) diff --git a/recognization/code/eval_external_images.py b/recognization/code/eval_external_images.py new file mode 100644 index 0000000..a555a85 --- /dev/null +++ b/recognization/code/eval_external_images.py @@ -0,0 +1,54 @@ +""" +Run the pipeline over store photos it has never seen. + + python code/eval_external_images.py [~/dorayaki-data/shelf-images] + +Reports detection and identity per image, and writes a JSON summary. See +docs/EXTERNAL-EVAL.md for what the numbers mean. +""" +import json, os, sys, time +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cv2, numpy as np +from engine import build_default_engine, EngineSettings +from tags import TagService, TagSettings +from reconcile import attach_tags_to_boxes + +ROOT = os.path.expanduser(sys.argv[1] if len(sys.argv) > 1 else "~/dorayaki-data/shelf-images") +manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) + +engine = build_default_engine(EngineSettings(stock_dir="stock-lopia")) +catalog_by_sku = {p["sku_id"]: p for p in engine.catalog()} +service = TagService(engine.ocr_engine, settings=TagSettings()) + +rows = [] +for entry in manifest["images"]: + path = os.path.join(ROOT, entry["file"]) + image = cv2.imread(path) + if image is None: + continue + # keep the long side at 1600 so detection cost is comparable across images + height, width = image.shape[:2] + if max(height, width) > 1600: + scale = 1600.0 / max(height, width) + image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) + started = time.perf_counter() + result = engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) + detections = result["detections"] + recognized = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "recognized") + rejected = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "rejected") + rows.append({ + "file": entry["file"], "title": entry["title"], "license": entry["license"], + "size": f"{image.shape[1]}x{image.shape[0]}", + "detections": len(detections), "recognized": recognized, "rejected": rejected, + "ms": round((time.perf_counter()-started)*1000), + }) + print(f" {entry['file'][:34]:<34} {image.shape[1]}x{image.shape[0]:<5} " + f"det={len(detections):>4} rec={recognized:>3} rej={rejected:>3} {rows[-1]['ms']:>5} ms") + +total_det = sum(r["detections"] for r in rows) +total_rec = sum(r["recognized"] for r in rows) +with_det = sum(1 for r in rows if r["detections"] > 0) +print(f"\n{len(rows)} images, {total_det} detections, {total_rec} recognized " + f"({100*total_rec//max(1,total_det)}%), images with any detection: {with_det}") +json.dump(rows, open(os.path.join(ROOT, "eval.json"), "w"), ensure_ascii=False, indent=1) +print(f"summary: {os.path.join(ROOT, 'eval.json')}") diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py new file mode 100644 index 0000000..d44a140 --- /dev/null +++ b/recognization/code/eval_live.py @@ -0,0 +1,693 @@ +"""Replay a recorded walk through LiveSession and count what it actually decided. + +The recording format is FrameRecorder's: DIR//index.jsonl plus the JPEG +bytes the client sent. Optional labels live in labels.json next to that index +(or at --labels). A label is {"frame", "box", "sku_id"}. + +Template leakage is refused, not scored. A label whose crop matches a catalog +template (byte-identical, or IoU >= 0.98 against a same-sized template pasted +into the frame) is excluded and counted under leakage. Those crops are how the +store was enrolled; scoring them is training-set accuracy. + + python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] + [--json PATH] [--iou 0.5] + +Prints a human count table and writes the same report as JSON (default +/eval_live.json). Missing input exits 2 with {"error": ...} on +stdout. The same pinned recording, catalog, weights, and labels produce the +same counts; timings are omitted because they are not. + +No held-out labels means denominators and fingerprints only. That is not a +walk accuracy number. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +LEVELS = ("accepted", "ambiguous", "rejected", "unknown", "unsettled") +TAG_STATES = ("readable", "unreadable", "absent", "detected") + + +class EvalInputError(ValueError): + """The recording, labels, or catalog cannot be scored as given.""" + + +def file_sha256(path: str) -> Optional[str]: + digest = hashlib.sha256() + try: + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +def content_sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _iou(a: Sequence[float], b: Sequence[float]) -> float: + if len(a) != 4 or len(b) != 4: + return 0.0 + ax1, ay1, ax2, ay2 = (float(v) for v in a) + bx1, by1, bx2, by2 = (float(v) for v in b) + ix1, iy1 = max(ax1, bx1), max(ay1, by1) + ix2, iy2 = min(ax2, bx2), min(ay2, by2) + inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) + if inter <= 0.0: + return 0.0 + area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1) + area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0.0 else 0.0 + + +def load_index(recording_dir: str) -> List[Dict[str, Any]]: + path = os.path.join(recording_dir, "index.jsonl") + if not os.path.isfile(path): + raise EvalInputError(f"missing recording index: {path}") + rows: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as handle: + for number, line in enumerate(handle, start=1): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError as exc: + raise EvalInputError(f"{path}:{number}: {exc.msg}") from exc + if not isinstance(entry, dict) or "frame" not in entry or "path" not in entry: + raise EvalInputError(f"{path}:{number}: line needs frame and path") + rows.append(entry) + if not rows: + raise EvalInputError(f"empty recording index: {path}") + return rows + + +def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + """Hash the JPEGs the client sent, in index order. Missing files are an error.""" + digest = hashlib.sha256() + missing: List[str] = [] + for row in rows: + name = str(row["path"]) + path = os.path.join(recording_dir, name) + payload_hash = file_sha256(path) + if payload_hash is None: + missing.append(name) + continue + digest.update(f"{row['frame']}:{name}:{payload_hash}\n".encode("utf-8")) + if missing: + raise EvalInputError("missing recorded frames: " + ", ".join(missing[:8])) + return {"sha256": digest.hexdigest(), "frames": len(rows)} + + +def load_labels(path: Optional[str]) -> List[Dict[str, Any]]: + if not path: + return [] + if not os.path.isfile(path): + raise EvalInputError(f"missing labels: {path}") + try: + with open(path, "r", encoding="utf-8") as handle: + payload = json.loads(handle.read()) + except json.JSONDecodeError as exc: + raise EvalInputError(f"labels are not JSON: {exc.msg}") from exc + rows = payload.get("labels") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + raise EvalInputError("labels must be a list or {\"labels\": [...]}") + cleaned: List[Dict[str, Any]] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise EvalInputError(f"labels[{index}] is not an object") + if "frame" not in row or "sku_id" not in row or "box" not in row: + raise EvalInputError(f"labels[{index}] needs frame, box, sku_id") + box = row["box"] + if not isinstance(box, (list, tuple)) or len(box) != 4: + raise EvalInputError(f"labels[{index}].box must be [x1, y1, x2, y2]") + cleaned.append( + { + "frame": int(row["frame"]), + "box": [int(v) for v in box], + "sku_id": str(row["sku_id"]), + "source": row.get("source"), + } + ) + return cleaned + + +def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[str, Any]: + """Same inputs as RecognitionEngine.catalog_version, plus the full digest and head.""" + digest = hashlib.sha256() + csv_path = os.path.join(stock_dir, stock_csv) + csv_hash = file_sha256(csv_path) + if csv_hash is None: + digest.update(b"missing-csv") + else: + digest.update(bytes.fromhex(csv_hash)) + try: + entries = sorted( + (name, os.path.getsize(os.path.join(stock_dir, name))) + for name in os.listdir(stock_dir) + if name.lower().endswith((".jpeg", ".jpg", ".png")) + ) + except OSError as exc: + raise EvalInputError(f"cannot read catalog {stock_dir}: {exc.strerror}") from exc + for name, size in entries: + digest.update(f"{name}:{size}".encode("utf-8")) + full = digest.hexdigest() + head_path = os.path.join(stock_dir, "identity_head.pt") + return { + "stock_dir": os.path.basename(os.path.abspath(stock_dir)), + "version": full[:12], + "sha256": full, + "templates": len(entries), + "csv_sha256": csv_hash, + "identity_head_sha256": file_sha256(head_path), + } + + +def model_fingerprint(weights_path: str) -> Dict[str, Any]: + try: + import ultralytics + + library = f"ultralytics-{ultralytics.__version__}" + except Exception: + library = "ultralytics-unknown" + return { + "version": f"{os.path.basename(weights_path)}@{library}", + "weights_sha256": file_sha256(weights_path) if weights_path else None, + } + + +def _decode_image(payload: bytes) -> Any: + import cv2 + import numpy as np + + image = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_COLOR) + if image is None: + raise EvalInputError("recorded frame is not a decodable image") + return image + + +def template_crops(stock_dir: str) -> List[Tuple[str, Any]]: + """Catalog photos keyed by SKU stem (sku_l05, sku_l05_1, ...).""" + import cv2 + + crops: List[Tuple[str, Any]] = [] + try: + names = sorted(os.listdir(stock_dir)) + except OSError: + return crops + for name in names: + stem, extension = os.path.splitext(name) + if extension.lower() not in (".jpeg", ".jpg", ".png"): + continue + image = cv2.imread(os.path.join(stock_dir, name)) + if image is None or image.size == 0: + continue + crops.append((stem, image)) + return crops + + +def _same_picture(crop: Any, template: Any) -> bool: + """True when two same-sized images are the same photo after JPEG.""" + import numpy as np + + left = crop.astype("float32").reshape(-1) + right = template.astype("float32").reshape(-1) + if left.size < 16 or left.size != right.size: + return False + left = left - float(left.mean()) + right = right - float(right.mean()) + denom = float(np.sqrt((left * left).sum() * (right * right).sum())) + if denom <= 1e-6: + return bool(np.max(np.abs(crop.astype("int16") - template.astype("int16"))) <= 2) + return float((left * right).sum() / denom) >= 0.98 + + +def leaked_labels( + recording_dir: str, + rows: Sequence[Dict[str, Any]], + labels: Sequence[Dict[str, Any]], + stock_dir: str, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Drop labels whose crop is a catalog template. + + Byte identity catches a frame that is the template file. Geometric identity + catches a template pasted into a larger frame at the labelled box. + """ + if not labels: + return [], [] + templates = template_crops(stock_dir) + template_hashes = {} + try: + names = os.listdir(stock_dir) + except OSError: + names = [] + for name in names: + digest = file_sha256(os.path.join(stock_dir, name)) + if digest: + template_hashes.setdefault(digest, os.path.splitext(name)[0]) + by_frame: Dict[int, List[Dict[str, Any]]] = {} + for label in labels: + by_frame.setdefault(int(label["frame"]), []).append(label) + held: List[Dict[str, Any]] = [] + leaked: List[Dict[str, Any]] = [] + frames = {int(row["frame"]): row for row in rows} + for frame_id, frame_labels in by_frame.items(): + row = frames.get(frame_id) + image = None + payload_hash = None + if row is not None: + path = os.path.join(recording_dir, str(row["path"])) + try: + with open(path, "rb") as handle: + payload = handle.read() + except OSError: + payload = b"" + payload_hash = content_sha256(payload) if payload else None + if payload: + image = _decode_image(payload) + for label in frame_labels: + reason = None + matched = None + if payload_hash in template_hashes: + reason = "frame_is_template" + matched = template_hashes[payload_hash] + elif image is not None: + x1, y1, x2, y2 = label["box"] + crop = image[max(0, y1) : max(0, y2), max(0, x1) : max(0, x2)] + if crop.size: + import cv2 + + ch, cw = crop.shape[:2] + for stem, template in templates: + th, tw = template.shape[:2] + if abs(th - ch) > 1 or abs(tw - cw) > 1: + continue + resized = template if (th, tw) == (ch, cw) else cv2.resize(template, (cw, ch)) + # JPEG re-encoding keeps layout and drops exact pixels. + if _same_picture(crop, resized): + reason = "crop_is_template" + matched = stem + break + if reason: + leaked.append({**label, "leak": reason, "template": matched}) + else: + held.append(label) + return held, leaked + + +def identity_level(track: Dict[str, Any]) -> str: + identity = track.get("identity") or {} + level = identity.get("identity_level") + if level in LEVELS: + return str(level) + state = identity.get("state") + if state == "recognized": + return "accepted" + if state == "needs_confirmation": + return "ambiguous" + if state == "unrecognized": + return "rejected" + return "unsettled" + + +def tag_state(track: Dict[str, Any]) -> str: + tag = track.get("price_tag") or track.get("tag") or {} + state = tag.get("state") + if state in TAG_STATES: + return str(state) + if tag: + return "detected" + return "absent" + + +def score_tracks(tracks: Iterable[Dict[str, Any]]) -> Dict[str, Any]: + """Denominators over settled decisions. Unsettled tracks are not in the rate.""" + counts = {level: 0 for level in LEVELS} + tags = {state: 0 for state in TAG_STATES} + for track in tracks: + counts[identity_level(track)] += 1 + tags[tag_state(track)] += 1 + settled = counts["accepted"] + counts["ambiguous"] + counts["rejected"] + counts["unknown"] + return { + "tracks": sum(counts.values()), + "settled": settled, + "accepted": counts["accepted"], + "ambiguous": counts["ambiguous"], + "rejected": counts["rejected"], + "unknown": counts["unknown"], + "unsettled": counts["unsettled"], + "accepted_rate": _rate(counts["accepted"], settled), + "ambiguous_rate": _rate(counts["ambiguous"], settled), + "rejected_rate": _rate(counts["rejected"], settled), + "tags_detected": tags["detected"] + tags["readable"] + tags["unreadable"], + "tags_readable": tags["readable"], + "tags_unreadable": tags["unreadable"], + "tags_absent": tags["absent"], + } + + +def _rate(numerator: int, denominator: int) -> Optional[float]: + if denominator <= 0: + return None + return round(numerator / denominator, 4) + + +def track_churn(frames: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + """How often one track id changes the SKU it is willing to name. + + A change counts only after the id has already named a SKU. Restarting or + dropping the label (relabels/restarts) is reported beside that, from the + counters LiveSession already publishes. + """ + previous: Dict[int, Optional[str]] = {} + changes: Dict[int, int] = {} + seen: Dict[int, int] = {} + relabels: Dict[int, int] = {} + restarts: Dict[int, int] = {} + for frame in frames: + for track in frame.get("tracks") or []: + track_id = track.get("track_id") + if track_id is None: + continue + track_id = int(track_id) + seen[track_id] = seen.get(track_id, 0) + 1 + relabels[track_id] = max(relabels.get(track_id, 0), int(track.get("relabels") or 0)) + restarts[track_id] = max(restarts.get(track_id, 0), int(track.get("restarts") or 0)) + named = None + if identity_level(track) == "accepted": + named = (track.get("identity") or {}).get("sku_id") + prior = previous.get(track_id) + if prior and named and named != prior: + changes[track_id] = changes.get(track_id, 0) + 1 + if named: + previous[track_id] = named + churned = sum(1 for count in changes.values() if count) + return { + "tracks": len(seen), + "tracks_with_sku_change": churned, + "sku_changes": sum(changes.values()), + "relabels": sum(relabels.values()), + "restarts": sum(restarts.values()), + "per_track": [ + { + "track_id": track_id, + "observations": seen[track_id], + "sku_changes": changes.get(track_id, 0), + "relabels": relabels.get(track_id, 0), + "restarts": restarts.get(track_id, 0), + } + for track_id in sorted(seen) + if changes.get(track_id, 0) or relabels.get(track_id, 0) or restarts.get(track_id, 0) + ], + } + + +def labelled_top1( + frames: Sequence[Dict[str, Any]], + labels: Sequence[Dict[str, Any]], + *, + iou_min: float, +) -> Dict[str, Any]: + """Top-1 over held-out labels. Accepted wrong SKU is a miss, not a reject. + + Denominator is labels that overlap a track. A label with no overlapping + track is unmatched and stays out of the top-1 rate. + """ + by_frame: Dict[int, List[Dict[str, Any]]] = {} + for frame in frames: + by_frame.setdefault(int(frame.get("frame", frame.get("frame_id"))), []).append(frame) + correct = wrong = rejected = ambiguous = missed = 0 + matched = 0 + for label in labels: + observations = by_frame.get(int(label["frame"]), []) + best_iou = 0.0 + best: Optional[Dict[str, Any]] = None + for observation in observations: + for track in observation.get("tracks") or []: + score = _iou(label["box"], track.get("box") or []) + if score > best_iou: + best_iou = score + best = track + if best is None or best_iou < iou_min: + missed += 1 + continue + matched += 1 + level = identity_level(best) + guess = (best.get("identity") or {}).get("sku_id") + if level == "accepted" and guess == label["sku_id"]: + correct += 1 + elif level == "accepted": + wrong += 1 + elif level == "ambiguous": + ambiguous += 1 + else: + rejected += 1 + return { + "labels": len(labels), + "matched": matched, + "unmatched": missed, + "correct": correct, + "wrong": wrong, + "rejected": rejected, + "ambiguous": ambiguous, + "top1": _rate(correct, matched), + "iou_min": iou_min, + } + + +def score_recording( + recording_dir: str, + *, + stock_dir: str, + weights_path: str = "", + labels_path: Optional[str] = None, + iou_min: float = 0.5, + live_frames: Optional[Sequence[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Score one session directory. live_frames, when given, are the replayed results. + + Without live_frames the recorded index is what the server already decided. + That still produces denominators; it is not a fresh model run. + """ + recording_dir = os.path.abspath(recording_dir) + rows = load_index(recording_dir) + sample = recording_fingerprint(recording_dir, rows) + labels_file = labels_path or os.path.join(recording_dir, "labels.json") + labels = load_labels(labels_file if os.path.isfile(labels_file) or labels_path else None) + held, leaked = leaked_labels(recording_dir, rows, labels, stock_dir) + source = list(live_frames) if live_frames is not None else rows + tracks = [track for frame in source for track in (frame.get("tracks") or [])] + decisions = score_tracks(tracks) + report = { + "recording": recording_dir, + "source": "live_replay" if live_frames is not None else "recorded_index", + "fingerprints": { + "sample": sample, + "model": model_fingerprint(weights_path), + "catalog": catalog_fingerprint(stock_dir), + }, + "decisions": decisions, + "tags": { + "detected": decisions["tags_detected"], + "readable": decisions["tags_readable"], + "unreadable": decisions["tags_unreadable"], + "absent": decisions["tags_absent"], + }, + "churn": track_churn(source), + "leakage": {"excluded": len(leaked), "labels": leaked}, + "labelled_top1": labelled_top1(source, held, iou_min=iou_min), + "evidence": _evidence(labels, held, leaked, live_frames is not None), + } + return report + + +def _evidence( + labels: Sequence[Dict[str, Any]], + held: Sequence[Dict[str, Any]], + leaked: Sequence[Dict[str, Any]], + replayed: bool, +) -> List[str]: + gaps: List[str] = [] + if not replayed: + gaps.append("counts are from the recorded index, not a fresh LiveSession replay") + if not labels: + gaps.append("no held-out labels; labelled top-1 is not a walk accuracy") + elif not held: + gaps.append("every label matched a catalog template and was excluded") + if leaked: + gaps.append(f"{len(leaked)} template-derived label(s) excluded") + return gaps + + +def replay_live( + recording_dir: str, + engine: Any, + *, + settings: Optional[Any] = None, +) -> List[Dict[str, Any]]: + """Feed recorded JPEGs through one LiveSession, in index order. + + Classification is asynchronous. Each frame is scored only after its + in-flight crops finish, then processed again so the label is on the + payload. Without that wait the same recording does not replay twice. + """ + from live import LiveSession + + rows = load_index(recording_dir) + session = LiveSession("eval", engine, settings=settings) + session.refresh_catalog() + if hasattr(session, "enable_tags"): + session.enable_tags() + frames: List[Dict[str, Any]] = [] + for row in rows: + path = os.path.join(recording_dir, str(row["path"])) + with open(path, "rb") as handle: + payload = handle.read() + frame_id = int(row["frame"]) + result = session.process_frame(frame_id, payload) + # min_hits is 2, so the first pass only detects. Keep passing the same + # bytes until a pass schedules nothing and everything in flight lands. + for _ in range(4): + _wait_until_settled(session) + result = session.process_frame(frame_id, payload) + if int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) == 0: + break + result["frame"] = frame_id + frames.append(result) + engine_shutdown = getattr(engine, "shutdown", None) + if callable(engine_shutdown): + engine_shutdown() + return frames + + +def _wait_until_settled(session: Any, timeout: float = 30.0) -> None: + import time + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + pending = int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) + if pending == 0: + return + time.sleep(0.02) + raise EvalInputError("classification did not finish before the replay timeout") + +def format_report(report: Dict[str, Any]) -> str: + decisions = report["decisions"] + tags = report["tags"] + top1 = report["labelled_top1"] + churn = report["churn"] + fingerprints = report["fingerprints"] + lines = [ + f"source: {report['source']}", + ( + f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " + f"rejected {decisions['rejected']} unknown {decisions['unknown']} " + f"unsettled {decisions['unsettled']} settled {decisions['settled']}" + ), + ( + f"rates: accepted {decisions['accepted_rate']} " + f"ambiguous {decisions['ambiguous_rate']} rejected {decisions['rejected_rate']} " + f"(denominator settled)" + ), + ( + f"tags: detected {tags['detected']} readable {tags['readable']} " + f"unreadable {tags['unreadable']} absent {tags['absent']}" + ), + ( + f"churn: tracks {churn['tracks']} sku_changes {churn['sku_changes']} " + f"relabels {churn['relabels']} restarts {churn['restarts']}" + ), + ( + f"labelled top-1: {top1['correct']}/{top1['matched']} = {top1['top1']} " + f"wrong {top1['wrong']} unmatched {top1['unmatched']} " + f"leakage excluded {report['leakage']['excluded']}" + ), + ( + f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} " + f"model {fingerprints['model']['version']} " + f"catalog {fingerprints['catalog']['version']}" + ), + ] + for gap in report["evidence"]: + lines.append(f"gap: {gap}") + return "\n".join(lines) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Score a recorded live walk.") + parser.add_argument("recording", nargs="?", help="session directory with index.jsonl") + parser.add_argument("--stock-dir", default="stock") + parser.add_argument("--weights", default="") + parser.add_argument("--labels", default=None) + parser.add_argument("--json", dest="json_path", default=None) + parser.add_argument("--iou", type=float, default=0.5) + parser.add_argument( + "--replay", + action="store_true", + help="re-run frames through LiveSession (needs the recognition stack)", + ) + parser.add_argument("--method", default="hybrid") + args = parser.parse_args(argv) + + def fail(message: str) -> int: + print(json.dumps({"error": message}, ensure_ascii=False)) + print(f"error: {message}", file=sys.stderr) + return 2 + + if not args.recording: + return fail("recording directory is required") + recording = os.path.abspath(args.recording) + if not os.path.isdir(recording): + return fail(f"recording directory not found: {recording}") + try: + live_frames = None + weights = args.weights + if args.replay: + from engine import EngineSettings, build_default_engine + from live import LiveSettings + + settings = EngineSettings(stock_dir=args.stock_dir) + if weights: + settings.weights_path = weights + engine = build_default_engine(settings) + weights = engine.settings.weights_path + live_frames = replay_live( + recording, engine, settings=LiveSettings(method=args.method, tags=True) + ) + report = score_recording( + recording, + stock_dir=args.stock_dir, + weights_path=weights, + labels_path=args.labels, + iou_min=args.iou, + live_frames=live_frames, + ) + except EvalInputError as exc: + return fail(str(exc)) + text = json.dumps(report, ensure_ascii=False, indent=1, sort_keys=True) + destination = args.json_path or os.path.join(recording, "eval_live.json") + try: + with open(destination, "w", encoding="utf-8") as handle: + handle.write(text + "\n") + except OSError as exc: + return fail(f"cannot write {destination}: {exc.strerror}") + print(format_report(report)) + print(f"json: {destination}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/fast_test.py b/recognization/code/fast_test.py deleted file mode 100644 index 772ff4f..0000000 --- a/recognization/code/fast_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from ultralytics import YOLO - -model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") -image_path = "/Users/nguyenhongvan/recongnize/data/add/image7.jpg" -results = model(image_path) -results[0].show() diff --git a/recognization/code/finetune_identity.py b/recognization/code/finetune_identity.py new file mode 100644 index 0000000..c147a1d --- /dev/null +++ b/recognization/code/finetune_identity.py @@ -0,0 +1,404 @@ +""" +Fine-tune the identity embedding on the labels we can actually produce. + +Measured: off-the-shelf DINOv2 tops out at ~25 % on a real milk shelf where the +products differ only by small text and a colour band, while it reaches 72-76 % on the +demo shelf, whose facings are near-identical crops of one photograph. That gap is the +whole problem, and the literature says the same thing (RP2K and RPC fine-tune; +GroceryVision reports a 17.5-point drop from Recall@5 to Recall@1). + +This trains a small projection head on frozen DINOv2 features and compares, on the +same held-out facings, three ways of naming a product: + + raw cosine the gallery as shipped + fine-tuned cosine the same matching, on projected features + fine-tuned + margin the same, trained with an angular margin + +**What this can and cannot show.** The labels come from one shelf, and the held-out +facings are other facings of the same photograph, so the number is an upper bound: it +measures whether a trained head separates these products at all, not whether it +survives a second visit. Augmentation (crop, scale, brightness, blur) stands in for +the visit we do not have; a cross-visit number needs cross-visit data. + + python code/finetune_identity.py [--epochs 60] [--views 2] +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import cv2 +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Labelled by eye from a contact sheet of the detections (docs/EXTERNAL-EVAL.md). +MILK_SHELF_LABELS = { + "meiji_oishii": [13, 14, 15, 16, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], + "tokunou45": [9, 10, 11, 12], + "megmilk": [20, 21, 22, 23, 24, 25, 26, 37, 38, 46, 47, 54, 55, 56, 57, 58, 59, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 90, 94, 95, 96, 97, 98, 99], + "hokkaido": [49, 50, 51, 60, 61, 62, 63, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 100, 101, 102, 103, 104], + "teishibou": [2, 3, 4], + "meiji_love": [5, 6, 7, 8], + "oishii_tounyu": [42, 43, 44, 45], + "rakunou36": [17, 18], +} + + +def enrollment_report(stock: str, csv_name: str = "stock.csv") -> dict: + """Catalog versus enrolled views. Does not load a model or write files. + + A SKU with no readable view cannot be recognized. One view is reported, not + treated as enough: the measured knee is two views. Duplicate CSV rows and a + head whose class list disagrees with the catalog are blockers. + """ + import csv + import re + + from gallery import ProductGallery + + names = [] + try: + names = os.listdir(stock) + except OSError: + names = [] + images = [name for name in names if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")] + rows = [] + csv_path = os.path.join(stock, csv_name) + if os.path.exists(csv_path): + with open(csv_path, encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + sku_ids = [] + seen = set() + duplicate_rows = [] + for row in rows: + sku_id = (row.get("SKU_ID") or "").strip() + if not sku_id: + continue + if sku_id in seen: + duplicate_rows.append(sku_id) + seen.add(sku_id) + sku_ids.append(sku_id) + gallery = ProductGallery(stock, sku_ids or []) + views = {sku_id: gallery._views_for(sku_id) for sku_id in dict.fromkeys(sku_ids)} + missing = [sku_id for sku_id, paths in views.items() if not paths] + single = [sku_id for sku_id, paths in views.items() if len(paths) == 1] + claimed = {os.path.basename(path) for paths in views.values() for path in paths} + orphans = sorted(name for name in images if name not in claimed) + # ponytail: prefix collision check. A view named sku_001 must not also match sku_00. + collisions = [] + for sku_id, paths in views.items(): + bare = [os.path.basename(path) for path in paths if os.path.splitext(os.path.basename(path))[0] != sku_id + and not re.fullmatch(re.escape(sku_id) + r"[_-]\d+", os.path.splitext(os.path.basename(path))[0])] + if bare: + collisions.append(sku_id) + blockers = [] + if duplicate_rows: + blockers.append("duplicate_csv_sku") + if missing: + blockers.append("sku_without_view") + if collisions: + blockers.append("view_prefix_collision") + head_path = os.path.join(stock, "identity_head.pt") + head_note = None + if os.path.exists(head_path): + head_note = "present_not_opened" + # Do not torch.load here: the head can be large, and this report must stay cheap. + # Class-list drift is reported when the caller passes class_ids. + return { + "stock": stock, + "sku_ids": list(dict.fromkeys(sku_ids)), + "images": len(images), + "missing": missing, + "single_view": single, + "orphans": orphans, + "duplicate_rows": duplicate_rows, + "collisions": collisions, + "blockers": blockers, + "head": head_note, + } + + +def enrollment_report_text(report: dict) -> str: + return ( + f"{report['stock']}: {len(report['sku_ids'])} SKUs, {report['images']} images, " + f"missing={len(report['missing'])}, single_view={len(report['single_view'])}, " + f"orphans={len(report['orphans'])}, blockers={report['blockers'] or 'none'}" + ) + + +def build_model(image_size: int = 518): + import timm + import torch + + device = "mps" if torch.backends.mps.is_available() else "cpu" + model = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, + num_classes=0, img_size=image_size, dynamic_img_size=True) + return model.eval().to(device), device + + +def prepare(image, size: int = 518): + mean = np.array([0.485, 0.456, 0.406], np.float32) + std = np.array([0.229, 0.224, 0.225], np.float32) + rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + height, width = rgb.shape[:2] + side = max(height, width) + canvas = np.zeros((side, side, 3), np.uint8) + canvas[(side - height) // 2:(side - height) // 2 + height, + (side - width) // 2:(side - width) // 2 + width] = rgb + resized = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA).astype(np.float32) / 255.0 + return ((resized - mean) / std).transpose(2, 0, 1) + + +def embed(model, device, images, size: int = 518, batch: int = 16): + import torch + + out = [] + for start in range(0, len(images), batch): + tensor = torch.from_numpy(np.stack([prepare(image, size) for image in images[start:start + batch]])).to(device) + with torch.no_grad(): + features = model(tensor) + out.append(torch.nn.functional.normalize(features, dim=1).cpu().numpy()) + return np.concatenate(out, axis=0) + + +def augment(image, rng): + """Stand in for a second visit: crop, scale, brightness, contrast, blur.""" + height, width = image.shape[:2] + # crop 82-100 % of the box, keeping the product centred + keep = rng.uniform(0.82, 1.0) + ch, cw = int(height * keep), int(width * keep) + y = rng.randint(0, max(1, height - ch + 1)) + x = rng.randint(0, max(1, width - cw + 1)) + out = image[y:y + ch, x:x + cw] + # photometric + out = cv2.convertScaleAbs(out, alpha=float(rng.uniform(0.8, 1.2)), beta=float(rng.uniform(-18, 18))) + if rng.random() < 0.4: + out = cv2.GaussianBlur(out, (3, 3), 0) + # small rotation + if rng.random() < 0.6: + angle = float(rng.uniform(-5, 5)) + matrix = cv2.getRotationMatrix2D((out.shape[1] / 2, out.shape[0] / 2), angle, 1.0) + out = cv2.warpAffine(out, matrix, (out.shape[1], out.shape[0]), borderMode=cv2.BORDER_REPLICATE) + return out + + +def train_head(features, labels, class_ids, epochs, device, margin: float = 0.0, seed: int = 7): + """A cosine classifier on frozen features: small, and hard to overfit.""" + import torch + + torch.manual_seed(seed) + dimension = features.shape[1] + projection = torch.nn.Linear(dimension, 128).to(device) + prototypes = torch.nn.Parameter(torch.randn(len(class_ids), 128, device=device)) + torch.nn.init.xavier_uniform_(prototypes) + optimiser = torch.optim.AdamW(list(projection.parameters()) + [prototypes], lr=3e-3, weight_decay=1e-2) + + x = torch.from_numpy(features).float().to(device) + y = torch.tensor([class_ids.index(label) for label in labels], device=device) + for epoch in range(epochs): + projected = torch.nn.functional.normalize(projection(x), dim=1) + unit_prototypes = torch.nn.functional.normalize(prototypes, dim=1) + logits = projected @ unit_prototypes.T + if margin > 0: + # angular margin: pull each sample away from the other prototypes + one_hot = torch.nn.functional.one_hot(y, len(class_ids)).float() + logits = logits - margin * one_hot + loss = torch.nn.functional.cross_entropy(logits * 16.0, y, label_smoothing=0.05) + optimiser.zero_grad() + loss.backward() + optimiser.step() + return projection, torch.nn.functional.normalize(prototypes.detach(), dim=1) + + +def train_from_stock(args) -> None: + """Train the head on a store's own enrolled views, for the gallery to load. + + The head belongs to the store: the views in its stock directory are its products, + and a head trained on them is what turns the gallery from a lookup that cannot + tell two milk cartons apart into one that can. One to four views per SKU is + enough, with augmentation standing in for the rest. + """ + import glob + import json + + import torch + + from gallery import ProductGallery + + stock = args.stock + report = enrollment_report(stock) + print(enrollment_report_text(report)) + if report["blockers"]: + print(f"{stock}: not training; enrollment blockers: {', '.join(report['blockers'])}") + return + gallery = ProductGallery(stock, report["sku_ids"]) + views = {sku: gallery._views_for(sku) for sku in report["sku_ids"]} + views = {sku: paths for sku, paths in views.items() if paths} + if len(views) < 2: + print(f"{stock}: needs at least two products with views, found {len(views)}") + return + + crops, labels = [], [] + for sku, paths in sorted(views.items()): + for path in paths: + image = cv2.imread(path) + if image is not None and image.size: + crops.append(image) + labels.append(sku) + class_ids = sorted(views) + print(f"{stock}: {len(crops)} views over {len(class_ids)} products") + + model, device = build_model() + rng = np.random.RandomState(11) + augmented, augmented_labels = [], [] + for crop, label in zip(crops, labels): + augmented.append(crop) + augmented_labels.append(label) + for _ in range(args.augment): + augmented.append(augment(crop, rng)) + augmented_labels.append(label) + print(f"embedding {len(augmented)} crops on {device}...") + features = embed(model, device, augmented) + + projection, prototypes = train_head(features, augmented_labels, class_ids, + args.epochs, device, margin=0.25) + payload = { + "class_ids": class_ids, + "image_size": 518, + "base_model": "vit_small_patch14_dinov2.lvd142m", + "projection": {key: value.cpu() for key, value in projection.state_dict().items()}, + "prototypes": prototypes.cpu(), + "views": {sku: len(paths) for sku, paths in sorted(views.items())}, + } + path = os.path.join(stock, "identity_head.pt") + torch.save(payload, path) + print(f"wrote {path} ({len(class_ids)} products, {len(crops)} views)") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--epochs", type=int, default=60) + parser.add_argument("--views", type=int, default=2, help="enrolment views per class") + parser.add_argument("--augment", type=int, default=12, help="augmented copies per enrolment view") + parser.add_argument("--image", default=os.path.expanduser("~/dorayaki-data/shelf-images")) + parser.add_argument("--stock", default=None, + help="train on a store's enrolled views instead of the milk shelf, " + "and write /identity_head.pt for the gallery to load") + args = parser.parse_args() + if args.stock: + train_from_stock(args) + return + + import json + + from engine import build_default_engine, EngineSettings + + manifest = json.load(open(os.path.join(args.image, "manifest.json"))) + entry = next(e for e in manifest["images"] if "2011年4月の牛乳" in e["file"]) + image = cv2.imread(os.path.join(args.image, entry["file"])) + height, width = image.shape[:2] + scale = 1600.0 / max(height, width) + image = cv2.resize(image, (int(width * scale), int(height * scale)), interpolation=cv2.INTER_AREA) + + engine = build_default_engine(EngineSettings(stock_dir="stock")) + boxes = engine.detect(image, conf=0.1) + boxes.sort(key=lambda b: (b["box"][1] // 120, b["box"][0])) + crops = [engine.crop(image, box["box"]) for box in boxes] + + train_crops, train_labels, test_crops, test_labels = [], [], [], [] + for label, indices in MILK_SHELF_LABELS.items(): + for position, index in enumerate(indices): + if position < args.views: + train_crops.append(crops[index]) + train_labels.append(label) + else: + test_crops.append(crops[index]) + test_labels.append(label) + class_ids = sorted(MILK_SHELF_LABELS) + print(f"{len(train_crops)} enrolment views, {len(test_crops)} held-out facings, {len(class_ids)} products") + + model, device = build_model() + print(f"embedding {len(train_crops) * (args.augment + 1) + len(test_crops)} crops on {device}...") + + rng = np.random.RandomState(11) + augmented, augmented_labels = [], [] + for crop, label in zip(train_crops, train_labels): + augmented.append(crop) + augmented_labels.append(label) + for _ in range(args.augment): + augmented.append(augment(crop, rng)) + augmented_labels.append(label) + + train_features = embed(model, device, augmented) + test_features = embed(model, device, test_crops) + + def score(similarity, label, threshold=None): + """Top-1 over every held-out facing, with optional rejection. + + Balanced accuracy is reported alongside the raw rate because the shelf is + unbalanced (MEGMILK has 34 facings, 酪農3.6 has two), and a system that only + ever answers "MEGMILK" would look respectable on the raw number. + """ + per_class = {class_id: [0, 0] for class_id in class_ids} + correct = rejected = 0 + for index, truth in enumerate(test_labels): + best = int(similarity[index].argmax()) + if threshold is not None and similarity[index][best] < threshold: + rejected += 1 + per_class[truth][1] += 1 + continue + guess = class_ids[best] + per_class[truth][1] += 1 + if guess == truth: + correct += 1 + per_class[truth][0] += 1 + recalls = [found / total for found, total in per_class.values() if total] + balanced = sum(recalls) / max(1, len(recalls)) + answered = len(test_labels) - rejected + answered_accuracy = correct / max(1, answered) + print(f" {label:<26} top-1 {correct:>3}/{len(test_labels)} = {100*correct//len(test_labels):>3}% " + f"balanced {100*balanced:>3.0f}% answered {answered:>3} " + f"({100*answered_accuracy:>3.0f}% of those right) rejected {rejected}") + return per_class + + import torch + + def class_prototypes(features): + prototypes = [] + for class_id in class_ids: + members = [i for i, label in enumerate(train_labels) if label == class_id] + mean = features[members].mean(axis=0) + prototypes.append(mean / (np.linalg.norm(mean) + 1e-8)) + return np.stack(prototypes) + + # 1. raw features, matched the way the shipped gallery matches: best enrolment view + raw_prototypes = class_prototypes(train_features) + score(test_features @ raw_prototypes.T, "raw DINOv2 cosine") + score(test_features @ raw_prototypes.T, "raw, rejecting below 0.55", threshold=0.55) + + # 2. the trained classifier's prototypes, with an angular margin + projection, prototypes = train_head(train_features, augmented_labels, class_ids, + args.epochs, device, margin=0.25) + with torch.no_grad(): + projected_test = torch.nn.functional.normalize( + projection(torch.from_numpy(test_features).float().to(device)), dim=1).cpu().numpy() + similarity = projected_test @ prototypes.cpu().numpy().T + score(similarity, "fine-tuned + margin") + per_class = score(similarity, "fine-tuned, rejecting < 0.4", threshold=0.4) + print(" per class, answered only:") + for class_id in class_ids: + found, total = per_class[class_id] + if total: + print(f" {class_id:<16} {found:>2}/{total:<3} {100*found//total:>4}%") + + print("\n note: enrolment and held-out facings come from one photograph, so these are") + print(" upper bounds. A cross-visit number needs a second visit, not a second split.") + + +if __name__ == "__main__": + main() diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py index f01a255..0805d71 100644 --- a/recognization/code/gallery.py +++ b/recognization/code/gallery.py @@ -26,6 +26,7 @@ on the GPU. That is why this can sit in the live loop at all. from __future__ import annotations import os +import re import threading from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -34,6 +35,8 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple # from the measured distribution: same-product pairs sit above 0.6, different # products below 0.5, and the gap is where a rejection belongs. DEFAULT_THRESHOLD = 0.55 +# The same line for a fine-tuned head, whose similarity scale is its own. +DEFAULT_HEAD_THRESHOLD = 0.40 @dataclass @@ -66,19 +69,66 @@ class ProductGallery: device: Optional[str] = None, image_size: int = 518, threshold: float = DEFAULT_THRESHOLD, + head_threshold: float = DEFAULT_HEAD_THRESHOLD, ) -> None: self.stock_dir = stock_dir self.sku_ids = list(sku_ids) self.image_size = int(image_size) self.threshold = float(threshold) + # A trained head produces a different similarity scale than raw features, so + # it needs its own line. Measured on a real milk shelf: the head answers 18 of + # 72 facings above 0.4 and is right on 94 % of them, while the raw threshold + # (0.55) rejects almost everything the head says. + self.head_threshold = float(head_threshold) self.device = device self._model = None self._lock = threading.Lock() self._entries: List[GalleryEntry] = [] self._matrix = None self._available = False + # A fine-tuned projection head, if the store has one (see + # finetune_identity.py). Measured on a real milk shelf: raw features put 1 % + # balanced accuracy on fine-grained products, a trained head 73 %. + self._head = None + self._head_path = os.path.join(stock_dir, "identity_head.pt") # ------------------------------------------------------------------ loading + def load_head(self) -> bool: + """Load the store's fine-tuned head, if it has one. + + weights_only: the file is tensors and strings, not executable code. + A payload that is not that head leaves the raw gallery in place. + """ + import torch + + if not os.path.exists(self._head_path): + return False + try: + payload = torch.load(self._head_path, map_location="cpu", weights_only=True) + if not isinstance(payload, dict): + return False + projection_state = payload.get("projection") + prototypes = payload.get("prototypes") + class_ids = payload.get("class_ids") + if not isinstance(projection_state, dict) or not torch.is_tensor(prototypes): + return False + if not isinstance(class_ids, (list, tuple)) or prototypes.ndim != 2: + return False + if prototypes.shape[0] != len(class_ids): + return False + projection = torch.nn.Linear(384, 128) + projection.load_state_dict(projection_state) + projection.eval() + self._head = { + "projection": projection, + "prototypes": torch.nn.functional.normalize(prototypes.float(), dim=1), + "class_ids": [str(sku_id) for sku_id in class_ids], + } + except Exception: + self._head = None + return False + return True + def _load_model(self) -> Any: if self._model is not None: return self._model @@ -104,8 +154,12 @@ class ProductGallery: return self._available def _views_for(self, sku_id: str) -> List[str]: - """Every image in the stock directory that belongs to this SKU.""" + """Images whose stem is the SKU or SKU plus _N / -N. + + A bare prefix would enroll sku_001 as a view of sku_00. + """ found: List[str] = [] + extra = re.compile(rf"{re.escape(sku_id)}[_-]\d+$") try: names = sorted(os.listdir(self.stock_dir)) except OSError: @@ -114,48 +168,57 @@ class ProductGallery: stem, extension = os.path.splitext(name) if extension.lower() not in (".jpeg", ".jpg", ".png"): continue - if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"): + if stem == sku_id or extra.fullmatch(stem): found.append(os.path.join(self.stock_dir, name)) return found def load(self) -> bool: - """Embed every enrolled view. Returns False when there is nothing to load.""" + """Embed every enrolled view. Returns False when there is nothing to load. + + One lock covers the whole load: two callers cannot each embed the catalog. + """ + with self._lock: + if self._available and self._matrix is not None: + return True + return self._load_locked() + + def _load_locked(self) -> bool: import cv2 import numpy as np import torch paths: List[Tuple[str, str]] = [] + kept: List[Any] = [] for sku_id in self.sku_ids: for path in self._views_for(sku_id): + image = cv2.imread(path) + if image is None or image.size == 0: + continue paths.append((sku_id, path)) - if not paths: + kept.append(_prepare(image, self.image_size)) + if not kept: self._available = False + self._entries = [] + self._matrix = None return False try: model = self._load_model() + tensor = torch.from_numpy(np.stack(kept)).to(self.device) + with torch.no_grad(): + features = model(tensor) + features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() except Exception: self._available = False + self._entries = [] + self._matrix = None return False - - batch = [] - for _, path in paths: - image = cv2.imread(path) - if image is None or image.size == 0: - continue - batch.append(_prepare(image, self.image_size)) - if not batch: - self._available = False - return False - tensor = torch.from_numpy(np.stack(batch)).to(self.device) - with torch.no_grad(): - features = model(tensor) - features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() self._entries = [ GalleryEntry(sku_id=sku_id, view=path, embedding=features[index]) for index, (sku_id, path) in enumerate(paths) ] self._matrix = features self._available = True + self.load_head() return True # ------------------------------------------------------------------ querying @@ -171,13 +234,43 @@ class ProductGallery: return features[0] def match(self, crop: Any, threshold: Optional[float] = None) -> GalleryMatch: - """Nearest enrolled view, with a rejection when nothing is close enough.""" + """Nearest enrolled product, with a rejection when nothing is close enough.""" if not self._available or self._matrix is None: return GalleryMatch(sku_id=None, similarity=0.0, rejected=True) import numpy as np - limit = self.threshold if threshold is None else float(threshold) + limit = threshold + if limit is None: + limit = self.head_threshold if self._head is not None else self.threshold + limit = float(limit) embedding = self.embed(crop) + if self._head is not None: + import torch + if self._head is not None: + # Two stages, because the two mechanisms fail in opposite directions. + # Measured: raw features reject strangers well (5 % false accepts) but + # rank fine-grained products badly (2 % top-1); a trained head ranks them + # well (73 % balanced) but is confidently wrong on strangers (44 % false + # accepts), because a discriminative head has no "none of the above". + # So the raw features decide *whether* to answer, and the head decides + # *what* to answer. + raw_similarity = self._matrix @ embedding + best_view = float(raw_similarity.max()) if len(raw_similarity) else 0.0 + if best_view < self.threshold: + return GalleryMatch(sku_id=None, similarity=best_view, rejected=True) + with torch.no_grad(): + projected = torch.nn.functional.normalize( + self._head["projection"](torch.from_numpy(embedding).float()), dim=0 + ) + similarity = (self._head["prototypes"] @ projected).numpy() + ranked = sorted(zip(self._head["class_ids"], similarity), key=lambda kv: kv[1], reverse=True) + top_sku, top_score = ranked[0] + runner_sku, runner_score = ranked[1] if len(ranked) > 1 else (None, 0.0) + if float(top_score) < limit: + return GalleryMatch(sku_id=None, similarity=float(top_score), runner_up=top_sku, + runner_up_similarity=float(top_score), rejected=True) + return GalleryMatch(sku_id=top_sku, similarity=float(top_score), runner_up=runner_sku, + runner_up_similarity=float(runner_score)) similarity = self._matrix @ embedding # Several views per SKU: a SKU's score is its best view. best_by_sku: Dict[str, float] = {} @@ -196,6 +289,9 @@ class ProductGallery: return GalleryMatch(sku_id=top_sku, similarity=top_score, runner_up=runner_sku, runner_up_similarity=runner_score) + def head_loaded(self) -> bool: + return self._head is not None + def view_count(self) -> int: return len(self._entries) diff --git a/recognization/code/gui_app.py b/recognization/code/gui_app.py deleted file mode 100644 index df92251..0000000 --- a/recognization/code/gui_app.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Native Desktop GUI for Supermarket Shelf Product Recognition -Opens a desktop OpenCV window displaying image.jpg with bounding boxes only. -Users can click on any product directly on screen to inspect price, stock, -and feature matching inliers in real-time. -""" - -import os -import sys -import cv2 -import numpy as np - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from pipeline import ShelfRecognitionPipeline - - -def main(): - print("=" * 60) - print("STARTING NATIVE DESKTOP INTERFACE") - print("Image: image.jpg | Mode: Hybrid (Paper Method)") - print("Instructions: Click on ANY product to identify & view info.") - print("Press 'q' or 'ESC' on the window to exit.") - print("=" * 60) - - pipeline = ShelfRecognitionPipeline() - init_res = pipeline.set_active_shelf_image("image.jpg") - - if init_res.get("status") != "success": - print("[Error] Failed to load image.jpg") - return - - boxes = init_res["boxes"] - base_image = pipeline.current_image.copy() - - # Draw ONLY bounding boxes (no class names, no confidence labels) - display_image = pipeline.detector.draw_detections(base_image, boxes, draw_labels=False) - - window_name = "Smart Shelf - Click Any Product (Press Q to quit)" - cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) - cv2.resizeWindow(window_name, 1280, 850) - - def on_mouse(event, x, y, flags, param): - if event == cv2.EVENT_LBUTTONDOWN: - print(f"\n[Click detected at coordinates: ({x}, {y})]") - res = pipeline.process_click(x, y, method="hybrid") - - if res.get("status") == "success": - prod = res["inventory"] - cls_info = res["classification"] - box_info = res["target_box"] - - print("-" * 55) - print(f"📦 Product Identified : {prod['product_name']}") - print(f"🏷️ SKU ID : {prod['sku_id']}") - print(f"💰 Price : ¥{prod['price']}") - print(f"📊 In Stock (On-Shelf): {prod['number_on']} units") - print(f"🚚 Received / Sold : {prod['number_in']} in / {prod['number_out']} out") - print(f"📅 Expiration Date : {prod['expiration_date']}") - print(f"🎯 Match Confidence : {cls_info['confidence'] * 100:.1f}% (Score: {cls_info['score']})") - print("-" * 55) - - # Update main window with selected box highlight (in red) - selected_display = pipeline.detector.draw_detections( - base_image, boxes, selected_box_id=box_info["id"], draw_labels=False - ) - cv2.imshow(window_name, selected_display) - - # Show visual match comparison window if available - if res.get("match_visualization") is not None: - match_win = "Feature Matcher (Shelf Crop vs Stock Template)" - cv2.namedWindow(match_win, cv2.WINDOW_NORMAL) - cv2.resizeWindow(match_win, 800, 400) - cv2.imshow(match_win, res["match_visualization"]) - else: - print(f"[Info] {res.get('message', 'No product found at coordinates.')}") - - cv2.setMouseCallback(window_name, on_mouse) - - while True: - cv2.imshow(window_name, display_image) - key = cv2.waitKey(20) & 0xFF - if key in [ord('q'), ord('Q'), 27]: - break - - cv2.destroyAllWindows() - - -if __name__ == "__main__": - main() diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 2514f91..6fab9c0 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -25,6 +25,128 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float: return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4) +# Cosine scale (gallery). Same-product pairs were measured above 0.6 and +# different products below 0.5. These are not a new calibration. +COSINE_ACCEPT = 0.60 +COSINE_AMBIGUOUS = 0.50 +COSINE_MARGIN = 0.04 +VOTE_CAP = 3 + + +def _finite(value: Any) -> Optional[float]: + try: + number = float(value) + except (TypeError, ValueError): + return None + if number != number or number in (float("inf"), float("-inf")): + return None + return number + + +def aggregate_votes( + observations: Sequence[Dict[str, float]], + *, + cap: int = VOTE_CAP, +) -> Dict[str, float]: + """Mean of the newest observations, at most `cap`. + + Older frames are dropped, not averaged in. An empty observation still + counts, so a later miss can outvote an earlier weak hit. Inlier and + cosine scores must not be mixed in one list. + """ + window = list(observations)[-max(1, int(cap)):] + if not window: + return {} + totals: Dict[str, float] = {} + for scores in window: + for sku_id, score in (scores or {}).items(): + number = _finite(score) + if number is None: + continue + totals[str(sku_id)] = totals.get(str(sku_id), 0.0) + number + count = len(window) + return {sku_id: total / count for sku_id, total in totals.items()} + + +def _cosine_decision( + sorted_candidates: List[Tuple[str, float]], + *, + method: str, + details: Optional[Dict[str, Any]], + accept: float, + ambiguous: float, + margin_min: float, +) -> Dict[str, Any]: + top_sku, top_score = sorted_candidates[0] + runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0 + margin = top_score - runner_score + if top_score >= accept and margin >= margin_min: + level = IDENTITY_ACCEPTED + elif top_score >= ambiguous: + level = IDENTITY_AMBIGUOUS + else: + level = IDENTITY_REJECTED + return { + "status": level, + "method": method, + "best_sku": top_sku if level == IDENTITY_ACCEPTED else None, + "score": float(top_score), + "confidence": float(top_score), + "identity_level": level, + "ranked_candidates": [ + {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3] + ], + "details": details or {}, + "message": None, + } + + +def decision_for_match( + match: Any, + *, + method: str = "gallery", + margin: float = COSINE_MARGIN, +) -> Dict[str, Any]: + """Rank one gallery match. Rejection is the gallery's, not a second threshold. + + `match()` already applied the raw 0.55 line or the head's 0.40 line. Re-ranking + those scores against COSINE_ACCEPT (0.60) would refuse a head match the gallery + accepted. The only extra gate here is the published cosine margin. + """ + similarity = _finite(getattr(match, "similarity", 0.0)) or 0.0 + if match is None or getattr(match, "rejected", False) or not getattr(match, "sku_id", None): + return { + "status": IDENTITY_REJECTED, + "method": method, + "best_sku": None, + "score": similarity, + "confidence": similarity, + "identity_level": IDENTITY_REJECTED, + "ranked_candidates": [], + "details": {}, + "message": "no enrolled product is close enough", + } + runner = getattr(match, "runner_up", None) + runner_score = _finite(getattr(match, "runner_up_similarity", 0.0)) or 0.0 + gap = similarity - runner_score + close = bool(runner) and gap < margin + level = IDENTITY_AMBIGUOUS if close else IDENTITY_ACCEPTED + ranked = [{"sku_id": str(match.sku_id), "score": similarity}] + if runner: + ranked.append({"sku_id": str(runner), "score": runner_score}) + return { + "status": level, + "method": method, + "best_sku": None if close else str(match.sku_id), + "score": similarity, + "confidence": similarity, + "identity_level": level, + "ranked_candidates": ranked, + "details": {}, + "message": None, + } + + def decide_identity( scores: Dict[str, float], *, @@ -33,6 +155,7 @@ def decide_identity( min_confidence: float = DEFAULT_MIN_CONFIDENCE, details: Optional[Dict[str, Any]] = None, method: str = "hybrid", + scale: str = "inlier", ) -> Dict[str, Any]: """Rank catalog scores into accepted / ambiguous / unknown / rejected. @@ -40,7 +163,20 @@ def decide_identity( unknown. Two close scores are ambiguous. Only a unique, above-threshold score is accepted. """ - if not scores: + if scale == "cosine": + if min_score == DEFAULT_MIN_SCORE: + min_score = COSINE_ACCEPT + if min_margin == DEFAULT_MIN_MARGIN: + min_margin = COSINE_MARGIN + if min_confidence == DEFAULT_MIN_CONFIDENCE: + min_confidence = COSINE_AMBIGUOUS + cleaned: Dict[str, float] = {} + for sku_id, score in (scores or {}).items(): + number = _finite(score) + if number is None: + continue + cleaned[str(sku_id)] = number + if not cleaned: return { "status": IDENTITY_REJECTED, "method": method, @@ -54,8 +190,17 @@ def decide_identity( } sorted_candidates: List[Tuple[str, float]] = sorted( - scores.items(), key=lambda x: x[1], reverse=True + cleaned.items(), key=lambda item: item[1], reverse=True ) + if scale == "cosine": + return _cosine_decision( + sorted_candidates, + method=method, + details=details, + accept=min_score, + ambiguous=min_confidence, + margin_min=min_margin, + ) top_sku, top_score = sorted_candidates[0] runner_up_sku, runner_up_score = ( sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0) diff --git a/recognization/code/live.py b/recognization/code/live.py index f8f4bed..385f379 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -96,8 +96,9 @@ class TrackState: last_attempt_area: int = 0 score_sums: Dict[str, float] = field(default_factory=dict) score_n: int = 0 - identity: Optional[Dict[str, Any]] = None + observations: List[Dict[str, float]] = field(default_factory=list) candidates: List[Dict[str, Any]] = field(default_factory=list) + identity: Optional[Dict[str, Any]] = None confirmed: bool = False label_box: List[int] = field(default_factory=list) drift_frames: int = 0 @@ -184,6 +185,10 @@ class LiveSession: self._catalog_by_sku: Dict[str, Dict[str, Any]] = {} self._catalog_version = "" self._tags = None + # Shelf position of this frame's origin. Updated from a content shift + # before the session lock; tag cadence keys rows by it. Not a shelf map. + self._frame_origin = [0.0, 0.0] + self._shift_map = None self._compliance_tracker = None self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []} self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque( @@ -256,7 +261,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.score_n = 0 + track.observations = [] track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -286,6 +291,8 @@ class LiveSession: self._slots = [] self._latest_image = None self._latest_frame_id = None + self._frame_origin = [0.0, 0.0] + self._shift_map = None # ------------------------------------------------------------------ frame path def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]: @@ -300,6 +307,9 @@ class LiveSession: empty_enabled = self.settings.empty_slots generation = self._generation + + shelf_offset = self._note_frame_shift(image) + boxes = self.engine.detect(image, conf=conf) detected_at = time.perf_counter() tracks = self.tracker.update( @@ -327,7 +337,13 @@ class LiveSession: raw_box = list(detection["box"]) confidence = float(detection["confidence"]) if raw_box is None: - raw_box = list(track.box) + # Low-score second pass, or a Kalman box with no detection. + # Keep the last measured crop. Do not invent one from the prediction. + raw_box = list(state.raw_box) if state is not None and state.raw_box else [] + confidence = float(state.confidence) if state is not None else confidence + low_score = bool(getattr(track, "low_score", False)) or ( + track.det_index is None and raw_box == [] + ) previous_seq = self._last_seen_seq.get(track.track_id) if state is None: # An id that comes back after an absence may be a recycled id: @@ -369,8 +385,12 @@ class LiveSession: state.last_seq = seq self._last_seen_seq[track.track_id] = seq state.box = list(track.box) - state.raw_box = raw_box - state.confidence = confidence + if raw_box and not low_score: + state.raw_box = raw_box + state.confidence = confidence + elif not state.raw_box and raw_box: + state.raw_box = raw_box + state.confidence = confidence self._drop_stale_label(state) active[track.track_id] = state self._tracks = active @@ -405,7 +425,7 @@ class LiveSession: for state in self._tracks.values() if state.box ] - tag_bands = self._tags.plan(tag_boxes, image.shape) + tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset) # Collect the classification work under the lock; submit it after # releasing it. add_done_callback can run the callback inline when the @@ -443,39 +463,44 @@ class LiveSession: } return result + def _inherit_labels(self, seq: int) -> None: """Give a fresh track the label of a recently recognized product it covers. - The same physical product can be tracked as several ids while the camera - pans. Remembering where a label was measured means each product is - classified once and its neighbours inherit the result instead of waiting - in the classification queue again. + Only a measured label, still overlapping its own box, may be copied. + An inherited label is not a measurement, so it cannot spread to a neighbour. + Blur, glare, and a too-far frame skip inheritance: the overlap is not + evidence when the frame itself failed the quality gate. """ if not self._label_memory: return + if (self._last_quality or {}).get("hint"): + return for track in self._tracks.values(): if track.state != STATE_DETECTED or track.confirmed: continue for entry in reversed(self._label_memory): if seq - entry["seq"] > LABEL_MEMORY_FRAMES: break - if _iou(track.box, entry["box"]) >= LABEL_MEMORY_IOU: - track.state = STATE_RECOGNIZED - track.confirmed = True - track.label_box = list(track.box) - track.identity = { - "sku_id": entry["sku_id"], - "confidence": entry["confidence"], - "score": entry["score"], - "attempts": 0, - "inherited": True, - } - # An inherited label is a guess from the same place, not a - # measurement of this object: count it as re-verified so a - # later correction is auditable rather than a silent flip. - track.relabels = self._relabel_count.get(track.track_id, 0) + 1 - self._relabel_count[track.track_id] = track.relabels - break + if entry.get("inherited"): + continue + if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU: + continue + if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU: + continue + track.state = STATE_RECOGNIZED + track.confirmed = True + track.label_box = list(track.box) + track.identity = { + "sku_id": entry["sku_id"], + "confidence": entry["confidence"], + "score": entry["score"], + "attempts": 0, + "inherited": True, + } + track.relabels = self._relabel_count.get(track.track_id, 0) + 1 + self._relabel_count[track.track_id] = track.relabels + break def _remember_label(self, track: TrackState, box: List[int], seq: int) -> None: """Must be called with the session lock held.""" @@ -484,10 +509,12 @@ class LiveSession: self._label_memory.append( { "box": list(box), + "label_box": list(track.label_box or box), "sku_id": track.identity.get("sku_id"), "confidence": track.identity.get("confidence"), "score": track.identity.get("score"), "seq": seq, + "inherited": bool(track.identity.get("inherited")), } ) @@ -516,7 +543,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.score_n = 0 + track.observations = [] track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -539,11 +566,16 @@ class LiveSession: return [] eligible: List[TrackState] = [] + poor_frame = bool((self._last_quality or {}).get("hint")) for track in self._tracks.values(): if track.state != STATE_DETECTED or track.in_flight > 0: continue + if poor_frame: + continue if track.hits < self.settings.min_hits: continue + if len(track.raw_box) != 4: + continue x1, y1, x2, y2 = track.raw_box if min(x2 - x1, y2 - y1) < self.settings.min_crop_px: continue @@ -563,9 +595,8 @@ class LiveSession: eligible.append(track) # Newest first: while a camera pans, the products still worth labelling - # are the ones that just entered the frame. Sorting by area (the obvious - # default) spends the budget on products that are already leaving. - eligible.sort(key=lambda track: (track.first_seq, -track.area())) + # are the ones that just entered the frame. Area is only the tie-break. + eligible.sort(key=lambda track: (-track.first_seq, -track.area())) planned: List[Tuple[TrackState, Any, int]] = [] for track in eligible[:capacity]: crop = self.engine.crop(image, track.raw_box) @@ -623,13 +654,18 @@ class LiveSession: return track.in_flight = max(0, track.in_flight - 1) track.attempts += 1 - for sku_id, score in (scores or {}).items(): + observation = { + str(sku_id): float(score) for sku_id, score in (scores or {}).items() + } + track.observations.append(observation) + track.observations = track.observations[-3:] + for sku_id, score in observation.items(): track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score) - if scores: + if observation: track.score_n += 1 self._classify_times.append(finished_at - submitted_at) - if not scores: + if not any(track.observations): if track.attempts >= self.settings.max_attempts: track.state = STATE_UNRECOGNIZED track.confirmed = True @@ -638,12 +674,12 @@ class LiveSession: track.candidates = [] return - from identity import decide_identity + from identity import aggregate_votes, decide_identity - mean_scores = { - sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items() - } - decision = decide_identity(mean_scores, method=method) + mean_scores = aggregate_votes(track.observations) + decision = decide_identity( + mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier") + ) track.candidates = [ {"sku_id": row.get("sku_id"), "score": row.get("score")} for row in (decision.get("ranked_candidates") or [])[:3] @@ -836,7 +872,7 @@ class LiveSession: gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var()) glare = float(np.mean(np.all(small > self.settings.glare_pixel, axis=2))) - heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if state.raw_box] + heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if len(state.raw_box) == 4] median_height = float(np.median(heights)) if heights else 0.0 too_far = bool( heights @@ -845,7 +881,8 @@ class LiveSession: or sum(1 for h in heights if h < self.settings.min_crop_px) > len(heights) / 2 ) ) - blur = sharpness < self._blur_threshold() + # A black frame has Laplacian 0. That is not blur; a real shelf is not black. + blur = sharpness < self._blur_threshold() and float(gray.mean()) > 1.0 hint = None if glare > self.settings.glare_fraction: hint = "glare" @@ -896,6 +933,24 @@ class LiveSession: self._tags = TagService(ocr_engine, settings=settings or TagSettings()) return True + def _note_frame_shift(self, image: Any) -> Tuple[float, float]: + """Shelf position of this frame's origin. Phase correlation only. + + Runs before the session lock. A failed registration keeps the last origin + rather than inventing a jump. The content shift moves the origin the + other way, same sign as ShelfMap. + """ + if self._shift_map is None: + from shelf_map import ShelfMap + + self._shift_map = ShelfMap() + shift = self._shift_map.register(image) + with self._lock: + if shift is not None: + self._frame_origin[0] -= float(shift[0]) + self._frame_origin[1] -= float(shift[1]) + return (self._frame_origin[0], self._frame_origin[1]) + def _run_tag_passes(self, image: Any, bands: Sequence[Any]) -> None: """Submit tag passes to the engine's tag worker (never the frame loop).""" service = self._tags diff --git a/recognization/code/orb.py b/recognization/code/orb.py deleted file mode 100644 index e69de29..0000000 diff --git a/recognization/code/shelf_map.py b/recognization/code/shelf_map.py new file mode 100644 index 0000000..c644dfb --- /dev/null +++ b/recognization/code/shelf_map.py @@ -0,0 +1,348 @@ +""" +The shelf as a plane, not a stream of boxes. + +Every measurement so far treats a frame as the world: a facing is "a track id", +and a track id dies the moment the camera pans past and comes back. Label +inheritance, drift checks and restart counters were each correct fixes for that, +and all of them are symptoms. The shelf does not move; the camera does. + +This module keeps a per-session **shelf map**. Each frame is registered to the map +by its content shift, every detection is placed in shelf coordinates, and a +detection that lands where a facing already is *merges* into it. Identity votes and +tag readings accumulate on the physical facing rather than on a transient track. + +That is what the current literature does for retail: RetailGlue (CVPR 2026W) builds +shelf panoramas by matching products across frames and reports 88.4 % stitching F1, +and the 7-Eleven Taiwan system (Sci. Rep. 2025, 7,000+ stores) stitches "virtual +shelves" before checking planograms. + +**What is measured, and what is not.** The synthetic walk registers to within +0.01 px of its ground truth - but it is a pure translation of a single photograph, +so that validates the harness, not handheld video. The number that matters (real +registration error, and the duplicate rate that follows from it) needs real +footage, which `record.py` exists to collect. +""" + +from __future__ import annotations + +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +# A frame whose content shift cannot be estimated this confidently is treated as +# discontinuous: the map starts a new section rather than guessing a position. +MIN_REGISTRATION_RESPONSE = 0.20 +# Two detections are the same physical facing when their shelf boxes overlap this +# much. Measured: adjacent facings of one product overlap ~0 by construction. +MERGE_IOU = 0.45 + + +def _iou(a: Sequence[int], b: Sequence[int]) -> float: + x1, y1 = max(a[0], b[0]), max(a[1], b[1]) + x2, y2 = min(a[2], b[2]), min(a[3], b[3]) + intersection = max(0, x2 - x1) * max(0, y2 - y1) + area_a = max(0, a[2] - a[0]) * max(0, a[3] - a[1]) + area_b = max(0, b[2] - b[0]) * max(0, b[3] - b[1]) + return intersection / max(1.0, float(area_a + area_b - intersection)) + + +@dataclass +class Facing: + """One physical position on the shelf, with everything observed there.""" + + facing_id: str + shelf_box: List[int] + row: int = 0 + observations: int = 0 + first_seq: int = 0 + last_seq: int = 0 + identity_votes: Dict[str, float] = field(default_factory=dict) + tag_votes: Dict[str, float] = field(default_factory=dict) + tag_readings: List[Dict[str, Any]] = field(default_factory=list) + seen_track_ids: set = field(default_factory=set) + + def vote_identity(self, sku_id: Optional[str], weight: float = 1.0) -> None: + if sku_id: + self.identity_votes[sku_id] = self.identity_votes.get(sku_id, 0.0) + weight + + def vote_tag(self, sku_id: Optional[str], reading: Optional[Dict[str, Any]] = None) -> None: + if sku_id: + self.tag_votes[sku_id] = self.tag_votes.get(sku_id, 0.0) + 1.0 + if reading: + self.tag_readings.append(reading) + + def best_identity(self) -> Tuple[Optional[str], float]: + """The identity the facing's pooled evidence supports, and its agreement.""" + if not self.identity_votes: + return None, 0.0 + sku_id, weight = max(self.identity_votes.items(), key=lambda item: item[1]) + total = sum(self.identity_votes.values()) + return sku_id, weight / max(1e-6, total) + + def best_tag_identity(self) -> Tuple[Optional[str], float]: + if not self.tag_votes: + return None, 0.0 + sku_id, weight = max(self.tag_votes.items(), key=lambda item: item[1]) + total = sum(self.tag_votes.values()) + return sku_id, weight / max(1e-6, total) + + def as_payload(self) -> Dict[str, Any]: + identity, agreement = self.best_identity() + tag_identity, tag_agreement = self.best_tag_identity() + return { + "facing_id": self.facing_id, + "box": list(self.shelf_box), + "row": self.row, + "observations": self.observations, + "track_ids": len(self.seen_track_ids), + "sku_id": identity, + "identity_agreement": round(agreement, 3), + "tag_sku_id": tag_identity, + "tag_agreement": round(tag_agreement, 3), + "tag_readings": self.tag_readings[-3:], + } + + +class ShelfMap: + """Registers frames and accumulates facings in shelf coordinates.""" + + def __init__(self, settings: Optional[Any] = None) -> None: + self._lock = threading.Lock() + self._prev_gray: Optional[Any] = None + self._offset: List[float] = [0.0, 0.0] # shelf position of the frame origin + self._facings: List[Facing] = [] + self._orb = None + self.registrations = 0 + self.registration_failures = 0 + self.anchors = 0 + self.continuations = 0 + self.detections = 0 + self.merges = 0 + self.responses: List[float] = [] + + # ------------------------------------------------------------- registration + def _gray(self, frame: Any, scale: float = 0.5) -> Any: + import cv2 + + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + if scale != 1.0: + gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA) + return gray + + def register(self, frame: Any) -> Optional[Tuple[float, float]]: + """Content shift from the previous frame, in full-resolution pixels. + + Phase correlation first: it is the right tool for a translating scene and + costs a couple of milliseconds. ORB + RANSAC only when the correlation is + weak (blur, a large jump, a scene with no texture), because that is the case + where a fast estimator quietly returns nonsense. + """ + import cv2 + import numpy as np + + gray = self._gray(frame) + with self._lock: + previous = self._prev_gray + self._prev_gray = gray + if previous is None or previous.shape != gray.shape: + return (0.0, 0.0) + window = cv2.createHanningWindow((gray.shape[1], gray.shape[0]), cv2.CV_32F) + (dx, dy), response = cv2.phaseCorrelate(previous.astype(np.float32), gray.astype(np.float32), window) + self.responses.append(float(response)) + if response >= MIN_REGISTRATION_RESPONSE: + self.registrations += 1 + return (dx / 0.5, dy / 0.5) # back to full resolution + + shift = self._orb_shift(previous, gray) + if shift is None: + self.registration_failures += 1 + return None + self.registrations += 1 + return (shift[0] / 0.5, shift[1] / 0.5) + + def _orb_shift(self, previous: Any, current: Any) -> Optional[Tuple[float, float]]: + import cv2 + import numpy as np + + if self._orb is None: + self._orb = cv2.ORB_create(nfeatures=1500, fastThreshold=10) + key_a, desc_a = self._orb.detectAndCompute(previous, None) + key_b, desc_b = self._orb.detectAndCompute(current, None) + if desc_a is None or desc_b is None or len(key_a) < 20 or len(key_b) < 20: + return None + matches = cv2.BFMatcher(cv2.NORM_HAMMING).knnMatch(desc_a, desc_b, k=2) + good = [m for m, n in matches if m.distance < 0.75 * n.distance] + if len(good) < 12: + return None + src = np.float32([key_a[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) + dst = np.float32([key_b[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + matrix, _ = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=3.0) + if matrix is None: + return None + return (float(matrix[0, 2]), float(matrix[1, 2])) + + # ------------------------------------------------------------------ facings + def _anchor_to_map( + self, detections: Sequence[Sequence[int]], offset_x: float, offset_y: float + ) -> Tuple[float, float]: + """Correct the chained offset by matching detections against the map. + + Chaining pairwise shifts drifts: measured over a 79-step walk with *perfect* + pairwise registration, the accumulated x offset reached 2047 px on a 1500 px + shelf, the same facing landed in two places, and a shelf of ~60 facings + became 334 with 113 of them seen once. Drift is inherent to chaining - the + fix is to anchor the frame to the structure already on the map, which is + what the retail-stitching literature does with product correspondences. + + One ICP-style step is enough: for every detection that has a facing nearby, + the correction it implies is a vote, and the median vote wins. + """ + if not self._facings or len(detections) < 4: + return offset_x, offset_y + corrections = [] + search = 40.0 + for box in detections: + centre_x = (box[0] + box[2]) / 2.0 + offset_x + centre_y = (box[1] + box[3]) / 2.0 + offset_y + best, best_distance = None, search + for facing in self._facings: + facing_x = (facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 + facing_y = (facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 + distance = max(abs(facing_x - centre_x), abs(facing_y - centre_y)) + if distance < best_distance: + best, best_distance = facing, distance + if best is not None: + corrections.append( + ( + (best.shelf_box[0] + best.shelf_box[2]) / 2.0 - ((box[0] + box[2]) / 2.0 + offset_x), + (best.shelf_box[1] + best.shelf_box[3]) / 2.0 - ((box[1] + box[3]) / 2.0 + offset_y), + ) + ) + if len(corrections) < 4: + return offset_x, offset_y + dx = sorted(c[0] for c in corrections)[len(corrections) // 2] + dy = sorted(c[1] for c in corrections)[len(corrections) // 2] + self.anchors += 1 + return offset_x + dx, offset_y + dy + + def update(self, seq: int, frame: Any, tracks: Sequence[Dict[str, Any]]) -> None: + """Register the frame, then place or merge every detection on the shelf.""" + shift = self.register(frame) + boxes = [ + [int(v) for v in (track.get("box") or track.get("raw_box"))] + for track in tracks + if (track.get("box") or track.get("raw_box")) and len(track.get("box") or track.get("raw_box")) == 4 + ] + with self._lock: + if shift is None: + # A discontinuous frame: keep the last known offset rather than + # inventing one, so its detections land where the shelf is. + shift = (0.0, 0.0) + # `shift` is how far the CONTENT moved in the frame, so the frame's + # origin moves the opposite way along the shelf. Adding it instead + # (measured) walks the map backwards: the same facing lands 50 px away + # in the next frame and a shelf of ~60 facings becomes 857. + self._offset[0] -= shift[0] + self._offset[1] -= shift[1] + if self._facings: + self._offset[0], self._offset[1] = self._anchor_to_map( + boxes, self._offset[0], self._offset[1] + ) + offset_x, offset_y = self._offset + for track in tracks: + box = track.get("box") or track.get("raw_box") + if not box or len(box) != 4: + continue + self.detections += 1 + shelf_box = [ + int(round(box[0] + offset_x)), int(round(box[1] + offset_y)), + int(round(box[2] + offset_x)), int(round(box[3] + offset_y)), + ] + track_id = track.get("track_id") + facing = self._find(shelf_box, int(track_id) if track_id is not None else None) + if facing is None: + facing = Facing( + facing_id=f"face_{uuid.uuid4().hex[:8]}", + shelf_box=shelf_box, + first_seq=seq, + ) + self._facings.append(facing) + else: + self.merges += 1 + # Average the box: repeated observations of a static object + # should converge, and the average is what the eye expects. + for index in range(4): + facing.shelf_box[index] = int(round((facing.shelf_box[index] * facing.observations + shelf_box[index]) / (facing.observations + 1))) + facing.observations += 1 + facing.last_seq = seq + identity = track.get("identity") or {} + if identity.get("state") == "recognized": + facing.vote_identity(identity.get("sku_id"), float(identity.get("score") or 1.0)) + tag_identity = track.get("tag_identity") or {} + tag = track.get("price_tag") or {} + if tag_identity.get("sku_id") or tag.get("state") == "readable": + facing.vote_tag(tag_identity.get("sku_id"), tag if tag else None) + if track.get("track_id") is not None: + facing.seen_track_ids.add(int(track["track_id"])) + + def _find(self, shelf_box: Sequence[int], track_id: Optional[int] = None) -> Optional[Facing]: + """The facing a detection belongs to. + + Continuity first: if this track id was already placed nearby, it is the same + facing, and no overlap test is needed - the tracker already decided that. + Geometry only when the id is new (a pan that outran the tracker, or the + tracker renumbering). Measured: with geometry alone, box jitter split the + same facing often enough that the map held 210 facings where the tracker + held 156 ids - it was adding duplicates, not removing them. + """ + width = max(1, shelf_box[2] - shelf_box[0]) + height = max(1, shelf_box[3] - shelf_box[1]) + if track_id is not None: + radius = 0.75 * max(width, height) + best, best_distance = None, radius + for facing in self._facings: + if track_id not in facing.seen_track_ids: + continue + distance = max( + abs((facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 - (shelf_box[0] + shelf_box[2]) / 2.0), + abs((facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 - (shelf_box[1] + shelf_box[3]) / 2.0), + ) + if distance < best_distance: + best, best_distance = facing, distance + if best is not None: + self.continuations += 1 + return best + best, best_overlap = None, MERGE_IOU + for facing in self._facings: + overlap = _iou(shelf_box, facing.shelf_box) + if overlap > best_overlap: + best, best_overlap = facing, overlap + return best + + # ------------------------------------------------------------------ reporting + def facings(self) -> List[Facing]: + with self._lock: + return list(self._facings) + + def stats(self) -> Dict[str, Any]: + with self._lock: + facings = list(self._facings) + responses = list(self.responses[-200:]) + consistent = 0 + for facing in facings: + sku_id, agreement = facing.best_identity() + if sku_id and agreement >= 0.8: + consistent += 1 + return { + "facings": len(facings), + "detections": self.detections, + "merges": self.merges, + "continuations": self.continuations, + "registrations": self.registrations, + "anchored": self.anchors, + "registration_failures": self.registration_failures, + "median_response": round(sorted(responses)[len(responses) // 2], 3) if responses else None, + "identity_consistent": consistent, + "offset": [round(value, 1) for value in self._offset], + } diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index 67d0dd6..8e4b6ae 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -124,8 +124,7 @@ def main() -> None: args = parser.parse_args() url = args.url - if args.token and "token=" not in url: - url = f"{url}{'&' if '?' in url else '?'}token={args.token}" + headers = {"Authorization": f"Bearer {args.token}"} if args.token else None source = FrameSource(args.source, width=args.width, pan_step=args.pan) rtts: List[float] = [] @@ -147,7 +146,7 @@ def main() -> None: in_flight: Dict[int, float] = {} pending_frame: Optional[int] = None - with connect(url, max_size=8 * 1024 * 1024) as socket: + with connect(url, max_size=8 * 1024 * 1024, additional_headers=headers) as socket: hello = json.loads(socket.recv(timeout=10)) if hello.get("type") != "hello": raise SystemExit(f"unexpected first message: {hello}") diff --git a/recognization/code/suft.py b/recognization/code/suft.py deleted file mode 100644 index 7e78d6d..0000000 --- a/recognization/code/suft.py +++ /dev/null @@ -1,84 +0,0 @@ -import cv2 -import matplotlib.pyplot as plt -import numpy as np - -def suft_function(image): - image1 = cv2.imread(image) - # Convert the training image to RGB - training_image = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB) - - # Convert the training image to gray scale - training_gray = cv2.cvtColor(training_image, cv2.COLOR_RGB2GRAY) - - # Create test image by adding Scale Invariance and Rotational Invariance - test_image = cv2.pyrDown(training_image) - test_image = cv2.pyrDown(test_image) - num_rows, num_cols = test_image.shape[:2] - - rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), 30, 1) - test_image = cv2.warpAffine(test_image, rotation_matrix, (num_cols, num_rows)) - - test_gray = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY) - - # Display traning image and testing image - fx, plots = plt.subplots(1, 2, figsize=(20,10)) - - plots[0].set_title("Training Image") - plots[0].imshow(training_image) - - plots[1].set_title("Testing Image") - plots[1].imshow(test_image) - - - # ## Detect keypoints and Create Descriptor - - surf = cv2.xfeatures2d.SURF_create(800) - - train_keypoints, train_descriptor = surf.detectAndCompute(training_gray, None) - test_keypoints, test_descriptor = surf.detectAndCompute(test_gray, None) - - keypoints_without_size = np.copy(training_image) - keypoints_with_size = np.copy(training_image) - - cv2.drawKeypoints(training_image, train_keypoints, keypoints_without_size, color = (0, 255, 0)) - - cv2.drawKeypoints(training_image, train_keypoints, keypoints_with_size, flags = cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) - - # Display image with and without keypoints size - fx, plots = plt.subplots(1, 2, figsize=(20,10)) - - plots[0].set_title("Train keypoints With Size") - plots[0].imshow(keypoints_with_size, cmap='gray') - - plots[1].set_title("Train keypoints Without Size") - plots[1].imshow(keypoints_without_size, cmap='gray') - - # Print the number of keypoints detected in the training image - print("Number of Keypoints Detected In The Training Image: ", len(train_keypoints)) - - # Print the number of keypoints detected in the query image - print("Number of Keypoints Detected In The Query Image: ", len(test_keypoints)) - - - # ## Matching Keypoints - - # Create a Brute Force Matcher object. - bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck = False) - - # Perform the matching between the SURF descriptors of the training image and the test image - matches = bf.match(train_descriptor, test_descriptor) - - # The matches with shorter distance are the ones we want. - matches = sorted(matches, key = lambda x : x.distance) - - result = cv2.drawMatches(training_image, train_keypoints, test_gray, test_keypoints, matches, test_gray, flags = 2) - - # Display the best matching points - plt.rcParams['figure.figsize'] = [14.0, 7.0] - plt.title('Best Matching Points') - plt.imshow(result) - plt.show() - - # Print total number of matching points between the training and query images - print("\nNumber of Matching Keypoints Between The Training and Query Images: ", len(matches)) - return keypoints_with_size \ No newline at end of file diff --git a/recognization/code/tag_cards.py b/recognization/code/tag_cards.py index f0a8a2a..20f55c2 100644 --- a/recognization/code/tag_cards.py +++ b/recognization/code/tag_cards.py @@ -29,19 +29,116 @@ from the small one and marked as derived rather than measured. from __future__ import annotations +import json import math +import os import re from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple -# The consumption tax used by the tags in the demo images. A tag whose small -# price is exactly 1.08x its big price is self-consistent; other stores and eras -# use 1.10 or a per-100g unit price instead, so this is a check, never a rule. -TAX_RATIOS = (1.08, 1.10) +# Defaults when a store has no store.json. Measured on the demo LOPIA tags: +# 8% consumption tax, brick-red rails, and shelf prices from ¥50. A store file +# replaces these for that store only. +DEFAULT_TAX_RATIOS = (1.08, 1.10) +DEFAULT_MIN_PRICE = 50 +# OpenCV hue is 0-180. Brick red wraps past 0, so a pixel is rail-coloured when +# its hue is below hue_low or above hue_high. +DEFAULT_RAIL_HSV = {"hue_low": 14, "hue_high": 166, "sat_min": 70, "val_min": 70} +# Aliases the parser reads. load_store_policy() republishes them per store. +TAX_RATIOS = DEFAULT_TAX_RATIOS +MIN_PRICE = DEFAULT_MIN_PRICE + +_RAIL: Optional["StorePolicy"] = None + + +def _remember_rail(policy: "StorePolicy") -> None: + global _RAIL + _RAIL = policy + + + +class StorePolicyError(ValueError): + """store.json exists but cannot be used. Never borrow another store's file.""" + + +@dataclass(frozen=True) +class StorePolicy: + tax_ratios: Tuple[float, ...] = DEFAULT_TAX_RATIOS + min_price: int = DEFAULT_MIN_PRICE + rail_hue_low: int = 14 + rail_hue_high: int = 166 + rail_sat_min: int = 70 + rail_val_min: int = 70 + store_id: Optional[str] = None + + def active(self) -> None: + global TAX_RATIOS, MIN_PRICE + TAX_RATIOS = self.tax_ratios + MIN_PRICE = self.min_price + _remember_rail(self) + +def default_policy() -> StorePolicy: + policy = StorePolicy() + policy.active() + return policy + + +def _as_ratio(value: Any) -> float: + number = float(value) + if not 1.0 < number <= 1.5: + raise StorePolicyError(f"tax ratio {value!r} is outside 1-1.5") + return number + + +def _as_bound(value: Any, name: str, low: int, high: int) -> int: + number = int(value) + if not low <= number <= high: + raise StorePolicyError(f"{name} {value!r} is outside {low}-{high}") + return number + + +def load_store_policy(stock_dir: Optional[str]) -> StorePolicy: + """Read `/store.json`. A missing file uses the defaults. + + A file that is present but invalid raises StorePolicyError naming that path. + """ + if not stock_dir: + return StorePolicy() + path = os.path.join(stock_dir, "store.json") + if not os.path.isfile(path): + return StorePolicy() + try: + with open(path, encoding="utf-8") as handle: + raw = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + raise StorePolicyError(f"{path}: {exc}") from exc + if not isinstance(raw, dict): + raise StorePolicyError(f"{path}: expected an object") + try: + ratios = raw.get("tax_ratios", list(DEFAULT_TAX_RATIOS)) + if not isinstance(ratios, list) or not ratios: + raise StorePolicyError(f"{path}: tax_ratios must be a non-empty list") + rail = raw.get("rail_hsv", DEFAULT_RAIL_HSV) + if not isinstance(rail, dict): + raise StorePolicyError(f"{path}: rail_hsv must be an object") + policy = StorePolicy( + tax_ratios=tuple(_as_ratio(item) for item in ratios), + min_price=_as_bound(raw.get("min_price", DEFAULT_MIN_PRICE), "min_price", 1, 100000), + rail_hue_low=_as_bound(rail.get("hue_low", 14), "hue_low", 0, 180), + rail_hue_high=_as_bound(rail.get("hue_high", 166), "hue_high", 0, 180), + rail_sat_min=_as_bound(rail.get("sat_min", 70), "sat_min", 0, 255), + rail_val_min=_as_bound(rail.get("val_min", 70), "val_min", 0, 255), + store_id=str(raw["store_id"]) if raw.get("store_id") else None, + ) + except (TypeError, ValueError) as exc: + if isinstance(exc, StorePolicyError): + raise + raise StorePolicyError(f"{path}: {exc}") from exc + policy.active() + return policy HEADER_PATTERN = re.compile(r"お買得|お買い得|特価|価格|セール") -# A shelf price below this is not a price (measured live reads included 20 and 23). -MIN_PRICE = 50 + # The tag truncates its discount to a tenth of a 割, so agreement is one step. DISCOUNT_TOLERANCE = 0.05 # A regular price above this multiple of the sale price is a misread, not a discount. @@ -201,22 +298,33 @@ def cluster_rows(lines: Sequence[OcrLine], max_gap: int = 26) -> List[List[OcrLi return rows -def tag_row_score(row: Sequence[OcrLine]) -> float: +def tag_row_score(row: Sequence[OcrLine], header_pattern: Any = None) -> float: """How much a row looks like a line of price tags. Tags carry a header, a discount, and a decimal (tax-included) price; product - packaging rarely carries all three on one line. + packaging rarely carries all three on one line. The header pattern is bound + here: a missing module global must not fail the tag pass. """ + def pattern(name: str, fallback: str) -> Any: + found = globals().get(name) + return found if found is not None else re.compile(fallback) + + header = header_pattern if header_pattern is not None else pattern( + "HEADER_PATTERN", r"お買得|お買い得|特価|価格|セール" + ) + discount = pattern("DISCOUNT_PATTERN", r"(\d+(?:\.\d)?)\s*割\s*引") + decimal = pattern("DECIMAL_PATTERN", r"(\d{1,4})[.,](\d{2})(?!\d)") + jan = pattern("JAN3_PATTERN", r"(? TagFields: name_candidates.sort(key=lambda item: (-len(item[0]), -item[1].confidence)) fields.name = name_candidates[0][0] - # Layout, not digit count, separates the price from the JAN: the price is the # LARGEST text on the tag (its line is roughly twice as tall as the small # print), and the JAN's last three digits sit on a small line of their own. # Getting this order wrong made a 269 price look like a JAN and disappear. @@ -534,7 +641,12 @@ def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, i hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) hue, sat, val = cv2.split(hsv) - brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8) + rail = _RAIL or StorePolicy() + brick = ( + ((hue < rail.rail_hue_low) | (hue > rail.rail_hue_high)) + & (sat > rail.rail_sat_min) + & (val > rail.rail_val_min) + ).astype(np.uint8) row_fraction = brick.mean(axis=1) height, width = image.shape[:2] rails: List[Tuple[int, int, int, int]] = [] diff --git a/recognization/code/tags.py b/recognization/code/tags.py index 8d5fc5b..b4ad79d 100644 --- a/recognization/code/tags.py +++ b/recognization/code/tags.py @@ -36,6 +36,7 @@ from ocr import OcrEngine, OcrLine, OcrResult, OcrUnavailable # noqa: E402 from tag_cards import ( # noqa: E402 CORROBORATION_FLAGS, MIN_PRICE, + StorePolicy, TagFields as TagFieldsModel, corroborate, best_tag_rows, @@ -49,6 +50,13 @@ from tag_cards import ( # noqa: E402 TAG_STATE_READABLE = "readable" TAG_STATE_UNREADABLE = "unreadable" TAG_STATE_ABSENT = "absent" +# Seen on the rail, OCR returned nothing. Not the same as absent (no tag) or +# unreadable (text came back but did not corroborate). +TAG_STATE_DETECTED = "detected" + +# Two shelf rows closer than this are the same rail. A screen row index is not +# an identity: panning a new shelf into view reuses row 0. +SHELF_ROW_BIN_PX = 80 # Measured tag shape (add/image6.jpg): ~300x80 px, aspect ~3.75:1, digits ~22 px. TAG_MIN_WIDTH_RATIO = 0.035 @@ -114,6 +122,7 @@ class TagRecord: tag_id: str box: List[int] band_index: int + shelf_key: Tuple[str, int] = ("screen", 0) observations: List[TagObservation] = field(default_factory=list) first_seen: float = 0.0 last_seen: float = 0.0 @@ -126,6 +135,7 @@ class TagRecord: discount: Optional[float] = None unit_price: Optional[float] = None name: Optional[str] = None + price_source: str = "none" flags: List[str] = field(default_factory=list) confidence: float = 0.0 @@ -135,17 +145,18 @@ class TagRecord: Measured live: without this, single uncorroborated reads of the stylised digits were published as readable tags with prices of 20, 23, 72 and 4508. + A block seen with no text is `detected`; text that does not parse is + `unreadable`. Neither is `absent`. """ - if not self.observations: - return TAG_STATE_UNREADABLE - if self.price is None or self.price < MIN_PRICE: + if self.price is None: + saw_text = any(observation.lines for observation in self.observations) + return TAG_STATE_UNREADABLE if saw_text else TAG_STATE_DETECTED + if self.price < MIN_PRICE: return TAG_STATE_UNREADABLE corroborated = bool( {"tax_consistent", "tax_pair_consistent", "discount_consistent"} & set(self.flags) ) - if not corroborated: - return TAG_STATE_UNREADABLE - if self.regular_price is None and self.name is None: + if not corroborated or (self.regular_price is None and self.name is None): return TAG_STATE_UNREADABLE return TAG_STATE_READABLE @@ -159,8 +170,6 @@ class TagRecord: "discount": self.discount, "unit_price": self.unit_price, "name": self.name, - "regular_price": self.regular_price, - "price_source": self.price_source, "flags": list(self.flags), "confidence": round(float(self.confidence), 3), "observations": len(self.observations), @@ -386,6 +395,20 @@ class TagSettings: settle_passes: int = SETTLE_PASSES min_band_width_ratio: float = 0.10 max_tags: int = 120 + policy: Optional[StorePolicy] = None + + +def band_key(band: RailBand, shelf_offset: Optional[Sequence[float]] = None) -> Tuple[str, int]: + """Which rail this band is, stable across a pan. + + No shelf origin: the screen row, which is all a still has. With one (the + shelf position of the frame origin), the row's shelf y, so a new shelf that + lands on screen row 0 is not the row that already settled. + """ + if not shelf_offset or len(shelf_offset) < 2: + return ("screen", int(band.row_index)) + shelf_y = (band.box[1] + band.box[3]) / 2.0 + float(shelf_offset[1]) + return ("shelf", int(round(shelf_y / SHELF_ROW_BIN_PX))) class TagService: @@ -410,7 +433,7 @@ class TagService: self._tags: Dict[str, TagRecord] = {} self._pass_id = 0 self._frame_seq = 0 - self._last_pass_seq: Dict[int, int] = {} + self._last_pass_seq: Dict[Tuple[str, int], int] = {} self._pending_bands: set = set() self.passes_run = 0 self.passes_failed = 0 @@ -418,29 +441,37 @@ class TagService: self.passes_skipped = 0 # ------------------------------------------------------------------ frame loop - def plan(self, boxes: Sequence[Dict[str, Any]], image_shape: Sequence[int]) -> List[RailBand]: + def plan( + self, + boxes: Sequence[Dict[str, Any]], + image_shape: Sequence[int], + shelf_offset: Optional[Sequence[float]] = None, + ) -> List[RailBand]: """Bands worth OCR-ing right now (must be called from the frame loop). - A band is a shelf row, so its key is the row - not its on-screen position. - Keying by position meant a walking camera minted a fresh key every few - pixels of pan and re-read the same shelf continuously, which starved - classification of CPU (measured: 10x slower per crop while OCR ran). + The key is the shelf row when the caller passes the frame origin, and the + screen row otherwise. Keying by on-screen pixels re-read the same shelf + on every pan step (measured: 10x slower per crop while OCR ran); keying + by the screen row index instead hides a new shelf that reuses row 0. """ if not self.settings.enabled or not self.engine.available(): return [] + if self.settings.policy is not None: + self.settings.policy.active() bands = rail_bands(boxes, image_shape) width = float(image_shape[1]) bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width] with self._lock: + self._shelf_offset = shelf_offset self._frame_seq += 1 seq = self._frame_seq chosen: List[RailBand] = [] for band in bands: - key = band.row_index + key = band_key(band, shelf_offset) if key in self._pending_bands: continue last = self._last_pass_seq.get(key) - settled = self._band_settled(band.row_index) + settled = self._band_settled(key) if settled and last is not None: continue if last is not None and seq - last < self.settings.pass_interval_frames: @@ -449,8 +480,9 @@ class TagService: if len(chosen) >= self.settings.max_bands_per_pass: break for band in chosen: - self._last_pass_seq[band.row_index] = seq - self._pending_bands.add(band.row_index) + key = band_key(band, shelf_offset) + self._last_pass_seq[key] = seq + self._pending_bands.add(key) return chosen def run_pass(self, image: Any, band: RailBand, scale_bias: float = 1.0) -> List[TagObservation]: @@ -460,7 +492,7 @@ class TagService: independent reads of the same tag are what makes the vote meaningful (identical pixels would produce identical output). """ - key = band.row_index + key = band_key(band, getattr(self, "_shelf_offset", None)) try: x1, y1, x2, y2 = band.box crop = image[y1:y2, x1:x2] @@ -537,7 +569,7 @@ class TagService: ) card = card_from_lines(pooled) if pooled else seed card.box = box - if not card.fields.is_readable() and card.fields.price is None: + if not card.fields.is_readable() and card.fields.price is None and not group: continue cards.append( { @@ -593,8 +625,8 @@ class TagService: ) return extra - def _band_settled(self, row_index: int) -> bool: - row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index] + def _band_settled(self, key: Tuple[str, int]) -> bool: + row_tags = [tag for tag in self._tags.values() if tag.shelf_key == key] if not row_tags: return False return all(tag.settled for tag in row_tags) @@ -604,6 +636,7 @@ class TagService: box = candidate["box"] lines = candidate["lines"] fields = candidate["fields"] + shelf_key = band_key(band, getattr(self, "_shelf_offset", None)) observation = TagObservation( box=list(box), lines=list(lines), @@ -613,12 +646,13 @@ class TagService: observed_at=time.time(), ) - record = self._match_record(box) + record = self._match_record(box, shelf_key) if record is None: record = TagRecord( tag_id=f"tag_{uuid.uuid4().hex[:8]}", box=list(box), band_index=band.row_index, + shelf_key=shelf_key, first_seen=observation.observed_at, ) if len(self._tags) >= self.settings.max_tags: @@ -636,10 +670,12 @@ class TagService: record.settled = True return observation - def _match_record(self, box: Sequence[int]) -> Optional[TagRecord]: + def _match_record(self, box: Sequence[int], shelf_key: Tuple[str, int]) -> Optional[TagRecord]: best = None best_iou = 0.35 for record in self._tags.values(): + if record.shelf_key != shelf_key: + continue iou = _iou(box, record.box) if iou >= best_iou: best_iou = iou @@ -769,7 +805,12 @@ class TagService: "settled": False, } - counts = {TAG_STATE_READABLE: 0, TAG_STATE_UNREADABLE: 0, TAG_STATE_ABSENT: 0} + counts = { + TAG_STATE_READABLE: 0, + TAG_STATE_UNREADABLE: 0, + TAG_STATE_DETECTED: 0, + TAG_STATE_ABSENT: 0, + } for payload in by_track.values(): counts[payload["state"]] = counts.get(payload["state"], 0) + 1 return by_track, tags, counts @@ -817,6 +858,7 @@ class TagService: self._pending_bands.clear() self._pass_id = 0 self._frame_seq = 0 + self._shelf_offset = None def _iou(a: Sequence[int], b: Sequence[int]) -> float: diff --git a/recognization/code/test.py b/recognization/code/test.py deleted file mode 100644 index 5eb7553..0000000 --- a/recognization/code/test.py +++ /dev/null @@ -1,25 +0,0 @@ -from ultralytics import YOLO -import cv2 - -model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") - -image_path = "/Users/nguyenhongvan/recongnize/data/add/image9.jpg" - -results = model(image_path) - -image = cv2.imread(image_path) - -for box in results[0].boxes.xyxy.cpu().numpy(): - x1, y1, x2, y2 = map(int, box) - - cv2.rectangle( - image, - (x1, y1), - (x2, y2), - (0, 255, 0), - 2 - ) - -cv2.imshow("Bounding Boxes Only", image) -cv2.waitKey(0) -cv2.destroyAllWindows() \ No newline at end of file diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index dc74def..bc21027 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -155,6 +155,47 @@ class RestContractTests(unittest.TestCase): self.assertEqual(response.status_code, 415) self.assertEqual(response.json()["error"]["code"], "UNSUPPORTED_MEDIA_TYPE") + def test_a_lying_content_length_is_rejected_while_streaming(self): + """The ASGI receive stops once the body passes the cap, before decode.""" + import anyio + + from transport import BodyLimit + + cap = 32 + seen = {"bytes": 0} + + async def app(scope, receive, send): + while True: + message = await receive() + if message["type"] != "http.request": + continue + seen["bytes"] += len(message.get("body") or b"") + if not message.get("more_body"): + break + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + chunks = [b"x" * 20, b"y" * 20, b"z" * 20] + + async def receive(): + body = chunks.pop(0) if chunks else b"" + return {"type": "http.request", "body": body, "more_body": bool(chunks)} + + sent = [] + + async def send(message): + sent.append(message) + + async def run(): + await BodyLimit(app, cap)( + {"type": "http"}, receive, send + ) + + anyio.run(run) + self.assertLessEqual(seen["bytes"], cap + 20) + self.assertEqual(sent[0]["status"], 413) + self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"]) + def test_classify_and_inspect_roundtrip(self): recognized = self.client.post( "/v1/recognize?classify=false", content=jpeg_bytes(), @@ -285,13 +326,26 @@ class WebSocketProtocolTests(unittest.TestCase): self.assertEqual(missing["code"], "TRACK_NOT_IN_VIEW") def test_token_is_enforced_on_the_socket(self): + """Header or the first post-hello auth frame. A query token is rejected.""" client, engine = make_app(token="s3cret") try: with self.assertRaises(Exception): with client.websocket_connect("/v1/stream") as socket: + self.assertEqual(socket.receive_json()["type"], "hello") + socket.send_json({"type": "ping", "t": 1}) socket.receive_json() - with client.websocket_connect("/v1/stream?token=s3cret") as socket: + with self.assertRaises(Exception): + with client.websocket_connect("/v1/stream?token=s3cret") as socket: + socket.receive_json() + with client.websocket_connect( + "/v1/stream", headers={"Authorization": "Bearer s3cret"} + ) as socket: self.assertEqual(socket.receive_json()["type"], "hello") + with client.websocket_connect("/v1/stream") as socket: + self.assertEqual(socket.receive_json()["type"], "hello") + socket.send_json({"type": "auth", "token": "s3cret"}) + socket.send_json({"type": "ping", "t": 1}) + self.assertEqual(socket.receive_json(), {"type": "pong", "t": 1}) finally: engine.shutdown() diff --git a/recognization/code/test_demo.py b/recognization/code/test_demo.py deleted file mode 100644 index b62ca5c..0000000 --- a/recognization/code/test_demo.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Contract tests for the demo camera (the synthetic store walk). - -The walk is what a demo shows when there is no camera, so its geometry has to be -predictable: inside the image, moving, serpentine, and wrapping forever. -""" - -from __future__ import annotations - -import os -import sys -import unittest - -current_dir = os.path.dirname(os.path.abspath(__file__)) -project_root = os.path.dirname(current_dir) -if os.path.exists(os.path.join(project_root, "stock")): - os.chdir(project_root) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import numpy as np # noqa: E402 - -from demo import DemoCamera, DemoSourceError # noqa: E402 - - -class FakeImageCamera(DemoCamera): - """A camera over a synthetic image, so the geometry is testable without files.""" - - def __init__(self, width=1500, height=2000, window=(960, 540)): - super().__init__({"synthetic": {"id": "synthetic", "path": "synthetic", "label": "synthetic"}}, window) - self._image_cache = np.zeros((height, width, 3), dtype="uint8") - self._size_cache = (width, height) - - def _size(self, sample_id): # noqa: D102 - if sample_id != "synthetic": - raise DemoSourceError(f"unknown sample '{sample_id}'") - return self._size_cache - - def _image(self, sample_id): # noqa: D102 - return self._image_cache - - -class WalkGeometryTests(unittest.TestCase): - def setUp(self): - self.camera = FakeImageCamera() - self.plan = self.camera.plan(["synthetic"]) - - def test_plan_is_deterministic_and_covers_the_shelf(self): - self.assertEqual(self.plan["steps"], self.camera.plan(["synthetic"])["steps"]) - self.assertGreater(self.plan["steps"], 20, "a shelf walk is not two steps long") - self.assertEqual(self.plan["bays"][0]["sample"], "synthetic") - - def test_window_never_leaves_the_image(self): - width, height = 1500, 2000 - for step in range(self.plan["steps"]): - position = self.camera.window_at(["synthetic"], step) - self.assertGreaterEqual(position["x"], 0) - self.assertGreaterEqual(position["y"], 0) - self.assertLessEqual(position["x"] + position["width"], width) - self.assertLessEqual(position["y"] + position["height"], height) - - def test_the_camera_actually_moves(self): - xs = [self.camera.window_at(["synthetic"], step)["x"] for step in range(6)] - ys = [self.camera.window_at(["synthetic"], step)["y"] for step in range(self.plan["steps"])] - self.assertGreater(len(set(xs)), 1, "the window must pan, not sit still") - self.assertGreater(len(set(ys)), 1, "the walk must climb rows") - - def test_it_ascends_the_shelf(self): - first = self.camera.window_at(["synthetic"], 0)["y"] - last_band = self.plan["steps"] - 1 - self.assertGreater(first, self.camera.window_at(["synthetic"], last_band)["y"], - "the walk starts at the bottom row and climbs") - - def test_step_wraps(self): - first = self.camera.window_at(["synthetic"], 0) - wrapped = self.camera.window_at(["synthetic"], self.plan["steps"]) - self.assertEqual((first["x"], first["y"]), (wrapped["x"], wrapped["y"])) - - def test_frames_are_deterministic_jpegs(self): - payload, position = self.camera.frame(["synthetic"], 3, quality=70) - again, _ = self.camera.frame(["synthetic"], 3, quality=70) - self.assertTrue(payload.startswith(b"\xff\xd8"), "JPEG magic") - self.assertEqual(payload, again, "the same step must render the same frame") - self.assertIn("sample", position) - - def test_unknown_sample_is_rejected(self): - with self.assertRaises(DemoSourceError): - self.camera.plan(["nope"]) - - def test_window_larger_than_the_image_is_clamped(self): - camera = FakeImageCamera(width=800, height=600, window=(960, 540)) - position = camera.window_at(["synthetic"], 0) - self.assertLessEqual(position["width"], 800) - self.assertLessEqual(position["height"], 600) - self.assertEqual(position["x"], 0) - # The walk starts at the bottom of the shelf, so the window is bottom-aligned. - self.assertEqual(position["y"], 600 - position["height"]) - - -class RealShelfTests(unittest.TestCase): - """One check against the actual demo asset, skipped when it is absent.""" - - def test_real_sample_walks(self): - from app import get_available_shelf_images - - samples = {item["id"]: item for item in get_available_shelf_images("add")} - if "image6" not in samples: - self.skipTest("demo sample image6 not present") - camera = DemoCamera(samples) - plan = camera.plan(["image6"], (1280, 720)) - self.assertGreater(plan["steps"], 20, "a 1500x2000 shelf gives a multi-row walk") - payload, position = camera.frame(["image6"], 5, (1280, 720)) - self.assertGreater(len(payload), 10_000) - self.assertLessEqual(position["x"] + position["width"], plan["bays"][0]["width"]) - self.assertLessEqual(position["y"] + position["height"], plan["bays"][0]["height"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/recognization/code/test_empty.py b/recognization/code/test_empty.py deleted file mode 100644 index 0781650..0000000 --- a/recognization/code/test_empty.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys -import os - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from empty_shelf_detector import detect_and_draw_empty_slots - -if __name__ == "__main__": - image_path = "image.jpg" - print(f"Processing empty shelf detection on {image_path}...") - annotated, empty_slots = detect_and_draw_empty_slots( - image_path, - output_path="detected_empty_slots.jpg" - ) - print(f"Detected {len(empty_slots)} empty shelf slots:") - for s in empty_slots: - print(f" - Slot [{s['id']}]: Row {s['row_index']}, Box={s['box']}, Width={s['width']}px, Est. Missing Items={s['missing_items_est']}") - print(f"Visualization saved to: detected_empty_slots.jpg") diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py new file mode 100644 index 0000000..f993328 --- /dev/null +++ b/recognization/code/test_eval_live.py @@ -0,0 +1,241 @@ +"""Metric contracts for eval_live. No model, no real walk, no fabricated accuracy.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 +import numpy as np # noqa: E402 + +from eval_live import ( # noqa: E402 + EvalInputError, + identity_level, + labelled_top1, + leaked_labels, + load_index, + score_recording, + score_tracks, + track_churn, +) + + +def track(level, sku=None, track_id=1, box=None, relabels=0, restarts=0, tag=None): + identity = {"identity_level": level, "state": "recognized" if level == "accepted" else "unrecognized"} + if level == "accepted": + identity["sku_id"] = sku + row = { + "track_id": track_id, + "box": box or [0, 0, 10, 10], + "identity": identity, + "relabels": relabels, + "restarts": restarts, + } + if tag is not None: + row["price_tag"] = tag + return row + + +def write_jpeg(path, image): + ok, encoded = cv2.imencode(".jpg", image) + assert ok + with open(path, "wb") as handle: + handle.write(encoded.tobytes()) + + +class DecisionDenominatorTests(unittest.TestCase): + def test_settled_excludes_unsettled_and_rates_use_that_denominator(self): + scored = score_tracks( + [ + track("accepted", "sku_a"), + track("ambiguous"), + track("rejected"), + track("unknown"), + {"identity": {"state": "unconfirmed", "identity_level": None}}, + ] + ) + self.assertEqual(scored["settled"], 4) + self.assertEqual(scored["unsettled"], 1) + self.assertEqual(scored["accepted_rate"], 0.25) + self.assertIsNone(score_tracks([{"identity": {"state": "unconfirmed"}}])["accepted_rate"]) + + def test_wire_state_fills_a_missing_level(self): + self.assertEqual( + identity_level({"identity": {"state": "needs_confirmation"}}), "ambiguous" + ) + self.assertEqual(identity_level({"identity": {"state": "unconfirmed"}}), "unsettled") + + def test_readable_tags_are_counted_apart_from_detected(self): + scored = score_tracks( + [ + track("accepted", "sku_a", tag={"state": "readable"}), + track("rejected", tag={"state": "unreadable"}), + track("rejected", tag={"state": "absent"}), + track("rejected"), + ] + ) + self.assertEqual(scored["tags_readable"], 1) + self.assertEqual(scored["tags_detected"], 2) + self.assertEqual(scored["tags_absent"], 2) + + +class Top1AndChurnTests(unittest.TestCase): + def test_top1_counts_only_accepted_matches_over_overlapping_labels(self): + frames = [ + { + "frame": 1, + "tracks": [ + track("accepted", "sku_a", box=[0, 0, 10, 10]), + track("accepted", "sku_b", track_id=2, box=[100, 100, 110, 110]), + ], + }, + {"frame": 2, "tracks": [track("rejected", box=[0, 0, 10, 10])]}, + {"frame": 3, "tracks": []}, + ] + labels = [ + {"frame": 1, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + {"frame": 1, "box": [100, 100, 110, 110], "sku_id": "sku_a"}, + {"frame": 2, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + {"frame": 3, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + ] + scored = labelled_top1(frames, labels, iou_min=0.5) + self.assertEqual((scored["correct"], scored["wrong"], scored["rejected"], scored["unmatched"]), (1, 1, 1, 1)) + self.assertEqual(scored["top1"], 0.3333) + + def test_sku_churn_counts_a_renamed_track_not_the_first_name(self): + frames = [ + {"tracks": [track("accepted", "sku_a", track_id=7)]}, + {"tracks": [track("accepted", "sku_a", track_id=7)]}, + {"tracks": [track("accepted", "sku_b", track_id=7, relabels=1)]}, + {"tracks": [track("rejected", track_id=8, restarts=2)]}, + ] + churn = track_churn(frames) + self.assertEqual(churn["sku_changes"], 1) + self.assertEqual(churn["tracks_with_sku_change"], 1) + self.assertEqual(churn["relabels"], 1) + self.assertEqual(churn["restarts"], 2) + self.assertEqual(churn["per_track"][0]["track_id"], 7) + + +class LeakageTests(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + + def test_a_pasted_catalog_crop_is_excluded_and_a_real_facing_is_not(self): + stock = os.path.join(self.root, "stock") + os.makedirs(stock) + template = np.full((20, 16, 3), 40, dtype=np.uint8) + template[2:8, 2:8] = (255, 0, 0) + write_jpeg(os.path.join(stock, "sku_a.jpeg"), template) + frame = np.zeros((40, 40, 3), dtype=np.uint8) + frame[5:25, 10:26] = template + recording = os.path.join(self.root, "walk") + os.makedirs(recording) + write_jpeg(os.path.join(recording, "000000.jpg"), frame) + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps({"frame": 0, "path": "000000.jpg", "tracks": []}) + "\n") + labels = [ + {"frame": 0, "box": [10, 5, 26, 25], "sku_id": "sku_a"}, + {"frame": 0, "box": [0, 0, 8, 8], "sku_id": "sku_b"}, + ] + held, leaked = leaked_labels(recording, load_index(recording), labels, stock) + self.assertEqual(len(leaked), 1) + self.assertEqual(leaked[0]["leak"], "crop_is_template") + self.assertEqual([row["sku_id"] for row in held], ["sku_b"]) + + def test_score_recording_names_the_leak_and_refuses_it_as_top1(self): + stock = os.path.join(self.root, "stock") + os.makedirs(stock) + image = np.full((12, 12, 3), 9, dtype=np.uint8) + write_jpeg(os.path.join(stock, "sku_a.jpg"), image) + with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: + handle.write("SKU_ID,product_name\nsku_a,A\n") + recording = os.path.join(self.root, "session") + os.makedirs(recording) + shutil.copy(os.path.join(stock, "sku_a.jpg"), os.path.join(recording, "000000.jpg")) + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "frame": 1, + "path": "000000.jpg", + "tracks": [track("accepted", "sku_a", box=[0, 0, 12, 12])], + } + ) + + "\n" + ) + with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: + json.dump({"labels": [{"frame": 1, "box": [0, 0, 12, 12], "sku_id": "sku_a"}]}, handle) + report = score_recording(recording, stock_dir=stock) + self.assertEqual(report["leakage"]["excluded"], 1) + self.assertEqual(report["labelled_top1"]["labels"], 0) + self.assertEqual(report["labelled_top1"]["top1"], None) + self.assertEqual(report["decisions"]["accepted"], 1) + self.assertTrue(report["fingerprints"]["sample"]["sha256"]) + self.assertEqual(len(report["fingerprints"]["catalog"]["version"]), 12) + self.assertTrue(any("template-derived" in gap for gap in report["evidence"])) + + +class CliTests(unittest.TestCase): + def test_missing_recording_is_an_explicit_error(self): + completed = subprocess.run( + [sys.executable, os.path.join(current_dir, "eval_live.py"), "/no/such/recording"], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("error", json.loads(completed.stdout)) + + def test_the_same_pinned_recording_scores_twice_the_same(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + stock = os.path.join(root, "stock") + os.makedirs(stock) + with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: + handle.write("SKU_ID\nsku_a\n") + recording = os.path.join(root, "session") + os.makedirs(recording) + payload = np.zeros((8, 8, 3), dtype=np.uint8) + write_jpeg(os.path.join(recording, "000000.jpg"), payload) + line = { + "frame": 4, + "path": "000000.jpg", + "tracks": [ + track("accepted", "sku_a", tag={"state": "readable"}), + track("ambiguous", track_id=2), + ], + } + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps(line) + "\n") + with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: + json.dump({"labels": [{"frame": 4, "box": [0, 0, 8, 8], "sku_id": "sku_a"}]}, handle) + first = score_recording(recording, stock_dir=stock) + second = score_recording(recording, stock_dir=stock) + self.assertEqual(first, second) + self.assertEqual(first["decisions"]["accepted"], 1) + self.assertEqual(first["decisions"]["ambiguous"], 1) + self.assertEqual(first["labelled_top1"]["correct"], 1) + self.assertIsNone(first["fingerprints"]["model"]["weights_sha256"]) + + def test_a_broken_index_line_is_an_input_error(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + with open(os.path.join(root, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write("{not json}\n") + with self.assertRaises(EvalInputError): + load_index(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/code/test_image.py b/recognization/code/test_image.py deleted file mode 100644 index 57f138f..0000000 --- a/recognization/code/test_image.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Test Script for image.jpg -1. Runs YOLO detection on image.jpg. -2. Draws ONLY bounding box rectangles (no class names, no confidence labels). -3. Saves output to detected_image.jpg. -4. Matches the detected bounding boxes against stock templates. -""" - -import os -import sys -import cv2 -import numpy as np - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from detector import ShelfProductDetector -from feature_matcher import ProductFeatureMatcher -from inventory import InventoryManager - - -def main(): - image_path = "image.jpg" - print(f"[1/4] Loading {image_path}...") - image = cv2.imread(image_path) - if image is None: - print(f"[Error] Failed to read {image_path}") - return - - h, w = image.shape[:2] - print(f" Image resolution: {w}x{h}") - - print("[2/4] Detecting products using YOLO...") - detector = ShelfProductDetector("runs/detect/train/weights/best.pt") - # You can adjust conf_threshold as needed (default 0.25) - det_res = detector.detect(image, conf_threshold=0.25) - boxes = det_res.get("boxes", []) - print(f" Total detected items: {len(boxes)}") - - print("[3/4] Drawing bounding boxes ONLY (no class names, no confidence)...") - annotated = detector.draw_detections(image, boxes, draw_labels=False) - output_path = "detected_image.jpg" - cv2.imwrite(output_path, annotated) - print(f" Saved detection visualization to: {output_path}") - - print("[4/4] Matching against stock catalogue (SURF, ORB, BRISK, Hybrid)...") - matcher = ProductFeatureMatcher("stock") - inv = InventoryManager("stock") - - print("\n--- Stock Templates Present in image.jpg ---") - matched_skus = set() - sample_matches = [] - - # Test top detections to find corresponding stock SKUs - for b in boxes: - crop = detector.crop_roi(image, b["box"]) - # Use hybrid method from the paper - res = matcher.classify_crop(crop, method="hybrid") - if res.get("status") == "success": - sku_id = res["best_sku"] - score = res["score"] - conf = res["confidence"] - prod = inv.get_product(sku_id) - if prod: - matched_skus.add(sku_id) - sample_matches.append({ - "box_id": b["id"], - "box": b["box"], - "sku_id": sku_id, - "product_name": prod["product_name"], - "price": prod["price"], - "number_on": prod["number_on"], - "score": score, - "confidence": conf - }) - - # Sort matches by confidence - sample_matches.sort(key=lambda x: x["score"], reverse=True) - - print(f"\nSuccessfully matched {len(matched_skus)} distinct SKUs across detected shelf boxes:") - for sm in sample_matches[:10]: - print(f"Box #{sm['box_id']:<3} -> [{sm['sku_id']}] {sm['product_name']:<30} | Price: ¥{sm['price']:<5} | On-Shelf: {sm['number_on']} | Score: {sm['score']}") - - print("\n[Complete] Tested image.jpg successfully.") - - -if __name__ == "__main__": - main() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index cec10cf..d1355c5 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -276,6 +276,36 @@ class TagPassIntegrationTests(unittest.TestCase): self.assertNotIn("price_tag", result["tracks"][0]) engine.shutdown() + def test_a_panned_shelf_reusing_the_screen_row_is_planned(self): + """The same screen row on a shelf that has moved is a new rail.""" + session, engine, _ = self.make_tagged_session() + session._tags.settings.pass_interval_frames = 0 + session._tags.settings.settle_observations = 1 + session._tags.settings.settle_passes = 1 + + class Panned: + def __init__(self): + self.calls = 0 + + def register(self, image): + self.calls += 1 + # Content moved up 400 px, so the frame origin moves down the shelf. + return (0.0, -400.0) if self.calls == 2 else (0.0, 0.0) + + session._shift_map = Panned() + process(session, 1) + time.sleep(0.05) + before = session._tags.stats()["passes_run"] + self.assertGreaterEqual(before, 1) + process(session, 2) + time.sleep(0.1) + self.assertGreaterEqual(session._frame_origin[1], 400.0) + self.assertGreater( + session._tags.stats()["passes_run"], before, + "a shelf 400 px away reuses screen row 0 and must be read again", + ) + engine.shutdown() + class FramePipelineTests(unittest.TestCase): def test_latest_frame_wins_and_drops_are_counted(self): diff --git a/recognization/code/test_ocr_tags.py b/recognization/code/test_ocr_tags.py index b48ea2f..25d911d 100644 --- a/recognization/code/test_ocr_tags.py +++ b/recognization/code/test_ocr_tags.py @@ -21,6 +21,7 @@ import numpy as np # noqa: E402 from ocr import FakeOcr, NullOcr, OcrLine, OcrResult, OcrUnavailable, build_ocr_engine # noqa: E402 from tags import ( # noqa: E402 TAG_STATE_ABSENT, + TAG_STATE_DETECTED, TAG_STATE_READABLE, TAG_STATE_UNREADABLE, TagService, @@ -30,6 +31,8 @@ from tags import ( # noqa: E402 parse_price_tag, rail_bands, ) +from tag_cards import StorePolicy, StorePolicyError, load_store_policy # noqa: E402 +import tag_cards # noqa: E402 def image(width=1000, height=600): @@ -275,6 +278,108 @@ class TagServiceTests(unittest.TestCase): service.reset() self.assertEqual(service.stats()["tags"], 0) + def test_a_new_shelf_reusing_row_zero_is_read(self): + """Screen row 0 on a newly panned shelf is not the row that settled.""" + service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + array = image() + for _ in range(3): + for band in service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)): + service.run_pass(array, band) + self.assertEqual( + service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)), + [], + "the same shelf row stays settled", + ) + again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0)) + self.assertTrue(again, "a shelf 400 px away reuses the screen row and must be read") + + def test_a_seen_tag_with_no_text_is_detected_not_absent(self): + def reader(crop): + return [] + + service = TagService(FakeOcr(reader=reader), settings=TagSettings(pass_interval_frames=0)) + array = image() + # A bright paper block on a brick rail, under the product row. OCR reads nothing. + array[300:420, :] = (40, 40, 180) + array[330:390, 120:280] = (235, 235, 235) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + bands = service.plan(boxes, array.shape) + self.assertTrue(bands) + service.run_pass(array, bands[0]) + by_track, tags, counts = service.attach([{"track_id": 0, "box": [100, 100, 200, 300]}]) + self.assertTrue(tags, "the paper block is a tag even though OCR returned nothing") + self.assertEqual(tags[0]["state"], TAG_STATE_DETECTED) + self.assertEqual(by_track[0]["state"], TAG_STATE_DETECTED) + self.assertEqual(counts[TAG_STATE_DETECTED], 1) + self.assertNotEqual(by_track[0]["state"], TAG_STATE_ABSENT) + + def test_a_tag_pass_survives_a_missing_header_pattern(self): + """Evaluation replay died with NameError: HEADER_PATTERN.""" + import tag_cards + from tag_cards import OcrLine as CardLine + + row = [CardLine(text="毎日お買得", box=(0, 0, 40, 16), confidence=0.9)] + saved = tag_cards.__dict__.pop("HEADER_PATTERN", None) + try: + score = tag_cards.tag_row_score(row) + finally: + if saved is not None: + tag_cards.HEADER_PATTERN = saved + self.assertGreater(score, 0, "a sale header still marks the row when the global is missing") + service, _ = self.make(pass_interval_frames=0) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + bands = service.plan(boxes, image().shape) + self.assertTrue(bands) + service.run_pass(image(), bands[0]) + self.assertGreaterEqual(service.stats()["passes_run"], 1) + self.assertIsNone(service.stats()["last_pass_error"]) + + +class StorePolicyTests(unittest.TestCase): + def test_a_missing_file_uses_the_defaults(self): + policy = load_store_policy(None) + policy.active() + self.assertEqual(policy.tax_ratios, (1.08, 1.10)) + self.assertEqual(policy.min_price, 50) + self.assertEqual(tag_cards.MIN_PRICE, 50) + + def test_a_store_file_does_not_change_another_store(self): + import json + import tempfile + + with tempfile.TemporaryDirectory() as root: + good = os.path.join(root, "lopia") + bad = os.path.join(root, "other") + os.makedirs(good) + os.makedirs(bad) + with open(os.path.join(good, "store.json"), "w", encoding="utf-8") as handle: + json.dump({"store_id": "lopia", "tax_ratios": [1.08], "min_price": 80}, handle) + with open(os.path.join(bad, "store.json"), "w", encoding="utf-8") as handle: + handle.write("{not json") + loaded = load_store_policy(good) + self.assertEqual(loaded.min_price, 80) + self.assertEqual(tag_cards.MIN_PRICE, 80) + with self.assertRaises(StorePolicyError) as raised: + load_store_policy(bad) + self.assertIn(bad, str(raised.exception)) + self.assertEqual(tag_cards.MIN_PRICE, 80, "a bad file must not replace the store already loaded") + again = load_store_policy(good) + self.assertEqual(again.min_price, 80) + + def test_a_ratio_outside_range_fails_clearly(self): + import json + import tempfile + + load_store_policy(None) + with tempfile.TemporaryDirectory() as root: + with open(os.path.join(root, "store.json"), "w", encoding="utf-8") as handle: + json.dump({"tax_ratios": [2.5], "min_price": 999}, handle) + with self.assertRaises(StorePolicyError) as raised: + load_store_policy(root) + self.assertIn("tax ratio", str(raised.exception)) + self.assertEqual(tag_cards.MIN_PRICE, 50) + class EngineSelectionTests(unittest.TestCase): def test_explicit_none_and_fake(self): diff --git a/recognization/code/test_shelf_map.py b/recognization/code/test_shelf_map.py new file mode 100644 index 0000000..664ecce --- /dev/null +++ b/recognization/code/test_shelf_map.py @@ -0,0 +1,125 @@ +""" +Tests for the shelf map: registration, merging, and evidence pooling. + +The sign convention is tested explicitly, because getting it backwards is silent: +the map still produces boxes, they just land in the wrong place, and a shelf of +~60 facings becomes 857. That happened during development. +""" + +from __future__ import annotations + +import os +import sys +import unittest + +import numpy as np + +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(current_dir) +if os.path.exists(os.path.join(project_root, "stock")): + os.chdir(project_root) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from shelf_map import Facing, ShelfMap, _iou # noqa: E402 + + +def a_frame(shift_x=0, shift_y=0, size=(160, 240)): + """A textured frame, optionally a shifted copy of another.""" + height, width = size + rng = np.random.RandomState(7) + base = rng.randint(0, 255, (height * 2, width * 2, 3), dtype=np.uint8) + return base[height - shift_y:height * 2 - shift_y, width - shift_x:width * 2 - shift_x] + + +def a_track(track_id, box, sku=None, score=None, tag_sku=None): + track = {"track_id": track_id, "box": list(box)} + if sku: + track["identity"] = {"state": "recognized", "sku_id": sku, "score": score or 1.0} + else: + track["identity"] = {"state": "unconfirmed"} + if tag_sku: + track["tag_identity"] = {"sku_id": tag_sku, "score": 0.7} + return track + + +class IouTests(unittest.TestCase): + def test_overlap_maths(self): + self.assertAlmostEqual(_iou([0, 0, 10, 10], [0, 0, 10, 10]), 1.0) + self.assertEqual(_iou([0, 0, 10, 10], [20, 20, 30, 30]), 0.0) + + +class RegistrationTests(unittest.TestCase): + def test_the_frame_origin_moves_opposite_to_the_content(self): + """The sign that matters: content moving left means the camera moved right. + + Getting this backwards walks the map the wrong way. It is silent - boxes + still appear - and it turned a shelf of ~60 facings into 857. + """ + shelf = ShelfMap() + shelf.register(a_frame()) + shift = shelf.register(a_frame(shift_x=10)) # content moved right 10 + self.assertIsNotNone(shift) + self.assertAlmostEqual(shift[0], 10, delta=1.0) + + # Content moving right means the camera moved LEFT, so the frame's origin + # sits 10 px further back along the shelf. + shelf.update(0, a_frame(), [a_track(1, [0, 0, 20, 20])]) + offset_before = list(shelf._offset) + shelf.update(1, a_frame(shift_x=10), [a_track(2, [0, 0, 20, 20])]) + self.assertLess(shelf._offset[0], offset_before[0] + 1, + "the offset must follow the camera, not the content") + + def test_a_first_frame_has_no_shift(self): + shelf = ShelfMap() + self.assertEqual(shelf.register(a_frame()), (0.0, 0.0)) + + +class MergingTests(unittest.TestCase): + def test_a_facing_seen_twice_is_one_facing(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200])]) + shelf.update(1, a_frame(), [a_track(2, [12, 11, 112, 201])]) + self.assertEqual(len(shelf.facings()), 1) + self.assertEqual(shelf.stats()["merges"], 1) + + def test_two_neighbouring_facings_stay_separate(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 100, 200]), a_track(2, [110, 10, 200, 200])]) + self.assertEqual(len(shelf.facings()), 2) + + def test_identity_votes_pool_across_track_ids(self): + """The point of the map: one facing, many transient track ids, one answer.""" + shelf = ShelfMap() + for index, (track_id, sku) in enumerate([(1, "sku_a"), (2, "sku_a"), (3, "sku_a"), (4, "sku_b")]): + shelf.update(index, a_frame(), [a_track(track_id, [10, 10, 110, 200], sku=sku)]) + facing = shelf.facings()[0] + self.assertEqual(facing.observations, 4) + self.assertEqual(len(facing.seen_track_ids), 4, "four tracks, one facing") + sku_id, agreement = facing.best_identity() + self.assertEqual(sku_id, "sku_a") + self.assertAlmostEqual(agreement, 0.75) + + def test_tag_evidence_pools_separately_from_packaging(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200], sku="sku_a", tag_sku="sku_a")]) + shelf.update(1, a_frame(), [a_track(2, [10, 10, 110, 200], tag_sku="sku_a")]) + facing = shelf.facings()[0] + self.assertEqual(facing.best_identity()[0], "sku_a") + self.assertEqual(facing.best_tag_identity()[0], "sku_a") + self.assertEqual(len(facing.tag_readings), 0, "no readable tag payload was attached") + + +class PayloadTests(unittest.TestCase): + def test_a_facing_reports_its_pooled_evidence(self): + facing = Facing(facing_id="face_1", shelf_box=[0, 0, 10, 10]) + facing.vote_identity("sku_a", 2.0) + facing.vote_identity("sku_b", 1.0) + payload = facing.as_payload() + self.assertEqual(payload["sku_id"], "sku_a") + self.assertAlmostEqual(payload["identity_agreement"], 2 / 3, places=3) + self.assertIsNone(payload["tag_sku_id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/code/tracking.py b/recognization/code/tracking.py index a77bc61..42b7b2a 100644 --- a/recognization/code/tracking.py +++ b/recognization/code/tracking.py @@ -122,16 +122,30 @@ class _SessionByteTrack(BYTETracker): class TrackOut: """One tracked object for a frame.""" - __slots__ = ("track_id", "box", "score", "det_index") + __slots__ = ("track_id", "box", "score", "det_index", "low_score") - def __init__(self, track_id: int, box: List[int], score: float, det_index: Optional[int]): + def __init__( + self, + track_id: int, + box: List[int], + score: float, + det_index: Optional[int], + *, + low_score: bool = False, + ): self.track_id = int(track_id) self.box = [int(v) for v in box] self.score = float(score) self.det_index = None if det_index is None or det_index < 0 else int(det_index) + # ByteTrack's second pass keeps an id alive on a low-score box. That box + # is not a new measurement: callers must not crop or re-label from it. + self.low_score = bool(low_score) def __repr__(self) -> str: # pragma: no cover - debug helper - return f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, det={self.det_index})" + return ( + f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, " + f"det={self.det_index}, low={self.low_score})" + ) class SessionTracker: @@ -171,14 +185,17 @@ class SessionTracker: outputs: List[TrackOut] = [] if rows is None or len(rows) == 0: return outputs + high = float(self.args.track_high_thresh) for row in np.asarray(rows): # [x1, y1, x2, y2, track_id, score, cls, idx] + score = float(row[5]) outputs.append( TrackOut( track_id=int(row[4]), box=[int(round(row[0])), int(round(row[1])), int(round(row[2])), int(round(row[3]))], - score=float(row[5]), + score=score, det_index=int(row[7]) if len(row) > 7 else None, + low_score=score < high, ) ) return outputs diff --git a/recognization/code/transport.py b/recognization/code/transport.py new file mode 100644 index 0000000..66e94a1 --- /dev/null +++ b/recognization/code/transport.py @@ -0,0 +1,250 @@ +""" +Request bounds and the LAN token check. + +A client can declare a tiny Content-Length and then keep writing, so the +declared length is a hint and the read itself stops at the cap. The token +never travels in a query string: proxies and access logs keep those. +""" + +from __future__ import annotations + +import hmac +import ipaddress +import os +import shutil +import signal +import socket +import subprocess +from typing import Any, Dict, Optional + +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +def is_loopback_host(host: Optional[str]) -> bool: + text = (host or "").strip().lower() + if text.startswith("[") and text.endswith("]"): + text = text[1:-1] + if "%" in text: # fe80::1%lo0 + text = text.split("%", 1)[0] + if text in LOOPBACK_HOSTS: + return True + try: + return ipaddress.ip_address(text).is_loopback + except ValueError: + return False + + +def require_token_for_bind(host: Optional[str], token: Optional[str]) -> None: + """A non-loopback bind with no token is reachable by anyone on that network.""" + if token or is_loopback_host(host): + return + raise SystemExit( + f"refusing to bind {host} without DORAYAKI_API_TOKEN " + "(loopback is the only bind that may run with auth off)" + ) + + +def token_matches(presented: Optional[str], token: Optional[str]) -> bool: + if not presented or not token: + return False + # compare_digest raises when the lengths differ. Fold the length in so a + # short guess neither throws nor returns early. + return hmac.compare_digest( + len(presented).to_bytes(8, "big") + presented.encode(), + len(token).to_bytes(8, "big") + token.encode(), + ) + + +def bearer_matches(authorization: Optional[str], token: str) -> bool: + if not authorization or not token: + return False + scheme, _, presented = authorization.partition(" ") + if scheme.lower() != "bearer" or not presented or " " in presented: + return False + return token_matches(presented, token) + + +def declared_length(header: Optional[str]) -> Optional[int]: + if header is None or header == "": + return None + try: + length = int(header) + except (TypeError, ValueError): + return None + return length if length >= 0 else None + + +async def read_bounded(source, max_bytes: int) -> bytes: + """Stop once the cap is passed. The caller rejects the oversize body. + + A request stream is async-iterable. An UploadFile is not: it has read(n). + """ + out = bytearray() + read = getattr(source, "read", None) + if read is not None and not hasattr(source, "__aiter__"): + while True: + chunk = await read(65536) + if not chunk: + break + out += chunk + if len(out) > max_bytes: + break + return bytes(out) + async for chunk in source: + if not chunk: + continue + out += chunk + if len(out) > max_bytes: + break + return bytes(out) + + +class BodyLimit: + """Stop the ASGI receive once the body passes the cap. + + A handler that checks Content-Length, or reads request.body(), runs after + the server has already buffered the stream. This counts bytes at receive + and returns 413 without handing the rest to the app. WebSocket frames are + not HTTP bodies and are left alone. + """ + + def __init__(self, app: Any, max_bytes: int) -> None: + self.app = app + self.max_bytes = int(max_bytes) + + async def __call__(self, scope, receive, send) -> None: + if scope.get("type") != "http": + await self.app(scope, receive, send) + return + class TooLarge(Exception): + pass + + seen = 0 + async def limited() -> Dict[str, Any]: + nonlocal seen + message = await receive() + if message.get("type") != "http.request": + return message + body = message.get("body") or b"" + seen += len(body) + if seen > self.max_bytes: + raise TooLarge + return message + + try: + await self.app(scope, limited, send) + except TooLarge: + await send( + { + "type": "http.response.start", + "status": 413, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": b'{"error":{"code":"PAYLOAD_TOO_LARGE","message":"request body too large"}}', + } + ) + + +SERVICE_TYPE = "_dorayaki._tcp" + + +def _lan_address() -> Optional[str]: + """Address a phone on this network can open. Loopback is not one. + + The demo launcher uses `ipconfig getifaddr en0` then `en1`. A UDP connect + is the fallback when those interfaces are absent. + """ + if shutil.which("ipconfig"): + for iface in ("en0", "en1"): + try: + out = subprocess.check_output( + ["ipconfig", "getifaddr", iface], stderr=subprocess.DEVNULL, text=True + ).strip() + except (subprocess.CalledProcessError, OSError): + continue + if out and not is_loopback_host(out): + return out + try: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.connect(("192.0.2.1", 9)) + found = probe.getsockname()[0] + finally: + probe.close() + except OSError: + return None + if not found or is_loopback_host(found): + return None + return found + + +def advertise_host(bind_host: Optional[str]) -> Optional[str]: + """The address clients should browse. A loopback bind is not advertised. + + A wildcard bind (0.0.0.0, ::) has no address of its own, so the record + uses the LAN address the demo launcher would hand a phone. + """ + text = (bind_host or "").strip() + if text.startswith("[") and text.endswith("]"): + text = text[1:-1] + if is_loopback_host(text): + return None + if text in ("", "0.0.0.0", "::"): + return _lan_address() + try: + parsed = ipaddress.ip_address(text) + except ValueError: + return None + if parsed.is_loopback or parsed.is_unspecified: + return None if parsed.is_loopback else _lan_address() + return str(parsed) + + +class ServiceAdvertiser: + """Register `_dorayaki._tcp` with the system mDNS daemon and drop it on close. + + ponytail: macOS `dns-sd` only. A Linux host needs `avahi-publish` wired the + same way; do not add a Python mDNS package for this one record. + """ + + def __init__(self, host: str, port: int, name: str = "Dorayaki") -> None: + self.host = host + self.port = int(port) + self.name = name + self._process: Optional[subprocess.Popen] = None + + def start(self) -> bool: + if self._process is not None: + return True + binary = shutil.which("dns-sd") + if not binary: + return False + self._process = subprocess.Popen( + [binary, "-P", self.name, SERVICE_TYPE, "local", str(self.port), self.host, self.host], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True + + def close(self) -> None: + process = self._process + self._process = None + if process is None or process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + process.wait(timeout=2) diff --git a/recognization/code/web/app.js b/recognization/code/web/app.js index 6661b0e..5eb75cd 100644 --- a/recognization/code/web/app.js +++ b/recognization/code/web/app.js @@ -3,12 +3,24 @@ * * A client of the recognition server, nothing else: live frames go out over * /v1/stream, stills go out over /v1/recognize. Point it at another server with - * ?server=https://host:port and pass a token with ?token=... + * ?server=https://host:port. A token, when the server requires one, comes from + * sessionStorage (key dorayaki.token) or the fragment #token=..., never the query. */ const params = new URLSearchParams(location.search); const SERVER = (params.get('server') || '').replace(/\/$/, ''); -const TOKEN = params.get('token') || ''; +const TOKEN = readToken(); + +function readToken() { + const stored = sessionStorage.getItem('dorayaki.token') || ''; + const fragment = new URLSearchParams((location.hash || '').replace(/^#/, '')); + const fromHash = fragment.get('token') || ''; + if (fromHash) { + sessionStorage.setItem('dorayaki.token', fromHash); + history.replaceState(null, '', location.pathname + location.search); + } + return fromHash || stored; +} const I18N = { en: { @@ -363,7 +375,7 @@ function thumbUrl(skuId) { async function listCameras() { const select = el('sourceSelect'); - select.innerHTML = ''; + select.replaceChildren(); if (!navigator.mediaDevices || !window.isSecureContext) { const option = document.createElement('option'); option.value = 'file'; @@ -390,13 +402,16 @@ function connectStream() { if (state.ws && (state.ws.readyState === WebSocket.OPEN || state.ws.readyState === WebSocket.CONNECTING)) return; setConnection('connecting'); const base = SERVER || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}`; - const url = `${base.replace(/^http/, 'ws')}/v1/stream${TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ''}`; + const url = `${base.replace(/^http/, 'ws')}/v1/stream`; const socket = new WebSocket(url); socket.binaryType = 'arraybuffer'; state.ws = socket; socket.onopen = () => { state.reconnectDelay = 1000; + // Browsers cannot set Authorization on a WebSocket. The server hello + // arrives first; this frame is the token, and it is not a URL. + if (TOKEN) socket.send(JSON.stringify({ type: 'auth', token: TOKEN })); setConnection('live'); socket.send(JSON.stringify({ type: 'config', method: state.method, empty_slots: state.showEmpty })); startSending(); @@ -464,25 +479,25 @@ function sendFrame() { function renderCompliance(compliance) { const strip = el('complianceStrip'); + strip.replaceChildren(); if (!compliance) { strip.hidden = true; return; } const rows = compliance.rows || []; - const items = rows.slice(0, 3).map( - (row) => `⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})` - ); - if (rows.length > 3) { - items.push(`+${rows.length - 3} ${t('more')}`); - } const hints = compliance.hints || []; - if (!items.length && hints.length) { - strip.hidden = false; - strip.dataset.kind = 'hint'; - strip.innerHTML = hints.map((hint) => `${hint.detail}`).join(''); - return; + const chips = []; + for (const row of rows.slice(0, 3)) { + chips.push(`⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})`); } - if (!items.length) { strip.hidden = true; return; } + if (rows.length > 3) chips.push(`+${rows.length - 3} ${t('more')}`); + const texts = chips.length ? chips : hints.map((hint) => hint.detail); + if (!texts.length) { strip.hidden = true; return; } strip.hidden = false; - strip.dataset.kind = 'violation'; - strip.innerHTML = items.join(''); + strip.dataset.kind = chips.length ? 'violation' : 'hint'; + for (const text of texts) { + const item = document.createElement('span'); + item.className = 'compliance-item'; + item.textContent = text; + strip.appendChild(item); + } } function handleResult(message) { @@ -536,7 +551,7 @@ async function loadSamples() { const response = await fetch(api('/v1/samples'), { headers: headers() }); const data = await response.json(); const select = el('sampleSelect'); - select.innerHTML = ''; + select.replaceChildren(); for (const sample of data.samples) { const option = document.createElement('option'); option.value = sample.id; @@ -634,24 +649,42 @@ function renderInView() { } } } + list.replaceChildren(); if (!counts.size) { - list.innerHTML = `
${t('inviewPlaceholder')}
`; + const empty = document.createElement('div'); + empty.className = 'placeholder small'; + empty.textContent = t('inviewPlaceholder'); + list.appendChild(empty); return; } - const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([skuId, facings]) => { + const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]); + for (const [skuId, facings] of ranked) { const product = productFor(skuId); const name = product ? product.product_name : skuId; const price = product ? `¥${Math.round(product.price)}` : ''; - return `
- -
${name}
-
${price} · ${t('facings')} ${facings}
-
${facings}
`; - }).join(''); - list.innerHTML = rows; - list.querySelectorAll('.inview-row').forEach((row) => { - row.addEventListener('click', () => selectBySku(row.dataset.sku)); - }); + const row = document.createElement('div'); + row.className = 'inview-row'; + row.dataset.sku = skuId; + const thumb = document.createElement('img'); + thumb.className = 'inview-thumb'; + thumb.alt = ''; + thumb.src = thumbUrl(skuId); + const text = document.createElement('div'); + text.className = 'inview-text'; + const nameEl = document.createElement('div'); + nameEl.className = 'inview-name'; + nameEl.textContent = name; + const meta = document.createElement('div'); + meta.className = 'inview-meta'; + meta.textContent = `${price} · ${t('facings')} ${facings}`; + text.append(nameEl, meta); + const count = document.createElement('div'); + count.className = 'inview-facings'; + count.textContent = String(facings); + row.append(thumb, text, count); + row.addEventListener('click', () => selectBySku(skuId)); + list.appendChild(row); + } } function selectBySku(skuId) { @@ -715,9 +748,19 @@ function renderOcrSection(tag, reconciliation) { if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]); } if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']); - rows.innerHTML = lines - .map(([key, value, bad]) => `
${key}${value}
`) - .join(''); + rows.replaceChildren(); + for (const [key, value, bad] of lines) { + const row = document.createElement('div'); + row.className = 'ocr-row'; + const keyEl = document.createElement('span'); + keyEl.className = 'ocr-key'; + keyEl.textContent = key; + const valueEl = document.createElement('span'); + valueEl.className = bad ? 'ocr-val bad' : 'ocr-val'; + valueEl.textContent = value; + row.append(keyEl, valueEl); + rows.appendChild(row); + } } function showProductCard(product, identity) { @@ -755,12 +798,22 @@ function showCandidatesCard(identity) { el('metricReceipt').textContent = '—'; el('metricExpiry').textContent = '—'; const candidates = el('candidates'); - const rows = (identity.candidates || []).map((candidate) => { + candidates.replaceChildren(); + const title = document.createElement('div'); + title.className = 'candidates-title'; + title.textContent = t('candidatesTitle'); + candidates.appendChild(title); + for (const candidate of identity.candidates || []) { const product = productFor(candidate.sku_id); - const name = product ? product.product_name : candidate.sku_id; - return `
${name}${candidate.score}
`; - }).join(''); - candidates.innerHTML = `
${t('candidatesTitle')}
${rows}`; + const row = document.createElement('div'); + row.className = 'candidate-row'; + const name = document.createElement('span'); + name.textContent = product ? product.product_name : candidate.sku_id; + const score = document.createElement('span'); + score.textContent = String(candidate.score); + row.append(name, score); + candidates.appendChild(row); + } candidates.hidden = false; el('showMatchBtn').disabled = false; renderOcrSection(state.currentTag, state.currentReconciliation); @@ -889,10 +942,18 @@ function handleInspect(result) { if (result.crop_jpeg_b64) { el('productThumb').src = result.crop_jpeg_b64; } + const box = el('matchBox'); + box.replaceChildren(); if (result.match_visualization_jpeg_b64) { - el('matchBox').innerHTML = `feature matches`; + const image = document.createElement('img'); + image.alt = 'feature matches'; + image.src = result.match_visualization_jpeg_b64; + box.appendChild(image); } else { - el('matchBox').innerHTML = `${t('matchPlaceholder')}`; + const placeholder = document.createElement('span'); + placeholder.className = 'match-placeholder'; + placeholder.textContent = t('matchPlaceholder'); + box.appendChild(placeholder); } const identity = result.identity || {}; const parts = []; diff --git a/recognization/demo-images/ATTRIBUTION.json b/recognization/demo-images/ATTRIBUTION.json new file mode 100644 index 0000000..2712674 --- /dev/null +++ b/recognization/demo-images/ATTRIBUTION.json @@ -0,0 +1,16 @@ +[ + { + "file": "demo-image-1-mediya-dairy.jpg", + "title": "File:Azabudai-Hills---2025-04-05 015.jpg", + "license": "CC0", + "artist": "RuinDig/Yuki Uchida", + "source": "https://commons.wikimedia.org/wiki/File%3AAzabudai-Hills---2025-04-05_015.jpg" + }, + { + "file": "demo-image-2-milk-case.jpg", + "title": "File:Milk (2655574568).jpg", + "license": "CC BY 2.0", + "artist": "yumtan", + "source": "https://commons.wikimedia.org/wiki/File%3AMilk_%282655574568%29.jpg" + } +] \ No newline at end of file diff --git a/recognization/demo-images/README.md b/recognization/demo-images/README.md new file mode 100644 index 0000000..e5a118d --- /dev/null +++ b/recognization/demo-images/README.md @@ -0,0 +1,63 @@ +# The two demo shelves + +Two real store photographs, chosen because they are dense and their products are +visually distinct, with a store catalog enrolled for each. Both come from Wikimedia +Commons under licences that allow reuse; `demo-images/ATTRIBUTION.json` records the +author, licence and source of each. + +| | image 1 | image 2 | +| --- | --- | --- | +| file | `demo-images/demo-image-1-mediya-dairy.jpg` | `demo-images/demo-image-2-milk-case.jpg` | +| scene | a MEDIYA dairy case | a supermarket milk case | +| licence | **CC0** | **CC BY 2.0** | +| detections | 21 products (13 price cards filtered out) | 56 products | +| recognised | **16, covering all 14 products** | **49, covering all 13 products** | +| store | `stock-mediya/` | `stock-milk150/` | + +## How to run them + +```bash +cd recognization +YOLO_OFFLINE=1 .venv/bin/python code/app.py \ + --port 8090 --identity gallery --live-conf 0.25 \ + --stock-dir stock-mediya # or stock-milk150 +``` + +Then POST the image, or open the web client and use **Photo** mode: + +```bash +curl -s -X POST --data-binary @demo-images/demo-image-1-mediya-dairy.jpg \ + -H 'content-type: image/jpeg' \ + 'http://127.0.0.1:8090/v1/recognize?classify=true&conf=0.25&tags=true' | python3 -m json.tool | head -30 +``` + +## What the demo shows + +* **Every product on the shelf is boxed and named** - 小岩井 低脂肪牛乳, + 荻山酪農 カフェ・オ・レ, 丹那 3.6牛乳, 木次 ミルクコーヒー, 木次 パスチャライズ牛乳, + キスキ ブラウン・スイス牛乳, 丹那 飲んでやさしいヨーグルト, 農場3.7牛乳【特選】, + MINOR FIGURES OAT, 北海道 低脂肪乳, タカナシ 低温殺菌牛乳, RUDE HEALTH Oat Organic, + 小岩井 無脂肪牛乳, きすき ノンホモ牛乳 on the first; 明治 おいしい牛乳, ゆうパック + 優良牛乳, 酪農家限定 低温殺菌牛乳, 北海道牛乳, MEGMILK, 毎日牛乳, タカナシ 低温殺菌牛乳, + タカナシ 殺菌低脂肪牛乳, 毎日骨太, カルシウム入り低脂肪乳, 無脂肪牛乳, 低脂肪牛乳, ミルク + on the second. +* **Price cards are not boxed as products.** The detector finds them (a card is the + shape of a pack); their text gives them away - a card's characters are 30-44 % + digits and it prints the tax-excluded price (本体), a product's are 3-19 %. Measured + on image 1: 13 cards dropped, 21 products kept. +* **What it cannot name, it does not invent.** Boxes on the dark shelf strips and on + cards come back as "check" (amber), not as a product name. + +## The honest caveat, for the demo + +Both shelves were **enrolled** - their products were read off the shelf and their +crops became the store's catalog and its fine-tuned head, which is the product's own +onboarding flow (a store is enrolled once, then scanned). So the recognition rate +above is **in-distribution**: it is what the app does on a shelf it has been enrolled +on, not on a shelf it has never seen. On shelves it has not been enrolled on, the +measured behaviour is: detection transfers (46 of 54 unseen photos), identity does not +(off-the-shelf embeddings reach ~25 % top-1 on fine-grained products; see +`docs/EXTERNAL-EVAL.md`). + +The first image is CC0, so it can be used in any recording without attribution; the +second is CC BY 2.0 and needs the credit in `ATTRIBUTION.json` if it appears on screen. diff --git a/recognization/demo-images/demo-image-1-mediya-dairy.jpg b/recognization/demo-images/demo-image-1-mediya-dairy.jpg new file mode 100644 index 0000000..0bba5fb Binary files /dev/null and b/recognization/demo-images/demo-image-1-mediya-dairy.jpg differ diff --git a/recognization/demo-images/demo-image-2-milk-case.jpg b/recognization/demo-images/demo-image-2-milk-case.jpg new file mode 100644 index 0000000..3e5cf1d Binary files /dev/null and b/recognization/demo-images/demo-image-2-milk-case.jpg differ diff --git a/recognization/detected_empty_slots.jpg b/recognization/detected_empty_slots.jpg deleted file mode 100644 index d24ce1e..0000000 Binary files a/recognization/detected_empty_slots.jpg and /dev/null differ diff --git a/recognization/detected_image.jpg b/recognization/detected_image.jpg deleted file mode 100644 index ce539b1..0000000 Binary files a/recognization/detected_image.jpg and /dev/null differ diff --git a/recognization/docs/API.md b/recognization/docs/API.md index d25fd88..989c2c0 100644 --- a/recognization/docs/API.md +++ b/recognization/docs/API.md @@ -377,12 +377,9 @@ The protocol is capture-source agnostic; an iPad/phone client needs no server ch ## Tests ```bash -cd recognization/code -YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_observe test_engine test_tracking test_live test_api -v -DORAYAKI_REAL_MODEL=1 YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_real_model -v -cd .. && YOLO_OFFLINE=1 .venv/bin/python code/test_suite.py # the original CLI suite +cd recognization +YOLO_OFFLINE=1 YOLO_AUTOINSTALL=false python -m unittest discover -s code -p 'test_*.py' -v ``` -Run tests with an explicit module list: `unittest discover` would import -`test.py`/`fast_test.py`, which are scratch scripts with hardcoded paths from another -machine and a blocking OpenCV window. +`test_real_model` is discovered and skipped unless `DORAYAKI_REAL_MODEL=1`. +CI installs `requirements.lock` and runs the same command. diff --git a/recognization/docs/EXTERNAL-EVAL.md b/recognization/docs/EXTERNAL-EVAL.md new file mode 100644 index 0000000..4bb0d9a --- /dev/null +++ b/recognization/docs/EXTERNAL-EVAL.md @@ -0,0 +1,285 @@ +# Evaluation on real store photos from Wikimedia Commons + +Everything else in this repository is measured on two photographs. This evaluates the +pipeline on **54 real store photos it has never seen**, collected from Wikimedia +Commons with their licences (CC0 / CC BY / CC BY-SA / public domain) recorded in a +manifest. + +Reproduce: + +```bash +python code/collect_shelf_images.py ~/dorayaki-data/shelf-images # needs network +python code/eval_external_images.py ~/dorayaki-data/shelf-images +``` + +## 1. Detection transfers, and well + +| | | +| --- | --- | +| Images with at least one detection | **46 of 54** | +| Total detections | **2,258** | +| Book scans, illustrations, a photo of a shop front | 0 detections (correct) | + +The detector is the teammate's single-class YOLO trained on one shelf, and it found +individual cartons, bottles and packets on dairy cases, soy-sauce shelves, spice +racks, a confectionery aisle, an Okinawan supermarket and a Hong Kong grocery - with +tight boxes. Detection is not the weak link. + +## 2. Identity transfers when the product is genuinely present + +The original 21-SKU catalog contains 明治 おいしい牛乳. On +`File:2011年4月の牛乳 (5588310773).jpg` - a milk case photographed in 2011, entirely +unseen - the pipeline found **6 facings of 明治 おいしい牛乳 and labelled them +correctly**, and on `File:Kawasaki-Inryo-1.jpg` it found 野菜生活100 and +プロビオヨーグルト R-1. So the matcher does transfer to new photographs of products it +has templates for. + +## 3. Precision is the failure, and rejection is missing entirely + +The same run labelled these as 明治 おいしい牛乳, in the same photograph: + +* 低脂肪殺菌牛乳 (a タカナシ carton, top left) +* 有機豆乳 (top left corner) +* 特濃4.5 (three cartons, top middle) +* 無調整豆乳 (top right) +* 北海道牛乳 (several, bottom right) + +They are all white milk cartons. The matcher has no way to say "unknown", so every +look-alike becomes a claim. + +Measured over 24 foreign shelves whose products are *not* in the catalog: **451 +facings accepted as recognised products**, with the score distribution overlapping +the real one and its extremes landing on the wrong images: + +| | accepted | median score | p10 | p90 | +| --- | --- | --- | --- | --- | +| The shelf the catalog describes (`image6`) | 71 | 18.2 | 5.2 | 372.4 | +| Foreign shelves (24 images) | 451 | 6.0 | — | — | +| A bento (「198円 中華弁当」) | 1 | 50.5 | | | +| A supermarket interior | 1 | 80.6 | | | +| Deer cookies at Nara | 6 | 33.1 | | 189.9 | + +**No threshold on this score separates the two**: it would have to sit above 5 (to +catch the foreign p10) and below 6 (to keep the real median), and the worst offenders +score higher than any correct match on the demo shelf. + +## 4. What this means for the plan + +It confirms, on real data, the two conclusions the literature and the review pointed +at - and it makes them concrete: + +1. **The matcher's raw inlier score is not a confidence.** It cannot be used for + rejection or for ranking, and it is the reason packaging recognition cannot name + products on its own. +2. **Fine-grained look-alikes are the failure mode**: milk cartons, soy-milk cartons, + and the twenty near-identical mayonnaise packs from the earlier benchmark. The + GroceryVision study of 190 open VLMs found the same shape (94.5 % Recall@5, a + 17.5-point drop at Recall@1): global embeddings cluster categories well and rank + visually similar SKUs badly. + +So step 4 of the plan is not a refinement, it is the fix: a **shortlist followed by +local re-ranking** (PRISM's shape: SigLIP shortlist, then LightGlue verification) and a +**calibrated rejection** on an embedding distance where same-product and +different-product are actually separable (measured earlier: above 0.6 and below 0.5 +respectively, unlike this score). + +## Attribution + +The images are from Wikimedia Commons under their individual licences (CC0, CC BY, +CC BY-SA, public domain); `manifest.json` records the author, licence and source URL +of each one. They are evaluation material and are not redistributed here. + + +--- + +# Part 2: the identity stack, measured on a shelf where the product IS present + +Part 1 measured shelves whose products are absent. This part uses +`File:2011年4月の牛乳 (5588310773).jpg` - a milk case where **14 of 107 facings are +明治 おいしい牛乳**, a product the original catalog holds - so precision and recall can +be computed instead of just false accepts. The 107 crops were labelled by eye from a +contact sheet. + +## The shipped matcher + +| | claims | correct | precision | recall | +| --- | --- | --- | --- | --- | +| matcher | 31 | 12 | **39 %** | **86 %** | + +It finds almost every true facing and makes 19 false claims, on cartons of 特濃4.5, +低脂肪殺菌牛乳, 明治ラブ and 北海道牛乳. Correct where the product is, wrong on the +look-alikes: the same shape as Part 1. + +## The embedding gallery: rejection, but not precision + +| threshold | false accepts on foreign shelves | milk shelf: precision | recall | +| --- | --- | --- | --- | +| 0.45 | 29 % | 28 % | 79 % | +| 0.50 | 18 % | 29 % | 64 % | +| 0.55 | 11 % | 35 % | 50 % | +| **0.60** | **5 %** | 45 % | 36 % | +| 0.65 | 2 % | 0 % | 0 % | + +The gallery's value is the **controllable rejection**: on shelves where nothing +matches, false accepts fall from the matcher's 47.8 % to **5 %** at 0.60, and the +curve is smooth enough to pick an operating point. But on the shelf where the product +*is* present its precision (28-45 %) is no better than the matcher's 39 % - it buys +honesty about the unknown, not finer discrimination. + +## Enrolling a shelf from itself does not fix it + +The flywheel's first step is to enroll a store's own shelf. Measured by taking 1-3 +views per product from the labelled milk shelf and testing on the rest: + +| views per SKU | top-1 | wrong | rejected | +| --- | --- | --- | --- | +| 1 | **26 %** | 30 | 29 | +| 2 | 25 % | 27 | 27 | +| 3 | 23 % | 24 | 26 | + +Flat at ~25 %, and far below the 72-76 % measured on the demo shelf with the same +number of views. The demo shelf's facings are near-identical crops of the same +photograph; real milk cartons differ by small text and a colour band. **Off-the-shelf +DINOv2 is not discriminative enough for fine-grained SKUs**, which is what the +literature says (RP2K and RPC both fine-tune; GroceryVision reports a 17.5-point drop +from Recall@5 to Recall@1). Fine-tuning is not a refinement of the plan, it is a +requirement - and its labels are what the tag channel and the confirmation loop +produce. + +## The packaging's own text is a real channel, and resolution-bound + +The cartons differ by the text printed on them, so OCR was tested as an identity +channel - first single-scale, then with the tag reader's trick (two upscales, the red +channel, pooled): + +| product | single scale | multi-scale + red | +| --- | --- | --- | +| MEGMILK (large Latin text) | 85 % | **85 %** | +| 北海道牛乳 (large kanji) | 65 % | **65 %** | +| 明治ラブ | 0 % | **75 %** | +| 特濃4.5 | 0 % | **50 %** | +| 明治 おいしい牛乳 (small vertical text) | 7 % | **28 %** | +| 低脂肪殺菌牛乳 / 無調整豆乳 / 酪農3.6 | 0 % | 0 % | +| **overall** | 51 % | **60 %** | + +Text works where it is large and fails where it is small - the same resolution wall as +the price tags. This is a capture-spec question, not a model question. + +## The identity architecture this implies + +| channel | what it is good for | measured | +| --- | --- | --- | +| Detection | finding facings on unseen shelves | 46/54 images, tight boxes | +| Tag (corroborated) | exact identity and price, where readable | 15/15 fields on the demo shelf; sparse in the field | +| Packaging text | brand-level discrimination | 60 % recall, higher when text is large | +| Embedding gallery | **rejection** - saying "unknown" | 5 % false accepts at 0.60 | +| Fine-tuned embedding | fine-grained SKUs (milk cartons) | not built; off-the-shelf is ~25 % | +| Confirmation loop | everything the above cannot settle | not built | + +So the honest design is: **reject by default**, name the product from the tag when it +is readable, from the packaging text when it is legible, from a fine-tuned embedding +otherwise - and hand the remainder to the operator, whose answers become the next +enrollment. + + +--- + +# Part 3: fine-tuning the embedding, and what it buys + +Part 2 ended on the number that matters: enrolling a real shelf from itself leaves +off-the-shelf DINOv2 at ~25 % top-1, because real milk cartons differ by small text +and a colour band. The literature's answer is fine-tuning, so this measures it. + +`code/finetune_identity.py` trains a small projection head (384 -> 128, cosine +classifier, angular margin) on frozen DINOv2 features, with augmentation standing in +for the second visit we do not have. Two enrolment views per product, twelve +augmented copies each, 72 held-out facings, 8 products. Balanced accuracy is reported +because the shelf is unbalanced (MEGMILK has 34 facings, 酪農3.6 has two - a system +that always answered "MEGMILK" would look respectable on the raw rate). + +| | top-1 | balanced | answered | of those right | +| --- | --- | --- | --- | --- | +| raw DINOv2 cosine | 2/72 (2 %) | 1 % | 72 | 3 % | +| raw, rejecting below 0.55 | 2/72 (2 %) | 1 % | 44 | 5 % | +| **fine-tuned + margin** | 28/72 (38 %) | **73 %** | 72 | 39 % | +| **fine-tuned, rejecting below 0.4** | 17/72 (23 %) | 58 % | 18 | **94 %** | + +Three things follow. + +1. **Fine-tuning is not a refinement, it is the difference between working and not.** + Balanced accuracy goes from **1 % to 73 %** on the same held-out facings. Off-the- + shelf features cannot separate these products; a head trained on sixteen crops can. +2. **With a rejection threshold it becomes precise**: answering only 18 of 72 facings + and being right on 94 % of them is exactly the operating point a store wants - + name what you are sure of, and hand the rest to the operator. The raw model cannot + do this at any threshold (5 % of answered). +3. **Where it fails is where the labels are coarse.** Per class, with rejection: + 明治ラブ 2/2, おいしい無調整豆乳 2/2, 低脂肪 1/1, 特濃4.5 1/2, 北海道牛乳 8/21, + 明治 おいしい牛乳 2/12, and **MEGMILK 1/32**. MEGMILK is the class I labelled most + coarsely - the contact sheet shows what are probably several different products + under one brand - so the class is internally inconsistent and the model correctly + refuses most of it. Better labels, not a better model, is what that needs. + +## The honest limit + +Enrolment and held-out facings come from **one photograph**. Augmentation stands in +for a second visit, which is not the same thing: a real second visit changes the +light, the angle and the stock. So 73 % balanced and 94 % precision are **upper +bounds**, and the only way to know the real number is to record a second visit +(`docs/DATA.md`). + +What is settled is the mechanism: **a head trained on labels the pipeline can produce +itself** - tag-named facings, plus operator confirmations - moves fine-grained +identity from useless to usable, and a rejection threshold turns it into something a +store can act on. + + +--- + +# Part 4: making it usable, and the trap that came with it + +The head is now part of a store: `python code/finetune_identity.py --stock stock-lopia` +trains on that store's enrolled views and writes `/identity_head.pt`, which +`ProductGallery` loads automatically. The first attempt exposed a trap worth writing +down. + +## A trained head destroys rejection + +Measured on the same two sets (the home shelf it was enrolled from, and 20 shelves +whose products are absent): + +| | home shelf accepted | foreign shelves accepted | +| --- | --- | --- | +| raw features, threshold 0.55 | 78 % | **5 %** | +| trained head, threshold 0.40 | 58 % | **44 %** | + +A discriminative head has no "none of the above": trained with an angular margin to +separate seven products, it maps *everything* confidently onto its seven prototypes, +including cartons it has never seen. Rejection, which was the gallery's whole point, +disappears. + +## The fix is two stages, and it needs no retraining + +The two mechanisms fail in opposite directions, so each does what it is good at: + +* **raw features decide whether to answer** (they generalise: 5 % false accepts on + strangers), +* **the head decides what to answer** (it discriminates: 73 % balanced on + fine-grained products against 1 % raw). + +| | home shelf accepted | foreign shelves accepted | +| --- | --- | --- | +| **raw gate, then head ranking** | 47 % | **5 %** | + +The home-shelf acceptance is lower than raw-only because the gate is strict, which is +the honest trade: fewer claims, and the claims are worth more. + +## Still open + +* A joint accuracy number on the LOPIA store needs held-out labels there; its + enrolment views came from the facings the tags named, so scoring on those would be + scoring on the training set. +* Training the head *with* out-of-distribution negatives (crops from other stores) + should let it keep rejecting on its own, which would remove the need for the gate. +* Everything above is still one photograph per store. The second visit remains the + measurement that settles it. diff --git a/recognization/pyproject.toml b/recognization/pyproject.toml new file mode 100644 index 0000000..a97a54a --- /dev/null +++ b/recognization/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "dorayaki-recognition" +version = "0.1.0" +description = "Dorayaki shelf recognition server" +requires-python = ">=3.10,<3.13" +readme = "README.md" +dependencies = [ + "torch>=2.0.0", + "torchvision>=0.15.0", + "ultralytics==8.4.153", + "lap>=0.5.12", + "websockets>=13", + "opencv-python>=4.8.0", + "numpy>=1.23.0,<2.0.0", + "fastapi>=0.100.0", + "uvicorn>=0.20.0", + "python-multipart>=0.0.6", + "requests>=2.28.0", + "pyobjc-framework-Vision>=10; sys_platform == 'darwin'", + "pyobjc-framework-Quartz>=10; sys_platform == 'darwin'", +] + +[project.optional-dependencies] +dev = ["httpx>=0.27"] + +[tool.pytest.ini_options] +testpaths = ["code"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +addopts = ["-q"] + +[tool.setuptools] +py-modules = [] diff --git a/recognization/requirements.lock b/recognization/requirements.lock new file mode 100644 index 0000000..06fa8eb --- /dev/null +++ b/recognization/requirements.lock @@ -0,0 +1,79 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.txt requirements-dev.txt --universal --python-version 3.12 -o requirements.lock --no-annotate +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +cloudpickle==3.1.2 +contourpy==1.3.3 +cuda-bindings==13.4.3 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-pathfinder==1.8.2 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-toolkit==13.0.3.0 ; sys_platform == 'linux' +cycler==0.12.1 +fastapi==0.141.1 +filelock==4.0.4 +fonttools==4.66.0 +fsspec==2026.9.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +jinja2==3.1.6 +kiwisolver==1.5.1 +lap==0.5.13 +markupsafe==3.0.3 +matplotlib==3.11.2 +mpmath==1.3.0 +networkx==3.7 +numpy==1.26.4 +nvidia-cublas==13.1.1.3 ; sys_platform == 'linux' +nvidia-cuda-cupti==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cuda-nvrtc==13.0.88 ; sys_platform == 'linux' +nvidia-cuda-runtime==13.0.96 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cudnn-cu13==9.24.0.43 ; sys_platform == 'linux' +nvidia-cufft==12.0.0.61 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cufile==1.15.1.6 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-curand==10.4.0.35 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusolver==12.0.4.66 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparse==12.6.3.3 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' +nvidia-ml-py==13.615.71 +nvidia-nccl-cu13==2.30.7 ; sys_platform == 'linux' +nvidia-nvjitlink==13.4.92 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' +nvidia-nvtx==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +opencv-python==4.11.0.86 +packaging==26.3 +pillow==12.3.0 +polars==1.44.2 +polars-runtime-32==1.44.2 +psutil==7.2.2 +pydantic==2.13.5 +pydantic-core==2.46.5 +pyobjc-core==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-cocoa==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-coreml==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-quartz==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-vision==12.2.2 ; sys_platform == 'darwin' +pyparsing==3.3.3 +python-dateutil==2.9.0.post0 +python-multipart==0.0.32 +pyyaml==6.0.3 +requests==2.34.2 +setuptools==84.0.0 +six==1.17.0 +starlette==1.7.0 +sympy==1.14.0 +torch==2.14.0 +torchvision==0.29.0 +triton==3.8.0 ; python_full_version < '3.15' and sys_platform == 'linux' +typing-extensions==4.16.0 +typing-inspection==0.4.4 +ultralytics==8.4.153 +ultralytics-platform==0.1.65 +ultralytics-thop==2.2.1 +urllib3==2.8.0 +uvicorn==0.54.0 +websockets==17.1 diff --git a/recognization/run-demo-server.sh b/recognization/run-demo-server.sh new file mode 100755 index 0000000..62084e6 --- /dev/null +++ b/recognization/run-demo-server.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Start the recognition server and write ignored local app endpoints. +# +# ./run-demo-server.sh +# ./run-demo-server.sh stock-milk150 +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$(dirname "$0")" + +STOCK="${1:-stock-demo}" +PORT="${PORT:-8099}" +TOKEN_FILE=".demo-token" + +if [ ! -d "$STOCK" ]; then + echo "no such store: $STOCK" >&2 + exit 1 +fi + +IP="$(ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null || true)" +if [ -z "$IP" ]; then + echo "this Mac has no LAN address; the phone cannot reach it" >&2 + exit 1 +fi + +if [ ! -s "$TOKEN_FILE" ]; then + umask 077 + openssl rand -hex 32 > "$TOKEN_FILE" +fi +TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")" + +umask 077 +cat > "$ROOT/iosApp/Configuration/Config.local.xcconfig" < config/local.env < "$TMP" +mv "$TMP" "$PROPERTIES" + +cat <