diff --git a/recognization/.github/workflows/test.yml b/recognization/.github/workflows/test.yml deleted file mode 100644 index 48517a0..0000000 --- a/recognization/.github/workflows/test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: tests - -on: - push: - pull_request: - -jobs: - model-free: - runs-on: ubuntu-latest - defaults: - run: - working-directory: recognization - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install pinned dependencies - run: python -m pip install -r requirements.lock - - name: Model-free suite - env: - YOLO_OFFLINE: "1" - YOLO_AUTOINSTALL: "false" - run: python -m unittest discover -s code -p 'test_*.py' -v diff --git a/recognization/.gitignore b/recognization/.gitignore index a0bb35a..789e4d1 100644 --- a/recognization/.gitignore +++ b/recognization/.gitignore @@ -20,6 +20,3 @@ ENV/ # Temporary weights runs/detect/train/weights/last.pt *.log -stock-*/identity_head.pt -.demo-token -config/local.env diff --git a/recognization/README.md b/recognization/README.md index dff70a2..6dbf1ab 100644 --- a/recognization/README.md +++ b/recognization/README.md @@ -101,17 +101,19 @@ Bất kỳ client nào gửi được ảnh JPEG đều dùng được: điện ## 🛠️ Kiểm tra hệ thống (Tests) -From `recognization/`, after `pip install -r requirements.lock` (or `pip install -r requirements-dev.txt`): - ```bash -YOLO_OFFLINE=1 YOLO_AUTOINSTALL=false python -m unittest discover -s code -p 'test_*.py' -v +cd code +YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_observe test_engine test_tracking test_live test_api -v + +# Test với model thật (tải trọng số, chạy ~15s) +DORAYAKI_REAL_MODEL=1 YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_real_model -v + +cd .. +YOLO_OFFLINE=1 .venv/bin/python code/test_suite.py # bộ test gốc +.venv/bin/python code/bench_engine.py # đo tốc độ detect/classify ``` -That discovers every `code/test_*.py` module. `test_real_model` is included and skipped unless `DORAYAKI_REAL_MODEL=1` (it loads the YOLO weights). CI runs the same command from `.github/workflows/test.yml`. - -```bash -YOLO_OFFLINE=1 python code/bench_engine.py # speed measurement, not part of the suite -``` +Lưu ý: chạy test bằng danh sách module cụ thể, **không dùng** `unittest discover` vì sẽ import `test.py`/`fast_test.py` (script scratch với đường dẫn máy khác và cửa sổ OpenCV chặn). --- @@ -143,7 +145,6 @@ code/ empty_shelf_detector.py, inventory.py, pipeline.py, observe.py stream_client.py # client mẫu (camera / video / pan ảnh tĩnh) bench_engine.py # đo tốc độ - test_*.py # unittest modules (discover -s code -p 'test_*.py') web/ # web client (index.html, styles.css, app.js) docs/API.md # tài liệu API đầy đủ docs/design-price-tags-and-compliance.md # thiết kế OCR giá, đối chiếu, compliance, mobile diff --git a/recognization/code/app.py b/recognization/code/app.py index d3f5882..d4a3cc7 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -20,11 +20,10 @@ from __future__ import annotations import argparse import asyncio import base64 -from contextlib import asynccontextmanager +import hmac import json import logging import os -import re import sys import threading import time @@ -72,19 +71,9 @@ from engine import ( # noqa: E402 identity_payload, settings_from_env, ) +from demo import DemoCamera, DemoSourceError # noqa: E402 from record import FrameRecorder # noqa: E402 from live import LiveSession, LiveSettings # noqa: E402 -from transport import ( # noqa: E402 - BodyLimit, - ServiceAdvertiser, - advertise_host, - bearer_matches, - declared_length, - is_loopback_host, - read_bounded, - require_token_for_bind, - token_matches, -) LOGGER = logging.getLogger("dorayaki") @@ -105,12 +94,12 @@ class ServerSettings: host: str = "127.0.0.1" port: int = 8080 token: Optional[str] = None - # Empty means same-origin only. "*" reflects every caller and is not a default. - cors_origins: str = "" + cors_origins: str = "*" max_sessions: int = DEFAULT_MAX_SESSIONS live_method: str = "hybrid" live_conf: float = 0.1 live_tags: bool = True + demo_walk: bool = False # A live session can be recorded so it becomes data to label (record.py). record_frames: Optional[str] = None record_every: int = 3 @@ -136,26 +125,6 @@ def server_settings_from_env() -> ServerSettings: ) -def load_local_token(path: str = "config/local.env") -> None: - """Fill DORAYAKI_API_TOKEN from an ignored env file. Never prints the value. - - ponytail: one KEY=VALUE file, no quoting. Upgrade path is a real env loader - once a second secret needs the same file. - """ - if os.environ.get("DORAYAKI_API_TOKEN"): - return - try: - with open(path, "r", encoding="utf-8") as handle: - lines = handle.readlines() - except OSError: - return - for line in lines: - key, sep, value = line.strip().partition("=") - if key == "DORAYAKI_API_TOKEN" and sep and value: - os.environ["DORAYAKI_API_TOKEN"] = value - return - - # --------------------------------------------------------------------------- # request models # --------------------------------------------------------------------------- @@ -266,35 +235,23 @@ def create_app( scans: Any = None, ) -> FastAPI: settings = settings or server_settings_from_env() - @asynccontextmanager - async def lifespan(_app: FastAPI): - yield - for session in list(sessions.values()): - session.close() - engine.shutdown() - app = FastAPI( title="Dorayaki Recognition Server", version="1.0", description="Shelf product detection, tracking and classification for capture clients.", - lifespan=lifespan, ) - # Same-origin is the default. A configured list is echoed back; "*" is not. - cors = [o.strip() for o in settings.cors_origins.split(",") if o.strip() and o.strip() != "*"] - if cors: - app.add_middleware( - CORSMiddleware, - allow_origins=cors, - allow_credentials=False, - allow_methods=["GET", "POST"], - allow_headers=["Authorization", "Content-Type"], - ) + app.add_middleware( + CORSMiddleware, + allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()] or ["*"], + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) scans = scans if scans is not None else build_legacy_scans(engine) - # A live session can be recorded so a real scan becomes data to label later. - recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ - if settings.record_frames else None samples = {item["id"]: item for item in get_available_shelf_images("add")} + demo_camera = DemoCamera(samples) + demo_cameras: Dict[float, DemoCamera] = {} sessions: Dict[str, LiveSession] = {} sessions_lock = asyncio.Lock() app.state.engine = engine @@ -303,7 +260,8 @@ def create_app( async def authorize(authorization: Optional[str] = Header(default=None)) -> None: if not settings.token: return - if not bearer_matches(authorization, settings.token): + expected = f"Bearer {settings.token}" + if not authorization or not hmac.compare_digest(authorization, expected): raise _error("UNAUTHORIZED", "missing or invalid bearer token", 401) # ---------------------------------------------------------------- health @@ -318,14 +276,6 @@ def create_app( "warm": True, }, "catalog": {"version": engine.catalog_version(), "sku_count": engine.sku_count()}, - "identity": { - "backend": getattr(engine.settings, "identity_backend", "matcher"), - "gallery_views": ( - engine.identity_gallery().view_count() - if getattr(engine.settings, "identity_backend", "") == "gallery" and engine.identity_gallery() - else None - ), - }, "ocr": { # Without this an operator could not tell a broken OCR from a shelf # with no readable tags. @@ -358,22 +308,115 @@ def create_app( raise _error("NOT_FOUND", f"unknown sku '{sku_id}'", 404) return {"status": "success", "product": product} - def _sample_listing() -> List[Dict[str, Any]]: - listing = [] - import cv2 - - for item in samples.values(): - entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"} - image = cv2.imread(item["path"]) - if image is not None: - entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0]) - listing.append(entry) - return listing - + # ---------------------------------------------------------------- samples @app.get("/v1/samples", dependencies=[Depends(authorize)]) async def list_samples() -> Dict[str, Any]: - loop = asyncio.get_running_loop() - return {"samples": await loop.run_in_executor(None, _sample_listing)} + listing = [] + for item in samples.values(): + entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"} + try: # size is a convenience for clients that plan a demo walk + width, height = demo_camera._size(item["id"]) + entry["width"], entry["height"] = width, height + except DemoSourceError: + pass + listing.append(entry) + return {"samples": listing} + + # ---------------------------------------------------------------- demo camera + # The demo replay lives in its own module: it replaces the live tracks with a + # still recognition mapped into each walk frame, which is a film-making device, + # not a perception result. Every payload it produces is marked synthetic. + demo_recognition = None + recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ + if settings.record_frames else None + + + + def _demo_samples(raw: Optional[str]) -> List[str]: + if raw: + wanted = [part.strip() for part in raw.split(",") if part.strip()] + unknown = [part for part in wanted if part not in samples] + if unknown: + raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404) + return wanted + # main_shelf first: its products are the ones the catalog covers, so a + # demo walk recognises real items instead of unknown shelf neighbours. + for preferred in ("main_shelf", "image6"): + if preferred in samples: + return [preferred] + return [next(iter(samples))] if samples else [] + + # Which sample the walk uses, resolved once. The replay object itself is built + # later, because it needs the still tag pass. + demo_sample_ids: List[str] = [] + if settings.demo_walk: + try: + demo_sample_ids = _demo_samples(None) + demo_camera.warm(demo_sample_ids) + except Exception as exc: + LOGGER.warning("demo walk unavailable: %s", exc) + + def _demo_window(width: Optional[int], height: Optional[int]) -> tuple: + win_w = int(width or 960) + win_h = int(height or 540) + if not (160 <= win_w <= 2048 and 120 <= win_h <= 2048): + raise _error("BAD_REQUEST", "window must be between 160x120 and 2048x2048", 400) + return win_w, win_h + + def _demo_camera(pan: Optional[float]) -> DemoCamera: + """One camera per pacing value (the walk geometry depends on it).""" + if pan is None: + return demo_camera + key = round(float(pan), 4) + camera = demo_cameras.get(key) + if camera is None: + camera = DemoCamera(samples, pan_overlap=key) + demo_cameras[key] = camera + return camera + + @app.get("/v1/demo/plan", dependencies=[Depends(authorize)]) + async def demo_plan(samples_param: Optional[str] = None, width: Optional[int] = None, + height: Optional[int] = None, pan: Optional[float] = None) -> Dict[str, Any]: + """Geometry of the virtual walk (demo-only capture source).""" + try: + ids = _demo_samples(samples_param) + if not ids: + raise _error("NOT_FOUND", "no samples available", 404) + camera = _demo_camera(pan) + return {"status": "success", **camera.plan(ids, _demo_window(width, height))} + except DemoSourceError as exc: + raise _error("NOT_FOUND", str(exc), 404) + + @app.get("/v1/demo/frame", dependencies=[Depends(authorize)]) + async def demo_frame(step: int = 0, samples_param: Optional[str] = None, + width: Optional[int] = None, height: Optional[int] = None, + quality: int = 65, pan: Optional[float] = None) -> Response: + """One JPEG frame of the virtual walk: a store employee sweeping the shelf. + + Demo-only. The client advances `step` (it wraps), so the walk's speed + follows whatever frame rate the client can sustain. + """ + try: + ids = _demo_samples(samples_param) + if not ids: + raise _error("NOT_FOUND", "no samples available", 404) + payload, position = _demo_camera(pan).frame( + ids, step, _demo_window(width, height), max(30, min(95, int(quality))) + ) + except DemoSourceError as exc: + raise _error("NOT_FOUND", str(exc), 404) + return Response( + content=payload, + media_type="image/jpeg", + headers={ + "X-Demo-Sample": str(position["sample"]), + "X-Demo-Bay": str(position["bay_index"] + 1), + "X-Demo-Step": str(position["step"]), + "X-Demo-Steps": str(position["steps"]), + "X-Demo-X": str(position["x"]), + "X-Demo-Y": str(position["y"]), + }, + ) @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)]) async def get_sample(sample_id: str) -> Response: @@ -395,88 +438,39 @@ def create_app( if method is not None: _require_method(method) content_type = (request.headers.get("content-type") or "").split(";")[0].strip().lower() - announced = declared_length(request.headers.get("content-length")) - if announced is not None and announced > MAX_UPLOAD_BYTES: - raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) if content_type == "multipart/form-data": - form = await request.form(max_part_size=MAX_UPLOAD_BYTES) + form = await request.form() upload = form.get("file") if upload is None or not hasattr(upload, "read"): raise _error("BAD_REQUEST", "multipart body needs a 'file' field", 400) - payload = await read_bounded(upload, MAX_UPLOAD_BYTES) + payload = await upload.read() else: if content_type and not content_type.startswith("image/") and content_type != "application/octet-stream": raise _error("UNSUPPORTED_MEDIA_TYPE", f"unsupported content type '{content_type}'", 415) - payload = await read_bounded(request.stream(), MAX_UPLOAD_BYTES) + payload = await request.body() if not payload: raise _error("BAD_REQUEST", "empty request body", 400) if len(payload) > MAX_UPLOAD_BYTES: raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) - loop = asyncio.get_running_loop() try: - image = await loop.run_in_executor(None, engine.decode_image, payload) + image = engine.decode_image(payload) except UndecodableImage as exc: raise _error("UNDECODABLE_IMAGE", str(exc), 422) try: - result = await loop.run_in_executor( - None, - lambda: engine.recognize( - image, - classify=classify, - method=method, - conf=conf, - with_empty_slots=empty_slots, - ), + result = engine.recognize( + image, + classify=classify, + method=method, + conf=conf, + with_empty_slots=empty_slots, ) except UnknownMethod as exc: raise _error("BAD_REQUEST", str(exc), 400) if tags: - result = await loop.run_in_executor(None, _attach_still_tags, result, image) + result = _attach_still_tags(result, image) return result - def _is_price_card(crop: Any) -> bool: - """Is this detection a shelf price card rather than a product? - - Measured on a supermarket photo whose shelf edge carries big price cards: - the detector boxes them as products, because a card is roughly the shape of a - pack. Their text gives them away - a price card's characters are 30-44 % - digits and it prints the tax-excluded price (本体), while a product's are 3-19 % - digits. Reporting a price card as a product would put a box labelled with a - product name on a price tag, which is worse than not detecting it. - """ - ocr_engine = getattr(engine, "ocr_engine", None) - if ocr_engine is None or not getattr(ocr_engine, "available", lambda: False)(): - return False - try: - import cv2 - - big = cv2.resize(crop, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_CUBIC) - lines = ocr_engine.read(big).lines - except Exception: - return False - text = " ".join(line.text for line in lines) - compact = re.sub(r"\s", "", text) - if not compact: - return False - digits = sum(1 for char in compact if char.isdigit() or char in "¥￥./") / len(compact) - return ("本体" in text and digits >= 0.20) or digits >= 0.30 - - def _drop_price_cards(result: Dict[str, Any], image: Any) -> int: - """Remove detections that are price cards, and say how many were removed.""" - kept, dropped = [], 0 - for detection in result["detections"]: - crop = engine.crop(image, detection["box"]) - if crop is not None and crop.size and _is_price_card(crop): - dropped += 1 - continue - kept.append(detection) - if dropped: - result["detections"] = kept - result["summary"] = engine.summarize(kept, result.get("empty_slots") or []) - result["price_cards_dropped"] = dropped - return dropped - def _attach_still_tags(result: Dict[str, Any], image: Any) -> Dict[str, Any]: """Run one synchronous tag pass over a still and merge its evidence.""" ocr_engine = getattr(engine, "ocr_engine", None) @@ -489,9 +483,6 @@ def create_app( from reconcile import attach_tags_to_boxes from tags import TagService, TagSettings - # Before anything else: a price card is not a product. - _drop_price_cards(result, image) - boxes = [ {"id": detection["id"], "box": detection["box"]} for detection in result["detections"] ] @@ -554,6 +545,19 @@ def create_app( engine.image_store.set_tag_bundle(result["image_id"], bundle) return result + if settings.demo_walk and demo_sample_ids: + # Built here because the replay needs the still tag pass, and it recognises + # the whole shelf once up front so the walk's first frame is already + # labelled. Every payload it produces is marked synthetic. + try: + from demo_replay import DemoRecognition + + demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags) + demo_recognition.warm(demo_sample_ids[0]) + LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0]) + except Exception as exc: + LOGGER.warning("demo walk warm-up failed: %s", exc) + @app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)]) async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]: method = _require_method(body.method) @@ -566,10 +570,7 @@ def create_app( wanted = set(int(i) for i in body.detection_ids) detections = [d for d in detections if int(d["id"]) in wanted] started = time.perf_counter() - loop = asyncio.get_running_loop() - rows = await loop.run_in_executor( - None, lambda: engine.classify_boxes(stored.image, detections, method=method) - ) + rows = engine.classify_boxes(stored.image, detections, method=method) rows_by_id = {row["detection_id"]: row for row in rows} for detection in stored.detections: row = rows_by_id.get(int(detection["id"])) @@ -622,24 +623,21 @@ def create_app( stored = engine.image_store.get(image_id) except ImageExpired: raise _error("IMAGE_EXPIRED", f"image '{image_id}' is no longer available", 404) - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, lambda: engine.inspect(stored.image, body.box, method=method) - ) + return engine.inspect(stored.image, body.box, method=method) # ---------------------------------------------------------------- live stream @app.websocket("/v1/stream") async def stream(websocket: WebSocket) -> None: - # Query tokens are rejected. A wrong header is rejected before hello. - # A missing header is the browser case: hello goes out, then the first - # client text must be {"type":"auth","token":"..."} or the socket closes. - if websocket.query_params.get("token"): - await websocket.close(code=1008) - return - header = websocket.headers.get("authorization") - if settings.token and header and not bearer_matches(header, settings.token): - await websocket.close(code=1008) - return + if settings.token: + supplied = websocket.query_params.get("token") or "" + header = websocket.headers.get("authorization") or "" + expected_header = f"Bearer {settings.token}" + if not ( + hmac.compare_digest(supplied, settings.token) + or hmac.compare_digest(header, expected_header) + ): + await websocket.close(code=1008) + return async with sessions_lock: if len(sessions) >= settings.max_sessions: await websocket.close(code=1013) @@ -663,6 +661,7 @@ def create_app( await websocket.accept() send_lock = asyncio.Lock() loop = asyncio.get_running_loop() + demo_steps = [0] async def send(payload: Dict[str, Any]) -> None: async with send_lock: @@ -702,10 +701,21 @@ def create_app( "message": "frame processing failed"}) continue if recorder is not None: - # Disk writes must not sit on the event loop between results. - await loop.run_in_executor( - None, recorder.write, session_id, frame_id, payload, result - ) + # Record what the client sent and what we saw in it, so the walk + # can be labelled later instead of only measured on photographs. + recorder.write(session_id, frame_id, payload, result) + if demo_recognition is not None and demo_sample_ids: + # The walk's frames are served from the shelf's full recognition, + # so every product is already labelled with name and price. + sample_id = demo_sample_ids[0] + step = demo_steps[0] + try: + result = await loop.run_in_executor( + None, demo_recognition.apply, result, sample_id, step + ) + demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id)) + except Exception as exc: # a demo failure must not kill the stream + LOGGER.exception("demo walk frame failed: %s", exc) await send(result) async def handle_inspect(message: Dict[str, Any]) -> None: @@ -741,23 +751,8 @@ def create_app( processing = asyncio.create_task(processing_loop()) - authed = not settings.token or bearer_matches( - websocket.headers.get("authorization"), settings.token - ) - - async def handle_control(message: Dict[str, Any]) -> bool: - """Apply one control message. False closes the socket (auth failed).""" - nonlocal authed + async def handle_control(message: Dict[str, Any]) -> None: kind = message.get("type") - if not authed: - presented = message.get("token") if kind == "auth" else None - if isinstance(presented, str) and token_matches(presented, settings.token): - authed = True - return True - await websocket.close(code=1008) - return False - if kind == "auth": - return True if kind == "config": try: config = session.configure( @@ -768,7 +763,7 @@ def create_app( ) except (TypeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "bad config values"}) - return True + return await send({"type": "config_ack", "config": config}) elif kind == "reset": session.reset() @@ -779,7 +774,6 @@ def create_app( await handle_inspect(message) else: await send({"type": "error", "code": "BAD_MESSAGE", "message": f"unknown type '{kind}'"}) - return True try: while True: @@ -792,9 +786,6 @@ def create_app( text = message.get("text") data = message.get("bytes") if data is not None: - if not authed: - await websocket.close(code=1008) - break if len(data) < 5: await send({"type": "error", "code": "BAD_FRAME", "message": "frame shorter than 5 bytes"}) continue @@ -811,8 +802,7 @@ def create_app( except (json.JSONDecodeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "invalid JSON"}) continue - if not await handle_control(parsed): - break + await handle_control(parsed) except WebSocketDisconnect: pass finally: @@ -826,7 +816,7 @@ def create_app( sessions.pop(session_id, None) # ---------------------------------------------------------------- legacy session routes - @app.post("/api/sessions", dependencies=[Depends(authorize)]) + @app.post("/api/sessions") async def legacy_start_session(payload: Dict[str, Any] = None) -> Dict[str, Any]: payload = payload or {} session = scans.start_session( @@ -836,7 +826,7 @@ def create_app( ) return {"status": "success", "session": session.snapshot()} - @app.post("/api/sessions/{session_id}/frames", dependencies=[Depends(authorize)]) + @app.post("/api/sessions/{session_id}/frames") async def legacy_observe_frame(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import FrameRequest, ObserveError @@ -844,19 +834,11 @@ def create_app( try: session = scans.get_session(session_id) raw = payload.get("frame_bytes_b64") or payload.get("image_base64") or "" - if not isinstance(raw, str): - raw = "" if "," in raw: raw = raw.split(",", 1)[1] - # Base64 expands ~4/3. Reject the text before decoding it. - if len(raw) > (MAX_FRAME_BYTES * 4) // 3 + 8: - raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) - frame_bytes = base64.b64decode(raw) if raw else b"" - if len(frame_bytes) > MAX_FRAME_BYTES: - raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) request = FrameRequest( frame_id=str(payload.get("frame_id") or ""), - frame_bytes=frame_bytes, + frame_bytes=base64.b64decode(raw) if raw else b"", sequence=int(payload.get("sequence", 0)), captured_at=float(payload.get("captured_at") or 0), session_id=session.session_id, @@ -868,13 +850,12 @@ def create_app( ) if not request.frame_id: raise _error("BAD_REQUEST", "frame_id required", 400) - loop = asyncio.get_running_loop() - observation = await loop.run_in_executor(None, scans.observe_frame, request) + observation = scans.observe_frame(request) return {"status": "success", "observation": observation.as_dict(), "session": session.snapshot()} except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.get("/api/sessions/{session_id}", dependencies=[Depends(authorize)]) + @app.get("/api/sessions/{session_id}") async def legacy_get_session(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -883,7 +864,7 @@ def create_app( except ObserveError as exc: raise _error("NOT_FOUND", str(exc), 404) - @app.post("/api/sessions/{session_id}/review", dependencies=[Depends(authorize)]) + @app.post("/api/sessions/{session_id}/review") async def legacy_review(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -900,7 +881,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/finalize", dependencies=[Depends(authorize)]) + @app.post("/api/sessions/{session_id}/finalize") async def legacy_finalize(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -909,7 +890,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/cancel", dependencies=[Depends(authorize)]) + @app.post("/api/sessions/{session_id}/cancel") async def legacy_cancel(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -918,7 +899,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/rescan", dependencies=[Depends(authorize)]) + @app.post("/api/sessions/{session_id}/rescan") async def legacy_rescan(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -927,7 +908,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/compare_live_vs_still", dependencies=[Depends(authorize)]) + @app.post("/api/compare_live_vs_still") async def legacy_compare(payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -969,11 +950,14 @@ def create_app( # ---------------------------------------------------------------- static web client resolved_web_dir = web_dir or os.path.join(current_dir, "web") - app.add_middleware(BodyLimit, max_bytes=MAX_UPLOAD_BYTES) if os.path.isdir(resolved_web_dir): app.mount("/", StaticFiles(directory=resolved_web_dir, html=True), name="web") - + @app.on_event("shutdown") + async def _shutdown() -> None: + for session in list(sessions.values()): + session.close() + engine.shutdown() return app @@ -1003,16 +987,16 @@ def main() -> None: parser.add_argument("--classify-workers", type=int, default=None) parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) - parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"], - help="which mechanism names products (default matcher)") parser.add_argument("--live-inflight", type=int, default=None, - help="classifications in flight per session") + help="classifications in flight per session (demo walks want a larger budget)") parser.add_argument("--live-attempts", type=int, default=None, help="classification attempts per track") parser.add_argument("--record-frames", type=str, default=None, help="write processed frames and their results under this directory") parser.add_argument("--record-every", type=int, default=None, help="record one frame in every N (default 3)") + parser.add_argument("--demo-walk", action="store_true", + help="serve the demo walk's pre-recognised shelf as live tracks") parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR") parser.add_argument("--max-sessions", type=int, default=None) parser.add_argument("--ssl-certfile", type=str, default=None) @@ -1036,6 +1020,8 @@ def main() -> None: server.live_method = args.live_method if args.live_conf is not None: server.live_conf = args.live_conf + if args.demo_walk: + server.demo_walk = True if args.no_tags: server.live_tags = False if args.ssl_certfile is not None: @@ -1045,18 +1031,11 @@ def main() -> None: if args.log_level is not None: server.log_level = args.log_level - load_local_token() - if not server.token: - server.token = os.environ.get("DORAYAKI_API_TOKEN") or None - require_token_for_bind(server.host, server.token) - engine_settings = settings_from_env() if args.record_frames is not None: server.record_frames = args.record_frames if args.record_every is not None: server.record_every = args.record_every - if args.identity is not None: - engine_settings.identity_backend = args.identity if args.live_inflight is not None: engine_settings.live_inflight = args.live_inflight if args.live_attempts is not None: @@ -1086,34 +1065,23 @@ def main() -> None: scheme = "https" if server.ssl_certfile else "http" LOGGER.info("Dorayaki recognition server on %s://%s:%d (token %s)", scheme, server.host, server.port, "required" if server.token else "off") - if is_loopback_host(server.host): + if server.host in ("127.0.0.1", "localhost"): LOGGER.info("Open %s://127.0.0.1:%d in a browser for the web client", scheme, server.port) else: LOGGER.info("LAN mode: live camera needs https (secure context); use --ssl-certfile or a tunnel") import uvicorn - advertised = advertise_host(server.host) - advertiser = ServiceAdvertiser(advertised, server.port) if advertised else None - if advertiser is not None and advertiser.start(): - LOGGER.info("Advertising _dorayaki._tcp on %s:%d", advertised, server.port) - elif advertised: - LOGGER.info("No dns-sd on this host; _dorayaki._tcp was not advertised") - - try: - uvicorn.run( - app, - host=server.host, - port=server.port, - log_level=(server.log_level or "info").lower(), - access_log=False, # request lines can carry Authorization; keep them off - ws="websockets-sansio", - ssl_certfile=server.ssl_certfile, - ssl_keyfile=server.ssl_keyfile, - ) - finally: - if advertiser is not None: - advertiser.close() + uvicorn.run( + app, + host=server.host, + port=server.port, + log_level=(server.log_level or "info").lower(), + access_log=False, # access log would print ?token=... query strings + ws="websockets-sansio", + ssl_certfile=server.ssl_certfile, + ssl_keyfile=server.ssl_keyfile, + ) if __name__ == "__main__": diff --git a/recognization/code/benchmark.py b/recognization/code/benchmark.py new file mode 100644 index 0000000..b54f6ca --- /dev/null +++ b/recognization/code/benchmark.py @@ -0,0 +1,103 @@ +""" +Benchmark Suite for Supermarket Product Classification +Compares SURF, ORB, BRISK, and Hybrid methods following +MDPI Electronics 2023, 12(17), 3640. +""" + +import os +import sys +import time +import cv2 +import numpy as np +from typing import Dict, List, Any + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from feature_matcher import ProductFeatureMatcher + + +def run_benchmark(stock_dir: str = "stock", num_runs: int = 3): + print("=" * 70) + print("RETAIL SHELF PRODUCT RECOGNITION - ALGORITHM BENCHMARK") + print("Methods Evaluated: SURF | ORB | BRISK | HYBRID (Paper Method)") + print("=" * 70) + + matcher = ProductFeatureMatcher(stock_dir=stock_dir) + templates = matcher.db.templates + + if not templates: + print("[Error] No templates found in stock directory.") + return + + methods = ["surf", "orb", "brisk", "hybrid"] + results = {m: {"latencies": [], "inliers": [], "keypoints_query": []} for m in methods} + + # Test images: We will evaluate using crops synthesized with slight transformations + # (scale, rotation, noise) as well as the template images themselves + print(f"\nEvaluating across {len(templates)} catalog SKUs (repeat count: {num_runs})...\n") + + for sku_id, tmpl in templates.items(): + original_img = tmpl["image"] + h, w = original_img.shape[:2] + + # Apply synthetic shelf condition (rotation by 10 deg, resize by 0.85) + M = cv2.getRotationMatrix2D((w / 2, h / 2), 10, 0.85) + query_img = cv2.warpAffine(original_img, M, (w, h)) + + for method in methods: + times = [] + for _ in range(num_runs): + t0 = time.perf_counter() + res = matcher.classify_crop(query_img, method=method) + t1 = time.perf_counter() + times.append((t1 - t0) * 1000.0) # ms + + avg_latency = np.mean(times) + results[method]["latencies"].append(avg_latency) + + # Record inliers/score + if res.get("status") == "success": + top_sku = res["best_sku"] + score = res["score"] + results[method]["inliers"].append(score) + + # Summarize metrics + summary_rows = [] + for method in methods: + latencies = results[method]["latencies"] + scores = results[method]["inliers"] + + avg_lat = np.mean(latencies) if latencies else 0.0 + p95_lat = np.percentile(latencies, 95) if latencies else 0.0 + avg_score = np.mean(scores) if scores else 0.0 + + # Paper characteristics notes + notes = { + "surf": "Scale & rotation robust; Hessian gradient features", + "orb": "Ultra fast binary descriptor; FAST + BRIEF", + "brisk": "Scale-space AGAST; circular pattern descriptor", + "hybrid": "Paper fusion (SURF+ORB+BRISK); optimal accuracy/speed" + }[method] + + summary_rows.append([ + method.upper(), + f"{avg_lat:.2f} ms", + f"{p95_lat:.2f} ms", + f"{avg_score:.1f}", + notes + ]) + + headers = ["Method", "Avg Latency (ms)", "P95 Latency (ms)", "Avg Match Score", "Characteristics / Paper Findings"] + print("\n" + "-" * 105) + print(f"{headers[0]:<10} | {headers[1]:<18} | {headers[2]:<18} | {headers[3]:<16} | {headers[4]}") + print("-" * 105) + for row in summary_rows: + print(f"{row[0]:<10} | {row[1]:<18} | {row[2]:<18} | {row[3]:<16} | {row[4]}") + print("-" * 105) + print("\n[Benchmark Complete] All methods operational.") + + +if __name__ == "__main__": + run_benchmark() diff --git a/recognization/code/build_store_from_shelf.py b/recognization/code/build_store_from_shelf.py deleted file mode 100644 index 589328d..0000000 --- a/recognization/code/build_store_from_shelf.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Enroll a shelf: the products read off its crops become the store's catalog. - -This is the onboarding flow a real store would go through once - detect the facings, -name the products, and keep their crops as the gallery that recognition matches -against. The two demo shelves were built with it; the product lists live in this file -because reading a contact sheet is a human step. - - python code/build_store_from_shelf.py -""" -import json, os, shutil, sys -sys.path.insert(0, "code") -import cv2 - -ROOT = os.path.expanduser("~/dorayaki-data/shelf-images") -manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) -entry = next(e for e in manifest["images"] if "Azabudai-Hills" in e["file"]) -image = cv2.imread(os.path.join(ROOT, entry["file"])) -height, width = image.shape[:2] -scale = min(1.0, 1600.0 / max(height, width)) -image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) -boxes = json.load(open("/tmp/dorayaki-verify/mediya-boxes.json"))["boxes"] - -# read off the numbered contact sheet; prices come from the shelf's own tags -PRODUCTS = [ - ("sku_m01", "小岩井 低脂肪牛乳", [9, 23, 31]), - ("sku_m02", "荻山酪農 カフェ・オ・レ", [10]), - ("sku_m03", "丹那 3.6牛乳", [11]), - ("sku_m04", "木次 ミルクコーヒー", [12]), - ("sku_m05", "木次 パスチャライズ牛乳", [13]), - ("sku_m06", "キスキ ブラウン・スイス牛乳", [14]), - ("sku_m07", "丹那 飲んでやさしいヨーグルト", [15]), - ("sku_m08", "農場3.7牛乳 特選", [24]), - ("sku_m09", "MINOR FIGURES OAT", [26]), - ("sku_m10", "北海道 低脂肪乳", [27]), - ("sku_m11", "タカナシ 低温殺菌牛乳", [28]), - ("sku_m12", "RUDE HEALTH Oat Organic", [29]), - ("sku_m13", "小岩井 無脂肪牛乳", [30]), - ("sku_m14", "きすき ノンホモ牛乳", [8]), -] -PRICES = {"sku_m01": 186, "sku_m02": 397, "sku_m03": 214, "sku_m04": 195, "sku_m05": 240, - "sku_m06": 254, "sku_m07": 225, "sku_m08": 216, "sku_m09": 300, "sku_m10": 214, - "sku_m11": 396, "sku_m12": 400, "sku_m13": 178, "sku_m14": 198} - -store = "stock-mediya" -shutil.rmtree(store, ignore_errors=True) -os.makedirs(store, exist_ok=True) -rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"] -written = [] -for sku, name, indices in PRODUCTS: - saved = 0 - for position, index in enumerate(indices): - box = boxes[index] - x1, y1, x2, y2 = [int(v) for v in box] - crop = image[max(0, y1):y2, max(0, x1):x2] - if crop.size == 0: - continue - suffix = "" if saved == 0 else f"_{saved}" - cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop) - saved += 1 - if saved == 0: - raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view") - written.append(sku) - rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}") -open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n") -print(f"{store}: {len(written)} products, csv written") diff --git a/recognization/code/collect_shelf_images.py b/recognization/code/collect_shelf_images.py deleted file mode 100644 index 56e20a2..0000000 --- a/recognization/code/collect_shelf_images.py +++ /dev/null @@ -1,190 +0,0 @@ -""" -Collect openly licensed store-shelf photographs for evaluation. - -Everything measured so far comes from two photographs. This pulls real store photos -from Wikimedia Commons - scriptable, licensed, and no authentication - so the -pipeline can be measured on images it has never seen. - -Only permissively licensed files are kept (CC0, CC BY, CC BY-SA, public domain), and -each one is written to a manifest with its author, licence and source URL, so an -attribution page can be produced from it. - - python code/collect_shelf_images.py [output_dir] -""" - -from __future__ import annotations - -import json -import os -import ssl -import sys -import time -import urllib.parse -import urllib.request - -import certifi - -CONTEXT = ssl.create_default_context(cafile=certifi.where()) -API = "https://commons.wikimedia.org/w/api.php" -USER_AGENT = "dorayaki-eval/0.1 (shelf recognition research; contact: repo owner)" -# Commons asks for a polite pace and offers thumbnails instead of originals. Both -# matter: a burst of original downloads got rate limited (HTTP 429) on the first run. -THUMBNAIL_WIDTH = 1600 -REQUEST_DELAY = 1.2 -MAX_RETRIES = 4 - -# Shelf photos of the categories the catalog covers: drinks and dairy, rice -# crackers and snacks, sauces and condiments. -QUERIES = [ - # the catalog's own products first: drinks, dairy, rice crackers - "野菜生活100", - "伊藤園 1日分の野菜", - "Japanese vegetable juice shelf", - "Japanese milk cartons supermarket", - "meiji oishii milk", - "せんべい 棚", - "Japanese rice cracker shop display", - "Japanese confectionery shelf", - "Japanese supermarket beverage aisle", - "Japanese convenience store refrigerator drinks", - "Japanese supermarket dairy case", - "スーパー 飲料 陳列", - "お菓子 売り場", - "Japanese snack aisle supermarket", - "Japanese supermarket shelves drinks bottles", - "Japanese supermarket interior shelves", -] - -PERMISSIVE = ("cc0", "cc-by", "cc by", "public domain", "pd-", "attribution") - - -def fetch(url: str, timeout: int = 60) -> bytes: - """GET with backoff: Commons answers 429 when a client is too eager.""" - for attempt in range(MAX_RETRIES): - try: - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request, timeout=timeout, context=CONTEXT) as response: - return response.read() - except urllib.error.HTTPError as error: - if error.code != 429 or attempt == MAX_RETRIES - 1: - raise - time.sleep(REQUEST_DELAY * (2 ** attempt)) - raise RuntimeError("unreachable") - - -def api(params: dict) -> dict: - url = API + "?" + urllib.parse.urlencode(params) - time.sleep(REQUEST_DELAY) - return json.loads(fetch(url, timeout=30).decode("utf-8")) - - -def search_files() -> dict: - """Candidate files, with how many queries found each one.""" - found: dict = {} - for query in QUERIES: - try: - result = api({"action": "query", "list": "search", "srsearch": query, - "srnamespace": "6", "srlimit": "25", "format": "json"}) - except Exception as error: - print(f" search failed for {query!r}: {error}") - continue - for hit in result.get("query", {}).get("search", []): - title = hit["title"] - if title.lower().endswith((".svg", ".pdf", ".ogv", ".webm")): - continue - found[title] = found.get(title, 0) + 1 - time.sleep(0.2) - return found - - -def image_info(titles: list) -> dict: - """URL, size and licence for each file, in batches of 50.""" - info: dict = {} - for start in range(0, len(titles), 50): - batch = titles[start:start + 50] - try: - result = api({"action": "query", "titles": "|".join(batch), "prop": "imageinfo", - "iiprop": "url|size|extmetadata", "iiurlwidth": str(THUMBNAIL_WIDTH), - "format": "json"}) - except Exception as error: - print(f" info failed for batch {start}: {error}") - continue - for page in (result.get("query", {}).get("pages") or {}).values(): - entries = page.get("imageinfo") or [] - if not entries: - continue - entry = entries[0] - meta = entry.get("extmetadata") or {} - info[page["title"]] = { - "url": entry.get("thumburl") or entry.get("url"), - "width": entry.get("width"), - "height": entry.get("height"), - "license": (meta.get("LicenseShortName") or {}).get("value", ""), - "artist": (meta.get("Artist") or {}).get("value", ""), - "description": (meta.get("ImageDescription") or {}).get("value", "")[:200], - "page": f"https://commons.wikimedia.org/wiki/{urllib.parse.quote(page['title'].replace(' ', '_'))}", - } - time.sleep(0.2) - return info - - -def main() -> None: - root = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/dorayaki-data/shelf-images") - os.makedirs(root, exist_ok=True) - - print("searching Commons...") - found = search_files() - print(f" {len(found)} distinct candidate files") - - info = image_info(sorted(found)) - print(f" metadata for {len(info)} files") - - kept, skipped = [], {"license": 0, "size": 0, "missing": 0} - for title, entry in sorted(info.items()): - if not entry.get("url"): - skipped["missing"] += 1 - continue - licence = (entry.get("license") or "").lower() - if not any(token in licence for token in PERMISSIVE): - skipped["license"] += 1 - continue - if (entry.get("width") or 0) < 1000 or (entry.get("height") or 0) < 700: - skipped["size"] += 1 - continue - kept.append((title, entry)) - - print(f" keeping {len(kept)} (skipped: {skipped})") - manifest = [] - for index, (title, entry) in enumerate(kept): - stem = os.path.basename(urllib.parse.urlparse(entry["url"]).path).replace(" ", "_") - stem = urllib.parse.unquote(stem)[:60] - name = f"{index:03d}_{stem}" - path = os.path.join(root, name) - if not os.path.exists(path): - try: - payload = fetch(entry["url"]) - with open(path, "wb") as handle: - handle.write(payload) - except Exception as error: - print(f" download failed for {title}: {error}") - continue - manifest.append({ - "file": name, - "title": title, - "width": entry["width"], - "height": entry["height"], - "license": entry["license"], - "artist": entry["artist"], - "source": entry["page"], - "queries": found.get(title, 0), - }) - print(f" [{index:>3}] {entry['width']}x{entry['height']} {entry['license']:<14} {title}") - - manifest_path = os.path.join(root, "manifest.json") - with open(manifest_path, "w", encoding="utf-8") as handle: - json.dump({"source": "Wikimedia Commons", "images": manifest}, handle, ensure_ascii=False, indent=1) - print(f"\n{len(manifest)} images in {root}\nmanifest: {manifest_path}") - - -if __name__ == "__main__": - main() diff --git a/recognization/code/demo.py b/recognization/code/demo.py new file mode 100644 index 0000000..8613b21 --- /dev/null +++ b/recognization/code/demo.py @@ -0,0 +1,254 @@ +""" +Synthetic camera for demos: a store employee walking a shelf. + +A simulator has no camera, and a still image makes a poor demo - nothing moves, so +nothing new is ever recognised. This module turns the demo shelf photos into a +virtual walk: a window pans left to right along a row, steps up to the next row, +and pans back the other way (serpentine, ascending), exactly the way a person +sweeps a shelf with a tablet. + +The client drives only `step`; all geometry lives here, so the web client, the +phone app, and the reference client see the same walk. `step` wraps, so an +unattended demo loops forever. + +Demo-only: this is a capture source, not a perception feature. `/v1/demo/*` is +documented as such. +""" + +from __future__ import annotations + +import math +import os +import threading +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Tuple + +DEFAULT_WINDOW = (960, 540) +DEFAULT_QUALITY = 65 +# How far the window advances per step, as a fraction of its own size. This is +# the demo's most important number: advance too fast and a product is in view for +# one or two frames, so the classifier never gets a chance and nothing is ever +# labelled (measured: 0.22 gave 9 labelled tracks in 20 s; the shelf swept past +# faster than a person could read it). 0.06 keeps a facing in view for roughly a +# second at demo frame rates - a deliberate, human sweep. +PAN_OVERLAP = 0.06 +ROW_OVERLAP = 0.25 +# Vertical steps spent moving between rows (the "look up, then keep going" beat). +TRANSITION_STEPS = 6 + + +class DemoSourceError(ValueError): + """Unknown sample or an impossible window.""" + + +@dataclass +class Bay: + """One shelf photo in the walk, with its geometry.""" + + sample_id: str + path: str + width: int + height: int + steps: int + + +class DemoCamera: + """Plans and renders the virtual walk over one or more shelf photos.""" + + def __init__(self, samples: Dict[str, Dict[str, str]], window: Tuple[int, int] = DEFAULT_WINDOW, + pan_overlap: float = PAN_OVERLAP): + self._samples = samples + self._window = window + # How far the window advances per step. The demo's pacing knob: a smaller + # value is a slower, more deliberate sweep (more steps, a facing in view + # for longer). Clamped so a client cannot ask for a frozen or teleporting + # camera. + self.pan_overlap = max(0.005, min(0.5, float(pan_overlap))) + self._lock = threading.Lock() + self._images: Dict[str, Any] = {} + self._sizes: Dict[str, Tuple[int, int]] = {} + self._plans: Dict[Tuple[str, ...], List[Bay]] = {} + + # ------------------------------------------------------------------ planning + def _size(self, sample_id: str) -> Tuple[int, int]: + cached = self._sizes.get(sample_id) + if cached is not None: + return cached + item = self._samples.get(sample_id) + if item is None or not os.path.exists(item["path"]): + raise DemoSourceError(f"unknown sample '{sample_id}'") + import cv2 + + image = cv2.imread(item["path"]) + if image is None: + raise DemoSourceError(f"cannot read sample '{sample_id}'") + with self._lock: + self._images[sample_id] = image + self._sizes[sample_id] = (int(image.shape[1]), int(image.shape[0])) + return self._sizes[sample_id] + + def _image(self, sample_id: str) -> Any: + if sample_id not in self._images: + self._size(sample_id) + return self._images[sample_id] + + def steps_for(self, sample_id: str, window: Tuple[int, int]) -> int: + width, height = self._size(sample_id) + win_w, win_h = window + bands = self._bands(height, win_h) + per_band = self._pan_steps(width, win_w) + # Every band pans, and every band after the first needs the step-up. + return bands * per_band + max(0, bands - 1) * TRANSITION_STEPS + + def _bands(self, image_height: int, window_height: int) -> int: + if window_height >= image_height: + return 1 + stride = max(1, int(window_height * (1.0 - ROW_OVERLAP))) + return max(1, int(math.ceil((image_height - window_height) / stride)) + 1) + + def _pan_steps(self, image_width: int, window_width: int) -> int: + if window_width >= image_width: + return 1 + stride = max(1, int(window_width * self.pan_overlap)) + return max(1, int(math.ceil((image_width - window_width) / stride)) + 1) + + def plan(self, sample_ids: Sequence[str], window: Optional[Tuple[int, int]] = None) -> Dict[str, Any]: + window = window or self._window + key = (tuple(sample_ids), window) + bays = self._plans.get(key) + if bays is None: + bays = [] + for sample_id in sample_ids: + width, height = self._size(sample_id) + bays.append( + Bay( + sample_id=sample_id, + path=self._samples[sample_id]["path"], + width=width, + height=height, + steps=self.steps_for(sample_id, window), + ) + ) + self._plans[key] = bays + total = sum(bay.steps for bay in bays) + return { + "samples": [bay.sample_id for bay in bays], + "window": {"width": window[0], "height": window[1]}, + "bays": [ + {"sample": bay.sample_id, "width": bay.width, "height": bay.height, + "steps": bay.steps, "first_step": sum(b.steps for b in bays[:index])} + for index, bay in enumerate(bays) + ], + "steps": total, + "row_overlap": ROW_OVERLAP, + "pan_overlap": self.pan_overlap, + } + + # ------------------------------------------------------------------ rendering + def window_at( + self, sample_ids: Sequence[str], step: int, window: Optional[Tuple[int, int]] = None + ) -> Dict[str, Any]: + """Where the virtual camera is at this step (wraps around the walk).""" + window = window or self._window + plan = self.plan(sample_ids, window) + total = plan["steps"] + step = int(step) % max(1, total) + offset = 0 + for index, bay in enumerate(plan["bays"]): + if step < offset + bay["steps"]: + local = step - offset + x, y = self._position(local, bay["width"], bay["height"], window) + return { + "sample": bay["sample"], + "bay_index": index, + "step": step, + "steps": total, + "local_step": local, + "bay_steps": bay["steps"], + "x": x, + "y": y, + "width": min(window[0], bay["width"]), + "height": min(window[1], bay["height"]), + } + offset += bay["steps"] + raise DemoSourceError("step outside the walk") + + def _position(self, local_step: int, image_width: int, image_height: int, window: Tuple[int, int]) -> Tuple[int, int]: + """Serpentine: pan a row, step up, pan back.""" + win_w = min(window[0], image_width) + win_h = min(window[1], image_height) + bands = self._bands(image_height, win_h) + per_band = self._pan_steps(image_width, win_w) + cycle = per_band + TRANSITION_STEPS + band = min(bands - 1, local_step // cycle) + within = local_step - band * cycle + stride_y = 0 if bands == 1 else max(1, int(win_h * (1.0 - ROW_OVERLAP))) + # Ascending: band 0 is the bottom of the shelf. + y_bottom = min(image_height - win_h, band * stride_y) + y = max(0, image_height - win_h - y_bottom) + stride_x = max(1, int(win_w * self.pan_overlap)) + max_x = max(0, image_width - win_w) + if within < per_band: + pan = within + x = min(max_x, pan * stride_x) + else: + # The step-up beat: hold x at the end of the row and climb. + transition = min(TRANSITION_STEPS, max(1, within - per_band + 1)) + pan = per_band - 1 + x = min(max_x, pan * stride_x) + next_band = min(bands - 1, band + 1) + next_y_bottom = min(image_height - win_h, next_band * stride_y) + ratio = transition / float(TRANSITION_STEPS) + y = int(round(y - (next_y_bottom - y_bottom) * ratio)) + if band % 2 == 1 and within < per_band: + # Odd bands pan back the other way, like a person reversing direction. + x = max(0, max_x - x) + return int(max(0, min(max_x, x))), int(max(0, min(image_height - win_h, y))) + + def frame( + self, + sample_ids: Sequence[str], + step: int, + window: Optional[Tuple[int, int]] = None, + quality: int = DEFAULT_QUALITY, + ) -> Tuple[bytes, Dict[str, Any]]: + """The JPEG frame the virtual camera sees at this step, plus its geometry.""" + import cv2 + + position = self.window_at(sample_ids, step, window) + image = self._image(position["sample"]) + x, y = position["x"], position["y"] + w, h = position["width"], position["height"] + crop = image[y:y + h, x:x + w] + if crop.size == 0: + raise DemoSourceError("empty crop") + ok, buffer = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, int(quality)]) + if not ok: + raise DemoSourceError("JPEG encode failed") + return bytes(buffer), position + + def warm(self, sample_ids: Sequence[str]) -> None: + """Decode the bays up front so the first demo frame is not the slow one.""" + for sample_id in sample_ids: + self._size(sample_id) + + +if __name__ == "__main__": # smoke: python code/demo.py [sample ...] + import sys + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from app import get_available_shelf_images + + samples = {item["id"]: item for item in get_available_shelf_images("add")} + camera = DemoCamera(samples) + ids = sys.argv[1:] or ["image6"] + plan = camera.plan(ids) + print(f"walk over {plan['samples']}: {plan['steps']} steps, window {plan['window']}") + started = time.perf_counter() + for step in range(min(plan["steps"], 24)): + payload, position = camera.frame(ids, step) + print(f" step {step:3d} {position['sample']} x={position['x']:4d} y={position['y']:4d} " + f"{len(payload) // 1024} KB") + elapsed = (time.perf_counter() - started) * 1000 / max(1, min(plan["steps"], 24)) + print(f"{elapsed:.1f} ms per frame") diff --git a/recognization/code/demo_replay.py b/recognization/code/demo_replay.py new file mode 100644 index 0000000..78ad9c6 --- /dev/null +++ b/recognization/code/demo_replay.py @@ -0,0 +1,162 @@ +""" +Demo-only: replay a shelf's recognition as if it were live tracking. + +`DemoRecognition` recognises the whole shelf once, with the real model and the real +matcher, and then hands those results to the walk's frames as if the live session +had produced them. That is useful for filming and for exercising a client without a +camera, and it is *not* a perception result: the boxes do not come from the frame +being processed, and no tracking or classification happened on it. Every payload it +touches is marked `synthetic: true` so a client can tell. + +It lives here rather than in `app.py` because it was committed inside an unrelated +feature once, which made a film-making device look like part of the server. +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Optional + +from demo import DemoSourceError + + +class DemoRecognition: + """The demo shortcut. + + A real walk recognises each facing as it appears, which on a shelf the + catalog only partly covers leaves most boxes unlabelled. For a demo the + shelf is recognised ONCE, in full, with the same model and matcher, and + those results are then handed to the walk's frames as if the live session + had just produced them - every box already green, with name and price. + Demo-only: enabled by --demo-walk. + """ + + _STILL_TO_LIVE = { + # The still payload already speaks the live vocabulary for its states; + # the decision-level names are kept for older shapes. + "recognized": "recognized", + "needs_confirmation": "needs_confirmation", + "unrecognized": "unrecognized", + "accepted": "recognized", + "ambiguous": "needs_confirmation", + "unknown": "unrecognized", + "rejected": "unrecognized", + } + + def __init__(self, engine: Any, camera: Any, samples: Dict[str, Any], tag_attacher: Any) -> None: + self._engine = engine + self._camera = camera + self._samples = samples + self._tag_attacher = tag_attacher + self._catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()} + self._cache: Dict[str, Any] = {} + self._lock = threading.Lock() + + def steps(self, sample_id: str) -> int: + return self._camera.plan([sample_id], (1280, 720)).get("steps", 0) + + def recognition(self, sample_id: str) -> Any: + cached = self._cache.get(sample_id) + if cached is not None: + return cached + import cv2 + + path = self._samples[sample_id]["path"] + image = cv2.imread(path) + if image is None: + raise DemoSourceError(f"cannot read sample '{sample_id}'") + result = self._engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) + if self._tag_attacher is not None: + result = self._tag_attacher(result, image) + with self._lock: + self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0])) + return self._cache[sample_id] + + def warm(self, sample_id: str) -> None: + self.recognition(sample_id) + + def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]: + """Swap the live tracks for the shelf's recognised ones, in frame space.""" + still, _, _ = self.recognition(sample_id) + position = self._camera.window_at([sample_id], step) + ox, oy = position["x"], position["y"] + win_w, win_h = position["width"], position["height"] + + tracks = [] + for index, detection in enumerate(still.get("detections") or []): + box = [int(v) for v in detection.get("box") or []] + if len(box) != 4: + continue + x1, y1, x2, y2 = box + if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: + continue + frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] + identity = detection.get("identity") or {} + state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized") + sku_id = identity.get("sku_id") + product = self._catalog_by_sku.get(sku_id or "") if sku_id else None + if state == "recognized" and product is None: + state = "unrecognized" + payload: Dict[str, Any] = { + "track_id": index + 1, + "box": frame_box, + "raw_box": frame_box, + "confidence": round(float(detection.get("confidence") or 0.0), 4), + "first_seen_seq": step, + "relabels": 0, + "restarts": 0, + "inherited": False, + "identity": { + "state": state, + "sku_id": sku_id if state == "recognized" else None, + "identity_level": identity.get("identity_level"), + "confidence": identity.get("confidence"), + "score": identity.get("score"), + "candidates": list(identity.get("candidates") or []), + "attempts": 1, + "method": "hybrid", + }, + } + if product is not None and state == "recognized": + payload["product"] = { + "sku_id": product.get("sku_id"), + "product_name": product.get("product_name"), + "price": product.get("price"), + } + tracks.append(payload) + + slots = [] + for slot in still.get("empty_slots") or []: + box = [int(v) for v in slot.get("box") or []] + if len(box) != 4: + continue + x1, y1, x2, y2 = box + if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: + continue + entry = dict(slot) + entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] + slots.append(entry) + + result["tracks"] = tracks + result["empty_slots"] = slots + counts: Dict[str, int] = {} + for track in tracks: + counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1 + result["summary"] = { + "in_view": len(tracks), + "recognized": counts.get("recognized", 0), + "needs_confirmation": counts.get("needs_confirmation", 0), + "unrecognized": counts.get("unrecognized", 0), + "detected": 0, + "unconfirmed": counts.get("unconfirmed", 0), + "identifying": 0, + "empty_slots": len(slots), + "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}), + "reconciliation": (still.get("reconciliation") or {}), + "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}), + } + # A client must be able to tell these boxes did not come from the frame it + # just sent. + result["synthetic"] = True + result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1} + return result diff --git a/recognization/code/download_stock.py b/recognization/code/download_stock.py new file mode 100644 index 0000000..889e2f5 --- /dev/null +++ b/recognization/code/download_stock.py @@ -0,0 +1,41 @@ +import os +import sys +import io +import urllib.request +from PIL import Image + +# Ensure stdout handles utf-8 cleanly +if sys.platform.startswith('win'): + sys.stdout.reconfigure(encoding='utf-8') + +urls = { + 'sku_011': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_apple@2x_v2.png', '野菜生活100 アップルサラダ'), + 'sku_012': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_lemon@2x_v2.png', '野菜生活100 レモンサラダ'), + 'sku_013': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/green-product-pic_2.png', '野菜生活100 Smoothie グリーンスムージー'), + 'sku_014': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/orange-product-pic_2.png', '野菜生活100 Smoothie ビタミンスムージー'), + 'sku_015': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/yellow-product-pic_2.png', '野菜生活100 Smoothie バナナ＆アーモンドmix'), + 'sku_016': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/red-product-pic_2.png', '野菜生活100 Smoothie アサイー＆ベリーmix') +} + +os.makedirs('stock', exist_ok=True) + +for sku, (url, name) in urls.items(): + print(f'Fetching {sku}: {name} from {url}...') + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + data = resp.read() + img = Image.open(io.BytesIO(data)) + if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info): + bg = Image.new('RGB', img.size, (255, 255, 255)) + if img.mode != 'RGBA': + img = img.convert('RGBA') + bg.paste(img, mask=img.split()[3]) + img = bg + else: + img = img.convert('RGB') + out_path = f'stock/{sku}.jpeg' + img.save(out_path, 'JPEG', quality=95) + print(f'Successfully saved {sku}: {name} -> {out_path} ({img.size})') + except Exception as e: + print(f'Error downloading {sku} ({url}): {e}') diff --git a/recognization/code/engine.py b/recognization/code/engine.py index f63e714..74117ee 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -21,7 +21,6 @@ from __future__ import annotations import base64 import collections -import logging import hashlib import json import os @@ -51,9 +50,6 @@ STATE_UNRECOGNIZED = "unrecognized" CLASSIFY_METHODS = ("surf", "orb", "brisk", "hybrid", "hybrid_fast") -LOGGER = logging.getLogger("dorayaki.engine") - - class UndecodableImage(ValueError): """Raised when an uploaded payload is not a readable image.""" @@ -85,16 +81,6 @@ class EngineSettings: # over a nearly static shelf wants more; that is a CLI override, not a default. live_inflight: int = 16 live_attempts: int = 2 - # Which mechanism names products: the hand-crafted local-feature matcher, or the - # embedding gallery with a calibrated rejection. Measured on 54 unseen store - # photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its - # raw inlier score cannot be thresholded; the gallery can say "unknown". - identity_backend: str = "matcher" - # Cosine similarity below which the gallery calls a facing unknown. Measured: - # same-product pairs sit above 0.6, different products below 0.5. - gallery_threshold: float = 0.55 - gallery_head_threshold: float = 0.40 - gallery_image_size: int = 518 imgsz: int = 640 # 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine). classify_workers: int = 5 @@ -248,10 +234,6 @@ class RecognitionEngine: self.matcher = matcher self.inventory = inventory self.settings = settings or EngineSettings() - # The embedding gallery is built on first use (it loads a model and every - # enrolled view), and cached as False when it cannot be built at all. - self._gallery: Any = None - self._gallery_lock = threading.Lock() workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2) self.classify_workers = int(workers) self._executor = classify_executor or ThreadPoolExecutor( @@ -412,28 +394,7 @@ class RecognitionEngine: return slots # ------------------------------------------------------------------ classification - @property - def identity_scale(self) -> str: - """What the scores from `score_crop` mean: inlier counts or cosine similarity.""" - return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier" - def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: - """Score one crop. Gallery never falls back to the matcher. - - A missing gallery returns no scores (unknown), not inlier counts. Mixing - those scales is how a live scan accepted a look-alike the still path rejected. - """ - if self.identity_scale == "cosine": - gallery = self.identity_gallery() - if not gallery: - return {} - match = gallery.match(crop) - if match.rejected or not match.sku_id: - return {} - scores = {match.sku_id: match.similarity} - if match.runner_up: - scores[match.runner_up] = match.runner_up_similarity - return {str(sku): float(score) for sku, score in scores.items()} method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) @@ -443,44 +404,6 @@ class RecognitionEngine: scores, _ = self.matcher.score_crop(crop, method=method) return {str(sku): float(score) for sku, score in scores.items()} - def identity_gallery(self) -> Any: - """The embedding gallery, loaded once (None when unavailable).""" - if self._gallery is False: - return None - if self._gallery is not None: - return self._gallery - with self._gallery_lock: - if self._gallery is False: - return None - if self._gallery is not None: - return self._gallery - from gallery import ProductGallery - - gallery = ProductGallery( - self.settings.stock_dir, - [product["sku_id"] for product in self.catalog()], - threshold=self.settings.gallery_threshold, - head_threshold=self.settings.gallery_head_threshold, - image_size=self.settings.gallery_image_size, - ) - try: - loaded = gallery.load() - except Exception as exc: # torch/timm missing, no weights, no network - LOGGER.warning("identity gallery unavailable: %s", exc) - loaded = False - # False means tried and failed. Do not retry and do not use the matcher. - self._gallery = gallery if loaded else False - if loaded: - LOGGER.info( - "identity gallery: %d views over %d SKUs on %s", - gallery.view_count(), - len(gallery.sku_ids), - gallery.device, - ) - else: - LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized") - return gallery if loaded else None - def classify_boxes( self, image: Any, @@ -504,11 +427,6 @@ class RecognitionEngine: ordered = sorted(boxes, key=area_of, reverse=True) crops = [self.crop(image, box["box"]) for box in ordered] - - if (self.settings.identity_backend or "matcher").lower() == "gallery": - gallery = self.identity_gallery() - # No matcher fallback: an unavailable gallery is unrecognized, not a different model. - return self._classify_with_gallery(gallery, ordered, crops) if method == "hybrid_fast": futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops] else: @@ -530,47 +448,26 @@ class RecognitionEngine: return results def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]: - """Full single-crop result, using the same backend as recognize/live.""" - from identity import decide_identity, decision_for_match - - crop = self.crop(image, box) - if self.identity_scale == "cosine": - gallery = self.identity_gallery() - match = gallery.match(crop) if gallery else None - # match() already applied the gallery's own threshold. Ranking again - # with the published cosine gates must not invent a second line. - decision = decision_for_match(match) - return self._inspect_payload(box, "gallery", decision, crop, visualize=False) + """Full single-crop result: decision, per-method details, crop, visualization.""" + from identity import decide_identity method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) + crop = self.crop(image, box) if method == "hybrid_fast": scores, details = self.matcher.score_crop_fast(crop) - visual_method = "hybrid" else: scores, details = self.matcher.score_crop(crop, method=method) - visual_method = method decision = decide_identity(scores, details=details, method=method) - return self._inspect_payload(box, method, decision, crop, visual_method=visual_method) - - def _inspect_payload( - self, - box: Sequence[int], - method: str, - decision: Dict[str, Any], - crop: Any, - *, - visual_method: Optional[str] = None, - visualize: bool = True, - ) -> Dict[str, Any]: identity = identity_payload(decision) + top_details = decision.get("details") or {} sub_details = None if isinstance(top_details, dict): raw = top_details.get("sub_details") if raw: - sub_details = {key: dict(value) for key, value in raw.items()} + sub_details = {k: dict(v) for k, v in raw.items()} product = None visualization = None @@ -582,16 +479,16 @@ class RecognitionEngine: elif identity["candidates"]: visualized_sku = identity["candidates"][0]["sku_id"] is_candidate = True - if visualize and visualized_sku and visual_method: + if visualized_sku: try: visualization = self.matcher.generate_match_visualization( - crop, visualized_sku, method=visual_method + crop, visualized_sku, method=method if method != "hybrid_fast" else "hybrid" ) except Exception: visualization = None return { - "box": [int(value) for value in box], + "box": [int(v) for v in box], "method": method, "identity": identity, "sub_details": sub_details, @@ -746,38 +643,6 @@ class RecognitionEngine: }, } - def _classify_with_gallery( - self, - gallery: Any, - ordered: Sequence[Dict[str, Any]], - crops: Sequence[Any], - ) -> List[Dict[str, Any]]: - """Identity by retrieval: nearest enrolled view, or an honest unknown. - - The gallery scores a SKU by its best view and rejects when nothing is close - enough, which is the behaviour the matcher cannot provide - measured on 54 - unseen store photos, the matcher labelled milk cartons as mayonnaise and its - score could not be thresholded to stop it. - """ - from identity import decision_for_match - - results: List[Dict[str, Any]] = [] - for box, crop in zip(ordered, crops): - match = None - if gallery is not None: - try: - match = gallery.match(crop) - except Exception as exc: # a bad crop must not fail the batch - match = None - LOGGER.debug("gallery match failed: %s", exc) - decision = decision_for_match(match) - results.append({ - "detection_id": int(box["id"]), - "box": list(box["box"]), - "identity": identity_payload(decision), - }) - return results - @staticmethod def summarize( detections: Sequence[Dict[str, Any]], @@ -877,7 +742,6 @@ def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings: live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)), live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)), live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)), - identity_backend=env.get("DORAYAKI_IDENTITY", base.identity_backend), detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)), ) diff --git a/recognization/code/eval_external_images.py b/recognization/code/eval_external_images.py deleted file mode 100644 index a555a85..0000000 --- a/recognization/code/eval_external_images.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Run the pipeline over store photos it has never seen. - - python code/eval_external_images.py [~/dorayaki-data/shelf-images] - -Reports detection and identity per image, and writes a JSON summary. See -docs/EXTERNAL-EVAL.md for what the numbers mean. -""" -import json, os, sys, time -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import cv2, numpy as np -from engine import build_default_engine, EngineSettings -from tags import TagService, TagSettings -from reconcile import attach_tags_to_boxes - -ROOT = os.path.expanduser(sys.argv[1] if len(sys.argv) > 1 else "~/dorayaki-data/shelf-images") -manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) - -engine = build_default_engine(EngineSettings(stock_dir="stock-lopia")) -catalog_by_sku = {p["sku_id"]: p for p in engine.catalog()} -service = TagService(engine.ocr_engine, settings=TagSettings()) - -rows = [] -for entry in manifest["images"]: - path = os.path.join(ROOT, entry["file"]) - image = cv2.imread(path) - if image is None: - continue - # keep the long side at 1600 so detection cost is comparable across images - height, width = image.shape[:2] - if max(height, width) > 1600: - scale = 1600.0 / max(height, width) - image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) - started = time.perf_counter() - result = engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) - detections = result["detections"] - recognized = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "recognized") - rejected = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "rejected") - rows.append({ - "file": entry["file"], "title": entry["title"], "license": entry["license"], - "size": f"{image.shape[1]}x{image.shape[0]}", - "detections": len(detections), "recognized": recognized, "rejected": rejected, - "ms": round((time.perf_counter()-started)*1000), - }) - print(f" {entry['file'][:34]:<34} {image.shape[1]}x{image.shape[0]:<5} " - f"det={len(detections):>4} rec={recognized:>3} rej={rejected:>3} {rows[-1]['ms']:>5} ms") - -total_det = sum(r["detections"] for r in rows) -total_rec = sum(r["recognized"] for r in rows) -with_det = sum(1 for r in rows if r["detections"] > 0) -print(f"\n{len(rows)} images, {total_det} detections, {total_rec} recognized " - f"({100*total_rec//max(1,total_det)}%), images with any detection: {with_det}") -json.dump(rows, open(os.path.join(ROOT, "eval.json"), "w"), ensure_ascii=False, indent=1) -print(f"summary: {os.path.join(ROOT, 'eval.json')}") diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py deleted file mode 100644 index d44a140..0000000 --- a/recognization/code/eval_live.py +++ /dev/null @@ -1,693 +0,0 @@ -"""Replay a recorded walk through LiveSession and count what it actually decided. - -The recording format is FrameRecorder's: DIR//index.jsonl plus the JPEG -bytes the client sent. Optional labels live in labels.json next to that index -(or at --labels). A label is {"frame", "box", "sku_id"}. - -Template leakage is refused, not scored. A label whose crop matches a catalog -template (byte-identical, or IoU >= 0.98 against a same-sized template pasted -into the frame) is excluded and counted under leakage. Those crops are how the -store was enrolled; scoring them is training-set accuracy. - - python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] - [--json PATH] [--iou 0.5] - -Prints a human count table and writes the same report as JSON (default -/eval_live.json). Missing input exits 2 with {"error": ...} on -stdout. The same pinned recording, catalog, weights, and labels produce the -same counts; timings are omitted because they are not. - -No held-out labels means denominators and fingerprints only. That is not a -walk accuracy number. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import sys -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -LEVELS = ("accepted", "ambiguous", "rejected", "unknown", "unsettled") -TAG_STATES = ("readable", "unreadable", "absent", "detected") - - -class EvalInputError(ValueError): - """The recording, labels, or catalog cannot be scored as given.""" - - -def file_sha256(path: str) -> Optional[str]: - digest = hashlib.sha256() - try: - with open(path, "rb") as handle: - for chunk in iter(lambda: handle.read(1 << 20), b""): - digest.update(chunk) - except OSError: - return None - return digest.hexdigest() - - -def content_sha256(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def _iou(a: Sequence[float], b: Sequence[float]) -> float: - if len(a) != 4 or len(b) != 4: - return 0.0 - ax1, ay1, ax2, ay2 = (float(v) for v in a) - bx1, by1, bx2, by2 = (float(v) for v in b) - ix1, iy1 = max(ax1, bx1), max(ay1, by1) - ix2, iy2 = min(ax2, bx2), min(ay2, by2) - inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) - if inter <= 0.0: - return 0.0 - area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1) - area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1) - union = area_a + area_b - inter - return inter / union if union > 0.0 else 0.0 - - -def load_index(recording_dir: str) -> List[Dict[str, Any]]: - path = os.path.join(recording_dir, "index.jsonl") - if not os.path.isfile(path): - raise EvalInputError(f"missing recording index: {path}") - rows: List[Dict[str, Any]] = [] - with open(path, "r", encoding="utf-8") as handle: - for number, line in enumerate(handle, start=1): - line = line.strip() - if not line: - continue - try: - entry = json.loads(line) - except json.JSONDecodeError as exc: - raise EvalInputError(f"{path}:{number}: {exc.msg}") from exc - if not isinstance(entry, dict) or "frame" not in entry or "path" not in entry: - raise EvalInputError(f"{path}:{number}: line needs frame and path") - rows.append(entry) - if not rows: - raise EvalInputError(f"empty recording index: {path}") - return rows - - -def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: - """Hash the JPEGs the client sent, in index order. Missing files are an error.""" - digest = hashlib.sha256() - missing: List[str] = [] - for row in rows: - name = str(row["path"]) - path = os.path.join(recording_dir, name) - payload_hash = file_sha256(path) - if payload_hash is None: - missing.append(name) - continue - digest.update(f"{row['frame']}:{name}:{payload_hash}\n".encode("utf-8")) - if missing: - raise EvalInputError("missing recorded frames: " + ", ".join(missing[:8])) - return {"sha256": digest.hexdigest(), "frames": len(rows)} - - -def load_labels(path: Optional[str]) -> List[Dict[str, Any]]: - if not path: - return [] - if not os.path.isfile(path): - raise EvalInputError(f"missing labels: {path}") - try: - with open(path, "r", encoding="utf-8") as handle: - payload = json.loads(handle.read()) - except json.JSONDecodeError as exc: - raise EvalInputError(f"labels are not JSON: {exc.msg}") from exc - rows = payload.get("labels") if isinstance(payload, dict) else payload - if not isinstance(rows, list): - raise EvalInputError("labels must be a list or {\"labels\": [...]}") - cleaned: List[Dict[str, Any]] = [] - for index, row in enumerate(rows): - if not isinstance(row, dict): - raise EvalInputError(f"labels[{index}] is not an object") - if "frame" not in row or "sku_id" not in row or "box" not in row: - raise EvalInputError(f"labels[{index}] needs frame, box, sku_id") - box = row["box"] - if not isinstance(box, (list, tuple)) or len(box) != 4: - raise EvalInputError(f"labels[{index}].box must be [x1, y1, x2, y2]") - cleaned.append( - { - "frame": int(row["frame"]), - "box": [int(v) for v in box], - "sku_id": str(row["sku_id"]), - "source": row.get("source"), - } - ) - return cleaned - - -def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[str, Any]: - """Same inputs as RecognitionEngine.catalog_version, plus the full digest and head.""" - digest = hashlib.sha256() - csv_path = os.path.join(stock_dir, stock_csv) - csv_hash = file_sha256(csv_path) - if csv_hash is None: - digest.update(b"missing-csv") - else: - digest.update(bytes.fromhex(csv_hash)) - try: - entries = sorted( - (name, os.path.getsize(os.path.join(stock_dir, name))) - for name in os.listdir(stock_dir) - if name.lower().endswith((".jpeg", ".jpg", ".png")) - ) - except OSError as exc: - raise EvalInputError(f"cannot read catalog {stock_dir}: {exc.strerror}") from exc - for name, size in entries: - digest.update(f"{name}:{size}".encode("utf-8")) - full = digest.hexdigest() - head_path = os.path.join(stock_dir, "identity_head.pt") - return { - "stock_dir": os.path.basename(os.path.abspath(stock_dir)), - "version": full[:12], - "sha256": full, - "templates": len(entries), - "csv_sha256": csv_hash, - "identity_head_sha256": file_sha256(head_path), - } - - -def model_fingerprint(weights_path: str) -> Dict[str, Any]: - try: - import ultralytics - - library = f"ultralytics-{ultralytics.__version__}" - except Exception: - library = "ultralytics-unknown" - return { - "version": f"{os.path.basename(weights_path)}@{library}", - "weights_sha256": file_sha256(weights_path) if weights_path else None, - } - - -def _decode_image(payload: bytes) -> Any: - import cv2 - import numpy as np - - image = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_COLOR) - if image is None: - raise EvalInputError("recorded frame is not a decodable image") - return image - - -def template_crops(stock_dir: str) -> List[Tuple[str, Any]]: - """Catalog photos keyed by SKU stem (sku_l05, sku_l05_1, ...).""" - import cv2 - - crops: List[Tuple[str, Any]] = [] - try: - names = sorted(os.listdir(stock_dir)) - except OSError: - return crops - for name in names: - stem, extension = os.path.splitext(name) - if extension.lower() not in (".jpeg", ".jpg", ".png"): - continue - image = cv2.imread(os.path.join(stock_dir, name)) - if image is None or image.size == 0: - continue - crops.append((stem, image)) - return crops - - -def _same_picture(crop: Any, template: Any) -> bool: - """True when two same-sized images are the same photo after JPEG.""" - import numpy as np - - left = crop.astype("float32").reshape(-1) - right = template.astype("float32").reshape(-1) - if left.size < 16 or left.size != right.size: - return False - left = left - float(left.mean()) - right = right - float(right.mean()) - denom = float(np.sqrt((left * left).sum() * (right * right).sum())) - if denom <= 1e-6: - return bool(np.max(np.abs(crop.astype("int16") - template.astype("int16"))) <= 2) - return float((left * right).sum() / denom) >= 0.98 - - -def leaked_labels( - recording_dir: str, - rows: Sequence[Dict[str, Any]], - labels: Sequence[Dict[str, Any]], - stock_dir: str, -) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: - """Drop labels whose crop is a catalog template. - - Byte identity catches a frame that is the template file. Geometric identity - catches a template pasted into a larger frame at the labelled box. - """ - if not labels: - return [], [] - templates = template_crops(stock_dir) - template_hashes = {} - try: - names = os.listdir(stock_dir) - except OSError: - names = [] - for name in names: - digest = file_sha256(os.path.join(stock_dir, name)) - if digest: - template_hashes.setdefault(digest, os.path.splitext(name)[0]) - by_frame: Dict[int, List[Dict[str, Any]]] = {} - for label in labels: - by_frame.setdefault(int(label["frame"]), []).append(label) - held: List[Dict[str, Any]] = [] - leaked: List[Dict[str, Any]] = [] - frames = {int(row["frame"]): row for row in rows} - for frame_id, frame_labels in by_frame.items(): - row = frames.get(frame_id) - image = None - payload_hash = None - if row is not None: - path = os.path.join(recording_dir, str(row["path"])) - try: - with open(path, "rb") as handle: - payload = handle.read() - except OSError: - payload = b"" - payload_hash = content_sha256(payload) if payload else None - if payload: - image = _decode_image(payload) - for label in frame_labels: - reason = None - matched = None - if payload_hash in template_hashes: - reason = "frame_is_template" - matched = template_hashes[payload_hash] - elif image is not None: - x1, y1, x2, y2 = label["box"] - crop = image[max(0, y1) : max(0, y2), max(0, x1) : max(0, x2)] - if crop.size: - import cv2 - - ch, cw = crop.shape[:2] - for stem, template in templates: - th, tw = template.shape[:2] - if abs(th - ch) > 1 or abs(tw - cw) > 1: - continue - resized = template if (th, tw) == (ch, cw) else cv2.resize(template, (cw, ch)) - # JPEG re-encoding keeps layout and drops exact pixels. - if _same_picture(crop, resized): - reason = "crop_is_template" - matched = stem - break - if reason: - leaked.append({**label, "leak": reason, "template": matched}) - else: - held.append(label) - return held, leaked - - -def identity_level(track: Dict[str, Any]) -> str: - identity = track.get("identity") or {} - level = identity.get("identity_level") - if level in LEVELS: - return str(level) - state = identity.get("state") - if state == "recognized": - return "accepted" - if state == "needs_confirmation": - return "ambiguous" - if state == "unrecognized": - return "rejected" - return "unsettled" - - -def tag_state(track: Dict[str, Any]) -> str: - tag = track.get("price_tag") or track.get("tag") or {} - state = tag.get("state") - if state in TAG_STATES: - return str(state) - if tag: - return "detected" - return "absent" - - -def score_tracks(tracks: Iterable[Dict[str, Any]]) -> Dict[str, Any]: - """Denominators over settled decisions. Unsettled tracks are not in the rate.""" - counts = {level: 0 for level in LEVELS} - tags = {state: 0 for state in TAG_STATES} - for track in tracks: - counts[identity_level(track)] += 1 - tags[tag_state(track)] += 1 - settled = counts["accepted"] + counts["ambiguous"] + counts["rejected"] + counts["unknown"] - return { - "tracks": sum(counts.values()), - "settled": settled, - "accepted": counts["accepted"], - "ambiguous": counts["ambiguous"], - "rejected": counts["rejected"], - "unknown": counts["unknown"], - "unsettled": counts["unsettled"], - "accepted_rate": _rate(counts["accepted"], settled), - "ambiguous_rate": _rate(counts["ambiguous"], settled), - "rejected_rate": _rate(counts["rejected"], settled), - "tags_detected": tags["detected"] + tags["readable"] + tags["unreadable"], - "tags_readable": tags["readable"], - "tags_unreadable": tags["unreadable"], - "tags_absent": tags["absent"], - } - - -def _rate(numerator: int, denominator: int) -> Optional[float]: - if denominator <= 0: - return None - return round(numerator / denominator, 4) - - -def track_churn(frames: Sequence[Dict[str, Any]]) -> Dict[str, Any]: - """How often one track id changes the SKU it is willing to name. - - A change counts only after the id has already named a SKU. Restarting or - dropping the label (relabels/restarts) is reported beside that, from the - counters LiveSession already publishes. - """ - previous: Dict[int, Optional[str]] = {} - changes: Dict[int, int] = {} - seen: Dict[int, int] = {} - relabels: Dict[int, int] = {} - restarts: Dict[int, int] = {} - for frame in frames: - for track in frame.get("tracks") or []: - track_id = track.get("track_id") - if track_id is None: - continue - track_id = int(track_id) - seen[track_id] = seen.get(track_id, 0) + 1 - relabels[track_id] = max(relabels.get(track_id, 0), int(track.get("relabels") or 0)) - restarts[track_id] = max(restarts.get(track_id, 0), int(track.get("restarts") or 0)) - named = None - if identity_level(track) == "accepted": - named = (track.get("identity") or {}).get("sku_id") - prior = previous.get(track_id) - if prior and named and named != prior: - changes[track_id] = changes.get(track_id, 0) + 1 - if named: - previous[track_id] = named - churned = sum(1 for count in changes.values() if count) - return { - "tracks": len(seen), - "tracks_with_sku_change": churned, - "sku_changes": sum(changes.values()), - "relabels": sum(relabels.values()), - "restarts": sum(restarts.values()), - "per_track": [ - { - "track_id": track_id, - "observations": seen[track_id], - "sku_changes": changes.get(track_id, 0), - "relabels": relabels.get(track_id, 0), - "restarts": restarts.get(track_id, 0), - } - for track_id in sorted(seen) - if changes.get(track_id, 0) or relabels.get(track_id, 0) or restarts.get(track_id, 0) - ], - } - - -def labelled_top1( - frames: Sequence[Dict[str, Any]], - labels: Sequence[Dict[str, Any]], - *, - iou_min: float, -) -> Dict[str, Any]: - """Top-1 over held-out labels. Accepted wrong SKU is a miss, not a reject. - - Denominator is labels that overlap a track. A label with no overlapping - track is unmatched and stays out of the top-1 rate. - """ - by_frame: Dict[int, List[Dict[str, Any]]] = {} - for frame in frames: - by_frame.setdefault(int(frame.get("frame", frame.get("frame_id"))), []).append(frame) - correct = wrong = rejected = ambiguous = missed = 0 - matched = 0 - for label in labels: - observations = by_frame.get(int(label["frame"]), []) - best_iou = 0.0 - best: Optional[Dict[str, Any]] = None - for observation in observations: - for track in observation.get("tracks") or []: - score = _iou(label["box"], track.get("box") or []) - if score > best_iou: - best_iou = score - best = track - if best is None or best_iou < iou_min: - missed += 1 - continue - matched += 1 - level = identity_level(best) - guess = (best.get("identity") or {}).get("sku_id") - if level == "accepted" and guess == label["sku_id"]: - correct += 1 - elif level == "accepted": - wrong += 1 - elif level == "ambiguous": - ambiguous += 1 - else: - rejected += 1 - return { - "labels": len(labels), - "matched": matched, - "unmatched": missed, - "correct": correct, - "wrong": wrong, - "rejected": rejected, - "ambiguous": ambiguous, - "top1": _rate(correct, matched), - "iou_min": iou_min, - } - - -def score_recording( - recording_dir: str, - *, - stock_dir: str, - weights_path: str = "", - labels_path: Optional[str] = None, - iou_min: float = 0.5, - live_frames: Optional[Sequence[Dict[str, Any]]] = None, -) -> Dict[str, Any]: - """Score one session directory. live_frames, when given, are the replayed results. - - Without live_frames the recorded index is what the server already decided. - That still produces denominators; it is not a fresh model run. - """ - recording_dir = os.path.abspath(recording_dir) - rows = load_index(recording_dir) - sample = recording_fingerprint(recording_dir, rows) - labels_file = labels_path or os.path.join(recording_dir, "labels.json") - labels = load_labels(labels_file if os.path.isfile(labels_file) or labels_path else None) - held, leaked = leaked_labels(recording_dir, rows, labels, stock_dir) - source = list(live_frames) if live_frames is not None else rows - tracks = [track for frame in source for track in (frame.get("tracks") or [])] - decisions = score_tracks(tracks) - report = { - "recording": recording_dir, - "source": "live_replay" if live_frames is not None else "recorded_index", - "fingerprints": { - "sample": sample, - "model": model_fingerprint(weights_path), - "catalog": catalog_fingerprint(stock_dir), - }, - "decisions": decisions, - "tags": { - "detected": decisions["tags_detected"], - "readable": decisions["tags_readable"], - "unreadable": decisions["tags_unreadable"], - "absent": decisions["tags_absent"], - }, - "churn": track_churn(source), - "leakage": {"excluded": len(leaked), "labels": leaked}, - "labelled_top1": labelled_top1(source, held, iou_min=iou_min), - "evidence": _evidence(labels, held, leaked, live_frames is not None), - } - return report - - -def _evidence( - labels: Sequence[Dict[str, Any]], - held: Sequence[Dict[str, Any]], - leaked: Sequence[Dict[str, Any]], - replayed: bool, -) -> List[str]: - gaps: List[str] = [] - if not replayed: - gaps.append("counts are from the recorded index, not a fresh LiveSession replay") - if not labels: - gaps.append("no held-out labels; labelled top-1 is not a walk accuracy") - elif not held: - gaps.append("every label matched a catalog template and was excluded") - if leaked: - gaps.append(f"{len(leaked)} template-derived label(s) excluded") - return gaps - - -def replay_live( - recording_dir: str, - engine: Any, - *, - settings: Optional[Any] = None, -) -> List[Dict[str, Any]]: - """Feed recorded JPEGs through one LiveSession, in index order. - - Classification is asynchronous. Each frame is scored only after its - in-flight crops finish, then processed again so the label is on the - payload. Without that wait the same recording does not replay twice. - """ - from live import LiveSession - - rows = load_index(recording_dir) - session = LiveSession("eval", engine, settings=settings) - session.refresh_catalog() - if hasattr(session, "enable_tags"): - session.enable_tags() - frames: List[Dict[str, Any]] = [] - for row in rows: - path = os.path.join(recording_dir, str(row["path"])) - with open(path, "rb") as handle: - payload = handle.read() - frame_id = int(row["frame"]) - result = session.process_frame(frame_id, payload) - # min_hits is 2, so the first pass only detects. Keep passing the same - # bytes until a pass schedules nothing and everything in flight lands. - for _ in range(4): - _wait_until_settled(session) - result = session.process_frame(frame_id, payload) - if int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) == 0: - break - result["frame"] = frame_id - frames.append(result) - engine_shutdown = getattr(engine, "shutdown", None) - if callable(engine_shutdown): - engine_shutdown() - return frames - - -def _wait_until_settled(session: Any, timeout: float = 30.0) -> None: - import time - - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - pending = int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) - if pending == 0: - return - time.sleep(0.02) - raise EvalInputError("classification did not finish before the replay timeout") - -def format_report(report: Dict[str, Any]) -> str: - decisions = report["decisions"] - tags = report["tags"] - top1 = report["labelled_top1"] - churn = report["churn"] - fingerprints = report["fingerprints"] - lines = [ - f"source: {report['source']}", - ( - f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " - f"rejected {decisions['rejected']} unknown {decisions['unknown']} " - f"unsettled {decisions['unsettled']} settled {decisions['settled']}" - ), - ( - f"rates: accepted {decisions['accepted_rate']} " - f"ambiguous {decisions['ambiguous_rate']} rejected {decisions['rejected_rate']} " - f"(denominator settled)" - ), - ( - f"tags: detected {tags['detected']} readable {tags['readable']} " - f"unreadable {tags['unreadable']} absent {tags['absent']}" - ), - ( - f"churn: tracks {churn['tracks']} sku_changes {churn['sku_changes']} " - f"relabels {churn['relabels']} restarts {churn['restarts']}" - ), - ( - f"labelled top-1: {top1['correct']}/{top1['matched']} = {top1['top1']} " - f"wrong {top1['wrong']} unmatched {top1['unmatched']} " - f"leakage excluded {report['leakage']['excluded']}" - ), - ( - f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} " - f"model {fingerprints['model']['version']} " - f"catalog {fingerprints['catalog']['version']}" - ), - ] - for gap in report["evidence"]: - lines.append(f"gap: {gap}") - return "\n".join(lines) - - -def main(argv: Optional[Sequence[str]] = None) -> int: - parser = argparse.ArgumentParser(description="Score a recorded live walk.") - parser.add_argument("recording", nargs="?", help="session directory with index.jsonl") - parser.add_argument("--stock-dir", default="stock") - parser.add_argument("--weights", default="") - parser.add_argument("--labels", default=None) - parser.add_argument("--json", dest="json_path", default=None) - parser.add_argument("--iou", type=float, default=0.5) - parser.add_argument( - "--replay", - action="store_true", - help="re-run frames through LiveSession (needs the recognition stack)", - ) - parser.add_argument("--method", default="hybrid") - args = parser.parse_args(argv) - - def fail(message: str) -> int: - print(json.dumps({"error": message}, ensure_ascii=False)) - print(f"error: {message}", file=sys.stderr) - return 2 - - if not args.recording: - return fail("recording directory is required") - recording = os.path.abspath(args.recording) - if not os.path.isdir(recording): - return fail(f"recording directory not found: {recording}") - try: - live_frames = None - weights = args.weights - if args.replay: - from engine import EngineSettings, build_default_engine - from live import LiveSettings - - settings = EngineSettings(stock_dir=args.stock_dir) - if weights: - settings.weights_path = weights - engine = build_default_engine(settings) - weights = engine.settings.weights_path - live_frames = replay_live( - recording, engine, settings=LiveSettings(method=args.method, tags=True) - ) - report = score_recording( - recording, - stock_dir=args.stock_dir, - weights_path=weights, - labels_path=args.labels, - iou_min=args.iou, - live_frames=live_frames, - ) - except EvalInputError as exc: - return fail(str(exc)) - text = json.dumps(report, ensure_ascii=False, indent=1, sort_keys=True) - destination = args.json_path or os.path.join(recording, "eval_live.json") - try: - with open(destination, "w", encoding="utf-8") as handle: - handle.write(text + "\n") - except OSError as exc: - return fail(f"cannot write {destination}: {exc.strerror}") - print(format_report(report)) - print(f"json: {destination}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/recognization/code/fast_test.py b/recognization/code/fast_test.py new file mode 100644 index 0000000..772ff4f --- /dev/null +++ b/recognization/code/fast_test.py @@ -0,0 +1,6 @@ +from ultralytics import YOLO + +model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") +image_path = "/Users/nguyenhongvan/recongnize/data/add/image7.jpg" +results = model(image_path) +results[0].show() diff --git a/recognization/code/finetune_identity.py b/recognization/code/finetune_identity.py deleted file mode 100644 index c147a1d..0000000 --- a/recognization/code/finetune_identity.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -Fine-tune the identity embedding on the labels we can actually produce. - -Measured: off-the-shelf DINOv2 tops out at ~25 % on a real milk shelf where the -products differ only by small text and a colour band, while it reaches 72-76 % on the -demo shelf, whose facings are near-identical crops of one photograph. That gap is the -whole problem, and the literature says the same thing (RP2K and RPC fine-tune; -GroceryVision reports a 17.5-point drop from Recall@5 to Recall@1). - -This trains a small projection head on frozen DINOv2 features and compares, on the -same held-out facings, three ways of naming a product: - - raw cosine the gallery as shipped - fine-tuned cosine the same matching, on projected features - fine-tuned + margin the same, trained with an angular margin - -**What this can and cannot show.** The labels come from one shelf, and the held-out -facings are other facings of the same photograph, so the number is an upper bound: it -measures whether a trained head separates these products at all, not whether it -survives a second visit. Augmentation (crop, scale, brightness, blur) stands in for -the visit we do not have; a cross-visit number needs cross-visit data. - - python code/finetune_identity.py [--epochs 60] [--views 2] -""" - -from __future__ import annotations - -import argparse -import os -import sys - -import cv2 -import numpy as np - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# Labelled by eye from a contact sheet of the detections (docs/EXTERNAL-EVAL.md). -MILK_SHELF_LABELS = { - "meiji_oishii": [13, 14, 15, 16, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], - "tokunou45": [9, 10, 11, 12], - "megmilk": [20, 21, 22, 23, 24, 25, 26, 37, 38, 46, 47, 54, 55, 56, 57, 58, 59, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 90, 94, 95, 96, 97, 98, 99], - "hokkaido": [49, 50, 51, 60, 61, 62, 63, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, - 100, 101, 102, 103, 104], - "teishibou": [2, 3, 4], - "meiji_love": [5, 6, 7, 8], - "oishii_tounyu": [42, 43, 44, 45], - "rakunou36": [17, 18], -} - - -def enrollment_report(stock: str, csv_name: str = "stock.csv") -> dict: - """Catalog versus enrolled views. Does not load a model or write files. - - A SKU with no readable view cannot be recognized. One view is reported, not - treated as enough: the measured knee is two views. Duplicate CSV rows and a - head whose class list disagrees with the catalog are blockers. - """ - import csv - import re - - from gallery import ProductGallery - - names = [] - try: - names = os.listdir(stock) - except OSError: - names = [] - images = [name for name in names if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")] - rows = [] - csv_path = os.path.join(stock, csv_name) - if os.path.exists(csv_path): - with open(csv_path, encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - sku_ids = [] - seen = set() - duplicate_rows = [] - for row in rows: - sku_id = (row.get("SKU_ID") or "").strip() - if not sku_id: - continue - if sku_id in seen: - duplicate_rows.append(sku_id) - seen.add(sku_id) - sku_ids.append(sku_id) - gallery = ProductGallery(stock, sku_ids or []) - views = {sku_id: gallery._views_for(sku_id) for sku_id in dict.fromkeys(sku_ids)} - missing = [sku_id for sku_id, paths in views.items() if not paths] - single = [sku_id for sku_id, paths in views.items() if len(paths) == 1] - claimed = {os.path.basename(path) for paths in views.values() for path in paths} - orphans = sorted(name for name in images if name not in claimed) - # ponytail: prefix collision check. A view named sku_001 must not also match sku_00. - collisions = [] - for sku_id, paths in views.items(): - bare = [os.path.basename(path) for path in paths if os.path.splitext(os.path.basename(path))[0] != sku_id - and not re.fullmatch(re.escape(sku_id) + r"[_-]\d+", os.path.splitext(os.path.basename(path))[0])] - if bare: - collisions.append(sku_id) - blockers = [] - if duplicate_rows: - blockers.append("duplicate_csv_sku") - if missing: - blockers.append("sku_without_view") - if collisions: - blockers.append("view_prefix_collision") - head_path = os.path.join(stock, "identity_head.pt") - head_note = None - if os.path.exists(head_path): - head_note = "present_not_opened" - # Do not torch.load here: the head can be large, and this report must stay cheap. - # Class-list drift is reported when the caller passes class_ids. - return { - "stock": stock, - "sku_ids": list(dict.fromkeys(sku_ids)), - "images": len(images), - "missing": missing, - "single_view": single, - "orphans": orphans, - "duplicate_rows": duplicate_rows, - "collisions": collisions, - "blockers": blockers, - "head": head_note, - } - - -def enrollment_report_text(report: dict) -> str: - return ( - f"{report['stock']}: {len(report['sku_ids'])} SKUs, {report['images']} images, " - f"missing={len(report['missing'])}, single_view={len(report['single_view'])}, " - f"orphans={len(report['orphans'])}, blockers={report['blockers'] or 'none'}" - ) - - -def build_model(image_size: int = 518): - import timm - import torch - - device = "mps" if torch.backends.mps.is_available() else "cpu" - model = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, - num_classes=0, img_size=image_size, dynamic_img_size=True) - return model.eval().to(device), device - - -def prepare(image, size: int = 518): - mean = np.array([0.485, 0.456, 0.406], np.float32) - std = np.array([0.229, 0.224, 0.225], np.float32) - rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - height, width = rgb.shape[:2] - side = max(height, width) - canvas = np.zeros((side, side, 3), np.uint8) - canvas[(side - height) // 2:(side - height) // 2 + height, - (side - width) // 2:(side - width) // 2 + width] = rgb - resized = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA).astype(np.float32) / 255.0 - return ((resized - mean) / std).transpose(2, 0, 1) - - -def embed(model, device, images, size: int = 518, batch: int = 16): - import torch - - out = [] - for start in range(0, len(images), batch): - tensor = torch.from_numpy(np.stack([prepare(image, size) for image in images[start:start + batch]])).to(device) - with torch.no_grad(): - features = model(tensor) - out.append(torch.nn.functional.normalize(features, dim=1).cpu().numpy()) - return np.concatenate(out, axis=0) - - -def augment(image, rng): - """Stand in for a second visit: crop, scale, brightness, contrast, blur.""" - height, width = image.shape[:2] - # crop 82-100 % of the box, keeping the product centred - keep = rng.uniform(0.82, 1.0) - ch, cw = int(height * keep), int(width * keep) - y = rng.randint(0, max(1, height - ch + 1)) - x = rng.randint(0, max(1, width - cw + 1)) - out = image[y:y + ch, x:x + cw] - # photometric - out = cv2.convertScaleAbs(out, alpha=float(rng.uniform(0.8, 1.2)), beta=float(rng.uniform(-18, 18))) - if rng.random() < 0.4: - out = cv2.GaussianBlur(out, (3, 3), 0) - # small rotation - if rng.random() < 0.6: - angle = float(rng.uniform(-5, 5)) - matrix = cv2.getRotationMatrix2D((out.shape[1] / 2, out.shape[0] / 2), angle, 1.0) - out = cv2.warpAffine(out, matrix, (out.shape[1], out.shape[0]), borderMode=cv2.BORDER_REPLICATE) - return out - - -def train_head(features, labels, class_ids, epochs, device, margin: float = 0.0, seed: int = 7): - """A cosine classifier on frozen features: small, and hard to overfit.""" - import torch - - torch.manual_seed(seed) - dimension = features.shape[1] - projection = torch.nn.Linear(dimension, 128).to(device) - prototypes = torch.nn.Parameter(torch.randn(len(class_ids), 128, device=device)) - torch.nn.init.xavier_uniform_(prototypes) - optimiser = torch.optim.AdamW(list(projection.parameters()) + [prototypes], lr=3e-3, weight_decay=1e-2) - - x = torch.from_numpy(features).float().to(device) - y = torch.tensor([class_ids.index(label) for label in labels], device=device) - for epoch in range(epochs): - projected = torch.nn.functional.normalize(projection(x), dim=1) - unit_prototypes = torch.nn.functional.normalize(prototypes, dim=1) - logits = projected @ unit_prototypes.T - if margin > 0: - # angular margin: pull each sample away from the other prototypes - one_hot = torch.nn.functional.one_hot(y, len(class_ids)).float() - logits = logits - margin * one_hot - loss = torch.nn.functional.cross_entropy(logits * 16.0, y, label_smoothing=0.05) - optimiser.zero_grad() - loss.backward() - optimiser.step() - return projection, torch.nn.functional.normalize(prototypes.detach(), dim=1) - - -def train_from_stock(args) -> None: - """Train the head on a store's own enrolled views, for the gallery to load. - - The head belongs to the store: the views in its stock directory are its products, - and a head trained on them is what turns the gallery from a lookup that cannot - tell two milk cartons apart into one that can. One to four views per SKU is - enough, with augmentation standing in for the rest. - """ - import glob - import json - - import torch - - from gallery import ProductGallery - - stock = args.stock - report = enrollment_report(stock) - print(enrollment_report_text(report)) - if report["blockers"]: - print(f"{stock}: not training; enrollment blockers: {', '.join(report['blockers'])}") - return - gallery = ProductGallery(stock, report["sku_ids"]) - views = {sku: gallery._views_for(sku) for sku in report["sku_ids"]} - views = {sku: paths for sku, paths in views.items() if paths} - if len(views) < 2: - print(f"{stock}: needs at least two products with views, found {len(views)}") - return - - crops, labels = [], [] - for sku, paths in sorted(views.items()): - for path in paths: - image = cv2.imread(path) - if image is not None and image.size: - crops.append(image) - labels.append(sku) - class_ids = sorted(views) - print(f"{stock}: {len(crops)} views over {len(class_ids)} products") - - model, device = build_model() - rng = np.random.RandomState(11) - augmented, augmented_labels = [], [] - for crop, label in zip(crops, labels): - augmented.append(crop) - augmented_labels.append(label) - for _ in range(args.augment): - augmented.append(augment(crop, rng)) - augmented_labels.append(label) - print(f"embedding {len(augmented)} crops on {device}...") - features = embed(model, device, augmented) - - projection, prototypes = train_head(features, augmented_labels, class_ids, - args.epochs, device, margin=0.25) - payload = { - "class_ids": class_ids, - "image_size": 518, - "base_model": "vit_small_patch14_dinov2.lvd142m", - "projection": {key: value.cpu() for key, value in projection.state_dict().items()}, - "prototypes": prototypes.cpu(), - "views": {sku: len(paths) for sku, paths in sorted(views.items())}, - } - path = os.path.join(stock, "identity_head.pt") - torch.save(payload, path) - print(f"wrote {path} ({len(class_ids)} products, {len(crops)} views)") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--epochs", type=int, default=60) - parser.add_argument("--views", type=int, default=2, help="enrolment views per class") - parser.add_argument("--augment", type=int, default=12, help="augmented copies per enrolment view") - parser.add_argument("--image", default=os.path.expanduser("~/dorayaki-data/shelf-images")) - parser.add_argument("--stock", default=None, - help="train on a store's enrolled views instead of the milk shelf, " - "and write /identity_head.pt for the gallery to load") - args = parser.parse_args() - if args.stock: - train_from_stock(args) - return - - import json - - from engine import build_default_engine, EngineSettings - - manifest = json.load(open(os.path.join(args.image, "manifest.json"))) - entry = next(e for e in manifest["images"] if "2011年4月の牛乳" in e["file"]) - image = cv2.imread(os.path.join(args.image, entry["file"])) - height, width = image.shape[:2] - scale = 1600.0 / max(height, width) - image = cv2.resize(image, (int(width * scale), int(height * scale)), interpolation=cv2.INTER_AREA) - - engine = build_default_engine(EngineSettings(stock_dir="stock")) - boxes = engine.detect(image, conf=0.1) - boxes.sort(key=lambda b: (b["box"][1] // 120, b["box"][0])) - crops = [engine.crop(image, box["box"]) for box in boxes] - - train_crops, train_labels, test_crops, test_labels = [], [], [], [] - for label, indices in MILK_SHELF_LABELS.items(): - for position, index in enumerate(indices): - if position < args.views: - train_crops.append(crops[index]) - train_labels.append(label) - else: - test_crops.append(crops[index]) - test_labels.append(label) - class_ids = sorted(MILK_SHELF_LABELS) - print(f"{len(train_crops)} enrolment views, {len(test_crops)} held-out facings, {len(class_ids)} products") - - model, device = build_model() - print(f"embedding {len(train_crops) * (args.augment + 1) + len(test_crops)} crops on {device}...") - - rng = np.random.RandomState(11) - augmented, augmented_labels = [], [] - for crop, label in zip(train_crops, train_labels): - augmented.append(crop) - augmented_labels.append(label) - for _ in range(args.augment): - augmented.append(augment(crop, rng)) - augmented_labels.append(label) - - train_features = embed(model, device, augmented) - test_features = embed(model, device, test_crops) - - def score(similarity, label, threshold=None): - """Top-1 over every held-out facing, with optional rejection. - - Balanced accuracy is reported alongside the raw rate because the shelf is - unbalanced (MEGMILK has 34 facings, 酪農3.6 has two), and a system that only - ever answers "MEGMILK" would look respectable on the raw number. - """ - per_class = {class_id: [0, 0] for class_id in class_ids} - correct = rejected = 0 - for index, truth in enumerate(test_labels): - best = int(similarity[index].argmax()) - if threshold is not None and similarity[index][best] < threshold: - rejected += 1 - per_class[truth][1] += 1 - continue - guess = class_ids[best] - per_class[truth][1] += 1 - if guess == truth: - correct += 1 - per_class[truth][0] += 1 - recalls = [found / total for found, total in per_class.values() if total] - balanced = sum(recalls) / max(1, len(recalls)) - answered = len(test_labels) - rejected - answered_accuracy = correct / max(1, answered) - print(f" {label:<26} top-1 {correct:>3}/{len(test_labels)} = {100*correct//len(test_labels):>3}% " - f"balanced {100*balanced:>3.0f}% answered {answered:>3} " - f"({100*answered_accuracy:>3.0f}% of those right) rejected {rejected}") - return per_class - - import torch - - def class_prototypes(features): - prototypes = [] - for class_id in class_ids: - members = [i for i, label in enumerate(train_labels) if label == class_id] - mean = features[members].mean(axis=0) - prototypes.append(mean / (np.linalg.norm(mean) + 1e-8)) - return np.stack(prototypes) - - # 1. raw features, matched the way the shipped gallery matches: best enrolment view - raw_prototypes = class_prototypes(train_features) - score(test_features @ raw_prototypes.T, "raw DINOv2 cosine") - score(test_features @ raw_prototypes.T, "raw, rejecting below 0.55", threshold=0.55) - - # 2. the trained classifier's prototypes, with an angular margin - projection, prototypes = train_head(train_features, augmented_labels, class_ids, - args.epochs, device, margin=0.25) - with torch.no_grad(): - projected_test = torch.nn.functional.normalize( - projection(torch.from_numpy(test_features).float().to(device)), dim=1).cpu().numpy() - similarity = projected_test @ prototypes.cpu().numpy().T - score(similarity, "fine-tuned + margin") - per_class = score(similarity, "fine-tuned, rejecting < 0.4", threshold=0.4) - print(" per class, answered only:") - for class_id in class_ids: - found, total = per_class[class_id] - if total: - print(f" {class_id:<16} {found:>2}/{total:<3} {100*found//total:>4}%") - - print("\n note: enrolment and held-out facings come from one photograph, so these are") - print(" upper bounds. A cross-visit number needs a second visit, not a second split.") - - -if __name__ == "__main__": - main() diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py index 0805d71..f01a255 100644 --- a/recognization/code/gallery.py +++ b/recognization/code/gallery.py @@ -26,7 +26,6 @@ on the GPU. That is why this can sit in the live loop at all. from __future__ import annotations import os -import re import threading from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -35,8 +34,6 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple # from the measured distribution: same-product pairs sit above 0.6, different # products below 0.5, and the gap is where a rejection belongs. DEFAULT_THRESHOLD = 0.55 -# The same line for a fine-tuned head, whose similarity scale is its own. -DEFAULT_HEAD_THRESHOLD = 0.40 @dataclass @@ -69,66 +66,19 @@ class ProductGallery: device: Optional[str] = None, image_size: int = 518, threshold: float = DEFAULT_THRESHOLD, - head_threshold: float = DEFAULT_HEAD_THRESHOLD, ) -> None: self.stock_dir = stock_dir self.sku_ids = list(sku_ids) self.image_size = int(image_size) self.threshold = float(threshold) - # A trained head produces a different similarity scale than raw features, so - # it needs its own line. Measured on a real milk shelf: the head answers 18 of - # 72 facings above 0.4 and is right on 94 % of them, while the raw threshold - # (0.55) rejects almost everything the head says. - self.head_threshold = float(head_threshold) self.device = device self._model = None self._lock = threading.Lock() self._entries: List[GalleryEntry] = [] self._matrix = None self._available = False - # A fine-tuned projection head, if the store has one (see - # finetune_identity.py). Measured on a real milk shelf: raw features put 1 % - # balanced accuracy on fine-grained products, a trained head 73 %. - self._head = None - self._head_path = os.path.join(stock_dir, "identity_head.pt") # ------------------------------------------------------------------ loading - def load_head(self) -> bool: - """Load the store's fine-tuned head, if it has one. - - weights_only: the file is tensors and strings, not executable code. - A payload that is not that head leaves the raw gallery in place. - """ - import torch - - if not os.path.exists(self._head_path): - return False - try: - payload = torch.load(self._head_path, map_location="cpu", weights_only=True) - if not isinstance(payload, dict): - return False - projection_state = payload.get("projection") - prototypes = payload.get("prototypes") - class_ids = payload.get("class_ids") - if not isinstance(projection_state, dict) or not torch.is_tensor(prototypes): - return False - if not isinstance(class_ids, (list, tuple)) or prototypes.ndim != 2: - return False - if prototypes.shape[0] != len(class_ids): - return False - projection = torch.nn.Linear(384, 128) - projection.load_state_dict(projection_state) - projection.eval() - self._head = { - "projection": projection, - "prototypes": torch.nn.functional.normalize(prototypes.float(), dim=1), - "class_ids": [str(sku_id) for sku_id in class_ids], - } - except Exception: - self._head = None - return False - return True - def _load_model(self) -> Any: if self._model is not None: return self._model @@ -154,12 +104,8 @@ class ProductGallery: return self._available def _views_for(self, sku_id: str) -> List[str]: - """Images whose stem is the SKU or SKU plus _N / -N. - - A bare prefix would enroll sku_001 as a view of sku_00. - """ + """Every image in the stock directory that belongs to this SKU.""" found: List[str] = [] - extra = re.compile(rf"{re.escape(sku_id)}[_-]\d+$") try: names = sorted(os.listdir(self.stock_dir)) except OSError: @@ -168,57 +114,48 @@ class ProductGallery: stem, extension = os.path.splitext(name) if extension.lower() not in (".jpeg", ".jpg", ".png"): continue - if stem == sku_id or extra.fullmatch(stem): + if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"): found.append(os.path.join(self.stock_dir, name)) return found def load(self) -> bool: - """Embed every enrolled view. Returns False when there is nothing to load. - - One lock covers the whole load: two callers cannot each embed the catalog. - """ - with self._lock: - if self._available and self._matrix is not None: - return True - return self._load_locked() - - def _load_locked(self) -> bool: + """Embed every enrolled view. Returns False when there is nothing to load.""" import cv2 import numpy as np import torch paths: List[Tuple[str, str]] = [] - kept: List[Any] = [] for sku_id in self.sku_ids: for path in self._views_for(sku_id): - image = cv2.imread(path) - if image is None or image.size == 0: - continue paths.append((sku_id, path)) - kept.append(_prepare(image, self.image_size)) - if not kept: + if not paths: self._available = False - self._entries = [] - self._matrix = None return False try: model = self._load_model() - tensor = torch.from_numpy(np.stack(kept)).to(self.device) - with torch.no_grad(): - features = model(tensor) - features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() except Exception: self._available = False - self._entries = [] - self._matrix = None return False + + batch = [] + for _, path in paths: + image = cv2.imread(path) + if image is None or image.size == 0: + continue + batch.append(_prepare(image, self.image_size)) + if not batch: + self._available = False + return False + tensor = torch.from_numpy(np.stack(batch)).to(self.device) + with torch.no_grad(): + features = model(tensor) + features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() self._entries = [ GalleryEntry(sku_id=sku_id, view=path, embedding=features[index]) for index, (sku_id, path) in enumerate(paths) ] self._matrix = features self._available = True - self.load_head() return True # ------------------------------------------------------------------ querying @@ -234,43 +171,13 @@ class ProductGallery: return features[0] def match(self, crop: Any, threshold: Optional[float] = None) -> GalleryMatch: - """Nearest enrolled product, with a rejection when nothing is close enough.""" + """Nearest enrolled view, with a rejection when nothing is close enough.""" if not self._available or self._matrix is None: return GalleryMatch(sku_id=None, similarity=0.0, rejected=True) import numpy as np - limit = threshold - if limit is None: - limit = self.head_threshold if self._head is not None else self.threshold - limit = float(limit) + limit = self.threshold if threshold is None else float(threshold) embedding = self.embed(crop) - if self._head is not None: - import torch - if self._head is not None: - # Two stages, because the two mechanisms fail in opposite directions. - # Measured: raw features reject strangers well (5 % false accepts) but - # rank fine-grained products badly (2 % top-1); a trained head ranks them - # well (73 % balanced) but is confidently wrong on strangers (44 % false - # accepts), because a discriminative head has no "none of the above". - # So the raw features decide *whether* to answer, and the head decides - # *what* to answer. - raw_similarity = self._matrix @ embedding - best_view = float(raw_similarity.max()) if len(raw_similarity) else 0.0 - if best_view < self.threshold: - return GalleryMatch(sku_id=None, similarity=best_view, rejected=True) - with torch.no_grad(): - projected = torch.nn.functional.normalize( - self._head["projection"](torch.from_numpy(embedding).float()), dim=0 - ) - similarity = (self._head["prototypes"] @ projected).numpy() - ranked = sorted(zip(self._head["class_ids"], similarity), key=lambda kv: kv[1], reverse=True) - top_sku, top_score = ranked[0] - runner_sku, runner_score = ranked[1] if len(ranked) > 1 else (None, 0.0) - if float(top_score) < limit: - return GalleryMatch(sku_id=None, similarity=float(top_score), runner_up=top_sku, - runner_up_similarity=float(top_score), rejected=True) - return GalleryMatch(sku_id=top_sku, similarity=float(top_score), runner_up=runner_sku, - runner_up_similarity=float(runner_score)) similarity = self._matrix @ embedding # Several views per SKU: a SKU's score is its best view. best_by_sku: Dict[str, float] = {} @@ -289,9 +196,6 @@ class ProductGallery: return GalleryMatch(sku_id=top_sku, similarity=top_score, runner_up=runner_sku, runner_up_similarity=runner_score) - def head_loaded(self) -> bool: - return self._head is not None - def view_count(self) -> int: return len(self._entries) diff --git a/recognization/code/gui_app.py b/recognization/code/gui_app.py new file mode 100644 index 0000000..df92251 --- /dev/null +++ b/recognization/code/gui_app.py @@ -0,0 +1,92 @@ +""" +Native Desktop GUI for Supermarket Shelf Product Recognition +Opens a desktop OpenCV window displaying image.jpg with bounding boxes only. +Users can click on any product directly on screen to inspect price, stock, +and feature matching inliers in real-time. +""" + +import os +import sys +import cv2 +import numpy as np + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from pipeline import ShelfRecognitionPipeline + + +def main(): + print("=" * 60) + print("STARTING NATIVE DESKTOP INTERFACE") + print("Image: image.jpg | Mode: Hybrid (Paper Method)") + print("Instructions: Click on ANY product to identify & view info.") + print("Press 'q' or 'ESC' on the window to exit.") + print("=" * 60) + + pipeline = ShelfRecognitionPipeline() + init_res = pipeline.set_active_shelf_image("image.jpg") + + if init_res.get("status") != "success": + print("[Error] Failed to load image.jpg") + return + + boxes = init_res["boxes"] + base_image = pipeline.current_image.copy() + + # Draw ONLY bounding boxes (no class names, no confidence labels) + display_image = pipeline.detector.draw_detections(base_image, boxes, draw_labels=False) + + window_name = "Smart Shelf - Click Any Product (Press Q to quit)" + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(window_name, 1280, 850) + + def on_mouse(event, x, y, flags, param): + if event == cv2.EVENT_LBUTTONDOWN: + print(f"\n[Click detected at coordinates: ({x}, {y})]") + res = pipeline.process_click(x, y, method="hybrid") + + if res.get("status") == "success": + prod = res["inventory"] + cls_info = res["classification"] + box_info = res["target_box"] + + print("-" * 55) + print(f"📦 Product Identified : {prod['product_name']}") + print(f"🏷️ SKU ID : {prod['sku_id']}") + print(f"💰 Price : ¥{prod['price']}") + print(f"📊 In Stock (On-Shelf): {prod['number_on']} units") + print(f"🚚 Received / Sold : {prod['number_in']} in / {prod['number_out']} out") + print(f"📅 Expiration Date : {prod['expiration_date']}") + print(f"🎯 Match Confidence : {cls_info['confidence'] * 100:.1f}% (Score: {cls_info['score']})") + print("-" * 55) + + # Update main window with selected box highlight (in red) + selected_display = pipeline.detector.draw_detections( + base_image, boxes, selected_box_id=box_info["id"], draw_labels=False + ) + cv2.imshow(window_name, selected_display) + + # Show visual match comparison window if available + if res.get("match_visualization") is not None: + match_win = "Feature Matcher (Shelf Crop vs Stock Template)" + cv2.namedWindow(match_win, cv2.WINDOW_NORMAL) + cv2.resizeWindow(match_win, 800, 400) + cv2.imshow(match_win, res["match_visualization"]) + else: + print(f"[Info] {res.get('message', 'No product found at coordinates.')}") + + cv2.setMouseCallback(window_name, on_mouse) + + while True: + cv2.imshow(window_name, display_image) + key = cv2.waitKey(20) & 0xFF + if key in [ord('q'), ord('Q'), 27]: + break + + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 6fab9c0..2514f91 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -25,128 +25,6 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float: return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4) -# Cosine scale (gallery). Same-product pairs were measured above 0.6 and -# different products below 0.5. These are not a new calibration. -COSINE_ACCEPT = 0.60 -COSINE_AMBIGUOUS = 0.50 -COSINE_MARGIN = 0.04 -VOTE_CAP = 3 - - -def _finite(value: Any) -> Optional[float]: - try: - number = float(value) - except (TypeError, ValueError): - return None - if number != number or number in (float("inf"), float("-inf")): - return None - return number - - -def aggregate_votes( - observations: Sequence[Dict[str, float]], - *, - cap: int = VOTE_CAP, -) -> Dict[str, float]: - """Mean of the newest observations, at most `cap`. - - Older frames are dropped, not averaged in. An empty observation still - counts, so a later miss can outvote an earlier weak hit. Inlier and - cosine scores must not be mixed in one list. - """ - window = list(observations)[-max(1, int(cap)):] - if not window: - return {} - totals: Dict[str, float] = {} - for scores in window: - for sku_id, score in (scores or {}).items(): - number = _finite(score) - if number is None: - continue - totals[str(sku_id)] = totals.get(str(sku_id), 0.0) + number - count = len(window) - return {sku_id: total / count for sku_id, total in totals.items()} - - -def _cosine_decision( - sorted_candidates: List[Tuple[str, float]], - *, - method: str, - details: Optional[Dict[str, Any]], - accept: float, - ambiguous: float, - margin_min: float, -) -> Dict[str, Any]: - top_sku, top_score = sorted_candidates[0] - runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0 - margin = top_score - runner_score - if top_score >= accept and margin >= margin_min: - level = IDENTITY_ACCEPTED - elif top_score >= ambiguous: - level = IDENTITY_AMBIGUOUS - else: - level = IDENTITY_REJECTED - return { - "status": level, - "method": method, - "best_sku": top_sku if level == IDENTITY_ACCEPTED else None, - "score": float(top_score), - "confidence": float(top_score), - "identity_level": level, - "ranked_candidates": [ - {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3] - ], - "details": details or {}, - "message": None, - } - - -def decision_for_match( - match: Any, - *, - method: str = "gallery", - margin: float = COSINE_MARGIN, -) -> Dict[str, Any]: - """Rank one gallery match. Rejection is the gallery's, not a second threshold. - - `match()` already applied the raw 0.55 line or the head's 0.40 line. Re-ranking - those scores against COSINE_ACCEPT (0.60) would refuse a head match the gallery - accepted. The only extra gate here is the published cosine margin. - """ - similarity = _finite(getattr(match, "similarity", 0.0)) or 0.0 - if match is None or getattr(match, "rejected", False) or not getattr(match, "sku_id", None): - return { - "status": IDENTITY_REJECTED, - "method": method, - "best_sku": None, - "score": similarity, - "confidence": similarity, - "identity_level": IDENTITY_REJECTED, - "ranked_candidates": [], - "details": {}, - "message": "no enrolled product is close enough", - } - runner = getattr(match, "runner_up", None) - runner_score = _finite(getattr(match, "runner_up_similarity", 0.0)) or 0.0 - gap = similarity - runner_score - close = bool(runner) and gap < margin - level = IDENTITY_AMBIGUOUS if close else IDENTITY_ACCEPTED - ranked = [{"sku_id": str(match.sku_id), "score": similarity}] - if runner: - ranked.append({"sku_id": str(runner), "score": runner_score}) - return { - "status": level, - "method": method, - "best_sku": None if close else str(match.sku_id), - "score": similarity, - "confidence": similarity, - "identity_level": level, - "ranked_candidates": ranked, - "details": {}, - "message": None, - } - - def decide_identity( scores: Dict[str, float], *, @@ -155,7 +33,6 @@ def decide_identity( min_confidence: float = DEFAULT_MIN_CONFIDENCE, details: Optional[Dict[str, Any]] = None, method: str = "hybrid", - scale: str = "inlier", ) -> Dict[str, Any]: """Rank catalog scores into accepted / ambiguous / unknown / rejected. @@ -163,20 +40,7 @@ def decide_identity( unknown. Two close scores are ambiguous. Only a unique, above-threshold score is accepted. """ - if scale == "cosine": - if min_score == DEFAULT_MIN_SCORE: - min_score = COSINE_ACCEPT - if min_margin == DEFAULT_MIN_MARGIN: - min_margin = COSINE_MARGIN - if min_confidence == DEFAULT_MIN_CONFIDENCE: - min_confidence = COSINE_AMBIGUOUS - cleaned: Dict[str, float] = {} - for sku_id, score in (scores or {}).items(): - number = _finite(score) - if number is None: - continue - cleaned[str(sku_id)] = number - if not cleaned: + if not scores: return { "status": IDENTITY_REJECTED, "method": method, @@ -190,17 +54,8 @@ def decide_identity( } sorted_candidates: List[Tuple[str, float]] = sorted( - cleaned.items(), key=lambda item: item[1], reverse=True + scores.items(), key=lambda x: x[1], reverse=True ) - if scale == "cosine": - return _cosine_decision( - sorted_candidates, - method=method, - details=details, - accept=min_score, - ambiguous=min_confidence, - margin_min=min_margin, - ) top_sku, top_score = sorted_candidates[0] runner_up_sku, runner_up_score = ( sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0) diff --git a/recognization/code/live.py b/recognization/code/live.py index 385f379..f8f4bed 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -96,9 +96,8 @@ class TrackState: last_attempt_area: int = 0 score_sums: Dict[str, float] = field(default_factory=dict) score_n: int = 0 - observations: List[Dict[str, float]] = field(default_factory=list) - candidates: List[Dict[str, Any]] = field(default_factory=list) identity: Optional[Dict[str, Any]] = None + candidates: List[Dict[str, Any]] = field(default_factory=list) confirmed: bool = False label_box: List[int] = field(default_factory=list) drift_frames: int = 0 @@ -185,10 +184,6 @@ class LiveSession: self._catalog_by_sku: Dict[str, Dict[str, Any]] = {} self._catalog_version = "" self._tags = None - # Shelf position of this frame's origin. Updated from a content shift - # before the session lock; tag cadence keys rows by it. Not a shelf map. - self._frame_origin = [0.0, 0.0] - self._shift_map = None self._compliance_tracker = None self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []} self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque( @@ -261,7 +256,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.observations = [] + track.score_n = 0 track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -291,8 +286,6 @@ class LiveSession: self._slots = [] self._latest_image = None self._latest_frame_id = None - self._frame_origin = [0.0, 0.0] - self._shift_map = None # ------------------------------------------------------------------ frame path def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]: @@ -307,9 +300,6 @@ class LiveSession: empty_enabled = self.settings.empty_slots generation = self._generation - - shelf_offset = self._note_frame_shift(image) - boxes = self.engine.detect(image, conf=conf) detected_at = time.perf_counter() tracks = self.tracker.update( @@ -337,13 +327,7 @@ class LiveSession: raw_box = list(detection["box"]) confidence = float(detection["confidence"]) if raw_box is None: - # Low-score second pass, or a Kalman box with no detection. - # Keep the last measured crop. Do not invent one from the prediction. - raw_box = list(state.raw_box) if state is not None and state.raw_box else [] - confidence = float(state.confidence) if state is not None else confidence - low_score = bool(getattr(track, "low_score", False)) or ( - track.det_index is None and raw_box == [] - ) + raw_box = list(track.box) previous_seq = self._last_seen_seq.get(track.track_id) if state is None: # An id that comes back after an absence may be a recycled id: @@ -385,12 +369,8 @@ class LiveSession: state.last_seq = seq self._last_seen_seq[track.track_id] = seq state.box = list(track.box) - if raw_box and not low_score: - state.raw_box = raw_box - state.confidence = confidence - elif not state.raw_box and raw_box: - state.raw_box = raw_box - state.confidence = confidence + state.raw_box = raw_box + state.confidence = confidence self._drop_stale_label(state) active[track.track_id] = state self._tracks = active @@ -425,7 +405,7 @@ class LiveSession: for state in self._tracks.values() if state.box ] - tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset) + tag_bands = self._tags.plan(tag_boxes, image.shape) # Collect the classification work under the lock; submit it after # releasing it. add_done_callback can run the callback inline when the @@ -463,44 +443,39 @@ class LiveSession: } return result - def _inherit_labels(self, seq: int) -> None: """Give a fresh track the label of a recently recognized product it covers. - Only a measured label, still overlapping its own box, may be copied. - An inherited label is not a measurement, so it cannot spread to a neighbour. - Blur, glare, and a too-far frame skip inheritance: the overlap is not - evidence when the frame itself failed the quality gate. + The same physical product can be tracked as several ids while the camera + pans. Remembering where a label was measured means each product is + classified once and its neighbours inherit the result instead of waiting + in the classification queue again. """ if not self._label_memory: return - if (self._last_quality or {}).get("hint"): - return for track in self._tracks.values(): if track.state != STATE_DETECTED or track.confirmed: continue for entry in reversed(self._label_memory): if seq - entry["seq"] > LABEL_MEMORY_FRAMES: break - if entry.get("inherited"): - continue - if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU: - continue - if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU: - continue - track.state = STATE_RECOGNIZED - track.confirmed = True - track.label_box = list(track.box) - track.identity = { - "sku_id": entry["sku_id"], - "confidence": entry["confidence"], - "score": entry["score"], - "attempts": 0, - "inherited": True, - } - track.relabels = self._relabel_count.get(track.track_id, 0) + 1 - self._relabel_count[track.track_id] = track.relabels - break + if _iou(track.box, entry["box"]) >= LABEL_MEMORY_IOU: + track.state = STATE_RECOGNIZED + track.confirmed = True + track.label_box = list(track.box) + track.identity = { + "sku_id": entry["sku_id"], + "confidence": entry["confidence"], + "score": entry["score"], + "attempts": 0, + "inherited": True, + } + # An inherited label is a guess from the same place, not a + # measurement of this object: count it as re-verified so a + # later correction is auditable rather than a silent flip. + track.relabels = self._relabel_count.get(track.track_id, 0) + 1 + self._relabel_count[track.track_id] = track.relabels + break def _remember_label(self, track: TrackState, box: List[int], seq: int) -> None: """Must be called with the session lock held.""" @@ -509,12 +484,10 @@ class LiveSession: self._label_memory.append( { "box": list(box), - "label_box": list(track.label_box or box), "sku_id": track.identity.get("sku_id"), "confidence": track.identity.get("confidence"), "score": track.identity.get("score"), "seq": seq, - "inherited": bool(track.identity.get("inherited")), } ) @@ -543,7 +516,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.observations = [] + track.score_n = 0 track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -566,16 +539,11 @@ class LiveSession: return [] eligible: List[TrackState] = [] - poor_frame = bool((self._last_quality or {}).get("hint")) for track in self._tracks.values(): if track.state != STATE_DETECTED or track.in_flight > 0: continue - if poor_frame: - continue if track.hits < self.settings.min_hits: continue - if len(track.raw_box) != 4: - continue x1, y1, x2, y2 = track.raw_box if min(x2 - x1, y2 - y1) < self.settings.min_crop_px: continue @@ -595,8 +563,9 @@ class LiveSession: eligible.append(track) # Newest first: while a camera pans, the products still worth labelling - # are the ones that just entered the frame. Area is only the tie-break. - eligible.sort(key=lambda track: (-track.first_seq, -track.area())) + # are the ones that just entered the frame. Sorting by area (the obvious + # default) spends the budget on products that are already leaving. + eligible.sort(key=lambda track: (track.first_seq, -track.area())) planned: List[Tuple[TrackState, Any, int]] = [] for track in eligible[:capacity]: crop = self.engine.crop(image, track.raw_box) @@ -654,18 +623,13 @@ class LiveSession: return track.in_flight = max(0, track.in_flight - 1) track.attempts += 1 - observation = { - str(sku_id): float(score) for sku_id, score in (scores or {}).items() - } - track.observations.append(observation) - track.observations = track.observations[-3:] - for sku_id, score in observation.items(): + for sku_id, score in (scores or {}).items(): track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score) - if observation: + if scores: track.score_n += 1 self._classify_times.append(finished_at - submitted_at) - if not any(track.observations): + if not scores: if track.attempts >= self.settings.max_attempts: track.state = STATE_UNRECOGNIZED track.confirmed = True @@ -674,12 +638,12 @@ class LiveSession: track.candidates = [] return - from identity import aggregate_votes, decide_identity + from identity import decide_identity - mean_scores = aggregate_votes(track.observations) - decision = decide_identity( - mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier") - ) + mean_scores = { + sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items() + } + decision = decide_identity(mean_scores, method=method) track.candidates = [ {"sku_id": row.get("sku_id"), "score": row.get("score")} for row in (decision.get("ranked_candidates") or [])[:3] @@ -872,7 +836,7 @@ class LiveSession: gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var()) glare = float(np.mean(np.all(small > self.settings.glare_pixel, axis=2))) - heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if len(state.raw_box) == 4] + heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if state.raw_box] median_height = float(np.median(heights)) if heights else 0.0 too_far = bool( heights @@ -881,8 +845,7 @@ class LiveSession: or sum(1 for h in heights if h < self.settings.min_crop_px) > len(heights) / 2 ) ) - # A black frame has Laplacian 0. That is not blur; a real shelf is not black. - blur = sharpness < self._blur_threshold() and float(gray.mean()) > 1.0 + blur = sharpness < self._blur_threshold() hint = None if glare > self.settings.glare_fraction: hint = "glare" @@ -933,24 +896,6 @@ class LiveSession: self._tags = TagService(ocr_engine, settings=settings or TagSettings()) return True - def _note_frame_shift(self, image: Any) -> Tuple[float, float]: - """Shelf position of this frame's origin. Phase correlation only. - - Runs before the session lock. A failed registration keeps the last origin - rather than inventing a jump. The content shift moves the origin the - other way, same sign as ShelfMap. - """ - if self._shift_map is None: - from shelf_map import ShelfMap - - self._shift_map = ShelfMap() - shift = self._shift_map.register(image) - with self._lock: - if shift is not None: - self._frame_origin[0] -= float(shift[0]) - self._frame_origin[1] -= float(shift[1]) - return (self._frame_origin[0], self._frame_origin[1]) - def _run_tag_passes(self, image: Any, bands: Sequence[Any]) -> None: """Submit tag passes to the engine's tag worker (never the frame loop).""" service = self._tags diff --git a/recognization/code/orb.py b/recognization/code/orb.py new file mode 100644 index 0000000..e69de29 diff --git a/recognization/code/shelf_map.py b/recognization/code/shelf_map.py deleted file mode 100644 index c644dfb..0000000 --- a/recognization/code/shelf_map.py +++ /dev/null @@ -1,348 +0,0 @@ -""" -The shelf as a plane, not a stream of boxes. - -Every measurement so far treats a frame as the world: a facing is "a track id", -and a track id dies the moment the camera pans past and comes back. Label -inheritance, drift checks and restart counters were each correct fixes for that, -and all of them are symptoms. The shelf does not move; the camera does. - -This module keeps a per-session **shelf map**. Each frame is registered to the map -by its content shift, every detection is placed in shelf coordinates, and a -detection that lands where a facing already is *merges* into it. Identity votes and -tag readings accumulate on the physical facing rather than on a transient track. - -That is what the current literature does for retail: RetailGlue (CVPR 2026W) builds -shelf panoramas by matching products across frames and reports 88.4 % stitching F1, -and the 7-Eleven Taiwan system (Sci. Rep. 2025, 7,000+ stores) stitches "virtual -shelves" before checking planograms. - -**What is measured, and what is not.** The synthetic walk registers to within -0.01 px of its ground truth - but it is a pure translation of a single photograph, -so that validates the harness, not handheld video. The number that matters (real -registration error, and the duplicate rate that follows from it) needs real -footage, which `record.py` exists to collect. -""" - -from __future__ import annotations - -import threading -import uuid -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -# A frame whose content shift cannot be estimated this confidently is treated as -# discontinuous: the map starts a new section rather than guessing a position. -MIN_REGISTRATION_RESPONSE = 0.20 -# Two detections are the same physical facing when their shelf boxes overlap this -# much. Measured: adjacent facings of one product overlap ~0 by construction. -MERGE_IOU = 0.45 - - -def _iou(a: Sequence[int], b: Sequence[int]) -> float: - x1, y1 = max(a[0], b[0]), max(a[1], b[1]) - x2, y2 = min(a[2], b[2]), min(a[3], b[3]) - intersection = max(0, x2 - x1) * max(0, y2 - y1) - area_a = max(0, a[2] - a[0]) * max(0, a[3] - a[1]) - area_b = max(0, b[2] - b[0]) * max(0, b[3] - b[1]) - return intersection / max(1.0, float(area_a + area_b - intersection)) - - -@dataclass -class Facing: - """One physical position on the shelf, with everything observed there.""" - - facing_id: str - shelf_box: List[int] - row: int = 0 - observations: int = 0 - first_seq: int = 0 - last_seq: int = 0 - identity_votes: Dict[str, float] = field(default_factory=dict) - tag_votes: Dict[str, float] = field(default_factory=dict) - tag_readings: List[Dict[str, Any]] = field(default_factory=list) - seen_track_ids: set = field(default_factory=set) - - def vote_identity(self, sku_id: Optional[str], weight: float = 1.0) -> None: - if sku_id: - self.identity_votes[sku_id] = self.identity_votes.get(sku_id, 0.0) + weight - - def vote_tag(self, sku_id: Optional[str], reading: Optional[Dict[str, Any]] = None) -> None: - if sku_id: - self.tag_votes[sku_id] = self.tag_votes.get(sku_id, 0.0) + 1.0 - if reading: - self.tag_readings.append(reading) - - def best_identity(self) -> Tuple[Optional[str], float]: - """The identity the facing's pooled evidence supports, and its agreement.""" - if not self.identity_votes: - return None, 0.0 - sku_id, weight = max(self.identity_votes.items(), key=lambda item: item[1]) - total = sum(self.identity_votes.values()) - return sku_id, weight / max(1e-6, total) - - def best_tag_identity(self) -> Tuple[Optional[str], float]: - if not self.tag_votes: - return None, 0.0 - sku_id, weight = max(self.tag_votes.items(), key=lambda item: item[1]) - total = sum(self.tag_votes.values()) - return sku_id, weight / max(1e-6, total) - - def as_payload(self) -> Dict[str, Any]: - identity, agreement = self.best_identity() - tag_identity, tag_agreement = self.best_tag_identity() - return { - "facing_id": self.facing_id, - "box": list(self.shelf_box), - "row": self.row, - "observations": self.observations, - "track_ids": len(self.seen_track_ids), - "sku_id": identity, - "identity_agreement": round(agreement, 3), - "tag_sku_id": tag_identity, - "tag_agreement": round(tag_agreement, 3), - "tag_readings": self.tag_readings[-3:], - } - - -class ShelfMap: - """Registers frames and accumulates facings in shelf coordinates.""" - - def __init__(self, settings: Optional[Any] = None) -> None: - self._lock = threading.Lock() - self._prev_gray: Optional[Any] = None - self._offset: List[float] = [0.0, 0.0] # shelf position of the frame origin - self._facings: List[Facing] = [] - self._orb = None - self.registrations = 0 - self.registration_failures = 0 - self.anchors = 0 - self.continuations = 0 - self.detections = 0 - self.merges = 0 - self.responses: List[float] = [] - - # ------------------------------------------------------------- registration - def _gray(self, frame: Any, scale: float = 0.5) -> Any: - import cv2 - - gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - if scale != 1.0: - gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA) - return gray - - def register(self, frame: Any) -> Optional[Tuple[float, float]]: - """Content shift from the previous frame, in full-resolution pixels. - - Phase correlation first: it is the right tool for a translating scene and - costs a couple of milliseconds. ORB + RANSAC only when the correlation is - weak (blur, a large jump, a scene with no texture), because that is the case - where a fast estimator quietly returns nonsense. - """ - import cv2 - import numpy as np - - gray = self._gray(frame) - with self._lock: - previous = self._prev_gray - self._prev_gray = gray - if previous is None or previous.shape != gray.shape: - return (0.0, 0.0) - window = cv2.createHanningWindow((gray.shape[1], gray.shape[0]), cv2.CV_32F) - (dx, dy), response = cv2.phaseCorrelate(previous.astype(np.float32), gray.astype(np.float32), window) - self.responses.append(float(response)) - if response >= MIN_REGISTRATION_RESPONSE: - self.registrations += 1 - return (dx / 0.5, dy / 0.5) # back to full resolution - - shift = self._orb_shift(previous, gray) - if shift is None: - self.registration_failures += 1 - return None - self.registrations += 1 - return (shift[0] / 0.5, shift[1] / 0.5) - - def _orb_shift(self, previous: Any, current: Any) -> Optional[Tuple[float, float]]: - import cv2 - import numpy as np - - if self._orb is None: - self._orb = cv2.ORB_create(nfeatures=1500, fastThreshold=10) - key_a, desc_a = self._orb.detectAndCompute(previous, None) - key_b, desc_b = self._orb.detectAndCompute(current, None) - if desc_a is None or desc_b is None or len(key_a) < 20 or len(key_b) < 20: - return None - matches = cv2.BFMatcher(cv2.NORM_HAMMING).knnMatch(desc_a, desc_b, k=2) - good = [m for m, n in matches if m.distance < 0.75 * n.distance] - if len(good) < 12: - return None - src = np.float32([key_a[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) - dst = np.float32([key_b[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) - matrix, _ = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=3.0) - if matrix is None: - return None - return (float(matrix[0, 2]), float(matrix[1, 2])) - - # ------------------------------------------------------------------ facings - def _anchor_to_map( - self, detections: Sequence[Sequence[int]], offset_x: float, offset_y: float - ) -> Tuple[float, float]: - """Correct the chained offset by matching detections against the map. - - Chaining pairwise shifts drifts: measured over a 79-step walk with *perfect* - pairwise registration, the accumulated x offset reached 2047 px on a 1500 px - shelf, the same facing landed in two places, and a shelf of ~60 facings - became 334 with 113 of them seen once. Drift is inherent to chaining - the - fix is to anchor the frame to the structure already on the map, which is - what the retail-stitching literature does with product correspondences. - - One ICP-style step is enough: for every detection that has a facing nearby, - the correction it implies is a vote, and the median vote wins. - """ - if not self._facings or len(detections) < 4: - return offset_x, offset_y - corrections = [] - search = 40.0 - for box in detections: - centre_x = (box[0] + box[2]) / 2.0 + offset_x - centre_y = (box[1] + box[3]) / 2.0 + offset_y - best, best_distance = None, search - for facing in self._facings: - facing_x = (facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 - facing_y = (facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 - distance = max(abs(facing_x - centre_x), abs(facing_y - centre_y)) - if distance < best_distance: - best, best_distance = facing, distance - if best is not None: - corrections.append( - ( - (best.shelf_box[0] + best.shelf_box[2]) / 2.0 - ((box[0] + box[2]) / 2.0 + offset_x), - (best.shelf_box[1] + best.shelf_box[3]) / 2.0 - ((box[1] + box[3]) / 2.0 + offset_y), - ) - ) - if len(corrections) < 4: - return offset_x, offset_y - dx = sorted(c[0] for c in corrections)[len(corrections) // 2] - dy = sorted(c[1] for c in corrections)[len(corrections) // 2] - self.anchors += 1 - return offset_x + dx, offset_y + dy - - def update(self, seq: int, frame: Any, tracks: Sequence[Dict[str, Any]]) -> None: - """Register the frame, then place or merge every detection on the shelf.""" - shift = self.register(frame) - boxes = [ - [int(v) for v in (track.get("box") or track.get("raw_box"))] - for track in tracks - if (track.get("box") or track.get("raw_box")) and len(track.get("box") or track.get("raw_box")) == 4 - ] - with self._lock: - if shift is None: - # A discontinuous frame: keep the last known offset rather than - # inventing one, so its detections land where the shelf is. - shift = (0.0, 0.0) - # `shift` is how far the CONTENT moved in the frame, so the frame's - # origin moves the opposite way along the shelf. Adding it instead - # (measured) walks the map backwards: the same facing lands 50 px away - # in the next frame and a shelf of ~60 facings becomes 857. - self._offset[0] -= shift[0] - self._offset[1] -= shift[1] - if self._facings: - self._offset[0], self._offset[1] = self._anchor_to_map( - boxes, self._offset[0], self._offset[1] - ) - offset_x, offset_y = self._offset - for track in tracks: - box = track.get("box") or track.get("raw_box") - if not box or len(box) != 4: - continue - self.detections += 1 - shelf_box = [ - int(round(box[0] + offset_x)), int(round(box[1] + offset_y)), - int(round(box[2] + offset_x)), int(round(box[3] + offset_y)), - ] - track_id = track.get("track_id") - facing = self._find(shelf_box, int(track_id) if track_id is not None else None) - if facing is None: - facing = Facing( - facing_id=f"face_{uuid.uuid4().hex[:8]}", - shelf_box=shelf_box, - first_seq=seq, - ) - self._facings.append(facing) - else: - self.merges += 1 - # Average the box: repeated observations of a static object - # should converge, and the average is what the eye expects. - for index in range(4): - facing.shelf_box[index] = int(round((facing.shelf_box[index] * facing.observations + shelf_box[index]) / (facing.observations + 1))) - facing.observations += 1 - facing.last_seq = seq - identity = track.get("identity") or {} - if identity.get("state") == "recognized": - facing.vote_identity(identity.get("sku_id"), float(identity.get("score") or 1.0)) - tag_identity = track.get("tag_identity") or {} - tag = track.get("price_tag") or {} - if tag_identity.get("sku_id") or tag.get("state") == "readable": - facing.vote_tag(tag_identity.get("sku_id"), tag if tag else None) - if track.get("track_id") is not None: - facing.seen_track_ids.add(int(track["track_id"])) - - def _find(self, shelf_box: Sequence[int], track_id: Optional[int] = None) -> Optional[Facing]: - """The facing a detection belongs to. - - Continuity first: if this track id was already placed nearby, it is the same - facing, and no overlap test is needed - the tracker already decided that. - Geometry only when the id is new (a pan that outran the tracker, or the - tracker renumbering). Measured: with geometry alone, box jitter split the - same facing often enough that the map held 210 facings where the tracker - held 156 ids - it was adding duplicates, not removing them. - """ - width = max(1, shelf_box[2] - shelf_box[0]) - height = max(1, shelf_box[3] - shelf_box[1]) - if track_id is not None: - radius = 0.75 * max(width, height) - best, best_distance = None, radius - for facing in self._facings: - if track_id not in facing.seen_track_ids: - continue - distance = max( - abs((facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 - (shelf_box[0] + shelf_box[2]) / 2.0), - abs((facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 - (shelf_box[1] + shelf_box[3]) / 2.0), - ) - if distance < best_distance: - best, best_distance = facing, distance - if best is not None: - self.continuations += 1 - return best - best, best_overlap = None, MERGE_IOU - for facing in self._facings: - overlap = _iou(shelf_box, facing.shelf_box) - if overlap > best_overlap: - best, best_overlap = facing, overlap - return best - - # ------------------------------------------------------------------ reporting - def facings(self) -> List[Facing]: - with self._lock: - return list(self._facings) - - def stats(self) -> Dict[str, Any]: - with self._lock: - facings = list(self._facings) - responses = list(self.responses[-200:]) - consistent = 0 - for facing in facings: - sku_id, agreement = facing.best_identity() - if sku_id and agreement >= 0.8: - consistent += 1 - return { - "facings": len(facings), - "detections": self.detections, - "merges": self.merges, - "continuations": self.continuations, - "registrations": self.registrations, - "anchored": self.anchors, - "registration_failures": self.registration_failures, - "median_response": round(sorted(responses)[len(responses) // 2], 3) if responses else None, - "identity_consistent": consistent, - "offset": [round(value, 1) for value in self._offset], - } diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index 8e4b6ae..67d0dd6 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -124,7 +124,8 @@ def main() -> None: args = parser.parse_args() url = args.url - headers = {"Authorization": f"Bearer {args.token}"} if args.token else None + if args.token and "token=" not in url: + url = f"{url}{'&' if '?' in url else '?'}token={args.token}" source = FrameSource(args.source, width=args.width, pan_step=args.pan) rtts: List[float] = [] @@ -146,7 +147,7 @@ def main() -> None: in_flight: Dict[int, float] = {} pending_frame: Optional[int] = None - with connect(url, max_size=8 * 1024 * 1024, additional_headers=headers) as socket: + with connect(url, max_size=8 * 1024 * 1024) as socket: hello = json.loads(socket.recv(timeout=10)) if hello.get("type") != "hello": raise SystemExit(f"unexpected first message: {hello}") diff --git a/recognization/code/suft.py b/recognization/code/suft.py new file mode 100644 index 0000000..7e78d6d --- /dev/null +++ b/recognization/code/suft.py @@ -0,0 +1,84 @@ +import cv2 +import matplotlib.pyplot as plt +import numpy as np + +def suft_function(image): + image1 = cv2.imread(image) + # Convert the training image to RGB + training_image = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB) + + # Convert the training image to gray scale + training_gray = cv2.cvtColor(training_image, cv2.COLOR_RGB2GRAY) + + # Create test image by adding Scale Invariance and Rotational Invariance + test_image = cv2.pyrDown(training_image) + test_image = cv2.pyrDown(test_image) + num_rows, num_cols = test_image.shape[:2] + + rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), 30, 1) + test_image = cv2.warpAffine(test_image, rotation_matrix, (num_cols, num_rows)) + + test_gray = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY) + + # Display traning image and testing image + fx, plots = plt.subplots(1, 2, figsize=(20,10)) + + plots[0].set_title("Training Image") + plots[0].imshow(training_image) + + plots[1].set_title("Testing Image") + plots[1].imshow(test_image) + + + # ## Detect keypoints and Create Descriptor + + surf = cv2.xfeatures2d.SURF_create(800) + + train_keypoints, train_descriptor = surf.detectAndCompute(training_gray, None) + test_keypoints, test_descriptor = surf.detectAndCompute(test_gray, None) + + keypoints_without_size = np.copy(training_image) + keypoints_with_size = np.copy(training_image) + + cv2.drawKeypoints(training_image, train_keypoints, keypoints_without_size, color = (0, 255, 0)) + + cv2.drawKeypoints(training_image, train_keypoints, keypoints_with_size, flags = cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) + + # Display image with and without keypoints size + fx, plots = plt.subplots(1, 2, figsize=(20,10)) + + plots[0].set_title("Train keypoints With Size") + plots[0].imshow(keypoints_with_size, cmap='gray') + + plots[1].set_title("Train keypoints Without Size") + plots[1].imshow(keypoints_without_size, cmap='gray') + + # Print the number of keypoints detected in the training image + print("Number of Keypoints Detected In The Training Image: ", len(train_keypoints)) + + # Print the number of keypoints detected in the query image + print("Number of Keypoints Detected In The Query Image: ", len(test_keypoints)) + + + # ## Matching Keypoints + + # Create a Brute Force Matcher object. + bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck = False) + + # Perform the matching between the SURF descriptors of the training image and the test image + matches = bf.match(train_descriptor, test_descriptor) + + # The matches with shorter distance are the ones we want. + matches = sorted(matches, key = lambda x : x.distance) + + result = cv2.drawMatches(training_image, train_keypoints, test_gray, test_keypoints, matches, test_gray, flags = 2) + + # Display the best matching points + plt.rcParams['figure.figsize'] = [14.0, 7.0] + plt.title('Best Matching Points') + plt.imshow(result) + plt.show() + + # Print total number of matching points between the training and query images + print("\nNumber of Matching Keypoints Between The Training and Query Images: ", len(matches)) + return keypoints_with_size \ No newline at end of file diff --git a/recognization/code/tag_cards.py b/recognization/code/tag_cards.py index 20f55c2..f0a8a2a 100644 --- a/recognization/code/tag_cards.py +++ b/recognization/code/tag_cards.py @@ -29,116 +29,19 @@ from the small one and marked as derived rather than measured. from __future__ import annotations -import json import math -import os import re from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple -# Defaults when a store has no store.json. Measured on the demo LOPIA tags: -# 8% consumption tax, brick-red rails, and shelf prices from ¥50. A store file -# replaces these for that store only. -DEFAULT_TAX_RATIOS = (1.08, 1.10) -DEFAULT_MIN_PRICE = 50 -# OpenCV hue is 0-180. Brick red wraps past 0, so a pixel is rail-coloured when -# its hue is below hue_low or above hue_high. -DEFAULT_RAIL_HSV = {"hue_low": 14, "hue_high": 166, "sat_min": 70, "val_min": 70} -# Aliases the parser reads. load_store_policy() republishes them per store. -TAX_RATIOS = DEFAULT_TAX_RATIOS -MIN_PRICE = DEFAULT_MIN_PRICE - -_RAIL: Optional["StorePolicy"] = None - - -def _remember_rail(policy: "StorePolicy") -> None: - global _RAIL - _RAIL = policy - - - -class StorePolicyError(ValueError): - """store.json exists but cannot be used. Never borrow another store's file.""" - - -@dataclass(frozen=True) -class StorePolicy: - tax_ratios: Tuple[float, ...] = DEFAULT_TAX_RATIOS - min_price: int = DEFAULT_MIN_PRICE - rail_hue_low: int = 14 - rail_hue_high: int = 166 - rail_sat_min: int = 70 - rail_val_min: int = 70 - store_id: Optional[str] = None - - def active(self) -> None: - global TAX_RATIOS, MIN_PRICE - TAX_RATIOS = self.tax_ratios - MIN_PRICE = self.min_price - _remember_rail(self) - -def default_policy() -> StorePolicy: - policy = StorePolicy() - policy.active() - return policy - - -def _as_ratio(value: Any) -> float: - number = float(value) - if not 1.0 < number <= 1.5: - raise StorePolicyError(f"tax ratio {value!r} is outside 1-1.5") - return number - - -def _as_bound(value: Any, name: str, low: int, high: int) -> int: - number = int(value) - if not low <= number <= high: - raise StorePolicyError(f"{name} {value!r} is outside {low}-{high}") - return number - - -def load_store_policy(stock_dir: Optional[str]) -> StorePolicy: - """Read `/store.json`. A missing file uses the defaults. - - A file that is present but invalid raises StorePolicyError naming that path. - """ - if not stock_dir: - return StorePolicy() - path = os.path.join(stock_dir, "store.json") - if not os.path.isfile(path): - return StorePolicy() - try: - with open(path, encoding="utf-8") as handle: - raw = json.load(handle) - except (OSError, json.JSONDecodeError) as exc: - raise StorePolicyError(f"{path}: {exc}") from exc - if not isinstance(raw, dict): - raise StorePolicyError(f"{path}: expected an object") - try: - ratios = raw.get("tax_ratios", list(DEFAULT_TAX_RATIOS)) - if not isinstance(ratios, list) or not ratios: - raise StorePolicyError(f"{path}: tax_ratios must be a non-empty list") - rail = raw.get("rail_hsv", DEFAULT_RAIL_HSV) - if not isinstance(rail, dict): - raise StorePolicyError(f"{path}: rail_hsv must be an object") - policy = StorePolicy( - tax_ratios=tuple(_as_ratio(item) for item in ratios), - min_price=_as_bound(raw.get("min_price", DEFAULT_MIN_PRICE), "min_price", 1, 100000), - rail_hue_low=_as_bound(rail.get("hue_low", 14), "hue_low", 0, 180), - rail_hue_high=_as_bound(rail.get("hue_high", 166), "hue_high", 0, 180), - rail_sat_min=_as_bound(rail.get("sat_min", 70), "sat_min", 0, 255), - rail_val_min=_as_bound(rail.get("val_min", 70), "val_min", 0, 255), - store_id=str(raw["store_id"]) if raw.get("store_id") else None, - ) - except (TypeError, ValueError) as exc: - if isinstance(exc, StorePolicyError): - raise - raise StorePolicyError(f"{path}: {exc}") from exc - policy.active() - return policy +# The consumption tax used by the tags in the demo images. A tag whose small +# price is exactly 1.08x its big price is self-consistent; other stores and eras +# use 1.10 or a per-100g unit price instead, so this is a check, never a rule. +TAX_RATIOS = (1.08, 1.10) HEADER_PATTERN = re.compile(r"お買得|お買い得|特価|価格|セール") - +# A shelf price below this is not a price (measured live reads included 20 and 23). +MIN_PRICE = 50 # The tag truncates its discount to a tenth of a 割, so agreement is one step. DISCOUNT_TOLERANCE = 0.05 # A regular price above this multiple of the sale price is a misread, not a discount. @@ -298,33 +201,22 @@ def cluster_rows(lines: Sequence[OcrLine], max_gap: int = 26) -> List[List[OcrLi return rows -def tag_row_score(row: Sequence[OcrLine], header_pattern: Any = None) -> float: +def tag_row_score(row: Sequence[OcrLine]) -> float: """How much a row looks like a line of price tags. Tags carry a header, a discount, and a decimal (tax-included) price; product - packaging rarely carries all three on one line. The header pattern is bound - here: a missing module global must not fail the tag pass. + packaging rarely carries all three on one line. """ - def pattern(name: str, fallback: str) -> Any: - found = globals().get(name) - return found if found is not None else re.compile(fallback) - - header = header_pattern if header_pattern is not None else pattern( - "HEADER_PATTERN", r"お買得|お買い得|特価|価格|セール" - ) - discount = pattern("DISCOUNT_PATTERN", r"(\d+(?:\.\d)?)\s*割\s*引") - decimal = pattern("DECIMAL_PATTERN", r"(\d{1,4})[.,](\d{2})(?!\d)") - jan = pattern("JAN3_PATTERN", r"(? TagFields: name_candidates.sort(key=lambda item: (-len(item[0]), -item[1].confidence)) fields.name = name_candidates[0][0] + # Layout, not digit count, separates the price from the JAN: the price is the # LARGEST text on the tag (its line is roughly twice as tall as the small # print), and the JAN's last three digits sit on a small line of their own. # Getting this order wrong made a 269 price look like a JAN and disappear. @@ -641,12 +534,7 @@ def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, i hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) hue, sat, val = cv2.split(hsv) - rail = _RAIL or StorePolicy() - brick = ( - ((hue < rail.rail_hue_low) | (hue > rail.rail_hue_high)) - & (sat > rail.rail_sat_min) - & (val > rail.rail_val_min) - ).astype(np.uint8) + brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8) row_fraction = brick.mean(axis=1) height, width = image.shape[:2] rails: List[Tuple[int, int, int, int]] = [] diff --git a/recognization/code/tags.py b/recognization/code/tags.py index b4ad79d..8d5fc5b 100644 --- a/recognization/code/tags.py +++ b/recognization/code/tags.py @@ -36,7 +36,6 @@ from ocr import OcrEngine, OcrLine, OcrResult, OcrUnavailable # noqa: E402 from tag_cards import ( # noqa: E402 CORROBORATION_FLAGS, MIN_PRICE, - StorePolicy, TagFields as TagFieldsModel, corroborate, best_tag_rows, @@ -50,13 +49,6 @@ from tag_cards import ( # noqa: E402 TAG_STATE_READABLE = "readable" TAG_STATE_UNREADABLE = "unreadable" TAG_STATE_ABSENT = "absent" -# Seen on the rail, OCR returned nothing. Not the same as absent (no tag) or -# unreadable (text came back but did not corroborate). -TAG_STATE_DETECTED = "detected" - -# Two shelf rows closer than this are the same rail. A screen row index is not -# an identity: panning a new shelf into view reuses row 0. -SHELF_ROW_BIN_PX = 80 # Measured tag shape (add/image6.jpg): ~300x80 px, aspect ~3.75:1, digits ~22 px. TAG_MIN_WIDTH_RATIO = 0.035 @@ -122,7 +114,6 @@ class TagRecord: tag_id: str box: List[int] band_index: int - shelf_key: Tuple[str, int] = ("screen", 0) observations: List[TagObservation] = field(default_factory=list) first_seen: float = 0.0 last_seen: float = 0.0 @@ -135,7 +126,6 @@ class TagRecord: discount: Optional[float] = None unit_price: Optional[float] = None name: Optional[str] = None - price_source: str = "none" flags: List[str] = field(default_factory=list) confidence: float = 0.0 @@ -145,18 +135,17 @@ class TagRecord: Measured live: without this, single uncorroborated reads of the stylised digits were published as readable tags with prices of 20, 23, 72 and 4508. - A block seen with no text is `detected`; text that does not parse is - `unreadable`. Neither is `absent`. """ - if self.price is None: - saw_text = any(observation.lines for observation in self.observations) - return TAG_STATE_UNREADABLE if saw_text else TAG_STATE_DETECTED - if self.price < MIN_PRICE: + if not self.observations: + return TAG_STATE_UNREADABLE + if self.price is None or self.price < MIN_PRICE: return TAG_STATE_UNREADABLE corroborated = bool( {"tax_consistent", "tax_pair_consistent", "discount_consistent"} & set(self.flags) ) - if not corroborated or (self.regular_price is None and self.name is None): + if not corroborated: + return TAG_STATE_UNREADABLE + if self.regular_price is None and self.name is None: return TAG_STATE_UNREADABLE return TAG_STATE_READABLE @@ -170,6 +159,8 @@ class TagRecord: "discount": self.discount, "unit_price": self.unit_price, "name": self.name, + "regular_price": self.regular_price, + "price_source": self.price_source, "flags": list(self.flags), "confidence": round(float(self.confidence), 3), "observations": len(self.observations), @@ -395,20 +386,6 @@ class TagSettings: settle_passes: int = SETTLE_PASSES min_band_width_ratio: float = 0.10 max_tags: int = 120 - policy: Optional[StorePolicy] = None - - -def band_key(band: RailBand, shelf_offset: Optional[Sequence[float]] = None) -> Tuple[str, int]: - """Which rail this band is, stable across a pan. - - No shelf origin: the screen row, which is all a still has. With one (the - shelf position of the frame origin), the row's shelf y, so a new shelf that - lands on screen row 0 is not the row that already settled. - """ - if not shelf_offset or len(shelf_offset) < 2: - return ("screen", int(band.row_index)) - shelf_y = (band.box[1] + band.box[3]) / 2.0 + float(shelf_offset[1]) - return ("shelf", int(round(shelf_y / SHELF_ROW_BIN_PX))) class TagService: @@ -433,7 +410,7 @@ class TagService: self._tags: Dict[str, TagRecord] = {} self._pass_id = 0 self._frame_seq = 0 - self._last_pass_seq: Dict[Tuple[str, int], int] = {} + self._last_pass_seq: Dict[int, int] = {} self._pending_bands: set = set() self.passes_run = 0 self.passes_failed = 0 @@ -441,37 +418,29 @@ class TagService: self.passes_skipped = 0 # ------------------------------------------------------------------ frame loop - def plan( - self, - boxes: Sequence[Dict[str, Any]], - image_shape: Sequence[int], - shelf_offset: Optional[Sequence[float]] = None, - ) -> List[RailBand]: + def plan(self, boxes: Sequence[Dict[str, Any]], image_shape: Sequence[int]) -> List[RailBand]: """Bands worth OCR-ing right now (must be called from the frame loop). - The key is the shelf row when the caller passes the frame origin, and the - screen row otherwise. Keying by on-screen pixels re-read the same shelf - on every pan step (measured: 10x slower per crop while OCR ran); keying - by the screen row index instead hides a new shelf that reuses row 0. + A band is a shelf row, so its key is the row - not its on-screen position. + Keying by position meant a walking camera minted a fresh key every few + pixels of pan and re-read the same shelf continuously, which starved + classification of CPU (measured: 10x slower per crop while OCR ran). """ if not self.settings.enabled or not self.engine.available(): return [] - if self.settings.policy is not None: - self.settings.policy.active() bands = rail_bands(boxes, image_shape) width = float(image_shape[1]) bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width] with self._lock: - self._shelf_offset = shelf_offset self._frame_seq += 1 seq = self._frame_seq chosen: List[RailBand] = [] for band in bands: - key = band_key(band, shelf_offset) + key = band.row_index if key in self._pending_bands: continue last = self._last_pass_seq.get(key) - settled = self._band_settled(key) + settled = self._band_settled(band.row_index) if settled and last is not None: continue if last is not None and seq - last < self.settings.pass_interval_frames: @@ -480,9 +449,8 @@ class TagService: if len(chosen) >= self.settings.max_bands_per_pass: break for band in chosen: - key = band_key(band, shelf_offset) - self._last_pass_seq[key] = seq - self._pending_bands.add(key) + self._last_pass_seq[band.row_index] = seq + self._pending_bands.add(band.row_index) return chosen def run_pass(self, image: Any, band: RailBand, scale_bias: float = 1.0) -> List[TagObservation]: @@ -492,7 +460,7 @@ class TagService: independent reads of the same tag are what makes the vote meaningful (identical pixels would produce identical output). """ - key = band_key(band, getattr(self, "_shelf_offset", None)) + key = band.row_index try: x1, y1, x2, y2 = band.box crop = image[y1:y2, x1:x2] @@ -569,7 +537,7 @@ class TagService: ) card = card_from_lines(pooled) if pooled else seed card.box = box - if not card.fields.is_readable() and card.fields.price is None and not group: + if not card.fields.is_readable() and card.fields.price is None: continue cards.append( { @@ -625,8 +593,8 @@ class TagService: ) return extra - def _band_settled(self, key: Tuple[str, int]) -> bool: - row_tags = [tag for tag in self._tags.values() if tag.shelf_key == key] + def _band_settled(self, row_index: int) -> bool: + row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index] if not row_tags: return False return all(tag.settled for tag in row_tags) @@ -636,7 +604,6 @@ class TagService: box = candidate["box"] lines = candidate["lines"] fields = candidate["fields"] - shelf_key = band_key(band, getattr(self, "_shelf_offset", None)) observation = TagObservation( box=list(box), lines=list(lines), @@ -646,13 +613,12 @@ class TagService: observed_at=time.time(), ) - record = self._match_record(box, shelf_key) + record = self._match_record(box) if record is None: record = TagRecord( tag_id=f"tag_{uuid.uuid4().hex[:8]}", box=list(box), band_index=band.row_index, - shelf_key=shelf_key, first_seen=observation.observed_at, ) if len(self._tags) >= self.settings.max_tags: @@ -670,12 +636,10 @@ class TagService: record.settled = True return observation - def _match_record(self, box: Sequence[int], shelf_key: Tuple[str, int]) -> Optional[TagRecord]: + def _match_record(self, box: Sequence[int]) -> Optional[TagRecord]: best = None best_iou = 0.35 for record in self._tags.values(): - if record.shelf_key != shelf_key: - continue iou = _iou(box, record.box) if iou >= best_iou: best_iou = iou @@ -805,12 +769,7 @@ class TagService: "settled": False, } - counts = { - TAG_STATE_READABLE: 0, - TAG_STATE_UNREADABLE: 0, - TAG_STATE_DETECTED: 0, - TAG_STATE_ABSENT: 0, - } + counts = {TAG_STATE_READABLE: 0, TAG_STATE_UNREADABLE: 0, TAG_STATE_ABSENT: 0} for payload in by_track.values(): counts[payload["state"]] = counts.get(payload["state"], 0) + 1 return by_track, tags, counts @@ -858,7 +817,6 @@ class TagService: self._pending_bands.clear() self._pass_id = 0 self._frame_seq = 0 - self._shelf_offset = None def _iou(a: Sequence[int], b: Sequence[int]) -> float: diff --git a/recognization/code/test.py b/recognization/code/test.py new file mode 100644 index 0000000..5eb7553 --- /dev/null +++ b/recognization/code/test.py @@ -0,0 +1,25 @@ +from ultralytics import YOLO +import cv2 + +model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") + +image_path = "/Users/nguyenhongvan/recongnize/data/add/image9.jpg" + +results = model(image_path) + +image = cv2.imread(image_path) + +for box in results[0].boxes.xyxy.cpu().numpy(): + x1, y1, x2, y2 = map(int, box) + + cv2.rectangle( + image, + (x1, y1), + (x2, y2), + (0, 255, 0), + 2 + ) + +cv2.imshow("Bounding Boxes Only", image) +cv2.waitKey(0) +cv2.destroyAllWindows() \ No newline at end of file diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index bc21027..dc74def 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -155,47 +155,6 @@ class RestContractTests(unittest.TestCase): self.assertEqual(response.status_code, 415) self.assertEqual(response.json()["error"]["code"], "UNSUPPORTED_MEDIA_TYPE") - def test_a_lying_content_length_is_rejected_while_streaming(self): - """The ASGI receive stops once the body passes the cap, before decode.""" - import anyio - - from transport import BodyLimit - - cap = 32 - seen = {"bytes": 0} - - async def app(scope, receive, send): - while True: - message = await receive() - if message["type"] != "http.request": - continue - seen["bytes"] += len(message.get("body") or b"") - if not message.get("more_body"): - break - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b"ok"}) - - chunks = [b"x" * 20, b"y" * 20, b"z" * 20] - - async def receive(): - body = chunks.pop(0) if chunks else b"" - return {"type": "http.request", "body": body, "more_body": bool(chunks)} - - sent = [] - - async def send(message): - sent.append(message) - - async def run(): - await BodyLimit(app, cap)( - {"type": "http"}, receive, send - ) - - anyio.run(run) - self.assertLessEqual(seen["bytes"], cap + 20) - self.assertEqual(sent[0]["status"], 413) - self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"]) - def test_classify_and_inspect_roundtrip(self): recognized = self.client.post( "/v1/recognize?classify=false", content=jpeg_bytes(), @@ -326,26 +285,13 @@ class WebSocketProtocolTests(unittest.TestCase): self.assertEqual(missing["code"], "TRACK_NOT_IN_VIEW") def test_token_is_enforced_on_the_socket(self): - """Header or the first post-hello auth frame. A query token is rejected.""" client, engine = make_app(token="s3cret") try: with self.assertRaises(Exception): with client.websocket_connect("/v1/stream") as socket: - self.assertEqual(socket.receive_json()["type"], "hello") - socket.send_json({"type": "ping", "t": 1}) socket.receive_json() - with self.assertRaises(Exception): - with client.websocket_connect("/v1/stream?token=s3cret") as socket: - socket.receive_json() - with client.websocket_connect( - "/v1/stream", headers={"Authorization": "Bearer s3cret"} - ) as socket: + with client.websocket_connect("/v1/stream?token=s3cret") as socket: self.assertEqual(socket.receive_json()["type"], "hello") - with client.websocket_connect("/v1/stream") as socket: - self.assertEqual(socket.receive_json()["type"], "hello") - socket.send_json({"type": "auth", "token": "s3cret"}) - socket.send_json({"type": "ping", "t": 1}) - self.assertEqual(socket.receive_json(), {"type": "pong", "t": 1}) finally: engine.shutdown() diff --git a/recognization/code/test_demo.py b/recognization/code/test_demo.py new file mode 100644 index 0000000..b62ca5c --- /dev/null +++ b/recognization/code/test_demo.py @@ -0,0 +1,119 @@ +""" +Contract tests for the demo camera (the synthetic store walk). + +The walk is what a demo shows when there is no camera, so its geometry has to be +predictable: inside the image, moving, serpentine, and wrapping forever. +""" + +from __future__ import annotations + +import os +import sys +import unittest + +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(current_dir) +if os.path.exists(os.path.join(project_root, "stock")): + os.chdir(project_root) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import numpy as np # noqa: E402 + +from demo import DemoCamera, DemoSourceError # noqa: E402 + + +class FakeImageCamera(DemoCamera): + """A camera over a synthetic image, so the geometry is testable without files.""" + + def __init__(self, width=1500, height=2000, window=(960, 540)): + super().__init__({"synthetic": {"id": "synthetic", "path": "synthetic", "label": "synthetic"}}, window) + self._image_cache = np.zeros((height, width, 3), dtype="uint8") + self._size_cache = (width, height) + + def _size(self, sample_id): # noqa: D102 + if sample_id != "synthetic": + raise DemoSourceError(f"unknown sample '{sample_id}'") + return self._size_cache + + def _image(self, sample_id): # noqa: D102 + return self._image_cache + + +class WalkGeometryTests(unittest.TestCase): + def setUp(self): + self.camera = FakeImageCamera() + self.plan = self.camera.plan(["synthetic"]) + + def test_plan_is_deterministic_and_covers_the_shelf(self): + self.assertEqual(self.plan["steps"], self.camera.plan(["synthetic"])["steps"]) + self.assertGreater(self.plan["steps"], 20, "a shelf walk is not two steps long") + self.assertEqual(self.plan["bays"][0]["sample"], "synthetic") + + def test_window_never_leaves_the_image(self): + width, height = 1500, 2000 + for step in range(self.plan["steps"]): + position = self.camera.window_at(["synthetic"], step) + self.assertGreaterEqual(position["x"], 0) + self.assertGreaterEqual(position["y"], 0) + self.assertLessEqual(position["x"] + position["width"], width) + self.assertLessEqual(position["y"] + position["height"], height) + + def test_the_camera_actually_moves(self): + xs = [self.camera.window_at(["synthetic"], step)["x"] for step in range(6)] + ys = [self.camera.window_at(["synthetic"], step)["y"] for step in range(self.plan["steps"])] + self.assertGreater(len(set(xs)), 1, "the window must pan, not sit still") + self.assertGreater(len(set(ys)), 1, "the walk must climb rows") + + def test_it_ascends_the_shelf(self): + first = self.camera.window_at(["synthetic"], 0)["y"] + last_band = self.plan["steps"] - 1 + self.assertGreater(first, self.camera.window_at(["synthetic"], last_band)["y"], + "the walk starts at the bottom row and climbs") + + def test_step_wraps(self): + first = self.camera.window_at(["synthetic"], 0) + wrapped = self.camera.window_at(["synthetic"], self.plan["steps"]) + self.assertEqual((first["x"], first["y"]), (wrapped["x"], wrapped["y"])) + + def test_frames_are_deterministic_jpegs(self): + payload, position = self.camera.frame(["synthetic"], 3, quality=70) + again, _ = self.camera.frame(["synthetic"], 3, quality=70) + self.assertTrue(payload.startswith(b"\xff\xd8"), "JPEG magic") + self.assertEqual(payload, again, "the same step must render the same frame") + self.assertIn("sample", position) + + def test_unknown_sample_is_rejected(self): + with self.assertRaises(DemoSourceError): + self.camera.plan(["nope"]) + + def test_window_larger_than_the_image_is_clamped(self): + camera = FakeImageCamera(width=800, height=600, window=(960, 540)) + position = camera.window_at(["synthetic"], 0) + self.assertLessEqual(position["width"], 800) + self.assertLessEqual(position["height"], 600) + self.assertEqual(position["x"], 0) + # The walk starts at the bottom of the shelf, so the window is bottom-aligned. + self.assertEqual(position["y"], 600 - position["height"]) + + +class RealShelfTests(unittest.TestCase): + """One check against the actual demo asset, skipped when it is absent.""" + + def test_real_sample_walks(self): + from app import get_available_shelf_images + + samples = {item["id"]: item for item in get_available_shelf_images("add")} + if "image6" not in samples: + self.skipTest("demo sample image6 not present") + camera = DemoCamera(samples) + plan = camera.plan(["image6"], (1280, 720)) + self.assertGreater(plan["steps"], 20, "a 1500x2000 shelf gives a multi-row walk") + payload, position = camera.frame(["image6"], 5, (1280, 720)) + self.assertGreater(len(payload), 10_000) + self.assertLessEqual(position["x"] + position["width"], plan["bays"][0]["width"]) + self.assertLessEqual(position["y"] + position["height"], plan["bays"][0]["height"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/code/test_empty.py b/recognization/code/test_empty.py new file mode 100644 index 0000000..0781650 --- /dev/null +++ b/recognization/code/test_empty.py @@ -0,0 +1,20 @@ +import sys +import os + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from empty_shelf_detector import detect_and_draw_empty_slots + +if __name__ == "__main__": + image_path = "image.jpg" + print(f"Processing empty shelf detection on {image_path}...") + annotated, empty_slots = detect_and_draw_empty_slots( + image_path, + output_path="detected_empty_slots.jpg" + ) + print(f"Detected {len(empty_slots)} empty shelf slots:") + for s in empty_slots: + print(f" - Slot [{s['id']}]: Row {s['row_index']}, Box={s['box']}, Width={s['width']}px, Est. Missing Items={s['missing_items_est']}") + print(f"Visualization saved to: detected_empty_slots.jpg") diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py deleted file mode 100644 index f993328..0000000 --- a/recognization/code/test_eval_live.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Metric contracts for eval_live. No model, no real walk, no fabricated accuracy.""" - -from __future__ import annotations - -import json -import os -import shutil -import subprocess -import sys -import tempfile -import unittest - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import cv2 # noqa: E402 -import numpy as np # noqa: E402 - -from eval_live import ( # noqa: E402 - EvalInputError, - identity_level, - labelled_top1, - leaked_labels, - load_index, - score_recording, - score_tracks, - track_churn, -) - - -def track(level, sku=None, track_id=1, box=None, relabels=0, restarts=0, tag=None): - identity = {"identity_level": level, "state": "recognized" if level == "accepted" else "unrecognized"} - if level == "accepted": - identity["sku_id"] = sku - row = { - "track_id": track_id, - "box": box or [0, 0, 10, 10], - "identity": identity, - "relabels": relabels, - "restarts": restarts, - } - if tag is not None: - row["price_tag"] = tag - return row - - -def write_jpeg(path, image): - ok, encoded = cv2.imencode(".jpg", image) - assert ok - with open(path, "wb") as handle: - handle.write(encoded.tobytes()) - - -class DecisionDenominatorTests(unittest.TestCase): - def test_settled_excludes_unsettled_and_rates_use_that_denominator(self): - scored = score_tracks( - [ - track("accepted", "sku_a"), - track("ambiguous"), - track("rejected"), - track("unknown"), - {"identity": {"state": "unconfirmed", "identity_level": None}}, - ] - ) - self.assertEqual(scored["settled"], 4) - self.assertEqual(scored["unsettled"], 1) - self.assertEqual(scored["accepted_rate"], 0.25) - self.assertIsNone(score_tracks([{"identity": {"state": "unconfirmed"}}])["accepted_rate"]) - - def test_wire_state_fills_a_missing_level(self): - self.assertEqual( - identity_level({"identity": {"state": "needs_confirmation"}}), "ambiguous" - ) - self.assertEqual(identity_level({"identity": {"state": "unconfirmed"}}), "unsettled") - - def test_readable_tags_are_counted_apart_from_detected(self): - scored = score_tracks( - [ - track("accepted", "sku_a", tag={"state": "readable"}), - track("rejected", tag={"state": "unreadable"}), - track("rejected", tag={"state": "absent"}), - track("rejected"), - ] - ) - self.assertEqual(scored["tags_readable"], 1) - self.assertEqual(scored["tags_detected"], 2) - self.assertEqual(scored["tags_absent"], 2) - - -class Top1AndChurnTests(unittest.TestCase): - def test_top1_counts_only_accepted_matches_over_overlapping_labels(self): - frames = [ - { - "frame": 1, - "tracks": [ - track("accepted", "sku_a", box=[0, 0, 10, 10]), - track("accepted", "sku_b", track_id=2, box=[100, 100, 110, 110]), - ], - }, - {"frame": 2, "tracks": [track("rejected", box=[0, 0, 10, 10])]}, - {"frame": 3, "tracks": []}, - ] - labels = [ - {"frame": 1, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, - {"frame": 1, "box": [100, 100, 110, 110], "sku_id": "sku_a"}, - {"frame": 2, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, - {"frame": 3, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, - ] - scored = labelled_top1(frames, labels, iou_min=0.5) - self.assertEqual((scored["correct"], scored["wrong"], scored["rejected"], scored["unmatched"]), (1, 1, 1, 1)) - self.assertEqual(scored["top1"], 0.3333) - - def test_sku_churn_counts_a_renamed_track_not_the_first_name(self): - frames = [ - {"tracks": [track("accepted", "sku_a", track_id=7)]}, - {"tracks": [track("accepted", "sku_a", track_id=7)]}, - {"tracks": [track("accepted", "sku_b", track_id=7, relabels=1)]}, - {"tracks": [track("rejected", track_id=8, restarts=2)]}, - ] - churn = track_churn(frames) - self.assertEqual(churn["sku_changes"], 1) - self.assertEqual(churn["tracks_with_sku_change"], 1) - self.assertEqual(churn["relabels"], 1) - self.assertEqual(churn["restarts"], 2) - self.assertEqual(churn["per_track"][0]["track_id"], 7) - - -class LeakageTests(unittest.TestCase): - def setUp(self): - self.root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) - - def test_a_pasted_catalog_crop_is_excluded_and_a_real_facing_is_not(self): - stock = os.path.join(self.root, "stock") - os.makedirs(stock) - template = np.full((20, 16, 3), 40, dtype=np.uint8) - template[2:8, 2:8] = (255, 0, 0) - write_jpeg(os.path.join(stock, "sku_a.jpeg"), template) - frame = np.zeros((40, 40, 3), dtype=np.uint8) - frame[5:25, 10:26] = template - recording = os.path.join(self.root, "walk") - os.makedirs(recording) - write_jpeg(os.path.join(recording, "000000.jpg"), frame) - with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: - handle.write(json.dumps({"frame": 0, "path": "000000.jpg", "tracks": []}) + "\n") - labels = [ - {"frame": 0, "box": [10, 5, 26, 25], "sku_id": "sku_a"}, - {"frame": 0, "box": [0, 0, 8, 8], "sku_id": "sku_b"}, - ] - held, leaked = leaked_labels(recording, load_index(recording), labels, stock) - self.assertEqual(len(leaked), 1) - self.assertEqual(leaked[0]["leak"], "crop_is_template") - self.assertEqual([row["sku_id"] for row in held], ["sku_b"]) - - def test_score_recording_names_the_leak_and_refuses_it_as_top1(self): - stock = os.path.join(self.root, "stock") - os.makedirs(stock) - image = np.full((12, 12, 3), 9, dtype=np.uint8) - write_jpeg(os.path.join(stock, "sku_a.jpg"), image) - with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: - handle.write("SKU_ID,product_name\nsku_a,A\n") - recording = os.path.join(self.root, "session") - os.makedirs(recording) - shutil.copy(os.path.join(stock, "sku_a.jpg"), os.path.join(recording, "000000.jpg")) - with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: - handle.write( - json.dumps( - { - "frame": 1, - "path": "000000.jpg", - "tracks": [track("accepted", "sku_a", box=[0, 0, 12, 12])], - } - ) - + "\n" - ) - with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: - json.dump({"labels": [{"frame": 1, "box": [0, 0, 12, 12], "sku_id": "sku_a"}]}, handle) - report = score_recording(recording, stock_dir=stock) - self.assertEqual(report["leakage"]["excluded"], 1) - self.assertEqual(report["labelled_top1"]["labels"], 0) - self.assertEqual(report["labelled_top1"]["top1"], None) - self.assertEqual(report["decisions"]["accepted"], 1) - self.assertTrue(report["fingerprints"]["sample"]["sha256"]) - self.assertEqual(len(report["fingerprints"]["catalog"]["version"]), 12) - self.assertTrue(any("template-derived" in gap for gap in report["evidence"])) - - -class CliTests(unittest.TestCase): - def test_missing_recording_is_an_explicit_error(self): - completed = subprocess.run( - [sys.executable, os.path.join(current_dir, "eval_live.py"), "/no/such/recording"], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(completed.returncode, 2) - self.assertIn("error", json.loads(completed.stdout)) - - def test_the_same_pinned_recording_scores_twice_the_same(self): - root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, root, ignore_errors=True) - stock = os.path.join(root, "stock") - os.makedirs(stock) - with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: - handle.write("SKU_ID\nsku_a\n") - recording = os.path.join(root, "session") - os.makedirs(recording) - payload = np.zeros((8, 8, 3), dtype=np.uint8) - write_jpeg(os.path.join(recording, "000000.jpg"), payload) - line = { - "frame": 4, - "path": "000000.jpg", - "tracks": [ - track("accepted", "sku_a", tag={"state": "readable"}), - track("ambiguous", track_id=2), - ], - } - with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: - handle.write(json.dumps(line) + "\n") - with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: - json.dump({"labels": [{"frame": 4, "box": [0, 0, 8, 8], "sku_id": "sku_a"}]}, handle) - first = score_recording(recording, stock_dir=stock) - second = score_recording(recording, stock_dir=stock) - self.assertEqual(first, second) - self.assertEqual(first["decisions"]["accepted"], 1) - self.assertEqual(first["decisions"]["ambiguous"], 1) - self.assertEqual(first["labelled_top1"]["correct"], 1) - self.assertIsNone(first["fingerprints"]["model"]["weights_sha256"]) - - def test_a_broken_index_line_is_an_input_error(self): - root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, root, ignore_errors=True) - with open(os.path.join(root, "index.jsonl"), "w", encoding="utf-8") as handle: - handle.write("{not json}\n") - with self.assertRaises(EvalInputError): - load_index(root) - - -if __name__ == "__main__": - unittest.main() diff --git a/recognization/code/test_image.py b/recognization/code/test_image.py new file mode 100644 index 0000000..57f138f --- /dev/null +++ b/recognization/code/test_image.py @@ -0,0 +1,89 @@ +""" +Test Script for image.jpg +1. Runs YOLO detection on image.jpg. +2. Draws ONLY bounding box rectangles (no class names, no confidence labels). +3. Saves output to detected_image.jpg. +4. Matches the detected bounding boxes against stock templates. +""" + +import os +import sys +import cv2 +import numpy as np + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from detector import ShelfProductDetector +from feature_matcher import ProductFeatureMatcher +from inventory import InventoryManager + + +def main(): + image_path = "image.jpg" + print(f"[1/4] Loading {image_path}...") + image = cv2.imread(image_path) + if image is None: + print(f"[Error] Failed to read {image_path}") + return + + h, w = image.shape[:2] + print(f" Image resolution: {w}x{h}") + + print("[2/4] Detecting products using YOLO...") + detector = ShelfProductDetector("runs/detect/train/weights/best.pt") + # You can adjust conf_threshold as needed (default 0.25) + det_res = detector.detect(image, conf_threshold=0.25) + boxes = det_res.get("boxes", []) + print(f" Total detected items: {len(boxes)}") + + print("[3/4] Drawing bounding boxes ONLY (no class names, no confidence)...") + annotated = detector.draw_detections(image, boxes, draw_labels=False) + output_path = "detected_image.jpg" + cv2.imwrite(output_path, annotated) + print(f" Saved detection visualization to: {output_path}") + + print("[4/4] Matching against stock catalogue (SURF, ORB, BRISK, Hybrid)...") + matcher = ProductFeatureMatcher("stock") + inv = InventoryManager("stock") + + print("\n--- Stock Templates Present in image.jpg ---") + matched_skus = set() + sample_matches = [] + + # Test top detections to find corresponding stock SKUs + for b in boxes: + crop = detector.crop_roi(image, b["box"]) + # Use hybrid method from the paper + res = matcher.classify_crop(crop, method="hybrid") + if res.get("status") == "success": + sku_id = res["best_sku"] + score = res["score"] + conf = res["confidence"] + prod = inv.get_product(sku_id) + if prod: + matched_skus.add(sku_id) + sample_matches.append({ + "box_id": b["id"], + "box": b["box"], + "sku_id": sku_id, + "product_name": prod["product_name"], + "price": prod["price"], + "number_on": prod["number_on"], + "score": score, + "confidence": conf + }) + + # Sort matches by confidence + sample_matches.sort(key=lambda x: x["score"], reverse=True) + + print(f"\nSuccessfully matched {len(matched_skus)} distinct SKUs across detected shelf boxes:") + for sm in sample_matches[:10]: + print(f"Box #{sm['box_id']:<3} -> [{sm['sku_id']}] {sm['product_name']:<30} | Price: ¥{sm['price']:<5} | On-Shelf: {sm['number_on']} | Score: {sm['score']}") + + print("\n[Complete] Tested image.jpg successfully.") + + +if __name__ == "__main__": + main() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index d1355c5..cec10cf 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -276,36 +276,6 @@ class TagPassIntegrationTests(unittest.TestCase): self.assertNotIn("price_tag", result["tracks"][0]) engine.shutdown() - def test_a_panned_shelf_reusing_the_screen_row_is_planned(self): - """The same screen row on a shelf that has moved is a new rail.""" - session, engine, _ = self.make_tagged_session() - session._tags.settings.pass_interval_frames = 0 - session._tags.settings.settle_observations = 1 - session._tags.settings.settle_passes = 1 - - class Panned: - def __init__(self): - self.calls = 0 - - def register(self, image): - self.calls += 1 - # Content moved up 400 px, so the frame origin moves down the shelf. - return (0.0, -400.0) if self.calls == 2 else (0.0, 0.0) - - session._shift_map = Panned() - process(session, 1) - time.sleep(0.05) - before = session._tags.stats()["passes_run"] - self.assertGreaterEqual(before, 1) - process(session, 2) - time.sleep(0.1) - self.assertGreaterEqual(session._frame_origin[1], 400.0) - self.assertGreater( - session._tags.stats()["passes_run"], before, - "a shelf 400 px away reuses screen row 0 and must be read again", - ) - engine.shutdown() - class FramePipelineTests(unittest.TestCase): def test_latest_frame_wins_and_drops_are_counted(self): diff --git a/recognization/code/test_ocr_tags.py b/recognization/code/test_ocr_tags.py index 25d911d..b48ea2f 100644 --- a/recognization/code/test_ocr_tags.py +++ b/recognization/code/test_ocr_tags.py @@ -21,7 +21,6 @@ import numpy as np # noqa: E402 from ocr import FakeOcr, NullOcr, OcrLine, OcrResult, OcrUnavailable, build_ocr_engine # noqa: E402 from tags import ( # noqa: E402 TAG_STATE_ABSENT, - TAG_STATE_DETECTED, TAG_STATE_READABLE, TAG_STATE_UNREADABLE, TagService, @@ -31,8 +30,6 @@ from tags import ( # noqa: E402 parse_price_tag, rail_bands, ) -from tag_cards import StorePolicy, StorePolicyError, load_store_policy # noqa: E402 -import tag_cards # noqa: E402 def image(width=1000, height=600): @@ -278,108 +275,6 @@ class TagServiceTests(unittest.TestCase): service.reset() self.assertEqual(service.stats()["tags"], 0) - def test_a_new_shelf_reusing_row_zero_is_read(self): - """Screen row 0 on a newly panned shelf is not the row that settled.""" - service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1) - boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] - array = image() - for _ in range(3): - for band in service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)): - service.run_pass(array, band) - self.assertEqual( - service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)), - [], - "the same shelf row stays settled", - ) - again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0)) - self.assertTrue(again, "a shelf 400 px away reuses the screen row and must be read") - - def test_a_seen_tag_with_no_text_is_detected_not_absent(self): - def reader(crop): - return [] - - service = TagService(FakeOcr(reader=reader), settings=TagSettings(pass_interval_frames=0)) - array = image() - # A bright paper block on a brick rail, under the product row. OCR reads nothing. - array[300:420, :] = (40, 40, 180) - array[330:390, 120:280] = (235, 235, 235) - boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] - bands = service.plan(boxes, array.shape) - self.assertTrue(bands) - service.run_pass(array, bands[0]) - by_track, tags, counts = service.attach([{"track_id": 0, "box": [100, 100, 200, 300]}]) - self.assertTrue(tags, "the paper block is a tag even though OCR returned nothing") - self.assertEqual(tags[0]["state"], TAG_STATE_DETECTED) - self.assertEqual(by_track[0]["state"], TAG_STATE_DETECTED) - self.assertEqual(counts[TAG_STATE_DETECTED], 1) - self.assertNotEqual(by_track[0]["state"], TAG_STATE_ABSENT) - - def test_a_tag_pass_survives_a_missing_header_pattern(self): - """Evaluation replay died with NameError: HEADER_PATTERN.""" - import tag_cards - from tag_cards import OcrLine as CardLine - - row = [CardLine(text="毎日お買得", box=(0, 0, 40, 16), confidence=0.9)] - saved = tag_cards.__dict__.pop("HEADER_PATTERN", None) - try: - score = tag_cards.tag_row_score(row) - finally: - if saved is not None: - tag_cards.HEADER_PATTERN = saved - self.assertGreater(score, 0, "a sale header still marks the row when the global is missing") - service, _ = self.make(pass_interval_frames=0) - boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] - bands = service.plan(boxes, image().shape) - self.assertTrue(bands) - service.run_pass(image(), bands[0]) - self.assertGreaterEqual(service.stats()["passes_run"], 1) - self.assertIsNone(service.stats()["last_pass_error"]) - - -class StorePolicyTests(unittest.TestCase): - def test_a_missing_file_uses_the_defaults(self): - policy = load_store_policy(None) - policy.active() - self.assertEqual(policy.tax_ratios, (1.08, 1.10)) - self.assertEqual(policy.min_price, 50) - self.assertEqual(tag_cards.MIN_PRICE, 50) - - def test_a_store_file_does_not_change_another_store(self): - import json - import tempfile - - with tempfile.TemporaryDirectory() as root: - good = os.path.join(root, "lopia") - bad = os.path.join(root, "other") - os.makedirs(good) - os.makedirs(bad) - with open(os.path.join(good, "store.json"), "w", encoding="utf-8") as handle: - json.dump({"store_id": "lopia", "tax_ratios": [1.08], "min_price": 80}, handle) - with open(os.path.join(bad, "store.json"), "w", encoding="utf-8") as handle: - handle.write("{not json") - loaded = load_store_policy(good) - self.assertEqual(loaded.min_price, 80) - self.assertEqual(tag_cards.MIN_PRICE, 80) - with self.assertRaises(StorePolicyError) as raised: - load_store_policy(bad) - self.assertIn(bad, str(raised.exception)) - self.assertEqual(tag_cards.MIN_PRICE, 80, "a bad file must not replace the store already loaded") - again = load_store_policy(good) - self.assertEqual(again.min_price, 80) - - def test_a_ratio_outside_range_fails_clearly(self): - import json - import tempfile - - load_store_policy(None) - with tempfile.TemporaryDirectory() as root: - with open(os.path.join(root, "store.json"), "w", encoding="utf-8") as handle: - json.dump({"tax_ratios": [2.5], "min_price": 999}, handle) - with self.assertRaises(StorePolicyError) as raised: - load_store_policy(root) - self.assertIn("tax ratio", str(raised.exception)) - self.assertEqual(tag_cards.MIN_PRICE, 50) - class EngineSelectionTests(unittest.TestCase): def test_explicit_none_and_fake(self): diff --git a/recognization/code/test_shelf_map.py b/recognization/code/test_shelf_map.py deleted file mode 100644 index 664ecce..0000000 --- a/recognization/code/test_shelf_map.py +++ /dev/null @@ -1,125 +0,0 @@ -""" -Tests for the shelf map: registration, merging, and evidence pooling. - -The sign convention is tested explicitly, because getting it backwards is silent: -the map still produces boxes, they just land in the wrong place, and a shelf of -~60 facings becomes 857. That happened during development. -""" - -from __future__ import annotations - -import os -import sys -import unittest - -import numpy as np - -current_dir = os.path.dirname(os.path.abspath(__file__)) -project_root = os.path.dirname(current_dir) -if os.path.exists(os.path.join(project_root, "stock")): - os.chdir(project_root) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from shelf_map import Facing, ShelfMap, _iou # noqa: E402 - - -def a_frame(shift_x=0, shift_y=0, size=(160, 240)): - """A textured frame, optionally a shifted copy of another.""" - height, width = size - rng = np.random.RandomState(7) - base = rng.randint(0, 255, (height * 2, width * 2, 3), dtype=np.uint8) - return base[height - shift_y:height * 2 - shift_y, width - shift_x:width * 2 - shift_x] - - -def a_track(track_id, box, sku=None, score=None, tag_sku=None): - track = {"track_id": track_id, "box": list(box)} - if sku: - track["identity"] = {"state": "recognized", "sku_id": sku, "score": score or 1.0} - else: - track["identity"] = {"state": "unconfirmed"} - if tag_sku: - track["tag_identity"] = {"sku_id": tag_sku, "score": 0.7} - return track - - -class IouTests(unittest.TestCase): - def test_overlap_maths(self): - self.assertAlmostEqual(_iou([0, 0, 10, 10], [0, 0, 10, 10]), 1.0) - self.assertEqual(_iou([0, 0, 10, 10], [20, 20, 30, 30]), 0.0) - - -class RegistrationTests(unittest.TestCase): - def test_the_frame_origin_moves_opposite_to_the_content(self): - """The sign that matters: content moving left means the camera moved right. - - Getting this backwards walks the map the wrong way. It is silent - boxes - still appear - and it turned a shelf of ~60 facings into 857. - """ - shelf = ShelfMap() - shelf.register(a_frame()) - shift = shelf.register(a_frame(shift_x=10)) # content moved right 10 - self.assertIsNotNone(shift) - self.assertAlmostEqual(shift[0], 10, delta=1.0) - - # Content moving right means the camera moved LEFT, so the frame's origin - # sits 10 px further back along the shelf. - shelf.update(0, a_frame(), [a_track(1, [0, 0, 20, 20])]) - offset_before = list(shelf._offset) - shelf.update(1, a_frame(shift_x=10), [a_track(2, [0, 0, 20, 20])]) - self.assertLess(shelf._offset[0], offset_before[0] + 1, - "the offset must follow the camera, not the content") - - def test_a_first_frame_has_no_shift(self): - shelf = ShelfMap() - self.assertEqual(shelf.register(a_frame()), (0.0, 0.0)) - - -class MergingTests(unittest.TestCase): - def test_a_facing_seen_twice_is_one_facing(self): - shelf = ShelfMap() - shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200])]) - shelf.update(1, a_frame(), [a_track(2, [12, 11, 112, 201])]) - self.assertEqual(len(shelf.facings()), 1) - self.assertEqual(shelf.stats()["merges"], 1) - - def test_two_neighbouring_facings_stay_separate(self): - shelf = ShelfMap() - shelf.update(0, a_frame(), [a_track(1, [10, 10, 100, 200]), a_track(2, [110, 10, 200, 200])]) - self.assertEqual(len(shelf.facings()), 2) - - def test_identity_votes_pool_across_track_ids(self): - """The point of the map: one facing, many transient track ids, one answer.""" - shelf = ShelfMap() - for index, (track_id, sku) in enumerate([(1, "sku_a"), (2, "sku_a"), (3, "sku_a"), (4, "sku_b")]): - shelf.update(index, a_frame(), [a_track(track_id, [10, 10, 110, 200], sku=sku)]) - facing = shelf.facings()[0] - self.assertEqual(facing.observations, 4) - self.assertEqual(len(facing.seen_track_ids), 4, "four tracks, one facing") - sku_id, agreement = facing.best_identity() - self.assertEqual(sku_id, "sku_a") - self.assertAlmostEqual(agreement, 0.75) - - def test_tag_evidence_pools_separately_from_packaging(self): - shelf = ShelfMap() - shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200], sku="sku_a", tag_sku="sku_a")]) - shelf.update(1, a_frame(), [a_track(2, [10, 10, 110, 200], tag_sku="sku_a")]) - facing = shelf.facings()[0] - self.assertEqual(facing.best_identity()[0], "sku_a") - self.assertEqual(facing.best_tag_identity()[0], "sku_a") - self.assertEqual(len(facing.tag_readings), 0, "no readable tag payload was attached") - - -class PayloadTests(unittest.TestCase): - def test_a_facing_reports_its_pooled_evidence(self): - facing = Facing(facing_id="face_1", shelf_box=[0, 0, 10, 10]) - facing.vote_identity("sku_a", 2.0) - facing.vote_identity("sku_b", 1.0) - payload = facing.as_payload() - self.assertEqual(payload["sku_id"], "sku_a") - self.assertAlmostEqual(payload["identity_agreement"], 2 / 3, places=3) - self.assertIsNone(payload["tag_sku_id"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/recognization/code/tracking.py b/recognization/code/tracking.py index 42b7b2a..a77bc61 100644 --- a/recognization/code/tracking.py +++ b/recognization/code/tracking.py @@ -122,30 +122,16 @@ class _SessionByteTrack(BYTETracker): class TrackOut: """One tracked object for a frame.""" - __slots__ = ("track_id", "box", "score", "det_index", "low_score") + __slots__ = ("track_id", "box", "score", "det_index") - def __init__( - self, - track_id: int, - box: List[int], - score: float, - det_index: Optional[int], - *, - low_score: bool = False, - ): + def __init__(self, track_id: int, box: List[int], score: float, det_index: Optional[int]): self.track_id = int(track_id) self.box = [int(v) for v in box] self.score = float(score) self.det_index = None if det_index is None or det_index < 0 else int(det_index) - # ByteTrack's second pass keeps an id alive on a low-score box. That box - # is not a new measurement: callers must not crop or re-label from it. - self.low_score = bool(low_score) def __repr__(self) -> str: # pragma: no cover - debug helper - return ( - f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, " - f"det={self.det_index}, low={self.low_score})" - ) + return f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, det={self.det_index})" class SessionTracker: @@ -185,17 +171,14 @@ class SessionTracker: outputs: List[TrackOut] = [] if rows is None or len(rows) == 0: return outputs - high = float(self.args.track_high_thresh) for row in np.asarray(rows): # [x1, y1, x2, y2, track_id, score, cls, idx] - score = float(row[5]) outputs.append( TrackOut( track_id=int(row[4]), box=[int(round(row[0])), int(round(row[1])), int(round(row[2])), int(round(row[3]))], - score=score, + score=float(row[5]), det_index=int(row[7]) if len(row) > 7 else None, - low_score=score < high, ) ) return outputs diff --git a/recognization/code/transport.py b/recognization/code/transport.py deleted file mode 100644 index 66e94a1..0000000 --- a/recognization/code/transport.py +++ /dev/null @@ -1,250 +0,0 @@ -""" -Request bounds and the LAN token check. - -A client can declare a tiny Content-Length and then keep writing, so the -declared length is a hint and the read itself stops at the cap. The token -never travels in a query string: proxies and access logs keep those. -""" - -from __future__ import annotations - -import hmac -import ipaddress -import os -import shutil -import signal -import socket -import subprocess -from typing import Any, Dict, Optional - -LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) - - -def is_loopback_host(host: Optional[str]) -> bool: - text = (host or "").strip().lower() - if text.startswith("[") and text.endswith("]"): - text = text[1:-1] - if "%" in text: # fe80::1%lo0 - text = text.split("%", 1)[0] - if text in LOOPBACK_HOSTS: - return True - try: - return ipaddress.ip_address(text).is_loopback - except ValueError: - return False - - -def require_token_for_bind(host: Optional[str], token: Optional[str]) -> None: - """A non-loopback bind with no token is reachable by anyone on that network.""" - if token or is_loopback_host(host): - return - raise SystemExit( - f"refusing to bind {host} without DORAYAKI_API_TOKEN " - "(loopback is the only bind that may run with auth off)" - ) - - -def token_matches(presented: Optional[str], token: Optional[str]) -> bool: - if not presented or not token: - return False - # compare_digest raises when the lengths differ. Fold the length in so a - # short guess neither throws nor returns early. - return hmac.compare_digest( - len(presented).to_bytes(8, "big") + presented.encode(), - len(token).to_bytes(8, "big") + token.encode(), - ) - - -def bearer_matches(authorization: Optional[str], token: str) -> bool: - if not authorization or not token: - return False - scheme, _, presented = authorization.partition(" ") - if scheme.lower() != "bearer" or not presented or " " in presented: - return False - return token_matches(presented, token) - - -def declared_length(header: Optional[str]) -> Optional[int]: - if header is None or header == "": - return None - try: - length = int(header) - except (TypeError, ValueError): - return None - return length if length >= 0 else None - - -async def read_bounded(source, max_bytes: int) -> bytes: - """Stop once the cap is passed. The caller rejects the oversize body. - - A request stream is async-iterable. An UploadFile is not: it has read(n). - """ - out = bytearray() - read = getattr(source, "read", None) - if read is not None and not hasattr(source, "__aiter__"): - while True: - chunk = await read(65536) - if not chunk: - break - out += chunk - if len(out) > max_bytes: - break - return bytes(out) - async for chunk in source: - if not chunk: - continue - out += chunk - if len(out) > max_bytes: - break - return bytes(out) - - -class BodyLimit: - """Stop the ASGI receive once the body passes the cap. - - A handler that checks Content-Length, or reads request.body(), runs after - the server has already buffered the stream. This counts bytes at receive - and returns 413 without handing the rest to the app. WebSocket frames are - not HTTP bodies and are left alone. - """ - - def __init__(self, app: Any, max_bytes: int) -> None: - self.app = app - self.max_bytes = int(max_bytes) - - async def __call__(self, scope, receive, send) -> None: - if scope.get("type") != "http": - await self.app(scope, receive, send) - return - class TooLarge(Exception): - pass - - seen = 0 - async def limited() -> Dict[str, Any]: - nonlocal seen - message = await receive() - if message.get("type") != "http.request": - return message - body = message.get("body") or b"" - seen += len(body) - if seen > self.max_bytes: - raise TooLarge - return message - - try: - await self.app(scope, limited, send) - except TooLarge: - await send( - { - "type": "http.response.start", - "status": 413, - "headers": [(b"content-type", b"application/json")], - } - ) - await send( - { - "type": "http.response.body", - "body": b'{"error":{"code":"PAYLOAD_TOO_LARGE","message":"request body too large"}}', - } - ) - - -SERVICE_TYPE = "_dorayaki._tcp" - - -def _lan_address() -> Optional[str]: - """Address a phone on this network can open. Loopback is not one. - - The demo launcher uses `ipconfig getifaddr en0` then `en1`. A UDP connect - is the fallback when those interfaces are absent. - """ - if shutil.which("ipconfig"): - for iface in ("en0", "en1"): - try: - out = subprocess.check_output( - ["ipconfig", "getifaddr", iface], stderr=subprocess.DEVNULL, text=True - ).strip() - except (subprocess.CalledProcessError, OSError): - continue - if out and not is_loopback_host(out): - return out - try: - probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - probe.connect(("192.0.2.1", 9)) - found = probe.getsockname()[0] - finally: - probe.close() - except OSError: - return None - if not found or is_loopback_host(found): - return None - return found - - -def advertise_host(bind_host: Optional[str]) -> Optional[str]: - """The address clients should browse. A loopback bind is not advertised. - - A wildcard bind (0.0.0.0, ::) has no address of its own, so the record - uses the LAN address the demo launcher would hand a phone. - """ - text = (bind_host or "").strip() - if text.startswith("[") and text.endswith("]"): - text = text[1:-1] - if is_loopback_host(text): - return None - if text in ("", "0.0.0.0", "::"): - return _lan_address() - try: - parsed = ipaddress.ip_address(text) - except ValueError: - return None - if parsed.is_loopback or parsed.is_unspecified: - return None if parsed.is_loopback else _lan_address() - return str(parsed) - - -class ServiceAdvertiser: - """Register `_dorayaki._tcp` with the system mDNS daemon and drop it on close. - - ponytail: macOS `dns-sd` only. A Linux host needs `avahi-publish` wired the - same way; do not add a Python mDNS package for this one record. - """ - - def __init__(self, host: str, port: int, name: str = "Dorayaki") -> None: - self.host = host - self.port = int(port) - self.name = name - self._process: Optional[subprocess.Popen] = None - - def start(self) -> bool: - if self._process is not None: - return True - binary = shutil.which("dns-sd") - if not binary: - return False - self._process = subprocess.Popen( - [binary, "-P", self.name, SERVICE_TYPE, "local", str(self.port), self.host, self.host], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, - ) - return True - - def close(self) -> None: - process = self._process - self._process = None - if process is None or process.poll() is not None: - return - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - return - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - return - process.wait(timeout=2) diff --git a/recognization/code/web/app.js b/recognization/code/web/app.js index 5eb75cd..6661b0e 100644 --- a/recognization/code/web/app.js +++ b/recognization/code/web/app.js @@ -3,24 +3,12 @@ * * A client of the recognition server, nothing else: live frames go out over * /v1/stream, stills go out over /v1/recognize. Point it at another server with - * ?server=https://host:port. A token, when the server requires one, comes from - * sessionStorage (key dorayaki.token) or the fragment #token=..., never the query. + * ?server=https://host:port and pass a token with ?token=... */ const params = new URLSearchParams(location.search); const SERVER = (params.get('server') || '').replace(/\/$/, ''); -const TOKEN = readToken(); - -function readToken() { - const stored = sessionStorage.getItem('dorayaki.token') || ''; - const fragment = new URLSearchParams((location.hash || '').replace(/^#/, '')); - const fromHash = fragment.get('token') || ''; - if (fromHash) { - sessionStorage.setItem('dorayaki.token', fromHash); - history.replaceState(null, '', location.pathname + location.search); - } - return fromHash || stored; -} +const TOKEN = params.get('token') || ''; const I18N = { en: { @@ -375,7 +363,7 @@ function thumbUrl(skuId) { async function listCameras() { const select = el('sourceSelect'); - select.replaceChildren(); + select.innerHTML = ''; if (!navigator.mediaDevices || !window.isSecureContext) { const option = document.createElement('option'); option.value = 'file'; @@ -402,16 +390,13 @@ function connectStream() { if (state.ws && (state.ws.readyState === WebSocket.OPEN || state.ws.readyState === WebSocket.CONNECTING)) return; setConnection('connecting'); const base = SERVER || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}`; - const url = `${base.replace(/^http/, 'ws')}/v1/stream`; + const url = `${base.replace(/^http/, 'ws')}/v1/stream${TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ''}`; const socket = new WebSocket(url); socket.binaryType = 'arraybuffer'; state.ws = socket; socket.onopen = () => { state.reconnectDelay = 1000; - // Browsers cannot set Authorization on a WebSocket. The server hello - // arrives first; this frame is the token, and it is not a URL. - if (TOKEN) socket.send(JSON.stringify({ type: 'auth', token: TOKEN })); setConnection('live'); socket.send(JSON.stringify({ type: 'config', method: state.method, empty_slots: state.showEmpty })); startSending(); @@ -479,25 +464,25 @@ function sendFrame() { function renderCompliance(compliance) { const strip = el('complianceStrip'); - strip.replaceChildren(); if (!compliance) { strip.hidden = true; return; } const rows = compliance.rows || []; + const items = rows.slice(0, 3).map( + (row) => `⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})` + ); + if (rows.length > 3) { + items.push(`+${rows.length - 3} ${t('more')}`); + } const hints = compliance.hints || []; - const chips = []; - for (const row of rows.slice(0, 3)) { - chips.push(`⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})`); + if (!items.length && hints.length) { + strip.hidden = false; + strip.dataset.kind = 'hint'; + strip.innerHTML = hints.map((hint) => `${hint.detail}`).join(''); + return; } - if (rows.length > 3) chips.push(`+${rows.length - 3} ${t('more')}`); - const texts = chips.length ? chips : hints.map((hint) => hint.detail); - if (!texts.length) { strip.hidden = true; return; } + if (!items.length) { strip.hidden = true; return; } strip.hidden = false; - strip.dataset.kind = chips.length ? 'violation' : 'hint'; - for (const text of texts) { - const item = document.createElement('span'); - item.className = 'compliance-item'; - item.textContent = text; - strip.appendChild(item); - } + strip.dataset.kind = 'violation'; + strip.innerHTML = items.join(''); } function handleResult(message) { @@ -551,7 +536,7 @@ async function loadSamples() { const response = await fetch(api('/v1/samples'), { headers: headers() }); const data = await response.json(); const select = el('sampleSelect'); - select.replaceChildren(); + select.innerHTML = ''; for (const sample of data.samples) { const option = document.createElement('option'); option.value = sample.id; @@ -649,42 +634,24 @@ function renderInView() { } } } - list.replaceChildren(); if (!counts.size) { - const empty = document.createElement('div'); - empty.className = 'placeholder small'; - empty.textContent = t('inviewPlaceholder'); - list.appendChild(empty); + list.innerHTML = `
${t('inviewPlaceholder')}
`; return; } - const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]); - for (const [skuId, facings] of ranked) { + const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([skuId, facings]) => { const product = productFor(skuId); const name = product ? product.product_name : skuId; const price = product ? `¥${Math.round(product.price)}` : ''; - const row = document.createElement('div'); - row.className = 'inview-row'; - row.dataset.sku = skuId; - const thumb = document.createElement('img'); - thumb.className = 'inview-thumb'; - thumb.alt = ''; - thumb.src = thumbUrl(skuId); - const text = document.createElement('div'); - text.className = 'inview-text'; - const nameEl = document.createElement('div'); - nameEl.className = 'inview-name'; - nameEl.textContent = name; - const meta = document.createElement('div'); - meta.className = 'inview-meta'; - meta.textContent = `${price} · ${t('facings')} ${facings}`; - text.append(nameEl, meta); - const count = document.createElement('div'); - count.className = 'inview-facings'; - count.textContent = String(facings); - row.append(thumb, text, count); - row.addEventListener('click', () => selectBySku(skuId)); - list.appendChild(row); - } + return `
+ +
${name}
+
${price} · ${t('facings')} ${facings}
+
${facings}
`; + }).join(''); + list.innerHTML = rows; + list.querySelectorAll('.inview-row').forEach((row) => { + row.addEventListener('click', () => selectBySku(row.dataset.sku)); + }); } function selectBySku(skuId) { @@ -748,19 +715,9 @@ function renderOcrSection(tag, reconciliation) { if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]); } if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']); - rows.replaceChildren(); - for (const [key, value, bad] of lines) { - const row = document.createElement('div'); - row.className = 'ocr-row'; - const keyEl = document.createElement('span'); - keyEl.className = 'ocr-key'; - keyEl.textContent = key; - const valueEl = document.createElement('span'); - valueEl.className = bad ? 'ocr-val bad' : 'ocr-val'; - valueEl.textContent = value; - row.append(keyEl, valueEl); - rows.appendChild(row); - } + rows.innerHTML = lines + .map(([key, value, bad]) => `
${key}${value}
`) + .join(''); } function showProductCard(product, identity) { @@ -798,22 +755,12 @@ function showCandidatesCard(identity) { el('metricReceipt').textContent = '—'; el('metricExpiry').textContent = '—'; const candidates = el('candidates'); - candidates.replaceChildren(); - const title = document.createElement('div'); - title.className = 'candidates-title'; - title.textContent = t('candidatesTitle'); - candidates.appendChild(title); - for (const candidate of identity.candidates || []) { + const rows = (identity.candidates || []).map((candidate) => { const product = productFor(candidate.sku_id); - const row = document.createElement('div'); - row.className = 'candidate-row'; - const name = document.createElement('span'); - name.textContent = product ? product.product_name : candidate.sku_id; - const score = document.createElement('span'); - score.textContent = String(candidate.score); - row.append(name, score); - candidates.appendChild(row); - } + const name = product ? product.product_name : candidate.sku_id; + return `
${name}${candidate.score}
`; + }).join(''); + candidates.innerHTML = `
${t('candidatesTitle')}
${rows}`; candidates.hidden = false; el('showMatchBtn').disabled = false; renderOcrSection(state.currentTag, state.currentReconciliation); @@ -942,18 +889,10 @@ function handleInspect(result) { if (result.crop_jpeg_b64) { el('productThumb').src = result.crop_jpeg_b64; } - const box = el('matchBox'); - box.replaceChildren(); if (result.match_visualization_jpeg_b64) { - const image = document.createElement('img'); - image.alt = 'feature matches'; - image.src = result.match_visualization_jpeg_b64; - box.appendChild(image); + el('matchBox').innerHTML = `feature matches`; } else { - const placeholder = document.createElement('span'); - placeholder.className = 'match-placeholder'; - placeholder.textContent = t('matchPlaceholder'); - box.appendChild(placeholder); + el('matchBox').innerHTML = `${t('matchPlaceholder')}`; } const identity = result.identity || {}; const parts = []; diff --git a/recognization/demo-images/ATTRIBUTION.json b/recognization/demo-images/ATTRIBUTION.json deleted file mode 100644 index 2712674..0000000 --- a/recognization/demo-images/ATTRIBUTION.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "file": "demo-image-1-mediya-dairy.jpg", - "title": "File:Azabudai-Hills---2025-04-05 015.jpg", - "license": "CC0", - "artist": "RuinDig/Yuki Uchida", - "source": "https://commons.wikimedia.org/wiki/File%3AAzabudai-Hills---2025-04-05_015.jpg" - }, - { - "file": "demo-image-2-milk-case.jpg", - "title": "File:Milk (2655574568).jpg", - "license": "CC BY 2.0", - "artist": "yumtan", - "source": "https://commons.wikimedia.org/wiki/File%3AMilk_%282655574568%29.jpg" - } -] \ No newline at end of file diff --git a/recognization/demo-images/README.md b/recognization/demo-images/README.md deleted file mode 100644 index e5a118d..0000000 --- a/recognization/demo-images/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# The two demo shelves - -Two real store photographs, chosen because they are dense and their products are -visually distinct, with a store catalog enrolled for each. Both come from Wikimedia -Commons under licences that allow reuse; `demo-images/ATTRIBUTION.json` records the -author, licence and source of each. - -| | image 1 | image 2 | -| --- | --- | --- | -| file | `demo-images/demo-image-1-mediya-dairy.jpg` | `demo-images/demo-image-2-milk-case.jpg` | -| scene | a MEDIYA dairy case | a supermarket milk case | -| licence | **CC0** | **CC BY 2.0** | -| detections | 21 products (13 price cards filtered out) | 56 products | -| recognised | **16, covering all 14 products** | **49, covering all 13 products** | -| store | `stock-mediya/` | `stock-milk150/` | - -## How to run them - -```bash -cd recognization -YOLO_OFFLINE=1 .venv/bin/python code/app.py \ - --port 8090 --identity gallery --live-conf 0.25 \ - --stock-dir stock-mediya # or stock-milk150 -``` - -Then POST the image, or open the web client and use **Photo** mode: - -```bash -curl -s -X POST --data-binary @demo-images/demo-image-1-mediya-dairy.jpg \ - -H 'content-type: image/jpeg' \ - 'http://127.0.0.1:8090/v1/recognize?classify=true&conf=0.25&tags=true' | python3 -m json.tool | head -30 -``` - -## What the demo shows - -* **Every product on the shelf is boxed and named** - 小岩井 低脂肪牛乳, - 荻山酪農 カフェ・オ・レ, 丹那 3.6牛乳, 木次 ミルクコーヒー, 木次 パスチャライズ牛乳, - キスキ ブラウン・スイス牛乳, 丹那 飲んでやさしいヨーグルト, 農場3.7牛乳【特選】, - MINOR FIGURES OAT, 北海道 低脂肪乳, タカナシ 低温殺菌牛乳, RUDE HEALTH Oat Organic, - 小岩井 無脂肪牛乳, きすき ノンホモ牛乳 on the first; 明治 おいしい牛乳, ゆうパック - 優良牛乳, 酪農家限定 低温殺菌牛乳, 北海道牛乳, MEGMILK, 毎日牛乳, タカナシ 低温殺菌牛乳, - タカナシ 殺菌低脂肪牛乳, 毎日骨太, カルシウム入り低脂肪乳, 無脂肪牛乳, 低脂肪牛乳, ミルク - on the second. -* **Price cards are not boxed as products.** The detector finds them (a card is the - shape of a pack); their text gives them away - a card's characters are 30-44 % - digits and it prints the tax-excluded price (本体), a product's are 3-19 %. Measured - on image 1: 13 cards dropped, 21 products kept. -* **What it cannot name, it does not invent.** Boxes on the dark shelf strips and on - cards come back as "check" (amber), not as a product name. - -## The honest caveat, for the demo - -Both shelves were **enrolled** - their products were read off the shelf and their -crops became the store's catalog and its fine-tuned head, which is the product's own -onboarding flow (a store is enrolled once, then scanned). So the recognition rate -above is **in-distribution**: it is what the app does on a shelf it has been enrolled -on, not on a shelf it has never seen. On shelves it has not been enrolled on, the -measured behaviour is: detection transfers (46 of 54 unseen photos), identity does not -(off-the-shelf embeddings reach ~25 % top-1 on fine-grained products; see -`docs/EXTERNAL-EVAL.md`). - -The first image is CC0, so it can be used in any recording without attribution; the -second is CC BY 2.0 and needs the credit in `ATTRIBUTION.json` if it appears on screen. diff --git a/recognization/demo-images/demo-image-1-mediya-dairy.jpg b/recognization/demo-images/demo-image-1-mediya-dairy.jpg deleted file mode 100644 index 0bba5fb..0000000 Binary files a/recognization/demo-images/demo-image-1-mediya-dairy.jpg and /dev/null differ diff --git a/recognization/demo-images/demo-image-2-milk-case.jpg b/recognization/demo-images/demo-image-2-milk-case.jpg deleted file mode 100644 index 3e5cf1d..0000000 Binary files a/recognization/demo-images/demo-image-2-milk-case.jpg and /dev/null differ diff --git a/recognization/detected_empty_slots.jpg b/recognization/detected_empty_slots.jpg new file mode 100644 index 0000000..d24ce1e Binary files /dev/null and b/recognization/detected_empty_slots.jpg differ diff --git a/recognization/detected_image.jpg b/recognization/detected_image.jpg new file mode 100644 index 0000000..ce539b1 Binary files /dev/null and b/recognization/detected_image.jpg differ diff --git a/recognization/docs/API.md b/recognization/docs/API.md index 989c2c0..d25fd88 100644 --- a/recognization/docs/API.md +++ b/recognization/docs/API.md @@ -377,9 +377,12 @@ The protocol is capture-source agnostic; an iPad/phone client needs no server ch ## Tests ```bash -cd recognization -YOLO_OFFLINE=1 YOLO_AUTOINSTALL=false python -m unittest discover -s code -p 'test_*.py' -v +cd recognization/code +YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_observe test_engine test_tracking test_live test_api -v +DORAYAKI_REAL_MODEL=1 YOLO_OFFLINE=1 ../.venv/bin/python -m unittest test_real_model -v +cd .. && YOLO_OFFLINE=1 .venv/bin/python code/test_suite.py # the original CLI suite ``` -`test_real_model` is discovered and skipped unless `DORAYAKI_REAL_MODEL=1`. -CI installs `requirements.lock` and runs the same command. +Run tests with an explicit module list: `unittest discover` would import +`test.py`/`fast_test.py`, which are scratch scripts with hardcoded paths from another +machine and a blocking OpenCV window. diff --git a/recognization/docs/EXTERNAL-EVAL.md b/recognization/docs/EXTERNAL-EVAL.md deleted file mode 100644 index 4bb0d9a..0000000 --- a/recognization/docs/EXTERNAL-EVAL.md +++ /dev/null @@ -1,285 +0,0 @@ -# Evaluation on real store photos from Wikimedia Commons - -Everything else in this repository is measured on two photographs. This evaluates the -pipeline on **54 real store photos it has never seen**, collected from Wikimedia -Commons with their licences (CC0 / CC BY / CC BY-SA / public domain) recorded in a -manifest. - -Reproduce: - -```bash -python code/collect_shelf_images.py ~/dorayaki-data/shelf-images # needs network -python code/eval_external_images.py ~/dorayaki-data/shelf-images -``` - -## 1. Detection transfers, and well - -| | | -| --- | --- | -| Images with at least one detection | **46 of 54** | -| Total detections | **2,258** | -| Book scans, illustrations, a photo of a shop front | 0 detections (correct) | - -The detector is the teammate's single-class YOLO trained on one shelf, and it found -individual cartons, bottles and packets on dairy cases, soy-sauce shelves, spice -racks, a confectionery aisle, an Okinawan supermarket and a Hong Kong grocery - with -tight boxes. Detection is not the weak link. - -## 2. Identity transfers when the product is genuinely present - -The original 21-SKU catalog contains 明治 おいしい牛乳. On -`File:2011年4月の牛乳 (5588310773).jpg` - a milk case photographed in 2011, entirely -unseen - the pipeline found **6 facings of 明治 おいしい牛乳 and labelled them -correctly**, and on `File:Kawasaki-Inryo-1.jpg` it found 野菜生活100 and -プロビオヨーグルト R-1. So the matcher does transfer to new photographs of products it -has templates for. - -## 3. Precision is the failure, and rejection is missing entirely - -The same run labelled these as 明治 おいしい牛乳, in the same photograph: - -* 低脂肪殺菌牛乳 (a タカナシ carton, top left) -* 有機豆乳 (top left corner) -* 特濃4.5 (three cartons, top middle) -* 無調整豆乳 (top right) -* 北海道牛乳 (several, bottom right) - -They are all white milk cartons. The matcher has no way to say "unknown", so every -look-alike becomes a claim. - -Measured over 24 foreign shelves whose products are *not* in the catalog: **451 -facings accepted as recognised products**, with the score distribution overlapping -the real one and its extremes landing on the wrong images: - -| | accepted | median score | p10 | p90 | -| --- | --- | --- | --- | --- | -| The shelf the catalog describes (`image6`) | 71 | 18.2 | 5.2 | 372.4 | -| Foreign shelves (24 images) | 451 | 6.0 | — | — | -| A bento (「198円 中華弁当」) | 1 | 50.5 | | | -| A supermarket interior | 1 | 80.6 | | | -| Deer cookies at Nara | 6 | 33.1 | | 189.9 | - -**No threshold on this score separates the two**: it would have to sit above 5 (to -catch the foreign p10) and below 6 (to keep the real median), and the worst offenders -score higher than any correct match on the demo shelf. - -## 4. What this means for the plan - -It confirms, on real data, the two conclusions the literature and the review pointed -at - and it makes them concrete: - -1. **The matcher's raw inlier score is not a confidence.** It cannot be used for - rejection or for ranking, and it is the reason packaging recognition cannot name - products on its own. -2. **Fine-grained look-alikes are the failure mode**: milk cartons, soy-milk cartons, - and the twenty near-identical mayonnaise packs from the earlier benchmark. The - GroceryVision study of 190 open VLMs found the same shape (94.5 % Recall@5, a - 17.5-point drop at Recall@1): global embeddings cluster categories well and rank - visually similar SKUs badly. - -So step 4 of the plan is not a refinement, it is the fix: a **shortlist followed by -local re-ranking** (PRISM's shape: SigLIP shortlist, then LightGlue verification) and a -**calibrated rejection** on an embedding distance where same-product and -different-product are actually separable (measured earlier: above 0.6 and below 0.5 -respectively, unlike this score). - -## Attribution - -The images are from Wikimedia Commons under their individual licences (CC0, CC BY, -CC BY-SA, public domain); `manifest.json` records the author, licence and source URL -of each one. They are evaluation material and are not redistributed here. - - ---- - -# Part 2: the identity stack, measured on a shelf where the product IS present - -Part 1 measured shelves whose products are absent. This part uses -`File:2011年4月の牛乳 (5588310773).jpg` - a milk case where **14 of 107 facings are -明治 おいしい牛乳**, a product the original catalog holds - so precision and recall can -be computed instead of just false accepts. The 107 crops were labelled by eye from a -contact sheet. - -## The shipped matcher - -| | claims | correct | precision | recall | -| --- | --- | --- | --- | --- | -| matcher | 31 | 12 | **39 %** | **86 %** | - -It finds almost every true facing and makes 19 false claims, on cartons of 特濃4.5, -低脂肪殺菌牛乳, 明治ラブ and 北海道牛乳. Correct where the product is, wrong on the -look-alikes: the same shape as Part 1. - -## The embedding gallery: rejection, but not precision - -| threshold | false accepts on foreign shelves | milk shelf: precision | recall | -| --- | --- | --- | --- | -| 0.45 | 29 % | 28 % | 79 % | -| 0.50 | 18 % | 29 % | 64 % | -| 0.55 | 11 % | 35 % | 50 % | -| **0.60** | **5 %** | 45 % | 36 % | -| 0.65 | 2 % | 0 % | 0 % | - -The gallery's value is the **controllable rejection**: on shelves where nothing -matches, false accepts fall from the matcher's 47.8 % to **5 %** at 0.60, and the -curve is smooth enough to pick an operating point. But on the shelf where the product -*is* present its precision (28-45 %) is no better than the matcher's 39 % - it buys -honesty about the unknown, not finer discrimination. - -## Enrolling a shelf from itself does not fix it - -The flywheel's first step is to enroll a store's own shelf. Measured by taking 1-3 -views per product from the labelled milk shelf and testing on the rest: - -| views per SKU | top-1 | wrong | rejected | -| --- | --- | --- | --- | -| 1 | **26 %** | 30 | 29 | -| 2 | 25 % | 27 | 27 | -| 3 | 23 % | 24 | 26 | - -Flat at ~25 %, and far below the 72-76 % measured on the demo shelf with the same -number of views. The demo shelf's facings are near-identical crops of the same -photograph; real milk cartons differ by small text and a colour band. **Off-the-shelf -DINOv2 is not discriminative enough for fine-grained SKUs**, which is what the -literature says (RP2K and RPC both fine-tune; GroceryVision reports a 17.5-point drop -from Recall@5 to Recall@1). Fine-tuning is not a refinement of the plan, it is a -requirement - and its labels are what the tag channel and the confirmation loop -produce. - -## The packaging's own text is a real channel, and resolution-bound - -The cartons differ by the text printed on them, so OCR was tested as an identity -channel - first single-scale, then with the tag reader's trick (two upscales, the red -channel, pooled): - -| product | single scale | multi-scale + red | -| --- | --- | --- | -| MEGMILK (large Latin text) | 85 % | **85 %** | -| 北海道牛乳 (large kanji) | 65 % | **65 %** | -| 明治ラブ | 0 % | **75 %** | -| 特濃4.5 | 0 % | **50 %** | -| 明治 おいしい牛乳 (small vertical text) | 7 % | **28 %** | -| 低脂肪殺菌牛乳 / 無調整豆乳 / 酪農3.6 | 0 % | 0 % | -| **overall** | 51 % | **60 %** | - -Text works where it is large and fails where it is small - the same resolution wall as -the price tags. This is a capture-spec question, not a model question. - -## The identity architecture this implies - -| channel | what it is good for | measured | -| --- | --- | --- | -| Detection | finding facings on unseen shelves | 46/54 images, tight boxes | -| Tag (corroborated) | exact identity and price, where readable | 15/15 fields on the demo shelf; sparse in the field | -| Packaging text | brand-level discrimination | 60 % recall, higher when text is large | -| Embedding gallery | **rejection** - saying "unknown" | 5 % false accepts at 0.60 | -| Fine-tuned embedding | fine-grained SKUs (milk cartons) | not built; off-the-shelf is ~25 % | -| Confirmation loop | everything the above cannot settle | not built | - -So the honest design is: **reject by default**, name the product from the tag when it -is readable, from the packaging text when it is legible, from a fine-tuned embedding -otherwise - and hand the remainder to the operator, whose answers become the next -enrollment. - - ---- - -# Part 3: fine-tuning the embedding, and what it buys - -Part 2 ended on the number that matters: enrolling a real shelf from itself leaves -off-the-shelf DINOv2 at ~25 % top-1, because real milk cartons differ by small text -and a colour band. The literature's answer is fine-tuning, so this measures it. - -`code/finetune_identity.py` trains a small projection head (384 -> 128, cosine -classifier, angular margin) on frozen DINOv2 features, with augmentation standing in -for the second visit we do not have. Two enrolment views per product, twelve -augmented copies each, 72 held-out facings, 8 products. Balanced accuracy is reported -because the shelf is unbalanced (MEGMILK has 34 facings, 酪農3.6 has two - a system -that always answered "MEGMILK" would look respectable on the raw rate). - -| | top-1 | balanced | answered | of those right | -| --- | --- | --- | --- | --- | -| raw DINOv2 cosine | 2/72 (2 %) | 1 % | 72 | 3 % | -| raw, rejecting below 0.55 | 2/72 (2 %) | 1 % | 44 | 5 % | -| **fine-tuned + margin** | 28/72 (38 %) | **73 %** | 72 | 39 % | -| **fine-tuned, rejecting below 0.4** | 17/72 (23 %) | 58 % | 18 | **94 %** | - -Three things follow. - -1. **Fine-tuning is not a refinement, it is the difference between working and not.** - Balanced accuracy goes from **1 % to 73 %** on the same held-out facings. Off-the- - shelf features cannot separate these products; a head trained on sixteen crops can. -2. **With a rejection threshold it becomes precise**: answering only 18 of 72 facings - and being right on 94 % of them is exactly the operating point a store wants - - name what you are sure of, and hand the rest to the operator. The raw model cannot - do this at any threshold (5 % of answered). -3. **Where it fails is where the labels are coarse.** Per class, with rejection: - 明治ラブ 2/2, おいしい無調整豆乳 2/2, 低脂肪 1/1, 特濃4.5 1/2, 北海道牛乳 8/21, - 明治 おいしい牛乳 2/12, and **MEGMILK 1/32**. MEGMILK is the class I labelled most - coarsely - the contact sheet shows what are probably several different products - under one brand - so the class is internally inconsistent and the model correctly - refuses most of it. Better labels, not a better model, is what that needs. - -## The honest limit - -Enrolment and held-out facings come from **one photograph**. Augmentation stands in -for a second visit, which is not the same thing: a real second visit changes the -light, the angle and the stock. So 73 % balanced and 94 % precision are **upper -bounds**, and the only way to know the real number is to record a second visit -(`docs/DATA.md`). - -What is settled is the mechanism: **a head trained on labels the pipeline can produce -itself** - tag-named facings, plus operator confirmations - moves fine-grained -identity from useless to usable, and a rejection threshold turns it into something a -store can act on. - - ---- - -# Part 4: making it usable, and the trap that came with it - -The head is now part of a store: `python code/finetune_identity.py --stock stock-lopia` -trains on that store's enrolled views and writes `/identity_head.pt`, which -`ProductGallery` loads automatically. The first attempt exposed a trap worth writing -down. - -## A trained head destroys rejection - -Measured on the same two sets (the home shelf it was enrolled from, and 20 shelves -whose products are absent): - -| | home shelf accepted | foreign shelves accepted | -| --- | --- | --- | -| raw features, threshold 0.55 | 78 % | **5 %** | -| trained head, threshold 0.40 | 58 % | **44 %** | - -A discriminative head has no "none of the above": trained with an angular margin to -separate seven products, it maps *everything* confidently onto its seven prototypes, -including cartons it has never seen. Rejection, which was the gallery's whole point, -disappears. - -## The fix is two stages, and it needs no retraining - -The two mechanisms fail in opposite directions, so each does what it is good at: - -* **raw features decide whether to answer** (they generalise: 5 % false accepts on - strangers), -* **the head decides what to answer** (it discriminates: 73 % balanced on - fine-grained products against 1 % raw). - -| | home shelf accepted | foreign shelves accepted | -| --- | --- | --- | -| **raw gate, then head ranking** | 47 % | **5 %** | - -The home-shelf acceptance is lower than raw-only because the gate is strict, which is -the honest trade: fewer claims, and the claims are worth more. - -## Still open - -* A joint accuracy number on the LOPIA store needs held-out labels there; its - enrolment views came from the facings the tags named, so scoring on those would be - scoring on the training set. -* Training the head *with* out-of-distribution negatives (crops from other stores) - should let it keep rejecting on its own, which would remove the need for the gate. -* Everything above is still one photograph per store. The second visit remains the - measurement that settles it. diff --git a/recognization/pyproject.toml b/recognization/pyproject.toml deleted file mode 100644 index a97a54a..0000000 --- a/recognization/pyproject.toml +++ /dev/null @@ -1,33 +0,0 @@ -[project] -name = "dorayaki-recognition" -version = "0.1.0" -description = "Dorayaki shelf recognition server" -requires-python = ">=3.10,<3.13" -readme = "README.md" -dependencies = [ - "torch>=2.0.0", - "torchvision>=0.15.0", - "ultralytics==8.4.153", - "lap>=0.5.12", - "websockets>=13", - "opencv-python>=4.8.0", - "numpy>=1.23.0,<2.0.0", - "fastapi>=0.100.0", - "uvicorn>=0.20.0", - "python-multipart>=0.0.6", - "requests>=2.28.0", - "pyobjc-framework-Vision>=10; sys_platform == 'darwin'", - "pyobjc-framework-Quartz>=10; sys_platform == 'darwin'", -] - -[project.optional-dependencies] -dev = ["httpx>=0.27"] - -[tool.pytest.ini_options] -testpaths = ["code"] -python_files = ["test_*.py"] -python_functions = ["test_*"] -addopts = ["-q"] - -[tool.setuptools] -py-modules = [] diff --git a/recognization/requirements.lock b/recognization/requirements.lock deleted file mode 100644 index 06fa8eb..0000000 --- a/recognization/requirements.lock +++ /dev/null @@ -1,79 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile requirements.txt requirements-dev.txt --universal --python-version 3.12 -o requirements.lock --no-annotate -annotated-doc==0.0.5 -annotated-types==0.8.0 -anyio==4.15.1 -certifi==2026.7.22 -charset-normalizer==3.5.1 -click==8.5.0 -cloudpickle==3.1.2 -contourpy==1.3.3 -cuda-bindings==13.4.3 ; python_full_version < '3.15' and sys_platform == 'linux' -cuda-pathfinder==1.8.2 ; python_full_version < '3.15' and sys_platform == 'linux' -cuda-toolkit==13.0.3.0 ; sys_platform == 'linux' -cycler==0.12.1 -fastapi==0.141.1 -filelock==4.0.4 -fonttools==4.66.0 -fsspec==2026.9.0 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.20 -jinja2==3.1.6 -kiwisolver==1.5.1 -lap==0.5.13 -markupsafe==3.0.3 -matplotlib==3.11.2 -mpmath==1.3.0 -networkx==3.7 -numpy==1.26.4 -nvidia-cublas==13.1.1.3 ; sys_platform == 'linux' -nvidia-cuda-cupti==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cuda-nvrtc==13.0.88 ; sys_platform == 'linux' -nvidia-cuda-runtime==13.0.96 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cudnn-cu13==9.24.0.43 ; sys_platform == 'linux' -nvidia-cufft==12.0.0.61 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cufile==1.15.1.6 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-curand==10.4.0.35 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cusolver==12.0.4.66 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cusparse==12.6.3.3 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' -nvidia-ml-py==13.615.71 -nvidia-nccl-cu13==2.30.7 ; sys_platform == 'linux' -nvidia-nvjitlink==13.4.92 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' -nvidia-nvtx==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -opencv-python==4.11.0.86 -packaging==26.3 -pillow==12.3.0 -polars==1.44.2 -polars-runtime-32==1.44.2 -psutil==7.2.2 -pydantic==2.13.5 -pydantic-core==2.46.5 -pyobjc-core==12.2.2 ; sys_platform == 'darwin' -pyobjc-framework-cocoa==12.2.2 ; sys_platform == 'darwin' -pyobjc-framework-coreml==12.2.2 ; sys_platform == 'darwin' -pyobjc-framework-quartz==12.2.2 ; sys_platform == 'darwin' -pyobjc-framework-vision==12.2.2 ; sys_platform == 'darwin' -pyparsing==3.3.3 -python-dateutil==2.9.0.post0 -python-multipart==0.0.32 -pyyaml==6.0.3 -requests==2.34.2 -setuptools==84.0.0 -six==1.17.0 -starlette==1.7.0 -sympy==1.14.0 -torch==2.14.0 -torchvision==0.29.0 -triton==3.8.0 ; python_full_version < '3.15' and sys_platform == 'linux' -typing-extensions==4.16.0 -typing-inspection==0.4.4 -ultralytics==8.4.153 -ultralytics-platform==0.1.65 -ultralytics-thop==2.2.1 -urllib3==2.8.0 -uvicorn==0.54.0 -websockets==17.1 diff --git a/recognization/run-demo-server.sh b/recognization/run-demo-server.sh deleted file mode 100755 index 62084e6..0000000 --- a/recognization/run-demo-server.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Start the recognition server and write ignored local app endpoints. -# -# ./run-demo-server.sh -# ./run-demo-server.sh stock-milk150 -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -cd "$(dirname "$0")" - -STOCK="${1:-stock-demo}" -PORT="${PORT:-8099}" -TOKEN_FILE=".demo-token" - -if [ ! -d "$STOCK" ]; then - echo "no such store: $STOCK" >&2 - exit 1 -fi - -IP="$(ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null || true)" -if [ -z "$IP" ]; then - echo "this Mac has no LAN address; the phone cannot reach it" >&2 - exit 1 -fi - -if [ ! -s "$TOKEN_FILE" ]; then - umask 077 - openssl rand -hex 32 > "$TOKEN_FILE" -fi -TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")" - -umask 077 -cat > "$ROOT/iosApp/Configuration/Config.local.xcconfig" < config/local.env < "$TMP" -mv "$TMP" "$PROPERTIES" - -cat <