diff --git a/recognization/.github/workflows/test.yml b/recognization/.github/workflows/test.yml new file mode 100644 index 0000000..48517a0 --- /dev/null +++ b/recognization/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: tests + +on: + push: + pull_request: + +jobs: + model-free: + runs-on: ubuntu-latest + defaults: + run: + working-directory: recognization + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install pinned dependencies + run: python -m pip install -r requirements.lock + - name: Model-free suite + env: + YOLO_OFFLINE: "1" + YOLO_AUTOINSTALL: "false" + run: python -m unittest discover -s code -p 'test_*.py' -v diff --git a/recognization/.gitignore b/recognization/.gitignore index 789e4d1..a0bb35a 100644 --- a/recognization/.gitignore +++ b/recognization/.gitignore @@ -20,3 +20,6 @@ ENV/ # Temporary weights runs/detect/train/weights/last.pt *.log +stock-*/identity_head.pt +.demo-token +config/local.env diff --git a/recognization/code/app.py b/recognization/code/app.py index 5ae917b..d3f5882 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -20,7 +20,7 @@ from __future__ import annotations import argparse import asyncio import base64 -import hmac +from contextlib import asynccontextmanager import json import logging import os @@ -74,6 +74,17 @@ from engine import ( # noqa: E402 ) from record import FrameRecorder # noqa: E402 from live import LiveSession, LiveSettings # noqa: E402 +from transport import ( # noqa: E402 + BodyLimit, + ServiceAdvertiser, + advertise_host, + bearer_matches, + declared_length, + is_loopback_host, + read_bounded, + require_token_for_bind, + token_matches, +) LOGGER = logging.getLogger("dorayaki") @@ -94,7 +105,8 @@ class ServerSettings: host: str = "127.0.0.1" port: int = 8080 token: Optional[str] = None - cors_origins: str = "*" + # Empty means same-origin only. "*" reflects every caller and is not a default. + cors_origins: str = "" max_sessions: int = DEFAULT_MAX_SESSIONS live_method: str = "hybrid" live_conf: float = 0.1 @@ -124,6 +136,26 @@ def server_settings_from_env() -> ServerSettings: ) +def load_local_token(path: str = "config/local.env") -> None: + """Fill DORAYAKI_API_TOKEN from an ignored env file. Never prints the value. + + ponytail: one KEY=VALUE file, no quoting. Upgrade path is a real env loader + once a second secret needs the same file. + """ + if os.environ.get("DORAYAKI_API_TOKEN"): + return + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.readlines() + except OSError: + return + for line in lines: + key, sep, value = line.strip().partition("=") + if key == "DORAYAKI_API_TOKEN" and sep and value: + os.environ["DORAYAKI_API_TOKEN"] = value + return + + # --------------------------------------------------------------------------- # request models # --------------------------------------------------------------------------- @@ -234,18 +266,29 @@ def create_app( scans: Any = None, ) -> FastAPI: settings = settings or server_settings_from_env() + @asynccontextmanager + async def lifespan(_app: FastAPI): + yield + for session in list(sessions.values()): + session.close() + engine.shutdown() + app = FastAPI( title="Dorayaki Recognition Server", version="1.0", description="Shelf product detection, tracking and classification for capture clients.", + lifespan=lifespan, ) - app.add_middleware( - CORSMiddleware, - allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()] or ["*"], - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], - ) + # Same-origin is the default. A configured list is echoed back; "*" is not. + cors = [o.strip() for o in settings.cors_origins.split(",") if o.strip() and o.strip() != "*"] + if cors: + app.add_middleware( + CORSMiddleware, + allow_origins=cors, + allow_credentials=False, + allow_methods=["GET", "POST"], + allow_headers=["Authorization", "Content-Type"], + ) scans = scans if scans is not None else build_legacy_scans(engine) # A live session can be recorded so a real scan becomes data to label later. @@ -260,8 +303,7 @@ def create_app( async def authorize(authorization: Optional[str] = Header(default=None)) -> None: if not settings.token: return - expected = f"Bearer {settings.token}" - if not authorization or not hmac.compare_digest(authorization, expected): + if not bearer_matches(authorization, settings.token): raise _error("UNAUTHORIZED", "missing or invalid bearer token", 401) # ---------------------------------------------------------------- health @@ -316,19 +358,22 @@ def create_app( raise _error("NOT_FOUND", f"unknown sku '{sku_id}'", 404) return {"status": "success", "product": product} - # ---------------------------------------------------------------- samples - @app.get("/v1/samples", dependencies=[Depends(authorize)]) - async def list_samples() -> Dict[str, Any]: + def _sample_listing() -> List[Dict[str, Any]]: listing = [] + import cv2 + for item in samples.values(): entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"} - import cv2 - image = cv2.imread(item["path"]) if image is not None: entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0]) listing.append(entry) - return {"samples": listing} + return listing + + @app.get("/v1/samples", dependencies=[Depends(authorize)]) + async def list_samples() -> Dict[str, Any]: + loop = asyncio.get_running_loop() + return {"samples": await loop.run_in_executor(None, _sample_listing)} @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)]) async def get_sample(sample_id: str) -> Response: @@ -350,37 +395,44 @@ def create_app( if method is not None: _require_method(method) content_type = (request.headers.get("content-type") or "").split(";")[0].strip().lower() + announced = declared_length(request.headers.get("content-length")) + if announced is not None and announced > MAX_UPLOAD_BYTES: + raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) if content_type == "multipart/form-data": - form = await request.form() + form = await request.form(max_part_size=MAX_UPLOAD_BYTES) upload = form.get("file") if upload is None or not hasattr(upload, "read"): raise _error("BAD_REQUEST", "multipart body needs a 'file' field", 400) - payload = await upload.read() + payload = await read_bounded(upload, MAX_UPLOAD_BYTES) else: if content_type and not content_type.startswith("image/") and content_type != "application/octet-stream": raise _error("UNSUPPORTED_MEDIA_TYPE", f"unsupported content type '{content_type}'", 415) - payload = await request.body() + payload = await read_bounded(request.stream(), MAX_UPLOAD_BYTES) if not payload: raise _error("BAD_REQUEST", "empty request body", 400) if len(payload) > MAX_UPLOAD_BYTES: raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413) + loop = asyncio.get_running_loop() try: - image = engine.decode_image(payload) + image = await loop.run_in_executor(None, engine.decode_image, payload) except UndecodableImage as exc: raise _error("UNDECODABLE_IMAGE", str(exc), 422) try: - result = engine.recognize( - image, - classify=classify, - method=method, - conf=conf, - with_empty_slots=empty_slots, + result = await loop.run_in_executor( + None, + lambda: engine.recognize( + image, + classify=classify, + method=method, + conf=conf, + with_empty_slots=empty_slots, + ), ) except UnknownMethod as exc: raise _error("BAD_REQUEST", str(exc), 400) if tags: - result = _attach_still_tags(result, image) + result = await loop.run_in_executor(None, _attach_still_tags, result, image) return result def _is_price_card(crop: Any) -> bool: @@ -514,7 +566,10 @@ def create_app( wanted = set(int(i) for i in body.detection_ids) detections = [d for d in detections if int(d["id"]) in wanted] started = time.perf_counter() - rows = engine.classify_boxes(stored.image, detections, method=method) + loop = asyncio.get_running_loop() + rows = await loop.run_in_executor( + None, lambda: engine.classify_boxes(stored.image, detections, method=method) + ) rows_by_id = {row["detection_id"]: row for row in rows} for detection in stored.detections: row = rows_by_id.get(int(detection["id"])) @@ -567,21 +622,24 @@ def create_app( stored = engine.image_store.get(image_id) except ImageExpired: raise _error("IMAGE_EXPIRED", f"image '{image_id}' is no longer available", 404) - return engine.inspect(stored.image, body.box, method=method) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, lambda: engine.inspect(stored.image, body.box, method=method) + ) # ---------------------------------------------------------------- live stream @app.websocket("/v1/stream") async def stream(websocket: WebSocket) -> None: - if settings.token: - supplied = websocket.query_params.get("token") or "" - header = websocket.headers.get("authorization") or "" - expected_header = f"Bearer {settings.token}" - if not ( - hmac.compare_digest(supplied, settings.token) - or hmac.compare_digest(header, expected_header) - ): - await websocket.close(code=1008) - return + # Query tokens are rejected. A wrong header is rejected before hello. + # A missing header is the browser case: hello goes out, then the first + # client text must be {"type":"auth","token":"..."} or the socket closes. + if websocket.query_params.get("token"): + await websocket.close(code=1008) + return + header = websocket.headers.get("authorization") + if settings.token and header and not bearer_matches(header, settings.token): + await websocket.close(code=1008) + return async with sessions_lock: if len(sessions) >= settings.max_sessions: await websocket.close(code=1013) @@ -644,9 +702,10 @@ def create_app( "message": "frame processing failed"}) continue if recorder is not None: - # Record what the client sent and what we saw in it, so a real - # scan can be labelled later instead of only measured on photos. - recorder.write(session_id, frame_id, payload, result) + # Disk writes must not sit on the event loop between results. + await loop.run_in_executor( + None, recorder.write, session_id, frame_id, payload, result + ) await send(result) async def handle_inspect(message: Dict[str, Any]) -> None: @@ -682,8 +741,23 @@ def create_app( processing = asyncio.create_task(processing_loop()) - async def handle_control(message: Dict[str, Any]) -> None: + authed = not settings.token or bearer_matches( + websocket.headers.get("authorization"), settings.token + ) + + async def handle_control(message: Dict[str, Any]) -> bool: + """Apply one control message. False closes the socket (auth failed).""" + nonlocal authed kind = message.get("type") + if not authed: + presented = message.get("token") if kind == "auth" else None + if isinstance(presented, str) and token_matches(presented, settings.token): + authed = True + return True + await websocket.close(code=1008) + return False + if kind == "auth": + return True if kind == "config": try: config = session.configure( @@ -694,7 +768,7 @@ def create_app( ) except (TypeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "bad config values"}) - return + return True await send({"type": "config_ack", "config": config}) elif kind == "reset": session.reset() @@ -705,6 +779,7 @@ def create_app( await handle_inspect(message) else: await send({"type": "error", "code": "BAD_MESSAGE", "message": f"unknown type '{kind}'"}) + return True try: while True: @@ -717,6 +792,9 @@ def create_app( text = message.get("text") data = message.get("bytes") if data is not None: + if not authed: + await websocket.close(code=1008) + break if len(data) < 5: await send({"type": "error", "code": "BAD_FRAME", "message": "frame shorter than 5 bytes"}) continue @@ -733,7 +811,8 @@ def create_app( except (json.JSONDecodeError, ValueError): await send({"type": "error", "code": "BAD_MESSAGE", "message": "invalid JSON"}) continue - await handle_control(parsed) + if not await handle_control(parsed): + break except WebSocketDisconnect: pass finally: @@ -747,7 +826,7 @@ def create_app( sessions.pop(session_id, None) # ---------------------------------------------------------------- legacy session routes - @app.post("/api/sessions") + @app.post("/api/sessions", dependencies=[Depends(authorize)]) async def legacy_start_session(payload: Dict[str, Any] = None) -> Dict[str, Any]: payload = payload or {} session = scans.start_session( @@ -757,7 +836,7 @@ def create_app( ) return {"status": "success", "session": session.snapshot()} - @app.post("/api/sessions/{session_id}/frames") + @app.post("/api/sessions/{session_id}/frames", dependencies=[Depends(authorize)]) async def legacy_observe_frame(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import FrameRequest, ObserveError @@ -765,11 +844,19 @@ def create_app( try: session = scans.get_session(session_id) raw = payload.get("frame_bytes_b64") or payload.get("image_base64") or "" + if not isinstance(raw, str): + raw = "" if "," in raw: raw = raw.split(",", 1)[1] + # Base64 expands ~4/3. Reject the text before decoding it. + if len(raw) > (MAX_FRAME_BYTES * 4) // 3 + 8: + raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) + frame_bytes = base64.b64decode(raw) if raw else b"" + if len(frame_bytes) > MAX_FRAME_BYTES: + raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413) request = FrameRequest( frame_id=str(payload.get("frame_id") or ""), - frame_bytes=base64.b64decode(raw) if raw else b"", + frame_bytes=frame_bytes, sequence=int(payload.get("sequence", 0)), captured_at=float(payload.get("captured_at") or 0), session_id=session.session_id, @@ -781,12 +868,13 @@ def create_app( ) if not request.frame_id: raise _error("BAD_REQUEST", "frame_id required", 400) - observation = scans.observe_frame(request) + loop = asyncio.get_running_loop() + observation = await loop.run_in_executor(None, scans.observe_frame, request) return {"status": "success", "observation": observation.as_dict(), "session": session.snapshot()} except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.get("/api/sessions/{session_id}") + @app.get("/api/sessions/{session_id}", dependencies=[Depends(authorize)]) async def legacy_get_session(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -795,7 +883,7 @@ def create_app( except ObserveError as exc: raise _error("NOT_FOUND", str(exc), 404) - @app.post("/api/sessions/{session_id}/review") + @app.post("/api/sessions/{session_id}/review", dependencies=[Depends(authorize)]) async def legacy_review(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -812,7 +900,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/finalize") + @app.post("/api/sessions/{session_id}/finalize", dependencies=[Depends(authorize)]) async def legacy_finalize(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -821,7 +909,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/cancel") + @app.post("/api/sessions/{session_id}/cancel", dependencies=[Depends(authorize)]) async def legacy_cancel(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -830,7 +918,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/sessions/{session_id}/rescan") + @app.post("/api/sessions/{session_id}/rescan", dependencies=[Depends(authorize)]) async def legacy_rescan(session_id: str) -> Dict[str, Any]: from observe import ObserveError @@ -839,7 +927,7 @@ def create_app( except ObserveError as exc: raise _error("CONFLICT", str(exc), 409) - @app.post("/api/compare_live_vs_still") + @app.post("/api/compare_live_vs_still", dependencies=[Depends(authorize)]) async def legacy_compare(payload: Dict[str, Any] = None) -> Dict[str, Any]: from observe import ObserveError @@ -881,14 +969,11 @@ def create_app( # ---------------------------------------------------------------- static web client resolved_web_dir = web_dir or os.path.join(current_dir, "web") + app.add_middleware(BodyLimit, max_bytes=MAX_UPLOAD_BYTES) if os.path.isdir(resolved_web_dir): app.mount("/", StaticFiles(directory=resolved_web_dir, html=True), name="web") - @app.on_event("shutdown") - async def _shutdown() -> None: - for session in list(sessions.values()): - session.close() - engine.shutdown() + return app @@ -960,6 +1045,11 @@ def main() -> None: if args.log_level is not None: server.log_level = args.log_level + load_local_token() + if not server.token: + server.token = os.environ.get("DORAYAKI_API_TOKEN") or None + require_token_for_bind(server.host, server.token) + engine_settings = settings_from_env() if args.record_frames is not None: server.record_frames = args.record_frames @@ -996,23 +1086,34 @@ def main() -> None: scheme = "https" if server.ssl_certfile else "http" LOGGER.info("Dorayaki recognition server on %s://%s:%d (token %s)", scheme, server.host, server.port, "required" if server.token else "off") - if server.host in ("127.0.0.1", "localhost"): + if is_loopback_host(server.host): LOGGER.info("Open %s://127.0.0.1:%d in a browser for the web client", scheme, server.port) else: LOGGER.info("LAN mode: live camera needs https (secure context); use --ssl-certfile or a tunnel") import uvicorn - uvicorn.run( - app, - host=server.host, - port=server.port, - log_level=(server.log_level or "info").lower(), - access_log=False, # access log would print ?token=... query strings - ws="websockets-sansio", - ssl_certfile=server.ssl_certfile, - ssl_keyfile=server.ssl_keyfile, - ) + advertised = advertise_host(server.host) + advertiser = ServiceAdvertiser(advertised, server.port) if advertised else None + if advertiser is not None and advertiser.start(): + LOGGER.info("Advertising _dorayaki._tcp on %s:%d", advertised, server.port) + elif advertised: + LOGGER.info("No dns-sd on this host; _dorayaki._tcp was not advertised") + + try: + uvicorn.run( + app, + host=server.host, + port=server.port, + log_level=(server.log_level or "info").lower(), + access_log=False, # request lines can carry Authorization; keep them off + ws="websockets-sansio", + ssl_certfile=server.ssl_certfile, + ssl_keyfile=server.ssl_keyfile, + ) + finally: + if advertiser is not None: + advertiser.close() if __name__ == "__main__": diff --git a/recognization/code/benchmark.py b/recognization/code/benchmark.py deleted file mode 100644 index b54f6ca..0000000 --- a/recognization/code/benchmark.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Benchmark Suite for Supermarket Product Classification -Compares SURF, ORB, BRISK, and Hybrid methods following -MDPI Electronics 2023, 12(17), 3640. -""" - -import os -import sys -import time -import cv2 -import numpy as np -from typing import Dict, List, Any - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from feature_matcher import ProductFeatureMatcher - - -def run_benchmark(stock_dir: str = "stock", num_runs: int = 3): - print("=" * 70) - print("RETAIL SHELF PRODUCT RECOGNITION - ALGORITHM BENCHMARK") - print("Methods Evaluated: SURF | ORB | BRISK | HYBRID (Paper Method)") - print("=" * 70) - - matcher = ProductFeatureMatcher(stock_dir=stock_dir) - templates = matcher.db.templates - - if not templates: - print("[Error] No templates found in stock directory.") - return - - methods = ["surf", "orb", "brisk", "hybrid"] - results = {m: {"latencies": [], "inliers": [], "keypoints_query": []} for m in methods} - - # Test images: We will evaluate using crops synthesized with slight transformations - # (scale, rotation, noise) as well as the template images themselves - print(f"\nEvaluating across {len(templates)} catalog SKUs (repeat count: {num_runs})...\n") - - for sku_id, tmpl in templates.items(): - original_img = tmpl["image"] - h, w = original_img.shape[:2] - - # Apply synthetic shelf condition (rotation by 10 deg, resize by 0.85) - M = cv2.getRotationMatrix2D((w / 2, h / 2), 10, 0.85) - query_img = cv2.warpAffine(original_img, M, (w, h)) - - for method in methods: - times = [] - for _ in range(num_runs): - t0 = time.perf_counter() - res = matcher.classify_crop(query_img, method=method) - t1 = time.perf_counter() - times.append((t1 - t0) * 1000.0) # ms - - avg_latency = np.mean(times) - results[method]["latencies"].append(avg_latency) - - # Record inliers/score - if res.get("status") == "success": - top_sku = res["best_sku"] - score = res["score"] - results[method]["inliers"].append(score) - - # Summarize metrics - summary_rows = [] - for method in methods: - latencies = results[method]["latencies"] - scores = results[method]["inliers"] - - avg_lat = np.mean(latencies) if latencies else 0.0 - p95_lat = np.percentile(latencies, 95) if latencies else 0.0 - avg_score = np.mean(scores) if scores else 0.0 - - # Paper characteristics notes - notes = { - "surf": "Scale & rotation robust; Hessian gradient features", - "orb": "Ultra fast binary descriptor; FAST + BRIEF", - "brisk": "Scale-space AGAST; circular pattern descriptor", - "hybrid": "Paper fusion (SURF+ORB+BRISK); optimal accuracy/speed" - }[method] - - summary_rows.append([ - method.upper(), - f"{avg_lat:.2f} ms", - f"{p95_lat:.2f} ms", - f"{avg_score:.1f}", - notes - ]) - - headers = ["Method", "Avg Latency (ms)", "P95 Latency (ms)", "Avg Match Score", "Characteristics / Paper Findings"] - print("\n" + "-" * 105) - print(f"{headers[0]:<10} | {headers[1]:<18} | {headers[2]:<18} | {headers[3]:<16} | {headers[4]}") - print("-" * 105) - for row in summary_rows: - print(f"{row[0]:<10} | {row[1]:<18} | {row[2]:<18} | {row[3]:<16} | {row[4]}") - print("-" * 105) - print("\n[Benchmark Complete] All methods operational.") - - -if __name__ == "__main__": - run_benchmark() diff --git a/recognization/code/build_store_from_shelf.py b/recognization/code/build_store_from_shelf.py index 40d66a3..589328d 100644 --- a/recognization/code/build_store_from_shelf.py +++ b/recognization/code/build_store_from_shelf.py @@ -46,16 +46,21 @@ store = "stock-mediya" shutil.rmtree(store, ignore_errors=True) os.makedirs(store, exist_ok=True) rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"] +written = [] for sku, name, indices in PRODUCTS: + saved = 0 for position, index in enumerate(indices): box = boxes[index] x1, y1, x2, y2 = [int(v) for v in box] - crop = image[max(0,y1):y2, max(0,x1):x2] + crop = image[max(0, y1):y2, max(0, x1):x2] if crop.size == 0: continue - suffix = "" if position == 0 else f"_{position}" + suffix = "" if saved == 0 else f"_{saved}" cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop) + saved += 1 + if saved == 0: + raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view") + written.append(sku) rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}") open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n") -print(f"{store}: {len(PRODUCTS)} products, " - f"{sum(len(i) for _, _, i in PRODUCTS)} views, csv written") +print(f"{store}: {len(written)} products, csv written") diff --git a/recognization/code/download_stock.py b/recognization/code/download_stock.py deleted file mode 100644 index 889e2f5..0000000 --- a/recognization/code/download_stock.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import sys -import io -import urllib.request -from PIL import Image - -# Ensure stdout handles utf-8 cleanly -if sys.platform.startswith('win'): - sys.stdout.reconfigure(encoding='utf-8') - -urls = { - 'sku_011': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_apple@2x_v2.png', '野菜生活100 アップルサラダ'), - 'sku_012': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_lemon@2x_v2.png', '野菜生活100 レモンサラダ'), - 'sku_013': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/green-product-pic_2.png', '野菜生活100 Smoothie グリーンスムージー'), - 'sku_014': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/orange-product-pic_2.png', '野菜生活100 Smoothie ビタミンスムージー'), - 'sku_015': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/yellow-product-pic_2.png', '野菜生活100 Smoothie バナナ＆アーモンドmix'), - 'sku_016': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/red-product-pic_2.png', '野菜生活100 Smoothie アサイー＆ベリーmix') -} - -os.makedirs('stock', exist_ok=True) - -for sku, (url, name) in urls.items(): - print(f'Fetching {sku}: {name} from {url}...') - req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}) - try: - with urllib.request.urlopen(req, timeout=15) as resp: - data = resp.read() - img = Image.open(io.BytesIO(data)) - if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info): - bg = Image.new('RGB', img.size, (255, 255, 255)) - if img.mode != 'RGBA': - img = img.convert('RGBA') - bg.paste(img, mask=img.split()[3]) - img = bg - else: - img = img.convert('RGB') - out_path = f'stock/{sku}.jpeg' - img.save(out_path, 'JPEG', quality=95) - print(f'Successfully saved {sku}: {name} -> {out_path} ({img.size})') - except Exception as e: - print(f'Error downloading {sku} ({url}): {e}') diff --git a/recognization/code/engine.py b/recognization/code/engine.py index c284c12..f63e714 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -251,6 +251,7 @@ class RecognitionEngine: # The embedding gallery is built on first use (it loads a model and every # enrolled view), and cached as False when it cannot be built at all. self._gallery: Any = None + self._gallery_lock = threading.Lock() workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2) self.classify_workers = int(workers) self._executor = classify_executor or ThreadPoolExecutor( @@ -417,23 +418,22 @@ class RecognitionEngine: return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier" def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: - """Score one crop against the catalog, in whatever scale this engine uses. + """Score one crop. Gallery never falls back to the matcher. - The live session calls this per track. With the gallery backend it must go - through the gallery, or a live scan would use the matcher while the still path - used the enrolled store - measured: the matcher accepts look-alikes and cannot - say "unknown", which is exactly what a real shelf needs it to do. + A missing gallery returns no scores (unknown), not inlier counts. Mixing + those scales is how a live scan accepted a look-alike the still path rejected. """ if self.identity_scale == "cosine": gallery = self.identity_gallery() - if gallery: - match = gallery.match(crop) - if match.rejected or not match.sku_id: - return {} - scores = {match.sku_id: match.similarity} - if match.runner_up: - scores[match.runner_up] = match.runner_up_similarity - return {str(sku): float(score) for sku, score in scores.items()} + if not gallery: + return {} + match = gallery.match(crop) + if match.rejected or not match.sku_id: + return {} + scores = {match.sku_id: match.similarity} + if match.runner_up: + scores[match.runner_up] = match.runner_up_similarity + return {str(sku): float(score) for sku, score in scores.items()} method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) @@ -445,27 +445,41 @@ class RecognitionEngine: def identity_gallery(self) -> Any: """The embedding gallery, loaded once (None when unavailable).""" + if self._gallery is False: + return None if self._gallery is not None: return self._gallery - from gallery import ProductGallery + with self._gallery_lock: + if self._gallery is False: + return None + if self._gallery is not None: + return self._gallery + from gallery import ProductGallery - gallery = ProductGallery( - self.settings.stock_dir, - [product["sku_id"] for product in self.catalog()], - threshold=self.settings.gallery_threshold, - head_threshold=self.settings.gallery_head_threshold, - image_size=self.settings.gallery_image_size, - ) - try: - loaded = gallery.load() - except Exception as exc: # torch/timm missing, no weights, no network - LOGGER.warning("identity gallery unavailable: %s", exc) - loaded = False - self._gallery = gallery if loaded else False - if loaded: - LOGGER.info("identity gallery: %d views over %d SKUs on %s", - gallery.view_count(), len(self.settings.stock_dir and gallery.sku_ids), gallery.device) - return self._gallery + gallery = ProductGallery( + self.settings.stock_dir, + [product["sku_id"] for product in self.catalog()], + threshold=self.settings.gallery_threshold, + head_threshold=self.settings.gallery_head_threshold, + image_size=self.settings.gallery_image_size, + ) + try: + loaded = gallery.load() + except Exception as exc: # torch/timm missing, no weights, no network + LOGGER.warning("identity gallery unavailable: %s", exc) + loaded = False + # False means tried and failed. Do not retry and do not use the matcher. + self._gallery = gallery if loaded else False + if loaded: + LOGGER.info( + "identity gallery: %d views over %d SKUs on %s", + gallery.view_count(), + len(gallery.sku_ids), + gallery.device, + ) + else: + LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized") + return gallery if loaded else None def classify_boxes( self, @@ -493,8 +507,8 @@ class RecognitionEngine: if (self.settings.identity_backend or "matcher").lower() == "gallery": gallery = self.identity_gallery() - if gallery: - return self._classify_with_gallery(gallery, ordered, crops) + # No matcher fallback: an unavailable gallery is unrecognized, not a different model. + return self._classify_with_gallery(gallery, ordered, crops) if method == "hybrid_fast": futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops] else: @@ -516,26 +530,47 @@ class RecognitionEngine: return results def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]: - """Full single-crop result: decision, per-method details, crop, visualization.""" - from identity import decide_identity + """Full single-crop result, using the same backend as recognize/live.""" + from identity import decide_identity, decision_for_match + + crop = self.crop(image, box) + if self.identity_scale == "cosine": + gallery = self.identity_gallery() + match = gallery.match(crop) if gallery else None + # match() already applied the gallery's own threshold. Ranking again + # with the published cosine gates must not invent a second line. + decision = decision_for_match(match) + return self._inspect_payload(box, "gallery", decision, crop, visualize=False) method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) - crop = self.crop(image, box) if method == "hybrid_fast": scores, details = self.matcher.score_crop_fast(crop) + visual_method = "hybrid" else: scores, details = self.matcher.score_crop(crop, method=method) + visual_method = method decision = decide_identity(scores, details=details, method=method) - identity = identity_payload(decision) + return self._inspect_payload(box, method, decision, crop, visual_method=visual_method) + def _inspect_payload( + self, + box: Sequence[int], + method: str, + decision: Dict[str, Any], + crop: Any, + *, + visual_method: Optional[str] = None, + visualize: bool = True, + ) -> Dict[str, Any]: + identity = identity_payload(decision) top_details = decision.get("details") or {} sub_details = None if isinstance(top_details, dict): raw = top_details.get("sub_details") if raw: - sub_details = {k: dict(v) for k, v in raw.items()} + sub_details = {key: dict(value) for key, value in raw.items()} product = None visualization = None @@ -547,16 +582,16 @@ class RecognitionEngine: elif identity["candidates"]: visualized_sku = identity["candidates"][0]["sku_id"] is_candidate = True - if visualized_sku: + if visualize and visualized_sku and visual_method: try: visualization = self.matcher.generate_match_visualization( - crop, visualized_sku, method=method if method != "hybrid_fast" else "hybrid" + crop, visualized_sku, method=visual_method ) except Exception: visualization = None return { - "box": [int(v) for v in box], + "box": [int(value) for value in box], "method": method, "identity": identity, "sub_details": sub_details, @@ -724,48 +759,22 @@ class RecognitionEngine: unseen store photos, the matcher labelled milk cartons as mayonnaise and its score could not be thresholded to stop it. """ + from identity import decision_for_match + results: List[Dict[str, Any]] = [] for box, crop in zip(ordered, crops): - try: - match = gallery.match(crop) - except Exception as exc: # a bad crop must not fail the batch - match = None - LOGGER.debug("gallery match failed: %s", exc) - if match is None or match.rejected: - similarity = float(match.similarity) if match is not None else 0.0 - results.append({ - "detection_id": int(box["id"]), - "box": list(box["box"]), - "identity": { - "state": "unrecognized", - "sku_id": None, - "identity_level": "rejected", - "confidence": round(similarity, 3), - "score": round(similarity, 3), - "candidates": [], - "attempts": 1, - "method": "gallery", - "message": "no enrolled product is close enough", - }, - }) - continue - runner_up = ( - {"sku_id": match.runner_up, "score": round(match.runner_up_similarity, 3)} - if match.runner_up else {} - ) + match = None + if gallery is not None: + try: + match = gallery.match(crop) + except Exception as exc: # a bad crop must not fail the batch + match = None + LOGGER.debug("gallery match failed: %s", exc) + decision = decision_for_match(match) results.append({ "detection_id": int(box["id"]), "box": list(box["box"]), - "identity": { - "state": "recognized", - "sku_id": match.sku_id, - "identity_level": "accepted", - "confidence": round(match.similarity, 3), - "score": round(match.similarity, 3), - "candidates": [runner_up] if runner_up else [], - "attempts": 1, - "method": "gallery", - }, + "identity": identity_payload(decision), }) return results diff --git a/recognization/code/eval_live.py b/recognization/code/eval_live.py new file mode 100644 index 0000000..d44a140 --- /dev/null +++ b/recognization/code/eval_live.py @@ -0,0 +1,693 @@ +"""Replay a recorded walk through LiveSession and count what it actually decided. + +The recording format is FrameRecorder's: DIR//index.jsonl plus the JPEG +bytes the client sent. Optional labels live in labels.json next to that index +(or at --labels). A label is {"frame", "box", "sku_id"}. + +Template leakage is refused, not scored. A label whose crop matches a catalog +template (byte-identical, or IoU >= 0.98 against a same-sized template pasted +into the frame) is excluded and counted under leakage. Those crops are how the +store was enrolled; scoring them is training-set accuracy. + + python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH] + [--json PATH] [--iou 0.5] + +Prints a human count table and writes the same report as JSON (default +/eval_live.json). Missing input exits 2 with {"error": ...} on +stdout. The same pinned recording, catalog, weights, and labels produce the +same counts; timings are omitted because they are not. + +No held-out labels means denominators and fingerprints only. That is not a +walk accuracy number. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +LEVELS = ("accepted", "ambiguous", "rejected", "unknown", "unsettled") +TAG_STATES = ("readable", "unreadable", "absent", "detected") + + +class EvalInputError(ValueError): + """The recording, labels, or catalog cannot be scored as given.""" + + +def file_sha256(path: str) -> Optional[str]: + digest = hashlib.sha256() + try: + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +def content_sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _iou(a: Sequence[float], b: Sequence[float]) -> float: + if len(a) != 4 or len(b) != 4: + return 0.0 + ax1, ay1, ax2, ay2 = (float(v) for v in a) + bx1, by1, bx2, by2 = (float(v) for v in b) + ix1, iy1 = max(ax1, bx1), max(ay1, by1) + ix2, iy2 = min(ax2, bx2), min(ay2, by2) + inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) + if inter <= 0.0: + return 0.0 + area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1) + area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0.0 else 0.0 + + +def load_index(recording_dir: str) -> List[Dict[str, Any]]: + path = os.path.join(recording_dir, "index.jsonl") + if not os.path.isfile(path): + raise EvalInputError(f"missing recording index: {path}") + rows: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as handle: + for number, line in enumerate(handle, start=1): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError as exc: + raise EvalInputError(f"{path}:{number}: {exc.msg}") from exc + if not isinstance(entry, dict) or "frame" not in entry or "path" not in entry: + raise EvalInputError(f"{path}:{number}: line needs frame and path") + rows.append(entry) + if not rows: + raise EvalInputError(f"empty recording index: {path}") + return rows + + +def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + """Hash the JPEGs the client sent, in index order. Missing files are an error.""" + digest = hashlib.sha256() + missing: List[str] = [] + for row in rows: + name = str(row["path"]) + path = os.path.join(recording_dir, name) + payload_hash = file_sha256(path) + if payload_hash is None: + missing.append(name) + continue + digest.update(f"{row['frame']}:{name}:{payload_hash}\n".encode("utf-8")) + if missing: + raise EvalInputError("missing recorded frames: " + ", ".join(missing[:8])) + return {"sha256": digest.hexdigest(), "frames": len(rows)} + + +def load_labels(path: Optional[str]) -> List[Dict[str, Any]]: + if not path: + return [] + if not os.path.isfile(path): + raise EvalInputError(f"missing labels: {path}") + try: + with open(path, "r", encoding="utf-8") as handle: + payload = json.loads(handle.read()) + except json.JSONDecodeError as exc: + raise EvalInputError(f"labels are not JSON: {exc.msg}") from exc + rows = payload.get("labels") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + raise EvalInputError("labels must be a list or {\"labels\": [...]}") + cleaned: List[Dict[str, Any]] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise EvalInputError(f"labels[{index}] is not an object") + if "frame" not in row or "sku_id" not in row or "box" not in row: + raise EvalInputError(f"labels[{index}] needs frame, box, sku_id") + box = row["box"] + if not isinstance(box, (list, tuple)) or len(box) != 4: + raise EvalInputError(f"labels[{index}].box must be [x1, y1, x2, y2]") + cleaned.append( + { + "frame": int(row["frame"]), + "box": [int(v) for v in box], + "sku_id": str(row["sku_id"]), + "source": row.get("source"), + } + ) + return cleaned + + +def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[str, Any]: + """Same inputs as RecognitionEngine.catalog_version, plus the full digest and head.""" + digest = hashlib.sha256() + csv_path = os.path.join(stock_dir, stock_csv) + csv_hash = file_sha256(csv_path) + if csv_hash is None: + digest.update(b"missing-csv") + else: + digest.update(bytes.fromhex(csv_hash)) + try: + entries = sorted( + (name, os.path.getsize(os.path.join(stock_dir, name))) + for name in os.listdir(stock_dir) + if name.lower().endswith((".jpeg", ".jpg", ".png")) + ) + except OSError as exc: + raise EvalInputError(f"cannot read catalog {stock_dir}: {exc.strerror}") from exc + for name, size in entries: + digest.update(f"{name}:{size}".encode("utf-8")) + full = digest.hexdigest() + head_path = os.path.join(stock_dir, "identity_head.pt") + return { + "stock_dir": os.path.basename(os.path.abspath(stock_dir)), + "version": full[:12], + "sha256": full, + "templates": len(entries), + "csv_sha256": csv_hash, + "identity_head_sha256": file_sha256(head_path), + } + + +def model_fingerprint(weights_path: str) -> Dict[str, Any]: + try: + import ultralytics + + library = f"ultralytics-{ultralytics.__version__}" + except Exception: + library = "ultralytics-unknown" + return { + "version": f"{os.path.basename(weights_path)}@{library}", + "weights_sha256": file_sha256(weights_path) if weights_path else None, + } + + +def _decode_image(payload: bytes) -> Any: + import cv2 + import numpy as np + + image = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_COLOR) + if image is None: + raise EvalInputError("recorded frame is not a decodable image") + return image + + +def template_crops(stock_dir: str) -> List[Tuple[str, Any]]: + """Catalog photos keyed by SKU stem (sku_l05, sku_l05_1, ...).""" + import cv2 + + crops: List[Tuple[str, Any]] = [] + try: + names = sorted(os.listdir(stock_dir)) + except OSError: + return crops + for name in names: + stem, extension = os.path.splitext(name) + if extension.lower() not in (".jpeg", ".jpg", ".png"): + continue + image = cv2.imread(os.path.join(stock_dir, name)) + if image is None or image.size == 0: + continue + crops.append((stem, image)) + return crops + + +def _same_picture(crop: Any, template: Any) -> bool: + """True when two same-sized images are the same photo after JPEG.""" + import numpy as np + + left = crop.astype("float32").reshape(-1) + right = template.astype("float32").reshape(-1) + if left.size < 16 or left.size != right.size: + return False + left = left - float(left.mean()) + right = right - float(right.mean()) + denom = float(np.sqrt((left * left).sum() * (right * right).sum())) + if denom <= 1e-6: + return bool(np.max(np.abs(crop.astype("int16") - template.astype("int16"))) <= 2) + return float((left * right).sum() / denom) >= 0.98 + + +def leaked_labels( + recording_dir: str, + rows: Sequence[Dict[str, Any]], + labels: Sequence[Dict[str, Any]], + stock_dir: str, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Drop labels whose crop is a catalog template. + + Byte identity catches a frame that is the template file. Geometric identity + catches a template pasted into a larger frame at the labelled box. + """ + if not labels: + return [], [] + templates = template_crops(stock_dir) + template_hashes = {} + try: + names = os.listdir(stock_dir) + except OSError: + names = [] + for name in names: + digest = file_sha256(os.path.join(stock_dir, name)) + if digest: + template_hashes.setdefault(digest, os.path.splitext(name)[0]) + by_frame: Dict[int, List[Dict[str, Any]]] = {} + for label in labels: + by_frame.setdefault(int(label["frame"]), []).append(label) + held: List[Dict[str, Any]] = [] + leaked: List[Dict[str, Any]] = [] + frames = {int(row["frame"]): row for row in rows} + for frame_id, frame_labels in by_frame.items(): + row = frames.get(frame_id) + image = None + payload_hash = None + if row is not None: + path = os.path.join(recording_dir, str(row["path"])) + try: + with open(path, "rb") as handle: + payload = handle.read() + except OSError: + payload = b"" + payload_hash = content_sha256(payload) if payload else None + if payload: + image = _decode_image(payload) + for label in frame_labels: + reason = None + matched = None + if payload_hash in template_hashes: + reason = "frame_is_template" + matched = template_hashes[payload_hash] + elif image is not None: + x1, y1, x2, y2 = label["box"] + crop = image[max(0, y1) : max(0, y2), max(0, x1) : max(0, x2)] + if crop.size: + import cv2 + + ch, cw = crop.shape[:2] + for stem, template in templates: + th, tw = template.shape[:2] + if abs(th - ch) > 1 or abs(tw - cw) > 1: + continue + resized = template if (th, tw) == (ch, cw) else cv2.resize(template, (cw, ch)) + # JPEG re-encoding keeps layout and drops exact pixels. + if _same_picture(crop, resized): + reason = "crop_is_template" + matched = stem + break + if reason: + leaked.append({**label, "leak": reason, "template": matched}) + else: + held.append(label) + return held, leaked + + +def identity_level(track: Dict[str, Any]) -> str: + identity = track.get("identity") or {} + level = identity.get("identity_level") + if level in LEVELS: + return str(level) + state = identity.get("state") + if state == "recognized": + return "accepted" + if state == "needs_confirmation": + return "ambiguous" + if state == "unrecognized": + return "rejected" + return "unsettled" + + +def tag_state(track: Dict[str, Any]) -> str: + tag = track.get("price_tag") or track.get("tag") or {} + state = tag.get("state") + if state in TAG_STATES: + return str(state) + if tag: + return "detected" + return "absent" + + +def score_tracks(tracks: Iterable[Dict[str, Any]]) -> Dict[str, Any]: + """Denominators over settled decisions. Unsettled tracks are not in the rate.""" + counts = {level: 0 for level in LEVELS} + tags = {state: 0 for state in TAG_STATES} + for track in tracks: + counts[identity_level(track)] += 1 + tags[tag_state(track)] += 1 + settled = counts["accepted"] + counts["ambiguous"] + counts["rejected"] + counts["unknown"] + return { + "tracks": sum(counts.values()), + "settled": settled, + "accepted": counts["accepted"], + "ambiguous": counts["ambiguous"], + "rejected": counts["rejected"], + "unknown": counts["unknown"], + "unsettled": counts["unsettled"], + "accepted_rate": _rate(counts["accepted"], settled), + "ambiguous_rate": _rate(counts["ambiguous"], settled), + "rejected_rate": _rate(counts["rejected"], settled), + "tags_detected": tags["detected"] + tags["readable"] + tags["unreadable"], + "tags_readable": tags["readable"], + "tags_unreadable": tags["unreadable"], + "tags_absent": tags["absent"], + } + + +def _rate(numerator: int, denominator: int) -> Optional[float]: + if denominator <= 0: + return None + return round(numerator / denominator, 4) + + +def track_churn(frames: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + """How often one track id changes the SKU it is willing to name. + + A change counts only after the id has already named a SKU. Restarting or + dropping the label (relabels/restarts) is reported beside that, from the + counters LiveSession already publishes. + """ + previous: Dict[int, Optional[str]] = {} + changes: Dict[int, int] = {} + seen: Dict[int, int] = {} + relabels: Dict[int, int] = {} + restarts: Dict[int, int] = {} + for frame in frames: + for track in frame.get("tracks") or []: + track_id = track.get("track_id") + if track_id is None: + continue + track_id = int(track_id) + seen[track_id] = seen.get(track_id, 0) + 1 + relabels[track_id] = max(relabels.get(track_id, 0), int(track.get("relabels") or 0)) + restarts[track_id] = max(restarts.get(track_id, 0), int(track.get("restarts") or 0)) + named = None + if identity_level(track) == "accepted": + named = (track.get("identity") or {}).get("sku_id") + prior = previous.get(track_id) + if prior and named and named != prior: + changes[track_id] = changes.get(track_id, 0) + 1 + if named: + previous[track_id] = named + churned = sum(1 for count in changes.values() if count) + return { + "tracks": len(seen), + "tracks_with_sku_change": churned, + "sku_changes": sum(changes.values()), + "relabels": sum(relabels.values()), + "restarts": sum(restarts.values()), + "per_track": [ + { + "track_id": track_id, + "observations": seen[track_id], + "sku_changes": changes.get(track_id, 0), + "relabels": relabels.get(track_id, 0), + "restarts": restarts.get(track_id, 0), + } + for track_id in sorted(seen) + if changes.get(track_id, 0) or relabels.get(track_id, 0) or restarts.get(track_id, 0) + ], + } + + +def labelled_top1( + frames: Sequence[Dict[str, Any]], + labels: Sequence[Dict[str, Any]], + *, + iou_min: float, +) -> Dict[str, Any]: + """Top-1 over held-out labels. Accepted wrong SKU is a miss, not a reject. + + Denominator is labels that overlap a track. A label with no overlapping + track is unmatched and stays out of the top-1 rate. + """ + by_frame: Dict[int, List[Dict[str, Any]]] = {} + for frame in frames: + by_frame.setdefault(int(frame.get("frame", frame.get("frame_id"))), []).append(frame) + correct = wrong = rejected = ambiguous = missed = 0 + matched = 0 + for label in labels: + observations = by_frame.get(int(label["frame"]), []) + best_iou = 0.0 + best: Optional[Dict[str, Any]] = None + for observation in observations: + for track in observation.get("tracks") or []: + score = _iou(label["box"], track.get("box") or []) + if score > best_iou: + best_iou = score + best = track + if best is None or best_iou < iou_min: + missed += 1 + continue + matched += 1 + level = identity_level(best) + guess = (best.get("identity") or {}).get("sku_id") + if level == "accepted" and guess == label["sku_id"]: + correct += 1 + elif level == "accepted": + wrong += 1 + elif level == "ambiguous": + ambiguous += 1 + else: + rejected += 1 + return { + "labels": len(labels), + "matched": matched, + "unmatched": missed, + "correct": correct, + "wrong": wrong, + "rejected": rejected, + "ambiguous": ambiguous, + "top1": _rate(correct, matched), + "iou_min": iou_min, + } + + +def score_recording( + recording_dir: str, + *, + stock_dir: str, + weights_path: str = "", + labels_path: Optional[str] = None, + iou_min: float = 0.5, + live_frames: Optional[Sequence[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Score one session directory. live_frames, when given, are the replayed results. + + Without live_frames the recorded index is what the server already decided. + That still produces denominators; it is not a fresh model run. + """ + recording_dir = os.path.abspath(recording_dir) + rows = load_index(recording_dir) + sample = recording_fingerprint(recording_dir, rows) + labels_file = labels_path or os.path.join(recording_dir, "labels.json") + labels = load_labels(labels_file if os.path.isfile(labels_file) or labels_path else None) + held, leaked = leaked_labels(recording_dir, rows, labels, stock_dir) + source = list(live_frames) if live_frames is not None else rows + tracks = [track for frame in source for track in (frame.get("tracks") or [])] + decisions = score_tracks(tracks) + report = { + "recording": recording_dir, + "source": "live_replay" if live_frames is not None else "recorded_index", + "fingerprints": { + "sample": sample, + "model": model_fingerprint(weights_path), + "catalog": catalog_fingerprint(stock_dir), + }, + "decisions": decisions, + "tags": { + "detected": decisions["tags_detected"], + "readable": decisions["tags_readable"], + "unreadable": decisions["tags_unreadable"], + "absent": decisions["tags_absent"], + }, + "churn": track_churn(source), + "leakage": {"excluded": len(leaked), "labels": leaked}, + "labelled_top1": labelled_top1(source, held, iou_min=iou_min), + "evidence": _evidence(labels, held, leaked, live_frames is not None), + } + return report + + +def _evidence( + labels: Sequence[Dict[str, Any]], + held: Sequence[Dict[str, Any]], + leaked: Sequence[Dict[str, Any]], + replayed: bool, +) -> List[str]: + gaps: List[str] = [] + if not replayed: + gaps.append("counts are from the recorded index, not a fresh LiveSession replay") + if not labels: + gaps.append("no held-out labels; labelled top-1 is not a walk accuracy") + elif not held: + gaps.append("every label matched a catalog template and was excluded") + if leaked: + gaps.append(f"{len(leaked)} template-derived label(s) excluded") + return gaps + + +def replay_live( + recording_dir: str, + engine: Any, + *, + settings: Optional[Any] = None, +) -> List[Dict[str, Any]]: + """Feed recorded JPEGs through one LiveSession, in index order. + + Classification is asynchronous. Each frame is scored only after its + in-flight crops finish, then processed again so the label is on the + payload. Without that wait the same recording does not replay twice. + """ + from live import LiveSession + + rows = load_index(recording_dir) + session = LiveSession("eval", engine, settings=settings) + session.refresh_catalog() + if hasattr(session, "enable_tags"): + session.enable_tags() + frames: List[Dict[str, Any]] = [] + for row in rows: + path = os.path.join(recording_dir, str(row["path"])) + with open(path, "rb") as handle: + payload = handle.read() + frame_id = int(row["frame"]) + result = session.process_frame(frame_id, payload) + # min_hits is 2, so the first pass only detects. Keep passing the same + # bytes until a pass schedules nothing and everything in flight lands. + for _ in range(4): + _wait_until_settled(session) + result = session.process_frame(frame_id, payload) + if int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) == 0: + break + result["frame"] = frame_id + frames.append(result) + engine_shutdown = getattr(engine, "shutdown", None) + if callable(engine_shutdown): + engine_shutdown() + return frames + + +def _wait_until_settled(session: Any, timeout: float = 30.0) -> None: + import time + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + pending = int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) + if pending == 0: + return + time.sleep(0.02) + raise EvalInputError("classification did not finish before the replay timeout") + +def format_report(report: Dict[str, Any]) -> str: + decisions = report["decisions"] + tags = report["tags"] + top1 = report["labelled_top1"] + churn = report["churn"] + fingerprints = report["fingerprints"] + lines = [ + f"source: {report['source']}", + ( + f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} " + f"rejected {decisions['rejected']} unknown {decisions['unknown']} " + f"unsettled {decisions['unsettled']} settled {decisions['settled']}" + ), + ( + f"rates: accepted {decisions['accepted_rate']} " + f"ambiguous {decisions['ambiguous_rate']} rejected {decisions['rejected_rate']} " + f"(denominator settled)" + ), + ( + f"tags: detected {tags['detected']} readable {tags['readable']} " + f"unreadable {tags['unreadable']} absent {tags['absent']}" + ), + ( + f"churn: tracks {churn['tracks']} sku_changes {churn['sku_changes']} " + f"relabels {churn['relabels']} restarts {churn['restarts']}" + ), + ( + f"labelled top-1: {top1['correct']}/{top1['matched']} = {top1['top1']} " + f"wrong {top1['wrong']} unmatched {top1['unmatched']} " + f"leakage excluded {report['leakage']['excluded']}" + ), + ( + f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} " + f"model {fingerprints['model']['version']} " + f"catalog {fingerprints['catalog']['version']}" + ), + ] + for gap in report["evidence"]: + lines.append(f"gap: {gap}") + return "\n".join(lines) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Score a recorded live walk.") + parser.add_argument("recording", nargs="?", help="session directory with index.jsonl") + parser.add_argument("--stock-dir", default="stock") + parser.add_argument("--weights", default="") + parser.add_argument("--labels", default=None) + parser.add_argument("--json", dest="json_path", default=None) + parser.add_argument("--iou", type=float, default=0.5) + parser.add_argument( + "--replay", + action="store_true", + help="re-run frames through LiveSession (needs the recognition stack)", + ) + parser.add_argument("--method", default="hybrid") + args = parser.parse_args(argv) + + def fail(message: str) -> int: + print(json.dumps({"error": message}, ensure_ascii=False)) + print(f"error: {message}", file=sys.stderr) + return 2 + + if not args.recording: + return fail("recording directory is required") + recording = os.path.abspath(args.recording) + if not os.path.isdir(recording): + return fail(f"recording directory not found: {recording}") + try: + live_frames = None + weights = args.weights + if args.replay: + from engine import EngineSettings, build_default_engine + from live import LiveSettings + + settings = EngineSettings(stock_dir=args.stock_dir) + if weights: + settings.weights_path = weights + engine = build_default_engine(settings) + weights = engine.settings.weights_path + live_frames = replay_live( + recording, engine, settings=LiveSettings(method=args.method, tags=True) + ) + report = score_recording( + recording, + stock_dir=args.stock_dir, + weights_path=weights, + labels_path=args.labels, + iou_min=args.iou, + live_frames=live_frames, + ) + except EvalInputError as exc: + return fail(str(exc)) + text = json.dumps(report, ensure_ascii=False, indent=1, sort_keys=True) + destination = args.json_path or os.path.join(recording, "eval_live.json") + try: + with open(destination, "w", encoding="utf-8") as handle: + handle.write(text + "\n") + except OSError as exc: + return fail(f"cannot write {destination}: {exc.strerror}") + print(format_report(report)) + print(f"json: {destination}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/fast_test.py b/recognization/code/fast_test.py deleted file mode 100644 index 772ff4f..0000000 --- a/recognization/code/fast_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from ultralytics import YOLO - -model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") -image_path = "/Users/nguyenhongvan/recongnize/data/add/image7.jpg" -results = model(image_path) -results[0].show() diff --git a/recognization/code/finetune_identity.py b/recognization/code/finetune_identity.py index cc7e519..c147a1d 100644 --- a/recognization/code/finetune_identity.py +++ b/recognization/code/finetune_identity.py @@ -49,6 +49,88 @@ MILK_SHELF_LABELS = { } +def enrollment_report(stock: str, csv_name: str = "stock.csv") -> dict: + """Catalog versus enrolled views. Does not load a model or write files. + + A SKU with no readable view cannot be recognized. One view is reported, not + treated as enough: the measured knee is two views. Duplicate CSV rows and a + head whose class list disagrees with the catalog are blockers. + """ + import csv + import re + + from gallery import ProductGallery + + names = [] + try: + names = os.listdir(stock) + except OSError: + names = [] + images = [name for name in names if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")] + rows = [] + csv_path = os.path.join(stock, csv_name) + if os.path.exists(csv_path): + with open(csv_path, encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + sku_ids = [] + seen = set() + duplicate_rows = [] + for row in rows: + sku_id = (row.get("SKU_ID") or "").strip() + if not sku_id: + continue + if sku_id in seen: + duplicate_rows.append(sku_id) + seen.add(sku_id) + sku_ids.append(sku_id) + gallery = ProductGallery(stock, sku_ids or []) + views = {sku_id: gallery._views_for(sku_id) for sku_id in dict.fromkeys(sku_ids)} + missing = [sku_id for sku_id, paths in views.items() if not paths] + single = [sku_id for sku_id, paths in views.items() if len(paths) == 1] + claimed = {os.path.basename(path) for paths in views.values() for path in paths} + orphans = sorted(name for name in images if name not in claimed) + # ponytail: prefix collision check. A view named sku_001 must not also match sku_00. + collisions = [] + for sku_id, paths in views.items(): + bare = [os.path.basename(path) for path in paths if os.path.splitext(os.path.basename(path))[0] != sku_id + and not re.fullmatch(re.escape(sku_id) + r"[_-]\d+", os.path.splitext(os.path.basename(path))[0])] + if bare: + collisions.append(sku_id) + blockers = [] + if duplicate_rows: + blockers.append("duplicate_csv_sku") + if missing: + blockers.append("sku_without_view") + if collisions: + blockers.append("view_prefix_collision") + head_path = os.path.join(stock, "identity_head.pt") + head_note = None + if os.path.exists(head_path): + head_note = "present_not_opened" + # Do not torch.load here: the head can be large, and this report must stay cheap. + # Class-list drift is reported when the caller passes class_ids. + return { + "stock": stock, + "sku_ids": list(dict.fromkeys(sku_ids)), + "images": len(images), + "missing": missing, + "single_view": single, + "orphans": orphans, + "duplicate_rows": duplicate_rows, + "collisions": collisions, + "blockers": blockers, + "head": head_note, + } + + +def enrollment_report_text(report: dict) -> str: + return ( + f"{report['stock']}: {len(report['sku_ids'])} SKUs, {report['images']} images, " + f"missing={len(report['missing'])}, single_view={len(report['single_view'])}, " + f"orphans={len(report['orphans'])}, blockers={report['blockers'] or 'none'}" + ) + + def build_model(image_size: int = 518): import timm import torch @@ -149,13 +231,13 @@ def train_from_stock(args) -> None: from gallery import ProductGallery stock = args.stock - gallery = ProductGallery(stock, []) - # the SKUs are whatever the stock directory holds - skus = sorted({os.path.splitext(name)[0].rsplit("_", 1)[0] if os.path.splitext(name)[0].rsplit("_", 1)[-1].isdigit() - else os.path.splitext(name)[0] - for name in os.listdir(stock) - if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")}) - views = {sku: gallery._views_for(sku) for sku in skus} + report = enrollment_report(stock) + print(enrollment_report_text(report)) + if report["blockers"]: + print(f"{stock}: not training; enrollment blockers: {', '.join(report['blockers'])}") + return + gallery = ProductGallery(stock, report["sku_ids"]) + views = {sku: gallery._views_for(sku) for sku in report["sku_ids"]} views = {sku: paths for sku, paths in views.items() if paths} if len(views) < 2: print(f"{stock}: needs at least two products with views, found {len(views)}") diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py index 313538d..0805d71 100644 --- a/recognization/code/gallery.py +++ b/recognization/code/gallery.py @@ -26,6 +26,7 @@ on the GPU. That is why this can sit in the live loop at all. from __future__ import annotations import os +import re import threading from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Tuple @@ -93,23 +94,39 @@ class ProductGallery: # ------------------------------------------------------------------ loading def load_head(self) -> bool: - """Load the store's fine-tuned head, if it has one.""" + """Load the store's fine-tuned head, if it has one. + + weights_only: the file is tensors and strings, not executable code. + A payload that is not that head leaves the raw gallery in place. + """ import torch if not os.path.exists(self._head_path): return False try: - payload = torch.load(self._head_path, map_location="cpu", weights_only=False) + payload = torch.load(self._head_path, map_location="cpu", weights_only=True) + if not isinstance(payload, dict): + return False + projection_state = payload.get("projection") + prototypes = payload.get("prototypes") + class_ids = payload.get("class_ids") + if not isinstance(projection_state, dict) or not torch.is_tensor(prototypes): + return False + if not isinstance(class_ids, (list, tuple)) or prototypes.ndim != 2: + return False + if prototypes.shape[0] != len(class_ids): + return False + projection = torch.nn.Linear(384, 128) + projection.load_state_dict(projection_state) + projection.eval() + self._head = { + "projection": projection, + "prototypes": torch.nn.functional.normalize(prototypes.float(), dim=1), + "class_ids": [str(sku_id) for sku_id in class_ids], + } except Exception: + self._head = None return False - projection = torch.nn.Linear(384, 128) - projection.load_state_dict(payload["projection"]) - projection.eval() - self._head = { - "projection": projection, - "prototypes": torch.nn.functional.normalize(payload["prototypes"].float(), dim=1), - "class_ids": list(payload["class_ids"]), - } return True def _load_model(self) -> Any: @@ -137,8 +154,12 @@ class ProductGallery: return self._available def _views_for(self, sku_id: str) -> List[str]: - """Every image in the stock directory that belongs to this SKU.""" + """Images whose stem is the SKU or SKU plus _N / -N. + + A bare prefix would enroll sku_001 as a view of sku_00. + """ found: List[str] = [] + extra = re.compile(rf"{re.escape(sku_id)}[_-]\d+$") try: names = sorted(os.listdir(self.stock_dir)) except OSError: @@ -147,42 +168,50 @@ class ProductGallery: stem, extension = os.path.splitext(name) if extension.lower() not in (".jpeg", ".jpg", ".png"): continue - if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"): + if stem == sku_id or extra.fullmatch(stem): found.append(os.path.join(self.stock_dir, name)) return found def load(self) -> bool: - """Embed every enrolled view. Returns False when there is nothing to load.""" + """Embed every enrolled view. Returns False when there is nothing to load. + + One lock covers the whole load: two callers cannot each embed the catalog. + """ + with self._lock: + if self._available and self._matrix is not None: + return True + return self._load_locked() + + def _load_locked(self) -> bool: import cv2 import numpy as np import torch paths: List[Tuple[str, str]] = [] + kept: List[Any] = [] for sku_id in self.sku_ids: for path in self._views_for(sku_id): + image = cv2.imread(path) + if image is None or image.size == 0: + continue paths.append((sku_id, path)) - if not paths: + kept.append(_prepare(image, self.image_size)) + if not kept: self._available = False + self._entries = [] + self._matrix = None return False try: model = self._load_model() + tensor = torch.from_numpy(np.stack(kept)).to(self.device) + with torch.no_grad(): + features = model(tensor) + features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() except Exception: self._available = False + self._entries = [] + self._matrix = None return False - - batch = [] - for _, path in paths: - image = cv2.imread(path) - if image is None or image.size == 0: - continue - batch.append(_prepare(image, self.image_size)) - if not batch: - self._available = False - return False - tensor = torch.from_numpy(np.stack(batch)).to(self.device) - with torch.no_grad(): - features = model(tensor) - features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() self._entries = [ GalleryEntry(sku_id=sku_id, view=path, embedding=features[index]) for index, (sku_id, path) in enumerate(paths) @@ -209,13 +238,14 @@ class ProductGallery: if not self._available or self._matrix is None: return GalleryMatch(sku_id=None, similarity=0.0, rejected=True) import numpy as np - import torch limit = threshold if limit is None: limit = self.head_threshold if self._head is not None else self.threshold limit = float(limit) embedding = self.embed(crop) + if self._head is not None: + import torch if self._head is not None: # Two stages, because the two mechanisms fail in opposite directions. # Measured: raw features reject strangers well (5 % false accepts) but diff --git a/recognization/code/gui_app.py b/recognization/code/gui_app.py deleted file mode 100644 index df92251..0000000 --- a/recognization/code/gui_app.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Native Desktop GUI for Supermarket Shelf Product Recognition -Opens a desktop OpenCV window displaying image.jpg with bounding boxes only. -Users can click on any product directly on screen to inspect price, stock, -and feature matching inliers in real-time. -""" - -import os -import sys -import cv2 -import numpy as np - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from pipeline import ShelfRecognitionPipeline - - -def main(): - print("=" * 60) - print("STARTING NATIVE DESKTOP INTERFACE") - print("Image: image.jpg | Mode: Hybrid (Paper Method)") - print("Instructions: Click on ANY product to identify & view info.") - print("Press 'q' or 'ESC' on the window to exit.") - print("=" * 60) - - pipeline = ShelfRecognitionPipeline() - init_res = pipeline.set_active_shelf_image("image.jpg") - - if init_res.get("status") != "success": - print("[Error] Failed to load image.jpg") - return - - boxes = init_res["boxes"] - base_image = pipeline.current_image.copy() - - # Draw ONLY bounding boxes (no class names, no confidence labels) - display_image = pipeline.detector.draw_detections(base_image, boxes, draw_labels=False) - - window_name = "Smart Shelf - Click Any Product (Press Q to quit)" - cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) - cv2.resizeWindow(window_name, 1280, 850) - - def on_mouse(event, x, y, flags, param): - if event == cv2.EVENT_LBUTTONDOWN: - print(f"\n[Click detected at coordinates: ({x}, {y})]") - res = pipeline.process_click(x, y, method="hybrid") - - if res.get("status") == "success": - prod = res["inventory"] - cls_info = res["classification"] - box_info = res["target_box"] - - print("-" * 55) - print(f"📦 Product Identified : {prod['product_name']}") - print(f"🏷️ SKU ID : {prod['sku_id']}") - print(f"💰 Price : ¥{prod['price']}") - print(f"📊 In Stock (On-Shelf): {prod['number_on']} units") - print(f"🚚 Received / Sold : {prod['number_in']} in / {prod['number_out']} out") - print(f"📅 Expiration Date : {prod['expiration_date']}") - print(f"🎯 Match Confidence : {cls_info['confidence'] * 100:.1f}% (Score: {cls_info['score']})") - print("-" * 55) - - # Update main window with selected box highlight (in red) - selected_display = pipeline.detector.draw_detections( - base_image, boxes, selected_box_id=box_info["id"], draw_labels=False - ) - cv2.imshow(window_name, selected_display) - - # Show visual match comparison window if available - if res.get("match_visualization") is not None: - match_win = "Feature Matcher (Shelf Crop vs Stock Template)" - cv2.namedWindow(match_win, cv2.WINDOW_NORMAL) - cv2.resizeWindow(match_win, 800, 400) - cv2.imshow(match_win, res["match_visualization"]) - else: - print(f"[Info] {res.get('message', 'No product found at coordinates.')}") - - cv2.setMouseCallback(window_name, on_mouse) - - while True: - cv2.imshow(window_name, display_image) - key = cv2.waitKey(20) & 0xFF - if key in [ord('q'), ord('Q'), 27]: - break - - cv2.destroyAllWindows() - - -if __name__ == "__main__": - main() diff --git a/recognization/code/identity.py b/recognization/code/identity.py index fdc597e..6fab9c0 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -25,11 +25,126 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float: return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4) -# The gallery's scale is a cosine similarity, where same-product pairs sit above 0.6 -# and different products below 0.5 - not an inlier count, which has no such meaning. +# Cosine scale (gallery). Same-product pairs were measured above 0.6 and +# different products below 0.5. These are not a new calibration. COSINE_ACCEPT = 0.60 COSINE_AMBIGUOUS = 0.50 COSINE_MARGIN = 0.04 +VOTE_CAP = 3 + + +def _finite(value: Any) -> Optional[float]: + try: + number = float(value) + except (TypeError, ValueError): + return None + if number != number or number in (float("inf"), float("-inf")): + return None + return number + + +def aggregate_votes( + observations: Sequence[Dict[str, float]], + *, + cap: int = VOTE_CAP, +) -> Dict[str, float]: + """Mean of the newest observations, at most `cap`. + + Older frames are dropped, not averaged in. An empty observation still + counts, so a later miss can outvote an earlier weak hit. Inlier and + cosine scores must not be mixed in one list. + """ + window = list(observations)[-max(1, int(cap)):] + if not window: + return {} + totals: Dict[str, float] = {} + for scores in window: + for sku_id, score in (scores or {}).items(): + number = _finite(score) + if number is None: + continue + totals[str(sku_id)] = totals.get(str(sku_id), 0.0) + number + count = len(window) + return {sku_id: total / count for sku_id, total in totals.items()} + + +def _cosine_decision( + sorted_candidates: List[Tuple[str, float]], + *, + method: str, + details: Optional[Dict[str, Any]], + accept: float, + ambiguous: float, + margin_min: float, +) -> Dict[str, Any]: + top_sku, top_score = sorted_candidates[0] + runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0 + margin = top_score - runner_score + if top_score >= accept and margin >= margin_min: + level = IDENTITY_ACCEPTED + elif top_score >= ambiguous: + level = IDENTITY_AMBIGUOUS + else: + level = IDENTITY_REJECTED + return { + "status": level, + "method": method, + "best_sku": top_sku if level == IDENTITY_ACCEPTED else None, + "score": float(top_score), + "confidence": float(top_score), + "identity_level": level, + "ranked_candidates": [ + {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3] + ], + "details": details or {}, + "message": None, + } + + +def decision_for_match( + match: Any, + *, + method: str = "gallery", + margin: float = COSINE_MARGIN, +) -> Dict[str, Any]: + """Rank one gallery match. Rejection is the gallery's, not a second threshold. + + `match()` already applied the raw 0.55 line or the head's 0.40 line. Re-ranking + those scores against COSINE_ACCEPT (0.60) would refuse a head match the gallery + accepted. The only extra gate here is the published cosine margin. + """ + similarity = _finite(getattr(match, "similarity", 0.0)) or 0.0 + if match is None or getattr(match, "rejected", False) or not getattr(match, "sku_id", None): + return { + "status": IDENTITY_REJECTED, + "method": method, + "best_sku": None, + "score": similarity, + "confidence": similarity, + "identity_level": IDENTITY_REJECTED, + "ranked_candidates": [], + "details": {}, + "message": "no enrolled product is close enough", + } + runner = getattr(match, "runner_up", None) + runner_score = _finite(getattr(match, "runner_up_similarity", 0.0)) or 0.0 + gap = similarity - runner_score + close = bool(runner) and gap < margin + level = IDENTITY_AMBIGUOUS if close else IDENTITY_ACCEPTED + ranked = [{"sku_id": str(match.sku_id), "score": similarity}] + if runner: + ranked.append({"sku_id": str(runner), "score": runner_score}) + return { + "status": level, + "method": method, + "best_sku": None if close else str(match.sku_id), + "score": similarity, + "confidence": similarity, + "identity_level": level, + "ranked_candidates": ranked, + "details": {}, + "message": None, + } def decide_identity( @@ -48,7 +163,20 @@ def decide_identity( unknown. Two close scores are ambiguous. Only a unique, above-threshold score is accepted. """ - if not scores: + if scale == "cosine": + if min_score == DEFAULT_MIN_SCORE: + min_score = COSINE_ACCEPT + if min_margin == DEFAULT_MIN_MARGIN: + min_margin = COSINE_MARGIN + if min_confidence == DEFAULT_MIN_CONFIDENCE: + min_confidence = COSINE_AMBIGUOUS + cleaned: Dict[str, float] = {} + for sku_id, score in (scores or {}).items(): + number = _finite(score) + if number is None: + continue + cleaned[str(sku_id)] = number + if not cleaned: return { "status": IDENTITY_REJECTED, "method": method, @@ -62,32 +190,17 @@ def decide_identity( } sorted_candidates: List[Tuple[str, float]] = sorted( - scores.items(), key=lambda x: x[1], reverse=True + cleaned.items(), key=lambda item: item[1], reverse=True ) if scale == "cosine": - # Thresholds that mean something on a similarity scale. - top_sku, top_score = sorted_candidates[0] - runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0 - margin = top_score - runner_score - if top_score >= COSINE_ACCEPT and margin >= COSINE_MARGIN: - level = IDENTITY_ACCEPTED - elif top_score >= COSINE_AMBIGUOUS: - level = IDENTITY_AMBIGUOUS - else: - level = IDENTITY_REJECTED - return { - "status": level, - "method": method, - "best_sku": top_sku if level == IDENTITY_ACCEPTED else None, - "score": float(top_score), - "confidence": float(top_score), - "identity_level": level, - "ranked_candidates": [ - {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3] - ], - "details": details or {}, - "message": None, - } + return _cosine_decision( + sorted_candidates, + method=method, + details=details, + accept=min_score, + ambiguous=min_confidence, + margin_min=min_margin, + ) top_sku, top_score = sorted_candidates[0] runner_up_sku, runner_up_score = ( sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0) diff --git a/recognization/code/live.py b/recognization/code/live.py index 9cab6ce..385f379 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -96,8 +96,9 @@ class TrackState: last_attempt_area: int = 0 score_sums: Dict[str, float] = field(default_factory=dict) score_n: int = 0 - identity: Optional[Dict[str, Any]] = None + observations: List[Dict[str, float]] = field(default_factory=list) candidates: List[Dict[str, Any]] = field(default_factory=list) + identity: Optional[Dict[str, Any]] = None confirmed: bool = False label_box: List[int] = field(default_factory=list) drift_frames: int = 0 @@ -184,6 +185,10 @@ class LiveSession: self._catalog_by_sku: Dict[str, Dict[str, Any]] = {} self._catalog_version = "" self._tags = None + # Shelf position of this frame's origin. Updated from a content shift + # before the session lock; tag cadence keys rows by it. Not a shelf map. + self._frame_origin = [0.0, 0.0] + self._shift_map = None self._compliance_tracker = None self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []} self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque( @@ -256,7 +261,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.score_n = 0 + track.observations = [] track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -286,6 +291,8 @@ class LiveSession: self._slots = [] self._latest_image = None self._latest_frame_id = None + self._frame_origin = [0.0, 0.0] + self._shift_map = None # ------------------------------------------------------------------ frame path def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]: @@ -300,6 +307,9 @@ class LiveSession: empty_enabled = self.settings.empty_slots generation = self._generation + + shelf_offset = self._note_frame_shift(image) + boxes = self.engine.detect(image, conf=conf) detected_at = time.perf_counter() tracks = self.tracker.update( @@ -327,7 +337,13 @@ class LiveSession: raw_box = list(detection["box"]) confidence = float(detection["confidence"]) if raw_box is None: - raw_box = list(track.box) + # Low-score second pass, or a Kalman box with no detection. + # Keep the last measured crop. Do not invent one from the prediction. + raw_box = list(state.raw_box) if state is not None and state.raw_box else [] + confidence = float(state.confidence) if state is not None else confidence + low_score = bool(getattr(track, "low_score", False)) or ( + track.det_index is None and raw_box == [] + ) previous_seq = self._last_seen_seq.get(track.track_id) if state is None: # An id that comes back after an absence may be a recycled id: @@ -369,8 +385,12 @@ class LiveSession: state.last_seq = seq self._last_seen_seq[track.track_id] = seq state.box = list(track.box) - state.raw_box = raw_box - state.confidence = confidence + if raw_box and not low_score: + state.raw_box = raw_box + state.confidence = confidence + elif not state.raw_box and raw_box: + state.raw_box = raw_box + state.confidence = confidence self._drop_stale_label(state) active[track.track_id] = state self._tracks = active @@ -405,7 +425,7 @@ class LiveSession: for state in self._tracks.values() if state.box ] - tag_bands = self._tags.plan(tag_boxes, image.shape) + tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset) # Collect the classification work under the lock; submit it after # releasing it. add_done_callback can run the callback inline when the @@ -443,39 +463,44 @@ class LiveSession: } return result + def _inherit_labels(self, seq: int) -> None: """Give a fresh track the label of a recently recognized product it covers. - The same physical product can be tracked as several ids while the camera - pans. Remembering where a label was measured means each product is - classified once and its neighbours inherit the result instead of waiting - in the classification queue again. + Only a measured label, still overlapping its own box, may be copied. + An inherited label is not a measurement, so it cannot spread to a neighbour. + Blur, glare, and a too-far frame skip inheritance: the overlap is not + evidence when the frame itself failed the quality gate. """ if not self._label_memory: return + if (self._last_quality or {}).get("hint"): + return for track in self._tracks.values(): if track.state != STATE_DETECTED or track.confirmed: continue for entry in reversed(self._label_memory): if seq - entry["seq"] > LABEL_MEMORY_FRAMES: break - if _iou(track.box, entry["box"]) >= LABEL_MEMORY_IOU: - track.state = STATE_RECOGNIZED - track.confirmed = True - track.label_box = list(track.box) - track.identity = { - "sku_id": entry["sku_id"], - "confidence": entry["confidence"], - "score": entry["score"], - "attempts": 0, - "inherited": True, - } - # An inherited label is a guess from the same place, not a - # measurement of this object: count it as re-verified so a - # later correction is auditable rather than a silent flip. - track.relabels = self._relabel_count.get(track.track_id, 0) + 1 - self._relabel_count[track.track_id] = track.relabels - break + if entry.get("inherited"): + continue + if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU: + continue + if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU: + continue + track.state = STATE_RECOGNIZED + track.confirmed = True + track.label_box = list(track.box) + track.identity = { + "sku_id": entry["sku_id"], + "confidence": entry["confidence"], + "score": entry["score"], + "attempts": 0, + "inherited": True, + } + track.relabels = self._relabel_count.get(track.track_id, 0) + 1 + self._relabel_count[track.track_id] = track.relabels + break def _remember_label(self, track: TrackState, box: List[int], seq: int) -> None: """Must be called with the session lock held.""" @@ -484,10 +509,12 @@ class LiveSession: self._label_memory.append( { "box": list(box), + "label_box": list(track.label_box or box), "sku_id": track.identity.get("sku_id"), "confidence": track.identity.get("confidence"), "score": track.identity.get("score"), "seq": seq, + "inherited": bool(track.identity.get("inherited")), } ) @@ -516,7 +543,7 @@ class LiveSession: track.identity = None track.candidates = [] track.score_sums = {} - track.score_n = 0 + track.observations = [] track.attempts = 0 track.in_flight = 0 track.last_attempt_seq = 0 @@ -539,11 +566,16 @@ class LiveSession: return [] eligible: List[TrackState] = [] + poor_frame = bool((self._last_quality or {}).get("hint")) for track in self._tracks.values(): if track.state != STATE_DETECTED or track.in_flight > 0: continue + if poor_frame: + continue if track.hits < self.settings.min_hits: continue + if len(track.raw_box) != 4: + continue x1, y1, x2, y2 = track.raw_box if min(x2 - x1, y2 - y1) < self.settings.min_crop_px: continue @@ -563,9 +595,8 @@ class LiveSession: eligible.append(track) # Newest first: while a camera pans, the products still worth labelling - # are the ones that just entered the frame. Sorting by area (the obvious - # default) spends the budget on products that are already leaving. - eligible.sort(key=lambda track: (track.first_seq, -track.area())) + # are the ones that just entered the frame. Area is only the tie-break. + eligible.sort(key=lambda track: (-track.first_seq, -track.area())) planned: List[Tuple[TrackState, Any, int]] = [] for track in eligible[:capacity]: crop = self.engine.crop(image, track.raw_box) @@ -623,13 +654,18 @@ class LiveSession: return track.in_flight = max(0, track.in_flight - 1) track.attempts += 1 - for sku_id, score in (scores or {}).items(): + observation = { + str(sku_id): float(score) for sku_id, score in (scores or {}).items() + } + track.observations.append(observation) + track.observations = track.observations[-3:] + for sku_id, score in observation.items(): track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score) - if scores: + if observation: track.score_n += 1 self._classify_times.append(finished_at - submitted_at) - if not scores: + if not any(track.observations): if track.attempts >= self.settings.max_attempts: track.state = STATE_UNRECOGNIZED track.confirmed = True @@ -638,11 +674,9 @@ class LiveSession: track.candidates = [] return - from identity import decide_identity + from identity import aggregate_votes, decide_identity - mean_scores = { - sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items() - } + mean_scores = aggregate_votes(track.observations) decision = decide_identity( mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier") ) @@ -838,7 +872,7 @@ class LiveSession: gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var()) glare = float(np.mean(np.all(small > self.settings.glare_pixel, axis=2))) - heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if state.raw_box] + heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if len(state.raw_box) == 4] median_height = float(np.median(heights)) if heights else 0.0 too_far = bool( heights @@ -847,7 +881,8 @@ class LiveSession: or sum(1 for h in heights if h < self.settings.min_crop_px) > len(heights) / 2 ) ) - blur = sharpness < self._blur_threshold() + # A black frame has Laplacian 0. That is not blur; a real shelf is not black. + blur = sharpness < self._blur_threshold() and float(gray.mean()) > 1.0 hint = None if glare > self.settings.glare_fraction: hint = "glare" @@ -898,6 +933,24 @@ class LiveSession: self._tags = TagService(ocr_engine, settings=settings or TagSettings()) return True + def _note_frame_shift(self, image: Any) -> Tuple[float, float]: + """Shelf position of this frame's origin. Phase correlation only. + + Runs before the session lock. A failed registration keeps the last origin + rather than inventing a jump. The content shift moves the origin the + other way, same sign as ShelfMap. + """ + if self._shift_map is None: + from shelf_map import ShelfMap + + self._shift_map = ShelfMap() + shift = self._shift_map.register(image) + with self._lock: + if shift is not None: + self._frame_origin[0] -= float(shift[0]) + self._frame_origin[1] -= float(shift[1]) + return (self._frame_origin[0], self._frame_origin[1]) + def _run_tag_passes(self, image: Any, bands: Sequence[Any]) -> None: """Submit tag passes to the engine's tag worker (never the frame loop).""" service = self._tags diff --git a/recognization/code/orb.py b/recognization/code/orb.py deleted file mode 100644 index e69de29..0000000 diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index 67d0dd6..8e4b6ae 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -124,8 +124,7 @@ def main() -> None: args = parser.parse_args() url = args.url - if args.token and "token=" not in url: - url = f"{url}{'&' if '?' in url else '?'}token={args.token}" + headers = {"Authorization": f"Bearer {args.token}"} if args.token else None source = FrameSource(args.source, width=args.width, pan_step=args.pan) rtts: List[float] = [] @@ -147,7 +146,7 @@ def main() -> None: in_flight: Dict[int, float] = {} pending_frame: Optional[int] = None - with connect(url, max_size=8 * 1024 * 1024) as socket: + with connect(url, max_size=8 * 1024 * 1024, additional_headers=headers) as socket: hello = json.loads(socket.recv(timeout=10)) if hello.get("type") != "hello": raise SystemExit(f"unexpected first message: {hello}") diff --git a/recognization/code/suft.py b/recognization/code/suft.py deleted file mode 100644 index 7e78d6d..0000000 --- a/recognization/code/suft.py +++ /dev/null @@ -1,84 +0,0 @@ -import cv2 -import matplotlib.pyplot as plt -import numpy as np - -def suft_function(image): - image1 = cv2.imread(image) - # Convert the training image to RGB - training_image = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB) - - # Convert the training image to gray scale - training_gray = cv2.cvtColor(training_image, cv2.COLOR_RGB2GRAY) - - # Create test image by adding Scale Invariance and Rotational Invariance - test_image = cv2.pyrDown(training_image) - test_image = cv2.pyrDown(test_image) - num_rows, num_cols = test_image.shape[:2] - - rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), 30, 1) - test_image = cv2.warpAffine(test_image, rotation_matrix, (num_cols, num_rows)) - - test_gray = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY) - - # Display traning image and testing image - fx, plots = plt.subplots(1, 2, figsize=(20,10)) - - plots[0].set_title("Training Image") - plots[0].imshow(training_image) - - plots[1].set_title("Testing Image") - plots[1].imshow(test_image) - - - # ## Detect keypoints and Create Descriptor - - surf = cv2.xfeatures2d.SURF_create(800) - - train_keypoints, train_descriptor = surf.detectAndCompute(training_gray, None) - test_keypoints, test_descriptor = surf.detectAndCompute(test_gray, None) - - keypoints_without_size = np.copy(training_image) - keypoints_with_size = np.copy(training_image) - - cv2.drawKeypoints(training_image, train_keypoints, keypoints_without_size, color = (0, 255, 0)) - - cv2.drawKeypoints(training_image, train_keypoints, keypoints_with_size, flags = cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) - - # Display image with and without keypoints size - fx, plots = plt.subplots(1, 2, figsize=(20,10)) - - plots[0].set_title("Train keypoints With Size") - plots[0].imshow(keypoints_with_size, cmap='gray') - - plots[1].set_title("Train keypoints Without Size") - plots[1].imshow(keypoints_without_size, cmap='gray') - - # Print the number of keypoints detected in the training image - print("Number of Keypoints Detected In The Training Image: ", len(train_keypoints)) - - # Print the number of keypoints detected in the query image - print("Number of Keypoints Detected In The Query Image: ", len(test_keypoints)) - - - # ## Matching Keypoints - - # Create a Brute Force Matcher object. - bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck = False) - - # Perform the matching between the SURF descriptors of the training image and the test image - matches = bf.match(train_descriptor, test_descriptor) - - # The matches with shorter distance are the ones we want. - matches = sorted(matches, key = lambda x : x.distance) - - result = cv2.drawMatches(training_image, train_keypoints, test_gray, test_keypoints, matches, test_gray, flags = 2) - - # Display the best matching points - plt.rcParams['figure.figsize'] = [14.0, 7.0] - plt.title('Best Matching Points') - plt.imshow(result) - plt.show() - - # Print total number of matching points between the training and query images - print("\nNumber of Matching Keypoints Between The Training and Query Images: ", len(matches)) - return keypoints_with_size \ No newline at end of file diff --git a/recognization/code/tag_cards.py b/recognization/code/tag_cards.py index f0a8a2a..20f55c2 100644 --- a/recognization/code/tag_cards.py +++ b/recognization/code/tag_cards.py @@ -29,19 +29,116 @@ from the small one and marked as derived rather than measured. from __future__ import annotations +import json import math +import os import re from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple -# The consumption tax used by the tags in the demo images. A tag whose small -# price is exactly 1.08x its big price is self-consistent; other stores and eras -# use 1.10 or a per-100g unit price instead, so this is a check, never a rule. -TAX_RATIOS = (1.08, 1.10) +# Defaults when a store has no store.json. Measured on the demo LOPIA tags: +# 8% consumption tax, brick-red rails, and shelf prices from ¥50. A store file +# replaces these for that store only. +DEFAULT_TAX_RATIOS = (1.08, 1.10) +DEFAULT_MIN_PRICE = 50 +# OpenCV hue is 0-180. Brick red wraps past 0, so a pixel is rail-coloured when +# its hue is below hue_low or above hue_high. +DEFAULT_RAIL_HSV = {"hue_low": 14, "hue_high": 166, "sat_min": 70, "val_min": 70} +# Aliases the parser reads. load_store_policy() republishes them per store. +TAX_RATIOS = DEFAULT_TAX_RATIOS +MIN_PRICE = DEFAULT_MIN_PRICE + +_RAIL: Optional["StorePolicy"] = None + + +def _remember_rail(policy: "StorePolicy") -> None: + global _RAIL + _RAIL = policy + + + +class StorePolicyError(ValueError): + """store.json exists but cannot be used. Never borrow another store's file.""" + + +@dataclass(frozen=True) +class StorePolicy: + tax_ratios: Tuple[float, ...] = DEFAULT_TAX_RATIOS + min_price: int = DEFAULT_MIN_PRICE + rail_hue_low: int = 14 + rail_hue_high: int = 166 + rail_sat_min: int = 70 + rail_val_min: int = 70 + store_id: Optional[str] = None + + def active(self) -> None: + global TAX_RATIOS, MIN_PRICE + TAX_RATIOS = self.tax_ratios + MIN_PRICE = self.min_price + _remember_rail(self) + +def default_policy() -> StorePolicy: + policy = StorePolicy() + policy.active() + return policy + + +def _as_ratio(value: Any) -> float: + number = float(value) + if not 1.0 < number <= 1.5: + raise StorePolicyError(f"tax ratio {value!r} is outside 1-1.5") + return number + + +def _as_bound(value: Any, name: str, low: int, high: int) -> int: + number = int(value) + if not low <= number <= high: + raise StorePolicyError(f"{name} {value!r} is outside {low}-{high}") + return number + + +def load_store_policy(stock_dir: Optional[str]) -> StorePolicy: + """Read `/store.json`. A missing file uses the defaults. + + A file that is present but invalid raises StorePolicyError naming that path. + """ + if not stock_dir: + return StorePolicy() + path = os.path.join(stock_dir, "store.json") + if not os.path.isfile(path): + return StorePolicy() + try: + with open(path, encoding="utf-8") as handle: + raw = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + raise StorePolicyError(f"{path}: {exc}") from exc + if not isinstance(raw, dict): + raise StorePolicyError(f"{path}: expected an object") + try: + ratios = raw.get("tax_ratios", list(DEFAULT_TAX_RATIOS)) + if not isinstance(ratios, list) or not ratios: + raise StorePolicyError(f"{path}: tax_ratios must be a non-empty list") + rail = raw.get("rail_hsv", DEFAULT_RAIL_HSV) + if not isinstance(rail, dict): + raise StorePolicyError(f"{path}: rail_hsv must be an object") + policy = StorePolicy( + tax_ratios=tuple(_as_ratio(item) for item in ratios), + min_price=_as_bound(raw.get("min_price", DEFAULT_MIN_PRICE), "min_price", 1, 100000), + rail_hue_low=_as_bound(rail.get("hue_low", 14), "hue_low", 0, 180), + rail_hue_high=_as_bound(rail.get("hue_high", 166), "hue_high", 0, 180), + rail_sat_min=_as_bound(rail.get("sat_min", 70), "sat_min", 0, 255), + rail_val_min=_as_bound(rail.get("val_min", 70), "val_min", 0, 255), + store_id=str(raw["store_id"]) if raw.get("store_id") else None, + ) + except (TypeError, ValueError) as exc: + if isinstance(exc, StorePolicyError): + raise + raise StorePolicyError(f"{path}: {exc}") from exc + policy.active() + return policy HEADER_PATTERN = re.compile(r"お買得|お買い得|特価|価格|セール") -# A shelf price below this is not a price (measured live reads included 20 and 23). -MIN_PRICE = 50 + # The tag truncates its discount to a tenth of a 割, so agreement is one step. DISCOUNT_TOLERANCE = 0.05 # A regular price above this multiple of the sale price is a misread, not a discount. @@ -201,22 +298,33 @@ def cluster_rows(lines: Sequence[OcrLine], max_gap: int = 26) -> List[List[OcrLi return rows -def tag_row_score(row: Sequence[OcrLine]) -> float: +def tag_row_score(row: Sequence[OcrLine], header_pattern: Any = None) -> float: """How much a row looks like a line of price tags. Tags carry a header, a discount, and a decimal (tax-included) price; product - packaging rarely carries all three on one line. + packaging rarely carries all three on one line. The header pattern is bound + here: a missing module global must not fail the tag pass. """ + def pattern(name: str, fallback: str) -> Any: + found = globals().get(name) + return found if found is not None else re.compile(fallback) + + header = header_pattern if header_pattern is not None else pattern( + "HEADER_PATTERN", r"お買得|お買い得|特価|価格|セール" + ) + discount = pattern("DISCOUNT_PATTERN", r"(\d+(?:\.\d)?)\s*割\s*引") + decimal = pattern("DECIMAL_PATTERN", r"(\d{1,4})[.,](\d{2})(?!\d)") + jan = pattern("JAN3_PATTERN", r"(? TagFields: name_candidates.sort(key=lambda item: (-len(item[0]), -item[1].confidence)) fields.name = name_candidates[0][0] - # Layout, not digit count, separates the price from the JAN: the price is the # LARGEST text on the tag (its line is roughly twice as tall as the small # print), and the JAN's last three digits sit on a small line of their own. # Getting this order wrong made a 269 price look like a JAN and disappear. @@ -534,7 +641,12 @@ def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, i hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) hue, sat, val = cv2.split(hsv) - brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8) + rail = _RAIL or StorePolicy() + brick = ( + ((hue < rail.rail_hue_low) | (hue > rail.rail_hue_high)) + & (sat > rail.rail_sat_min) + & (val > rail.rail_val_min) + ).astype(np.uint8) row_fraction = brick.mean(axis=1) height, width = image.shape[:2] rails: List[Tuple[int, int, int, int]] = [] diff --git a/recognization/code/tags.py b/recognization/code/tags.py index 8d5fc5b..b4ad79d 100644 --- a/recognization/code/tags.py +++ b/recognization/code/tags.py @@ -36,6 +36,7 @@ from ocr import OcrEngine, OcrLine, OcrResult, OcrUnavailable # noqa: E402 from tag_cards import ( # noqa: E402 CORROBORATION_FLAGS, MIN_PRICE, + StorePolicy, TagFields as TagFieldsModel, corroborate, best_tag_rows, @@ -49,6 +50,13 @@ from tag_cards import ( # noqa: E402 TAG_STATE_READABLE = "readable" TAG_STATE_UNREADABLE = "unreadable" TAG_STATE_ABSENT = "absent" +# Seen on the rail, OCR returned nothing. Not the same as absent (no tag) or +# unreadable (text came back but did not corroborate). +TAG_STATE_DETECTED = "detected" + +# Two shelf rows closer than this are the same rail. A screen row index is not +# an identity: panning a new shelf into view reuses row 0. +SHELF_ROW_BIN_PX = 80 # Measured tag shape (add/image6.jpg): ~300x80 px, aspect ~3.75:1, digits ~22 px. TAG_MIN_WIDTH_RATIO = 0.035 @@ -114,6 +122,7 @@ class TagRecord: tag_id: str box: List[int] band_index: int + shelf_key: Tuple[str, int] = ("screen", 0) observations: List[TagObservation] = field(default_factory=list) first_seen: float = 0.0 last_seen: float = 0.0 @@ -126,6 +135,7 @@ class TagRecord: discount: Optional[float] = None unit_price: Optional[float] = None name: Optional[str] = None + price_source: str = "none" flags: List[str] = field(default_factory=list) confidence: float = 0.0 @@ -135,17 +145,18 @@ class TagRecord: Measured live: without this, single uncorroborated reads of the stylised digits were published as readable tags with prices of 20, 23, 72 and 4508. + A block seen with no text is `detected`; text that does not parse is + `unreadable`. Neither is `absent`. """ - if not self.observations: - return TAG_STATE_UNREADABLE - if self.price is None or self.price < MIN_PRICE: + if self.price is None: + saw_text = any(observation.lines for observation in self.observations) + return TAG_STATE_UNREADABLE if saw_text else TAG_STATE_DETECTED + if self.price < MIN_PRICE: return TAG_STATE_UNREADABLE corroborated = bool( {"tax_consistent", "tax_pair_consistent", "discount_consistent"} & set(self.flags) ) - if not corroborated: - return TAG_STATE_UNREADABLE - if self.regular_price is None and self.name is None: + if not corroborated or (self.regular_price is None and self.name is None): return TAG_STATE_UNREADABLE return TAG_STATE_READABLE @@ -159,8 +170,6 @@ class TagRecord: "discount": self.discount, "unit_price": self.unit_price, "name": self.name, - "regular_price": self.regular_price, - "price_source": self.price_source, "flags": list(self.flags), "confidence": round(float(self.confidence), 3), "observations": len(self.observations), @@ -386,6 +395,20 @@ class TagSettings: settle_passes: int = SETTLE_PASSES min_band_width_ratio: float = 0.10 max_tags: int = 120 + policy: Optional[StorePolicy] = None + + +def band_key(band: RailBand, shelf_offset: Optional[Sequence[float]] = None) -> Tuple[str, int]: + """Which rail this band is, stable across a pan. + + No shelf origin: the screen row, which is all a still has. With one (the + shelf position of the frame origin), the row's shelf y, so a new shelf that + lands on screen row 0 is not the row that already settled. + """ + if not shelf_offset or len(shelf_offset) < 2: + return ("screen", int(band.row_index)) + shelf_y = (band.box[1] + band.box[3]) / 2.0 + float(shelf_offset[1]) + return ("shelf", int(round(shelf_y / SHELF_ROW_BIN_PX))) class TagService: @@ -410,7 +433,7 @@ class TagService: self._tags: Dict[str, TagRecord] = {} self._pass_id = 0 self._frame_seq = 0 - self._last_pass_seq: Dict[int, int] = {} + self._last_pass_seq: Dict[Tuple[str, int], int] = {} self._pending_bands: set = set() self.passes_run = 0 self.passes_failed = 0 @@ -418,29 +441,37 @@ class TagService: self.passes_skipped = 0 # ------------------------------------------------------------------ frame loop - def plan(self, boxes: Sequence[Dict[str, Any]], image_shape: Sequence[int]) -> List[RailBand]: + def plan( + self, + boxes: Sequence[Dict[str, Any]], + image_shape: Sequence[int], + shelf_offset: Optional[Sequence[float]] = None, + ) -> List[RailBand]: """Bands worth OCR-ing right now (must be called from the frame loop). - A band is a shelf row, so its key is the row - not its on-screen position. - Keying by position meant a walking camera minted a fresh key every few - pixels of pan and re-read the same shelf continuously, which starved - classification of CPU (measured: 10x slower per crop while OCR ran). + The key is the shelf row when the caller passes the frame origin, and the + screen row otherwise. Keying by on-screen pixels re-read the same shelf + on every pan step (measured: 10x slower per crop while OCR ran); keying + by the screen row index instead hides a new shelf that reuses row 0. """ if not self.settings.enabled or not self.engine.available(): return [] + if self.settings.policy is not None: + self.settings.policy.active() bands = rail_bands(boxes, image_shape) width = float(image_shape[1]) bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width] with self._lock: + self._shelf_offset = shelf_offset self._frame_seq += 1 seq = self._frame_seq chosen: List[RailBand] = [] for band in bands: - key = band.row_index + key = band_key(band, shelf_offset) if key in self._pending_bands: continue last = self._last_pass_seq.get(key) - settled = self._band_settled(band.row_index) + settled = self._band_settled(key) if settled and last is not None: continue if last is not None and seq - last < self.settings.pass_interval_frames: @@ -449,8 +480,9 @@ class TagService: if len(chosen) >= self.settings.max_bands_per_pass: break for band in chosen: - self._last_pass_seq[band.row_index] = seq - self._pending_bands.add(band.row_index) + key = band_key(band, shelf_offset) + self._last_pass_seq[key] = seq + self._pending_bands.add(key) return chosen def run_pass(self, image: Any, band: RailBand, scale_bias: float = 1.0) -> List[TagObservation]: @@ -460,7 +492,7 @@ class TagService: independent reads of the same tag are what makes the vote meaningful (identical pixels would produce identical output). """ - key = band.row_index + key = band_key(band, getattr(self, "_shelf_offset", None)) try: x1, y1, x2, y2 = band.box crop = image[y1:y2, x1:x2] @@ -537,7 +569,7 @@ class TagService: ) card = card_from_lines(pooled) if pooled else seed card.box = box - if not card.fields.is_readable() and card.fields.price is None: + if not card.fields.is_readable() and card.fields.price is None and not group: continue cards.append( { @@ -593,8 +625,8 @@ class TagService: ) return extra - def _band_settled(self, row_index: int) -> bool: - row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index] + def _band_settled(self, key: Tuple[str, int]) -> bool: + row_tags = [tag for tag in self._tags.values() if tag.shelf_key == key] if not row_tags: return False return all(tag.settled for tag in row_tags) @@ -604,6 +636,7 @@ class TagService: box = candidate["box"] lines = candidate["lines"] fields = candidate["fields"] + shelf_key = band_key(band, getattr(self, "_shelf_offset", None)) observation = TagObservation( box=list(box), lines=list(lines), @@ -613,12 +646,13 @@ class TagService: observed_at=time.time(), ) - record = self._match_record(box) + record = self._match_record(box, shelf_key) if record is None: record = TagRecord( tag_id=f"tag_{uuid.uuid4().hex[:8]}", box=list(box), band_index=band.row_index, + shelf_key=shelf_key, first_seen=observation.observed_at, ) if len(self._tags) >= self.settings.max_tags: @@ -636,10 +670,12 @@ class TagService: record.settled = True return observation - def _match_record(self, box: Sequence[int]) -> Optional[TagRecord]: + def _match_record(self, box: Sequence[int], shelf_key: Tuple[str, int]) -> Optional[TagRecord]: best = None best_iou = 0.35 for record in self._tags.values(): + if record.shelf_key != shelf_key: + continue iou = _iou(box, record.box) if iou >= best_iou: best_iou = iou @@ -769,7 +805,12 @@ class TagService: "settled": False, } - counts = {TAG_STATE_READABLE: 0, TAG_STATE_UNREADABLE: 0, TAG_STATE_ABSENT: 0} + counts = { + TAG_STATE_READABLE: 0, + TAG_STATE_UNREADABLE: 0, + TAG_STATE_DETECTED: 0, + TAG_STATE_ABSENT: 0, + } for payload in by_track.values(): counts[payload["state"]] = counts.get(payload["state"], 0) + 1 return by_track, tags, counts @@ -817,6 +858,7 @@ class TagService: self._pending_bands.clear() self._pass_id = 0 self._frame_seq = 0 + self._shelf_offset = None def _iou(a: Sequence[int], b: Sequence[int]) -> float: diff --git a/recognization/code/test.py b/recognization/code/test.py deleted file mode 100644 index 5eb7553..0000000 --- a/recognization/code/test.py +++ /dev/null @@ -1,25 +0,0 @@ -from ultralytics import YOLO -import cv2 - -model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt") - -image_path = "/Users/nguyenhongvan/recongnize/data/add/image9.jpg" - -results = model(image_path) - -image = cv2.imread(image_path) - -for box in results[0].boxes.xyxy.cpu().numpy(): - x1, y1, x2, y2 = map(int, box) - - cv2.rectangle( - image, - (x1, y1), - (x2, y2), - (0, 255, 0), - 2 - ) - -cv2.imshow("Bounding Boxes Only", image) -cv2.waitKey(0) -cv2.destroyAllWindows() \ No newline at end of file diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index dc74def..bc21027 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -155,6 +155,47 @@ class RestContractTests(unittest.TestCase): self.assertEqual(response.status_code, 415) self.assertEqual(response.json()["error"]["code"], "UNSUPPORTED_MEDIA_TYPE") + def test_a_lying_content_length_is_rejected_while_streaming(self): + """The ASGI receive stops once the body passes the cap, before decode.""" + import anyio + + from transport import BodyLimit + + cap = 32 + seen = {"bytes": 0} + + async def app(scope, receive, send): + while True: + message = await receive() + if message["type"] != "http.request": + continue + seen["bytes"] += len(message.get("body") or b"") + if not message.get("more_body"): + break + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + chunks = [b"x" * 20, b"y" * 20, b"z" * 20] + + async def receive(): + body = chunks.pop(0) if chunks else b"" + return {"type": "http.request", "body": body, "more_body": bool(chunks)} + + sent = [] + + async def send(message): + sent.append(message) + + async def run(): + await BodyLimit(app, cap)( + {"type": "http"}, receive, send + ) + + anyio.run(run) + self.assertLessEqual(seen["bytes"], cap + 20) + self.assertEqual(sent[0]["status"], 413) + self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"]) + def test_classify_and_inspect_roundtrip(self): recognized = self.client.post( "/v1/recognize?classify=false", content=jpeg_bytes(), @@ -285,13 +326,26 @@ class WebSocketProtocolTests(unittest.TestCase): self.assertEqual(missing["code"], "TRACK_NOT_IN_VIEW") def test_token_is_enforced_on_the_socket(self): + """Header or the first post-hello auth frame. A query token is rejected.""" client, engine = make_app(token="s3cret") try: with self.assertRaises(Exception): with client.websocket_connect("/v1/stream") as socket: + self.assertEqual(socket.receive_json()["type"], "hello") + socket.send_json({"type": "ping", "t": 1}) socket.receive_json() - with client.websocket_connect("/v1/stream?token=s3cret") as socket: + with self.assertRaises(Exception): + with client.websocket_connect("/v1/stream?token=s3cret") as socket: + socket.receive_json() + with client.websocket_connect( + "/v1/stream", headers={"Authorization": "Bearer s3cret"} + ) as socket: self.assertEqual(socket.receive_json()["type"], "hello") + with client.websocket_connect("/v1/stream") as socket: + self.assertEqual(socket.receive_json()["type"], "hello") + socket.send_json({"type": "auth", "token": "s3cret"}) + socket.send_json({"type": "ping", "t": 1}) + self.assertEqual(socket.receive_json(), {"type": "pong", "t": 1}) finally: engine.shutdown() diff --git a/recognization/code/test_empty.py b/recognization/code/test_empty.py deleted file mode 100644 index 0781650..0000000 --- a/recognization/code/test_empty.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys -import os - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from empty_shelf_detector import detect_and_draw_empty_slots - -if __name__ == "__main__": - image_path = "image.jpg" - print(f"Processing empty shelf detection on {image_path}...") - annotated, empty_slots = detect_and_draw_empty_slots( - image_path, - output_path="detected_empty_slots.jpg" - ) - print(f"Detected {len(empty_slots)} empty shelf slots:") - for s in empty_slots: - print(f" - Slot [{s['id']}]: Row {s['row_index']}, Box={s['box']}, Width={s['width']}px, Est. Missing Items={s['missing_items_est']}") - print(f"Visualization saved to: detected_empty_slots.jpg") diff --git a/recognization/code/test_eval_live.py b/recognization/code/test_eval_live.py new file mode 100644 index 0000000..f993328 --- /dev/null +++ b/recognization/code/test_eval_live.py @@ -0,0 +1,241 @@ +"""Metric contracts for eval_live. No model, no real walk, no fabricated accuracy.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +import cv2 # noqa: E402 +import numpy as np # noqa: E402 + +from eval_live import ( # noqa: E402 + EvalInputError, + identity_level, + labelled_top1, + leaked_labels, + load_index, + score_recording, + score_tracks, + track_churn, +) + + +def track(level, sku=None, track_id=1, box=None, relabels=0, restarts=0, tag=None): + identity = {"identity_level": level, "state": "recognized" if level == "accepted" else "unrecognized"} + if level == "accepted": + identity["sku_id"] = sku + row = { + "track_id": track_id, + "box": box or [0, 0, 10, 10], + "identity": identity, + "relabels": relabels, + "restarts": restarts, + } + if tag is not None: + row["price_tag"] = tag + return row + + +def write_jpeg(path, image): + ok, encoded = cv2.imencode(".jpg", image) + assert ok + with open(path, "wb") as handle: + handle.write(encoded.tobytes()) + + +class DecisionDenominatorTests(unittest.TestCase): + def test_settled_excludes_unsettled_and_rates_use_that_denominator(self): + scored = score_tracks( + [ + track("accepted", "sku_a"), + track("ambiguous"), + track("rejected"), + track("unknown"), + {"identity": {"state": "unconfirmed", "identity_level": None}}, + ] + ) + self.assertEqual(scored["settled"], 4) + self.assertEqual(scored["unsettled"], 1) + self.assertEqual(scored["accepted_rate"], 0.25) + self.assertIsNone(score_tracks([{"identity": {"state": "unconfirmed"}}])["accepted_rate"]) + + def test_wire_state_fills_a_missing_level(self): + self.assertEqual( + identity_level({"identity": {"state": "needs_confirmation"}}), "ambiguous" + ) + self.assertEqual(identity_level({"identity": {"state": "unconfirmed"}}), "unsettled") + + def test_readable_tags_are_counted_apart_from_detected(self): + scored = score_tracks( + [ + track("accepted", "sku_a", tag={"state": "readable"}), + track("rejected", tag={"state": "unreadable"}), + track("rejected", tag={"state": "absent"}), + track("rejected"), + ] + ) + self.assertEqual(scored["tags_readable"], 1) + self.assertEqual(scored["tags_detected"], 2) + self.assertEqual(scored["tags_absent"], 2) + + +class Top1AndChurnTests(unittest.TestCase): + def test_top1_counts_only_accepted_matches_over_overlapping_labels(self): + frames = [ + { + "frame": 1, + "tracks": [ + track("accepted", "sku_a", box=[0, 0, 10, 10]), + track("accepted", "sku_b", track_id=2, box=[100, 100, 110, 110]), + ], + }, + {"frame": 2, "tracks": [track("rejected", box=[0, 0, 10, 10])]}, + {"frame": 3, "tracks": []}, + ] + labels = [ + {"frame": 1, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + {"frame": 1, "box": [100, 100, 110, 110], "sku_id": "sku_a"}, + {"frame": 2, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + {"frame": 3, "box": [0, 0, 10, 10], "sku_id": "sku_a"}, + ] + scored = labelled_top1(frames, labels, iou_min=0.5) + self.assertEqual((scored["correct"], scored["wrong"], scored["rejected"], scored["unmatched"]), (1, 1, 1, 1)) + self.assertEqual(scored["top1"], 0.3333) + + def test_sku_churn_counts_a_renamed_track_not_the_first_name(self): + frames = [ + {"tracks": [track("accepted", "sku_a", track_id=7)]}, + {"tracks": [track("accepted", "sku_a", track_id=7)]}, + {"tracks": [track("accepted", "sku_b", track_id=7, relabels=1)]}, + {"tracks": [track("rejected", track_id=8, restarts=2)]}, + ] + churn = track_churn(frames) + self.assertEqual(churn["sku_changes"], 1) + self.assertEqual(churn["tracks_with_sku_change"], 1) + self.assertEqual(churn["relabels"], 1) + self.assertEqual(churn["restarts"], 2) + self.assertEqual(churn["per_track"][0]["track_id"], 7) + + +class LeakageTests(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + + def test_a_pasted_catalog_crop_is_excluded_and_a_real_facing_is_not(self): + stock = os.path.join(self.root, "stock") + os.makedirs(stock) + template = np.full((20, 16, 3), 40, dtype=np.uint8) + template[2:8, 2:8] = (255, 0, 0) + write_jpeg(os.path.join(stock, "sku_a.jpeg"), template) + frame = np.zeros((40, 40, 3), dtype=np.uint8) + frame[5:25, 10:26] = template + recording = os.path.join(self.root, "walk") + os.makedirs(recording) + write_jpeg(os.path.join(recording, "000000.jpg"), frame) + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps({"frame": 0, "path": "000000.jpg", "tracks": []}) + "\n") + labels = [ + {"frame": 0, "box": [10, 5, 26, 25], "sku_id": "sku_a"}, + {"frame": 0, "box": [0, 0, 8, 8], "sku_id": "sku_b"}, + ] + held, leaked = leaked_labels(recording, load_index(recording), labels, stock) + self.assertEqual(len(leaked), 1) + self.assertEqual(leaked[0]["leak"], "crop_is_template") + self.assertEqual([row["sku_id"] for row in held], ["sku_b"]) + + def test_score_recording_names_the_leak_and_refuses_it_as_top1(self): + stock = os.path.join(self.root, "stock") + os.makedirs(stock) + image = np.full((12, 12, 3), 9, dtype=np.uint8) + write_jpeg(os.path.join(stock, "sku_a.jpg"), image) + with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: + handle.write("SKU_ID,product_name\nsku_a,A\n") + recording = os.path.join(self.root, "session") + os.makedirs(recording) + shutil.copy(os.path.join(stock, "sku_a.jpg"), os.path.join(recording, "000000.jpg")) + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "frame": 1, + "path": "000000.jpg", + "tracks": [track("accepted", "sku_a", box=[0, 0, 12, 12])], + } + ) + + "\n" + ) + with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: + json.dump({"labels": [{"frame": 1, "box": [0, 0, 12, 12], "sku_id": "sku_a"}]}, handle) + report = score_recording(recording, stock_dir=stock) + self.assertEqual(report["leakage"]["excluded"], 1) + self.assertEqual(report["labelled_top1"]["labels"], 0) + self.assertEqual(report["labelled_top1"]["top1"], None) + self.assertEqual(report["decisions"]["accepted"], 1) + self.assertTrue(report["fingerprints"]["sample"]["sha256"]) + self.assertEqual(len(report["fingerprints"]["catalog"]["version"]), 12) + self.assertTrue(any("template-derived" in gap for gap in report["evidence"])) + + +class CliTests(unittest.TestCase): + def test_missing_recording_is_an_explicit_error(self): + completed = subprocess.run( + [sys.executable, os.path.join(current_dir, "eval_live.py"), "/no/such/recording"], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("error", json.loads(completed.stdout)) + + def test_the_same_pinned_recording_scores_twice_the_same(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + stock = os.path.join(root, "stock") + os.makedirs(stock) + with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle: + handle.write("SKU_ID\nsku_a\n") + recording = os.path.join(root, "session") + os.makedirs(recording) + payload = np.zeros((8, 8, 3), dtype=np.uint8) + write_jpeg(os.path.join(recording, "000000.jpg"), payload) + line = { + "frame": 4, + "path": "000000.jpg", + "tracks": [ + track("accepted", "sku_a", tag={"state": "readable"}), + track("ambiguous", track_id=2), + ], + } + with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write(json.dumps(line) + "\n") + with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle: + json.dump({"labels": [{"frame": 4, "box": [0, 0, 8, 8], "sku_id": "sku_a"}]}, handle) + first = score_recording(recording, stock_dir=stock) + second = score_recording(recording, stock_dir=stock) + self.assertEqual(first, second) + self.assertEqual(first["decisions"]["accepted"], 1) + self.assertEqual(first["decisions"]["ambiguous"], 1) + self.assertEqual(first["labelled_top1"]["correct"], 1) + self.assertIsNone(first["fingerprints"]["model"]["weights_sha256"]) + + def test_a_broken_index_line_is_an_input_error(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + with open(os.path.join(root, "index.jsonl"), "w", encoding="utf-8") as handle: + handle.write("{not json}\n") + with self.assertRaises(EvalInputError): + load_index(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/code/test_image.py b/recognization/code/test_image.py deleted file mode 100644 index 57f138f..0000000 --- a/recognization/code/test_image.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Test Script for image.jpg -1. Runs YOLO detection on image.jpg. -2. Draws ONLY bounding box rectangles (no class names, no confidence labels). -3. Saves output to detected_image.jpg. -4. Matches the detected bounding boxes against stock templates. -""" - -import os -import sys -import cv2 -import numpy as np - -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -from detector import ShelfProductDetector -from feature_matcher import ProductFeatureMatcher -from inventory import InventoryManager - - -def main(): - image_path = "image.jpg" - print(f"[1/4] Loading {image_path}...") - image = cv2.imread(image_path) - if image is None: - print(f"[Error] Failed to read {image_path}") - return - - h, w = image.shape[:2] - print(f" Image resolution: {w}x{h}") - - print("[2/4] Detecting products using YOLO...") - detector = ShelfProductDetector("runs/detect/train/weights/best.pt") - # You can adjust conf_threshold as needed (default 0.25) - det_res = detector.detect(image, conf_threshold=0.25) - boxes = det_res.get("boxes", []) - print(f" Total detected items: {len(boxes)}") - - print("[3/4] Drawing bounding boxes ONLY (no class names, no confidence)...") - annotated = detector.draw_detections(image, boxes, draw_labels=False) - output_path = "detected_image.jpg" - cv2.imwrite(output_path, annotated) - print(f" Saved detection visualization to: {output_path}") - - print("[4/4] Matching against stock catalogue (SURF, ORB, BRISK, Hybrid)...") - matcher = ProductFeatureMatcher("stock") - inv = InventoryManager("stock") - - print("\n--- Stock Templates Present in image.jpg ---") - matched_skus = set() - sample_matches = [] - - # Test top detections to find corresponding stock SKUs - for b in boxes: - crop = detector.crop_roi(image, b["box"]) - # Use hybrid method from the paper - res = matcher.classify_crop(crop, method="hybrid") - if res.get("status") == "success": - sku_id = res["best_sku"] - score = res["score"] - conf = res["confidence"] - prod = inv.get_product(sku_id) - if prod: - matched_skus.add(sku_id) - sample_matches.append({ - "box_id": b["id"], - "box": b["box"], - "sku_id": sku_id, - "product_name": prod["product_name"], - "price": prod["price"], - "number_on": prod["number_on"], - "score": score, - "confidence": conf - }) - - # Sort matches by confidence - sample_matches.sort(key=lambda x: x["score"], reverse=True) - - print(f"\nSuccessfully matched {len(matched_skus)} distinct SKUs across detected shelf boxes:") - for sm in sample_matches[:10]: - print(f"Box #{sm['box_id']:<3} -> [{sm['sku_id']}] {sm['product_name']:<30} | Price: ¥{sm['price']:<5} | On-Shelf: {sm['number_on']} | Score: {sm['score']}") - - print("\n[Complete] Tested image.jpg successfully.") - - -if __name__ == "__main__": - main() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index cec10cf..d1355c5 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -276,6 +276,36 @@ class TagPassIntegrationTests(unittest.TestCase): self.assertNotIn("price_tag", result["tracks"][0]) engine.shutdown() + def test_a_panned_shelf_reusing_the_screen_row_is_planned(self): + """The same screen row on a shelf that has moved is a new rail.""" + session, engine, _ = self.make_tagged_session() + session._tags.settings.pass_interval_frames = 0 + session._tags.settings.settle_observations = 1 + session._tags.settings.settle_passes = 1 + + class Panned: + def __init__(self): + self.calls = 0 + + def register(self, image): + self.calls += 1 + # Content moved up 400 px, so the frame origin moves down the shelf. + return (0.0, -400.0) if self.calls == 2 else (0.0, 0.0) + + session._shift_map = Panned() + process(session, 1) + time.sleep(0.05) + before = session._tags.stats()["passes_run"] + self.assertGreaterEqual(before, 1) + process(session, 2) + time.sleep(0.1) + self.assertGreaterEqual(session._frame_origin[1], 400.0) + self.assertGreater( + session._tags.stats()["passes_run"], before, + "a shelf 400 px away reuses screen row 0 and must be read again", + ) + engine.shutdown() + class FramePipelineTests(unittest.TestCase): def test_latest_frame_wins_and_drops_are_counted(self): diff --git a/recognization/code/test_ocr_tags.py b/recognization/code/test_ocr_tags.py index b48ea2f..25d911d 100644 --- a/recognization/code/test_ocr_tags.py +++ b/recognization/code/test_ocr_tags.py @@ -21,6 +21,7 @@ import numpy as np # noqa: E402 from ocr import FakeOcr, NullOcr, OcrLine, OcrResult, OcrUnavailable, build_ocr_engine # noqa: E402 from tags import ( # noqa: E402 TAG_STATE_ABSENT, + TAG_STATE_DETECTED, TAG_STATE_READABLE, TAG_STATE_UNREADABLE, TagService, @@ -30,6 +31,8 @@ from tags import ( # noqa: E402 parse_price_tag, rail_bands, ) +from tag_cards import StorePolicy, StorePolicyError, load_store_policy # noqa: E402 +import tag_cards # noqa: E402 def image(width=1000, height=600): @@ -275,6 +278,108 @@ class TagServiceTests(unittest.TestCase): service.reset() self.assertEqual(service.stats()["tags"], 0) + def test_a_new_shelf_reusing_row_zero_is_read(self): + """Screen row 0 on a newly panned shelf is not the row that settled.""" + service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + array = image() + for _ in range(3): + for band in service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)): + service.run_pass(array, band) + self.assertEqual( + service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)), + [], + "the same shelf row stays settled", + ) + again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0)) + self.assertTrue(again, "a shelf 400 px away reuses the screen row and must be read") + + def test_a_seen_tag_with_no_text_is_detected_not_absent(self): + def reader(crop): + return [] + + service = TagService(FakeOcr(reader=reader), settings=TagSettings(pass_interval_frames=0)) + array = image() + # A bright paper block on a brick rail, under the product row. OCR reads nothing. + array[300:420, :] = (40, 40, 180) + array[330:390, 120:280] = (235, 235, 235) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + bands = service.plan(boxes, array.shape) + self.assertTrue(bands) + service.run_pass(array, bands[0]) + by_track, tags, counts = service.attach([{"track_id": 0, "box": [100, 100, 200, 300]}]) + self.assertTrue(tags, "the paper block is a tag even though OCR returned nothing") + self.assertEqual(tags[0]["state"], TAG_STATE_DETECTED) + self.assertEqual(by_track[0]["state"], TAG_STATE_DETECTED) + self.assertEqual(counts[TAG_STATE_DETECTED], 1) + self.assertNotEqual(by_track[0]["state"], TAG_STATE_ABSENT) + + def test_a_tag_pass_survives_a_missing_header_pattern(self): + """Evaluation replay died with NameError: HEADER_PATTERN.""" + import tag_cards + from tag_cards import OcrLine as CardLine + + row = [CardLine(text="毎日お買得", box=(0, 0, 40, 16), confidence=0.9)] + saved = tag_cards.__dict__.pop("HEADER_PATTERN", None) + try: + score = tag_cards.tag_row_score(row) + finally: + if saved is not None: + tag_cards.HEADER_PATTERN = saved + self.assertGreater(score, 0, "a sale header still marks the row when the global is missing") + service, _ = self.make(pass_interval_frames=0) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + bands = service.plan(boxes, image().shape) + self.assertTrue(bands) + service.run_pass(image(), bands[0]) + self.assertGreaterEqual(service.stats()["passes_run"], 1) + self.assertIsNone(service.stats()["last_pass_error"]) + + +class StorePolicyTests(unittest.TestCase): + def test_a_missing_file_uses_the_defaults(self): + policy = load_store_policy(None) + policy.active() + self.assertEqual(policy.tax_ratios, (1.08, 1.10)) + self.assertEqual(policy.min_price, 50) + self.assertEqual(tag_cards.MIN_PRICE, 50) + + def test_a_store_file_does_not_change_another_store(self): + import json + import tempfile + + with tempfile.TemporaryDirectory() as root: + good = os.path.join(root, "lopia") + bad = os.path.join(root, "other") + os.makedirs(good) + os.makedirs(bad) + with open(os.path.join(good, "store.json"), "w", encoding="utf-8") as handle: + json.dump({"store_id": "lopia", "tax_ratios": [1.08], "min_price": 80}, handle) + with open(os.path.join(bad, "store.json"), "w", encoding="utf-8") as handle: + handle.write("{not json") + loaded = load_store_policy(good) + self.assertEqual(loaded.min_price, 80) + self.assertEqual(tag_cards.MIN_PRICE, 80) + with self.assertRaises(StorePolicyError) as raised: + load_store_policy(bad) + self.assertIn(bad, str(raised.exception)) + self.assertEqual(tag_cards.MIN_PRICE, 80, "a bad file must not replace the store already loaded") + again = load_store_policy(good) + self.assertEqual(again.min_price, 80) + + def test_a_ratio_outside_range_fails_clearly(self): + import json + import tempfile + + load_store_policy(None) + with tempfile.TemporaryDirectory() as root: + with open(os.path.join(root, "store.json"), "w", encoding="utf-8") as handle: + json.dump({"tax_ratios": [2.5], "min_price": 999}, handle) + with self.assertRaises(StorePolicyError) as raised: + load_store_policy(root) + self.assertIn("tax ratio", str(raised.exception)) + self.assertEqual(tag_cards.MIN_PRICE, 50) + class EngineSelectionTests(unittest.TestCase): def test_explicit_none_and_fake(self): diff --git a/recognization/code/tracking.py b/recognization/code/tracking.py index a77bc61..42b7b2a 100644 --- a/recognization/code/tracking.py +++ b/recognization/code/tracking.py @@ -122,16 +122,30 @@ class _SessionByteTrack(BYTETracker): class TrackOut: """One tracked object for a frame.""" - __slots__ = ("track_id", "box", "score", "det_index") + __slots__ = ("track_id", "box", "score", "det_index", "low_score") - def __init__(self, track_id: int, box: List[int], score: float, det_index: Optional[int]): + def __init__( + self, + track_id: int, + box: List[int], + score: float, + det_index: Optional[int], + *, + low_score: bool = False, + ): self.track_id = int(track_id) self.box = [int(v) for v in box] self.score = float(score) self.det_index = None if det_index is None or det_index < 0 else int(det_index) + # ByteTrack's second pass keeps an id alive on a low-score box. That box + # is not a new measurement: callers must not crop or re-label from it. + self.low_score = bool(low_score) def __repr__(self) -> str: # pragma: no cover - debug helper - return f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, det={self.det_index})" + return ( + f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, " + f"det={self.det_index}, low={self.low_score})" + ) class SessionTracker: @@ -171,14 +185,17 @@ class SessionTracker: outputs: List[TrackOut] = [] if rows is None or len(rows) == 0: return outputs + high = float(self.args.track_high_thresh) for row in np.asarray(rows): # [x1, y1, x2, y2, track_id, score, cls, idx] + score = float(row[5]) outputs.append( TrackOut( track_id=int(row[4]), box=[int(round(row[0])), int(round(row[1])), int(round(row[2])), int(round(row[3]))], - score=float(row[5]), + score=score, det_index=int(row[7]) if len(row) > 7 else None, + low_score=score < high, ) ) return outputs diff --git a/recognization/code/transport.py b/recognization/code/transport.py new file mode 100644 index 0000000..292a95e --- /dev/null +++ b/recognization/code/transport.py @@ -0,0 +1,254 @@ +""" +Request bounds and the LAN token check. + +A client can declare a tiny Content-Length and then keep writing, so the +declared length is a hint and the read itself stops at the cap. The token +never travels in a query string: proxies and access logs keep those. +""" + +from __future__ import annotations + +import hmac +import ipaddress +import os +import shutil +import signal +import socket +import subprocess +from typing import Any, Dict, Optional + +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +def is_loopback_host(host: Optional[str]) -> bool: + text = (host or "").strip().lower() + if text.startswith("[") and text.endswith("]"): + text = text[1:-1] + if "%" in text: # fe80::1%lo0 + text = text.split("%", 1)[0] + if text in LOOPBACK_HOSTS: + return True + try: + return ipaddress.ip_address(text).is_loopback + except ValueError: + return False + + +def require_token_for_bind(host: Optional[str], token: Optional[str]) -> None: + """A non-loopback bind with no token is reachable by anyone on that network.""" + if token or is_loopback_host(host): + return + raise SystemExit( + f"refusing to bind {host} without DORAYAKI_API_TOKEN " + "(loopback is the only bind that may run with auth off)" + ) + + +def token_matches(presented: Optional[str], token: Optional[str]) -> bool: + if not presented or not token: + return False + # compare_digest raises when the lengths differ. Fold the length in so a + # short guess neither throws nor returns early. + return hmac.compare_digest( + len(presented).to_bytes(8, "big") + presented.encode(), + len(token).to_bytes(8, "big") + token.encode(), + ) + + +def bearer_matches(authorization: Optional[str], token: str) -> bool: + if not authorization or not token: + return False + scheme, _, presented = authorization.partition(" ") + if scheme.lower() != "bearer" or not presented or " " in presented: + return False + return token_matches(presented, token) + + +def declared_length(header: Optional[str]) -> Optional[int]: + if header is None or header == "": + return None + try: + length = int(header) + except (TypeError, ValueError): + return None + return length if length >= 0 else None + + +async def read_bounded(source, max_bytes: int) -> bytes: + """Stop once the cap is passed. The caller rejects the oversize body. + + A request stream is async-iterable. An UploadFile is not: it has read(n). + """ + out = bytearray() + read = getattr(source, "read", None) + if read is not None and not hasattr(source, "__aiter__"): + while True: + chunk = await read(65536) + if not chunk: + break + out += chunk + if len(out) > max_bytes: + break + return bytes(out) + async for chunk in source: + if not chunk: + continue + out += chunk + if len(out) > max_bytes: + break + return bytes(out) + + +class BodyLimit: + """Stop the ASGI receive once the body passes the cap. + + A handler that checks Content-Length, or reads request.body(), runs after + the server has already buffered the stream. This counts bytes at receive + and returns 413 without handing the rest to the app. WebSocket frames are + not HTTP bodies and are left alone. + """ + + def __init__(self, app: Any, max_bytes: int) -> None: + self.app = app + self.max_bytes = int(max_bytes) + + async def __call__(self, scope, receive, send) -> None: + if scope.get("type") != "http": + await self.app(scope, receive, send) + return + seen = 0 + blocked = False + + async def limited() -> Dict[str, Any]: + nonlocal seen, blocked + if blocked: + return {"type": "http.request", "body": b"", "more_body": False} + message = await receive() + if message.get("type") != "http.request": + return message + body = message.get("body") or b"" + seen += len(body) + if seen > self.max_bytes: + blocked = True + await send( + { + "type": "http.response.start", + "status": 413, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": b'{"error":{"code":"PAYLOAD_TOO_LARGE","message":"request body too large"}}', + } + ) + return {"type": "http.request", "body": b"", "more_body": False} + return message + + try: + await self.app(scope, limited, send) + except RuntimeError: + if not blocked: + raise + + +SERVICE_TYPE = "_dorayaki._tcp" + + +def _lan_address() -> Optional[str]: + """Address a phone on this network can open. Loopback is not one. + + The demo launcher uses `ipconfig getifaddr en0` then `en1`. A UDP connect + is the fallback when those interfaces are absent. + """ + if shutil.which("ipconfig"): + for iface in ("en0", "en1"): + try: + out = subprocess.check_output( + ["ipconfig", "getifaddr", iface], stderr=subprocess.DEVNULL, text=True + ).strip() + except (subprocess.CalledProcessError, OSError): + continue + if out and not is_loopback_host(out): + return out + try: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.connect(("192.0.2.1", 9)) + found = probe.getsockname()[0] + finally: + probe.close() + except OSError: + return None + if not found or is_loopback_host(found): + return None + return found + + +def advertise_host(bind_host: Optional[str]) -> Optional[str]: + """The address clients should browse. A loopback bind is not advertised. + + A wildcard bind (0.0.0.0, ::) has no address of its own, so the record + uses the LAN address the demo launcher would hand a phone. + """ + text = (bind_host or "").strip() + if text.startswith("[") and text.endswith("]"): + text = text[1:-1] + if is_loopback_host(text): + return None + if text in ("", "0.0.0.0", "::"): + return _lan_address() + try: + parsed = ipaddress.ip_address(text) + except ValueError: + return None + if parsed.is_loopback or parsed.is_unspecified: + return None if parsed.is_loopback else _lan_address() + return str(parsed) + + +class ServiceAdvertiser: + """Register `_dorayaki._tcp` with the system mDNS daemon and drop it on close. + + ponytail: macOS `dns-sd` only. A Linux host needs `avahi-publish` wired the + same way; do not add a Python mDNS package for this one record. + """ + + def __init__(self, host: str, port: int, name: str = "Dorayaki") -> None: + self.host = host + self.port = int(port) + self.name = name + self._process: Optional[subprocess.Popen] = None + + def start(self) -> bool: + if self._process is not None: + return True + binary = shutil.which("dns-sd") + if not binary: + return False + self._process = subprocess.Popen( + [binary, "-P", self.name, SERVICE_TYPE, "local", str(self.port), self.host, self.host], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True + + def close(self) -> None: + process = self._process + self._process = None + if process is None or process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + process.wait(timeout=2) diff --git a/recognization/code/web/app.js b/recognization/code/web/app.js index 6661b0e..5eb75cd 100644 --- a/recognization/code/web/app.js +++ b/recognization/code/web/app.js @@ -3,12 +3,24 @@ * * A client of the recognition server, nothing else: live frames go out over * /v1/stream, stills go out over /v1/recognize. Point it at another server with - * ?server=https://host:port and pass a token with ?token=... + * ?server=https://host:port. A token, when the server requires one, comes from + * sessionStorage (key dorayaki.token) or the fragment #token=..., never the query. */ const params = new URLSearchParams(location.search); const SERVER = (params.get('server') || '').replace(/\/$/, ''); -const TOKEN = params.get('token') || ''; +const TOKEN = readToken(); + +function readToken() { + const stored = sessionStorage.getItem('dorayaki.token') || ''; + const fragment = new URLSearchParams((location.hash || '').replace(/^#/, '')); + const fromHash = fragment.get('token') || ''; + if (fromHash) { + sessionStorage.setItem('dorayaki.token', fromHash); + history.replaceState(null, '', location.pathname + location.search); + } + return fromHash || stored; +} const I18N = { en: { @@ -363,7 +375,7 @@ function thumbUrl(skuId) { async function listCameras() { const select = el('sourceSelect'); - select.innerHTML = ''; + select.replaceChildren(); if (!navigator.mediaDevices || !window.isSecureContext) { const option = document.createElement('option'); option.value = 'file'; @@ -390,13 +402,16 @@ function connectStream() { if (state.ws && (state.ws.readyState === WebSocket.OPEN || state.ws.readyState === WebSocket.CONNECTING)) return; setConnection('connecting'); const base = SERVER || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}`; - const url = `${base.replace(/^http/, 'ws')}/v1/stream${TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ''}`; + const url = `${base.replace(/^http/, 'ws')}/v1/stream`; const socket = new WebSocket(url); socket.binaryType = 'arraybuffer'; state.ws = socket; socket.onopen = () => { state.reconnectDelay = 1000; + // Browsers cannot set Authorization on a WebSocket. The server hello + // arrives first; this frame is the token, and it is not a URL. + if (TOKEN) socket.send(JSON.stringify({ type: 'auth', token: TOKEN })); setConnection('live'); socket.send(JSON.stringify({ type: 'config', method: state.method, empty_slots: state.showEmpty })); startSending(); @@ -464,25 +479,25 @@ function sendFrame() { function renderCompliance(compliance) { const strip = el('complianceStrip'); + strip.replaceChildren(); if (!compliance) { strip.hidden = true; return; } const rows = compliance.rows || []; - const items = rows.slice(0, 3).map( - (row) => `⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})` - ); - if (rows.length > 3) { - items.push(`+${rows.length - 3} ${t('more')}`); - } const hints = compliance.hints || []; - if (!items.length && hints.length) { - strip.hidden = false; - strip.dataset.kind = 'hint'; - strip.innerHTML = hints.map((hint) => `${hint.detail}`).join(''); - return; + const chips = []; + for (const row of rows.slice(0, 3)) { + chips.push(`⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})`); } - if (!items.length) { strip.hidden = true; return; } + if (rows.length > 3) chips.push(`+${rows.length - 3} ${t('more')}`); + const texts = chips.length ? chips : hints.map((hint) => hint.detail); + if (!texts.length) { strip.hidden = true; return; } strip.hidden = false; - strip.dataset.kind = 'violation'; - strip.innerHTML = items.join(''); + strip.dataset.kind = chips.length ? 'violation' : 'hint'; + for (const text of texts) { + const item = document.createElement('span'); + item.className = 'compliance-item'; + item.textContent = text; + strip.appendChild(item); + } } function handleResult(message) { @@ -536,7 +551,7 @@ async function loadSamples() { const response = await fetch(api('/v1/samples'), { headers: headers() }); const data = await response.json(); const select = el('sampleSelect'); - select.innerHTML = ''; + select.replaceChildren(); for (const sample of data.samples) { const option = document.createElement('option'); option.value = sample.id; @@ -634,24 +649,42 @@ function renderInView() { } } } + list.replaceChildren(); if (!counts.size) { - list.innerHTML = `
${t('inviewPlaceholder')}
`; + const empty = document.createElement('div'); + empty.className = 'placeholder small'; + empty.textContent = t('inviewPlaceholder'); + list.appendChild(empty); return; } - const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([skuId, facings]) => { + const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]); + for (const [skuId, facings] of ranked) { const product = productFor(skuId); const name = product ? product.product_name : skuId; const price = product ? `¥${Math.round(product.price)}` : ''; - return `
- -
${name}
-
${price} · ${t('facings')} ${facings}
-
${facings}
`; - }).join(''); - list.innerHTML = rows; - list.querySelectorAll('.inview-row').forEach((row) => { - row.addEventListener('click', () => selectBySku(row.dataset.sku)); - }); + const row = document.createElement('div'); + row.className = 'inview-row'; + row.dataset.sku = skuId; + const thumb = document.createElement('img'); + thumb.className = 'inview-thumb'; + thumb.alt = ''; + thumb.src = thumbUrl(skuId); + const text = document.createElement('div'); + text.className = 'inview-text'; + const nameEl = document.createElement('div'); + nameEl.className = 'inview-name'; + nameEl.textContent = name; + const meta = document.createElement('div'); + meta.className = 'inview-meta'; + meta.textContent = `${price} · ${t('facings')} ${facings}`; + text.append(nameEl, meta); + const count = document.createElement('div'); + count.className = 'inview-facings'; + count.textContent = String(facings); + row.append(thumb, text, count); + row.addEventListener('click', () => selectBySku(skuId)); + list.appendChild(row); + } } function selectBySku(skuId) { @@ -715,9 +748,19 @@ function renderOcrSection(tag, reconciliation) { if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]); } if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']); - rows.innerHTML = lines - .map(([key, value, bad]) => `
${key}${value}
`) - .join(''); + rows.replaceChildren(); + for (const [key, value, bad] of lines) { + const row = document.createElement('div'); + row.className = 'ocr-row'; + const keyEl = document.createElement('span'); + keyEl.className = 'ocr-key'; + keyEl.textContent = key; + const valueEl = document.createElement('span'); + valueEl.className = bad ? 'ocr-val bad' : 'ocr-val'; + valueEl.textContent = value; + row.append(keyEl, valueEl); + rows.appendChild(row); + } } function showProductCard(product, identity) { @@ -755,12 +798,22 @@ function showCandidatesCard(identity) { el('metricReceipt').textContent = '—'; el('metricExpiry').textContent = '—'; const candidates = el('candidates'); - const rows = (identity.candidates || []).map((candidate) => { + candidates.replaceChildren(); + const title = document.createElement('div'); + title.className = 'candidates-title'; + title.textContent = t('candidatesTitle'); + candidates.appendChild(title); + for (const candidate of identity.candidates || []) { const product = productFor(candidate.sku_id); - const name = product ? product.product_name : candidate.sku_id; - return `
${name}${candidate.score}
`; - }).join(''); - candidates.innerHTML = `
${t('candidatesTitle')}
${rows}`; + const row = document.createElement('div'); + row.className = 'candidate-row'; + const name = document.createElement('span'); + name.textContent = product ? product.product_name : candidate.sku_id; + const score = document.createElement('span'); + score.textContent = String(candidate.score); + row.append(name, score); + candidates.appendChild(row); + } candidates.hidden = false; el('showMatchBtn').disabled = false; renderOcrSection(state.currentTag, state.currentReconciliation); @@ -889,10 +942,18 @@ function handleInspect(result) { if (result.crop_jpeg_b64) { el('productThumb').src = result.crop_jpeg_b64; } + const box = el('matchBox'); + box.replaceChildren(); if (result.match_visualization_jpeg_b64) { - el('matchBox').innerHTML = `feature matches`; + const image = document.createElement('img'); + image.alt = 'feature matches'; + image.src = result.match_visualization_jpeg_b64; + box.appendChild(image); } else { - el('matchBox').innerHTML = `${t('matchPlaceholder')}`; + const placeholder = document.createElement('span'); + placeholder.className = 'match-placeholder'; + placeholder.textContent = t('matchPlaceholder'); + box.appendChild(placeholder); } const identity = result.identity || {}; const parts = []; diff --git a/recognization/detected_empty_slots.jpg b/recognization/detected_empty_slots.jpg deleted file mode 100644 index d24ce1e..0000000 Binary files a/recognization/detected_empty_slots.jpg and /dev/null differ diff --git a/recognization/detected_image.jpg b/recognization/detected_image.jpg deleted file mode 100644 index ce539b1..0000000 Binary files a/recognization/detected_image.jpg and /dev/null differ diff --git a/recognization/pyproject.toml b/recognization/pyproject.toml new file mode 100644 index 0000000..a97a54a --- /dev/null +++ b/recognization/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "dorayaki-recognition" +version = "0.1.0" +description = "Dorayaki shelf recognition server" +requires-python = ">=3.10,<3.13" +readme = "README.md" +dependencies = [ + "torch>=2.0.0", + "torchvision>=0.15.0", + "ultralytics==8.4.153", + "lap>=0.5.12", + "websockets>=13", + "opencv-python>=4.8.0", + "numpy>=1.23.0,<2.0.0", + "fastapi>=0.100.0", + "uvicorn>=0.20.0", + "python-multipart>=0.0.6", + "requests>=2.28.0", + "pyobjc-framework-Vision>=10; sys_platform == 'darwin'", + "pyobjc-framework-Quartz>=10; sys_platform == 'darwin'", +] + +[project.optional-dependencies] +dev = ["httpx>=0.27"] + +[tool.pytest.ini_options] +testpaths = ["code"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +addopts = ["-q"] + +[tool.setuptools] +py-modules = [] diff --git a/recognization/requirements.lock b/recognization/requirements.lock new file mode 100644 index 0000000..06fa8eb --- /dev/null +++ b/recognization/requirements.lock @@ -0,0 +1,79 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.txt requirements-dev.txt --universal --python-version 3.12 -o requirements.lock --no-annotate +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +cloudpickle==3.1.2 +contourpy==1.3.3 +cuda-bindings==13.4.3 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-pathfinder==1.8.2 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-toolkit==13.0.3.0 ; sys_platform == 'linux' +cycler==0.12.1 +fastapi==0.141.1 +filelock==4.0.4 +fonttools==4.66.0 +fsspec==2026.9.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +jinja2==3.1.6 +kiwisolver==1.5.1 +lap==0.5.13 +markupsafe==3.0.3 +matplotlib==3.11.2 +mpmath==1.3.0 +networkx==3.7 +numpy==1.26.4 +nvidia-cublas==13.1.1.3 ; sys_platform == 'linux' +nvidia-cuda-cupti==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cuda-nvrtc==13.0.88 ; sys_platform == 'linux' +nvidia-cuda-runtime==13.0.96 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cudnn-cu13==9.24.0.43 ; sys_platform == 'linux' +nvidia-cufft==12.0.0.61 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cufile==1.15.1.6 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-curand==10.4.0.35 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusolver==12.0.4.66 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparse==12.6.3.3 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' +nvidia-ml-py==13.615.71 +nvidia-nccl-cu13==2.30.7 ; sys_platform == 'linux' +nvidia-nvjitlink==13.4.92 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' +nvidia-nvtx==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +opencv-python==4.11.0.86 +packaging==26.3 +pillow==12.3.0 +polars==1.44.2 +polars-runtime-32==1.44.2 +psutil==7.2.2 +pydantic==2.13.5 +pydantic-core==2.46.5 +pyobjc-core==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-cocoa==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-coreml==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-quartz==12.2.2 ; sys_platform == 'darwin' +pyobjc-framework-vision==12.2.2 ; sys_platform == 'darwin' +pyparsing==3.3.3 +python-dateutil==2.9.0.post0 +python-multipart==0.0.32 +pyyaml==6.0.3 +requests==2.34.2 +setuptools==84.0.0 +six==1.17.0 +starlette==1.7.0 +sympy==1.14.0 +torch==2.14.0 +torchvision==0.29.0 +triton==3.8.0 ; python_full_version < '3.15' and sys_platform == 'linux' +typing-extensions==4.16.0 +typing-inspection==0.4.4 +ultralytics==8.4.153 +ultralytics-platform==0.1.65 +ultralytics-thop==2.2.1 +urllib3==2.8.0 +uvicorn==0.54.0 +websockets==17.1 diff --git a/recognization/run-demo-server.sh b/recognization/run-demo-server.sh index 6f0e5c2..62084e6 100755 --- a/recognization/run-demo-server.sh +++ b/recognization/run-demo-server.sh @@ -1,18 +1,16 @@ #!/usr/bin/env bash -# The demo server: one command, bound to the network so a phone can reach it. +# Start the recognition server and write ignored local app endpoints. # -# ./run-demo-server.sh # both demo shelves enrolled (stock-demo) +# ./run-demo-server.sh # ./run-demo-server.sh stock-milk150 -# -# The iPhone must be on the same Wi-Fi as this Mac, and the app must point at this -# Mac's address - Config.xcconfig's APP_RECOGNITION_BASE_URL, which the app reads at -# build time. The script prints the exact line to use. set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$(dirname "$0")" STOCK="${1:-stock-demo}" PORT="${PORT:-8099}" +TOKEN_FILE=".demo-token" if [ ! -d "$STOCK" ]; then echo "no such store: $STOCK" >&2 @@ -25,21 +23,49 @@ if [ -z "$IP" ]; then exit 1 fi +if [ ! -s "$TOKEN_FILE" ]; then + umask 077 + openssl rand -hex 32 > "$TOKEN_FILE" +fi +TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")" + +umask 077 +cat > "$ROOT/iosApp/Configuration/Config.local.xcconfig" < config/local.env < "$TMP" +mv "$TMP" "$PROPERTIES" + cat <