diff --git a/recognization/code/app.py b/recognization/code/app.py index 7053f87..5ae917b 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -72,7 +72,6 @@ from engine import ( # noqa: E402 identity_payload, settings_from_env, ) -from demo import DemoCamera, DemoSourceError # noqa: E402 from record import FrameRecorder # noqa: E402 from live import LiveSession, LiveSettings # noqa: E402 @@ -100,7 +99,6 @@ class ServerSettings: live_method: str = "hybrid" live_conf: float = 0.1 live_tags: bool = True - demo_walk: bool = False # A live session can be recorded so it becomes data to label (record.py). record_frames: Optional[str] = None record_every: int = 3 @@ -250,9 +248,10 @@ def create_app( ) scans = scans if scans is not None else build_legacy_scans(engine) + # A live session can be recorded so a real scan becomes data to label later. + recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ + if settings.record_frames else None samples = {item["id"]: item for item in get_available_shelf_images("add")} - demo_camera = DemoCamera(samples) - demo_cameras: Dict[float, DemoCamera] = {} sessions: Dict[str, LiveSession] = {} sessions_lock = asyncio.Lock() app.state.engine = engine @@ -323,110 +322,14 @@ def create_app( listing = [] for item in samples.values(): entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"} - try: # size is a convenience for clients that plan a demo walk - width, height = demo_camera._size(item["id"]) - entry["width"], entry["height"] = width, height - except DemoSourceError: - pass + import cv2 + + image = cv2.imread(item["path"]) + if image is not None: + entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0]) listing.append(entry) return {"samples": listing} - # ---------------------------------------------------------------- demo camera - # The demo replay lives in its own module: it replaces the live tracks with a - # still recognition mapped into each walk frame, which is a film-making device, - # not a perception result. Every payload it produces is marked synthetic. - demo_recognition = None - recorder = FrameRecorder(settings.record_frames, every=settings.record_every) \ - if settings.record_frames else None - - - - def _demo_samples(raw: Optional[str]) -> List[str]: - if raw: - wanted = [part.strip() for part in raw.split(",") if part.strip()] - unknown = [part for part in wanted if part not in samples] - if unknown: - raise _error("NOT_FOUND", f"unknown sample(s): {', '.join(unknown)}", 404) - return wanted - # main_shelf first: its products are the ones the catalog covers, so a - # demo walk recognises real items instead of unknown shelf neighbours. - for preferred in ("main_shelf", "image6"): - if preferred in samples: - return [preferred] - return [next(iter(samples))] if samples else [] - - # Which sample the walk uses, resolved once. The replay object itself is built - # later, because it needs the still tag pass. - demo_sample_ids: List[str] = [] - if settings.demo_walk: - try: - demo_sample_ids = _demo_samples(None) - demo_camera.warm(demo_sample_ids) - except Exception as exc: - LOGGER.warning("demo walk unavailable: %s", exc) - - def _demo_window(width: Optional[int], height: Optional[int]) -> tuple: - win_w = int(width or 960) - win_h = int(height or 540) - if not (160 <= win_w <= 2048 and 120 <= win_h <= 2048): - raise _error("BAD_REQUEST", "window must be between 160x120 and 2048x2048", 400) - return win_w, win_h - - def _demo_camera(pan: Optional[float]) -> DemoCamera: - """One camera per pacing value (the walk geometry depends on it).""" - if pan is None: - return demo_camera - key = round(float(pan), 4) - camera = demo_cameras.get(key) - if camera is None: - camera = DemoCamera(samples, pan_overlap=key) - demo_cameras[key] = camera - return camera - - @app.get("/v1/demo/plan", dependencies=[Depends(authorize)]) - async def demo_plan(samples_param: Optional[str] = None, width: Optional[int] = None, - height: Optional[int] = None, pan: Optional[float] = None) -> Dict[str, Any]: - """Geometry of the virtual walk (demo-only capture source).""" - try: - ids = _demo_samples(samples_param) - if not ids: - raise _error("NOT_FOUND", "no samples available", 404) - camera = _demo_camera(pan) - return {"status": "success", **camera.plan(ids, _demo_window(width, height))} - except DemoSourceError as exc: - raise _error("NOT_FOUND", str(exc), 404) - - @app.get("/v1/demo/frame", dependencies=[Depends(authorize)]) - async def demo_frame(step: int = 0, samples_param: Optional[str] = None, - width: Optional[int] = None, height: Optional[int] = None, - quality: int = 65, pan: Optional[float] = None) -> Response: - """One JPEG frame of the virtual walk: a store employee sweeping the shelf. - - Demo-only. The client advances `step` (it wraps), so the walk's speed - follows whatever frame rate the client can sustain. - """ - try: - ids = _demo_samples(samples_param) - if not ids: - raise _error("NOT_FOUND", "no samples available", 404) - payload, position = _demo_camera(pan).frame( - ids, step, _demo_window(width, height), max(30, min(95, int(quality))) - ) - except DemoSourceError as exc: - raise _error("NOT_FOUND", str(exc), 404) - return Response( - content=payload, - media_type="image/jpeg", - headers={ - "X-Demo-Sample": str(position["sample"]), - "X-Demo-Bay": str(position["bay_index"] + 1), - "X-Demo-Step": str(position["step"]), - "X-Demo-Steps": str(position["steps"]), - "X-Demo-X": str(position["x"]), - "X-Demo-Y": str(position["y"]), - }, - ) - @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)]) async def get_sample(sample_id: str) -> Response: item = samples.get(sample_id) @@ -599,19 +502,6 @@ def create_app( engine.image_store.set_tag_bundle(result["image_id"], bundle) return result - if settings.demo_walk and demo_sample_ids: - # Built here because the replay needs the still tag pass, and it recognises - # the whole shelf once up front so the walk's first frame is already - # labelled. Every payload it produces is marked synthetic. - try: - from demo_replay import DemoRecognition - - demo_recognition = DemoRecognition(engine, demo_camera, samples, _attach_still_tags) - demo_recognition.warm(demo_sample_ids[0]) - LOGGER.info("demo walk: '%s' recognised up front", demo_sample_ids[0]) - except Exception as exc: - LOGGER.warning("demo walk warm-up failed: %s", exc) - @app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)]) async def classify_image(image_id: str, body: ClassifyBody) -> Dict[str, Any]: method = _require_method(body.method) @@ -715,7 +605,6 @@ def create_app( await websocket.accept() send_lock = asyncio.Lock() loop = asyncio.get_running_loop() - demo_steps = [0] async def send(payload: Dict[str, Any]) -> None: async with send_lock: @@ -755,21 +644,9 @@ def create_app( "message": "frame processing failed"}) continue if recorder is not None: - # Record what the client sent and what we saw in it, so the walk - # can be labelled later instead of only measured on photographs. + # Record what the client sent and what we saw in it, so a real + # scan can be labelled later instead of only measured on photos. recorder.write(session_id, frame_id, payload, result) - if demo_recognition is not None and demo_sample_ids: - # The walk's frames are served from the shelf's full recognition, - # so every product is already labelled with name and price. - sample_id = demo_sample_ids[0] - step = demo_steps[0] - try: - result = await loop.run_in_executor( - None, demo_recognition.apply, result, sample_id, step - ) - demo_steps[0] = (step + 1) % max(1, demo_recognition.steps(sample_id)) - except Exception as exc: # a demo failure must not kill the stream - LOGGER.exception("demo walk frame failed: %s", exc) await send(result) async def handle_inspect(message: Dict[str, Any]) -> None: @@ -1044,15 +921,13 @@ def main() -> None: parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"], help="which mechanism names products (default matcher)") parser.add_argument("--live-inflight", type=int, default=None, - help="classifications in flight per session (demo walks want a larger budget)") + help="classifications in flight per session") parser.add_argument("--live-attempts", type=int, default=None, help="classification attempts per track") parser.add_argument("--record-frames", type=str, default=None, help="write processed frames and their results under this directory") parser.add_argument("--record-every", type=int, default=None, help="record one frame in every N (default 3)") - parser.add_argument("--demo-walk", action="store_true", - help="serve the demo walk's pre-recognised shelf as live tracks") parser.add_argument("--no-tags", action="store_true", help="disable price-tag OCR") parser.add_argument("--max-sessions", type=int, default=None) parser.add_argument("--ssl-certfile", type=str, default=None) @@ -1076,8 +951,6 @@ def main() -> None: server.live_method = args.live_method if args.live_conf is not None: server.live_conf = args.live_conf - if args.demo_walk: - server.demo_walk = True if args.no_tags: server.live_tags = False if args.ssl_certfile is not None: diff --git a/recognization/code/bench_shelf_map.py b/recognization/code/bench_shelf_map.py deleted file mode 100644 index 3988b64..0000000 --- a/recognization/code/bench_shelf_map.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -What does a shelf map buy over track ids? - -Walks the demo camera once, feeds every frame through both the live session and a -ShelfMap, and reports what each counts. Needs a running server on :8077. - - STEPS=58 python code/bench_shelf_map.py -""" -import json, sys, time, urllib.request -sys.path.insert(0, "code") -import cv2, numpy as np -from websockets.sync.client import connect -from shelf_map import ShelfMap - -BASE = "http://127.0.0.1:8077" -STEPS = int(__import__("os").environ.get("STEPS", "58")) # one walk over image6 at pan 0.02 - -def fetch(step): - url = f"{BASE}/v1/demo/frame?samples_param=image6&step={step}&width=1280&height=720&quality=70&pan=0.02" - return urllib.request.urlopen(url, timeout=10).read() - -shelf = ShelfMap() -track_ids, frames, detections = set(), 0, 0 -started = time.perf_counter() -with connect("ws://127.0.0.1:8077/v1/stream") as ws: - ws.recv(timeout=15) - for step in range(STEPS): - payload = fetch(step) - frame = cv2.imdecode(np.frombuffer(payload, np.uint8), cv2.IMREAD_COLOR) - ws.send(step.to_bytes(4, "big") + payload) - msg = json.loads(ws.recv(timeout=25)) - if msg.get("type") != "result": - continue - frames += 1 - tracks = msg["tracks"] - detections += len(tracks) - for track in tracks: - track_ids.add(track["track_id"]) - shelf.update(step, frame, tracks) - time.sleep(0.05) -elapsed = time.perf_counter() - started - -stats = shelf.stats() -print(f"{frames} frames in {elapsed:.1f} s, {detections} detections") -print(f" track ids seen (what the live session counts): {len(track_ids)}") -print(f" facings on the shelf map (what is actually there): {stats['facings']}") -print(f" merges {stats['merges']}, registrations {stats['registrations']}, failures {stats['registration_failures']}") -print(f" median registration response {stats['median_response']}") -print(f" facings whose pooled identity agrees with itself >= 0.8: {stats['identity_consistent']}") - -facings = [f for f in shelf.facings() if f.observations >= 3] -print(f"\nfacings seen at least 3 times: {len(facings)}") -for facing in sorted(facings, key=lambda f: (f.shelf_box[1], f.shelf_box[0]))[:10]: - sku, agreement = facing.best_identity() - tag_sku, tag_agreement = facing.best_tag_identity() - print(f" shelf x={facing.shelf_box[0]:>5} y={facing.shelf_box[1]:>5} obs={facing.observations:>2} " - f"tracks={len(facing.seen_track_ids):>2} identity={sku} ({agreement:.2f}) tag={tag_sku} ({tag_agreement:.2f})") - -# render the map so duplicates are visible rather than inferred -import numpy as np -facings = shelf.facings() -xs = [f.shelf_box[0] for f in facings] + [f.shelf_box[2] for f in facings] -ys = [f.shelf_box[1] for f in facings] + [f.shelf_box[3] for f in facings] -x0, x1 = min(xs), max(xs) -y0, y1 = min(ys), max(ys) -scale = 0.6 -canvas = np.full((int((y1 - y0) * scale) + 20, int((x1 - x0) * scale) + 20, 3), 20, np.uint8) -for facing in facings: - bx = [(facing.shelf_box[0] - x0) * scale + 10, (facing.shelf_box[1] - y0) * scale + 10, - (facing.shelf_box[2] - x0) * scale + 10, (facing.shelf_box[3] - y0) * scale + 10] - colour = (0, 200, 0) if facing.observations >= 5 else (0, 120, 255) - cv2.rectangle(canvas, (int(bx[0]), int(bx[1])), (int(bx[2]), int(bx[3])), colour, 1) -cv2.imwrite("/tmp/dorayaki-verify/shelf-map.png", canvas) -print(f"\nmap canvas {canvas.shape[1]}x{canvas.shape[0]} px covering shelf x {x0}..{x1}, y {y0}..{y1}") -from collections import Counter -print("observations per facing:", Counter(min(f.observations, 20) for f in facings).most_common(6)) -print("facings per y band (100 px):", Counter((f.shelf_box[1] - y0) // 100 for f in facings).most_common(8)) diff --git a/recognization/code/collect_shelf_images.py b/recognization/code/collect_shelf_images.py index 402d2fc..56e20a2 100644 --- a/recognization/code/collect_shelf_images.py +++ b/recognization/code/collect_shelf_images.py @@ -36,22 +36,23 @@ MAX_RETRIES = 4 # Shelf photos of the categories the catalog covers: drinks and dairy, rice # crackers and snacks, sauces and condiments. QUERIES = [ - "supermarket shelf Japan", - "Japanese supermarket beverages shelf", - "Japanese convenience store drink shelf", - "milk shelf Japan", - "drink shelf Japan", - "Japanese supermarket snack shelf", - "rice crackers Japan shop", - "せんべい 売り場", - "飲料 棚 スーパー", - "牛乳 棚", - "スーパーマーケット 陳列", - "Japanese grocery store interior", - "Japanese supermarket dairy", - "菓子 棚", - "supermarket drinks Japan", - "Japanese supermarket soft drinks", + # the catalog's own products first: drinks, dairy, rice crackers + "野菜生活100", + "伊藤園 1日分の野菜", + "Japanese vegetable juice shelf", + "Japanese milk cartons supermarket", + "meiji oishii milk", + "せんべい 棚", + "Japanese rice cracker shop display", + "Japanese confectionery shelf", + "Japanese supermarket beverage aisle", + "Japanese convenience store refrigerator drinks", + "Japanese supermarket dairy case", + "スーパー 飲料 陳列", + "お菓子 売り場", + "Japanese snack aisle supermarket", + "Japanese supermarket shelves drinks bottles", + "Japanese supermarket interior shelves", ] PERMISSIVE = ("cc0", "cc-by", "cc by", "public domain", "pd-", "attribution") diff --git a/recognization/code/demo.py b/recognization/code/demo.py deleted file mode 100644 index 8613b21..0000000 --- a/recognization/code/demo.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -Synthetic camera for demos: a store employee walking a shelf. - -A simulator has no camera, and a still image makes a poor demo - nothing moves, so -nothing new is ever recognised. This module turns the demo shelf photos into a -virtual walk: a window pans left to right along a row, steps up to the next row, -and pans back the other way (serpentine, ascending), exactly the way a person -sweeps a shelf with a tablet. - -The client drives only `step`; all geometry lives here, so the web client, the -phone app, and the reference client see the same walk. `step` wraps, so an -unattended demo loops forever. - -Demo-only: this is a capture source, not a perception feature. `/v1/demo/*` is -documented as such. -""" - -from __future__ import annotations - -import math -import os -import threading -import time -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple - -DEFAULT_WINDOW = (960, 540) -DEFAULT_QUALITY = 65 -# How far the window advances per step, as a fraction of its own size. This is -# the demo's most important number: advance too fast and a product is in view for -# one or two frames, so the classifier never gets a chance and nothing is ever -# labelled (measured: 0.22 gave 9 labelled tracks in 20 s; the shelf swept past -# faster than a person could read it). 0.06 keeps a facing in view for roughly a -# second at demo frame rates - a deliberate, human sweep. -PAN_OVERLAP = 0.06 -ROW_OVERLAP = 0.25 -# Vertical steps spent moving between rows (the "look up, then keep going" beat). -TRANSITION_STEPS = 6 - - -class DemoSourceError(ValueError): - """Unknown sample or an impossible window.""" - - -@dataclass -class Bay: - """One shelf photo in the walk, with its geometry.""" - - sample_id: str - path: str - width: int - height: int - steps: int - - -class DemoCamera: - """Plans and renders the virtual walk over one or more shelf photos.""" - - def __init__(self, samples: Dict[str, Dict[str, str]], window: Tuple[int, int] = DEFAULT_WINDOW, - pan_overlap: float = PAN_OVERLAP): - self._samples = samples - self._window = window - # How far the window advances per step. The demo's pacing knob: a smaller - # value is a slower, more deliberate sweep (more steps, a facing in view - # for longer). Clamped so a client cannot ask for a frozen or teleporting - # camera. - self.pan_overlap = max(0.005, min(0.5, float(pan_overlap))) - self._lock = threading.Lock() - self._images: Dict[str, Any] = {} - self._sizes: Dict[str, Tuple[int, int]] = {} - self._plans: Dict[Tuple[str, ...], List[Bay]] = {} - - # ------------------------------------------------------------------ planning - def _size(self, sample_id: str) -> Tuple[int, int]: - cached = self._sizes.get(sample_id) - if cached is not None: - return cached - item = self._samples.get(sample_id) - if item is None or not os.path.exists(item["path"]): - raise DemoSourceError(f"unknown sample '{sample_id}'") - import cv2 - - image = cv2.imread(item["path"]) - if image is None: - raise DemoSourceError(f"cannot read sample '{sample_id}'") - with self._lock: - self._images[sample_id] = image - self._sizes[sample_id] = (int(image.shape[1]), int(image.shape[0])) - return self._sizes[sample_id] - - def _image(self, sample_id: str) -> Any: - if sample_id not in self._images: - self._size(sample_id) - return self._images[sample_id] - - def steps_for(self, sample_id: str, window: Tuple[int, int]) -> int: - width, height = self._size(sample_id) - win_w, win_h = window - bands = self._bands(height, win_h) - per_band = self._pan_steps(width, win_w) - # Every band pans, and every band after the first needs the step-up. - return bands * per_band + max(0, bands - 1) * TRANSITION_STEPS - - def _bands(self, image_height: int, window_height: int) -> int: - if window_height >= image_height: - return 1 - stride = max(1, int(window_height * (1.0 - ROW_OVERLAP))) - return max(1, int(math.ceil((image_height - window_height) / stride)) + 1) - - def _pan_steps(self, image_width: int, window_width: int) -> int: - if window_width >= image_width: - return 1 - stride = max(1, int(window_width * self.pan_overlap)) - return max(1, int(math.ceil((image_width - window_width) / stride)) + 1) - - def plan(self, sample_ids: Sequence[str], window: Optional[Tuple[int, int]] = None) -> Dict[str, Any]: - window = window or self._window - key = (tuple(sample_ids), window) - bays = self._plans.get(key) - if bays is None: - bays = [] - for sample_id in sample_ids: - width, height = self._size(sample_id) - bays.append( - Bay( - sample_id=sample_id, - path=self._samples[sample_id]["path"], - width=width, - height=height, - steps=self.steps_for(sample_id, window), - ) - ) - self._plans[key] = bays - total = sum(bay.steps for bay in bays) - return { - "samples": [bay.sample_id for bay in bays], - "window": {"width": window[0], "height": window[1]}, - "bays": [ - {"sample": bay.sample_id, "width": bay.width, "height": bay.height, - "steps": bay.steps, "first_step": sum(b.steps for b in bays[:index])} - for index, bay in enumerate(bays) - ], - "steps": total, - "row_overlap": ROW_OVERLAP, - "pan_overlap": self.pan_overlap, - } - - # ------------------------------------------------------------------ rendering - def window_at( - self, sample_ids: Sequence[str], step: int, window: Optional[Tuple[int, int]] = None - ) -> Dict[str, Any]: - """Where the virtual camera is at this step (wraps around the walk).""" - window = window or self._window - plan = self.plan(sample_ids, window) - total = plan["steps"] - step = int(step) % max(1, total) - offset = 0 - for index, bay in enumerate(plan["bays"]): - if step < offset + bay["steps"]: - local = step - offset - x, y = self._position(local, bay["width"], bay["height"], window) - return { - "sample": bay["sample"], - "bay_index": index, - "step": step, - "steps": total, - "local_step": local, - "bay_steps": bay["steps"], - "x": x, - "y": y, - "width": min(window[0], bay["width"]), - "height": min(window[1], bay["height"]), - } - offset += bay["steps"] - raise DemoSourceError("step outside the walk") - - def _position(self, local_step: int, image_width: int, image_height: int, window: Tuple[int, int]) -> Tuple[int, int]: - """Serpentine: pan a row, step up, pan back.""" - win_w = min(window[0], image_width) - win_h = min(window[1], image_height) - bands = self._bands(image_height, win_h) - per_band = self._pan_steps(image_width, win_w) - cycle = per_band + TRANSITION_STEPS - band = min(bands - 1, local_step // cycle) - within = local_step - band * cycle - stride_y = 0 if bands == 1 else max(1, int(win_h * (1.0 - ROW_OVERLAP))) - # Ascending: band 0 is the bottom of the shelf. - y_bottom = min(image_height - win_h, band * stride_y) - y = max(0, image_height - win_h - y_bottom) - stride_x = max(1, int(win_w * self.pan_overlap)) - max_x = max(0, image_width - win_w) - if within < per_band: - pan = within - x = min(max_x, pan * stride_x) - else: - # The step-up beat: hold x at the end of the row and climb. - transition = min(TRANSITION_STEPS, max(1, within - per_band + 1)) - pan = per_band - 1 - x = min(max_x, pan * stride_x) - next_band = min(bands - 1, band + 1) - next_y_bottom = min(image_height - win_h, next_band * stride_y) - ratio = transition / float(TRANSITION_STEPS) - y = int(round(y - (next_y_bottom - y_bottom) * ratio)) - if band % 2 == 1 and within < per_band: - # Odd bands pan back the other way, like a person reversing direction. - x = max(0, max_x - x) - return int(max(0, min(max_x, x))), int(max(0, min(image_height - win_h, y))) - - def frame( - self, - sample_ids: Sequence[str], - step: int, - window: Optional[Tuple[int, int]] = None, - quality: int = DEFAULT_QUALITY, - ) -> Tuple[bytes, Dict[str, Any]]: - """The JPEG frame the virtual camera sees at this step, plus its geometry.""" - import cv2 - - position = self.window_at(sample_ids, step, window) - image = self._image(position["sample"]) - x, y = position["x"], position["y"] - w, h = position["width"], position["height"] - crop = image[y:y + h, x:x + w] - if crop.size == 0: - raise DemoSourceError("empty crop") - ok, buffer = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, int(quality)]) - if not ok: - raise DemoSourceError("JPEG encode failed") - return bytes(buffer), position - - def warm(self, sample_ids: Sequence[str]) -> None: - """Decode the bays up front so the first demo frame is not the slow one.""" - for sample_id in sample_ids: - self._size(sample_id) - - -if __name__ == "__main__": # smoke: python code/demo.py [sample ...] - import sys - - sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - from app import get_available_shelf_images - - samples = {item["id"]: item for item in get_available_shelf_images("add")} - camera = DemoCamera(samples) - ids = sys.argv[1:] or ["image6"] - plan = camera.plan(ids) - print(f"walk over {plan['samples']}: {plan['steps']} steps, window {plan['window']}") - started = time.perf_counter() - for step in range(min(plan["steps"], 24)): - payload, position = camera.frame(ids, step) - print(f" step {step:3d} {position['sample']} x={position['x']:4d} y={position['y']:4d} " - f"{len(payload) // 1024} KB") - elapsed = (time.perf_counter() - started) * 1000 / max(1, min(plan["steps"], 24)) - print(f"{elapsed:.1f} ms per frame") diff --git a/recognization/code/demo_replay.py b/recognization/code/demo_replay.py deleted file mode 100644 index 78ad9c6..0000000 --- a/recognization/code/demo_replay.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Demo-only: replay a shelf's recognition as if it were live tracking. - -`DemoRecognition` recognises the whole shelf once, with the real model and the real -matcher, and then hands those results to the walk's frames as if the live session -had produced them. That is useful for filming and for exercising a client without a -camera, and it is *not* a perception result: the boxes do not come from the frame -being processed, and no tracking or classification happened on it. Every payload it -touches is marked `synthetic: true` so a client can tell. - -It lives here rather than in `app.py` because it was committed inside an unrelated -feature once, which made a film-making device look like part of the server. -""" - -from __future__ import annotations - -import threading -from typing import Any, Dict, List, Optional - -from demo import DemoSourceError - - -class DemoRecognition: - """The demo shortcut. - - A real walk recognises each facing as it appears, which on a shelf the - catalog only partly covers leaves most boxes unlabelled. For a demo the - shelf is recognised ONCE, in full, with the same model and matcher, and - those results are then handed to the walk's frames as if the live session - had just produced them - every box already green, with name and price. - Demo-only: enabled by --demo-walk. - """ - - _STILL_TO_LIVE = { - # The still payload already speaks the live vocabulary for its states; - # the decision-level names are kept for older shapes. - "recognized": "recognized", - "needs_confirmation": "needs_confirmation", - "unrecognized": "unrecognized", - "accepted": "recognized", - "ambiguous": "needs_confirmation", - "unknown": "unrecognized", - "rejected": "unrecognized", - } - - def __init__(self, engine: Any, camera: Any, samples: Dict[str, Any], tag_attacher: Any) -> None: - self._engine = engine - self._camera = camera - self._samples = samples - self._tag_attacher = tag_attacher - self._catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()} - self._cache: Dict[str, Any] = {} - self._lock = threading.Lock() - - def steps(self, sample_id: str) -> int: - return self._camera.plan([sample_id], (1280, 720)).get("steps", 0) - - def recognition(self, sample_id: str) -> Any: - cached = self._cache.get(sample_id) - if cached is not None: - return cached - import cv2 - - path = self._samples[sample_id]["path"] - image = cv2.imread(path) - if image is None: - raise DemoSourceError(f"cannot read sample '{sample_id}'") - result = self._engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) - if self._tag_attacher is not None: - result = self._tag_attacher(result, image) - with self._lock: - self._cache[sample_id] = (result, int(image.shape[1]), int(image.shape[0])) - return self._cache[sample_id] - - def warm(self, sample_id: str) -> None: - self.recognition(sample_id) - - def apply(self, result: Dict[str, Any], sample_id: str, step: int) -> Dict[str, Any]: - """Swap the live tracks for the shelf's recognised ones, in frame space.""" - still, _, _ = self.recognition(sample_id) - position = self._camera.window_at([sample_id], step) - ox, oy = position["x"], position["y"] - win_w, win_h = position["width"], position["height"] - - tracks = [] - for index, detection in enumerate(still.get("detections") or []): - box = [int(v) for v in detection.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - frame_box = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - identity = detection.get("identity") or {} - state = self._STILL_TO_LIVE.get(identity.get("state") or "", "unrecognized") - sku_id = identity.get("sku_id") - product = self._catalog_by_sku.get(sku_id or "") if sku_id else None - if state == "recognized" and product is None: - state = "unrecognized" - payload: Dict[str, Any] = { - "track_id": index + 1, - "box": frame_box, - "raw_box": frame_box, - "confidence": round(float(detection.get("confidence") or 0.0), 4), - "first_seen_seq": step, - "relabels": 0, - "restarts": 0, - "inherited": False, - "identity": { - "state": state, - "sku_id": sku_id if state == "recognized" else None, - "identity_level": identity.get("identity_level"), - "confidence": identity.get("confidence"), - "score": identity.get("score"), - "candidates": list(identity.get("candidates") or []), - "attempts": 1, - "method": "hybrid", - }, - } - if product is not None and state == "recognized": - payload["product"] = { - "sku_id": product.get("sku_id"), - "product_name": product.get("product_name"), - "price": product.get("price"), - } - tracks.append(payload) - - slots = [] - for slot in still.get("empty_slots") or []: - box = [int(v) for v in slot.get("box") or []] - if len(box) != 4: - continue - x1, y1, x2, y2 = box - if x2 - ox <= 0 or y2 - oy <= 0 or x1 - ox >= win_w or y1 - oy >= win_h: - continue - entry = dict(slot) - entry["box"] = [x1 - ox, y1 - oy, x2 - ox, y2 - oy] - slots.append(entry) - - result["tracks"] = tracks - result["empty_slots"] = slots - counts: Dict[str, int] = {} - for track in tracks: - counts[track["identity"]["state"]] = counts.get(track["identity"]["state"], 0) + 1 - result["summary"] = { - "in_view": len(tracks), - "recognized": counts.get("recognized", 0), - "needs_confirmation": counts.get("needs_confirmation", 0), - "unrecognized": counts.get("unrecognized", 0), - "detected": 0, - "unconfirmed": counts.get("unconfirmed", 0), - "identifying": 0, - "empty_slots": len(slots), - "price_tags": (still.get("price_tags") or {"readable": 0, "unreadable": 0}), - "reconciliation": (still.get("reconciliation") or {}), - "compliance": (still.get("compliance") or {"violations": 0, "rows": [], "hints": []}), - } - # A client must be able to tell these boxes did not come from the frame it - # just sent. - result["synthetic"] = True - result["demo"] = {"sample": sample_id, "step": step, "bay": position["bay_index"] + 1} - return result diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 2d5c4e1..c284c12 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -411,7 +411,29 @@ class RecognitionEngine: return slots # ------------------------------------------------------------------ classification + @property + def identity_scale(self) -> str: + """What the scores from `score_crop` mean: inlier counts or cosine similarity.""" + return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier" + def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]: + """Score one crop against the catalog, in whatever scale this engine uses. + + The live session calls this per track. With the gallery backend it must go + through the gallery, or a live scan would use the matcher while the still path + used the enrolled store - measured: the matcher accepts look-alikes and cannot + say "unknown", which is exactly what a real shelf needs it to do. + """ + if self.identity_scale == "cosine": + gallery = self.identity_gallery() + if gallery: + match = gallery.match(crop) + if match.rejected or not match.sku_id: + return {} + scores = {match.sku_id: match.similarity} + if match.runner_up: + scores[match.runner_up] = match.runner_up_similarity + return {str(sku): float(score) for sku, score in scores.items()} method = (method or self.settings.live_method).lower() if method not in CLASSIFY_METHODS: raise UnknownMethod(method) diff --git a/recognization/code/identity.py b/recognization/code/identity.py index 2514f91..fdc597e 100644 --- a/recognization/code/identity.py +++ b/recognization/code/identity.py @@ -25,6 +25,13 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float: return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4) +# The gallery's scale is a cosine similarity, where same-product pairs sit above 0.6 +# and different products below 0.5 - not an inlier count, which has no such meaning. +COSINE_ACCEPT = 0.60 +COSINE_AMBIGUOUS = 0.50 +COSINE_MARGIN = 0.04 + + def decide_identity( scores: Dict[str, float], *, @@ -33,6 +40,7 @@ def decide_identity( min_confidence: float = DEFAULT_MIN_CONFIDENCE, details: Optional[Dict[str, Any]] = None, method: str = "hybrid", + scale: str = "inlier", ) -> Dict[str, Any]: """Rank catalog scores into accepted / ambiguous / unknown / rejected. @@ -56,6 +64,30 @@ def decide_identity( sorted_candidates: List[Tuple[str, float]] = sorted( scores.items(), key=lambda x: x[1], reverse=True ) + if scale == "cosine": + # Thresholds that mean something on a similarity scale. + top_sku, top_score = sorted_candidates[0] + runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0 + margin = top_score - runner_score + if top_score >= COSINE_ACCEPT and margin >= COSINE_MARGIN: + level = IDENTITY_ACCEPTED + elif top_score >= COSINE_AMBIGUOUS: + level = IDENTITY_AMBIGUOUS + else: + level = IDENTITY_REJECTED + return { + "status": level, + "method": method, + "best_sku": top_sku if level == IDENTITY_ACCEPTED else None, + "score": float(top_score), + "confidence": float(top_score), + "identity_level": level, + "ranked_candidates": [ + {"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3] + ], + "details": details or {}, + "message": None, + } top_sku, top_score = sorted_candidates[0] runner_up_sku, runner_up_score = ( sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0) diff --git a/recognization/code/live.py b/recognization/code/live.py index f8f4bed..9cab6ce 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -643,7 +643,9 @@ class LiveSession: mean_scores = { sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items() } - decision = decide_identity(mean_scores, method=method) + decision = decide_identity( + mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier") + ) track.candidates = [ {"sku_id": row.get("sku_id"), "score": row.get("score")} for row in (decision.get("ranked_candidates") or [])[:3] diff --git a/recognization/code/test_demo.py b/recognization/code/test_demo.py deleted file mode 100644 index b62ca5c..0000000 --- a/recognization/code/test_demo.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Contract tests for the demo camera (the synthetic store walk). - -The walk is what a demo shows when there is no camera, so its geometry has to be -predictable: inside the image, moving, serpentine, and wrapping forever. -""" - -from __future__ import annotations - -import os -import sys -import unittest - -current_dir = os.path.dirname(os.path.abspath(__file__)) -project_root = os.path.dirname(current_dir) -if os.path.exists(os.path.join(project_root, "stock")): - os.chdir(project_root) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -import numpy as np # noqa: E402 - -from demo import DemoCamera, DemoSourceError # noqa: E402 - - -class FakeImageCamera(DemoCamera): - """A camera over a synthetic image, so the geometry is testable without files.""" - - def __init__(self, width=1500, height=2000, window=(960, 540)): - super().__init__({"synthetic": {"id": "synthetic", "path": "synthetic", "label": "synthetic"}}, window) - self._image_cache = np.zeros((height, width, 3), dtype="uint8") - self._size_cache = (width, height) - - def _size(self, sample_id): # noqa: D102 - if sample_id != "synthetic": - raise DemoSourceError(f"unknown sample '{sample_id}'") - return self._size_cache - - def _image(self, sample_id): # noqa: D102 - return self._image_cache - - -class WalkGeometryTests(unittest.TestCase): - def setUp(self): - self.camera = FakeImageCamera() - self.plan = self.camera.plan(["synthetic"]) - - def test_plan_is_deterministic_and_covers_the_shelf(self): - self.assertEqual(self.plan["steps"], self.camera.plan(["synthetic"])["steps"]) - self.assertGreater(self.plan["steps"], 20, "a shelf walk is not two steps long") - self.assertEqual(self.plan["bays"][0]["sample"], "synthetic") - - def test_window_never_leaves_the_image(self): - width, height = 1500, 2000 - for step in range(self.plan["steps"]): - position = self.camera.window_at(["synthetic"], step) - self.assertGreaterEqual(position["x"], 0) - self.assertGreaterEqual(position["y"], 0) - self.assertLessEqual(position["x"] + position["width"], width) - self.assertLessEqual(position["y"] + position["height"], height) - - def test_the_camera_actually_moves(self): - xs = [self.camera.window_at(["synthetic"], step)["x"] for step in range(6)] - ys = [self.camera.window_at(["synthetic"], step)["y"] for step in range(self.plan["steps"])] - self.assertGreater(len(set(xs)), 1, "the window must pan, not sit still") - self.assertGreater(len(set(ys)), 1, "the walk must climb rows") - - def test_it_ascends_the_shelf(self): - first = self.camera.window_at(["synthetic"], 0)["y"] - last_band = self.plan["steps"] - 1 - self.assertGreater(first, self.camera.window_at(["synthetic"], last_band)["y"], - "the walk starts at the bottom row and climbs") - - def test_step_wraps(self): - first = self.camera.window_at(["synthetic"], 0) - wrapped = self.camera.window_at(["synthetic"], self.plan["steps"]) - self.assertEqual((first["x"], first["y"]), (wrapped["x"], wrapped["y"])) - - def test_frames_are_deterministic_jpegs(self): - payload, position = self.camera.frame(["synthetic"], 3, quality=70) - again, _ = self.camera.frame(["synthetic"], 3, quality=70) - self.assertTrue(payload.startswith(b"\xff\xd8"), "JPEG magic") - self.assertEqual(payload, again, "the same step must render the same frame") - self.assertIn("sample", position) - - def test_unknown_sample_is_rejected(self): - with self.assertRaises(DemoSourceError): - self.camera.plan(["nope"]) - - def test_window_larger_than_the_image_is_clamped(self): - camera = FakeImageCamera(width=800, height=600, window=(960, 540)) - position = camera.window_at(["synthetic"], 0) - self.assertLessEqual(position["width"], 800) - self.assertLessEqual(position["height"], 600) - self.assertEqual(position["x"], 0) - # The walk starts at the bottom of the shelf, so the window is bottom-aligned. - self.assertEqual(position["y"], 600 - position["height"]) - - -class RealShelfTests(unittest.TestCase): - """One check against the actual demo asset, skipped when it is absent.""" - - def test_real_sample_walks(self): - from app import get_available_shelf_images - - samples = {item["id"]: item for item in get_available_shelf_images("add")} - if "image6" not in samples: - self.skipTest("demo sample image6 not present") - camera = DemoCamera(samples) - plan = camera.plan(["image6"], (1280, 720)) - self.assertGreater(plan["steps"], 20, "a 1500x2000 shelf gives a multi-row walk") - payload, position = camera.frame(["image6"], 5, (1280, 720)) - self.assertGreater(len(payload), 10_000) - self.assertLessEqual(position["x"] + position["width"], plan["bays"][0]["width"]) - self.assertLessEqual(position["y"] + position["height"], plan["bays"][0]["height"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/recognization/run-demo-server.sh b/recognization/run-demo-server.sh new file mode 100755 index 0000000..6f0e5c2 --- /dev/null +++ b/recognization/run-demo-server.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# The demo server: one command, bound to the network so a phone can reach it. +# +# ./run-demo-server.sh # both demo shelves enrolled (stock-demo) +# ./run-demo-server.sh stock-milk150 +# +# The iPhone must be on the same Wi-Fi as this Mac, and the app must point at this +# Mac's address - Config.xcconfig's APP_RECOGNITION_BASE_URL, which the app reads at +# build time. The script prints the exact line to use. +set -euo pipefail + +cd "$(dirname "$0")" + +STOCK="${1:-stock-demo}" +PORT="${PORT:-8099}" + +if [ ! -d "$STOCK" ]; then + echo "no such store: $STOCK" >&2 + exit 1 +fi + +IP="$(ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null || true)" +if [ -z "$IP" ]; then + echo "this Mac has no LAN address; the phone cannot reach it" >&2 + exit 1 +fi + +cat <