From 4ce41ecb171887b978a5d930e53d4f696c9bd9d7 Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Tue, 29 Sep 2026 09:28:07 +0900 Subject: [PATCH] feat(server): walk the replayed sample instead of freezing it A still replayed as frames gave every frame the same boxes, so the live path never saw a track move. /v1/samples/{id}?pan=N now returns a portrait 720x1280 window slid N*PAN_STEP pixels along the sample, reversing at each edge, at the frame size and JPEG quality the phone sends. The decoded image is cached by (mtime, size): re-reading a 1500x2000 JPEG per frame cost more than the encode and made the walk slower than PAN_STEP says. Measured after the cache: 3.3 ms mean per panned frame, 301 frames/s ceiling. Two tests, both verified to fail before the change. --- recognization/.gitignore | 3 ++ recognization/code/app.py | 81 +++++++++++++++++++++++++++++++++- recognization/code/test_api.py | 24 ++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/recognization/.gitignore b/recognization/.gitignore index 2d86673..20e714e 100644 --- a/recognization/.gitignore +++ b/recognization/.gitignore @@ -36,3 +36,6 @@ data/external-shelves/*/*.png data/packshots/research/*/images/ data/packshots/research/*/pages/ data/packshots/research/*/*.jpg + +# Rebuildable sweep cache (code/sweep_accept_lines.py --cache). +*.pkl diff --git a/recognization/code/app.py b/recognization/code/app.py index 274346a..8739970 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -96,6 +96,14 @@ WS_IDLE_TIMEOUT_S = 120.0 CATALOG_VERSION_FALLBACK = "unknown" MAX_CROP_PIXELS = 4_000_000 +# The replayed-sample pan: a portrait window at the phone's frame size, JPEG at +# the quality the device camera sends. `PAN_STEP` is pixels per frame, so at the +# scan's ~10 fps the shelf walks ~80 px/s - brisk enough to cover the shelf in +# about 9 s without jumping past a facing between frames. +PAN_LONG_EDGE = 1280 +PAN_STEP = 8 +PAN_JPEG_QUALITY = 70 + def _jpeg_dimensions(data: bytes) -> Optional[Tuple[int, int]]: """Read JPEG SOF dimensions without allocating a decoded image.""" @@ -299,6 +307,9 @@ def create_app( # path -> (mtime_ns, size, width, height). Dropped when the file changes. sample_dimensions: Dict[str, Tuple[int, int, int, int]] = {} sample_dimensions_lock = threading.Lock() + # path -> ((mtime_ns, size), decoded image). The pan re-cuts this per frame. + sample_images: Dict[str, Any] = {} + sample_image_lock = threading.Lock() sessions: Dict[str, LiveSession] = {} sessions_lock = asyncio.Lock() app.state.engine = engine @@ -398,6 +409,61 @@ def create_app( sample_dimensions[path] = (key[0], key[1], width, height) return width, height + def _panned_jpeg(path: str, index: int) -> Optional[bytes]: + """One portrait window of `path`, slid `index` steps along the shelf. + + The window is the shape the phone sends - long edge `PAN_LONG_EDGE`, + 9:16 - so a simulator sees the frame size the live path really gets. + It steps `PAN_STEP` pixels per frame and reverses at each edge, so the + walk is continuous rather than a jump back to the start. + + The decoded image is cached: re-reading a 1500x2000 JPEG per frame cost + more than the encode, which made the walk slower than `PAN_STEP` says. + """ + import cv2 + + image = _decoded_sample(path) + if image is None: + return None + height, width = image.shape[:2] + window_h = min(PAN_LONG_EDGE, height) + window_w = min(int(window_h * 9 / 16), width) + max_x = max(0, width - window_w) + travel = index * PAN_STEP + if max_x == 0: + x = 0 + else: + # Triangle wave: pan out to the edge, then back, forever. + period = 2 * max_x + phase = travel % period + x = phase if phase <= max_x else period - phase + y = max(0, (height - window_h) // 2) + window = image[y:y + window_h, x:x + window_w] + ok, encoded = cv2.imencode(".jpg", window, [int(cv2.IMWRITE_JPEG_QUALITY), PAN_JPEG_QUALITY]) + return encoded.tobytes() if ok else None + + def _decoded_sample(path: str) -> Any: + """The sample decoded once, re-read only when the file changes.""" + try: + stat = os.stat(path) + except OSError: + with sample_image_lock: + sample_images.pop(path, None) + return None + key = (stat.st_mtime_ns, stat.st_size) + with sample_image_lock: + cached = sample_images.get(path) + if cached is not None and cached[0] == key: + return cached[1] + import cv2 + + image = cv2.imread(path) + if image is None: + return None + with sample_image_lock: + sample_images[path] = (key, image) + return image + def _sample_listing() -> List[Dict[str, Any]]: listing = [] for item in samples.values(): @@ -414,11 +480,22 @@ def create_app( return {"samples": await loop.run_in_executor(None, _sample_listing)} @app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)]) - async def get_sample(sample_id: str) -> Response: + async def get_sample(sample_id: str, pan: Optional[int] = None) -> Response: item = samples.get(sample_id) if not item or not await asyncio.get_running_loop().run_in_executor(None, os.path.exists, item["path"]): raise _error("NOT_FOUND", f"no sample '{sample_id}'", 404) - return FileResponse(item["path"]) + if pan is None: + return FileResponse(item["path"]) + # A still replayed as frames is a frozen shelf: every frame carries the + # same boxes, so the live path never exercises a track that moves. `pan` + # is the frame index, and the window slides one step per frame and + # reverses at the edges - the same walk `stream_client.py` drives, so a + # simulator sees what a real slow pan produces. + loop = asyncio.get_running_loop() + jpeg = await loop.run_in_executor(None, _panned_jpeg, item["path"], max(0, pan)) + if jpeg is None: + raise _error("NOT_FOUND", f"cannot read sample '{sample_id}'", 404) + return Response(content=jpeg, media_type="image/jpeg") # ---------------------------------------------------------------- recognize @app.post("/v1/recognize", dependencies=[Depends(authorize)]) diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index 49cc070..bef9813 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -69,6 +69,30 @@ class RestContractTests(unittest.TestCase): self.assertIn("version", body["model"]) self.assertEqual(body["catalog"]["sku_count"], 2) + def test_a_panned_sample_walks_and_stays_portrait(self): + # The debug replay source walks a shelf by asking for a frame index. A + # pan that returned the same window every time would freeze the stage + # while the boxes moved, which is the bug this pins. + first = self.client.get("/v1/samples/image6?pan=0") + self.assertEqual(first.status_code, 200) + self.assertEqual(first.headers["content-type"], "image/jpeg") + + later = self.client.get("/v1/samples/image6?pan=40") + self.assertEqual(later.status_code, 200) + self.assertNotEqual(first.content, later.content, "a later pan must show a different window") + + import cv2 + import numpy as np + + decoded = cv2.imdecode(np.frombuffer(later.content, dtype="uint8"), cv2.IMREAD_COLOR) + height, width = decoded.shape[:2] + self.assertEqual((width, height), (720, 1280), "the window must match the phone's portrait frame") + + def test_a_sample_without_pan_is_the_whole_image(self): + response = self.client.get("/v1/samples/image6") + self.assertEqual(response.status_code, 200) + self.assertGreater(len(response.content), 1024) + def test_catalog_shape_has_no_local_paths(self): body = self.client.get("/v1/catalog").json() product = body["products"][0]