diff --git a/recognization/code/app.py b/recognization/code/app.py index 8739970..47524ec 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -97,11 +97,12 @@ CATALOG_VERSION_FALLBACK = "unknown" MAX_CROP_PIXELS = 4_000_000 # The replayed-sample pan: a portrait window at the phone's frame size, JPEG at -# the quality the device camera sends. `PAN_STEP` is pixels per frame, so at the -# scan's ~10 fps the shelf walks ~80 px/s - brisk enough to cover the shelf in -# about 9 s without jumping past a facing between frames. +# the quality the device camera sends. `PAN_STEP` is pixels per frame **in the +# sent frame**, so it is divided by the downscale before slicing: the window is +# the sample's full height scaled to `PAN_LONG_EDGE`, and a step measured in +# source pixels would shrink with that scale and silently change the pace. PAN_LONG_EDGE = 1280 -PAN_STEP = 8 +PAN_STEP = 5.6 PAN_JPEG_QUALITY = 70 @@ -412,8 +413,16 @@ def create_app( def _panned_jpeg(path: str, index: int) -> Optional[bytes]: """One portrait window of `path`, slid `index` steps along the shelf. - The window is the shape the phone sends - long edge `PAN_LONG_EDGE`, - 9:16 - so a simulator sees the frame size the live path really gets. + The window is the shape the phone sends - 9:16, long edge + `PAN_LONG_EDGE` - but it spans the sample's **full height** and pans + horizontally. A window that is only `PAN_LONG_EDGE` tall shows one + shelf row: on `image6` (1500x2000, rows at y 200/560/870/1150/1440) a + 720 px band centred at 640-1360 reaches 52 of 66 enrolled facings and + gives every other SKU 0% coverage, which reads as a matcher failure + when it is only the crop. Full height reaches all of them, and the + downscale to `PAN_LONG_EDGE` is the same 0.64 the evaluator's live + resolution uses. + It steps `PAN_STEP` pixels per frame and reverses at each edge, so the walk is continuous rather than a jump back to the start. @@ -426,19 +435,27 @@ def create_app( if image is None: return None height, width = image.shape[:2] - window_h = min(PAN_LONG_EDGE, height) + window_h = height window_w = min(int(window_h * 9 / 16), width) max_x = max(0, width - window_w) - travel = index * PAN_STEP + # `PAN_STEP` is in sent-frame pixels; the slice is in source pixels. + scale = min(1.0, PAN_LONG_EDGE / float(window_h)) + step = PAN_STEP / scale if scale > 0 else PAN_STEP + travel = index * step if max_x == 0: x = 0 else: # Triangle wave: pan out to the edge, then back, forever. period = 2 * max_x phase = travel % period - x = phase if phase <= max_x else period - phase - y = max(0, (height - window_h) // 2) - window = image[y:y + window_h, x:x + window_w] + x = int(phase if phase <= max_x else period - phase) + window = image[:, x:x + window_w] + if scale < 1.0: + window = cv2.resize( + window, + (int(round(window.shape[1] * scale)), int(round(window.shape[0] * scale))), + interpolation=cv2.INTER_AREA, + ) ok, encoded = cv2.imencode(".jpg", window, [int(cv2.IMWRITE_JPEG_QUALITY), PAN_JPEG_QUALITY]) return encoded.tobytes() if ok else None diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index ddb1406..cbf4d7f 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -56,6 +56,10 @@ class FrameSource: self.image = None self.pan_x = 0 self.pan_direction = 1 + # The downscale `frame()` applied to the window. A box the server + # measured is in the sent frame's pixels, so mapping it back to the + # source divides by this. + self.scale = 1.0 # Where the last frame was cut from the source image. (0, 0) for a # camera or a video, whose frames are the whole image. self.offset = (0, 0) @@ -83,17 +87,36 @@ class FrameSource: """ if self.image is not None: height, width = self.image.shape[:2] - window_w = min(self.pan_width, width) - window_h = min(int(window_w * 9 / 16), height) + # Full height, 9:16 wide: the same window the app's replay source + # sends. A window only `pan_width` tall shows one shelf row, which + # reads as a matcher failure when it is only the crop. + window_h = height + window_w = min(int(window_h * 9 / 16), width) max_x = max(0, width - window_w) - x = min(self.pan_x, max_x) - y = max(0, (height - window_h) // 2) - window = self.image[y:y + window_h, x:x + window_w] - self.offset = (x, y) - self.windows.append((x, y, x + window_w, y + window_h)) - self.pan_x += self.pan_direction * self.pan_step - if self.pan_x >= max_x or self.pan_x <= 0: - self.pan_direction *= -1 + x = int(max(0, min(self.pan_x, max_x))) + window = self.image[:, x:x + window_w] + # Scale to the send size here, so the frame this yields is exactly + # what the app sends and `encode` has nothing left to do. The pan + # step is in sent-frame pixels, so it is divided by this scale. + self.scale = min(1.0, self.width / float(window_w)) if self.width else 1.0 + if self.scale < 1.0: + window = cv2.resize( + window, + (int(round(window.shape[1] * self.scale)), int(round(window.shape[0] * self.scale))), + interpolation=cv2.INTER_AREA, + ) + self.offset = (x, 0) + self.windows.append((x, 0, x + window_w, window_h)) + self.pan_x += self.pan_direction * self.pan_step / self.scale + # Clamp the accumulator, not just the slice origin: a negative + # `pan_x` would satisfy `<= 0` on every later frame and flip the + # direction each time, parking the window at the left edge. + if self.pan_x >= max_x: + self.pan_x = float(max_x) + self.pan_direction = -1 + elif self.pan_x <= 0: + self.pan_x = 0.0 + self.pan_direction = 1 return window if self.capture is not None: ok, frame = self.capture.read() @@ -126,7 +149,7 @@ def main() -> None: parser.add_argument("--url", default="ws://127.0.0.1:8080/v1/stream") parser.add_argument("--token", default=os.environ.get("DORAYAKI_API_TOKEN")) parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path") - parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image") + parser.add_argument("--pan", type=float, default=12.0, help="pixels per frame when panning a still image") parser.add_argument("--seconds", type=float, default=20.0) parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table") parser.add_argument("--tracks", default=None, @@ -161,6 +184,11 @@ def main() -> None: # --tracks so an evaluator can match labels to ground truth. track_records: Dict[int, Dict[str, Any]] = {} frame_offsets: Dict[int, Tuple[int, int]] = {} + # The scale `encode` applied to that frame. A box the server measured is in + # the sent frame's pixels, so mapping it back to the source needs both the + # pan offset and this factor; without it a `--width` below the source size + # shrinks every box and the evaluator matches the wrong facing. + frame_scales: Dict[int, float] = {} label_changes = 0 unexplained_changes = 0 relabel_seen: Dict[int, int] = {} @@ -231,14 +259,15 @@ def main() -> None: # offset comes from the frame the box was measured on, not the # frame this result arrived with: the pan has moved on since. offset = frame_offsets.get(track.get("label_frame_id"), (0, 0)) + inv = 1.0 / frame_scales.get(track.get("label_frame_id"), 1.0) label_box = track.get("label_box") or track.get("box") or [] if len(label_box) == 4: track_records[track_id] = { "track_id": track_id, "sku_id": sku, "label_box": [ - label_box[0] + offset[0], label_box[1] + offset[1], - label_box[2] + offset[0], label_box[3] + offset[1], + label_box[0] * inv + offset[0], label_box[1] * inv + offset[1], + label_box[2] * inv + offset[0], label_box[3] * inv + offset[1], ], "label_frame_id": track.get("label_frame_id"), "result_frame_id": message["frame_id"], @@ -254,7 +283,6 @@ def main() -> None: f"unconfirmed={last_summary.get('unconfirmed', 0):3d} " f"detect={message['timing_ms']['detect']:5.1f}ms " f"rtt={rtts[-1]:6.1f}ms fps={results / (now - started):5.2f}") - frame_id = 0 index = 0 while time.perf_counter() - started < args.seconds: @@ -264,6 +292,7 @@ def main() -> None: index += 1 frame_id = (frame_id + 1) & 0xFFFFFFFF frame_offsets[frame_id] = source.offset + frame_scales[frame_id] = source.scale payload = encode(frame, args.width, args.quality) if len(in_flight) >= args.max_inflight: if pending_frame is not None: