vnguyen/product-recognition-demo
1
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix(replay): walk the sample at the app's real frame geometry

Browse source 
The replayed walk and the harness that measures it disagreed with the
phone in two ways, so every live number taken from them was measuring a
different pipeline than the one that ships.

The server cut a window `PAN_LONG_EDGE` tall, which on image6 shows one
shelf row: a 720 px band reaches 52 of 66 enrolled facings and gives
every other SKU 0% coverage, which reads as a matcher failure when it is
only the crop. It now spans the sample's full height and pans
horizontally, and `PAN_STEP` is expressed in sent-frame pixels so the
pace survives the downscale.

`stream_client.py` had the same window bug plus a box-mapping bug: it
added the pan offset but never divided by the downscale, so a `--width`
below the source size shrank every box and the evaluator matched the
wrong facing. It now scales the window itself (so the frame it sends is
byte-for-byte the app's 720x1280), records that scale per frame, and
divides by it when mapping a label box back to source coordinates. The
pan accumulator is clamped at the reversal: a negative `pan_x` satisfied
`<= 0` on every later frame and flipped the direction each time, parking
the window at the left edge.

Measured through the real server at the app's geometry: 57 of 66
enrolled facings covered (86%) at 94% precision, 9 distinct SKUs.
This commit is contained in:
kiyo9w 2026-09-29 10:09:22 +09:00
parent 7736465bee
commit e52684c37e
2 changed files with 71 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

39
recognization/code/app.py
View file

@ -97,11 +97,12 @@ CATALOG_VERSION_FALLBACK = "unknown"
MAX_CROP_PIXELS = 4_000_000
# The replayed-sample pan: a portrait window at the phone's frame size, JPEG at
# the quality the device camera sends. `PAN_STEP` is pixels per frame, so at the
# scan's ~10 fps the shelf walks ~80 px/s - brisk enough to cover the shelf in
# about 9 s without jumping past a facing between frames.
# the quality the device camera sends. `PAN_STEP` is pixels per frame **in the
# sent frame**, so it is divided by the downscale before slicing: the window is
# the sample's full height scaled to `PAN_LONG_EDGE`, and a step measured in
# source pixels would shrink with that scale and silently change the pace.
PAN_LONG_EDGE = 1280
PAN_STEP = 8
PAN_STEP = 5.6
PAN_JPEG_QUALITY = 70
@ -412,8 +413,16 @@ def create_app(
def _panned_jpeg(path: str, index: int) -> Optional[bytes]:
"""One portrait window of `path`, slid `index` steps along the shelf.
The window is the shape the phone sends - long edge `PAN_LONG_EDGE`,
9:16 - so a simulator sees the frame size the live path really gets.
The window is the shape the phone sends - 9:16, long edge
`PAN_LONG_EDGE` - but it spans the sample's **full height** and pans
horizontally. A window that is only `PAN_LONG_EDGE` tall shows one
shelf row: on `image6` (1500x2000, rows at y 200/560/870/1150/1440) a
720 px band centred at 640-1360 reaches 52 of 66 enrolled facings and
gives every other SKU 0% coverage, which reads as a matcher failure
when it is only the crop. Full height reaches all of them, and the
downscale to `PAN_LONG_EDGE` is the same 0.64 the evaluator's live
resolution uses.
It steps `PAN_STEP` pixels per frame and reverses at each edge, so the
walk is continuous rather than a jump back to the start.
@ -426,19 +435,27 @@ def create_app(
if image is None:
return None
height, width = image.shape[:2]
window_h = min(PAN_LONG_EDGE, height)
window_h = height
window_w = min(int(window_h * 9 / 16), width)
max_x = max(0, width - window_w)
travel = index * PAN_STEP
# `PAN_STEP` is in sent-frame pixels; the slice is in source pixels.
scale = min(1.0, PAN_LONG_EDGE / float(window_h))
step = PAN_STEP / scale if scale > 0 else PAN_STEP
travel = index * step
if max_x == 0:
x = 0
else:
# Triangle wave: pan out to the edge, then back, forever.
period = 2 * max_x
phase = travel % period
x = phase if phase <= max_x else period - phase
y = max(0, (height - window_h) // 2)
window = image[y:y + window_h, x:x + window_w]
x = int(phase if phase <= max_x else period - phase)
window = image[:, x:x + window_w]
if scale < 1.0:
window = cv2.resize(
window,
(int(round(window.shape[1] * scale)), int(round(window.shape[0] * scale))),
interpolation=cv2.INTER_AREA,
)
ok, encoded = cv2.imencode(".jpg", window, [int(cv2.IMWRITE_JPEG_QUALITY), PAN_JPEG_QUALITY])
return encoded.tobytes() if ok else None

57
recognization/code/stream_client.py
View file

@ -56,6 +56,10 @@ class FrameSource:
self.image = None
self.pan_x = 0
self.pan_direction = 1
# The downscale `frame()` applied to the window. A box the server
# measured is in the sent frame's pixels, so mapping it back to the
# source divides by this.
self.scale = 1.0
# Where the last frame was cut from the source image. (0, 0) for a
# camera or a video, whose frames are the whole image.
self.offset = (0, 0)
@ -83,17 +87,36 @@ class FrameSource:
"""
if self.image is not None:
height, width = self.image.shape[:2]
window_w = min(self.pan_width, width)
window_h = min(int(window_w * 9 / 16), height)
# Full height, 9:16 wide: the same window the app's replay source
# sends. A window only `pan_width` tall shows one shelf row, which
# reads as a matcher failure when it is only the crop.
window_h = height
window_w = min(int(window_h * 9 / 16), width)
max_x = max(0, width - window_w)
x = min(self.pan_x, max_x)
y = max(0, (height - window_h) // 2)
window = self.image[y:y + window_h, x:x + window_w]
self.offset = (x, y)
self.windows.append((x, y, x + window_w, y + window_h))
self.pan_x += self.pan_direction * self.pan_step
if self.pan_x >= max_x or self.pan_x <= 0:
self.pan_direction *= -1
x = int(max(0, min(self.pan_x, max_x)))
window = self.image[:, x:x + window_w]
# Scale to the send size here, so the frame this yields is exactly
# what the app sends and `encode` has nothing left to do. The pan
# step is in sent-frame pixels, so it is divided by this scale.
self.scale = min(1.0, self.width / float(window_w)) if self.width else 1.0
if self.scale < 1.0:
window = cv2.resize(
window,
(int(round(window.shape[1] * self.scale)), int(round(window.shape[0] * self.scale))),
interpolation=cv2.INTER_AREA,
)
self.offset = (x, 0)
self.windows.append((x, 0, x + window_w, window_h))
self.pan_x += self.pan_direction * self.pan_step / self.scale
# Clamp the accumulator, not just the slice origin: a negative
# `pan_x` would satisfy `<= 0` on every later frame and flip the
# direction each time, parking the window at the left edge.
if self.pan_x >= max_x:
self.pan_x = float(max_x)
self.pan_direction = -1
elif self.pan_x <= 0:
self.pan_x = 0.0
self.pan_direction = 1
return window
if self.capture is not None:
ok, frame = self.capture.read()
@ -126,7 +149,7 @@ def main() -> None:
parser.add_argument("--url", default="ws://127.0.0.1:8080/v1/stream")
parser.add_argument("--token", default=os.environ.get("DORAYAKI_API_TOKEN"))
parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path")
parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image")
parser.add_argument("--pan", type=float, default=12.0, help="pixels per frame when panning a still image")
parser.add_argument("--seconds", type=float, default=20.0)
parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table")
parser.add_argument("--tracks", default=None,
@ -161,6 +184,11 @@ def main() -> None:
# --tracks so an evaluator can match labels to ground truth.
track_records: Dict[int, Dict[str, Any]] = {}
frame_offsets: Dict[int, Tuple[int, int]] = {}
# The scale `encode` applied to that frame. A box the server measured is in
# the sent frame's pixels, so mapping it back to the source needs both the
# pan offset and this factor; without it a `--width` below the source size
# shrinks every box and the evaluator matches the wrong facing.
frame_scales: Dict[int, float] = {}
label_changes = 0
unexplained_changes = 0
relabel_seen: Dict[int, int] = {}
@ -231,14 +259,15 @@ def main() -> None:
# offset comes from the frame the box was measured on, not the
# frame this result arrived with: the pan has moved on since.
offset = frame_offsets.get(track.get("label_frame_id"), (0, 0))
inv = 1.0 / frame_scales.get(track.get("label_frame_id"), 1.0)
label_box = track.get("label_box") or track.get("box") or []
if len(label_box) == 4:
track_records[track_id] = {
"track_id": track_id,
"sku_id": sku,
"label_box": [
label_box[0] + offset[0], label_box[1] + offset[1],
label_box[2] + offset[0], label_box[3] + offset[1],
label_box[0] * inv + offset[0], label_box[1] * inv + offset[1],
label_box[2] * inv + offset[0], label_box[3] * inv + offset[1],
],
"label_frame_id": track.get("label_frame_id"),
"result_frame_id": message["frame_id"],
@ -254,7 +283,6 @@ def main() -> None:
f"unconfirmed={last_summary.get('unconfirmed', 0):3d} "
f"detect={message['timing_ms']['detect']:5.1f}ms "
f"rtt={rtts[-1]:6.1f}ms fps={results / (now - started):5.2f}")
frame_id = 0
index = 0
while time.perf_counter() - started < args.seconds:
@ -264,6 +292,7 @@ def main() -> None:
index += 1
frame_id = (frame_id + 1) & 0xFFFFFFFF
frame_offsets[frame_id] = source.offset
frame_scales[frame_id] = source.scale
payload = encode(frame, args.width, args.quality)
if len(in_flight) >= args.max_inflight:
if pending_frame is not None: