tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

Fix label frame mapping and reference upscaling; add the stream G1 harness

Browse source 
Three defects, all found by scoring the live path against ground truth.

Inherited labels carried frame id 0. `_inherit_labels` set `label_box` but
never `label_frame_id`, so every inherited label was mapped back to the source
image with the first frame's offset. During a pan that puts the box a whole
pan-distance from the product it names, which is why most streamed tracks
matched no labelled facing. It now stamps the current frame and anchors to the
raw detection box rather than the smoothed track box, which lags a pan.

The reference lane upscaled views smaller than a pyramid level. `sku_l09.jpg`
is 407x305, so its 400 px level was a 305->400 blow-up: a blur with no
high-frequency detail to disagree with, which won 31 of 54 crops on the shelf
it was cut from. A view is now used only at the levels it actually has.

`score_stream.py` measured the wrong thing. It counted any facing touching the
window by one pixel (23 of 47 "in-catalog" facings were less than half
visible), scored a correct text-lane answer as a false accept because it
checked enrollment before correctness, and counted tracks rather than facings,
which reported 138% coverage of a 24-facing window. It now takes the union of
the window rects the client showed, keeps facings at least half inside, checks
correctness first, and reports facings named separately from labels correct.

Measured on add/image6.jpg, stock-lopia-clean, 20 s per run:

  pan  backend   facings named   precision
  0    van        11/24 (46%)     52%
  0    local      11/24 (46%)    100%
  2    van        12/24 (50%)     45%
  2    local      18/24 (75%)    100%
  12   van        11/24 (46%)     28%
  12   local      16/24 (67%)     83%

eval_identity on the same catalog is unchanged by the upscale fix (local 61%
end-to-end at 88% precision full, 59% at 95% live), and 1 of 602 distractor
crops from the 11 out-of-catalog shelf photos reaches 10 inliers, none reach 20.
This commit is contained in:
tngo 2026-09-29 00:30:17 +09:00
parent ff11529476
commit 8ab17ccd5d
4 changed files with 325 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

66
recognization/code/stream_client.py
View file

@ -56,6 +56,15 @@ class FrameSource:
self.image = None
self.pan_x = 0
self.pan_direction = 1
# Where the last frame was cut from the source image. (0, 0) for a
# camera or a video, whose frames are the whole image.
self.offset = (0, 0)
# The source-image rects the window showed, so an evaluator can restrict
# its denominator to the facings this run could actually have seen.
# A list, because a panned run shows a different rect each frame and the
# reachable set is their union. Both axes matter: a 1280-wide window
# over a 1500-wide photo is x-limited as well as y-limited.
self.windows: List[Tuple[int, int, int, int]] = []
if source.isdigit():
self.capture = cv2.VideoCapture(int(source))
elif os.path.isfile(source) and source.lower().endswith((".mp4", ".mov", ".avi", ".mkv", ".webm")):
@ -66,6 +75,12 @@ class FrameSource:
raise SystemExit(f"cannot read source {source}")
def frame(self, index: int):
"""One frame, plus where it was cut from the source image.
The offset is what lets an evaluator map a track's box back to the
source image's coordinates: a label measured on a pan window is only
comparable to ground truth once the window's origin is added back.
"""
if self.image is not None:
height, width = self.image.shape[:2]
window_w = min(self.pan_width, width)
@ -74,6 +89,8 @@ class FrameSource:
x = min(self.pan_x, max_x)
y = max(0, (height - window_h) // 2)
window = self.image[y:y + window_h, x:x + window_w]
self.offset = (x, y)
self.windows.append((x, y, x + window_w, y + window_h))
self.pan_x += self.pan_direction * self.pan_step
if self.pan_x >= max_x or self.pan_x <= 0:
self.pan_direction *= -1
@ -111,6 +128,11 @@ def main() -> None:
parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path")
parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image")
parser.add_argument("--seconds", type=float, default=20.0)
parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table")
parser.add_argument("--tracks", default=None,
help="write per-track labels and their boxes (source-image "
"coordinates) as JSON, for scoring against ground truth")
parser.add_argument("--quiet", action="store_true", help="only print the final summary")
parser.add_argument("--width", type=int, default=1280, help="send width")
parser.add_argument("--quality", type=int, default=75)
parser.add_argument("--max-inflight", type=int, default=1)
@ -119,8 +141,6 @@ def main() -> None:
"'result': send the next frame only after a result, so the "
"pan stays continuous (the mode to use for stability checks)")
parser.add_argument("--method", default=None, help="surf | orb | brisk | hybrid")
parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table")
parser.add_argument("--quiet", action="store_true", help="only print the final summary")
args = parser.parse_args()
url = args.url
@ -135,6 +155,12 @@ def main() -> None:
first_label: Dict[int, float] = {}
label_latencies: List[float] = []
labels: Dict[int, str] = {}
pending_frame: Optional[int] = None
# Per track: the label it settled on, the box that label was measured on,
# and the pan offset of the frame that box lives in. Written out with
# --tracks so an evaluator can match labels to ground truth.
track_records: Dict[int, Dict[str, Any]] = {}
frame_offsets: Dict[int, Tuple[int, int]] = {}
label_changes = 0
unexplained_changes = 0
relabel_seen: Dict[int, int] = {}
@ -192,15 +218,33 @@ def main() -> None:
first_label[track_id] = now
if not inherited:
label_latencies.append((now - started) * 1000.0)
elif labels.get(track_id) != sku:
# Only a measured-to-measured change on the same object is a
# label change; inherited labels describe the same product
# under a new track id.
# Only a measured-to-measured change on the same object is a
# label change; inherited labels describe the same product
# under a new track id.
if track_id in labels and labels[track_id] != sku:
if not inherited and not inherited_seen.get(track_id):
label_changes += 1
if relabels <= relabel_seen.get(track_id, 0):
unexplained_changes += 1
labels[track_id] = sku
# The box the label is about, in source-image coordinates. The
# offset comes from the frame the box was measured on, not the
# frame this result arrived with: the pan has moved on since.
offset = frame_offsets.get(track.get("label_frame_id"), (0, 0))
label_box = track.get("label_box") or track.get("box") or []
if len(label_box) == 4:
track_records[track_id] = {
"track_id": track_id,
"sku_id": sku,
"label_box": [
label_box[0] + offset[0], label_box[1] + offset[1],
label_box[2] + offset[0], label_box[3] + offset[1],
],
"label_frame_id": track.get("label_frame_id"),
"result_frame_id": message["frame_id"],
"inherited": inherited,
"restarts": restarts,
}
inherited_seen[track_id] = inherited
relabel_seen[track_id] = relabels
restart_seen[track_id] = restarts
@ -219,6 +263,7 @@ def main() -> None:
break
index += 1
frame_id = (frame_id + 1) & 0xFFFFFFFF
frame_offsets[frame_id] = source.offset
payload = encode(frame, args.width, args.quality)
if len(in_flight) >= args.max_inflight:
if pending_frame is not None:
@ -266,6 +311,15 @@ def main() -> None:
"first_label_ms_p50": round(statistics.median(label_latencies), 0) if label_latencies else None,
"final_summary": last_summary,
}
if args.tracks:
with open(args.tracks, "w") as handle:
json.dump({
"windows": [list(w) for w in source.windows] or None,
"source": args.source,
"pace": args.pace,
"seconds": round(elapsed, 1),
"tracks": sorted(track_records.values(), key=lambda row: row["track_id"]),
}, handle, indent=2)
if args.json:
print(json.dumps(summary, indent=2))
else: