vnguyen/product-recognition-demo
1
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Fix label frame mapping and reference upscaling; add the stream G1 harness

Browse source 
Three defects, all found by scoring the live path against ground truth.

Inherited labels carried frame id 0. `_inherit_labels` set `label_box` but
never `label_frame_id`, so every inherited label was mapped back to the source
image with the first frame's offset. During a pan that puts the box a whole
pan-distance from the product it names, which is why most streamed tracks
matched no labelled facing. It now stamps the current frame and anchors to the
raw detection box rather than the smoothed track box, which lags a pan.

The reference lane upscaled views smaller than a pyramid level. `sku_l09.jpg`
is 407x305, so its 400 px level was a 305->400 blow-up: a blur with no
high-frequency detail to disagree with, which won 31 of 54 crops on the shelf
it was cut from. A view is now used only at the levels it actually has.

`score_stream.py` measured the wrong thing. It counted any facing touching the
window by one pixel (23 of 47 "in-catalog" facings were less than half
visible), scored a correct text-lane answer as a false accept because it
checked enrollment before correctness, and counted tracks rather than facings,
which reported 138% coverage of a 24-facing window. It now takes the union of
the window rects the client showed, keeps facings at least half inside, checks
correctness first, and reports facings named separately from labels correct.

Measured on add/image6.jpg, stock-lopia-clean, 20 s per run:

  pan  backend   facings named   precision
  0    van        11/24 (46%)     52%
  0    local      11/24 (46%)    100%
  2    van        12/24 (50%)     45%
  2    local      18/24 (75%)    100%
  12   van        11/24 (46%)     28%
  12   local      16/24 (67%)     83%

eval_identity on the same catalog is unchanged by the upscale fix (local 61%
end-to-end at 88% precision full, 59% at 95% live), and 1 of 602 distractor
crops from the 11 out-of-catalog shelf photos reaches 10 inliers, none reach 20.
This commit is contained in:
tngo 2026-09-29 00:30:17 +09:00
parent ff11529476
commit 8ab17ccd5d
4 changed files with 325 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

8
recognization/code/identity_lanes.py
View file

@ -134,6 +134,14 @@ class LocalLane:
if sku not in self.sku_ids:
self.sku_ids.append(sku)
for height in heights:
if height > image.shape[0]:
# Never upscale a reference. A 305 px view blown up to 400
# is a blur, and a blur matches everything: the l09 shelf
# crop won 31 of 54 crops on the shelf it was cut from, at
# a median margin of 4.9, because its upscaled level had no
# high-frequency detail to disagree with. A view is used at
# the levels it actually has.
continue
scaled = cv2.resize(
image,
(max(1, int(image.shape[1] * height / image.shape[0])), int(height)),

61
recognization/code/live.py
View file

@ -154,6 +154,10 @@ class TrackState:
# with the rest of the evidence when the object generation changes.
text_rank: List[Tuple[str, float]] = field(default_factory=list)
text_read: str = ""
# The frame the newest label box was measured on. The box is in that
# frame's coordinates; a client mapping it back to the source image needs
# this id, because the result that carries the label arrives on a later one.
label_frame_id: int = 0
def area(self) -> int:
if not self.raw_box:
@ -615,11 +619,10 @@ class LiveSession:
if state.box
]
tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset)
# Collect the classification work under the lock; submit it after
# releasing it. add_done_callback can run the callback inline when the
# future already finished, and the callback takes this same lock.
submissions = self._plan_classification(image, seq)
submissions = self._plan_classification(image, seq, int(frame_id))
crop_requests = self._plan_crop_requests(image.shape, seq, int(frame_id), measured)
summary = self._summary_locked()
@ -635,7 +638,7 @@ class LiveSession:
summary["price_tags"] = tag_summary["tags"]
summary["compliance"] = tag_summary["compliance"]
self._submit_classification(submissions, method, generation)
self._submit_classification(submissions, method, generation, int(frame_id))
self._run_tag_passes(image, tag_bands)
result = {
@ -685,7 +688,16 @@ class LiveSession:
continue
track.state = STATE_RECOGNIZED
track.confirmed = True
track.label_box = list(track.box)
# The raw detection box, not the smoothed track box: smoothing
# lags a pan by about one step, and the label is about where the
# product was measured, not where the filter has drifted to.
track.label_box = list(track.raw_box or track.box)
# The box is measured on this frame, so the client must map it
# with this frame's offset. Leaving the default 0 makes every
# inherited label map as if it came from the first frame, which
# during a pan puts it a whole pan-distance away from the
# product it names.
track.label_frame_id = int(self._latest_frame_id)
track.last_source = SOURCE_INHERITED
track.identity = {
"sku_id": entry["sku_id"],
@ -762,11 +774,17 @@ class LiveSession:
track.crop_attempts = 0
# ------------------------------------------------------------------ classification
def _plan_classification(self, image: Any, seq: int) -> List[Tuple[TrackState, Any, int]]:
def _plan_classification(
self, image: Any, seq: int, frame_id: int
) -> List[Tuple[TrackState, Any, int]]:
"""Pick what to classify now. Must be called with the session lock held.
Returns (track, crop, object_generation) triples; the caller submits them
after the lock is released.
after the lock is released. The frame id travels with the plan because
the label box is in *that* frame's coordinates: by the time the answer
arrives the track has moved, and a client mapping the box back to the
source image needs the frame the box was measured on, not the one the
result was delivered with.
"""
frame_h, frame_w = image.shape[:2]
margin_y = int(frame_h * self.settings.edge_margin_ratio)
@ -812,12 +830,17 @@ class LiveSession:
crop = self.engine.crop(image, track.raw_box)
track.in_flight += 1
track.last_attempt_seq = seq
track.label_frame_id = int(frame_id)
track.last_attempt_area = track.area()
planned.append((track, crop, track.object_generation))
return planned
def _submit_classification(
self, planned: List[Tuple[TrackState, Any, int]], method: str, generation: int
self,
planned: List[Tuple[TrackState, Any, int]],
method: str,
generation: int,
frame_id: int,
) -> None:
"""Submit planned crops. Called WITHOUT the session lock held."""
for track, crop, object_generation in planned:
@ -841,9 +864,8 @@ class LiveSession:
continue
future.add_done_callback(
lambda fut, tid=track_id, gen=generation, m=method, started=submitted_at,
obj=object_generation, box=submitted_box: self._on_classified(
fut, tid, gen, m, started, obj, box
)
obj=object_generation, box=submitted_box, fid=int(frame_id):
self._on_classified(fut, tid, gen, m, started, obj, box, fid)
)
# ------------------------------------------------------------------ crop lane
@ -1023,9 +1045,8 @@ class LiveSession:
self._crops_accepted += 1
future.add_done_callback(
lambda fut, tid=track_id_out, gen=generation, m=method, started=submitted_at,
obj=object_generation, box=box: self._on_classified(
fut, tid, gen, m, started, obj, box, SOURCE_CROP
)
obj=object_generation, box=box, fid=int(record["frame_id"]):
self._on_classified(fut, tid, gen, m, started, obj, box, fid, SOURCE_CROP)
)
return {
"type": "crop_ack",
@ -1117,6 +1138,7 @@ class LiveSession:
submitted_at: float,
object_generation: int,
submitted_box: List[int],
label_frame_id: int = 0,
source: str = SOURCE_FRAME,
) -> None:
try:
@ -1174,6 +1196,7 @@ class LiveSession:
track.state = STATE_UNRECOGNIZED
track.confirmed = True
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
track.identity = None
track.candidates = []
return
@ -1223,6 +1246,7 @@ class LiveSession:
# The label belongs to the box the crop came from, not to wherever
# the track has moved by the time classification finishes.
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
track.drift_frames = 0
track.identity = {
"sku_id": decision.get("best_sku"),
@ -1239,6 +1263,7 @@ class LiveSession:
)
track.confirmed = True
track.label_box = list(submitted_box)
track.label_frame_id = int(label_frame_id)
track.drift_frames = 0
track.identity = None
else:
@ -1278,10 +1303,20 @@ class LiveSession:
)
payload = {
"track_id": track.track_id,
# The frame `label_box` is in. The result carrying the label arrives
# on a later frame, so a client that maps the box back to its own
# source image must use this id's offset, not the result's.
"label_frame_id": int(track.label_frame_id),
# box: smoothed track box, for drawing. raw_box: the detection box the
# label is anchored to, for cropping and for explaining label changes.
"box": list(track.box),
"raw_box": list(track.raw_box),
# The box the label was measured on, in frame coordinates. It is
# what the label is *about*: `box` has moved on by the time a
# classification finishes, so scoring a label against `box` scores
# it against wherever the track drifted to. An evaluator matching
# labels to ground truth needs this one.
"label_box": list(track.label_box or track.box),
"confidence": round(float(track.confidence), 4),
"first_seen_seq": track.first_seq,
"relabels": track.relabels,

209
recognization/code/score_stream.py Normal file
View file

@ -0,0 +1,209 @@
"""Score a streamed clip's labels against ground truth: the G1 number.
The stream path was reported as "53 of 131 tracks", which is not comparable to
Van's 57% of labelled facings - the two count different things. But the clip is
built from `add/image6.jpg`, which has labels, so matching each confirmed
track's `label_box` to the labelled facings gives stream-path coverage directly.
python code/score_stream.py TRACKS.json --labels eval/labels/image6.json
`TRACKS.json` is what `stream_client.py --tracks` writes: one row per track that
settled on a label, with the box that label was measured on, already mapped back
to source-image coordinates.
Reported per run: correct / in-catalog facings, precision, unknown accepts, and
tracks that matched no labelled facing. A track whose box overlaps a facing the
catalog does not enrol is an unenrolled false accept - the dangerous kind, and
the one the evaluator's own precision denominator used to hide.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
# A facing clipped by the window edge is not something a run could have named,
# so the denominator keeps only facings at least this far inside the window.
VISIBLE_MIN = 0.5
def iou(a: Sequence[int], b: Sequence[int]) -> float:
ax1, ay1, ax2, ay2 = a
bx1, by1, bx2, by2 = b
inter_w = max(0, min(ax2, bx2) - max(ax1, bx1))
inter_h = max(0, min(ay2, by2) - max(ay1, by1))
inter = inter_w * inter_h
area_a = max(0, ax2 - ax1) * max(0, ay2 - ay1)
area_b = max(0, bx2 - bx1) * max(0, by2 - by1)
union = area_a + area_b - inter
return inter / union if union > 0 else 0.0
def load_facings(labels_path: str) -> List[Dict[str, Any]]:
with open(labels_path) as handle:
payload = json.load(handle)
return payload["facings"]
def enrolled_skus(stock_dir: str) -> set:
"""Canonical SKUs with at least one enrolled view."""
from rebuild_catalog import sku_of
return {
sku_of(name) for name in os.listdir(stock_dir)
if name.lower().endswith((".jpg", ".jpeg", ".png"))
}
def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float:
"""Share of a facing's area inside the window the client actually showed."""
wx0, wy0, wx1, wy1 = window
bx0, by0, bx1, by1 = box
inter_w = max(0, min(bx1, wx1) - max(bx0, wx0))
inter_h = max(0, min(by1, wy1) - max(by0, wy0))
area = max(0, bx1 - bx0) * max(0, by1 - by0)
return (inter_w * inter_h) / area if area > 0 else 0.0
def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
stock_dir: str, iou_min: float,
windows: Optional[Sequence[Sequence[int]]] = None
) -> Dict[str, Any]:
"""Score the labels a stream settled on.
`windows` are the source-image rects the client showed, as
[x0, y0, x1, y1] each. The denominator is the facings at least half inside
one of them, in x and y both: a 1280x720 window over a 1500x2000 photo sees
24 of the 63 in-catalog facings, and a facing clipped by the window edge is
not something any run could have named. A panned run shows a different rect
each frame, so the reachable set is the union. Without it the denominator is
every in-catalog facing.
A correct answer is counted before the unenrolled and unknown branches. The
text lane names SKUs the image lane has no view for, which is the point of
it, and scoring that as a false accept punishes the lane for working.
"""
if windows:
facings = [
f for f in facings
if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN
]
in_catalog = [f for f in facings if f["label"] != "unknown"]
enrolled = enrolled_skus(stock_dir)
correct = 0
wrong = 0
unenrolled_false = 0
unknown_false = 0
unmatched = 0
rows: List[Dict[str, Any]] = []
# Coverage is per facing, precision is per label. A track that restarts on
# the same product, or two tracks on one facing, is two claims but one
# facing: counting tracks as coverage reports 138% of a 24-facing window.
named: Dict[int, str] = {}
for track in tracks:
box = track.get("label_box") or []
if len(box) != 4:
unmatched += 1
continue
best, best_iou = None, 0.0
for facing in facings:
value = iou(box, facing["box"])
if value > best_iou:
best, best_iou = facing, value
if best is None or best_iou < iou_min:
unmatched += 1
rows.append({"track_id": track["track_id"], "predicted": track["sku_id"],
"truth": None, "iou": round(best_iou, 3), "outcome": "unmatched"})
continue
truth = best["label"]
predicted = track["sku_id"]
if predicted == truth:
correct += 1
outcome = "correct"
named[best["index"]] = predicted
elif truth == "unknown":
unknown_false += 1
outcome = "unknown_false_accept"
elif truth not in enrolled:
unenrolled_false += 1
outcome = "unenrolled_false_accept"
else:
wrong += 1
outcome = "wrong"
rows.append({"track_id": track["track_id"], "predicted": predicted,
"truth": truth, "iou": round(best_iou, 3), "outcome": outcome})
shown = correct + wrong + unenrolled_false + unknown_false
return {
"in_catalog_facings": len(in_catalog),
"facings_named": len(named),
"tracks_scored": len(tracks),
"correct": correct,
"wrong": wrong,
"unenrolled_false_accepts": unenrolled_false,
"unknown_false_accepts": unknown_false,
"unmatched_tracks": unmatched,
"shown": shown,
"coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0,
"precision": round(correct / shown, 4) if shown else 0.0,
"rows": rows,
}
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description="Score streamed labels against ground truth")
parser.add_argument("tracks", help="JSON written by stream_client.py --tracks")
parser.add_argument("--labels", default="eval/labels/image6.json")
parser.add_argument("--stock-dir", default="stock-lopia-clean")
parser.add_argument("--iou", type=float, default=0.5)
parser.add_argument("--json", dest="json_path", default=None)
parser.add_argument("--window", default=None,
help="x0,y0,x1,y1 the client showed; the denominator is "
"restricted to facings at least half inside it")
parser.add_argument("--verbose", action="store_true", help="print every scored track")
args = parser.parse_args(argv)
with open(args.tracks) as handle:
payload = json.load(handle)
windows = None
if args.window:
windows = [tuple(int(part) for part in args.window.split(","))]
elif payload.get("windows"):
windows = [tuple(w) for w in payload["windows"]]
report = score(payload["tracks"], load_facings(args.labels), args.stock_dir,
args.iou, windows)
report["source"] = payload.get("source")
report["pace"] = payload.get("pace")
report["seconds"] = payload.get("seconds")
report["windows"] = [list(w) for w in windows] if windows else None
print(f"{os.path.basename(args.tracks)} pace={report['pace']} "
f"{report['seconds']}s")
print(f" facings named {report['facings_named']}/{report['in_catalog_facings']} "
f"in-catalog ({report['coverage']:.0%}) labels correct {report['correct']} "
f"precision {report['precision']:.0%} shown {report['shown']}")
print(f" wrong {report['wrong']} unenrolled false accepts "
f"{report['unenrolled_false_accepts']} unknown false accepts "
f"{report['unknown_false_accepts']} unmatched tracks {report['unmatched_tracks']}")
if args.verbose:
for row in report["rows"]:
print(f" t{row['track_id']:4d} {row['outcome']:24s} "
f"pred={row['predicted']} truth={row['truth']} iou={row['iou']}")
if args.json_path:
with open(args.json_path, "w") as handle:
json.dump(report, handle, indent=2)
return 0
if __name__ == "__main__":
sys.exit(main())

60
recognization/code/stream_client.py
View file

@ -56,6 +56,15 @@ class FrameSource:
self.image = None
self.pan_x = 0
self.pan_direction = 1
# Where the last frame was cut from the source image. (0, 0) for a
# camera or a video, whose frames are the whole image.
self.offset = (0, 0)
# The source-image rects the window showed, so an evaluator can restrict
# its denominator to the facings this run could actually have seen.
# A list, because a panned run shows a different rect each frame and the
# reachable set is their union. Both axes matter: a 1280-wide window
# over a 1500-wide photo is x-limited as well as y-limited.
self.windows: List[Tuple[int, int, int, int]] = []
if source.isdigit():
self.capture = cv2.VideoCapture(int(source))
elif os.path.isfile(source) and source.lower().endswith((".mp4", ".mov", ".avi", ".mkv", ".webm")):
@ -66,6 +75,12 @@ class FrameSource:
raise SystemExit(f"cannot read source {source}")
def frame(self, index: int):
"""One frame, plus where it was cut from the source image.
The offset is what lets an evaluator map a track's box back to the
source image's coordinates: a label measured on a pan window is only
comparable to ground truth once the window's origin is added back.
"""
if self.image is not None:
height, width = self.image.shape[:2]
window_w = min(self.pan_width, width)
@ -74,6 +89,8 @@ class FrameSource:
x = min(self.pan_x, max_x)
y = max(0, (height - window_h) // 2)
window = self.image[y:y + window_h, x:x + window_w]
self.offset = (x, y)
self.windows.append((x, y, x + window_w, y + window_h))
self.pan_x += self.pan_direction * self.pan_step
if self.pan_x >= max_x or self.pan_x <= 0:
self.pan_direction *= -1
@ -111,6 +128,11 @@ def main() -> None:
parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path")
parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image")
parser.add_argument("--seconds", type=float, default=20.0)
parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table")
parser.add_argument("--tracks", default=None,
help="write per-track labels and their boxes (source-image "
"coordinates) as JSON, for scoring against ground truth")
parser.add_argument("--quiet", action="store_true", help="only print the final summary")
parser.add_argument("--width", type=int, default=1280, help="send width")
parser.add_argument("--quality", type=int, default=75)
parser.add_argument("--max-inflight", type=int, default=1)
@ -119,8 +141,6 @@ def main() -> None:
"'result': send the next frame only after a result, so the "
"pan stays continuous (the mode to use for stability checks)")
parser.add_argument("--method", default=None, help="surf | orb | brisk | hybrid")
parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table")
parser.add_argument("--quiet", action="store_true", help="only print the final summary")
args = parser.parse_args()
url = args.url
@ -135,6 +155,12 @@ def main() -> None:
first_label: Dict[int, float] = {}
label_latencies: List[float] = []
labels: Dict[int, str] = {}
pending_frame: Optional[int] = None
# Per track: the label it settled on, the box that label was measured on,
# and the pan offset of the frame that box lives in. Written out with
# --tracks so an evaluator can match labels to ground truth.
track_records: Dict[int, Dict[str, Any]] = {}
frame_offsets: Dict[int, Tuple[int, int]] = {}
label_changes = 0
unexplained_changes = 0
relabel_seen: Dict[int, int] = {}
@ -192,15 +218,33 @@ def main() -> None:
first_label[track_id] = now
if not inherited:
label_latencies.append((now - started) * 1000.0)
elif labels.get(track_id) != sku:
# Only a measured-to-measured change on the same object is a
# label change; inherited labels describe the same product
# under a new track id.
if track_id in labels and labels[track_id] != sku:
if not inherited and not inherited_seen.get(track_id):
label_changes += 1
if relabels <= relabel_seen.get(track_id, 0):
unexplained_changes += 1
labels[track_id] = sku
# The box the label is about, in source-image coordinates. The
# offset comes from the frame the box was measured on, not the
# frame this result arrived with: the pan has moved on since.
offset = frame_offsets.get(track.get("label_frame_id"), (0, 0))
label_box = track.get("label_box") or track.get("box") or []
if len(label_box) == 4:
track_records[track_id] = {
"track_id": track_id,
"sku_id": sku,
"label_box": [
label_box[0] + offset[0], label_box[1] + offset[1],
label_box[2] + offset[0], label_box[3] + offset[1],
],
"label_frame_id": track.get("label_frame_id"),
"result_frame_id": message["frame_id"],
"inherited": inherited,
"restarts": restarts,
}
inherited_seen[track_id] = inherited
relabel_seen[track_id] = relabels
restart_seen[track_id] = restarts
@ -219,6 +263,7 @@ def main() -> None:
break
index += 1
frame_id = (frame_id + 1) & 0xFFFFFFFF
frame_offsets[frame_id] = source.offset
payload = encode(frame, args.width, args.quality)
if len(in_flight) >= args.max_inflight:
if pending_frame is not None:
@ -266,6 +311,15 @@ def main() -> None:
"first_label_ms_p50": round(statistics.median(label_latencies), 0) if label_latencies else None,
"final_summary": last_summary,
}
if args.tracks:
with open(args.tracks, "w") as handle:
json.dump({
"windows": [list(w) for w in source.windows] or None,
"source": args.source,
"pace": args.pace,
"seconds": round(elapsed, 1),
"tracks": sorted(track_records.values(), key=lambda row: row["track_id"]),
}, handle, indent=2)
if args.json:
print(json.dumps(summary, indent=2))
else: