Fix label frame mapping and reference upscaling; add the stream G1 harness
Three defects, all found by scoring the live path against ground truth. Inherited labels carried frame id 0. `_inherit_labels` set `label_box` but never `label_frame_id`, so every inherited label was mapped back to the source image with the first frame's offset. During a pan that puts the box a whole pan-distance from the product it names, which is why most streamed tracks matched no labelled facing. It now stamps the current frame and anchors to the raw detection box rather than the smoothed track box, which lags a pan. The reference lane upscaled views smaller than a pyramid level. `sku_l09.jpg` is 407x305, so its 400 px level was a 305->400 blow-up: a blur with no high-frequency detail to disagree with, which won 31 of 54 crops on the shelf it was cut from. A view is now used only at the levels it actually has. `score_stream.py` measured the wrong thing. It counted any facing touching the window by one pixel (23 of 47 "in-catalog" facings were less than half visible), scored a correct text-lane answer as a false accept because it checked enrollment before correctness, and counted tracks rather than facings, which reported 138% coverage of a 24-facing window. It now takes the union of the window rects the client showed, keeps facings at least half inside, checks correctness first, and reports facings named separately from labels correct. Measured on add/image6.jpg, stock-lopia-clean, 20 s per run: pan backend facings named precision 0 van 11/24 (46%) 52% 0 local 11/24 (46%) 100% 2 van 12/24 (50%) 45% 2 local 18/24 (75%) 100% 12 van 11/24 (46%) 28% 12 local 16/24 (67%) 83% eval_identity on the same catalog is unchanged by the upscale fix (local 61% end-to-end at 88% precision full, 59% at 95% live), and 1 of 602 distractor crops from the 11 out-of-catalog shelf photos reaches 10 inliers, none reach 20.
This commit is contained in:
parent
ff11529476
commit
8ab17ccd5d
4 changed files with 325 additions and 19 deletions
|
|
@ -154,6 +154,10 @@ class TrackState:
|
|||
# with the rest of the evidence when the object generation changes.
|
||||
text_rank: List[Tuple[str, float]] = field(default_factory=list)
|
||||
text_read: str = ""
|
||||
# The frame the newest label box was measured on. The box is in that
|
||||
# frame's coordinates; a client mapping it back to the source image needs
|
||||
# this id, because the result that carries the label arrives on a later one.
|
||||
label_frame_id: int = 0
|
||||
|
||||
def area(self) -> int:
|
||||
if not self.raw_box:
|
||||
|
|
@ -615,11 +619,10 @@ class LiveSession:
|
|||
if state.box
|
||||
]
|
||||
tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset)
|
||||
|
||||
# Collect the classification work under the lock; submit it after
|
||||
# releasing it. add_done_callback can run the callback inline when the
|
||||
# future already finished, and the callback takes this same lock.
|
||||
submissions = self._plan_classification(image, seq)
|
||||
submissions = self._plan_classification(image, seq, int(frame_id))
|
||||
crop_requests = self._plan_crop_requests(image.shape, seq, int(frame_id), measured)
|
||||
|
||||
summary = self._summary_locked()
|
||||
|
|
@ -635,7 +638,7 @@ class LiveSession:
|
|||
summary["price_tags"] = tag_summary["tags"]
|
||||
summary["compliance"] = tag_summary["compliance"]
|
||||
|
||||
self._submit_classification(submissions, method, generation)
|
||||
self._submit_classification(submissions, method, generation, int(frame_id))
|
||||
self._run_tag_passes(image, tag_bands)
|
||||
|
||||
result = {
|
||||
|
|
@ -685,7 +688,16 @@ class LiveSession:
|
|||
continue
|
||||
track.state = STATE_RECOGNIZED
|
||||
track.confirmed = True
|
||||
track.label_box = list(track.box)
|
||||
# The raw detection box, not the smoothed track box: smoothing
|
||||
# lags a pan by about one step, and the label is about where the
|
||||
# product was measured, not where the filter has drifted to.
|
||||
track.label_box = list(track.raw_box or track.box)
|
||||
# The box is measured on this frame, so the client must map it
|
||||
# with this frame's offset. Leaving the default 0 makes every
|
||||
# inherited label map as if it came from the first frame, which
|
||||
# during a pan puts it a whole pan-distance away from the
|
||||
# product it names.
|
||||
track.label_frame_id = int(self._latest_frame_id)
|
||||
track.last_source = SOURCE_INHERITED
|
||||
track.identity = {
|
||||
"sku_id": entry["sku_id"],
|
||||
|
|
@ -762,11 +774,17 @@ class LiveSession:
|
|||
track.crop_attempts = 0
|
||||
|
||||
# ------------------------------------------------------------------ classification
|
||||
def _plan_classification(self, image: Any, seq: int) -> List[Tuple[TrackState, Any, int]]:
|
||||
def _plan_classification(
|
||||
self, image: Any, seq: int, frame_id: int
|
||||
) -> List[Tuple[TrackState, Any, int]]:
|
||||
"""Pick what to classify now. Must be called with the session lock held.
|
||||
|
||||
Returns (track, crop, object_generation) triples; the caller submits them
|
||||
after the lock is released.
|
||||
after the lock is released. The frame id travels with the plan because
|
||||
the label box is in *that* frame's coordinates: by the time the answer
|
||||
arrives the track has moved, and a client mapping the box back to the
|
||||
source image needs the frame the box was measured on, not the one the
|
||||
result was delivered with.
|
||||
"""
|
||||
frame_h, frame_w = image.shape[:2]
|
||||
margin_y = int(frame_h * self.settings.edge_margin_ratio)
|
||||
|
|
@ -812,12 +830,17 @@ class LiveSession:
|
|||
crop = self.engine.crop(image, track.raw_box)
|
||||
track.in_flight += 1
|
||||
track.last_attempt_seq = seq
|
||||
track.label_frame_id = int(frame_id)
|
||||
track.last_attempt_area = track.area()
|
||||
planned.append((track, crop, track.object_generation))
|
||||
return planned
|
||||
|
||||
def _submit_classification(
|
||||
self, planned: List[Tuple[TrackState, Any, int]], method: str, generation: int
|
||||
self,
|
||||
planned: List[Tuple[TrackState, Any, int]],
|
||||
method: str,
|
||||
generation: int,
|
||||
frame_id: int,
|
||||
) -> None:
|
||||
"""Submit planned crops. Called WITHOUT the session lock held."""
|
||||
for track, crop, object_generation in planned:
|
||||
|
|
@ -841,9 +864,8 @@ class LiveSession:
|
|||
continue
|
||||
future.add_done_callback(
|
||||
lambda fut, tid=track_id, gen=generation, m=method, started=submitted_at,
|
||||
obj=object_generation, box=submitted_box: self._on_classified(
|
||||
fut, tid, gen, m, started, obj, box
|
||||
)
|
||||
obj=object_generation, box=submitted_box, fid=int(frame_id):
|
||||
self._on_classified(fut, tid, gen, m, started, obj, box, fid)
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ crop lane
|
||||
|
|
@ -1023,9 +1045,8 @@ class LiveSession:
|
|||
self._crops_accepted += 1
|
||||
future.add_done_callback(
|
||||
lambda fut, tid=track_id_out, gen=generation, m=method, started=submitted_at,
|
||||
obj=object_generation, box=box: self._on_classified(
|
||||
fut, tid, gen, m, started, obj, box, SOURCE_CROP
|
||||
)
|
||||
obj=object_generation, box=box, fid=int(record["frame_id"]):
|
||||
self._on_classified(fut, tid, gen, m, started, obj, box, fid, SOURCE_CROP)
|
||||
)
|
||||
return {
|
||||
"type": "crop_ack",
|
||||
|
|
@ -1117,6 +1138,7 @@ class LiveSession:
|
|||
submitted_at: float,
|
||||
object_generation: int,
|
||||
submitted_box: List[int],
|
||||
label_frame_id: int = 0,
|
||||
source: str = SOURCE_FRAME,
|
||||
) -> None:
|
||||
try:
|
||||
|
|
@ -1174,6 +1196,7 @@ class LiveSession:
|
|||
track.state = STATE_UNRECOGNIZED
|
||||
track.confirmed = True
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
track.identity = None
|
||||
track.candidates = []
|
||||
return
|
||||
|
|
@ -1223,6 +1246,7 @@ class LiveSession:
|
|||
# The label belongs to the box the crop came from, not to wherever
|
||||
# the track has moved by the time classification finishes.
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
track.drift_frames = 0
|
||||
track.identity = {
|
||||
"sku_id": decision.get("best_sku"),
|
||||
|
|
@ -1239,6 +1263,7 @@ class LiveSession:
|
|||
)
|
||||
track.confirmed = True
|
||||
track.label_box = list(submitted_box)
|
||||
track.label_frame_id = int(label_frame_id)
|
||||
track.drift_frames = 0
|
||||
track.identity = None
|
||||
else:
|
||||
|
|
@ -1278,10 +1303,20 @@ class LiveSession:
|
|||
)
|
||||
payload = {
|
||||
"track_id": track.track_id,
|
||||
# The frame `label_box` is in. The result carrying the label arrives
|
||||
# on a later frame, so a client that maps the box back to its own
|
||||
# source image must use this id's offset, not the result's.
|
||||
"label_frame_id": int(track.label_frame_id),
|
||||
# box: smoothed track box, for drawing. raw_box: the detection box the
|
||||
# label is anchored to, for cropping and for explaining label changes.
|
||||
"box": list(track.box),
|
||||
"raw_box": list(track.raw_box),
|
||||
# The box the label was measured on, in frame coordinates. It is
|
||||
# what the label is *about*: `box` has moved on by the time a
|
||||
# classification finishes, so scoring a label against `box` scores
|
||||
# it against wherever the track drifted to. An evaluator matching
|
||||
# labels to ground truth needs this one.
|
||||
"label_box": list(track.label_box or track.box),
|
||||
"confidence": round(float(track.confidence), 4),
|
||||
"first_seen_seq": track.first_seq,
|
||||
"relabels": track.relabels,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue