diff --git a/recognization/code/identity_lanes.py b/recognization/code/identity_lanes.py index ba6b74c..ea6f231 100644 --- a/recognization/code/identity_lanes.py +++ b/recognization/code/identity_lanes.py @@ -134,6 +134,14 @@ class LocalLane: if sku not in self.sku_ids: self.sku_ids.append(sku) for height in heights: + if height > image.shape[0]: + # Never upscale a reference. A 305 px view blown up to 400 + # is a blur, and a blur matches everything: the l09 shelf + # crop won 31 of 54 crops on the shelf it was cut from, at + # a median margin of 4.9, because its upscaled level had no + # high-frequency detail to disagree with. A view is used at + # the levels it actually has. + continue scaled = cv2.resize( image, (max(1, int(image.shape[1] * height / image.shape[0])), int(height)), diff --git a/recognization/code/live.py b/recognization/code/live.py index 31bf465..d4240ac 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -154,6 +154,10 @@ class TrackState: # with the rest of the evidence when the object generation changes. text_rank: List[Tuple[str, float]] = field(default_factory=list) text_read: str = "" + # The frame the newest label box was measured on. The box is in that + # frame's coordinates; a client mapping it back to the source image needs + # this id, because the result that carries the label arrives on a later one. + label_frame_id: int = 0 def area(self) -> int: if not self.raw_box: @@ -615,11 +619,10 @@ class LiveSession: if state.box ] tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset) - # Collect the classification work under the lock; submit it after # releasing it. add_done_callback can run the callback inline when the # future already finished, and the callback takes this same lock. - submissions = self._plan_classification(image, seq) + submissions = self._plan_classification(image, seq, int(frame_id)) crop_requests = self._plan_crop_requests(image.shape, seq, int(frame_id), measured) summary = self._summary_locked() @@ -635,7 +638,7 @@ class LiveSession: summary["price_tags"] = tag_summary["tags"] summary["compliance"] = tag_summary["compliance"] - self._submit_classification(submissions, method, generation) + self._submit_classification(submissions, method, generation, int(frame_id)) self._run_tag_passes(image, tag_bands) result = { @@ -685,7 +688,16 @@ class LiveSession: continue track.state = STATE_RECOGNIZED track.confirmed = True - track.label_box = list(track.box) + # The raw detection box, not the smoothed track box: smoothing + # lags a pan by about one step, and the label is about where the + # product was measured, not where the filter has drifted to. + track.label_box = list(track.raw_box or track.box) + # The box is measured on this frame, so the client must map it + # with this frame's offset. Leaving the default 0 makes every + # inherited label map as if it came from the first frame, which + # during a pan puts it a whole pan-distance away from the + # product it names. + track.label_frame_id = int(self._latest_frame_id) track.last_source = SOURCE_INHERITED track.identity = { "sku_id": entry["sku_id"], @@ -762,11 +774,17 @@ class LiveSession: track.crop_attempts = 0 # ------------------------------------------------------------------ classification - def _plan_classification(self, image: Any, seq: int) -> List[Tuple[TrackState, Any, int]]: + def _plan_classification( + self, image: Any, seq: int, frame_id: int + ) -> List[Tuple[TrackState, Any, int]]: """Pick what to classify now. Must be called with the session lock held. Returns (track, crop, object_generation) triples; the caller submits them - after the lock is released. + after the lock is released. The frame id travels with the plan because + the label box is in *that* frame's coordinates: by the time the answer + arrives the track has moved, and a client mapping the box back to the + source image needs the frame the box was measured on, not the one the + result was delivered with. """ frame_h, frame_w = image.shape[:2] margin_y = int(frame_h * self.settings.edge_margin_ratio) @@ -812,12 +830,17 @@ class LiveSession: crop = self.engine.crop(image, track.raw_box) track.in_flight += 1 track.last_attempt_seq = seq + track.label_frame_id = int(frame_id) track.last_attempt_area = track.area() planned.append((track, crop, track.object_generation)) return planned def _submit_classification( - self, planned: List[Tuple[TrackState, Any, int]], method: str, generation: int + self, + planned: List[Tuple[TrackState, Any, int]], + method: str, + generation: int, + frame_id: int, ) -> None: """Submit planned crops. Called WITHOUT the session lock held.""" for track, crop, object_generation in planned: @@ -841,9 +864,8 @@ class LiveSession: continue future.add_done_callback( lambda fut, tid=track_id, gen=generation, m=method, started=submitted_at, - obj=object_generation, box=submitted_box: self._on_classified( - fut, tid, gen, m, started, obj, box - ) + obj=object_generation, box=submitted_box, fid=int(frame_id): + self._on_classified(fut, tid, gen, m, started, obj, box, fid) ) # ------------------------------------------------------------------ crop lane @@ -1023,9 +1045,8 @@ class LiveSession: self._crops_accepted += 1 future.add_done_callback( lambda fut, tid=track_id_out, gen=generation, m=method, started=submitted_at, - obj=object_generation, box=box: self._on_classified( - fut, tid, gen, m, started, obj, box, SOURCE_CROP - ) + obj=object_generation, box=box, fid=int(record["frame_id"]): + self._on_classified(fut, tid, gen, m, started, obj, box, fid, SOURCE_CROP) ) return { "type": "crop_ack", @@ -1117,6 +1138,7 @@ class LiveSession: submitted_at: float, object_generation: int, submitted_box: List[int], + label_frame_id: int = 0, source: str = SOURCE_FRAME, ) -> None: try: @@ -1174,6 +1196,7 @@ class LiveSession: track.state = STATE_UNRECOGNIZED track.confirmed = True track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.identity = None track.candidates = [] return @@ -1223,6 +1246,7 @@ class LiveSession: # The label belongs to the box the crop came from, not to wherever # the track has moved by the time classification finishes. track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.drift_frames = 0 track.identity = { "sku_id": decision.get("best_sku"), @@ -1239,6 +1263,7 @@ class LiveSession: ) track.confirmed = True track.label_box = list(submitted_box) + track.label_frame_id = int(label_frame_id) track.drift_frames = 0 track.identity = None else: @@ -1278,10 +1303,20 @@ class LiveSession: ) payload = { "track_id": track.track_id, + # The frame `label_box` is in. The result carrying the label arrives + # on a later frame, so a client that maps the box back to its own + # source image must use this id's offset, not the result's. + "label_frame_id": int(track.label_frame_id), # box: smoothed track box, for drawing. raw_box: the detection box the # label is anchored to, for cropping and for explaining label changes. "box": list(track.box), "raw_box": list(track.raw_box), + # The box the label was measured on, in frame coordinates. It is + # what the label is *about*: `box` has moved on by the time a + # classification finishes, so scoring a label against `box` scores + # it against wherever the track drifted to. An evaluator matching + # labels to ground truth needs this one. + "label_box": list(track.label_box or track.box), "confidence": round(float(track.confidence), 4), "first_seen_seq": track.first_seq, "relabels": track.relabels, diff --git a/recognization/code/score_stream.py b/recognization/code/score_stream.py new file mode 100644 index 0000000..e5bbbb6 --- /dev/null +++ b/recognization/code/score_stream.py @@ -0,0 +1,209 @@ +"""Score a streamed clip's labels against ground truth: the G1 number. + +The stream path was reported as "53 of 131 tracks", which is not comparable to +Van's 57% of labelled facings - the two count different things. But the clip is +built from `add/image6.jpg`, which has labels, so matching each confirmed +track's `label_box` to the labelled facings gives stream-path coverage directly. + + python code/score_stream.py TRACKS.json --labels eval/labels/image6.json + +`TRACKS.json` is what `stream_client.py --tracks` writes: one row per track that +settled on a label, with the box that label was measured on, already mapped back +to source-image coordinates. + +Reported per run: correct / in-catalog facings, precision, unknown accepts, and +tracks that matched no labelled facing. A track whose box overlaps a facing the +catalog does not enrol is an unenrolled false accept - the dangerous kind, and +the one the evaluator's own precision denominator used to hide. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List, Optional, Sequence, Tuple + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +# A facing clipped by the window edge is not something a run could have named, +# so the denominator keeps only facings at least this far inside the window. +VISIBLE_MIN = 0.5 + + +def iou(a: Sequence[int], b: Sequence[int]) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + inter_w = max(0, min(ax2, bx2) - max(ax1, bx1)) + inter_h = max(0, min(ay2, by2) - max(ay1, by1)) + inter = inter_w * inter_h + area_a = max(0, ax2 - ax1) * max(0, ay2 - ay1) + area_b = max(0, bx2 - bx1) * max(0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def load_facings(labels_path: str) -> List[Dict[str, Any]]: + with open(labels_path) as handle: + payload = json.load(handle) + return payload["facings"] + + +def enrolled_skus(stock_dir: str) -> set: + """Canonical SKUs with at least one enrolled view.""" + from rebuild_catalog import sku_of + + return { + sku_of(name) for name in os.listdir(stock_dir) + if name.lower().endswith((".jpg", ".jpeg", ".png")) + } + + +def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float: + """Share of a facing's area inside the window the client actually showed.""" + wx0, wy0, wx1, wy1 = window + bx0, by0, bx1, by1 = box + inter_w = max(0, min(bx1, wx1) - max(bx0, wx0)) + inter_h = max(0, min(by1, wy1) - max(by0, wy0)) + area = max(0, bx1 - bx0) * max(0, by1 - by0) + return (inter_w * inter_h) / area if area > 0 else 0.0 + + +def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]], + stock_dir: str, iou_min: float, + windows: Optional[Sequence[Sequence[int]]] = None + ) -> Dict[str, Any]: + """Score the labels a stream settled on. + + `windows` are the source-image rects the client showed, as + [x0, y0, x1, y1] each. The denominator is the facings at least half inside + one of them, in x and y both: a 1280x720 window over a 1500x2000 photo sees + 24 of the 63 in-catalog facings, and a facing clipped by the window edge is + not something any run could have named. A panned run shows a different rect + each frame, so the reachable set is the union. Without it the denominator is + every in-catalog facing. + + A correct answer is counted before the unenrolled and unknown branches. The + text lane names SKUs the image lane has no view for, which is the point of + it, and scoring that as a false accept punishes the lane for working. + """ + if windows: + facings = [ + f for f in facings + if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN + ] + in_catalog = [f for f in facings if f["label"] != "unknown"] + enrolled = enrolled_skus(stock_dir) + + correct = 0 + wrong = 0 + unenrolled_false = 0 + unknown_false = 0 + unmatched = 0 + rows: List[Dict[str, Any]] = [] + # Coverage is per facing, precision is per label. A track that restarts on + # the same product, or two tracks on one facing, is two claims but one + # facing: counting tracks as coverage reports 138% of a 24-facing window. + named: Dict[int, str] = {} + + for track in tracks: + box = track.get("label_box") or [] + if len(box) != 4: + unmatched += 1 + continue + best, best_iou = None, 0.0 + for facing in facings: + value = iou(box, facing["box"]) + if value > best_iou: + best, best_iou = facing, value + if best is None or best_iou < iou_min: + unmatched += 1 + rows.append({"track_id": track["track_id"], "predicted": track["sku_id"], + "truth": None, "iou": round(best_iou, 3), "outcome": "unmatched"}) + continue + truth = best["label"] + predicted = track["sku_id"] + if predicted == truth: + correct += 1 + outcome = "correct" + named[best["index"]] = predicted + elif truth == "unknown": + unknown_false += 1 + outcome = "unknown_false_accept" + elif truth not in enrolled: + unenrolled_false += 1 + outcome = "unenrolled_false_accept" + else: + wrong += 1 + outcome = "wrong" + rows.append({"track_id": track["track_id"], "predicted": predicted, + "truth": truth, "iou": round(best_iou, 3), "outcome": outcome}) + + shown = correct + wrong + unenrolled_false + unknown_false + return { + "in_catalog_facings": len(in_catalog), + "facings_named": len(named), + "tracks_scored": len(tracks), + "correct": correct, + "wrong": wrong, + "unenrolled_false_accepts": unenrolled_false, + "unknown_false_accepts": unknown_false, + "unmatched_tracks": unmatched, + "shown": shown, + "coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0, + "precision": round(correct / shown, 4) if shown else 0.0, + "rows": rows, + } + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Score streamed labels against ground truth") + parser.add_argument("tracks", help="JSON written by stream_client.py --tracks") + parser.add_argument("--labels", default="eval/labels/image6.json") + parser.add_argument("--stock-dir", default="stock-lopia-clean") + parser.add_argument("--iou", type=float, default=0.5) + parser.add_argument("--json", dest="json_path", default=None) + parser.add_argument("--window", default=None, + help="x0,y0,x1,y1 the client showed; the denominator is " + "restricted to facings at least half inside it") + parser.add_argument("--verbose", action="store_true", help="print every scored track") + args = parser.parse_args(argv) + + with open(args.tracks) as handle: + payload = json.load(handle) + windows = None + if args.window: + windows = [tuple(int(part) for part in args.window.split(","))] + elif payload.get("windows"): + windows = [tuple(w) for w in payload["windows"]] + report = score(payload["tracks"], load_facings(args.labels), args.stock_dir, + args.iou, windows) + report["source"] = payload.get("source") + report["pace"] = payload.get("pace") + report["seconds"] = payload.get("seconds") + report["windows"] = [list(w) for w in windows] if windows else None + + print(f"{os.path.basename(args.tracks)} pace={report['pace']} " + f"{report['seconds']}s") + print(f" facings named {report['facings_named']}/{report['in_catalog_facings']} " + f"in-catalog ({report['coverage']:.0%}) labels correct {report['correct']} " + f"precision {report['precision']:.0%} shown {report['shown']}") + print(f" wrong {report['wrong']} unenrolled false accepts " + f"{report['unenrolled_false_accepts']} unknown false accepts " + f"{report['unknown_false_accepts']} unmatched tracks {report['unmatched_tracks']}") + if args.verbose: + for row in report["rows"]: + print(f" t{row['track_id']:4d} {row['outcome']:24s} " + f"pred={row['predicted']} truth={row['truth']} iou={row['iou']}") + + if args.json_path: + with open(args.json_path, "w") as handle: + json.dump(report, handle, indent=2) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recognization/code/stream_client.py b/recognization/code/stream_client.py index 8e4b6ae..ddb1406 100644 --- a/recognization/code/stream_client.py +++ b/recognization/code/stream_client.py @@ -56,6 +56,15 @@ class FrameSource: self.image = None self.pan_x = 0 self.pan_direction = 1 + # Where the last frame was cut from the source image. (0, 0) for a + # camera or a video, whose frames are the whole image. + self.offset = (0, 0) + # The source-image rects the window showed, so an evaluator can restrict + # its denominator to the facings this run could actually have seen. + # A list, because a panned run shows a different rect each frame and the + # reachable set is their union. Both axes matter: a 1280-wide window + # over a 1500-wide photo is x-limited as well as y-limited. + self.windows: List[Tuple[int, int, int, int]] = [] if source.isdigit(): self.capture = cv2.VideoCapture(int(source)) elif os.path.isfile(source) and source.lower().endswith((".mp4", ".mov", ".avi", ".mkv", ".webm")): @@ -66,6 +75,12 @@ class FrameSource: raise SystemExit(f"cannot read source {source}") def frame(self, index: int): + """One frame, plus where it was cut from the source image. + + The offset is what lets an evaluator map a track's box back to the + source image's coordinates: a label measured on a pan window is only + comparable to ground truth once the window's origin is added back. + """ if self.image is not None: height, width = self.image.shape[:2] window_w = min(self.pan_width, width) @@ -74,6 +89,8 @@ class FrameSource: x = min(self.pan_x, max_x) y = max(0, (height - window_h) // 2) window = self.image[y:y + window_h, x:x + window_w] + self.offset = (x, y) + self.windows.append((x, y, x + window_w, y + window_h)) self.pan_x += self.pan_direction * self.pan_step if self.pan_x >= max_x or self.pan_x <= 0: self.pan_direction *= -1 @@ -111,6 +128,11 @@ def main() -> None: parser.add_argument("--source", default="image.jpg", help="camera index, video path, or image path") parser.add_argument("--pan", type=int, default=12, help="pixels per frame when panning a still image") parser.add_argument("--seconds", type=float, default=20.0) + parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table") + parser.add_argument("--tracks", default=None, + help="write per-track labels and their boxes (source-image " + "coordinates) as JSON, for scoring against ground truth") + parser.add_argument("--quiet", action="store_true", help="only print the final summary") parser.add_argument("--width", type=int, default=1280, help="send width") parser.add_argument("--quality", type=int, default=75) parser.add_argument("--max-inflight", type=int, default=1) @@ -119,8 +141,6 @@ def main() -> None: "'result': send the next frame only after a result, so the " "pan stays continuous (the mode to use for stability checks)") parser.add_argument("--method", default=None, help="surf | orb | brisk | hybrid") - parser.add_argument("--json", action="store_true", help="print a JSON summary instead of the table") - parser.add_argument("--quiet", action="store_true", help="only print the final summary") args = parser.parse_args() url = args.url @@ -135,6 +155,12 @@ def main() -> None: first_label: Dict[int, float] = {} label_latencies: List[float] = [] labels: Dict[int, str] = {} + pending_frame: Optional[int] = None + # Per track: the label it settled on, the box that label was measured on, + # and the pan offset of the frame that box lives in. Written out with + # --tracks so an evaluator can match labels to ground truth. + track_records: Dict[int, Dict[str, Any]] = {} + frame_offsets: Dict[int, Tuple[int, int]] = {} label_changes = 0 unexplained_changes = 0 relabel_seen: Dict[int, int] = {} @@ -192,15 +218,33 @@ def main() -> None: first_label[track_id] = now if not inherited: label_latencies.append((now - started) * 1000.0) - elif labels.get(track_id) != sku: - # Only a measured-to-measured change on the same object is a - # label change; inherited labels describe the same product - # under a new track id. + # Only a measured-to-measured change on the same object is a + # label change; inherited labels describe the same product + # under a new track id. + if track_id in labels and labels[track_id] != sku: if not inherited and not inherited_seen.get(track_id): label_changes += 1 if relabels <= relabel_seen.get(track_id, 0): unexplained_changes += 1 labels[track_id] = sku + # The box the label is about, in source-image coordinates. The + # offset comes from the frame the box was measured on, not the + # frame this result arrived with: the pan has moved on since. + offset = frame_offsets.get(track.get("label_frame_id"), (0, 0)) + label_box = track.get("label_box") or track.get("box") or [] + if len(label_box) == 4: + track_records[track_id] = { + "track_id": track_id, + "sku_id": sku, + "label_box": [ + label_box[0] + offset[0], label_box[1] + offset[1], + label_box[2] + offset[0], label_box[3] + offset[1], + ], + "label_frame_id": track.get("label_frame_id"), + "result_frame_id": message["frame_id"], + "inherited": inherited, + "restarts": restarts, + } inherited_seen[track_id] = inherited relabel_seen[track_id] = relabels restart_seen[track_id] = restarts @@ -219,6 +263,7 @@ def main() -> None: break index += 1 frame_id = (frame_id + 1) & 0xFFFFFFFF + frame_offsets[frame_id] = source.offset payload = encode(frame, args.width, args.quality) if len(in_flight) >= args.max_inflight: if pending_frame is not None: @@ -266,6 +311,15 @@ def main() -> None: "first_label_ms_p50": round(statistics.median(label_latencies), 0) if label_latencies else None, "final_summary": last_summary, } + if args.tracks: + with open(args.tracks, "w") as handle: + json.dump({ + "windows": [list(w) for w in source.windows] or None, + "source": args.source, + "pace": args.pace, + "seconds": round(elapsed, 1), + "tracks": sorted(track_records.values(), key=lambda row: row["track_id"]), + }, handle, indent=2) if args.json: print(json.dumps(summary, indent=2)) else: