fix: reject crop evidence after track drift

recognization/code/live.py

@@ -162,6 +162,19 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float:
     return inter / union if union > 0 else 0.0
 
 
+def _crop_off_product(requested: Sequence[int], current: Sequence[int]) -> bool:
+    """True when a track no longer covers the box a crop was taken from.
+
+    The measure a settled label already uses: below DRIFT_IOU the track has
+    slid onto a different product, and a crop of the old one says nothing about
+    what is there now. A box that was never measured cannot vouch either, so
+    it counts as off product.
+    """
+    if len(requested) != 4 or len(current) != 4:
+        return True
+    return _iou(current, requested) < DRIFT_IOU
+
+
 class _CropRejected(Exception):
     """A returned crop that may not be used, carrying the reply to send back."""
 
@@ -806,10 +819,14 @@ class LiveSession:
                 continue
             self._crop_counter += 1
             request_id = "c%d" % self._crop_counter
+            # The detection box this crop is a photograph of. The answer is only
+            # about that product, so the box travels with the request.
+            requested_box = list(track.raw_box)
             self._crop_requests[request_id] = {
                 "request_id": request_id,
                 "track_id": track.track_id,
                 "frame_id": int(frame_id),
+                "box": requested_box,
                 "object_generation": track.object_generation,
                 "generation": self._generation,
                 "seq": seq,
@@ -824,7 +841,7 @@ class LiveSession:
                 "request_id": request_id,
                 "track_id": track.track_id,
                 "frame_id": int(frame_id),
-                "box": list(track.raw_box),
+                "box": list(requested_box),
             }
         return planned
 
@@ -944,11 +961,19 @@ class LiveSession:
             if track.last_seq < record["seq"]:
                 self._release_crop_slot(record)
                 raise _CropRejected("CROP_STALE", "the track is no longer in the requested frame")
+            if _crop_off_product(record["box"], track.raw_box):
+                # The crop is of the product the phone was asked to photograph.
+                # If the track has since slid onto a neighbour the answer would
+                # be a label for a product nobody cropped.
+                self._release_crop_slot(record)
+                raise _CropRejected("CROP_STALE", "the track moved off the product that was cropped")
             limit = self.settings.max_crop_bytes
             if not payload or len(payload) > limit:
                 self._release_crop_slot(record)
                 raise _CropRejected("CROP_TOO_LARGE", "crop is empty or over %d bytes" % limit)
-            return record, list(track.box), payload
+            # The label anchors to the box the crop was taken from, not to where
+            # the track is by the time the answer lands.
+            return record, list(record["box"]), payload
 
     def _on_classified(
         self,
@@ -975,6 +1000,12 @@ class LiveSession:
                 track.in_flight = max(0, track.in_flight - 1)
                 return
             track.in_flight = max(0, track.in_flight - 1)
+            if source == SOURCE_CROP and _crop_off_product(submitted_box, track.raw_box):
+                # The answer was measured on the box the track has since left.
+                # Dropping it leaves the track to be decided from a later frame
+                # crop instead of putting the old product's label on whatever
+                # moved into its place.
+                return
             track.attempts += 1
             # A crop is an attempt against the same budget as a frame crop, so
             # the lane cannot be used to re-roll the dice on one product.