fix: restore reliable authenticated live recognition
This commit is contained in:
kiyo9w
parent
0e23425ef0
commit
934615b593
40 changed files with 2556 additions and 820 deletions
|
|
@ -96,8 +96,9 @@ class TrackState:
|
|||
last_attempt_area: int = 0
|
||||
score_sums: Dict[str, float] = field(default_factory=dict)
|
||||
score_n: int = 0
|
||||
identity: Optional[Dict[str, Any]] = None
|
||||
observations: List[Dict[str, float]] = field(default_factory=list)
|
||||
candidates: List[Dict[str, Any]] = field(default_factory=list)
|
||||
identity: Optional[Dict[str, Any]] = None
|
||||
confirmed: bool = False
|
||||
label_box: List[int] = field(default_factory=list)
|
||||
drift_frames: int = 0
|
||||
|
|
@ -184,6 +185,10 @@ class LiveSession:
|
|||
self._catalog_by_sku: Dict[str, Dict[str, Any]] = {}
|
||||
self._catalog_version = ""
|
||||
self._tags = None
|
||||
# Shelf position of this frame's origin. Updated from a content shift
|
||||
# before the session lock; tag cadence keys rows by it. Not a shelf map.
|
||||
self._frame_origin = [0.0, 0.0]
|
||||
self._shift_map = None
|
||||
self._compliance_tracker = None
|
||||
self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []}
|
||||
self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque(
|
||||
|
|
@ -256,7 +261,7 @@ class LiveSession:
|
|||
track.identity = None
|
||||
track.candidates = []
|
||||
track.score_sums = {}
|
||||
track.score_n = 0
|
||||
track.observations = []
|
||||
track.attempts = 0
|
||||
track.in_flight = 0
|
||||
track.last_attempt_seq = 0
|
||||
|
|
@ -286,6 +291,8 @@ class LiveSession:
|
|||
self._slots = []
|
||||
self._latest_image = None
|
||||
self._latest_frame_id = None
|
||||
self._frame_origin = [0.0, 0.0]
|
||||
self._shift_map = None
|
||||
|
||||
# ------------------------------------------------------------------ frame path
|
||||
def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]:
|
||||
|
|
@ -300,6 +307,9 @@ class LiveSession:
|
|||
empty_enabled = self.settings.empty_slots
|
||||
generation = self._generation
|
||||
|
||||
|
||||
shelf_offset = self._note_frame_shift(image)
|
||||
|
||||
boxes = self.engine.detect(image, conf=conf)
|
||||
detected_at = time.perf_counter()
|
||||
tracks = self.tracker.update(
|
||||
|
|
@ -327,7 +337,13 @@ class LiveSession:
|
|||
raw_box = list(detection["box"])
|
||||
confidence = float(detection["confidence"])
|
||||
if raw_box is None:
|
||||
raw_box = list(track.box)
|
||||
# Low-score second pass, or a Kalman box with no detection.
|
||||
# Keep the last measured crop. Do not invent one from the prediction.
|
||||
raw_box = list(state.raw_box) if state is not None and state.raw_box else []
|
||||
confidence = float(state.confidence) if state is not None else confidence
|
||||
low_score = bool(getattr(track, "low_score", False)) or (
|
||||
track.det_index is None and raw_box == []
|
||||
)
|
||||
previous_seq = self._last_seen_seq.get(track.track_id)
|
||||
if state is None:
|
||||
# An id that comes back after an absence may be a recycled id:
|
||||
|
|
@ -369,8 +385,12 @@ class LiveSession:
|
|||
state.last_seq = seq
|
||||
self._last_seen_seq[track.track_id] = seq
|
||||
state.box = list(track.box)
|
||||
state.raw_box = raw_box
|
||||
state.confidence = confidence
|
||||
if raw_box and not low_score:
|
||||
state.raw_box = raw_box
|
||||
state.confidence = confidence
|
||||
elif not state.raw_box and raw_box:
|
||||
state.raw_box = raw_box
|
||||
state.confidence = confidence
|
||||
self._drop_stale_label(state)
|
||||
active[track.track_id] = state
|
||||
self._tracks = active
|
||||
|
|
@ -405,7 +425,7 @@ class LiveSession:
|
|||
for state in self._tracks.values()
|
||||
if state.box
|
||||
]
|
||||
tag_bands = self._tags.plan(tag_boxes, image.shape)
|
||||
tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset)
|
||||
|
||||
# Collect the classification work under the lock; submit it after
|
||||
# releasing it. add_done_callback can run the callback inline when the
|
||||
|
|
@ -443,39 +463,44 @@ class LiveSession:
|
|||
}
|
||||
return result
|
||||
|
||||
|
||||
def _inherit_labels(self, seq: int) -> None:
|
||||
"""Give a fresh track the label of a recently recognized product it covers.
|
||||
|
||||
The same physical product can be tracked as several ids while the camera
|
||||
pans. Remembering where a label was measured means each product is
|
||||
classified once and its neighbours inherit the result instead of waiting
|
||||
in the classification queue again.
|
||||
Only a measured label, still overlapping its own box, may be copied.
|
||||
An inherited label is not a measurement, so it cannot spread to a neighbour.
|
||||
Blur, glare, and a too-far frame skip inheritance: the overlap is not
|
||||
evidence when the frame itself failed the quality gate.
|
||||
"""
|
||||
if not self._label_memory:
|
||||
return
|
||||
if (self._last_quality or {}).get("hint"):
|
||||
return
|
||||
for track in self._tracks.values():
|
||||
if track.state != STATE_DETECTED or track.confirmed:
|
||||
continue
|
||||
for entry in reversed(self._label_memory):
|
||||
if seq - entry["seq"] > LABEL_MEMORY_FRAMES:
|
||||
break
|
||||
if _iou(track.box, entry["box"]) >= LABEL_MEMORY_IOU:
|
||||
track.state = STATE_RECOGNIZED
|
||||
track.confirmed = True
|
||||
track.label_box = list(track.box)
|
||||
track.identity = {
|
||||
"sku_id": entry["sku_id"],
|
||||
"confidence": entry["confidence"],
|
||||
"score": entry["score"],
|
||||
"attempts": 0,
|
||||
"inherited": True,
|
||||
}
|
||||
# An inherited label is a guess from the same place, not a
|
||||
# measurement of this object: count it as re-verified so a
|
||||
# later correction is auditable rather than a silent flip.
|
||||
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
|
||||
self._relabel_count[track.track_id] = track.relabels
|
||||
break
|
||||
if entry.get("inherited"):
|
||||
continue
|
||||
if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU:
|
||||
continue
|
||||
if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU:
|
||||
continue
|
||||
track.state = STATE_RECOGNIZED
|
||||
track.confirmed = True
|
||||
track.label_box = list(track.box)
|
||||
track.identity = {
|
||||
"sku_id": entry["sku_id"],
|
||||
"confidence": entry["confidence"],
|
||||
"score": entry["score"],
|
||||
"attempts": 0,
|
||||
"inherited": True,
|
||||
}
|
||||
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
|
||||
self._relabel_count[track.track_id] = track.relabels
|
||||
break
|
||||
|
||||
def _remember_label(self, track: TrackState, box: List[int], seq: int) -> None:
|
||||
"""Must be called with the session lock held."""
|
||||
|
|
@ -484,10 +509,12 @@ class LiveSession:
|
|||
self._label_memory.append(
|
||||
{
|
||||
"box": list(box),
|
||||
"label_box": list(track.label_box or box),
|
||||
"sku_id": track.identity.get("sku_id"),
|
||||
"confidence": track.identity.get("confidence"),
|
||||
"score": track.identity.get("score"),
|
||||
"seq": seq,
|
||||
"inherited": bool(track.identity.get("inherited")),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -516,7 +543,7 @@ class LiveSession:
|
|||
track.identity = None
|
||||
track.candidates = []
|
||||
track.score_sums = {}
|
||||
track.score_n = 0
|
||||
track.observations = []
|
||||
track.attempts = 0
|
||||
track.in_flight = 0
|
||||
track.last_attempt_seq = 0
|
||||
|
|
@ -539,11 +566,16 @@ class LiveSession:
|
|||
return []
|
||||
|
||||
eligible: List[TrackState] = []
|
||||
poor_frame = bool((self._last_quality or {}).get("hint"))
|
||||
for track in self._tracks.values():
|
||||
if track.state != STATE_DETECTED or track.in_flight > 0:
|
||||
continue
|
||||
if poor_frame:
|
||||
continue
|
||||
if track.hits < self.settings.min_hits:
|
||||
continue
|
||||
if len(track.raw_box) != 4:
|
||||
continue
|
||||
x1, y1, x2, y2 = track.raw_box
|
||||
if min(x2 - x1, y2 - y1) < self.settings.min_crop_px:
|
||||
continue
|
||||
|
|
@ -563,9 +595,8 @@ class LiveSession:
|
|||
eligible.append(track)
|
||||
|
||||
# Newest first: while a camera pans, the products still worth labelling
|
||||
# are the ones that just entered the frame. Sorting by area (the obvious
|
||||
# default) spends the budget on products that are already leaving.
|
||||
eligible.sort(key=lambda track: (track.first_seq, -track.area()))
|
||||
# are the ones that just entered the frame. Area is only the tie-break.
|
||||
eligible.sort(key=lambda track: (-track.first_seq, -track.area()))
|
||||
planned: List[Tuple[TrackState, Any, int]] = []
|
||||
for track in eligible[:capacity]:
|
||||
crop = self.engine.crop(image, track.raw_box)
|
||||
|
|
@ -623,13 +654,18 @@ class LiveSession:
|
|||
return
|
||||
track.in_flight = max(0, track.in_flight - 1)
|
||||
track.attempts += 1
|
||||
for sku_id, score in (scores or {}).items():
|
||||
observation = {
|
||||
str(sku_id): float(score) for sku_id, score in (scores or {}).items()
|
||||
}
|
||||
track.observations.append(observation)
|
||||
track.observations = track.observations[-3:]
|
||||
for sku_id, score in observation.items():
|
||||
track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score)
|
||||
if scores:
|
||||
if observation:
|
||||
track.score_n += 1
|
||||
self._classify_times.append(finished_at - submitted_at)
|
||||
|
||||
if not scores:
|
||||
if not any(track.observations):
|
||||
if track.attempts >= self.settings.max_attempts:
|
||||
track.state = STATE_UNRECOGNIZED
|
||||
track.confirmed = True
|
||||
|
|
@ -638,11 +674,9 @@ class LiveSession:
|
|||
track.candidates = []
|
||||
return
|
||||
|
||||
from identity import decide_identity
|
||||
from identity import aggregate_votes, decide_identity
|
||||
|
||||
mean_scores = {
|
||||
sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items()
|
||||
}
|
||||
mean_scores = aggregate_votes(track.observations)
|
||||
decision = decide_identity(
|
||||
mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier")
|
||||
)
|
||||
|
|
@ -838,7 +872,7 @@ class LiveSession:
|
|||
gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY)
|
||||
sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var())
|
||||
glare = float(np.mean(np.all(small > self.settings.glare_pixel, axis=2)))
|
||||
heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if state.raw_box]
|
||||
heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if len(state.raw_box) == 4]
|
||||
median_height = float(np.median(heights)) if heights else 0.0
|
||||
too_far = bool(
|
||||
heights
|
||||
|
|
@ -847,7 +881,8 @@ class LiveSession:
|
|||
or sum(1 for h in heights if h < self.settings.min_crop_px) > len(heights) / 2
|
||||
)
|
||||
)
|
||||
blur = sharpness < self._blur_threshold()
|
||||
# A black frame has Laplacian 0. That is not blur; a real shelf is not black.
|
||||
blur = sharpness < self._blur_threshold() and float(gray.mean()) > 1.0
|
||||
hint = None
|
||||
if glare > self.settings.glare_fraction:
|
||||
hint = "glare"
|
||||
|
|
@ -898,6 +933,24 @@ class LiveSession:
|
|||
self._tags = TagService(ocr_engine, settings=settings or TagSettings())
|
||||
return True
|
||||
|
||||
def _note_frame_shift(self, image: Any) -> Tuple[float, float]:
|
||||
"""Shelf position of this frame's origin. Phase correlation only.
|
||||
|
||||
Runs before the session lock. A failed registration keeps the last origin
|
||||
rather than inventing a jump. The content shift moves the origin the
|
||||
other way, same sign as ShelfMap.
|
||||
"""
|
||||
if self._shift_map is None:
|
||||
from shelf_map import ShelfMap
|
||||
|
||||
self._shift_map = ShelfMap()
|
||||
shift = self._shift_map.register(image)
|
||||
with self._lock:
|
||||
if shift is not None:
|
||||
self._frame_origin[0] -= float(shift[0])
|
||||
self._frame_origin[1] -= float(shift[1])
|
||||
return (self._frame_origin[0], self._frame_origin[1])
|
||||
|
||||
def _run_tag_passes(self, image: Any, bands: Sequence[Any]) -> None:
|
||||
"""Submit tag passes to the engine's tag worker (never the frame loop)."""
|
||||
service = self._tags
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue