tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

feat: add bounded crop evidence and keyed gallery cache

Browse source
This commit is contained in:
kiyo9w 2026-09-28 15:49:34 +09:00
parent 69de9ea418
commit 0d0e9f82b9
8 changed files with 1065 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

273
recognization/code/live.py
View file

@ -58,6 +58,14 @@ STATS_WINDOW = 30
# still overlaps the labelled product; a neighbour-slide crosses it.
DRIFT_IOU = 0.5
STATE_UNCONFIRMED = "unconfirmed"
# Which evidence produced the newest observation on a track. A label is only ever
# shown for an accepted match, so this says how that match was measured.
SOURCE_FRAME = "frame"
SOURCE_CROP = "crop"
SOURCE_INHERITED = "inherited"
# A crop the client returns is base64 inside a text control message, so it is
# bounded on its own rather than riding the binary frame path.
MAX_CROP_BYTES = 1024 * 1024
@dataclass
@ -79,6 +87,18 @@ class LiveSettings:
glare_fraction: float = 0.02
glare_pixel: int = 245
too_far_height_ratio: float = 0.08
# Full-resolution crop lane. Off until a client advertises it, so a
# protocol-v1 client is never asked for a crop and its traffic is unchanged.
crops: bool = False
max_crop_bytes: int = 128 * 1024
# A crop answered this many processed frames late describes a product that
# may have moved on, so it is dropped instead of relabelling the track.
crop_ttl_frames: int = 6
# Requests outstanding at once. Bounds what one phone can owe us.
max_crop_inflight: int = 4
# Ask for a native-resolution crop while the box is still this small. At the
# camera's own resolution it may still be large enough to classify.
crop_below_px: int = 64
@dataclass
@ -115,6 +135,11 @@ class TrackState:
# or pruned then re-created). Restarting resets the label, so it is always a
# re-verification rather than an unexplained label change.
restarts: int = 0
# A crop is asked for at most once per object. A recycled id gets a fresh
# TrackState, so the new object may be asked again.
crop_requested: bool = False
# Which evidence produced the newest observation: frame or crop.
last_source: str = SOURCE_FRAME
def area(self) -> int:
if not self.raw_box:
@ -137,6 +162,19 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float:
return inter / union if union > 0 else 0.0
class _CropRejected(Exception):
"""A returned crop that may not be used, carrying the reply to send back."""
def __init__(self, code: str, message: str):
super().__init__(message)
self.reply = _crop_error(code, message)
def _crop_error(code: str, message: str) -> Dict[str, Any]:
"""The one shape a refused crop answers with, recoverable like any other."""
return {"type": "crop_error", "code": code, "message": message}
class LiveSession:
"""Stateful live session for one streaming client."""
@ -195,6 +233,14 @@ class LiveSession:
self._frame_origin = [0.0, 0.0]
self._shift_map = None
self._compliance_tracker = None
# Full-resolution crop lane. A request pins the frame and the track
# object it was minted for, so a late or borrowed crop cannot relabel a
# product the client never photographed.
self._crop_requests: Dict[str, Dict[str, Any]] = {}
self._crop_counter = 0
self._crops_requested = 0
self._crops_accepted = 0
self._crops_rejected = 0
self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []}
self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque(
maxlen=LABEL_MEMORY_MAX
@ -325,6 +371,10 @@ class LiveSession:
track.in_flight = 0
track.last_attempt_seq = 0
track.last_attempt_area = 0
# Outstanding crops were minted against the old method and the
# old generation; answering one now would be evidence measured
# under settings the session no longer holds.
self._crop_requests.clear()
return self.config()
def config(self) -> Dict[str, Any]:
@ -348,6 +398,7 @@ class LiveSession:
self._restart_count.clear()
self._slot_history.clear()
self._label_memory.clear()
self._crop_requests.clear()
self._slots = []
self._latest_image = None
self._latest_frame_id = None
@ -492,10 +543,16 @@ class LiveSession:
# releasing it. add_done_callback can run the callback inline when the
# future already finished, and the callback takes this same lock.
submissions = self._plan_classification(image, seq)
crop_requests = self._plan_crop_requests(seq, int(frame_id))
summary = self._summary_locked()
stats = self._stats_locked()
track_payload = [self._track_payload(state) for state in self._tracks.values()]
if crop_requests:
for row in track_payload:
request = crop_requests.get(row["track_id"])
if request is not None:
row["crop_request"] = request
tag_summary = self._apply_tags(seq, track_payload, tag_bands, shelf_offset, image.shape)
if tag_summary is not None:
summary["price_tags"] = tag_summary["tags"]
@ -552,6 +609,7 @@ class LiveSession:
track.state = STATE_RECOGNIZED
track.confirmed = True
track.label_box = list(track.box)
track.last_source = SOURCE_INHERITED
track.identity = {
"sku_id": entry["sku_id"],
"confidence": entry["confidence"],
@ -690,6 +748,208 @@ class LiveSession:
)
)
# ------------------------------------------------------------------ crop lane
def capabilities(self) -> Dict[str, Any]:
"""What this client is allowed to ask the phone for.
A client that never sends `capabilities` keeps the protocol-v1 defaults:
no crop lane, so no result ever carries a crop request.
"""
return {
"crop": bool(self.settings.crops),
"max_crop_bytes": int(self.settings.max_crop_bytes),
}
def enable_crops(self, max_crop_bytes: Optional[int] = None) -> Dict[str, Any]:
"""Turn the lane on for a client that advertised it. The client states the
largest crop it will produce; the server keeps the lower of the two."""
with self._lock:
self.settings.crops = True
if max_crop_bytes is not None:
try:
stated = int(max_crop_bytes)
except (TypeError, ValueError):
stated = 0
self.settings.max_crop_bytes = max(1024, min(stated, MAX_CROP_BYTES))
return self.capabilities()
def _plan_crop_requests(self, seq: int, frame_id: int) -> Dict[int, Dict[str, Any]]:
"""Ask the phone for a fuller crop of tracks the frame crop could not use.
The box is in the pixel space of `frame_id`, the frame the client just
sent, so the phone can answer from the buffer behind that exact frame
rather than from whatever is on screen now. One request per track object,
at most `max_crop_inflight` outstanding, and only for a client that
advertised the lane.
"""
self._expire_crop_requests(seq)
if not self.settings.crops:
return {}
room = self.settings.max_crop_inflight - len(self._crop_requests)
if room <= 0:
return {}
planned: Dict[int, Dict[str, Any]] = {}
for track in self._tracks.values():
if len(planned) >= room:
break
if track.crop_requested or track.in_flight > 0:
continue
if track.state != STATE_DETECTED or track.confirmed:
continue
if track.hits < self.settings.min_hits or len(track.raw_box) != 4:
continue
x1, y1, x2, y2 = track.raw_box
# The frame crop is too small to say anything about this product. At
# the camera's own resolution it may still be worth a decision, and
# that is the only thing a crop can add here.
if min(x2 - x1, y2 - y1) >= self.settings.crop_below_px:
continue
self._crop_counter += 1
request_id = "c%d" % self._crop_counter
self._crop_requests[request_id] = {
"request_id": request_id,
"track_id": track.track_id,
"frame_id": int(frame_id),
"object_generation": track.object_generation,
"generation": self._generation,
"seq": seq,
"method": self.settings.method,
}
track.crop_requested = True
# Hold the track while the phone answers, so the frame path does not
# spend a second attempt on the same box at the same time.
track.in_flight += 1
self._crops_requested += 1
planned[track.track_id] = {
"request_id": request_id,
"track_id": track.track_id,
"frame_id": int(frame_id),
"box": list(track.raw_box),
}
return planned
def _expire_crop_requests(self, seq: int) -> None:
"""Drop requests the phone did not answer in time.
A request holds the track busy while it is out, so an unanswered one has
to give that slot back or the track could never be classified at all.
"""
stale = [
request_id for request_id, record in self._crop_requests.items()
if seq - record["seq"] > self.settings.crop_ttl_frames
]
for request_id in stale:
record = self._crop_requests.pop(request_id)
self._crops_rejected += 1
self._release_crop_slot(record)
def _release_crop_slot(self, record: Dict[str, Any]) -> None:
"""Give back the busy slot a crop request took. Lock must be held."""
track = self._tracks.get(record["track_id"])
if track is not None and track.object_generation == record["object_generation"]:
track.in_flight = max(0, track.in_flight - 1)
def submit_crop(
self,
request_id: Any,
track_id: Any,
frame_id: Any,
payload: Optional[bytes],
) -> Dict[str, Any]:
"""Score a crop the client returned for a track we asked about.
The crop is evidence about one track object, in one frame, decided under
one configuration. Anything that breaks that binding is refused before a
single pixel reaches the classifier, and the frame crop stays the only
evidence for the track.
"""
try:
record, box, payload = self._prepare_crop(request_id, track_id, frame_id, payload)
except _CropRejected as refused:
with self._lock:
self._crops_rejected += 1
return refused.reply
try:
crop = self.engine.decode_image(payload)
except Exception:
# The busy slot is the request's to give back, whoever failed on it.
with self._lock:
self._release_crop_slot(record)
self._crops_rejected += 1
return _crop_error("CROP_UNDECODABLE", "crop is not a readable image")
submitted_at = time.perf_counter()
track_id_out = record["track_id"]
generation = record["generation"]
method = record["method"]
object_generation = record["object_generation"]
try:
future = self._executor.submit(self.engine.score_crop, crop, method)
except RuntimeError: # executor already shut down
with self._lock:
self._release_crop_slot(record)
self._crops_rejected += 1
return _crop_error("CROP_UNAVAILABLE", "classifier is shutting down")
with self._lock:
self._crops_accepted += 1
future.add_done_callback(
lambda fut, tid=track_id_out, gen=generation, m=method, started=submitted_at,
obj=object_generation, box=box: self._on_classified(
fut, tid, gen, m, started, obj, box, SOURCE_CROP
)
)
return {
"type": "crop_ack",
"request_id": record["request_id"],
"track_id": track_id_out,
"frame_id": record["frame_id"],
}
def _prepare_crop(
self, request_id: Any, track_id: Any, frame_id: Any, payload: Optional[bytes]
) -> Tuple[Dict[str, Any], List[int], bytes]:
"""Check a returned crop against the request it claims to answer.
Raises [_CropRejected] with the reply to send. The request is consumed
either way: it is single use, so a rejected crop cannot be retried
against a track that has since moved.
"""
with self._lock:
record = self._crop_requests.pop(str(request_id), None) if request_id else None
if record is None:
raise _CropRejected("CROP_UNKNOWN_REQUEST", "no crop is waiting for that request id")
try:
answered_track = int(track_id)
answered_frame = int(frame_id)
except (TypeError, ValueError):
self._release_crop_slot(record)
raise _CropRejected("CROP_BAD_MESSAGE", "track_id and frame_id are required")
if answered_track != record["track_id"] or answered_frame != record["frame_id"]:
# A crop of some other track, or of another frame, says nothing
# about this one however well it matches.
self._release_crop_slot(record)
raise _CropRejected("CROP_MISMATCH", "crop does not match the request it answers")
if self._seq - record["seq"] > self.settings.crop_ttl_frames:
self._release_crop_slot(record)
raise _CropRejected("CROP_STALE", "crop answered too late to describe the track")
if record["generation"] != self._generation:
self._release_crop_slot(record)
raise _CropRejected("CROP_STALE", "the session changed after the request")
track = self._tracks.get(record["track_id"])
if track is None or track.object_generation != record["object_generation"]:
self._release_crop_slot(record)
raise _CropRejected("CROP_STALE", "the track id was recycled after the request")
# The sender keeps streaming while the requested crop is encoded.
# A newer frame is normal; object generation and the bounded frame
# TTL, not exact latest-sequence equality, guard against reuse.
if track.last_seq < record["seq"]:
self._release_crop_slot(record)
raise _CropRejected("CROP_STALE", "the track is no longer in the requested frame")
limit = self.settings.max_crop_bytes
if not payload or len(payload) > limit:
self._release_crop_slot(record)
raise _CropRejected("CROP_TOO_LARGE", "crop is empty or over %d bytes" % limit)
return record, list(track.box), payload
def _on_classified(
self,
future: Future,
@ -699,6 +959,7 @@ class LiveSession:
submitted_at: float,
object_generation: int,
submitted_box: List[int],
source: str = SOURCE_FRAME,
) -> None:
try:
scores = future.result()
@ -715,6 +976,9 @@ class LiveSession:
return
track.in_flight = max(0, track.in_flight - 1)
track.attempts += 1
# A crop is an attempt against the same budget as a frame crop, so
# the lane cannot be used to re-roll the dice on one product.
track.last_source = source
observation = {
str(sku_id): float(score) for sku_id, score in (scores or {}).items()
}
@ -788,6 +1052,10 @@ class LiveSession:
"candidates": list(track.candidates),
"attempts": track.attempts,
"method": self.settings.method,
# Which evidence produced the newest observation. A client that only
# reads this can tell a label measured on the frame it sent from one
# measured on a crop it sent back.
"source": track.last_source,
}
product = None
if track.state == STATE_RECOGNIZED and track.identity:
@ -865,6 +1133,11 @@ class LiveSession:
if self._classify_times:
classify_p50 = round(statistics.median(self._classify_times) * 1000.0, 2)
return {
"crops": {
"requested": self._crops_requested,
"accepted": self._crops_accepted,
"rejected": self._crops_rejected,
},
"frames_received": self._frames_received,
"frames_processed": self._frames_processed,
"frames_dropped": self._frames_dropped,