feat: a shelf map, so a facing outlives the track id that found it
The camera moves and the shelf does not, but every measurement so far treats a frame as the world: a facing is "a track id", and that id dies the moment the camera pans away and comes back. Label inheritance, drift checks and restart counters were each correct fixes for that, and all of them are symptoms. `shelf_map.py` keeps a per-session map of the shelf plane. Frames are registered by their content shift, detections are placed in shelf coordinates, and a detection that lands where a facing already is merges into it - so identity votes and tag readings pool on the physical facing. Measured, and reported honestly: - Registration is exact on the synthetic walk: 0.01 px median error against the demo camera's own offsets, over 23 consecutive pairs, with both phase correlation and ORB + RANSAC. That validates the harness, not the method - the walk is a pure translation of one photograph. Real video needs step 1's footage. - Pooling works: a facing seen 15 times across 2 track ids reports one identity at 1.00 agreement. - De-duplication is NOT yet better than the tracker: 179 facings against 156 track ids, where the shelf holds roughly 70. Both over-count, the map slightly more. Two bugs were found by measuring, and both were silent: the content shift is the negation of the camera's motion (getting it backwards turned ~60 facings into 857), and matching purely by box overlap lets jitter split a facing (210 facings, 103 of them seen once; matching by track continuity first brought it to 179 with 38). 175 tests pass. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a718acdc15
commit
b451e7096f
3 changed files with 550 additions and 0 deletions
77
recognization/code/bench_shelf_map.py
Normal file
77
recognization/code/bench_shelf_map.py
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
"""
|
||||||
|
What does a shelf map buy over track ids?
|
||||||
|
|
||||||
|
Walks the demo camera once, feeds every frame through both the live session and a
|
||||||
|
ShelfMap, and reports what each counts. Needs a running server on :8077.
|
||||||
|
|
||||||
|
STEPS=58 python code/bench_shelf_map.py
|
||||||
|
"""
|
||||||
|
import json, sys, time, urllib.request
|
||||||
|
sys.path.insert(0, "code")
|
||||||
|
import cv2, numpy as np
|
||||||
|
from websockets.sync.client import connect
|
||||||
|
from shelf_map import ShelfMap
|
||||||
|
|
||||||
|
BASE = "http://127.0.0.1:8077"
|
||||||
|
STEPS = int(__import__("os").environ.get("STEPS", "58")) # one walk over image6 at pan 0.02
|
||||||
|
|
||||||
|
def fetch(step):
|
||||||
|
url = f"{BASE}/v1/demo/frame?samples_param=image6&step={step}&width=1280&height=720&quality=70&pan=0.02"
|
||||||
|
return urllib.request.urlopen(url, timeout=10).read()
|
||||||
|
|
||||||
|
shelf = ShelfMap()
|
||||||
|
track_ids, frames, detections = set(), 0, 0
|
||||||
|
started = time.perf_counter()
|
||||||
|
with connect("ws://127.0.0.1:8077/v1/stream") as ws:
|
||||||
|
ws.recv(timeout=15)
|
||||||
|
for step in range(STEPS):
|
||||||
|
payload = fetch(step)
|
||||||
|
frame = cv2.imdecode(np.frombuffer(payload, np.uint8), cv2.IMREAD_COLOR)
|
||||||
|
ws.send(step.to_bytes(4, "big") + payload)
|
||||||
|
msg = json.loads(ws.recv(timeout=25))
|
||||||
|
if msg.get("type") != "result":
|
||||||
|
continue
|
||||||
|
frames += 1
|
||||||
|
tracks = msg["tracks"]
|
||||||
|
detections += len(tracks)
|
||||||
|
for track in tracks:
|
||||||
|
track_ids.add(track["track_id"])
|
||||||
|
shelf.update(step, frame, tracks)
|
||||||
|
time.sleep(0.05)
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
|
||||||
|
stats = shelf.stats()
|
||||||
|
print(f"{frames} frames in {elapsed:.1f} s, {detections} detections")
|
||||||
|
print(f" track ids seen (what the live session counts): {len(track_ids)}")
|
||||||
|
print(f" facings on the shelf map (what is actually there): {stats['facings']}")
|
||||||
|
print(f" merges {stats['merges']}, registrations {stats['registrations']}, failures {stats['registration_failures']}")
|
||||||
|
print(f" median registration response {stats['median_response']}")
|
||||||
|
print(f" facings whose pooled identity agrees with itself >= 0.8: {stats['identity_consistent']}")
|
||||||
|
|
||||||
|
facings = [f for f in shelf.facings() if f.observations >= 3]
|
||||||
|
print(f"\nfacings seen at least 3 times: {len(facings)}")
|
||||||
|
for facing in sorted(facings, key=lambda f: (f.shelf_box[1], f.shelf_box[0]))[:10]:
|
||||||
|
sku, agreement = facing.best_identity()
|
||||||
|
tag_sku, tag_agreement = facing.best_tag_identity()
|
||||||
|
print(f" shelf x={facing.shelf_box[0]:>5} y={facing.shelf_box[1]:>5} obs={facing.observations:>2} "
|
||||||
|
f"tracks={len(facing.seen_track_ids):>2} identity={sku} ({agreement:.2f}) tag={tag_sku} ({tag_agreement:.2f})")
|
||||||
|
|
||||||
|
# render the map so duplicates are visible rather than inferred
|
||||||
|
import numpy as np
|
||||||
|
facings = shelf.facings()
|
||||||
|
xs = [f.shelf_box[0] for f in facings] + [f.shelf_box[2] for f in facings]
|
||||||
|
ys = [f.shelf_box[1] for f in facings] + [f.shelf_box[3] for f in facings]
|
||||||
|
x0, x1 = min(xs), max(xs)
|
||||||
|
y0, y1 = min(ys), max(ys)
|
||||||
|
scale = 0.6
|
||||||
|
canvas = np.full((int((y1 - y0) * scale) + 20, int((x1 - x0) * scale) + 20, 3), 20, np.uint8)
|
||||||
|
for facing in facings:
|
||||||
|
bx = [(facing.shelf_box[0] - x0) * scale + 10, (facing.shelf_box[1] - y0) * scale + 10,
|
||||||
|
(facing.shelf_box[2] - x0) * scale + 10, (facing.shelf_box[3] - y0) * scale + 10]
|
||||||
|
colour = (0, 200, 0) if facing.observations >= 5 else (0, 120, 255)
|
||||||
|
cv2.rectangle(canvas, (int(bx[0]), int(bx[1])), (int(bx[2]), int(bx[3])), colour, 1)
|
||||||
|
cv2.imwrite("/tmp/dorayaki-verify/shelf-map.png", canvas)
|
||||||
|
print(f"\nmap canvas {canvas.shape[1]}x{canvas.shape[0]} px covering shelf x {x0}..{x1}, y {y0}..{y1}")
|
||||||
|
from collections import Counter
|
||||||
|
print("observations per facing:", Counter(min(f.observations, 20) for f in facings).most_common(6))
|
||||||
|
print("facings per y band (100 px):", Counter((f.shelf_box[1] - y0) // 100 for f in facings).most_common(8))
|
||||||
348
recognization/code/shelf_map.py
Normal file
348
recognization/code/shelf_map.py
Normal file
|
|
@ -0,0 +1,348 @@
|
||||||
|
"""
|
||||||
|
The shelf as a plane, not a stream of boxes.
|
||||||
|
|
||||||
|
Every measurement so far treats a frame as the world: a facing is "a track id",
|
||||||
|
and a track id dies the moment the camera pans past and comes back. Label
|
||||||
|
inheritance, drift checks and restart counters were each correct fixes for that,
|
||||||
|
and all of them are symptoms. The shelf does not move; the camera does.
|
||||||
|
|
||||||
|
This module keeps a per-session **shelf map**. Each frame is registered to the map
|
||||||
|
by its content shift, every detection is placed in shelf coordinates, and a
|
||||||
|
detection that lands where a facing already is *merges* into it. Identity votes and
|
||||||
|
tag readings accumulate on the physical facing rather than on a transient track.
|
||||||
|
|
||||||
|
That is what the current literature does for retail: RetailGlue (CVPR 2026W) builds
|
||||||
|
shelf panoramas by matching products across frames and reports 88.4 % stitching F1,
|
||||||
|
and the 7-Eleven Taiwan system (Sci. Rep. 2025, 7,000+ stores) stitches "virtual
|
||||||
|
shelves" before checking planograms.
|
||||||
|
|
||||||
|
**What is measured, and what is not.** The synthetic walk registers to within
|
||||||
|
0.01 px of its ground truth - but it is a pure translation of a single photograph,
|
||||||
|
so that validates the harness, not handheld video. The number that matters (real
|
||||||
|
registration error, and the duplicate rate that follows from it) needs real
|
||||||
|
footage, which `record.py` exists to collect.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||||
|
|
||||||
|
# A frame whose content shift cannot be estimated this confidently is treated as
|
||||||
|
# discontinuous: the map starts a new section rather than guessing a position.
|
||||||
|
MIN_REGISTRATION_RESPONSE = 0.20
|
||||||
|
# Two detections are the same physical facing when their shelf boxes overlap this
|
||||||
|
# much. Measured: adjacent facings of one product overlap ~0 by construction.
|
||||||
|
MERGE_IOU = 0.45
|
||||||
|
|
||||||
|
|
||||||
|
def _iou(a: Sequence[int], b: Sequence[int]) -> float:
|
||||||
|
x1, y1 = max(a[0], b[0]), max(a[1], b[1])
|
||||||
|
x2, y2 = min(a[2], b[2]), min(a[3], b[3])
|
||||||
|
intersection = max(0, x2 - x1) * max(0, y2 - y1)
|
||||||
|
area_a = max(0, a[2] - a[0]) * max(0, a[3] - a[1])
|
||||||
|
area_b = max(0, b[2] - b[0]) * max(0, b[3] - b[1])
|
||||||
|
return intersection / max(1.0, float(area_a + area_b - intersection))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Facing:
|
||||||
|
"""One physical position on the shelf, with everything observed there."""
|
||||||
|
|
||||||
|
facing_id: str
|
||||||
|
shelf_box: List[int]
|
||||||
|
row: int = 0
|
||||||
|
observations: int = 0
|
||||||
|
first_seq: int = 0
|
||||||
|
last_seq: int = 0
|
||||||
|
identity_votes: Dict[str, float] = field(default_factory=dict)
|
||||||
|
tag_votes: Dict[str, float] = field(default_factory=dict)
|
||||||
|
tag_readings: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
seen_track_ids: set = field(default_factory=set)
|
||||||
|
|
||||||
|
def vote_identity(self, sku_id: Optional[str], weight: float = 1.0) -> None:
|
||||||
|
if sku_id:
|
||||||
|
self.identity_votes[sku_id] = self.identity_votes.get(sku_id, 0.0) + weight
|
||||||
|
|
||||||
|
def vote_tag(self, sku_id: Optional[str], reading: Optional[Dict[str, Any]] = None) -> None:
|
||||||
|
if sku_id:
|
||||||
|
self.tag_votes[sku_id] = self.tag_votes.get(sku_id, 0.0) + 1.0
|
||||||
|
if reading:
|
||||||
|
self.tag_readings.append(reading)
|
||||||
|
|
||||||
|
def best_identity(self) -> Tuple[Optional[str], float]:
|
||||||
|
"""The identity the facing's pooled evidence supports, and its agreement."""
|
||||||
|
if not self.identity_votes:
|
||||||
|
return None, 0.0
|
||||||
|
sku_id, weight = max(self.identity_votes.items(), key=lambda item: item[1])
|
||||||
|
total = sum(self.identity_votes.values())
|
||||||
|
return sku_id, weight / max(1e-6, total)
|
||||||
|
|
||||||
|
def best_tag_identity(self) -> Tuple[Optional[str], float]:
|
||||||
|
if not self.tag_votes:
|
||||||
|
return None, 0.0
|
||||||
|
sku_id, weight = max(self.tag_votes.items(), key=lambda item: item[1])
|
||||||
|
total = sum(self.tag_votes.values())
|
||||||
|
return sku_id, weight / max(1e-6, total)
|
||||||
|
|
||||||
|
def as_payload(self) -> Dict[str, Any]:
|
||||||
|
identity, agreement = self.best_identity()
|
||||||
|
tag_identity, tag_agreement = self.best_tag_identity()
|
||||||
|
return {
|
||||||
|
"facing_id": self.facing_id,
|
||||||
|
"box": list(self.shelf_box),
|
||||||
|
"row": self.row,
|
||||||
|
"observations": self.observations,
|
||||||
|
"track_ids": len(self.seen_track_ids),
|
||||||
|
"sku_id": identity,
|
||||||
|
"identity_agreement": round(agreement, 3),
|
||||||
|
"tag_sku_id": tag_identity,
|
||||||
|
"tag_agreement": round(tag_agreement, 3),
|
||||||
|
"tag_readings": self.tag_readings[-3:],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ShelfMap:
|
||||||
|
"""Registers frames and accumulates facings in shelf coordinates."""
|
||||||
|
|
||||||
|
def __init__(self, settings: Optional[Any] = None) -> None:
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._prev_gray: Optional[Any] = None
|
||||||
|
self._offset: List[float] = [0.0, 0.0] # shelf position of the frame origin
|
||||||
|
self._facings: List[Facing] = []
|
||||||
|
self._orb = None
|
||||||
|
self.registrations = 0
|
||||||
|
self.registration_failures = 0
|
||||||
|
self.anchors = 0
|
||||||
|
self.continuations = 0
|
||||||
|
self.detections = 0
|
||||||
|
self.merges = 0
|
||||||
|
self.responses: List[float] = []
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- registration
|
||||||
|
def _gray(self, frame: Any, scale: float = 0.5) -> Any:
|
||||||
|
import cv2
|
||||||
|
|
||||||
|
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||||
|
if scale != 1.0:
|
||||||
|
gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA)
|
||||||
|
return gray
|
||||||
|
|
||||||
|
def register(self, frame: Any) -> Optional[Tuple[float, float]]:
|
||||||
|
"""Content shift from the previous frame, in full-resolution pixels.
|
||||||
|
|
||||||
|
Phase correlation first: it is the right tool for a translating scene and
|
||||||
|
costs a couple of milliseconds. ORB + RANSAC only when the correlation is
|
||||||
|
weak (blur, a large jump, a scene with no texture), because that is the case
|
||||||
|
where a fast estimator quietly returns nonsense.
|
||||||
|
"""
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
gray = self._gray(frame)
|
||||||
|
with self._lock:
|
||||||
|
previous = self._prev_gray
|
||||||
|
self._prev_gray = gray
|
||||||
|
if previous is None or previous.shape != gray.shape:
|
||||||
|
return (0.0, 0.0)
|
||||||
|
window = cv2.createHanningWindow((gray.shape[1], gray.shape[0]), cv2.CV_32F)
|
||||||
|
(dx, dy), response = cv2.phaseCorrelate(previous.astype(np.float32), gray.astype(np.float32), window)
|
||||||
|
self.responses.append(float(response))
|
||||||
|
if response >= MIN_REGISTRATION_RESPONSE:
|
||||||
|
self.registrations += 1
|
||||||
|
return (dx / 0.5, dy / 0.5) # back to full resolution
|
||||||
|
|
||||||
|
shift = self._orb_shift(previous, gray)
|
||||||
|
if shift is None:
|
||||||
|
self.registration_failures += 1
|
||||||
|
return None
|
||||||
|
self.registrations += 1
|
||||||
|
return (shift[0] / 0.5, shift[1] / 0.5)
|
||||||
|
|
||||||
|
def _orb_shift(self, previous: Any, current: Any) -> Optional[Tuple[float, float]]:
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
if self._orb is None:
|
||||||
|
self._orb = cv2.ORB_create(nfeatures=1500, fastThreshold=10)
|
||||||
|
key_a, desc_a = self._orb.detectAndCompute(previous, None)
|
||||||
|
key_b, desc_b = self._orb.detectAndCompute(current, None)
|
||||||
|
if desc_a is None or desc_b is None or len(key_a) < 20 or len(key_b) < 20:
|
||||||
|
return None
|
||||||
|
matches = cv2.BFMatcher(cv2.NORM_HAMMING).knnMatch(desc_a, desc_b, k=2)
|
||||||
|
good = [m for m, n in matches if m.distance < 0.75 * n.distance]
|
||||||
|
if len(good) < 12:
|
||||||
|
return None
|
||||||
|
src = np.float32([key_a[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
|
||||||
|
dst = np.float32([key_b[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
|
||||||
|
matrix, _ = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=3.0)
|
||||||
|
if matrix is None:
|
||||||
|
return None
|
||||||
|
return (float(matrix[0, 2]), float(matrix[1, 2]))
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ facings
|
||||||
|
def _anchor_to_map(
|
||||||
|
self, detections: Sequence[Sequence[int]], offset_x: float, offset_y: float
|
||||||
|
) -> Tuple[float, float]:
|
||||||
|
"""Correct the chained offset by matching detections against the map.
|
||||||
|
|
||||||
|
Chaining pairwise shifts drifts: measured over a 79-step walk with *perfect*
|
||||||
|
pairwise registration, the accumulated x offset reached 2047 px on a 1500 px
|
||||||
|
shelf, the same facing landed in two places, and a shelf of ~60 facings
|
||||||
|
became 334 with 113 of them seen once. Drift is inherent to chaining - the
|
||||||
|
fix is to anchor the frame to the structure already on the map, which is
|
||||||
|
what the retail-stitching literature does with product correspondences.
|
||||||
|
|
||||||
|
One ICP-style step is enough: for every detection that has a facing nearby,
|
||||||
|
the correction it implies is a vote, and the median vote wins.
|
||||||
|
"""
|
||||||
|
if not self._facings or len(detections) < 4:
|
||||||
|
return offset_x, offset_y
|
||||||
|
corrections = []
|
||||||
|
search = 40.0
|
||||||
|
for box in detections:
|
||||||
|
centre_x = (box[0] + box[2]) / 2.0 + offset_x
|
||||||
|
centre_y = (box[1] + box[3]) / 2.0 + offset_y
|
||||||
|
best, best_distance = None, search
|
||||||
|
for facing in self._facings:
|
||||||
|
facing_x = (facing.shelf_box[0] + facing.shelf_box[2]) / 2.0
|
||||||
|
facing_y = (facing.shelf_box[1] + facing.shelf_box[3]) / 2.0
|
||||||
|
distance = max(abs(facing_x - centre_x), abs(facing_y - centre_y))
|
||||||
|
if distance < best_distance:
|
||||||
|
best, best_distance = facing, distance
|
||||||
|
if best is not None:
|
||||||
|
corrections.append(
|
||||||
|
(
|
||||||
|
(best.shelf_box[0] + best.shelf_box[2]) / 2.0 - ((box[0] + box[2]) / 2.0 + offset_x),
|
||||||
|
(best.shelf_box[1] + best.shelf_box[3]) / 2.0 - ((box[1] + box[3]) / 2.0 + offset_y),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(corrections) < 4:
|
||||||
|
return offset_x, offset_y
|
||||||
|
dx = sorted(c[0] for c in corrections)[len(corrections) // 2]
|
||||||
|
dy = sorted(c[1] for c in corrections)[len(corrections) // 2]
|
||||||
|
self.anchors += 1
|
||||||
|
return offset_x + dx, offset_y + dy
|
||||||
|
|
||||||
|
def update(self, seq: int, frame: Any, tracks: Sequence[Dict[str, Any]]) -> None:
|
||||||
|
"""Register the frame, then place or merge every detection on the shelf."""
|
||||||
|
shift = self.register(frame)
|
||||||
|
boxes = [
|
||||||
|
[int(v) for v in (track.get("box") or track.get("raw_box"))]
|
||||||
|
for track in tracks
|
||||||
|
if (track.get("box") or track.get("raw_box")) and len(track.get("box") or track.get("raw_box")) == 4
|
||||||
|
]
|
||||||
|
with self._lock:
|
||||||
|
if shift is None:
|
||||||
|
# A discontinuous frame: keep the last known offset rather than
|
||||||
|
# inventing one, so its detections land where the shelf is.
|
||||||
|
shift = (0.0, 0.0)
|
||||||
|
# `shift` is how far the CONTENT moved in the frame, so the frame's
|
||||||
|
# origin moves the opposite way along the shelf. Adding it instead
|
||||||
|
# (measured) walks the map backwards: the same facing lands 50 px away
|
||||||
|
# in the next frame and a shelf of ~60 facings becomes 857.
|
||||||
|
self._offset[0] -= shift[0]
|
||||||
|
self._offset[1] -= shift[1]
|
||||||
|
if self._facings:
|
||||||
|
self._offset[0], self._offset[1] = self._anchor_to_map(
|
||||||
|
boxes, self._offset[0], self._offset[1]
|
||||||
|
)
|
||||||
|
offset_x, offset_y = self._offset
|
||||||
|
for track in tracks:
|
||||||
|
box = track.get("box") or track.get("raw_box")
|
||||||
|
if not box or len(box) != 4:
|
||||||
|
continue
|
||||||
|
self.detections += 1
|
||||||
|
shelf_box = [
|
||||||
|
int(round(box[0] + offset_x)), int(round(box[1] + offset_y)),
|
||||||
|
int(round(box[2] + offset_x)), int(round(box[3] + offset_y)),
|
||||||
|
]
|
||||||
|
track_id = track.get("track_id")
|
||||||
|
facing = self._find(shelf_box, int(track_id) if track_id is not None else None)
|
||||||
|
if facing is None:
|
||||||
|
facing = Facing(
|
||||||
|
facing_id=f"face_{uuid.uuid4().hex[:8]}",
|
||||||
|
shelf_box=shelf_box,
|
||||||
|
first_seq=seq,
|
||||||
|
)
|
||||||
|
self._facings.append(facing)
|
||||||
|
else:
|
||||||
|
self.merges += 1
|
||||||
|
# Average the box: repeated observations of a static object
|
||||||
|
# should converge, and the average is what the eye expects.
|
||||||
|
for index in range(4):
|
||||||
|
facing.shelf_box[index] = int(round((facing.shelf_box[index] * facing.observations + shelf_box[index]) / (facing.observations + 1)))
|
||||||
|
facing.observations += 1
|
||||||
|
facing.last_seq = seq
|
||||||
|
identity = track.get("identity") or {}
|
||||||
|
if identity.get("state") == "recognized":
|
||||||
|
facing.vote_identity(identity.get("sku_id"), float(identity.get("score") or 1.0))
|
||||||
|
tag_identity = track.get("tag_identity") or {}
|
||||||
|
tag = track.get("price_tag") or {}
|
||||||
|
if tag_identity.get("sku_id") or tag.get("state") == "readable":
|
||||||
|
facing.vote_tag(tag_identity.get("sku_id"), tag if tag else None)
|
||||||
|
if track.get("track_id") is not None:
|
||||||
|
facing.seen_track_ids.add(int(track["track_id"]))
|
||||||
|
|
||||||
|
def _find(self, shelf_box: Sequence[int], track_id: Optional[int] = None) -> Optional[Facing]:
|
||||||
|
"""The facing a detection belongs to.
|
||||||
|
|
||||||
|
Continuity first: if this track id was already placed nearby, it is the same
|
||||||
|
facing, and no overlap test is needed - the tracker already decided that.
|
||||||
|
Geometry only when the id is new (a pan that outran the tracker, or the
|
||||||
|
tracker renumbering). Measured: with geometry alone, box jitter split the
|
||||||
|
same facing often enough that the map held 210 facings where the tracker
|
||||||
|
held 156 ids - it was adding duplicates, not removing them.
|
||||||
|
"""
|
||||||
|
width = max(1, shelf_box[2] - shelf_box[0])
|
||||||
|
height = max(1, shelf_box[3] - shelf_box[1])
|
||||||
|
if track_id is not None:
|
||||||
|
radius = 0.75 * max(width, height)
|
||||||
|
best, best_distance = None, radius
|
||||||
|
for facing in self._facings:
|
||||||
|
if track_id not in facing.seen_track_ids:
|
||||||
|
continue
|
||||||
|
distance = max(
|
||||||
|
abs((facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 - (shelf_box[0] + shelf_box[2]) / 2.0),
|
||||||
|
abs((facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 - (shelf_box[1] + shelf_box[3]) / 2.0),
|
||||||
|
)
|
||||||
|
if distance < best_distance:
|
||||||
|
best, best_distance = facing, distance
|
||||||
|
if best is not None:
|
||||||
|
self.continuations += 1
|
||||||
|
return best
|
||||||
|
best, best_overlap = None, MERGE_IOU
|
||||||
|
for facing in self._facings:
|
||||||
|
overlap = _iou(shelf_box, facing.shelf_box)
|
||||||
|
if overlap > best_overlap:
|
||||||
|
best, best_overlap = facing, overlap
|
||||||
|
return best
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ reporting
|
||||||
|
def facings(self) -> List[Facing]:
|
||||||
|
with self._lock:
|
||||||
|
return list(self._facings)
|
||||||
|
|
||||||
|
def stats(self) -> Dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
facings = list(self._facings)
|
||||||
|
responses = list(self.responses[-200:])
|
||||||
|
consistent = 0
|
||||||
|
for facing in facings:
|
||||||
|
sku_id, agreement = facing.best_identity()
|
||||||
|
if sku_id and agreement >= 0.8:
|
||||||
|
consistent += 1
|
||||||
|
return {
|
||||||
|
"facings": len(facings),
|
||||||
|
"detections": self.detections,
|
||||||
|
"merges": self.merges,
|
||||||
|
"continuations": self.continuations,
|
||||||
|
"registrations": self.registrations,
|
||||||
|
"anchored": self.anchors,
|
||||||
|
"registration_failures": self.registration_failures,
|
||||||
|
"median_response": round(sorted(responses)[len(responses) // 2], 3) if responses else None,
|
||||||
|
"identity_consistent": consistent,
|
||||||
|
"offset": [round(value, 1) for value in self._offset],
|
||||||
|
}
|
||||||
125
recognization/code/test_shelf_map.py
Normal file
125
recognization/code/test_shelf_map.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
"""
|
||||||
|
Tests for the shelf map: registration, merging, and evidence pooling.
|
||||||
|
|
||||||
|
The sign convention is tested explicitly, because getting it backwards is silent:
|
||||||
|
the map still produces boxes, they just land in the wrong place, and a shelf of
|
||||||
|
~60 facings becomes 857. That happened during development.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
project_root = os.path.dirname(current_dir)
|
||||||
|
if os.path.exists(os.path.join(project_root, "stock")):
|
||||||
|
os.chdir(project_root)
|
||||||
|
if current_dir not in sys.path:
|
||||||
|
sys.path.insert(0, current_dir)
|
||||||
|
|
||||||
|
from shelf_map import Facing, ShelfMap, _iou # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def a_frame(shift_x=0, shift_y=0, size=(160, 240)):
|
||||||
|
"""A textured frame, optionally a shifted copy of another."""
|
||||||
|
height, width = size
|
||||||
|
rng = np.random.RandomState(7)
|
||||||
|
base = rng.randint(0, 255, (height * 2, width * 2, 3), dtype=np.uint8)
|
||||||
|
return base[height - shift_y:height * 2 - shift_y, width - shift_x:width * 2 - shift_x]
|
||||||
|
|
||||||
|
|
||||||
|
def a_track(track_id, box, sku=None, score=None, tag_sku=None):
|
||||||
|
track = {"track_id": track_id, "box": list(box)}
|
||||||
|
if sku:
|
||||||
|
track["identity"] = {"state": "recognized", "sku_id": sku, "score": score or 1.0}
|
||||||
|
else:
|
||||||
|
track["identity"] = {"state": "unconfirmed"}
|
||||||
|
if tag_sku:
|
||||||
|
track["tag_identity"] = {"sku_id": tag_sku, "score": 0.7}
|
||||||
|
return track
|
||||||
|
|
||||||
|
|
||||||
|
class IouTests(unittest.TestCase):
|
||||||
|
def test_overlap_maths(self):
|
||||||
|
self.assertAlmostEqual(_iou([0, 0, 10, 10], [0, 0, 10, 10]), 1.0)
|
||||||
|
self.assertEqual(_iou([0, 0, 10, 10], [20, 20, 30, 30]), 0.0)
|
||||||
|
|
||||||
|
|
||||||
|
class RegistrationTests(unittest.TestCase):
|
||||||
|
def test_the_frame_origin_moves_opposite_to_the_content(self):
|
||||||
|
"""The sign that matters: content moving left means the camera moved right.
|
||||||
|
|
||||||
|
Getting this backwards walks the map the wrong way. It is silent - boxes
|
||||||
|
still appear - and it turned a shelf of ~60 facings into 857.
|
||||||
|
"""
|
||||||
|
shelf = ShelfMap()
|
||||||
|
shelf.register(a_frame())
|
||||||
|
shift = shelf.register(a_frame(shift_x=10)) # content moved right 10
|
||||||
|
self.assertIsNotNone(shift)
|
||||||
|
self.assertAlmostEqual(shift[0], 10, delta=1.0)
|
||||||
|
|
||||||
|
# Content moving right means the camera moved LEFT, so the frame's origin
|
||||||
|
# sits 10 px further back along the shelf.
|
||||||
|
shelf.update(0, a_frame(), [a_track(1, [0, 0, 20, 20])])
|
||||||
|
offset_before = list(shelf._offset)
|
||||||
|
shelf.update(1, a_frame(shift_x=10), [a_track(2, [0, 0, 20, 20])])
|
||||||
|
self.assertLess(shelf._offset[0], offset_before[0] + 1,
|
||||||
|
"the offset must follow the camera, not the content")
|
||||||
|
|
||||||
|
def test_a_first_frame_has_no_shift(self):
|
||||||
|
shelf = ShelfMap()
|
||||||
|
self.assertEqual(shelf.register(a_frame()), (0.0, 0.0))
|
||||||
|
|
||||||
|
|
||||||
|
class MergingTests(unittest.TestCase):
|
||||||
|
def test_a_facing_seen_twice_is_one_facing(self):
|
||||||
|
shelf = ShelfMap()
|
||||||
|
shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200])])
|
||||||
|
shelf.update(1, a_frame(), [a_track(2, [12, 11, 112, 201])])
|
||||||
|
self.assertEqual(len(shelf.facings()), 1)
|
||||||
|
self.assertEqual(shelf.stats()["merges"], 1)
|
||||||
|
|
||||||
|
def test_two_neighbouring_facings_stay_separate(self):
|
||||||
|
shelf = ShelfMap()
|
||||||
|
shelf.update(0, a_frame(), [a_track(1, [10, 10, 100, 200]), a_track(2, [110, 10, 200, 200])])
|
||||||
|
self.assertEqual(len(shelf.facings()), 2)
|
||||||
|
|
||||||
|
def test_identity_votes_pool_across_track_ids(self):
|
||||||
|
"""The point of the map: one facing, many transient track ids, one answer."""
|
||||||
|
shelf = ShelfMap()
|
||||||
|
for index, (track_id, sku) in enumerate([(1, "sku_a"), (2, "sku_a"), (3, "sku_a"), (4, "sku_b")]):
|
||||||
|
shelf.update(index, a_frame(), [a_track(track_id, [10, 10, 110, 200], sku=sku)])
|
||||||
|
facing = shelf.facings()[0]
|
||||||
|
self.assertEqual(facing.observations, 4)
|
||||||
|
self.assertEqual(len(facing.seen_track_ids), 4, "four tracks, one facing")
|
||||||
|
sku_id, agreement = facing.best_identity()
|
||||||
|
self.assertEqual(sku_id, "sku_a")
|
||||||
|
self.assertAlmostEqual(agreement, 0.75)
|
||||||
|
|
||||||
|
def test_tag_evidence_pools_separately_from_packaging(self):
|
||||||
|
shelf = ShelfMap()
|
||||||
|
shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200], sku="sku_a", tag_sku="sku_a")])
|
||||||
|
shelf.update(1, a_frame(), [a_track(2, [10, 10, 110, 200], tag_sku="sku_a")])
|
||||||
|
facing = shelf.facings()[0]
|
||||||
|
self.assertEqual(facing.best_identity()[0], "sku_a")
|
||||||
|
self.assertEqual(facing.best_tag_identity()[0], "sku_a")
|
||||||
|
self.assertEqual(len(facing.tag_readings), 0, "no readable tag payload was attached")
|
||||||
|
|
||||||
|
|
||||||
|
class PayloadTests(unittest.TestCase):
|
||||||
|
def test_a_facing_reports_its_pooled_evidence(self):
|
||||||
|
facing = Facing(facing_id="face_1", shelf_box=[0, 0, 10, 10])
|
||||||
|
facing.vote_identity("sku_a", 2.0)
|
||||||
|
facing.vote_identity("sku_b", 1.0)
|
||||||
|
payload = facing.as_payload()
|
||||||
|
self.assertEqual(payload["sku_id"], "sku_a")
|
||||||
|
self.assertAlmostEqual(payload["identity_agreement"], 2 / 3, places=3)
|
||||||
|
self.assertIsNone(payload["tag_sku_id"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Add table
Add a link
Reference in a new issue