diff --git a/recognization/code/bench_shelf_map.py b/recognization/code/bench_shelf_map.py new file mode 100644 index 0000000..3988b64 --- /dev/null +++ b/recognization/code/bench_shelf_map.py @@ -0,0 +1,77 @@ +""" +What does a shelf map buy over track ids? + +Walks the demo camera once, feeds every frame through both the live session and a +ShelfMap, and reports what each counts. Needs a running server on :8077. + + STEPS=58 python code/bench_shelf_map.py +""" +import json, sys, time, urllib.request +sys.path.insert(0, "code") +import cv2, numpy as np +from websockets.sync.client import connect +from shelf_map import ShelfMap + +BASE = "http://127.0.0.1:8077" +STEPS = int(__import__("os").environ.get("STEPS", "58")) # one walk over image6 at pan 0.02 + +def fetch(step): + url = f"{BASE}/v1/demo/frame?samples_param=image6&step={step}&width=1280&height=720&quality=70&pan=0.02" + return urllib.request.urlopen(url, timeout=10).read() + +shelf = ShelfMap() +track_ids, frames, detections = set(), 0, 0 +started = time.perf_counter() +with connect("ws://127.0.0.1:8077/v1/stream") as ws: + ws.recv(timeout=15) + for step in range(STEPS): + payload = fetch(step) + frame = cv2.imdecode(np.frombuffer(payload, np.uint8), cv2.IMREAD_COLOR) + ws.send(step.to_bytes(4, "big") + payload) + msg = json.loads(ws.recv(timeout=25)) + if msg.get("type") != "result": + continue + frames += 1 + tracks = msg["tracks"] + detections += len(tracks) + for track in tracks: + track_ids.add(track["track_id"]) + shelf.update(step, frame, tracks) + time.sleep(0.05) +elapsed = time.perf_counter() - started + +stats = shelf.stats() +print(f"{frames} frames in {elapsed:.1f} s, {detections} detections") +print(f" track ids seen (what the live session counts): {len(track_ids)}") +print(f" facings on the shelf map (what is actually there): {stats['facings']}") +print(f" merges {stats['merges']}, registrations {stats['registrations']}, failures {stats['registration_failures']}") +print(f" median registration response {stats['median_response']}") +print(f" facings whose pooled identity agrees with itself >= 0.8: {stats['identity_consistent']}") + +facings = [f for f in shelf.facings() if f.observations >= 3] +print(f"\nfacings seen at least 3 times: {len(facings)}") +for facing in sorted(facings, key=lambda f: (f.shelf_box[1], f.shelf_box[0]))[:10]: + sku, agreement = facing.best_identity() + tag_sku, tag_agreement = facing.best_tag_identity() + print(f" shelf x={facing.shelf_box[0]:>5} y={facing.shelf_box[1]:>5} obs={facing.observations:>2} " + f"tracks={len(facing.seen_track_ids):>2} identity={sku} ({agreement:.2f}) tag={tag_sku} ({tag_agreement:.2f})") + +# render the map so duplicates are visible rather than inferred +import numpy as np +facings = shelf.facings() +xs = [f.shelf_box[0] for f in facings] + [f.shelf_box[2] for f in facings] +ys = [f.shelf_box[1] for f in facings] + [f.shelf_box[3] for f in facings] +x0, x1 = min(xs), max(xs) +y0, y1 = min(ys), max(ys) +scale = 0.6 +canvas = np.full((int((y1 - y0) * scale) + 20, int((x1 - x0) * scale) + 20, 3), 20, np.uint8) +for facing in facings: + bx = [(facing.shelf_box[0] - x0) * scale + 10, (facing.shelf_box[1] - y0) * scale + 10, + (facing.shelf_box[2] - x0) * scale + 10, (facing.shelf_box[3] - y0) * scale + 10] + colour = (0, 200, 0) if facing.observations >= 5 else (0, 120, 255) + cv2.rectangle(canvas, (int(bx[0]), int(bx[1])), (int(bx[2]), int(bx[3])), colour, 1) +cv2.imwrite("/tmp/dorayaki-verify/shelf-map.png", canvas) +print(f"\nmap canvas {canvas.shape[1]}x{canvas.shape[0]} px covering shelf x {x0}..{x1}, y {y0}..{y1}") +from collections import Counter +print("observations per facing:", Counter(min(f.observations, 20) for f in facings).most_common(6)) +print("facings per y band (100 px):", Counter((f.shelf_box[1] - y0) // 100 for f in facings).most_common(8)) diff --git a/recognization/code/shelf_map.py b/recognization/code/shelf_map.py new file mode 100644 index 0000000..c644dfb --- /dev/null +++ b/recognization/code/shelf_map.py @@ -0,0 +1,348 @@ +""" +The shelf as a plane, not a stream of boxes. + +Every measurement so far treats a frame as the world: a facing is "a track id", +and a track id dies the moment the camera pans past and comes back. Label +inheritance, drift checks and restart counters were each correct fixes for that, +and all of them are symptoms. The shelf does not move; the camera does. + +This module keeps a per-session **shelf map**. Each frame is registered to the map +by its content shift, every detection is placed in shelf coordinates, and a +detection that lands where a facing already is *merges* into it. Identity votes and +tag readings accumulate on the physical facing rather than on a transient track. + +That is what the current literature does for retail: RetailGlue (CVPR 2026W) builds +shelf panoramas by matching products across frames and reports 88.4 % stitching F1, +and the 7-Eleven Taiwan system (Sci. Rep. 2025, 7,000+ stores) stitches "virtual +shelves" before checking planograms. + +**What is measured, and what is not.** The synthetic walk registers to within +0.01 px of its ground truth - but it is a pure translation of a single photograph, +so that validates the harness, not handheld video. The number that matters (real +registration error, and the duplicate rate that follows from it) needs real +footage, which `record.py` exists to collect. +""" + +from __future__ import annotations + +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +# A frame whose content shift cannot be estimated this confidently is treated as +# discontinuous: the map starts a new section rather than guessing a position. +MIN_REGISTRATION_RESPONSE = 0.20 +# Two detections are the same physical facing when their shelf boxes overlap this +# much. Measured: adjacent facings of one product overlap ~0 by construction. +MERGE_IOU = 0.45 + + +def _iou(a: Sequence[int], b: Sequence[int]) -> float: + x1, y1 = max(a[0], b[0]), max(a[1], b[1]) + x2, y2 = min(a[2], b[2]), min(a[3], b[3]) + intersection = max(0, x2 - x1) * max(0, y2 - y1) + area_a = max(0, a[2] - a[0]) * max(0, a[3] - a[1]) + area_b = max(0, b[2] - b[0]) * max(0, b[3] - b[1]) + return intersection / max(1.0, float(area_a + area_b - intersection)) + + +@dataclass +class Facing: + """One physical position on the shelf, with everything observed there.""" + + facing_id: str + shelf_box: List[int] + row: int = 0 + observations: int = 0 + first_seq: int = 0 + last_seq: int = 0 + identity_votes: Dict[str, float] = field(default_factory=dict) + tag_votes: Dict[str, float] = field(default_factory=dict) + tag_readings: List[Dict[str, Any]] = field(default_factory=list) + seen_track_ids: set = field(default_factory=set) + + def vote_identity(self, sku_id: Optional[str], weight: float = 1.0) -> None: + if sku_id: + self.identity_votes[sku_id] = self.identity_votes.get(sku_id, 0.0) + weight + + def vote_tag(self, sku_id: Optional[str], reading: Optional[Dict[str, Any]] = None) -> None: + if sku_id: + self.tag_votes[sku_id] = self.tag_votes.get(sku_id, 0.0) + 1.0 + if reading: + self.tag_readings.append(reading) + + def best_identity(self) -> Tuple[Optional[str], float]: + """The identity the facing's pooled evidence supports, and its agreement.""" + if not self.identity_votes: + return None, 0.0 + sku_id, weight = max(self.identity_votes.items(), key=lambda item: item[1]) + total = sum(self.identity_votes.values()) + return sku_id, weight / max(1e-6, total) + + def best_tag_identity(self) -> Tuple[Optional[str], float]: + if not self.tag_votes: + return None, 0.0 + sku_id, weight = max(self.tag_votes.items(), key=lambda item: item[1]) + total = sum(self.tag_votes.values()) + return sku_id, weight / max(1e-6, total) + + def as_payload(self) -> Dict[str, Any]: + identity, agreement = self.best_identity() + tag_identity, tag_agreement = self.best_tag_identity() + return { + "facing_id": self.facing_id, + "box": list(self.shelf_box), + "row": self.row, + "observations": self.observations, + "track_ids": len(self.seen_track_ids), + "sku_id": identity, + "identity_agreement": round(agreement, 3), + "tag_sku_id": tag_identity, + "tag_agreement": round(tag_agreement, 3), + "tag_readings": self.tag_readings[-3:], + } + + +class ShelfMap: + """Registers frames and accumulates facings in shelf coordinates.""" + + def __init__(self, settings: Optional[Any] = None) -> None: + self._lock = threading.Lock() + self._prev_gray: Optional[Any] = None + self._offset: List[float] = [0.0, 0.0] # shelf position of the frame origin + self._facings: List[Facing] = [] + self._orb = None + self.registrations = 0 + self.registration_failures = 0 + self.anchors = 0 + self.continuations = 0 + self.detections = 0 + self.merges = 0 + self.responses: List[float] = [] + + # ------------------------------------------------------------- registration + def _gray(self, frame: Any, scale: float = 0.5) -> Any: + import cv2 + + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + if scale != 1.0: + gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA) + return gray + + def register(self, frame: Any) -> Optional[Tuple[float, float]]: + """Content shift from the previous frame, in full-resolution pixels. + + Phase correlation first: it is the right tool for a translating scene and + costs a couple of milliseconds. ORB + RANSAC only when the correlation is + weak (blur, a large jump, a scene with no texture), because that is the case + where a fast estimator quietly returns nonsense. + """ + import cv2 + import numpy as np + + gray = self._gray(frame) + with self._lock: + previous = self._prev_gray + self._prev_gray = gray + if previous is None or previous.shape != gray.shape: + return (0.0, 0.0) + window = cv2.createHanningWindow((gray.shape[1], gray.shape[0]), cv2.CV_32F) + (dx, dy), response = cv2.phaseCorrelate(previous.astype(np.float32), gray.astype(np.float32), window) + self.responses.append(float(response)) + if response >= MIN_REGISTRATION_RESPONSE: + self.registrations += 1 + return (dx / 0.5, dy / 0.5) # back to full resolution + + shift = self._orb_shift(previous, gray) + if shift is None: + self.registration_failures += 1 + return None + self.registrations += 1 + return (shift[0] / 0.5, shift[1] / 0.5) + + def _orb_shift(self, previous: Any, current: Any) -> Optional[Tuple[float, float]]: + import cv2 + import numpy as np + + if self._orb is None: + self._orb = cv2.ORB_create(nfeatures=1500, fastThreshold=10) + key_a, desc_a = self._orb.detectAndCompute(previous, None) + key_b, desc_b = self._orb.detectAndCompute(current, None) + if desc_a is None or desc_b is None or len(key_a) < 20 or len(key_b) < 20: + return None + matches = cv2.BFMatcher(cv2.NORM_HAMMING).knnMatch(desc_a, desc_b, k=2) + good = [m for m, n in matches if m.distance < 0.75 * n.distance] + if len(good) < 12: + return None + src = np.float32([key_a[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) + dst = np.float32([key_b[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + matrix, _ = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=3.0) + if matrix is None: + return None + return (float(matrix[0, 2]), float(matrix[1, 2])) + + # ------------------------------------------------------------------ facings + def _anchor_to_map( + self, detections: Sequence[Sequence[int]], offset_x: float, offset_y: float + ) -> Tuple[float, float]: + """Correct the chained offset by matching detections against the map. + + Chaining pairwise shifts drifts: measured over a 79-step walk with *perfect* + pairwise registration, the accumulated x offset reached 2047 px on a 1500 px + shelf, the same facing landed in two places, and a shelf of ~60 facings + became 334 with 113 of them seen once. Drift is inherent to chaining - the + fix is to anchor the frame to the structure already on the map, which is + what the retail-stitching literature does with product correspondences. + + One ICP-style step is enough: for every detection that has a facing nearby, + the correction it implies is a vote, and the median vote wins. + """ + if not self._facings or len(detections) < 4: + return offset_x, offset_y + corrections = [] + search = 40.0 + for box in detections: + centre_x = (box[0] + box[2]) / 2.0 + offset_x + centre_y = (box[1] + box[3]) / 2.0 + offset_y + best, best_distance = None, search + for facing in self._facings: + facing_x = (facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 + facing_y = (facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 + distance = max(abs(facing_x - centre_x), abs(facing_y - centre_y)) + if distance < best_distance: + best, best_distance = facing, distance + if best is not None: + corrections.append( + ( + (best.shelf_box[0] + best.shelf_box[2]) / 2.0 - ((box[0] + box[2]) / 2.0 + offset_x), + (best.shelf_box[1] + best.shelf_box[3]) / 2.0 - ((box[1] + box[3]) / 2.0 + offset_y), + ) + ) + if len(corrections) < 4: + return offset_x, offset_y + dx = sorted(c[0] for c in corrections)[len(corrections) // 2] + dy = sorted(c[1] for c in corrections)[len(corrections) // 2] + self.anchors += 1 + return offset_x + dx, offset_y + dy + + def update(self, seq: int, frame: Any, tracks: Sequence[Dict[str, Any]]) -> None: + """Register the frame, then place or merge every detection on the shelf.""" + shift = self.register(frame) + boxes = [ + [int(v) for v in (track.get("box") or track.get("raw_box"))] + for track in tracks + if (track.get("box") or track.get("raw_box")) and len(track.get("box") or track.get("raw_box")) == 4 + ] + with self._lock: + if shift is None: + # A discontinuous frame: keep the last known offset rather than + # inventing one, so its detections land where the shelf is. + shift = (0.0, 0.0) + # `shift` is how far the CONTENT moved in the frame, so the frame's + # origin moves the opposite way along the shelf. Adding it instead + # (measured) walks the map backwards: the same facing lands 50 px away + # in the next frame and a shelf of ~60 facings becomes 857. + self._offset[0] -= shift[0] + self._offset[1] -= shift[1] + if self._facings: + self._offset[0], self._offset[1] = self._anchor_to_map( + boxes, self._offset[0], self._offset[1] + ) + offset_x, offset_y = self._offset + for track in tracks: + box = track.get("box") or track.get("raw_box") + if not box or len(box) != 4: + continue + self.detections += 1 + shelf_box = [ + int(round(box[0] + offset_x)), int(round(box[1] + offset_y)), + int(round(box[2] + offset_x)), int(round(box[3] + offset_y)), + ] + track_id = track.get("track_id") + facing = self._find(shelf_box, int(track_id) if track_id is not None else None) + if facing is None: + facing = Facing( + facing_id=f"face_{uuid.uuid4().hex[:8]}", + shelf_box=shelf_box, + first_seq=seq, + ) + self._facings.append(facing) + else: + self.merges += 1 + # Average the box: repeated observations of a static object + # should converge, and the average is what the eye expects. + for index in range(4): + facing.shelf_box[index] = int(round((facing.shelf_box[index] * facing.observations + shelf_box[index]) / (facing.observations + 1))) + facing.observations += 1 + facing.last_seq = seq + identity = track.get("identity") or {} + if identity.get("state") == "recognized": + facing.vote_identity(identity.get("sku_id"), float(identity.get("score") or 1.0)) + tag_identity = track.get("tag_identity") or {} + tag = track.get("price_tag") or {} + if tag_identity.get("sku_id") or tag.get("state") == "readable": + facing.vote_tag(tag_identity.get("sku_id"), tag if tag else None) + if track.get("track_id") is not None: + facing.seen_track_ids.add(int(track["track_id"])) + + def _find(self, shelf_box: Sequence[int], track_id: Optional[int] = None) -> Optional[Facing]: + """The facing a detection belongs to. + + Continuity first: if this track id was already placed nearby, it is the same + facing, and no overlap test is needed - the tracker already decided that. + Geometry only when the id is new (a pan that outran the tracker, or the + tracker renumbering). Measured: with geometry alone, box jitter split the + same facing often enough that the map held 210 facings where the tracker + held 156 ids - it was adding duplicates, not removing them. + """ + width = max(1, shelf_box[2] - shelf_box[0]) + height = max(1, shelf_box[3] - shelf_box[1]) + if track_id is not None: + radius = 0.75 * max(width, height) + best, best_distance = None, radius + for facing in self._facings: + if track_id not in facing.seen_track_ids: + continue + distance = max( + abs((facing.shelf_box[0] + facing.shelf_box[2]) / 2.0 - (shelf_box[0] + shelf_box[2]) / 2.0), + abs((facing.shelf_box[1] + facing.shelf_box[3]) / 2.0 - (shelf_box[1] + shelf_box[3]) / 2.0), + ) + if distance < best_distance: + best, best_distance = facing, distance + if best is not None: + self.continuations += 1 + return best + best, best_overlap = None, MERGE_IOU + for facing in self._facings: + overlap = _iou(shelf_box, facing.shelf_box) + if overlap > best_overlap: + best, best_overlap = facing, overlap + return best + + # ------------------------------------------------------------------ reporting + def facings(self) -> List[Facing]: + with self._lock: + return list(self._facings) + + def stats(self) -> Dict[str, Any]: + with self._lock: + facings = list(self._facings) + responses = list(self.responses[-200:]) + consistent = 0 + for facing in facings: + sku_id, agreement = facing.best_identity() + if sku_id and agreement >= 0.8: + consistent += 1 + return { + "facings": len(facings), + "detections": self.detections, + "merges": self.merges, + "continuations": self.continuations, + "registrations": self.registrations, + "anchored": self.anchors, + "registration_failures": self.registration_failures, + "median_response": round(sorted(responses)[len(responses) // 2], 3) if responses else None, + "identity_consistent": consistent, + "offset": [round(value, 1) for value in self._offset], + } diff --git a/recognization/code/test_shelf_map.py b/recognization/code/test_shelf_map.py new file mode 100644 index 0000000..664ecce --- /dev/null +++ b/recognization/code/test_shelf_map.py @@ -0,0 +1,125 @@ +""" +Tests for the shelf map: registration, merging, and evidence pooling. + +The sign convention is tested explicitly, because getting it backwards is silent: +the map still produces boxes, they just land in the wrong place, and a shelf of +~60 facings becomes 857. That happened during development. +""" + +from __future__ import annotations + +import os +import sys +import unittest + +import numpy as np + +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(current_dir) +if os.path.exists(os.path.join(project_root, "stock")): + os.chdir(project_root) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from shelf_map import Facing, ShelfMap, _iou # noqa: E402 + + +def a_frame(shift_x=0, shift_y=0, size=(160, 240)): + """A textured frame, optionally a shifted copy of another.""" + height, width = size + rng = np.random.RandomState(7) + base = rng.randint(0, 255, (height * 2, width * 2, 3), dtype=np.uint8) + return base[height - shift_y:height * 2 - shift_y, width - shift_x:width * 2 - shift_x] + + +def a_track(track_id, box, sku=None, score=None, tag_sku=None): + track = {"track_id": track_id, "box": list(box)} + if sku: + track["identity"] = {"state": "recognized", "sku_id": sku, "score": score or 1.0} + else: + track["identity"] = {"state": "unconfirmed"} + if tag_sku: + track["tag_identity"] = {"sku_id": tag_sku, "score": 0.7} + return track + + +class IouTests(unittest.TestCase): + def test_overlap_maths(self): + self.assertAlmostEqual(_iou([0, 0, 10, 10], [0, 0, 10, 10]), 1.0) + self.assertEqual(_iou([0, 0, 10, 10], [20, 20, 30, 30]), 0.0) + + +class RegistrationTests(unittest.TestCase): + def test_the_frame_origin_moves_opposite_to_the_content(self): + """The sign that matters: content moving left means the camera moved right. + + Getting this backwards walks the map the wrong way. It is silent - boxes + still appear - and it turned a shelf of ~60 facings into 857. + """ + shelf = ShelfMap() + shelf.register(a_frame()) + shift = shelf.register(a_frame(shift_x=10)) # content moved right 10 + self.assertIsNotNone(shift) + self.assertAlmostEqual(shift[0], 10, delta=1.0) + + # Content moving right means the camera moved LEFT, so the frame's origin + # sits 10 px further back along the shelf. + shelf.update(0, a_frame(), [a_track(1, [0, 0, 20, 20])]) + offset_before = list(shelf._offset) + shelf.update(1, a_frame(shift_x=10), [a_track(2, [0, 0, 20, 20])]) + self.assertLess(shelf._offset[0], offset_before[0] + 1, + "the offset must follow the camera, not the content") + + def test_a_first_frame_has_no_shift(self): + shelf = ShelfMap() + self.assertEqual(shelf.register(a_frame()), (0.0, 0.0)) + + +class MergingTests(unittest.TestCase): + def test_a_facing_seen_twice_is_one_facing(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200])]) + shelf.update(1, a_frame(), [a_track(2, [12, 11, 112, 201])]) + self.assertEqual(len(shelf.facings()), 1) + self.assertEqual(shelf.stats()["merges"], 1) + + def test_two_neighbouring_facings_stay_separate(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 100, 200]), a_track(2, [110, 10, 200, 200])]) + self.assertEqual(len(shelf.facings()), 2) + + def test_identity_votes_pool_across_track_ids(self): + """The point of the map: one facing, many transient track ids, one answer.""" + shelf = ShelfMap() + for index, (track_id, sku) in enumerate([(1, "sku_a"), (2, "sku_a"), (3, "sku_a"), (4, "sku_b")]): + shelf.update(index, a_frame(), [a_track(track_id, [10, 10, 110, 200], sku=sku)]) + facing = shelf.facings()[0] + self.assertEqual(facing.observations, 4) + self.assertEqual(len(facing.seen_track_ids), 4, "four tracks, one facing") + sku_id, agreement = facing.best_identity() + self.assertEqual(sku_id, "sku_a") + self.assertAlmostEqual(agreement, 0.75) + + def test_tag_evidence_pools_separately_from_packaging(self): + shelf = ShelfMap() + shelf.update(0, a_frame(), [a_track(1, [10, 10, 110, 200], sku="sku_a", tag_sku="sku_a")]) + shelf.update(1, a_frame(), [a_track(2, [10, 10, 110, 200], tag_sku="sku_a")]) + facing = shelf.facings()[0] + self.assertEqual(facing.best_identity()[0], "sku_a") + self.assertEqual(facing.best_tag_identity()[0], "sku_a") + self.assertEqual(len(facing.tag_readings), 0, "no readable tag payload was attached") + + +class PayloadTests(unittest.TestCase): + def test_a_facing_reports_its_pooled_evidence(self): + facing = Facing(facing_id="face_1", shelf_box=[0, 0, 10, 10]) + facing.vote_identity("sku_a", 2.0) + facing.vote_identity("sku_b", 1.0) + payload = facing.as_payload() + self.assertEqual(payload["sku_id"], "sku_a") + self.assertAlmostEqual(payload["identity_agreement"], 2 / 3, places=3) + self.assertIsNone(payload["tag_sku_id"]) + + +if __name__ == "__main__": + unittest.main()