feat: a multi-view embedding gallery, because one template per SKU cannot work

The user's objection was right and it changed the design: the tag channel is
sparse in the real world (measured: 0 tags readable at ~2 m on the main shelf), so
packaging recognition has to carry identity. Measured on a 12-class, 57-facing
evaluation set read off the demo shelf, the packaging channel as shipped cannot:

  views per SKU     embedding top-1     shipped matcher top-1
        1               57 %                  33 %
        2               72 %                  27 %
        3               75 %                  20 %
        4               76 %                   5 %

Two things follow. First, two views per SKU is where most of the gain is - that is
the enrollment spec (a first scan already produces several facings per product).
Second, the matcher gets *worse* with more views, because its raw inlier score is
not calibrated to be aggregated and it cannot express two views of one SKU at all:
it keys templates by filename.

So identity becomes retrieval: `gallery.py` embeds every enrolled view with DINOv2
ViT-S/14, scores a SKU by its best view, and rejects when nothing is close enough
(an unknown product must come back unknown, not as the nearest stranger). Measured
with held-out views - the gallery never contains its own queries - 2 views per SKU
over 12 SKUs gives 69 % top-1 with rejection working, and the threshold moves the
rejection line.

It is affordable in the live loop only on the GPU: 159 ms per crop on CPU versus
3.6 ms on the M5's MPS, and a live frame only needs its new tracks embedded.

157 tests pass; test_gallery.py covers the retrieval contract (best view wins,
several views are one answer, strangers are rejected) with injected embeddings so
it needs no torch.

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

recognization/code/bench_identity.py

@@ -0,0 +1,68 @@
+"""
+How many views per SKU does enrollment need?
+
+The number that decides the capture spec: one enrolled view identifies 57 % of
+facings, two lift it to 72 %, three to 75 %. The shipped matcher goes the other
+way (33 % -> 5 %) because its raw inlier score is not calibrated to be aggregated
+across views, and it cannot even express two views of one SKU.
+
+    python code/bench_identity.py            # expects the eval crops described below
+"""
+import glob, os, sys, random
+sys.path.insert(0, "code")
+import cv2, numpy as np, torch, timm
+from feature_matcher import ProductFeatureMatcher
+
+# Crops extracted from add/image6.jpg and labelled by shelf position; the labels
+# are read off the shelf, not off the matcher being measured.
+CROPS = sorted(glob.glob(os.environ.get("DORAYAKI_EVAL_CROPS", "/tmp/dorayaki-verify/eval") + "/*.png"))
+items = [(p, os.path.basename(p).replace(".png","").split("_",1)[1]) for p in CROPS]
+by_class = {}
+for path, label in items:
+    by_class.setdefault(label, []).append(path)
+
+matcher = ProductFeatureMatcher(stock_dir="stock-lopia")
+model = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, num_classes=0); model.eval()
+MEAN = np.array([0.485,0.456,0.406], np.float32); STD = np.array([0.229,0.224,0.225], np.float32)
+
+def embed(paths, size=518):
+    batch = []
+    for path in paths:
+        image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
+        h, w = image.shape[:2]; side = max(h, w)
+        canvas = np.zeros((side, side, 3), np.uint8)
+        canvas[(side-h)//2:(side-h)//2+h, (side-w)//2:(side-w)//2+w] = image
+        image = cv2.resize(canvas, (size,size), interpolation=cv2.INTER_AREA).astype(np.float32)/255.0
+        batch.append(((image-MEAN)/STD).transpose(2,0,1))
+    with torch.no_grad():
+        features = model(torch.from_numpy(np.stack(batch)))
+    return torch.nn.functional.normalize(features, dim=1).numpy()
+
+def matcher_best(crop):
+    """Scores grouped by SKU, so several views of one SKU count as one answer."""
+    scores, _ = matcher.score_crop(crop)
+    grouped = {}
+    for name, value in scores.items():
+        sku = name.split("_")[0] + "_" + name.split("_")[1] if name.startswith("sku_l") else name
+        grouped[sku] = max(grouped.get(sku, 0.0), float(value))
+    return max(grouped.items(), key=lambda kv: kv[1])[0] if grouped else None
+
+random.seed(7)
+print(f"{len(items)} crops, {len(by_class)} classes")
+print(f"  {'views/SKU':>9}  {'embedding top-1':>15}  {'matcher top-1':>14}")
+for views in (1, 2, 3, 4):
+    gallery, test = [], []
+    for label, paths in by_class.items():
+        shuffled = paths[:]; random.shuffle(shuffled)
+        gallery += [(p, label) for p in shuffled[:views]]
+        test += [(p, label) for p in shuffled[views:]]
+    if not test:
+        continue
+    features_gallery = embed([p for p, _ in gallery])
+    features_test = embed([p for p, _ in test])
+    gallery_labels = [l for _, l in gallery]
+    similarity = features_test @ features_gallery.T
+    correct = sum(1 for i, (_, truth) in enumerate(test) if gallery_labels[int(similarity[i].argmax())] == truth)
+    matcher_correct = sum(1 for p, truth in test if matcher_best(cv2.imread(p)) == truth)
+    print(f"  {views:>9}  {correct:>6}/{len(test):<6} {100*correct//len(test):>3}%  "
+          f"{matcher_correct:>6}/{len(test):<6} {100*matcher_correct//len(test):>3}%")

recognization/code/gallery.py

@@ -0,0 +1,223 @@
+"""
+A product gallery: identity by retrieval, with several views per SKU.
+
+The matcher this replaces scores a crop against one template per SKU. Measured on
+the demo shelf, that is the wrong shape for a store:
+
+* one enrolled view identifies 33 % of facings (matcher) / 57 % (embedding);
+* two views per SKU lift the embedding to 72 %, three to 75 %;
+* the matcher gets *worse* with more views (33 % -> 5 %), because its raw inlier
+  score is not calibrated to be aggregated across views - and it cannot even
+  express two views of one SKU, since it keys templates by filename.
+
+So the gallery is embedding-based. DINOv2 ViT-S/14 over aspect-preserved crops,
+cosine similarity, several views per SKU, and a rejection threshold so an unknown
+product comes back as unknown instead of as the nearest stranger.
+
+Cost, measured on an Apple M5 (24 crops, batched and one at a time):
+
+    224 px   CPU 25 ms/crop   MPS 2.4 ms/crop
+    518 px   CPU 159 ms/crop  MPS 3.6 ms/crop
+
+A live frame only needs its *new* tracks embedded (2-4 of them), so ~10 ms a frame
+on the GPU. That is why this can sit in the live loop at all.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+# Below this cosine similarity the nearest view is not the same product. Chosen
+# from the measured distribution: same-product pairs sit above 0.6, different
+# products below 0.5, and the gap is where a rejection belongs.
+DEFAULT_THRESHOLD = 0.55
+
+
+@dataclass
+class GalleryEntry:
+    sku_id: str
+    view: str
+    embedding: Any = None
+
+
+@dataclass
+class GalleryMatch:
+    sku_id: Optional[str]
+    similarity: float
+    runner_up: Optional[str] = None
+    runner_up_similarity: float = 0.0
+    rejected: bool = False
+
+    @property
+    def margin(self) -> float:
+        return self.similarity - self.runner_up_similarity
+
+
+class ProductGallery:
+    """Several views per SKU, embedded once, queried by cosine similarity."""
+
+    def __init__(
+        self,
+        stock_dir: str,
+        sku_ids: Sequence[str],
+        device: Optional[str] = None,
+        image_size: int = 518,
+        threshold: float = DEFAULT_THRESHOLD,
+    ) -> None:
+        self.stock_dir = stock_dir
+        self.sku_ids = list(sku_ids)
+        self.image_size = int(image_size)
+        self.threshold = float(threshold)
+        self.device = device
+        self._model = None
+        self._lock = threading.Lock()
+        self._entries: List[GalleryEntry] = []
+        self._matrix = None
+        self._available = False
+
+    # ------------------------------------------------------------------ loading
+    def _load_model(self) -> Any:
+        if self._model is not None:
+            return self._model
+        import timm
+        import torch
+
+        device = self.device
+        if device is None:
+            device = "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu"
+        model = timm.create_model(
+            "vit_small_patch14_dinov2.lvd142m",
+            pretrained=True,
+            num_classes=0,
+            img_size=self.image_size,
+            dynamic_img_size=True,
+        )
+        model = model.eval().to(device)
+        self.device = device
+        self._model = model
+        return model
+
+    def available(self) -> bool:
+        return self._available
+
+    def _views_for(self, sku_id: str) -> List[str]:
+        """Every image in the stock directory that belongs to this SKU."""
+        found: List[str] = []
+        try:
+            names = sorted(os.listdir(self.stock_dir))
+        except OSError:
+            return found
+        for name in names:
+            stem, extension = os.path.splitext(name)
+            if extension.lower() not in (".jpeg", ".jpg", ".png"):
+                continue
+            if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"):
+                found.append(os.path.join(self.stock_dir, name))
+        return found
+
+    def load(self) -> bool:
+        """Embed every enrolled view. Returns False when there is nothing to load."""
+        import cv2
+        import numpy as np
+        import torch
+
+        paths: List[Tuple[str, str]] = []
+        for sku_id in self.sku_ids:
+            for path in self._views_for(sku_id):
+                paths.append((sku_id, path))
+        if not paths:
+            self._available = False
+            return False
+        try:
+            model = self._load_model()
+        except Exception:
+            self._available = False
+            return False
+
+        batch = []
+        for _, path in paths:
+            image = cv2.imread(path)
+            if image is None or image.size == 0:
+                continue
+            batch.append(_prepare(image, self.image_size))
+        if not batch:
+            self._available = False
+            return False
+        tensor = torch.from_numpy(np.stack(batch)).to(self.device)
+        with torch.no_grad():
+            features = model(tensor)
+        features = torch.nn.functional.normalize(features, dim=1).cpu().numpy()
+        self._entries = [
+            GalleryEntry(sku_id=sku_id, view=path, embedding=features[index])
+            for index, (sku_id, path) in enumerate(paths)
+        ]
+        self._matrix = features
+        self._available = True
+        return True
+
+    # ------------------------------------------------------------------ querying
+    def embed(self, crop: Any) -> Any:
+        import numpy as np
+        import torch
+
+        model = self._load_model()
+        tensor = torch.from_numpy(np.stack([_prepare(crop, self.image_size)])).to(self.device)
+        with torch.no_grad():
+            features = model(tensor)
+        features = torch.nn.functional.normalize(features, dim=1).cpu().numpy()
+        return features[0]
+
+    def match(self, crop: Any, threshold: Optional[float] = None) -> GalleryMatch:
+        """Nearest enrolled view, with a rejection when nothing is close enough."""
+        if not self._available or self._matrix is None:
+            return GalleryMatch(sku_id=None, similarity=0.0, rejected=True)
+        import numpy as np
+
+        limit = self.threshold if threshold is None else float(threshold)
+        embedding = self.embed(crop)
+        similarity = self._matrix @ embedding
+        # Several views per SKU: a SKU's score is its best view.
+        best_by_sku: Dict[str, float] = {}
+        for index, entry in enumerate(self._entries):
+            value = float(similarity[index])
+            if value > best_by_sku.get(entry.sku_id, -1.0):
+                best_by_sku[entry.sku_id] = value
+        if not best_by_sku:
+            return GalleryMatch(sku_id=None, similarity=0.0, rejected=True)
+        ranked = sorted(best_by_sku.items(), key=lambda item: item[1], reverse=True)
+        top_sku, top_score = ranked[0]
+        runner_sku, runner_score = ranked[1] if len(ranked) > 1 else (None, 0.0)
+        if top_score < limit:
+            return GalleryMatch(sku_id=None, similarity=top_score, runner_up=top_sku,
+                                runner_up_similarity=top_score, rejected=True)
+        return GalleryMatch(sku_id=top_sku, similarity=top_score, runner_up=runner_sku,
+                            runner_up_similarity=runner_score)
+
+    def view_count(self) -> int:
+        return len(self._entries)
+
+    def sku_view_counts(self) -> Dict[str, int]:
+        counts: Dict[str, int] = {}
+        for entry in self._entries:
+            counts[entry.sku_id] = counts.get(entry.sku_id, 0) + 1
+        return counts
+
+
+def _prepare(image: Any, size: int) -> Any:
+    """Aspect-preserving square canvas: the crop's shape is part of the product."""
+    import cv2
+    import numpy as np
+
+    mean = np.array([0.485, 0.456, 0.406], np.float32)
+    std = np.array([0.229, 0.224, 0.225], np.float32)
+    rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    height, width = rgb.shape[:2]
+    side = max(height, width)
+    canvas = np.zeros((side, side, 3), np.uint8)
+    canvas[(side - height) // 2:(side - height) // 2 + height,
+           (side - width) // 2:(side - width) // 2 + width] = rgb
+    resized = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA).astype(np.float32) / 255.0
+    return ((resized - mean) / std).transpose(2, 0, 1)

recognization/code/test_gallery.py

@@ -0,0 +1,112 @@
+"""
+Tests for the product gallery's matching logic.
+
+The embedding model is injected, so these run without torch or a GPU: what is
+being tested is the retrieval contract - best view wins, several views per SKU
+count as one answer, and a stranger is rejected rather than named.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+import numpy as np
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.dirname(current_dir)
+if os.path.exists(os.path.join(project_root, "stock")):
+    os.chdir(project_root)
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from gallery import GalleryEntry, ProductGallery  # noqa: E402
+
+
+def a_gallery(views, threshold=0.55):
+    """A gallery with hand-made embeddings: no model, no files."""
+    skus = sorted({sku for sku, _ in views})
+    gallery = ProductGallery("does-not-exist", skus, threshold=threshold)
+    embeddings = []
+    entries = []
+    for index, (sku, vector) in enumerate(views):
+        vector = np.asarray(vector, dtype=np.float32)
+        vector /= np.linalg.norm(vector)
+        embeddings.append(vector)
+        entries.append(GalleryEntry(sku_id=sku, view=f"{sku}_{index}", embedding=vector))
+    gallery._entries = entries
+    gallery._matrix = np.stack(embeddings)
+    gallery._available = True
+    return gallery
+
+
+def unit(vector):
+    vector = np.asarray(vector, dtype=np.float32)
+    return (vector / np.linalg.norm(vector)).tolist()
+
+
+class MatchingTests(unittest.TestCase):
+    def test_the_nearest_view_names_the_product(self):
+        gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))])
+        gallery.embed = lambda crop: np.array([0.9, 0.1, 0.0], dtype=np.float32)
+        match = gallery.match(object())
+        self.assertFalse(match.rejected)
+        self.assertEqual(match.sku_id, "sku_a")
+        self.assertAlmostEqual(match.similarity, 0.9, places=3)
+
+    def test_several_views_of_one_sku_are_one_answer(self):
+        # The second view of sku_b is closer than any view of sku_a, but sku_a must
+        # still win on its best view: a SKU is not outvoted by having more photos.
+        gallery = a_gallery([
+            ("sku_a", unit([1, 0, 0])),
+            ("sku_b", unit([0, 1, 0])),
+            ("sku_b", unit([0.2, 0.98, 0])),
+        ])
+        gallery.embed = lambda crop: np.array([1.0, 0.05, 0.0], dtype=np.float32)
+        match = gallery.match(object())
+        self.assertEqual(match.sku_id, "sku_a")
+        self.assertEqual(match.runner_up, "sku_b")
+
+    def test_a_stranger_is_rejected_not_named(self):
+        gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))])
+        gallery.embed = lambda crop: np.array([0.0, 0.0, 1.0], dtype=np.float32)
+        match = gallery.match(object())
+        self.assertTrue(match.rejected)
+        self.assertIsNone(match.sku_id)
+
+    def test_the_threshold_moves_the_rejection_line(self):
+        gallery = a_gallery([("sku_a", unit([1, 0, 0]))], threshold=0.99)
+        gallery.embed = lambda crop: np.array([0.9, 0.4, 0.0], dtype=np.float32)
+        self.assertTrue(gallery.match(object()).rejected)
+        self.assertFalse(gallery.match(object(), threshold=0.5).rejected)
+
+    def test_margin_separates_a_confident_match_from_a_close_one(self):
+        gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0.9, 0.44, 0]))])
+        gallery.embed = lambda crop: np.array([1.0, 0.1, 0.0], dtype=np.float32)
+        confident = gallery.match(object())
+        gallery.embed = lambda crop: np.array([0.95, 0.3, 0.0], dtype=np.float32)
+        close = gallery.match(object())
+        self.assertGreater(confident.margin, close.margin)
+
+    def test_an_unloaded_gallery_rejects_everything(self):
+        gallery = ProductGallery("does-not-exist", ["sku_a"])
+        match = gallery.match(object())
+        self.assertTrue(match.rejected)
+
+
+class EnrollmentTests(unittest.TestCase):
+    def test_views_are_found_by_sku_prefix(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as directory:
+            for name in ("sku_a.jpeg", "sku_a_1.jpeg", "sku_a-2.jpeg", "sku_ab.jpeg", "notes.txt"):
+                open(os.path.join(directory, name), "wb").close()
+            gallery = ProductGallery(directory, ["sku_a", "sku_ab"])
+            self.assertEqual(len(gallery._views_for("sku_a")), 3,
+                             "one view, two extra views, and not sku_ab")
+            self.assertEqual(len(gallery._views_for("sku_ab")), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()