diff --git a/recognization/code/bench_identity.py b/recognization/code/bench_identity.py new file mode 100644 index 0000000..6755ecf --- /dev/null +++ b/recognization/code/bench_identity.py @@ -0,0 +1,68 @@ +""" +How many views per SKU does enrollment need? + +The number that decides the capture spec: one enrolled view identifies 57 % of +facings, two lift it to 72 %, three to 75 %. The shipped matcher goes the other +way (33 % -> 5 %) because its raw inlier score is not calibrated to be aggregated +across views, and it cannot even express two views of one SKU. + + python code/bench_identity.py # expects the eval crops described below +""" +import glob, os, sys, random +sys.path.insert(0, "code") +import cv2, numpy as np, torch, timm +from feature_matcher import ProductFeatureMatcher + +# Crops extracted from add/image6.jpg and labelled by shelf position; the labels +# are read off the shelf, not off the matcher being measured. +CROPS = sorted(glob.glob(os.environ.get("DORAYAKI_EVAL_CROPS", "/tmp/dorayaki-verify/eval") + "/*.png")) +items = [(p, os.path.basename(p).replace(".png","").split("_",1)[1]) for p in CROPS] +by_class = {} +for path, label in items: + by_class.setdefault(label, []).append(path) + +matcher = ProductFeatureMatcher(stock_dir="stock-lopia") +model = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, num_classes=0); model.eval() +MEAN = np.array([0.485,0.456,0.406], np.float32); STD = np.array([0.229,0.224,0.225], np.float32) + +def embed(paths, size=518): + batch = [] + for path in paths: + image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB) + h, w = image.shape[:2]; side = max(h, w) + canvas = np.zeros((side, side, 3), np.uint8) + canvas[(side-h)//2:(side-h)//2+h, (side-w)//2:(side-w)//2+w] = image + image = cv2.resize(canvas, (size,size), interpolation=cv2.INTER_AREA).astype(np.float32)/255.0 + batch.append(((image-MEAN)/STD).transpose(2,0,1)) + with torch.no_grad(): + features = model(torch.from_numpy(np.stack(batch))) + return torch.nn.functional.normalize(features, dim=1).numpy() + +def matcher_best(crop): + """Scores grouped by SKU, so several views of one SKU count as one answer.""" + scores, _ = matcher.score_crop(crop) + grouped = {} + for name, value in scores.items(): + sku = name.split("_")[0] + "_" + name.split("_")[1] if name.startswith("sku_l") else name + grouped[sku] = max(grouped.get(sku, 0.0), float(value)) + return max(grouped.items(), key=lambda kv: kv[1])[0] if grouped else None + +random.seed(7) +print(f"{len(items)} crops, {len(by_class)} classes") +print(f" {'views/SKU':>9} {'embedding top-1':>15} {'matcher top-1':>14}") +for views in (1, 2, 3, 4): + gallery, test = [], [] + for label, paths in by_class.items(): + shuffled = paths[:]; random.shuffle(shuffled) + gallery += [(p, label) for p in shuffled[:views]] + test += [(p, label) for p in shuffled[views:]] + if not test: + continue + features_gallery = embed([p for p, _ in gallery]) + features_test = embed([p for p, _ in test]) + gallery_labels = [l for _, l in gallery] + similarity = features_test @ features_gallery.T + correct = sum(1 for i, (_, truth) in enumerate(test) if gallery_labels[int(similarity[i].argmax())] == truth) + matcher_correct = sum(1 for p, truth in test if matcher_best(cv2.imread(p)) == truth) + print(f" {views:>9} {correct:>6}/{len(test):<6} {100*correct//len(test):>3}% " + f"{matcher_correct:>6}/{len(test):<6} {100*matcher_correct//len(test):>3}%") diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py new file mode 100644 index 0000000..f01a255 --- /dev/null +++ b/recognization/code/gallery.py @@ -0,0 +1,223 @@ +""" +A product gallery: identity by retrieval, with several views per SKU. + +The matcher this replaces scores a crop against one template per SKU. Measured on +the demo shelf, that is the wrong shape for a store: + +* one enrolled view identifies 33 % of facings (matcher) / 57 % (embedding); +* two views per SKU lift the embedding to 72 %, three to 75 %; +* the matcher gets *worse* with more views (33 % -> 5 %), because its raw inlier + score is not calibrated to be aggregated across views - and it cannot even + express two views of one SKU, since it keys templates by filename. + +So the gallery is embedding-based. DINOv2 ViT-S/14 over aspect-preserved crops, +cosine similarity, several views per SKU, and a rejection threshold so an unknown +product comes back as unknown instead of as the nearest stranger. + +Cost, measured on an Apple M5 (24 crops, batched and one at a time): + + 224 px CPU 25 ms/crop MPS 2.4 ms/crop + 518 px CPU 159 ms/crop MPS 3.6 ms/crop + +A live frame only needs its *new* tracks embedded (2-4 of them), so ~10 ms a frame +on the GPU. That is why this can sit in the live loop at all. +""" + +from __future__ import annotations + +import os +import threading +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +# Below this cosine similarity the nearest view is not the same product. Chosen +# from the measured distribution: same-product pairs sit above 0.6, different +# products below 0.5, and the gap is where a rejection belongs. +DEFAULT_THRESHOLD = 0.55 + + +@dataclass +class GalleryEntry: + sku_id: str + view: str + embedding: Any = None + + +@dataclass +class GalleryMatch: + sku_id: Optional[str] + similarity: float + runner_up: Optional[str] = None + runner_up_similarity: float = 0.0 + rejected: bool = False + + @property + def margin(self) -> float: + return self.similarity - self.runner_up_similarity + + +class ProductGallery: + """Several views per SKU, embedded once, queried by cosine similarity.""" + + def __init__( + self, + stock_dir: str, + sku_ids: Sequence[str], + device: Optional[str] = None, + image_size: int = 518, + threshold: float = DEFAULT_THRESHOLD, + ) -> None: + self.stock_dir = stock_dir + self.sku_ids = list(sku_ids) + self.image_size = int(image_size) + self.threshold = float(threshold) + self.device = device + self._model = None + self._lock = threading.Lock() + self._entries: List[GalleryEntry] = [] + self._matrix = None + self._available = False + + # ------------------------------------------------------------------ loading + def _load_model(self) -> Any: + if self._model is not None: + return self._model + import timm + import torch + + device = self.device + if device is None: + device = "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu" + model = timm.create_model( + "vit_small_patch14_dinov2.lvd142m", + pretrained=True, + num_classes=0, + img_size=self.image_size, + dynamic_img_size=True, + ) + model = model.eval().to(device) + self.device = device + self._model = model + return model + + def available(self) -> bool: + return self._available + + def _views_for(self, sku_id: str) -> List[str]: + """Every image in the stock directory that belongs to this SKU.""" + found: List[str] = [] + try: + names = sorted(os.listdir(self.stock_dir)) + except OSError: + return found + for name in names: + stem, extension = os.path.splitext(name) + if extension.lower() not in (".jpeg", ".jpg", ".png"): + continue + if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"): + found.append(os.path.join(self.stock_dir, name)) + return found + + def load(self) -> bool: + """Embed every enrolled view. Returns False when there is nothing to load.""" + import cv2 + import numpy as np + import torch + + paths: List[Tuple[str, str]] = [] + for sku_id in self.sku_ids: + for path in self._views_for(sku_id): + paths.append((sku_id, path)) + if not paths: + self._available = False + return False + try: + model = self._load_model() + except Exception: + self._available = False + return False + + batch = [] + for _, path in paths: + image = cv2.imread(path) + if image is None or image.size == 0: + continue + batch.append(_prepare(image, self.image_size)) + if not batch: + self._available = False + return False + tensor = torch.from_numpy(np.stack(batch)).to(self.device) + with torch.no_grad(): + features = model(tensor) + features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() + self._entries = [ + GalleryEntry(sku_id=sku_id, view=path, embedding=features[index]) + for index, (sku_id, path) in enumerate(paths) + ] + self._matrix = features + self._available = True + return True + + # ------------------------------------------------------------------ querying + def embed(self, crop: Any) -> Any: + import numpy as np + import torch + + model = self._load_model() + tensor = torch.from_numpy(np.stack([_prepare(crop, self.image_size)])).to(self.device) + with torch.no_grad(): + features = model(tensor) + features = torch.nn.functional.normalize(features, dim=1).cpu().numpy() + return features[0] + + def match(self, crop: Any, threshold: Optional[float] = None) -> GalleryMatch: + """Nearest enrolled view, with a rejection when nothing is close enough.""" + if not self._available or self._matrix is None: + return GalleryMatch(sku_id=None, similarity=0.0, rejected=True) + import numpy as np + + limit = self.threshold if threshold is None else float(threshold) + embedding = self.embed(crop) + similarity = self._matrix @ embedding + # Several views per SKU: a SKU's score is its best view. + best_by_sku: Dict[str, float] = {} + for index, entry in enumerate(self._entries): + value = float(similarity[index]) + if value > best_by_sku.get(entry.sku_id, -1.0): + best_by_sku[entry.sku_id] = value + if not best_by_sku: + return GalleryMatch(sku_id=None, similarity=0.0, rejected=True) + ranked = sorted(best_by_sku.items(), key=lambda item: item[1], reverse=True) + top_sku, top_score = ranked[0] + runner_sku, runner_score = ranked[1] if len(ranked) > 1 else (None, 0.0) + if top_score < limit: + return GalleryMatch(sku_id=None, similarity=top_score, runner_up=top_sku, + runner_up_similarity=top_score, rejected=True) + return GalleryMatch(sku_id=top_sku, similarity=top_score, runner_up=runner_sku, + runner_up_similarity=runner_score) + + def view_count(self) -> int: + return len(self._entries) + + def sku_view_counts(self) -> Dict[str, int]: + counts: Dict[str, int] = {} + for entry in self._entries: + counts[entry.sku_id] = counts.get(entry.sku_id, 0) + 1 + return counts + + +def _prepare(image: Any, size: int) -> Any: + """Aspect-preserving square canvas: the crop's shape is part of the product.""" + import cv2 + import numpy as np + + mean = np.array([0.485, 0.456, 0.406], np.float32) + std = np.array([0.229, 0.224, 0.225], np.float32) + rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + height, width = rgb.shape[:2] + side = max(height, width) + canvas = np.zeros((side, side, 3), np.uint8) + canvas[(side - height) // 2:(side - height) // 2 + height, + (side - width) // 2:(side - width) // 2 + width] = rgb + resized = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA).astype(np.float32) / 255.0 + return ((resized - mean) / std).transpose(2, 0, 1) diff --git a/recognization/code/test_gallery.py b/recognization/code/test_gallery.py new file mode 100644 index 0000000..afc6585 --- /dev/null +++ b/recognization/code/test_gallery.py @@ -0,0 +1,112 @@ +""" +Tests for the product gallery's matching logic. + +The embedding model is injected, so these run without torch or a GPU: what is +being tested is the retrieval contract - best view wins, several views per SKU +count as one answer, and a stranger is rejected rather than named. +""" + +from __future__ import annotations + +import os +import sys +import unittest + +import numpy as np + +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(current_dir) +if os.path.exists(os.path.join(project_root, "stock")): + os.chdir(project_root) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from gallery import GalleryEntry, ProductGallery # noqa: E402 + + +def a_gallery(views, threshold=0.55): + """A gallery with hand-made embeddings: no model, no files.""" + skus = sorted({sku for sku, _ in views}) + gallery = ProductGallery("does-not-exist", skus, threshold=threshold) + embeddings = [] + entries = [] + for index, (sku, vector) in enumerate(views): + vector = np.asarray(vector, dtype=np.float32) + vector /= np.linalg.norm(vector) + embeddings.append(vector) + entries.append(GalleryEntry(sku_id=sku, view=f"{sku}_{index}", embedding=vector)) + gallery._entries = entries + gallery._matrix = np.stack(embeddings) + gallery._available = True + return gallery + + +def unit(vector): + vector = np.asarray(vector, dtype=np.float32) + return (vector / np.linalg.norm(vector)).tolist() + + +class MatchingTests(unittest.TestCase): + def test_the_nearest_view_names_the_product(self): + gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))]) + gallery.embed = lambda crop: np.array([0.9, 0.1, 0.0], dtype=np.float32) + match = gallery.match(object()) + self.assertFalse(match.rejected) + self.assertEqual(match.sku_id, "sku_a") + self.assertAlmostEqual(match.similarity, 0.9, places=3) + + def test_several_views_of_one_sku_are_one_answer(self): + # The second view of sku_b is closer than any view of sku_a, but sku_a must + # still win on its best view: a SKU is not outvoted by having more photos. + gallery = a_gallery([ + ("sku_a", unit([1, 0, 0])), + ("sku_b", unit([0, 1, 0])), + ("sku_b", unit([0.2, 0.98, 0])), + ]) + gallery.embed = lambda crop: np.array([1.0, 0.05, 0.0], dtype=np.float32) + match = gallery.match(object()) + self.assertEqual(match.sku_id, "sku_a") + self.assertEqual(match.runner_up, "sku_b") + + def test_a_stranger_is_rejected_not_named(self): + gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))]) + gallery.embed = lambda crop: np.array([0.0, 0.0, 1.0], dtype=np.float32) + match = gallery.match(object()) + self.assertTrue(match.rejected) + self.assertIsNone(match.sku_id) + + def test_the_threshold_moves_the_rejection_line(self): + gallery = a_gallery([("sku_a", unit([1, 0, 0]))], threshold=0.99) + gallery.embed = lambda crop: np.array([0.9, 0.4, 0.0], dtype=np.float32) + self.assertTrue(gallery.match(object()).rejected) + self.assertFalse(gallery.match(object(), threshold=0.5).rejected) + + def test_margin_separates_a_confident_match_from_a_close_one(self): + gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0.9, 0.44, 0]))]) + gallery.embed = lambda crop: np.array([1.0, 0.1, 0.0], dtype=np.float32) + confident = gallery.match(object()) + gallery.embed = lambda crop: np.array([0.95, 0.3, 0.0], dtype=np.float32) + close = gallery.match(object()) + self.assertGreater(confident.margin, close.margin) + + def test_an_unloaded_gallery_rejects_everything(self): + gallery = ProductGallery("does-not-exist", ["sku_a"]) + match = gallery.match(object()) + self.assertTrue(match.rejected) + + +class EnrollmentTests(unittest.TestCase): + def test_views_are_found_by_sku_prefix(self): + import tempfile + + with tempfile.TemporaryDirectory() as directory: + for name in ("sku_a.jpeg", "sku_a_1.jpeg", "sku_a-2.jpeg", "sku_ab.jpeg", "notes.txt"): + open(os.path.join(directory, name), "wb").close() + gallery = ProductGallery(directory, ["sku_a", "sku_ab"]) + self.assertEqual(len(gallery._views_for("sku_a")), 3, + "one view, two extra views, and not sku_ab") + self.assertEqual(len(gallery._views_for("sku_ab")), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/recognization/stock-lopia/sku_l01_1.jpeg b/recognization/stock-lopia/sku_l01_1.jpeg new file mode 100644 index 0000000..ba0d51f Binary files /dev/null and b/recognization/stock-lopia/sku_l01_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l01_2.jpeg b/recognization/stock-lopia/sku_l01_2.jpeg new file mode 100644 index 0000000..5a079a9 Binary files /dev/null and b/recognization/stock-lopia/sku_l01_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l02_1.jpeg b/recognization/stock-lopia/sku_l02_1.jpeg new file mode 100644 index 0000000..e2e95d5 Binary files /dev/null and b/recognization/stock-lopia/sku_l02_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l02_2.jpeg b/recognization/stock-lopia/sku_l02_2.jpeg new file mode 100644 index 0000000..55dbd78 Binary files /dev/null and b/recognization/stock-lopia/sku_l02_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l02_3.jpeg b/recognization/stock-lopia/sku_l02_3.jpeg new file mode 100644 index 0000000..b3d1e4b Binary files /dev/null and b/recognization/stock-lopia/sku_l02_3.jpeg differ diff --git a/recognization/stock-lopia/sku_l03_1.jpeg b/recognization/stock-lopia/sku_l03_1.jpeg new file mode 100644 index 0000000..4760c54 Binary files /dev/null and b/recognization/stock-lopia/sku_l03_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l03_2.jpeg b/recognization/stock-lopia/sku_l03_2.jpeg new file mode 100644 index 0000000..a14ede9 Binary files /dev/null and b/recognization/stock-lopia/sku_l03_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l03_3.jpeg b/recognization/stock-lopia/sku_l03_3.jpeg new file mode 100644 index 0000000..2fcb71d Binary files /dev/null and b/recognization/stock-lopia/sku_l03_3.jpeg differ diff --git a/recognization/stock-lopia/sku_l04_1.jpeg b/recognization/stock-lopia/sku_l04_1.jpeg new file mode 100644 index 0000000..e3392e2 Binary files /dev/null and b/recognization/stock-lopia/sku_l04_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l04_2.jpeg b/recognization/stock-lopia/sku_l04_2.jpeg new file mode 100644 index 0000000..0eaa1ae Binary files /dev/null and b/recognization/stock-lopia/sku_l04_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l04_3.jpeg b/recognization/stock-lopia/sku_l04_3.jpeg new file mode 100644 index 0000000..0ae19b8 Binary files /dev/null and b/recognization/stock-lopia/sku_l04_3.jpeg differ diff --git a/recognization/stock-lopia/sku_l05_1.jpeg b/recognization/stock-lopia/sku_l05_1.jpeg new file mode 100644 index 0000000..efdabef Binary files /dev/null and b/recognization/stock-lopia/sku_l05_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l05_2.jpeg b/recognization/stock-lopia/sku_l05_2.jpeg new file mode 100644 index 0000000..25df422 Binary files /dev/null and b/recognization/stock-lopia/sku_l05_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l05_3.jpeg b/recognization/stock-lopia/sku_l05_3.jpeg new file mode 100644 index 0000000..84675cb Binary files /dev/null and b/recognization/stock-lopia/sku_l05_3.jpeg differ diff --git a/recognization/stock-lopia/sku_l07_1.jpeg b/recognization/stock-lopia/sku_l07_1.jpeg new file mode 100644 index 0000000..27d2a98 Binary files /dev/null and b/recognization/stock-lopia/sku_l07_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l07_2.jpeg b/recognization/stock-lopia/sku_l07_2.jpeg new file mode 100644 index 0000000..a6f91af Binary files /dev/null and b/recognization/stock-lopia/sku_l07_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l07_3.jpeg b/recognization/stock-lopia/sku_l07_3.jpeg new file mode 100644 index 0000000..e15f69e Binary files /dev/null and b/recognization/stock-lopia/sku_l07_3.jpeg differ diff --git a/recognization/stock-lopia/sku_l09_1.jpeg b/recognization/stock-lopia/sku_l09_1.jpeg new file mode 100644 index 0000000..d2284de Binary files /dev/null and b/recognization/stock-lopia/sku_l09_1.jpeg differ diff --git a/recognization/stock-lopia/sku_l09_2.jpeg b/recognization/stock-lopia/sku_l09_2.jpeg new file mode 100644 index 0000000..4f068c4 Binary files /dev/null and b/recognization/stock-lopia/sku_l09_2.jpeg differ diff --git a/recognization/stock-lopia/sku_l09_3.jpeg b/recognization/stock-lopia/sku_l09_3.jpeg new file mode 100644 index 0000000..14fe787 Binary files /dev/null and b/recognization/stock-lopia/sku_l09_3.jpeg differ