e854aea23d
The user's objection was right and it changed the design: the tag channel is
sparse in the real world (measured: 0 tags readable at ~2 m on the main shelf), so
packaging recognition has to carry identity. Measured on a 12-class, 57-facing
evaluation set read off the demo shelf, the packaging channel as shipped cannot:
views per SKU embedding top-1 shipped matcher top-1
1 57 % 33 %
2 72 % 27 %
3 75 % 20 %
4 76 % 5 %
Two things follow. First, two views per SKU is where most of the gain is - that is
the enrollment spec (a first scan already produces several facings per product).
Second, the matcher gets *worse* with more views, because its raw inlier score is
not calibrated to be aggregated and it cannot express two views of one SKU at all:
it keys templates by filename.
So identity becomes retrieval: `gallery.py` embeds every enrolled view with DINOv2
ViT-S/14, scores a SKU by its best view, and rejects when nothing is close enough
(an unknown product must come back unknown, not as the nearest stranger). Measured
with held-out views - the gallery never contains its own queries - 2 views per SKU
over 12 SKUs gives 69 % top-1 with rejection working, and the threshold moves the
rejection line.
It is affordable in the live loop only on the GPU: 159 ms per crop on CPU versus
3.6 ms on the M5's MPS, and a live frame only needs its new tracks embedded.
157 tests pass; test_gallery.py covers the retrieval contract (best view wins,
several views are one answer, strangers are rejected) with injected embeddings so
it needs no torch.
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
112 lines
4.5 KiB
Python
112 lines
4.5 KiB
Python
"""
|
|
Tests for the product gallery's matching logic.
|
|
|
|
The embedding model is injected, so these run without torch or a GPU: what is
|
|
being tested is the retrieval contract - best view wins, several views per SKU
|
|
count as one answer, and a stranger is rejected rather than named.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import unittest
|
|
|
|
import numpy as np
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
project_root = os.path.dirname(current_dir)
|
|
if os.path.exists(os.path.join(project_root, "stock")):
|
|
os.chdir(project_root)
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
from gallery import GalleryEntry, ProductGallery # noqa: E402
|
|
|
|
|
|
def a_gallery(views, threshold=0.55):
|
|
"""A gallery with hand-made embeddings: no model, no files."""
|
|
skus = sorted({sku for sku, _ in views})
|
|
gallery = ProductGallery("does-not-exist", skus, threshold=threshold)
|
|
embeddings = []
|
|
entries = []
|
|
for index, (sku, vector) in enumerate(views):
|
|
vector = np.asarray(vector, dtype=np.float32)
|
|
vector /= np.linalg.norm(vector)
|
|
embeddings.append(vector)
|
|
entries.append(GalleryEntry(sku_id=sku, view=f"{sku}_{index}", embedding=vector))
|
|
gallery._entries = entries
|
|
gallery._matrix = np.stack(embeddings)
|
|
gallery._available = True
|
|
return gallery
|
|
|
|
|
|
def unit(vector):
|
|
vector = np.asarray(vector, dtype=np.float32)
|
|
return (vector / np.linalg.norm(vector)).tolist()
|
|
|
|
|
|
class MatchingTests(unittest.TestCase):
|
|
def test_the_nearest_view_names_the_product(self):
|
|
gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))])
|
|
gallery.embed = lambda crop: np.array([0.9, 0.1, 0.0], dtype=np.float32)
|
|
match = gallery.match(object())
|
|
self.assertFalse(match.rejected)
|
|
self.assertEqual(match.sku_id, "sku_a")
|
|
self.assertAlmostEqual(match.similarity, 0.9, places=3)
|
|
|
|
def test_several_views_of_one_sku_are_one_answer(self):
|
|
# The second view of sku_b is closer than any view of sku_a, but sku_a must
|
|
# still win on its best view: a SKU is not outvoted by having more photos.
|
|
gallery = a_gallery([
|
|
("sku_a", unit([1, 0, 0])),
|
|
("sku_b", unit([0, 1, 0])),
|
|
("sku_b", unit([0.2, 0.98, 0])),
|
|
])
|
|
gallery.embed = lambda crop: np.array([1.0, 0.05, 0.0], dtype=np.float32)
|
|
match = gallery.match(object())
|
|
self.assertEqual(match.sku_id, "sku_a")
|
|
self.assertEqual(match.runner_up, "sku_b")
|
|
|
|
def test_a_stranger_is_rejected_not_named(self):
|
|
gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0, 1, 0]))])
|
|
gallery.embed = lambda crop: np.array([0.0, 0.0, 1.0], dtype=np.float32)
|
|
match = gallery.match(object())
|
|
self.assertTrue(match.rejected)
|
|
self.assertIsNone(match.sku_id)
|
|
|
|
def test_the_threshold_moves_the_rejection_line(self):
|
|
gallery = a_gallery([("sku_a", unit([1, 0, 0]))], threshold=0.99)
|
|
gallery.embed = lambda crop: np.array([0.9, 0.4, 0.0], dtype=np.float32)
|
|
self.assertTrue(gallery.match(object()).rejected)
|
|
self.assertFalse(gallery.match(object(), threshold=0.5).rejected)
|
|
|
|
def test_margin_separates_a_confident_match_from_a_close_one(self):
|
|
gallery = a_gallery([("sku_a", unit([1, 0, 0])), ("sku_b", unit([0.9, 0.44, 0]))])
|
|
gallery.embed = lambda crop: np.array([1.0, 0.1, 0.0], dtype=np.float32)
|
|
confident = gallery.match(object())
|
|
gallery.embed = lambda crop: np.array([0.95, 0.3, 0.0], dtype=np.float32)
|
|
close = gallery.match(object())
|
|
self.assertGreater(confident.margin, close.margin)
|
|
|
|
def test_an_unloaded_gallery_rejects_everything(self):
|
|
gallery = ProductGallery("does-not-exist", ["sku_a"])
|
|
match = gallery.match(object())
|
|
self.assertTrue(match.rejected)
|
|
|
|
|
|
class EnrollmentTests(unittest.TestCase):
|
|
def test_views_are_found_by_sku_prefix(self):
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
for name in ("sku_a.jpeg", "sku_a_1.jpeg", "sku_a-2.jpeg", "sku_ab.jpeg", "notes.txt"):
|
|
open(os.path.join(directory, name), "wb").close()
|
|
gallery = ProductGallery(directory, ["sku_a", "sku_ab"])
|
|
self.assertEqual(len(gallery._views_for("sku_a")), 3,
|
|
"one view, two extra views, and not sku_ab")
|
|
self.assertEqual(len(gallery._views_for("sku_ab")), 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|