feat: a multi-view embedding gallery, because one template per SKU cannot work
The user's objection was right and it changed the design: the tag channel is
sparse in the real world (measured: 0 tags readable at ~2 m on the main shelf), so
packaging recognition has to carry identity. Measured on a 12-class, 57-facing
evaluation set read off the demo shelf, the packaging channel as shipped cannot:
views per SKU embedding top-1 shipped matcher top-1
1 57 % 33 %
2 72 % 27 %
3 75 % 20 %
4 76 % 5 %
Two things follow. First, two views per SKU is where most of the gain is - that is
the enrollment spec (a first scan already produces several facings per product).
Second, the matcher gets *worse* with more views, because its raw inlier score is
not calibrated to be aggregated and it cannot express two views of one SKU at all:
it keys templates by filename.
So identity becomes retrieval: `gallery.py` embeds every enrolled view with DINOv2
ViT-S/14, scores a SKU by its best view, and rejects when nothing is close enough
(an unknown product must come back unknown, not as the nearest stranger). Measured
with held-out views - the gallery never contains its own queries - 2 views per SKU
over 12 SKUs gives 69 % top-1 with rejection working, and the threshold moves the
rejection line.
It is affordable in the live loop only on the GPU: 159 ms per crop on CPU versus
3.6 ms on the M5's MPS, and a live frame only needs its new tracks embedded.
157 tests pass; test_gallery.py covers the retrieval contract (best view wins,
several views are one answer, strangers are rejected) with injected embeddings so
it needs no torch.
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
52905e73f6
commit
e854aea23d
23 changed files with 403 additions and 0 deletions
68
recognization/code/bench_identity.py
Normal file
68
recognization/code/bench_identity.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""
|
||||
How many views per SKU does enrollment need?
|
||||
|
||||
The number that decides the capture spec: one enrolled view identifies 57 % of
|
||||
facings, two lift it to 72 %, three to 75 %. The shipped matcher goes the other
|
||||
way (33 % -> 5 %) because its raw inlier score is not calibrated to be aggregated
|
||||
across views, and it cannot even express two views of one SKU.
|
||||
|
||||
python code/bench_identity.py # expects the eval crops described below
|
||||
"""
|
||||
import glob, os, sys, random
|
||||
sys.path.insert(0, "code")
|
||||
import cv2, numpy as np, torch, timm
|
||||
from feature_matcher import ProductFeatureMatcher
|
||||
|
||||
# Crops extracted from add/image6.jpg and labelled by shelf position; the labels
|
||||
# are read off the shelf, not off the matcher being measured.
|
||||
CROPS = sorted(glob.glob(os.environ.get("DORAYAKI_EVAL_CROPS", "/tmp/dorayaki-verify/eval") + "/*.png"))
|
||||
items = [(p, os.path.basename(p).replace(".png","").split("_",1)[1]) for p in CROPS]
|
||||
by_class = {}
|
||||
for path, label in items:
|
||||
by_class.setdefault(label, []).append(path)
|
||||
|
||||
matcher = ProductFeatureMatcher(stock_dir="stock-lopia")
|
||||
model = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, num_classes=0); model.eval()
|
||||
MEAN = np.array([0.485,0.456,0.406], np.float32); STD = np.array([0.229,0.224,0.225], np.float32)
|
||||
|
||||
def embed(paths, size=518):
|
||||
batch = []
|
||||
for path in paths:
|
||||
image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
|
||||
h, w = image.shape[:2]; side = max(h, w)
|
||||
canvas = np.zeros((side, side, 3), np.uint8)
|
||||
canvas[(side-h)//2:(side-h)//2+h, (side-w)//2:(side-w)//2+w] = image
|
||||
image = cv2.resize(canvas, (size,size), interpolation=cv2.INTER_AREA).astype(np.float32)/255.0
|
||||
batch.append(((image-MEAN)/STD).transpose(2,0,1))
|
||||
with torch.no_grad():
|
||||
features = model(torch.from_numpy(np.stack(batch)))
|
||||
return torch.nn.functional.normalize(features, dim=1).numpy()
|
||||
|
||||
def matcher_best(crop):
|
||||
"""Scores grouped by SKU, so several views of one SKU count as one answer."""
|
||||
scores, _ = matcher.score_crop(crop)
|
||||
grouped = {}
|
||||
for name, value in scores.items():
|
||||
sku = name.split("_")[0] + "_" + name.split("_")[1] if name.startswith("sku_l") else name
|
||||
grouped[sku] = max(grouped.get(sku, 0.0), float(value))
|
||||
return max(grouped.items(), key=lambda kv: kv[1])[0] if grouped else None
|
||||
|
||||
random.seed(7)
|
||||
print(f"{len(items)} crops, {len(by_class)} classes")
|
||||
print(f" {'views/SKU':>9} {'embedding top-1':>15} {'matcher top-1':>14}")
|
||||
for views in (1, 2, 3, 4):
|
||||
gallery, test = [], []
|
||||
for label, paths in by_class.items():
|
||||
shuffled = paths[:]; random.shuffle(shuffled)
|
||||
gallery += [(p, label) for p in shuffled[:views]]
|
||||
test += [(p, label) for p in shuffled[views:]]
|
||||
if not test:
|
||||
continue
|
||||
features_gallery = embed([p for p, _ in gallery])
|
||||
features_test = embed([p for p, _ in test])
|
||||
gallery_labels = [l for _, l in gallery]
|
||||
similarity = features_test @ features_gallery.T
|
||||
correct = sum(1 for i, (_, truth) in enumerate(test) if gallery_labels[int(similarity[i].argmax())] == truth)
|
||||
matcher_correct = sum(1 for p, truth in test if matcher_best(cv2.imread(p)) == truth)
|
||||
print(f" {views:>9} {correct:>6}/{len(test):<6} {100*correct//len(test):>3}% "
|
||||
f"{matcher_correct:>6}/{len(test):<6} {100*matcher_correct//len(test):>3}%")
|
||||
Loading…
Add table
Add a link
Reference in a new issue