feat: an embedding identity backend with a calibrated rejection
The external evaluation showed the matcher cannot say "unknown": on 20 shelves whose
products are absent it claimed 47.8% of facings as recognised, and its raw inlier
score cannot be thresholded (foreign shelves' median 6.0 overlaps the real shelf's
5.2 p10, and the worst offenders score 50-190). So the gallery becomes a selectable
identity backend:
--identity gallery (or DORAYAKI_IDENTITY=gallery)
Measured on the same 20 foreign shelves: false accepts fall from 47.8% to 13.8% at
the default threshold and to **5%** at 0.60, with a smooth curve to pick an operating
point from. Detection is unchanged (the gallery only names boxes).
What it does NOT fix, measured on a milk shelf where 14 of 107 facings really are
明治 おいしい牛乳 (labelled by eye): precision is 28-45%, no better than the matcher's
39%, and enrolling the shelf from itself (1-3 views per product) is flat at ~25%
top-1. Off-the-shelf DINOv2 is not discriminative enough for fine-grained SKUs - the
same conclusion the literature reaches, now measured here.
The packaging's own text is a real channel and resolution-bound: 51% recall
single-scale, 60% with the tag reader's multi-scale + red-channel trick, 85% where
the text is large (MEGMILK) and 28% where it is small vertical Japanese.
docs/EXTERNAL-EVAL.md records all of it, including the identity architecture it
implies: reject by default, name from the tag when readable, from the packaging text
when legible, from a fine-tuned embedding otherwise, and hand the rest to the
operator.
175 tests pass.
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w
parent
f74366d6a4
commit
7d01de8062
3 changed files with 207 additions and 0 deletions
|
|
@ -21,6 +21,7 @@ from __future__ import annotations
|
|||
|
||||
import base64
|
||||
import collections
|
||||
import logging
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
|
|
@ -50,6 +51,9 @@ STATE_UNRECOGNIZED = "unrecognized"
|
|||
CLASSIFY_METHODS = ("surf", "orb", "brisk", "hybrid", "hybrid_fast")
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("dorayaki.engine")
|
||||
|
||||
|
||||
class UndecodableImage(ValueError):
|
||||
"""Raised when an uploaded payload is not a readable image."""
|
||||
|
||||
|
|
@ -81,6 +85,15 @@ class EngineSettings:
|
|||
# over a nearly static shelf wants more; that is a CLI override, not a default.
|
||||
live_inflight: int = 16
|
||||
live_attempts: int = 2
|
||||
# Which mechanism names products: the hand-crafted local-feature matcher, or the
|
||||
# embedding gallery with a calibrated rejection. Measured on 54 unseen store
|
||||
# photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its
|
||||
# raw inlier score cannot be thresholded; the gallery can say "unknown".
|
||||
identity_backend: str = "matcher"
|
||||
# Cosine similarity below which the gallery calls a facing unknown. Measured:
|
||||
# same-product pairs sit above 0.6, different products below 0.5.
|
||||
gallery_threshold: float = 0.55
|
||||
gallery_image_size: int = 518
|
||||
imgsz: int = 640
|
||||
# 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine).
|
||||
classify_workers: int = 5
|
||||
|
|
@ -234,6 +247,9 @@ class RecognitionEngine:
|
|||
self.matcher = matcher
|
||||
self.inventory = inventory
|
||||
self.settings = settings or EngineSettings()
|
||||
# The embedding gallery is built on first use (it loads a model and every
|
||||
# enrolled view), and cached as False when it cannot be built at all.
|
||||
self._gallery: Any = None
|
||||
workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2)
|
||||
self.classify_workers = int(workers)
|
||||
self._executor = classify_executor or ThreadPoolExecutor(
|
||||
|
|
@ -404,6 +420,29 @@ class RecognitionEngine:
|
|||
scores, _ = self.matcher.score_crop(crop, method=method)
|
||||
return {str(sku): float(score) for sku, score in scores.items()}
|
||||
|
||||
def identity_gallery(self) -> Any:
|
||||
"""The embedding gallery, loaded once (None when unavailable)."""
|
||||
if self._gallery is not None:
|
||||
return self._gallery
|
||||
from gallery import ProductGallery
|
||||
|
||||
gallery = ProductGallery(
|
||||
self.settings.stock_dir,
|
||||
[product["sku_id"] for product in self.catalog()],
|
||||
threshold=self.settings.gallery_threshold,
|
||||
image_size=self.settings.gallery_image_size,
|
||||
)
|
||||
try:
|
||||
loaded = gallery.load()
|
||||
except Exception as exc: # torch/timm missing, no weights, no network
|
||||
LOGGER.warning("identity gallery unavailable: %s", exc)
|
||||
loaded = False
|
||||
self._gallery = gallery if loaded else False
|
||||
if loaded:
|
||||
LOGGER.info("identity gallery: %d views over %d SKUs on %s",
|
||||
gallery.view_count(), len(self.settings.stock_dir and gallery.sku_ids), gallery.device)
|
||||
return self._gallery
|
||||
|
||||
def classify_boxes(
|
||||
self,
|
||||
image: Any,
|
||||
|
|
@ -427,6 +466,11 @@ class RecognitionEngine:
|
|||
|
||||
ordered = sorted(boxes, key=area_of, reverse=True)
|
||||
crops = [self.crop(image, box["box"]) for box in ordered]
|
||||
|
||||
if (self.settings.identity_backend or "matcher").lower() == "gallery":
|
||||
gallery = self.identity_gallery()
|
||||
if gallery:
|
||||
return self._classify_with_gallery(gallery, ordered, crops)
|
||||
if method == "hybrid_fast":
|
||||
futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops]
|
||||
else:
|
||||
|
|
@ -643,6 +687,64 @@ class RecognitionEngine:
|
|||
},
|
||||
}
|
||||
|
||||
def _classify_with_gallery(
|
||||
self,
|
||||
gallery: Any,
|
||||
ordered: Sequence[Dict[str, Any]],
|
||||
crops: Sequence[Any],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Identity by retrieval: nearest enrolled view, or an honest unknown.
|
||||
|
||||
The gallery scores a SKU by its best view and rejects when nothing is close
|
||||
enough, which is the behaviour the matcher cannot provide - measured on 54
|
||||
unseen store photos, the matcher labelled milk cartons as mayonnaise and its
|
||||
score could not be thresholded to stop it.
|
||||
"""
|
||||
results: List[Dict[str, Any]] = []
|
||||
for box, crop in zip(ordered, crops):
|
||||
try:
|
||||
match = gallery.match(crop)
|
||||
except Exception as exc: # a bad crop must not fail the batch
|
||||
match = None
|
||||
LOGGER.debug("gallery match failed: %s", exc)
|
||||
if match is None or match.rejected:
|
||||
similarity = float(match.similarity) if match is not None else 0.0
|
||||
results.append({
|
||||
"detection_id": int(box["id"]),
|
||||
"box": list(box["box"]),
|
||||
"identity": {
|
||||
"state": "unrecognized",
|
||||
"sku_id": None,
|
||||
"identity_level": "rejected",
|
||||
"confidence": round(similarity, 3),
|
||||
"score": round(similarity, 3),
|
||||
"candidates": [],
|
||||
"attempts": 1,
|
||||
"method": "gallery",
|
||||
"message": "no enrolled product is close enough",
|
||||
},
|
||||
})
|
||||
continue
|
||||
runner_up = (
|
||||
{"sku_id": match.runner_up, "score": round(match.runner_up_similarity, 3)}
|
||||
if match.runner_up else {}
|
||||
)
|
||||
results.append({
|
||||
"detection_id": int(box["id"]),
|
||||
"box": list(box["box"]),
|
||||
"identity": {
|
||||
"state": "recognized",
|
||||
"sku_id": match.sku_id,
|
||||
"identity_level": "accepted",
|
||||
"confidence": round(match.similarity, 3),
|
||||
"score": round(match.similarity, 3),
|
||||
"candidates": [runner_up] if runner_up else [],
|
||||
"attempts": 1,
|
||||
"method": "gallery",
|
||||
},
|
||||
})
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def summarize(
|
||||
detections: Sequence[Dict[str, Any]],
|
||||
|
|
@ -742,6 +844,7 @@ def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings:
|
|||
live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)),
|
||||
live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)),
|
||||
live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)),
|
||||
identity_backend=env.get("DORAYAKI_IDENTITY", base.identity_backend),
|
||||
detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)),
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue