feat(identity): local-feature matcher plus text lane as the default backend
Replaces the DINO gallery as the recognition path with the measured design from docs/research/2026-09-28-recognition-architecture/DECISION.md: RootSIFT over a per-SKU reference pyramid, similarity-constrained RANSAC and a hue check, fused with an OCR text lane that vetoes a match whose package text names a different catalog row and answers on its own when geometry finds nothing. - identity_lanes.py: LocalLane, TextLane, DecisionPolicy. The text lane builds each row's signature from the catalog name plus its packshot OCR, so a row with no enrolled image is still answerable. - engine.py: every backend groups scores by canonical SKU, so no view-suffixed id reaches the wire and a name and price always resolve. decide_crop fuses the lanes; score_crop_lanes feeds the live path. Both lanes are built in warm_up (non-fatally) so the first frame is not the slow one, and the text lane shares the engine's OCR lock. - live.py: decides per crop on the lane's own rank order. The lane's order comes from its rank key and its value is only the gate, so re-sorting by value named a different SKU than the lane chose; averaging in an empty observation also diluted a good confidence below the accept line. - identity.py: a `local` score scale whose gate is one absolute line, because the value is already a margin over the runner-up. - app.py, run-demo-server.sh: `--identity local` is the default; the demo server no longer starts on the leaked `stock-lopia` catalog. - eval_live.py: the replay vocabulary accepts `local`, so a recording from the shipped server can be scored. - README.md: documents the three backends and what each is for. Measured on the corrected image6 labels at live resolution: 36/61 in-catalog facings named at 95% precision, 0 of 48 unknowns accepted, 11 labels sourced from the text lane alone. Through the real server: /v1/recognize 90% precision, /v1/stream 100% precision, no view-suffixed ids on either.
This commit is contained in:
parent
25dbf132e7
commit
aaed5d388f
44 changed files with 7847 additions and 60 deletions
|
|
@ -85,11 +85,13 @@ class EngineSettings:
|
|||
# over a nearly static shelf wants more; that is a CLI override, not a default.
|
||||
live_inflight: int = 16
|
||||
live_attempts: int = 2
|
||||
# Which mechanism names products: the hand-crafted local-feature matcher, or the
|
||||
# embedding gallery with a calibrated rejection. Measured on 54 unseen store
|
||||
# photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its
|
||||
# raw inlier score cannot be thresholded; the gallery can say "unknown".
|
||||
identity_backend: str = "matcher"
|
||||
# Which mechanism names products. `local` is the RootSIFT lane behind a
|
||||
# calibrated open-set gate: measured on the corrected image6 labels it shows
|
||||
# 36 of 61 in-catalog facings at 95% precision with the text lane, against
|
||||
# 35 at 35% for Van's always-argmax matcher. `matcher` keeps Van's composite
|
||||
# as a selectable baseline lane (E-1). `gallery` is fenced off: raw DINOv2
|
||||
# is 12-21% top-1 on 7 SKUs and its recall@10 falls to 7/42 at 493 SKUs.
|
||||
identity_backend: str = "local"
|
||||
# Cosine similarity below which the gallery calls a facing unknown. Measured:
|
||||
# same-product pairs sit above 0.6, different products below 0.5.
|
||||
gallery_threshold: float = 0.55
|
||||
|
|
@ -121,6 +123,24 @@ class StoredImage:
|
|||
tag_bundle: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
def group_by_sku(scores: Dict[str, float]) -> Dict[str, float]:
|
||||
"""Collapse per-view template names onto their canonical SKU.
|
||||
|
||||
A SKU's score is its best view, which is also how the gallery ranks several
|
||||
views. `sku_l01_1` and `sku_l01_2` become one `sku_l01`, so two views of one
|
||||
product can no longer compete with each other and no view-suffixed id can
|
||||
reach the wire.
|
||||
"""
|
||||
from rebuild_catalog import sku_of
|
||||
|
||||
grouped: Dict[str, float] = {}
|
||||
for name, value in (scores or {}).items():
|
||||
number = float(value)
|
||||
sku = sku_of(str(name))
|
||||
grouped[sku] = max(grouped.get(sku, 0.0), number)
|
||||
return grouped
|
||||
|
||||
|
||||
def identity_state_from_level(level: Optional[str]) -> str:
|
||||
"""Map a decide_identity() level to the wire state vocabulary."""
|
||||
if level == "accepted":
|
||||
|
|
@ -252,6 +272,13 @@ class RecognitionEngine:
|
|||
# enrolled view), and cached as False when it cannot be built at all.
|
||||
self._gallery: Any = None
|
||||
self._gallery_lock = threading.Lock()
|
||||
# The local identity lane, built on first use. It loads every reference
|
||||
# view and its descriptors, which is why it is lazy like the gallery.
|
||||
self._lane: Any = None
|
||||
self._lane_lock = threading.Lock()
|
||||
# The text lane, built on first use from the catalog and the OCR engine.
|
||||
self._text_lane: Any = None
|
||||
self._text_lane_lock = threading.Lock()
|
||||
workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2)
|
||||
self.classify_workers = int(workers)
|
||||
self._executor = classify_executor or ThreadPoolExecutor(
|
||||
|
|
@ -301,12 +328,25 @@ class RecognitionEngine:
|
|||
return self.ocr_engine.read(image)
|
||||
|
||||
def warm_up(self) -> float:
|
||||
"""One dummy predict so the first real frame is not the slow one."""
|
||||
"""One dummy predict so the first real frame is not the slow one.
|
||||
|
||||
The identity lanes are built here too. `TextLane.from_catalog` OCRs
|
||||
every enrolled packshot, which is seconds of work; left lazy, the first
|
||||
live frame would pay it inside a classify worker while the phone waits.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
blank = np.zeros((self.settings.imgsz, self.settings.imgsz, 3), dtype="uint8")
|
||||
started = time.perf_counter()
|
||||
self.detect(blank, conf=self.settings.detect_conf)
|
||||
# A lane that cannot build here is not fatal: the lazy builders retry it
|
||||
# on first use, so a bad stock dir or a Vision failure degrades the
|
||||
# backend instead of refusing to start the server.
|
||||
for name, build in (("local", self.identity_lane), ("text", self.identity_text_lane)):
|
||||
try:
|
||||
build()
|
||||
except Exception as exc:
|
||||
LOGGER.warning("identity %s lane not warmed: %s", name, exc)
|
||||
return (time.perf_counter() - started) * 1000.0
|
||||
|
||||
def shutdown(self) -> None:
|
||||
|
|
@ -418,14 +458,113 @@ class RecognitionEngine:
|
|||
# ------------------------------------------------------------------ classification
|
||||
@property
|
||||
def identity_scale(self) -> str:
|
||||
"""What the scores from `score_crop` mean: inlier counts or cosine similarity."""
|
||||
return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier"
|
||||
"""What the scores from `score_crop` mean.
|
||||
|
||||
`local` is the RootSIFT lane's margin-over-runner-up, `cosine` the
|
||||
gallery's similarity, `inlier` Van's composite. The live path reads this
|
||||
to pick the matching gate, so the three must never be mixed.
|
||||
"""
|
||||
backend = (self.settings.identity_backend or "matcher").lower()
|
||||
if backend == "gallery":
|
||||
return "cosine"
|
||||
if backend == "local":
|
||||
return "local"
|
||||
return "inlier"
|
||||
|
||||
def identity_text_lane(self) -> Any:
|
||||
"""The text lane, built once from the catalog. None without OCR.
|
||||
|
||||
Signatures come from the catalog rows (their printed names plus the OCR
|
||||
of their enrolled packshots), never from an evaluation photo, so a row
|
||||
with no image can still be named. That is what lets the lane reject the
|
||||
near-unknown siblings no geometric threshold can separate.
|
||||
"""
|
||||
if self.identity_scale != "local":
|
||||
return None
|
||||
if self._text_lane is None:
|
||||
with self._text_lane_lock:
|
||||
if self._text_lane is None:
|
||||
from identity_lanes import TextLane
|
||||
|
||||
self._text_lane = TextLane.from_catalog(
|
||||
self.catalog(), self.settings.stock_dir, self.ocr_engine,
|
||||
lock=self.ocr_lock,
|
||||
)
|
||||
LOGGER.info("identity text lane: %d signatures", len(self._text_lane.signatures))
|
||||
return self._text_lane
|
||||
|
||||
def decide_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""One identity decision for one crop, fusing every lane that is on.
|
||||
|
||||
The local lane answers first and the text lane vetoes or fills in, which
|
||||
is the measured design: geometry ranks fine-grained siblings correctly
|
||||
and text is the only thing that rejects a sibling the catalog does not
|
||||
know about. Without OCR the local lane stands alone.
|
||||
"""
|
||||
from identity import decide_identity
|
||||
from identity_lanes import DecisionPolicy
|
||||
|
||||
if self.identity_scale != "local":
|
||||
return decide_identity(
|
||||
self.score_crop(crop, method), method=method or self.settings.live_method,
|
||||
scale=self.identity_scale,
|
||||
)
|
||||
lane = self.identity_lane()
|
||||
if lane is None:
|
||||
return decide_identity({}, method="local", scale="local")
|
||||
text_lane = self.identity_text_lane()
|
||||
text_rank: Any = None
|
||||
if text_lane is not None:
|
||||
read = text_lane.read(crop)
|
||||
text_rank = sorted(
|
||||
text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
|
||||
)
|
||||
return DecisionPolicy().decide(lane.rank(crop), text_rank)
|
||||
|
||||
def score_crop_lanes(
|
||||
self, crop: Any, method: Optional[str] = None
|
||||
) -> Tuple[Dict[str, float], List[Tuple[str, float]]]:
|
||||
"""Both lanes' evidence for one crop: local scores and text ranking.
|
||||
|
||||
The live path accumulates evidence across looks, so it needs the lanes
|
||||
separately rather than one fused decision: it averages the local scores
|
||||
over the track's window and keeps the newest text reading, then decides
|
||||
once. The still path calls `decide_crop`, which fuses a single crop.
|
||||
"""
|
||||
scores = self.score_crop(crop, method)
|
||||
text_lane = self.identity_text_lane()
|
||||
if text_lane is None:
|
||||
return scores, []
|
||||
read = text_lane.read(crop)
|
||||
return scores, sorted(
|
||||
text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
|
||||
)
|
||||
|
||||
def identity_lane(self) -> Any:
|
||||
"""The local lane, built once. None when the backend is not `local`."""
|
||||
if self.identity_scale != "local":
|
||||
return None
|
||||
if self._lane is None:
|
||||
with self._lane_lock:
|
||||
if self._lane is None:
|
||||
from identity_lanes import LocalLane
|
||||
|
||||
self._lane = LocalLane(self.settings.stock_dir)
|
||||
LOGGER.info(
|
||||
"identity lane: %d reference views over %d SKUs",
|
||||
self._lane.view_count(),
|
||||
len(self._lane.sku_ids),
|
||||
)
|
||||
return self._lane
|
||||
|
||||
def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Score one crop. Gallery never falls back to the matcher.
|
||||
"""Score one crop, keyed by canonical SKU.
|
||||
|
||||
A missing gallery returns no scores (unknown), not inlier counts. Mixing
|
||||
those scales is how a live scan accepted a look-alike the still path rejected.
|
||||
Every backend groups by SKU before returning. Scoring each reference
|
||||
view separately is what made the demo's default catalog report 22-29 of
|
||||
109 facings as "ambiguous" when the top two candidates were two views of
|
||||
the same product, and what put view-suffixed ids like `sku_l01_1` on the
|
||||
wire, where no catalog row exists to supply a name or price.
|
||||
"""
|
||||
if self.identity_scale == "cosine":
|
||||
gallery = self.identity_gallery()
|
||||
|
|
@ -438,6 +577,11 @@ class RecognitionEngine:
|
|||
if match.runner_up:
|
||||
scores[match.runner_up] = match.runner_up_similarity
|
||||
return {str(sku): float(score) for sku, score in scores.items()}
|
||||
if self.identity_scale == "local":
|
||||
lane = self.identity_lane()
|
||||
if lane is None:
|
||||
return {}
|
||||
return {sku: float(value) for sku, value in lane.rank(crop)}
|
||||
method = (method or self.settings.live_method).lower()
|
||||
if method not in CLASSIFY_METHODS:
|
||||
raise UnknownMethod(method)
|
||||
|
|
@ -445,7 +589,7 @@ class RecognitionEngine:
|
|||
scores, _ = self.matcher.score_crop_fast(crop)
|
||||
else:
|
||||
scores, _ = self.matcher.score_crop(crop, method=method)
|
||||
return {str(sku): float(score) for sku, score in scores.items()}
|
||||
return group_by_sku(scores)
|
||||
|
||||
def identity_gallery(self) -> Any:
|
||||
"""The embedding gallery, loaded once (None when unavailable)."""
|
||||
|
|
@ -494,8 +638,6 @@ class RecognitionEngine:
|
|||
method: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Classifies every box in parallel; returns one entry per box, ordered by area."""
|
||||
from identity import decide_identity
|
||||
|
||||
method = (method or self.settings.live_method).lower()
|
||||
if method not in CLASSIFY_METHODS:
|
||||
raise UnknownMethod(method)
|
||||
|
|
@ -511,33 +653,37 @@ class RecognitionEngine:
|
|||
ordered = sorted(boxes, key=area_of, reverse=True)
|
||||
crops = [self.crop(image, box["box"]) for box in ordered]
|
||||
|
||||
if (self.settings.identity_backend or "matcher").lower() == "gallery":
|
||||
if self.identity_scale == "cosine":
|
||||
gallery = self.identity_gallery()
|
||||
# No matcher fallback: an unavailable gallery is unrecognized, not a different model.
|
||||
return self._classify_with_gallery(gallery, ordered, crops)
|
||||
if method == "hybrid_fast":
|
||||
futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops]
|
||||
else:
|
||||
futures = [self._executor.submit(self.matcher.score_crop, crop, method) for crop in crops]
|
||||
from identity import decide_identity
|
||||
|
||||
# Every backend goes through decide_crop, so the scores are keyed by
|
||||
# canonical SKU and the text lane is fused in whichever lane produced them.
|
||||
futures = [self._executor.submit(self.decide_crop, crop, method) for crop in crops]
|
||||
results: List[Dict[str, Any]] = []
|
||||
for box, future in zip(ordered, futures):
|
||||
try:
|
||||
scores, details = future.result()
|
||||
decision = future.result()
|
||||
except Exception as exc: # a single bad crop must not fail the batch
|
||||
scores, details = {}, {"error": str(exc)}
|
||||
decision = decide_identity(scores, details=details, method=method)
|
||||
LOGGER.debug("classification failed for box %s: %s", box.get("id"), exc)
|
||||
# A rejected decision, not the "not yet classified" payload: an
|
||||
# operational failure must stay distinguishable from absence.
|
||||
decision = decide_identity({}, method=method, scale=self.identity_scale)
|
||||
decision = identity_payload(decision)
|
||||
results.append(
|
||||
{
|
||||
"detection_id": int(box["id"]),
|
||||
"box": list(box["box"]),
|
||||
"identity": identity_payload(decision),
|
||||
"identity": decision,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Full single-crop result, using the same backend as recognize/live."""
|
||||
from identity import decide_identity, decision_for_match
|
||||
from identity import decision_for_match
|
||||
|
||||
crop = self.crop(image, box)
|
||||
if self.identity_scale == "cosine":
|
||||
|
|
@ -547,17 +693,11 @@ class RecognitionEngine:
|
|||
# with the published cosine gates must not invent a second line.
|
||||
decision = decision_for_match(match)
|
||||
return self._inspect_payload(box, "gallery", decision, crop, visualize=False)
|
||||
|
||||
method = (method or self.settings.live_method).lower()
|
||||
if method not in CLASSIFY_METHODS:
|
||||
raise UnknownMethod(method)
|
||||
if method == "hybrid_fast":
|
||||
scores, details = self.matcher.score_crop_fast(crop)
|
||||
visual_method = "hybrid"
|
||||
else:
|
||||
scores, details = self.matcher.score_crop(crop, method=method)
|
||||
visual_method = method
|
||||
decision = decide_identity(scores, details=details, method=method)
|
||||
decision = self.decide_crop(crop, method)
|
||||
visual_method = "hybrid" if method == "hybrid_fast" else method
|
||||
return self._inspect_payload(box, method, decision, crop, visual_method=visual_method)
|
||||
|
||||
def _inspect_payload(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue