tngo
aaed5d388f
Replaces the DINO gallery as the recognition path with the measured design from docs/research/2026-09-28-recognition-architecture/DECISION.md: RootSIFT over a per-SKU reference pyramid, similarity-constrained RANSAC and a hue check, fused with an OCR text lane that vetoes a match whose package text names a different catalog row and answers on its own when geometry finds nothing. - identity_lanes.py: LocalLane, TextLane, DecisionPolicy. The text lane builds each row's signature from the catalog name plus its packshot OCR, so a row with no enrolled image is still answerable. - engine.py: every backend groups scores by canonical SKU, so no view-suffixed id reaches the wire and a name and price always resolve. decide_crop fuses the lanes; score_crop_lanes feeds the live path. Both lanes are built in warm_up (non-fatally) so the first frame is not the slow one, and the text lane shares the engine's OCR lock. - live.py: decides per crop on the lane's own rank order. The lane's order comes from its rank key and its value is only the gate, so re-sorting by value named a different SKU than the lane chose; averaging in an empty observation also diluted a good confidence below the accept line. - identity.py: a `local` score scale whose gate is one absolute line, because the value is already a margin over the runner-up. - app.py, run-demo-server.sh: `--identity local` is the default; the demo server no longer starts on the leaked `stock-lopia` catalog. - eval_live.py: the replay vocabulary accepts `local`, so a recording from the shipped server can be scored. - README.md: documents the three backends and what each is for. Measured on the corrected image6 labels at live resolution: 36/61 in-catalog facings named at 95% precision, 0 of 48 unknowns accepted, 11 labels sourced from the text lane alone. Through the real server: /v1/recognize 90% precision, /v1/stream 100% precision, no view-suffixed ids on either.
1069 lines
44 KiB
Python
1069 lines
44 KiB
Python
"""
|
|
Recognition engine for the Dorayaki shelf-recognition server.
|
|
|
|
One process-wide, thread-safe engine. Every call is stateless from the caller's
|
|
point of view: the active image is never stored here, so any number of clients
|
|
can call concurrently.
|
|
|
|
Owns:
|
|
- the YOLO product detector (single instance, every predict under a lock),
|
|
- the feature-matching classifier (thread-local OpenCV extractors, a classify
|
|
thread pool),
|
|
- the stock catalog (InventoryManager, mutations under a lock),
|
|
- empty-slot geometry (resolution-normalized),
|
|
- a small LRU store of recent still images for later classify/inspect calls.
|
|
|
|
Heavy imports (torch, ultralytics, cv2) are pulled in lazily by
|
|
build_default_engine so tests can inject fakes without loading weights.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import collections
|
|
import logging
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass, field, replace
|
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
|
# Keep the demo box off the network before ultralytics is imported anywhere.
|
|
os.environ.setdefault("YOLO_OFFLINE", "1")
|
|
os.environ.setdefault("YOLO_AUTOINSTALL", "false")
|
|
|
|
COORDINATE_CONVENTION = "pixel-xyxy-origin-top-left"
|
|
REFERENCE_WIDTH = 2048.0
|
|
MAX_UPSCALE = 2.0
|
|
DEFAULT_IMAGE_STORE_MAX = 16
|
|
DEFAULT_IMAGE_STORE_TTL_S = 15 * 60
|
|
DEFAULT_IMAGE_STORE_MAX_BYTES = 256 * 1024 * 1024
|
|
|
|
STATE_DETECTED = "detected"
|
|
STATE_IDENTIFYING = "identifying"
|
|
STATE_RECOGNIZED = "recognized"
|
|
STATE_NEEDS_CONFIRMATION = "needs_confirmation"
|
|
STATE_UNRECOGNIZED = "unrecognized"
|
|
|
|
CLASSIFY_METHODS = ("surf", "orb", "brisk", "hybrid", "hybrid_fast")
|
|
|
|
|
|
LOGGER = logging.getLogger("dorayaki.engine")
|
|
|
|
|
|
class UndecodableImage(ValueError):
|
|
"""Raised when an uploaded payload is not a readable image."""
|
|
|
|
|
|
class UnknownMethod(ValueError):
|
|
"""Raised for a classification method outside CLASSIFY_METHODS."""
|
|
|
|
|
|
class ImageExpired(KeyError):
|
|
"""Raised when an image_id is no longer in the store."""
|
|
|
|
|
|
@dataclass
|
|
class EngineSettings:
|
|
weights_path: str = "runs/detect/train/weights/best.pt"
|
|
stock_dir: str = "stock"
|
|
stock_csv: str = "stock.csv"
|
|
# Measured on this project (Apple M5, bench_engine.py): CPU detects in ~27 ms
|
|
# p50 while MPS takes ~112 ms with identical boxes, so CPU is the default.
|
|
# Override with --device / DORAYAKI_DEVICE (cpu | mps | cuda | cuda:0).
|
|
device: Optional[str] = "cpu"
|
|
# Live-mode defaults, measured with stream_client.py --pace result:
|
|
# hybrid classifies at ~44 ms/crop p50 and is the most accurate method
|
|
# (21/21 SKUs on image.jpg). hybrid_fast (ORB shortlist) is 3.4x faster but
|
|
# agreed with hybrid on only 47% of accepted crops, so it is NOT the default
|
|
# and stays available for experiments only (bench_engine.py --compare).
|
|
# Measured for a moving camera with hybrid classification: 16 in flight keeps
|
|
# the queue draining while detection keeps its share of the CPU. A demo walk
|
|
# over a nearly static shelf wants more; that is a CLI override, not a default.
|
|
live_inflight: int = 16
|
|
live_attempts: int = 2
|
|
# Which mechanism names products. `local` is the RootSIFT lane behind a
|
|
# calibrated open-set gate: measured on the corrected image6 labels it shows
|
|
# 36 of 61 in-catalog facings at 95% precision with the text lane, against
|
|
# 35 at 35% for Van's always-argmax matcher. `matcher` keeps Van's composite
|
|
# as a selectable baseline lane (E-1). `gallery` is fenced off: raw DINOv2
|
|
# is 12-21% top-1 on 7 SKUs and its recall@10 falls to 7/42 at 493 SKUs.
|
|
identity_backend: str = "local"
|
|
# Cosine similarity below which the gallery calls a facing unknown. Measured:
|
|
# same-product pairs sit above 0.6, different products below 0.5.
|
|
gallery_threshold: float = 0.55
|
|
gallery_head_threshold: float = 0.40
|
|
gallery_image_size: int = 518
|
|
imgsz: int = 640
|
|
# 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine).
|
|
classify_workers: int = 5
|
|
live_method: str = "hybrid"
|
|
live_conf: float = 0.1
|
|
detect_conf: float = 0.25
|
|
detect_iou: float = 0.45
|
|
image_store_max: int = DEFAULT_IMAGE_STORE_MAX
|
|
image_store_ttl_s: float = DEFAULT_IMAGE_STORE_TTL_S
|
|
image_store_max_bytes: int = DEFAULT_IMAGE_STORE_MAX_BYTES
|
|
|
|
|
|
@dataclass
|
|
class StoredImage:
|
|
image_id: str
|
|
image: Any # numpy BGR array
|
|
width: int
|
|
height: int
|
|
detections: List[Dict[str, Any]] = field(default_factory=list)
|
|
created_at: float = 0.0
|
|
nbytes: int = 0
|
|
# The tag pass for this still (tags, bands, box->tag), kept so a later
|
|
# classify can re-reconcile against the identities it just produced.
|
|
tag_bundle: Optional[Dict[str, Any]] = None
|
|
|
|
|
|
def group_by_sku(scores: Dict[str, float]) -> Dict[str, float]:
|
|
"""Collapse per-view template names onto their canonical SKU.
|
|
|
|
A SKU's score is its best view, which is also how the gallery ranks several
|
|
views. `sku_l01_1` and `sku_l01_2` become one `sku_l01`, so two views of one
|
|
product can no longer compete with each other and no view-suffixed id can
|
|
reach the wire.
|
|
"""
|
|
from rebuild_catalog import sku_of
|
|
|
|
grouped: Dict[str, float] = {}
|
|
for name, value in (scores or {}).items():
|
|
number = float(value)
|
|
sku = sku_of(str(name))
|
|
grouped[sku] = max(grouped.get(sku, 0.0), number)
|
|
return grouped
|
|
|
|
|
|
def identity_state_from_level(level: Optional[str]) -> str:
|
|
"""Map a decide_identity() level to the wire state vocabulary."""
|
|
if level == "accepted":
|
|
return STATE_RECOGNIZED
|
|
if level in ("ambiguous", "unknown"):
|
|
return STATE_NEEDS_CONFIRMATION
|
|
return STATE_UNRECOGNIZED
|
|
|
|
|
|
def identity_payload(decision: Optional[Dict[str, Any]], *, attempts: int = 1) -> Dict[str, Any]:
|
|
"""One wire-shaped identity object from a decide_identity() result."""
|
|
if not decision:
|
|
return {
|
|
"state": STATE_DETECTED,
|
|
"sku_id": None,
|
|
"identity_level": None,
|
|
"confidence": None,
|
|
"score": None,
|
|
"candidates": [],
|
|
"attempts": 0,
|
|
"method": None,
|
|
}
|
|
level = decision.get("identity_level")
|
|
state = identity_state_from_level(level)
|
|
candidates = [
|
|
{"sku_id": row.get("sku_id"), "score": row.get("score")}
|
|
for row in (decision.get("ranked_candidates") or [])[:3]
|
|
]
|
|
return {
|
|
"state": state,
|
|
"sku_id": decision.get("best_sku") if state == STATE_RECOGNIZED else None,
|
|
"identity_level": level,
|
|
"confidence": decision.get("confidence"),
|
|
"score": decision.get("score"),
|
|
"candidates": candidates,
|
|
"attempts": attempts,
|
|
"method": decision.get("method"),
|
|
}
|
|
|
|
|
|
class ImageStore:
|
|
"""Bounded LRU of decoded still images keyed by a random image_id."""
|
|
|
|
def __init__(self, max_items: int = DEFAULT_IMAGE_STORE_MAX,
|
|
ttl_s: float = DEFAULT_IMAGE_STORE_TTL_S,
|
|
max_bytes: int = DEFAULT_IMAGE_STORE_MAX_BYTES):
|
|
self.max_items = max(1, int(max_items))
|
|
self.ttl_s = float(ttl_s)
|
|
self.max_bytes = max(1, int(max_bytes))
|
|
self._items: "collections.OrderedDict[str, StoredImage]" = collections.OrderedDict()
|
|
self._bytes = 0
|
|
self._lock = threading.Lock()
|
|
|
|
def put(self, image: Any, detections: Optional[List[Dict[str, Any]]] = None) -> StoredImage:
|
|
height, width = image.shape[:2]
|
|
stored = StoredImage(
|
|
image_id="img_" + os.urandom(8).hex(),
|
|
image=image,
|
|
width=int(width),
|
|
height=int(height),
|
|
detections=list(detections or []),
|
|
created_at=time.time(),
|
|
nbytes=int(image.nbytes),
|
|
)
|
|
with self._lock:
|
|
self._evict_locked()
|
|
self._items[stored.image_id] = stored
|
|
self._bytes += stored.nbytes
|
|
while len(self._items) > self.max_items or self._bytes > self.max_bytes:
|
|
if len(self._items) <= 1:
|
|
break
|
|
_, dropped = self._items.popitem(last=False)
|
|
self._bytes -= dropped.nbytes
|
|
return stored
|
|
|
|
def get(self, image_id: str) -> StoredImage:
|
|
with self._lock:
|
|
stored = self._items.get(image_id)
|
|
if stored is None or time.time() - stored.created_at > self.ttl_s:
|
|
if stored is not None:
|
|
self._items.pop(image_id, None)
|
|
self._bytes -= stored.nbytes
|
|
raise ImageExpired(image_id)
|
|
self._items.move_to_end(image_id)
|
|
return stored
|
|
|
|
def update_detections(self, image_id: str, detections: List[Dict[str, Any]]) -> None:
|
|
with self._lock:
|
|
stored = self._items.get(image_id)
|
|
if stored is not None:
|
|
stored.detections = list(detections)
|
|
|
|
def set_tag_bundle(self, image_id: str, bundle: Dict[str, Any]) -> None:
|
|
with self._lock:
|
|
stored = self._items.get(image_id)
|
|
if stored is not None:
|
|
stored.tag_bundle = bundle
|
|
|
|
def get_tag_bundle(self, image_id: str) -> Optional[Dict[str, Any]]:
|
|
with self._lock:
|
|
stored = self._items.get(image_id)
|
|
return stored.tag_bundle if stored is not None else None
|
|
|
|
def _evict_locked(self) -> None:
|
|
now = time.time()
|
|
for image_id in [k for k, v in self._items.items() if now - v.created_at > self.ttl_s]:
|
|
dropped = self._items.pop(image_id)
|
|
self._bytes -= dropped.nbytes
|
|
|
|
|
|
class RecognitionEngine:
|
|
"""Thread-safe facade over detector + matcher + catalog."""
|
|
|
|
def __init__(
|
|
self,
|
|
detector: Any,
|
|
matcher: Any,
|
|
inventory: Any,
|
|
*,
|
|
settings: Optional[EngineSettings] = None,
|
|
classify_executor: Optional[ThreadPoolExecutor] = None,
|
|
owns_executor: bool = True,
|
|
):
|
|
self.detector = detector
|
|
self.matcher = matcher
|
|
self.inventory = inventory
|
|
self.settings = settings or EngineSettings()
|
|
# The embedding gallery is built on first use (it loads a model and every
|
|
# enrolled view), and cached as False when it cannot be built at all.
|
|
self._gallery: Any = None
|
|
self._gallery_lock = threading.Lock()
|
|
# The local identity lane, built on first use. It loads every reference
|
|
# view and its descriptors, which is why it is lazy like the gallery.
|
|
self._lane: Any = None
|
|
self._lane_lock = threading.Lock()
|
|
# The text lane, built on first use from the catalog and the OCR engine.
|
|
self._text_lane: Any = None
|
|
self._text_lane_lock = threading.Lock()
|
|
workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2)
|
|
self.classify_workers = int(workers)
|
|
self._executor = classify_executor or ThreadPoolExecutor(
|
|
max_workers=self.classify_workers, thread_name_prefix="classify"
|
|
)
|
|
self._owns_executor = owns_executor
|
|
self._detect_lock = threading.Lock()
|
|
self._inventory_lock = threading.Lock()
|
|
# OCR is the slowest stage (0.2-3.4 s). It gets its own executor so tag
|
|
# passes can never occupy a classification worker, and a lock so several
|
|
# live sessions do not pile OCR requests onto the CPU at once.
|
|
self.ocr_engine: Optional[Any] = None
|
|
self.ocr_lock = threading.Lock()
|
|
self._tag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="tags")
|
|
self._owns_tag_executor = True
|
|
self.image_store = ImageStore(
|
|
self.settings.image_store_max,
|
|
self.settings.image_store_ttl_s,
|
|
self.settings.image_store_max_bytes,
|
|
)
|
|
self._catalog_cache: Optional[Tuple[str, List[Dict[str, Any]]]] = None
|
|
# (version, csv mtime_ns, csv size, sorted image name:size). Version is
|
|
# the content digest; the rest is the cheap fingerprint that invalidates it.
|
|
self._catalog_version_cache: Optional[Tuple[str, int, int, Tuple[Tuple[str, int], ...]]] = None
|
|
self._catalog_version_lock = threading.Lock()
|
|
self._started_at = time.time()
|
|
|
|
# ------------------------------------------------------------------ lifecycle
|
|
@property
|
|
def executor(self) -> ThreadPoolExecutor:
|
|
return self._executor
|
|
|
|
@property
|
|
def tag_executor(self) -> ThreadPoolExecutor:
|
|
return self._tag_executor
|
|
|
|
def set_ocr_engine(self, engine: Any) -> None:
|
|
"""Attach an OCR backend; None disables tag reading."""
|
|
self.ocr_engine = engine
|
|
|
|
def read_tags(self, image: Any) -> Any:
|
|
"""OCR one crop under the shared lock (Vision is happy with concurrency,
|
|
but the CPU is not: one OCR at a time keeps detection responsive)."""
|
|
if self.ocr_engine is None:
|
|
return None
|
|
with self.ocr_lock:
|
|
return self.ocr_engine.read(image)
|
|
|
|
def warm_up(self) -> float:
|
|
"""One dummy predict so the first real frame is not the slow one.
|
|
|
|
The identity lanes are built here too. `TextLane.from_catalog` OCRs
|
|
every enrolled packshot, which is seconds of work; left lazy, the first
|
|
live frame would pay it inside a classify worker while the phone waits.
|
|
"""
|
|
import numpy as np
|
|
|
|
blank = np.zeros((self.settings.imgsz, self.settings.imgsz, 3), dtype="uint8")
|
|
started = time.perf_counter()
|
|
self.detect(blank, conf=self.settings.detect_conf)
|
|
# A lane that cannot build here is not fatal: the lazy builders retry it
|
|
# on first use, so a bad stock dir or a Vision failure degrades the
|
|
# backend instead of refusing to start the server.
|
|
for name, build in (("local", self.identity_lane), ("text", self.identity_text_lane)):
|
|
try:
|
|
build()
|
|
except Exception as exc:
|
|
LOGGER.warning("identity %s lane not warmed: %s", name, exc)
|
|
return (time.perf_counter() - started) * 1000.0
|
|
|
|
def shutdown(self) -> None:
|
|
if self._owns_executor:
|
|
self._executor.shutdown(wait=False, cancel_futures=True)
|
|
if self._owns_tag_executor:
|
|
self._tag_executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
# ------------------------------------------------------------------ images
|
|
def decode_image(self, payload: bytes) -> Any:
|
|
import cv2
|
|
import numpy as np
|
|
|
|
if not payload:
|
|
raise UndecodableImage("empty payload")
|
|
array = np.frombuffer(payload, dtype=np.uint8)
|
|
image = cv2.imdecode(array, cv2.IMREAD_COLOR)
|
|
if image is None or image.size == 0:
|
|
raise UndecodableImage("payload is not a decodable image")
|
|
return image
|
|
|
|
def encode_jpeg(self, image: Any, quality: int = 85) -> bytes:
|
|
import cv2
|
|
|
|
ok, buffer = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, quality])
|
|
if not ok:
|
|
return b""
|
|
return bytes(buffer)
|
|
|
|
def encode_jpeg_data_url(self, image: Any, quality: int = 85) -> str:
|
|
payload = self.encode_jpeg(image, quality=quality)
|
|
if not payload:
|
|
return ""
|
|
return "data:image/jpeg;base64," + base64.b64encode(payload).decode("ascii")
|
|
|
|
# ------------------------------------------------------------------ detection
|
|
def detect(self, image: Any, conf: Optional[float] = None) -> List[Dict[str, Any]]:
|
|
"""Runs the detector under a lock; returns normalized box dicts."""
|
|
threshold = self.settings.detect_conf if conf is None else float(conf)
|
|
with self._detect_lock:
|
|
result = self.detector.detect(
|
|
image, conf_threshold=threshold, iou_threshold=self.settings.detect_iou
|
|
)
|
|
boxes = result.get("boxes") or []
|
|
normalized: List[Dict[str, Any]] = []
|
|
for index, box in enumerate(boxes):
|
|
x1, y1, x2, y2 = [int(v) for v in box["box"]]
|
|
if x2 <= x1 or y2 <= y1:
|
|
continue
|
|
normalized.append(
|
|
{
|
|
"id": index,
|
|
"box": [x1, y1, x2, y2],
|
|
"confidence": float(box.get("confidence", 0.0)),
|
|
"class_id": int(box.get("class_id", 0)),
|
|
"width": x2 - x1,
|
|
"height": y2 - y1,
|
|
"center": [(x1 + x2) // 2, (y1 + y2) // 2],
|
|
}
|
|
)
|
|
for index, box in enumerate(normalized):
|
|
box["id"] = index
|
|
return normalized
|
|
|
|
def crop(self, image: Any, box: Sequence[int]) -> Any:
|
|
x1, y1, x2, y2 = [int(v) for v in box]
|
|
height, width = image.shape[:2]
|
|
x1 = max(0, min(x1, width))
|
|
x2 = max(0, min(x2, width))
|
|
y1 = max(0, min(y1, height))
|
|
y2 = max(0, min(y2, height))
|
|
return image[y1:y2, x1:x2].copy()
|
|
|
|
# ------------------------------------------------------------------ empty slots
|
|
def empty_slots(self, image_shape: Sequence[int], boxes: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
"""Empty-shelf gaps, computed in a 2048 px-wide reference frame so the
|
|
detector's absolute pixel thresholds do not change with capture size."""
|
|
height, width = int(image_shape[0]), int(image_shape[1])
|
|
if not boxes or width <= 0 or height <= 0:
|
|
return []
|
|
scale = min(MAX_UPSCALE, REFERENCE_WIDTH / float(width))
|
|
ref_boxes = []
|
|
for box in boxes:
|
|
x1, y1, x2, y2 = [int(v) for v in box["box"]]
|
|
ref_boxes.append(
|
|
{
|
|
"box": [
|
|
int(round(x1 * scale)),
|
|
int(round(y1 * scale)),
|
|
int(round(x2 * scale)),
|
|
int(round(y2 * scale)),
|
|
]
|
|
}
|
|
)
|
|
ref_shape = (int(round(height * scale)), int(round(width * scale)))
|
|
slots = self.detector.detect_empty_slots(ref_shape, ref_boxes)
|
|
if scale == 1.0:
|
|
return slots
|
|
inv = 1.0 / scale
|
|
for slot in slots:
|
|
x1, y1, x2, y2 = slot["box"]
|
|
slot["box"] = [int(round(x1 * inv)), int(round(y1 * inv)),
|
|
int(round(x2 * inv)), int(round(y2 * inv))]
|
|
slot["width"] = int(round(slot["width"] * inv))
|
|
slot["height"] = int(round(slot["height"] * inv))
|
|
slot["center"] = [int(round(slot["center"][0] * inv)), int(round(slot["center"][1] * inv))]
|
|
return slots
|
|
|
|
# ------------------------------------------------------------------ classification
|
|
@property
|
|
def identity_scale(self) -> str:
|
|
"""What the scores from `score_crop` mean.
|
|
|
|
`local` is the RootSIFT lane's margin-over-runner-up, `cosine` the
|
|
gallery's similarity, `inlier` Van's composite. The live path reads this
|
|
to pick the matching gate, so the three must never be mixed.
|
|
"""
|
|
backend = (self.settings.identity_backend or "matcher").lower()
|
|
if backend == "gallery":
|
|
return "cosine"
|
|
if backend == "local":
|
|
return "local"
|
|
return "inlier"
|
|
|
|
def identity_text_lane(self) -> Any:
|
|
"""The text lane, built once from the catalog. None without OCR.
|
|
|
|
Signatures come from the catalog rows (their printed names plus the OCR
|
|
of their enrolled packshots), never from an evaluation photo, so a row
|
|
with no image can still be named. That is what lets the lane reject the
|
|
near-unknown siblings no geometric threshold can separate.
|
|
"""
|
|
if self.identity_scale != "local":
|
|
return None
|
|
if self._text_lane is None:
|
|
with self._text_lane_lock:
|
|
if self._text_lane is None:
|
|
from identity_lanes import TextLane
|
|
|
|
self._text_lane = TextLane.from_catalog(
|
|
self.catalog(), self.settings.stock_dir, self.ocr_engine,
|
|
lock=self.ocr_lock,
|
|
)
|
|
LOGGER.info("identity text lane: %d signatures", len(self._text_lane.signatures))
|
|
return self._text_lane
|
|
|
|
def decide_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, Any]:
|
|
"""One identity decision for one crop, fusing every lane that is on.
|
|
|
|
The local lane answers first and the text lane vetoes or fills in, which
|
|
is the measured design: geometry ranks fine-grained siblings correctly
|
|
and text is the only thing that rejects a sibling the catalog does not
|
|
know about. Without OCR the local lane stands alone.
|
|
"""
|
|
from identity import decide_identity
|
|
from identity_lanes import DecisionPolicy
|
|
|
|
if self.identity_scale != "local":
|
|
return decide_identity(
|
|
self.score_crop(crop, method), method=method or self.settings.live_method,
|
|
scale=self.identity_scale,
|
|
)
|
|
lane = self.identity_lane()
|
|
if lane is None:
|
|
return decide_identity({}, method="local", scale="local")
|
|
text_lane = self.identity_text_lane()
|
|
text_rank: Any = None
|
|
if text_lane is not None:
|
|
read = text_lane.read(crop)
|
|
text_rank = sorted(
|
|
text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
|
|
)
|
|
return DecisionPolicy().decide(lane.rank(crop), text_rank)
|
|
|
|
def score_crop_lanes(
|
|
self, crop: Any, method: Optional[str] = None
|
|
) -> Tuple[Dict[str, float], List[Tuple[str, float]]]:
|
|
"""Both lanes' evidence for one crop: local scores and text ranking.
|
|
|
|
The live path accumulates evidence across looks, so it needs the lanes
|
|
separately rather than one fused decision: it averages the local scores
|
|
over the track's window and keeps the newest text reading, then decides
|
|
once. The still path calls `decide_crop`, which fuses a single crop.
|
|
"""
|
|
scores = self.score_crop(crop, method)
|
|
text_lane = self.identity_text_lane()
|
|
if text_lane is None:
|
|
return scores, []
|
|
read = text_lane.read(crop)
|
|
return scores, sorted(
|
|
text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
|
|
)
|
|
|
|
def identity_lane(self) -> Any:
|
|
"""The local lane, built once. None when the backend is not `local`."""
|
|
if self.identity_scale != "local":
|
|
return None
|
|
if self._lane is None:
|
|
with self._lane_lock:
|
|
if self._lane is None:
|
|
from identity_lanes import LocalLane
|
|
|
|
self._lane = LocalLane(self.settings.stock_dir)
|
|
LOGGER.info(
|
|
"identity lane: %d reference views over %d SKUs",
|
|
self._lane.view_count(),
|
|
len(self._lane.sku_ids),
|
|
)
|
|
return self._lane
|
|
|
|
def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
|
|
"""Score one crop, keyed by canonical SKU.
|
|
|
|
Every backend groups by SKU before returning. Scoring each reference
|
|
view separately is what made the demo's default catalog report 22-29 of
|
|
109 facings as "ambiguous" when the top two candidates were two views of
|
|
the same product, and what put view-suffixed ids like `sku_l01_1` on the
|
|
wire, where no catalog row exists to supply a name or price.
|
|
"""
|
|
if self.identity_scale == "cosine":
|
|
gallery = self.identity_gallery()
|
|
if not gallery:
|
|
return {}
|
|
match = gallery.match(crop)
|
|
if match.rejected or not match.sku_id:
|
|
return {}
|
|
scores = {match.sku_id: match.similarity}
|
|
if match.runner_up:
|
|
scores[match.runner_up] = match.runner_up_similarity
|
|
return {str(sku): float(score) for sku, score in scores.items()}
|
|
if self.identity_scale == "local":
|
|
lane = self.identity_lane()
|
|
if lane is None:
|
|
return {}
|
|
return {sku: float(value) for sku, value in lane.rank(crop)}
|
|
method = (method or self.settings.live_method).lower()
|
|
if method not in CLASSIFY_METHODS:
|
|
raise UnknownMethod(method)
|
|
if method == "hybrid_fast":
|
|
scores, _ = self.matcher.score_crop_fast(crop)
|
|
else:
|
|
scores, _ = self.matcher.score_crop(crop, method=method)
|
|
return group_by_sku(scores)
|
|
|
|
def identity_gallery(self) -> Any:
|
|
"""The embedding gallery, loaded once (None when unavailable)."""
|
|
if self._gallery is False:
|
|
return None
|
|
if self._gallery is not None:
|
|
return self._gallery
|
|
with self._gallery_lock:
|
|
if self._gallery is False:
|
|
return None
|
|
if self._gallery is not None:
|
|
return self._gallery
|
|
from gallery import ProductGallery
|
|
|
|
gallery = ProductGallery(
|
|
self.settings.stock_dir,
|
|
[product["sku_id"] for product in self.catalog()],
|
|
threshold=self.settings.gallery_threshold,
|
|
head_threshold=self.settings.gallery_head_threshold,
|
|
image_size=self.settings.gallery_image_size,
|
|
)
|
|
try:
|
|
loaded = gallery.load()
|
|
except Exception as exc: # torch/timm missing, no weights, no network
|
|
LOGGER.warning("identity gallery unavailable: %s", exc)
|
|
loaded = False
|
|
# False means tried and failed. Do not retry and do not use the matcher.
|
|
self._gallery = gallery if loaded else False
|
|
if loaded:
|
|
LOGGER.info(
|
|
"identity gallery: %d views over %d SKUs on %s",
|
|
gallery.view_count(),
|
|
len(gallery.sku_ids),
|
|
gallery.device,
|
|
)
|
|
if gallery.head_reason != "ok":
|
|
LOGGER.warning("identity head: %s (%s)", gallery.head_reason, gallery._head_path)
|
|
else:
|
|
LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized")
|
|
return gallery if loaded else None
|
|
|
|
def classify_boxes(
|
|
self,
|
|
image: Any,
|
|
boxes: Sequence[Dict[str, Any]],
|
|
method: Optional[str] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""Classifies every box in parallel; returns one entry per box, ordered by area."""
|
|
method = (method or self.settings.live_method).lower()
|
|
if method not in CLASSIFY_METHODS:
|
|
raise UnknownMethod(method)
|
|
if not boxes:
|
|
return []
|
|
|
|
def area_of(box: Dict[str, Any]) -> int:
|
|
if "width" in box and "height" in box:
|
|
return int(box["width"]) * int(box["height"])
|
|
x1, y1, x2, y2 = [int(v) for v in box["box"]]
|
|
return max(0, x2 - x1) * max(0, y2 - y1)
|
|
|
|
ordered = sorted(boxes, key=area_of, reverse=True)
|
|
crops = [self.crop(image, box["box"]) for box in ordered]
|
|
|
|
if self.identity_scale == "cosine":
|
|
gallery = self.identity_gallery()
|
|
# No matcher fallback: an unavailable gallery is unrecognized, not a different model.
|
|
return self._classify_with_gallery(gallery, ordered, crops)
|
|
from identity import decide_identity
|
|
|
|
# Every backend goes through decide_crop, so the scores are keyed by
|
|
# canonical SKU and the text lane is fused in whichever lane produced them.
|
|
futures = [self._executor.submit(self.decide_crop, crop, method) for crop in crops]
|
|
results: List[Dict[str, Any]] = []
|
|
for box, future in zip(ordered, futures):
|
|
try:
|
|
decision = future.result()
|
|
except Exception as exc: # a single bad crop must not fail the batch
|
|
LOGGER.debug("classification failed for box %s: %s", box.get("id"), exc)
|
|
# A rejected decision, not the "not yet classified" payload: an
|
|
# operational failure must stay distinguishable from absence.
|
|
decision = decide_identity({}, method=method, scale=self.identity_scale)
|
|
decision = identity_payload(decision)
|
|
results.append(
|
|
{
|
|
"detection_id": int(box["id"]),
|
|
"box": list(box["box"]),
|
|
"identity": decision,
|
|
}
|
|
)
|
|
return results
|
|
|
|
def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Full single-crop result, using the same backend as recognize/live."""
|
|
from identity import decision_for_match
|
|
|
|
crop = self.crop(image, box)
|
|
if self.identity_scale == "cosine":
|
|
gallery = self.identity_gallery()
|
|
match = gallery.match(crop) if gallery else None
|
|
# match() already applied the gallery's own threshold. Ranking again
|
|
# with the published cosine gates must not invent a second line.
|
|
decision = decision_for_match(match)
|
|
return self._inspect_payload(box, "gallery", decision, crop, visualize=False)
|
|
method = (method or self.settings.live_method).lower()
|
|
if method not in CLASSIFY_METHODS:
|
|
raise UnknownMethod(method)
|
|
decision = self.decide_crop(crop, method)
|
|
visual_method = "hybrid" if method == "hybrid_fast" else method
|
|
return self._inspect_payload(box, method, decision, crop, visual_method=visual_method)
|
|
|
|
def _inspect_payload(
|
|
self,
|
|
box: Sequence[int],
|
|
method: str,
|
|
decision: Dict[str, Any],
|
|
crop: Any,
|
|
*,
|
|
visual_method: Optional[str] = None,
|
|
visualize: bool = True,
|
|
) -> Dict[str, Any]:
|
|
identity = identity_payload(decision)
|
|
top_details = decision.get("details") or {}
|
|
sub_details = None
|
|
if isinstance(top_details, dict):
|
|
raw = top_details.get("sub_details")
|
|
if raw:
|
|
sub_details = {key: dict(value) for key, value in raw.items()}
|
|
|
|
product = None
|
|
visualization = None
|
|
visualized_sku = None
|
|
is_candidate = False
|
|
if identity["state"] == STATE_RECOGNIZED:
|
|
product = self.product(identity["sku_id"])
|
|
visualized_sku = identity["sku_id"]
|
|
elif identity["candidates"]:
|
|
visualized_sku = identity["candidates"][0]["sku_id"]
|
|
is_candidate = True
|
|
if visualize and visualized_sku and visual_method:
|
|
try:
|
|
visualization = self.matcher.generate_match_visualization(
|
|
crop, visualized_sku, method=visual_method
|
|
)
|
|
except Exception:
|
|
visualization = None
|
|
|
|
return {
|
|
"box": [int(value) for value in box],
|
|
"method": method,
|
|
"identity": identity,
|
|
"sub_details": sub_details,
|
|
"product": product,
|
|
"visualized_sku_id": visualized_sku,
|
|
"is_candidate": is_candidate,
|
|
"crop_jpeg_b64": self.encode_jpeg_data_url(crop),
|
|
"match_visualization_jpeg_b64": (
|
|
self.encode_jpeg_data_url(visualization) if visualization is not None else None
|
|
),
|
|
}
|
|
|
|
# ------------------------------------------------------------------ catalog
|
|
def catalog_version(self) -> str:
|
|
"""12-hex digest of stock.csv bytes plus template image names and sizes.
|
|
|
|
Unchanged files reuse the digest. A stock update that has not been written
|
|
back to the CSV still changes the digest, because the in-memory fingerprint
|
|
is dropped. Same-size content rewritten without a new mtime is not detected
|
|
until the next stock update or a real stat change.
|
|
"""
|
|
csv_path = os.path.join(self.settings.stock_dir, self.settings.stock_csv)
|
|
try:
|
|
stat = os.stat(csv_path)
|
|
csv_key = (stat.st_mtime_ns, stat.st_size)
|
|
except OSError:
|
|
csv_key = None
|
|
try:
|
|
entries = tuple(sorted(
|
|
(name, os.path.getsize(os.path.join(self.settings.stock_dir, name)))
|
|
for name in os.listdir(self.settings.stock_dir)
|
|
if name.lower().endswith((".jpeg", ".jpg", ".png"))
|
|
))
|
|
except OSError:
|
|
entries = ()
|
|
fingerprint = (
|
|
csv_key[0] if csv_key else -1,
|
|
csv_key[1] if csv_key else -1,
|
|
entries,
|
|
)
|
|
with self._catalog_version_lock:
|
|
cached = self._catalog_version_cache
|
|
if cached is not None and csv_key is not None and cached[1:] == fingerprint:
|
|
return cached[0]
|
|
digest = hashlib.sha256()
|
|
if csv_key is None:
|
|
digest.update(b"missing-csv")
|
|
else:
|
|
try:
|
|
with open(csv_path, "rb") as handle:
|
|
digest.update(handle.read())
|
|
except OSError:
|
|
digest.update(b"missing-csv")
|
|
csv_key = None
|
|
for name, size in entries:
|
|
digest.update(f"{name}:{size}".encode("utf-8"))
|
|
version = digest.hexdigest()[:12]
|
|
if csv_key is not None:
|
|
with self._catalog_version_lock:
|
|
self._catalog_version_cache = (version, csv_key[0], csv_key[1], entries)
|
|
return version
|
|
|
|
def model_version(self) -> str:
|
|
try:
|
|
import ultralytics
|
|
|
|
library = f"ultralytics-{ultralytics.__version__}"
|
|
except Exception:
|
|
library = "ultralytics-unknown"
|
|
return f"{os.path.basename(self.settings.weights_path)}@{library}"
|
|
|
|
def catalog(self) -> List[Dict[str, Any]]:
|
|
version = self.catalog_version()
|
|
cached = self._catalog_cache
|
|
if cached and cached[0] == version:
|
|
return cached[1]
|
|
products: List[Dict[str, Any]] = []
|
|
for sku_id in sorted(self.inventory.list_all_skus()):
|
|
product = self.inventory.get_product(sku_id) or {}
|
|
products.append(self._product_payload(sku_id, product))
|
|
self._catalog_cache = (version, products)
|
|
return products
|
|
|
|
def product(self, sku_id: Optional[str]) -> Optional[Dict[str, Any]]:
|
|
if not sku_id:
|
|
return None
|
|
with self._inventory_lock:
|
|
product = self.inventory.get_product(sku_id)
|
|
if product is None:
|
|
return None
|
|
payload = self._product_payload(sku_id, product)
|
|
self._catalog_cache = None
|
|
return payload
|
|
|
|
def _product_payload(self, sku_id: str, product: Dict[str, Any]) -> Dict[str, Any]:
|
|
return {
|
|
"sku_id": sku_id,
|
|
"product_name": product.get("product_name", ""),
|
|
"price": product.get("price", 0.0),
|
|
"number_in": int(product.get("number_in", 0)),
|
|
"number_out": int(product.get("number_out", 0)),
|
|
"number_on": int(product.get("number_on", 0)),
|
|
"receipt_date": product.get("receipt_date", ""),
|
|
"expiration_date": product.get("expiration_date", ""),
|
|
"image_url": f"/v1/catalog/{sku_id}/image",
|
|
}
|
|
|
|
def catalog_image_path(self, sku_id: str) -> Optional[str]:
|
|
"""Resolve a template image path through the inventory map only."""
|
|
with self._inventory_lock:
|
|
product = self.inventory.get_product(sku_id)
|
|
if product is None:
|
|
return None
|
|
path = product.get("local_image_path")
|
|
if not path or not os.path.exists(path):
|
|
return None
|
|
return path
|
|
|
|
def update_stock(self, sku_id: str, delta: int) -> Optional[Dict[str, Any]]:
|
|
with self._inventory_lock:
|
|
ok = self.inventory.update_stock(sku_id, int(delta))
|
|
if not ok:
|
|
return None
|
|
product = self.inventory.get_product(sku_id) or {}
|
|
payload = self._product_payload(sku_id, product)
|
|
self._catalog_cache = None
|
|
with self._catalog_version_lock:
|
|
self._catalog_version_cache = None
|
|
return payload
|
|
|
|
def sku_count(self) -> int:
|
|
return len(self.inventory.list_all_skus())
|
|
|
|
def uptime_s(self) -> float:
|
|
return time.time() - self._started_at
|
|
|
|
# ------------------------------------------------------------------ recognize
|
|
def recognize(
|
|
self,
|
|
image: Any,
|
|
*,
|
|
classify: bool = True,
|
|
method: Optional[str] = None,
|
|
conf: Optional[float] = None,
|
|
with_empty_slots: bool = True,
|
|
) -> Dict[str, Any]:
|
|
"""Stateless still-image recognition (photo mode and one-shot scans)."""
|
|
started = time.perf_counter()
|
|
boxes = self.detect(image, conf=conf)
|
|
detected_at = time.perf_counter()
|
|
detections: List[Dict[str, Any]] = []
|
|
identities: List[Dict[str, Any]] = []
|
|
if classify and boxes:
|
|
identities = self.classify_boxes(image, boxes, method=method)
|
|
identities_by_id = {row["detection_id"]: row["identity"] for row in identities}
|
|
for box in boxes:
|
|
detections.append(
|
|
{
|
|
"id": box["id"],
|
|
"box": box["box"],
|
|
"confidence": box["confidence"],
|
|
"identity": identities_by_id.get(box["id"], identity_payload(None)),
|
|
}
|
|
)
|
|
slots = self.empty_slots(image.shape, boxes) if with_empty_slots else []
|
|
classified_at = time.perf_counter()
|
|
summary = self.summarize(detections, slots)
|
|
stored = self.image_store.put(image, detections)
|
|
return {
|
|
"image_id": stored.image_id,
|
|
"width": stored.width,
|
|
"height": stored.height,
|
|
"catalog_version": self.catalog_version(),
|
|
"model_version": self.model_version(),
|
|
"coordinate_convention": COORDINATE_CONVENTION,
|
|
"detections": detections,
|
|
"empty_slots": slots,
|
|
"summary": summary,
|
|
"timing_ms": {
|
|
"detect": round((detected_at - started) * 1000.0, 2),
|
|
"classify": round((classified_at - detected_at) * 1000.0, 2),
|
|
"total": round((classified_at - started) * 1000.0, 2),
|
|
},
|
|
}
|
|
|
|
def _classify_with_gallery(
|
|
self,
|
|
gallery: Any,
|
|
ordered: Sequence[Dict[str, Any]],
|
|
crops: Sequence[Any],
|
|
) -> List[Dict[str, Any]]:
|
|
"""Identity by retrieval: nearest enrolled view, or an honest unknown.
|
|
|
|
The gallery scores a SKU by its best view and rejects when nothing is close
|
|
enough, which is the behaviour the matcher cannot provide - measured on 54
|
|
unseen store photos, the matcher labelled milk cartons as mayonnaise and its
|
|
score could not be thresholded to stop it.
|
|
"""
|
|
from identity import decision_for_match
|
|
|
|
results: List[Dict[str, Any]] = []
|
|
for box, crop in zip(ordered, crops):
|
|
match = None
|
|
if gallery is not None:
|
|
try:
|
|
match = gallery.match(crop)
|
|
except Exception as exc: # a bad crop must not fail the batch
|
|
match = None
|
|
LOGGER.debug("gallery match failed: %s", exc)
|
|
decision = decision_for_match(match)
|
|
results.append({
|
|
"detection_id": int(box["id"]),
|
|
"box": list(box["box"]),
|
|
"identity": identity_payload(decision),
|
|
})
|
|
return results
|
|
|
|
@staticmethod
|
|
def summarize(
|
|
detections: Sequence[Dict[str, Any]],
|
|
slots: Sequence[Dict[str, Any]],
|
|
*,
|
|
include_identifying: bool = False,
|
|
) -> Dict[str, Any]:
|
|
counts = {
|
|
STATE_RECOGNIZED: 0,
|
|
STATE_NEEDS_CONFIRMATION: 0,
|
|
STATE_UNRECOGNIZED: 0,
|
|
STATE_DETECTED: 0,
|
|
}
|
|
if include_identifying:
|
|
counts[STATE_IDENTIFYING] = 0
|
|
facings: Dict[str, int] = {}
|
|
for row in detections:
|
|
state = (row.get("identity") or {}).get("state") or STATE_DETECTED
|
|
if state not in counts:
|
|
state = STATE_DETECTED
|
|
counts[state] += 1
|
|
sku_id = (row.get("identity") or {}).get("sku_id")
|
|
if state == STATE_RECOGNIZED and sku_id:
|
|
facings[sku_id] = facings.get(sku_id, 0) + 1
|
|
summary: Dict[str, Any] = {
|
|
"in_view": len(detections),
|
|
"recognized": counts[STATE_RECOGNIZED],
|
|
"needs_confirmation": counts[STATE_NEEDS_CONFIRMATION],
|
|
"unrecognized": counts[STATE_UNRECOGNIZED],
|
|
"detected": counts[STATE_DETECTED],
|
|
"empty_slots": len(slots),
|
|
"by_sku": [
|
|
{"sku_id": sku, "facings": count}
|
|
for sku, count in sorted(facings.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
],
|
|
}
|
|
if include_identifying:
|
|
summary["identifying"] = counts[STATE_IDENTIFYING]
|
|
return summary
|
|
|
|
|
|
def build_default_engine(settings: Optional[EngineSettings] = None) -> RecognitionEngine:
|
|
"""Construct the real detector/matcher/inventory. Heavy imports happen here."""
|
|
import sys
|
|
|
|
settings = settings or EngineSettings()
|
|
# Detection and classification share the CPU. torch defaults to every core,
|
|
# which starves the classification pool (and the pool then slows detection
|
|
# back). Measured on the M5: capping torch at 4 threads and classifying with
|
|
# 5 workers gave 12.2 results/s single-client (46 ms detect) versus 11.1/s
|
|
# (49 ms) uncapped, and 8.5/s each for two clients. Override with
|
|
# DORAYAKI_TORCH_THREADS (0 = leave torch's default).
|
|
torch_threads = int(os.environ.get("DORAYAKI_TORCH_THREADS", "4") or 0)
|
|
if torch_threads > 0:
|
|
import torch
|
|
|
|
torch.set_num_threads(torch_threads)
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
from detector import ShelfProductDetector
|
|
from feature_matcher import ProductFeatureMatcher
|
|
from inventory import InventoryManager
|
|
|
|
detector = ShelfProductDetector(
|
|
weights_path=settings.weights_path, device=settings.device, imgsz=settings.imgsz
|
|
)
|
|
matcher = ProductFeatureMatcher(stock_dir=settings.stock_dir)
|
|
inventory = InventoryManager(stock_dir=settings.stock_dir, csv_filename=settings.stock_csv)
|
|
engine = RecognitionEngine(detector, matcher, inventory, settings=settings)
|
|
|
|
from ocr import build_ocr_engine
|
|
|
|
try:
|
|
ocr_engine = build_ocr_engine()
|
|
except Exception as exc: # a broken OCR install must not stop recognition
|
|
print(f"[Engine] OCR disabled: {exc}")
|
|
ocr_engine = None
|
|
engine.set_ocr_engine(ocr_engine)
|
|
print(f"[Engine] OCR engine: {getattr(ocr_engine, 'name', 'none')}")
|
|
return engine
|
|
|
|
|
|
def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings:
|
|
"""Build EngineSettings from DORAYAKI_* environment variables."""
|
|
env = dict(os.environ if env is None else env)
|
|
base = EngineSettings()
|
|
return replace(
|
|
base,
|
|
weights_path=env.get("DORAYAKI_WEIGHTS", base.weights_path),
|
|
stock_dir=env.get("DORAYAKI_STOCK_DIR", base.stock_dir),
|
|
device=env.get("DORAYAKI_DEVICE") or base.device,
|
|
imgsz=int(env.get("DORAYAKI_IMGSZ", base.imgsz)),
|
|
classify_workers=int(env.get("DORAYAKI_CLASSIFY_WORKERS", base.classify_workers)),
|
|
live_method=env.get("DORAYAKI_LIVE_METHOD", base.live_method),
|
|
live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)),
|
|
live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)),
|
|
live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)),
|
|
identity_backend=env.get("DORAYAKI_IDENTITY", base.identity_backend),
|
|
detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__": # tiny smoke check: python code/engine.py
|
|
engine = build_default_engine(settings_from_env())
|
|
print("[Engine] warmup ms:", round(engine.warm_up(), 1))
|
|
print("[Engine] catalog:", engine.sku_count(), "skus, version", engine.catalog_version())
|
|
result = engine.recognize(engine.decode_image(open("image.jpg", "rb").read()))
|
|
print("[Engine] detected:", len(result["detections"]), "recognized:", result["summary"]["recognized"])
|
|
print("[Engine] timing:", json.dumps(result["timing_ms"]))
|
|
engine.shutdown()
|