diff --git a/recognization/code/app.py b/recognization/code/app.py index d4a3cc7..0da33e3 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -276,6 +276,14 @@ def create_app( "warm": True, }, "catalog": {"version": engine.catalog_version(), "sku_count": engine.sku_count()}, + "identity": { + "backend": getattr(engine.settings, "identity_backend", "matcher"), + "gallery_views": ( + engine.identity_gallery().view_count() + if getattr(engine.settings, "identity_backend", "") == "gallery" and engine.identity_gallery() + else None + ), + }, "ocr": { # Without this an operator could not tell a broken OCR from a shelf # with no readable tags. @@ -987,6 +995,8 @@ def main() -> None: parser.add_argument("--classify-workers", type=int, default=None) parser.add_argument("--live-method", type=str, default=None, choices=list(CLASSIFY_METHODS)) parser.add_argument("--live-conf", type=float, default=None) + parser.add_argument("--identity", type=str, default=None, choices=["matcher", "gallery"], + help="which mechanism names products (default matcher)") parser.add_argument("--live-inflight", type=int, default=None, help="classifications in flight per session (demo walks want a larger budget)") parser.add_argument("--live-attempts", type=int, default=None, @@ -1036,6 +1046,8 @@ def main() -> None: server.record_frames = args.record_frames if args.record_every is not None: server.record_every = args.record_every + if args.identity is not None: + engine_settings.identity_backend = args.identity if args.live_inflight is not None: engine_settings.live_inflight = args.live_inflight if args.live_attempts is not None: diff --git a/recognization/code/engine.py b/recognization/code/engine.py index 74117ee..7da5f13 100644 --- a/recognization/code/engine.py +++ b/recognization/code/engine.py @@ -21,6 +21,7 @@ from __future__ import annotations import base64 import collections +import logging import hashlib import json import os @@ -50,6 +51,9 @@ STATE_UNRECOGNIZED = "unrecognized" CLASSIFY_METHODS = ("surf", "orb", "brisk", "hybrid", "hybrid_fast") +LOGGER = logging.getLogger("dorayaki.engine") + + class UndecodableImage(ValueError): """Raised when an uploaded payload is not a readable image.""" @@ -81,6 +85,15 @@ class EngineSettings: # over a nearly static shelf wants more; that is a CLI override, not a default. live_inflight: int = 16 live_attempts: int = 2 + # Which mechanism names products: the hand-crafted local-feature matcher, or the + # embedding gallery with a calibrated rejection. Measured on 54 unseen store + # photos, the matcher accepts look-alikes (milk cartons as mayonnaise) and its + # raw inlier score cannot be thresholded; the gallery can say "unknown". + identity_backend: str = "matcher" + # Cosine similarity below which the gallery calls a facing unknown. Measured: + # same-product pairs sit above 0.6, different products below 0.5. + gallery_threshold: float = 0.55 + gallery_image_size: int = 518 imgsz: int = 640 # 0 -> max(2, cpu_count - 2). Measured default below (see build_default_engine). classify_workers: int = 5 @@ -234,6 +247,9 @@ class RecognitionEngine: self.matcher = matcher self.inventory = inventory self.settings = settings or EngineSettings() + # The embedding gallery is built on first use (it loads a model and every + # enrolled view), and cached as False when it cannot be built at all. + self._gallery: Any = None workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2) self.classify_workers = int(workers) self._executor = classify_executor or ThreadPoolExecutor( @@ -404,6 +420,29 @@ class RecognitionEngine: scores, _ = self.matcher.score_crop(crop, method=method) return {str(sku): float(score) for sku, score in scores.items()} + def identity_gallery(self) -> Any: + """The embedding gallery, loaded once (None when unavailable).""" + if self._gallery is not None: + return self._gallery + from gallery import ProductGallery + + gallery = ProductGallery( + self.settings.stock_dir, + [product["sku_id"] for product in self.catalog()], + threshold=self.settings.gallery_threshold, + image_size=self.settings.gallery_image_size, + ) + try: + loaded = gallery.load() + except Exception as exc: # torch/timm missing, no weights, no network + LOGGER.warning("identity gallery unavailable: %s", exc) + loaded = False + self._gallery = gallery if loaded else False + if loaded: + LOGGER.info("identity gallery: %d views over %d SKUs on %s", + gallery.view_count(), len(self.settings.stock_dir and gallery.sku_ids), gallery.device) + return self._gallery + def classify_boxes( self, image: Any, @@ -427,6 +466,11 @@ class RecognitionEngine: ordered = sorted(boxes, key=area_of, reverse=True) crops = [self.crop(image, box["box"]) for box in ordered] + + if (self.settings.identity_backend or "matcher").lower() == "gallery": + gallery = self.identity_gallery() + if gallery: + return self._classify_with_gallery(gallery, ordered, crops) if method == "hybrid_fast": futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops] else: @@ -643,6 +687,64 @@ class RecognitionEngine: }, } + def _classify_with_gallery( + self, + gallery: Any, + ordered: Sequence[Dict[str, Any]], + crops: Sequence[Any], + ) -> List[Dict[str, Any]]: + """Identity by retrieval: nearest enrolled view, or an honest unknown. + + The gallery scores a SKU by its best view and rejects when nothing is close + enough, which is the behaviour the matcher cannot provide - measured on 54 + unseen store photos, the matcher labelled milk cartons as mayonnaise and its + score could not be thresholded to stop it. + """ + results: List[Dict[str, Any]] = [] + for box, crop in zip(ordered, crops): + try: + match = gallery.match(crop) + except Exception as exc: # a bad crop must not fail the batch + match = None + LOGGER.debug("gallery match failed: %s", exc) + if match is None or match.rejected: + similarity = float(match.similarity) if match is not None else 0.0 + results.append({ + "detection_id": int(box["id"]), + "box": list(box["box"]), + "identity": { + "state": "unrecognized", + "sku_id": None, + "identity_level": "rejected", + "confidence": round(similarity, 3), + "score": round(similarity, 3), + "candidates": [], + "attempts": 1, + "method": "gallery", + "message": "no enrolled product is close enough", + }, + }) + continue + runner_up = ( + {"sku_id": match.runner_up, "score": round(match.runner_up_similarity, 3)} + if match.runner_up else {} + ) + results.append({ + "detection_id": int(box["id"]), + "box": list(box["box"]), + "identity": { + "state": "recognized", + "sku_id": match.sku_id, + "identity_level": "accepted", + "confidence": round(match.similarity, 3), + "score": round(match.similarity, 3), + "candidates": [runner_up] if runner_up else [], + "attempts": 1, + "method": "gallery", + }, + }) + return results + @staticmethod def summarize( detections: Sequence[Dict[str, Any]], @@ -742,6 +844,7 @@ def settings_from_env(env: Optional[Dict[str, str]] = None) -> EngineSettings: live_conf=float(env.get("DORAYAKI_LIVE_CONF", base.live_conf)), live_inflight=int(env.get("DORAYAKI_LIVE_INFLIGHT", base.live_inflight)), live_attempts=int(env.get("DORAYAKI_LIVE_ATTEMPTS", base.live_attempts)), + identity_backend=env.get("DORAYAKI_IDENTITY", base.identity_backend), detect_conf=float(env.get("DORAYAKI_DETECT_CONF", base.detect_conf)), ) diff --git a/recognization/docs/EXTERNAL-EVAL.md b/recognization/docs/EXTERNAL-EVAL.md index b9c32fe..b5c5f78 100644 --- a/recognization/docs/EXTERNAL-EVAL.md +++ b/recognization/docs/EXTERNAL-EVAL.md @@ -88,3 +88,95 @@ respectively, unlike this score). The images are from Wikimedia Commons under their individual licences (CC0, CC BY, CC BY-SA, public domain); `manifest.json` records the author, licence and source URL of each one. They are evaluation material and are not redistributed here. + + +--- + +# Part 2: the identity stack, measured on a shelf where the product IS present + +Part 1 measured shelves whose products are absent. This part uses +`File:2011年4月の牛乳 (5588310773).jpg` - a milk case where **14 of 107 facings are +明治 おいしい牛乳**, a product the original catalog holds - so precision and recall can +be computed instead of just false accepts. The 107 crops were labelled by eye from a +contact sheet. + +## The shipped matcher + +| | claims | correct | precision | recall | +| --- | --- | --- | --- | --- | +| matcher | 31 | 12 | **39 %** | **86 %** | + +It finds almost every true facing and makes 19 false claims, on cartons of 特濃4.5, +低脂肪殺菌牛乳, 明治ラブ and 北海道牛乳. Correct where the product is, wrong on the +look-alikes: the same shape as Part 1. + +## The embedding gallery: rejection, but not precision + +| threshold | false accepts on foreign shelves | milk shelf: precision | recall | +| --- | --- | --- | --- | +| 0.45 | 29 % | 28 % | 79 % | +| 0.50 | 18 % | 29 % | 64 % | +| 0.55 | 11 % | 35 % | 50 % | +| **0.60** | **5 %** | 45 % | 36 % | +| 0.65 | 2 % | 0 % | 0 % | + +The gallery's value is the **controllable rejection**: on shelves where nothing +matches, false accepts fall from the matcher's 47.8 % to **5 %** at 0.60, and the +curve is smooth enough to pick an operating point. But on the shelf where the product +*is* present its precision (28-45 %) is no better than the matcher's 39 % - it buys +honesty about the unknown, not finer discrimination. + +## Enrolling a shelf from itself does not fix it + +The flywheel's first step is to enroll a store's own shelf. Measured by taking 1-3 +views per product from the labelled milk shelf and testing on the rest: + +| views per SKU | top-1 | wrong | rejected | +| --- | --- | --- | --- | +| 1 | **26 %** | 30 | 29 | +| 2 | 25 % | 27 | 27 | +| 3 | 23 % | 24 | 26 | + +Flat at ~25 %, and far below the 72-76 % measured on the demo shelf with the same +number of views. The demo shelf's facings are near-identical crops of the same +photograph; real milk cartons differ by small text and a colour band. **Off-the-shelf +DINOv2 is not discriminative enough for fine-grained SKUs**, which is what the +literature says (RP2K and RPC both fine-tune; GroceryVision reports a 17.5-point drop +from Recall@5 to Recall@1). Fine-tuning is not a refinement of the plan, it is a +requirement - and its labels are what the tag channel and the confirmation loop +produce. + +## The packaging's own text is a real channel, and resolution-bound + +The cartons differ by the text printed on them, so OCR was tested as an identity +channel - first single-scale, then with the tag reader's trick (two upscales, the red +channel, pooled): + +| product | single scale | multi-scale + red | +| --- | --- | --- | +| MEGMILK (large Latin text) | 85 % | **85 %** | +| 北海道牛乳 (large kanji) | 65 % | **65 %** | +| 明治ラブ | 0 % | **75 %** | +| 特濃4.5 | 0 % | **50 %** | +| 明治 おいしい牛乳 (small vertical text) | 7 % | **28 %** | +| 低脂肪殺菌牛乳 / 無調整豆乳 / 酪農3.6 | 0 % | 0 % | +| **overall** | 51 % | **60 %** | + +Text works where it is large and fails where it is small - the same resolution wall as +the price tags. This is a capture-spec question, not a model question. + +## The identity architecture this implies + +| channel | what it is good for | measured | +| --- | --- | --- | +| Detection | finding facings on unseen shelves | 46/54 images, tight boxes | +| Tag (corroborated) | exact identity and price, where readable | 15/15 fields on the demo shelf; sparse in the field | +| Packaging text | brand-level discrimination | 60 % recall, higher when text is large | +| Embedding gallery | **rejection** - saying "unknown" | 5 % false accepts at 0.60 | +| Fine-tuned embedding | fine-grained SKUs (milk cartons) | not built; off-the-shelf is ~25 % | +| Confirmation loop | everything the above cannot settle | not built | + +So the honest design is: **reject by default**, name the product from the tag when it +is readable, from the packaging text when it is legible, from a fine-tuned embedding +otherwise - and hand the remainder to the operator, whose answers become the next +enrollment.