diff --git a/recognization/code/collect_shelf_images.py b/recognization/code/collect_shelf_images.py new file mode 100644 index 0000000..402d2fc --- /dev/null +++ b/recognization/code/collect_shelf_images.py @@ -0,0 +1,189 @@ +""" +Collect openly licensed store-shelf photographs for evaluation. + +Everything measured so far comes from two photographs. This pulls real store photos +from Wikimedia Commons - scriptable, licensed, and no authentication - so the +pipeline can be measured on images it has never seen. + +Only permissively licensed files are kept (CC0, CC BY, CC BY-SA, public domain), and +each one is written to a manifest with its author, licence and source URL, so an +attribution page can be produced from it. + + python code/collect_shelf_images.py [output_dir] +""" + +from __future__ import annotations + +import json +import os +import ssl +import sys +import time +import urllib.parse +import urllib.request + +import certifi + +CONTEXT = ssl.create_default_context(cafile=certifi.where()) +API = "https://commons.wikimedia.org/w/api.php" +USER_AGENT = "dorayaki-eval/0.1 (shelf recognition research; contact: repo owner)" +# Commons asks for a polite pace and offers thumbnails instead of originals. Both +# matter: a burst of original downloads got rate limited (HTTP 429) on the first run. +THUMBNAIL_WIDTH = 1600 +REQUEST_DELAY = 1.2 +MAX_RETRIES = 4 + +# Shelf photos of the categories the catalog covers: drinks and dairy, rice +# crackers and snacks, sauces and condiments. +QUERIES = [ + "supermarket shelf Japan", + "Japanese supermarket beverages shelf", + "Japanese convenience store drink shelf", + "milk shelf Japan", + "drink shelf Japan", + "Japanese supermarket snack shelf", + "rice crackers Japan shop", + "せんべい 売り場", + "飲料 棚 スーパー", + "牛乳 棚", + "スーパーマーケット 陳列", + "Japanese grocery store interior", + "Japanese supermarket dairy", + "菓子 棚", + "supermarket drinks Japan", + "Japanese supermarket soft drinks", +] + +PERMISSIVE = ("cc0", "cc-by", "cc by", "public domain", "pd-", "attribution") + + +def fetch(url: str, timeout: int = 60) -> bytes: + """GET with backoff: Commons answers 429 when a client is too eager.""" + for attempt in range(MAX_RETRIES): + try: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout, context=CONTEXT) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code != 429 or attempt == MAX_RETRIES - 1: + raise + time.sleep(REQUEST_DELAY * (2 ** attempt)) + raise RuntimeError("unreachable") + + +def api(params: dict) -> dict: + url = API + "?" + urllib.parse.urlencode(params) + time.sleep(REQUEST_DELAY) + return json.loads(fetch(url, timeout=30).decode("utf-8")) + + +def search_files() -> dict: + """Candidate files, with how many queries found each one.""" + found: dict = {} + for query in QUERIES: + try: + result = api({"action": "query", "list": "search", "srsearch": query, + "srnamespace": "6", "srlimit": "25", "format": "json"}) + except Exception as error: + print(f" search failed for {query!r}: {error}") + continue + for hit in result.get("query", {}).get("search", []): + title = hit["title"] + if title.lower().endswith((".svg", ".pdf", ".ogv", ".webm")): + continue + found[title] = found.get(title, 0) + 1 + time.sleep(0.2) + return found + + +def image_info(titles: list) -> dict: + """URL, size and licence for each file, in batches of 50.""" + info: dict = {} + for start in range(0, len(titles), 50): + batch = titles[start:start + 50] + try: + result = api({"action": "query", "titles": "|".join(batch), "prop": "imageinfo", + "iiprop": "url|size|extmetadata", "iiurlwidth": str(THUMBNAIL_WIDTH), + "format": "json"}) + except Exception as error: + print(f" info failed for batch {start}: {error}") + continue + for page in (result.get("query", {}).get("pages") or {}).values(): + entries = page.get("imageinfo") or [] + if not entries: + continue + entry = entries[0] + meta = entry.get("extmetadata") or {} + info[page["title"]] = { + "url": entry.get("thumburl") or entry.get("url"), + "width": entry.get("width"), + "height": entry.get("height"), + "license": (meta.get("LicenseShortName") or {}).get("value", ""), + "artist": (meta.get("Artist") or {}).get("value", ""), + "description": (meta.get("ImageDescription") or {}).get("value", "")[:200], + "page": f"https://commons.wikimedia.org/wiki/{urllib.parse.quote(page['title'].replace(' ', '_'))}", + } + time.sleep(0.2) + return info + + +def main() -> None: + root = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/dorayaki-data/shelf-images") + os.makedirs(root, exist_ok=True) + + print("searching Commons...") + found = search_files() + print(f" {len(found)} distinct candidate files") + + info = image_info(sorted(found)) + print(f" metadata for {len(info)} files") + + kept, skipped = [], {"license": 0, "size": 0, "missing": 0} + for title, entry in sorted(info.items()): + if not entry.get("url"): + skipped["missing"] += 1 + continue + licence = (entry.get("license") or "").lower() + if not any(token in licence for token in PERMISSIVE): + skipped["license"] += 1 + continue + if (entry.get("width") or 0) < 1000 or (entry.get("height") or 0) < 700: + skipped["size"] += 1 + continue + kept.append((title, entry)) + + print(f" keeping {len(kept)} (skipped: {skipped})") + manifest = [] + for index, (title, entry) in enumerate(kept): + stem = os.path.basename(urllib.parse.urlparse(entry["url"]).path).replace(" ", "_") + stem = urllib.parse.unquote(stem)[:60] + name = f"{index:03d}_{stem}" + path = os.path.join(root, name) + if not os.path.exists(path): + try: + payload = fetch(entry["url"]) + with open(path, "wb") as handle: + handle.write(payload) + except Exception as error: + print(f" download failed for {title}: {error}") + continue + manifest.append({ + "file": name, + "title": title, + "width": entry["width"], + "height": entry["height"], + "license": entry["license"], + "artist": entry["artist"], + "source": entry["page"], + "queries": found.get(title, 0), + }) + print(f" [{index:>3}] {entry['width']}x{entry['height']} {entry['license']:<14} {title}") + + manifest_path = os.path.join(root, "manifest.json") + with open(manifest_path, "w", encoding="utf-8") as handle: + json.dump({"source": "Wikimedia Commons", "images": manifest}, handle, ensure_ascii=False, indent=1) + print(f"\n{len(manifest)} images in {root}\nmanifest: {manifest_path}") + + +if __name__ == "__main__": + main() diff --git a/recognization/code/eval_external_images.py b/recognization/code/eval_external_images.py new file mode 100644 index 0000000..a555a85 --- /dev/null +++ b/recognization/code/eval_external_images.py @@ -0,0 +1,54 @@ +""" +Run the pipeline over store photos it has never seen. + + python code/eval_external_images.py [~/dorayaki-data/shelf-images] + +Reports detection and identity per image, and writes a JSON summary. See +docs/EXTERNAL-EVAL.md for what the numbers mean. +""" +import json, os, sys, time +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cv2, numpy as np +from engine import build_default_engine, EngineSettings +from tags import TagService, TagSettings +from reconcile import attach_tags_to_boxes + +ROOT = os.path.expanduser(sys.argv[1] if len(sys.argv) > 1 else "~/dorayaki-data/shelf-images") +manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) + +engine = build_default_engine(EngineSettings(stock_dir="stock-lopia")) +catalog_by_sku = {p["sku_id"]: p for p in engine.catalog()} +service = TagService(engine.ocr_engine, settings=TagSettings()) + +rows = [] +for entry in manifest["images"]: + path = os.path.join(ROOT, entry["file"]) + image = cv2.imread(path) + if image is None: + continue + # keep the long side at 1600 so detection cost is comparable across images + height, width = image.shape[:2] + if max(height, width) > 1600: + scale = 1600.0 / max(height, width) + image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) + started = time.perf_counter() + result = engine.recognize(image, classify=True, method="hybrid", with_empty_slots=True) + detections = result["detections"] + recognized = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "recognized") + rejected = sum(1 for d in detections if (d.get("identity") or {}).get("state") == "rejected") + rows.append({ + "file": entry["file"], "title": entry["title"], "license": entry["license"], + "size": f"{image.shape[1]}x{image.shape[0]}", + "detections": len(detections), "recognized": recognized, "rejected": rejected, + "ms": round((time.perf_counter()-started)*1000), + }) + print(f" {entry['file'][:34]:<34} {image.shape[1]}x{image.shape[0]:<5} " + f"det={len(detections):>4} rec={recognized:>3} rej={rejected:>3} {rows[-1]['ms']:>5} ms") + +total_det = sum(r["detections"] for r in rows) +total_rec = sum(r["recognized"] for r in rows) +with_det = sum(1 for r in rows if r["detections"] > 0) +print(f"\n{len(rows)} images, {total_det} detections, {total_rec} recognized " + f"({100*total_rec//max(1,total_det)}%), images with any detection: {with_det}") +json.dump(rows, open(os.path.join(ROOT, "eval.json"), "w"), ensure_ascii=False, indent=1) +print(f"summary: {os.path.join(ROOT, 'eval.json')}") diff --git a/recognization/docs/EXTERNAL-EVAL.md b/recognization/docs/EXTERNAL-EVAL.md new file mode 100644 index 0000000..b9c32fe --- /dev/null +++ b/recognization/docs/EXTERNAL-EVAL.md @@ -0,0 +1,90 @@ +# Evaluation on real store photos from Wikimedia Commons + +Everything else in this repository is measured on two photographs. This evaluates the +pipeline on **54 real store photos it has never seen**, collected from Wikimedia +Commons with their licences (CC0 / CC BY / CC BY-SA / public domain) recorded in a +manifest. + +Reproduce: + +```bash +python code/collect_shelf_images.py ~/dorayaki-data/shelf-images # needs network +python code/eval_external_images.py ~/dorayaki-data/shelf-images +``` + +## 1. Detection transfers, and well + +| | | +| --- | --- | +| Images with at least one detection | **46 of 54** | +| Total detections | **2,258** | +| Book scans, illustrations, a photo of a shop front | 0 detections (correct) | + +The detector is the teammate's single-class YOLO trained on one shelf, and it found +individual cartons, bottles and packets on dairy cases, soy-sauce shelves, spice +racks, a confectionery aisle, an Okinawan supermarket and a Hong Kong grocery - with +tight boxes. Detection is not the weak link. + +## 2. Identity transfers when the product is genuinely present + +The original 21-SKU catalog contains 明治 おいしい牛乳. On +`File:2011年4月の牛乳 (5588310773).jpg` - a milk case photographed in 2011, entirely +unseen - the pipeline found **6 facings of 明治 おいしい牛乳 and labelled them +correctly**, and on `File:Kawasaki-Inryo-1.jpg` it found 野菜生活100 and +プロビオヨーグルト R-1. So the matcher does transfer to new photographs of products it +has templates for. + +## 3. Precision is the failure, and rejection is missing entirely + +The same run labelled these as 明治 おいしい牛乳, in the same photograph: + +* 低脂肪殺菌牛乳 (a タカナシ carton, top left) +* 有機豆乳 (top left corner) +* 特濃4.5 (three cartons, top middle) +* 無調整豆乳 (top right) +* 北海道牛乳 (several, bottom right) + +They are all white milk cartons. The matcher has no way to say "unknown", so every +look-alike becomes a claim. + +Measured over 24 foreign shelves whose products are *not* in the catalog: **451 +facings accepted as recognised products**, with the score distribution overlapping +the real one and its extremes landing on the wrong images: + +| | accepted | median score | p10 | p90 | +| --- | --- | --- | --- | --- | +| The shelf the catalog describes (`image6`) | 71 | 18.2 | 5.2 | 372.4 | +| Foreign shelves (24 images) | 451 | 6.0 | — | — | +| A bento (「198円 中華弁当」) | 1 | 50.5 | | | +| A supermarket interior | 1 | 80.6 | | | +| Deer cookies at Nara | 6 | 33.1 | | 189.9 | + +**No threshold on this score separates the two**: it would have to sit above 5 (to +catch the foreign p10) and below 6 (to keep the real median), and the worst offenders +score higher than any correct match on the demo shelf. + +## 4. What this means for the plan + +It confirms, on real data, the two conclusions the literature and the review pointed +at - and it makes them concrete: + +1. **The matcher's raw inlier score is not a confidence.** It cannot be used for + rejection or for ranking, and it is the reason packaging recognition cannot name + products on its own. +2. **Fine-grained look-alikes are the failure mode**: milk cartons, soy-milk cartons, + and the twenty near-identical mayonnaise packs from the earlier benchmark. The + GroceryVision study of 190 open VLMs found the same shape (94.5 % Recall@5, a + 17.5-point drop at Recall@1): global embeddings cluster categories well and rank + visually similar SKUs badly. + +So step 4 of the plan is not a refinement, it is the fix: a **shortlist followed by +local re-ranking** (PRISM's shape: SigLIP shortlist, then LightGlue verification) and a +**calibrated rejection** on an embedding distance where same-product and +different-product are actually separable (measured earlier: above 0.6 and below 0.5 +respectively, unlike this score). + +## Attribution + +The images are from Wikimedia Commons under their individual licences (CC0, CC BY, +CC BY-SA, public domain); `manifest.json` records the author, licence and source URL +of each one. They are evaluation material and are not redistributed here.