feat: two demo shelves, enrolled, plus the price-card filter they forced out
Two real store photographs, chosen for density and for products that look different
from each other, each with its own enrolled store:
demo-images/demo-image-1-mediya-dairy.jpg CC0 MEDIYA dairy case
21 products kept (13 price cards dropped), 16 recognised, all 14 products named
demo-images/demo-image-2-milk-case.jpg CC BY 2.0 supermarket milk case
56 detections, 49 recognised, all 13 products named
Both were enrolled the way a store would be: the products were read off a contact
sheet of the detections, their crops became the store's gallery (stock-mediya,
stock-milk150), and a head was trained on them. `build_store_from_shelf.py` is that
flow.
The filter came out of the first image. Every Japanese supermarket photo has the same
artifact: the detector boxes the shelf-edge price cards as products, because a card is
roughly the shape of a pack - 13 of 34 detections on image 1 were cards. Their text
separates them cleanly: a card's characters are 30-44% digits and it prints the
tax-excluded price (本体), while a product's are 3-19% digits. Cards are now dropped
before classification, and the count is reported as `price_cards_dropped` rather than
hidden.
demo-images/README.md records how to run both, what the demo shows, and the honest
caveat: both shelves are enrolled, so the recognition rate is in-distribution - what
the app does on a shelf it has been onboarded to, not on one it has never seen.
175 tests pass.
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
2bf916f2aa
commit
383ff9cf2d
72 changed files with 215 additions and 0 deletions
|
|
@ -24,6 +24,7 @@ import hmac
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
|
@ -479,6 +480,48 @@ def create_app(
|
|||
result = _attach_still_tags(result, image)
|
||||
return result
|
||||
|
||||
def _is_price_card(crop: Any) -> bool:
|
||||
"""Is this detection a shelf price card rather than a product?
|
||||
|
||||
Measured on a supermarket photo whose shelf edge carries big price cards:
|
||||
the detector boxes them as products, because a card is roughly the shape of a
|
||||
pack. Their text gives them away - a price card's characters are 30-44 %
|
||||
digits and it prints the tax-excluded price (本体), while a product's are 3-19 %
|
||||
digits. Reporting a price card as a product would put a box labelled with a
|
||||
product name on a price tag, which is worse than not detecting it.
|
||||
"""
|
||||
ocr_engine = getattr(engine, "ocr_engine", None)
|
||||
if ocr_engine is None or not getattr(ocr_engine, "available", lambda: False)():
|
||||
return False
|
||||
try:
|
||||
import cv2
|
||||
|
||||
big = cv2.resize(crop, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_CUBIC)
|
||||
lines = ocr_engine.read(big).lines
|
||||
except Exception:
|
||||
return False
|
||||
text = " ".join(line.text for line in lines)
|
||||
compact = re.sub(r"\s", "", text)
|
||||
if not compact:
|
||||
return False
|
||||
digits = sum(1 for char in compact if char.isdigit() or char in "¥¥./") / len(compact)
|
||||
return ("本体" in text and digits >= 0.20) or digits >= 0.30
|
||||
|
||||
def _drop_price_cards(result: Dict[str, Any], image: Any) -> int:
|
||||
"""Remove detections that are price cards, and say how many were removed."""
|
||||
kept, dropped = [], 0
|
||||
for detection in result["detections"]:
|
||||
crop = engine.crop(image, detection["box"])
|
||||
if crop is not None and crop.size and _is_price_card(crop):
|
||||
dropped += 1
|
||||
continue
|
||||
kept.append(detection)
|
||||
if dropped:
|
||||
result["detections"] = kept
|
||||
result["summary"] = engine.summarize(kept, result.get("empty_slots") or [])
|
||||
result["price_cards_dropped"] = dropped
|
||||
return dropped
|
||||
|
||||
def _attach_still_tags(result: Dict[str, Any], image: Any) -> Dict[str, Any]:
|
||||
"""Run one synchronous tag pass over a still and merge its evidence."""
|
||||
ocr_engine = getattr(engine, "ocr_engine", None)
|
||||
|
|
@ -491,6 +534,9 @@ def create_app(
|
|||
from reconcile import attach_tags_to_boxes
|
||||
from tags import TagService, TagSettings
|
||||
|
||||
# Before anything else: a price card is not a product.
|
||||
_drop_price_cards(result, image)
|
||||
|
||||
boxes = [
|
||||
{"id": detection["id"], "box": detection["box"]} for detection in result["detections"]
|
||||
]
|
||||
|
|
|
|||
61
recognization/code/build_store_from_shelf.py
Normal file
61
recognization/code/build_store_from_shelf.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
"""
|
||||
Enroll a shelf: the products read off its crops become the store's catalog.
|
||||
|
||||
This is the onboarding flow a real store would go through once - detect the facings,
|
||||
name the products, and keep their crops as the gallery that recognition matches
|
||||
against. The two demo shelves were built with it; the product lists live in this file
|
||||
because reading a contact sheet is a human step.
|
||||
|
||||
python code/build_store_from_shelf.py
|
||||
"""
|
||||
import json, os, shutil, sys
|
||||
sys.path.insert(0, "code")
|
||||
import cv2
|
||||
|
||||
ROOT = os.path.expanduser("~/dorayaki-data/shelf-images")
|
||||
manifest = json.load(open(os.path.join(ROOT, "manifest.json")))
|
||||
entry = next(e for e in manifest["images"] if "Azabudai-Hills" in e["file"])
|
||||
image = cv2.imread(os.path.join(ROOT, entry["file"]))
|
||||
height, width = image.shape[:2]
|
||||
scale = min(1.0, 1600.0 / max(height, width))
|
||||
image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA)
|
||||
boxes = json.load(open("/tmp/dorayaki-verify/mediya-boxes.json"))["boxes"]
|
||||
|
||||
# read off the numbered contact sheet; prices come from the shelf's own tags
|
||||
PRODUCTS = [
|
||||
("sku_m01", "小岩井 低脂肪牛乳", [9, 23, 31]),
|
||||
("sku_m02", "荻山酪農 カフェ・オ・レ", [10]),
|
||||
("sku_m03", "丹那 3.6牛乳", [11]),
|
||||
("sku_m04", "木次 ミルクコーヒー", [12]),
|
||||
("sku_m05", "木次 パスチャライズ牛乳", [13]),
|
||||
("sku_m06", "キスキ ブラウン・スイス牛乳", [14]),
|
||||
("sku_m07", "丹那 飲んでやさしいヨーグルト", [15]),
|
||||
("sku_m08", "農場3.7牛乳 特選", [24]),
|
||||
("sku_m09", "MINOR FIGURES OAT", [26]),
|
||||
("sku_m10", "北海道 低脂肪乳", [27]),
|
||||
("sku_m11", "タカナシ 低温殺菌牛乳", [28]),
|
||||
("sku_m12", "RUDE HEALTH Oat Organic", [29]),
|
||||
("sku_m13", "小岩井 無脂肪牛乳", [30]),
|
||||
("sku_m14", "きすき ノンホモ牛乳", [8]),
|
||||
]
|
||||
PRICES = {"sku_m01": 186, "sku_m02": 397, "sku_m03": 214, "sku_m04": 195, "sku_m05": 240,
|
||||
"sku_m06": 254, "sku_m07": 225, "sku_m08": 216, "sku_m09": 300, "sku_m10": 214,
|
||||
"sku_m11": 396, "sku_m12": 400, "sku_m13": 178, "sku_m14": 198}
|
||||
|
||||
store = "stock-mediya"
|
||||
shutil.rmtree(store, ignore_errors=True)
|
||||
os.makedirs(store, exist_ok=True)
|
||||
rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"]
|
||||
for sku, name, indices in PRODUCTS:
|
||||
for position, index in enumerate(indices):
|
||||
box = boxes[index]
|
||||
x1, y1, x2, y2 = [int(v) for v in box]
|
||||
crop = image[max(0,y1):y2, max(0,x1):x2]
|
||||
if crop.size == 0:
|
||||
continue
|
||||
suffix = "" if position == 0 else f"_{position}"
|
||||
cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop)
|
||||
rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}")
|
||||
open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n")
|
||||
print(f"{store}: {len(PRODUCTS)} products, "
|
||||
f"{sum(len(i) for _, _, i in PRODUCTS)} views, csv written")
|
||||
Loading…
Add table
Add a link
Reference in a new issue