tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

Compare commits

base: tngo:19af7ce54535987e5496c90373239adc2b5f8f82
Branches Tags
tngo:main
tngo:fix/live-recognition-quality
vnguyen:main
..
compare: tngo:06f9fb4819af22c6883ba66965601facdc5cde8f
Branches Tags
tngo:fix/live-recognition-quality
tngo:main
vnguyen:main

No commits in common. "19af7ce54535987e5496c90373239adc2b5f8f82" and "06f9fb4819af22c6883ba66965601facdc5cde8f" have entirely different histories.
19af7ce545 ... 06f9fb4819

12 changed files with 176 additions and 625 deletions
Download patch file Download diff file Expand all files Collapse all files

1
recognization/.gitignore vendored
View file

@ -19,7 +19,6 @@ ENV/
# Temporary weights
runs/detect/train/weights/last.pt
weights/dinov2_vits14_lvd142m.safetensors
*.log
stock-*/identity_head.pt
.demo-token

15
recognization/README.md
View file

@ -25,7 +25,6 @@ Cần có đầy đủ:
- `code/` - mã nguồn server, client web, client mẫu.
- `runs/` - trọng số YOLO đã train (`runs/detect/train/weights/best.pt`).
- `weights/dinov2_vits14_lvd142m.safetensors` - DINOv2 backbone for the identity gallery. Not in Git. See "DINOv2 weights" below.
- `stock/` - catalog `stock.csv` và ảnh mẫu sản phẩm `sku_001` -> `sku_022`.
- `add/` và `image.jpg` - ảnh kệ mẫu để test/demo.
- `requirements.txt`, `requirements-dev.txt`.
@ -50,20 +49,6 @@ source .venv/bin/activate # Windows: .venv\Scripts\activate
pip install -r requirements-dev.txt
```
### DINOv2 weights
The identity gallery embeds crops with DINOv2 ViT-S/14 (LVD-142M), Apache-2.0. It loads only a local file and does not contact Hugging Face, including when `YOLO_OFFLINE` is unset. Without the file, gallery identity fails and the server log says the weights are missing.
The pinned file is `model.safetensors` from `timm/vit_small_patch14_dinov2.lvd142m`, sha256 `04d27f3400d059fc0cfd7d17dd1909a75bf3ea8fb3eeb48b97cb99e57ee20081` (~84 MB). Copy a cache you already have; do not fetch one from an unknown URL:
```bash
mkdir -p weights
cp "$HOME/.cache/huggingface/hub/models--timm--vit_small_patch14_dinov2.lvd142m/snapshots/"*/model.safetensors \
weights/dinov2_vits14_lvd142m.safetensors
```
`DORAYAKI_DINOV2_WEIGHTS` overrides that path. A file with any other sha256 is refused.
(`requirements-dev.txt` = `requirements.txt` + `httpx` cho test.)
### Bước 4: Khởi chạy recognition server

8
recognization/code/app.py
View file

@ -270,8 +270,7 @@ def create_app(
async def lifespan(_app: FastAPI):
yield
for session in list(sessions.values()):
if session is not None:
session.close()
session.close()
engine.shutdown()
app = FastAPI(
@ -1112,11 +1111,6 @@ def main() -> None:
LOGGER.info("Loading model and catalog (device=%s, imgsz=%s)...",
engine_settings.device or "auto", engine_settings.imgsz)
engine = build_default_engine(engine_settings)
if (engine_settings.identity_backend or "").lower() == "gallery":
if engine.identity_gallery() is None:
raise RuntimeError(
"gallery identity backend requires local DINOv2 weights; refusing to start"
)
warm_ms = engine.warm_up()
LOGGER.info(
"Ready: %d SKUs, catalog %s, warmup %.0f ms, classify workers %d",

303
recognization/code/build_store_from_shelf.py
View file

@ -1,253 +1,66 @@
"""
Crop a labelled shelf photo into a stock directory.
Enroll a shelf: the products read off its crops become the store's catalog.
This is a one-shot offline helper. SKU names and prices are supplied by a human
who read a contact sheet; this script does not enroll a phone recording and
does not talk to the recognition server.
This is the onboarding flow a real store would go through once - detect the facings,
name the products, and keep their crops as the gallery that recognition matches
against. The two demo shelves were built with it; the product lists live in this file
because reading a contact sheet is a human step.
The Meidiya dairy labels that built stock-mediya are kept as --example mediya.
Point --image, --boxes, and --output at real paths. There is no default home
directory or /tmp path.
python code/build_store_from_shelf.py \\
--image shelf.jpg --boxes boxes.json --output /path/to/new-stock \\
--labels labels.json
boxes.json is {"boxes": [[x1, y1, x2, y2], ...]} in the image's own pixels;
this script does not rescale the photo. labels.json is
{"products": [{"sku", "name", "price", "boxes": [index, ...]}]}.
An existing output directory is left untouched unless --replace is passed,
and even then the old directory is renamed to "<output>.previous", not deleted.
python code/build_store_from_shelf.py
"""
from __future__ import annotations
import json, os, shutil, sys
sys.path.insert(0, "code")
import cv2
import argparse
import csv
import json
import os
import shutil
import sys
import tempfile
ROOT = os.path.expanduser("~/dorayaki-data/shelf-images")
manifest = json.load(open(os.path.join(ROOT, "manifest.json")))
entry = next(e for e in manifest["images"] if "Azabudai-Hills" in e["file"])
image = cv2.imread(os.path.join(ROOT, entry["file"]))
height, width = image.shape[:2]
scale = min(1.0, 1600.0 / max(height, width))
image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA)
boxes = json.load(open("/tmp/dorayaki-verify/mediya-boxes.json"))["boxes"]
# ponytail: demo counts/dates are the Meidiya sheet's bookkeeping, not a catalog API.
EXAMPLE_RECEIPT = "2026-09-25"
EXAMPLE_EXPIRATION = "2026-10-20"
EXAMPLE_COUNTS = (24, 8, 16)
# Named example only. These are the labels read off the Meidiya contact sheet,
# not an enrollment of a phone walk.
MEDIYA_PRODUCTS = [
{"sku": "sku_m01", "name": "小岩井 低脂肪牛乳", "price": 186, "boxes": [9, 23, 31]},
{"sku": "sku_m02", "name": "荻山酪農 カフェ・オ・レ", "price": 397, "boxes": [10]},
{"sku": "sku_m03", "name": "丹那 3.6牛乳", "price": 214, "boxes": [11]},
{"sku": "sku_m04", "name": "木次 ミルクコーヒー", "price": 195, "boxes": [12]},
{"sku": "sku_m05", "name": "木次 パスチャライズ牛乳", "price": 240, "boxes": [13]},
{"sku": "sku_m06", "name": "キスキ ブラウン・スイス牛乳", "price": 254, "boxes": [14]},
{"sku": "sku_m07", "name": "丹那 飲んでやさしいヨーグルト", "price": 225, "boxes": [15]},
{"sku": "sku_m08", "name": "農場3.7牛乳 特選", "price": 216, "boxes": [24]},
{"sku": "sku_m09", "name": "MINOR FIGURES OAT", "price": 300, "boxes": [26]},
{"sku": "sku_m10", "name": "北海道 低脂肪乳", "price": 214, "boxes": [27]},
{"sku": "sku_m11", "name": "タカナシ 低温殺菌牛乳", "price": 396, "boxes": [28]},
{"sku": "sku_m12", "name": "RUDE HEALTH Oat Organic", "price": 400, "boxes": [29]},
{"sku": "sku_m13", "name": "小岩井 無脂肪牛乳", "price": 178, "boxes": [30]},
{"sku": "sku_m14", "name": "きすき ノンホモ牛乳", "price": 198, "boxes": [8]},
# read off the numbered contact sheet; prices come from the shelf's own tags
PRODUCTS = [
("sku_m01", "小岩井 低脂肪牛乳", [9, 23, 31]),
("sku_m02", "荻山酪農 カフェ・オ・レ", [10]),
("sku_m03", "丹那 3.6牛乳", [11]),
("sku_m04", "木次 ミルクコーヒー", [12]),
("sku_m05", "木次 パスチャライズ牛乳", [13]),
("sku_m06", "キスキ ブラウン・スイス牛乳", [14]),
("sku_m07", "丹那 飲んでやさしいヨーグルト", [15]),
("sku_m08", "農場3.7牛乳 特選", [24]),
("sku_m09", "MINOR FIGURES OAT", [26]),
("sku_m10", "北海道 低脂肪乳", [27]),
("sku_m11", "タカナシ 低温殺菌牛乳", [28]),
("sku_m12", "RUDE HEALTH Oat Organic", [29]),
("sku_m13", "小岩井 無脂肪牛乳", [30]),
("sku_m14", "きすき ノンホモ牛乳", [8]),
]
PRICES = {"sku_m01": 186, "sku_m02": 397, "sku_m03": 214, "sku_m04": 195, "sku_m05": 240,
"sku_m06": 254, "sku_m07": 225, "sku_m08": 216, "sku_m09": 300, "sku_m10": 214,
"sku_m11": 396, "sku_m12": 400, "sku_m13": 178, "sku_m14": 198}
class BuildStoreError(Exception):
"""Refused input. The destination, if it already existed, is unchanged."""
def _is_within(path: str, root: str) -> bool:
path = os.path.realpath(path)
root = os.path.realpath(root)
return path == root or path.startswith(root + os.sep)
def _load_json(path: str):
try:
with open(path, encoding="utf-8") as handle:
return json.load(handle)
except (OSError, json.JSONDecodeError) as exc:
raise BuildStoreError(f"{path}: {exc}") from exc
def load_boxes(path: str) -> list:
payload = _load_json(path)
boxes = payload.get("boxes") if isinstance(payload, dict) else None
if not isinstance(boxes, list) or not boxes:
raise BuildStoreError(f"{path}: expected a non-empty {{\"boxes\": [...]}} list")
parsed = []
for index, box in enumerate(boxes):
if not isinstance(box, (list, tuple)) or len(box) != 4:
raise BuildStoreError(f"{path}: box {index} is not [x1, y1, x2, y2]")
try:
coords = [int(value) for value in box]
except (TypeError, ValueError) as exc:
raise BuildStoreError(f"{path}: box {index} is not four numbers") from exc
x1, y1, x2, y2 = coords
if x2 <= x1 or y2 <= y1:
raise BuildStoreError(f"{path}: box {index} has no area")
parsed.append(coords)
return parsed
def load_labels(path: str | None, example: str | None) -> list:
if example and path:
raise BuildStoreError("pass either --labels or --example, not both")
if example == "mediya":
return [dict(item) for item in MEDIYA_PRODUCTS]
if example:
raise BuildStoreError(f"unknown example {example!r}; the only named example is mediya")
if not path:
raise BuildStoreError("pass --labels, or --example mediya for the demo sheet")
payload = _load_json(path)
products = payload.get("products") if isinstance(payload, dict) else None
if not isinstance(products, list) or not products:
raise BuildStoreError(f"{path}: expected a non-empty {{\"products\": [...]}} list")
return products
def _sku_name(value) -> str:
text = str(value).strip()
if not text or text in {".", ".."} or "/" in text or "\\" in text or "\x00" in text:
raise BuildStoreError(f"refusing sku {value!r}: it is not a single file name")
return text
def _products(labels: list) -> list:
seen = set()
products = []
for item in labels:
if not isinstance(item, dict):
raise BuildStoreError("each product must be an object with sku, name, price, boxes")
sku = _sku_name(item.get("sku", ""))
if sku in seen:
raise BuildStoreError(f"duplicate sku {sku}")
seen.add(sku)
name = str(item.get("name", "")).strip()
if not name or any(char in name for char in ",\n\r"):
raise BuildStoreError(f"{sku}: name must be one CSV field")
try:
price = int(item["price"])
indices = [int(index) for index in item["boxes"]]
except (KeyError, TypeError, ValueError) as exc:
raise BuildStoreError(f"{sku}: price and boxes must be integers") from exc
if price < 0 or not indices:
raise BuildStoreError(f"{sku}: price and boxes must be present")
products.append({"sku": sku, "name": name, "price": price, "boxes": indices})
return products
def check_destination(output: str, image: str, boxes_path: str, labels_path: str | None, replace: bool) -> str:
output = os.path.abspath(output)
real_output = os.path.realpath(output)
if real_output == os.path.realpath(os.sep) or real_output == os.path.realpath(os.getcwd()):
raise BuildStoreError("refusing to use the filesystem root or the current directory as the store")
parent = os.path.dirname(output)
if not parent or not os.path.isdir(parent):
raise BuildStoreError(f"{output}: parent directory does not exist")
for source in (image, boxes_path, labels_path):
if source and (_is_within(source, real_output) or _is_within(real_output, os.path.realpath(source))):
raise BuildStoreError(f"refusing to write {output}: it aliases input {source}")
if os.path.lexists(output):
if not replace:
raise BuildStoreError(f"{output}: already exists; pass --replace to move it aside")
if os.path.islink(output) or not os.path.isdir(output):
raise BuildStoreError(f"{output}: --replace only moves aside a real directory")
previous = output + ".previous"
if os.path.lexists(previous):
raise BuildStoreError(f"{previous}: already exists; move it away before --replace")
return output
def build_store(image_path: str, boxes: list, products: list, output: str, replace: bool,
boxes_path: str, labels_path: str | None) -> int:
output = check_destination(output, image_path, boxes_path, labels_path, replace)
import cv2
image = cv2.imread(image_path)
if image is None or getattr(image, "size", 0) == 0:
raise BuildStoreError(f"{image_path}: not a readable image")
height, width = image.shape[:2]
parent = os.path.dirname(output)
staging = tempfile.mkdtemp(prefix=".build-store-", dir=parent)
try:
rows = [[
"SKU_ID", "product_name", "receipt_date", "expiration_date",
"number_in", "number_out", "number_on", "image_link", "price",
]]
written = []
for product in products:
saved = 0
for index in product["boxes"]:
if index < 0 or index >= len(boxes):
raise BuildStoreError(f"{product['sku']}: box {index} is outside the boxes file")
x1, y1, x2, y2 = boxes[index]
x1, y1 = max(0, x1), max(0, y1)
x2, y2 = min(width, x2), min(height, y2)
crop = image[y1:y2, x1:x2] if x2 > x1 and y2 > y1 else image[0:0, 0:0]
if crop.size == 0:
raise BuildStoreError(f"{product['sku']}: box {index} does not crop the image")
suffix = "" if saved == 0 else f"_{saved}"
target = os.path.join(staging, f"{product['sku']}{suffix}.jpeg")
if not cv2.imwrite(target, crop):
raise BuildStoreError(f"{product['sku']}: failed to write crop {index}")
saved += 1
written.append(product["sku"])
number_in, number_out, number_on = EXAMPLE_COUNTS
rows.append([
product["sku"], product["name"], EXAMPLE_RECEIPT, EXAMPLE_EXPIRATION,
number_in, number_out, number_on, "", product["price"],
])
with open(os.path.join(staging, "stock.csv"), "w", encoding="utf-8", newline="") as handle:
csv.writer(handle).writerows(rows)
if not os.path.lexists(output):
os.rename(staging, output)
staging = None
else:
previous = output + ".previous"
os.rename(output, previous)
try:
os.rename(staging, output)
except OSError:
os.rename(previous, output)
raise
staging = None
print(f"previous store kept at {previous}", file=sys.stderr)
return len(written)
finally:
if staging and os.path.isdir(staging):
shutil.rmtree(staging)
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--image", required=True, help="shelf photo whose pixels match the box coordinates")
parser.add_argument("--boxes", required=True, help='JSON file {"boxes": [[x1, y1, x2, y2], ...]}')
parser.add_argument("--output", required=True, help="new stock directory to create")
parser.add_argument("--labels", help='JSON file {"products": [{"sku", "name", "price", "boxes"}]}')
parser.add_argument("--example", choices=["mediya"], help="use the named demo contact-sheet labels")
parser.add_argument("--replace", action="store_true", help="replace an existing output directory")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
try:
products = _products(load_labels(args.labels, args.example))
count = build_store(
args.image, load_boxes(args.boxes), products, args.output, args.replace,
args.boxes, args.labels,
)
except BuildStoreError as exc:
print(f"build_store_from_shelf: {exc}", file=sys.stderr)
return 1
print(f"{os.path.abspath(args.output)}: {count} products, csv written")
print("phone recordings are not enrolled by this script; labels stay a manual contact sheet")
return 0
if __name__ == "__main__":
raise SystemExit(main())
store = "stock-mediya"
shutil.rmtree(store, ignore_errors=True)
os.makedirs(store, exist_ok=True)
rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"]
written = []
for sku, name, indices in PRODUCTS:
saved = 0
for position, index in enumerate(indices):
box = boxes[index]
x1, y1, x2, y2 = [int(v) for v in box]
crop = image[max(0, y1):y2, max(0, x1):x2]
if crop.size == 0:
continue
suffix = "" if saved == 0 else f"_{saved}"
cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop)
saved += 1
if saved == 0:
raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view")
written.append(sku)
rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}")
open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n")
print(f"{store}: {len(written)} products, csv written")

80
recognization/code/gallery.py
View file

@ -38,78 +38,6 @@ DEFAULT_THRESHOLD = 0.55
# The same line for a fine-tuned head, whose similarity scale is its own.
DEFAULT_HEAD_THRESHOLD = 0.40
# DINOv2 ViT-S/14, LVD-142M. timm's default source is the Hugging Face copy of
# these weights (Apache-2.0); pretrained=True fetches it even when YOLO_OFFLINE=1.
# The pin is the sha256 of that model.safetensors (timm hub snapshot 4610ca14).
# Not in Git (~84 MB). Place it at weights/dinov2_vits14_lvd142m.safetensors, or
# point DORAYAKI_DINOV2_WEIGHTS at a copy. See README "DINOv2 weights".
DINOV2_MODEL = "vit_small_patch14_dinov2.lvd142m"
DINOV2_SHA256 = "04d27f3400d059fc0cfd7d17dd1909a75bf3ea8fb3eeb48b97cb99e57ee20081"
DINOV2_FILENAME = "dinov2_vits14_lvd142m.safetensors"
def dinov2_weights_path() -> str:
override = os.environ.get("DORAYAKI_DINOV2_WEIGHTS")
if override:
return override
here = os.path.dirname(os.path.abspath(__file__))
return os.path.join(os.path.dirname(here), "weights", DINOV2_FILENAME)
def _sha256(path: str) -> str:
import hashlib
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def load_dinov2(image_size: int) -> Any:
"""Build the gallery backbone from the pinned local file. Never downloads.
A missing or altered file raises FileNotFoundError naming the path and the
expected sha256. The head in finetune_identity.py is a Linear(384, 128) on
this model's features; a different backbone will not load that head.
"""
import timm
from safetensors.torch import load_file
from timm.models.vision_transformer import checkpoint_filter_fn
path = dinov2_weights_path()
if not os.path.isfile(path):
raise FileNotFoundError(
f"DINOv2 weights not found at {path}. "
f"Copy model.safetensors (sha256 {DINOV2_SHA256}) from the local "
f"Hugging Face cache of timm/{DINOV2_MODEL}, or set DORAYAKI_DINOV2_WEIGHTS. "
f"The gallery does not download weights."
)
digest = _sha256(path)
if digest != DINOV2_SHA256:
raise FileNotFoundError(
f"DINOv2 weights at {path} are sha256 {digest}, expected {DINOV2_SHA256}."
)
model = timm.create_model(
DINOV2_MODEL,
pretrained=False,
num_classes=0,
img_size=int(image_size),
dynamic_img_size=True,
)
# The file is the 518px checkpoint. timm's own filter resizes pos_embed when
# the caller asks for another size; pretrained=True would do this and also
# download, so it stays off.
state = checkpoint_filter_fn(load_file(path), model)
missing, unexpected = model.load_state_dict(state, strict=False)
unexpected = [key for key in unexpected if not key.startswith("head.")]
if missing or unexpected:
raise RuntimeError(
f"DINOv2 weights at {path} do not match {DINOV2_MODEL} "
f"(missing={missing}, unexpected={unexpected})."
)
return model
@dataclass
class GalleryEntry:
@ -210,7 +138,13 @@ class ProductGallery:
device = self.device
if device is None:
device = "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu"
model = load_dinov2(self.image_size)
model = timm.create_model(
"vit_small_patch14_dinov2.lvd142m",
pretrained=True,
num_classes=0,
img_size=self.image_size,
dynamic_img_size=True,
)
model = model.eval().to(device)
self.device = device
self._model = model

28
recognization/code/live.py
View file

@ -191,6 +191,10 @@ class LiveSession:
self._shift_map = None
self._compliance_tracker = None
self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []}
# Settled tag + reconciliation from the last result, keyed by track.
# Inspect copies these; it does not re-run OCR.
self._settled_tags: Dict[int, Dict[str, Any]] = {}
self._settled_tag_seq = 0
self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque(
maxlen=LABEL_MEMORY_MAX
)
@ -293,6 +297,8 @@ class LiveSession:
self._latest_frame_id = None
self._frame_origin = [0.0, 0.0]
self._shift_map = None
self._settled_tags = {}
self._settled_tag_seq = 0
# ------------------------------------------------------------------ frame path
def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]:
@ -439,6 +445,17 @@ class LiveSession:
if tag_summary is not None:
summary["price_tags"] = tag_summary["tags"]
summary["compliance"] = tag_summary["compliance"]
settled: Dict[int, Dict[str, Any]] = {}
for payload in track_payload:
snapshot: Dict[str, Any] = {}
if "price_tag" in payload:
snapshot["price_tag"] = payload["price_tag"]
if "reconciliation" in payload:
snapshot["reconciliation"] = payload["reconciliation"]
if snapshot:
settled[int(payload["track_id"])] = snapshot
self._settled_tags = settled
self._settled_tag_seq = seq
self._submit_classification(submissions, method, generation)
self._run_tag_passes(image, tag_bands)
@ -912,7 +929,16 @@ class LiveSession:
if track is None or image is None:
raise KeyError("TRACK_NOT_IN_VIEW")
box = list(track.raw_box)
return self.engine.inspect(image, box, method=chosen_method)
# Snapshot is the frame current at tap time. engine.inspect runs
# outside the lock, so a newer frame may land first; the reply still
# carries the tag from the frame the tap matched, not that later one.
settled = None
if self._settled_tag_seq == self._latest_seq:
settled = self._settled_tags.get(track_id)
result = self.engine.inspect(image, box, method=chosen_method)
if settled:
result.update(settled)
return result
# ------------------------------------------------------------------ catalog cache
def refresh_catalog(self) -> None:

183
recognization/code/tags.py
View file

@ -84,19 +84,11 @@ _TAX_RE = re.compile(r"税(込|抜|不|別)")
@dataclass
class RailBand:
"""One horizontal strip of shelf rail, derived from a product row.
`box` is this frame's pixels (the crop `run_pass` reads). `origin` is the
shelf position of the frame origin at the moment `plan` chose the band, and
`key` is the rail that origin names. Both travel with the band: a later
frame's origin must not relabel a pass that is still queued.
"""
"""One horizontal strip of shelf rail, derived from a product row."""
box: List[int]
row_index: int
product_ids: List[int] = field(default_factory=list)
origin: Optional[Tuple[float, float]] = None
key: Optional[Tuple[str, int]] = None
# The field model lives with the card reader; the pipeline consumes it unchanged.
@ -447,9 +439,6 @@ class TagService:
self.passes_failed = 0
self.last_pass_error: Optional[str] = None
self.passes_skipped = 0
# Latest frame only. A queued pass must not read these; its origin is on the band.
self._view_origin: Optional[Tuple[float, float]] = None
self._view_shape: Optional[Tuple[int, int]] = None
# ------------------------------------------------------------------ frame loop
def plan(
@ -473,8 +462,7 @@ class TagService:
width = float(image_shape[1])
bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width]
with self._lock:
self._view_origin = _origin_of(shelf_offset)
self._view_shape = (int(image_shape[0]), int(image_shape[1]))
self._shelf_offset = shelf_offset
self._frame_seq += 1
seq = self._frame_seq
chosen: List[RailBand] = []
@ -493,8 +481,6 @@ class TagService:
break
for band in chosen:
key = band_key(band, shelf_offset)
band.origin = _origin_of(shelf_offset)
band.key = key
self._last_pass_seq[key] = seq
self._pending_bands.add(key)
return chosen
@ -506,7 +492,7 @@ class TagService:
independent reads of the same tag are what makes the vote meaningful
(identical pixels would produce identical output).
"""
key = _planned_key(band)
key = band_key(band, getattr(self, "_shelf_offset", None))
try:
x1, y1, x2, y2 = band.box
crop = image[y1:y2, x1:x2]
@ -646,19 +632,13 @@ class TagService:
return all(tag.settled for tag in row_tags)
def _absorb(self, candidate: Dict[str, Any], band: RailBand, pass_id: int) -> TagObservation:
"""Fold one clustered tag into the cache; must be called with the lock held.
The candidate box is this frame's pixels. The record keeps shelf pixels,
using the origin `plan` stamped on the band, so a later pan cannot make
this read land on a different rail.
"""
"""Fold one clustered tag into the cache; must be called with the lock held."""
box = candidate["box"]
lines = candidate["lines"]
fields = candidate["fields"]
shelf_key = _planned_key(band)
shelf_box = _to_shelf(box, band.origin)
shelf_key = band_key(band, getattr(self, "_shelf_offset", None))
observation = TagObservation(
box=list(shelf_box),
box=list(box),
lines=list(lines),
fields=fields,
band_index=band.row_index,
@ -666,11 +646,11 @@ class TagService:
observed_at=time.time(),
)
record = self._match_record(shelf_box, shelf_key)
record = self._match_record(box, shelf_key)
if record is None:
record = TagRecord(
tag_id=f"tag_{uuid.uuid4().hex[:8]}",
box=list(shelf_box),
box=list(box),
band_index=band.row_index,
shelf_key=shelf_key,
first_seen=observation.observed_at,
@ -678,7 +658,7 @@ class TagService:
if len(self._tags) >= self.settings.max_tags:
return observation
self._tags[record.tag_id] = record
record.box = list(shelf_box)
record.box = list(box)
record.last_seen = observation.observed_at
record.observations.append(observation)
record.passes += 1
@ -778,47 +758,52 @@ class TagService:
# ------------------------------------------------------------------ attach
def attach(
self,
tracks: Sequence[Dict[str, Any]],
shelf_offset: Optional[Sequence[float]] = None,
image_shape: Optional[Sequence[int]] = None,
self, tracks: Sequence[Dict[str, Any]]
) -> Tuple[Dict[int, Dict[str, Any]], List[Dict[str, Any]], Dict[str, int]]:
"""Give each track its tag for this frame.
Records live in shelf coordinates. They are projected into this frame
before association, and a tag that does not overlap the frame is left
out, so an old tag cannot bind a new facing that reused its screen box.
Returned boxes stay in screen pixels. `shelf_offset` defaults to the
origin of the latest `plan` (a still, or a caller that has not panned).
Returns (tag_by_track_id, all_tags, state_counts). A track whose tag was
never read gets state 'unreadable' when a tag was found near it, and
'absent' when nothing tag-shaped was seen there.
"""
with self._lock:
origin = _origin_of(shelf_offset if shelf_offset is not None else self._view_origin)
shape = image_shape if image_shape is not None else self._view_shape
visible = _visible_records(list(self._tags.values()), origin, shape)
tags = [record.as_dict() for record in self._tags.values()]
records = list(self._tags.values())
by_track: Dict[int, Dict[str, Any]] = {}
used: set = set()
records = [record for record, _screen in visible]
screen_of = {record.tag_id: screen for record, screen in visible}
tags = [_screen_dict(record, screen_of[record.tag_id]) for record in records]
for track in tracks:
box = track.get("box") or track.get("raw_box")
if not box:
continue
record = _tag_below(box, records, used, screen_of)
record = _tag_below(box, records, used)
if record is not None:
used.add(record.tag_id)
payload = _screen_dict(record, screen_of[record.tag_id])
payload = record.as_dict()
payload["associated"] = True
by_track[int(track["track_id"])] = payload
elif self._band_had_readable(record=None, box=box, records=records, screen_of=screen_of):
by_track[int(track["track_id"])] = _empty_tag(TAG_STATE_UNREADABLE)
elif self._band_had_readable(record=None, box=box, records=records):
by_track[int(track["track_id"])] = {
"id": None,
"box": None,
"state": TAG_STATE_UNREADABLE,
"price": None,
"name": None,
"confidence": 0.0,
"observations": 0,
"settled": False,
}
else:
by_track[int(track["track_id"])] = _empty_tag(TAG_STATE_ABSENT)
by_track[int(track["track_id"])] = {
"id": None,
"box": None,
"state": TAG_STATE_ABSENT,
"price": None,
"name": None,
"confidence": 0.0,
"observations": 0,
"settled": False,
}
counts = {
TAG_STATE_READABLE: 0,
@ -830,20 +815,13 @@ class TagService:
counts[payload["state"]] = counts.get(payload["state"], 0) + 1
return by_track, tags, counts
def _band_had_readable(
self,
record,
box: Sequence[int],
records: Sequence[TagRecord],
screen_of: Optional[Dict[str, List[int]]] = None,
) -> bool:
def _band_had_readable(self, record, box: Sequence[int], records: Sequence[TagRecord]) -> bool:
"""Was there any tag-shaped evidence near this facing (readable or not)?"""
x1, y1, x2, y2 = box
width = max(1, x2 - x1)
for candidate in records:
shown = (screen_of or {}).get(candidate.tag_id, candidate.box)
cx = (shown[0] + shown[2]) / 2.0
cy = (shown[1] + shown[3]) / 2.0
cx = (candidate.box[0] + candidate.box[2]) / 2.0
cy = (candidate.box[1] + candidate.box[3]) / 2.0
if x1 - width * 0.3 <= cx <= x2 + width * 0.3 and y1 <= cy <= y2 + (y2 - y1) * 0.5:
return True
return False
@ -880,8 +858,7 @@ class TagService:
self._pending_bands.clear()
self._pass_id = 0
self._frame_seq = 0
self._view_origin = None
self._view_shape = None
self._shelf_offset = None
def _iou(a: Sequence[int], b: Sequence[int]) -> float:
@ -899,17 +876,8 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float:
return inter / union if union > 0 else 0.0
def _tag_below(
box: Sequence[int],
records: Sequence[TagRecord],
used: set,
screen_of: Optional[Dict[str, List[int]]] = None,
) -> Optional[TagRecord]:
"""The tag that belongs to this facing: below it, horizontally overlapping.
`screen_of` is the record projected into the current frame. Without it the
stored shelf box is used, which is the screen box when no origin was known.
"""
def _tag_below(box: Sequence[int], records: Sequence[TagRecord], used: set) -> Optional[TagRecord]:
"""The tag that belongs to this facing: below it, horizontally overlapping."""
x1, y1, x2, y2 = box
width = max(1, x2 - x1)
height = max(1, y2 - y1)
@ -918,7 +886,7 @@ def _tag_below(
for record in records:
if record.tag_id in used:
continue
rx1, ry1, rx2, ry2 = (screen_of or {}).get(record.tag_id, record.box)
rx1, ry1, rx2, ry2 = record.box
center_x = (rx1 + rx2) / 2.0
overlap = max(0, min(x2, rx2) - max(x1, rx1))
overlap_ratio = overlap / max(1, rx2 - rx1)
@ -939,73 +907,6 @@ def _tag_below(
return best
def _origin_of(shelf_offset: Optional[Sequence[float]]) -> Optional[Tuple[float, float]]:
if not shelf_offset or len(shelf_offset) < 2:
return None
return (float(shelf_offset[0]), float(shelf_offset[1]))
def _planned_key(band: RailBand) -> Tuple[str, int]:
"""The rail `plan` chose. A band built without `plan` falls back to its screen row."""
if band.key is not None:
return band.key
return band_key(band, band.origin)
def _shift_box(box: Sequence[int], dx: float, dy: float) -> List[int]:
return [int(round(box[0] + dx)), int(round(box[1] + dy)), int(round(box[2] + dx)), int(round(box[3] + dy))]
def _to_shelf(box: Sequence[int], origin: Optional[Tuple[float, float]]) -> List[int]:
if origin is None:
return [int(v) for v in box]
return _shift_box(box, origin[0], origin[1])
def _to_screen(box: Sequence[int], origin: Optional[Tuple[float, float]]) -> List[int]:
if origin is None:
return [int(v) for v in box]
return _shift_box(box, -origin[0], -origin[1])
def _overlaps_frame(box: Sequence[int], image_shape: Sequence[int]) -> bool:
height, width = int(image_shape[0]), int(image_shape[1])
return box[2] > 0 and box[0] < width and box[3] > 0 and box[1] < height
def _visible_records(
records: Sequence[TagRecord],
origin: Optional[Tuple[float, float]],
image_shape: Optional[Sequence[int]],
) -> List[Tuple[TagRecord, List[int]]]:
shown: List[Tuple[TagRecord, List[int]]] = []
for record in records:
screen = _to_screen(record.box, origin)
if image_shape is not None and not _overlaps_frame(screen, image_shape):
continue
shown.append((record, screen))
return shown
def _screen_dict(record: TagRecord, screen_box: Sequence[int]) -> Dict[str, Any]:
payload = record.as_dict()
payload["box"] = list(screen_box)
return payload
def _empty_tag(state: str) -> Dict[str, Any]:
return {
"id": None,
"box": None,
"state": state,
"price": None,
"name": None,
"confidence": 0.0,
"observations": 0,
"settled": False,
}
if __name__ == "__main__": # smoke: python code/tags.py <image>
import argparse

35
recognization/code/test_api.py
View file

@ -196,31 +196,6 @@ class RestContractTests(unittest.TestCase):
self.assertEqual(sent[0]["status"], 413)
self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"])
def test_overflow_after_response_start_sends_no_second_start(self):
"""A body that overflows after headers began must not start again."""
import anyio
from transport import BodyLimit
async def app(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await receive()
await send({"type": "http.response.body", "body": b"ok"})
sent = []
async def receive():
return {"type": "http.request", "body": b"x" * 40, "more_body": False}
async def send(message):
sent.append(message)
anyio.run(BodyLimit(app, 8), {"type": "http"}, receive, send)
starts = [message for message in sent if message.get("type") == "http.response.start"]
self.assertEqual(len(starts), 1)
self.assertEqual(starts[0]["status"], 200)
self.assertFalse(any(message.get("status") == 413 for message in sent))
def test_classify_and_inspect_roundtrip(self):
recognized = self.client.post(
"/v1/recognize?classify=false", content=jpeg_bytes(),
@ -385,16 +360,6 @@ class WebSocketProtocolTests(unittest.TestCase):
finally:
engine.shutdown()
def test_lifespan_shutdown_skips_a_reserved_slot(self):
"""A reservation is None until setup finishes. Shutdown must not call it."""
client, engine = make_app()
client.app.state.sessions["reserved"] = None
try:
with client:
pass
finally:
engine.shutdown()
class LegacyRouteTests(unittest.TestCase):
def test_legacy_sessions_still_respond(self):

38
recognization/code/test_gallery.py
View file

@ -21,13 +21,7 @@ if os.path.exists(os.path.join(project_root, "stock")):
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from gallery import ( # noqa: E402
DINOV2_SHA256,
GalleryEntry,
ProductGallery,
dinov2_weights_path,
load_dinov2,
)
from gallery import GalleryEntry, ProductGallery # noqa: E402
def a_gallery(views, threshold=0.55):
@ -114,35 +108,5 @@ class EnrollmentTests(unittest.TestCase):
self.assertEqual(len(gallery._views_for("sku_ab")), 1)
class OfflineWeightsTests(unittest.TestCase):
def test_a_missing_weights_file_names_the_path_and_does_not_fetch(self):
previous = os.environ.get("DORAYAKI_DINOV2_WEIGHTS")
os.environ["DORAYAKI_DINOV2_WEIGHTS"] = os.path.join(
"does-not-exist", "dinov2_vits14_lvd142m.safetensors"
)
try:
with self.assertRaises(FileNotFoundError) as raised:
load_dinov2(224)
finally:
if previous is None:
os.environ.pop("DORAYAKI_DINOV2_WEIGHTS", None)
else:
os.environ["DORAYAKI_DINOV2_WEIGHTS"] = previous
message = str(raised.exception)
self.assertIn("does-not-exist", message)
self.assertIn(DINOV2_SHA256, message)
self.assertIn("does not download", message)
def test_the_default_path_stays_inside_the_repo(self):
previous = os.environ.pop("DORAYAKI_DINOV2_WEIGHTS", None)
try:
self.assertTrue(dinov2_weights_path().endswith(
os.path.join("weights", "dinov2_vits14_lvd142m.safetensors")
))
finally:
if previous is not None:
os.environ["DORAYAKI_DINOV2_WEIGHTS"] = previous
if __name__ == "__main__":
unittest.main()

38
recognization/code/test_live.py
View file

@ -385,6 +385,44 @@ class ClassificationBudgetTests(unittest.TestCase):
with self.assertRaises(KeyError):
session.inspect(999)
def test_inspect_includes_settled_tag_from_the_latest_frame(self):
session, engine, _, _ = make_session()
tag = {"state": "readable", "price": 150, "box": [100, 270, 200, 320]}
class Tagged:
def plan(self, boxes, image_shape, shelf_offset=None):
return []
def attach(self, tracks):
by_track = {int(track["track_id"]): dict(tag) for track in tracks}
return by_track, list(by_track.values()), {"readable": len(by_track)}
def stats(self):
return {}
session._tags = Tagged()
result = process(session, 1)
track = result["tracks"][0]
track_id = track["track_id"]
self.assertEqual(track["price_tag"]["price"], 150)
self.assertIn("reconciliation", track)
inspected = session.inspect(track_id)
self.assertEqual(inspected["price_tag"], track["price_tag"])
self.assertEqual(inspected["reconciliation"], track["reconciliation"])
self.assertIn("identity", inspected)
session._tags = None
bare = process(session, 2)
self.assertNotIn("price_tag", bare["tracks"][0])
inspected_bare = session.inspect(track_id)
self.assertNotIn("price_tag", inspected_bare)
self.assertNotIn("reconciliation", inspected_bare)
session.reset()
with self.assertRaises(KeyError):
session.inspect(track_id)
engine.shutdown()
def test_close_stops_the_loop(self):
session, engine, _, _ = make_session()

50
recognization/code/test_ocr_tags.py
View file

@ -335,56 +335,6 @@ class TagServiceTests(unittest.TestCase):
self.assertGreaterEqual(service.stats()["passes_run"], 1)
self.assertIsNone(service.stats()["last_pass_error"])
def test_a_queued_pass_keeps_the_shelf_it_was_planned_on(self):
"""OCR that finishes after a pan must clear its own rail, not the new one."""
service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1)
boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)]
array = image()
planned = service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0))
self.assertEqual(service.stats()["pending_bands"], 1)
moved = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0))
self.assertTrue(moved, "the new shelf is a different rail and must be queued too")
self.assertEqual(service.stats()["pending_bands"], 2)
service.run_pass(array, planned[0])
self.assertEqual(service.stats()["pending_bands"], 1, "the finished pass clears only its own key")
self.assertEqual(service._tags[next(iter(service._tags))].shelf_key[0], "shelf")
self.assertNotEqual(
service._tags[next(iter(service._tags))].shelf_key,
moved[0].key,
"the late read must not be filed under the shelf now in frame",
)
service.run_pass(array, moved[0])
self.assertEqual(service.stats()["pending_bands"], 0)
self.assertEqual(len({tag.shelf_key for tag in service._tags.values()}), 2)
def test_an_offscreen_tag_does_not_bind_the_new_facing(self):
"""A pan that reuses screen coordinates must not hand the old tag to the new facing."""
service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1)
boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)]
array = image()
first = service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0))
service.run_pass(array, first[0])
by_track, tags, _ = service.attach(
[{"track_id": 0, "box": [100, 100, 200, 300]}],
shelf_offset=(0.0, 0.0),
image_shape=array.shape,
)
self.assertEqual(by_track[0]["state"], TAG_STATE_READABLE)
self.assertEqual(by_track[0]["price"], 269)
self.assertGreaterEqual(tags[0]["box"][1], 0)
self.assertLess(tags[0]["box"][1], array.shape[0])
again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0))
self.assertTrue(again)
by_track, tags, _ = service.attach(
[{"track_id": 9, "box": [100, 100, 200, 300]}],
shelf_offset=(0.0, 400.0),
image_shape=array.shape,
)
self.assertEqual(tags, [], "the old tag is above this frame")
self.assertEqual(by_track[9]["state"], TAG_STATE_ABSENT)
self.assertIsNone(by_track[9]["id"])
class StorePolicyTests(unittest.TestCase):
def test_a_missing_file_uses_the_defaults(self):

22
recognization/code/transport.py
View file

@ -104,10 +104,7 @@ class BodyLimit:
A handler that checks Content-Length, or reads request.body(), runs after
the server has already buffered the stream. This counts bytes at receive
and returns 413 without handing the rest to the app. Content-Length is
not trusted: the stream is what counts, and reading stops at the cap.
If the app already started the response, the connection is aborted
instead of sending a second http.response.start. WebSocket frames are
and returns 413 without handing the rest to the app. WebSocket frames are
not HTTP bodies and are left alone.
"""
@ -119,13 +116,10 @@ class BodyLimit:
if scope.get("type") != "http":
await self.app(scope, receive, send)
return
class TooLarge(Exception):
pass
seen = 0
started = False
async def limited() -> Dict[str, Any]:
nonlocal seen
message = await receive()
@ -134,24 +128,12 @@ class BodyLimit:
body = message.get("body") or b""
seen += len(body)
if seen > self.max_bytes:
# The app must not see this chunk or any later one.
raise TooLarge
return message
async def tracked(message: Dict[str, Any]) -> None:
nonlocal started
if message.get("type") == "http.response.start":
started = True
await send(message)
try:
await self.app(scope, limited, tracked)
await self.app(scope, limited, send)
except TooLarge:
if started:
# The status line is already on the wire. Returning leaves the
# response unfinished so the server closes the connection
# instead of accepting a second http.response.start.
return
await send(
{
"type": "http.response.start",