diff --git a/recognization/.gitignore b/recognization/.gitignore index a0bb35a..013e35c 100644 --- a/recognization/.gitignore +++ b/recognization/.gitignore @@ -19,6 +19,7 @@ ENV/ # Temporary weights runs/detect/train/weights/last.pt +weights/dinov2_vits14_lvd142m.safetensors *.log stock-*/identity_head.pt .demo-token diff --git a/recognization/README.md b/recognization/README.md index b64e7d7..bcff2d4 100644 --- a/recognization/README.md +++ b/recognization/README.md @@ -25,6 +25,7 @@ Cần có đầy đủ: - `code/` - mã nguồn server, client web, client mẫu. - `runs/` - trọng số YOLO đã train (`runs/detect/train/weights/best.pt`). +- `weights/dinov2_vits14_lvd142m.safetensors` - DINOv2 backbone for the identity gallery. Not in Git. See "DINOv2 weights" below. - `stock/` - catalog `stock.csv` và ảnh mẫu sản phẩm `sku_001` -> `sku_022`. - `add/` và `image.jpg` - ảnh kệ mẫu để test/demo. - `requirements.txt`, `requirements-dev.txt`. @@ -49,6 +50,20 @@ source .venv/bin/activate # Windows: .venv\Scripts\activate pip install -r requirements-dev.txt ``` +### DINOv2 weights + +The identity gallery embeds crops with DINOv2 ViT-S/14 (LVD-142M), Apache-2.0. It loads only a local file and does not contact Hugging Face, including when `YOLO_OFFLINE` is unset. Without the file, gallery identity fails and the server log says the weights are missing. + +The pinned file is `model.safetensors` from `timm/vit_small_patch14_dinov2.lvd142m`, sha256 `04d27f3400d059fc0cfd7d17dd1909a75bf3ea8fb3eeb48b97cb99e57ee20081` (~84 MB). Copy a cache you already have; do not fetch one from an unknown URL: + +```bash +mkdir -p weights +cp "$HOME/.cache/huggingface/hub/models--timm--vit_small_patch14_dinov2.lvd142m/snapshots/"*/model.safetensors \ + weights/dinov2_vits14_lvd142m.safetensors +``` + +`DORAYAKI_DINOV2_WEIGHTS` overrides that path. A file with any other sha256 is refused. + (`requirements-dev.txt` = `requirements.txt` + `httpx` cho test.) ### Bước 4: Khởi chạy recognition server diff --git a/recognization/code/app.py b/recognization/code/app.py index 5c16dbd..bcbd1ff 100644 --- a/recognization/code/app.py +++ b/recognization/code/app.py @@ -270,7 +270,8 @@ def create_app( async def lifespan(_app: FastAPI): yield for session in list(sessions.values()): - session.close() + if session is not None: + session.close() engine.shutdown() app = FastAPI( @@ -1111,6 +1112,11 @@ def main() -> None: LOGGER.info("Loading model and catalog (device=%s, imgsz=%s)...", engine_settings.device or "auto", engine_settings.imgsz) engine = build_default_engine(engine_settings) + if (engine_settings.identity_backend or "").lower() == "gallery": + if engine.identity_gallery() is None: + raise RuntimeError( + "gallery identity backend requires local DINOv2 weights; refusing to start" + ) warm_ms = engine.warm_up() LOGGER.info( "Ready: %d SKUs, catalog %s, warmup %.0f ms, classify workers %d", diff --git a/recognization/code/build_store_from_shelf.py b/recognization/code/build_store_from_shelf.py index 589328d..f65ac02 100644 --- a/recognization/code/build_store_from_shelf.py +++ b/recognization/code/build_store_from_shelf.py @@ -1,66 +1,253 @@ """ -Enroll a shelf: the products read off its crops become the store's catalog. +Crop a labelled shelf photo into a stock directory. -This is the onboarding flow a real store would go through once - detect the facings, -name the products, and keep their crops as the gallery that recognition matches -against. The two demo shelves were built with it; the product lists live in this file -because reading a contact sheet is a human step. +This is a one-shot offline helper. SKU names and prices are supplied by a human +who read a contact sheet; this script does not enroll a phone recording and +does not talk to the recognition server. - python code/build_store_from_shelf.py +The Meidiya dairy labels that built stock-mediya are kept as --example mediya. +Point --image, --boxes, and --output at real paths. There is no default home +directory or /tmp path. + + python code/build_store_from_shelf.py \\ + --image shelf.jpg --boxes boxes.json --output /path/to/new-stock \\ + --labels labels.json + +boxes.json is {"boxes": [[x1, y1, x2, y2], ...]} in the image's own pixels; +this script does not rescale the photo. labels.json is +{"products": [{"sku", "name", "price", "boxes": [index, ...]}]}. +An existing output directory is left untouched unless --replace is passed, +and even then the old directory is renamed to ".previous", not deleted. """ -import json, os, shutil, sys -sys.path.insert(0, "code") -import cv2 +from __future__ import annotations -ROOT = os.path.expanduser("~/dorayaki-data/shelf-images") -manifest = json.load(open(os.path.join(ROOT, "manifest.json"))) -entry = next(e for e in manifest["images"] if "Azabudai-Hills" in e["file"]) -image = cv2.imread(os.path.join(ROOT, entry["file"])) -height, width = image.shape[:2] -scale = min(1.0, 1600.0 / max(height, width)) -image = cv2.resize(image, (int(width*scale), int(height*scale)), interpolation=cv2.INTER_AREA) -boxes = json.load(open("/tmp/dorayaki-verify/mediya-boxes.json"))["boxes"] +import argparse +import csv +import json +import os +import shutil +import sys +import tempfile -# read off the numbered contact sheet; prices come from the shelf's own tags -PRODUCTS = [ - ("sku_m01", "小岩井 低脂肪牛乳", [9, 23, 31]), - ("sku_m02", "荻山酪農 カフェ・オ・レ", [10]), - ("sku_m03", "丹那 3.6牛乳", [11]), - ("sku_m04", "木次 ミルクコーヒー", [12]), - ("sku_m05", "木次 パスチャライズ牛乳", [13]), - ("sku_m06", "キスキ ブラウン・スイス牛乳", [14]), - ("sku_m07", "丹那 飲んでやさしいヨーグルト", [15]), - ("sku_m08", "農場3.7牛乳 特選", [24]), - ("sku_m09", "MINOR FIGURES OAT", [26]), - ("sku_m10", "北海道 低脂肪乳", [27]), - ("sku_m11", "タカナシ 低温殺菌牛乳", [28]), - ("sku_m12", "RUDE HEALTH Oat Organic", [29]), - ("sku_m13", "小岩井 無脂肪牛乳", [30]), - ("sku_m14", "きすき ノンホモ牛乳", [8]), +# ponytail: demo counts/dates are the Meidiya sheet's bookkeeping, not a catalog API. +EXAMPLE_RECEIPT = "2026-09-25" +EXAMPLE_EXPIRATION = "2026-10-20" +EXAMPLE_COUNTS = (24, 8, 16) + +# Named example only. These are the labels read off the Meidiya contact sheet, +# not an enrollment of a phone walk. +MEDIYA_PRODUCTS = [ + {"sku": "sku_m01", "name": "小岩井 低脂肪牛乳", "price": 186, "boxes": [9, 23, 31]}, + {"sku": "sku_m02", "name": "荻山酪農 カフェ・オ・レ", "price": 397, "boxes": [10]}, + {"sku": "sku_m03", "name": "丹那 3.6牛乳", "price": 214, "boxes": [11]}, + {"sku": "sku_m04", "name": "木次 ミルクコーヒー", "price": 195, "boxes": [12]}, + {"sku": "sku_m05", "name": "木次 パスチャライズ牛乳", "price": 240, "boxes": [13]}, + {"sku": "sku_m06", "name": "キスキ ブラウン・スイス牛乳", "price": 254, "boxes": [14]}, + {"sku": "sku_m07", "name": "丹那 飲んでやさしいヨーグルト", "price": 225, "boxes": [15]}, + {"sku": "sku_m08", "name": "農場3.7牛乳 特選", "price": 216, "boxes": [24]}, + {"sku": "sku_m09", "name": "MINOR FIGURES OAT", "price": 300, "boxes": [26]}, + {"sku": "sku_m10", "name": "北海道 低脂肪乳", "price": 214, "boxes": [27]}, + {"sku": "sku_m11", "name": "タカナシ 低温殺菌牛乳", "price": 396, "boxes": [28]}, + {"sku": "sku_m12", "name": "RUDE HEALTH Oat Organic", "price": 400, "boxes": [29]}, + {"sku": "sku_m13", "name": "小岩井 無脂肪牛乳", "price": 178, "boxes": [30]}, + {"sku": "sku_m14", "name": "きすき ノンホモ牛乳", "price": 198, "boxes": [8]}, ] -PRICES = {"sku_m01": 186, "sku_m02": 397, "sku_m03": 214, "sku_m04": 195, "sku_m05": 240, - "sku_m06": 254, "sku_m07": 225, "sku_m08": 216, "sku_m09": 300, "sku_m10": 214, - "sku_m11": 396, "sku_m12": 400, "sku_m13": 178, "sku_m14": 198} -store = "stock-mediya" -shutil.rmtree(store, ignore_errors=True) -os.makedirs(store, exist_ok=True) -rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"] -written = [] -for sku, name, indices in PRODUCTS: - saved = 0 - for position, index in enumerate(indices): - box = boxes[index] - x1, y1, x2, y2 = [int(v) for v in box] - crop = image[max(0, y1):y2, max(0, x1):x2] - if crop.size == 0: - continue - suffix = "" if saved == 0 else f"_{saved}" - cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop) - saved += 1 - if saved == 0: - raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view") - written.append(sku) - rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}") -open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n") -print(f"{store}: {len(written)} products, csv written") + +class BuildStoreError(Exception): + """Refused input. The destination, if it already existed, is unchanged.""" + + +def _is_within(path: str, root: str) -> bool: + path = os.path.realpath(path) + root = os.path.realpath(root) + return path == root or path.startswith(root + os.sep) + + +def _load_json(path: str): + try: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + raise BuildStoreError(f"{path}: {exc}") from exc + + +def load_boxes(path: str) -> list: + payload = _load_json(path) + boxes = payload.get("boxes") if isinstance(payload, dict) else None + if not isinstance(boxes, list) or not boxes: + raise BuildStoreError(f"{path}: expected a non-empty {{\"boxes\": [...]}} list") + parsed = [] + for index, box in enumerate(boxes): + if not isinstance(box, (list, tuple)) or len(box) != 4: + raise BuildStoreError(f"{path}: box {index} is not [x1, y1, x2, y2]") + try: + coords = [int(value) for value in box] + except (TypeError, ValueError) as exc: + raise BuildStoreError(f"{path}: box {index} is not four numbers") from exc + x1, y1, x2, y2 = coords + if x2 <= x1 or y2 <= y1: + raise BuildStoreError(f"{path}: box {index} has no area") + parsed.append(coords) + return parsed + + +def load_labels(path: str | None, example: str | None) -> list: + if example and path: + raise BuildStoreError("pass either --labels or --example, not both") + if example == "mediya": + return [dict(item) for item in MEDIYA_PRODUCTS] + if example: + raise BuildStoreError(f"unknown example {example!r}; the only named example is mediya") + if not path: + raise BuildStoreError("pass --labels, or --example mediya for the demo sheet") + payload = _load_json(path) + products = payload.get("products") if isinstance(payload, dict) else None + if not isinstance(products, list) or not products: + raise BuildStoreError(f"{path}: expected a non-empty {{\"products\": [...]}} list") + return products + + +def _sku_name(value) -> str: + text = str(value).strip() + if not text or text in {".", ".."} or "/" in text or "\\" in text or "\x00" in text: + raise BuildStoreError(f"refusing sku {value!r}: it is not a single file name") + return text + + +def _products(labels: list) -> list: + seen = set() + products = [] + for item in labels: + if not isinstance(item, dict): + raise BuildStoreError("each product must be an object with sku, name, price, boxes") + sku = _sku_name(item.get("sku", "")) + if sku in seen: + raise BuildStoreError(f"duplicate sku {sku}") + seen.add(sku) + name = str(item.get("name", "")).strip() + if not name or any(char in name for char in ",\n\r"): + raise BuildStoreError(f"{sku}: name must be one CSV field") + try: + price = int(item["price"]) + indices = [int(index) for index in item["boxes"]] + except (KeyError, TypeError, ValueError) as exc: + raise BuildStoreError(f"{sku}: price and boxes must be integers") from exc + if price < 0 or not indices: + raise BuildStoreError(f"{sku}: price and boxes must be present") + products.append({"sku": sku, "name": name, "price": price, "boxes": indices}) + return products + + +def check_destination(output: str, image: str, boxes_path: str, labels_path: str | None, replace: bool) -> str: + output = os.path.abspath(output) + real_output = os.path.realpath(output) + if real_output == os.path.realpath(os.sep) or real_output == os.path.realpath(os.getcwd()): + raise BuildStoreError("refusing to use the filesystem root or the current directory as the store") + parent = os.path.dirname(output) + if not parent or not os.path.isdir(parent): + raise BuildStoreError(f"{output}: parent directory does not exist") + for source in (image, boxes_path, labels_path): + if source and (_is_within(source, real_output) or _is_within(real_output, os.path.realpath(source))): + raise BuildStoreError(f"refusing to write {output}: it aliases input {source}") + if os.path.lexists(output): + if not replace: + raise BuildStoreError(f"{output}: already exists; pass --replace to move it aside") + if os.path.islink(output) or not os.path.isdir(output): + raise BuildStoreError(f"{output}: --replace only moves aside a real directory") + previous = output + ".previous" + if os.path.lexists(previous): + raise BuildStoreError(f"{previous}: already exists; move it away before --replace") + return output + + +def build_store(image_path: str, boxes: list, products: list, output: str, replace: bool, + boxes_path: str, labels_path: str | None) -> int: + output = check_destination(output, image_path, boxes_path, labels_path, replace) + import cv2 + + image = cv2.imread(image_path) + if image is None or getattr(image, "size", 0) == 0: + raise BuildStoreError(f"{image_path}: not a readable image") + height, width = image.shape[:2] + + parent = os.path.dirname(output) + staging = tempfile.mkdtemp(prefix=".build-store-", dir=parent) + try: + rows = [[ + "SKU_ID", "product_name", "receipt_date", "expiration_date", + "number_in", "number_out", "number_on", "image_link", "price", + ]] + written = [] + for product in products: + saved = 0 + for index in product["boxes"]: + if index < 0 or index >= len(boxes): + raise BuildStoreError(f"{product['sku']}: box {index} is outside the boxes file") + x1, y1, x2, y2 = boxes[index] + x1, y1 = max(0, x1), max(0, y1) + x2, y2 = min(width, x2), min(height, y2) + crop = image[y1:y2, x1:x2] if x2 > x1 and y2 > y1 else image[0:0, 0:0] + if crop.size == 0: + raise BuildStoreError(f"{product['sku']}: box {index} does not crop the image") + suffix = "" if saved == 0 else f"_{saved}" + target = os.path.join(staging, f"{product['sku']}{suffix}.jpeg") + if not cv2.imwrite(target, crop): + raise BuildStoreError(f"{product['sku']}: failed to write crop {index}") + saved += 1 + written.append(product["sku"]) + number_in, number_out, number_on = EXAMPLE_COUNTS + rows.append([ + product["sku"], product["name"], EXAMPLE_RECEIPT, EXAMPLE_EXPIRATION, + number_in, number_out, number_on, "", product["price"], + ]) + with open(os.path.join(staging, "stock.csv"), "w", encoding="utf-8", newline="") as handle: + csv.writer(handle).writerows(rows) + if not os.path.lexists(output): + os.rename(staging, output) + staging = None + else: + previous = output + ".previous" + os.rename(output, previous) + try: + os.rename(staging, output) + except OSError: + os.rename(previous, output) + raise + staging = None + print(f"previous store kept at {previous}", file=sys.stderr) + return len(written) + finally: + if staging and os.path.isdir(staging): + shutil.rmtree(staging) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--image", required=True, help="shelf photo whose pixels match the box coordinates") + parser.add_argument("--boxes", required=True, help='JSON file {"boxes": [[x1, y1, x2, y2], ...]}') + parser.add_argument("--output", required=True, help="new stock directory to create") + parser.add_argument("--labels", help='JSON file {"products": [{"sku", "name", "price", "boxes"}]}') + parser.add_argument("--example", choices=["mediya"], help="use the named demo contact-sheet labels") + parser.add_argument("--replace", action="store_true", help="replace an existing output directory") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + products = _products(load_labels(args.labels, args.example)) + count = build_store( + args.image, load_boxes(args.boxes), products, args.output, args.replace, + args.boxes, args.labels, + ) + except BuildStoreError as exc: + print(f"build_store_from_shelf: {exc}", file=sys.stderr) + return 1 + print(f"{os.path.abspath(args.output)}: {count} products, csv written") + print("phone recordings are not enrolled by this script; labels stay a manual contact sheet") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recognization/code/gallery.py b/recognization/code/gallery.py index 0805d71..5d5f5dd 100644 --- a/recognization/code/gallery.py +++ b/recognization/code/gallery.py @@ -38,6 +38,78 @@ DEFAULT_THRESHOLD = 0.55 # The same line for a fine-tuned head, whose similarity scale is its own. DEFAULT_HEAD_THRESHOLD = 0.40 +# DINOv2 ViT-S/14, LVD-142M. timm's default source is the Hugging Face copy of +# these weights (Apache-2.0); pretrained=True fetches it even when YOLO_OFFLINE=1. +# The pin is the sha256 of that model.safetensors (timm hub snapshot 4610ca14). +# Not in Git (~84 MB). Place it at weights/dinov2_vits14_lvd142m.safetensors, or +# point DORAYAKI_DINOV2_WEIGHTS at a copy. See README "DINOv2 weights". +DINOV2_MODEL = "vit_small_patch14_dinov2.lvd142m" +DINOV2_SHA256 = "04d27f3400d059fc0cfd7d17dd1909a75bf3ea8fb3eeb48b97cb99e57ee20081" +DINOV2_FILENAME = "dinov2_vits14_lvd142m.safetensors" + + +def dinov2_weights_path() -> str: + override = os.environ.get("DORAYAKI_DINOV2_WEIGHTS") + if override: + return override + here = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(os.path.dirname(here), "weights", DINOV2_FILENAME) + + +def _sha256(path: str) -> str: + import hashlib + + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_dinov2(image_size: int) -> Any: + """Build the gallery backbone from the pinned local file. Never downloads. + + A missing or altered file raises FileNotFoundError naming the path and the + expected sha256. The head in finetune_identity.py is a Linear(384, 128) on + this model's features; a different backbone will not load that head. + """ + import timm + from safetensors.torch import load_file + from timm.models.vision_transformer import checkpoint_filter_fn + + path = dinov2_weights_path() + if not os.path.isfile(path): + raise FileNotFoundError( + f"DINOv2 weights not found at {path}. " + f"Copy model.safetensors (sha256 {DINOV2_SHA256}) from the local " + f"Hugging Face cache of timm/{DINOV2_MODEL}, or set DORAYAKI_DINOV2_WEIGHTS. " + f"The gallery does not download weights." + ) + digest = _sha256(path) + if digest != DINOV2_SHA256: + raise FileNotFoundError( + f"DINOv2 weights at {path} are sha256 {digest}, expected {DINOV2_SHA256}." + ) + model = timm.create_model( + DINOV2_MODEL, + pretrained=False, + num_classes=0, + img_size=int(image_size), + dynamic_img_size=True, + ) + # The file is the 518px checkpoint. timm's own filter resizes pos_embed when + # the caller asks for another size; pretrained=True would do this and also + # download, so it stays off. + state = checkpoint_filter_fn(load_file(path), model) + missing, unexpected = model.load_state_dict(state, strict=False) + unexpected = [key for key in unexpected if not key.startswith("head.")] + if missing or unexpected: + raise RuntimeError( + f"DINOv2 weights at {path} do not match {DINOV2_MODEL} " + f"(missing={missing}, unexpected={unexpected})." + ) + return model + @dataclass class GalleryEntry: @@ -138,13 +210,7 @@ class ProductGallery: device = self.device if device is None: device = "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu" - model = timm.create_model( - "vit_small_patch14_dinov2.lvd142m", - pretrained=True, - num_classes=0, - img_size=self.image_size, - dynamic_img_size=True, - ) + model = load_dinov2(self.image_size) model = model.eval().to(device) self.device = device self._model = model diff --git a/recognization/code/live.py b/recognization/code/live.py index 8e1308b..385f379 100644 --- a/recognization/code/live.py +++ b/recognization/code/live.py @@ -191,10 +191,6 @@ class LiveSession: self._shift_map = None self._compliance_tracker = None self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []} - # Settled tag + reconciliation from the last result, keyed by track. - # Inspect copies these; it does not re-run OCR. - self._settled_tags: Dict[int, Dict[str, Any]] = {} - self._settled_tag_seq = 0 self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque( maxlen=LABEL_MEMORY_MAX ) @@ -297,8 +293,6 @@ class LiveSession: self._latest_frame_id = None self._frame_origin = [0.0, 0.0] self._shift_map = None - self._settled_tags = {} - self._settled_tag_seq = 0 # ------------------------------------------------------------------ frame path def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]: @@ -445,17 +439,6 @@ class LiveSession: if tag_summary is not None: summary["price_tags"] = tag_summary["tags"] summary["compliance"] = tag_summary["compliance"] - settled: Dict[int, Dict[str, Any]] = {} - for payload in track_payload: - snapshot: Dict[str, Any] = {} - if "price_tag" in payload: - snapshot["price_tag"] = payload["price_tag"] - if "reconciliation" in payload: - snapshot["reconciliation"] = payload["reconciliation"] - if snapshot: - settled[int(payload["track_id"])] = snapshot - self._settled_tags = settled - self._settled_tag_seq = seq self._submit_classification(submissions, method, generation) self._run_tag_passes(image, tag_bands) @@ -929,16 +912,7 @@ class LiveSession: if track is None or image is None: raise KeyError("TRACK_NOT_IN_VIEW") box = list(track.raw_box) - # Snapshot is the frame current at tap time. engine.inspect runs - # outside the lock, so a newer frame may land first; the reply still - # carries the tag from the frame the tap matched, not that later one. - settled = None - if self._settled_tag_seq == self._latest_seq: - settled = self._settled_tags.get(track_id) - result = self.engine.inspect(image, box, method=chosen_method) - if settled: - result.update(settled) - return result + return self.engine.inspect(image, box, method=chosen_method) # ------------------------------------------------------------------ catalog cache def refresh_catalog(self) -> None: diff --git a/recognization/code/tags.py b/recognization/code/tags.py index b4ad79d..2a7d8a0 100644 --- a/recognization/code/tags.py +++ b/recognization/code/tags.py @@ -84,11 +84,19 @@ _TAX_RE = re.compile(r"税(込|抜|不|別)") @dataclass class RailBand: - """One horizontal strip of shelf rail, derived from a product row.""" + """One horizontal strip of shelf rail, derived from a product row. + + `box` is this frame's pixels (the crop `run_pass` reads). `origin` is the + shelf position of the frame origin at the moment `plan` chose the band, and + `key` is the rail that origin names. Both travel with the band: a later + frame's origin must not relabel a pass that is still queued. + """ box: List[int] row_index: int product_ids: List[int] = field(default_factory=list) + origin: Optional[Tuple[float, float]] = None + key: Optional[Tuple[str, int]] = None # The field model lives with the card reader; the pipeline consumes it unchanged. @@ -439,6 +447,9 @@ class TagService: self.passes_failed = 0 self.last_pass_error: Optional[str] = None self.passes_skipped = 0 + # Latest frame only. A queued pass must not read these; its origin is on the band. + self._view_origin: Optional[Tuple[float, float]] = None + self._view_shape: Optional[Tuple[int, int]] = None # ------------------------------------------------------------------ frame loop def plan( @@ -462,7 +473,8 @@ class TagService: width = float(image_shape[1]) bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width] with self._lock: - self._shelf_offset = shelf_offset + self._view_origin = _origin_of(shelf_offset) + self._view_shape = (int(image_shape[0]), int(image_shape[1])) self._frame_seq += 1 seq = self._frame_seq chosen: List[RailBand] = [] @@ -481,6 +493,8 @@ class TagService: break for band in chosen: key = band_key(band, shelf_offset) + band.origin = _origin_of(shelf_offset) + band.key = key self._last_pass_seq[key] = seq self._pending_bands.add(key) return chosen @@ -492,7 +506,7 @@ class TagService: independent reads of the same tag are what makes the vote meaningful (identical pixels would produce identical output). """ - key = band_key(band, getattr(self, "_shelf_offset", None)) + key = _planned_key(band) try: x1, y1, x2, y2 = band.box crop = image[y1:y2, x1:x2] @@ -632,13 +646,19 @@ class TagService: return all(tag.settled for tag in row_tags) def _absorb(self, candidate: Dict[str, Any], band: RailBand, pass_id: int) -> TagObservation: - """Fold one clustered tag into the cache; must be called with the lock held.""" + """Fold one clustered tag into the cache; must be called with the lock held. + + The candidate box is this frame's pixels. The record keeps shelf pixels, + using the origin `plan` stamped on the band, so a later pan cannot make + this read land on a different rail. + """ box = candidate["box"] lines = candidate["lines"] fields = candidate["fields"] - shelf_key = band_key(band, getattr(self, "_shelf_offset", None)) + shelf_key = _planned_key(band) + shelf_box = _to_shelf(box, band.origin) observation = TagObservation( - box=list(box), + box=list(shelf_box), lines=list(lines), fields=fields, band_index=band.row_index, @@ -646,11 +666,11 @@ class TagService: observed_at=time.time(), ) - record = self._match_record(box, shelf_key) + record = self._match_record(shelf_box, shelf_key) if record is None: record = TagRecord( tag_id=f"tag_{uuid.uuid4().hex[:8]}", - box=list(box), + box=list(shelf_box), band_index=band.row_index, shelf_key=shelf_key, first_seen=observation.observed_at, @@ -658,7 +678,7 @@ class TagService: if len(self._tags) >= self.settings.max_tags: return observation self._tags[record.tag_id] = record - record.box = list(box) + record.box = list(shelf_box) record.last_seen = observation.observed_at record.observations.append(observation) record.passes += 1 @@ -758,52 +778,47 @@ class TagService: # ------------------------------------------------------------------ attach def attach( - self, tracks: Sequence[Dict[str, Any]] + self, + tracks: Sequence[Dict[str, Any]], + shelf_offset: Optional[Sequence[float]] = None, + image_shape: Optional[Sequence[int]] = None, ) -> Tuple[Dict[int, Dict[str, Any]], List[Dict[str, Any]], Dict[str, int]]: """Give each track its tag for this frame. + Records live in shelf coordinates. They are projected into this frame + before association, and a tag that does not overlap the frame is left + out, so an old tag cannot bind a new facing that reused its screen box. + Returned boxes stay in screen pixels. `shelf_offset` defaults to the + origin of the latest `plan` (a still, or a caller that has not panned). + Returns (tag_by_track_id, all_tags, state_counts). A track whose tag was never read gets state 'unreadable' when a tag was found near it, and 'absent' when nothing tag-shaped was seen there. """ with self._lock: - tags = [record.as_dict() for record in self._tags.values()] - records = list(self._tags.values()) + origin = _origin_of(shelf_offset if shelf_offset is not None else self._view_origin) + shape = image_shape if image_shape is not None else self._view_shape + visible = _visible_records(list(self._tags.values()), origin, shape) by_track: Dict[int, Dict[str, Any]] = {} used: set = set() + records = [record for record, _screen in visible] + screen_of = {record.tag_id: screen for record, screen in visible} + tags = [_screen_dict(record, screen_of[record.tag_id]) for record in records] for track in tracks: box = track.get("box") or track.get("raw_box") if not box: continue - record = _tag_below(box, records, used) + record = _tag_below(box, records, used, screen_of) if record is not None: used.add(record.tag_id) - payload = record.as_dict() + payload = _screen_dict(record, screen_of[record.tag_id]) payload["associated"] = True by_track[int(track["track_id"])] = payload - elif self._band_had_readable(record=None, box=box, records=records): - by_track[int(track["track_id"])] = { - "id": None, - "box": None, - "state": TAG_STATE_UNREADABLE, - "price": None, - "name": None, - "confidence": 0.0, - "observations": 0, - "settled": False, - } + elif self._band_had_readable(record=None, box=box, records=records, screen_of=screen_of): + by_track[int(track["track_id"])] = _empty_tag(TAG_STATE_UNREADABLE) else: - by_track[int(track["track_id"])] = { - "id": None, - "box": None, - "state": TAG_STATE_ABSENT, - "price": None, - "name": None, - "confidence": 0.0, - "observations": 0, - "settled": False, - } + by_track[int(track["track_id"])] = _empty_tag(TAG_STATE_ABSENT) counts = { TAG_STATE_READABLE: 0, @@ -815,13 +830,20 @@ class TagService: counts[payload["state"]] = counts.get(payload["state"], 0) + 1 return by_track, tags, counts - def _band_had_readable(self, record, box: Sequence[int], records: Sequence[TagRecord]) -> bool: + def _band_had_readable( + self, + record, + box: Sequence[int], + records: Sequence[TagRecord], + screen_of: Optional[Dict[str, List[int]]] = None, + ) -> bool: """Was there any tag-shaped evidence near this facing (readable or not)?""" x1, y1, x2, y2 = box width = max(1, x2 - x1) for candidate in records: - cx = (candidate.box[0] + candidate.box[2]) / 2.0 - cy = (candidate.box[1] + candidate.box[3]) / 2.0 + shown = (screen_of or {}).get(candidate.tag_id, candidate.box) + cx = (shown[0] + shown[2]) / 2.0 + cy = (shown[1] + shown[3]) / 2.0 if x1 - width * 0.3 <= cx <= x2 + width * 0.3 and y1 <= cy <= y2 + (y2 - y1) * 0.5: return True return False @@ -858,7 +880,8 @@ class TagService: self._pending_bands.clear() self._pass_id = 0 self._frame_seq = 0 - self._shelf_offset = None + self._view_origin = None + self._view_shape = None def _iou(a: Sequence[int], b: Sequence[int]) -> float: @@ -876,8 +899,17 @@ def _iou(a: Sequence[int], b: Sequence[int]) -> float: return inter / union if union > 0 else 0.0 -def _tag_below(box: Sequence[int], records: Sequence[TagRecord], used: set) -> Optional[TagRecord]: - """The tag that belongs to this facing: below it, horizontally overlapping.""" +def _tag_below( + box: Sequence[int], + records: Sequence[TagRecord], + used: set, + screen_of: Optional[Dict[str, List[int]]] = None, +) -> Optional[TagRecord]: + """The tag that belongs to this facing: below it, horizontally overlapping. + + `screen_of` is the record projected into the current frame. Without it the + stored shelf box is used, which is the screen box when no origin was known. + """ x1, y1, x2, y2 = box width = max(1, x2 - x1) height = max(1, y2 - y1) @@ -886,7 +918,7 @@ def _tag_below(box: Sequence[int], records: Sequence[TagRecord], used: set) -> O for record in records: if record.tag_id in used: continue - rx1, ry1, rx2, ry2 = record.box + rx1, ry1, rx2, ry2 = (screen_of or {}).get(record.tag_id, record.box) center_x = (rx1 + rx2) / 2.0 overlap = max(0, min(x2, rx2) - max(x1, rx1)) overlap_ratio = overlap / max(1, rx2 - rx1) @@ -907,6 +939,73 @@ def _tag_below(box: Sequence[int], records: Sequence[TagRecord], used: set) -> O return best +def _origin_of(shelf_offset: Optional[Sequence[float]]) -> Optional[Tuple[float, float]]: + if not shelf_offset or len(shelf_offset) < 2: + return None + return (float(shelf_offset[0]), float(shelf_offset[1])) + + +def _planned_key(band: RailBand) -> Tuple[str, int]: + """The rail `plan` chose. A band built without `plan` falls back to its screen row.""" + if band.key is not None: + return band.key + return band_key(band, band.origin) + + +def _shift_box(box: Sequence[int], dx: float, dy: float) -> List[int]: + return [int(round(box[0] + dx)), int(round(box[1] + dy)), int(round(box[2] + dx)), int(round(box[3] + dy))] + + +def _to_shelf(box: Sequence[int], origin: Optional[Tuple[float, float]]) -> List[int]: + if origin is None: + return [int(v) for v in box] + return _shift_box(box, origin[0], origin[1]) + + +def _to_screen(box: Sequence[int], origin: Optional[Tuple[float, float]]) -> List[int]: + if origin is None: + return [int(v) for v in box] + return _shift_box(box, -origin[0], -origin[1]) + + +def _overlaps_frame(box: Sequence[int], image_shape: Sequence[int]) -> bool: + height, width = int(image_shape[0]), int(image_shape[1]) + return box[2] > 0 and box[0] < width and box[3] > 0 and box[1] < height + + +def _visible_records( + records: Sequence[TagRecord], + origin: Optional[Tuple[float, float]], + image_shape: Optional[Sequence[int]], +) -> List[Tuple[TagRecord, List[int]]]: + shown: List[Tuple[TagRecord, List[int]]] = [] + for record in records: + screen = _to_screen(record.box, origin) + if image_shape is not None and not _overlaps_frame(screen, image_shape): + continue + shown.append((record, screen)) + return shown + + +def _screen_dict(record: TagRecord, screen_box: Sequence[int]) -> Dict[str, Any]: + payload = record.as_dict() + payload["box"] = list(screen_box) + return payload + + +def _empty_tag(state: str) -> Dict[str, Any]: + return { + "id": None, + "box": None, + "state": state, + "price": None, + "name": None, + "confidence": 0.0, + "observations": 0, + "settled": False, + } + + if __name__ == "__main__": # smoke: python code/tags.py import argparse diff --git a/recognization/code/test_api.py b/recognization/code/test_api.py index bc21027..f7cf60a 100644 --- a/recognization/code/test_api.py +++ b/recognization/code/test_api.py @@ -196,6 +196,31 @@ class RestContractTests(unittest.TestCase): self.assertEqual(sent[0]["status"], 413) self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"]) + def test_overflow_after_response_start_sends_no_second_start(self): + """A body that overflows after headers began must not start again.""" + import anyio + + from transport import BodyLimit + + async def app(scope, receive, send): + await send({"type": "http.response.start", "status": 200, "headers": []}) + await receive() + await send({"type": "http.response.body", "body": b"ok"}) + + sent = [] + + async def receive(): + return {"type": "http.request", "body": b"x" * 40, "more_body": False} + + async def send(message): + sent.append(message) + + anyio.run(BodyLimit(app, 8), {"type": "http"}, receive, send) + starts = [message for message in sent if message.get("type") == "http.response.start"] + self.assertEqual(len(starts), 1) + self.assertEqual(starts[0]["status"], 200) + self.assertFalse(any(message.get("status") == 413 for message in sent)) + def test_classify_and_inspect_roundtrip(self): recognized = self.client.post( "/v1/recognize?classify=false", content=jpeg_bytes(), @@ -360,6 +385,16 @@ class WebSocketProtocolTests(unittest.TestCase): finally: engine.shutdown() + def test_lifespan_shutdown_skips_a_reserved_slot(self): + """A reservation is None until setup finishes. Shutdown must not call it.""" + client, engine = make_app() + client.app.state.sessions["reserved"] = None + try: + with client: + pass + finally: + engine.shutdown() + class LegacyRouteTests(unittest.TestCase): def test_legacy_sessions_still_respond(self): diff --git a/recognization/code/test_gallery.py b/recognization/code/test_gallery.py index afc6585..815365e 100644 --- a/recognization/code/test_gallery.py +++ b/recognization/code/test_gallery.py @@ -21,7 +21,13 @@ if os.path.exists(os.path.join(project_root, "stock")): if current_dir not in sys.path: sys.path.insert(0, current_dir) -from gallery import GalleryEntry, ProductGallery # noqa: E402 +from gallery import ( # noqa: E402 + DINOV2_SHA256, + GalleryEntry, + ProductGallery, + dinov2_weights_path, + load_dinov2, +) def a_gallery(views, threshold=0.55): @@ -108,5 +114,35 @@ class EnrollmentTests(unittest.TestCase): self.assertEqual(len(gallery._views_for("sku_ab")), 1) + +class OfflineWeightsTests(unittest.TestCase): + def test_a_missing_weights_file_names_the_path_and_does_not_fetch(self): + previous = os.environ.get("DORAYAKI_DINOV2_WEIGHTS") + os.environ["DORAYAKI_DINOV2_WEIGHTS"] = os.path.join( + "does-not-exist", "dinov2_vits14_lvd142m.safetensors" + ) + try: + with self.assertRaises(FileNotFoundError) as raised: + load_dinov2(224) + finally: + if previous is None: + os.environ.pop("DORAYAKI_DINOV2_WEIGHTS", None) + else: + os.environ["DORAYAKI_DINOV2_WEIGHTS"] = previous + message = str(raised.exception) + self.assertIn("does-not-exist", message) + self.assertIn(DINOV2_SHA256, message) + self.assertIn("does not download", message) + + def test_the_default_path_stays_inside_the_repo(self): + previous = os.environ.pop("DORAYAKI_DINOV2_WEIGHTS", None) + try: + self.assertTrue(dinov2_weights_path().endswith( + os.path.join("weights", "dinov2_vits14_lvd142m.safetensors") + )) + finally: + if previous is not None: + os.environ["DORAYAKI_DINOV2_WEIGHTS"] = previous + if __name__ == "__main__": unittest.main() diff --git a/recognization/code/test_live.py b/recognization/code/test_live.py index a5a4eab..1e9ed14 100644 --- a/recognization/code/test_live.py +++ b/recognization/code/test_live.py @@ -385,44 +385,6 @@ class ClassificationBudgetTests(unittest.TestCase): with self.assertRaises(KeyError): session.inspect(999) - def test_inspect_includes_settled_tag_from_the_latest_frame(self): - session, engine, _, _ = make_session() - tag = {"state": "readable", "price": 150, "box": [100, 270, 200, 320]} - - class Tagged: - def plan(self, boxes, image_shape, shelf_offset=None): - return [] - - def attach(self, tracks): - by_track = {int(track["track_id"]): dict(tag) for track in tracks} - return by_track, list(by_track.values()), {"readable": len(by_track)} - - def stats(self): - return {} - - session._tags = Tagged() - result = process(session, 1) - track = result["tracks"][0] - track_id = track["track_id"] - self.assertEqual(track["price_tag"]["price"], 150) - self.assertIn("reconciliation", track) - - inspected = session.inspect(track_id) - self.assertEqual(inspected["price_tag"], track["price_tag"]) - self.assertEqual(inspected["reconciliation"], track["reconciliation"]) - self.assertIn("identity", inspected) - - session._tags = None - bare = process(session, 2) - self.assertNotIn("price_tag", bare["tracks"][0]) - inspected_bare = session.inspect(track_id) - self.assertNotIn("price_tag", inspected_bare) - self.assertNotIn("reconciliation", inspected_bare) - - session.reset() - with self.assertRaises(KeyError): - session.inspect(track_id) - engine.shutdown() def test_close_stops_the_loop(self): session, engine, _, _ = make_session() diff --git a/recognization/code/test_ocr_tags.py b/recognization/code/test_ocr_tags.py index 25d911d..26c6401 100644 --- a/recognization/code/test_ocr_tags.py +++ b/recognization/code/test_ocr_tags.py @@ -335,6 +335,56 @@ class TagServiceTests(unittest.TestCase): self.assertGreaterEqual(service.stats()["passes_run"], 1) self.assertIsNone(service.stats()["last_pass_error"]) + def test_a_queued_pass_keeps_the_shelf_it_was_planned_on(self): + """OCR that finishes after a pan must clear its own rail, not the new one.""" + service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + array = image() + planned = service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)) + self.assertEqual(service.stats()["pending_bands"], 1) + moved = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0)) + self.assertTrue(moved, "the new shelf is a different rail and must be queued too") + self.assertEqual(service.stats()["pending_bands"], 2) + service.run_pass(array, planned[0]) + self.assertEqual(service.stats()["pending_bands"], 1, "the finished pass clears only its own key") + self.assertEqual(service._tags[next(iter(service._tags))].shelf_key[0], "shelf") + self.assertNotEqual( + service._tags[next(iter(service._tags))].shelf_key, + moved[0].key, + "the late read must not be filed under the shelf now in frame", + ) + service.run_pass(array, moved[0]) + self.assertEqual(service.stats()["pending_bands"], 0) + self.assertEqual(len({tag.shelf_key for tag in service._tags.values()}), 2) + + def test_an_offscreen_tag_does_not_bind_the_new_facing(self): + """A pan that reuses screen coordinates must not hand the old tag to the new facing.""" + service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1) + boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)] + array = image() + first = service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)) + service.run_pass(array, first[0]) + by_track, tags, _ = service.attach( + [{"track_id": 0, "box": [100, 100, 200, 300]}], + shelf_offset=(0.0, 0.0), + image_shape=array.shape, + ) + self.assertEqual(by_track[0]["state"], TAG_STATE_READABLE) + self.assertEqual(by_track[0]["price"], 269) + self.assertGreaterEqual(tags[0]["box"][1], 0) + self.assertLess(tags[0]["box"][1], array.shape[0]) + + again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0)) + self.assertTrue(again) + by_track, tags, _ = service.attach( + [{"track_id": 9, "box": [100, 100, 200, 300]}], + shelf_offset=(0.0, 400.0), + image_shape=array.shape, + ) + self.assertEqual(tags, [], "the old tag is above this frame") + self.assertEqual(by_track[9]["state"], TAG_STATE_ABSENT) + self.assertIsNone(by_track[9]["id"]) + class StorePolicyTests(unittest.TestCase): def test_a_missing_file_uses_the_defaults(self): diff --git a/recognization/code/transport.py b/recognization/code/transport.py index 66e94a1..0322306 100644 --- a/recognization/code/transport.py +++ b/recognization/code/transport.py @@ -104,7 +104,10 @@ class BodyLimit: A handler that checks Content-Length, or reads request.body(), runs after the server has already buffered the stream. This counts bytes at receive - and returns 413 without handing the rest to the app. WebSocket frames are + and returns 413 without handing the rest to the app. Content-Length is + not trusted: the stream is what counts, and reading stops at the cap. + If the app already started the response, the connection is aborted + instead of sending a second http.response.start. WebSocket frames are not HTTP bodies and are left alone. """ @@ -116,10 +119,13 @@ class BodyLimit: if scope.get("type") != "http": await self.app(scope, receive, send) return + class TooLarge(Exception): pass seen = 0 + started = False + async def limited() -> Dict[str, Any]: nonlocal seen message = await receive() @@ -128,12 +134,24 @@ class BodyLimit: body = message.get("body") or b"" seen += len(body) if seen > self.max_bytes: + # The app must not see this chunk or any later one. raise TooLarge return message + async def tracked(message: Dict[str, Any]) -> None: + nonlocal started + if message.get("type") == "http.response.start": + started = True + await send(message) + try: - await self.app(scope, limited, send) + await self.app(scope, limited, tracked) except TooLarge: + if started: + # The status line is already on the wire. Returning leaves the + # response unfinished so the server closes the connection + # instead of accepting a second http.response.start. + return await send( { "type": "http.response.start",