tngo/product-recognition-demo
1
0
Fork 0
forked from vnguyen/product-recognition-demo
Code Pull requests Activity

fix: restore reliable authenticated live recognition

Browse source
This commit is contained in:
kiyo9w 2026-09-28 12:38:15 +09:00
parent 0e23425ef0
commit 934615b593
40 changed files with 2556 additions and 820 deletions
Download patch file Download diff file Expand all files Collapse all files

245
recognization/code/app.py
View file

@ -20,7 +20,7 @@ from __future__ import annotations
import argparse
import asyncio
import base64
import hmac
from contextlib import asynccontextmanager
import json
import logging
import os
@ -74,6 +74,17 @@ from engine import ( # noqa: E402
)
from record import FrameRecorder # noqa: E402
from live import LiveSession, LiveSettings # noqa: E402
from transport import ( # noqa: E402
BodyLimit,
ServiceAdvertiser,
advertise_host,
bearer_matches,
declared_length,
is_loopback_host,
read_bounded,
require_token_for_bind,
token_matches,
)
LOGGER = logging.getLogger("dorayaki")
@ -94,7 +105,8 @@ class ServerSettings:
host: str = "127.0.0.1"
port: int = 8080
token: Optional[str] = None
cors_origins: str = "*"
# Empty means same-origin only. "*" reflects every caller and is not a default.
cors_origins: str = ""
max_sessions: int = DEFAULT_MAX_SESSIONS
live_method: str = "hybrid"
live_conf: float = 0.1
@ -124,6 +136,26 @@ def server_settings_from_env() -> ServerSettings:
)
def load_local_token(path: str = "config/local.env") -> None:
"""Fill DORAYAKI_API_TOKEN from an ignored env file. Never prints the value.
ponytail: one KEY=VALUE file, no quoting. Upgrade path is a real env loader
once a second secret needs the same file.
"""
if os.environ.get("DORAYAKI_API_TOKEN"):
return
try:
with open(path, "r", encoding="utf-8") as handle:
lines = handle.readlines()
except OSError:
return
for line in lines:
key, sep, value = line.strip().partition("=")
if key == "DORAYAKI_API_TOKEN" and sep and value:
os.environ["DORAYAKI_API_TOKEN"] = value
return
# ---------------------------------------------------------------------------
# request models
# ---------------------------------------------------------------------------
@ -234,18 +266,29 @@ def create_app(
scans: Any = None,
) -> FastAPI:
settings = settings or server_settings_from_env()
@asynccontextmanager
async def lifespan(_app: FastAPI):
yield
for session in list(sessions.values()):
session.close()
engine.shutdown()
app = FastAPI(
title="Dorayaki Recognition Server",
version="1.0",
description="Shelf product detection, tracking and classification for capture clients.",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()] or ["*"],
allow_credentials=False,
allow_methods=["*"],
allow_headers=["*"],
)
# Same-origin is the default. A configured list is echoed back; "*" is not.
cors = [o.strip() for o in settings.cors_origins.split(",") if o.strip() and o.strip() != "*"]
if cors:
app.add_middleware(
CORSMiddleware,
allow_origins=cors,
allow_credentials=False,
allow_methods=["GET", "POST"],
allow_headers=["Authorization", "Content-Type"],
)
scans = scans if scans is not None else build_legacy_scans(engine)
# A live session can be recorded so a real scan becomes data to label later.
@ -260,8 +303,7 @@ def create_app(
async def authorize(authorization: Optional[str] = Header(default=None)) -> None:
if not settings.token:
return
expected = f"Bearer {settings.token}"
if not authorization or not hmac.compare_digest(authorization, expected):
if not bearer_matches(authorization, settings.token):
raise _error("UNAUTHORIZED", "missing or invalid bearer token", 401)
# ---------------------------------------------------------------- health
@ -316,19 +358,22 @@ def create_app(
raise _error("NOT_FOUND", f"unknown sku '{sku_id}'", 404)
return {"status": "success", "product": product}
# ---------------------------------------------------------------- samples
@app.get("/v1/samples", dependencies=[Depends(authorize)])
async def list_samples() -> Dict[str, Any]:
def _sample_listing() -> List[Dict[str, Any]]:
listing = []
import cv2
for item in samples.values():
entry = {"id": item["id"], "label": item["label"], "url": f"/v1/samples/{item['id']}"}
import cv2
image = cv2.imread(item["path"])
if image is not None:
entry["width"], entry["height"] = int(image.shape[1]), int(image.shape[0])
listing.append(entry)
return {"samples": listing}
return listing
@app.get("/v1/samples", dependencies=[Depends(authorize)])
async def list_samples() -> Dict[str, Any]:
loop = asyncio.get_running_loop()
return {"samples": await loop.run_in_executor(None, _sample_listing)}
@app.get("/v1/samples/{sample_id}", dependencies=[Depends(authorize)])
async def get_sample(sample_id: str) -> Response:
@ -350,37 +395,44 @@ def create_app(
if method is not None:
_require_method(method)
content_type = (request.headers.get("content-type") or "").split(";")[0].strip().lower()
announced = declared_length(request.headers.get("content-length"))
if announced is not None and announced > MAX_UPLOAD_BYTES:
raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413)
if content_type == "multipart/form-data":
form = await request.form()
form = await request.form(max_part_size=MAX_UPLOAD_BYTES)
upload = form.get("file")
if upload is None or not hasattr(upload, "read"):
raise _error("BAD_REQUEST", "multipart body needs a 'file' field", 400)
payload = await upload.read()
payload = await read_bounded(upload, MAX_UPLOAD_BYTES)
else:
if content_type and not content_type.startswith("image/") and content_type != "application/octet-stream":
raise _error("UNSUPPORTED_MEDIA_TYPE", f"unsupported content type '{content_type}'", 415)
payload = await request.body()
payload = await read_bounded(request.stream(), MAX_UPLOAD_BYTES)
if not payload:
raise _error("BAD_REQUEST", "empty request body", 400)
if len(payload) > MAX_UPLOAD_BYTES:
raise _error("PAYLOAD_TOO_LARGE", f"image larger than {MAX_UPLOAD_BYTES} bytes", 413)
loop = asyncio.get_running_loop()
try:
image = engine.decode_image(payload)
image = await loop.run_in_executor(None, engine.decode_image, payload)
except UndecodableImage as exc:
raise _error("UNDECODABLE_IMAGE", str(exc), 422)
try:
result = engine.recognize(
image,
classify=classify,
method=method,
conf=conf,
with_empty_slots=empty_slots,
result = await loop.run_in_executor(
None,
lambda: engine.recognize(
image,
classify=classify,
method=method,
conf=conf,
with_empty_slots=empty_slots,
),
)
except UnknownMethod as exc:
raise _error("BAD_REQUEST", str(exc), 400)
if tags:
result = _attach_still_tags(result, image)
result = await loop.run_in_executor(None, _attach_still_tags, result, image)
return result
def _is_price_card(crop: Any) -> bool:
@ -514,7 +566,10 @@ def create_app(
wanted = set(int(i) for i in body.detection_ids)
detections = [d for d in detections if int(d["id"]) in wanted]
started = time.perf_counter()
rows = engine.classify_boxes(stored.image, detections, method=method)
loop = asyncio.get_running_loop()
rows = await loop.run_in_executor(
None, lambda: engine.classify_boxes(stored.image, detections, method=method)
)
rows_by_id = {row["detection_id"]: row for row in rows}
for detection in stored.detections:
row = rows_by_id.get(int(detection["id"]))
@ -567,21 +622,24 @@ def create_app(
stored = engine.image_store.get(image_id)
except ImageExpired:
raise _error("IMAGE_EXPIRED", f"image '{image_id}' is no longer available", 404)
return engine.inspect(stored.image, body.box, method=method)
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
None, lambda: engine.inspect(stored.image, body.box, method=method)
)
# ---------------------------------------------------------------- live stream
@app.websocket("/v1/stream")
async def stream(websocket: WebSocket) -> None:
if settings.token:
supplied = websocket.query_params.get("token") or ""
header = websocket.headers.get("authorization") or ""
expected_header = f"Bearer {settings.token}"
if not (
hmac.compare_digest(supplied, settings.token)
or hmac.compare_digest(header, expected_header)
):
await websocket.close(code=1008)
return
# Query tokens are rejected. A wrong header is rejected before hello.
# A missing header is the browser case: hello goes out, then the first
# client text must be {"type":"auth","token":"..."} or the socket closes.
if websocket.query_params.get("token"):
await websocket.close(code=1008)
return
header = websocket.headers.get("authorization")
if settings.token and header and not bearer_matches(header, settings.token):
await websocket.close(code=1008)
return
async with sessions_lock:
if len(sessions) >= settings.max_sessions:
await websocket.close(code=1013)
@ -644,9 +702,10 @@ def create_app(
"message": "frame processing failed"})
continue
if recorder is not None:
# Record what the client sent and what we saw in it, so a real
# scan can be labelled later instead of only measured on photos.
recorder.write(session_id, frame_id, payload, result)
# Disk writes must not sit on the event loop between results.
await loop.run_in_executor(
None, recorder.write, session_id, frame_id, payload, result
)
await send(result)
async def handle_inspect(message: Dict[str, Any]) -> None:
@ -682,8 +741,23 @@ def create_app(
processing = asyncio.create_task(processing_loop())
async def handle_control(message: Dict[str, Any]) -> None:
authed = not settings.token or bearer_matches(
websocket.headers.get("authorization"), settings.token
)
async def handle_control(message: Dict[str, Any]) -> bool:
"""Apply one control message. False closes the socket (auth failed)."""
nonlocal authed
kind = message.get("type")
if not authed:
presented = message.get("token") if kind == "auth" else None
if isinstance(presented, str) and token_matches(presented, settings.token):
authed = True
return True
await websocket.close(code=1008)
return False
if kind == "auth":
return True
if kind == "config":
try:
config = session.configure(
@ -694,7 +768,7 @@ def create_app(
)
except (TypeError, ValueError):
await send({"type": "error", "code": "BAD_MESSAGE", "message": "bad config values"})
return
return True
await send({"type": "config_ack", "config": config})
elif kind == "reset":
session.reset()
@ -705,6 +779,7 @@ def create_app(
await handle_inspect(message)
else:
await send({"type": "error", "code": "BAD_MESSAGE", "message": f"unknown type '{kind}'"})
return True
try:
while True:
@ -717,6 +792,9 @@ def create_app(
text = message.get("text")
data = message.get("bytes")
if data is not None:
if not authed:
await websocket.close(code=1008)
break
if len(data) < 5:
await send({"type": "error", "code": "BAD_FRAME", "message": "frame shorter than 5 bytes"})
continue
@ -733,7 +811,8 @@ def create_app(
except (json.JSONDecodeError, ValueError):
await send({"type": "error", "code": "BAD_MESSAGE", "message": "invalid JSON"})
continue
await handle_control(parsed)
if not await handle_control(parsed):
break
except WebSocketDisconnect:
pass
finally:
@ -747,7 +826,7 @@ def create_app(
sessions.pop(session_id, None)
# ---------------------------------------------------------------- legacy session routes
@app.post("/api/sessions")
@app.post("/api/sessions", dependencies=[Depends(authorize)])
async def legacy_start_session(payload: Dict[str, Any] = None) -> Dict[str, Any]:
payload = payload or {}
session = scans.start_session(
@ -757,7 +836,7 @@ def create_app(
)
return {"status": "success", "session": session.snapshot()}
@app.post("/api/sessions/{session_id}/frames")
@app.post("/api/sessions/{session_id}/frames", dependencies=[Depends(authorize)])
async def legacy_observe_frame(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]:
from observe import FrameRequest, ObserveError
@ -765,11 +844,19 @@ def create_app(
try:
session = scans.get_session(session_id)
raw = payload.get("frame_bytes_b64") or payload.get("image_base64") or ""
if not isinstance(raw, str):
raw = ""
if "," in raw:
raw = raw.split(",", 1)[1]
# Base64 expands ~4/3. Reject the text before decoding it.
if len(raw) > (MAX_FRAME_BYTES * 4) // 3 + 8:
raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413)
frame_bytes = base64.b64decode(raw) if raw else b""
if len(frame_bytes) > MAX_FRAME_BYTES:
raise _error("PAYLOAD_TOO_LARGE", f"frame larger than {MAX_FRAME_BYTES} bytes", 413)
request = FrameRequest(
frame_id=str(payload.get("frame_id") or ""),
frame_bytes=base64.b64decode(raw) if raw else b"",
frame_bytes=frame_bytes,
sequence=int(payload.get("sequence", 0)),
captured_at=float(payload.get("captured_at") or 0),
session_id=session.session_id,
@ -781,12 +868,13 @@ def create_app(
)
if not request.frame_id:
raise _error("BAD_REQUEST", "frame_id required", 400)
observation = scans.observe_frame(request)
loop = asyncio.get_running_loop()
observation = await loop.run_in_executor(None, scans.observe_frame, request)
return {"status": "success", "observation": observation.as_dict(), "session": session.snapshot()}
except ObserveError as exc:
raise _error("CONFLICT", str(exc), 409)
@app.get("/api/sessions/{session_id}")
@app.get("/api/sessions/{session_id}", dependencies=[Depends(authorize)])
async def legacy_get_session(session_id: str) -> Dict[str, Any]:
from observe import ObserveError
@ -795,7 +883,7 @@ def create_app(
except ObserveError as exc:
raise _error("NOT_FOUND", str(exc), 404)
@app.post("/api/sessions/{session_id}/review")
@app.post("/api/sessions/{session_id}/review", dependencies=[Depends(authorize)])
async def legacy_review(session_id: str, payload: Dict[str, Any] = None) -> Dict[str, Any]:
from observe import ObserveError
@ -812,7 +900,7 @@ def create_app(
except ObserveError as exc:
raise _error("CONFLICT", str(exc), 409)
@app.post("/api/sessions/{session_id}/finalize")
@app.post("/api/sessions/{session_id}/finalize", dependencies=[Depends(authorize)])
async def legacy_finalize(session_id: str) -> Dict[str, Any]:
from observe import ObserveError
@ -821,7 +909,7 @@ def create_app(
except ObserveError as exc:
raise _error("CONFLICT", str(exc), 409)
@app.post("/api/sessions/{session_id}/cancel")
@app.post("/api/sessions/{session_id}/cancel", dependencies=[Depends(authorize)])
async def legacy_cancel(session_id: str) -> Dict[str, Any]:
from observe import ObserveError
@ -830,7 +918,7 @@ def create_app(
except ObserveError as exc:
raise _error("CONFLICT", str(exc), 409)
@app.post("/api/sessions/{session_id}/rescan")
@app.post("/api/sessions/{session_id}/rescan", dependencies=[Depends(authorize)])
async def legacy_rescan(session_id: str) -> Dict[str, Any]:
from observe import ObserveError
@ -839,7 +927,7 @@ def create_app(
except ObserveError as exc:
raise _error("CONFLICT", str(exc), 409)
@app.post("/api/compare_live_vs_still")
@app.post("/api/compare_live_vs_still", dependencies=[Depends(authorize)])
async def legacy_compare(payload: Dict[str, Any] = None) -> Dict[str, Any]:
from observe import ObserveError
@ -881,14 +969,11 @@ def create_app(
# ---------------------------------------------------------------- static web client
resolved_web_dir = web_dir or os.path.join(current_dir, "web")
app.add_middleware(BodyLimit, max_bytes=MAX_UPLOAD_BYTES)
if os.path.isdir(resolved_web_dir):
app.mount("/", StaticFiles(directory=resolved_web_dir, html=True), name="web")
@app.on_event("shutdown")
async def _shutdown() -> None:
for session in list(sessions.values()):
session.close()
engine.shutdown()
return app
@ -960,6 +1045,11 @@ def main() -> None:
if args.log_level is not None:
server.log_level = args.log_level
load_local_token()
if not server.token:
server.token = os.environ.get("DORAYAKI_API_TOKEN") or None
require_token_for_bind(server.host, server.token)
engine_settings = settings_from_env()
if args.record_frames is not None:
server.record_frames = args.record_frames
@ -996,23 +1086,34 @@ def main() -> None:
scheme = "https" if server.ssl_certfile else "http"
LOGGER.info("Dorayaki recognition server on %s://%s:%d (token %s)",
scheme, server.host, server.port, "required" if server.token else "off")
if server.host in ("127.0.0.1", "localhost"):
if is_loopback_host(server.host):
LOGGER.info("Open %s://127.0.0.1:%d in a browser for the web client", scheme, server.port)
else:
LOGGER.info("LAN mode: live camera needs https (secure context); use --ssl-certfile or a tunnel")
import uvicorn
uvicorn.run(
app,
host=server.host,
port=server.port,
log_level=(server.log_level or "info").lower(),
access_log=False, # access log would print ?token=... query strings
ws="websockets-sansio",
ssl_certfile=server.ssl_certfile,
ssl_keyfile=server.ssl_keyfile,
)
advertised = advertise_host(server.host)
advertiser = ServiceAdvertiser(advertised, server.port) if advertised else None
if advertiser is not None and advertiser.start():
LOGGER.info("Advertising _dorayaki._tcp on %s:%d", advertised, server.port)
elif advertised:
LOGGER.info("No dns-sd on this host; _dorayaki._tcp was not advertised")
try:
uvicorn.run(
app,
host=server.host,
port=server.port,
log_level=(server.log_level or "info").lower(),
access_log=False, # request lines can carry Authorization; keep them off
ws="websockets-sansio",
ssl_certfile=server.ssl_certfile,
ssl_keyfile=server.ssl_keyfile,
)
finally:
if advertiser is not None:
advertiser.close()
if __name__ == "__main__":

103
recognization/code/benchmark.py
View file

@ -1,103 +0,0 @@
"""
Benchmark Suite for Supermarket Product Classification
Compares SURF, ORB, BRISK, and Hybrid methods following
MDPI Electronics 2023, 12(17), 3640.
"""
import os
import sys
import time
import cv2
import numpy as np
from typing import Dict, List, Any
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from feature_matcher import ProductFeatureMatcher
def run_benchmark(stock_dir: str = "stock", num_runs: int = 3):
print("=" * 70)
print("RETAIL SHELF PRODUCT RECOGNITION - ALGORITHM BENCHMARK")
print("Methods Evaluated: SURF | ORB | BRISK | HYBRID (Paper Method)")
print("=" * 70)
matcher = ProductFeatureMatcher(stock_dir=stock_dir)
templates = matcher.db.templates
if not templates:
print("[Error] No templates found in stock directory.")
return
methods = ["surf", "orb", "brisk", "hybrid"]
results = {m: {"latencies": [], "inliers": [], "keypoints_query": []} for m in methods}
# Test images: We will evaluate using crops synthesized with slight transformations
# (scale, rotation, noise) as well as the template images themselves
print(f"\nEvaluating across {len(templates)} catalog SKUs (repeat count: {num_runs})...\n")
for sku_id, tmpl in templates.items():
original_img = tmpl["image"]
h, w = original_img.shape[:2]
# Apply synthetic shelf condition (rotation by 10 deg, resize by 0.85)
M = cv2.getRotationMatrix2D((w / 2, h / 2), 10, 0.85)
query_img = cv2.warpAffine(original_img, M, (w, h))
for method in methods:
times = []
for _ in range(num_runs):
t0 = time.perf_counter()
res = matcher.classify_crop(query_img, method=method)
t1 = time.perf_counter()
times.append((t1 - t0) * 1000.0) # ms
avg_latency = np.mean(times)
results[method]["latencies"].append(avg_latency)
# Record inliers/score
if res.get("status") == "success":
top_sku = res["best_sku"]
score = res["score"]
results[method]["inliers"].append(score)
# Summarize metrics
summary_rows = []
for method in methods:
latencies = results[method]["latencies"]
scores = results[method]["inliers"]
avg_lat = np.mean(latencies) if latencies else 0.0
p95_lat = np.percentile(latencies, 95) if latencies else 0.0
avg_score = np.mean(scores) if scores else 0.0
# Paper characteristics notes
notes = {
"surf": "Scale & rotation robust; Hessian gradient features",
"orb": "Ultra fast binary descriptor; FAST + BRIEF",
"brisk": "Scale-space AGAST; circular pattern descriptor",
"hybrid": "Paper fusion (SURF+ORB+BRISK); optimal accuracy/speed"
}[method]
summary_rows.append([
method.upper(),
f"{avg_lat:.2f} ms",
f"{p95_lat:.2f} ms",
f"{avg_score:.1f}",
notes
])
headers = ["Method", "Avg Latency (ms)", "P95 Latency (ms)", "Avg Match Score", "Characteristics / Paper Findings"]
print("\n" + "-" * 105)
print(f"{headers[0]:<10} | {headers[1]:<18} | {headers[2]:<18} | {headers[3]:<16} | {headers[4]}")
print("-" * 105)
for row in summary_rows:
print(f"{row[0]:<10} | {row[1]:<18} | {row[2]:<18} | {row[3]:<16} | {row[4]}")
print("-" * 105)
print("\n[Benchmark Complete] All methods operational.")
if __name__ == "__main__":
run_benchmark()

13
recognization/code/build_store_from_shelf.py
View file

@ -46,16 +46,21 @@ store = "stock-mediya"
shutil.rmtree(store, ignore_errors=True)
os.makedirs(store, exist_ok=True)
rows = ["SKU_ID,product_name,receipt_date,expiration_date,number_in,number_out,number_on,image_link,price"]
written = []
for sku, name, indices in PRODUCTS:
saved = 0
for position, index in enumerate(indices):
box = boxes[index]
x1, y1, x2, y2 = [int(v) for v in box]
crop = image[max(0,y1):y2, max(0,x1):x2]
crop = image[max(0, y1):y2, max(0, x1):x2]
if crop.size == 0:
continue
suffix = "" if position == 0 else f"_{position}"
suffix = "" if saved == 0 else f"_{saved}"
cv2.imwrite(os.path.join(store, f"{sku}{suffix}.jpeg"), crop)
saved += 1
if saved == 0:
raise SystemExit(f"{sku}: no readable crop; refusing a catalog row with no view")
written.append(sku)
rows.append(f"{sku},{name},2026-09-25,2026-10-20,24,8,16,'',{PRICES[sku]}")
open(os.path.join(store, "stock.csv"), "w").write("\n".join(rows) + "\n")
print(f"{store}: {len(PRODUCTS)} products, "
f"{sum(len(i) for _, _, i in PRODUCTS)} views, csv written")
print(f"{store}: {len(written)} products, csv written")

41
recognization/code/download_stock.py
View file

@ -1,41 +0,0 @@
import os
import sys
import io
import urllib.request
from PIL import Image
# Ensure stdout handles utf-8 cleanly
if sys.platform.startswith('win'):
sys.stdout.reconfigure(encoding='utf-8')
urls = {
'sku_011': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_apple@2x_v2.png', '野菜生活100 アップルサラダ'),
'sku_012': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_lemon@2x_v2.png', '野菜生活100 レモンサラダ'),
'sku_013': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/green-product-pic_2.png', '野菜生活100 Smoothie グリーンスムージー'),
'sku_014': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/orange-product-pic_2.png', '野菜生活100 Smoothie ビタミンスムージー'),
'sku_015': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/yellow-product-pic_2.png', '野菜生活100 Smoothie バナナ＆アーモンドmix'),
'sku_016': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/red-product-pic_2.png', '野菜生活100 Smoothie アサイー＆ベリーmix')
}
os.makedirs('stock', exist_ok=True)
for sku, (url, name) in urls.items():
print(f'Fetching {sku}: {name} from {url}...')
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
try:
with urllib.request.urlopen(req, timeout=15) as resp:
data = resp.read()
img = Image.open(io.BytesIO(data))
if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
bg = Image.new('RGB', img.size, (255, 255, 255))
if img.mode != 'RGBA':
img = img.convert('RGBA')
bg.paste(img, mask=img.split()[3])
img = bg
else:
img = img.convert('RGB')
out_path = f'stock/{sku}.jpeg'
img.save(out_path, 'JPEG', quality=95)
print(f'Successfully saved {sku}: {name} -> {out_path} ({img.size})')
except Exception as e:
print(f'Error downloading {sku} ({url}): {e}')

165
recognization/code/engine.py
View file

@ -251,6 +251,7 @@ class RecognitionEngine:
# The embedding gallery is built on first use (it loads a model and every
# enrolled view), and cached as False when it cannot be built at all.
self._gallery: Any = None
self._gallery_lock = threading.Lock()
workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2)
self.classify_workers = int(workers)
self._executor = classify_executor or ThreadPoolExecutor(
@ -417,23 +418,22 @@ class RecognitionEngine:
return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier"
def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
"""Score one crop against the catalog, in whatever scale this engine uses.
"""Score one crop. Gallery never falls back to the matcher.
The live session calls this per track. With the gallery backend it must go
through the gallery, or a live scan would use the matcher while the still path
used the enrolled store - measured: the matcher accepts look-alikes and cannot
say "unknown", which is exactly what a real shelf needs it to do.
A missing gallery returns no scores (unknown), not inlier counts. Mixing
those scales is how a live scan accepted a look-alike the still path rejected.
"""
if self.identity_scale == "cosine":
gallery = self.identity_gallery()
if gallery:
match = gallery.match(crop)
if match.rejected or not match.sku_id:
return {}
scores = {match.sku_id: match.similarity}
if match.runner_up:
scores[match.runner_up] = match.runner_up_similarity
return {str(sku): float(score) for sku, score in scores.items()}
if not gallery:
return {}
match = gallery.match(crop)
if match.rejected or not match.sku_id:
return {}
scores = {match.sku_id: match.similarity}
if match.runner_up:
scores[match.runner_up] = match.runner_up_similarity
return {str(sku): float(score) for sku, score in scores.items()}
method = (method or self.settings.live_method).lower()
if method not in CLASSIFY_METHODS:
raise UnknownMethod(method)
@ -445,27 +445,41 @@ class RecognitionEngine:
def identity_gallery(self) -> Any:
"""The embedding gallery, loaded once (None when unavailable)."""
if self._gallery is False:
return None
if self._gallery is not None:
return self._gallery
from gallery import ProductGallery
with self._gallery_lock:
if self._gallery is False:
return None
if self._gallery is not None:
return self._gallery
from gallery import ProductGallery
gallery = ProductGallery(
self.settings.stock_dir,
[product["sku_id"] for product in self.catalog()],
threshold=self.settings.gallery_threshold,
head_threshold=self.settings.gallery_head_threshold,
image_size=self.settings.gallery_image_size,
)
try:
loaded = gallery.load()
except Exception as exc: # torch/timm missing, no weights, no network
LOGGER.warning("identity gallery unavailable: %s", exc)
loaded = False
self._gallery = gallery if loaded else False
if loaded:
LOGGER.info("identity gallery: %d views over %d SKUs on %s",
gallery.view_count(), len(self.settings.stock_dir and gallery.sku_ids), gallery.device)
return self._gallery
gallery = ProductGallery(
self.settings.stock_dir,
[product["sku_id"] for product in self.catalog()],
threshold=self.settings.gallery_threshold,
head_threshold=self.settings.gallery_head_threshold,
image_size=self.settings.gallery_image_size,
)
try:
loaded = gallery.load()
except Exception as exc: # torch/timm missing, no weights, no network
LOGGER.warning("identity gallery unavailable: %s", exc)
loaded = False
# False means tried and failed. Do not retry and do not use the matcher.
self._gallery = gallery if loaded else False
if loaded:
LOGGER.info(
"identity gallery: %d views over %d SKUs on %s",
gallery.view_count(),
len(gallery.sku_ids),
gallery.device,
)
else:
LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized")
return gallery if loaded else None
def classify_boxes(
self,
@ -493,8 +507,8 @@ class RecognitionEngine:
if (self.settings.identity_backend or "matcher").lower() == "gallery":
gallery = self.identity_gallery()
if gallery:
return self._classify_with_gallery(gallery, ordered, crops)
# No matcher fallback: an unavailable gallery is unrecognized, not a different model.
return self._classify_with_gallery(gallery, ordered, crops)
if method == "hybrid_fast":
futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops]
else:
@ -516,26 +530,47 @@ class RecognitionEngine:
return results
def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]:
"""Full single-crop result: decision, per-method details, crop, visualization."""
from identity import decide_identity
"""Full single-crop result, using the same backend as recognize/live."""
from identity import decide_identity, decision_for_match
crop = self.crop(image, box)
if self.identity_scale == "cosine":
gallery = self.identity_gallery()
match = gallery.match(crop) if gallery else None
# match() already applied the gallery's own threshold. Ranking again
# with the published cosine gates must not invent a second line.
decision = decision_for_match(match)
return self._inspect_payload(box, "gallery", decision, crop, visualize=False)
method = (method or self.settings.live_method).lower()
if method not in CLASSIFY_METHODS:
raise UnknownMethod(method)
crop = self.crop(image, box)
if method == "hybrid_fast":
scores, details = self.matcher.score_crop_fast(crop)
visual_method = "hybrid"
else:
scores, details = self.matcher.score_crop(crop, method=method)
visual_method = method
decision = decide_identity(scores, details=details, method=method)
identity = identity_payload(decision)
return self._inspect_payload(box, method, decision, crop, visual_method=visual_method)
def _inspect_payload(
self,
box: Sequence[int],
method: str,
decision: Dict[str, Any],
crop: Any,
*,
visual_method: Optional[str] = None,
visualize: bool = True,
) -> Dict[str, Any]:
identity = identity_payload(decision)
top_details = decision.get("details") or {}
sub_details = None
if isinstance(top_details, dict):
raw = top_details.get("sub_details")
if raw:
sub_details = {k: dict(v) for k, v in raw.items()}
sub_details = {key: dict(value) for key, value in raw.items()}
product = None
visualization = None
@ -547,16 +582,16 @@ class RecognitionEngine:
elif identity["candidates"]:
visualized_sku = identity["candidates"][0]["sku_id"]
is_candidate = True
if visualized_sku:
if visualize and visualized_sku and visual_method:
try:
visualization = self.matcher.generate_match_visualization(
crop, visualized_sku, method=method if method != "hybrid_fast" else "hybrid"
crop, visualized_sku, method=visual_method
)
except Exception:
visualization = None
return {
"box": [int(v) for v in box],
"box": [int(value) for value in box],
"method": method,
"identity": identity,
"sub_details": sub_details,
@ -724,48 +759,22 @@ class RecognitionEngine:
unseen store photos, the matcher labelled milk cartons as mayonnaise and its
score could not be thresholded to stop it.
"""
from identity import decision_for_match
results: List[Dict[str, Any]] = []
for box, crop in zip(ordered, crops):
try:
match = gallery.match(crop)
except Exception as exc: # a bad crop must not fail the batch
match = None
LOGGER.debug("gallery match failed: %s", exc)
if match is None or match.rejected:
similarity = float(match.similarity) if match is not None else 0.0
results.append({
"detection_id": int(box["id"]),
"box": list(box["box"]),
"identity": {
"state": "unrecognized",
"sku_id": None,
"identity_level": "rejected",
"confidence": round(similarity, 3),
"score": round(similarity, 3),
"candidates": [],
"attempts": 1,
"method": "gallery",
"message": "no enrolled product is close enough",
},
})
continue
runner_up = (
{"sku_id": match.runner_up, "score": round(match.runner_up_similarity, 3)}
if match.runner_up else {}
)
match = None
if gallery is not None:
try:
match = gallery.match(crop)
except Exception as exc: # a bad crop must not fail the batch
match = None
LOGGER.debug("gallery match failed: %s", exc)
decision = decision_for_match(match)
results.append({
"detection_id": int(box["id"]),
"box": list(box["box"]),
"identity": {
"state": "recognized",
"sku_id": match.sku_id,
"identity_level": "accepted",
"confidence": round(match.similarity, 3),
"score": round(match.similarity, 3),
"candidates": [runner_up] if runner_up else [],
"attempts": 1,
"method": "gallery",
},
"identity": identity_payload(decision),
})
return results

693
recognization/code/eval_live.py Normal file
View file

@ -0,0 +1,693 @@
"""Replay a recorded walk through LiveSession and count what it actually decided.
The recording format is FrameRecorder's: DIR/<session>/index.jsonl plus the JPEG
bytes the client sent. Optional labels live in labels.json next to that index
(or at --labels). A label is {"frame", "box", "sku_id"}.
Template leakage is refused, not scored. A label whose crop matches a catalog
template (byte-identical, or IoU >= 0.98 against a same-sized template pasted
into the frame) is excluded and counted under leakage. Those crops are how the
store was enrolled; scoring them is training-set accuracy.
python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH]
[--json PATH] [--iou 0.5]
Prints a human count table and writes the same report as JSON (default
<recording>/eval_live.json). Missing input exits 2 with {"error": ...} on
stdout. The same pinned recording, catalog, weights, and labels produce the
same counts; timings are omitted because they are not.
No held-out labels means denominators and fingerprints only. That is not a
walk accuracy number.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sys
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
LEVELS = ("accepted", "ambiguous", "rejected", "unknown", "unsettled")
TAG_STATES = ("readable", "unreadable", "absent", "detected")
class EvalInputError(ValueError):
"""The recording, labels, or catalog cannot be scored as given."""
def file_sha256(path: str) -> Optional[str]:
digest = hashlib.sha256()
try:
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
except OSError:
return None
return digest.hexdigest()
def content_sha256(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _iou(a: Sequence[float], b: Sequence[float]) -> float:
if len(a) != 4 or len(b) != 4:
return 0.0
ax1, ay1, ax2, ay2 = (float(v) for v in a)
bx1, by1, bx2, by2 = (float(v) for v in b)
ix1, iy1 = max(ax1, bx1), max(ay1, by1)
ix2, iy2 = min(ax2, bx2), min(ay2, by2)
inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1)
if inter <= 0.0:
return 0.0
area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)
area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)
union = area_a + area_b - inter
return inter / union if union > 0.0 else 0.0
def load_index(recording_dir: str) -> List[Dict[str, Any]]:
path = os.path.join(recording_dir, "index.jsonl")
if not os.path.isfile(path):
raise EvalInputError(f"missing recording index: {path}")
rows: List[Dict[str, Any]] = []
with open(path, "r", encoding="utf-8") as handle:
for number, line in enumerate(handle, start=1):
line = line.strip()
if not line:
continue
try:
entry = json.loads(line)
except json.JSONDecodeError as exc:
raise EvalInputError(f"{path}:{number}: {exc.msg}") from exc
if not isinstance(entry, dict) or "frame" not in entry or "path" not in entry:
raise EvalInputError(f"{path}:{number}: line needs frame and path")
rows.append(entry)
if not rows:
raise EvalInputError(f"empty recording index: {path}")
return rows
def recording_fingerprint(recording_dir: str, rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
"""Hash the JPEGs the client sent, in index order. Missing files are an error."""
digest = hashlib.sha256()
missing: List[str] = []
for row in rows:
name = str(row["path"])
path = os.path.join(recording_dir, name)
payload_hash = file_sha256(path)
if payload_hash is None:
missing.append(name)
continue
digest.update(f"{row['frame']}:{name}:{payload_hash}\n".encode("utf-8"))
if missing:
raise EvalInputError("missing recorded frames: " + ", ".join(missing[:8]))
return {"sha256": digest.hexdigest(), "frames": len(rows)}
def load_labels(path: Optional[str]) -> List[Dict[str, Any]]:
if not path:
return []
if not os.path.isfile(path):
raise EvalInputError(f"missing labels: {path}")
try:
with open(path, "r", encoding="utf-8") as handle:
payload = json.loads(handle.read())
except json.JSONDecodeError as exc:
raise EvalInputError(f"labels are not JSON: {exc.msg}") from exc
rows = payload.get("labels") if isinstance(payload, dict) else payload
if not isinstance(rows, list):
raise EvalInputError("labels must be a list or {\"labels\": [...]}")
cleaned: List[Dict[str, Any]] = []
for index, row in enumerate(rows):
if not isinstance(row, dict):
raise EvalInputError(f"labels[{index}] is not an object")
if "frame" not in row or "sku_id" not in row or "box" not in row:
raise EvalInputError(f"labels[{index}] needs frame, box, sku_id")
box = row["box"]
if not isinstance(box, (list, tuple)) or len(box) != 4:
raise EvalInputError(f"labels[{index}].box must be [x1, y1, x2, y2]")
cleaned.append(
{
"frame": int(row["frame"]),
"box": [int(v) for v in box],
"sku_id": str(row["sku_id"]),
"source": row.get("source"),
}
)
return cleaned
def catalog_fingerprint(stock_dir: str, stock_csv: str = "stock.csv") -> Dict[str, Any]:
"""Same inputs as RecognitionEngine.catalog_version, plus the full digest and head."""
digest = hashlib.sha256()
csv_path = os.path.join(stock_dir, stock_csv)
csv_hash = file_sha256(csv_path)
if csv_hash is None:
digest.update(b"missing-csv")
else:
digest.update(bytes.fromhex(csv_hash))
try:
entries = sorted(
(name, os.path.getsize(os.path.join(stock_dir, name)))
for name in os.listdir(stock_dir)
if name.lower().endswith((".jpeg", ".jpg", ".png"))
)
except OSError as exc:
raise EvalInputError(f"cannot read catalog {stock_dir}: {exc.strerror}") from exc
for name, size in entries:
digest.update(f"{name}:{size}".encode("utf-8"))
full = digest.hexdigest()
head_path = os.path.join(stock_dir, "identity_head.pt")
return {
"stock_dir": os.path.basename(os.path.abspath(stock_dir)),
"version": full[:12],
"sha256": full,
"templates": len(entries),
"csv_sha256": csv_hash,
"identity_head_sha256": file_sha256(head_path),
}
def model_fingerprint(weights_path: str) -> Dict[str, Any]:
try:
import ultralytics
library = f"ultralytics-{ultralytics.__version__}"
except Exception:
library = "ultralytics-unknown"
return {
"version": f"{os.path.basename(weights_path)}@{library}",
"weights_sha256": file_sha256(weights_path) if weights_path else None,
}
def _decode_image(payload: bytes) -> Any:
import cv2
import numpy as np
image = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_COLOR)
if image is None:
raise EvalInputError("recorded frame is not a decodable image")
return image
def template_crops(stock_dir: str) -> List[Tuple[str, Any]]:
"""Catalog photos keyed by SKU stem (sku_l05, sku_l05_1, ...)."""
import cv2
crops: List[Tuple[str, Any]] = []
try:
names = sorted(os.listdir(stock_dir))
except OSError:
return crops
for name in names:
stem, extension = os.path.splitext(name)
if extension.lower() not in (".jpeg", ".jpg", ".png"):
continue
image = cv2.imread(os.path.join(stock_dir, name))
if image is None or image.size == 0:
continue
crops.append((stem, image))
return crops
def _same_picture(crop: Any, template: Any) -> bool:
"""True when two same-sized images are the same photo after JPEG."""
import numpy as np
left = crop.astype("float32").reshape(-1)
right = template.astype("float32").reshape(-1)
if left.size < 16 or left.size != right.size:
return False
left = left - float(left.mean())
right = right - float(right.mean())
denom = float(np.sqrt((left * left).sum() * (right * right).sum()))
if denom <= 1e-6:
return bool(np.max(np.abs(crop.astype("int16") - template.astype("int16"))) <= 2)
return float((left * right).sum() / denom) >= 0.98
def leaked_labels(
recording_dir: str,
rows: Sequence[Dict[str, Any]],
labels: Sequence[Dict[str, Any]],
stock_dir: str,
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""Drop labels whose crop is a catalog template.
Byte identity catches a frame that is the template file. Geometric identity
catches a template pasted into a larger frame at the labelled box.
"""
if not labels:
return [], []
templates = template_crops(stock_dir)
template_hashes = {}
try:
names = os.listdir(stock_dir)
except OSError:
names = []
for name in names:
digest = file_sha256(os.path.join(stock_dir, name))
if digest:
template_hashes.setdefault(digest, os.path.splitext(name)[0])
by_frame: Dict[int, List[Dict[str, Any]]] = {}
for label in labels:
by_frame.setdefault(int(label["frame"]), []).append(label)
held: List[Dict[str, Any]] = []
leaked: List[Dict[str, Any]] = []
frames = {int(row["frame"]): row for row in rows}
for frame_id, frame_labels in by_frame.items():
row = frames.get(frame_id)
image = None
payload_hash = None
if row is not None:
path = os.path.join(recording_dir, str(row["path"]))
try:
with open(path, "rb") as handle:
payload = handle.read()
except OSError:
payload = b""
payload_hash = content_sha256(payload) if payload else None
if payload:
image = _decode_image(payload)
for label in frame_labels:
reason = None
matched = None
if payload_hash in template_hashes:
reason = "frame_is_template"
matched = template_hashes[payload_hash]
elif image is not None:
x1, y1, x2, y2 = label["box"]
crop = image[max(0, y1) : max(0, y2), max(0, x1) : max(0, x2)]
if crop.size:
import cv2
ch, cw = crop.shape[:2]
for stem, template in templates:
th, tw = template.shape[:2]
if abs(th - ch) > 1 or abs(tw - cw) > 1:
continue
resized = template if (th, tw) == (ch, cw) else cv2.resize(template, (cw, ch))
# JPEG re-encoding keeps layout and drops exact pixels.
if _same_picture(crop, resized):
reason = "crop_is_template"
matched = stem
break
if reason:
leaked.append({**label, "leak": reason, "template": matched})
else:
held.append(label)
return held, leaked
def identity_level(track: Dict[str, Any]) -> str:
identity = track.get("identity") or {}
level = identity.get("identity_level")
if level in LEVELS:
return str(level)
state = identity.get("state")
if state == "recognized":
return "accepted"
if state == "needs_confirmation":
return "ambiguous"
if state == "unrecognized":
return "rejected"
return "unsettled"
def tag_state(track: Dict[str, Any]) -> str:
tag = track.get("price_tag") or track.get("tag") or {}
state = tag.get("state")
if state in TAG_STATES:
return str(state)
if tag:
return "detected"
return "absent"
def score_tracks(tracks: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
"""Denominators over settled decisions. Unsettled tracks are not in the rate."""
counts = {level: 0 for level in LEVELS}
tags = {state: 0 for state in TAG_STATES}
for track in tracks:
counts[identity_level(track)] += 1
tags[tag_state(track)] += 1
settled = counts["accepted"] + counts["ambiguous"] + counts["rejected"] + counts["unknown"]
return {
"tracks": sum(counts.values()),
"settled": settled,
"accepted": counts["accepted"],
"ambiguous": counts["ambiguous"],
"rejected": counts["rejected"],
"unknown": counts["unknown"],
"unsettled": counts["unsettled"],
"accepted_rate": _rate(counts["accepted"], settled),
"ambiguous_rate": _rate(counts["ambiguous"], settled),
"rejected_rate": _rate(counts["rejected"], settled),
"tags_detected": tags["detected"] + tags["readable"] + tags["unreadable"],
"tags_readable": tags["readable"],
"tags_unreadable": tags["unreadable"],
"tags_absent": tags["absent"],
}
def _rate(numerator: int, denominator: int) -> Optional[float]:
if denominator <= 0:
return None
return round(numerator / denominator, 4)
def track_churn(frames: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
"""How often one track id changes the SKU it is willing to name.
A change counts only after the id has already named a SKU. Restarting or
dropping the label (relabels/restarts) is reported beside that, from the
counters LiveSession already publishes.
"""
previous: Dict[int, Optional[str]] = {}
changes: Dict[int, int] = {}
seen: Dict[int, int] = {}
relabels: Dict[int, int] = {}
restarts: Dict[int, int] = {}
for frame in frames:
for track in frame.get("tracks") or []:
track_id = track.get("track_id")
if track_id is None:
continue
track_id = int(track_id)
seen[track_id] = seen.get(track_id, 0) + 1
relabels[track_id] = max(relabels.get(track_id, 0), int(track.get("relabels") or 0))
restarts[track_id] = max(restarts.get(track_id, 0), int(track.get("restarts") or 0))
named = None
if identity_level(track) == "accepted":
named = (track.get("identity") or {}).get("sku_id")
prior = previous.get(track_id)
if prior and named and named != prior:
changes[track_id] = changes.get(track_id, 0) + 1
if named:
previous[track_id] = named
churned = sum(1 for count in changes.values() if count)
return {
"tracks": len(seen),
"tracks_with_sku_change": churned,
"sku_changes": sum(changes.values()),
"relabels": sum(relabels.values()),
"restarts": sum(restarts.values()),
"per_track": [
{
"track_id": track_id,
"observations": seen[track_id],
"sku_changes": changes.get(track_id, 0),
"relabels": relabels.get(track_id, 0),
"restarts": restarts.get(track_id, 0),
}
for track_id in sorted(seen)
if changes.get(track_id, 0) or relabels.get(track_id, 0) or restarts.get(track_id, 0)
],
}
def labelled_top1(
frames: Sequence[Dict[str, Any]],
labels: Sequence[Dict[str, Any]],
*,
iou_min: float,
) -> Dict[str, Any]:
"""Top-1 over held-out labels. Accepted wrong SKU is a miss, not a reject.
Denominator is labels that overlap a track. A label with no overlapping
track is unmatched and stays out of the top-1 rate.
"""
by_frame: Dict[int, List[Dict[str, Any]]] = {}
for frame in frames:
by_frame.setdefault(int(frame.get("frame", frame.get("frame_id"))), []).append(frame)
correct = wrong = rejected = ambiguous = missed = 0
matched = 0
for label in labels:
observations = by_frame.get(int(label["frame"]), [])
best_iou = 0.0
best: Optional[Dict[str, Any]] = None
for observation in observations:
for track in observation.get("tracks") or []:
score = _iou(label["box"], track.get("box") or [])
if score > best_iou:
best_iou = score
best = track
if best is None or best_iou < iou_min:
missed += 1
continue
matched += 1
level = identity_level(best)
guess = (best.get("identity") or {}).get("sku_id")
if level == "accepted" and guess == label["sku_id"]:
correct += 1
elif level == "accepted":
wrong += 1
elif level == "ambiguous":
ambiguous += 1
else:
rejected += 1
return {
"labels": len(labels),
"matched": matched,
"unmatched": missed,
"correct": correct,
"wrong": wrong,
"rejected": rejected,
"ambiguous": ambiguous,
"top1": _rate(correct, matched),
"iou_min": iou_min,
}
def score_recording(
recording_dir: str,
*,
stock_dir: str,
weights_path: str = "",
labels_path: Optional[str] = None,
iou_min: float = 0.5,
live_frames: Optional[Sequence[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""Score one session directory. live_frames, when given, are the replayed results.
Without live_frames the recorded index is what the server already decided.
That still produces denominators; it is not a fresh model run.
"""
recording_dir = os.path.abspath(recording_dir)
rows = load_index(recording_dir)
sample = recording_fingerprint(recording_dir, rows)
labels_file = labels_path or os.path.join(recording_dir, "labels.json")
labels = load_labels(labels_file if os.path.isfile(labels_file) or labels_path else None)
held, leaked = leaked_labels(recording_dir, rows, labels, stock_dir)
source = list(live_frames) if live_frames is not None else rows
tracks = [track for frame in source for track in (frame.get("tracks") or [])]
decisions = score_tracks(tracks)
report = {
"recording": recording_dir,
"source": "live_replay" if live_frames is not None else "recorded_index",
"fingerprints": {
"sample": sample,
"model": model_fingerprint(weights_path),
"catalog": catalog_fingerprint(stock_dir),
},
"decisions": decisions,
"tags": {
"detected": decisions["tags_detected"],
"readable": decisions["tags_readable"],
"unreadable": decisions["tags_unreadable"],
"absent": decisions["tags_absent"],
},
"churn": track_churn(source),
"leakage": {"excluded": len(leaked), "labels": leaked},
"labelled_top1": labelled_top1(source, held, iou_min=iou_min),
"evidence": _evidence(labels, held, leaked, live_frames is not None),
}
return report
def _evidence(
labels: Sequence[Dict[str, Any]],
held: Sequence[Dict[str, Any]],
leaked: Sequence[Dict[str, Any]],
replayed: bool,
) -> List[str]:
gaps: List[str] = []
if not replayed:
gaps.append("counts are from the recorded index, not a fresh LiveSession replay")
if not labels:
gaps.append("no held-out labels; labelled top-1 is not a walk accuracy")
elif not held:
gaps.append("every label matched a catalog template and was excluded")
if leaked:
gaps.append(f"{len(leaked)} template-derived label(s) excluded")
return gaps
def replay_live(
recording_dir: str,
engine: Any,
*,
settings: Optional[Any] = None,
) -> List[Dict[str, Any]]:
"""Feed recorded JPEGs through one LiveSession, in index order.
Classification is asynchronous. Each frame is scored only after its
in-flight crops finish, then processed again so the label is on the
payload. Without that wait the same recording does not replay twice.
"""
from live import LiveSession
rows = load_index(recording_dir)
session = LiveSession("eval", engine, settings=settings)
session.refresh_catalog()
if hasattr(session, "enable_tags"):
session.enable_tags()
frames: List[Dict[str, Any]] = []
for row in rows:
path = os.path.join(recording_dir, str(row["path"]))
with open(path, "rb") as handle:
payload = handle.read()
frame_id = int(row["frame"])
result = session.process_frame(frame_id, payload)
# min_hits is 2, so the first pass only detects. Keep passing the same
# bytes until a pass schedules nothing and everything in flight lands.
for _ in range(4):
_wait_until_settled(session)
result = session.process_frame(frame_id, payload)
if int((session.snapshot().get("stats") or {}).get("classify_pending") or 0) == 0:
break
result["frame"] = frame_id
frames.append(result)
engine_shutdown = getattr(engine, "shutdown", None)
if callable(engine_shutdown):
engine_shutdown()
return frames
def _wait_until_settled(session: Any, timeout: float = 30.0) -> None:
import time
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
pending = int((session.snapshot().get("stats") or {}).get("classify_pending") or 0)
if pending == 0:
return
time.sleep(0.02)
raise EvalInputError("classification did not finish before the replay timeout")
def format_report(report: Dict[str, Any]) -> str:
decisions = report["decisions"]
tags = report["tags"]
top1 = report["labelled_top1"]
churn = report["churn"]
fingerprints = report["fingerprints"]
lines = [
f"source: {report['source']}",
(
f"decisions: accepted {decisions['accepted']} ambiguous {decisions['ambiguous']} "
f"rejected {decisions['rejected']} unknown {decisions['unknown']} "
f"unsettled {decisions['unsettled']} settled {decisions['settled']}"
),
(
f"rates: accepted {decisions['accepted_rate']} "
f"ambiguous {decisions['ambiguous_rate']} rejected {decisions['rejected_rate']} "
f"(denominator settled)"
),
(
f"tags: detected {tags['detected']} readable {tags['readable']} "
f"unreadable {tags['unreadable']} absent {tags['absent']}"
),
(
f"churn: tracks {churn['tracks']} sku_changes {churn['sku_changes']} "
f"relabels {churn['relabels']} restarts {churn['restarts']}"
),
(
f"labelled top-1: {top1['correct']}/{top1['matched']} = {top1['top1']} "
f"wrong {top1['wrong']} unmatched {top1['unmatched']} "
f"leakage excluded {report['leakage']['excluded']}"
),
(
f"fingerprints: sample {fingerprints['sample']['sha256'][:12]} "
f"model {fingerprints['model']['version']} "
f"catalog {fingerprints['catalog']['version']}"
),
]
for gap in report["evidence"]:
lines.append(f"gap: {gap}")
return "\n".join(lines)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description="Score a recorded live walk.")
parser.add_argument("recording", nargs="?", help="session directory with index.jsonl")
parser.add_argument("--stock-dir", default="stock")
parser.add_argument("--weights", default="")
parser.add_argument("--labels", default=None)
parser.add_argument("--json", dest="json_path", default=None)
parser.add_argument("--iou", type=float, default=0.5)
parser.add_argument(
"--replay",
action="store_true",
help="re-run frames through LiveSession (needs the recognition stack)",
)
parser.add_argument("--method", default="hybrid")
args = parser.parse_args(argv)
def fail(message: str) -> int:
print(json.dumps({"error": message}, ensure_ascii=False))
print(f"error: {message}", file=sys.stderr)
return 2
if not args.recording:
return fail("recording directory is required")
recording = os.path.abspath(args.recording)
if not os.path.isdir(recording):
return fail(f"recording directory not found: {recording}")
try:
live_frames = None
weights = args.weights
if args.replay:
from engine import EngineSettings, build_default_engine
from live import LiveSettings
settings = EngineSettings(stock_dir=args.stock_dir)
if weights:
settings.weights_path = weights
engine = build_default_engine(settings)
weights = engine.settings.weights_path
live_frames = replay_live(
recording, engine, settings=LiveSettings(method=args.method, tags=True)
)
report = score_recording(
recording,
stock_dir=args.stock_dir,
weights_path=weights,
labels_path=args.labels,
iou_min=args.iou,
live_frames=live_frames,
)
except EvalInputError as exc:
return fail(str(exc))
text = json.dumps(report, ensure_ascii=False, indent=1, sort_keys=True)
destination = args.json_path or os.path.join(recording, "eval_live.json")
try:
with open(destination, "w", encoding="utf-8") as handle:
handle.write(text + "\n")
except OSError as exc:
return fail(f"cannot write {destination}: {exc.strerror}")
print(format_report(report))
print(f"json: {destination}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

6
recognization/code/fast_test.py
View file

@ -1,6 +0,0 @@
from ultralytics import YOLO
model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt")
image_path = "/Users/nguyenhongvan/recongnize/data/add/image7.jpg"
results = model(image_path)
results[0].show()

96
recognization/code/finetune_identity.py
View file

@ -49,6 +49,88 @@ MILK_SHELF_LABELS = {
}
def enrollment_report(stock: str, csv_name: str = "stock.csv") -> dict:
"""Catalog versus enrolled views. Does not load a model or write files.
A SKU with no readable view cannot be recognized. One view is reported, not
treated as enough: the measured knee is two views. Duplicate CSV rows and a
head whose class list disagrees with the catalog are blockers.
"""
import csv
import re
from gallery import ProductGallery
names = []
try:
names = os.listdir(stock)
except OSError:
names = []
images = [name for name in names if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")]
rows = []
csv_path = os.path.join(stock, csv_name)
if os.path.exists(csv_path):
with open(csv_path, encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
sku_ids = []
seen = set()
duplicate_rows = []
for row in rows:
sku_id = (row.get("SKU_ID") or "").strip()
if not sku_id:
continue
if sku_id in seen:
duplicate_rows.append(sku_id)
seen.add(sku_id)
sku_ids.append(sku_id)
gallery = ProductGallery(stock, sku_ids or [])
views = {sku_id: gallery._views_for(sku_id) for sku_id in dict.fromkeys(sku_ids)}
missing = [sku_id for sku_id, paths in views.items() if not paths]
single = [sku_id for sku_id, paths in views.items() if len(paths) == 1]
claimed = {os.path.basename(path) for paths in views.values() for path in paths}
orphans = sorted(name for name in images if name not in claimed)
# ponytail: prefix collision check. A view named sku_001 must not also match sku_00.
collisions = []
for sku_id, paths in views.items():
bare = [os.path.basename(path) for path in paths if os.path.splitext(os.path.basename(path))[0] != sku_id
and not re.fullmatch(re.escape(sku_id) + r"[_-]\d+", os.path.splitext(os.path.basename(path))[0])]
if bare:
collisions.append(sku_id)
blockers = []
if duplicate_rows:
blockers.append("duplicate_csv_sku")
if missing:
blockers.append("sku_without_view")
if collisions:
blockers.append("view_prefix_collision")
head_path = os.path.join(stock, "identity_head.pt")
head_note = None
if os.path.exists(head_path):
head_note = "present_not_opened"
# Do not torch.load here: the head can be large, and this report must stay cheap.
# Class-list drift is reported when the caller passes class_ids.
return {
"stock": stock,
"sku_ids": list(dict.fromkeys(sku_ids)),
"images": len(images),
"missing": missing,
"single_view": single,
"orphans": orphans,
"duplicate_rows": duplicate_rows,
"collisions": collisions,
"blockers": blockers,
"head": head_note,
}
def enrollment_report_text(report: dict) -> str:
return (
f"{report['stock']}: {len(report['sku_ids'])} SKUs, {report['images']} images, "
f"missing={len(report['missing'])}, single_view={len(report['single_view'])}, "
f"orphans={len(report['orphans'])}, blockers={report['blockers'] or 'none'}"
)
def build_model(image_size: int = 518):
import timm
import torch
@ -149,13 +231,13 @@ def train_from_stock(args) -> None:
from gallery import ProductGallery
stock = args.stock
gallery = ProductGallery(stock, [])
# the SKUs are whatever the stock directory holds
skus = sorted({os.path.splitext(name)[0].rsplit("_", 1)[0] if os.path.splitext(name)[0].rsplit("_", 1)[-1].isdigit()
else os.path.splitext(name)[0]
for name in os.listdir(stock)
if os.path.splitext(name)[1].lower() in (".jpeg", ".jpg", ".png")})
views = {sku: gallery._views_for(sku) for sku in skus}
report = enrollment_report(stock)
print(enrollment_report_text(report))
if report["blockers"]:
print(f"{stock}: not training; enrollment blockers: {', '.join(report['blockers'])}")
return
gallery = ProductGallery(stock, report["sku_ids"])
views = {sku: gallery._views_for(sku) for sku in report["sku_ids"]}
views = {sku: paths for sku, paths in views.items() if paths}
if len(views) < 2:
print(f"{stock}: needs at least two products with views, found {len(views)}")

88
recognization/code/gallery.py
View file

@ -26,6 +26,7 @@ on the GPU. That is why this can sit in the live loop at all.
from __future__ import annotations
import os
import re
import threading
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
@ -93,23 +94,39 @@ class ProductGallery:
# ------------------------------------------------------------------ loading
def load_head(self) -> bool:
"""Load the store's fine-tuned head, if it has one."""
"""Load the store's fine-tuned head, if it has one.
weights_only: the file is tensors and strings, not executable code.
A payload that is not that head leaves the raw gallery in place.
"""
import torch
if not os.path.exists(self._head_path):
return False
try:
payload = torch.load(self._head_path, map_location="cpu", weights_only=False)
payload = torch.load(self._head_path, map_location="cpu", weights_only=True)
if not isinstance(payload, dict):
return False
projection_state = payload.get("projection")
prototypes = payload.get("prototypes")
class_ids = payload.get("class_ids")
if not isinstance(projection_state, dict) or not torch.is_tensor(prototypes):
return False
if not isinstance(class_ids, (list, tuple)) or prototypes.ndim != 2:
return False
if prototypes.shape[0] != len(class_ids):
return False
projection = torch.nn.Linear(384, 128)
projection.load_state_dict(projection_state)
projection.eval()
self._head = {
"projection": projection,
"prototypes": torch.nn.functional.normalize(prototypes.float(), dim=1),
"class_ids": [str(sku_id) for sku_id in class_ids],
}
except Exception:
self._head = None
return False
projection = torch.nn.Linear(384, 128)
projection.load_state_dict(payload["projection"])
projection.eval()
self._head = {
"projection": projection,
"prototypes": torch.nn.functional.normalize(payload["prototypes"].float(), dim=1),
"class_ids": list(payload["class_ids"]),
}
return True
def _load_model(self) -> Any:
@ -137,8 +154,12 @@ class ProductGallery:
return self._available
def _views_for(self, sku_id: str) -> List[str]:
"""Every image in the stock directory that belongs to this SKU."""
"""Images whose stem is the SKU or SKU plus _N / -N.
A bare prefix would enroll sku_001 as a view of sku_00.
"""
found: List[str] = []
extra = re.compile(rf"{re.escape(sku_id)}[_-]\d+$")
try:
names = sorted(os.listdir(self.stock_dir))
except OSError:
@ -147,42 +168,50 @@ class ProductGallery:
stem, extension = os.path.splitext(name)
if extension.lower() not in (".jpeg", ".jpg", ".png"):
continue
if stem == sku_id or stem.startswith(f"{sku_id}_") or stem.startswith(f"{sku_id}-"):
if stem == sku_id or extra.fullmatch(stem):
found.append(os.path.join(self.stock_dir, name))
return found
def load(self) -> bool:
"""Embed every enrolled view. Returns False when there is nothing to load."""
"""Embed every enrolled view. Returns False when there is nothing to load.
One lock covers the whole load: two callers cannot each embed the catalog.
"""
with self._lock:
if self._available and self._matrix is not None:
return True
return self._load_locked()
def _load_locked(self) -> bool:
import cv2
import numpy as np
import torch
paths: List[Tuple[str, str]] = []
kept: List[Any] = []
for sku_id in self.sku_ids:
for path in self._views_for(sku_id):
image = cv2.imread(path)
if image is None or image.size == 0:
continue
paths.append((sku_id, path))
if not paths:
kept.append(_prepare(image, self.image_size))
if not kept:
self._available = False
self._entries = []
self._matrix = None
return False
try:
model = self._load_model()
tensor = torch.from_numpy(np.stack(kept)).to(self.device)
with torch.no_grad():
features = model(tensor)
features = torch.nn.functional.normalize(features, dim=1).cpu().numpy()
except Exception:
self._available = False
self._entries = []
self._matrix = None
return False
batch = []
for _, path in paths:
image = cv2.imread(path)
if image is None or image.size == 0:
continue
batch.append(_prepare(image, self.image_size))
if not batch:
self._available = False
return False
tensor = torch.from_numpy(np.stack(batch)).to(self.device)
with torch.no_grad():
features = model(tensor)
features = torch.nn.functional.normalize(features, dim=1).cpu().numpy()
self._entries = [
GalleryEntry(sku_id=sku_id, view=path, embedding=features[index])
for index, (sku_id, path) in enumerate(paths)
@ -209,13 +238,14 @@ class ProductGallery:
if not self._available or self._matrix is None:
return GalleryMatch(sku_id=None, similarity=0.0, rejected=True)
import numpy as np
import torch
limit = threshold
if limit is None:
limit = self.head_threshold if self._head is not None else self.threshold
limit = float(limit)
embedding = self.embed(crop)
if self._head is not None:
import torch
if self._head is not None:
# Two stages, because the two mechanisms fail in opposite directions.
# Measured: raw features reject strangers well (5 % false accepts) but

92
recognization/code/gui_app.py
View file

@ -1,92 +0,0 @@
"""
Native Desktop GUI for Supermarket Shelf Product Recognition
Opens a desktop OpenCV window displaying image.jpg with bounding boxes only.
Users can click on any product directly on screen to inspect price, stock,
and feature matching inliers in real-time.
"""
import os
import sys
import cv2
import numpy as np
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from pipeline import ShelfRecognitionPipeline
def main():
print("=" * 60)
print("STARTING NATIVE DESKTOP INTERFACE")
print("Image: image.jpg | Mode: Hybrid (Paper Method)")
print("Instructions: Click on ANY product to identify & view info.")
print("Press 'q' or 'ESC' on the window to exit.")
print("=" * 60)
pipeline = ShelfRecognitionPipeline()
init_res = pipeline.set_active_shelf_image("image.jpg")
if init_res.get("status") != "success":
print("[Error] Failed to load image.jpg")
return
boxes = init_res["boxes"]
base_image = pipeline.current_image.copy()
# Draw ONLY bounding boxes (no class names, no confidence labels)
display_image = pipeline.detector.draw_detections(base_image, boxes, draw_labels=False)
window_name = "Smart Shelf - Click Any Product (Press Q to quit)"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
cv2.resizeWindow(window_name, 1280, 850)
def on_mouse(event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
print(f"\n[Click detected at coordinates: ({x}, {y})]")
res = pipeline.process_click(x, y, method="hybrid")
if res.get("status") == "success":
prod = res["inventory"]
cls_info = res["classification"]
box_info = res["target_box"]
print("-" * 55)
print(f"📦 Product Identified : {prod['product_name']}")
print(f"🏷️ SKU ID : {prod['sku_id']}")
print(f"💰 Price : ¥{prod['price']}")
print(f"📊 In Stock (On-Shelf): {prod['number_on']} units")
print(f"🚚 Received / Sold : {prod['number_in']} in / {prod['number_out']} out")
print(f"📅 Expiration Date : {prod['expiration_date']}")
print(f"🎯 Match Confidence : {cls_info['confidence'] * 100:.1f}% (Score: {cls_info['score']})")
print("-" * 55)
# Update main window with selected box highlight (in red)
selected_display = pipeline.detector.draw_detections(
base_image, boxes, selected_box_id=box_info["id"], draw_labels=False
)
cv2.imshow(window_name, selected_display)
# Show visual match comparison window if available
if res.get("match_visualization") is not None:
match_win = "Feature Matcher (Shelf Crop vs Stock Template)"
cv2.namedWindow(match_win, cv2.WINDOW_NORMAL)
cv2.resizeWindow(match_win, 800, 400)
cv2.imshow(match_win, res["match_visualization"])
else:
print(f"[Info] {res.get('message', 'No product found at coordinates.')}")
cv2.setMouseCallback(window_name, on_mouse)
while True:
cv2.imshow(window_name, display_image)
key = cv2.waitKey(20) & 0xFF
if key in [ord('q'), ord('Q'), 27]:
break
cv2.destroyAllWindows()
if __name__ == "__main__":
main()

167
recognization/code/identity.py
View file

@ -25,11 +25,126 @@ def relative_confidence(top_score: float, runner_up_score: float) -> float:
return round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4)
# The gallery's scale is a cosine similarity, where same-product pairs sit above 0.6
# and different products below 0.5 - not an inlier count, which has no such meaning.
# Cosine scale (gallery). Same-product pairs were measured above 0.6 and
# different products below 0.5. These are not a new calibration.
COSINE_ACCEPT = 0.60
COSINE_AMBIGUOUS = 0.50
COSINE_MARGIN = 0.04
VOTE_CAP = 3
def _finite(value: Any) -> Optional[float]:
try:
number = float(value)
except (TypeError, ValueError):
return None
if number != number or number in (float("inf"), float("-inf")):
return None
return number
def aggregate_votes(
observations: Sequence[Dict[str, float]],
*,
cap: int = VOTE_CAP,
) -> Dict[str, float]:
"""Mean of the newest observations, at most `cap`.
Older frames are dropped, not averaged in. An empty observation still
counts, so a later miss can outvote an earlier weak hit. Inlier and
cosine scores must not be mixed in one list.
"""
window = list(observations)[-max(1, int(cap)):]
if not window:
return {}
totals: Dict[str, float] = {}
for scores in window:
for sku_id, score in (scores or {}).items():
number = _finite(score)
if number is None:
continue
totals[str(sku_id)] = totals.get(str(sku_id), 0.0) + number
count = len(window)
return {sku_id: total / count for sku_id, total in totals.items()}
def _cosine_decision(
sorted_candidates: List[Tuple[str, float]],
*,
method: str,
details: Optional[Dict[str, Any]],
accept: float,
ambiguous: float,
margin_min: float,
) -> Dict[str, Any]:
top_sku, top_score = sorted_candidates[0]
runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0
margin = top_score - runner_score
if top_score >= accept and margin >= margin_min:
level = IDENTITY_ACCEPTED
elif top_score >= ambiguous:
level = IDENTITY_AMBIGUOUS
else:
level = IDENTITY_REJECTED
return {
"status": level,
"method": method,
"best_sku": top_sku if level == IDENTITY_ACCEPTED else None,
"score": float(top_score),
"confidence": float(top_score),
"identity_level": level,
"ranked_candidates": [
{"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3]
],
"details": details or {},
"message": None,
}
def decision_for_match(
match: Any,
*,
method: str = "gallery",
margin: float = COSINE_MARGIN,
) -> Dict[str, Any]:
"""Rank one gallery match. Rejection is the gallery's, not a second threshold.
`match()` already applied the raw 0.55 line or the head's 0.40 line. Re-ranking
those scores against COSINE_ACCEPT (0.60) would refuse a head match the gallery
accepted. The only extra gate here is the published cosine margin.
"""
similarity = _finite(getattr(match, "similarity", 0.0)) or 0.0
if match is None or getattr(match, "rejected", False) or not getattr(match, "sku_id", None):
return {
"status": IDENTITY_REJECTED,
"method": method,
"best_sku": None,
"score": similarity,
"confidence": similarity,
"identity_level": IDENTITY_REJECTED,
"ranked_candidates": [],
"details": {},
"message": "no enrolled product is close enough",
}
runner = getattr(match, "runner_up", None)
runner_score = _finite(getattr(match, "runner_up_similarity", 0.0)) or 0.0
gap = similarity - runner_score
close = bool(runner) and gap < margin
level = IDENTITY_AMBIGUOUS if close else IDENTITY_ACCEPTED
ranked = [{"sku_id": str(match.sku_id), "score": similarity}]
if runner:
ranked.append({"sku_id": str(runner), "score": runner_score})
return {
"status": level,
"method": method,
"best_sku": None if close else str(match.sku_id),
"score": similarity,
"confidence": similarity,
"identity_level": level,
"ranked_candidates": ranked,
"details": {},
"message": None,
}
def decide_identity(
@ -48,7 +163,20 @@ def decide_identity(
unknown. Two close scores are ambiguous. Only a unique, above-threshold
score is accepted.
"""
if not scores:
if scale == "cosine":
if min_score == DEFAULT_MIN_SCORE:
min_score = COSINE_ACCEPT
if min_margin == DEFAULT_MIN_MARGIN:
min_margin = COSINE_MARGIN
if min_confidence == DEFAULT_MIN_CONFIDENCE:
min_confidence = COSINE_AMBIGUOUS
cleaned: Dict[str, float] = {}
for sku_id, score in (scores or {}).items():
number = _finite(score)
if number is None:
continue
cleaned[str(sku_id)] = number
if not cleaned:
return {
"status": IDENTITY_REJECTED,
"method": method,
@ -62,32 +190,17 @@ def decide_identity(
}
sorted_candidates: List[Tuple[str, float]] = sorted(
scores.items(), key=lambda x: x[1], reverse=True
cleaned.items(), key=lambda item: item[1], reverse=True
)
if scale == "cosine":
# Thresholds that mean something on a similarity scale.
top_sku, top_score = sorted_candidates[0]
runner_score = sorted_candidates[1][1] if len(sorted_candidates) > 1 else 0.0
margin = top_score - runner_score
if top_score >= COSINE_ACCEPT and margin >= COSINE_MARGIN:
level = IDENTITY_ACCEPTED
elif top_score >= COSINE_AMBIGUOUS:
level = IDENTITY_AMBIGUOUS
else:
level = IDENTITY_REJECTED
return {
"status": level,
"method": method,
"best_sku": top_sku if level == IDENTITY_ACCEPTED else None,
"score": float(top_score),
"confidence": float(top_score),
"identity_level": level,
"ranked_candidates": [
{"sku_id": sku, "score": float(value)} for sku, value in sorted_candidates[:3]
],
"details": details or {},
"message": None,
}
return _cosine_decision(
sorted_candidates,
method=method,
details=details,
accept=min_score,
ambiguous=min_confidence,
margin_min=min_margin,
)
top_sku, top_score = sorted_candidates[0]
runner_up_sku, runner_up_score = (
sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0)

133
recognization/code/live.py
View file

@ -96,8 +96,9 @@ class TrackState:
last_attempt_area: int = 0
score_sums: Dict[str, float] = field(default_factory=dict)
score_n: int = 0
identity: Optional[Dict[str, Any]] = None
observations: List[Dict[str, float]] = field(default_factory=list)
candidates: List[Dict[str, Any]] = field(default_factory=list)
identity: Optional[Dict[str, Any]] = None
confirmed: bool = False
label_box: List[int] = field(default_factory=list)
drift_frames: int = 0
@ -184,6 +185,10 @@ class LiveSession:
self._catalog_by_sku: Dict[str, Dict[str, Any]] = {}
self._catalog_version = ""
self._tags = None
# Shelf position of this frame's origin. Updated from a content shift
# before the session lock; tag cadence keys rows by it. Not a shelf map.
self._frame_origin = [0.0, 0.0]
self._shift_map = None
self._compliance_tracker = None
self._last_compliance: Dict[str, Any] = {"violations": 0, "by_rule": {}, "rows": []}
self._label_memory: "collections.deque[Dict[str, Any]]" = collections.deque(
@ -256,7 +261,7 @@ class LiveSession:
track.identity = None
track.candidates = []
track.score_sums = {}
track.score_n = 0
track.observations = []
track.attempts = 0
track.in_flight = 0
track.last_attempt_seq = 0
@ -286,6 +291,8 @@ class LiveSession:
self._slots = []
self._latest_image = None
self._latest_frame_id = None
self._frame_origin = [0.0, 0.0]
self._shift_map = None
# ------------------------------------------------------------------ frame path
def process_frame(self, frame_id: int, payload: bytes) -> Dict[str, Any]:
@ -300,6 +307,9 @@ class LiveSession:
empty_enabled = self.settings.empty_slots
generation = self._generation
shelf_offset = self._note_frame_shift(image)
boxes = self.engine.detect(image, conf=conf)
detected_at = time.perf_counter()
tracks = self.tracker.update(
@ -327,7 +337,13 @@ class LiveSession:
raw_box = list(detection["box"])
confidence = float(detection["confidence"])
if raw_box is None:
raw_box = list(track.box)
# Low-score second pass, or a Kalman box with no detection.
# Keep the last measured crop. Do not invent one from the prediction.
raw_box = list(state.raw_box) if state is not None and state.raw_box else []
confidence = float(state.confidence) if state is not None else confidence
low_score = bool(getattr(track, "low_score", False)) or (
track.det_index is None and raw_box == []
)
previous_seq = self._last_seen_seq.get(track.track_id)
if state is None:
# An id that comes back after an absence may be a recycled id:
@ -369,8 +385,12 @@ class LiveSession:
state.last_seq = seq
self._last_seen_seq[track.track_id] = seq
state.box = list(track.box)
state.raw_box = raw_box
state.confidence = confidence
if raw_box and not low_score:
state.raw_box = raw_box
state.confidence = confidence
elif not state.raw_box and raw_box:
state.raw_box = raw_box
state.confidence = confidence
self._drop_stale_label(state)
active[track.track_id] = state
self._tracks = active
@ -405,7 +425,7 @@ class LiveSession:
for state in self._tracks.values()
if state.box
]
tag_bands = self._tags.plan(tag_boxes, image.shape)
tag_bands = self._tags.plan(tag_boxes, image.shape, shelf_offset=shelf_offset)
# Collect the classification work under the lock; submit it after
# releasing it. add_done_callback can run the callback inline when the
@ -443,39 +463,44 @@ class LiveSession:
}
return result
def _inherit_labels(self, seq: int) -> None:
"""Give a fresh track the label of a recently recognized product it covers.
The same physical product can be tracked as several ids while the camera
pans. Remembering where a label was measured means each product is
classified once and its neighbours inherit the result instead of waiting
in the classification queue again.
Only a measured label, still overlapping its own box, may be copied.
An inherited label is not a measurement, so it cannot spread to a neighbour.
Blur, glare, and a too-far frame skip inheritance: the overlap is not
evidence when the frame itself failed the quality gate.
"""
if not self._label_memory:
return
if (self._last_quality or {}).get("hint"):
return
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.confirmed:
continue
for entry in reversed(self._label_memory):
if seq - entry["seq"] > LABEL_MEMORY_FRAMES:
break
if _iou(track.box, entry["box"]) >= LABEL_MEMORY_IOU:
track.state = STATE_RECOGNIZED
track.confirmed = True
track.label_box = list(track.box)
track.identity = {
"sku_id": entry["sku_id"],
"confidence": entry["confidence"],
"score": entry["score"],
"attempts": 0,
"inherited": True,
}
# An inherited label is a guess from the same place, not a
# measurement of this object: count it as re-verified so a
# later correction is auditable rather than a silent flip.
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
self._relabel_count[track.track_id] = track.relabels
break
if entry.get("inherited"):
continue
if _iou(entry["box"], entry.get("label_box") or entry["box"]) < DRIFT_IOU:
continue
if _iou(track.box, entry["box"]) < LABEL_MEMORY_IOU:
continue
track.state = STATE_RECOGNIZED
track.confirmed = True
track.label_box = list(track.box)
track.identity = {
"sku_id": entry["sku_id"],
"confidence": entry["confidence"],
"score": entry["score"],
"attempts": 0,
"inherited": True,
}
track.relabels = self._relabel_count.get(track.track_id, 0) + 1
self._relabel_count[track.track_id] = track.relabels
break
def _remember_label(self, track: TrackState, box: List[int], seq: int) -> None:
"""Must be called with the session lock held."""
@ -484,10 +509,12 @@ class LiveSession:
self._label_memory.append(
{
"box": list(box),
"label_box": list(track.label_box or box),
"sku_id": track.identity.get("sku_id"),
"confidence": track.identity.get("confidence"),
"score": track.identity.get("score"),
"seq": seq,
"inherited": bool(track.identity.get("inherited")),
}
)
@ -516,7 +543,7 @@ class LiveSession:
track.identity = None
track.candidates = []
track.score_sums = {}
track.score_n = 0
track.observations = []
track.attempts = 0
track.in_flight = 0
track.last_attempt_seq = 0
@ -539,11 +566,16 @@ class LiveSession:
return []
eligible: List[TrackState] = []
poor_frame = bool((self._last_quality or {}).get("hint"))
for track in self._tracks.values():
if track.state != STATE_DETECTED or track.in_flight > 0:
continue
if poor_frame:
continue
if track.hits < self.settings.min_hits:
continue
if len(track.raw_box) != 4:
continue
x1, y1, x2, y2 = track.raw_box
if min(x2 - x1, y2 - y1) < self.settings.min_crop_px:
continue
@ -563,9 +595,8 @@ class LiveSession:
eligible.append(track)
# Newest first: while a camera pans, the products still worth labelling
# are the ones that just entered the frame. Sorting by area (the obvious
# default) spends the budget on products that are already leaving.
eligible.sort(key=lambda track: (track.first_seq, -track.area()))
# are the ones that just entered the frame. Area is only the tie-break.
eligible.sort(key=lambda track: (-track.first_seq, -track.area()))
planned: List[Tuple[TrackState, Any, int]] = []
for track in eligible[:capacity]:
crop = self.engine.crop(image, track.raw_box)
@ -623,13 +654,18 @@ class LiveSession:
return
track.in_flight = max(0, track.in_flight - 1)
track.attempts += 1
for sku_id, score in (scores or {}).items():
observation = {
str(sku_id): float(score) for sku_id, score in (scores or {}).items()
}
track.observations.append(observation)
track.observations = track.observations[-3:]
for sku_id, score in observation.items():
track.score_sums[sku_id] = track.score_sums.get(sku_id, 0.0) + float(score)
if scores:
if observation:
track.score_n += 1
self._classify_times.append(finished_at - submitted_at)
if not scores:
if not any(track.observations):
if track.attempts >= self.settings.max_attempts:
track.state = STATE_UNRECOGNIZED
track.confirmed = True
@ -638,11 +674,9 @@ class LiveSession:
track.candidates = []
return
from identity import decide_identity
from identity import aggregate_votes, decide_identity
mean_scores = {
sku_id: total / max(1, track.score_n) for sku_id, total in track.score_sums.items()
}
mean_scores = aggregate_votes(track.observations)
decision = decide_identity(
mean_scores, method=method, scale=getattr(self.engine, "identity_scale", "inlier")
)
@ -838,7 +872,7 @@ class LiveSession:
gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY)
sharpness = float(cv2.Laplacian(gray, cv2.CV_64F).var())
glare = float(np.mean(np.all(small > self.settings.glare_pixel, axis=2)))
heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if state.raw_box]
heights = [state.raw_box[3] - state.raw_box[1] for state in tracks.values() if len(state.raw_box) == 4]
median_height = float(np.median(heights)) if heights else 0.0
too_far = bool(
heights
@ -847,7 +881,8 @@ class LiveSession:
or sum(1 for h in heights if h < self.settings.min_crop_px) > len(heights) / 2
)
)
blur = sharpness < self._blur_threshold()
# A black frame has Laplacian 0. That is not blur; a real shelf is not black.
blur = sharpness < self._blur_threshold() and float(gray.mean()) > 1.0
hint = None
if glare > self.settings.glare_fraction:
hint = "glare"
@ -898,6 +933,24 @@ class LiveSession:
self._tags = TagService(ocr_engine, settings=settings or TagSettings())
return True
def _note_frame_shift(self, image: Any) -> Tuple[float, float]:
"""Shelf position of this frame's origin. Phase correlation only.
Runs before the session lock. A failed registration keeps the last origin
rather than inventing a jump. The content shift moves the origin the
other way, same sign as ShelfMap.
"""
if self._shift_map is None:
from shelf_map import ShelfMap
self._shift_map = ShelfMap()
shift = self._shift_map.register(image)
with self._lock:
if shift is not None:
self._frame_origin[0] -= float(shift[0])
self._frame_origin[1] -= float(shift[1])
return (self._frame_origin[0], self._frame_origin[1])
def _run_tag_passes(self, image: Any, bands: Sequence[Any]) -> None:
"""Submit tag passes to the engine's tag worker (never the frame loop)."""
service = self._tags

0
recognization/code/orb.py
View file

5
recognization/code/stream_client.py
View file

@ -124,8 +124,7 @@ def main() -> None:
args = parser.parse_args()
url = args.url
if args.token and "token=" not in url:
url = f"{url}{'&' if '?' in url else '?'}token={args.token}"
headers = {"Authorization": f"Bearer {args.token}"} if args.token else None
source = FrameSource(args.source, width=args.width, pan_step=args.pan)
rtts: List[float] = []
@ -147,7 +146,7 @@ def main() -> None:
in_flight: Dict[int, float] = {}
pending_frame: Optional[int] = None
with connect(url, max_size=8 * 1024 * 1024) as socket:
with connect(url, max_size=8 * 1024 * 1024, additional_headers=headers) as socket:
hello = json.loads(socket.recv(timeout=10))
if hello.get("type") != "hello":
raise SystemExit(f"unexpected first message: {hello}")

84
recognization/code/suft.py
View file

@ -1,84 +0,0 @@
import cv2
import matplotlib.pyplot as plt
import numpy as np
def suft_function(image):
image1 = cv2.imread(image)
# Convert the training image to RGB
training_image = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
# Convert the training image to gray scale
training_gray = cv2.cvtColor(training_image, cv2.COLOR_RGB2GRAY)
# Create test image by adding Scale Invariance and Rotational Invariance
test_image = cv2.pyrDown(training_image)
test_image = cv2.pyrDown(test_image)
num_rows, num_cols = test_image.shape[:2]
rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), 30, 1)
test_image = cv2.warpAffine(test_image, rotation_matrix, (num_cols, num_rows))
test_gray = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY)
# Display traning image and testing image
fx, plots = plt.subplots(1, 2, figsize=(20,10))
plots[0].set_title("Training Image")
plots[0].imshow(training_image)
plots[1].set_title("Testing Image")
plots[1].imshow(test_image)
# ## Detect keypoints and Create Descriptor
surf = cv2.xfeatures2d.SURF_create(800)
train_keypoints, train_descriptor = surf.detectAndCompute(training_gray, None)
test_keypoints, test_descriptor = surf.detectAndCompute(test_gray, None)
keypoints_without_size = np.copy(training_image)
keypoints_with_size = np.copy(training_image)
cv2.drawKeypoints(training_image, train_keypoints, keypoints_without_size, color = (0, 255, 0))
cv2.drawKeypoints(training_image, train_keypoints, keypoints_with_size, flags = cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
# Display image with and without keypoints size
fx, plots = plt.subplots(1, 2, figsize=(20,10))
plots[0].set_title("Train keypoints With Size")
plots[0].imshow(keypoints_with_size, cmap='gray')
plots[1].set_title("Train keypoints Without Size")
plots[1].imshow(keypoints_without_size, cmap='gray')
# Print the number of keypoints detected in the training image
print("Number of Keypoints Detected In The Training Image: ", len(train_keypoints))
# Print the number of keypoints detected in the query image
print("Number of Keypoints Detected In The Query Image: ", len(test_keypoints))
# ## Matching Keypoints
# Create a Brute Force Matcher object.
bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck = False)
# Perform the matching between the SURF descriptors of the training image and the test image
matches = bf.match(train_descriptor, test_descriptor)
# The matches with shorter distance are the ones we want.
matches = sorted(matches, key = lambda x : x.distance)
result = cv2.drawMatches(training_image, train_keypoints, test_gray, test_keypoints, matches, test_gray, flags = 2)
# Display the best matching points
plt.rcParams['figure.figsize'] = [14.0, 7.0]
plt.title('Best Matching Points')
plt.imshow(result)
plt.show()
# Print total number of matching points between the training and query images
print("\nNumber of Matching Keypoints Between The Training and Query Images: ", len(matches))
return keypoints_with_size

140
recognization/code/tag_cards.py
View file

@ -29,19 +29,116 @@ from the small one and marked as derived rather than measured.
from __future__ import annotations
import json
import math
import os
import re
from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
# The consumption tax used by the tags in the demo images. A tag whose small
# price is exactly 1.08x its big price is self-consistent; other stores and eras
# use 1.10 or a per-100g unit price instead, so this is a check, never a rule.
TAX_RATIOS = (1.08, 1.10)
# Defaults when a store has no store.json. Measured on the demo LOPIA tags:
# 8% consumption tax, brick-red rails, and shelf prices from ¥50. A store file
# replaces these for that store only.
DEFAULT_TAX_RATIOS = (1.08, 1.10)
DEFAULT_MIN_PRICE = 50
# OpenCV hue is 0-180. Brick red wraps past 0, so a pixel is rail-coloured when
# its hue is below hue_low or above hue_high.
DEFAULT_RAIL_HSV = {"hue_low": 14, "hue_high": 166, "sat_min": 70, "val_min": 70}
# Aliases the parser reads. load_store_policy() republishes them per store.
TAX_RATIOS = DEFAULT_TAX_RATIOS
MIN_PRICE = DEFAULT_MIN_PRICE
_RAIL: Optional["StorePolicy"] = None
def _remember_rail(policy: "StorePolicy") -> None:
global _RAIL
_RAIL = policy
class StorePolicyError(ValueError):
"""store.json exists but cannot be used. Never borrow another store's file."""
@dataclass(frozen=True)
class StorePolicy:
tax_ratios: Tuple[float, ...] = DEFAULT_TAX_RATIOS
min_price: int = DEFAULT_MIN_PRICE
rail_hue_low: int = 14
rail_hue_high: int = 166
rail_sat_min: int = 70
rail_val_min: int = 70
store_id: Optional[str] = None
def active(self) -> None:
global TAX_RATIOS, MIN_PRICE
TAX_RATIOS = self.tax_ratios
MIN_PRICE = self.min_price
_remember_rail(self)
def default_policy() -> StorePolicy:
policy = StorePolicy()
policy.active()
return policy
def _as_ratio(value: Any) -> float:
number = float(value)
if not 1.0 < number <= 1.5:
raise StorePolicyError(f"tax ratio {value!r} is outside 1-1.5")
return number
def _as_bound(value: Any, name: str, low: int, high: int) -> int:
number = int(value)
if not low <= number <= high:
raise StorePolicyError(f"{name} {value!r} is outside {low}-{high}")
return number
def load_store_policy(stock_dir: Optional[str]) -> StorePolicy:
"""Read `<stock_dir>/store.json`. A missing file uses the defaults.
A file that is present but invalid raises StorePolicyError naming that path.
"""
if not stock_dir:
return StorePolicy()
path = os.path.join(stock_dir, "store.json")
if not os.path.isfile(path):
return StorePolicy()
try:
with open(path, encoding="utf-8") as handle:
raw = json.load(handle)
except (OSError, json.JSONDecodeError) as exc:
raise StorePolicyError(f"{path}: {exc}") from exc
if not isinstance(raw, dict):
raise StorePolicyError(f"{path}: expected an object")
try:
ratios = raw.get("tax_ratios", list(DEFAULT_TAX_RATIOS))
if not isinstance(ratios, list) or not ratios:
raise StorePolicyError(f"{path}: tax_ratios must be a non-empty list")
rail = raw.get("rail_hsv", DEFAULT_RAIL_HSV)
if not isinstance(rail, dict):
raise StorePolicyError(f"{path}: rail_hsv must be an object")
policy = StorePolicy(
tax_ratios=tuple(_as_ratio(item) for item in ratios),
min_price=_as_bound(raw.get("min_price", DEFAULT_MIN_PRICE), "min_price", 1, 100000),
rail_hue_low=_as_bound(rail.get("hue_low", 14), "hue_low", 0, 180),
rail_hue_high=_as_bound(rail.get("hue_high", 166), "hue_high", 0, 180),
rail_sat_min=_as_bound(rail.get("sat_min", 70), "sat_min", 0, 255),
rail_val_min=_as_bound(rail.get("val_min", 70), "val_min", 0, 255),
store_id=str(raw["store_id"]) if raw.get("store_id") else None,
)
except (TypeError, ValueError) as exc:
if isinstance(exc, StorePolicyError):
raise
raise StorePolicyError(f"{path}: {exc}") from exc
policy.active()
return policy
HEADER_PATTERN = re.compile(r"お買得|お買い得|特価|価格|セール")
# A shelf price below this is not a price (measured live reads included 20 and 23).
MIN_PRICE = 50
# The tag truncates its discount to a tenth of a 割, so agreement is one step.
DISCOUNT_TOLERANCE = 0.05
# A regular price above this multiple of the sale price is a misread, not a discount.
@ -201,22 +298,33 @@ def cluster_rows(lines: Sequence[OcrLine], max_gap: int = 26) -> List[List[OcrLi
return rows
def tag_row_score(row: Sequence[OcrLine]) -> float:
def tag_row_score(row: Sequence[OcrLine], header_pattern: Any = None) -> float:
"""How much a row looks like a line of price tags.
Tags carry a header, a discount, and a decimal (tax-included) price; product
packaging rarely carries all three on one line.
packaging rarely carries all three on one line. The header pattern is bound
here: a missing module global must not fail the tag pass.
"""
def pattern(name: str, fallback: str) -> Any:
found = globals().get(name)
return found if found is not None else re.compile(fallback)
header = header_pattern if header_pattern is not None else pattern(
"HEADER_PATTERN", r"お買得|お買い得|特価|価格|セール"
)
discount = pattern("DISCOUNT_PATTERN", r"(\d+(?:\.\d)?)\s*割\s*引")
decimal = pattern("DECIMAL_PATTERN", r"(\d{1,4})[.,](\d{2})(?!\d)")
jan = pattern("JAN3_PATTERN", r"(?<!\d)(\d{3})(?!\d)")
score = 0.0
for line in row:
text = line.text
if HEADER_PATTERN.search(text):
if header.search(text):
score += 2.0
if DISCOUNT_PATTERN.search(text):
if discount.search(text):
score += 1.5
if DECIMAL_PATTERN.search(text):
if decimal.search(text):
score += 1.5
if JAN3_PATTERN.search(text) and not DECIMAL_PATTERN.search(text):
if jan.search(text) and not decimal.search(text):
score += 0.5
return score
@ -448,7 +556,6 @@ def parse_tag_fields(lines: Sequence[OcrLine]) -> TagFields:
name_candidates.sort(key=lambda item: (-len(item[0]), -item[1].confidence))
fields.name = name_candidates[0][0]
# Layout, not digit count, separates the price from the JAN: the price is the
# LARGEST text on the tag (its line is roughly twice as tall as the small
# print), and the JAN's last three digits sit on a small line of their own.
# Getting this order wrong made a 269 price look like a JAN and disappear.
@ -534,7 +641,12 @@ def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, i
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
hue, sat, val = cv2.split(hsv)
brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8)
rail = _RAIL or StorePolicy()
brick = (
((hue < rail.rail_hue_low) | (hue > rail.rail_hue_high))
& (sat > rail.rail_sat_min)
& (val > rail.rail_val_min)
).astype(np.uint8)
row_fraction = brick.mean(axis=1)
height, width = image.shape[:2]
rails: List[Tuple[int, int, int, int]] = []

92
recognization/code/tags.py
View file

@ -36,6 +36,7 @@ from ocr import OcrEngine, OcrLine, OcrResult, OcrUnavailable # noqa: E402
from tag_cards import ( # noqa: E402
CORROBORATION_FLAGS,
MIN_PRICE,
StorePolicy,
TagFields as TagFieldsModel,
corroborate,
best_tag_rows,
@ -49,6 +50,13 @@ from tag_cards import ( # noqa: E402
TAG_STATE_READABLE = "readable"
TAG_STATE_UNREADABLE = "unreadable"
TAG_STATE_ABSENT = "absent"
# Seen on the rail, OCR returned nothing. Not the same as absent (no tag) or
# unreadable (text came back but did not corroborate).
TAG_STATE_DETECTED = "detected"
# Two shelf rows closer than this are the same rail. A screen row index is not
# an identity: panning a new shelf into view reuses row 0.
SHELF_ROW_BIN_PX = 80
# Measured tag shape (add/image6.jpg): ~300x80 px, aspect ~3.75:1, digits ~22 px.
TAG_MIN_WIDTH_RATIO = 0.035
@ -114,6 +122,7 @@ class TagRecord:
tag_id: str
box: List[int]
band_index: int
shelf_key: Tuple[str, int] = ("screen", 0)
observations: List[TagObservation] = field(default_factory=list)
first_seen: float = 0.0
last_seen: float = 0.0
@ -126,6 +135,7 @@ class TagRecord:
discount: Optional[float] = None
unit_price: Optional[float] = None
name: Optional[str] = None
price_source: str = "none"
flags: List[str] = field(default_factory=list)
confidence: float = 0.0
@ -135,17 +145,18 @@ class TagRecord:
Measured live: without this, single uncorroborated reads of the stylised
digits were published as readable tags with prices of 20, 23, 72 and 4508.
A block seen with no text is `detected`; text that does not parse is
`unreadable`. Neither is `absent`.
"""
if not self.observations:
return TAG_STATE_UNREADABLE
if self.price is None or self.price < MIN_PRICE:
if self.price is None:
saw_text = any(observation.lines for observation in self.observations)
return TAG_STATE_UNREADABLE if saw_text else TAG_STATE_DETECTED
if self.price < MIN_PRICE:
return TAG_STATE_UNREADABLE
corroborated = bool(
{"tax_consistent", "tax_pair_consistent", "discount_consistent"} & set(self.flags)
)
if not corroborated:
return TAG_STATE_UNREADABLE
if self.regular_price is None and self.name is None:
if not corroborated or (self.regular_price is None and self.name is None):
return TAG_STATE_UNREADABLE
return TAG_STATE_READABLE
@ -159,8 +170,6 @@ class TagRecord:
"discount": self.discount,
"unit_price": self.unit_price,
"name": self.name,
"regular_price": self.regular_price,
"price_source": self.price_source,
"flags": list(self.flags),
"confidence": round(float(self.confidence), 3),
"observations": len(self.observations),
@ -386,6 +395,20 @@ class TagSettings:
settle_passes: int = SETTLE_PASSES
min_band_width_ratio: float = 0.10
max_tags: int = 120
policy: Optional[StorePolicy] = None
def band_key(band: RailBand, shelf_offset: Optional[Sequence[float]] = None) -> Tuple[str, int]:
"""Which rail this band is, stable across a pan.
No shelf origin: the screen row, which is all a still has. With one (the
shelf position of the frame origin), the row's shelf y, so a new shelf that
lands on screen row 0 is not the row that already settled.
"""
if not shelf_offset or len(shelf_offset) < 2:
return ("screen", int(band.row_index))
shelf_y = (band.box[1] + band.box[3]) / 2.0 + float(shelf_offset[1])
return ("shelf", int(round(shelf_y / SHELF_ROW_BIN_PX)))
class TagService:
@ -410,7 +433,7 @@ class TagService:
self._tags: Dict[str, TagRecord] = {}
self._pass_id = 0
self._frame_seq = 0
self._last_pass_seq: Dict[int, int] = {}
self._last_pass_seq: Dict[Tuple[str, int], int] = {}
self._pending_bands: set = set()
self.passes_run = 0
self.passes_failed = 0
@ -418,29 +441,37 @@ class TagService:
self.passes_skipped = 0
# ------------------------------------------------------------------ frame loop
def plan(self, boxes: Sequence[Dict[str, Any]], image_shape: Sequence[int]) -> List[RailBand]:
def plan(
self,
boxes: Sequence[Dict[str, Any]],
image_shape: Sequence[int],
shelf_offset: Optional[Sequence[float]] = None,
) -> List[RailBand]:
"""Bands worth OCR-ing right now (must be called from the frame loop).
A band is a shelf row, so its key is the row - not its on-screen position.
Keying by position meant a walking camera minted a fresh key every few
pixels of pan and re-read the same shelf continuously, which starved
classification of CPU (measured: 10x slower per crop while OCR ran).
The key is the shelf row when the caller passes the frame origin, and the
screen row otherwise. Keying by on-screen pixels re-read the same shelf
on every pan step (measured: 10x slower per crop while OCR ran); keying
by the screen row index instead hides a new shelf that reuses row 0.
"""
if not self.settings.enabled or not self.engine.available():
return []
if self.settings.policy is not None:
self.settings.policy.active()
bands = rail_bands(boxes, image_shape)
width = float(image_shape[1])
bands = [band for band in bands if (band.box[2] - band.box[0]) >= self.settings.min_band_width_ratio * width]
with self._lock:
self._shelf_offset = shelf_offset
self._frame_seq += 1
seq = self._frame_seq
chosen: List[RailBand] = []
for band in bands:
key = band.row_index
key = band_key(band, shelf_offset)
if key in self._pending_bands:
continue
last = self._last_pass_seq.get(key)
settled = self._band_settled(band.row_index)
settled = self._band_settled(key)
if settled and last is not None:
continue
if last is not None and seq - last < self.settings.pass_interval_frames:
@ -449,8 +480,9 @@ class TagService:
if len(chosen) >= self.settings.max_bands_per_pass:
break
for band in chosen:
self._last_pass_seq[band.row_index] = seq
self._pending_bands.add(band.row_index)
key = band_key(band, shelf_offset)
self._last_pass_seq[key] = seq
self._pending_bands.add(key)
return chosen
def run_pass(self, image: Any, band: RailBand, scale_bias: float = 1.0) -> List[TagObservation]:
@ -460,7 +492,7 @@ class TagService:
independent reads of the same tag are what makes the vote meaningful
(identical pixels would produce identical output).
"""
key = band.row_index
key = band_key(band, getattr(self, "_shelf_offset", None))
try:
x1, y1, x2, y2 = band.box
crop = image[y1:y2, x1:x2]
@ -537,7 +569,7 @@ class TagService:
)
card = card_from_lines(pooled) if pooled else seed
card.box = box
if not card.fields.is_readable() and card.fields.price is None:
if not card.fields.is_readable() and card.fields.price is None and not group:
continue
cards.append(
{
@ -593,8 +625,8 @@ class TagService:
)
return extra
def _band_settled(self, row_index: int) -> bool:
row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index]
def _band_settled(self, key: Tuple[str, int]) -> bool:
row_tags = [tag for tag in self._tags.values() if tag.shelf_key == key]
if not row_tags:
return False
return all(tag.settled for tag in row_tags)
@ -604,6 +636,7 @@ class TagService:
box = candidate["box"]
lines = candidate["lines"]
fields = candidate["fields"]
shelf_key = band_key(band, getattr(self, "_shelf_offset", None))
observation = TagObservation(
box=list(box),
lines=list(lines),
@ -613,12 +646,13 @@ class TagService:
observed_at=time.time(),
)
record = self._match_record(box)
record = self._match_record(box, shelf_key)
if record is None:
record = TagRecord(
tag_id=f"tag_{uuid.uuid4().hex[:8]}",
box=list(box),
band_index=band.row_index,
shelf_key=shelf_key,
first_seen=observation.observed_at,
)
if len(self._tags) >= self.settings.max_tags:
@ -636,10 +670,12 @@ class TagService:
record.settled = True
return observation
def _match_record(self, box: Sequence[int]) -> Optional[TagRecord]:
def _match_record(self, box: Sequence[int], shelf_key: Tuple[str, int]) -> Optional[TagRecord]:
best = None
best_iou = 0.35
for record in self._tags.values():
if record.shelf_key != shelf_key:
continue
iou = _iou(box, record.box)
if iou >= best_iou:
best_iou = iou
@ -769,7 +805,12 @@ class TagService:
"settled": False,
}
counts = {TAG_STATE_READABLE: 0, TAG_STATE_UNREADABLE: 0, TAG_STATE_ABSENT: 0}
counts = {
TAG_STATE_READABLE: 0,
TAG_STATE_UNREADABLE: 0,
TAG_STATE_DETECTED: 0,
TAG_STATE_ABSENT: 0,
}
for payload in by_track.values():
counts[payload["state"]] = counts.get(payload["state"], 0) + 1
return by_track, tags, counts
@ -817,6 +858,7 @@ class TagService:
self._pending_bands.clear()
self._pass_id = 0
self._frame_seq = 0
self._shelf_offset = None
def _iou(a: Sequence[int], b: Sequence[int]) -> float:

25
recognization/code/test.py
View file

@ -1,25 +0,0 @@
from ultralytics import YOLO
import cv2
model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt")
image_path = "/Users/nguyenhongvan/recongnize/data/add/image9.jpg"
results = model(image_path)
image = cv2.imread(image_path)
for box in results[0].boxes.xyxy.cpu().numpy():
x1, y1, x2, y2 = map(int, box)
cv2.rectangle(
image,
(x1, y1),
(x2, y2),
(0, 255, 0),
2
)
cv2.imshow("Bounding Boxes Only", image)
cv2.waitKey(0)
cv2.destroyAllWindows()

56
recognization/code/test_api.py
View file

@ -155,6 +155,47 @@ class RestContractTests(unittest.TestCase):
self.assertEqual(response.status_code, 415)
self.assertEqual(response.json()["error"]["code"], "UNSUPPORTED_MEDIA_TYPE")
def test_a_lying_content_length_is_rejected_while_streaming(self):
"""The ASGI receive stops once the body passes the cap, before decode."""
import anyio
from transport import BodyLimit
cap = 32
seen = {"bytes": 0}
async def app(scope, receive, send):
while True:
message = await receive()
if message["type"] != "http.request":
continue
seen["bytes"] += len(message.get("body") or b"")
if not message.get("more_body"):
break
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
chunks = [b"x" * 20, b"y" * 20, b"z" * 20]
async def receive():
body = chunks.pop(0) if chunks else b""
return {"type": "http.request", "body": body, "more_body": bool(chunks)}
sent = []
async def send(message):
sent.append(message)
async def run():
await BodyLimit(app, cap)(
{"type": "http"}, receive, send
)
anyio.run(run)
self.assertLessEqual(seen["bytes"], cap + 20)
self.assertEqual(sent[0]["status"], 413)
self.assertIn(b"PAYLOAD_TOO_LARGE", sent[1]["body"])
def test_classify_and_inspect_roundtrip(self):
recognized = self.client.post(
"/v1/recognize?classify=false", content=jpeg_bytes(),
@ -285,13 +326,26 @@ class WebSocketProtocolTests(unittest.TestCase):
self.assertEqual(missing["code"], "TRACK_NOT_IN_VIEW")
def test_token_is_enforced_on_the_socket(self):
"""Header or the first post-hello auth frame. A query token is rejected."""
client, engine = make_app(token="s3cret")
try:
with self.assertRaises(Exception):
with client.websocket_connect("/v1/stream") as socket:
self.assertEqual(socket.receive_json()["type"], "hello")
socket.send_json({"type": "ping", "t": 1})
socket.receive_json()
with client.websocket_connect("/v1/stream?token=s3cret") as socket:
with self.assertRaises(Exception):
with client.websocket_connect("/v1/stream?token=s3cret") as socket:
socket.receive_json()
with client.websocket_connect(
"/v1/stream", headers={"Authorization": "Bearer s3cret"}
) as socket:
self.assertEqual(socket.receive_json()["type"], "hello")
with client.websocket_connect("/v1/stream") as socket:
self.assertEqual(socket.receive_json()["type"], "hello")
socket.send_json({"type": "auth", "token": "s3cret"})
socket.send_json({"type": "ping", "t": 1})
self.assertEqual(socket.receive_json(), {"type": "pong", "t": 1})
finally:
engine.shutdown()

20
recognization/code/test_empty.py
View file

@ -1,20 +0,0 @@
import sys
import os
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from empty_shelf_detector import detect_and_draw_empty_slots
if __name__ == "__main__":
image_path = "image.jpg"
print(f"Processing empty shelf detection on {image_path}...")
annotated, empty_slots = detect_and_draw_empty_slots(
image_path,
output_path="detected_empty_slots.jpg"
)
print(f"Detected {len(empty_slots)} empty shelf slots:")
for s in empty_slots:
print(f" - Slot [{s['id']}]: Row {s['row_index']}, Box={s['box']}, Width={s['width']}px, Est. Missing Items={s['missing_items_est']}")
print(f"Visualization saved to: detected_empty_slots.jpg")

241
recognization/code/test_eval_live.py Normal file
View file

@ -0,0 +1,241 @@
"""Metric contracts for eval_live. No model, no real walk, no fabricated accuracy."""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
import cv2 # noqa: E402
import numpy as np # noqa: E402
from eval_live import ( # noqa: E402
EvalInputError,
identity_level,
labelled_top1,
leaked_labels,
load_index,
score_recording,
score_tracks,
track_churn,
)
def track(level, sku=None, track_id=1, box=None, relabels=0, restarts=0, tag=None):
identity = {"identity_level": level, "state": "recognized" if level == "accepted" else "unrecognized"}
if level == "accepted":
identity["sku_id"] = sku
row = {
"track_id": track_id,
"box": box or [0, 0, 10, 10],
"identity": identity,
"relabels": relabels,
"restarts": restarts,
}
if tag is not None:
row["price_tag"] = tag
return row
def write_jpeg(path, image):
ok, encoded = cv2.imencode(".jpg", image)
assert ok
with open(path, "wb") as handle:
handle.write(encoded.tobytes())
class DecisionDenominatorTests(unittest.TestCase):
def test_settled_excludes_unsettled_and_rates_use_that_denominator(self):
scored = score_tracks(
[
track("accepted", "sku_a"),
track("ambiguous"),
track("rejected"),
track("unknown"),
{"identity": {"state": "unconfirmed", "identity_level": None}},
]
)
self.assertEqual(scored["settled"], 4)
self.assertEqual(scored["unsettled"], 1)
self.assertEqual(scored["accepted_rate"], 0.25)
self.assertIsNone(score_tracks([{"identity": {"state": "unconfirmed"}}])["accepted_rate"])
def test_wire_state_fills_a_missing_level(self):
self.assertEqual(
identity_level({"identity": {"state": "needs_confirmation"}}), "ambiguous"
)
self.assertEqual(identity_level({"identity": {"state": "unconfirmed"}}), "unsettled")
def test_readable_tags_are_counted_apart_from_detected(self):
scored = score_tracks(
[
track("accepted", "sku_a", tag={"state": "readable"}),
track("rejected", tag={"state": "unreadable"}),
track("rejected", tag={"state": "absent"}),
track("rejected"),
]
)
self.assertEqual(scored["tags_readable"], 1)
self.assertEqual(scored["tags_detected"], 2)
self.assertEqual(scored["tags_absent"], 2)
class Top1AndChurnTests(unittest.TestCase):
def test_top1_counts_only_accepted_matches_over_overlapping_labels(self):
frames = [
{
"frame": 1,
"tracks": [
track("accepted", "sku_a", box=[0, 0, 10, 10]),
track("accepted", "sku_b", track_id=2, box=[100, 100, 110, 110]),
],
},
{"frame": 2, "tracks": [track("rejected", box=[0, 0, 10, 10])]},
{"frame": 3, "tracks": []},
]
labels = [
{"frame": 1, "box": [0, 0, 10, 10], "sku_id": "sku_a"},
{"frame": 1, "box": [100, 100, 110, 110], "sku_id": "sku_a"},
{"frame": 2, "box": [0, 0, 10, 10], "sku_id": "sku_a"},
{"frame": 3, "box": [0, 0, 10, 10], "sku_id": "sku_a"},
]
scored = labelled_top1(frames, labels, iou_min=0.5)
self.assertEqual((scored["correct"], scored["wrong"], scored["rejected"], scored["unmatched"]), (1, 1, 1, 1))
self.assertEqual(scored["top1"], 0.3333)
def test_sku_churn_counts_a_renamed_track_not_the_first_name(self):
frames = [
{"tracks": [track("accepted", "sku_a", track_id=7)]},
{"tracks": [track("accepted", "sku_a", track_id=7)]},
{"tracks": [track("accepted", "sku_b", track_id=7, relabels=1)]},
{"tracks": [track("rejected", track_id=8, restarts=2)]},
]
churn = track_churn(frames)
self.assertEqual(churn["sku_changes"], 1)
self.assertEqual(churn["tracks_with_sku_change"], 1)
self.assertEqual(churn["relabels"], 1)
self.assertEqual(churn["restarts"], 2)
self.assertEqual(churn["per_track"][0]["track_id"], 7)
class LeakageTests(unittest.TestCase):
def setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.root, ignore_errors=True)
def test_a_pasted_catalog_crop_is_excluded_and_a_real_facing_is_not(self):
stock = os.path.join(self.root, "stock")
os.makedirs(stock)
template = np.full((20, 16, 3), 40, dtype=np.uint8)
template[2:8, 2:8] = (255, 0, 0)
write_jpeg(os.path.join(stock, "sku_a.jpeg"), template)
frame = np.zeros((40, 40, 3), dtype=np.uint8)
frame[5:25, 10:26] = template
recording = os.path.join(self.root, "walk")
os.makedirs(recording)
write_jpeg(os.path.join(recording, "000000.jpg"), frame)
with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write(json.dumps({"frame": 0, "path": "000000.jpg", "tracks": []}) + "\n")
labels = [
{"frame": 0, "box": [10, 5, 26, 25], "sku_id": "sku_a"},
{"frame": 0, "box": [0, 0, 8, 8], "sku_id": "sku_b"},
]
held, leaked = leaked_labels(recording, load_index(recording), labels, stock)
self.assertEqual(len(leaked), 1)
self.assertEqual(leaked[0]["leak"], "crop_is_template")
self.assertEqual([row["sku_id"] for row in held], ["sku_b"])
def test_score_recording_names_the_leak_and_refuses_it_as_top1(self):
stock = os.path.join(self.root, "stock")
os.makedirs(stock)
image = np.full((12, 12, 3), 9, dtype=np.uint8)
write_jpeg(os.path.join(stock, "sku_a.jpg"), image)
with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle:
handle.write("SKU_ID,product_name\nsku_a,A\n")
recording = os.path.join(self.root, "session")
os.makedirs(recording)
shutil.copy(os.path.join(stock, "sku_a.jpg"), os.path.join(recording, "000000.jpg"))
with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write(
json.dumps(
{
"frame": 1,
"path": "000000.jpg",
"tracks": [track("accepted", "sku_a", box=[0, 0, 12, 12])],
}
)
+ "\n"
)
with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle:
json.dump({"labels": [{"frame": 1, "box": [0, 0, 12, 12], "sku_id": "sku_a"}]}, handle)
report = score_recording(recording, stock_dir=stock)
self.assertEqual(report["leakage"]["excluded"], 1)
self.assertEqual(report["labelled_top1"]["labels"], 0)
self.assertEqual(report["labelled_top1"]["top1"], None)
self.assertEqual(report["decisions"]["accepted"], 1)
self.assertTrue(report["fingerprints"]["sample"]["sha256"])
self.assertEqual(len(report["fingerprints"]["catalog"]["version"]), 12)
self.assertTrue(any("template-derived" in gap for gap in report["evidence"]))
class CliTests(unittest.TestCase):
def test_missing_recording_is_an_explicit_error(self):
completed = subprocess.run(
[sys.executable, os.path.join(current_dir, "eval_live.py"), "/no/such/recording"],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(completed.returncode, 2)
self.assertIn("error", json.loads(completed.stdout))
def test_the_same_pinned_recording_scores_twice_the_same(self):
root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, root, ignore_errors=True)
stock = os.path.join(root, "stock")
os.makedirs(stock)
with open(os.path.join(stock, "stock.csv"), "w", encoding="utf-8") as handle:
handle.write("SKU_ID\nsku_a\n")
recording = os.path.join(root, "session")
os.makedirs(recording)
payload = np.zeros((8, 8, 3), dtype=np.uint8)
write_jpeg(os.path.join(recording, "000000.jpg"), payload)
line = {
"frame": 4,
"path": "000000.jpg",
"tracks": [
track("accepted", "sku_a", tag={"state": "readable"}),
track("ambiguous", track_id=2),
],
}
with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write(json.dumps(line) + "\n")
with open(os.path.join(recording, "labels.json"), "w", encoding="utf-8") as handle:
json.dump({"labels": [{"frame": 4, "box": [0, 0, 8, 8], "sku_id": "sku_a"}]}, handle)
first = score_recording(recording, stock_dir=stock)
second = score_recording(recording, stock_dir=stock)
self.assertEqual(first, second)
self.assertEqual(first["decisions"]["accepted"], 1)
self.assertEqual(first["decisions"]["ambiguous"], 1)
self.assertEqual(first["labelled_top1"]["correct"], 1)
self.assertIsNone(first["fingerprints"]["model"]["weights_sha256"])
def test_a_broken_index_line_is_an_input_error(self):
root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, root, ignore_errors=True)
with open(os.path.join(root, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write("{not json}\n")
with self.assertRaises(EvalInputError):
load_index(root)
if __name__ == "__main__":
unittest.main()

89
recognization/code/test_image.py
View file

@ -1,89 +0,0 @@
"""
Test Script for image.jpg
1. Runs YOLO detection on image.jpg.
2. Draws ONLY bounding box rectangles (no class names, no confidence labels).
3. Saves output to detected_image.jpg.
4. Matches the detected bounding boxes against stock templates.
"""
import os
import sys
import cv2
import numpy as np
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from detector import ShelfProductDetector
from feature_matcher import ProductFeatureMatcher
from inventory import InventoryManager
def main():
image_path = "image.jpg"
print(f"[1/4] Loading {image_path}...")
image = cv2.imread(image_path)
if image is None:
print(f"[Error] Failed to read {image_path}")
return
h, w = image.shape[:2]
print(f" Image resolution: {w}x{h}")
print("[2/4] Detecting products using YOLO...")
detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
# You can adjust conf_threshold as needed (default 0.25)
det_res = detector.detect(image, conf_threshold=0.25)
boxes = det_res.get("boxes", [])
print(f" Total detected items: {len(boxes)}")
print("[3/4] Drawing bounding boxes ONLY (no class names, no confidence)...")
annotated = detector.draw_detections(image, boxes, draw_labels=False)
output_path = "detected_image.jpg"
cv2.imwrite(output_path, annotated)
print(f" Saved detection visualization to: {output_path}")
print("[4/4] Matching against stock catalogue (SURF, ORB, BRISK, Hybrid)...")
matcher = ProductFeatureMatcher("stock")
inv = InventoryManager("stock")
print("\n--- Stock Templates Present in image.jpg ---")
matched_skus = set()
sample_matches = []
# Test top detections to find corresponding stock SKUs
for b in boxes:
crop = detector.crop_roi(image, b["box"])
# Use hybrid method from the paper
res = matcher.classify_crop(crop, method="hybrid")
if res.get("status") == "success":
sku_id = res["best_sku"]
score = res["score"]
conf = res["confidence"]
prod = inv.get_product(sku_id)
if prod:
matched_skus.add(sku_id)
sample_matches.append({
"box_id": b["id"],
"box": b["box"],
"sku_id": sku_id,
"product_name": prod["product_name"],
"price": prod["price"],
"number_on": prod["number_on"],
"score": score,
"confidence": conf
})
# Sort matches by confidence
sample_matches.sort(key=lambda x: x["score"], reverse=True)
print(f"\nSuccessfully matched {len(matched_skus)} distinct SKUs across detected shelf boxes:")
for sm in sample_matches[:10]:
print(f"Box #{sm['box_id']:<3} -> [{sm['sku_id']}] {sm['product_name']:<30} | Price: ¥{sm['price']:<5} | On-Shelf: {sm['number_on']} | Score: {sm['score']}")
print("\n[Complete] Tested image.jpg successfully.")
if __name__ == "__main__":
main()

30
recognization/code/test_live.py
View file

@ -276,6 +276,36 @@ class TagPassIntegrationTests(unittest.TestCase):
self.assertNotIn("price_tag", result["tracks"][0])
engine.shutdown()
def test_a_panned_shelf_reusing_the_screen_row_is_planned(self):
"""The same screen row on a shelf that has moved is a new rail."""
session, engine, _ = self.make_tagged_session()
session._tags.settings.pass_interval_frames = 0
session._tags.settings.settle_observations = 1
session._tags.settings.settle_passes = 1
class Panned:
def __init__(self):
self.calls = 0
def register(self, image):
self.calls += 1
# Content moved up 400 px, so the frame origin moves down the shelf.
return (0.0, -400.0) if self.calls == 2 else (0.0, 0.0)
session._shift_map = Panned()
process(session, 1)
time.sleep(0.05)
before = session._tags.stats()["passes_run"]
self.assertGreaterEqual(before, 1)
process(session, 2)
time.sleep(0.1)
self.assertGreaterEqual(session._frame_origin[1], 400.0)
self.assertGreater(
session._tags.stats()["passes_run"], before,
"a shelf 400 px away reuses screen row 0 and must be read again",
)
engine.shutdown()
class FramePipelineTests(unittest.TestCase):
def test_latest_frame_wins_and_drops_are_counted(self):

105
recognization/code/test_ocr_tags.py
View file

@ -21,6 +21,7 @@ import numpy as np # noqa: E402
from ocr import FakeOcr, NullOcr, OcrLine, OcrResult, OcrUnavailable, build_ocr_engine # noqa: E402
from tags import ( # noqa: E402
TAG_STATE_ABSENT,
TAG_STATE_DETECTED,
TAG_STATE_READABLE,
TAG_STATE_UNREADABLE,
TagService,
@ -30,6 +31,8 @@ from tags import ( # noqa: E402
parse_price_tag,
rail_bands,
)
from tag_cards import StorePolicy, StorePolicyError, load_store_policy # noqa: E402
import tag_cards # noqa: E402
def image(width=1000, height=600):
@ -275,6 +278,108 @@ class TagServiceTests(unittest.TestCase):
service.reset()
self.assertEqual(service.stats()["tags"], 0)
def test_a_new_shelf_reusing_row_zero_is_read(self):
"""Screen row 0 on a newly panned shelf is not the row that settled."""
service, _ = self.make(pass_interval_frames=0, settle_observations=1, settle_passes=1)
boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)]
array = image()
for _ in range(3):
for band in service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)):
service.run_pass(array, band)
self.assertEqual(
service.plan(boxes, array.shape, shelf_offset=(0.0, 0.0)),
[],
"the same shelf row stays settled",
)
again = service.plan(boxes, array.shape, shelf_offset=(0.0, 400.0))
self.assertTrue(again, "a shelf 400 px away reuses the screen row and must be read")
def test_a_seen_tag_with_no_text_is_detected_not_absent(self):
def reader(crop):
return []
service = TagService(FakeOcr(reader=reader), settings=TagSettings(pass_interval_frames=0))
array = image()
# A bright paper block on a brick rail, under the product row. OCR reads nothing.
array[300:420, :] = (40, 40, 180)
array[330:390, 120:280] = (235, 235, 235)
boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)]
bands = service.plan(boxes, array.shape)
self.assertTrue(bands)
service.run_pass(array, bands[0])
by_track, tags, counts = service.attach([{"track_id": 0, "box": [100, 100, 200, 300]}])
self.assertTrue(tags, "the paper block is a tag even though OCR returned nothing")
self.assertEqual(tags[0]["state"], TAG_STATE_DETECTED)
self.assertEqual(by_track[0]["state"], TAG_STATE_DETECTED)
self.assertEqual(counts[TAG_STATE_DETECTED], 1)
self.assertNotEqual(by_track[0]["state"], TAG_STATE_ABSENT)
def test_a_tag_pass_survives_a_missing_header_pattern(self):
"""Evaluation replay died with NameError: HEADER_PATTERN."""
import tag_cards
from tag_cards import OcrLine as CardLine
row = [CardLine(text="毎日お買得", box=(0, 0, 40, 16), confidence=0.9)]
saved = tag_cards.__dict__.pop("HEADER_PATTERN", None)
try:
score = tag_cards.tag_row_score(row)
finally:
if saved is not None:
tag_cards.HEADER_PATTERN = saved
self.assertGreater(score, 0, "a sale header still marks the row when the global is missing")
service, _ = self.make(pass_interval_frames=0)
boxes = [box(100, 100, 200, 300, 0), box(220, 100, 320, 300, 1), box(340, 100, 440, 300, 2)]
bands = service.plan(boxes, image().shape)
self.assertTrue(bands)
service.run_pass(image(), bands[0])
self.assertGreaterEqual(service.stats()["passes_run"], 1)
self.assertIsNone(service.stats()["last_pass_error"])
class StorePolicyTests(unittest.TestCase):
def test_a_missing_file_uses_the_defaults(self):
policy = load_store_policy(None)
policy.active()
self.assertEqual(policy.tax_ratios, (1.08, 1.10))
self.assertEqual(policy.min_price, 50)
self.assertEqual(tag_cards.MIN_PRICE, 50)
def test_a_store_file_does_not_change_another_store(self):
import json
import tempfile
with tempfile.TemporaryDirectory() as root:
good = os.path.join(root, "lopia")
bad = os.path.join(root, "other")
os.makedirs(good)
os.makedirs(bad)
with open(os.path.join(good, "store.json"), "w", encoding="utf-8") as handle:
json.dump({"store_id": "lopia", "tax_ratios": [1.08], "min_price": 80}, handle)
with open(os.path.join(bad, "store.json"), "w", encoding="utf-8") as handle:
handle.write("{not json")
loaded = load_store_policy(good)
self.assertEqual(loaded.min_price, 80)
self.assertEqual(tag_cards.MIN_PRICE, 80)
with self.assertRaises(StorePolicyError) as raised:
load_store_policy(bad)
self.assertIn(bad, str(raised.exception))
self.assertEqual(tag_cards.MIN_PRICE, 80, "a bad file must not replace the store already loaded")
again = load_store_policy(good)
self.assertEqual(again.min_price, 80)
def test_a_ratio_outside_range_fails_clearly(self):
import json
import tempfile
load_store_policy(None)
with tempfile.TemporaryDirectory() as root:
with open(os.path.join(root, "store.json"), "w", encoding="utf-8") as handle:
json.dump({"tax_ratios": [2.5], "min_price": 999}, handle)
with self.assertRaises(StorePolicyError) as raised:
load_store_policy(root)
self.assertIn("tax ratio", str(raised.exception))
self.assertEqual(tag_cards.MIN_PRICE, 50)
class EngineSelectionTests(unittest.TestCase):
def test_explicit_none_and_fake(self):

25
recognization/code/tracking.py
View file

@ -122,16 +122,30 @@ class _SessionByteTrack(BYTETracker):
class TrackOut:
"""One tracked object for a frame."""
__slots__ = ("track_id", "box", "score", "det_index")
__slots__ = ("track_id", "box", "score", "det_index", "low_score")
def __init__(self, track_id: int, box: List[int], score: float, det_index: Optional[int]):
def __init__(
self,
track_id: int,
box: List[int],
score: float,
det_index: Optional[int],
*,
low_score: bool = False,
):
self.track_id = int(track_id)
self.box = [int(v) for v in box]
self.score = float(score)
self.det_index = None if det_index is None or det_index < 0 else int(det_index)
# ByteTrack's second pass keeps an id alive on a low-score box. That box
# is not a new measurement: callers must not crop or re-label from it.
self.low_score = bool(low_score)
def __repr__(self) -> str: # pragma: no cover - debug helper
return f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, det={self.det_index})"
return (
f"TrackOut(id={self.track_id}, box={self.box}, score={self.score:.3f}, "
f"det={self.det_index}, low={self.low_score})"
)
class SessionTracker:
@ -171,14 +185,17 @@ class SessionTracker:
outputs: List[TrackOut] = []
if rows is None or len(rows) == 0:
return outputs
high = float(self.args.track_high_thresh)
for row in np.asarray(rows):
# [x1, y1, x2, y2, track_id, score, cls, idx]
score = float(row[5])
outputs.append(
TrackOut(
track_id=int(row[4]),
box=[int(round(row[0])), int(round(row[1])), int(round(row[2])), int(round(row[3]))],
score=float(row[5]),
score=score,
det_index=int(row[7]) if len(row) > 7 else None,
low_score=score < high,
)
)
return outputs

254
recognization/code/transport.py Normal file
View file

@ -0,0 +1,254 @@
"""
Request bounds and the LAN token check.
A client can declare a tiny Content-Length and then keep writing, so the
declared length is a hint and the read itself stops at the cap. The token
never travels in a query string: proxies and access logs keep those.
"""
from __future__ import annotations
import hmac
import ipaddress
import os
import shutil
import signal
import socket
import subprocess
from typing import Any, Dict, Optional
LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
def is_loopback_host(host: Optional[str]) -> bool:
text = (host or "").strip().lower()
if text.startswith("[") and text.endswith("]"):
text = text[1:-1]
if "%" in text: # fe80::1%lo0
text = text.split("%", 1)[0]
if text in LOOPBACK_HOSTS:
return True
try:
return ipaddress.ip_address(text).is_loopback
except ValueError:
return False
def require_token_for_bind(host: Optional[str], token: Optional[str]) -> None:
"""A non-loopback bind with no token is reachable by anyone on that network."""
if token or is_loopback_host(host):
return
raise SystemExit(
f"refusing to bind {host} without DORAYAKI_API_TOKEN "
"(loopback is the only bind that may run with auth off)"
)
def token_matches(presented: Optional[str], token: Optional[str]) -> bool:
if not presented or not token:
return False
# compare_digest raises when the lengths differ. Fold the length in so a
# short guess neither throws nor returns early.
return hmac.compare_digest(
len(presented).to_bytes(8, "big") + presented.encode(),
len(token).to_bytes(8, "big") + token.encode(),
)
def bearer_matches(authorization: Optional[str], token: str) -> bool:
if not authorization or not token:
return False
scheme, _, presented = authorization.partition(" ")
if scheme.lower() != "bearer" or not presented or " " in presented:
return False
return token_matches(presented, token)
def declared_length(header: Optional[str]) -> Optional[int]:
if header is None or header == "":
return None
try:
length = int(header)
except (TypeError, ValueError):
return None
return length if length >= 0 else None
async def read_bounded(source, max_bytes: int) -> bytes:
"""Stop once the cap is passed. The caller rejects the oversize body.
A request stream is async-iterable. An UploadFile is not: it has read(n).
"""
out = bytearray()
read = getattr(source, "read", None)
if read is not None and not hasattr(source, "__aiter__"):
while True:
chunk = await read(65536)
if not chunk:
break
out += chunk
if len(out) > max_bytes:
break
return bytes(out)
async for chunk in source:
if not chunk:
continue
out += chunk
if len(out) > max_bytes:
break
return bytes(out)
class BodyLimit:
"""Stop the ASGI receive once the body passes the cap.
A handler that checks Content-Length, or reads request.body(), runs after
the server has already buffered the stream. This counts bytes at receive
and returns 413 without handing the rest to the app. WebSocket frames are
not HTTP bodies and are left alone.
"""
def __init__(self, app: Any, max_bytes: int) -> None:
self.app = app
self.max_bytes = int(max_bytes)
async def __call__(self, scope, receive, send) -> None:
if scope.get("type") != "http":
await self.app(scope, receive, send)
return
seen = 0
blocked = False
async def limited() -> Dict[str, Any]:
nonlocal seen, blocked
if blocked:
return {"type": "http.request", "body": b"", "more_body": False}
message = await receive()
if message.get("type") != "http.request":
return message
body = message.get("body") or b""
seen += len(body)
if seen > self.max_bytes:
blocked = True
await send(
{
"type": "http.response.start",
"status": 413,
"headers": [(b"content-type", b"application/json")],
}
)
await send(
{
"type": "http.response.body",
"body": b'{"error":{"code":"PAYLOAD_TOO_LARGE","message":"request body too large"}}',
}
)
return {"type": "http.request", "body": b"", "more_body": False}
return message
try:
await self.app(scope, limited, send)
except RuntimeError:
if not blocked:
raise
SERVICE_TYPE = "_dorayaki._tcp"
def _lan_address() -> Optional[str]:
"""Address a phone on this network can open. Loopback is not one.
The demo launcher uses `ipconfig getifaddr en0` then `en1`. A UDP connect
is the fallback when those interfaces are absent.
"""
if shutil.which("ipconfig"):
for iface in ("en0", "en1"):
try:
out = subprocess.check_output(
["ipconfig", "getifaddr", iface], stderr=subprocess.DEVNULL, text=True
).strip()
except (subprocess.CalledProcessError, OSError):
continue
if out and not is_loopback_host(out):
return out
try:
probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
probe.connect(("192.0.2.1", 9))
found = probe.getsockname()[0]
finally:
probe.close()
except OSError:
return None
if not found or is_loopback_host(found):
return None
return found
def advertise_host(bind_host: Optional[str]) -> Optional[str]:
"""The address clients should browse. A loopback bind is not advertised.
A wildcard bind (0.0.0.0, ::) has no address of its own, so the record
uses the LAN address the demo launcher would hand a phone.
"""
text = (bind_host or "").strip()
if text.startswith("[") and text.endswith("]"):
text = text[1:-1]
if is_loopback_host(text):
return None
if text in ("", "0.0.0.0", "::"):
return _lan_address()
try:
parsed = ipaddress.ip_address(text)
except ValueError:
return None
if parsed.is_loopback or parsed.is_unspecified:
return None if parsed.is_loopback else _lan_address()
return str(parsed)
class ServiceAdvertiser:
"""Register `_dorayaki._tcp` with the system mDNS daemon and drop it on close.
ponytail: macOS `dns-sd` only. A Linux host needs `avahi-publish` wired the
same way; do not add a Python mDNS package for this one record.
"""
def __init__(self, host: str, port: int, name: str = "Dorayaki") -> None:
self.host = host
self.port = int(port)
self.name = name
self._process: Optional[subprocess.Popen] = None
def start(self) -> bool:
if self._process is not None:
return True
binary = shutil.which("dns-sd")
if not binary:
return False
self._process = subprocess.Popen(
[binary, "-P", self.name, SERVICE_TYPE, "local", str(self.port), self.host, self.host],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
return True
def close(self) -> None:
process = self._process
self._process = None
if process is None or process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
process.wait(timeout=2)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
process.wait(timeout=2)

143
recognization/code/web/app.js
View file

@ -3,12 +3,24 @@
*
* A client of the recognition server, nothing else: live frames go out over
* /v1/stream, stills go out over /v1/recognize. Point it at another server with
* ?server=https://host:port and pass a token with ?token=...
* ?server=https://host:port. A token, when the server requires one, comes from
* sessionStorage (key dorayaki.token) or the fragment #token=..., never the query.
*/
const params = new URLSearchParams(location.search);
const SERVER = (params.get('server') || '').replace(/\/$/, '');
const TOKEN = params.get('token') || '';
const TOKEN = readToken();
function readToken() {
const stored = sessionStorage.getItem('dorayaki.token') || '';
const fragment = new URLSearchParams((location.hash || '').replace(/^#/, ''));
const fromHash = fragment.get('token') || '';
if (fromHash) {
sessionStorage.setItem('dorayaki.token', fromHash);
history.replaceState(null, '', location.pathname + location.search);
}
return fromHash || stored;
}
const I18N = {
en: {
@ -363,7 +375,7 @@ function thumbUrl(skuId) {
async function listCameras() {
const select = el('sourceSelect');
select.innerHTML = '';
select.replaceChildren();
if (!navigator.mediaDevices || !window.isSecureContext) {
const option = document.createElement('option');
option.value = 'file';
@ -390,13 +402,16 @@ function connectStream() {
if (state.ws && (state.ws.readyState === WebSocket.OPEN || state.ws.readyState === WebSocket.CONNECTING)) return;
setConnection('connecting');
const base = SERVER || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}`;
const url = `${base.replace(/^http/, 'ws')}/v1/stream${TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ''}`;
const url = `${base.replace(/^http/, 'ws')}/v1/stream`;
const socket = new WebSocket(url);
socket.binaryType = 'arraybuffer';
state.ws = socket;
socket.onopen = () => {
state.reconnectDelay = 1000;
// Browsers cannot set Authorization on a WebSocket. The server hello
// arrives first; this frame is the token, and it is not a URL.
if (TOKEN) socket.send(JSON.stringify({ type: 'auth', token: TOKEN }));
setConnection('live');
socket.send(JSON.stringify({ type: 'config', method: state.method, empty_slots: state.showEmpty }));
startSending();
@ -464,25 +479,25 @@ function sendFrame() {
function renderCompliance(compliance) {
const strip = el('complianceStrip');
strip.replaceChildren();
if (!compliance) { strip.hidden = true; return; }
const rows = compliance.rows || [];
const items = rows.slice(0, 3).map(
(row) => `<span class="compliance-item">⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})</span>`
);
if (rows.length > 3) {
items.push(`<span class="compliance-item">+${rows.length - 3} ${t('more')}</span>`);
}
const hints = compliance.hints || [];
if (!items.length && hints.length) {
strip.hidden = false;
strip.dataset.kind = 'hint';
strip.innerHTML = hints.map((hint) => `<span class="compliance-item">${hint.detail}</span>`).join('');
return;
const chips = [];
for (const row of rows.slice(0, 3)) {
chips.push(`⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})`);
}
if (!items.length) { strip.hidden = true; return; }
if (rows.length > 3) chips.push(`+${rows.length - 3} ${t('more')}`);
const texts = chips.length ? chips : hints.map((hint) => hint.detail);
if (!texts.length) { strip.hidden = true; return; }
strip.hidden = false;
strip.dataset.kind = 'violation';
strip.innerHTML = items.join('');
strip.dataset.kind = chips.length ? 'violation' : 'hint';
for (const text of texts) {
const item = document.createElement('span');
item.className = 'compliance-item';
item.textContent = text;
strip.appendChild(item);
}
}
function handleResult(message) {
@ -536,7 +551,7 @@ async function loadSamples() {
const response = await fetch(api('/v1/samples'), { headers: headers() });
const data = await response.json();
const select = el('sampleSelect');
select.innerHTML = '';
select.replaceChildren();
for (const sample of data.samples) {
const option = document.createElement('option');
option.value = sample.id;
@ -634,24 +649,42 @@ function renderInView() {
}
}
}
list.replaceChildren();
if (!counts.size) {
list.innerHTML = `<div class="placeholder small">${t('inviewPlaceholder')}</div>`;
const empty = document.createElement('div');
empty.className = 'placeholder small';
empty.textContent = t('inviewPlaceholder');
list.appendChild(empty);
return;
}
const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([skuId, facings]) => {
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
for (const [skuId, facings] of ranked) {
const product = productFor(skuId);
const name = product ? product.product_name : skuId;
const price = product ? `¥${Math.round(product.price)}` : '';
return `<div class="inview-row" data-sku="${skuId}">
<img class="inview-thumb" src="${thumbUrl(skuId)}" alt="">
<div class="inview-text"><div class="inview-name">${name}</div>
<div class="inview-meta">${price} · ${t('facings')} ${facings}</div></div>
<div class="inview-facings">${facings}</div></div>`;
}).join('');
list.innerHTML = rows;
list.querySelectorAll('.inview-row').forEach((row) => {
row.addEventListener('click', () => selectBySku(row.dataset.sku));
});
const row = document.createElement('div');
row.className = 'inview-row';
row.dataset.sku = skuId;
const thumb = document.createElement('img');
thumb.className = 'inview-thumb';
thumb.alt = '';
thumb.src = thumbUrl(skuId);
const text = document.createElement('div');
text.className = 'inview-text';
const nameEl = document.createElement('div');
nameEl.className = 'inview-name';
nameEl.textContent = name;
const meta = document.createElement('div');
meta.className = 'inview-meta';
meta.textContent = `${price} · ${t('facings')} ${facings}`;
text.append(nameEl, meta);
const count = document.createElement('div');
count.className = 'inview-facings';
count.textContent = String(facings);
row.append(thumb, text, count);
row.addEventListener('click', () => selectBySku(skuId));
list.appendChild(row);
}
}
function selectBySku(skuId) {
@ -715,9 +748,19 @@ function renderOcrSection(tag, reconciliation) {
if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]);
}
if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']);
rows.innerHTML = lines
.map(([key, value, bad]) => `<div class="ocr-row"><span class="ocr-key">${key}</span><span class="ocr-val${bad ? ' bad' : ''}">${value}</span></div>`)
.join('');
rows.replaceChildren();
for (const [key, value, bad] of lines) {
const row = document.createElement('div');
row.className = 'ocr-row';
const keyEl = document.createElement('span');
keyEl.className = 'ocr-key';
keyEl.textContent = key;
const valueEl = document.createElement('span');
valueEl.className = bad ? 'ocr-val bad' : 'ocr-val';
valueEl.textContent = value;
row.append(keyEl, valueEl);
rows.appendChild(row);
}
}
function showProductCard(product, identity) {
@ -755,12 +798,22 @@ function showCandidatesCard(identity) {
el('metricReceipt').textContent = '—';
el('metricExpiry').textContent = '—';
const candidates = el('candidates');
const rows = (identity.candidates || []).map((candidate) => {
candidates.replaceChildren();
const title = document.createElement('div');
title.className = 'candidates-title';
title.textContent = t('candidatesTitle');
candidates.appendChild(title);
for (const candidate of identity.candidates || []) {
const product = productFor(candidate.sku_id);
const name = product ? product.product_name : candidate.sku_id;
return `<div class="candidate-row"><span>${name}</span><span>${candidate.score}</span></div>`;
}).join('');
candidates.innerHTML = `<div class="candidates-title">${t('candidatesTitle')}</div>${rows}`;
const row = document.createElement('div');
row.className = 'candidate-row';
const name = document.createElement('span');
name.textContent = product ? product.product_name : candidate.sku_id;
const score = document.createElement('span');
score.textContent = String(candidate.score);
row.append(name, score);
candidates.appendChild(row);
}
candidates.hidden = false;
el('showMatchBtn').disabled = false;
renderOcrSection(state.currentTag, state.currentReconciliation);
@ -889,10 +942,18 @@ function handleInspect(result) {
if (result.crop_jpeg_b64) {
el('productThumb').src = result.crop_jpeg_b64;
}
const box = el('matchBox');
box.replaceChildren();
if (result.match_visualization_jpeg_b64) {
el('matchBox').innerHTML = `<img src="${result.match_visualization_jpeg_b64}" alt="feature matches">`;
const image = document.createElement('img');
image.alt = 'feature matches';
image.src = result.match_visualization_jpeg_b64;
box.appendChild(image);
} else {
el('matchBox').innerHTML = `<span class="match-placeholder">${t('matchPlaceholder')}</span>`;
const placeholder = document.createElement('span');
placeholder.className = 'match-placeholder';
placeholder.textContent = t('matchPlaceholder');
box.appendChild(placeholder);
}
const identity = result.identity || {};
const parts = [];