07de2bd37d
The gates were tuned on image6 alone, where every crop is a facing of a product the shelf actually holds. Wikimedia supermarket photos of milk, soy and snack shelves share nothing with the LOPIA catalog, so every accept there is a false accept by construction - and at the old lines there were 83 of them. Text lane: TEXT_MIN_SHARED 2 -> 4. The vocabulary filter drops bigrams no name contains, so a short garbage read can end up with a perfect ratio on two coincidental pairs. 20 of 26 text-lane answers on those shelves rested on exactly two: '避くて おいしい' named sku_l12 through おい/いし. Local lane: LOCAL_ACCEPT 2.0 -> 4.0. The residual accepts were geometric matches with no text overlap at all, scoring 2.26-4.81. Measured, 2297 distractor crops and 109 image6 facings: accept floor image6 shown/correct precision false accepts 2.0 2 54 / 49 90.7% 83 (3.61%) 2.0 4 52 / 48 92.3% 30 (1.31%) 4.0 4 44 / 42 95.5% 8 (0.35%) 5.0 6 36 / 36 100.0% 0 (0.00%) 4.0/4 is the only region passing both G2 (>=95% precision) and G3 (<=1% far-unknown) with coverage left. On the shipped evaluator: 66% end-to-end at 95% precision, 2 false accepts, against 77%/91%/5 before. The two tests that pinned the old floors now pin the new ones, with the reads that separate them.
116 lines
4.4 KiB
Python
116 lines
4.4 KiB
Python
"""Open-world false-accept probe: unrelated shelf photos through the shipped lane.
|
|
|
|
data/external-shelves/ holds Wikimedia Commons supermarket photos of shelves
|
|
that share nothing with the LOPIA catalog. Every accept here is a false accept
|
|
by construction, which is what makes the set useful: the image6 shelf can only
|
|
measure false accepts on the products it happens to contain.
|
|
|
|
The stored eval.json predates the open-set gate, so it is re-run rather than
|
|
trusted.
|
|
|
|
.venv/bin/python code/eval_distractors.py --stock stock-lopia-clean \
|
|
data/external-shelves/wikimedia-commons
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from typing import Any, Dict, List
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
import cv2 # noqa: E402
|
|
|
|
from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter # noqa: E402
|
|
|
|
|
|
def main(argv: List[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("directories", nargs="+")
|
|
parser.add_argument("--stock", default="stock-lopia-clean")
|
|
parser.add_argument("--write", action="store_true")
|
|
parser.add_argument("--limit", type=int, default=0, help="images per set (0 = all)")
|
|
args = parser.parse_args(argv)
|
|
|
|
from engine import build_default_engine, EngineSettings
|
|
|
|
engine = build_default_engine(EngineSettings(stock_dir=args.stock))
|
|
lane = LocalLaneAdapter(args.stock)
|
|
for directory in args.directories:
|
|
manifest_path = os.path.join(directory, "manifest.json")
|
|
try:
|
|
with open(manifest_path) as handle:
|
|
manifest = json.load(handle)
|
|
except (OSError, ValueError):
|
|
manifest = {"images": []}
|
|
meta = {row["file"]: row for row in manifest.get("images", [])}
|
|
rows: List[Dict[str, Any]] = []
|
|
|
|
paths = sorted(
|
|
path for path in glob.glob(os.path.join(directory, "*"))
|
|
if os.path.splitext(path)[1].lower() in (".jpg", ".jpeg", ".png")
|
|
)
|
|
if args.limit:
|
|
paths = paths[: args.limit]
|
|
for path in paths:
|
|
name = os.path.basename(path)
|
|
image = cv2.imread(path)
|
|
if image is None:
|
|
continue
|
|
started = time.perf_counter()
|
|
detections = engine.detect(image)
|
|
accepted = 0
|
|
for detection in detections:
|
|
box = detection.get("box") or detection.get("bbox")
|
|
if not box:
|
|
continue
|
|
x1, y1, x2, y2 = [int(v) for v in box]
|
|
crop = image[max(0, y1):y2, max(0, x1):x2]
|
|
if crop.size == 0:
|
|
continue
|
|
decision = lane.decide(crop, lane.rank(crop), accept=LOCAL_ACCEPT,
|
|
margin=0.0, scale="full")
|
|
if decision.get("best_sku"):
|
|
accepted += 1
|
|
rows.append({
|
|
"file": name,
|
|
"title": meta.get(name, {}).get("title", ""),
|
|
"detections": len(detections),
|
|
"false_accepts": accepted,
|
|
"ms": round((time.perf_counter() - started) * 1000),
|
|
})
|
|
|
|
detections = sum(r["detections"] for r in rows)
|
|
accepts = sum(r["false_accepts"] for r in rows)
|
|
report = {
|
|
"dataset": os.path.basename(directory.rstrip("/")),
|
|
"catalog": args.stock,
|
|
"note": "Unrelated shelves: every accept is a false accept by construction.",
|
|
"images": len(rows),
|
|
"detections": detections,
|
|
"false_accepts": accepts,
|
|
"false_accept_rate": round(accepts / detections, 4) if detections else None,
|
|
"rows": rows,
|
|
}
|
|
print(f"{report['dataset']}: {len(rows)} images, {detections} detections, "
|
|
f"{accepts} false accepts ({report['false_accept_rate'] or 0:.2%})")
|
|
for row in sorted(rows, key=lambda r: -r["false_accepts"])[:5]:
|
|
if row["false_accepts"]:
|
|
print(f" {row['false_accepts']:>3} {row['file'][:55]}")
|
|
if args.write:
|
|
out = os.path.join(directory, "eval_result.json")
|
|
with open(out, "w") as handle:
|
|
json.dump(report, handle, ensure_ascii=False, indent=1)
|
|
print(f" wrote {out}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|