11826cc43c
A set total hides which products the lane can and cannot name. Every SKU in these sets has images from two independent source domains, so a SKU that scores the same on both is not one lucky photograph. The breakdown is what makes the two interesting results visible: l01, l02, l03, l07 and l09 are 2/2 correct on both their domains, l07 with no enrolled image at all, while l11 abstains on both and l04 abstains on all four of its images.
127 lines
5.3 KiB
Python
127 lines
5.3 KiB
Python
"""Score external-proxy image sets through the shipped identity lane.
|
|
|
|
These sets are not Tier 2 (see data/external-eval/README.md). They exist to test
|
|
the pipeline on data it had no part in building, so the one thing this script
|
|
must never do is score an image that has since been enrolled - that is
|
|
training-set accuracy wearing a held-out label.
|
|
|
|
`leaked()` is the same check the shelf evaluator runs, and it is applied here
|
|
for that reason: enrolling a packshot from an external set is a normal thing to
|
|
do (it is how F-12 was closed), and it silently converts one of these rows into
|
|
a leak unless something refuses it.
|
|
|
|
.venv/bin/python code/eval_external.py --stock stock-lopia-clean \
|
|
data/external-eval/kewpie data/external-eval/other-mayo
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
from typing import Any, Dict, List
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
import cv2 # noqa: E402
|
|
|
|
from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter, leaked # noqa: E402
|
|
|
|
|
|
def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]:
|
|
rows: List[Dict[str, Any]] = []
|
|
for path in sorted(glob.glob(os.path.join(directory, "*.jpg"))):
|
|
name = os.path.basename(path)
|
|
truth = "sku_" + name.split("_")[1]
|
|
image = cv2.imread(path)
|
|
if image is None:
|
|
rows.append({"file": name, "truth": truth, "verdict": "unreadable"})
|
|
continue
|
|
# The whole image is the crop: an external packshot is not a shelf facing.
|
|
box = (0, 0, image.shape[1], image.shape[0])
|
|
leak = leaked(image, box, stock_dir)
|
|
if leak:
|
|
rows.append({
|
|
"file": name, "truth": truth, "verdict": "leaked",
|
|
"leaked_into": leak,
|
|
"note": "enrolled in the catalog; scoring it would be training accuracy",
|
|
})
|
|
continue
|
|
decision = lane.decide(
|
|
image, lane.rank(image), accept=LOCAL_ACCEPT, margin=0.0, scale="full"
|
|
)
|
|
best = decision.get("best_sku")
|
|
verdict = "correct" if best == truth else ("abstain" if not best else "wrong")
|
|
rows.append({
|
|
"file": name, "truth": truth, "best": best,
|
|
"confidence": decision.get("confidence"),
|
|
"status": decision.get("status"),
|
|
"source": decision.get("source"),
|
|
"verdict": verdict,
|
|
})
|
|
|
|
scored = [r for r in rows if r["verdict"] in ("correct", "abstain", "wrong")]
|
|
counts = {v: sum(1 for r in scored if r["verdict"] == v)
|
|
for v in ("correct", "abstain", "wrong")}
|
|
# Per SKU, because a set total hides which products the lane can and cannot
|
|
# name. Every SKU here has images from two independent source domains, so a
|
|
# SKU that scores the same on both is not one lucky photograph.
|
|
by_sku: Dict[str, Dict[str, int]] = {}
|
|
for row in scored:
|
|
bucket = by_sku.setdefault(row["truth"], {"correct": 0, "abstain": 0, "wrong": 0})
|
|
bucket[row["verdict"]] += 1
|
|
return {
|
|
"dataset": os.path.basename(directory.rstrip("/")),
|
|
"catalog": stock_dir,
|
|
"note": "External proxy images, not Tier 2. Scored through the shipped "
|
|
"LocalLaneAdapter + DecisionPolicy.",
|
|
"rows": rows,
|
|
"scored": len(scored),
|
|
"leaked": sum(1 for r in rows if r["verdict"] == "leaked"),
|
|
"by_sku": by_sku,
|
|
**counts,
|
|
}
|
|
|
|
|
|
def main(argv: List[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("directories", nargs="+", help="external set directories")
|
|
parser.add_argument("--stock", default="stock-lopia-clean")
|
|
parser.add_argument("--write", action="store_true",
|
|
help="write eval_result.json into each set directory")
|
|
args = parser.parse_args(argv)
|
|
|
|
lane = LocalLaneAdapter(args.stock)
|
|
totals = {"correct": 0, "abstain": 0, "wrong": 0, "leaked": 0, "scored": 0}
|
|
for directory in args.directories:
|
|
report = score_set(lane, directory, args.stock)
|
|
for key in totals:
|
|
totals[key] += report[key]
|
|
print(f"{report['dataset']}: correct {report['correct']} | "
|
|
f"abstain {report['abstain']} | wrong {report['wrong']} | "
|
|
f"leaked {report['leaked']} of {report['scored']} scored")
|
|
for sku, bucket in sorted(report["by_sku"].items()):
|
|
print(f" {sku:<10} correct {bucket['correct']} "
|
|
f"abstain {bucket['abstain']} wrong {bucket['wrong']}")
|
|
for row in report["rows"]:
|
|
if row["verdict"] in ("wrong", "leaked"):
|
|
print(f" {row['verdict'].upper():<7} {row['file']} "
|
|
f"truth={row['truth']} best={row.get('best')} "
|
|
f"{row.get('leaked_into', '')}")
|
|
if args.write:
|
|
out = os.path.join(directory, "eval_result.json")
|
|
with open(out, "w") as handle:
|
|
json.dump(report, handle, ensure_ascii=False, indent=1)
|
|
print(f" wrote {out}")
|
|
print(f"\nTOTAL: {totals['scored']} scored, correct {totals['correct']}, "
|
|
f"abstain {totals['abstain']}, FALSE ACCEPTS {totals['wrong']}, "
|
|
f"leaked {totals['leaked']}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|