diff --git a/recognization/code/eval_external.py b/recognization/code/eval_external.py index 631c1ee..1635d98 100644 --- a/recognization/code/eval_external.py +++ b/recognization/code/eval_external.py @@ -67,6 +67,13 @@ def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]: scored = [r for r in rows if r["verdict"] in ("correct", "abstain", "wrong")] counts = {v: sum(1 for r in scored if r["verdict"] == v) for v in ("correct", "abstain", "wrong")} + # Per SKU, because a set total hides which products the lane can and cannot + # name. Every SKU here has images from two independent source domains, so a + # SKU that scores the same on both is not one lucky photograph. + by_sku: Dict[str, Dict[str, int]] = {} + for row in scored: + bucket = by_sku.setdefault(row["truth"], {"correct": 0, "abstain": 0, "wrong": 0}) + bucket[row["verdict"]] += 1 return { "dataset": os.path.basename(directory.rstrip("/")), "catalog": stock_dir, @@ -75,6 +82,7 @@ def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]: "rows": rows, "scored": len(scored), "leaked": sum(1 for r in rows if r["verdict"] == "leaked"), + "by_sku": by_sku, **counts, } @@ -96,6 +104,9 @@ def main(argv: List[str] | None = None) -> int: print(f"{report['dataset']}: correct {report['correct']} | " f"abstain {report['abstain']} | wrong {report['wrong']} | " f"leaked {report['leaked']} of {report['scored']} scored") + for sku, bucket in sorted(report["by_sku"].items()): + print(f" {sku:<10} correct {bucket['correct']} " + f"abstain {bucket['abstain']} wrong {bucket['wrong']}") for row in report["rows"]: if row["verdict"] in ("wrong", "leaked"): print(f" {row['verdict'].upper():<7} {row['file']} " diff --git a/recognization/data/external-eval/kewpie/eval_result.json b/recognization/data/external-eval/kewpie/eval_result.json index 91591b5..8937e85 100644 --- a/recognization/data/external-eval/kewpie/eval_result.json +++ b/recognization/data/external-eval/kewpie/eval_result.json @@ -112,6 +112,38 @@ ], "scored": 11, "leaked": 1, + "by_sku": { + "sku_l01": { + "correct": 2, + "abstain": 0, + "wrong": 0 + }, + "sku_l02": { + "correct": 2, + "abstain": 0, + "wrong": 0 + }, + "sku_l03": { + "correct": 2, + "abstain": 0, + "wrong": 0 + }, + "sku_l05": { + "correct": 1, + "abstain": 1, + "wrong": 0 + }, + "sku_l06": { + "correct": 1, + "abstain": 0, + "wrong": 0 + }, + "sku_l11": { + "correct": 0, + "abstain": 2, + "wrong": 0 + } + }, "correct": 8, "abstain": 3, "wrong": 0 diff --git a/recognization/data/external-eval/other-mayo/eval_result.json b/recognization/data/external-eval/other-mayo/eval_result.json index eed3924..075f01f 100644 --- a/recognization/data/external-eval/other-mayo/eval_result.json +++ b/recognization/data/external-eval/other-mayo/eval_result.json @@ -78,6 +78,23 @@ ], "scored": 8, "leaked": 0, + "by_sku": { + "sku_l04": { + "correct": 0, + "abstain": 4, + "wrong": 0 + }, + "sku_l07": { + "correct": 2, + "abstain": 0, + "wrong": 0 + }, + "sku_l09": { + "correct": 2, + "abstain": 0, + "wrong": 0 + } + }, "correct": 4, "abstain": 4, "wrong": 0