feat(eval): report the external benchmark per SKU

A set total hides which products the lane can and cannot name. Every SKU in
these sets has images from two independent source domains, so a SKU that scores
the same on both is not one lucky photograph.

The breakdown is what makes the two interesting results visible: l01, l02, l03,
l07 and l09 are 2/2 correct on both their domains, l07 with no enrolled image
at all, while l11 abstains on both and l04 abstains on all four of its images.

recognization/code/eval_external.py

@@ -67,6 +67,13 @@ def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]:
     scored = [r for r in rows if r["verdict"] in ("correct", "abstain", "wrong")]
     counts = {v: sum(1 for r in scored if r["verdict"] == v)
               for v in ("correct", "abstain", "wrong")}
+    # Per SKU, because a set total hides which products the lane can and cannot
+    # name. Every SKU here has images from two independent source domains, so a
+    # SKU that scores the same on both is not one lucky photograph.
+    by_sku: Dict[str, Dict[str, int]] = {}
+    for row in scored:
+        bucket = by_sku.setdefault(row["truth"], {"correct": 0, "abstain": 0, "wrong": 0})
+        bucket[row["verdict"]] += 1
     return {
         "dataset": os.path.basename(directory.rstrip("/")),
         "catalog": stock_dir,
@@ -75,6 +82,7 @@ def score_set(lane: Any, directory: str, stock_dir: str) -> Dict[str, Any]:
         "rows": rows,
         "scored": len(scored),
         "leaked": sum(1 for r in rows if r["verdict"] == "leaked"),
+        "by_sku": by_sku,
         **counts,
     }
 
@@ -96,6 +104,9 @@ def main(argv: List[str] | None = None) -> int:
         print(f"{report['dataset']}: correct {report['correct']} | "
               f"abstain {report['abstain']} | wrong {report['wrong']} | "
               f"leaked {report['leaked']} of {report['scored']} scored")
+        for sku, bucket in sorted(report["by_sku"].items()):
+            print(f"    {sku:<10} correct {bucket['correct']}  "
+                  f"abstain {bucket['abstain']}  wrong {bucket['wrong']}")
         for row in report["rows"]:
             if row["verdict"] in ("wrong", "leaked"):
                 print(f"    {row['verdict'].upper():<7} {row['file']} "

recognization/data/external-eval/kewpie/eval_result.json

@@ -112,6 +112,38 @@
  ],
  "scored": 11,
  "leaked": 1,
+ "by_sku": {
+  "sku_l01": {
+   "correct": 2,
+   "abstain": 0,
+   "wrong": 0
+  },
+  "sku_l02": {
+   "correct": 2,
+   "abstain": 0,
+   "wrong": 0
+  },
+  "sku_l03": {
+   "correct": 2,
+   "abstain": 0,
+   "wrong": 0
+  },
+  "sku_l05": {
+   "correct": 1,
+   "abstain": 1,
+   "wrong": 0
+  },
+  "sku_l06": {
+   "correct": 1,
+   "abstain": 0,
+   "wrong": 0
+  },
+  "sku_l11": {
+   "correct": 0,
+   "abstain": 2,
+   "wrong": 0
+  }
+ },
  "correct": 8,
  "abstain": 3,
  "wrong": 0

recognization/data/external-eval/other-mayo/eval_result.json

@@ -78,6 +78,23 @@
  ],
  "scored": 8,
  "leaked": 0,
+ "by_sku": {
+  "sku_l04": {
+   "correct": 0,
+   "abstain": 4,
+   "wrong": 0
+  },
+  "sku_l07": {
+   "correct": 2,
+   "abstain": 0,
+   "wrong": 0
+  },
+  "sku_l09": {
+   "correct": 2,
+   "abstain": 0,
+   "wrong": 0
+  }
+ },
  "correct": 4,
  "abstain": 4,
  "wrong": 0