diff --git a/recognization/code/eval_identity.py b/recognization/code/eval_identity.py index 05c7a30..2f9d5f8 100644 --- a/recognization/code/eval_identity.py +++ b/recognization/code/eval_identity.py @@ -487,6 +487,10 @@ def score_lane( unenrolled_count = len(in_catalog) - enrolled_count correct = wrong = rejected = top1_correct = 0 + # The decision ceiling is over enrolled facings only. The text lane can + # answer an unenrolled sibling correctly, and counting that in a ratio + # whose denominator is the enrolled count reports a ceiling above 100%. + enrolled_correct = 0 unknown_false = unenrolled_false = 0 accepts: Counter = Counter() per_sku: Dict[str, List[int]] = {} @@ -510,6 +514,8 @@ def score_lane( if answered == truth: correct += 1 bucket[0] += 1 + if truth in enrolled_skus: + enrolled_correct += 1 elif answered: wrong += 1 if truth not in enrolled_skus: @@ -537,7 +543,7 @@ def score_lane( "rejected": rejected, "catalog_ceiling": round(enrolled_count / len(in_catalog), 4) if in_catalog else None, "identity_ceiling": round(top1_correct / enrolled_count, 4) if enrolled_count else None, - "decision_ceiling": round(correct / enrolled_count, 4) if enrolled_count else None, + "decision_ceiling": round(enrolled_correct / enrolled_count, 4) if enrolled_count else None, "end_to_end_coverage": round(correct / len(in_catalog), 4) if in_catalog else None, "false_accepts": wrong, "unknown_false_accepts": unknown_false,