feat: refuse a label file whose rows carry no provenance
F-01's target was 'every scored label carries provenance', and the fields existed but nothing enforced them: a v1-style file with no 'source' loaded exactly like the audited one, which is how 23% wrong labels went unnoticed. load_labels now refuses such a file by name and index. The v2 labels load unchanged; a stripped copy is refused. 3 tests. Suite 281 -> 284.
This commit is contained in:
parent
b4d7106d28
commit
80e69e74df
2 changed files with 76 additions and 0 deletions
|
|
@ -81,10 +81,28 @@ LEAK_SIGNATURE_MAD = 20.0
|
||||||
|
|
||||||
|
|
||||||
def load_labels(path: str) -> Dict[str, Any]:
|
def load_labels(path: str) -> Dict[str, Any]:
|
||||||
|
"""Load a label file, refusing one whose rows carry no provenance.
|
||||||
|
|
||||||
|
F-01: the v1 labels were wrong on 23% of facings and nothing in the file
|
||||||
|
said where any of them came from, so a bad label was indistinguishable from
|
||||||
|
a good one. Every row must now name its `source`; a row that cannot be
|
||||||
|
traced is a row that cannot be argued with.
|
||||||
|
"""
|
||||||
with open(path) as handle:
|
with open(path) as handle:
|
||||||
payload = json.load(handle)
|
payload = json.load(handle)
|
||||||
if not isinstance(payload, dict) or "facings" not in payload:
|
if not isinstance(payload, dict) or "facings" not in payload:
|
||||||
raise SystemExit(f"{path}: expected {{'image', 'facings': [...]}}")
|
raise SystemExit(f"{path}: expected {{'image', 'facings': [...]}}")
|
||||||
|
missing = [
|
||||||
|
row.get("index", i)
|
||||||
|
for i, row in enumerate(payload["facings"])
|
||||||
|
if not row.get("source")
|
||||||
|
]
|
||||||
|
if missing:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{path}: {len(missing)} facings carry no 'source' "
|
||||||
|
f"(first: index {missing[0]}). A label without provenance cannot be "
|
||||||
|
"audited; see the enrollment protocol in DECISION.md."
|
||||||
|
)
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
58
recognization/code/test_eval_labels.py
Normal file
58
recognization/code/test_eval_labels.py
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
"""
|
||||||
|
The evaluator's label-file contract.
|
||||||
|
|
||||||
|
F-01: the v1 labels were wrong on 23% of facings and the file recorded nothing
|
||||||
|
about where any of them came from, so a bad label could not be told from a good
|
||||||
|
one. `load_labels` now refuses a file whose rows carry no provenance.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
if current_dir not in sys.path:
|
||||||
|
sys.path.insert(0, current_dir)
|
||||||
|
|
||||||
|
from eval_identity import load_labels # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def write_labels(payload) -> str:
|
||||||
|
handle = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False)
|
||||||
|
json.dump(payload, handle)
|
||||||
|
handle.close()
|
||||||
|
return handle.name
|
||||||
|
|
||||||
|
|
||||||
|
class LoadLabelsTests(unittest.TestCase):
|
||||||
|
def test_a_traced_label_file_loads(self):
|
||||||
|
path = write_labels({
|
||||||
|
"image": "add/image6.jpg",
|
||||||
|
"facings": [{"index": 0, "label": "sku_l01", "source": "visual-audit"}],
|
||||||
|
})
|
||||||
|
self.addCleanup(os.unlink, path)
|
||||||
|
self.assertEqual(len(load_labels(path)["facings"]), 1)
|
||||||
|
|
||||||
|
def test_a_label_without_provenance_is_refused(self):
|
||||||
|
path = write_labels({
|
||||||
|
"image": "add/image6.jpg",
|
||||||
|
"facings": [{"index": 0, "label": "sku_l01"}],
|
||||||
|
})
|
||||||
|
self.addCleanup(os.unlink, path)
|
||||||
|
with self.assertRaises(SystemExit) as caught:
|
||||||
|
load_labels(path)
|
||||||
|
self.assertIn("source", str(caught.exception))
|
||||||
|
|
||||||
|
def test_a_file_without_facings_is_refused(self):
|
||||||
|
path = write_labels({"image": "add/image6.jpg"})
|
||||||
|
self.addCleanup(os.unlink, path)
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
load_labels(path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Add table
Add a link
Reference in a new issue