5094b5f6ca
Every threshold was fitted on image6, where each crop is a facing of a product the shelf holds, so the set could not price a false accept. Measured on 4580 crops from 64 unrelated Wikimedia shelf photos at the shipped configuration: LOCAL_ACCEPT 2.0 -> 4.0 false accepts 60 (1.31%) -> 3 (0.07%) TEXT_MIN_SHARED 2 -> 4 the two-bigram reads that named sku_l12 via おい/いし On image6 that is 44 shown / 42 correct at full resolution - 95.5% precision, 65.6% coverage - the highest-coverage cell of the joint sweep that passes both G2 (>= 95%) and G3 (<= 1%). At live resolution 56% end-to-end at 97%. The text lane's evidence floor gates its answer only: zeroing the row's score would reorder the ranking and make the margin gate compare a different pair than the one it was measured on. code/sweep_accept_lines.py sweeps both axes jointly through the shipped DecisionPolicy; the earlier table swept one axis while pinning the other, so its baseline rows were not the old line.
328 lines
14 KiB
Python
328 lines
14 KiB
Python
"""Score a streamed clip's labels against ground truth: the G1 number.
|
|
|
|
The stream path was reported as "53 of 131 tracks", which is not comparable to
|
|
Van's 57% of labelled facings - the two count different things. But the clip is
|
|
built from `add/image6.jpg`, which has labels, so matching each confirmed
|
|
track's `label_box` to the labelled facings gives stream-path coverage directly.
|
|
|
|
python code/score_stream.py TRACKS.json --labels eval/labels/image6.json
|
|
|
|
`TRACKS.json` is what `stream_client.py --tracks` writes: one row per track that
|
|
settled on a label, with the box that label was measured on, already mapped back
|
|
to source-image coordinates.
|
|
|
|
Reported per run: correct / in-catalog facings, precision, unknown accepts, and
|
|
tracks that matched no labelled facing. A track whose box overlaps a facing the
|
|
catalog does not enrol is an unenrolled false accept - the dangerous kind, and
|
|
the one the evaluator's own precision denominator used to hide.
|
|
|
|
G1 has two halves - new covers at least as much as Van, and the lower bound of
|
|
the paired-bootstrap difference clears -2 pp - and the second half needs the
|
|
per-facing pairing, so every row carries the `facing` index it matched. Two
|
|
reports plus the labels then answer the gate on the same facings:
|
|
|
|
python code/score_stream.py NEW.json --van VAN.json
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import random
|
|
import sys
|
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
# A facing clipped by the window edge is not something a run could have named,
|
|
# so the denominator keeps only facings at least this far inside the window.
|
|
VISIBLE_MIN = 0.5
|
|
|
|
# G1's second half: the lower bound of the paired difference must clear -2 pp.
|
|
G1_BOUND_MIN = -0.02
|
|
|
|
# Resampling draws for the paired bootstrap, and the seed that makes the bound
|
|
# a number this repo can print twice.
|
|
BOOTSTRAP_DRAWS = 10000
|
|
BOOTSTRAP_SEED = 0
|
|
|
|
|
|
def iou(a: Sequence[int], b: Sequence[int]) -> float:
|
|
ax1, ay1, ax2, ay2 = a
|
|
bx1, by1, bx2, by2 = b
|
|
inter_w = max(0, min(ax2, bx2) - max(ax1, bx1))
|
|
inter_h = max(0, min(ay2, by2) - max(ay1, by1))
|
|
inter = inter_w * inter_h
|
|
area_a = max(0, ax2 - ax1) * max(0, ay2 - ay1)
|
|
area_b = max(0, bx2 - bx1) * max(0, by2 - by1)
|
|
union = area_a + area_b - inter
|
|
return inter / union if union > 0 else 0.0
|
|
|
|
|
|
def load_facings(labels_path: str) -> List[Dict[str, Any]]:
|
|
with open(labels_path) as handle:
|
|
payload = json.load(handle)
|
|
return payload["facings"]
|
|
|
|
|
|
def enrolled_skus(stock_dir: str) -> set:
|
|
"""Canonical SKUs with at least one enrolled view."""
|
|
from rebuild_catalog import sku_of
|
|
|
|
return {
|
|
sku_of(name) for name in os.listdir(stock_dir)
|
|
if name.lower().endswith((".jpg", ".jpeg", ".png"))
|
|
}
|
|
|
|
|
|
def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float:
|
|
"""Share of a facing's area inside the window the client actually showed."""
|
|
wx0, wy0, wx1, wy1 = window
|
|
bx0, by0, bx1, by1 = box
|
|
inter_w = max(0, min(bx1, wx1) - max(bx0, wx0))
|
|
inter_h = max(0, min(by1, wy1) - max(by0, wy0))
|
|
area = max(0, bx1 - bx0) * max(0, by1 - by0)
|
|
return (inter_w * inter_h) / area if area > 0 else 0.0
|
|
|
|
|
|
def reachable(facings: Sequence[Dict[str, Any]],
|
|
windows: Optional[Sequence[Sequence[int]]]
|
|
) -> List[Dict[str, Any]]:
|
|
"""Facings a run's windows showed well enough for the run to have named."""
|
|
if not windows:
|
|
return list(facings)
|
|
return [f for f in facings
|
|
if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN]
|
|
|
|
|
|
def correct_facings(rows: Sequence[Dict[str, Any]]) -> set:
|
|
"""Facing indices at least one track got right - one hit each, not one per track."""
|
|
return {row["facing"] for row in rows
|
|
if row["outcome"] == "correct" and row["facing"] is not None}
|
|
|
|
|
|
def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
|
|
stock_dir: str, iou_min: float,
|
|
windows: Optional[Sequence[Sequence[int]]] = None
|
|
) -> Dict[str, Any]:
|
|
"""Score the labels a stream settled on.
|
|
|
|
`windows` are the source-image rects the client showed, as
|
|
[x0, y0, x1, y1] each. The denominator is the facings at least half inside
|
|
one of them, in x and y both: a 1280x720 window over a 1500x2000 photo sees
|
|
24 of the 63 in-catalog facings, and a facing clipped by the window edge is
|
|
not something any run could have named. A panned run shows a different rect
|
|
each frame, so the reachable set is the union. Without it the denominator is
|
|
every in-catalog facing.
|
|
|
|
A correct answer is counted before the unenrolled and unknown branches. The
|
|
text lane names SKUs the image lane has no view for, which is the point of
|
|
it, and scoring that as a false accept punishes the lane for working.
|
|
"""
|
|
facings = reachable(facings, windows)
|
|
in_catalog = [f for f in facings if f["label"] != "unknown"]
|
|
enrolled = enrolled_skus(stock_dir)
|
|
|
|
correct = 0
|
|
wrong = 0
|
|
unenrolled_false = 0
|
|
unknown_false = 0
|
|
unmatched = 0
|
|
rows: List[Dict[str, Any]] = []
|
|
# Coverage is per facing, precision is per label. A track that restarts on
|
|
# the same product, or two tracks on one facing, is two claims but one
|
|
# facing: counting tracks as coverage reports 138% of a 24-facing window.
|
|
named: Dict[int, str] = {}
|
|
|
|
for track in tracks:
|
|
box = track.get("label_box") or []
|
|
if len(box) != 4:
|
|
unmatched += 1
|
|
continue
|
|
best, best_iou = None, 0.0
|
|
for facing in facings:
|
|
value = iou(box, facing["box"])
|
|
if value > best_iou:
|
|
best, best_iou = facing, value
|
|
if best is None or best_iou < iou_min:
|
|
unmatched += 1
|
|
rows.append({"track_id": track["track_id"], "predicted": track["sku_id"],
|
|
"facing": None, "truth": None, "iou": round(best_iou, 3),
|
|
"outcome": "unmatched"})
|
|
continue
|
|
truth = best["label"]
|
|
predicted = track["sku_id"]
|
|
if predicted == truth:
|
|
correct += 1
|
|
outcome = "correct"
|
|
named[best["index"]] = predicted
|
|
elif truth == "unknown":
|
|
unknown_false += 1
|
|
outcome = "unknown_false_accept"
|
|
elif truth not in enrolled:
|
|
unenrolled_false += 1
|
|
outcome = "unenrolled_false_accept"
|
|
else:
|
|
wrong += 1
|
|
outcome = "wrong"
|
|
rows.append({"track_id": track["track_id"], "predicted": predicted,
|
|
"facing": best["index"], "truth": truth,
|
|
"iou": round(best_iou, 3), "outcome": outcome})
|
|
|
|
shown = correct + wrong + unenrolled_false + unknown_false
|
|
return {
|
|
"in_catalog_facings": len(in_catalog),
|
|
"facings_named": len(named),
|
|
"tracks_scored": len(tracks),
|
|
"correct": correct,
|
|
"wrong": wrong,
|
|
"unenrolled_false_accepts": unenrolled_false,
|
|
"unknown_false_accepts": unknown_false,
|
|
"unmatched_tracks": unmatched,
|
|
"shown": shown,
|
|
"coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0,
|
|
"precision": round(correct / shown, 4) if shown else 0.0,
|
|
# The window set travels with the report: a paired comparison has to
|
|
# know what each run was shown, not just what it named.
|
|
"windows": [list(w) for w in windows] if windows else None,
|
|
"rows": rows,
|
|
}
|
|
|
|
|
|
def paired_bootstrap(van: Dict[str, Any], new: Dict[str, Any],
|
|
facings: Sequence[Dict[str, Any]],
|
|
draws: int = BOOTSTRAP_DRAWS,
|
|
seed: int = BOOTSTRAP_SEED) -> Dict[str, Any]:
|
|
"""G1's second half: new minus Van, per facing, with the 95 % lower bound.
|
|
|
|
Both reports are graded on one facing set - the in-catalog facings either
|
|
run's windows showed - so a facing one run panned past cannot flatter the
|
|
other. Resampling draws facings with replacement rather than comparing two
|
|
independent rates, because the two runs saw the same photo: pairing removes
|
|
the facing-to-facing variance that otherwise swamps a 27-facing sample.
|
|
|
|
Percentile method on the sorted draw differences; the seed is fixed so the
|
|
number in a report is one this repo reproduces.
|
|
"""
|
|
windows = list(van.get("windows") or []) + list(new.get("windows") or [])
|
|
shared = sorted(f["index"] for f in reachable(facings, windows)
|
|
if f["label"] != "unknown")
|
|
if not shared:
|
|
return {"facings": 0, "difference": 0.0, "lower_95": 0.0, "upper_95": 0.0,
|
|
"van": 0, "new": 0, "draws": draws, "seed": seed}
|
|
|
|
van_hits = correct_facings(van["rows"])
|
|
new_hits = correct_facings(new["rows"])
|
|
van_vector = [1 if i in van_hits else 0 for i in shared]
|
|
new_vector = [1 if i in new_hits else 0 for i in shared]
|
|
n = len(shared)
|
|
|
|
rng = random.Random(seed)
|
|
spread = []
|
|
for _ in range(draws):
|
|
van_sum = new_sum = 0
|
|
for _ in range(n):
|
|
# The same draw index into both vectors: that is the pairing.
|
|
drawn = rng.randrange(n)
|
|
van_sum += van_vector[drawn]
|
|
new_sum += new_vector[drawn]
|
|
spread.append((new_sum - van_sum) / n)
|
|
spread.sort()
|
|
tail = max(1, int(0.025 * draws))
|
|
return {
|
|
"facings": n,
|
|
"van": sum(van_vector),
|
|
"new": sum(new_vector),
|
|
"van_rate": round(sum(van_vector) / n, 4),
|
|
"new_rate": round(sum(new_vector) / n, 4),
|
|
"difference": round((sum(new_vector) - sum(van_vector)) / n, 4),
|
|
"lower_95": round(spread[tail - 1], 4),
|
|
"upper_95": round(spread[min(draws - 1, int(0.975 * draws))], 4),
|
|
"draws": draws,
|
|
"seed": seed,
|
|
}
|
|
|
|
|
|
def load_run(path: str, window: Optional[str] = None
|
|
) -> Tuple[Dict[str, Any], Optional[List[Tuple[int, ...]]]]:
|
|
"""A stream_client --tracks dump and the source-image windows it showed."""
|
|
with open(path) as handle:
|
|
payload = json.load(handle)
|
|
if window:
|
|
windows = [tuple(int(part) for part in window.split(","))]
|
|
elif payload.get("windows"):
|
|
windows = [tuple(w) for w in payload["windows"]]
|
|
else:
|
|
windows = None
|
|
return payload, windows
|
|
|
|
|
|
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Score streamed labels against ground truth")
|
|
parser.add_argument("tracks", help="JSON written by stream_client.py --tracks")
|
|
parser.add_argument("--labels", default="eval/labels/image6.json")
|
|
parser.add_argument("--stock-dir", default="stock-lopia-clean")
|
|
parser.add_argument("--iou", type=float, default=0.5)
|
|
parser.add_argument("--json", dest="json_path", default=None)
|
|
parser.add_argument("--window", default=None,
|
|
help="x0,y0,x1,y1 the client showed; the denominator is "
|
|
"restricted to facings at least half inside it")
|
|
parser.add_argument("--van", default=None,
|
|
help="Van baseline track dump on the same clip; scores G1's "
|
|
"paired-bootstrap half against it")
|
|
parser.add_argument("--draws", type=int, default=BOOTSTRAP_DRAWS)
|
|
parser.add_argument("--seed", type=int, default=BOOTSTRAP_SEED)
|
|
parser.add_argument("--verbose", action="store_true", help="print every scored track")
|
|
args = parser.parse_args(argv)
|
|
|
|
facings = load_facings(args.labels)
|
|
payload, windows = load_run(args.tracks, args.window)
|
|
report = score(payload["tracks"], facings, args.stock_dir, args.iou, windows)
|
|
report["source"] = payload.get("source")
|
|
report["pace"] = payload.get("pace")
|
|
report["seconds"] = payload.get("seconds")
|
|
|
|
print(f"{os.path.basename(args.tracks)} pace={report['pace']} "
|
|
f"{report['seconds']}s")
|
|
print(f" facings named {report['facings_named']}/{report['in_catalog_facings']} "
|
|
f"in-catalog ({report['coverage']:.0%}) labels correct {report['correct']} "
|
|
f"precision {report['precision']:.0%} shown {report['shown']}")
|
|
print(f" wrong {report['wrong']} unenrolled false accepts "
|
|
f"{report['unenrolled_false_accepts']} unknown false accepts "
|
|
f"{report['unknown_false_accepts']} unmatched tracks {report['unmatched_tracks']}")
|
|
if args.verbose:
|
|
for row in report["rows"]:
|
|
print(f" t{row['track_id']:4d} {row['outcome']:24s} "
|
|
f"facing={row['facing']} pred={row['predicted']} "
|
|
f"truth={row['truth']} iou={row['iou']}")
|
|
|
|
if args.van:
|
|
van_payload, van_windows = load_run(args.van, args.window)
|
|
van_report = score(van_payload["tracks"], facings, args.stock_dir,
|
|
args.iou, van_windows)
|
|
paired = paired_bootstrap(van_report, report, facings, args.draws, args.seed)
|
|
report["van"] = {k: van_report[k] for k in
|
|
("in_catalog_facings", "facings_named", "coverage",
|
|
"correct", "wrong", "precision", "shown")}
|
|
report["g1_paired"] = paired
|
|
print(f" G1 vs {os.path.basename(args.van)}: new {paired['new']}/"
|
|
f"{paired['facings']} ({paired['new_rate']:.0%}) - van {paired['van']}/"
|
|
f"{paired['facings']} ({paired['van_rate']:.0%}) = "
|
|
f"{paired['difference'] * 100:+.1f} pp")
|
|
print(f" paired bootstrap {paired['draws']} draws seed {paired['seed']}: "
|
|
f"95% bounds {paired['lower_95'] * 100:+.1f} pp to "
|
|
f"{paired['upper_95'] * 100:+.1f} pp; lower bound clears "
|
|
f"{G1_BOUND_MIN * 100:+.0f} pp: "
|
|
f"{'yes' if paired['lower_95'] >= G1_BOUND_MIN else 'NO'}")
|
|
|
|
if args.json_path:
|
|
with open(args.json_path, "w") as handle:
|
|
json.dump(report, handle, indent=2)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|