vnguyen/product-recognition-demo
1
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix(recognition): calibrate the accept lines against unrelated shelves

Browse source 
Every threshold was fitted on image6, where each crop is a facing of a product
the shelf holds, so the set could not price a false accept. Measured on 4580
crops from 64 unrelated Wikimedia shelf photos at the shipped configuration:

  LOCAL_ACCEPT 2.0 -> 4.0   false accepts 60 (1.31%) -> 3 (0.07%)
  TEXT_MIN_SHARED 2 -> 4    the two-bigram reads that named sku_l12 via おい/いし

On image6 that is 44 shown / 42 correct at full resolution - 95.5% precision,
65.6% coverage - the highest-coverage cell of the joint sweep that passes both
G2 (>= 95%) and G3 (<= 1%). At live resolution 56% end-to-end at 97%.

The text lane's evidence floor gates its answer only: zeroing the row's score
would reorder the ranking and make the margin gate compare a different pair
than the one it was measured on.

code/sweep_accept_lines.py sweeps both axes jointly through the shipped
DecisionPolicy; the earlier table swept one axis while pinning the other, so
its baseline rows were not the old line.
This commit is contained in:
kiyo9w 2026-09-29 09:28:13 +09:00
parent 4ce41ecb17
commit 5094b5f6ca
12 changed files with 815 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

30
recognization/code/eval_distractors.py
View file

@ -5,8 +5,11 @@ that share nothing with the LOPIA catalog. Every accept here is a false accept
by construction, which is what makes the set useful: the image6 shelf can only
measure false accepts on the products it happens to contain.
The stored eval.json predates the open-set gate, so it is re-run rather than
trusted.
Measured the way the server measures: a 1280-long-side frame detected at
`LiveSettings.conf`. The sources are 4032-4512 px photos, and detecting them at
full resolution and at the default `detect_conf` instead measures a
configuration the phone never runs, which is how a threshold change reads as a
pass on evidence the product does not produce.
.venv/bin/python code/eval_distractors.py --stock stock-lopia-clean \
data/external-shelves/wikimedia-commons
@ -28,7 +31,13 @@ if current_dir not in sys.path:
import cv2 # noqa: E402
from eval_identity import LOCAL_ACCEPT, LocalLaneAdapter # noqa: E402
from eval_identity import ( # noqa: E402
LIVE_LONG_SIDE,
LOCAL_ACCEPT,
LocalLaneAdapter,
at_live_resolution,
)
from identity_lanes import TEXT_MIN_SHARED # noqa: E402
def main(argv: List[str] | None = None) -> int:
@ -40,9 +49,11 @@ def main(argv: List[str] | None = None) -> int:
args = parser.parse_args(argv)
from engine import build_default_engine, EngineSettings
from live import LiveSettings
engine = build_default_engine(EngineSettings(stock_dir=args.stock))
lane = LocalLaneAdapter(args.stock)
conf = LiveSettings().conf
for directory in args.directories:
manifest_path = os.path.join(directory, "manifest.json")
try:
@ -65,14 +76,15 @@ def main(argv: List[str] | None = None) -> int:
if image is None:
continue
started = time.perf_counter()
detections = engine.detect(image)
frame = at_live_resolution(image)
detections = engine.detect(frame, conf=conf)
accepted = 0
for detection in detections:
box = detection.get("box") or detection.get("bbox")
if not box:
continue
x1, y1, x2, y2 = [int(v) for v in box]
crop = image[max(0, y1):y2, max(0, x1):x2]
crop = frame[max(0, y1):y2, max(0, x1):x2]
if crop.size == 0:
continue
decision = lane.decide(crop, lane.rank(crop), accept=LOCAL_ACCEPT,
@ -93,6 +105,14 @@ def main(argv: List[str] | None = None) -> int:
"dataset": os.path.basename(directory.rstrip("/")),
"catalog": args.stock,
"note": "Unrelated shelves: every accept is a false accept by construction.",
# The configuration the numbers above were measured under, so the
# stored evidence is checkable against the constants it claims.
"config": {
"frame_long_side": LIVE_LONG_SIDE,
"detect_conf": conf,
"local_accept": LOCAL_ACCEPT,
"text_min_shared": TEXT_MIN_SHARED,
},
"images": len(rows),
"detections": detections,
"false_accepts": accepts,

22
recognization/code/eval_identity.py
View file

@ -418,11 +418,16 @@ class LocalLaneAdapter(Lane):
self.lane = LocalLane(stock_dir)
# E-3 asks what the text lane is worth, which is only answerable by
# running the same crops without it. Building the lane anyway keeps the
# comparison to one variable: the OCR cost is paid either way.
# running the same crops without it. `--no-text` therefore drops the
# lane whole: nothing in the decision reads it, and an ablation that
# still had to build an OCR engine could not be run on a machine
# without one.
self.use_text = use_text
self.text_lane = TextLane.from_catalog(
_catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine()
self.text_lane = (
TextLane.from_catalog(
_catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine()
)
if use_text else None
)
def rank(self, crop: Any) -> List[Tuple[str, float]]:
@ -433,9 +438,10 @@ class LocalLaneAdapter(Lane):
from identity_lanes import DecisionPolicy
if not self.use_text:
from identity import decide_identity
return decide_identity(dict(ranked), scale="local", min_score=accept)
# E-3's other arm: the same policy, over the same ranking, with no
# text evidence. `dict(ranked)` would not do - it drops the lane's
# order, and the value is not the order (see `LocalLane.rank`).
return DecisionPolicy(local_accept=accept).decide(ranked)
read = self.text_lane.read(crop)
text_rank = sorted(
self.text_lane.scores_from_text(read).items(),
@ -523,8 +529,6 @@ def score_lane(
the SKU is on the shelf but has no reference, so an accept there is wrong
and is the failure the enrollment protocol exists to prevent.
"""
from identity import decide_identity
in_catalog = [f for f in facings if f["label"] != UNKNOWN]
out_catalog = [f for f in facings if f["label"] == UNKNOWN]
enrolled_count = sum(1 for f in in_catalog if f["label"] in enrolled_skus)

4
recognization/code/eval_live.py
View file

@ -11,7 +11,7 @@ store was enrolled; scoring them is training-set accuracy.
python code/eval_live.py RECORDING_DIR [--stock-dir DIR] [--labels PATH]
[--json PATH] [--iou 0.5]
[--replay] [--identity matcher|local|gallery] [--device cpu|mps|cuda]
[--replay] [--identity matcher|local|gallery|van] [--device cpu|mps|cuda]
Prints a human count table and writes the same report as JSON (default
<recording>/eval_live.json). Missing input exits 2 with {"error": ...} on
@ -66,7 +66,7 @@ def content_sha256(payload: bytes) -> str:
# Every backend the server can be started with, so a recording made by the
# shipped `run-demo-server.sh` (--identity local) can be replayed. `local` is
# the default; `matcher` is Van's scorer kept as the E-1 baseline lane.
IDENTITY_BACKENDS = ("matcher", "local", "gallery")
IDENTITY_BACKENDS = ("matcher", "local", "gallery", "van")
def _iou(a: Sequence[float], b: Sequence[float]) -> float:

57
recognization/code/identity_lanes.py
View file

@ -45,28 +45,38 @@ from rebuild_catalog import bigrams, normalize_text, signature_bigrams, sku_of
# wrong label on a shelf is worse than a missing one.
#
# 4.0 since 2026-09-29: the line was tuned on image6 alone, where every crop is
# a facing of a product the shelf actually holds. On 2297 crops from unrelated
# Wikimedia shelves, 2.0 accepted 25 of them on geometry alone (scores 2.26-4.81
# with no text overlap at all), a 1.1% far-unknown rate against G3's 1%. At 4.0
# that falls to 8 (0.35%) while image6 keeps 42 of 48 correct labels at 95.5%
# precision, which is the only region passing both G2 and G3 with coverage left.
# a facing of a product the shelf actually holds, so that set could not price
# it. Re-measured with `code/sweep_accept_lines.py` on 4580 crops from 64
# unrelated Wikimedia shelf photos - 1280-long-side frames at
# `LiveSettings.conf`, the configuration the phone runs - against
# `TEXT_MIN_SHARED = 4`: 2.0 accepts 60 of them (1.31%) and 4.0 accepts 3
# (0.07%), against G3's 1%. On image6 at full resolution 4.0 shows 44 labels of
# which 42 are right - 95.5% precision, and 65.6% of the 64 facings that hold a
# catalog row - and it is the highest-coverage cell in the sweep that passes
# both G2 and G3.
LOCAL_ACCEPT = 4.0
# The text lane's own accept line: a bigram overlap this high, with this much
# margin over the runner-up, names the row on its own. Measured: the text lane
# picks the right SKU on 41-42 of the 55 facings where it reads anything.
TEXT_ACCEPT = 0.5
TEXT_MARGIN = 0.15
# How many of the read's bigrams the winning row must actually contain. The
# ratio alone is not evidence: a two-pair read the row happens to contain scores
# 1.0, and "サソース" named sku_l08 that way. Two pairs was the old floor, and it
# is too low once the lane answers on unrelated shelves: on 2297 crops from
# Wikimedia milk, soy and snack shelves, 20 of 26 text-lane answers rested on
# exactly two shared bigrams ("避くて おいしい" -> sku_l12 via おい/いし), an
# 11.4% false accept rate against G3's 1%. Four is where the two sides separate
# once the local line is also at 4.0: false accepts fall to 8 (0.35%) while
# image6 keeps 42 of its 48 correct labels. It is a floor on evidence, not on
# the ratio, so a long read of shared brand copy still scores low and is still
# refused by the margin.
# How many of the read's bigrams the winning row must actually contain before
# that read may name a product. A floor on the amount of shared evidence, not
# on the ratio: a two-pair read the row happens to contain scores 1.0, and
# "サソース" named sku_l08 that way.
#
# It gates the text lane's *answer* and nothing else. Zeroing the row's score
# would be a different policy under the same name: it reorders the ranking, so
# a zeroed top row promotes a runner-up and the margin gate ends up comparing a
# different pair than the one it was measured on. It would also gate the veto,
# which is backwards - the veto only ever refuses a local accept, so a floor
# under it would let a two-bigram read rescue a match the local lane earned.
# `code/sweep_accept_lines.py` measures the line by calling the policy below
# with this value, so the sweep and the shipped lane cannot disagree about what
# it means.
#
# The ratio stays the ranking's own gate, so a long read of shared brand copy
# still scores low and is still refused by the margin.
TEXT_MIN_SHARED = 4
# Reference pyramid heights. A ~1000 px packshot against a 110-250 px crop is a
# 4-8x ratio, and ORB's 8x1.2 pyramid tops out at 3.6x, which is why ORB and
@ -348,8 +358,15 @@ def text_similarity(
the most real text to match were punished hardest. Bounded this way the
same reads score 0.86 and 1.0 against their true rows, and the garbage
read ties across three rows and is refused by the margin.
`vocabulary` defaults to the weight table's own keys, which are by
construction the bigrams some signature produces. A second copy of that set
on the lane is a second thing to keep in step, and it is the drift the
module already paid for once with the folding rules.
"""
read_grams = bigrams(read)
if vocabulary is None and weights is not None:
vocabulary = set(weights)
if vocabulary is not None:
read_grams &= vocabulary
if not read_grams:
@ -404,12 +421,6 @@ class TextLane:
# Bigram weights over the catalog's own names, so the lane can tell
# copy every sibling prints from the words that name one product.
self.weights = bigram_weights(list(self.signatures.values()))
# Every pair any name can produce. A read's pairs outside this set are
# OCR noise or word-boundary artifacts, and counting them in the
# denominator charges a row for text it could never have explained.
self.vocabulary: Set[str] = set()
for text in self.signatures.values():
self.vocabulary |= signature_bigrams(text)
self.ocr_engine = ocr_engine
# The engine's OCR lock. Vision tolerates concurrency but the CPU does
# not, and this lane runs one OCR per crop on the classify pool, so it
@ -472,7 +483,7 @@ class TextLane:
if len(normalize_text(read)) < 4:
return {}
return {
sku: text_similarity(read, signature, self.weights, self.vocabulary)
sku: text_similarity(read, signature, self.weights)
for sku, signature in self.signatures.items()
}

13
recognization/code/live.py
View file

@ -306,6 +306,9 @@ class LiveSession:
self._frames_dropped = 0
self._process_times: "collections.deque[float]" = collections.deque(maxlen=STATS_WINDOW)
self._classify_times: "collections.deque[float]" = collections.deque(maxlen=STATS_WINDOW)
# A lane that raised is an operational failure, not "nothing matched".
# Counted so a broken run is never read as a coverage change.
self._classify_failures = 0
self._slot_history: "collections.deque[List[Dict[str, Any]]]" = collections.deque(
maxlen=EMPTY_SLOT_HISTORY
)
@ -1198,7 +1201,14 @@ class LiveSession:
) -> None:
try:
scores, text_rank, text_read = future.result()
except Exception:
except Exception as exc:
# A lane that raised is an operational failure, not a crop that
# matched nothing. Swallowing it made a broken run (tracks, no
# labels, clean log) indistinguishable from a coverage change, so
# it is counted and logged before the empty answer is used.
with self._lock:
self._classify_failures += 1
LOGGER.exception("classification failed for track %s: %s", track_id, exc)
scores, text_rank, text_read = {}, [], ""
finished_at = time.perf_counter()
# Resolved before the session lock is taken. `identity_text_lane` builds
@ -1457,6 +1467,7 @@ class LiveSession:
"frames_dropped": self._frames_dropped,
"server_fps": fps,
"classify_pending": sum(track.in_flight for track in self._tracks.values()),
"classify_failures": self._classify_failures,
"classify_ms_p50": classify_p50,
}

155
recognization/code/score_stream.py
View file

@ -15,6 +15,13 @@ Reported per run: correct / in-catalog facings, precision, unknown accepts, and
tracks that matched no labelled facing. A track whose box overlaps a facing the
catalog does not enrol is an unenrolled false accept - the dangerous kind, and
the one the evaluator's own precision denominator used to hide.
G1 has two halves - new covers at least as much as Van, and the lower bound of
the paired-bootstrap difference clears -2 pp - and the second half needs the
per-facing pairing, so every row carries the `facing` index it matched. Two
reports plus the labels then answer the gate on the same facings:
python code/score_stream.py NEW.json --van VAN.json
"""
from __future__ import annotations
@ -22,6 +29,7 @@ from __future__ import annotations
import argparse
import json
import os
import random
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple
@ -33,6 +41,14 @@ if current_dir not in sys.path:
# so the denominator keeps only facings at least this far inside the window.
VISIBLE_MIN = 0.5
# G1's second half: the lower bound of the paired difference must clear -2 pp.
G1_BOUND_MIN = -0.02
# Resampling draws for the paired bootstrap, and the seed that makes the bound
# a number this repo can print twice.
BOOTSTRAP_DRAWS = 10000
BOOTSTRAP_SEED = 0
def iou(a: Sequence[int], b: Sequence[int]) -> float:
ax1, ay1, ax2, ay2 = a
@ -72,6 +88,22 @@ def visible_fraction(box: Sequence[int], window: Sequence[int]) -> float:
return (inter_w * inter_h) / area if area > 0 else 0.0
def reachable(facings: Sequence[Dict[str, Any]],
windows: Optional[Sequence[Sequence[int]]]
) -> List[Dict[str, Any]]:
"""Facings a run's windows showed well enough for the run to have named."""
if not windows:
return list(facings)
return [f for f in facings
if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN]
def correct_facings(rows: Sequence[Dict[str, Any]]) -> set:
"""Facing indices at least one track got right - one hit each, not one per track."""
return {row["facing"] for row in rows
if row["outcome"] == "correct" and row["facing"] is not None}
def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
stock_dir: str, iou_min: float,
windows: Optional[Sequence[Sequence[int]]] = None
@ -90,11 +122,7 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
text lane names SKUs the image lane has no view for, which is the point of
it, and scoring that as a false accept punishes the lane for working.
"""
if windows:
facings = [
f for f in facings
if max(visible_fraction(f["box"], w) for w in windows) >= VISIBLE_MIN
]
facings = reachable(facings, windows)
in_catalog = [f for f in facings if f["label"] != "unknown"]
enrolled = enrolled_skus(stock_dir)
@ -122,7 +150,8 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
if best is None or best_iou < iou_min:
unmatched += 1
rows.append({"track_id": track["track_id"], "predicted": track["sku_id"],
"truth": None, "iou": round(best_iou, 3), "outcome": "unmatched"})
"facing": None, "truth": None, "iou": round(best_iou, 3),
"outcome": "unmatched"})
continue
truth = best["label"]
predicted = track["sku_id"]
@ -140,7 +169,8 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
wrong += 1
outcome = "wrong"
rows.append({"track_id": track["track_id"], "predicted": predicted,
"truth": truth, "iou": round(best_iou, 3), "outcome": outcome})
"facing": best["index"], "truth": truth,
"iou": round(best_iou, 3), "outcome": outcome})
shown = correct + wrong + unenrolled_false + unknown_false
return {
@ -155,10 +185,81 @@ def score(tracks: Sequence[Dict[str, Any]], facings: Sequence[Dict[str, Any]],
"shown": shown,
"coverage": round(len(named) / len(in_catalog), 4) if in_catalog else 0.0,
"precision": round(correct / shown, 4) if shown else 0.0,
# The window set travels with the report: a paired comparison has to
# know what each run was shown, not just what it named.
"windows": [list(w) for w in windows] if windows else None,
"rows": rows,
}
def paired_bootstrap(van: Dict[str, Any], new: Dict[str, Any],
facings: Sequence[Dict[str, Any]],
draws: int = BOOTSTRAP_DRAWS,
seed: int = BOOTSTRAP_SEED) -> Dict[str, Any]:
"""G1's second half: new minus Van, per facing, with the 95 % lower bound.
Both reports are graded on one facing set - the in-catalog facings either
run's windows showed - so a facing one run panned past cannot flatter the
other. Resampling draws facings with replacement rather than comparing two
independent rates, because the two runs saw the same photo: pairing removes
the facing-to-facing variance that otherwise swamps a 27-facing sample.
Percentile method on the sorted draw differences; the seed is fixed so the
number in a report is one this repo reproduces.
"""
windows = list(van.get("windows") or []) + list(new.get("windows") or [])
shared = sorted(f["index"] for f in reachable(facings, windows)
if f["label"] != "unknown")
if not shared:
return {"facings": 0, "difference": 0.0, "lower_95": 0.0, "upper_95": 0.0,
"van": 0, "new": 0, "draws": draws, "seed": seed}
van_hits = correct_facings(van["rows"])
new_hits = correct_facings(new["rows"])
van_vector = [1 if i in van_hits else 0 for i in shared]
new_vector = [1 if i in new_hits else 0 for i in shared]
n = len(shared)
rng = random.Random(seed)
spread = []
for _ in range(draws):
van_sum = new_sum = 0
for _ in range(n):
# The same draw index into both vectors: that is the pairing.
drawn = rng.randrange(n)
van_sum += van_vector[drawn]
new_sum += new_vector[drawn]
spread.append((new_sum - van_sum) / n)
spread.sort()
tail = max(1, int(0.025 * draws))
return {
"facings": n,
"van": sum(van_vector),
"new": sum(new_vector),
"van_rate": round(sum(van_vector) / n, 4),
"new_rate": round(sum(new_vector) / n, 4),
"difference": round((sum(new_vector) - sum(van_vector)) / n, 4),
"lower_95": round(spread[tail - 1], 4),
"upper_95": round(spread[min(draws - 1, int(0.975 * draws))], 4),
"draws": draws,
"seed": seed,
}
def load_run(path: str, window: Optional[str] = None
) -> Tuple[Dict[str, Any], Optional[List[Tuple[int, ...]]]]:
"""A stream_client --tracks dump and the source-image windows it showed."""
with open(path) as handle:
payload = json.load(handle)
if window:
windows = [tuple(int(part) for part in window.split(","))]
elif payload.get("windows"):
windows = [tuple(w) for w in payload["windows"]]
else:
windows = None
return payload, windows
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description="Score streamed labels against ground truth")
parser.add_argument("tracks", help="JSON written by stream_client.py --tracks")
@ -169,22 +270,20 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
parser.add_argument("--window", default=None,
help="x0,y0,x1,y1 the client showed; the denominator is "
"restricted to facings at least half inside it")
parser.add_argument("--van", default=None,
help="Van baseline track dump on the same clip; scores G1's "
"paired-bootstrap half against it")
parser.add_argument("--draws", type=int, default=BOOTSTRAP_DRAWS)
parser.add_argument("--seed", type=int, default=BOOTSTRAP_SEED)
parser.add_argument("--verbose", action="store_true", help="print every scored track")
args = parser.parse_args(argv)
with open(args.tracks) as handle:
payload = json.load(handle)
windows = None
if args.window:
windows = [tuple(int(part) for part in args.window.split(","))]
elif payload.get("windows"):
windows = [tuple(w) for w in payload["windows"]]
report = score(payload["tracks"], load_facings(args.labels), args.stock_dir,
args.iou, windows)
facings = load_facings(args.labels)
payload, windows = load_run(args.tracks, args.window)
report = score(payload["tracks"], facings, args.stock_dir, args.iou, windows)
report["source"] = payload.get("source")
report["pace"] = payload.get("pace")
report["seconds"] = payload.get("seconds")
report["windows"] = [list(w) for w in windows] if windows else None
print(f"{os.path.basename(args.tracks)} pace={report['pace']} "
f"{report['seconds']}s")
@ -197,7 +296,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
if args.verbose:
for row in report["rows"]:
print(f" t{row['track_id']:4d} {row['outcome']:24s} "
f"pred={row['predicted']} truth={row['truth']} iou={row['iou']}")
f"facing={row['facing']} pred={row['predicted']} "
f"truth={row['truth']} iou={row['iou']}")
if args.van:
van_payload, van_windows = load_run(args.van, args.window)
van_report = score(van_payload["tracks"], facings, args.stock_dir,
args.iou, van_windows)
paired = paired_bootstrap(van_report, report, facings, args.draws, args.seed)
report["van"] = {k: van_report[k] for k in
("in_catalog_facings", "facings_named", "coverage",
"correct", "wrong", "precision", "shown")}
report["g1_paired"] = paired
print(f" G1 vs {os.path.basename(args.van)}: new {paired['new']}/"
f"{paired['facings']} ({paired['new_rate']:.0%}) - van {paired['van']}/"
f"{paired['facings']} ({paired['van_rate']:.0%}) = "
f"{paired['difference'] * 100:+.1f} pp")
print(f" paired bootstrap {paired['draws']} draws seed {paired['seed']}: "
f"95% bounds {paired['lower_95'] * 100:+.1f} pp to "
f"{paired['upper_95'] * 100:+.1f} pp; lower bound clears "
f"{G1_BOUND_MIN * 100:+.0f} pp: "
f"{'yes' if paired['lower_95'] >= G1_BOUND_MIN else 'NO'}")
if args.json_path:
with open(args.json_path, "w") as handle:

225
recognization/code/sweep_accept_lines.py Normal file
View file

@ -0,0 +1,225 @@
"""Joint sweep of the two open-set gates: LOCAL_ACCEPT x TEXT_MIN_SHARED.
Both lines were fitted on image6, where every crop is a facing of a product the
shelf actually holds. That set can only measure false accepts on the products it
happens to contain, so it cannot price a line on its own - the far-unknown rate
comes from `data/external-shelves/*`: Wikimedia Commons shelf photos that share
nothing with the catalog, where every accept is a false accept by construction.
Each cell is produced by calling the shipped `DecisionPolicy` with that cell's
values and the catalog signatures, never by re-implementing the gate in the
sweep: a sweep that scores its own copy of the policy reports on a machine the
server does not run. `TEXT_MIN_SHARED` is a floor on the evidence behind the
text lane's *answer*. It is deliberately not a floor on the score - zeroing a
row reorders the ranking, so the margin gate would compare a different pair -
and deliberately not a floor on the veto, which only ever refuses. The constant
in `identity_lanes.py` says the same thing next to the code that does it.
The distractor side is measured the way the phone measures it - a 1280-long-side
frame at `LiveSettings.conf`, the path `code/eval_distractors.py` runs. The
shelf is scored at both resolutions: the full-resolution column is the one the
published table quotes, the live column is what the phone sees.
RootSIFT and OCR dominate the runtime and neither depends on the gates, so the
per-crop evidence is cached. Without `--cache` the sweep rebuilds it every run
and prints the same numbers, only slower.
.venv/bin/python code/sweep_accept_lines.py \\
--cache data/evaluations/2026-09-29-session/scratch/accept_line_cache.pkl
"""
from __future__ import annotations
import argparse
import glob
import os
import pickle
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
import cv2 # noqa: E402
from eval_identity import ( # noqa: E402
EXCLUDE,
LIVE_LONG_SIDE,
UNKNOWN,
LocalLaneAdapter,
at_live_resolution,
crop_of,
load_labels,
)
from identity_lanes import LOCAL_ACCEPT, TEXT_MIN_SHARED, DecisionPolicy # noqa: E402
SHELF_LABELS = "eval/labels/image6.json"
SHELF_IMAGE = "add/image6.jpg"
DISTRACTOR_DIRS = (
"data/external-shelves/wikimedia-commons",
"data/external-shelves/wikimedia-commons-2",
)
LOCAL_ACCEPTS = (2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0)
TEXT_MIN_SHAREDS = (2, 3, 4, 5, 6)
CACHE_VERSION = 3
# One crop's evidence: the local lane's ranking and the text lane's raw read.
Evidence = Tuple[Any, str]
# One facing: its ground-truth label plus that evidence.
Facing = Tuple[str, Any, str]
def _evidence(lane: LocalLaneAdapter, frame: Any, box: Sequence[int]) -> Optional[Evidence]:
"""Rank and read one crop, or None when the box is off the frame."""
crop = crop_of(frame, box)
if crop.size == 0:
return None
return lane.rank(crop), lane.text_lane.read(crop)
def build_cache(stock_dir: str) -> Dict[str, Any]:
"""Every crop's evidence, gathered once and swept many times."""
from engine import EngineSettings, build_default_engine
from live import LiveSettings
lane = LocalLaneAdapter(stock_dir)
image = cv2.imread(SHELF_IMAGE)
live = at_live_resolution(image)
factor = live.shape[1] / float(image.shape[1])
shelf_full: List[Facing] = []
shelf_live: List[Facing] = []
for facing in load_labels(SHELF_LABELS)["facings"]:
label = facing.get("label")
if label in (None, EXCLUDE):
continue
full = _evidence(lane, image, facing["box"])
if full is None:
continue
shelf_full.append((label, full[0], full[1]))
# A facing the live frame cuts to nothing is dropped rather than
# scored at full resolution: the two columns have to be the same crops
# at two sizes, or the resolution is not the only variable.
small = _evidence(lane, live, [int(round(v * factor)) for v in facing["box"]])
if small is not None:
shelf_live.append((label, small[0], small[1]))
engine = build_default_engine(EngineSettings(stock_dir=stock_dir))
conf = LiveSettings().conf
distractors: List[Evidence] = []
photos = 0
for directory in DISTRACTOR_DIRS:
paths = sorted(
path for path in glob.glob(os.path.join(directory, "*"))
if os.path.splitext(path)[1].lower() in (".jpg", ".jpeg", ".png")
)
for path in paths:
shelf_photo = cv2.imread(path)
if shelf_photo is None:
continue
photos += 1
frame = at_live_resolution(shelf_photo)
for detection in engine.detect(frame, conf=conf):
box = detection.get("box") or detection.get("bbox")
if not box:
continue
item = _evidence(lane, frame, [int(v) for v in box])
if item is not None:
distractors.append(item)
return {
"version": CACHE_VERSION,
"conf": conf,
"photos": photos,
"shelf_full": shelf_full,
"shelf_live": shelf_live,
"distractors": distractors,
}
def _score(lane: LocalLaneAdapter, policy: DecisionPolicy, facings: Sequence[Facing]) -> Tuple[int, int]:
"""Shown and correct labels for one shelf column under one policy."""
shown = correct = 0
for label, ranked, read in facings:
text = sorted(
lane.text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
)
answer = policy.decide(ranked, text, text_read=read)["best_sku"]
if answer:
shown += 1
correct += answer == label
return shown, correct
def _false_accepts(lane: LocalLaneAdapter, policy: DecisionPolicy,
distractors: Sequence[Evidence]) -> int:
"""Every accept on an unrelated shelf is a false accept by construction."""
accepts = 0
for ranked, read in distractors:
text = sorted(
lane.text_lane.scores_from_text(read).items(), key=lambda kv: kv[1], reverse=True
)
if policy.decide(ranked, text, text_read=read)["best_sku"]:
accepts += 1
return accepts
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stock", default="stock-lopia-clean")
parser.add_argument("--cache", default="", help="build once, reuse on later runs")
parser.add_argument("--rebuild", action="store_true", help="ignore an existing cache")
args = parser.parse_args(argv)
cache: Optional[Dict[str, Any]] = None
if args.cache and os.path.exists(args.cache) and not args.rebuild:
with open(args.cache, "rb") as handle:
cache = pickle.load(handle)
if cache.get("version") != CACHE_VERSION:
cache = None
if cache is None:
cache = build_cache(args.stock)
if args.cache:
os.makedirs(os.path.dirname(os.path.abspath(args.cache)), exist_ok=True)
with open(args.cache, "wb") as handle:
pickle.dump(cache, handle)
lane = LocalLaneAdapter(args.stock)
shelf_full: List[Facing] = cache["shelf_full"]
shelf_live: List[Facing] = cache["shelf_live"]
distractors: List[Evidence] = cache["distractors"]
known = sum(1 for label, _, _ in shelf_full if label != UNKNOWN)
print(f"catalog {args.stock}, {LIVE_LONG_SIDE}-long-side frames at detect_conf {cache['conf']}")
print(f"image6: {len(shelf_full)} facings, {known} holding a catalog row, "
f"{len(shelf_live)} of them cut from the live frame")
print(f"distractors: {len(distractors)} crops from {cache['photos']} unrelated shelf photos")
print()
print(f"{'accept':>6} {'floor':>5} | {'shown':>5} {'corr':>4} {'prec':>6} {'cover':>6} "
f"| {'shown':>5} {'corr':>4} {'prec':>6} | {'false':>5} {'rate':>6}")
print(f"{'':>6} {'':>5} | {'--- full resolution ---':^25} "
f"| {'---- live resolution ----':^25} | {'-- unknown shelves --':^14}")
for accept in LOCAL_ACCEPTS:
for floor in TEXT_MIN_SHAREDS:
policy = DecisionPolicy(
local_accept=accept, text_min_shared=floor,
signatures=lane.text_lane.signatures,
)
full_shown, full_correct = _score(lane, policy, shelf_full)
live_shown, live_correct = _score(lane, policy, shelf_live)
false = _false_accepts(lane, policy, distractors)
shown, correct = full_shown, full_correct
rate = false / len(distractors) * 100
shipped = " <- shipped" if (accept, floor) == (LOCAL_ACCEPT, TEXT_MIN_SHARED) else ""
print(
f"{accept:>6.1f} {floor:>5} | {shown:>5} {correct:>4} "
f"{correct / shown * 100 if shown else 0:>5.1f}% {correct / known * 100:>5.1f}% "
f"| {live_shown:>5} {live_correct:>4} "
f"{live_correct / live_shown * 100 if live_shown else 0:>5.1f}% "
f"| {false:>5} {rate:>5.2f}%{shipped}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

38
recognization/code/test_eval_live.py
View file

@ -18,6 +18,7 @@ import cv2 # noqa: E402
import numpy as np # noqa: E402
from eval_live import ( # noqa: E402
IDENTITY_BACKENDS,
EvalInputError,
identity_level,
labelled_top1,
@ -296,5 +297,42 @@ class IdentityContractTests(unittest.TestCase):
{"identity_backend": "matcher"},
])
def test_van_is_accepted_by_the_cli_and_by_a_recording_that_names_it(self):
self.assertIn("van", IDENTITY_BACKENDS)
self.assertEqual(resolve_identity("van", None), "van")
self.assertEqual(recorded_identity([{"identity_backend": "van"}]), "van")
root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, root, ignore_errors=True)
stock = os.path.join(root, "stock")
os.makedirs(stock)
recording = os.path.join(root, "session")
os.makedirs(recording)
write_jpeg(os.path.join(recording, "000000.jpg"), np.zeros((8, 8, 3), dtype=np.uint8))
line = {"frame": 1, "path": "000000.jpg", "tracks": [], "identity_backend": "van"}
with open(os.path.join(recording, "index.jsonl"), "w", encoding="utf-8") as handle:
handle.write(json.dumps(line) + "\n")
for extra in ([], ["--identity", "van"]):
report_path = os.path.join(root, "eval_live.json")
completed = subprocess.run(
[
sys.executable,
os.path.join(current_dir, "eval_live.py"),
recording,
"--stock-dir",
stock,
"--json",
report_path,
*extra,
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr)
with open(report_path, "r", encoding="utf-8") as handle:
report = json.load(handle)
self.assertEqual(report["identity"]["backend"], "van")
self.assertEqual(report["fingerprints"]["model"]["identity"], "van")
if __name__ == "__main__":
unittest.main()

29
recognization/code/test_live.py
View file

@ -455,6 +455,35 @@ class LocalScaleTests(unittest.TestCase):
self.assertGreater(engine.ocr_calls, 1, "the size-only read must not be cached")
engine.shutdown()
def test_a_failing_lane_is_counted_and_logged(self):
"""A lane that raises is a failure, not a crop that matched nothing.
`_on_classified` substituted an empty answer for the exception, so a
run in which `score_crop_lanes` failed on every crop produced tracks,
zero labels and a clean log - indistinguishable from a real coverage
drop. The counter and the log line are what tell the two apart, and
the session has to keep serving frames either way.
"""
session, engine, _, _ = make_session(engine_cls=LocalScaleEngine)
def boom(crop, method=None, skip_ocr=False):
raise RuntimeError("lane unavailable")
engine.score_crop_lanes = boom
named = None
with self.assertLogs("dorayaki.live", level="ERROR") as logs:
for frame_id in range(1, 14):
tracks = process(session, frame_id)["tracks"]
self.assertTrue(tracks, "a failing lane must not take the session down")
named = named or next(
(t["identity"]["sku_id"] for t in tracks if t["identity"]["sku_id"]), None)
self.assertTrue(wait_for(lambda: bool(logs.output)),
"the exception must be logged, not swallowed")
self.assertGreaterEqual(session.snapshot()["stats"]["classify_failures"], 1,
"a lane failure must be counted in the session stats")
self.assertIsNone(named, "a failed lane must not name a product")
engine.shutdown()
class TagPassIntegrationTests(unittest.TestCase):
"""The tag pass must reach the live result without blocking frames."""

137
recognization/code/test_score_stream.py Normal file
View file

@ -0,0 +1,137 @@
"""
Tests for the G1 scorer.
The gate has two halves - the new lane covers at least as much as Van, and the
lower bound of the paired-bootstrap difference clears -2 pp - and the second
half is only computable if a scored row still says which facing it matched.
These tests pin that pairing and the bound it produces.
"""
from __future__ import annotations
import os
import sys
import tempfile
import unittest
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from score_stream import ( # noqa: E402
G1_BOUND_MIN,
correct_facings,
paired_bootstrap,
score,
)
def facing(index, box, label="sku_a"):
return {"index": index, "box": list(box), "label": label}
def track(track_id, box, sku="sku_a"):
return {"track_id": track_id, "sku_id": sku, "label_box": list(box)}
def report(rows, windows=None):
"""A score report shaped the way `score` returns one, without the boxes."""
return {"rows": rows, "windows": windows}
def correct(track_id, matched):
return {"track_id": track_id, "predicted": "sku_a", "facing": matched,
"truth": "sku_a", "iou": 1.0, "outcome": "correct"}
def grid(count):
return [facing(i, [i * 20, 0, i * 20 + 10, 10]) for i in range(count)]
class ScoreRowTests(unittest.TestCase):
def setUp(self):
self.stock = tempfile.TemporaryDirectory()
open(os.path.join(self.stock.name, "sku_a.jpg"), "wb").close()
self.addCleanup(self.stock.cleanup)
def test_rows_carry_the_matched_facing(self):
facings = [facing(7, [0, 0, 100, 100]), facing(8, [900, 0, 1000, 100])]
rows = score([track(1, [0, 0, 100, 100]), track(2, [400, 400, 500, 500])],
facings, self.stock.name, 0.5)["rows"]
by_track = {row["track_id"]: row for row in rows}
self.assertEqual(by_track[1]["facing"], 7)
self.assertEqual(by_track[1]["outcome"], "correct")
# A track that hit nothing is unpaired, and says so instead of guessing.
self.assertIsNone(by_track[2]["facing"])
self.assertEqual(by_track[2]["outcome"], "unmatched")
def test_correct_facings_counts_a_facing_once(self):
rows = [correct(1, 7), correct(2, 7), correct(3, 9)]
self.assertEqual(correct_facings(rows), {7, 9})
class PairedBootstrapTests(unittest.TestCase):
"""27 facings is a small sample, so the pairing is what carries the gate."""
def test_identical_systems_sit_at_zero_everywhere(self):
rows = [correct(1, i) for i in range(10)]
result = paired_bootstrap(report(rows), report(rows), [])
self.assertEqual(result["difference"], 0.0)
self.assertEqual(result["lower_95"], 0.0)
self.assertEqual(result["upper_95"], 0.0)
self.assertEqual(result["van"], result["new"])
def test_a_superset_of_vans_hits_can_never_bound_below_zero(self):
# The image6 shape: the new lane gets everything Van got, plus a facing.
van = [correct(i, i) for i in range(17)]
new = van + [correct(99, 17)]
result = paired_bootstrap(report(van), report(new), grid(27))
self.assertEqual(result["facings"], 27)
self.assertEqual(result["van"], 17)
self.assertEqual(result["new"], 18)
self.assertAlmostEqual(result["difference"], 1 / 27, places=4)
# No resample of these 27 can put a facing in the new run that Van's run
# did not already have, so the 2.5th percentile is a hard zero. A bound
# printed as +0.0 pp is the floor of this pairing, not a margin.
self.assertEqual(result["lower_95"], 0.0)
self.assertGreaterEqual(result["lower_95"], G1_BOUND_MIN)
def test_a_swapped_facing_spreads_across_zero(self):
van = [correct(i, i) for i in range(17)]
new = van[:16] + [correct(99, 17)]
result = paired_bootstrap(report(van), report(new), grid(27))
self.assertEqual(result["difference"], 0.0)
self.assertLess(result["lower_95"], G1_BOUND_MIN)
self.assertGreater(result["upper_95"], 0.0)
def test_both_runs_are_graded_on_what_either_window_reached(self):
# Two facings at opposite edges; each run panned to one of them. Grading
# on the union keeps both on the same facings; the intersection would
# score each run on its own pan and the pairing would be empty.
facings = [facing(0, [0, 0, 100, 100]), facing(1, [900, 0, 1000, 100])]
left, right = [0, 0, 500, 1000], [500, 0, 1000, 1000]
result = paired_bootstrap(report([correct(1, 0)], [left]),
report([correct(2, 1)], [right]), facings)
self.assertEqual(result["facings"], 2)
self.assertEqual((result["van"], result["new"]), (1, 1))
self.assertEqual(result["difference"], 0.0)
def test_the_bound_repeats_for_the_same_seed(self):
van = [correct(i, i) for i in range(17)]
new = van[:5] + [correct(90 + i, 20 + i) for i in range(9)]
facings = grid(27)
first = paired_bootstrap(report(van), report(new), facings)
self.assertEqual(first, paired_bootstrap(report(van), report(new), facings))
def test_no_reachable_facing_scores_zero_rather_than_dividing_by_zero(self):
# Both runs panned to a corner the only facing is not in: the pairing
# set is empty, and a rate is not a number there.
elsewhere = [facing(0, [0, 0, 10, 10])]
corner = [[500, 500, 600, 600]]
result = paired_bootstrap(report([], corner), report([], corner), elsewhere)
self.assertEqual(result["facings"], 0)
self.assertEqual(result["lower_95"], 0.0)
if __name__ == "__main__":
unittest.main()