feat(eval): add --no-text so the E-3 claim is reproducible
E-3 asks what the text lane is worth, and the record answered it with numbers that no committed command could produce - the evaluator had no way to run the local lane without the text lane. --no-text is that way, and it keeps the comparison to one variable by building the lane either way, so the OCR cost is paid in both arms. Measured: full resolution 77% end-to-end at 91% precision with text against 61% at 66% without (5 false accepts against 20); live 62% at 93% against 56% at 84% (3 against 7). The lane adds 16 pp and 6 pp of coverage while raising precision, against E-3's +5 pp bar. Also refreshes the re-measured block above it, which still printed the pre-correction 61/48 label counts and a stray 'HUB!' marker.
This commit is contained in:
parent
56cff09292
commit
a1c69f72fe
2 changed files with 25 additions and 14 deletions
|
|
@ -413,10 +413,14 @@ class LocalLaneAdapter(Lane):
|
||||||
|
|
||||||
name = "local"
|
name = "local"
|
||||||
|
|
||||||
def __init__(self, stock_dir: str, ocr: Any = None) -> None:
|
def __init__(self, stock_dir: str, ocr: Any = None, use_text: bool = True) -> None:
|
||||||
from identity_lanes import LocalLane, TextLane
|
from identity_lanes import LocalLane, TextLane
|
||||||
|
|
||||||
self.lane = LocalLane(stock_dir)
|
self.lane = LocalLane(stock_dir)
|
||||||
|
# E-3 asks what the text lane is worth, which is only answerable by
|
||||||
|
# running the same crops without it. Building the lane anyway keeps the
|
||||||
|
# comparison to one variable: the OCR cost is paid either way.
|
||||||
|
self.use_text = use_text
|
||||||
self.text_lane = TextLane.from_catalog(
|
self.text_lane = TextLane.from_catalog(
|
||||||
_catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine()
|
_catalog_rows(stock_dir), stock_dir, ocr if ocr is not None else _ocr_engine()
|
||||||
)
|
)
|
||||||
|
|
@ -428,6 +432,10 @@ class LocalLaneAdapter(Lane):
|
||||||
margin: float, scale: str) -> Dict[str, Any]:
|
margin: float, scale: str) -> Dict[str, Any]:
|
||||||
from identity_lanes import DecisionPolicy
|
from identity_lanes import DecisionPolicy
|
||||||
|
|
||||||
|
if not self.use_text:
|
||||||
|
from identity import decide_identity
|
||||||
|
|
||||||
|
return decide_identity(dict(ranked), scale="local", min_score=accept)
|
||||||
read = self.text_lane.read(crop)
|
read = self.text_lane.read(crop)
|
||||||
text_rank = sorted(
|
text_rank = sorted(
|
||||||
self.text_lane.scores_from_text(read).items(),
|
self.text_lane.scores_from_text(read).items(),
|
||||||
|
|
@ -641,6 +649,8 @@ def main() -> int:
|
||||||
parser.add_argument("--lanes", default="local,matcher,gallery,hybrid-verify")
|
parser.add_argument("--lanes", default="local,matcher,gallery,hybrid-verify")
|
||||||
parser.add_argument("--shortlist", type=int, default=5)
|
parser.add_argument("--shortlist", type=int, default=5)
|
||||||
parser.add_argument("--accept", type=float, default=None, help="override the lane's own floor")
|
parser.add_argument("--accept", type=float, default=None, help="override the lane's own floor")
|
||||||
|
parser.add_argument("--no-text", action="store_true",
|
||||||
|
help="run the local lane without the text lane (E-3)")
|
||||||
parser.add_argument("--margin", type=float, default=None)
|
parser.add_argument("--margin", type=float, default=None)
|
||||||
parser.add_argument("--live", action="store_true", help="also score at live resolution")
|
parser.add_argument("--live", action="store_true", help="also score at live resolution")
|
||||||
parser.add_argument("--json", help="write the report here")
|
parser.add_argument("--json", help="write the report here")
|
||||||
|
|
@ -680,7 +690,7 @@ def main() -> int:
|
||||||
if "van" in wanted:
|
if "van" in wanted:
|
||||||
lanes["van"] = VanLane(args.stock_dir)
|
lanes["van"] = VanLane(args.stock_dir)
|
||||||
if "local" in wanted:
|
if "local" in wanted:
|
||||||
lanes["local"] = LocalLaneAdapter(args.stock_dir)
|
lanes["local"] = LocalLaneAdapter(args.stock_dir, use_text=not args.no_text)
|
||||||
if "gallery" in wanted or "hybrid-verify" in wanted:
|
if "gallery" in wanted or "hybrid-verify" in wanted:
|
||||||
lanes["gallery"] = GalleryLane(args.stock_dir, catalog_skus)
|
lanes["gallery"] = GalleryLane(args.stock_dir, catalog_skus)
|
||||||
if "hybrid-verify" in wanted:
|
if "hybrid-verify" in wanted:
|
||||||
|
|
|
||||||
|
|
@ -758,23 +758,24 @@ unanswerable before and after.
|
||||||
The text lane's evidence floor (shared bigrams between the read and the winning
|
The text lane's evidence floor (shared bigrams between the read and the winning
|
||||||
row's name) and the local lane's signature pass-through landed, and the
|
row's name) and the local lane's signature pass-through landed, and the
|
||||||
decision ceiling was corrected: it divided a numerator that counted unenrolled
|
decision ceiling was corrected: it divided a numerator that counted unenrolled
|
||||||
facings by the enrolled count, so it read 117%. It now reads 86%.
|
facings by the enrolled count, so it read 117%.
|
||||||
|
|
||||||
full resolution — 109 scored facings (61 in-catalog, 48 unknown)
|
full resolution — 109 scored facings (64 in-catalog, 45 unknown)
|
||||||
lane catalog identity decision end-to-end precision false acc
|
lane catalog identity decision end-to-end precision false acc
|
||||||
van 69% 76% 76% 52% 29% 77
|
van 83% 77% 77% 64% 38% 68
|
||||||
matcher 69% 76% 71% 49% 39% 46 HUB!
|
matcher 83% 77% 74% 61% 51% 38
|
||||||
local 69% 93% 86% 80% 91% 5
|
local 83% 92% 83% 77% 91% 5
|
||||||
|
|
||||||
live resolution — 109 scored facings (61 in-catalog, 48 unknown)
|
live resolution — 109 scored facings (64 in-catalog, 45 unknown)
|
||||||
lane catalog identity decision end-to-end precision false acc
|
lane catalog identity decision end-to-end precision false acc
|
||||||
local 69% 86% 86% 66% 95% 2
|
local 83% 91% 64% 62% 93% 3
|
||||||
|
|
||||||
**E-3 answered.** Removing the text lane from the decision, same crops and
|
**E-3 answered.** `--no-text` runs the same crops and catalog without the text
|
||||||
catalog, drops the local lane from 80% to 49% coverage at full resolution
|
lane, which is the only way this question is answerable. Full resolution:
|
||||||
(91% → 79% precision, 5 → 8 false accepts) and from 66% to 43% at live
|
end-to-end 77% at 91% precision with text, 61% at 66% without (5 false accepts
|
||||||
(95% → 87%, 2 → 4). The lane adds 31 pp and 23 pp of coverage while *raising*
|
against 20). Live: 62% at 93% against 56% at 84% (3 against 7). The lane adds
|
||||||
precision, against E-3's +5 pp bar. It is not a veto-only lane.
|
16 pp and 6 pp of coverage while *raising* precision, against E-3's +5 pp bar.
|
||||||
|
It is not a veto-only lane.
|
||||||
|
|
||||||
**E-2 answered.** `LocalLane` is RootSIFT-only; ORB and BRISK survive only in
|
**E-2 answered.** `LocalLane` is RootSIFT-only; ORB and BRISK survive only in
|
||||||
the `matcher` baseline lane, so there is nothing to drop from the shipped path.
|
the `matcher` baseline lane, so there is nothing to drop from the shipped path.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue