fix(catalog): re-source sku_l09 and enforce the pyramid floor
sku_l09.jpg was 407x305, so its 400 px pyramid level was a 305->400 upscale: a blur with no high-frequency detail to disagree with, which won 31 of 54 crops on the shelf it was cut from. Re-sourced at 1000x1000. The floor is now a hard gate in rebuild_catalog.py - a view under 800 px tall is refused, not reported - because reporting it is what let the 305 px view ship. data/packshots/source/sku_l07.jpg (600x600) is refused on this rule. README records the real sizes and the provenance for every row.
This commit is contained in:
kiyo9w
parent
5094b5f6ca
commit
730a5e782b
4 changed files with 274 additions and 102 deletions
|
|
@ -7,7 +7,9 @@ its own exam.
|
|||
|
||||
This rebuilds a catalog directory from named packshots and refuses, per view:
|
||||
|
||||
* a view shorter than MIN_HEIGHT or narrower than MIN_WIDTH,
|
||||
* a view shorter than MIN_HEIGHT, narrower than MIN_WIDTH, or shorter than
|
||||
PYRAMID_FLOOR (a source under it cannot be stored as the protocol's
|
||||
200/400/800 px pyramid, so its scores would not be comparable with the rest),
|
||||
* a view whose capture session appears in --eval-manifest,
|
||||
* a packshot whose catalog name is not supported by its OCR text,
|
||||
* a view that is a near-duplicate of another view of the same SKU,
|
||||
|
|
@ -63,10 +65,16 @@ MIN_WIDTH = 120
|
|||
# The enrollment protocol's floor: a reference view under 160 px tall carries no
|
||||
# text geometry (facings under 60 px wide failed every measured lane).
|
||||
MIN_HEIGHT = 160
|
||||
# Correct packshots measured 0.79-1.00 of the catalog name's bigrams in their
|
||||
# OCR text; the wrong-size Kewpie 450g packshot tops out at 0.64 against every
|
||||
# name in the catalog. 0.70 sits between them with room for OCR noise.
|
||||
MIN_NAME_OCR = 0.70
|
||||
# The name/OCR gate, re-measured on the shipped packshots with the strict name
|
||||
# side in place, each image read four ways and scored against all 12 catalog
|
||||
# names: correct packshots 0.9167-1.0000 (seven read their whole name; l11's
|
||||
# read omits "マヨネーズ" and is the floor), the wrong-product packshot in
|
||||
# references/rejected/ 0.0476-0.5833. 0.75 is the midpoint: 0.167 of headroom
|
||||
# above every correct packshot and 0.167 below the wrong-product ceiling, so
|
||||
# OCR noise on either side does not move a verdict. Siblings are not this
|
||||
# gate's job - the コクうま packshot scores 0.85 against the マヨネーズ name,
|
||||
# which is what text_discriminated_siblings flags, not a gap a threshold closes.
|
||||
MIN_NAME_OCR = 0.75
|
||||
# Vision drops small print at native scale: the l11 packshot reads ハーフ only
|
||||
# when upscaled (0.57 -> 0.79). Reading at 2x is what makes the check about the
|
||||
# package rather than about Vision's recall.
|
||||
|
|
@ -75,10 +83,12 @@ OCR_SCALE = 2.0
|
|||
DUPLICATE_COSINE = 0.98
|
||||
|
||||
# The enrollment protocol's pyramid target: a source at least this tall can be
|
||||
# stored as a 200/400/800 px pyramid. Below it the view still enrolls - the hard
|
||||
# reject is MIN_HEIGHT - but its scores are not comparable with the rest, which
|
||||
# is the documented l09 defect: at 305 px its inlier scores dwarf every other
|
||||
# reference (85-129 against 10-40) and it wins facings it should not.
|
||||
# stored as a 200/400/800 px pyramid, so a view under it cannot be scored
|
||||
# against the rest. Refused, not reported: the l09 defect was a 305 px view
|
||||
# whose inlier scores (85-129) dwarfed every other reference (10-40) and it won
|
||||
# facings it should not, and a report is exactly the kind of thing that gets
|
||||
# read once and then ignored. The fix is re-sourcing, which is a data task
|
||||
# this tool should not let through the door in the first place.
|
||||
PYRAMID_FLOOR = 800
|
||||
|
||||
# Official pack-shot pages publish a single front view per JAN, so a catalog
|
||||
|
|
@ -317,6 +327,12 @@ def audit(
|
|||
refused.append({"sku": sku, "view": name,
|
||||
"reason": f"{image.shape[1]}px wide, below the {MIN_WIDTH}px floor"})
|
||||
continue
|
||||
if image.shape[0] < PYRAMID_FLOOR:
|
||||
refused.append({"sku": sku, "view": name,
|
||||
"reason": f"{image.shape[0]}px tall, below the {PYRAMID_FLOOR}px "
|
||||
"pyramid floor; its scores would not be comparable "
|
||||
"with the rest - re-source this view"})
|
||||
continue
|
||||
if ocr_engine is not None and names is not None:
|
||||
try:
|
||||
text = best_orientation_text(ocr_engine, image, names.get(sku, ""))
|
||||
|
|
@ -453,27 +469,6 @@ def load_shelf_skus(path: Optional[str]) -> Optional[Set[str]]:
|
|||
raise RebuildError(f"{path}: expected a JSON list or one SKU per line")
|
||||
|
||||
|
||||
def below_pyramid_floor(kept: Dict[str, List[str]], source: str) -> List[Dict[str, Any]]:
|
||||
"""Enrolled views too small to store as the protocol's pyramid.
|
||||
|
||||
Reported, not refused: the view is usable and dropping it would remove the
|
||||
SKU from the catalog entirely, which is worse than an incomparable score.
|
||||
The fix is re-sourcing the packshot, which is a data task.
|
||||
"""
|
||||
small: List[Dict[str, Any]] = []
|
||||
for sku, filenames in sorted(kept.items()):
|
||||
for filename in filenames:
|
||||
image = cv2.imread(os.path.join(source, filename))
|
||||
if image is not None and image.shape[0] < PYRAMID_FLOOR:
|
||||
small.append({
|
||||
"sku": sku,
|
||||
"view": filename,
|
||||
"height": int(image.shape[0]),
|
||||
"floor": PYRAMID_FLOOR,
|
||||
})
|
||||
return small
|
||||
|
||||
|
||||
def write_catalog(kept: Dict[str, List[str]], source: str, output: str, csv_path: Optional[str]) -> None:
|
||||
"""Copy the kept views and the whole CSV into a staging dir, then swap it in."""
|
||||
staging = tempfile.mkdtemp(prefix=".catalog-", dir=os.path.dirname(os.path.abspath(output)) or ".")
|
||||
|
|
@ -582,7 +577,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|||
# and vetoes it from the CSV alone, so it is a coverage gap, not a refusal.
|
||||
text_only = sorted(set(names) - set(kept))
|
||||
siblings = text_discriminated_siblings(kept, args.source, names)
|
||||
small = below_pyramid_floor(kept, args.source)
|
||||
no_source = unattributed(args.source)
|
||||
verdict = live_ready_verdict(kept, names, shelf_skus)
|
||||
report = {
|
||||
|
|
@ -596,7 +590,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|||
"ocr": engine.name if ocr_engine is not None else "skipped",
|
||||
"text_only_skus": text_only,
|
||||
"text_discriminated_siblings": siblings,
|
||||
"below_pyramid_floor": small,
|
||||
"unattributed_skus": no_source,
|
||||
"completeness": verdict,
|
||||
"skus": {sku: views for sku, views in sorted(kept.items())},
|
||||
|
|
@ -616,11 +609,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|||
for row in siblings:
|
||||
print(f" {row['sku']:<10} {row['view']:<20} ~ {row['sibling_sku']} "
|
||||
f"({row['sibling_view']}) cosine {row['cosine']}")
|
||||
if small:
|
||||
print(f"\nbelow the {PYRAMID_FLOOR}px pyramid floor ({len(small)}):")
|
||||
for row in small:
|
||||
print(f" {row['sku']:<10} {row['view']:<20} {row['height']}px tall - "
|
||||
"scores are not comparable with the rest; re-source this view")
|
||||
if no_source:
|
||||
print(f"\nno recorded source ({len(no_source)}): {', '.join(no_source)} - "
|
||||
"usable, but provenance is missing; re-sourcing would close it")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue