730a5e782b
sku_l09.jpg was 407x305, so its 400 px pyramid level was a 305->400 upscale: a blur with no high-frequency detail to disagree with, which won 31 of 54 crops on the shelf it was cut from. Re-sourced at 1000x1000. The floor is now a hard gate in rebuild_catalog.py - a view under 800 px tall is refused, not reported - because reporting it is what let the 305 px view ship. data/packshots/source/sku_l07.jpg (600x600) is refused on this rule. README records the real sizes and the provenance for every row.
629 lines
29 KiB
Python
629 lines
29 KiB
Python
"""Rebuild a store catalog from packshots, refusing unsafe enrollment.
|
|
|
|
The LOPIA catalog shipped with views cut from the test shelf itself: 71-119 px
|
|
wide, one 18 px sliver, duplicate crops, and five SKUs with no view at all. That
|
|
is why every dark package collapsed onto one SKU and why the head was trained on
|
|
its own exam.
|
|
|
|
This rebuilds a catalog directory from named packshots and refuses, per view:
|
|
|
|
* a view shorter than MIN_HEIGHT, narrower than MIN_WIDTH, or shorter than
|
|
PYRAMID_FLOOR (a source under it cannot be stored as the protocol's
|
|
200/400/800 px pyramid, so its scores would not be comparable with the rest),
|
|
* a view whose capture session appears in --eval-manifest,
|
|
* a packshot whose catalog name is not supported by its OCR text,
|
|
* a view that is a near-duplicate of another view of the same SKU,
|
|
* a SKU left with fewer than --min-views distinct views.
|
|
|
|
Every enrolled image must have a CSV row; a SKU with an image and no row refuses
|
|
the build. CSV rows with no image are kept and reported as text-only coverage
|
|
gaps, because the text lane names and vetoes those products from the CSV alone.
|
|
Capture session ids come from the source manifest (``manifest.json`` beside the
|
|
images by default), whose image entries use ``file`` plus
|
|
``capture_session_id`` or ``session_id``. Pixel equality is not a leak test:
|
|
re-cut crops pass it, so leakage is refused by capture session instead.
|
|
|
|
Two checks report rather than refuse, because both are properties of the
|
|
catalog as a whole: cross-SKU near-duplicates are flagged as
|
|
text-discriminated siblings (the near-unknown case only package text
|
|
separates), and ``--shelf-tags`` turns the completeness rule into a live-ready
|
|
verdict. Without that list the verdict is ``unknown``, not a guess.
|
|
|
|
python code/rebuild_catalog.py --source /tmp/lopia-clean \
|
|
--output stock-lopia-clean --csv stock-lopia/stock.csv
|
|
python code/rebuild_catalog.py --source /tmp/lopia-clean \
|
|
--output stock-lopia-clean --csv stock-lopia/stock.csv --replace
|
|
|
|
Views are named <sku>.jpg, <sku>_2.jpg, ... The CSV is copied byte-for-byte, so
|
|
text-only products stay in the catalog and its content hash does not move.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unicodedata
|
|
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
if current_dir not in sys.path:
|
|
sys.path.insert(0, current_dir)
|
|
|
|
import cv2 # noqa: E402
|
|
import numpy as np # noqa: E402
|
|
|
|
from ocr import OcrUnavailable, build_ocr_engine # noqa: E402
|
|
|
|
# A view narrower than this cannot carry the logo and text geometry that tells
|
|
# two fine-grained products apart. The shipped LOPIA views were 71-119 px.
|
|
MIN_WIDTH = 120
|
|
# The enrollment protocol's floor: a reference view under 160 px tall carries no
|
|
# text geometry (facings under 60 px wide failed every measured lane).
|
|
MIN_HEIGHT = 160
|
|
# The name/OCR gate, re-measured on the shipped packshots with the strict name
|
|
# side in place, each image read four ways and scored against all 12 catalog
|
|
# names: correct packshots 0.9167-1.0000 (seven read their whole name; l11's
|
|
# read omits "マヨネーズ" and is the floor), the wrong-product packshot in
|
|
# references/rejected/ 0.0476-0.5833. 0.75 is the midpoint: 0.167 of headroom
|
|
# above every correct packshot and 0.167 below the wrong-product ceiling, so
|
|
# OCR noise on either side does not move a verdict. Siblings are not this
|
|
# gate's job - the コクうま packshot scores 0.85 against the マヨネーズ name,
|
|
# which is what text_discriminated_siblings flags, not a gap a threshold closes.
|
|
MIN_NAME_OCR = 0.75
|
|
# Vision drops small print at native scale: the l11 packshot reads ハーフ only
|
|
# when upscaled (0.57 -> 0.79). Reading at 2x is what makes the check about the
|
|
# package rather than about Vision's recall.
|
|
OCR_SCALE = 2.0
|
|
# Two views of one SKU this similar are the same photograph counted twice.
|
|
DUPLICATE_COSINE = 0.98
|
|
|
|
# The enrollment protocol's pyramid target: a source at least this tall can be
|
|
# stored as a 200/400/800 px pyramid, so a view under it cannot be scored
|
|
# against the rest. Refused, not reported: the l09 defect was a 305 px view
|
|
# whose inlier scores (85-129) dwarfed every other reference (10-40) and it won
|
|
# facings it should not, and a report is exactly the kind of thing that gets
|
|
# read once and then ignored. The fix is re-sourcing, which is a data task
|
|
# this tool should not let through the door in the first place.
|
|
PYRAMID_FLOOR = 800
|
|
|
|
# Official pack-shot pages publish a single front view per JAN, so a catalog
|
|
# built from them runs at 1 and says so; the floor exists to catch accidental
|
|
# single-view enrollment from shelf crops, where several views are available.
|
|
MIN_VIEWS = 2
|
|
IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png")
|
|
SESSION_KEYS = ("capture_session_id", "session_id")
|
|
SESSION_LIST_KEYS = ("sessions", "session_ids", "evaluation_sessions")
|
|
|
|
|
|
class RebuildError(Exception):
|
|
"""Refused input. The destination, if it existed, is unchanged."""
|
|
|
|
|
|
def sku_of(filename: str) -> str:
|
|
"""`sku_l01_2.jpg` -> `sku_l01`; `sku_l01.jpg` -> `sku_l01`."""
|
|
stem = os.path.splitext(filename)[0]
|
|
parts = stem.split("_")
|
|
if len(parts) >= 2 and parts[0] == "sku":
|
|
return "_".join(parts[:2])
|
|
return stem
|
|
|
|
|
|
def view_rank(filename: str) -> int:
|
|
"""Order views so the primary view sorts first: `sku_l01.jpg` before `_2`."""
|
|
stem = os.path.splitext(filename)[0]
|
|
parts = stem.split("_")
|
|
return int(parts[2]) if len(parts) > 2 and parts[2].isdigit() else 1
|
|
|
|
|
|
def cosine(left: np.ndarray, right: np.ndarray) -> float:
|
|
a = left.astype(np.float32).ravel()
|
|
b = right.astype(np.float32).ravel()
|
|
denom = float(np.linalg.norm(a) * np.linalg.norm(b))
|
|
return float((a * b).sum() / denom) if denom else 0.0
|
|
|
|
|
|
def upscale(image: np.ndarray) -> np.ndarray:
|
|
"""Enlarge for OCR only; the stored view keeps its own pixels."""
|
|
return cv2.resize(image, None, fx=OCR_SCALE, fy=OCR_SCALE, interpolation=cv2.INTER_CUBIC)
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""NFKC, uppercase, small kana folded, alnum and `%` kept."""
|
|
value = unicodedata.normalize("NFKC", text).upper()
|
|
# Vision varies between small and full-size kana, and Kewpie prints
|
|
# キユーピー with a small ユ; folding both keeps the comparison about the
|
|
# product name rather than the typesetter.
|
|
value = value.translate(str.maketrans("ッャュョァィゥェォヮヵヶ", "ツヤユヨアイウエオワカケ"))
|
|
value = value.replace("キユーピー", "キユピ").replace("キューピー", "キユピ")
|
|
return "".join(char for char in value if char.isalnum() or char == "%")
|
|
|
|
|
|
def signature_bigrams(text: str) -> Set[str]:
|
|
"""Character pairs of a catalog name, never across a word boundary.
|
|
|
|
A pair that spans two words is not a substring of either one, so it can
|
|
only ever be noise - and expensive noise, because it is rare across the
|
|
catalog and IDF therefore weights it as highly distinctive. "キユーピー
|
|
アマニ油" used to contribute "ピア", which let the garbage read
|
|
"キユピアク5C" be accepted as that row.
|
|
"""
|
|
pairs: Set[str] = set()
|
|
for token in text.split():
|
|
value = normalize_text(token)
|
|
pairs.update(value[index:index + 2] for index in range(len(value) - 1))
|
|
return pairs
|
|
|
|
|
|
def bigrams(text: str) -> Set[str]:
|
|
"""Character pairs of a read, across whitespace.
|
|
|
|
Deliberately not `signature_bigrams`: OCR puts spaces where the package
|
|
has none, so "ピュア セレクト" has to keep matching the catalog's
|
|
"ピュアセレクト". The catalog name is authoritative about where its words
|
|
end; a read is noisy evidence about the same characters, and dropping its
|
|
spaces is the only way the two can meet. The read is the side that gets
|
|
the benefit of the doubt, so the fake-pair problem is fixed where it
|
|
originates - in the signature - rather than by refusing to match.
|
|
"""
|
|
value = normalize_text(text)
|
|
return {value[index:index + 2] for index in range(len(value) - 1)}
|
|
|
|
|
|
def name_ocr_similarity(name: str, ocr_text: str) -> float:
|
|
"""Share of the catalog name's bigrams that the OCR text explains."""
|
|
name_bigrams = signature_bigrams(name)
|
|
return len(name_bigrams & bigrams(ocr_text)) / max(1, len(name_bigrams))
|
|
|
|
|
|
def best_orientation_text(ocr_engine: Any, image: np.ndarray, name: str = "") -> str:
|
|
"""OCR text from whichever of the four right-angle rotations matches best.
|
|
|
|
A packshot's identifying text is not always horizontal: the やまや
|
|
めんたいマヨネーズ packshot reads 明太手屋さんが only at 270 degrees, and
|
|
the upright pass returns three stray characters. Checking one orientation
|
|
refused a correct image, which is the worst failure this check has - it
|
|
rejects good data while claiming to validate it.
|
|
|
|
The winner is the rotation that explains the most of the catalog name, not
|
|
the one that reads the most characters: the upright pass on that packshot
|
|
returns stray kana that concatenate longer than the real read, so length
|
|
picks the wrong one. With no name to compare against, length is the only
|
|
signal left.
|
|
"""
|
|
rotations = (
|
|
image,
|
|
cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE),
|
|
cv2.rotate(image, cv2.ROTATE_180),
|
|
cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE),
|
|
)
|
|
reads = [" ".join(line.text for line in ocr_engine.read(upscale(rotated)).lines)
|
|
for rotated in rotations]
|
|
if name:
|
|
return max(reads, key=lambda text: name_ocr_similarity(name, text))
|
|
return max(reads, key=lambda text: len(text.strip()))
|
|
|
|
|
|
def collect(source: str) -> Dict[str, List[str]]:
|
|
by_sku: Dict[str, List[str]] = {}
|
|
for name in sorted(os.listdir(source)):
|
|
if os.path.splitext(name)[1].lower() not in IMAGE_EXTENSIONS:
|
|
continue
|
|
by_sku.setdefault(sku_of(name), []).append(name)
|
|
for views in by_sku.values():
|
|
views.sort(key=view_rank)
|
|
return by_sku
|
|
|
|
|
|
def read_catalog(csv_path: str) -> Tuple[List[str], List[Dict[str, str]], Dict[str, str]]:
|
|
"""Columns, rows, and SKU -> product_name. Refuses a malformed catalog."""
|
|
with open(csv_path, newline="", encoding="utf-8-sig") as handle:
|
|
reader = csv.DictReader(handle)
|
|
fieldnames = list(reader.fieldnames or [])
|
|
rows = list(reader)
|
|
if "SKU_ID" not in fieldnames or "product_name" not in fieldnames:
|
|
raise RebuildError(f"{csv_path}: needs SKU_ID and product_name columns")
|
|
names: Dict[str, str] = {}
|
|
duplicates: Set[str] = set()
|
|
for row in rows:
|
|
sku = (row.get("SKU_ID") or "").strip()
|
|
if not sku:
|
|
raise RebuildError(f"{csv_path}: a row has an empty SKU_ID")
|
|
if sku in names:
|
|
duplicates.add(sku)
|
|
names[sku] = (row.get("product_name") or "").strip()
|
|
if duplicates:
|
|
raise RebuildError("duplicate CSV row(s): " + ", ".join(sorted(duplicates)))
|
|
return fieldnames, rows, names
|
|
|
|
|
|
def _session_values(payload: Any) -> Set[str]:
|
|
"""Every session id anywhere in a manifest, whatever its shape."""
|
|
found: Set[str] = set()
|
|
if isinstance(payload, dict):
|
|
for key, value in payload.items():
|
|
if key in SESSION_KEYS and isinstance(value, (str, int)):
|
|
found.add(str(value))
|
|
elif key in SESSION_LIST_KEYS and isinstance(value, list):
|
|
for item in value:
|
|
found.update([str(item)] if isinstance(item, (str, int)) else _session_values(item))
|
|
elif isinstance(value, (dict, list)):
|
|
found.update(_session_values(value))
|
|
elif isinstance(payload, list):
|
|
for item in payload:
|
|
found.update(_session_values(item))
|
|
return found
|
|
|
|
|
|
def load_eval_sessions(path: str) -> Set[str]:
|
|
"""Sessions an evaluation used. A manifest naming none is refused, not trusted."""
|
|
with open(path, encoding="utf-8") as handle:
|
|
sessions = _session_values(json.load(handle))
|
|
if not sessions:
|
|
raise RebuildError(f"{path}: evaluation manifest names no capture session")
|
|
return sessions
|
|
|
|
|
|
def load_capture_sessions(path: Optional[str]) -> Dict[str, str]:
|
|
"""Image basename -> capture session, from an enrollment manifest."""
|
|
if not path:
|
|
return {}
|
|
with open(path, encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
if isinstance(payload, dict):
|
|
entries = payload.get("images", [])
|
|
default = payload.get("capture_session_id") or payload.get("session_id")
|
|
else:
|
|
entries, default = payload, None
|
|
if not isinstance(entries, list):
|
|
raise RebuildError(f"{path}: 'images' must be a list")
|
|
sessions: Dict[str, str] = {}
|
|
for entry in entries:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
filename = entry.get("file") or entry.get("filename") or entry.get("image")
|
|
session = entry.get("capture_session_id") or entry.get("session_id") or default
|
|
if filename and session is not None:
|
|
sessions[os.path.basename(str(filename))] = str(session)
|
|
return sessions
|
|
|
|
|
|
def audit(
|
|
source: str,
|
|
min_views: int = MIN_VIEWS,
|
|
names: Optional[Dict[str, str]] = None,
|
|
ocr_engine: Any = None,
|
|
capture_sessions: Optional[Dict[str, str]] = None,
|
|
eval_sessions: Optional[Set[str]] = None,
|
|
) -> Tuple[Dict[str, List[str]], List[Dict[str, str]]]:
|
|
"""Keep the views that pass, and report every refusal with its reason."""
|
|
kept: Dict[str, List[str]] = {}
|
|
refused: List[Dict[str, str]] = []
|
|
capture_sessions = capture_sessions or {}
|
|
eval_sessions = eval_sessions or set()
|
|
for sku, views in sorted(collect(source).items()):
|
|
accepted: List[Tuple[str, np.ndarray]] = []
|
|
for name in views:
|
|
image = cv2.imread(os.path.join(source, name))
|
|
if image is None:
|
|
refused.append({"sku": sku, "view": name, "reason": "unreadable"})
|
|
continue
|
|
# Leakage first: a view from an evaluation session is refused even if
|
|
# it would fail another check, so the leak is never hidden by one.
|
|
session = capture_sessions.get(name)
|
|
if session and session in eval_sessions:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"capture session {session} is in the evaluation manifest"})
|
|
continue
|
|
if image.shape[0] < MIN_HEIGHT:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"{image.shape[0]}px tall, below the {MIN_HEIGHT}px floor"})
|
|
continue
|
|
if image.shape[1] < MIN_WIDTH:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"{image.shape[1]}px wide, below the {MIN_WIDTH}px floor"})
|
|
continue
|
|
if image.shape[0] < PYRAMID_FLOOR:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"{image.shape[0]}px tall, below the {PYRAMID_FLOOR}px "
|
|
"pyramid floor; its scores would not be comparable "
|
|
"with the rest - re-source this view"})
|
|
continue
|
|
if ocr_engine is not None and names is not None:
|
|
try:
|
|
text = best_orientation_text(ocr_engine, image, names.get(sku, ""))
|
|
except OcrUnavailable as error:
|
|
print(f"OCR skipped for {name}: {error}", file=sys.stderr)
|
|
else:
|
|
similarity = name_ocr_similarity(names.get(sku, ""), text)
|
|
if similarity < MIN_NAME_OCR:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"name/OCR {similarity:.2f} below {MIN_NAME_OCR:.2f}: "
|
|
f"{text[:100] or '<no text read>'}"})
|
|
continue
|
|
duplicate = next((other for other, pixels in accepted
|
|
if image.shape == pixels.shape and cosine(image, pixels) >= DUPLICATE_COSINE), None)
|
|
if duplicate:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"near-duplicate of {duplicate}"})
|
|
continue
|
|
accepted.append((name, image))
|
|
if len(accepted) < min_views:
|
|
for name, _ in accepted:
|
|
refused.append({"sku": sku, "view": name,
|
|
"reason": f"only {len(accepted)} usable view(s), {min_views} required"})
|
|
continue
|
|
kept[sku] = [name for name, _ in accepted]
|
|
return kept, refused
|
|
|
|
|
|
def unattributed(source: str) -> List[str]:
|
|
"""SKUs whose views have no entry in the source's ATTRIBUTION.json.
|
|
|
|
Reported, not refused. A view with no recorded source is still a usable
|
|
reference - the six that predate the attribution file are independent of
|
|
the evaluation photo and their OCR matches their catalog names - but the
|
|
provenance is gone, and provenance is what lets a later audit tell a
|
|
re-sourced packshot from a leaked shelf crop. Naming them keeps the gap
|
|
from growing quietly.
|
|
"""
|
|
path = os.path.join(source, "ATTRIBUTION.json")
|
|
try:
|
|
with open(path) as handle:
|
|
recorded = json.load(handle)
|
|
except (OSError, ValueError):
|
|
recorded = {}
|
|
return sorted(sku for sku in collect(source) if sku not in recorded)
|
|
|
|
|
|
def text_discriminated_siblings(
|
|
kept: Dict[str, List[str]], source: str, names: Optional[Dict[str, str]] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""Cross-SKU near-duplicates: the same artwork on two different products.
|
|
|
|
These are the near-unknown case no threshold rejects. The geometry is real
|
|
because the sibling shares the brand's artwork, so only the package text
|
|
separates them. They are kept - they are legitimately different products -
|
|
but flagged, because a catalog that enrolls one and not the other will
|
|
accept the missing one as the enrolled one.
|
|
"""
|
|
views: List[Tuple[str, str, np.ndarray]] = []
|
|
for sku, filenames in sorted(kept.items()):
|
|
for filename in filenames:
|
|
image = cv2.imread(os.path.join(source, filename))
|
|
if image is not None:
|
|
views.append((sku, filename, image))
|
|
flagged: List[Dict[str, Any]] = []
|
|
for index, (sku, filename, image) in enumerate(views):
|
|
for other_sku, other_name, other in views[index + 1:]:
|
|
if other_sku == sku or image.shape != other.shape:
|
|
continue
|
|
similarity = cosine(image, other)
|
|
if similarity < DUPLICATE_COSINE:
|
|
continue
|
|
flagged.append({
|
|
"sku": sku,
|
|
"view": filename,
|
|
"sibling_sku": other_sku,
|
|
"sibling_view": other_name,
|
|
"cosine": round(similarity, 4),
|
|
"name_similarity": round(
|
|
name_ocr_similarity(
|
|
(names or {}).get(sku, ""), (names or {}).get(other_sku, "")
|
|
), 3,
|
|
),
|
|
})
|
|
return flagged
|
|
|
|
|
|
def live_ready_verdict(
|
|
kept: Dict[str, List[str]], names: Optional[Dict[str, str]], shelf_skus: Optional[Set[str]]
|
|
) -> Dict[str, Any]:
|
|
"""The enrollment protocol's completeness rule, as a verdict.
|
|
|
|
A bay is live-ready when every SKU on its shelf tags is enrolled or
|
|
explicitly listed as text-only. Without a shelf-tag list the rule cannot be
|
|
evaluated, and the verdict says so rather than guessing.
|
|
"""
|
|
catalog = set(names or {})
|
|
enrolled = set(kept)
|
|
text_only = catalog - enrolled
|
|
if shelf_skus is None:
|
|
return {
|
|
"live_ready": None,
|
|
"reason": "no --shelf-tags given; the completeness rule needs the shelf's SKU list",
|
|
"enrolled": len(enrolled),
|
|
"text_only": sorted(text_only),
|
|
}
|
|
missing = sorted(shelf_skus - enrolled - text_only)
|
|
return {
|
|
"live_ready": not missing,
|
|
"reason": (
|
|
"every shelf SKU is enrolled or text-only"
|
|
if not missing
|
|
else "on the shelf but absent from the catalog: " + ", ".join(missing)
|
|
),
|
|
"enrolled": len(enrolled),
|
|
"text_only": sorted(text_only),
|
|
"shelf_skus": len(shelf_skus),
|
|
"missing_from_catalog": missing,
|
|
}
|
|
|
|
|
|
def load_shelf_skus(path: Optional[str]) -> Optional[Set[str]]:
|
|
"""SKUs on the shelf's tags: one per line, or a JSON list."""
|
|
if not path:
|
|
return None
|
|
with open(path, encoding="utf-8") as handle:
|
|
text = handle.read()
|
|
try:
|
|
payload = json.loads(text)
|
|
except json.JSONDecodeError:
|
|
return {line.strip() for line in text.splitlines() if line.strip()}
|
|
if isinstance(payload, list):
|
|
return {str(item).strip() for item in payload if str(item).strip()}
|
|
raise RebuildError(f"{path}: expected a JSON list or one SKU per line")
|
|
|
|
|
|
def write_catalog(kept: Dict[str, List[str]], source: str, output: str, csv_path: Optional[str]) -> None:
|
|
"""Copy the kept views and the whole CSV into a staging dir, then swap it in."""
|
|
staging = tempfile.mkdtemp(prefix=".catalog-", dir=os.path.dirname(os.path.abspath(output)) or ".")
|
|
try:
|
|
for views in kept.values():
|
|
for name in views:
|
|
shutil.copy2(os.path.join(source, name), os.path.join(staging, name))
|
|
if csv_path:
|
|
# Byte-for-byte, not re-serialized: gallery.catalog_digest hashes the
|
|
# raw file, and csv.DictWriter's default CRLF would move the digest
|
|
# and invalidate a trained identity head over a newline.
|
|
shutil.copy2(csv_path, os.path.join(staging, "stock.csv"))
|
|
if os.path.isdir(output):
|
|
# The documented rebuild runs with --source == --output, and the
|
|
# catalog directory also holds provenance this tool does not manage
|
|
# (README, references/rejected/, label files). Carry those over so a
|
|
# rebuild replaces the views and the CSV, not the record of where
|
|
# they came from.
|
|
for entry in os.listdir(output):
|
|
if entry == "stock.csv" or os.path.splitext(entry)[1].lower() in IMAGE_EXTENSIONS:
|
|
continue
|
|
origin, target = os.path.join(output, entry), os.path.join(staging, entry)
|
|
if os.path.isdir(origin):
|
|
shutil.copytree(origin, target)
|
|
else:
|
|
shutil.copy2(origin, target)
|
|
if os.path.exists(output):
|
|
shutil.rmtree(output)
|
|
os.replace(staging, output)
|
|
except BaseException:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
raise
|
|
|
|
|
|
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--source", required=True, help="directory of pack shots")
|
|
parser.add_argument("--output", required=True, help="catalog directory to write")
|
|
parser.add_argument("--csv", help="stock.csv to validate and carry over whole")
|
|
parser.add_argument("--replace", action="store_true", help="overwrite an existing output")
|
|
parser.add_argument("--min-views", type=int, default=MIN_VIEWS,
|
|
help=f"views required per SKU (default {MIN_VIEWS}; "
|
|
"official pack-shot pages publish one front view per JAN)")
|
|
parser.add_argument("--ocr", choices=("auto", "vision", "none"), default=None,
|
|
help="packshot/name OCR backend (default: DORAYAKI_OCR, else vision "
|
|
"on macOS; an unavailable backend is skipped, not fatal)")
|
|
parser.add_argument("--source-manifest",
|
|
help="image capture metadata (default: SOURCE/manifest.json when present)")
|
|
parser.add_argument("--eval-manifest",
|
|
help="manifest of evaluation sessions; images captured in them are refused")
|
|
parser.add_argument("--shelf-tags",
|
|
help="the shelf's SKU list (JSON list or one per line) for the "
|
|
"completeness rule; without it the verdict is 'unknown'")
|
|
parser.add_argument("--json", help="write the audit here")
|
|
args = parser.parse_args(argv)
|
|
|
|
if os.path.exists(args.output) and not args.replace:
|
|
raise SystemExit(f"{args.output} exists; pass --replace to overwrite")
|
|
|
|
try:
|
|
if args.csv:
|
|
_, _, names = read_catalog(args.csv)
|
|
missing_rows = set(collect(args.source)) - set(names)
|
|
if missing_rows:
|
|
raise RebuildError("enrolled image SKU(s) with no CSV row: " + ", ".join(sorted(missing_rows)))
|
|
else:
|
|
names = {}
|
|
print("no --csv: image/CSV match and name/OCR checks skipped", file=sys.stderr)
|
|
|
|
source_manifest = args.source_manifest
|
|
default_manifest = os.path.join(args.source, "manifest.json")
|
|
if source_manifest is None and os.path.isfile(default_manifest):
|
|
source_manifest = default_manifest
|
|
capture_sessions = load_capture_sessions(source_manifest)
|
|
eval_sessions = load_eval_sessions(args.eval_manifest) if args.eval_manifest else set()
|
|
if eval_sessions:
|
|
unlabelled = sorted(name for views in collect(args.source).values() for name in views
|
|
if name not in capture_sessions)
|
|
if unlabelled:
|
|
print("eval-session check cannot cover image(s) with no capture_session_id: "
|
|
+ ", ".join(unlabelled), file=sys.stderr)
|
|
|
|
# Read the shelf list before anything is written: a bad path or a
|
|
# malformed list must fail here, with the tool's own message, rather
|
|
# than after write_catalog has already replaced the catalog.
|
|
shelf_skus = load_shelf_skus(args.shelf_tags)
|
|
engine = build_ocr_engine(args.ocr)
|
|
ocr_engine = engine if engine.available() else None
|
|
if ocr_engine is None:
|
|
print("name/OCR check skipped: no OCR backend available", file=sys.stderr)
|
|
kept, refused = audit(args.source, min_views=args.min_views, names=names,
|
|
ocr_engine=ocr_engine, capture_sessions=capture_sessions,
|
|
eval_sessions=eval_sessions)
|
|
if refused:
|
|
print(f"refused {len(refused)} view(s):", file=sys.stderr)
|
|
for row in refused:
|
|
print(f" {row['sku']:<10} {row['view']:<20} {row['reason']}", file=sys.stderr)
|
|
if not kept:
|
|
raise RebuildError("no SKU has usable views; nothing written")
|
|
write_catalog(kept, args.source, args.output, args.csv)
|
|
except (OSError, ValueError, json.JSONDecodeError, RebuildError, OcrUnavailable) as error:
|
|
raise SystemExit(str(error)) from error
|
|
|
|
# A row with no kept image is a text-only entry: the text lane still names
|
|
# and vetoes it from the CSV alone, so it is a coverage gap, not a refusal.
|
|
text_only = sorted(set(names) - set(kept))
|
|
siblings = text_discriminated_siblings(kept, args.source, names)
|
|
no_source = unattributed(args.source)
|
|
verdict = live_ready_verdict(kept, names, shelf_skus)
|
|
report = {
|
|
"source": args.source,
|
|
"output": args.output,
|
|
"min_width": MIN_WIDTH,
|
|
"min_height": MIN_HEIGHT,
|
|
"min_views": args.min_views,
|
|
"duplicate_cosine": DUPLICATE_COSINE,
|
|
"min_name_ocr": MIN_NAME_OCR,
|
|
"ocr": engine.name if ocr_engine is not None else "skipped",
|
|
"text_only_skus": text_only,
|
|
"text_discriminated_siblings": siblings,
|
|
"unattributed_skus": no_source,
|
|
"completeness": verdict,
|
|
"skus": {sku: views for sku, views in sorted(kept.items())},
|
|
"refused": refused,
|
|
}
|
|
if args.json:
|
|
with open(args.json, "w", encoding="utf-8") as handle:
|
|
json.dump(report, handle, indent=1, ensure_ascii=False)
|
|
|
|
print(f"{len(kept)} SKU(s), {sum(len(v) for v in kept.values())} view(s) -> {args.output}")
|
|
for sku, views in sorted(kept.items()):
|
|
print(f" {sku:<10} {len(views)} view(s): {', '.join(views)}")
|
|
if text_only:
|
|
print(f"\ntext-only CSV row(s), no enrolled image ({len(text_only)}): {', '.join(text_only)}")
|
|
if siblings:
|
|
print(f"\ntext-discriminated sibling(s) ({len(siblings)}):")
|
|
for row in siblings:
|
|
print(f" {row['sku']:<10} {row['view']:<20} ~ {row['sibling_sku']} "
|
|
f"({row['sibling_view']}) cosine {row['cosine']}")
|
|
if no_source:
|
|
print(f"\nno recorded source ({len(no_source)}): {', '.join(no_source)} - "
|
|
"usable, but provenance is missing; re-sourcing would close it")
|
|
if verdict["live_ready"] is None:
|
|
print(f"\ncompleteness: unknown - {verdict['reason']}")
|
|
elif verdict["live_ready"]:
|
|
print(f"\ncompleteness: live-ready ({verdict['reason']})")
|
|
else:
|
|
print(f"\ncompleteness: NOT live-ready - {verdict['reason']}")
|
|
if refused:
|
|
print(f"\nrefused {len(refused)} view(s):")
|
|
for row in refused:
|
|
print(f" {row['sku']:<10} {row['view']:<20} {row['reason']}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|