product-recognition-demo/recognization/code/reconcile.py
kiyo9w 8f750c15c5 feat: price-tag UI, still-image tag passes, and honest tag evidence
Web client: price-tag boxes in a distinct style (teal, chip with the parsed
price), an OCR section in the product card showing the tag's price, name,
discount and the reconciliation verdict with both values, and a compliance strip
under the stage. The photo stage now draws the image into the canvas and uses the
same containment as the media, so boxes cannot drift from the product (they did:
the canvas stretched to the stage while the photo was contained).

Server fixes the browser runs forced out:
- Photo mode reconciled before identities existed, so every tag read "tag only".
  The tag bundle is cached with the still and classify re-reconciles against the
  identities it just produced, returning them per detection.
- The still path only ever read the first four bands (plan hands out two per
  call); it now drains every band, twice at different upscales so the vote has
  independent reads.
- Voting is confidence-weighted and a dead heat stays unread; the plausible
  shelf-price range is 10-9999, which stops a misread from becoming a ¥26,300
  price. The compliance coverage bar moved to 0.7 after 0.5 produced a false
  "group without price tag" on a shelf where every facing has one.

Measured on add/image6.jpg: 8 readable tags with names and prices, 942 ms for
the whole still, 7 real price mismatches, no false violations, and hints that
say plainly when a row could not be read.

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-09-24 14:11:23 +09:00

568 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Reconciliation (item vs price tag) and the first compliance rule family.
Two pure layers, no I/O:
- `reconcile()` compares one accepted identity with its tag and the catalog entry,
and publishes a state plus the individual signals. It never changes the product
identity: a disagreement is a second opinion, displayed next to the first.
- `evaluate_compliance()` turns settled evidence into findings. Every rule fails
closed on evidence quality: a row whose tags are merely unreadable produces a
capture hint, never a violation.
"""
from __future__ import annotations
import os
import re
import sys
import unicodedata
from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from tags import TAG_STATE_ABSENT, TAG_STATE_READABLE, TAG_STATE_UNREADABLE # noqa: E402
RECONCILED_CONFIRMED = "confirmed"
RECONCILED_MISMATCH = "mismatch"
RECONCILED_UNVERIFIED = "unverified"
RECONCILED_NO_TAG = "no_tag"
RECONCILED_TAG_ONLY = "tag_only"
RULE_MISSING_TAG = "missing_price_tag"
RULE_MISSING_TAG_GROUP = "missing_tag_group"
RULE_ORPHAN_TAG = "orphan_tag"
RULE_PRICE_MISMATCH = "price_mismatch"
RULE_DUPLICATE_TAG = "duplicate_tag"
RULE_UNREADABLE_TAGS = "unreadable_tags"
NAME_MATCH_THRESHOLD = 0.4
GROUP_MIN_FACINGS = 3
STABILITY_FRAMES = 3
STABILITY_WINDOW = 5
# ---------------------------------------------------------------------------
# text matching
# ---------------------------------------------------------------------------
def normalize_name(text: Optional[str]) -> str:
if not text:
return ""
text = unicodedata.normalize("NFKC", text)
text = re.sub(r"<[^>]*>", "", text)
text = re.sub(r"[\s・、。,.\-–—_/]+", "", text)
return text.lower()
def longest_common_substring_ratio(a: str, b: str) -> float:
"""Length of the longest common substring over the shorter string."""
if not a or not b:
return 0.0
shorter, longer = (a, b) if len(a) <= len(b) else (b, a)
best = 0
for start in range(len(shorter)):
for end in range(start + best + 1, len(shorter) + 1):
if shorter[start:end] in longer:
best = end - start
else:
break
return best / len(shorter)
# ---------------------------------------------------------------------------
# reconciliation
# ---------------------------------------------------------------------------
@dataclass
class Signal:
kind: str # "jan" | "name" | "price"
agrees: bool
item: Any
tag: Any
note: Optional[str] = None
def as_dict(self) -> Dict[str, Any]:
payload = {"kind": self.kind, "agrees": self.agrees, "item": self.item, "tag": self.tag}
if self.note:
payload["note"] = self.note
return payload
def reconcile(
identity: Optional[Dict[str, Any]],
tag: Optional[Dict[str, Any]],
product: Optional[Dict[str, Any]],
) -> Optional[Dict[str, Any]]:
"""Compare one facing with its tag.
Returns None when there is nothing to reconcile (no accepted identity and no
readable tag). A tag with no product above it is `tag_only`.
"""
identity_state = (identity or {}).get("state")
tag_state = (tag or {}).get("state", TAG_STATE_ABSENT)
accepted = identity_state == "recognized"
readable_tag = tag_state == TAG_STATE_READABLE
if not accepted and not readable_tag:
return None
if accepted and tag_state == TAG_STATE_ABSENT:
return {"state": RECONCILED_NO_TAG, "signals": [], "message": None}
if accepted and tag_state == TAG_STATE_UNREADABLE:
return {
"state": RECONCILED_UNVERIFIED,
"signals": [],
"message": "tag found but its text was not read",
}
if not accepted and readable_tag:
return {"state": RECONCILED_TAG_ONLY, "signals": [], "message": None}
signals: List[Signal] = []
# JAN is the only exact signal; a checksummed barcode is not a guess.
item_jan = str((product or {}).get("jan") or "").strip()
tag_jan = str((tag or {}).get("jan") or "").strip()
if item_jan and tag_jan:
signals.append(Signal("jan", item_jan == tag_jan, item_jan, tag_jan))
item_name = normalize_name((product or {}).get("product_name"))
tag_name = normalize_name((tag or {}).get("name"))
if item_name and tag_name:
ratio = longest_common_substring_ratio(item_name, tag_name)
signals.append(
Signal("name", ratio >= NAME_MATCH_THRESHOLD, item_name[:32], tag_name[:32],
note=f"match {ratio:.2f}")
)
item_price = (product or {}).get("price")
tag_price = (tag or {}).get("price")
tag_regular = (tag or {}).get("regular_price")
if item_price is not None and (tag_price is not None or tag_regular is not None):
candidates = [value for value in (tag_price, tag_regular) if value is not None]
agrees = any(abs(float(item_price) - float(value)) < 0.5 for value in candidates)
signals.append(Signal("price", agrees, item_price, tag_price if tag_price is not None else tag_regular))
if not signals:
return {
"state": RECONCILED_UNVERIFIED,
"signals": [],
"message": "tag read, but nothing comparable in the catalog",
}
if any(signal.kind == "jan" for signal in signals):
decided = [signal for signal in signals if signal.kind == "jan"][0]
state = RECONCILED_CONFIRMED if decided.agrees else RECONCILED_MISMATCH
elif any(signal.agrees for signal in signals):
state = RECONCILED_CONFIRMED
elif all(not signal.agrees for signal in signals):
state = RECONCILED_MISMATCH
else:
state = RECONCILED_UNVERIFIED
message = None
if state == RECONCILED_MISMATCH:
price_signal = next((signal for signal in signals if signal.kind == "price"), None)
name_signal = next((signal for signal in signals if signal.kind == "name"), None)
if price_signal is not None:
message = f"tag says {price_signal.tag}, catalog says {price_signal.item}"
elif name_signal is not None:
message = "tag name does not match the catalog entry"
return {"state": state, "signals": [signal.as_dict() for signal in signals], "message": message}
# ---------------------------------------------------------------------------
# compliance
# ---------------------------------------------------------------------------
@dataclass
class ComplianceSettings:
group_min_facings: int = GROUP_MIN_FACINGS
stability_frames: int = STABILITY_FRAMES
stability_window: int = STABILITY_WINDOW
# A row may only be judged when enough of its facings carry tag evidence
# (readable or unreadable). Below this, the row is "not evaluated": claiming a
# tag is missing because our detector found none would be a false violation.
# 0.7 measured: at 0.5 a row where our own tag detection was patchy produced a
# false "group without price tag" on a shelf where every facing has a tag.
min_row_tag_coverage: float = 0.7
@dataclass
class Finding:
rule: str
severity: str # "violation" | "hint"
row_index: Optional[int]
box: List[int]
count: int = 1
skus: List[str] = field(default_factory=list)
detail: Optional[str] = None
def key(self) -> Tuple[Any, ...]:
return (self.rule, self.row_index, self.box[0] // 32, self.box[1] // 32, self.count)
def as_dict(self) -> Dict[str, Any]:
payload = {
"rule": self.rule,
"severity": self.severity,
"row_index": self.row_index,
"box": list(self.box),
"count": self.count,
"skus": list(self.skus),
}
if self.detail:
payload["detail"] = self.detail
return payload
class ComplianceTracker:
"""Keeps findings only while they persist, so the UI never flickers."""
def __init__(self, settings: Optional[ComplianceSettings] = None):
self.settings = settings or ComplianceSettings()
self._history: List[set] = []
def update(self, findings: Sequence[Finding]) -> List[Finding]:
current = {finding.key(): finding for finding in findings}
self._history.append(set(current))
if len(self._history) > self.settings.stability_window:
self._history.pop(0)
stable: List[Finding] = []
for key, finding in current.items():
seen = sum(1 for frame in self._history if key in frame)
if seen >= self.settings.stability_frames:
stable.append(finding)
return stable
def reset(self) -> None:
self._history.clear()
def evaluate_compliance(
tracks: Sequence[Dict[str, Any]],
tag_by_track: Dict[int, Dict[str, Any]],
all_tags: Sequence[Dict[str, Any]],
bands: Sequence[Any],
reconciliations: Optional[Dict[int, Dict[str, Any]]] = None,
settings: Optional[ComplianceSettings] = None,
) -> List[Finding]:
"""Findings from settled evidence. Rules with unreadable evidence stay silent.
`tag_by_track` is the per-facing attachment; `all_tags` is every tag the pass
found, so a tag nobody claimed can be reported as an orphan.
"""
settings = settings or ComplianceSettings()
findings: List[Finding] = []
by_row: Dict[int, List[Dict[str, Any]]] = {}
for track in tracks:
row_index = _row_for(track.get("box") or track.get("raw_box"), bands)
by_row.setdefault(row_index, []).append(track)
for row_index, row_tracks in by_row.items():
row_tracks.sort(key=lambda track: (track.get("box") or [0])[0])
row_tags = [tag_by_track.get(int(track["track_id"]), {}) for track in row_tracks]
readable = [tag for tag in row_tags if tag.get("state") == TAG_STATE_READABLE]
evidenced = [
tag for tag in row_tags if tag.get("state") in (TAG_STATE_READABLE, TAG_STATE_UNREADABLE)
]
coverage = len(evidenced) / max(1, len(row_tracks))
if not readable or coverage < settings.min_row_tag_coverage:
# Fail closed on evidence quality. Either nothing in this row could be
# read, or too few facings carry any tag evidence at all - in both cases
# the honest answer is "not evaluated", never "tag missing".
if evidenced:
detail = (
"move closer: the price tags in this row could not be read"
if not readable
else f"tags detected for {len(evidenced)} of {len(row_tracks)} facings in this row"
)
findings.append(
Finding(
rule=RULE_UNREADABLE_TAGS,
severity="hint",
row_index=row_index,
box=_row_box(row_tracks),
count=len(row_tracks),
detail=detail,
)
)
continue
# missing_price_tag: one finding per row (the UI highlights the row), with
# the facings that lack a tag listed rather than one finding per facing.
missing = [
track
for track, tag in zip(row_tracks, row_tags)
if tag.get("state") == TAG_STATE_ABSENT
]
if missing:
findings.append(
Finding(
rule=RULE_MISSING_TAG,
severity="violation",
row_index=row_index,
box=_row_box(missing),
count=len(missing),
skus=[_sku_of(track) for track in missing],
detail=f"{len(missing)} of {len(row_tracks)} facings in this row have no price tag",
)
)
# missing_tag_group: consecutive facings without a tag
run: List[Dict[str, Any]] = []
for track, tag in zip(row_tracks, row_tags):
if tag.get("state") == TAG_STATE_ABSENT:
run.append(track)
continue
if len(run) >= settings.group_min_facings:
findings.append(_group_finding(row_index, run))
run = []
if len(run) >= settings.group_min_facings:
findings.append(_group_finding(row_index, run))
# price_mismatch from reconciliation
for track in tracks:
track_id = int(track["track_id"])
reconciliation = (reconciliations or {}).get(track_id)
if not reconciliation or reconciliation.get("state") != RECONCILED_MISMATCH:
continue
if not any(signal.get("kind") == "price" for signal in reconciliation.get("signals", [])):
continue
findings.append(
Finding(
rule=RULE_PRICE_MISMATCH,
severity="violation",
row_index=_row_for(track.get("box") or track.get("raw_box"), bands),
box=list(track.get("box") or track.get("raw_box") or [0, 0, 0, 0]),
count=1,
skus=[_sku_of(track)],
detail=reconciliation.get("message"),
)
)
# orphan_tag: a readable tag that no facing claims (a tag with no product above
# it is either an empty slot with a leftover tag or a mis-clustered read).
attached_ids = {
tag.get("id") for tag in tag_by_track.values() if tag.get("id")
}
for tag in all_tags:
if tag.get("state") != TAG_STATE_READABLE or not tag.get("id"):
continue
if tag["id"] in attached_ids:
continue
findings.append(
Finding(
rule=RULE_ORPHAN_TAG,
severity="violation",
row_index=None,
box=list(tag.get("box") or [0, 0, 0, 0]),
count=1,
detail="price tag with no product above it",
)
)
return findings
def _group_finding(row_index: int, run: Sequence[Dict[str, Any]]) -> Finding:
boxes = [track.get("box") or track.get("raw_box") or [0, 0, 0, 0] for track in run]
x1 = min(box[0] for box in boxes)
y1 = min(box[1] for box in boxes)
x2 = max(box[2] for box in boxes)
y2 = max(box[3] for box in boxes)
return Finding(
rule=RULE_MISSING_TAG_GROUP,
severity="violation",
row_index=row_index,
box=[x1, y1, x2, y2],
count=len(run),
skus=[_sku_of(track) for track in run],
detail=f"{len(run)} facings in a row without a price tag",
)
def _row_box(tracks: Sequence[Dict[str, Any]]) -> List[int]:
boxes = [track.get("box") or track.get("raw_box") or [0, 0, 0, 0] for track in tracks]
return [
min(box[0] for box in boxes),
min(box[1] for box in boxes),
max(box[2] for box in boxes),
max(box[3] for box in boxes),
]
def _row_for(box: Optional[Sequence[int]], bands: Sequence[Any]) -> int:
"""Which rail band (i.e. row) a facing sits above."""
if not box or not bands:
return 0
x_center = (box[0] + box[2]) / 2.0
y_bottom = box[3]
best_index = 0
best_distance = None
for band in bands:
bx1, by1, bx2, by2 = band.box
if not (bx1 - 40 <= x_center <= bx2 + 40):
continue
distance = abs(by1 - y_bottom)
if best_distance is None or distance < best_distance:
best_distance = distance
best_index = band.row_index
return best_index
def _sku_of(track: Dict[str, Any]) -> str:
identity = track.get("identity") or {}
return str(identity.get("sku_id") or "")
def attach_tags_to_boxes(
tag_service: Any,
image: Any,
boxes: Sequence[Dict[str, Any]],
catalog_by_sku: Dict[str, Dict[str, Any]],
identities: Optional[Dict[int, Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""One-shot tag pass for a still image (photo mode / `?tags=true`).
Runs the passes synchronously - a still is not a stream, so there is no frame
loop to protect - then attaches tags, reconciles, and evaluates compliance.
"""
image_shape = image.shape
# A still has no later frame to pick up the remaining bands, and `plan` only
# hands out a few per call (a stream must stay responsive). Drain them, with a
# cap so a huge shelf cannot turn one request into minutes of OCR. Draining
# also feeds the vote: a band read in two rounds is two observations.
bands: List[Any] = []
rounds = 0
while rounds < 4:
planned = tag_service.plan(boxes, image_shape)
if not planned:
break
for band in planned:
# Two independent reads (different upscale) so the vote has evidence
# to work with; a single read has nothing to vote against.
tag_service.run_pass(image, band, scale_bias=1.0)
tag_service.run_pass(image, band, scale_bias=1.7)
bands.append(band)
rounds += 1
tracks = [
{
"track_id": int(box["id"]),
"box": list(box["box"]),
"identity": (identities or {}).get(int(box["id"])) or {},
}
for box in boxes
]
tag_by_track, all_tags, counts = tag_service.attach(tracks)
reconciliations: Dict[int, Dict[str, Any]] = {}
for track in tracks:
track_id = track["track_id"]
tag = tag_by_track.get(track_id)
product = catalog_by_sku.get((track["identity"] or {}).get("sku_id") or "")
decision = reconcile(track["identity"], tag, product)
if decision is not None:
reconciliations[track_id] = decision
findings = evaluate_compliance(tracks, tag_by_track, all_tags, bands, reconciliations)
summary = summarize_findings(findings)
return {
"by_box": tag_by_track,
"tags": all_tags,
"counts": counts,
"reconciliations": reconciliations,
"bands": [{"row_index": band.row_index, "box": band.box} for band in bands],
"bands_objects": list(bands),
"compliance": {
"violations": summary["violations"],
"by_rule": summary["by_rule"],
"rows": [
{
"row_index": finding.row_index,
"rule": finding.rule,
"box": finding.box,
"missing": finding.count,
"skus": finding.skus,
}
for finding in findings
if finding.severity == "violation"
],
"hints": summary["hints"],
},
}
def reconcile_bundle(
bundle: Dict[str, Any],
detections: Sequence[Dict[str, Any]],
catalog_by_sku: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
"""Re-run reconciliation and compliance against a cached tag bundle.
Used when identities arrive after the tag pass (photo mode: recognize paints
boxes, classify names them). Without this, every tag would look like `tag_only`
because it was compared against an identity that did not exist yet.
"""
tracks = [
{
"track_id": int(detection["id"]),
"box": list(detection["box"]),
"identity": detection.get("identity") or {},
}
for detection in detections
]
tag_by_track = bundle.get("by_box") or {}
reconciliations: Dict[int, Dict[str, Any]] = {}
for track in tracks:
track_id = track["track_id"]
tag = tag_by_track.get(track_id)
product = catalog_by_sku.get((track["identity"] or {}).get("sku_id") or "")
decision = reconcile(track["identity"], tag, product)
if decision is not None:
reconciliations[track_id] = decision
findings = evaluate_compliance(
tracks, tag_by_track, bundle.get("tags") or [], bundle.get("bands_objects") or [], reconciliations
)
summary = summarize_findings(findings)
return {
"reconciliations": reconciliations,
"compliance": {
"violations": summary["violations"],
"by_rule": summary["by_rule"],
"rows": [
{
"row_index": finding.row_index,
"rule": finding.rule,
"box": finding.box,
"missing": finding.count,
"skus": finding.skus,
}
for finding in findings
if finding.severity == "violation"
],
"hints": summary["hints"],
},
}
def summarize_findings(findings: Sequence[Finding]) -> Dict[str, Any]:
by_rule: Dict[str, int] = {}
for finding in findings:
if finding.severity != "violation":
continue
by_rule[finding.rule] = by_rule.get(finding.rule, 0) + 1
return {
"violations": sum(by_rule.values()),
"by_rule": by_rule,
"hints": [finding.as_dict() for finding in findings if finding.severity != "violation"],
}