feat: price-tag UI, still-image tag passes, and honest tag evidence
Web client: price-tag boxes in a distinct style (teal, chip with the parsed price), an OCR section in the product card showing the tag's price, name, discount and the reconciliation verdict with both values, and a compliance strip under the stage. The photo stage now draws the image into the canvas and uses the same containment as the media, so boxes cannot drift from the product (they did: the canvas stretched to the stage while the photo was contained). Server fixes the browser runs forced out: - Photo mode reconciled before identities existed, so every tag read "tag only". The tag bundle is cached with the still and classify re-reconciles against the identities it just produced, returning them per detection. - The still path only ever read the first four bands (plan hands out two per call); it now drains every band, twice at different upscales so the vote has independent reads. - Voting is confidence-weighted and a dead heat stays unread; the plausible shelf-price range is 10-9999, which stops a misread from becoming a ¥26,300 price. The compliance coverage bar moved to 0.7 after 0.5 produced a false "group without price tag" on a shelf where every facing has one. Measured on add/image6.jpg: 8 readable tags with names and prices, 942 ms for the whole still, 7 real price mismatches, no false violations, and hints that say plainly when a row could not be read. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w
parent
a27841afa4
commit
8f750c15c5
8 changed files with 330 additions and 60 deletions
|
|
@ -392,6 +392,7 @@ def create_app(
|
|||
result["compliance"] = bundle["compliance"]
|
||||
result["ocr_engine"] = ocr_engine.name
|
||||
result["tag_timing_ms"] = round(elapsed_ms, 1)
|
||||
engine.image_store.set_tag_bundle(result["image_id"], bundle)
|
||||
return result
|
||||
|
||||
@app.post("/v1/images/{image_id}/classify", dependencies=[Depends(authorize)])
|
||||
|
|
@ -413,22 +414,44 @@ def create_app(
|
|||
if row is not None:
|
||||
detection["identity"] = row["identity"]
|
||||
engine.image_store.update_detections(image_id, stored.detections)
|
||||
identities = [
|
||||
{
|
||||
|
||||
# Tags were read before the identities existed; reconcile them now.
|
||||
compliance = None
|
||||
if stored.tag_bundle:
|
||||
from reconcile import reconcile_bundle
|
||||
|
||||
catalog_by_sku = {product["sku_id"]: product for product in engine.catalog()}
|
||||
refreshed = reconcile_bundle(stored.tag_bundle, stored.detections, catalog_by_sku)
|
||||
for detection in stored.detections:
|
||||
decision = refreshed["reconciliations"].get(int(detection["id"]))
|
||||
if decision is not None:
|
||||
detection["reconciliation"] = decision
|
||||
compliance = refreshed["compliance"]
|
||||
identities = []
|
||||
for detection in detections:
|
||||
row = {
|
||||
"detection_id": int(detection["id"]),
|
||||
"box": list(detection["box"]),
|
||||
"identity": detection["identity"],
|
||||
}
|
||||
for detection in detections
|
||||
]
|
||||
# Re-reconciled after classification; without it a client would keep
|
||||
# the `tag_only` verdict it got when identities did not exist yet.
|
||||
if detection.get("reconciliation") is not None:
|
||||
row["reconciliation"] = detection["reconciliation"]
|
||||
if detection.get("price_tag") is not None:
|
||||
row["price_tag"] = detection["price_tag"]
|
||||
identities.append(row)
|
||||
summary = engine.summarize(stored.detections, [])
|
||||
return {
|
||||
payload = {
|
||||
"image_id": image_id,
|
||||
"method": method,
|
||||
"identities": identities,
|
||||
"summary": summary,
|
||||
"timing_ms": {"classify": round((time.perf_counter() - started) * 1000.0, 2)},
|
||||
}
|
||||
if compliance is not None:
|
||||
payload["compliance"] = compliance
|
||||
return payload
|
||||
|
||||
@app.post("/v1/images/{image_id}/inspect", dependencies=[Depends(authorize)])
|
||||
async def inspect_image(image_id: str, body: InspectBody) -> Dict[str, Any]:
|
||||
|
|
|
|||
|
|
@ -98,6 +98,9 @@ class StoredImage:
|
|||
detections: List[Dict[str, Any]] = field(default_factory=list)
|
||||
created_at: float = 0.0
|
||||
nbytes: int = 0
|
||||
# The tag pass for this still (tags, bands, box->tag), kept so a later
|
||||
# classify can re-reconcile against the identities it just produced.
|
||||
tag_bundle: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
def identity_state_from_level(level: Optional[str]) -> str:
|
||||
|
|
@ -192,6 +195,17 @@ class ImageStore:
|
|||
if stored is not None:
|
||||
stored.detections = list(detections)
|
||||
|
||||
def set_tag_bundle(self, image_id: str, bundle: Dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
stored = self._items.get(image_id)
|
||||
if stored is not None:
|
||||
stored.tag_bundle = bundle
|
||||
|
||||
def get_tag_bundle(self, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
stored = self._items.get(image_id)
|
||||
return stored.tag_bundle if stored is not None else None
|
||||
|
||||
def _evict_locked(self) -> None:
|
||||
now = time.time()
|
||||
for image_id in [k for k, v in self._items.items() if now - v.created_at > self.ttl_s]:
|
||||
|
|
|
|||
|
|
@ -189,7 +189,9 @@ class ComplianceSettings:
|
|||
# A row may only be judged when enough of its facings carry tag evidence
|
||||
# (readable or unreadable). Below this, the row is "not evaluated": claiming a
|
||||
# tag is missing because our detector found none would be a false violation.
|
||||
min_row_tag_coverage: float = 0.5
|
||||
# 0.7 measured: at 0.5 a row where our own tag detection was patchy produced a
|
||||
# false "group without price tag" on a shelf where every facing has a tag.
|
||||
min_row_tag_coverage: float = 0.7
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -434,12 +436,23 @@ def attach_tags_to_boxes(
|
|||
loop to protect - then attaches tags, reconciles, and evaluates compliance.
|
||||
"""
|
||||
image_shape = image.shape
|
||||
bands = tag_service.plan(boxes, image_shape)
|
||||
for band in bands:
|
||||
tag_service.run_pass(image, band)
|
||||
# Plan again: a still has no later frame to pick up a second band.
|
||||
for band in tag_service.plan(boxes, image_shape):
|
||||
tag_service.run_pass(image, band)
|
||||
# A still has no later frame to pick up the remaining bands, and `plan` only
|
||||
# hands out a few per call (a stream must stay responsive). Drain them, with a
|
||||
# cap so a huge shelf cannot turn one request into minutes of OCR. Draining
|
||||
# also feeds the vote: a band read in two rounds is two observations.
|
||||
bands: List[Any] = []
|
||||
rounds = 0
|
||||
while rounds < 4:
|
||||
planned = tag_service.plan(boxes, image_shape)
|
||||
if not planned:
|
||||
break
|
||||
for band in planned:
|
||||
# Two independent reads (different upscale) so the vote has evidence
|
||||
# to work with; a single read has nothing to vote against.
|
||||
tag_service.run_pass(image, band, scale_bias=1.0)
|
||||
tag_service.run_pass(image, band, scale_bias=1.7)
|
||||
bands.append(band)
|
||||
rounds += 1
|
||||
|
||||
tracks = [
|
||||
{
|
||||
|
|
@ -468,6 +481,61 @@ def attach_tags_to_boxes(
|
|||
"counts": counts,
|
||||
"reconciliations": reconciliations,
|
||||
"bands": [{"row_index": band.row_index, "box": band.box} for band in bands],
|
||||
"bands_objects": list(bands),
|
||||
"compliance": {
|
||||
"violations": summary["violations"],
|
||||
"by_rule": summary["by_rule"],
|
||||
"rows": [
|
||||
{
|
||||
"row_index": finding.row_index,
|
||||
"rule": finding.rule,
|
||||
"box": finding.box,
|
||||
"missing": finding.count,
|
||||
"skus": finding.skus,
|
||||
}
|
||||
for finding in findings
|
||||
if finding.severity == "violation"
|
||||
],
|
||||
"hints": summary["hints"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def reconcile_bundle(
|
||||
bundle: Dict[str, Any],
|
||||
detections: Sequence[Dict[str, Any]],
|
||||
catalog_by_sku: Dict[str, Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Re-run reconciliation and compliance against a cached tag bundle.
|
||||
|
||||
Used when identities arrive after the tag pass (photo mode: recognize paints
|
||||
boxes, classify names them). Without this, every tag would look like `tag_only`
|
||||
because it was compared against an identity that did not exist yet.
|
||||
"""
|
||||
tracks = [
|
||||
{
|
||||
"track_id": int(detection["id"]),
|
||||
"box": list(detection["box"]),
|
||||
"identity": detection.get("identity") or {},
|
||||
}
|
||||
for detection in detections
|
||||
]
|
||||
tag_by_track = bundle.get("by_box") or {}
|
||||
reconciliations: Dict[int, Dict[str, Any]] = {}
|
||||
for track in tracks:
|
||||
track_id = track["track_id"]
|
||||
tag = tag_by_track.get(track_id)
|
||||
product = catalog_by_sku.get((track["identity"] or {}).get("sku_id") or "")
|
||||
decision = reconcile(track["identity"], tag, product)
|
||||
if decision is not None:
|
||||
reconciliations[track_id] = decision
|
||||
|
||||
findings = evaluate_compliance(
|
||||
tracks, tag_by_track, bundle.get("tags") or [], bundle.get("bands_objects") or [], reconciliations
|
||||
)
|
||||
summary = summarize_findings(findings)
|
||||
return {
|
||||
"reconciliations": reconciliations,
|
||||
"compliance": {
|
||||
"violations": summary["violations"],
|
||||
"by_rule": summary["by_rule"],
|
||||
|
|
|
|||
|
|
@ -309,7 +309,9 @@ def parse_price_tag(texts: Sequence[str], lines: Optional[Sequence[OcrLine]] = N
|
|||
value = int(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if not (1 <= value <= 99999):
|
||||
# A Japanese shelf tag is ¥10-¥9,999. Five-digit values are misreads
|
||||
# (a packaging size, a unit price, a barcode fragment).
|
||||
if not (10 <= value <= 9999):
|
||||
continue
|
||||
area = float(_text_area(line)) if line is not None else float(len(match.group(1)))
|
||||
price_candidates.append((value, area))
|
||||
|
|
@ -427,8 +429,13 @@ class TagService:
|
|||
self._pending_bands.add(key)
|
||||
return chosen
|
||||
|
||||
def run_pass(self, image: Any, band: RailBand) -> List[TagObservation]:
|
||||
"""OCR one band and fold the observations into the cache (worker thread)."""
|
||||
def run_pass(self, image: Any, band: RailBand, scale_bias: float = 1.0) -> List[TagObservation]:
|
||||
"""OCR one band and fold the observations into the cache (worker thread).
|
||||
|
||||
`scale_bias` lets a caller read the same band at a different upscale: two
|
||||
independent reads of the same tag are what makes the vote meaningful
|
||||
(identical pixels would produce identical output).
|
||||
"""
|
||||
key = (band.row_index, band.box[0] // 32, band.box[2] // 32)
|
||||
try:
|
||||
x1, y1, x2, y2 = band.box
|
||||
|
|
@ -440,6 +447,8 @@ class TagService:
|
|||
digit_estimate = crop.shape[0]
|
||||
if digit_estimate < 90:
|
||||
scale = min(2.5, 120.0 / max(1, digit_estimate))
|
||||
scale = max(1.0, min(3.0, scale * max(0.5, scale_bias)))
|
||||
if scale > 1.0:
|
||||
import cv2
|
||||
|
||||
crop = cv2.resize(crop, None, fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
|
||||
|
|
@ -525,32 +534,40 @@ class TagService:
|
|||
|
||||
def _vote(self, record: TagRecord) -> None:
|
||||
"""Majority vote over observations: the accuracy lever for digit misreads."""
|
||||
prices: Dict[int, int] = {}
|
||||
regulars: Dict[int, int] = {}
|
||||
names: Dict[str, int] = {}
|
||||
units: Dict[float, int] = {}
|
||||
discounts: Dict[float, int] = {}
|
||||
# Votes are weighted by the read's confidence: two equally common values
|
||||
# are then decided by which read was surer, never by which number is
|
||||
# bigger. A tie with no confident read stays unread.
|
||||
prices: Dict[int, float] = {}
|
||||
regulars: Dict[int, float] = {}
|
||||
names: Dict[str, float] = {}
|
||||
units: Dict[float, float] = {}
|
||||
discounts: Dict[float, float] = {}
|
||||
flags: set = set()
|
||||
confidences: List[float] = []
|
||||
for observation in record.observations:
|
||||
fields = observation.fields
|
||||
weight = max(0.05, float(observation.confidence))
|
||||
if fields.price is not None:
|
||||
prices[fields.price] = prices.get(fields.price, 0) + 1
|
||||
prices[fields.price] = prices.get(fields.price, 0.0) + weight
|
||||
if fields.regular_price is not None:
|
||||
regulars[fields.regular_price] = regulars.get(fields.regular_price, 0) + 1
|
||||
regulars[fields.regular_price] = regulars.get(fields.regular_price, 0.0) + weight
|
||||
if fields.name:
|
||||
names[fields.name] = names.get(fields.name, 0) + 1
|
||||
names[fields.name] = names.get(fields.name, 0.0) + weight
|
||||
if fields.unit_price is not None:
|
||||
units[fields.unit_price] = units.get(fields.unit_price, 0) + 1
|
||||
units[fields.unit_price] = units.get(fields.unit_price, 0.0) + weight
|
||||
if fields.discount is not None:
|
||||
discounts[fields.discount] = discounts.get(fields.discount, 0) + 1
|
||||
discounts[fields.discount] = discounts.get(fields.discount, 0.0) + weight
|
||||
flags.update(fields.flags)
|
||||
confidences.append(observation.confidence)
|
||||
|
||||
def top(counter: Dict[Any, int]) -> Any:
|
||||
def top(counter: Dict[Any, float]) -> Any:
|
||||
if not counter:
|
||||
return None
|
||||
return max(counter.items(), key=lambda item: (item[1], item[0]))[0]
|
||||
ranked = sorted(counter.items(), key=lambda item: item[1], reverse=True)
|
||||
if len(ranked) > 1 and abs(ranked[0][1] - ranked[1][1]) < 1e-9:
|
||||
# A dead heat between two different readings is not evidence.
|
||||
return None
|
||||
return ranked[0][0]
|
||||
|
||||
record.price = top(prices)
|
||||
record.regular_price = top(regulars)
|
||||
|
|
|
|||
|
|
@ -152,15 +152,15 @@ BANDS = [band(1, [0, 300, 700, 340]), band(2, [0, 600, 700, 640])]
|
|||
|
||||
class ComplianceTests(unittest.TestCase):
|
||||
def test_missing_tag_on_a_covered_row_is_a_violation(self):
|
||||
tracks = [
|
||||
track(1, [10, 100, 90, 300]),
|
||||
track(2, [110, 100, 190, 300]),
|
||||
]
|
||||
# 3 of 4 facings carry tag evidence (75% >= the 70% bar), one has none.
|
||||
tracks = [track(index, [10 + index * 150, 100, 140 + index * 150, 300]) for index in range(4)]
|
||||
tags = {
|
||||
1: tag(state=TAG_STATE_READABLE, price=168),
|
||||
0: tag(state=TAG_STATE_READABLE, price=168, id="tag_a"),
|
||||
1: tag(state=TAG_STATE_READABLE, price=178, id="tag_b"),
|
||||
2: tag(state=TAG_STATE_ABSENT),
|
||||
3: tag(state=TAG_STATE_READABLE, price=188, id="tag_c"),
|
||||
}
|
||||
findings = evaluate_compliance(tracks, tags, [tags[1]], BANDS)
|
||||
findings = evaluate_compliance(tracks, tags, [tags[0], tags[1], tags[3]], BANDS)
|
||||
rules = [finding.rule for finding in findings]
|
||||
self.assertIn(RULE_MISSING_TAG, rules)
|
||||
missing = [finding for finding in findings if finding.rule == RULE_MISSING_TAG][0]
|
||||
|
|
@ -201,18 +201,13 @@ class ComplianceTests(unittest.TestCase):
|
|||
self.assertEqual(findings, [])
|
||||
|
||||
def test_group_of_consecutive_facings_without_tags(self):
|
||||
# 5 facings: 2 tagged, 3 not. Coverage counts facings with *any* tag
|
||||
# evidence, so give the row enough evidence to be judged at all.
|
||||
tracks = [track(index, [10 + index * 100, 100, 90 + index * 100, 300]) for index in range(6)]
|
||||
tags = {
|
||||
0: tag(state=TAG_STATE_READABLE, price=168, id="tag_a"),
|
||||
1: tag(state=TAG_STATE_READABLE, price=178, id="tag_b"),
|
||||
2: tag(state=TAG_STATE_ABSENT),
|
||||
3: tag(state=TAG_STATE_ABSENT),
|
||||
4: tag(state=TAG_STATE_ABSENT),
|
||||
5: tag(state=TAG_STATE_READABLE, price=188, id="tag_c"),
|
||||
}
|
||||
findings = evaluate_compliance(tracks, tags, [tags[0], tags[1], tags[5]], BANDS)
|
||||
# Coverage counts facings with *any* tag evidence, so the row needs enough
|
||||
# of it to be judged at all (bar is 70%).
|
||||
tracks = [track(index, [10 + index * 70, 100, 70 + index * 70, 300]) for index in range(10)]
|
||||
tags = {index: tag(state=TAG_STATE_READABLE, price=168 + index, id=f"tag_{index}") for index in range(10)}
|
||||
for absent in (2, 3, 4):
|
||||
tags[absent] = tag(state=TAG_STATE_ABSENT, id=None)
|
||||
findings = evaluate_compliance(tracks, tags, [t for t in tags.values() if t["id"]], BANDS)
|
||||
groups = [finding for finding in findings if finding.rule == RULE_MISSING_TAG_GROUP]
|
||||
self.assertEqual(len(groups), 1)
|
||||
self.assertEqual(groups[0].count, 3)
|
||||
|
|
@ -220,13 +215,14 @@ class ComplianceTests(unittest.TestCase):
|
|||
self.assertIn("3 facings", groups[0].detail)
|
||||
|
||||
def test_two_missing_facings_are_not_a_group(self):
|
||||
tracks = [track(index, [10 + index * 100, 100, 90 + index * 100, 300]) for index in range(3)]
|
||||
tracks = [track(index, [10 + index * 150, 100, 140 + index * 150, 300]) for index in range(4)]
|
||||
tags = {
|
||||
0: tag(state=TAG_STATE_READABLE, price=168, id="tag_a"),
|
||||
1: tag(state=TAG_STATE_ABSENT),
|
||||
2: tag(state=TAG_STATE_READABLE, price=188, id="tag_b"),
|
||||
3: tag(state=TAG_STATE_READABLE, price=198, id="tag_c"),
|
||||
}
|
||||
findings = evaluate_compliance(tracks, tags, [tags[0], tags[2]], BANDS)
|
||||
findings = evaluate_compliance(tracks, tags, [tags[0], tags[2], tags[3]], BANDS)
|
||||
self.assertNotIn(RULE_MISSING_TAG_GROUP, [finding.rule for finding in findings])
|
||||
missing = [finding for finding in findings if finding.rule == RULE_MISSING_TAG]
|
||||
self.assertEqual(len(missing), 1)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,15 @@ const I18N = {
|
|||
connectionLost: 'Connection lost — retrying…', unknownMethod: 'Unknown algorithm',
|
||||
noServer: 'Server unreachable', confirmEmpty: 'Hide empty spaces', showEmpty: 'Show empty spaces',
|
||||
facings: 'facings', inView: 'in view', products: 'products',
|
||||
ocrTitle: 'Price tag', ocrPrice: 'Tag price', ocrRegular: 'Regular price',
|
||||
ocrDiscount: 'Discount', ocrUnit: 'Unit price', ocrName: 'Tag name',
|
||||
ocrState: 'Tag', ocrUnreadable: 'unreadable — move closer', ocrAbsent: 'no tag under this product',
|
||||
ocrNote: 'Note', row: 'Row', more: 'more',
|
||||
recon_confirmed: 'tag matches', recon_mismatch: 'tag differs', recon_unverified: 'tag unread',
|
||||
recon_no_tag: 'no tag', recon_tag_only: 'tag only',
|
||||
rule_missing_price_tag: 'missing price tag', rule_missing_tag_group: 'group without price tag',
|
||||
rule_orphan_tag: 'tag with no product', rule_price_mismatch: 'price differs from catalog',
|
||||
rule_duplicate_tag: 'tag shared by two products',
|
||||
},
|
||||
ja: {
|
||||
subtitle: 'スマートスーパーマーケット棚認識 & 在庫管理AI',
|
||||
|
|
@ -56,6 +65,15 @@ const I18N = {
|
|||
connectionLost: '接続が切れました — 再接続中…', unknownMethod: '不明なアルゴリズム',
|
||||
noServer: 'サーバーに接続できません', confirmEmpty: '空きスペースを隠す', showEmpty: '空きスペースを表示',
|
||||
facings: 'フェイス', inView: '表示中', products: '商品',
|
||||
ocrTitle: '値札', ocrPrice: '値札の価格', ocrRegular: '通常価格',
|
||||
ocrDiscount: '割引', ocrUnit: '単位価格', ocrName: '値札の商品名',
|
||||
ocrState: '値札', ocrUnreadable: '読み取れません — 近づいてください', ocrAbsent: '商品の下に値札がありません',
|
||||
ocrNote: '備考', row: '段', more: '件',
|
||||
recon_confirmed: '一致', recon_mismatch: '不一致', recon_unverified: '未読取',
|
||||
recon_no_tag: '値札なし', recon_tag_only: '値札のみ',
|
||||
rule_missing_price_tag: '値札なし', rule_missing_tag_group: '値札のない商品が連続',
|
||||
rule_orphan_tag: '商品のない値札', rule_price_mismatch: 'カタログ価格と不一致',
|
||||
rule_duplicate_tag: '同じ値札が2商品',
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -151,8 +169,13 @@ function render() {
|
|||
ctx.clearRect(0, 0, overlay.width, overlay.height);
|
||||
const now = performance.now();
|
||||
|
||||
if (state.mode === 'photo' && state.photoDetections.length) {
|
||||
drawPhotoDetections();
|
||||
if (state.mode === 'photo') {
|
||||
// Draw the photo into the canvas so annotations cannot drift from it.
|
||||
const photo = el('photo');
|
||||
if (photo.complete && photo.naturalWidth) {
|
||||
ctx.drawImage(photo, 0, 0, overlay.width, overlay.height);
|
||||
}
|
||||
if (state.photoDetections.length) drawPhotoDetections();
|
||||
}
|
||||
|
||||
for (const [trackId, track] of state.tracks) {
|
||||
|
|
@ -186,6 +209,24 @@ function render() {
|
|||
ctx.globalAlpha = 1;
|
||||
}
|
||||
|
||||
if (state.mode === 'live' || state.mode === 'photo') {
|
||||
const tagSource = state.mode === 'live'
|
||||
? [...state.tracks.values()].map((track) => track.identity && track.priceTag).filter(Boolean)
|
||||
: state.photoDetections.map((d) => d.price_tag).filter(Boolean);
|
||||
for (const tag of tagSource) {
|
||||
if (!tag.box) continue;
|
||||
const readable = tag.state === 'readable';
|
||||
drawBox(tag.box, {
|
||||
color: readable ? '#0ea5e9' : 'rgba(14,165,233,0.5)',
|
||||
width: readable ? 2 : 1,
|
||||
dashed: !readable,
|
||||
});
|
||||
if (readable && tag.price !== null && tag.price !== undefined) {
|
||||
chip(`¥${tag.price}`, tag.box[0], tag.box[1], '#0ea5e9', '#ffffff');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (state.showEmpty && state.mode === 'live') {
|
||||
for (const slot of state.slots) {
|
||||
drawBox(slot.box, { color: '#ef4444', width: 3, fill: 'rgba(239,68,68,0.20)' });
|
||||
|
|
@ -421,6 +462,29 @@ function sendFrame() {
|
|||
}, 'image/jpeg', state.quality);
|
||||
}
|
||||
|
||||
function renderCompliance(compliance) {
|
||||
const strip = el('complianceStrip');
|
||||
if (!compliance) { strip.hidden = true; return; }
|
||||
const rows = compliance.rows || [];
|
||||
const items = rows.slice(0, 3).map(
|
||||
(row) => `<span class="compliance-item">⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})</span>`
|
||||
);
|
||||
if (rows.length > 3) {
|
||||
items.push(`<span class="compliance-item">+${rows.length - 3} ${t('more')}</span>`);
|
||||
}
|
||||
const hints = compliance.hints || [];
|
||||
if (!items.length && hints.length) {
|
||||
strip.hidden = false;
|
||||
strip.dataset.kind = 'hint';
|
||||
strip.innerHTML = hints.map((hint) => `<span class="compliance-item">${hint.detail}</span>`).join('');
|
||||
return;
|
||||
}
|
||||
if (!items.length) { strip.hidden = true; return; }
|
||||
strip.hidden = false;
|
||||
strip.dataset.kind = 'violation';
|
||||
strip.innerHTML = items.join('');
|
||||
}
|
||||
|
||||
function handleResult(message) {
|
||||
state.results += 1;
|
||||
state.inFlight = false;
|
||||
|
|
@ -437,6 +501,8 @@ function handleResult(message) {
|
|||
box: existing ? existing.box : [...track.box],
|
||||
target: track.box,
|
||||
identity: track.identity,
|
||||
priceTag: track.price_tag || null,
|
||||
reconciliation: track.reconciliation || null,
|
||||
product: track.product || productFor(track.identity.sku_id),
|
||||
lastSeen: performance.now(),
|
||||
});
|
||||
|
|
@ -455,6 +521,7 @@ function handleResult(message) {
|
|||
else if (hint === 'too_far') showHint('warning', t('hintTooFar'));
|
||||
else showHint(null, '');
|
||||
}
|
||||
renderCompliance(message.summary && message.summary.compliance);
|
||||
renderInView();
|
||||
if (state.selectedTrack !== null && !state.cardPinned) {
|
||||
const track = state.tracks.get(state.selectedTrack);
|
||||
|
|
@ -494,12 +561,12 @@ async function recognizeBlob(blob, previewUrl) {
|
|||
setStatus('neutral', t('identifying'));
|
||||
const photo = el('photo');
|
||||
photo.src = previewUrl || URL.createObjectURL(blob);
|
||||
photo.hidden = false;
|
||||
photo.hidden = true; // the canvas draws it
|
||||
el('video').hidden = true;
|
||||
el('stageEmpty').hidden = true;
|
||||
await photo.decode().catch(() => {});
|
||||
|
||||
const response = await fetch(api('/v1/recognize?classify=false&empty_slots=true'), {
|
||||
const response = await fetch(api('/v1/recognize?classify=false&empty_slots=true&tags=true'), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'image/jpeg', ...headers() },
|
||||
body: blob,
|
||||
|
|
@ -514,8 +581,13 @@ async function recognizeBlob(blob, previewUrl) {
|
|||
state.photoWidth = data.width;
|
||||
state.photoHeight = data.height;
|
||||
state.slots = data.empty_slots || [];
|
||||
setCanvasSize(photo.naturalWidth || data.width, photo.naturalHeight || data.height);
|
||||
const cap = 2048;
|
||||
const naturalWidth = photo.naturalWidth || data.width;
|
||||
const naturalHeight = photo.naturalHeight || data.height;
|
||||
const scale = Math.min(1, cap / Math.max(1, naturalWidth));
|
||||
setCanvasSize(Math.round(naturalWidth * scale), Math.round(naturalHeight * scale));
|
||||
el('inViewCount').textContent = String(data.detections.length);
|
||||
renderCompliance(data.compliance);
|
||||
setStatus('neutral', t('detected'));
|
||||
classifyPhoto();
|
||||
}
|
||||
|
|
@ -530,11 +602,16 @@ async function classifyPhoto(detectionIds = null) {
|
|||
});
|
||||
if (!response.ok) { setStatus('danger', t('noServer')); return; }
|
||||
const data = await response.json();
|
||||
const byId = new Map(data.identities.map((row) => [row.detection_id, row.identity]));
|
||||
const byId = new Map(data.identities.map((row) => [row.detection_id, row]));
|
||||
for (const detection of state.photoDetections) {
|
||||
if (byId.has(detection.id)) detection.identity = byId.get(detection.id);
|
||||
const row = byId.get(detection.id);
|
||||
if (!row) continue;
|
||||
detection.identity = row.identity;
|
||||
if (row.reconciliation) detection.reconciliation = row.reconciliation;
|
||||
if (row.price_tag) detection.price_tag = row.price_tag;
|
||||
}
|
||||
const summary = data.summary || {};
|
||||
renderCompliance(data.compliance);
|
||||
setStatus('recognized', `${summary.recognized || 0} / ${data.identities.length}`);
|
||||
renderInView();
|
||||
}
|
||||
|
|
@ -601,6 +678,8 @@ function selectBySku(skuId) {
|
|||
|
||||
function showTrackCard(track) {
|
||||
const identity = track.identity || {};
|
||||
state.currentTag = track.priceTag || null;
|
||||
state.currentReconciliation = track.reconciliation || null;
|
||||
if (identity.state === 'recognized') {
|
||||
const product = track.product || productFor(identity.sku_id);
|
||||
showProductCard(product, identity);
|
||||
|
|
@ -613,6 +692,34 @@ function showTrackCard(track) {
|
|||
}
|
||||
}
|
||||
|
||||
function renderOcrSection(tag, reconciliation) {
|
||||
const section = el('ocrSection');
|
||||
const chipEl = el('reconChip');
|
||||
const rows = el('ocrRows');
|
||||
if (!tag && !reconciliation) { section.hidden = true; return; }
|
||||
section.hidden = false;
|
||||
const state = (reconciliation || {}).state;
|
||||
chipEl.dataset.state = state || 'unverified';
|
||||
chipEl.textContent = t(`recon_${state || 'unverified'}`);
|
||||
const lines = [];
|
||||
if (tag) {
|
||||
if (tag.price !== null && tag.price !== undefined) {
|
||||
const bad = state === 'mismatch';
|
||||
lines.push([t('ocrPrice'), `¥${tag.price}`, bad]);
|
||||
}
|
||||
if (tag.regular_price) lines.push([t('ocrRegular'), `¥${tag.regular_price}`, false]);
|
||||
if (tag.discount) lines.push([t('ocrDiscount'), `${Math.round(tag.discount * 100)}%`, false]);
|
||||
if (tag.unit_price) lines.push([t('ocrUnit'), `${tag.unit_price}`, false]);
|
||||
if (tag.name) lines.push([t('ocrName'), tag.name, false]);
|
||||
if (tag.state === 'unreadable') lines.push([t('ocrState'), t('ocrUnreadable'), true]);
|
||||
if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]);
|
||||
}
|
||||
if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']);
|
||||
rows.innerHTML = lines
|
||||
.map(([key, value, bad]) => `<div class="ocr-row"><span class="ocr-key">${key}</span><span class="ocr-val${bad ? ' bad' : ''}">${value}</span></div>`)
|
||||
.join('');
|
||||
}
|
||||
|
||||
function showProductCard(product, identity) {
|
||||
if (!product) { showPlainCard(t('detected'), ''); return; }
|
||||
el('cardPlaceholder').hidden = true;
|
||||
|
|
@ -631,6 +738,7 @@ function showProductCard(product, identity) {
|
|||
el('showMatchBtn').disabled = false;
|
||||
const confidence = identity.confidence ? ` · ${(identity.confidence * 100).toFixed(0)}%` : '';
|
||||
el('scoreLabel').textContent = identity.score ? `score ${identity.score}${confidence}` : '';
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('recognized', t('recognized'));
|
||||
}
|
||||
|
||||
|
|
@ -655,6 +763,7 @@ function showCandidatesCard(identity) {
|
|||
candidates.innerHTML = `<div class="candidates-title">${t('candidatesTitle')}</div>${rows}`;
|
||||
candidates.hidden = false;
|
||||
el('showMatchBtn').disabled = false;
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('needs', t('needsConfirm'));
|
||||
}
|
||||
|
||||
|
|
@ -672,6 +781,7 @@ function showPlainCard(title, note) {
|
|||
el('metricExpiry').textContent = '—';
|
||||
el('candidates').hidden = true;
|
||||
el('showMatchBtn').disabled = true;
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('danger', title);
|
||||
if (note) el('productPrice').textContent = note;
|
||||
}
|
||||
|
|
@ -705,7 +815,10 @@ overlay.addEventListener('click', (event) => {
|
|||
return;
|
||||
}
|
||||
if (state.mode === 'photo') {
|
||||
const [px, py] = photoPointToDetectionSpace(x, y);
|
||||
// The canvas backing store matches the server's pixel space (both scaled
|
||||
// from the same image), so no extra transform is needed.
|
||||
const px = x * (state.photoWidth / Math.max(1, overlay.width));
|
||||
const py = y * (state.photoHeight / Math.max(1, overlay.height));
|
||||
let best = null;
|
||||
for (const detection of state.photoDetections) {
|
||||
const [x1, y1, x2, y2] = detection.box;
|
||||
|
|
@ -735,6 +848,8 @@ function photoPointToDetectionSpace(x, y) {
|
|||
function selectPhotoDetection(detection) {
|
||||
state.selectedDetection = detection.id;
|
||||
state.selectedTrack = null;
|
||||
state.currentTag = detection.price_tag || null;
|
||||
state.currentReconciliation = detection.reconciliation || null;
|
||||
const identity = detection.identity || {};
|
||||
if (identity.state === 'recognized') showProductCard(productFor(identity.sku_id), identity);
|
||||
else if (identity.state === 'needs_confirmation') showCandidatesCard(identity);
|
||||
|
|
@ -813,7 +928,7 @@ function setMode(mode) {
|
|||
el('liveControls').hidden = mode !== 'live';
|
||||
el('photoControls').hidden = mode === 'live';
|
||||
el('video').hidden = mode !== 'live';
|
||||
el('photo').hidden = mode !== 'photo';
|
||||
el('photo').hidden = true;
|
||||
state.tracks.clear();
|
||||
state.slots = [];
|
||||
state.photoDetections = [];
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@
|
|||
<div id="t-stage-empty">Start the live scan, or open Photo mode to inspect a shelf image.</div>
|
||||
</div>
|
||||
<div class="hint-banner" id="hintBanner" hidden></div>
|
||||
<div class="compliance-strip" id="complianceStrip" hidden></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -128,6 +129,13 @@
|
|||
<div class="metric-val small warn" id="metricExpiry"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ocr-section" id="ocrSection" hidden>
|
||||
<div class="ocr-header">
|
||||
<span id="t-ocr-title">Price tag</span>
|
||||
<span class="recon-chip" id="reconChip"></span>
|
||||
</div>
|
||||
<div class="ocr-rows" id="ocrRows"></div>
|
||||
</div>
|
||||
<button class="action-btn" id="dispenseBtn" type="button">🛒 <span id="t-dispense">Dispense 1 unit</span></button>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -130,13 +130,20 @@ main {
|
|||
background: var(--night);
|
||||
border-radius: 14px;
|
||||
overflow: hidden;
|
||||
min-height: 540px;
|
||||
height: 70vh;
|
||||
min-height: 420px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.stage video, .stage img { max-width: 100%; max-height: 74vh; display: block; }
|
||||
.stage canvas { position: absolute; inset: 0; width: 100%; height: 100%; }
|
||||
.stage video, .stage img { max-width: 100%; max-height: 100%; display: block; }
|
||||
/* The canvas is the same contained box as the media it annotates: its backing
|
||||
store is the media's own pixel size, so server coordinates map 1:1 and the
|
||||
element rect (used for click mapping) is exactly the visible picture. */
|
||||
.stage canvas {
|
||||
position: absolute; left: 50%; top: 50%; transform: translate(-50%, -50%);
|
||||
max-width: 100%; max-height: 100%;
|
||||
}
|
||||
|
||||
.stage-empty { position: absolute; inset: 0; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 10px; color: #cbd5e1; font-size: 13px; text-align: center; padding: 24px; }
|
||||
.stage-empty-icon { font-size: 34px; }
|
||||
|
|
@ -220,5 +227,27 @@ main {
|
|||
|
||||
@media (max-width: 1024px) {
|
||||
main { grid-template-columns: 1fr; padding: 16px; }
|
||||
.stage { min-height: 380px; }
|
||||
.stage { height: 56vh; min-height: 320px; }
|
||||
}
|
||||
|
||||
.ocr-section { border-top: 1px solid var(--border-color); padding-top: 12px; display: flex; flex-direction: column; gap: 8px; }
|
||||
.ocr-header { display: flex; justify-content: space-between; align-items: center; font-size: 11px; font-weight: 800; color: #0369a1; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
.recon-chip { font-size: 10px; font-weight: 800; padding: 2px 8px; border-radius: 999px; text-transform: none; letter-spacing: 0; }
|
||||
.recon-chip[data-state="confirmed"] { background: var(--primary-light); color: var(--primary-dark); }
|
||||
.recon-chip[data-state="mismatch"] { background: var(--danger-soft); color: #b91c1c; }
|
||||
.recon-chip[data-state="unverified"], .recon-chip[data-state="no_tag"] { background: var(--slate-soft); color: var(--text-muted); }
|
||||
.recon-chip[data-state="tag_only"] { background: #e0f2fe; color: #0369a1; }
|
||||
.ocr-rows { display: flex; flex-direction: column; gap: 4px; }
|
||||
.ocr-row { display: flex; justify-content: space-between; gap: 10px; font-size: 12px; }
|
||||
.ocr-row .ocr-key { color: var(--text-muted); font-weight: 600; }
|
||||
.ocr-row .ocr-val { color: var(--text-main); font-weight: 700; text-align: right; }
|
||||
.ocr-row .ocr-val.bad { color: #b91c1c; }
|
||||
.compliance-strip {
|
||||
position: absolute; top: 14px; left: 14px; right: 14px;
|
||||
background: rgba(255, 255, 255, 0.94); backdrop-filter: blur(8px);
|
||||
border: 1px solid #fca5a5; border-radius: 12px; padding: 8px 14px;
|
||||
font-size: 12px; font-weight: 700; color: #b91c1c; display: flex; gap: 12px; flex-wrap: wrap;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08); pointer-events: none;
|
||||
}
|
||||
.compliance-strip[data-kind="hint"] { border-color: var(--amber); color: #92400e; }
|
||||
.compliance-strip .compliance-item { display: inline-flex; align-items: center; gap: 6px; }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue