feat: price-tag UI, still-image tag passes, and honest tag evidence
Web client: price-tag boxes in a distinct style (teal, chip with the parsed price), an OCR section in the product card showing the tag's price, name, discount and the reconciliation verdict with both values, and a compliance strip under the stage. The photo stage now draws the image into the canvas and uses the same containment as the media, so boxes cannot drift from the product (they did: the canvas stretched to the stage while the photo was contained). Server fixes the browser runs forced out: - Photo mode reconciled before identities existed, so every tag read "tag only". The tag bundle is cached with the still and classify re-reconciles against the identities it just produced, returning them per detection. - The still path only ever read the first four bands (plan hands out two per call); it now drains every band, twice at different upscales so the vote has independent reads. - Voting is confidence-weighted and a dead heat stays unread; the plausible shelf-price range is 10-9999, which stops a misread from becoming a ¥26,300 price. The compliance coverage bar moved to 0.7 after 0.5 produced a false "group without price tag" on a shelf where every facing has one. Measured on add/image6.jpg: 8 readable tags with names and prices, 942 ms for the whole still, 7 real price mismatches, no false violations, and hints that say plainly when a row could not be read. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w
parent
a27841afa4
commit
8f750c15c5
8 changed files with 330 additions and 60 deletions
|
|
@ -33,6 +33,15 @@ const I18N = {
|
|||
connectionLost: 'Connection lost — retrying…', unknownMethod: 'Unknown algorithm',
|
||||
noServer: 'Server unreachable', confirmEmpty: 'Hide empty spaces', showEmpty: 'Show empty spaces',
|
||||
facings: 'facings', inView: 'in view', products: 'products',
|
||||
ocrTitle: 'Price tag', ocrPrice: 'Tag price', ocrRegular: 'Regular price',
|
||||
ocrDiscount: 'Discount', ocrUnit: 'Unit price', ocrName: 'Tag name',
|
||||
ocrState: 'Tag', ocrUnreadable: 'unreadable — move closer', ocrAbsent: 'no tag under this product',
|
||||
ocrNote: 'Note', row: 'Row', more: 'more',
|
||||
recon_confirmed: 'tag matches', recon_mismatch: 'tag differs', recon_unverified: 'tag unread',
|
||||
recon_no_tag: 'no tag', recon_tag_only: 'tag only',
|
||||
rule_missing_price_tag: 'missing price tag', rule_missing_tag_group: 'group without price tag',
|
||||
rule_orphan_tag: 'tag with no product', rule_price_mismatch: 'price differs from catalog',
|
||||
rule_duplicate_tag: 'tag shared by two products',
|
||||
},
|
||||
ja: {
|
||||
subtitle: 'スマートスーパーマーケット棚認識 & 在庫管理AI',
|
||||
|
|
@ -56,6 +65,15 @@ const I18N = {
|
|||
connectionLost: '接続が切れました — 再接続中…', unknownMethod: '不明なアルゴリズム',
|
||||
noServer: 'サーバーに接続できません', confirmEmpty: '空きスペースを隠す', showEmpty: '空きスペースを表示',
|
||||
facings: 'フェイス', inView: '表示中', products: '商品',
|
||||
ocrTitle: '値札', ocrPrice: '値札の価格', ocrRegular: '通常価格',
|
||||
ocrDiscount: '割引', ocrUnit: '単位価格', ocrName: '値札の商品名',
|
||||
ocrState: '値札', ocrUnreadable: '読み取れません — 近づいてください', ocrAbsent: '商品の下に値札がありません',
|
||||
ocrNote: '備考', row: '段', more: '件',
|
||||
recon_confirmed: '一致', recon_mismatch: '不一致', recon_unverified: '未読取',
|
||||
recon_no_tag: '値札なし', recon_tag_only: '値札のみ',
|
||||
rule_missing_price_tag: '値札なし', rule_missing_tag_group: '値札のない商品が連続',
|
||||
rule_orphan_tag: '商品のない値札', rule_price_mismatch: 'カタログ価格と不一致',
|
||||
rule_duplicate_tag: '同じ値札が2商品',
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -151,8 +169,13 @@ function render() {
|
|||
ctx.clearRect(0, 0, overlay.width, overlay.height);
|
||||
const now = performance.now();
|
||||
|
||||
if (state.mode === 'photo' && state.photoDetections.length) {
|
||||
drawPhotoDetections();
|
||||
if (state.mode === 'photo') {
|
||||
// Draw the photo into the canvas so annotations cannot drift from it.
|
||||
const photo = el('photo');
|
||||
if (photo.complete && photo.naturalWidth) {
|
||||
ctx.drawImage(photo, 0, 0, overlay.width, overlay.height);
|
||||
}
|
||||
if (state.photoDetections.length) drawPhotoDetections();
|
||||
}
|
||||
|
||||
for (const [trackId, track] of state.tracks) {
|
||||
|
|
@ -186,6 +209,24 @@ function render() {
|
|||
ctx.globalAlpha = 1;
|
||||
}
|
||||
|
||||
if (state.mode === 'live' || state.mode === 'photo') {
|
||||
const tagSource = state.mode === 'live'
|
||||
? [...state.tracks.values()].map((track) => track.identity && track.priceTag).filter(Boolean)
|
||||
: state.photoDetections.map((d) => d.price_tag).filter(Boolean);
|
||||
for (const tag of tagSource) {
|
||||
if (!tag.box) continue;
|
||||
const readable = tag.state === 'readable';
|
||||
drawBox(tag.box, {
|
||||
color: readable ? '#0ea5e9' : 'rgba(14,165,233,0.5)',
|
||||
width: readable ? 2 : 1,
|
||||
dashed: !readable,
|
||||
});
|
||||
if (readable && tag.price !== null && tag.price !== undefined) {
|
||||
chip(`¥${tag.price}`, tag.box[0], tag.box[1], '#0ea5e9', '#ffffff');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (state.showEmpty && state.mode === 'live') {
|
||||
for (const slot of state.slots) {
|
||||
drawBox(slot.box, { color: '#ef4444', width: 3, fill: 'rgba(239,68,68,0.20)' });
|
||||
|
|
@ -421,6 +462,29 @@ function sendFrame() {
|
|||
}, 'image/jpeg', state.quality);
|
||||
}
|
||||
|
||||
function renderCompliance(compliance) {
|
||||
const strip = el('complianceStrip');
|
||||
if (!compliance) { strip.hidden = true; return; }
|
||||
const rows = compliance.rows || [];
|
||||
const items = rows.slice(0, 3).map(
|
||||
(row) => `<span class="compliance-item">⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})</span>`
|
||||
);
|
||||
if (rows.length > 3) {
|
||||
items.push(`<span class="compliance-item">+${rows.length - 3} ${t('more')}</span>`);
|
||||
}
|
||||
const hints = compliance.hints || [];
|
||||
if (!items.length && hints.length) {
|
||||
strip.hidden = false;
|
||||
strip.dataset.kind = 'hint';
|
||||
strip.innerHTML = hints.map((hint) => `<span class="compliance-item">${hint.detail}</span>`).join('');
|
||||
return;
|
||||
}
|
||||
if (!items.length) { strip.hidden = true; return; }
|
||||
strip.hidden = false;
|
||||
strip.dataset.kind = 'violation';
|
||||
strip.innerHTML = items.join('');
|
||||
}
|
||||
|
||||
function handleResult(message) {
|
||||
state.results += 1;
|
||||
state.inFlight = false;
|
||||
|
|
@ -437,6 +501,8 @@ function handleResult(message) {
|
|||
box: existing ? existing.box : [...track.box],
|
||||
target: track.box,
|
||||
identity: track.identity,
|
||||
priceTag: track.price_tag || null,
|
||||
reconciliation: track.reconciliation || null,
|
||||
product: track.product || productFor(track.identity.sku_id),
|
||||
lastSeen: performance.now(),
|
||||
});
|
||||
|
|
@ -455,6 +521,7 @@ function handleResult(message) {
|
|||
else if (hint === 'too_far') showHint('warning', t('hintTooFar'));
|
||||
else showHint(null, '');
|
||||
}
|
||||
renderCompliance(message.summary && message.summary.compliance);
|
||||
renderInView();
|
||||
if (state.selectedTrack !== null && !state.cardPinned) {
|
||||
const track = state.tracks.get(state.selectedTrack);
|
||||
|
|
@ -494,12 +561,12 @@ async function recognizeBlob(blob, previewUrl) {
|
|||
setStatus('neutral', t('identifying'));
|
||||
const photo = el('photo');
|
||||
photo.src = previewUrl || URL.createObjectURL(blob);
|
||||
photo.hidden = false;
|
||||
photo.hidden = true; // the canvas draws it
|
||||
el('video').hidden = true;
|
||||
el('stageEmpty').hidden = true;
|
||||
await photo.decode().catch(() => {});
|
||||
|
||||
const response = await fetch(api('/v1/recognize?classify=false&empty_slots=true'), {
|
||||
const response = await fetch(api('/v1/recognize?classify=false&empty_slots=true&tags=true'), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'image/jpeg', ...headers() },
|
||||
body: blob,
|
||||
|
|
@ -514,8 +581,13 @@ async function recognizeBlob(blob, previewUrl) {
|
|||
state.photoWidth = data.width;
|
||||
state.photoHeight = data.height;
|
||||
state.slots = data.empty_slots || [];
|
||||
setCanvasSize(photo.naturalWidth || data.width, photo.naturalHeight || data.height);
|
||||
const cap = 2048;
|
||||
const naturalWidth = photo.naturalWidth || data.width;
|
||||
const naturalHeight = photo.naturalHeight || data.height;
|
||||
const scale = Math.min(1, cap / Math.max(1, naturalWidth));
|
||||
setCanvasSize(Math.round(naturalWidth * scale), Math.round(naturalHeight * scale));
|
||||
el('inViewCount').textContent = String(data.detections.length);
|
||||
renderCompliance(data.compliance);
|
||||
setStatus('neutral', t('detected'));
|
||||
classifyPhoto();
|
||||
}
|
||||
|
|
@ -530,11 +602,16 @@ async function classifyPhoto(detectionIds = null) {
|
|||
});
|
||||
if (!response.ok) { setStatus('danger', t('noServer')); return; }
|
||||
const data = await response.json();
|
||||
const byId = new Map(data.identities.map((row) => [row.detection_id, row.identity]));
|
||||
const byId = new Map(data.identities.map((row) => [row.detection_id, row]));
|
||||
for (const detection of state.photoDetections) {
|
||||
if (byId.has(detection.id)) detection.identity = byId.get(detection.id);
|
||||
const row = byId.get(detection.id);
|
||||
if (!row) continue;
|
||||
detection.identity = row.identity;
|
||||
if (row.reconciliation) detection.reconciliation = row.reconciliation;
|
||||
if (row.price_tag) detection.price_tag = row.price_tag;
|
||||
}
|
||||
const summary = data.summary || {};
|
||||
renderCompliance(data.compliance);
|
||||
setStatus('recognized', `${summary.recognized || 0} / ${data.identities.length}`);
|
||||
renderInView();
|
||||
}
|
||||
|
|
@ -601,6 +678,8 @@ function selectBySku(skuId) {
|
|||
|
||||
function showTrackCard(track) {
|
||||
const identity = track.identity || {};
|
||||
state.currentTag = track.priceTag || null;
|
||||
state.currentReconciliation = track.reconciliation || null;
|
||||
if (identity.state === 'recognized') {
|
||||
const product = track.product || productFor(identity.sku_id);
|
||||
showProductCard(product, identity);
|
||||
|
|
@ -613,6 +692,34 @@ function showTrackCard(track) {
|
|||
}
|
||||
}
|
||||
|
||||
function renderOcrSection(tag, reconciliation) {
|
||||
const section = el('ocrSection');
|
||||
const chipEl = el('reconChip');
|
||||
const rows = el('ocrRows');
|
||||
if (!tag && !reconciliation) { section.hidden = true; return; }
|
||||
section.hidden = false;
|
||||
const state = (reconciliation || {}).state;
|
||||
chipEl.dataset.state = state || 'unverified';
|
||||
chipEl.textContent = t(`recon_${state || 'unverified'}`);
|
||||
const lines = [];
|
||||
if (tag) {
|
||||
if (tag.price !== null && tag.price !== undefined) {
|
||||
const bad = state === 'mismatch';
|
||||
lines.push([t('ocrPrice'), `¥${tag.price}`, bad]);
|
||||
}
|
||||
if (tag.regular_price) lines.push([t('ocrRegular'), `¥${tag.regular_price}`, false]);
|
||||
if (tag.discount) lines.push([t('ocrDiscount'), `${Math.round(tag.discount * 100)}%`, false]);
|
||||
if (tag.unit_price) lines.push([t('ocrUnit'), `${tag.unit_price}`, false]);
|
||||
if (tag.name) lines.push([t('ocrName'), tag.name, false]);
|
||||
if (tag.state === 'unreadable') lines.push([t('ocrState'), t('ocrUnreadable'), true]);
|
||||
if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]);
|
||||
}
|
||||
if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']);
|
||||
rows.innerHTML = lines
|
||||
.map(([key, value, bad]) => `<div class="ocr-row"><span class="ocr-key">${key}</span><span class="ocr-val${bad ? ' bad' : ''}">${value}</span></div>`)
|
||||
.join('');
|
||||
}
|
||||
|
||||
function showProductCard(product, identity) {
|
||||
if (!product) { showPlainCard(t('detected'), ''); return; }
|
||||
el('cardPlaceholder').hidden = true;
|
||||
|
|
@ -631,6 +738,7 @@ function showProductCard(product, identity) {
|
|||
el('showMatchBtn').disabled = false;
|
||||
const confidence = identity.confidence ? ` · ${(identity.confidence * 100).toFixed(0)}%` : '';
|
||||
el('scoreLabel').textContent = identity.score ? `score ${identity.score}${confidence}` : '';
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('recognized', t('recognized'));
|
||||
}
|
||||
|
||||
|
|
@ -655,6 +763,7 @@ function showCandidatesCard(identity) {
|
|||
candidates.innerHTML = `<div class="candidates-title">${t('candidatesTitle')}</div>${rows}`;
|
||||
candidates.hidden = false;
|
||||
el('showMatchBtn').disabled = false;
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('needs', t('needsConfirm'));
|
||||
}
|
||||
|
||||
|
|
@ -672,6 +781,7 @@ function showPlainCard(title, note) {
|
|||
el('metricExpiry').textContent = '—';
|
||||
el('candidates').hidden = true;
|
||||
el('showMatchBtn').disabled = true;
|
||||
renderOcrSection(state.currentTag, state.currentReconciliation);
|
||||
setStatus('danger', title);
|
||||
if (note) el('productPrice').textContent = note;
|
||||
}
|
||||
|
|
@ -705,7 +815,10 @@ overlay.addEventListener('click', (event) => {
|
|||
return;
|
||||
}
|
||||
if (state.mode === 'photo') {
|
||||
const [px, py] = photoPointToDetectionSpace(x, y);
|
||||
// The canvas backing store matches the server's pixel space (both scaled
|
||||
// from the same image), so no extra transform is needed.
|
||||
const px = x * (state.photoWidth / Math.max(1, overlay.width));
|
||||
const py = y * (state.photoHeight / Math.max(1, overlay.height));
|
||||
let best = null;
|
||||
for (const detection of state.photoDetections) {
|
||||
const [x1, y1, x2, y2] = detection.box;
|
||||
|
|
@ -735,6 +848,8 @@ function photoPointToDetectionSpace(x, y) {
|
|||
function selectPhotoDetection(detection) {
|
||||
state.selectedDetection = detection.id;
|
||||
state.selectedTrack = null;
|
||||
state.currentTag = detection.price_tag || null;
|
||||
state.currentReconciliation = detection.reconciliation || null;
|
||||
const identity = detection.identity || {};
|
||||
if (identity.state === 'recognized') showProductCard(productFor(identity.sku_id), identity);
|
||||
else if (identity.state === 'needs_confirmation') showCandidatesCard(identity);
|
||||
|
|
@ -813,7 +928,7 @@ function setMode(mode) {
|
|||
el('liveControls').hidden = mode !== 'live';
|
||||
el('photoControls').hidden = mode === 'live';
|
||||
el('video').hidden = mode !== 'live';
|
||||
el('photo').hidden = mode !== 'photo';
|
||||
el('photo').hidden = true;
|
||||
state.tracks.clear();
|
||||
state.slots = [];
|
||||
state.photoDetections = [];
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@
|
|||
<div id="t-stage-empty">Start the live scan, or open Photo mode to inspect a shelf image.</div>
|
||||
</div>
|
||||
<div class="hint-banner" id="hintBanner" hidden></div>
|
||||
<div class="compliance-strip" id="complianceStrip" hidden></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -128,6 +129,13 @@
|
|||
<div class="metric-val small warn" id="metricExpiry"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ocr-section" id="ocrSection" hidden>
|
||||
<div class="ocr-header">
|
||||
<span id="t-ocr-title">Price tag</span>
|
||||
<span class="recon-chip" id="reconChip"></span>
|
||||
</div>
|
||||
<div class="ocr-rows" id="ocrRows"></div>
|
||||
</div>
|
||||
<button class="action-btn" id="dispenseBtn" type="button">🛒 <span id="t-dispense">Dispense 1 unit</span></button>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -130,13 +130,20 @@ main {
|
|||
background: var(--night);
|
||||
border-radius: 14px;
|
||||
overflow: hidden;
|
||||
min-height: 540px;
|
||||
height: 70vh;
|
||||
min-height: 420px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.stage video, .stage img { max-width: 100%; max-height: 74vh; display: block; }
|
||||
.stage canvas { position: absolute; inset: 0; width: 100%; height: 100%; }
|
||||
.stage video, .stage img { max-width: 100%; max-height: 100%; display: block; }
|
||||
/* The canvas is the same contained box as the media it annotates: its backing
|
||||
store is the media's own pixel size, so server coordinates map 1:1 and the
|
||||
element rect (used for click mapping) is exactly the visible picture. */
|
||||
.stage canvas {
|
||||
position: absolute; left: 50%; top: 50%; transform: translate(-50%, -50%);
|
||||
max-width: 100%; max-height: 100%;
|
||||
}
|
||||
|
||||
.stage-empty { position: absolute; inset: 0; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 10px; color: #cbd5e1; font-size: 13px; text-align: center; padding: 24px; }
|
||||
.stage-empty-icon { font-size: 34px; }
|
||||
|
|
@ -220,5 +227,27 @@ main {
|
|||
|
||||
@media (max-width: 1024px) {
|
||||
main { grid-template-columns: 1fr; padding: 16px; }
|
||||
.stage { min-height: 380px; }
|
||||
.stage { height: 56vh; min-height: 320px; }
|
||||
}
|
||||
|
||||
.ocr-section { border-top: 1px solid var(--border-color); padding-top: 12px; display: flex; flex-direction: column; gap: 8px; }
|
||||
.ocr-header { display: flex; justify-content: space-between; align-items: center; font-size: 11px; font-weight: 800; color: #0369a1; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
.recon-chip { font-size: 10px; font-weight: 800; padding: 2px 8px; border-radius: 999px; text-transform: none; letter-spacing: 0; }
|
||||
.recon-chip[data-state="confirmed"] { background: var(--primary-light); color: var(--primary-dark); }
|
||||
.recon-chip[data-state="mismatch"] { background: var(--danger-soft); color: #b91c1c; }
|
||||
.recon-chip[data-state="unverified"], .recon-chip[data-state="no_tag"] { background: var(--slate-soft); color: var(--text-muted); }
|
||||
.recon-chip[data-state="tag_only"] { background: #e0f2fe; color: #0369a1; }
|
||||
.ocr-rows { display: flex; flex-direction: column; gap: 4px; }
|
||||
.ocr-row { display: flex; justify-content: space-between; gap: 10px; font-size: 12px; }
|
||||
.ocr-row .ocr-key { color: var(--text-muted); font-weight: 600; }
|
||||
.ocr-row .ocr-val { color: var(--text-main); font-weight: 700; text-align: right; }
|
||||
.ocr-row .ocr-val.bad { color: #b91c1c; }
|
||||
.compliance-strip {
|
||||
position: absolute; top: 14px; left: 14px; right: 14px;
|
||||
background: rgba(255, 255, 255, 0.94); backdrop-filter: blur(8px);
|
||||
border: 1px solid #fca5a5; border-radius: 12px; padding: 8px 14px;
|
||||
font-size: 12px; font-weight: 700; color: #b91c1c; display: flex; gap: 12px; flex-wrap: wrap;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08); pointer-events: none;
|
||||
}
|
||||
.compliance-strip[data-kind="hint"] { border-color: var(--amber); color: #92400e; }
|
||||
.compliance-strip .compliance-item { display: inline-flex; align-items: center; gap: 6px; }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue