/* * Dorayaki web client. * * A client of the recognition server, nothing else: live frames go out over * /v1/stream, stills go out over /v1/recognize. Point it at another server with * ?server=https://host:port and pass a token with ?token=... */ const params = new URLSearchParams(location.search); const SERVER = (params.get('server') || '').replace(/\/$/, ''); const TOKEN = params.get('token') || ''; const I18N = { en: { subtitle: 'Smart Supermarket Shelf AI & Inventory Recognition', live: 'Live', photo: 'Photo', start: 'Start', stop: 'Stop', capture: 'Take photo', upload: 'Upload image', empty: 'Empty spaces', stageEmpty: 'Start the live scan, or open Photo mode to inspect a shelf image.', cardTitle: 'Product details', cardPlaceholder: 'Tap a product on the shelf to see its details.', metricOn: 'On shelf', metricOut: 'Sold / dispatched', metricReceipt: 'Received', metricExpiry: 'Expires', dispense: 'Dispense 1 unit', matchTitle: 'Feature matching', matchPlaceholder: 'Tap a product, then “Show matching”.', showMatch: 'Show matching', inviewLive: 'In view', inviewPhoto: 'On this photo', inviewPlaceholder: 'Recognized products appear here.', cameraTitle: 'Take shelf photo', snapshot: 'Capture', offline: 'Offline', connecting: 'Connecting', liveBadge: 'Live', photoBadge: 'Photo', recognized: 'Recognized', needsConfirm: 'Check this', unrecognized: 'Not in catalog', identifying: 'Identifying…', detected: 'Detecting…', emptySlot: 'Empty shelf space', emptySlotNote: 'No product detected here', candidatesTitle: 'Possible matches', hintGlare: 'Glare — tilt the camera', hintBlur: 'Hold steady', hintTooFar: 'Move closer', noCamera: 'Live camera needs https or localhost.', cameraDenied: 'Camera access was denied.', connectionLost: 'Connection lost — retrying…', unknownMethod: 'Unknown algorithm', noServer: 'Server unreachable', confirmEmpty: 'Hide empty spaces', showEmpty: 'Show empty spaces', facings: 'facings', inView: 'in view', products: 'products', ocrTitle: 'Price tag', ocrPrice: 'Tag price', ocrRegular: 'Regular price', ocrDiscount: 'Discount', ocrUnit: 'Unit price', ocrName: 'Tag name', ocrState: 'Tag', ocrUnreadable: 'unreadable — move closer', ocrAbsent: 'no tag under this product', ocrNote: 'Note', row: 'Row', more: 'more', recon_confirmed: 'tag matches', recon_mismatch: 'tag differs', recon_unverified: 'tag unread', recon_no_tag: 'no tag', recon_tag_only: 'tag only', rule_missing_price_tag: 'missing price tag', rule_missing_tag_group: 'group without price tag', rule_orphan_tag: 'tag with no product', rule_price_mismatch: 'price differs from catalog', rule_duplicate_tag: 'tag shared by two products', }, ja: { subtitle: 'スマートスーパーマーケット棚認識 & 在庫管理AI', live: 'ライブ', photo: '写真', start: '開始', stop: '停止', capture: 'カメラ撮影', upload: '画像アップロード', empty: '空きスペース', stageEmpty: 'ライブスキャンを開始するか、写真モードで棚画像を確認してください。', cardTitle: '商品情報', cardPlaceholder: '棚の商品をタップすると詳細が表示されます。', metricOn: '陳列在庫数', metricOut: '販売済 / 出庫数', metricReceipt: '受入日', metricExpiry: '賞味期限', dispense: '1点出庫 / 購入する', matchTitle: '特徴点マッチング', matchPlaceholder: '商品をタップして「照合を表示」を押してください。', showMatch: '照合を表示', inviewLive: '表示中の商品', inviewPhoto: 'この写真の商品', inviewPlaceholder: '認識した商品がここに表示されます。', cameraTitle: '棚の写真を撮影', snapshot: '撮影する', offline: '未接続', connecting: '接続中', liveBadge: 'ライブ', photoBadge: '写真', recognized: '認識済み', needsConfirm: '要確認', unrecognized: '未登録', identifying: '識別中…', detected: '検出中…', emptySlot: '棚の空きスペース', emptySlotNote: 'ここに商品はありません', candidatesTitle: '候補', hintGlare: '反射しています — 角度を変えてください', hintBlur: 'カメラを固定してください', hintTooFar: '近づいてください', noCamera: 'ライブカメラには https または localhost が必要です。', cameraDenied: 'カメラの使用が許可されていません。', connectionLost: '接続が切れました — 再接続中…', unknownMethod: '不明なアルゴリズム', noServer: 'サーバーに接続できません', confirmEmpty: '空きスペースを隠す', showEmpty: '空きスペースを表示', facings: 'フェイス', inView: '表示中', products: '商品', ocrTitle: '値札', ocrPrice: '値札の価格', ocrRegular: '通常価格', ocrDiscount: '割引', ocrUnit: '単位価格', ocrName: '値札の商品名', ocrState: '値札', ocrUnreadable: '読み取れません — 近づいてください', ocrAbsent: '商品の下に値札がありません', ocrNote: '備考', row: '段', more: '件', recon_confirmed: '一致', recon_mismatch: '不一致', recon_unverified: '未読取', recon_no_tag: '値札なし', recon_tag_only: '値札のみ', rule_missing_price_tag: '値札なし', rule_missing_tag_group: '値札のない商品が連続', rule_orphan_tag: '商品のない値札', rule_price_mismatch: 'カタログ価格と不一致', rule_duplicate_tag: '同じ値札が2商品', }, }; const state = { lang: 'en', mode: 'live', method: 'hybrid', showEmpty: true, catalog: new Map(), catalogVersion: '', ws: null, wsState: 'idle', video: null, stream: null, frames: 0, results: 0, rtts: [], detectMs: [], dropped: 0, lastFrameAt: 0, inFlight: false, inflightLimit: 1, sendWidth: 1280, quality: 0.75, tracks: new Map(), // track_id -> {box, target, state, identity, product, lastSeen} slots: [], selectedTrack: null, selectedDetection: null, currentImageId: null, photoDetections: [], photoWidth: 0, photoHeight: 0, reconnectDelay: 1000, sendTimer: null, lastHint: null, }; const el = (id) => document.getElementById(id); const t = (key) => (I18N[state.lang] || I18N.en)[key] || key; function api(path) { return (SERVER || '') + path; } function headers() { return TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {}; } // ---------------------------------------------------------------- rendering const overlay = el('overlay'); const ctx = overlay.getContext('2d'); const videoEl = el('video'); const photoEl = el('photo'); state.video = videoEl; let displayWidth = 0; let displayHeight = 0; function setCanvasSize(width, height) { displayWidth = width; displayHeight = height; overlay.width = width; overlay.height = height; } function chip(text, x, y, background, color) { ctx.font = '700 12px "Plus Jakarta Sans", "Noto Sans JP", sans-serif'; const width = ctx.measureText(text).width + 12; const height = 18; const boxY = Math.max(0, y - height - 3); ctx.fillStyle = background; ctx.beginPath(); ctx.roundRect(x, boxY, width, height, 6); ctx.fill(); ctx.fillStyle = color; ctx.fillText(text, x + 6, boxY + 13); } function drawBox(box, { color, dashed = false, fill = null, width = 2 }) { ctx.save(); ctx.lineWidth = width; ctx.strokeStyle = color; if (dashed) ctx.setLineDash([6, 4]); if (fill) { ctx.fillStyle = fill; ctx.fillRect(box[0], box[1], box[2] - box[0], box[3] - box[1]); } ctx.strokeRect(box[0], box[1], box[2] - box[0], box[3] - box[1]); ctx.restore(); } function render() { ctx.clearRect(0, 0, overlay.width, overlay.height); const now = performance.now(); if (state.mode === 'photo') { // Draw the photo into the canvas so annotations cannot drift from it. const photo = el('photo'); if (photo.complete && photo.naturalWidth) { ctx.drawImage(photo, 0, 0, overlay.width, overlay.height); } if (state.photoDetections.length) drawPhotoDetections(); } for (const [trackId, track] of state.tracks) { const target = track.target; if (!track.box) track.box = [...target]; for (let i = 0; i < 4; i += 1) { track.box[i] += (target[i] - track.box[i]) * 0.35; } const age = now - track.lastSeen; const alpha = Math.max(0, 1 - age / 400); if (alpha <= 0.02) continue; const identity = track.identity || { state: 'unconfirmed' }; const selected = state.selectedTrack === trackId; ctx.globalAlpha = alpha; if (identity.state === 'recognized') { drawBox(track.box, { color: selected ? '#10b981' : '#22c55e', width: selected ? 4 : 2, fill: 'rgba(34,197,94,0.10)' }); const name = track.product ? track.product.product_name : identity.sku_id; const price = track.product ? ` ¥${Math.round(track.product.price)}` : ''; chip(`${name}${price}`, track.box[0], track.box[1], '#22c55e', '#ffffff'); } else if (identity.state === 'needs_confirmation') { drawBox(track.box, { color: '#f59e0b', width: 2, dashed: true, fill: 'rgba(245,158,11,0.08)' }); chip(t('needsConfirm'), track.box[0], track.box[1], '#f59e0b', '#ffffff'); } else if (identity.state === 'unrecognized') { drawBox(track.box, { color: '#94a3b8', width: 2 }); } else if (identity.state === 'unconfirmed') { drawBox(track.box, { color: 'rgba(226,232,240,0.55)', width: 1, dashed: true }); } else { drawBox(track.box, { color: 'rgba(148,163,184,0.8)', width: 1, dashed: true }); } ctx.globalAlpha = 1; } if (state.mode === 'live' || state.mode === 'photo') { const tagSource = state.mode === 'live' ? [...state.tracks.values()].map((track) => track.identity && track.priceTag).filter(Boolean) : state.photoDetections.map((d) => d.price_tag).filter(Boolean); for (const tag of tagSource) { if (!tag.box) continue; const readable = tag.state === 'readable'; drawBox(tag.box, { color: readable ? '#0ea5e9' : 'rgba(14,165,233,0.5)', width: readable ? 2 : 1, dashed: !readable, }); if (readable && tag.price !== null && tag.price !== undefined) { chip(`¥${tag.price}`, tag.box[0], tag.box[1], '#0ea5e9', '#ffffff'); } } } if (state.showEmpty && state.mode === 'live') { for (const slot of state.slots) { drawBox(slot.box, { color: '#ef4444', width: 3, fill: 'rgba(239,68,68,0.20)' }); chip(`EMPTY ~${slot.missing_items_est}`, slot.box[0], slot.box[1], '#ef4444', '#ffffff'); } } requestAnimationFrame(render); } requestAnimationFrame(render); function drawPhotoDetections() { for (const detection of state.photoDetections) { const identity = detection.identity || {}; const selected = state.selectedDetection === detection.id; if (identity.state === 'recognized') { drawBox(detection.box, { color: selected ? '#10b981' : '#22c55e', width: selected ? 4 : 2, fill: 'rgba(34,197,94,0.08)' }); chip(identity.sku_id || '', detection.box[0], detection.box[1], '#22c55e', '#fff'); } else if (identity.state === 'needs_confirmation') { drawBox(detection.box, { color: '#f59e0b', width: 2, dashed: true }); } else if (identity.state === 'unrecognized') { drawBox(detection.box, { color: '#94a3b8', width: 2 }); } else { drawBox(detection.box, { color: '#22c55e', width: 2 }); } } if (state.showEmpty) { for (const slot of state.slots) { drawBox(slot.box, { color: '#ef4444', width: 3, fill: 'rgba(239,68,68,0.20)' }); chip(`EMPTY ~${slot.missing_items_est}`, slot.box[0], slot.box[1], '#ef4444', '#ffffff'); } } } // ---------------------------------------------------------------- i18n / chrome function applyLanguage() { document.querySelectorAll('[id^="t-"]').forEach((node) => { const key = node.id.slice(2).replace(/-([a-z])/g, (m, c) => c.toUpperCase()); const value = t(key); if (value && value !== key) node.textContent = value; }); el('langText').textContent = state.lang === 'en' ? '日本語 / JA' : 'English / EN'; el('inViewCount').textContent = String(state.mode === 'live' ? state.tracks.size : state.photoDetections.length); updateInViewTitle(); updateEmptyButton(); setConnection(state.wsState); } function updateInViewTitle() { el('t-inview-title').textContent = state.mode === 'live' ? t('inviewLive') : t('inviewPhoto'); } function updateEmptyButton() { el('t-empty').textContent = state.showEmpty ? t('confirmEmpty') : t('showEmpty'); } function setConnection(status) { state.wsState = status; const pill = el('connPill'); pill.dataset.state = status; const label = { idle: t('offline'), connecting: t('connecting'), live: t('liveBadge'), photo: t('photoBadge'), error: t('noServer') }[status] || t('offline'); el('connText').textContent = label; } function setStatus(kind, text) { const badge = el('statusBadge'); badge.className = `status-badge badge-${{ recognized: 'success', needs: 'warning', danger: 'danger' }[kind] || 'neutral'}`; badge.textContent = text; } function showHint(kind, text) { const banner = el('hintBanner'); if (!text) { banner.hidden = true; return; } banner.hidden = false; banner.dataset.kind = kind || 'info'; banner.textContent = text; } // ---------------------------------------------------------------- HUD function updateHud() { const hud = el('hud'); if (state.mode !== 'live' || !state.results) { hud.textContent = state.mode === 'live' ? '—' : `${state.photoDetections.length} ${t('products')}`; return; } const rtt = state.rtts.length ? median(state.rtts.slice(-30)) : 0; const detect = state.detectMs.length ? median(state.detectMs.slice(-30)) : 0; hud.dataset.detail = hud.dataset.detail === '1' ? '1' : '0'; if (hud.dataset.detail === '1') { hud.textContent = `${state.results} frames · rtt ${rtt.toFixed(0)}ms · detect ${detect.toFixed(0)}ms · dropped ${state.dropped}`; } else { hud.textContent = `${(state.results / Math.max(1, (performance.now() - state.startedAt) / 1000)).toFixed(1)}/s · ${rtt.toFixed(0)}ms`; } } function median(values) { const sorted = [...values].sort((a, b) => a - b); return sorted[Math.floor(sorted.length / 2)] || 0; } setInterval(updateHud, 500); function clearCard() { el('cardPlaceholder').hidden = false; el('cardBody').hidden = true; el('scoreLabel').textContent = ''; setStatus('neutral', ''); } // ---------------------------------------------------------------- catalog async function loadCatalog() { try { const response = await fetch(api('/v1/catalog'), { headers: headers() }); if (!response.ok) throw new Error(`catalog ${response.status}`); const data = await response.json(); state.catalogVersion = data.catalog_version; state.catalog = new Map(data.products.map((product) => [product.sku_id, product])); renderInView(); } catch (error) { console.warn('catalog load failed', error); } } function productFor(skuId) { return skuId ? state.catalog.get(skuId) : null; } function thumbUrl(skuId) { return productFor(skuId) ? api(`/v1/catalog/${skuId}/image`) : ''; } // ---------------------------------------------------------------- live mode async function listCameras() { const select = el('sourceSelect'); select.innerHTML = ''; if (!navigator.mediaDevices || !window.isSecureContext) { const option = document.createElement('option'); option.value = 'file'; option.textContent = t('noCamera'); select.appendChild(option); } else { try { const devices = await navigator.mediaDevices.enumerateDevices(); for (const device of devices.filter((d) => d.kind === 'videoinput')) { const option = document.createElement('option'); option.value = device.deviceId; option.textContent = device.label || 'Camera'; select.appendChild(option); } } catch (error) { /* permission not granted yet */ } } const fileOption = document.createElement('option'); fileOption.value = 'file'; fileOption.textContent = '🎞 ' + (state.lang === 'en' ? 'Video file…' : '動画ファイル…'); select.appendChild(fileOption); } function connectStream() { if (state.ws && (state.ws.readyState === WebSocket.OPEN || state.ws.readyState === WebSocket.CONNECTING)) return; setConnection('connecting'); const base = SERVER || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}`; const url = `${base.replace(/^http/, 'ws')}/v1/stream${TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ''}`; const socket = new WebSocket(url); socket.binaryType = 'arraybuffer'; state.ws = socket; socket.onopen = () => { state.reconnectDelay = 1000; setConnection('live'); socket.send(JSON.stringify({ type: 'config', method: state.method, empty_slots: state.showEmpty })); startSending(); }; socket.onmessage = (event) => { const message = JSON.parse(event.data); if (message.type === 'result') handleResult(message); else if (message.type === 'inspect_result') handleInspect(message); else if (message.type === 'error' && message.message) console.warn('server error', message); }; socket.onclose = () => { stopSending(); if (state.mode === 'live' && state.stream) { setConnection('error'); setTimeout(connectStream, state.reconnectDelay); state.reconnectDelay = Math.min(10000, state.reconnectDelay * 2); } else { setConnection('idle'); } }; socket.onerror = () => setConnection('error'); } function startSending() { stopSending(); state.startedAt = performance.now(); state.sendTimer = setInterval(sendFrame, 1000 / 30); } function stopSending() { if (state.sendTimer) clearInterval(state.sendTimer); state.sendTimer = null; state.inFlight = false; } function sendFrame() { const video = state.video; if (!video || video.readyState < 2 || document.hidden) return; if (state.inFlight && state.inflightLimit <= 1) return; if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return; const sourceWidth = video.videoWidth; const sourceHeight = video.videoHeight; if (!sourceWidth || !sourceHeight) return; const width = Math.min(state.sendWidth, sourceWidth); const height = Math.round(sourceHeight * (width / sourceWidth)); const canvas = state.captureCanvas || (state.captureCanvas = document.createElement('canvas')); canvas.width = width; canvas.height = height; const captureCtx = canvas.getContext('2d'); captureCtx.drawImage(video, 0, 0, width, height); canvas.toBlob((blob) => { if (!blob) return; blob.arrayBuffer().then((buffer) => { const frameId = ++state.frames & 0xffffffff; const payload = new Uint8Array(4 + buffer.byteLength); new DataView(payload.buffer).setUint32(0, frameId, false); payload.set(new Uint8Array(buffer), 4); state.ws.send(payload); state.frameSentAt = new Map([[frameId, performance.now()]]); state.inFlight = true; }); }, 'image/jpeg', state.quality); } function renderCompliance(compliance) { const strip = el('complianceStrip'); if (!compliance) { strip.hidden = true; return; } const rows = compliance.rows || []; const items = rows.slice(0, 3).map( (row) => `⚠ ${t('rule_' + row.rule)} · ${t('row')} ${row.row_index} (${row.missing})` ); if (rows.length > 3) { items.push(`+${rows.length - 3} ${t('more')}`); } const hints = compliance.hints || []; if (!items.length && hints.length) { strip.hidden = false; strip.dataset.kind = 'hint'; strip.innerHTML = hints.map((hint) => `${hint.detail}`).join(''); return; } if (!items.length) { strip.hidden = true; return; } strip.hidden = false; strip.dataset.kind = 'violation'; strip.innerHTML = items.join(''); } function handleResult(message) { state.results += 1; state.inFlight = false; const sentAt = state.frameSentAt && state.frameSentAt.get(message.frame_id); if (sentAt) state.rtts.push(performance.now() - sentAt); state.detectMs.push(message.timing_ms.detect); state.dropped = message.stats.frames_dropped; const seen = new Set(); for (const track of message.tracks) { seen.add(track.track_id); const existing = state.tracks.get(track.track_id); state.tracks.set(track.track_id, { box: existing ? existing.box : [...track.box], target: track.box, identity: track.identity, priceTag: track.price_tag || null, reconciliation: track.reconciliation || null, product: track.product || productFor(track.identity.sku_id), lastSeen: performance.now(), }); } for (const [trackId, track] of state.tracks) { if (!seen.has(trackId) && performance.now() - track.lastSeen > 400) state.tracks.delete(trackId); } state.slots = message.empty_slots || []; el('inViewCount').textContent = String(message.summary.in_view); const hint = message.quality && message.quality.hint; if (hint !== state.lastHint) { state.lastHint = hint; if (hint === 'glare') showHint('warning', t('hintGlare')); else if (hint === 'blur') showHint('warning', t('hintBlur')); else if (hint === 'too_far') showHint('warning', t('hintTooFar')); else showHint(null, ''); } renderCompliance(message.summary && message.summary.compliance); renderInView(); if (state.selectedTrack !== null && !state.cardPinned) { const track = state.tracks.get(state.selectedTrack); if (track) showTrackCard(track); } } // ---------------------------------------------------------------- photo mode async function loadSamples() { try { const response = await fetch(api('/v1/samples'), { headers: headers() }); const data = await response.json(); const select = el('sampleSelect'); select.innerHTML = ''; for (const sample of data.samples) { const option = document.createElement('option'); option.value = sample.id; option.textContent = sample.label; select.appendChild(option); } if (data.samples.length) loadSample(data.samples[0].id); } catch (error) { console.warn('samples failed', error); } } async function loadSample(sampleId) { const response = await fetch(api(`/v1/samples/${sampleId}`), { headers: headers() }); const blob = await response.blob(); const url = URL.createObjectURL(blob); await recognizeBlob(blob, url); } async function recognizeBlob(blob, previewUrl) { setConnection('photo'); setStatus('neutral', t('identifying')); const photo = el('photo'); photo.src = previewUrl || URL.createObjectURL(blob); photo.hidden = true; // the canvas draws it el('video').hidden = true; el('stageEmpty').hidden = true; await photo.decode().catch(() => {}); const response = await fetch(api('/v1/recognize?classify=false&empty_slots=true&tags=true'), { method: 'POST', headers: { 'Content-Type': 'image/jpeg', ...headers() }, body: blob, }); if (!response.ok) { setStatus('danger', t('noServer')); return; } const data = await response.json(); state.currentImageId = data.image_id; state.photoDetections = data.detections; state.photoWidth = data.width; state.photoHeight = data.height; state.slots = data.empty_slots || []; const cap = 2048; const naturalWidth = photo.naturalWidth || data.width; const naturalHeight = photo.naturalHeight || data.height; const scale = Math.min(1, cap / Math.max(1, naturalWidth)); setCanvasSize(Math.round(naturalWidth * scale), Math.round(naturalHeight * scale)); el('inViewCount').textContent = String(data.detections.length); renderCompliance(data.compliance); setStatus('neutral', t('detected')); classifyPhoto(); } async function classifyPhoto(detectionIds = null) { if (!state.currentImageId) return; setStatus('neutral', t('identifying')); const response = await fetch(api(`/v1/images/${state.currentImageId}/classify`), { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers() }, body: JSON.stringify({ method: state.method, detection_ids: detectionIds }), }); if (!response.ok) { setStatus('danger', t('noServer')); return; } const data = await response.json(); const byId = new Map(data.identities.map((row) => [row.detection_id, row])); for (const detection of state.photoDetections) { const row = byId.get(detection.id); if (!row) continue; detection.identity = row.identity; if (row.reconciliation) detection.reconciliation = row.reconciliation; if (row.price_tag) detection.price_tag = row.price_tag; } const summary = data.summary || {}; renderCompliance(data.compliance); setStatus('recognized', `${summary.recognized || 0} / ${data.identities.length}`); renderInView(); } function renderInView() { const list = el('inViewList'); const counts = new Map(); if (state.mode === 'live') { for (const track of state.tracks.values()) { const identity = track.identity || {}; if (identity.state === 'recognized' && identity.sku_id) { counts.set(identity.sku_id, (counts.get(identity.sku_id) || 0) + 1); } } } else { for (const detection of state.photoDetections) { const identity = detection.identity || {}; if (identity.state === 'recognized' && identity.sku_id) { counts.set(identity.sku_id, (counts.get(identity.sku_id) || 0) + 1); } } } if (!counts.size) { list.innerHTML = `
${t('inviewPlaceholder')}
`; return; } const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([skuId, facings]) => { const product = productFor(skuId); const name = product ? product.product_name : skuId; const price = product ? `¥${Math.round(product.price)}` : ''; return `
${name}
${price} · ${t('facings')} ${facings}
${facings}
`; }).join(''); list.innerHTML = rows; list.querySelectorAll('.inview-row').forEach((row) => { row.addEventListener('click', () => selectBySku(row.dataset.sku)); }); } function selectBySku(skuId) { if (state.mode === 'photo') { const detection = state.photoDetections.find((d) => (d.identity || {}).sku_id === skuId); if (detection) selectPhotoDetection(detection); return; } for (const [trackId, track] of state.tracks) { const identity = track.identity || {}; if (identity.sku_id === skuId) { state.selectedTrack = trackId; // Show the product this row names, even if the track has since moved // to a neighbour: the row is the user's intent. Pinning stops the // per-frame refresh from replacing the card the user just opened. state.cardPinned = true; showProductCard(productFor(skuId), identity); break; } } } // ---------------------------------------------------------------- product card function showTrackCard(track) { const identity = track.identity || {}; state.currentTag = track.priceTag || null; state.currentReconciliation = track.reconciliation || null; if (identity.state === 'recognized') { const product = track.product || productFor(identity.sku_id); showProductCard(product, identity); } else if (identity.state === 'needs_confirmation') { showCandidatesCard(identity); } else if (identity.state === 'unrecognized') { showPlainCard(t('unrecognized'), t('emptySlotNote')); } else { showPlainCard(t('detected'), ''); } } function renderOcrSection(tag, reconciliation) { const section = el('ocrSection'); const chipEl = el('reconChip'); const rows = el('ocrRows'); if (!tag && !reconciliation) { section.hidden = true; return; } section.hidden = false; const state = (reconciliation || {}).state; chipEl.dataset.state = state || 'unverified'; chipEl.textContent = t(`recon_${state || 'unverified'}`); const lines = []; if (tag) { if (tag.price !== null && tag.price !== undefined) { const bad = state === 'mismatch'; lines.push([t('ocrPrice'), `¥${tag.price}`, bad]); } if (tag.regular_price) lines.push([t('ocrRegular'), `¥${tag.regular_price}`, false]); if (tag.discount) lines.push([t('ocrDiscount'), `${Math.round(tag.discount * 100)}%`, false]); if (tag.unit_price) lines.push([t('ocrUnit'), `${tag.unit_price}`, false]); if (tag.name) lines.push([t('ocrName'), tag.name, false]); if (tag.state === 'unreadable') lines.push([t('ocrState'), t('ocrUnreadable'), true]); if (tag.state === 'absent') lines.push([t('ocrState'), t('ocrAbsent'), false]); } if (reconciliation && reconciliation.message) lines.push([t('ocrNote'), reconciliation.message, state === 'mismatch']); rows.innerHTML = lines .map(([key, value, bad]) => `
${key}${value}
`) .join(''); } function showProductCard(product, identity) { if (!product) { showPlainCard(t('detected'), ''); return; } el('cardPlaceholder').hidden = true; el('cardBody').hidden = false; el('productSku').hidden = false; el('productSku').textContent = product.sku_id; el('productName').textContent = product.product_name; el('productPrice').textContent = `¥${Math.round(product.price)}`; el('productThumb').src = thumbUrl(product.sku_id); el('productThumb').hidden = false; el('metricOn').textContent = product.number_on; el('metricOut').textContent = product.number_out; el('metricReceipt').textContent = product.receipt_date || '—'; el('metricExpiry').textContent = product.expiration_date || '—'; el('candidates').hidden = true; el('showMatchBtn').disabled = false; const confidence = identity.confidence ? ` · ${(identity.confidence * 100).toFixed(0)}%` : ''; el('scoreLabel').textContent = identity.score ? `score ${identity.score}${confidence}` : ''; renderOcrSection(state.currentTag, state.currentReconciliation); setStatus('recognized', t('recognized')); } function showCandidatesCard(identity) { el('cardPlaceholder').hidden = true; el('cardBody').hidden = false; el('productSku').hidden = true; el('productThumb').hidden = true; el('productName').textContent = t('needsConfirm'); el('productPrice').textContent = '—'; el('productThumb').src = ''; el('metricOn').textContent = '—'; el('metricOut').textContent = '—'; el('metricReceipt').textContent = '—'; el('metricExpiry').textContent = '—'; const candidates = el('candidates'); const rows = (identity.candidates || []).map((candidate) => { const product = productFor(candidate.sku_id); const name = product ? product.product_name : candidate.sku_id; return `
${name}${candidate.score}
`; }).join(''); candidates.innerHTML = `
${t('candidatesTitle')}
${rows}`; candidates.hidden = false; el('showMatchBtn').disabled = false; renderOcrSection(state.currentTag, state.currentReconciliation); setStatus('needs', t('needsConfirm')); } function showPlainCard(title, note) { el('cardPlaceholder').hidden = true; el('cardBody').hidden = false; el('productSku').hidden = true; el('productThumb').hidden = true; el('productName').textContent = title; el('productPrice').textContent = ''; el('productThumb').src = ''; el('metricOn').textContent = '—'; el('metricOut').textContent = '—'; el('metricReceipt').textContent = '—'; el('metricExpiry').textContent = '—'; el('candidates').hidden = true; el('showMatchBtn').disabled = true; renderOcrSection(state.currentTag, state.currentReconciliation); setStatus('danger', title); if (note) el('productPrice').textContent = note; } // ---------------------------------------------------------------- interaction function stagePointToImage(event) { const rect = overlay.getBoundingClientRect(); const scaleX = overlay.width / rect.width; const scaleY = overlay.height / rect.height; return [(event.clientX - rect.left) * scaleX, (event.clientY - rect.top) * scaleY]; } overlay.addEventListener('click', (event) => { const [x, y] = stagePointToImage(event); if (state.mode === 'live') { let best = null; for (const [trackId, track] of state.tracks) { const [x1, y1, x2, y2] = track.target; if (x >= x1 && x <= x2 && y >= y1 && y <= y2) { const area = (x2 - x1) * (y2 - y1); if (!best || area < best.area) best = { trackId, track, area }; } } if (best) { state.selectedTrack = best.trackId; state.selectedDetection = null; state.cardPinned = false; showTrackCard(best.track); } return; } if (state.mode === 'photo') { // The canvas backing store matches the server's pixel space (both scaled // from the same image), so no extra transform is needed. const px = x * (state.photoWidth / Math.max(1, overlay.width)); const py = y * (state.photoHeight / Math.max(1, overlay.height)); let best = null; for (const detection of state.photoDetections) { const [x1, y1, x2, y2] = detection.box; if (px >= x1 && px <= x2 && py >= y1 && py <= y2) { const area = (x2 - x1) * (y2 - y1); if (!best || area < best.area) best = { detection, area }; } } if (best) selectPhotoDetection(best.detection); else { const slot = state.slots.find((s) => { const [x1, y1, x2, y2] = s.box; return px >= x1 && px <= x2 && py >= y1 && py <= y2; }); if (slot) showEmptySlotCard(slot); } } }); function photoPointToDetectionSpace(x, y) { const photo = el('photo'); const scaleX = state.photoWidth / (photo.naturalWidth || state.photoWidth); const scaleY = state.photoHeight / (photo.naturalHeight || state.photoHeight); return [x * scaleX, y * scaleY]; } function selectPhotoDetection(detection) { state.selectedDetection = detection.id; state.selectedTrack = null; state.currentTag = detection.price_tag || null; state.currentReconciliation = detection.reconciliation || null; const identity = detection.identity || {}; if (identity.state === 'recognized') showProductCard(productFor(identity.sku_id), identity); else if (identity.state === 'needs_confirmation') showCandidatesCard(identity); else showPlainCard(t(identity.state === 'unrecognized' ? 'unrecognized' : 'detected'), ''); inspectCurrent(detection.box); } function showEmptySlotCard(slot) { el('cardPlaceholder').hidden = true; el('cardBody').hidden = false; el('productSku').hidden = true; el('productThumb').hidden = true; el('productName').textContent = t('emptySlot'); el('productPrice').textContent = t('emptySlotNote'); el('productThumb').src = ''; el('metricOn').textContent = '0'; el('metricOut').textContent = `~${slot.missing_items_est}`; el('metricReceipt').textContent = `row ${slot.row_index}`; el('metricExpiry').textContent = `${slot.width}×${slot.height}px`; el('candidates').hidden = true; el('showMatchBtn').disabled = true; setStatus('danger', t('emptySlot')); } async function inspectCurrent(box) { if (!state.currentImageId || !box) return; const response = await fetch(api(`/v1/images/${state.currentImageId}/inspect`), { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers() }, body: JSON.stringify({ box, method: state.method }), }); if (!response.ok) return; handleInspect(await response.json()); } function handleInspect(result) { if (result.crop_jpeg_b64) { el('productThumb').src = result.crop_jpeg_b64; } if (result.match_visualization_jpeg_b64) { el('matchBox').innerHTML = `feature matches`; } else { el('matchBox').innerHTML = `${t('matchPlaceholder')}`; } const identity = result.identity || {}; const parts = []; if (identity.score) parts.push(`score ${identity.score}`); if (identity.confidence) parts.push(`confidence ${(identity.confidence * 100).toFixed(0)}%`); if (result.visualized_sku_id) parts.push(`${result.is_candidate ? 'candidate' : 'match'} ${result.visualized_sku_id}`); el('scoreLabel').textContent = parts.join(' · '); } el('showMatchBtn').addEventListener('click', () => { if (state.mode === 'live' && state.selectedTrack !== null) { const track = state.tracks.get(state.selectedTrack); if (track && state.ws && state.ws.readyState === WebSocket.OPEN) { state.ws.send(JSON.stringify({ type: 'inspect', track_id: state.selectedTrack, method: state.method, request_id: String(Date.now()) })); } } else if (state.mode === 'photo' && state.selectedDetection !== null) { const detection = state.photoDetections.find((d) => d.id === state.selectedDetection); if (detection) inspectCurrent(detection.box); } }); // ---------------------------------------------------------------- controls el('modeControl').addEventListener('click', (event) => { const button = event.target.closest('.seg-btn'); if (!button) return; setMode(button.dataset.mode); }); function setMode(mode) { state.mode = mode; document.querySelectorAll('#modeControl .seg-btn').forEach((b) => b.classList.toggle('active', b.dataset.mode === mode)); el('liveControls').hidden = mode !== 'live'; el('photoControls').hidden = mode === 'live'; el('video').hidden = mode !== 'live'; el('photo').hidden = true; state.tracks.clear(); state.slots = []; state.photoDetections = []; state.selectedTrack = null; state.selectedDetection = null; state.cardPinned = false; el('stageEmpty').hidden = false; clearCard(); updateInViewTitle(); if (mode === 'photo') { setConnection('photo'); loadSamples(); } else { stopCamera(); setConnection(state.ws && state.ws.readyState === WebSocket.OPEN ? 'live' : 'idle'); } el('inViewCount').textContent = '0'; renderInView(); } el('methodControl').addEventListener('click', async (event) => { const button = event.target.closest('.seg-btn'); if (!button) return; state.method = button.dataset.method; document.querySelectorAll('#methodControl .seg-btn').forEach((b) => b.classList.toggle('active', b === button)); if (state.mode === 'live' && state.ws && state.ws.readyState === WebSocket.OPEN) { state.ws.send(JSON.stringify({ type: 'config', method: state.method })); state.tracks.clear(); } else if (state.mode === 'photo') { await classifyPhoto(); } }); el('startBtn').addEventListener('click', async () => { const source = el('sourceSelect').value; if (source === 'file') { pickVideoFile(); return; } try { state.stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: source ? { exact: source } : undefined, facingMode: { ideal: 'environment' }, width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); } catch (error) { showHint('danger', error.name === 'NotAllowedError' ? t('cameraDenied') : t('noCamera')); return; } el('video').srcObject = state.stream; videoEl.hidden = false; photoEl.hidden = true; el('stageEmpty').hidden = true; await videoEl.play().catch(() => {}); setCanvasSize(videoEl.videoWidth || 1280, videoEl.videoHeight || 720); el('startBtn').disabled = true; el('stopBtn').disabled = false; connectStream(); listCameras(); }); el('stopBtn').addEventListener('click', () => { stopCamera(); if (state.ws) state.ws.close(); setConnection('idle'); el('stageEmpty').hidden = false; }); function stopCamera() { stopSending(); if (state.stream) { state.stream.getTracks().forEach((track) => track.stop()); state.stream = null; } el('video').srcObject = null; el('startBtn').disabled = false; el('stopBtn').disabled = true; } function pickVideoFile() { const input = document.createElement('input'); input.type = 'file'; input.accept = 'video/*'; input.onchange = async () => { const file = input.files[0]; if (!file) return; const url = URL.createObjectURL(file); const video = el('video'); video.srcObject = null; video.src = url; video.loop = true; video.muted = true; video.hidden = false; el('photo').hidden = true; el('stageEmpty').hidden = true; await video.play().catch(() => {}); setCanvasSize(video.videoWidth || 1280, video.videoHeight || 720); el('startBtn').disabled = true; el('stopBtn').disabled = false; connectStream(); }; input.click(); } el('emptyBtn').addEventListener('click', () => { state.showEmpty = !state.showEmpty; updateEmptyButton(); if (state.mode === 'live' && state.ws && state.ws.readyState === WebSocket.OPEN) { state.ws.send(JSON.stringify({ type: 'config', empty_slots: state.showEmpty })); } }); el('sampleSelect').addEventListener('change', (event) => loadSample(event.target.value)); el('uploadBtn').addEventListener('click', () => el('fileInput').click()); el('fileInput').addEventListener('change', (event) => { const file = event.target.files[0]; if (file) recognizeBlob(file); }); el('captureBtn').addEventListener('click', async () => { el('cameraModal').classList.add('open'); try { state.cameraStream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: { ideal: 'environment' }, width: { ideal: 1280 } }, audio: false }); el('cameraVideo').srcObject = state.cameraStream; } catch (error) { showHint('danger', t('cameraDenied')); el('cameraModal').classList.remove('open'); } }); el('cameraClose').addEventListener('click', closeCameraModal); el('snapshotBtn').addEventListener('click', () => { const video = el('cameraVideo'); const canvas = document.createElement('canvas'); canvas.width = video.videoWidth || 1280; canvas.height = video.videoHeight || 720; canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height); canvas.toBlob((blob) => { closeCameraModal(); if (blob) recognizeBlob(blob); }, 'image/jpeg', 0.9); }); function closeCameraModal() { el('cameraModal').classList.remove('open'); if (state.cameraStream) { state.cameraStream.getTracks().forEach((track) => track.stop()); state.cameraStream = null; } } el('dispenseBtn').addEventListener('click', async () => { const sku = el('productSku').textContent; if (!sku || sku === '—') return; const response = await fetch(api(`/v1/catalog/${sku}/stock`), { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers() }, body: JSON.stringify({ delta: -1 }), }); if (!response.ok) return; const data = await response.json(); el('metricOn').textContent = data.product.number_on; el('metricOut').textContent = data.product.number_out; state.catalog.set(data.product.sku_id, data.product); renderInView(); }); el('langBtn').addEventListener('click', () => { state.lang = state.lang === 'en' ? 'ja' : 'en'; applyLanguage(); listCameras(); }); el('hud').addEventListener('click', () => { el('hud').dataset.detail = el('hud').dataset.detail === '1' ? '0' : '1'; updateHud(); }); document.addEventListener('visibilitychange', () => { if (document.hidden) stopSending(); else if (state.mode === 'live' && state.stream) startSending(); }); // ---------------------------------------------------------------- boot applyLanguage(); loadCatalog(); setMode('live'); listCameras();