product-recognition-demo/recognization/code/app.py
2026-09-18 09:13:53 +09:00

1246 lines
48 KiB
Python

"""
Interactive Web Application: Dorayaki (どら焼き)
Supermarket Shelf Product Recognition & Inventory System
Features:
- Light, fresh pastel green visual design
- Brand Logo: Dorayaki
- Live camera photo capture + File upload + Select from D:\\recognization\\add
- Clean bounding box detection (NO class labels, NO confidence text)
- Dual language support (English / Japanese)
- Fine-grained classification: SURF, ORB, BRISK, Hybrid (Paper method)
"""
import os
import sys
import io
import cv2
import json
import base64
import argparse
import numpy as np
from typing import Optional, List, Dict
current_dir = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.dirname(current_dir)
if os.path.exists(os.path.join(project_root, "stock")):
os.chdir(project_root)
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from pipeline import ShelfRecognitionPipeline
def encode_image_base64(img_bgr: np.ndarray, quality: int = 85) -> str:
"""Encodes an OpenCV image to a base64 JPEG data URL."""
if img_bgr is None or img_bgr.size == 0:
return ""
success, buffer = cv2.imencode(".jpg", img_bgr, [cv2.IMWRITE_JPEG_QUALITY, quality])
if not success:
return ""
return f"data:image/jpeg;base64,{base64.b64encode(buffer).decode('utf-8')}"
def decode_image_base64(data_url: str) -> Optional[np.ndarray]:
"""Decodes a base64 data URL into an OpenCV BGR image."""
try:
if "," in data_url:
data_url = data_url.split(",")[1]
img_bytes = base64.b64decode(data_url)
np_arr = np.frombuffer(img_bytes, np.uint8)
return cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
except Exception as e:
print(f"[Error] Failed to decode base64 image: {e}")
return None
def get_available_shelf_images(add_dir: str = "add") -> List[Dict[str, str]]:
"""Lists images from D:\\recognization\\add and root image.jpg."""
items = []
if os.path.exists("image.jpg"):
items.append({"path": "image.jpg", "label": "image.jpg (Main Shelf)"})
if os.path.exists(add_dir):
valid_exts = (".webp", ".jpg", ".jpeg", ".png")
files = sorted(
[f for f in os.listdir(add_dir) if f.lower().endswith(valid_exts)],
key=lambda x: (len(x), x)
)
for f in files:
p = os.path.join(add_dir, f).replace("\\", "/")
items.append({"path": p, "label": f"{add_dir}/{f}"})
# Also include runs samples if present
mosaic = "runs/detect/train/train_batch0.jpg"
if os.path.exists(mosaic):
items.append({"path": mosaic, "label": "train_batch0.jpg (Mosaic)"})
return items
def create_fastapi_app(pipeline: ShelfRecognitionPipeline):
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.middleware.cors import CORSMiddleware
app = FastAPI(title="Dorayaki - Supermarket Shelf Intelligence")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
HTML_PAGE = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dorayaki - Smart Supermarket Shelf AI</title>
<link href="https://fonts.googleapis.com/css2?family=Plus+Jakarta+Sans:wght@400;500;600;700;800&family=Noto+Sans+JP:wght@400;500;700&display=swap" rel="stylesheet">
<style>
:root {
--bg-body: #f4fbf6;
--bg-card: #ffffff;
--bg-glass: rgba(255, 255, 255, 0.9);
--primary-green: #22c55e;
--primary-dark: #15803d;
--primary-light: #dcfce7;
--primary-subtle: #f0fdf4;
--accent-green: #86efac;
--border-color: #d1fae5;
--border-hover: #86efac;
--text-main: #0f172a;
--text-muted: #64748b;
--shadow-soft: 0 4px 20px -2px rgba(34, 197, 94, 0.08), 0 2px 6px -1px rgba(0, 0, 0, 0.04);
--shadow-hover: 0 10px 25px -3px rgba(34, 197, 94, 0.16);
}
* {
box-sizing: border-box;
margin: 0;
padding: 0;
font-family: 'Plus Jakarta Sans', 'Noto Sans JP', sans-serif;
}
body {
background-color: var(--bg-body);
color: var(--text-main);
min-height: 100vh;
display: flex;
flex-direction: column;
}
header {
background: var(--bg-card);
border-bottom: 1px solid var(--border-color);
padding: 14px 32px;
display: flex;
justify-content: space-between;
align-items: center;
position: sticky;
top: 0;
z-index: 100;
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.02);
}
.brand {
display: flex;
align-items: center;
gap: 14px;
}
.brand-logo-svg {
width: 44px;
height: 44px;
filter: drop-shadow(0 2px 6px rgba(34, 197, 94, 0.25));
}
.brand-title {
font-size: 20px;
font-weight: 800;
color: #14532d;
letter-spacing: -0.5px;
display: flex;
align-items: center;
gap: 6px;
}
.brand-jp {
font-size: 13px;
font-weight: 600;
color: #16a34a;
background: var(--primary-light);
padding: 2px 8px;
border-radius: 6px;
}
.brand-sub {
font-size: 12px;
color: var(--text-muted);
font-weight: 500;
}
.header-actions {
display: flex;
align-items: center;
gap: 12px;
}
.lang-switch-btn {
background: var(--primary-subtle);
border: 1px solid var(--border-color);
color: var(--primary-dark);
padding: 6px 14px;
border-radius: 999px;
font-size: 12px;
font-weight: 700;
cursor: pointer;
display: flex;
align-items: center;
gap: 6px;
transition: all 0.2s;
}
.lang-switch-btn:hover {
background: var(--primary-light);
border-color: var(--primary-green);
}
.agent-pills {
display: flex;
gap: 6px;
}
.pill {
font-size: 11px;
font-weight: 600;
padding: 4px 10px;
border-radius: 999px;
border: 1px solid var(--border-color);
background: var(--primary-subtle);
color: #166534;
display: flex;
align-items: center;
gap: 5px;
}
.dot {
width: 6px;
height: 6px;
border-radius: 50%;
background-color: var(--primary-green);
box-shadow: 0 0 6px var(--primary-green);
}
main {
flex: 1;
padding: 20px 32px;
display: grid;
grid-template-columns: 1fr 430px;
gap: 24px;
}
.shelf-panel {
background: var(--bg-card);
border: 1px solid var(--border-color);
border-radius: 18px;
padding: 20px;
display: flex;
flex-direction: column;
gap: 16px;
box-shadow: var(--shadow-soft);
}
.toolbar {
display: flex;
justify-content: space-between;
align-items: center;
flex-wrap: wrap;
gap: 12px;
padding-bottom: 12px;
border-bottom: 1px solid var(--border-color);
}
.toolbar-group {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.btn-group {
display: flex;
background: var(--primary-subtle);
border: 1px solid var(--border-color);
border-radius: 10px;
padding: 3px;
gap: 2px;
}
.method-btn {
background: transparent;
border: none;
color: var(--text-muted);
padding: 6px 12px;
border-radius: 8px;
font-size: 12px;
font-weight: 600;
cursor: pointer;
transition: all 0.2s;
}
.method-btn.active {
background: var(--primary-green);
color: white;
box-shadow: 0 2px 8px rgba(34, 197, 94, 0.3);
}
.tool-btn {
background: var(--bg-card);
border: 1px solid var(--border-color);
color: #15803d;
padding: 7px 14px;
border-radius: 10px;
font-size: 12px;
font-weight: 700;
cursor: pointer;
display: flex;
align-items: center;
gap: 6px;
transition: all 0.2s;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.04);
}
.tool-btn:hover {
background: var(--primary-subtle);
border-color: var(--primary-green);
transform: translateY(-1px);
}
select.shelf-select {
background: var(--bg-card);
color: var(--text-main);
border: 1px solid var(--border-color);
padding: 7px 12px;
border-radius: 10px;
font-size: 12px;
font-weight: 600;
outline: none;
cursor: pointer;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.04);
}
select.shelf-select:hover {
border-color: var(--primary-green);
}
.shelf-canvas-wrapper {
flex: 1;
position: relative;
background: #0f172a;
border-radius: 14px;
overflow: auto;
border: 1px solid var(--border-color);
display: flex;
align-items: center;
justify-content: center;
min-height: 560px;
cursor: crosshair;
}
#shelfCanvas {
max-width: 100%;
display: block;
}
.instruction-banner {
position: absolute;
bottom: 16px;
left: 50%;
transform: translateX(-50%);
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(8px);
border: 1px solid var(--primary-green);
color: #14532d;
padding: 8px 20px;
border-radius: 999px;
font-size: 12px;
font-weight: 700;
box-shadow: 0 4px 16px rgba(34, 197, 94, 0.2);
pointer-events: none;
display: flex;
align-items: center;
gap: 8px;
}
.side-panel {
display: flex;
flex-direction: column;
gap: 18px;
}
.card {
background: var(--bg-card);
border: 1px solid var(--border-color);
border-radius: 18px;
padding: 20px;
display: flex;
flex-direction: column;
gap: 14px;
box-shadow: var(--shadow-soft);
}
.card-header {
font-size: 13px;
font-weight: 800;
color: #166534;
text-transform: uppercase;
letter-spacing: 0.6px;
display: flex;
justify-content: space-between;
align-items: center;
padding-bottom: 8px;
border-bottom: 1px solid var(--border-color);
}
.product-hero {
display: flex;
gap: 16px;
align-items: center;
}
.product-thumb {
width: 95px;
height: 120px;
border-radius: 12px;
object-fit: cover;
border: 2px solid var(--primary-light);
background: #f8fafc;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.06);
}
.product-details {
flex: 1;
display: flex;
flex-direction: column;
gap: 6px;
}
.product-name {
font-size: 17px;
font-weight: 800;
line-height: 1.3;
color: #0f172a;
}
.product-sku {
font-size: 11px;
color: #16a34a;
font-family: monospace;
background: var(--primary-light);
padding: 2px 6px;
border-radius: 4px;
display: inline-block;
width: fit-content;
font-weight: 700;
}
.product-price {
font-size: 26px;
font-weight: 800;
color: #15803d;
}
.stock-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 10px;
margin-top: 4px;
}
.stock-metric {
background: var(--primary-subtle);
border: 1px solid var(--border-color);
border-radius: 12px;
padding: 10px 14px;
}
.metric-title {
font-size: 11px;
color: var(--text-muted);
margin-bottom: 4px;
font-weight: 600;
}
.metric-val {
font-size: 16px;
font-weight: 800;
color: #15803d;
}
.vis-match-box {
width: 100%;
height: 180px;
background: #0f172a;
border-radius: 12px;
border: 1px solid var(--border-color);
overflow: hidden;
display: flex;
align-items: center;
justify-content: center;
}
.vis-match-img {
width: 100%;
height: 100%;
object-fit: contain;
}
.action-btn {
width: 100%;
padding: 12px;
border: none;
border-radius: 12px;
background: linear-gradient(135deg, #22c55e, #16a34a);
color: white;
font-weight: 800;
font-size: 14px;
cursor: pointer;
box-shadow: 0 4px 14px rgba(34, 197, 94, 0.3);
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
}
.action-btn:hover {
transform: translateY(-1px);
box-shadow: 0 6px 20px rgba(34, 197, 94, 0.4);
}
.status-badge {
font-size: 11px;
padding: 3px 10px;
border-radius: 999px;
font-weight: 700;
}
.badge-success { background: #dcfce7; color: #15803d; }
.badge-warning { background: #fef3c7; color: #b45309; }
.badge-danger { background: #fee2e2; color: #b91c1c; }
/* Modal Styles */
.modal-backdrop {
display: none;
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(15, 23, 42, 0.6);
backdrop-filter: blur(4px);
z-index: 1000;
align-items: center;
justify-content: center;
}
.modal-window {
background: #ffffff;
border-radius: 20px;
width: 90%;
max-width: 600px;
padding: 24px;
box-shadow: 0 20px 40px rgba(0, 0, 0, 0.15);
display: flex;
flex-direction: column;
gap: 16px;
}
.modal-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.modal-title {
font-size: 18px;
font-weight: 800;
color: #14532d;
}
.close-btn {
background: none;
border: none;
font-size: 24px;
color: var(--text-muted);
cursor: pointer;
}
.video-box {
width: 100%;
height: 380px;
background: #000000;
border-radius: 12px;
overflow: hidden;
position: relative;
}
#cameraVideo {
width: 100%;
height: 100%;
object-fit: cover;
}
</style>
</head>
<body>
<header>
<div class="brand">
<!-- Dorayaki SVG Logo -->
<svg class="brand-logo-svg" viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg">
<defs>
<linearGradient id="pancakeTop" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" stop-color="#f59e0b"/>
<stop offset="100%" stop-color="#b45309"/>
</linearGradient>
<linearGradient id="ankoFill" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" stop-color="#450a0a"/>
<stop offset="100%" stop-color="#2d0606"/>
</linearGradient>
<linearGradient id="pancakeBottom" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" stop-color="#d97706"/>
<stop offset="100%" stop-color="#92400e"/>
</linearGradient>
</defs>
<!-- Bottom Pancake -->
<ellipse cx="50" cy="62" rx="42" ry="18" fill="url(#pancakeBottom)" />
<!-- Anko (Sweet Red Bean) Layer -->
<ellipse cx="50" cy="53" rx="43" ry="14" fill="url(#ankoFill)" />
<!-- Top Pancake -->
<ellipse cx="50" cy="42" rx="44" ry="22" fill="url(#pancakeTop)" />
<!-- Glaze highlight -->
<ellipse cx="44" cy="35" rx="20" ry="8" fill="#fde68a" opacity="0.4" />
<!-- Cute Leaf Garnish for fresh green theme -->
<path d="M 52,18 C 65,12 72,25 65,30 C 58,28 54,23 52,18 Z" fill="#22c55e" />
</svg>
<div>
<div class="brand-title">
Dorayaki <span class="brand-jp">どら焼き</span>
</div>
<div class="brand-sub" id="t-subtitle">Smart Supermarket Shelf AI & Inventory Recognition</div>
</div>
</div>
<div class="header-actions">
<button class="lang-switch-btn" onclick="toggleLanguage()">
<span id="langIcon">🌐</span> <span id="langText">日本語 / JA</span>
</button>
<div class="agent-pills">
<div class="pill"><span class="dot"></span> PM</div>
<div class="pill"><span class="dot"></span> AI</div>
<div class="pill"><span class="dot"></span> Dev</div>
<div class="pill"><span class="dot"></span> Reviewer</div>
</div>
</div>
</header>
<main>
<div class="shelf-panel">
<div class="toolbar">
<div class="toolbar-group">
<span style="font-size: 12px; font-weight: 700; color: #166534;" id="t-method-label">Algorithm:</span>
<div class="btn-group">
<button class="method-btn active" onclick="setMethod('hybrid')">Hybrid (Paper)</button>
<button class="method-btn" onclick="setMethod('surf')">SURF</button>
<button class="method-btn" onclick="setMethod('orb')">ORB</button>
<button class="method-btn" onclick="setMethod('brisk')">BRISK</button>
</div>
</div>
<div class="toolbar-group">
<select id="shelfSelect" class="shelf-select" onchange="loadSelectedShelf()">
<!-- Populated dynamically via API -->
</select>
<!-- Empty Shelf Slots Toggle -->
<button id="emptySlotsToggle" class="tool-btn" style="border-color: #fca5a5; background: #fef2f2; color: #b91c1c; font-weight: 700;" onclick="toggleEmptySlots()">
<span>🔴</span> <span id="t-empty-btn">Empty Spaces</span>
</button>
<!-- Camera Snapshot Button -->
<button class="tool-btn" onclick="openCameraModal()">
<span>📷</span> <span id="t-camera-btn">Capture Photo</span>
</button>
<!-- File Upload Button -->
<button class="tool-btn" onclick="triggerFileInput()">
<span>📁</span> <span id="t-upload-btn">Upload Image</span>
</button>
<input type="file" id="fileInput" accept="image/*" style="display: none;" onchange="handleFileUpload(event)">
</div>
</div>
<div class="shelf-canvas-wrapper" id="canvasWrapper">
<canvas id="shelfCanvas"></canvas>
<div class="instruction-banner">
<span>👆 <span id="t-instruction">Click on any product packaging to identify & view real-time inventory</span></span>
</div>
</div>
</div>
<div class="side-panel">
<div class="card" id="productCard">
<div class="card-header">
<span id="t-card-title">Product Details</span>
<span id="matchStatusBadge" class="status-badge badge-success">Ready</span>
</div>
<div id="cardContentPlaceholder" style="text-align: center; padding: 45px 10px; color: var(--text-muted); font-size: 13px;">
<div style="font-size: 36px; margin-bottom: 10px;">🏷️</div>
<span id="t-placeholder-text">No product selected.<br>Click any merchandise item on the shelf image to inspect details.</span>
</div>
<div id="cardContentDetails" style="display: none; flex-direction: column; gap: 14px;">
<div class="product-hero">
<img id="productThumb" class="product-thumb" src="" alt="Thumbnail">
<div class="product-details">
<span id="productSku" class="product-sku">SKU_001</span>
<div id="productName" class="product-name">Product Name</div>
<div id="productPrice" class="product-price">¥0</div>
</div>
</div>
<div class="stock-grid">
<div class="stock-metric">
<div class="metric-title" id="t-metric-on">On Shelf (In Stock)</div>
<div id="metricOn" class="metric-val">0</div>
</div>
<div class="stock-metric">
<div class="metric-title" id="t-metric-out">Sold / Dispatched</div>
<div id="metricOut" class="metric-val">0</div>
</div>
<div class="stock-metric">
<div class="metric-title" id="t-metric-receipt">Receipt Date</div>
<div id="metricReceipt" class="metric-val" style="font-size: 13px;">--</div>
</div>
<div class="stock-metric">
<div class="metric-title" id="t-metric-expiry">Expiration Date</div>
<div id="metricExpiry" class="metric-val" style="font-size: 13px; color: #b45309;">--</div>
</div>
</div>
<button class="action-btn" onclick="purchaseOne()">
<span>🛒</span> <span id="t-dispense-btn">Dispense / Purchase 1 Unit</span>
</button>
</div>
</div>
<div class="card">
<div class="card-header">
<span id="t-match-title">Feature Matching Inliers</span>
<span id="scoreLabel" style="font-size: 12px; color: #16a34a; font-weight: 700;">--</span>
</div>
<div class="vis-match-box" id="visMatchBox">
<span style="color: #94a3b8; font-size: 12px;" id="t-vis-placeholder">Visual keypoint matching will appear here</span>
</div>
</div>
</div>
</main>
<!-- Camera Capture Modal -->
<div class="modal-backdrop" id="cameraModal">
<div class="modal-window">
<div class="modal-header">
<div class="modal-title" id="t-modal-camera-title">📷 Take Shelf Photo</div>
<button class="close-btn" onclick="closeCameraModal()">&times;</button>
</div>
<div class="video-box">
<video id="cameraVideo" autoplay playsinline></video>
</div>
<button class="action-btn" onclick="captureSnapshot()">
<span>📸</span> <span id="t-take-photo-btn">Capture & Run Detection</span>
</button>
</div>
</div>
<script>
let currentLang = 'en'; // 'en' or 'ja'
let currentMethod = 'hybrid';
let detectedBoxes = [];
let emptySlots = [];
let showEmptySlots = true;
let selectedBoxId = null;
let selectedEmptyId = null;
let shelfImageObj = new Image();
let currentProductData = null;
let mediaStream = null;
const translations = {
en: {
subtitle: "Smart Supermarket Shelf AI & Inventory Recognition",
langToggle: "日本語 / JA",
methodLabel: "Algorithm:",
cameraBtn: "Capture Photo",
uploadBtn: "Upload Image",
emptyBtn: "Empty Spaces",
instruction: "Click on any product packaging or red empty slot to inspect details",
cardTitle: "Product Details",
emptySlotCardTitle: "Empty Shelf Space",
emptySlotBadge: "OUT OF STOCK",
placeholderText: "No product selected.<br>Click any merchandise item on the shelf image to inspect details.",
metricOn: "On Shelf (In Stock)",
metricOut: "Sold / Dispatched",
metricReceipt: "Receipt Date",
metricExpiry: "Expiration Date",
dispenseBtn: "Dispense / Purchase 1 Unit",
matchTitle: "Feature Matching Inliers",
visPlaceholder: "Visual keypoint matching will appear here",
cameraModalTitle: "📷 Take Shelf Photo",
takePhotoBtn: "Capture & Run Detection",
statusReady: "Ready",
statusAnalyzing: "Analyzing...",
statusRecognized: "Recognized",
statusNotFound: "Not Found"
},
ja: {
subtitle: "スマートスーパーマーケット棚認識 & 在庫管理AI",
langToggle: "English / EN",
methodLabel: "分類手法:",
cameraBtn: "カメラ撮影",
uploadBtn: "画像アップロード",
emptyBtn: "空きスペース",
instruction: "棚の画像上の商品または赤枠の空きスペースをクリックして詳細を表示します",
cardTitle: "商品情報",
emptySlotCardTitle: "棚の空きスペース (欠品)",
emptySlotBadge: "欠品 (在庫切れ)",
placeholderText: "商品が選択されていません。<br>棚画像上の商品をクリックしてください。",
metricOn: "陳列在庫数 (棚上)",
metricOut: "販売済 / 出庫数",
metricReceipt: "受入日",
metricExpiry: "賞味期限",
dispenseBtn: "1点出庫 / 購入する",
matchTitle: "特徴点マッチング結果",
visPlaceholder: "ここに特徴点の一致描画が表示されます",
cameraModalTitle: "📷 棚の写真を撮影",
takePhotoBtn: "撮影して物体検出を実行",
statusReady: "待機中",
statusAnalyzing: "解析中...",
statusRecognized: "認識完了",
statusNotFound: "未検出"
}
};
function toggleLanguage() {
currentLang = (currentLang === 'en') ? 'ja' : 'en';
applyLanguage();
}
function applyLanguage() {
const t = translations[currentLang];
document.getElementById('t-subtitle').innerText = t.subtitle;
document.getElementById('langText').innerText = t.langToggle;
document.getElementById('t-method-label').innerText = t.methodLabel;
document.getElementById('t-camera-btn').innerText = t.cameraBtn;
document.getElementById('t-upload-btn').innerText = t.uploadBtn;
document.getElementById('t-empty-btn').innerText = `${t.emptyBtn} (${emptySlots.length})`;
document.getElementById('t-instruction').innerText = t.instruction;
document.getElementById('t-card-title').innerText = t.cardTitle;
document.getElementById('t-placeholder-text').innerHTML = t.placeholderText;
document.getElementById('t-metric-on').innerText = t.metricOn;
document.getElementById('t-metric-out').innerText = t.metricOut;
document.getElementById('t-metric-receipt').innerText = t.metricReceipt;
document.getElementById('t-metric-expiry').innerText = t.metricExpiry;
document.getElementById('t-dispense-btn').innerText = t.dispenseBtn;
document.getElementById('t-match-title').innerText = t.matchTitle;
document.getElementById('t-vis-placeholder').innerText = t.visPlaceholder;
document.getElementById('t-modal-camera-title').innerText = t.cameraModalTitle;
document.getElementById('t-take-photo-btn').innerText = t.takePhotoBtn;
}
function toggleEmptySlots() {
showEmptySlots = !showEmptySlots;
const btn = document.getElementById('emptySlotsToggle');
if (showEmptySlots) {
btn.style.background = '#fef2f2';
btn.style.color = '#b91c1c';
btn.style.borderColor = '#fca5a5';
} else {
btn.style.background = '#ffffff';
btn.style.color = '#64748b';
btn.style.borderColor = '#e2e8f0';
}
drawCanvas();
}
const canvas = document.getElementById('shelfCanvas');
const ctx = canvas.getContext('2d');
function setMethod(method) {
currentMethod = method;
document.querySelectorAll('.method-btn').forEach(b => b.classList.remove('active'));
event.target.classList.add('active');
if (selectedBoxId !== null) {
const b = detectedBoxes.find(x => x.id === selectedBoxId);
if (b) queryClick(b.center[0], b.center[1]);
}
}
async function initSampleList() {
try {
const res = await fetch('/api/shelf_samples');
const data = await res.json();
const select = document.getElementById('shelfSelect');
select.innerHTML = '';
data.samples.forEach(item => {
const opt = document.createElement('option');
opt.value = item.path;
opt.innerText = item.label;
if (item.path === 'image.jpg') opt.selected = true;
select.appendChild(opt);
});
loadSelectedShelf();
} catch (e) {
console.error(e);
}
}
async function loadSelectedShelf() {
const path = document.getElementById('shelfSelect').value;
const res = await fetch('/api/load_shelf', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ image_path: path })
});
const data = await res.json();
if (data.status === 'success') {
detectedBoxes = data.boxes;
emptySlots = data.empty_slots || [];
selectedBoxId = null;
selectedEmptyId = null;
document.getElementById('t-empty-btn').innerText = `${translations[currentLang].emptyBtn} (${emptySlots.length})`;
shelfImageObj.onload = () => {
canvas.width = shelfImageObj.width;
canvas.height = shelfImageObj.height;
drawCanvas();
};
shelfImageObj.src = data.image_base64;
}
}
function triggerFileInput() {
document.getElementById('fileInput').click();
}
async function handleFileUpload(e) {
const file = e.target.files[0];
if (!file) return;
const reader = new FileReader();
reader.onload = async function(evt) {
const base64Data = evt.target.result;
await uploadAndDetect(base64Data);
};
reader.readAsDataURL(file);
}
async function uploadAndDetect(base64Data) {
document.getElementById('matchStatusBadge').innerText = translations[currentLang].statusAnalyzing;
document.getElementById('matchStatusBadge').className = 'status-badge badge-warning';
const res = await fetch('/api/upload_shelf', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ image_base64: base64Data })
});
const data = await res.json();
if (data.status === 'success') {
detectedBoxes = data.boxes;
emptySlots = data.empty_slots || [];
selectedBoxId = null;
selectedEmptyId = null;
document.getElementById('t-empty-btn').innerText = `${translations[currentLang].emptyBtn} (${emptySlots.length})`;
shelfImageObj.onload = () => {
canvas.width = shelfImageObj.width;
canvas.height = shelfImageObj.height;
drawCanvas();
};
shelfImageObj.src = data.image_base64;
document.getElementById('matchStatusBadge').innerText = translations[currentLang].statusReady;
document.getElementById('matchStatusBadge').className = 'status-badge badge-success';
}
}
// Camera Modal Functions
async function openCameraModal() {
const modal = document.getElementById('cameraModal');
modal.style.display = 'flex';
try {
mediaStream = await navigator.mediaDevices.getUserMedia({ video: { width: 1280, height: 720 } });
document.getElementById('cameraVideo').srcObject = mediaStream;
} catch (err) {
alert("Camera access failed: " + err.message);
closeCameraModal();
}
}
function closeCameraModal() {
const modal = document.getElementById('cameraModal');
modal.style.display = 'none';
if (mediaStream) {
mediaStream.getTracks().forEach(track => track.stop());
mediaStream = null;
}
}
async function captureSnapshot() {
const video = document.getElementById('cameraVideo');
const offscreen = document.createElement('canvas');
offscreen.width = video.videoWidth || 1280;
offscreen.height = video.videoHeight || 720;
const offCtx = offscreen.getContext('2d');
offCtx.drawImage(video, 0, 0, offscreen.width, offscreen.height);
const dataUrl = offscreen.toDataURL('image/jpeg', 0.9);
closeCameraModal();
await uploadAndDetect(dataUrl);
}
function drawCanvas() {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(shelfImageObj, 0, 0);
// 1. Detected Products: Clean bounding boxes without class labels or confidence text
detectedBoxes.forEach(b => {
const [x1, y1, x2, y2] = b.box;
const isSelected = (b.id === selectedBoxId);
ctx.lineWidth = isSelected ? 4 : 2;
ctx.strokeStyle = isSelected ? '#10b981' : '#22c55e';
ctx.fillStyle = isSelected ? 'rgba(16, 185, 129, 0.25)' : 'rgba(34, 197, 94, 0.05)';
ctx.fillRect(x1, y1, x2 - x1, y2 - y1);
ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
});
// 2. Empty Shelf Spaces: Clean RED bounding boxes
if (showEmptySlots && emptySlots && emptySlots.length > 0) {
emptySlots.forEach(slot => {
const [x1, y1, x2, y2] = slot.box;
const isSelected = (slot.id === selectedEmptyId);
ctx.lineWidth = isSelected ? 4 : 3;
ctx.strokeStyle = '#ef4444'; // Red bounding box
ctx.fillStyle = isSelected ? 'rgba(239, 68, 68, 0.38)' : 'rgba(239, 68, 68, 0.22)';
ctx.fillRect(x1, y1, x2 - x1, y2 - y1);
ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
// Red header tag
const tagText = `EMPTY (~${slot.missing_items_est})`;
ctx.font = 'bold 11px Plus Jakarta Sans, sans-serif';
const textW = ctx.measureText(tagText).width;
const tagH = 16;
const tagY = Math.max(0, y1 - tagH - 2);
ctx.fillStyle = '#ef4444';
ctx.fillRect(x1, tagY, textW + 8, tagH);
ctx.fillStyle = '#ffffff';
ctx.fillText(tagText, x1 + 4, tagY + 12);
});
}
}
canvas.addEventListener('click', (e) => {
const rect = canvas.getBoundingClientRect();
const scaleX = canvas.width / rect.width;
const scaleY = canvas.height / rect.height;
const clickX = Math.round((e.clientX - rect.left) * scaleX);
const clickY = Math.round((e.clientY - rect.top) * scaleY);
queryClick(clickX, clickY);
});
async function queryClick(x, y) {
document.getElementById('matchStatusBadge').innerText = translations[currentLang].statusAnalyzing;
document.getElementById('matchStatusBadge').className = 'status-badge badge-warning';
const res = await fetch('/api/click', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ x: x, y: y, method: currentMethod })
});
const data = await res.json();
if (data.status === 'success') {
selectedBoxId = data.target_box.id;
selectedEmptyId = null;
drawCanvas();
currentProductData = data.inventory;
document.getElementById('cardContentPlaceholder').style.display = 'none';
document.getElementById('cardContentDetails').style.display = 'flex';
document.getElementById('productSku').innerText = data.inventory.sku_id;
document.getElementById('productName').innerText = data.inventory.product_name;
document.getElementById('productPrice').innerText = '¥' + data.inventory.price;
document.getElementById('productPrice').style.color = 'var(--primary-dark)';
document.getElementById('productThumb').src = data.crop_base64;
document.getElementById('metricOn').innerText = data.inventory.number_on;
document.getElementById('metricOut').innerText = data.inventory.number_out;
document.getElementById('metricReceipt').innerText = data.inventory.receipt_date;
document.getElementById('metricExpiry').innerText = data.inventory.expiration_date;
document.getElementById('scoreLabel').innerText = `Score: ${data.classification.score} | Inliers: ${(data.classification.confidence * 100).toFixed(1)}%`;
document.getElementById('matchStatusBadge').innerText = translations[currentLang].statusRecognized;
document.getElementById('matchStatusBadge').className = 'status-badge badge-success';
if (data.vis_base64) {
document.getElementById('visMatchBox').innerHTML = `<img class="vis-match-img" src="${data.vis_base64}">`;
}
} else if (data.status === 'empty_slot') {
selectedBoxId = null;
selectedEmptyId = data.empty_slot.id;
drawCanvas();
currentProductData = null;
document.getElementById('cardContentPlaceholder').style.display = 'none';
document.getElementById('cardContentDetails').style.display = 'flex';
document.getElementById('productSku').innerText = data.empty_slot.id.toUpperCase();
document.getElementById('productName').innerText = `${translations[currentLang].emptySlotCardTitle} (Row ${data.empty_slot.row_index})`;
document.getElementById('productPrice').innerText = 'OUT OF STOCK';
document.getElementById('productPrice').style.color = '#dc2626';
document.getElementById('productThumb').src = "data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='80' height='80' viewBox='0 0 80 80'><rect width='80' height='80' rx='12' fill='%23fee2e2'/><text x='50%' y='55%' dominant-baseline='middle' text-anchor='middle' font-size='32'>🔴</text></svg>";
document.getElementById('metricOn').innerText = '0 (Empty)';
document.getElementById('metricOut').innerText = `~${data.empty_slot.missing_items_est} items`;
document.getElementById('metricReceipt').innerText = `Width: ${data.empty_slot.width}px`;
document.getElementById('metricExpiry').innerText = `Height: ${data.empty_slot.height}px`;
document.getElementById('scoreLabel').innerText = `Gap Width: ${data.empty_slot.width}px | Missing: ~${data.empty_slot.missing_items_est} units`;
document.getElementById('matchStatusBadge').innerText = translations[currentLang].emptySlotBadge;
document.getElementById('matchStatusBadge').className = 'status-badge badge-danger';
document.getElementById('visMatchBox').innerHTML = `<div style="text-align: center; padding: 24px 10px; color: #dc2626; font-size: 13px; font-weight: 600; background: #fef2f2; border-radius: 8px; border: 1px dashed #fca5a5;">⚠️ ${data.message}</div>`;
} else {
document.getElementById('matchStatusBadge').innerText = translations[currentLang].statusNotFound;
document.getElementById('matchStatusBadge').className = 'status-badge badge-warning';
}
}
async function purchaseOne() {
if (!currentProductData) return;
const res = await fetch('/api/update_stock', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ sku_id: currentProductData.sku_id, delta: -1 })
});
const data = await res.json();
if (data.status === 'success') {
document.getElementById('metricOn').innerText = data.number_on;
document.getElementById('metricOut').innerText = data.number_out;
}
}
// Initialize on load
window.addEventListener('DOMContentLoaded', () => {
initSampleList();
applyLanguage();
});
</script>
</body>
</html>
"""
@app.get("/", response_class=HTMLResponse)
async def index():
return HTMLResponse(content=HTML_PAGE)
@app.get("/api/shelf_samples")
async def get_shelf_samples():
samples = get_available_shelf_images("add")
return {"status": "success", "samples": samples}
@app.post("/api/load_shelf")
async def load_shelf(payload: dict):
img_path = payload.get("image_path", "image.jpg")
init_res = pipeline.set_active_shelf_image(img_path)
if init_res.get("status") == "success":
return {
"status": "success",
"boxes": init_res["boxes"],
"empty_slots": init_res.get("empty_slots", []),
"image_base64": encode_image_base64(pipeline.current_image)
}
return JSONResponse(status_code=400, content={"status": "error", "message": "Failed to load image"})
@app.post("/api/upload_shelf")
async def upload_shelf(payload: dict):
data_url = payload.get("image_base64", "")
img_bgr = decode_image_base64(data_url)
if img_bgr is None:
return JSONResponse(status_code=400, content={"status": "error", "message": "Failed to decode image"})
init_res = pipeline.set_active_shelf_image(img_bgr)
if init_res.get("status") == "success":
return {
"status": "success",
"boxes": init_res["boxes"],
"empty_slots": init_res.get("empty_slots", []),
"image_base64": encode_image_base64(pipeline.current_image)
}
return JSONResponse(status_code=400, content={"status": "error", "message": "Detection failed"})
@app.post("/api/click")
async def handle_click(payload: dict):
x = int(payload.get("x", 0))
y = int(payload.get("y", 0))
method = payload.get("method", "hybrid")
res = pipeline.process_click(x, y, method=method)
if res.get("status") == "success":
return {
"status": "success",
"target_box": res["target_box"],
"classification": res["classification"],
"inventory": res["inventory"],
"crop_base64": encode_image_base64(res["crop_image"]),
"vis_base64": encode_image_base64(res["match_visualization"])
}
elif res.get("status") == "empty_slot":
return {
"status": "empty_slot",
"empty_slot": res["empty_slot"],
"message": res["message"]
}
return JSONResponse(status_code=404, content=res)
@app.post("/api/update_stock")
async def update_stock(payload: dict):
sku_id = payload.get("sku_id")
delta = int(payload.get("delta", -1))
ok = pipeline.inventory.update_stock(sku_id, delta)
if ok:
p = pipeline.inventory.get_product(sku_id)
return {"status": "success", "number_on": p["number_on"], "number_out": p["number_out"]}
return JSONResponse(status_code=400, content={"status": "error"})
return app
def main():
parser = argparse.ArgumentParser(description="Dorayaki - Smart Supermarket Shelf AI")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--host", type=str, default="0.0.0.0")
args = parser.parse_args()
pipeline = ShelfRecognitionPipeline()
app = create_fastapi_app(pipeline)
import uvicorn
print(f"\n[Dorayaki Server] Starting on http://localhost:{args.port} (or http://127.0.0.1:{args.port})")
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
if __name__ == "__main__":
main()