feat: web client with live and photo modes, served by the server
The client is a pure API consumer: live frames go out over /v1/stream, stills
over /v1/recognize, and it can be pointed at another server with
?server=&token=. It keeps the team's visual identity (green palette, logo, EN/JA
toggle) and adds the live surface: state-coloured boxes with name/price chips,
interpolated track boxes, a results/s + latency HUD, quality hints, and an
in-view list that highlights a SKU's boxes.
Photo mode recognises the whole shelf at once instead of one click per product:
boxes paint immediately, identities fill in, tapping a product inspects it
(crop + feature-match visualization), the algorithm selector re-classifies, and
dispense updates stock.
Fixes found by browser verification with a synthetic camera feed:
- The WebSocket handler referenced its processing loop before definition, so the
socket died with NameError and live mode never received a result. Helpers are
now defined inside the handler.
- The client never kept a reference to the video element, so frame capture
returned early and nothing was ever sent.
- /v1/images/{id}/classify passed detection dicts where the engine expected box
dicts (KeyError: 'width').
- Placeholder chrome (thumb, dashes, hidden groups) leaked on first paint.
Verified in Chrome with a fake camera: live 8-12 results/s, ~60 products in
view, product card and feature matching populated, photo 105/170 recognised on
image.jpg and 59/111 on add/image6.jpg, 0 console errors.
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w
parent
5f84f4f2a7
commit
6902067322
5 changed files with 1454 additions and 50 deletions
|
|
@ -445,7 +445,57 @@ def create_app(
|
|||
}
|
||||
)
|
||||
|
||||
processing = asyncio.create_task(_processing_loop(session, send, loop))
|
||||
async def processing_loop() -> None:
|
||||
while not session.closed:
|
||||
pending = await loop.run_in_executor(None, session.take_pending, 0.25)
|
||||
if pending is None:
|
||||
continue
|
||||
frame_id, payload = pending
|
||||
try:
|
||||
result = await loop.run_in_executor(None, session.process_frame, frame_id, payload)
|
||||
except UndecodableImage:
|
||||
await send({"type": "error", "code": "UNDECODABLE_IMAGE", "frame_id": frame_id,
|
||||
"message": "frame is not a readable image"})
|
||||
continue
|
||||
except Exception as exc: # keep the stream alive on one bad frame
|
||||
LOGGER.exception("live frame failed: %s", exc)
|
||||
await send({"type": "error", "code": "INTERNAL", "frame_id": frame_id,
|
||||
"message": "frame processing failed"})
|
||||
continue
|
||||
await send(result)
|
||||
|
||||
async def handle_inspect(message: Dict[str, Any]) -> None:
|
||||
request_id = message.get("request_id")
|
||||
try:
|
||||
track_id = int(message.get("track_id"))
|
||||
except (TypeError, ValueError):
|
||||
await send({"type": "error", "code": "BAD_MESSAGE", "request_id": request_id,
|
||||
"message": "track_id required"})
|
||||
return
|
||||
method = message.get("method")
|
||||
if method is not None:
|
||||
try:
|
||||
method = _require_method(method)
|
||||
except HTTPException as exc:
|
||||
await send({"type": "error", "code": "BAD_MESSAGE", "request_id": request_id,
|
||||
"message": str(exc.detail)})
|
||||
return
|
||||
try:
|
||||
result = await loop.run_in_executor(None, session.inspect, track_id, method)
|
||||
except KeyError:
|
||||
await send({"type": "error", "code": "TRACK_NOT_IN_VIEW", "request_id": request_id,
|
||||
"message": "track is not in the latest frame"})
|
||||
return
|
||||
except Exception as exc:
|
||||
LOGGER.exception("inspect failed: %s", exc)
|
||||
await send({"type": "error", "code": "INTERNAL", "request_id": request_id,
|
||||
"message": "inspect failed"})
|
||||
return
|
||||
payload = {"type": "inspect_result", "request_id": request_id, "track_id": track_id}
|
||||
payload.update(result)
|
||||
await send(payload)
|
||||
|
||||
processing = asyncio.create_task(processing_loop())
|
||||
|
||||
async def handle_control(message: Dict[str, Any]) -> None:
|
||||
kind = message.get("type")
|
||||
|
|
@ -466,7 +516,7 @@ def create_app(
|
|||
elif kind == "ping":
|
||||
await send({"type": "pong", "t": message.get("t")})
|
||||
elif kind == "inspect":
|
||||
await _handle_inspect(session, message, send, loop)
|
||||
await handle_inspect(message)
|
||||
else:
|
||||
await send({"type": "error", "code": "BAD_MESSAGE", "message": f"unknown type '{kind}'"})
|
||||
|
||||
|
|
@ -510,53 +560,6 @@ def create_app(
|
|||
async with sessions_lock:
|
||||
sessions.pop(session_id, None)
|
||||
|
||||
async def _processing_loop(session: LiveSession, send, loop: asyncio.AbstractEventLoop) -> None:
|
||||
while not session.closed:
|
||||
pending = await loop.run_in_executor(None, session.take_pending, 0.25)
|
||||
if pending is None:
|
||||
continue
|
||||
frame_id, payload = pending
|
||||
try:
|
||||
result = await loop.run_in_executor(None, session.process_frame, frame_id, payload)
|
||||
except UndecodableImage:
|
||||
await send(
|
||||
{"type": "error", "code": "UNDECODABLE_IMAGE", "frame_id": frame_id, "message": "frame is not a readable image"}
|
||||
)
|
||||
continue
|
||||
except Exception as exc: # keep the stream alive on one bad frame
|
||||
LOGGER.exception("live frame failed: %s", exc)
|
||||
await send({"type": "error", "code": "INTERNAL", "frame_id": frame_id, "message": "frame processing failed"})
|
||||
continue
|
||||
await send(result)
|
||||
|
||||
async def _handle_inspect(session: LiveSession, message: Dict[str, Any], send, loop) -> None:
|
||||
request_id = message.get("request_id")
|
||||
track_id = message.get("track_id")
|
||||
try:
|
||||
track_id = int(track_id)
|
||||
except (TypeError, ValueError):
|
||||
await send({"type": "error", "code": "BAD_MESSAGE", "request_id": request_id, "message": "track_id required"})
|
||||
return
|
||||
method = message.get("method")
|
||||
if method is not None:
|
||||
try:
|
||||
method = _require_method(method)
|
||||
except HTTPException as exc:
|
||||
await send({"type": "error", "code": "BAD_MESSAGE", "request_id": request_id, "message": str(exc.detail)})
|
||||
return
|
||||
try:
|
||||
result = await loop.run_in_executor(None, session.inspect, track_id, method)
|
||||
except KeyError:
|
||||
await send({"type": "error", "code": "TRACK_NOT_IN_VIEW", "request_id": request_id, "message": "track is not in the latest frame"})
|
||||
return
|
||||
except Exception as exc:
|
||||
LOGGER.exception("inspect failed: %s", exc)
|
||||
await send({"type": "error", "code": "INTERNAL", "request_id": request_id, "message": "inspect failed"})
|
||||
return
|
||||
payload = {"type": "inspect_result", "request_id": request_id, "track_id": track_id}
|
||||
payload.update(result)
|
||||
await send(payload)
|
||||
|
||||
# ---------------------------------------------------------------- legacy session routes
|
||||
@app.post("/api/sessions")
|
||||
async def legacy_start_session(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue