5f84f4f2a7
app.py becomes the recognition server: /v1 REST (health, catalog + template images, samples, recognize, classify, inspect, stock) plus /v1/stream over WebSocket, with token auth, size limits, one consistent error shape, and the legacy /api/sessions routes kept but now backed by the real engine instead of the fake adapter. Default bind is 127.0.0.1; LAN and TLS are opt-in. stream_client.py is the reference non-browser client (camera, video file, or a pan across a still) and prints results/s, RTT, detect time and label stability. Live quality work driven by measurements: - Labels are remembered per place: a fresh track covering a recently recognized product inherits the label instead of queueing for classification again. On a continuous pan this raised recognized products in view from 9/66 to 17-21/66 and cut time-to-first-label to ~1.6 s. - Classification priority is newest-first, because sorting by area spends the budget on products that are already leaving the frame. - Per-session classification concurrency raised to 12. hybrid_fast (ORB shortlist then hybrid) is implemented but NOT the default: it is 3.4x faster (13 ms vs 44 ms p50) yet agreed with hybrid on only 47% of accepted crops across 553 crops, far below the 98% adoption rule. It stays available for experiments; the numbers are in the commit and the benchmark. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
417 lines
16 KiB
Python
417 lines
16 KiB
Python
"""
|
|
Feature Matching & Classification Module
|
|
Implements SURF, ORB, BRISK, and Hybrid matching according to
|
|
the methodology of Electronics 2023, 12(17), 3640.
|
|
"""
|
|
|
|
import os
|
|
import cv2
|
|
import numpy as np
|
|
import threading
|
|
from typing import Dict, List, Tuple, Optional, Any
|
|
from identity import decide_identity
|
|
|
|
|
|
class LocalFeatureExtractor:
|
|
"""Encapsulates individual OpenCV local feature algorithms."""
|
|
|
|
def __init__(self, method: str = "orb"):
|
|
self.method = method.lower()
|
|
self.detector = self._create_detector(self.method)
|
|
self.norm_type = self._get_norm_type(self.method)
|
|
|
|
def _create_detector(self, method: str):
|
|
if method == "surf":
|
|
# Check if SURF is supported in the current OpenCV build
|
|
if hasattr(cv2, "xfeatures2d") and hasattr(cv2.xfeatures2d, "SURF_create"):
|
|
try:
|
|
return cv2.xfeatures2d.SURF_create(hessianThreshold=400, nOctaves=4, nOctaveLayers=3)
|
|
except Exception as e:
|
|
print(f"[Warning] SURF initialization failed: {e}. Falling back to SIFT.")
|
|
else:
|
|
print("[Info] SURF not available in this OpenCV build. Using SIFT as high-grade floating point descriptor.")
|
|
return cv2.SIFT_create(nfeatures=1000)
|
|
|
|
elif method == "orb":
|
|
return cv2.ORB_create(
|
|
nfeatures=1500,
|
|
scaleFactor=1.2,
|
|
nlevels=8,
|
|
edgeThreshold=31,
|
|
firstLevel=0,
|
|
WTA_K=2,
|
|
scoreType=cv2.ORB_HARRIS_SCORE,
|
|
patchSize=31,
|
|
fastThreshold=20
|
|
)
|
|
|
|
elif method == "brisk":
|
|
return cv2.BRISK_create(thresh=30, octaves=3, patternScale=1.0)
|
|
|
|
elif method == "sift":
|
|
return cv2.SIFT_create(nfeatures=1000)
|
|
|
|
else:
|
|
raise ValueError(f"Unsupported feature extractor method: {method}")
|
|
|
|
def _get_norm_type(self, method: str):
|
|
# Binary descriptors use Hamming distance; floating-point descriptors use L2 distance
|
|
if method in ["orb", "brisk"]:
|
|
return cv2.NORM_HAMMING
|
|
return cv2.NORM_L2
|
|
|
|
def detect_and_compute(self, image_gray: np.ndarray) -> Tuple[List[cv2.KeyPoint], Optional[np.ndarray]]:
|
|
"""Detects keypoints and computes their descriptors."""
|
|
if image_gray is None or image_gray.size == 0:
|
|
return [], None
|
|
return self.detector.detectAndCompute(image_gray, None)
|
|
|
|
|
|
class TemplateFeatureDatabase:
|
|
"""Precomputes and caches local feature descriptors for all stock templates."""
|
|
|
|
def __init__(self, stock_dir: str = "stock"):
|
|
self.stock_dir = stock_dir
|
|
self.templates: Dict[str, Dict[str, Any]] = {}
|
|
self.extractors = {
|
|
"surf": LocalFeatureExtractor("surf"),
|
|
"orb": LocalFeatureExtractor("orb"),
|
|
"brisk": LocalFeatureExtractor("brisk")
|
|
}
|
|
self.load_and_cache_templates()
|
|
|
|
def load_and_cache_templates(self):
|
|
"""Loads all SKU images and extracts their multi-feature descriptors."""
|
|
if not os.path.exists(self.stock_dir):
|
|
return
|
|
|
|
image_files = [
|
|
f for f in os.listdir(self.stock_dir)
|
|
if f.lower().endswith((".jpeg", ".jpg", ".png"))
|
|
]
|
|
|
|
print(f"[FeatureDatabase] Pre-caching {len(image_files)} stock SKU templates...")
|
|
for filename in image_files:
|
|
sku_id = os.path.splitext(filename)[0]
|
|
filepath = os.path.join(self.stock_dir, filename)
|
|
img = cv2.imread(filepath)
|
|
if img is None:
|
|
continue
|
|
|
|
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
|
features = {}
|
|
for name, extractor in self.extractors.items():
|
|
kp, des = extractor.detect_and_compute(gray)
|
|
features[name] = {"keypoints": kp, "descriptors": des}
|
|
|
|
self.templates[sku_id] = {
|
|
"sku_id": sku_id,
|
|
"filepath": filepath,
|
|
"image": img,
|
|
"gray": gray,
|
|
"features": features
|
|
}
|
|
print(f"[FeatureDatabase] Successfully cached {len(self.templates)} templates.")
|
|
|
|
|
|
class ProductFeatureMatcher:
|
|
"""
|
|
Main product classification engine.
|
|
Matches a query cropped bounding box against cached SKU templates using
|
|
SURF, ORB, BRISK, or the Paper's Hybrid combination.
|
|
"""
|
|
|
|
def __init__(self, stock_dir: str = "stock"):
|
|
self.db = TemplateFeatureDatabase(stock_dir)
|
|
self.extractors = self.db.extractors
|
|
self._thread_local = threading.local()
|
|
|
|
def thread_extractors(self) -> Dict[str, "LocalFeatureExtractor"]:
|
|
"""One extractor set per thread: OpenCV Feature2D objects are not safe to
|
|
call concurrently from several threads."""
|
|
extractors = getattr(self._thread_local, "extractors", None)
|
|
if extractors is None:
|
|
extractors = {
|
|
name: LocalFeatureExtractor(name) for name in ("surf", "orb", "brisk")
|
|
}
|
|
self._thread_local.extractors = extractors
|
|
return extractors
|
|
|
|
def match_descriptors(
|
|
self,
|
|
des_query: Optional[np.ndarray],
|
|
des_template: Optional[np.ndarray],
|
|
norm_type: int,
|
|
ratio_thresh: float = 0.75
|
|
) -> List[cv2.DMatch]:
|
|
"""Performs k-NN matching with Lowe's ratio test."""
|
|
if des_query is None or des_template is None:
|
|
return []
|
|
if len(des_query) < 2 or len(des_template) < 2:
|
|
return []
|
|
|
|
bf = cv2.BFMatcher(norm_type, crossCheck=False)
|
|
try:
|
|
raw_matches = bf.knnMatch(des_query, des_template, k=2)
|
|
except Exception:
|
|
return []
|
|
|
|
good_matches = []
|
|
for match_pair in raw_matches:
|
|
if len(match_pair) == 2:
|
|
m, n = match_pair
|
|
if m.distance < ratio_thresh * n.distance:
|
|
good_matches.append(m)
|
|
return good_matches
|
|
|
|
def compute_geometric_inliers(
|
|
self,
|
|
kp_query: List[cv2.KeyPoint],
|
|
kp_template: List[cv2.KeyPoint],
|
|
good_matches: List[cv2.DMatch]
|
|
) -> Tuple[int, List[cv2.DMatch], Optional[np.ndarray]]:
|
|
"""Applies RANSAC Homography to verify geometric consistency."""
|
|
if len(good_matches) < 4:
|
|
return len(good_matches), good_matches, None
|
|
|
|
src_pts = np.float32([kp_query[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2)
|
|
dst_pts = np.float32([kp_template[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2)
|
|
|
|
try:
|
|
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
|
|
if mask is None:
|
|
return len(good_matches), good_matches, None
|
|
|
|
inlier_matches = [good_matches[i] for i in range(len(good_matches)) if mask[i][0] == 1]
|
|
return len(inlier_matches), inlier_matches, H
|
|
except Exception:
|
|
return len(good_matches), good_matches, None
|
|
|
|
def score_crop(
|
|
self,
|
|
crop_bgr: np.ndarray,
|
|
method: str = "hybrid"
|
|
) -> Tuple[Dict[str, float], Dict[str, Dict[str, Any]]]:
|
|
"""
|
|
Scores a product crop against every SKU template.
|
|
Returns (scores, details); raises ValueError for an unsupported method and
|
|
returns empty results for an empty crop.
|
|
Methods supported: 'surf', 'orb', 'brisk', 'hybrid'.
|
|
"""
|
|
if crop_bgr is None or crop_bgr.size == 0:
|
|
return {}, {}
|
|
|
|
crop_gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY)
|
|
method = method.lower()
|
|
extractors = self.thread_extractors()
|
|
|
|
scores: Dict[str, float] = {}
|
|
details: Dict[str, Dict[str, Any]] = {}
|
|
|
|
if method in ["surf", "orb", "brisk"]:
|
|
extractor = extractors.get(method)
|
|
if extractor is None:
|
|
extractor = LocalFeatureExtractor(method)
|
|
kp_query, des_query = extractor.detect_and_compute(crop_gray)
|
|
|
|
for sku_id, tmpl in self.db.templates.items():
|
|
kp_tmpl = tmpl["features"][method]["keypoints"]
|
|
des_tmpl = tmpl["features"][method]["descriptors"]
|
|
|
|
good_matches = self.match_descriptors(
|
|
des_query, des_tmpl, extractor.norm_type, ratio_thresh=0.75
|
|
)
|
|
inlier_count, inliers, _ = self.compute_geometric_inliers(
|
|
kp_query, kp_tmpl, good_matches
|
|
)
|
|
|
|
# Score based on geometric inliers & match ratio
|
|
denom = max(1, min(len(kp_query), len(kp_tmpl)))
|
|
inlier_ratio = inlier_count / denom
|
|
score = inlier_count * 2.0 + inlier_ratio * 100.0
|
|
|
|
scores[sku_id] = score
|
|
details[sku_id] = {
|
|
"inliers": inlier_count,
|
|
"inlier_matches": inliers,
|
|
"total_good": len(good_matches),
|
|
"kp_query": kp_query,
|
|
"kp_tmpl": kp_tmpl,
|
|
"score": score
|
|
}
|
|
|
|
elif method == "hybrid":
|
|
# Paper's hybrid strategy combining SURF, ORB, and BRISK
|
|
sub_methods = ["surf", "orb", "brisk"]
|
|
weights = {"surf": 0.45, "orb": 0.25, "brisk": 0.30}
|
|
|
|
# Pre-compute features for query crop across all 3 extractors
|
|
query_features = {}
|
|
for sm in sub_methods:
|
|
kp_q, des_q = extractors[sm].detect_and_compute(crop_gray)
|
|
query_features[sm] = (kp_q, des_q)
|
|
|
|
for sku_id, tmpl in self.db.templates.items():
|
|
composite_score = 0.0
|
|
sku_sub_details = {}
|
|
|
|
for sm in sub_methods:
|
|
kp_q, des_q = query_features[sm]
|
|
kp_t = tmpl["features"][sm]["keypoints"]
|
|
des_t = tmpl["features"][sm]["descriptors"]
|
|
norm_t = extractors[sm].norm_type
|
|
|
|
matches = self.match_descriptors(des_q, des_t, norm_t, ratio_thresh=0.75)
|
|
inliers, inlier_list, _ = self.compute_geometric_inliers(kp_q, kp_t, matches)
|
|
|
|
denom = max(1, min(len(kp_q), len(kp_t)))
|
|
sub_score = inliers * 1.5 + (inliers / denom) * 80.0
|
|
composite_score += weights[sm] * sub_score
|
|
|
|
sku_sub_details[sm] = {
|
|
"inliers": inliers,
|
|
"matches": len(matches),
|
|
"kp_q_count": len(kp_q),
|
|
"kp_t_count": len(kp_t),
|
|
"sub_score": round(sub_score, 2)
|
|
}
|
|
|
|
scores[sku_id] = composite_score
|
|
details[sku_id] = {
|
|
"score": composite_score,
|
|
"sub_details": sku_sub_details,
|
|
"kp_query": query_features["surf"][0],
|
|
"kp_tmpl": tmpl["features"]["surf"]["keypoints"]
|
|
}
|
|
else:
|
|
raise ValueError(f"Unknown method: {method}")
|
|
|
|
return scores, details
|
|
|
|
def score_crop_fast(
|
|
self,
|
|
crop_bgr: np.ndarray,
|
|
shortlist: int = 5
|
|
) -> Tuple[Dict[str, float], Dict[str, Dict[str, Any]]]:
|
|
"""
|
|
ORB-shortlist then hybrid-score the top candidates only.
|
|
|
|
Same scoring formula as 'hybrid', but the expensive SURF/ORB/BRISK
|
|
descriptor matching runs on `shortlist` templates instead of all of them.
|
|
Every other template scores 0.0, so decide_identity() still ranks the
|
|
whole catalog. This is a live-mode speed path; validate it against
|
|
'hybrid' with bench_engine.py --compare before trusting it.
|
|
"""
|
|
if crop_bgr is None or crop_bgr.size == 0:
|
|
return {}, {}
|
|
crop_gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY)
|
|
extractors = self.thread_extractors()
|
|
|
|
orb = extractors["orb"]
|
|
kp_orb, des_orb = orb.detect_and_compute(crop_gray)
|
|
orb_scores: Dict[str, float] = {}
|
|
for sku_id, tmpl in self.db.templates.items():
|
|
kp_t = tmpl["features"]["orb"]["keypoints"]
|
|
des_t = tmpl["features"]["orb"]["descriptors"]
|
|
matches = self.match_descriptors(des_orb, des_t, orb.norm_type, ratio_thresh=0.75)
|
|
inliers, _, _ = self.compute_geometric_inliers(kp_orb, kp_t, matches)
|
|
denom = max(1, min(len(kp_orb), len(kp_t)))
|
|
orb_scores[sku_id] = inliers * 1.5 + (inliers / denom) * 80.0
|
|
|
|
candidates = sorted(orb_scores.items(), key=lambda kv: kv[1], reverse=True)[:shortlist]
|
|
|
|
scores: Dict[str, float] = {sku_id: 0.0 for sku_id in self.db.templates}
|
|
details: Dict[str, Dict[str, Any]] = {}
|
|
sub_methods = ["surf", "orb", "brisk"]
|
|
weights = {"surf": 0.45, "orb": 0.25, "brisk": 0.30}
|
|
query_features = {}
|
|
for sm in sub_methods:
|
|
kp_q, des_q = extractors[sm].detect_and_compute(crop_gray)
|
|
query_features[sm] = (kp_q, des_q)
|
|
|
|
for sku_id, _ in candidates:
|
|
tmpl = self.db.templates[sku_id]
|
|
composite = 0.0
|
|
sku_sub_details = {}
|
|
for sm in sub_methods:
|
|
kp_q, des_q = query_features[sm]
|
|
kp_t = tmpl["features"][sm]["keypoints"]
|
|
des_t = tmpl["features"][sm]["descriptors"]
|
|
matches = self.match_descriptors(des_q, des_t, extractors[sm].norm_type, ratio_thresh=0.75)
|
|
inliers, _, _ = self.compute_geometric_inliers(kp_q, kp_t, matches)
|
|
denom = max(1, min(len(kp_q), len(kp_t)))
|
|
sub_score = inliers * 1.5 + (inliers / denom) * 80.0
|
|
composite += weights[sm] * sub_score
|
|
sku_sub_details[sm] = {
|
|
"inliers": inliers,
|
|
"matches": len(matches),
|
|
"kp_q_count": len(kp_q),
|
|
"kp_t_count": len(kp_t),
|
|
"sub_score": round(sub_score, 2),
|
|
}
|
|
scores[sku_id] = composite
|
|
details[sku_id] = {
|
|
"score": composite,
|
|
"sub_details": sku_sub_details,
|
|
"kp_query": query_features["surf"][0],
|
|
"kp_tmpl": tmpl["features"]["surf"]["keypoints"],
|
|
}
|
|
return scores, details
|
|
|
|
def classify_crop(
|
|
self,
|
|
crop_bgr: np.ndarray,
|
|
method: str = "hybrid"
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Classifies a product crop against all SKU templates.
|
|
Methods supported: 'surf', 'orb', 'brisk', 'hybrid', 'hybrid_fast'.
|
|
"""
|
|
if crop_bgr is None or crop_bgr.size == 0:
|
|
return {"status": "error", "message": "Empty crop image"}
|
|
|
|
method = method.lower()
|
|
if method == "hybrid_fast":
|
|
scores, details = self.score_crop_fast(crop_bgr)
|
|
elif method in ["surf", "orb", "brisk", "hybrid"]:
|
|
scores, details = self.score_crop(crop_bgr, method=method)
|
|
else:
|
|
return {"status": "error", "message": f"Unknown method: {method}"}
|
|
|
|
ranked = decide_identity(scores, details=details, method=method)
|
|
return ranked
|
|
|
|
def generate_match_visualization(
|
|
self,
|
|
crop_bgr: np.ndarray,
|
|
matched_sku: str,
|
|
method: str = "surf",
|
|
max_matches: int = 30
|
|
) -> Optional[np.ndarray]:
|
|
"""Draws visual matching lines between crop and matched template."""
|
|
if matched_sku not in self.db.templates:
|
|
return None
|
|
|
|
tmpl = self.db.templates[matched_sku]
|
|
crop_gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY)
|
|
eval_method = method if method in ["surf", "orb", "brisk"] else "surf"
|
|
|
|
extractor = self.thread_extractors()[eval_method]
|
|
kp_q, des_q = extractor.detect_and_compute(crop_gray)
|
|
kp_t = tmpl["features"][eval_method]["keypoints"]
|
|
des_t = tmpl["features"][eval_method]["descriptors"]
|
|
|
|
matches = self.match_descriptors(des_q, des_t, extractor.norm_type, ratio_thresh=0.75)
|
|
_, inliers, _ = self.compute_geometric_inliers(kp_q, kp_t, matches)
|
|
|
|
matches_to_draw = inliers[:max_matches] if inliers else matches[:max_matches]
|
|
|
|
vis = cv2.drawMatches(
|
|
crop_bgr, kp_q,
|
|
tmpl["image"], kp_t,
|
|
matches_to_draw, None,
|
|
matchColor=(0, 255, 0),
|
|
singlePointColor=(0, 0, 255),
|
|
flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS
|
|
)
|
|
return vis
|