fix: restore reliable authenticated live recognition
This commit is contained in:
kiyo9w
parent
0e23425ef0
commit
934615b593
40 changed files with 2556 additions and 820 deletions
|
|
@ -251,6 +251,7 @@ class RecognitionEngine:
|
|||
# The embedding gallery is built on first use (it loads a model and every
|
||||
# enrolled view), and cached as False when it cannot be built at all.
|
||||
self._gallery: Any = None
|
||||
self._gallery_lock = threading.Lock()
|
||||
workers = self.settings.classify_workers or max(2, (os.cpu_count() or 4) - 2)
|
||||
self.classify_workers = int(workers)
|
||||
self._executor = classify_executor or ThreadPoolExecutor(
|
||||
|
|
@ -417,23 +418,22 @@ class RecognitionEngine:
|
|||
return "cosine" if (self.settings.identity_backend or "matcher").lower() == "gallery" else "inlier"
|
||||
|
||||
def score_crop(self, crop: Any, method: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Score one crop against the catalog, in whatever scale this engine uses.
|
||||
"""Score one crop. Gallery never falls back to the matcher.
|
||||
|
||||
The live session calls this per track. With the gallery backend it must go
|
||||
through the gallery, or a live scan would use the matcher while the still path
|
||||
used the enrolled store - measured: the matcher accepts look-alikes and cannot
|
||||
say "unknown", which is exactly what a real shelf needs it to do.
|
||||
A missing gallery returns no scores (unknown), not inlier counts. Mixing
|
||||
those scales is how a live scan accepted a look-alike the still path rejected.
|
||||
"""
|
||||
if self.identity_scale == "cosine":
|
||||
gallery = self.identity_gallery()
|
||||
if gallery:
|
||||
match = gallery.match(crop)
|
||||
if match.rejected or not match.sku_id:
|
||||
return {}
|
||||
scores = {match.sku_id: match.similarity}
|
||||
if match.runner_up:
|
||||
scores[match.runner_up] = match.runner_up_similarity
|
||||
return {str(sku): float(score) for sku, score in scores.items()}
|
||||
if not gallery:
|
||||
return {}
|
||||
match = gallery.match(crop)
|
||||
if match.rejected or not match.sku_id:
|
||||
return {}
|
||||
scores = {match.sku_id: match.similarity}
|
||||
if match.runner_up:
|
||||
scores[match.runner_up] = match.runner_up_similarity
|
||||
return {str(sku): float(score) for sku, score in scores.items()}
|
||||
method = (method or self.settings.live_method).lower()
|
||||
if method not in CLASSIFY_METHODS:
|
||||
raise UnknownMethod(method)
|
||||
|
|
@ -445,27 +445,41 @@ class RecognitionEngine:
|
|||
|
||||
def identity_gallery(self) -> Any:
|
||||
"""The embedding gallery, loaded once (None when unavailable)."""
|
||||
if self._gallery is False:
|
||||
return None
|
||||
if self._gallery is not None:
|
||||
return self._gallery
|
||||
from gallery import ProductGallery
|
||||
with self._gallery_lock:
|
||||
if self._gallery is False:
|
||||
return None
|
||||
if self._gallery is not None:
|
||||
return self._gallery
|
||||
from gallery import ProductGallery
|
||||
|
||||
gallery = ProductGallery(
|
||||
self.settings.stock_dir,
|
||||
[product["sku_id"] for product in self.catalog()],
|
||||
threshold=self.settings.gallery_threshold,
|
||||
head_threshold=self.settings.gallery_head_threshold,
|
||||
image_size=self.settings.gallery_image_size,
|
||||
)
|
||||
try:
|
||||
loaded = gallery.load()
|
||||
except Exception as exc: # torch/timm missing, no weights, no network
|
||||
LOGGER.warning("identity gallery unavailable: %s", exc)
|
||||
loaded = False
|
||||
self._gallery = gallery if loaded else False
|
||||
if loaded:
|
||||
LOGGER.info("identity gallery: %d views over %d SKUs on %s",
|
||||
gallery.view_count(), len(self.settings.stock_dir and gallery.sku_ids), gallery.device)
|
||||
return self._gallery
|
||||
gallery = ProductGallery(
|
||||
self.settings.stock_dir,
|
||||
[product["sku_id"] for product in self.catalog()],
|
||||
threshold=self.settings.gallery_threshold,
|
||||
head_threshold=self.settings.gallery_head_threshold,
|
||||
image_size=self.settings.gallery_image_size,
|
||||
)
|
||||
try:
|
||||
loaded = gallery.load()
|
||||
except Exception as exc: # torch/timm missing, no weights, no network
|
||||
LOGGER.warning("identity gallery unavailable: %s", exc)
|
||||
loaded = False
|
||||
# False means tried and failed. Do not retry and do not use the matcher.
|
||||
self._gallery = gallery if loaded else False
|
||||
if loaded:
|
||||
LOGGER.info(
|
||||
"identity gallery: %d views over %d SKUs on %s",
|
||||
gallery.view_count(),
|
||||
len(gallery.sku_ids),
|
||||
gallery.device,
|
||||
)
|
||||
else:
|
||||
LOGGER.warning("identity gallery unavailable; gallery requests stay unrecognized")
|
||||
return gallery if loaded else None
|
||||
|
||||
def classify_boxes(
|
||||
self,
|
||||
|
|
@ -493,8 +507,8 @@ class RecognitionEngine:
|
|||
|
||||
if (self.settings.identity_backend or "matcher").lower() == "gallery":
|
||||
gallery = self.identity_gallery()
|
||||
if gallery:
|
||||
return self._classify_with_gallery(gallery, ordered, crops)
|
||||
# No matcher fallback: an unavailable gallery is unrecognized, not a different model.
|
||||
return self._classify_with_gallery(gallery, ordered, crops)
|
||||
if method == "hybrid_fast":
|
||||
futures = [self._executor.submit(self.matcher.score_crop_fast, crop) for crop in crops]
|
||||
else:
|
||||
|
|
@ -516,26 +530,47 @@ class RecognitionEngine:
|
|||
return results
|
||||
|
||||
def inspect(self, image: Any, box: Sequence[int], method: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Full single-crop result: decision, per-method details, crop, visualization."""
|
||||
from identity import decide_identity
|
||||
"""Full single-crop result, using the same backend as recognize/live."""
|
||||
from identity import decide_identity, decision_for_match
|
||||
|
||||
crop = self.crop(image, box)
|
||||
if self.identity_scale == "cosine":
|
||||
gallery = self.identity_gallery()
|
||||
match = gallery.match(crop) if gallery else None
|
||||
# match() already applied the gallery's own threshold. Ranking again
|
||||
# with the published cosine gates must not invent a second line.
|
||||
decision = decision_for_match(match)
|
||||
return self._inspect_payload(box, "gallery", decision, crop, visualize=False)
|
||||
|
||||
method = (method or self.settings.live_method).lower()
|
||||
if method not in CLASSIFY_METHODS:
|
||||
raise UnknownMethod(method)
|
||||
crop = self.crop(image, box)
|
||||
if method == "hybrid_fast":
|
||||
scores, details = self.matcher.score_crop_fast(crop)
|
||||
visual_method = "hybrid"
|
||||
else:
|
||||
scores, details = self.matcher.score_crop(crop, method=method)
|
||||
visual_method = method
|
||||
decision = decide_identity(scores, details=details, method=method)
|
||||
identity = identity_payload(decision)
|
||||
return self._inspect_payload(box, method, decision, crop, visual_method=visual_method)
|
||||
|
||||
def _inspect_payload(
|
||||
self,
|
||||
box: Sequence[int],
|
||||
method: str,
|
||||
decision: Dict[str, Any],
|
||||
crop: Any,
|
||||
*,
|
||||
visual_method: Optional[str] = None,
|
||||
visualize: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
identity = identity_payload(decision)
|
||||
top_details = decision.get("details") or {}
|
||||
sub_details = None
|
||||
if isinstance(top_details, dict):
|
||||
raw = top_details.get("sub_details")
|
||||
if raw:
|
||||
sub_details = {k: dict(v) for k, v in raw.items()}
|
||||
sub_details = {key: dict(value) for key, value in raw.items()}
|
||||
|
||||
product = None
|
||||
visualization = None
|
||||
|
|
@ -547,16 +582,16 @@ class RecognitionEngine:
|
|||
elif identity["candidates"]:
|
||||
visualized_sku = identity["candidates"][0]["sku_id"]
|
||||
is_candidate = True
|
||||
if visualized_sku:
|
||||
if visualize and visualized_sku and visual_method:
|
||||
try:
|
||||
visualization = self.matcher.generate_match_visualization(
|
||||
crop, visualized_sku, method=method if method != "hybrid_fast" else "hybrid"
|
||||
crop, visualized_sku, method=visual_method
|
||||
)
|
||||
except Exception:
|
||||
visualization = None
|
||||
|
||||
return {
|
||||
"box": [int(v) for v in box],
|
||||
"box": [int(value) for value in box],
|
||||
"method": method,
|
||||
"identity": identity,
|
||||
"sub_details": sub_details,
|
||||
|
|
@ -724,48 +759,22 @@ class RecognitionEngine:
|
|||
unseen store photos, the matcher labelled milk cartons as mayonnaise and its
|
||||
score could not be thresholded to stop it.
|
||||
"""
|
||||
from identity import decision_for_match
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
for box, crop in zip(ordered, crops):
|
||||
try:
|
||||
match = gallery.match(crop)
|
||||
except Exception as exc: # a bad crop must not fail the batch
|
||||
match = None
|
||||
LOGGER.debug("gallery match failed: %s", exc)
|
||||
if match is None or match.rejected:
|
||||
similarity = float(match.similarity) if match is not None else 0.0
|
||||
results.append({
|
||||
"detection_id": int(box["id"]),
|
||||
"box": list(box["box"]),
|
||||
"identity": {
|
||||
"state": "unrecognized",
|
||||
"sku_id": None,
|
||||
"identity_level": "rejected",
|
||||
"confidence": round(similarity, 3),
|
||||
"score": round(similarity, 3),
|
||||
"candidates": [],
|
||||
"attempts": 1,
|
||||
"method": "gallery",
|
||||
"message": "no enrolled product is close enough",
|
||||
},
|
||||
})
|
||||
continue
|
||||
runner_up = (
|
||||
{"sku_id": match.runner_up, "score": round(match.runner_up_similarity, 3)}
|
||||
if match.runner_up else {}
|
||||
)
|
||||
match = None
|
||||
if gallery is not None:
|
||||
try:
|
||||
match = gallery.match(crop)
|
||||
except Exception as exc: # a bad crop must not fail the batch
|
||||
match = None
|
||||
LOGGER.debug("gallery match failed: %s", exc)
|
||||
decision = decision_for_match(match)
|
||||
results.append({
|
||||
"detection_id": int(box["id"]),
|
||||
"box": list(box["box"]),
|
||||
"identity": {
|
||||
"state": "recognized",
|
||||
"sku_id": match.sku_id,
|
||||
"identity_level": "accepted",
|
||||
"confidence": round(match.similarity, 3),
|
||||
"score": round(match.similarity, 3),
|
||||
"candidates": [runner_up] if runner_up else [],
|
||||
"attempts": 1,
|
||||
"method": "gallery",
|
||||
},
|
||||
"identity": identity_payload(decision),
|
||||
})
|
||||
return results
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue