Initial commit for push folder

recognization/code/__init__.py

@@ -0,0 +1 @@
+"""Supermarket Shelf Recognition Package."""

recognization/code/benchmark.py

@@ -0,0 +1,103 @@
+"""
+Benchmark Suite for Supermarket Product Classification
+Compares SURF, ORB, BRISK, and Hybrid methods following
+MDPI Electronics 2023, 12(17), 3640.
+"""
+
+import os
+import sys
+import time
+import cv2
+import numpy as np
+from typing import Dict, List, Any
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from feature_matcher import ProductFeatureMatcher
+
+
+def run_benchmark(stock_dir: str = "stock", num_runs: int = 3):
+    print("=" * 70)
+    print("RETAIL SHELF PRODUCT RECOGNITION - ALGORITHM BENCHMARK")
+    print("Methods Evaluated: SURF | ORB | BRISK | HYBRID (Paper Method)")
+    print("=" * 70)
+
+    matcher = ProductFeatureMatcher(stock_dir=stock_dir)
+    templates = matcher.db.templates
+
+    if not templates:
+        print("[Error] No templates found in stock directory.")
+        return
+
+    methods = ["surf", "orb", "brisk", "hybrid"]
+    results = {m: {"latencies": [], "inliers": [], "keypoints_query": []} for m in methods}
+
+    # Test images: We will evaluate using crops synthesized with slight transformations
+    # (scale, rotation, noise) as well as the template images themselves
+    print(f"\nEvaluating across {len(templates)} catalog SKUs (repeat count: {num_runs})...\n")
+
+    for sku_id, tmpl in templates.items():
+        original_img = tmpl["image"]
+        h, w = original_img.shape[:2]
+
+        # Apply synthetic shelf condition (rotation by 10 deg, resize by 0.85)
+        M = cv2.getRotationMatrix2D((w / 2, h / 2), 10, 0.85)
+        query_img = cv2.warpAffine(original_img, M, (w, h))
+
+        for method in methods:
+            times = []
+            for _ in range(num_runs):
+                t0 = time.perf_counter()
+                res = matcher.classify_crop(query_img, method=method)
+                t1 = time.perf_counter()
+                times.append((t1 - t0) * 1000.0)  # ms
+
+            avg_latency = np.mean(times)
+            results[method]["latencies"].append(avg_latency)
+
+            # Record inliers/score
+            if res.get("status") == "success":
+                top_sku = res["best_sku"]
+                score = res["score"]
+                results[method]["inliers"].append(score)
+
+    # Summarize metrics
+    summary_rows = []
+    for method in methods:
+        latencies = results[method]["latencies"]
+        scores = results[method]["inliers"]
+
+        avg_lat = np.mean(latencies) if latencies else 0.0
+        p95_lat = np.percentile(latencies, 95) if latencies else 0.0
+        avg_score = np.mean(scores) if scores else 0.0
+
+        # Paper characteristics notes
+        notes = {
+            "surf": "Scale & rotation robust; Hessian gradient features",
+            "orb": "Ultra fast binary descriptor; FAST + BRIEF",
+            "brisk": "Scale-space AGAST; circular pattern descriptor",
+            "hybrid": "Paper fusion (SURF+ORB+BRISK); optimal accuracy/speed"
+        }[method]
+
+        summary_rows.append([
+            method.upper(),
+            f"{avg_lat:.2f} ms",
+            f"{p95_lat:.2f} ms",
+            f"{avg_score:.1f}",
+            notes
+        ])
+
+    headers = ["Method", "Avg Latency (ms)", "P95 Latency (ms)", "Avg Match Score", "Characteristics / Paper Findings"]
+    print("\n" + "-" * 105)
+    print(f"{headers[0]:<10} | {headers[1]:<18} | {headers[2]:<18} | {headers[3]:<16} | {headers[4]}")
+    print("-" * 105)
+    for row in summary_rows:
+        print(f"{row[0]:<10} | {row[1]:<18} | {row[2]:<18} | {row[3]:<16} | {row[4]}")
+    print("-" * 105)
+    print("\n[Benchmark Complete] All methods operational.")
+
+
+if __name__ == "__main__":
+    run_benchmark()

recognization/code/detector.py

@@ -0,0 +1,284 @@
+"""
+Object Detection Module using YOLO
+Handles inference, bounding box extraction, and spatial hit-testing for shelf images.
+"""
+
+import os
+import cv2
+import numpy as np
+from typing import List, Dict, Optional, Tuple, Any
+
+import torch
+import torch.nn as nn
+try:
+    import ultralytics.nn.modules.block as _B
+    import ultralytics.nn.modules as _M
+
+    if not hasattr(_B, "C3k"):
+        class C3k(_B.C3):
+            def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, k=3):
+                super().__init__(c1, c2, n, shortcut, g, e)
+                c_ = int(c2 * e)
+                self.m = nn.Sequential(*(_B.Bottleneck(c_, c_, shortcut, g, k=((k, k), (k, k)), e=1.0) for _ in range(n)))
+
+        class C3k2(_B.C2f):
+            def __init__(self, c1, c2, n=1, c3k=False, e=0.5, g=1, shortcut=True):
+                super().__init__(c1, c2, n, shortcut, g, e)
+                self.m = nn.ModuleList(
+                    C3k(self.c, self.c, 2, shortcut, g) if c3k else _B.Bottleneck(self.c, self.c, shortcut, g) for _ in range(n)
+                )
+
+        class Attention(nn.Module):
+            def __init__(self, dim, num_heads=8, attn_ratio=0.5):
+                super().__init__()
+                self.num_heads = num_heads
+                self.head_dim = dim // num_heads
+                self.key_dim = int(self.head_dim * attn_ratio)
+                self.scale = self.key_dim ** -0.5
+                nh_kd = self.key_dim * num_heads
+                h = dim + nh_kd * 2
+                self.qkv = _B.Conv(dim, h, 1, act=False)
+                self.proj = _B.Conv(dim, dim, 1, act=False)
+                self.pe = _B.Conv(dim, dim, 3, 1, g=dim, act=False)
+
+            def forward(self, x):
+                b, c, h, w = x.shape
+                n = h * w
+                qkv = self.qkv(x)
+                q, k, v = qkv.view(b, self.num_heads, self.key_dim * 2 + self.head_dim, n).split(
+                    [self.key_dim, self.key_dim, self.head_dim], dim=2
+                )
+                attn = (q.transpose(-2, -1) @ k) * self.scale
+                attn = attn.softmax(dim=-1)
+                x = (v @ attn.transpose(-2, -1)).view(b, c, h, w) + self.pe(x)
+                return self.proj(x)
+
+        class PSABlock(nn.Module):
+            def __init__(self, c, attn_ratio=0.5, num_heads=4, shortcut=True):
+                super().__init__()
+                self.attn = Attention(c, attn_ratio=attn_ratio, num_heads=num_heads)
+                self.ffn = nn.Sequential(_B.Conv(c, c * 2, 1), _B.Conv(c * 2, c, 1, act=False))
+                self.add = shortcut
+
+            def forward(self, x):
+                x = x + self.attn(x) if self.add else self.attn(x)
+                x = x + self.ffn(x) if self.add else self.ffn(x)
+                return x
+
+        class C2PSA(nn.Module):
+            def __init__(self, c1, c2, n=1, e=0.5):
+                super().__init__()
+                assert c1 == c2
+                self.c = int(c1 * e)
+                self.cv1 = _B.Conv(c1, 2 * self.c, 1, 1)
+                self.cv2 = _B.Conv(2 * self.c, c1, 1)
+                self.m = nn.Sequential(*(PSABlock(self.c) for _ in range(n)))
+
+            def forward(self, x):
+                a, b = self.cv1(x).split((self.c, self.c), dim=1)
+                b = self.m(b)
+                return self.cv2(torch.cat((a, b), 1))
+
+        _B.C3k = C3k
+        _B.C3k2 = C3k2
+        _B.C2PSA = C2PSA
+        _B.PSABlock = PSABlock
+        _B.Attention = Attention
+
+        _M.C3k = C3k
+        _M.C3k2 = C3k2
+        _M.C2PSA = C2PSA
+        _M.PSABlock = PSABlock
+        _M.Attention = Attention
+except Exception as _e:
+    print(f"[Warning] YOLO26 patch not applied: {_e}")
+
+from ultralytics import YOLO
+
+
+class ShelfProductDetector:
+    """Wraps the YOLO detector for supermarket shelf products."""
+
+    def __init__(self, weights_path: str = "runs/detect/train/weights/best.pt"):
+        self.weights_path = weights_path
+        if not os.path.exists(weights_path):
+            raise FileNotFoundError(f"YOLO weights not found at: {weights_path}")
+
+        print(f"[Detector] Loading YOLO model from {weights_path}...")
+        self.model = YOLO(weights_path)
+        print("[Detector] Model loaded successfully.")
+
+    def detect(
+        self,
+        image_input: Any,
+        conf_threshold: float = 0.25,
+        iou_threshold: float = 0.45
+    ) -> Dict[str, Any]:
+        """
+        Runs object detection on a shelf image.
+        image_input can be a filepath (str) or a numpy BGR array.
+        """
+        if isinstance(image_input, str):
+            if not os.path.exists(image_input):
+                raise FileNotFoundError(f"Image not found at: {image_input}")
+            image_bgr = cv2.imread(image_input)
+        else:
+            image_bgr = image_input
+
+        if image_bgr is None:
+            return {"status": "error", "message": "Failed to read image", "boxes": []}
+
+        h, w = image_bgr.shape[:2]
+        results = self.model(image_bgr, conf=conf_threshold, iou=iou_threshold, verbose=False)
+
+        boxes_list = []
+        if results and len(results) > 0:
+            boxes_data = results[0].boxes
+            for idx, box in enumerate(boxes_data):
+                xyxy = box.xyxy[0].cpu().numpy()
+                x1, y1, x2, y2 = map(int, xyxy)
+
+                # Clamp to image boundaries
+                x1, y1 = max(0, x1), max(0, y1)
+                x2, y2 = min(w, x2), min(h, y2)
+
+                conf = float(box.conf[0].cpu().numpy())
+                cls_id = int(box.cls[0].cpu().numpy()) if box.cls is not None else 0
+
+                boxes_list.append({
+                    "id": idx,
+                    "box": [x1, y1, x2, y2],
+                    "confidence": round(conf, 4),
+                    "class_id": cls_id,
+                    "width": x2 - x1,
+                    "height": y2 - y1,
+                    "center": [(x1 + x2) // 2, (y1 + y2) // 2]
+                })
+
+        return {
+            "status": "success",
+            "image_shape": (h, w),
+            "total_detected": len(boxes_list),
+            "boxes": boxes_list
+        }
+
+    def find_box_at_point(
+        self,
+        x: int,
+        y: int,
+        boxes: List[Dict[str, Any]],
+        tolerance_px: int = 40
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Finds the bounding box enclosing the click (x, y).
+        If clicked on overlapping boxes, selects the one with minimum distance to center.
+        If clicked slightly outside, selects the nearest box within tolerance_px.
+        """
+        enclosing_boxes = []
+        for b_info in boxes:
+            x1, y1, x2, y2 = b_info["box"]
+            if x1 <= x <= x2 and y1 <= y <= y2:
+                # Calculate distance from click to center
+                cx, cy = b_info["center"]
+                dist = np.sqrt((x - cx) ** 2 + (y - cy) ** 2)
+                enclosing_boxes.append((dist, b_info))
+
+        if enclosing_boxes:
+            # Return box closest to click center
+            enclosing_boxes.sort(key=lambda item: item[0])
+            return enclosing_boxes[0][1]
+
+        # Check nearest within tolerance
+        nearest_box = None
+        min_dist = float("inf")
+        for b_info in boxes:
+            x1, y1, x2, y2 = b_info["box"]
+            # Distance to rectangle
+            dx = max(x1 - x, 0, x - x2)
+            dy = max(y1 - y, 0, y - y2)
+            dist = np.sqrt(dx ** 2 + dy ** 2)
+            if dist < min_dist and dist <= tolerance_px:
+                min_dist = dist
+                nearest_box = b_info
+
+        return nearest_box
+
+    def crop_roi(self, image_bgr: np.ndarray, box: List[int]) -> np.ndarray:
+        """Crops a product ROI bounding box from the shelf image."""
+        x1, y1, x2, y2 = box
+        h, w = image_bgr.shape[:2]
+        x1, y1 = max(0, x1), max(0, y1)
+        x2, y2 = min(w, x2), min(h, y2)
+        return image_bgr[y1:y2, x1:x2].copy()
+
+    def draw_detections(
+        self,
+        image_bgr: np.ndarray,
+        boxes: List[Dict[str, Any]],
+        selected_box_id: Optional[int] = None,
+        draw_labels: bool = False
+    ) -> np.ndarray:
+        """
+        Draws bounding boxes on the shelf image.
+        By default (draw_labels=False), ONLY bounding boxes are drawn:
+        no class names, no confidence numbers.
+        """
+        annotated = image_bgr.copy()
+        for b in boxes:
+            x1, y1, x2, y2 = b["box"]
+            is_selected = (selected_box_id is not None and b["id"] == selected_box_id)
+
+            color = (0, 0, 255) if is_selected else (0, 255, 0)
+            thickness = 3 if is_selected else 2
+
+            # Draw ONLY bounding box
+            cv2.rectangle(annotated, (x1, y1), (x2, y2), color, thickness)
+
+            if draw_labels:
+                label = f"#{b['id']} ({b['confidence']:.2f})"
+                if is_selected:
+                    label = f"SELECTED #{b['id']}"
+                cv2.putText(
+                    annotated,
+                    label,
+                    (x1, max(15, y1 - 6)),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    0.5,
+                    color,
+                    2
+                )
+
+        return annotated
+
+    def detect_empty_slots(
+        self,
+        image_shape: Tuple[int, int],
+        boxes: List[Dict[str, Any]],
+        min_gap_ratio: float = 0.75
+    ) -> List[Dict[str, Any]]:
+        """
+        Detects empty shelf slots / out-of-stock gaps along shelf rows.
+        Uses planogram geometric row-clustering and architectural pillar exclusion.
+        """
+        from empty_shelf_detector import EmptyShelfDetector
+        empty_detector = EmptyShelfDetector(min_gap_ratio=min_gap_ratio)
+        return empty_detector.detect_empty_slots(image_shape, boxes)
+
+    def draw_empty_slots(
+        self,
+        image_bgr: np.ndarray,
+        empty_slots: List[Dict[str, Any]],
+        box_color: Tuple[int, int, int] = (0, 0, 255),
+        thickness: int = 3,
+        draw_fill: bool = True
+    ) -> np.ndarray:
+        """Draws RED bounding boxes around empty shelf spaces with semi-transparent fill."""
+        from empty_shelf_detector import EmptyShelfDetector
+        empty_detector = EmptyShelfDetector()
+        return empty_detector.draw_empty_slots(
+            image_bgr,
+            empty_slots,
+            box_color=box_color,
+            thickness=thickness,
+            draw_fill=draw_fill
+        )

recognization/code/download_stock.py

@@ -0,0 +1,41 @@
+import os
+import sys
+import io
+import urllib.request
+from PIL import Image
+
+# Ensure stdout handles utf-8 cleanly
+if sys.platform.startswith('win'):
+    sys.stdout.reconfigure(encoding='utf-8')
+
+urls = {
+    'sku_011': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_apple@2x_v2.png', '野菜生活100 アップルサラダ'),
+    'sku_012': ('https://www.kagome.co.jp/products/brand/ys100/regular/img/img_products_lemon@2x_v2.png', '野菜生活100 レモンサラダ'),
+    'sku_013': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/green-product-pic_2.png', '野菜生活100 Smoothie グリーンスムージー'),
+    'sku_014': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/orange-product-pic_2.png', '野菜生活100 Smoothie ビタミンスムージー'),
+    'sku_015': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/yellow-product-pic_2.png', '野菜生活100 Smoothie バナナ＆アーモンドmix'),
+    'sku_016': ('https://www.kagome.co.jp/products/brand/ys100/yssmoothie/img/red-product-pic_2.png', '野菜生活100 Smoothie アサイー＆ベリーmix')
+}
+
+os.makedirs('stock', exist_ok=True)
+
+for sku, (url, name) in urls.items():
+    print(f'Fetching {sku}: {name} from {url}...')
+    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            data = resp.read()
+            img = Image.open(io.BytesIO(data))
+            if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
+                bg = Image.new('RGB', img.size, (255, 255, 255))
+                if img.mode != 'RGBA':
+                    img = img.convert('RGBA')
+                bg.paste(img, mask=img.split()[3])
+                img = bg
+            else:
+                img = img.convert('RGB')
+            out_path = f'stock/{sku}.jpeg'
+            img.save(out_path, 'JPEG', quality=95)
+            print(f'Successfully saved {sku}: {name} -> {out_path} ({img.size})')
+    except Exception as e:
+        print(f'Error downloading {sku} ({url}): {e}')

recognization/code/empty_shelf_detector.py

@@ -0,0 +1,286 @@
+"""
+Empty Shelf / Out-of-Stock (OOS) Slot Detector
+Algorithm: Planogram Geometric Row-Clustering & Gap Analysis
+Methodology:
+1. Clusters YOLO product bounding boxes into horizontal shelf tiers/rows.
+2. Filters out vertical architectural dividers / central pillars.
+3. Identifies horizontal gaps exceeding expected product pitch.
+4. Computes precise red bounding boxes for empty shelf slots.
+"""
+
+import cv2
+import numpy as np
+from typing import List, Dict, Any, Tuple, Optional
+
+
+class EmptyShelfDetector:
+    def __init__(
+        self,
+        min_gap_ratio: float = 0.75,
+        min_row_products: int = 4,
+        row_height_tol: float = 0.35,
+        pillar_margin: int = 25
+    ):
+        """
+        Args:
+            min_gap_ratio: Minimum ratio of gap width to median item width in row (default 0.75).
+            min_row_products: Minimum products required to establish a valid shelf row.
+            row_height_tol: Vertical clustering tolerance relative to item height.
+            pillar_margin: Buffer zone around detected vertical pillars/dividers.
+        """
+        self.min_gap_ratio = min_gap_ratio
+        self.min_row_products = min_row_products
+        self.row_height_tol = row_height_tol
+        self.pillar_margin = pillar_margin
+
+    def cluster_rows(self, boxes: List[List[int]]) -> List[List[List[int]]]:
+        """Clusters bounding boxes [x1, y1, x2, y2] into horizontal shelf rows."""
+        if not boxes:
+            return []
+
+        # Sort by vertical center
+        sorted_boxes = sorted(boxes, key=lambda b: (b[1] + b[3]) / 2)
+        rows: List[List[List[int]]] = []
+
+        for b in sorted_boxes:
+            yc = (b[1] + b[3]) / 2
+            placed = False
+            for r in rows:
+                ryc = float(np.mean([(x[1] + x[3]) / 2 for x in r]))
+                rh = float(np.mean([x[3] - x[1] for x in r]))
+                if abs(yc - ryc) < self.row_height_tol * rh:
+                    r.append(b)
+                    placed = True
+                    break
+            if not placed:
+                rows.append([b])
+
+        # Filter rows with sufficient products and sort each row left-to-right
+        valid_rows = []
+        for r in rows:
+            if len(r) >= self.min_row_products:
+                r.sort(key=lambda b: b[0])
+                valid_rows.append(r)
+
+        return valid_rows
+
+    def detect_vertical_pillars(
+        self,
+        rows: List[List[List[int]]],
+        image_width: int
+    ) -> List[Tuple[int, int]]:
+        """
+        Identifies structural vertical shelf dividers (pillars) across rows.
+        A pillar exhibits consistent gaps across multiple vertical shelf rows.
+        """
+        if len(rows) < 2:
+            return []
+
+        # Count gap intersections across rows in 10px bins
+        bin_size = 10
+        num_bins = int(np.ceil(image_width / bin_size))
+        gap_votes = np.zeros(num_bins, dtype=int)
+
+        for row in rows:
+            for j in range(len(row) - 1):
+                x1 = row[j][2]
+                x2 = row[j + 1][0]
+                gap_w = x2 - x1
+                # Architectural dividers typically span 40px - 180px
+                if 40 <= gap_w <= 180:
+                    start_bin = int(x1 // bin_size)
+                    end_bin = min(num_bins, int(np.ceil(x2 / bin_size)))
+                    gap_votes[start_bin:end_bin] += 1
+
+        # A pillar should have gap votes in at least 50% of the shelf rows
+        min_votes = max(2, int(len(rows) * 0.5))
+        pillars = []
+        in_pillar = False
+        p_start = 0
+
+        for b in range(num_bins):
+            if gap_votes[b] >= min_votes:
+                if not in_pillar:
+                    in_pillar = True
+                    p_start = b * bin_size
+            else:
+                if in_pillar:
+                    in_pillar = False
+                    p_end = b * bin_size
+                    pillars.append((max(0, p_start - self.pillar_margin), min(image_width, p_end + self.pillar_margin)))
+
+        if in_pillar:
+            pillars.append((max(0, p_start - self.pillar_margin), image_width))
+
+        return pillars
+
+    def detect_empty_slots(
+        self,
+        image_shape: Tuple[int, int],
+        product_boxes: List[Any],
+        custom_pillars: Optional[List[Tuple[int, int]]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Detects empty shelf slots from product boxes.
+
+        Args:
+            image_shape: (height, width) of the shelf image.
+            product_boxes: List of bounding boxes [x1, y1, x2, y2] or dicts with 'box' key.
+            custom_pillars: Optional pre-defined pillar intervals [(x1, x2), ...].
+
+        Returns:
+            List of empty slot dictionaries:
+            [
+                {
+                    "id": "empty_1",
+                    "box": [x1, y1, x2, y2],
+                    "width": w,
+                    "height": h,
+                    "missing_items_est": int,
+                    "center": [cx, cy],
+                    "row_index": int
+                }, ...
+            ]
+        """
+        img_h, img_w = image_shape[:2]
+        if not product_boxes:
+            return []
+
+        # Standardize box format
+        boxes: List[List[int]] = []
+        for b in product_boxes:
+            if isinstance(b, dict) and "box" in b:
+                boxes.append([int(v) for v in b["box"]])
+            elif isinstance(b, (list, tuple)) and len(b) >= 4:
+                boxes.append([int(b[0]), int(b[1]), int(b[2]), int(b[3])])
+
+        rows = self.cluster_rows(boxes)
+        if not rows:
+            return []
+
+        # Detect structural vertical pillars/frames
+        pillars = custom_pillars if custom_pillars is not None else self.detect_vertical_pillars(rows, img_w)
+
+        empty_slots: List[Dict[str, Any]] = []
+        slot_counter = 1
+
+        for row_idx, row in enumerate(rows):
+            med_y1 = int(np.median([b[1] for b in row]))
+            med_y2 = int(np.median([b[3] for b in row]))
+            med_w = float(np.median([b[2] - b[0] for b in row]))
+            med_h = med_y2 - med_y1
+
+            for j in range(len(row) - 1):
+                x1 = row[j][2]
+                x2 = row[j + 1][0]
+                gap_w = x2 - x1
+
+                if gap_w < self.min_gap_ratio * med_w:
+                    continue
+
+                # Check if this gap overlaps significantly with a known structural pillar
+                is_pillar = False
+                for px1, px2 in pillars:
+                    overlap_start = max(x1, px1)
+                    overlap_end = min(x2, px2)
+                    if overlap_end > overlap_start:
+                        overlap_len = overlap_end - overlap_start
+                        if overlap_len >= 0.6 * gap_w or (x1 >= px1 and x2 <= px2) or (x1 <= px1 and x2 >= px2):
+                            is_pillar = True
+                            break
+
+                if is_pillar:
+                    continue
+
+                missing_est = max(1, int(round(gap_w / med_w)))
+                cx = int((x1 + x2) / 2)
+                cy = int((med_y1 + med_y2) / 2)
+
+                empty_slots.append({
+                    "id": f"empty_{slot_counter}",
+                    "box": [int(x1), int(med_y1), int(x2), int(med_y2)],
+                    "width": int(gap_w),
+                    "height": int(med_h),
+                    "missing_items_est": missing_est,
+                    "center": [cx, cy],
+                    "row_index": row_idx + 1
+                })
+                slot_counter += 1
+
+        return empty_slots
+
+    def draw_empty_slots(
+        self,
+        image: np.ndarray,
+        empty_slots: List[Dict[str, Any]],
+        box_color: Tuple[int, int, int] = (0, 0, 255),
+        thickness: int = 3,
+        draw_fill: bool = True,
+        draw_label: bool = False
+    ) -> np.ndarray:
+        """
+        Draws red bounding boxes around empty shelf slots.
+
+        Args:
+            image: BGR numpy image.
+            empty_slots: List of empty slot dictionaries.
+            box_color: BGR color (default Red: (0, 0, 255)).
+            thickness: Bounding box line thickness.
+            draw_fill: Whether to render a semi-transparent fill tint.
+            draw_label: Whether to render label text.
+        """
+        annotated = image.copy()
+
+        if draw_fill and empty_slots:
+            overlay = annotated.copy()
+            for slot in empty_slots:
+                x1, y1, x2, y2 = slot["box"]
+                cv2.rectangle(overlay, (x1, y1), (x2, y2), box_color, -1)
+            cv2.addWeighted(overlay, 0.22, annotated, 0.78, 0, annotated)
+
+        for slot in empty_slots:
+            x1, y1, x2, y2 = slot["box"]
+            cv2.rectangle(annotated, (x1, y1), (x2, y2), box_color, thickness)
+            if draw_label:
+                est = slot.get("missing_items_est", 1)
+                text = f"Empty (~{est})"
+                font = cv2.FONT_HERSHEY_SIMPLEX
+                font_scale = 0.5
+                tw, th = cv2.getTextSize(text, font, font_scale, 1)[0]
+                cv2.rectangle(annotated, (x1, max(0, y1 - th - 6)), (x1 + tw + 6, y1), box_color, -1)
+                cv2.putText(annotated, text, (x1 + 3, y1 - 4), font, font_scale, (255, 255, 255), 1, cv2.LINE_AA)
+
+        return annotated
+
+
+def detect_and_draw_empty_slots(
+    image_path: str,
+    detector_weights: str = "runs/detect/train/weights/best.pt",
+    output_path: Optional[str] = "detected_empty_slots.jpg"
+) -> Tuple[np.ndarray, List[Dict[str, Any]]]:
+    """
+    Convenience function: runs YOLO detection, extracts empty shelf slots,
+    and draws red bounding boxes.
+    """
+    from detector import ShelfProductDetector
+
+    image = cv2.imread(image_path)
+    if image is None:
+        raise ValueError(f"Failed to read image at: {image_path}")
+
+    detector = ShelfProductDetector(detector_weights)
+    det_res = detector.detect(image)
+    boxes = det_res.get("boxes", [])
+
+    empty_detector = EmptyShelfDetector()
+    empty_slots = empty_detector.detect_empty_slots(image.shape, boxes)
+
+    # Draw products in existing clean bounding boxes (green)
+    annotated = detector.draw_detections(image, boxes, draw_labels=False)
+    # Draw empty slots in RED
+    annotated = empty_detector.draw_empty_slots(annotated, empty_slots, box_color=(0, 0, 255), thickness=3)
+
+    if output_path:
+        cv2.imwrite(output_path, annotated)
+
+    return annotated, empty_slots

recognization/code/fast_test.py

@@ -0,0 +1,6 @@
+from ultralytics import YOLO
+
+model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt")
+image_path = "/Users/nguyenhongvan/recongnize/data/add/image7.jpg"
+results = model(image_path)
+results[0].show()

recognization/code/feature_matcher.py

@@ -0,0 +1,327 @@
+"""
+Feature Matching & Classification Module
+Implements SURF, ORB, BRISK, and Hybrid matching according to
+the methodology of Electronics 2023, 12(17), 3640.
+"""
+
+import os
+import cv2
+import numpy as np
+from typing import Dict, List, Tuple, Optional, Any
+
+
+class LocalFeatureExtractor:
+    """Encapsulates individual OpenCV local feature algorithms."""
+
+    def __init__(self, method: str = "orb"):
+        self.method = method.lower()
+        self.detector = self._create_detector(self.method)
+        self.norm_type = self._get_norm_type(self.method)
+
+    def _create_detector(self, method: str):
+        if method == "surf":
+            # Check if SURF is supported in the current OpenCV build
+            if hasattr(cv2, "xfeatures2d") and hasattr(cv2.xfeatures2d, "SURF_create"):
+                try:
+                    return cv2.xfeatures2d.SURF_create(hessianThreshold=400, nOctaves=4, nOctaveLayers=3)
+                except Exception as e:
+                    print(f"[Warning] SURF initialization failed: {e}. Falling back to SIFT.")
+            else:
+                print("[Info] SURF not available in this OpenCV build. Using SIFT as high-grade floating point descriptor.")
+            return cv2.SIFT_create(nfeatures=1000)
+
+        elif method == "orb":
+            return cv2.ORB_create(
+                nfeatures=1500,
+                scaleFactor=1.2,
+                nlevels=8,
+                edgeThreshold=31,
+                firstLevel=0,
+                WTA_K=2,
+                scoreType=cv2.ORB_HARRIS_SCORE,
+                patchSize=31,
+                fastThreshold=20
+            )
+
+        elif method == "brisk":
+            return cv2.BRISK_create(thresh=30, octaves=3, patternScale=1.0)
+
+        elif method == "sift":
+            return cv2.SIFT_create(nfeatures=1000)
+
+        else:
+            raise ValueError(f"Unsupported feature extractor method: {method}")
+
+    def _get_norm_type(self, method: str):
+        # Binary descriptors use Hamming distance; floating-point descriptors use L2 distance
+        if method in ["orb", "brisk"]:
+            return cv2.NORM_HAMMING
+        return cv2.NORM_L2
+
+    def detect_and_compute(self, image_gray: np.ndarray) -> Tuple[List[cv2.KeyPoint], Optional[np.ndarray]]:
+        """Detects keypoints and computes their descriptors."""
+        if image_gray is None or image_gray.size == 0:
+            return [], None
+        return self.detector.detectAndCompute(image_gray, None)
+
+
+class TemplateFeatureDatabase:
+    """Precomputes and caches local feature descriptors for all stock templates."""
+
+    def __init__(self, stock_dir: str = "stock"):
+        self.stock_dir = stock_dir
+        self.templates: Dict[str, Dict[str, Any]] = {}
+        self.extractors = {
+            "surf": LocalFeatureExtractor("surf"),
+            "orb": LocalFeatureExtractor("orb"),
+            "brisk": LocalFeatureExtractor("brisk")
+        }
+        self.load_and_cache_templates()
+
+    def load_and_cache_templates(self):
+        """Loads all SKU images and extracts their multi-feature descriptors."""
+        if not os.path.exists(self.stock_dir):
+            return
+
+        image_files = [
+            f for f in os.listdir(self.stock_dir)
+            if f.lower().endswith((".jpeg", ".jpg", ".png"))
+        ]
+
+        print(f"[FeatureDatabase] Pre-caching {len(image_files)} stock SKU templates...")
+        for filename in image_files:
+            sku_id = os.path.splitext(filename)[0]
+            filepath = os.path.join(self.stock_dir, filename)
+            img = cv2.imread(filepath)
+            if img is None:
+                continue
+
+            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+            features = {}
+            for name, extractor in self.extractors.items():
+                kp, des = extractor.detect_and_compute(gray)
+                features[name] = {"keypoints": kp, "descriptors": des}
+
+            self.templates[sku_id] = {
+                "sku_id": sku_id,
+                "filepath": filepath,
+                "image": img,
+                "gray": gray,
+                "features": features
+            }
+        print(f"[FeatureDatabase] Successfully cached {len(self.templates)} templates.")
+
+
+class ProductFeatureMatcher:
+    """
+    Main product classification engine.
+    Matches a query cropped bounding box against cached SKU templates using
+    SURF, ORB, BRISK, or the Paper's Hybrid combination.
+    """
+
+    def __init__(self, stock_dir: str = "stock"):
+        self.db = TemplateFeatureDatabase(stock_dir)
+        self.extractors = self.db.extractors
+
+    def match_descriptors(
+        self,
+        des_query: Optional[np.ndarray],
+        des_template: Optional[np.ndarray],
+        norm_type: int,
+        ratio_thresh: float = 0.75
+    ) -> List[cv2.DMatch]:
+        """Performs k-NN matching with Lowe's ratio test."""
+        if des_query is None or des_template is None:
+            return []
+        if len(des_query) < 2 or len(des_template) < 2:
+            return []
+
+        bf = cv2.BFMatcher(norm_type, crossCheck=False)
+        try:
+            raw_matches = bf.knnMatch(des_query, des_template, k=2)
+        except Exception:
+            return []
+
+        good_matches = []
+        for match_pair in raw_matches:
+            if len(match_pair) == 2:
+                m, n = match_pair
+                if m.distance < ratio_thresh * n.distance:
+                    good_matches.append(m)
+        return good_matches
+
+    def compute_geometric_inliers(
+        self,
+        kp_query: List[cv2.KeyPoint],
+        kp_template: List[cv2.KeyPoint],
+        good_matches: List[cv2.DMatch]
+    ) -> Tuple[int, List[cv2.DMatch], Optional[np.ndarray]]:
+        """Applies RANSAC Homography to verify geometric consistency."""
+        if len(good_matches) < 4:
+            return len(good_matches), good_matches, None
+
+        src_pts = np.float32([kp_query[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2)
+        dst_pts = np.float32([kp_template[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2)
+
+        try:
+            H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
+            if mask is None:
+                return len(good_matches), good_matches, None
+
+            inlier_matches = [good_matches[i] for i in range(len(good_matches)) if mask[i][0] == 1]
+            return len(inlier_matches), inlier_matches, H
+        except Exception:
+            return len(good_matches), good_matches, None
+
+    def classify_crop(
+        self,
+        crop_bgr: np.ndarray,
+        method: str = "hybrid"
+    ) -> Dict[str, Any]:
+        """
+        Classifies a product crop against all SKU templates.
+        Methods supported: 'surf', 'orb', 'brisk', 'hybrid'.
+        """
+        if crop_bgr is None or crop_bgr.size == 0:
+            return {"status": "error", "message": "Empty crop image"}
+
+        crop_gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY)
+        method = method.lower()
+
+        scores: Dict[str, float] = {}
+        details: Dict[str, Dict[str, Any]] = {}
+
+        if method in ["surf", "orb", "brisk"]:
+            extractor = self.extractors.get(method)
+            if extractor is None:
+                extractor = LocalFeatureExtractor(method)
+            kp_query, des_query = extractor.detect_and_compute(crop_gray)
+
+            for sku_id, tmpl in self.db.templates.items():
+                kp_tmpl = tmpl["features"][method]["keypoints"]
+                des_tmpl = tmpl["features"][method]["descriptors"]
+
+                good_matches = self.match_descriptors(
+                    des_query, des_tmpl, extractor.norm_type, ratio_thresh=0.75
+                )
+                inlier_count, inliers, _ = self.compute_geometric_inliers(
+                    kp_query, kp_tmpl, good_matches
+                )
+
+                # Score based on geometric inliers & match ratio
+                denom = max(1, min(len(kp_query), len(kp_tmpl)))
+                inlier_ratio = inlier_count / denom
+                score = inlier_count * 2.0 + inlier_ratio * 100.0
+
+                scores[sku_id] = score
+                details[sku_id] = {
+                    "inliers": inlier_count,
+                    "inlier_matches": inliers,
+                    "total_good": len(good_matches),
+                    "kp_query": kp_query,
+                    "kp_tmpl": kp_tmpl,
+                    "score": score
+                }
+
+        elif method == "hybrid":
+            # Paper's hybrid strategy combining SURF, ORB, and BRISK
+            sub_methods = ["surf", "orb", "brisk"]
+            weights = {"surf": 0.45, "orb": 0.25, "brisk": 0.30}
+
+            # Pre-compute features for query crop across all 3 extractors
+            query_features = {}
+            for sm in sub_methods:
+                kp_q, des_q = self.extractors[sm].detect_and_compute(crop_gray)
+                query_features[sm] = (kp_q, des_q)
+
+            for sku_id, tmpl in self.db.templates.items():
+                composite_score = 0.0
+                sku_sub_details = {}
+
+                for sm in sub_methods:
+                    kp_q, des_q = query_features[sm]
+                    kp_t = tmpl["features"][sm]["keypoints"]
+                    des_t = tmpl["features"][sm]["descriptors"]
+                    norm_t = self.extractors[sm].norm_type
+
+                    matches = self.match_descriptors(des_q, des_t, norm_t, ratio_thresh=0.75)
+                    inliers, inlier_list, _ = self.compute_geometric_inliers(kp_q, kp_t, matches)
+
+                    denom = max(1, min(len(kp_q), len(kp_t)))
+                    sub_score = inliers * 1.5 + (inliers / denom) * 80.0
+                    composite_score += weights[sm] * sub_score
+
+                    sku_sub_details[sm] = {
+                        "inliers": inliers,
+                        "matches": len(matches),
+                        "kp_q_count": len(kp_q),
+                        "kp_t_count": len(kp_t),
+                        "sub_score": round(sub_score, 2)
+                    }
+
+                scores[sku_id] = composite_score
+                details[sku_id] = {
+                    "score": composite_score,
+                    "sub_details": sku_sub_details,
+                    "kp_query": query_features["surf"][0],
+                    "kp_tmpl": tmpl["features"]["surf"]["keypoints"]
+                }
+        else:
+            return {"status": "error", "message": f"Unknown method: {method}"}
+
+        # Rank templates by score
+        if not scores:
+            return {"status": "error", "message": "No templates available"}
+
+        sorted_candidates = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+        top_sku, top_score = sorted_candidates[0]
+        runner_up_sku, runner_up_score = sorted_candidates[1] if len(sorted_candidates) > 1 else (None, 0.0)
+
+        confidence = round(min(1.0, top_score / max(1.0, (top_score + runner_up_score * 0.5))), 4)
+
+        return {
+            "status": "success",
+            "method": method,
+            "best_sku": top_sku,
+            "score": round(top_score, 2),
+            "confidence": confidence,
+            "ranked_candidates": [
+                {"sku_id": s, "score": round(sc, 2)} for s, sc in sorted_candidates[:5]
+            ],
+            "details": details[top_sku]
+        }
+
+    def generate_match_visualization(
+        self,
+        crop_bgr: np.ndarray,
+        matched_sku: str,
+        method: str = "surf",
+        max_matches: int = 30
+    ) -> Optional[np.ndarray]:
+        """Draws visual matching lines between crop and matched template."""
+        if matched_sku not in self.db.templates:
+            return None
+
+        tmpl = self.db.templates[matched_sku]
+        crop_gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY)
+        eval_method = method if method in ["surf", "orb", "brisk"] else "surf"
+
+        extractor = self.extractors[eval_method]
+        kp_q, des_q = extractor.detect_and_compute(crop_gray)
+        kp_t = tmpl["features"][eval_method]["keypoints"]
+        des_t = tmpl["features"][eval_method]["descriptors"]
+
+        matches = self.match_descriptors(des_q, des_t, extractor.norm_type, ratio_thresh=0.75)
+        _, inliers, _ = self.compute_geometric_inliers(kp_q, kp_t, matches)
+
+        matches_to_draw = inliers[:max_matches] if inliers else matches[:max_matches]
+
+        vis = cv2.drawMatches(
+            crop_bgr, kp_q,
+            tmpl["image"], kp_t,
+            matches_to_draw, None,
+            matchColor=(0, 255, 0),
+            singlePointColor=(0, 0, 255),
+            flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS
+        )
+        return vis

recognization/code/gui_app.py

@@ -0,0 +1,92 @@
+"""
+Native Desktop GUI for Supermarket Shelf Product Recognition
+Opens a desktop OpenCV window displaying image.jpg with bounding boxes only.
+Users can click on any product directly on screen to inspect price, stock,
+and feature matching inliers in real-time.
+"""
+
+import os
+import sys
+import cv2
+import numpy as np
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from pipeline import ShelfRecognitionPipeline
+
+
+def main():
+    print("=" * 60)
+    print("STARTING NATIVE DESKTOP INTERFACE")
+    print("Image: image.jpg | Mode: Hybrid (Paper Method)")
+    print("Instructions: Click on ANY product to identify & view info.")
+    print("Press 'q' or 'ESC' on the window to exit.")
+    print("=" * 60)
+
+    pipeline = ShelfRecognitionPipeline()
+    init_res = pipeline.set_active_shelf_image("image.jpg")
+
+    if init_res.get("status") != "success":
+        print("[Error] Failed to load image.jpg")
+        return
+
+    boxes = init_res["boxes"]
+    base_image = pipeline.current_image.copy()
+
+    # Draw ONLY bounding boxes (no class names, no confidence labels)
+    display_image = pipeline.detector.draw_detections(base_image, boxes, draw_labels=False)
+
+    window_name = "Smart Shelf - Click Any Product (Press Q to quit)"
+    cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
+    cv2.resizeWindow(window_name, 1280, 850)
+
+    def on_mouse(event, x, y, flags, param):
+        if event == cv2.EVENT_LBUTTONDOWN:
+            print(f"\n[Click detected at coordinates: ({x}, {y})]")
+            res = pipeline.process_click(x, y, method="hybrid")
+
+            if res.get("status") == "success":
+                prod = res["inventory"]
+                cls_info = res["classification"]
+                box_info = res["target_box"]
+
+                print("-" * 55)
+                print(f"📦 Product Identified : {prod['product_name']}")
+                print(f"🏷️  SKU ID             : {prod['sku_id']}")
+                print(f"💰 Price              : ¥{prod['price']}")
+                print(f"📊 In Stock (On-Shelf): {prod['number_on']} units")
+                print(f"🚚 Received / Sold    : {prod['number_in']} in / {prod['number_out']} out")
+                print(f"📅 Expiration Date    : {prod['expiration_date']}")
+                print(f"🎯 Match Confidence   : {cls_info['confidence'] * 100:.1f}% (Score: {cls_info['score']})")
+                print("-" * 55)
+
+                # Update main window with selected box highlight (in red)
+                selected_display = pipeline.detector.draw_detections(
+                    base_image, boxes, selected_box_id=box_info["id"], draw_labels=False
+                )
+                cv2.imshow(window_name, selected_display)
+
+                # Show visual match comparison window if available
+                if res.get("match_visualization") is not None:
+                    match_win = "Feature Matcher (Shelf Crop vs Stock Template)"
+                    cv2.namedWindow(match_win, cv2.WINDOW_NORMAL)
+                    cv2.resizeWindow(match_win, 800, 400)
+                    cv2.imshow(match_win, res["match_visualization"])
+            else:
+                print(f"[Info] {res.get('message', 'No product found at coordinates.')}")
+
+    cv2.setMouseCallback(window_name, on_mouse)
+
+    while True:
+        cv2.imshow(window_name, display_image)
+        key = cv2.waitKey(20) & 0xFF
+        if key in [ord('q'), ord('Q'), 27]:
+            break
+
+    cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()

recognization/code/inventory.py

@@ -0,0 +1,90 @@
+"""
+Inventory Management Module
+Handles reading and querying product metadata from stock.csv.
+"""
+
+import os
+import sys
+import csv
+from typing import Dict, Optional, Any, List
+
+if sys.platform.startswith('win'):
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+
+class InventoryManager:
+    def __init__(self, stock_dir: str = "stock", csv_filename: str = "stock.csv"):
+        self.stock_dir = stock_dir
+        self.csv_path = os.path.join(stock_dir, csv_filename)
+        self.products: Dict[str, Dict[str, Any]] = {}
+        self.load_inventory()
+
+    def load_inventory(self) -> None:
+        """Loads inventory data from the CSV file."""
+        if not os.path.exists(self.csv_path):
+            print(f"[Warning] Inventory CSV not found at: {self.csv_path}")
+            return
+
+        with open(self.csv_path, mode="r", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                sku_id = row.get("SKU_ID", "").strip()
+                if not sku_id:
+                    continue
+
+                # Locate the corresponding template image in stock_dir
+                local_image = self._find_template_image(sku_id)
+
+                self.products[sku_id] = {
+                    "sku_id": sku_id,
+                    "product_name": row.get("product_name", "").strip(),
+                    "receipt_date": row.get("receipt_date", "").strip(),
+                    "expiration_date": row.get("expiration_date", "").strip(),
+                    "number_in": int(row.get("number_in", 0) or 0),
+                    "number_out": int(row.get("number_out", 0) or 0),
+                    "number_on": int(row.get("number_on", 0) or 0),
+                    "price": float(row.get("price", 0) or 0),
+                    "raw_image_link": row.get("image_link", "").strip(),
+                    "local_image_path": local_image
+                }
+
+    def _find_template_image(self, sku_id: str) -> Optional[str]:
+        """Finds the local image file for the given SKU_ID."""
+        extensions = [".jpeg", ".jpg", ".png"]
+        for ext in extensions:
+            candidate = os.path.join(self.stock_dir, f"{sku_id}{ext}")
+            if os.path.exists(candidate):
+                return candidate
+        return None
+
+    def get_product(self, sku_id: str) -> Optional[Dict[str, Any]]:
+        """Retrieves product information for a given SKU_ID."""
+        return self.products.get(sku_id)
+
+    def list_all_skus(self) -> List[str]:
+        """Returns all available SKU IDs."""
+        return list(self.products.keys())
+
+    def update_stock(self, sku_id: str, quantity_delta: int) -> bool:
+        """Updates the on-shelf quantity (e.g., when purchased or restocked)."""
+        if sku_id in self.products:
+            current = self.products[sku_id]["number_on"]
+            new_val = max(0, current + quantity_delta)
+            self.products[sku_id]["number_on"] = new_val
+            if quantity_delta < 0:
+                self.products[sku_id]["number_out"] += abs(quantity_delta)
+            else:
+                self.products[sku_id]["number_in"] += quantity_delta
+            return True
+        return False
+
+
+if __name__ == "__main__":
+    inv = InventoryManager("stock")
+    print(f"Loaded {len(inv.products)} products.")
+    for sku in inv.list_all_skus():
+        p = inv.get_product(sku)
+        print(f"[{sku}] {p['product_name']} | Price: {p['price']} | In-Stock: {p['number_on']} | Image: {p['local_image_path']}")

recognization/code/pipeline.py

@@ -0,0 +1,181 @@
+"""
+End-to-End Supermarket Shelf Recognition Pipeline
+Integrates:
+1. YOLO object detector for shelf product localization
+2. SURF / ORB / BRISK / Hybrid local feature matcher for classification
+3. Stock CSV inventory lookup for real-time pricing and stock quantities
+"""
+
+import os
+import sys
+import cv2
+import numpy as np
+from typing import Dict, Optional, Any
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from detector import ShelfProductDetector
+from feature_matcher import ProductFeatureMatcher
+from inventory import InventoryManager
+
+
+class ShelfRecognitionPipeline:
+    """Master pipeline managing interactive supermarket shelf recognition."""
+
+    def __init__(
+        self,
+        weights_path: str = "runs/detect/train/weights/best.pt",
+        stock_dir: str = "stock",
+        stock_csv: str = "stock.csv"
+    ):
+        print("[Pipeline] Initializing Inventory Manager...")
+        self.inventory = InventoryManager(stock_dir=stock_dir, csv_filename=stock_csv)
+
+        print("[Pipeline] Initializing Feature Matcher...")
+        self.matcher = ProductFeatureMatcher(stock_dir=stock_dir)
+
+        print("[Pipeline] Initializing Shelf Detector...")
+        self.detector = ShelfProductDetector(weights_path=weights_path)
+
+        self.current_image_path: Optional[str] = None
+        self.current_image: Optional[np.ndarray] = None
+        self.current_detections: list = []
+        self.current_empty_slots: list = []
+
+    def set_active_shelf_image(self, image_input: Any) -> Dict[str, Any]:
+        """Loads and runs detection on a new shelf image."""
+        if isinstance(image_input, str):
+            self.current_image_path = image_input
+            self.current_image = cv2.imread(image_input)
+        else:
+            self.current_image_path = "memory_buffer"
+            self.current_image = image_input
+
+        if self.current_image is None:
+            return {"status": "error", "message": "Invalid or unreadable image"}
+
+        det_result = self.detector.detect(self.current_image)
+        self.current_detections = det_result.get("boxes", [])
+
+        # Detect empty shelf spaces (out-of-stock gaps)
+        self.current_empty_slots = self.detector.detect_empty_slots(
+            self.current_image.shape, self.current_detections
+        )
+
+        return {
+            "status": "success",
+            "image_shape": self.current_image.shape,
+            "total_detected": len(self.current_detections),
+            "boxes": self.current_detections,
+            "empty_slots": self.current_empty_slots
+        }
+
+    def process_click(
+        self,
+        x: int,
+        y: int,
+        method: str = "hybrid",
+        tolerance_px: int = 40
+    ) -> Dict[str, Any]:
+        """
+        Handles user click at coordinate (x, y) on the shelf image:
+        1. Identifies the bounding box
+        2. Crops product ROI
+        3. Classifies via SURF/ORB/BRISK/Hybrid matching
+        4. Queries inventory for price, quantity, and metadata
+        """
+        if self.current_image is None:
+            return {"status": "error", "message": "No active shelf image loaded."}
+
+        target_box = self.detector.find_box_at_point(
+            x, y, self.current_detections, tolerance_px=tolerance_px
+        )
+
+        if target_box is None:
+            # Check if user clicked an empty shelf space
+            for slot in getattr(self, "current_empty_slots", []):
+                sx1, sy1, sx2, sy2 = slot["box"]
+                if (sx1 - 10 <= x <= sx2 + 10) and (sy1 - 10 <= y <= sy2 + 10):
+                    return {
+                        "status": "empty_slot",
+                        "clicked_point": (x, y),
+                        "empty_slot": slot,
+                        "message": f"Chỗ trống trên kệ (Khu vực hàng hết / Row {slot['row_index']}). Ước tính thiếu ~{slot['missing_items_est']} sản phẩm."
+                    }
+
+            return {
+                "status": "not_found",
+                "clicked_point": (x, y),
+                "message": f"No product detected near coordinate ({x}, {y}). Please click inside a product packaging."
+            }
+
+        # Crop ROI
+        crop = self.detector.crop_roi(self.current_image, target_box["box"])
+
+        # Classify with selected method
+        cls_result = self.matcher.classify_crop(crop, method=method)
+        if cls_result.get("status") != "success":
+            return {
+                "status": "classification_failed",
+                "target_box": target_box,
+                "message": cls_result.get("message", "Feature matching failed.")
+            }
+
+        best_sku = cls_result["best_sku"]
+        product_info = self.inventory.get_product(best_sku)
+
+        # Generate match visualization
+        vis_match = self.matcher.generate_match_visualization(crop, best_sku, method=method)
+
+        # Generate annotated shelf image highlighting the selected item
+        annotated_shelf = self.detector.draw_detections(
+            self.current_image, self.current_detections, selected_box_id=target_box["id"]
+        )
+
+        return {
+            "status": "success",
+            "clicked_point": (x, y),
+            "target_box": target_box,
+            "classification": {
+                "method": method,
+                "matched_sku": best_sku,
+                "confidence": cls_result["confidence"],
+                "score": cls_result["score"],
+                "top_candidates": cls_result["ranked_candidates"],
+            },
+            "inventory": product_info,
+            "crop_image": crop,
+            "annotated_shelf": annotated_shelf,
+            "match_visualization": vis_match
+        }
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Test Shelf Recognition Pipeline")
+    parser.add_argument("--image", type=str, default="runs/detect/train/train_batch0.jpg")
+    parser.add_argument("--method", type=str, default="hybrid", choices=["surf", "orb", "brisk", "hybrid"])
+    args = parser.parse_args()
+
+    pipeline = ShelfRecognitionPipeline()
+    print(f"Loading image: {args.image}")
+    init_res = pipeline.set_active_shelf_image(args.image)
+    print(f"Total detected items: {init_res['total_detected']}")
+
+    if init_res['boxes']:
+        # Simulate click on the center of the first detected product
+        first_box = init_res['boxes'][0]
+        cx, cy = first_box['center']
+        print(f"Simulating click on Product #{first_box['id']} at ({cx}, {cy})...")
+        res = pipeline.process_click(cx, cy, method=args.method)
+        print("\n--- Recognition & Inventory Result ---")
+        print(f"Status: {res['status']}")
+        if res['status'] == 'success':
+            prod = res['inventory']
+            print(f"Matched SKU: {res['classification']['matched_sku']} (Score: {res['classification']['score']})")
+            print(f"Product Name: {prod['product_name']}")
+            print(f"Price: {prod['price']} JPY")
+            print(f"In Stock: {prod['number_on']} units (Received: {prod['number_in']}, Sold: {prod['number_out']})")
+            print(f"Expiration Date: {prod['expiration_date']}")

recognization/code/suft.py

@@ -0,0 +1,84 @@
+import cv2
+import matplotlib.pyplot as plt
+import numpy as np
+
+def suft_function(image):
+    image1 = cv2.imread(image)
+    # Convert the training image to RGB
+    training_image = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
+
+    # Convert the training image to gray scale
+    training_gray = cv2.cvtColor(training_image, cv2.COLOR_RGB2GRAY)
+
+    # Create test image by adding Scale Invariance and Rotational Invariance
+    test_image = cv2.pyrDown(training_image)
+    test_image = cv2.pyrDown(test_image)
+    num_rows, num_cols = test_image.shape[:2]
+
+    rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), 30, 1)
+    test_image = cv2.warpAffine(test_image, rotation_matrix, (num_cols, num_rows))
+
+    test_gray = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY)
+
+    # Display traning image and testing image
+    fx, plots = plt.subplots(1, 2, figsize=(20,10))
+
+    plots[0].set_title("Training Image")
+    plots[0].imshow(training_image)
+
+    plots[1].set_title("Testing Image")
+    plots[1].imshow(test_image)
+
+
+    # ## Detect keypoints and Create Descriptor
+
+    surf = cv2.xfeatures2d.SURF_create(800)
+
+    train_keypoints, train_descriptor = surf.detectAndCompute(training_gray, None)
+    test_keypoints, test_descriptor = surf.detectAndCompute(test_gray, None)
+
+    keypoints_without_size = np.copy(training_image)
+    keypoints_with_size = np.copy(training_image)
+
+    cv2.drawKeypoints(training_image, train_keypoints, keypoints_without_size, color = (0, 255, 0))
+
+    cv2.drawKeypoints(training_image, train_keypoints, keypoints_with_size, flags = cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
+
+    # Display image with and without keypoints size
+    fx, plots = plt.subplots(1, 2, figsize=(20,10))
+
+    plots[0].set_title("Train keypoints With Size")
+    plots[0].imshow(keypoints_with_size, cmap='gray')
+
+    plots[1].set_title("Train keypoints Without Size")
+    plots[1].imshow(keypoints_without_size, cmap='gray')
+
+    # Print the number of keypoints detected in the training image
+    print("Number of Keypoints Detected In The Training Image: ", len(train_keypoints))
+
+    # Print the number of keypoints detected in the query image
+    print("Number of Keypoints Detected In The Query Image: ", len(test_keypoints))
+
+
+    # ## Matching Keypoints
+
+    # Create a Brute Force Matcher object.
+    bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck = False)
+
+    # Perform the matching between the SURF descriptors of the training image and the test image
+    matches = bf.match(train_descriptor, test_descriptor)
+
+    # The matches with shorter distance are the ones we want.
+    matches = sorted(matches, key = lambda x : x.distance)
+
+    result = cv2.drawMatches(training_image, train_keypoints, test_gray, test_keypoints, matches, test_gray, flags = 2)
+
+    # Display the best matching points
+    plt.rcParams['figure.figsize'] = [14.0, 7.0]
+    plt.title('Best Matching Points')
+    plt.imshow(result)
+    plt.show()
+
+    # Print total number of matching points between the training and query images
+    print("\nNumber of Matching Keypoints Between The Training and Query Images: ", len(matches))
+    return keypoints_with_size

recognization/code/test.py

@@ -0,0 +1,25 @@
+from ultralytics import YOLO
+import cv2
+
+model = YOLO("/Users/nguyenhongvan/recongnize/runs/detect/train/weights/best.pt")
+
+image_path = "/Users/nguyenhongvan/recongnize/data/add/image9.jpg"
+
+results = model(image_path)
+
+image = cv2.imread(image_path)
+
+for box in results[0].boxes.xyxy.cpu().numpy():
+    x1, y1, x2, y2 = map(int, box)
+
+    cv2.rectangle(
+        image,
+        (x1, y1),
+        (x2, y2),
+        (0, 255, 0),
+        2
+    )
+
+cv2.imshow("Bounding Boxes Only", image)
+cv2.waitKey(0)
+cv2.destroyAllWindows()

recognization/code/test_empty.py

@@ -0,0 +1,20 @@
+import sys
+import os
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from empty_shelf_detector import detect_and_draw_empty_slots
+
+if __name__ == "__main__":
+    image_path = "image.jpg"
+    print(f"Processing empty shelf detection on {image_path}...")
+    annotated, empty_slots = detect_and_draw_empty_slots(
+        image_path,
+        output_path="detected_empty_slots.jpg"
+    )
+    print(f"Detected {len(empty_slots)} empty shelf slots:")
+    for s in empty_slots:
+        print(f"  - Slot [{s['id']}]: Row {s['row_index']}, Box={s['box']}, Width={s['width']}px, Est. Missing Items={s['missing_items_est']}")
+    print(f"Visualization saved to: detected_empty_slots.jpg")

recognization/code/test_image.py

@@ -0,0 +1,89 @@
+"""
+Test Script for image.jpg
+1. Runs YOLO detection on image.jpg.
+2. Draws ONLY bounding box rectangles (no class names, no confidence labels).
+3. Saves output to detected_image.jpg.
+4. Matches the detected bounding boxes against stock templates.
+"""
+
+import os
+import sys
+import cv2
+import numpy as np
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from detector import ShelfProductDetector
+from feature_matcher import ProductFeatureMatcher
+from inventory import InventoryManager
+
+
+def main():
+    image_path = "image.jpg"
+    print(f"[1/4] Loading {image_path}...")
+    image = cv2.imread(image_path)
+    if image is None:
+        print(f"[Error] Failed to read {image_path}")
+        return
+
+    h, w = image.shape[:2]
+    print(f"      Image resolution: {w}x{h}")
+
+    print("[2/4] Detecting products using YOLO...")
+    detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
+    # You can adjust conf_threshold as needed (default 0.25)
+    det_res = detector.detect(image, conf_threshold=0.25)
+    boxes = det_res.get("boxes", [])
+    print(f"      Total detected items: {len(boxes)}")
+
+    print("[3/4] Drawing bounding boxes ONLY (no class names, no confidence)...")
+    annotated = detector.draw_detections(image, boxes, draw_labels=False)
+    output_path = "detected_image.jpg"
+    cv2.imwrite(output_path, annotated)
+    print(f"      Saved detection visualization to: {output_path}")
+
+    print("[4/4] Matching against stock catalogue (SURF, ORB, BRISK, Hybrid)...")
+    matcher = ProductFeatureMatcher("stock")
+    inv = InventoryManager("stock")
+
+    print("\n--- Stock Templates Present in image.jpg ---")
+    matched_skus = set()
+    sample_matches = []
+
+    # Test top detections to find corresponding stock SKUs
+    for b in boxes:
+        crop = detector.crop_roi(image, b["box"])
+        # Use hybrid method from the paper
+        res = matcher.classify_crop(crop, method="hybrid")
+        if res.get("status") == "success":
+            sku_id = res["best_sku"]
+            score = res["score"]
+            conf = res["confidence"]
+            prod = inv.get_product(sku_id)
+            if prod:
+                matched_skus.add(sku_id)
+                sample_matches.append({
+                    "box_id": b["id"],
+                    "box": b["box"],
+                    "sku_id": sku_id,
+                    "product_name": prod["product_name"],
+                    "price": prod["price"],
+                    "number_on": prod["number_on"],
+                    "score": score,
+                    "confidence": conf
+                })
+
+    # Sort matches by confidence
+    sample_matches.sort(key=lambda x: x["score"], reverse=True)
+
+    print(f"\nSuccessfully matched {len(matched_skus)} distinct SKUs across detected shelf boxes:")
+    for sm in sample_matches[:10]:
+        print(f"Box #{sm['box_id']:<3} -> [{sm['sku_id']}] {sm['product_name']:<30} | Price: ¥{sm['price']:<5} | On-Shelf: {sm['number_on']} | Score: {sm['score']}")
+
+    print("\n[Complete] Tested image.jpg successfully.")
+
+
+if __name__ == "__main__":
+    main()

recognization/code/test_suite.py

@@ -0,0 +1,91 @@
+"""
+Unit and Integration Test Suite
+Validates InventoryManager, ShelfProductDetector, ProductFeatureMatcher, and Pipeline.
+"""
+
+import os
+import sys
+import cv2
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+from inventory import InventoryManager
+from feature_matcher import ProductFeatureMatcher
+from detector import ShelfProductDetector
+from pipeline import ShelfRecognitionPipeline
+
+
+def test_inventory():
+    print("[TEST] Testing InventoryManager...")
+    inv = InventoryManager("stock")
+    assert len(inv.products) >= 10, f"Expected products >= 10 in stock.csv, found {len(inv.products)}"
+    p1 = inv.get_product("sku_001")
+    assert p1 is not None, "sku_001 not found"
+    assert p1["product_name"] == "明治 おいしい牛乳"
+    assert p1["price"] == 258.0
+    assert p1["number_on"] == 73
+    print("  -> InventoryManager test: PASSED")
+
+
+def test_feature_matcher():
+    print("[TEST] Testing ProductFeatureMatcher...")
+    matcher = ProductFeatureMatcher("stock")
+    assert len(matcher.db.templates) >= 7, f"Expected templates >= 7, got {len(matcher.db.templates)}"
+
+    for method in ["hybrid", "orb", "brisk"]:
+        tmpl_img = matcher.db.templates["sku_001"]["image"]
+        res = matcher.classify_crop(tmpl_img, method=method)
+        assert res["status"] == "success", f"Method {method} failed"
+        assert res["best_sku"] == "sku_001", f"Expected sku_001 with {method}, got {res['best_sku']}"
+        print(f"  -> {method.upper()} classification test: PASSED (Score: {res['score']}, Conf: {res['confidence']})")
+
+
+def test_detector():
+    print("[TEST] Testing ShelfProductDetector...")
+    detector = ShelfProductDetector("runs/detect/train/weights/best.pt")
+    sample_shelf = "runs/detect/train/train_batch0.jpg"
+    res = detector.detect(sample_shelf)
+    assert res["status"] == "success", "Detection failed"
+    assert res["total_detected"] > 0, "No boxes detected"
+    print(f"  -> Detector test: PASSED ({res['total_detected']} items localized)")
+
+    first_box = res["boxes"][0]
+    cx, cy = first_box["center"]
+    hit = detector.find_box_at_point(cx, cy, res["boxes"])
+    assert hit is not None, "Hit test failed on center coordinate"
+    assert hit["id"] == first_box["id"], "Hit test returned incorrect box"
+    print("  -> Spatial hit-test: PASSED")
+
+
+def test_pipeline():
+    print("[TEST] Testing End-to-End Pipeline...")
+    pipeline = ShelfRecognitionPipeline(
+        weights_path="runs/detect/train/weights/best.pt",
+        stock_dir="stock"
+    )
+    init_res = pipeline.set_active_shelf_image("runs/detect/train/train_batch0.jpg")
+    assert init_res["status"] == "success"
+    assert len(init_res["boxes"]) > 0
+
+    first_box = init_res["boxes"][0]
+    cx, cy = first_box["center"]
+    click_res = pipeline.process_click(cx, cy, method="hybrid")
+    assert click_res["status"] == "success", f"Click processing failed: {click_res.get('message')}"
+    assert "inventory" in click_res
+    assert click_res["inventory"] is not None
+    print(f"  -> End-to-End Pipeline test: PASSED (Matched: {click_res['classification']['matched_sku']} - {click_res['inventory']['product_name']})")
+
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("STARTING TEST SUITE")
+    print("=" * 60)
+    test_inventory()
+    test_feature_matcher()
+    test_detector()
+    test_pipeline()
+    print("=" * 60)
+    print("ALL TESTS PASSED SUCCESSFULLY!")
+    print("=" * 60)

recognization/code/train.py

@@ -0,0 +1,32 @@
+from ultralytics import YOLO
+
+# Load a model
+model = YOLO("yolo26n.pt")  # load a pretrained model (recommended for training)
+EPOCHS = 10
+IMG_SIZE = 640
+BATCH_SIZE = 8
+WORKERS = 2
+PATIENCE = 5
+SEED = 42
+# Train the model
+results = model.train(data="/Users/nguyenhongvan/recongnize/data/SKU/data_kaggle.yaml", epochs=EPOCHS,
+    imgsz=IMG_SIZE,
+
+    batch=BATCH_SIZE,
+
+    workers=WORKERS,
+
+    patience=PATIENCE,
+
+    cache=True,
+
+    amp=True,
+
+    cos_lr=True,
+
+    optimizer="AdamW",
+
+    seed=SEED,
+
+    exist_ok=True
+)