product-recognition-demo/recognization/docs/research/2026-09-29-shelf-intelligence/collect.py

"""Frozen A-F observations; imports shipped lanes, never edits production.
Run from recognization/: .venv/bin/python docs/research/2026-09-29-shelf-intelligence/collect.py
"""
from pathlib import Path
import sys
import json
import time
from dataclasses import asdict

ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT / 'code'))
import cv2
import numpy as np
from identity_lanes import LocalLane, TextLane, Evidence, _root_sift, _hue_histogram
from ocr import build_ocr_engine
from eval_identity import catalog_hash, file_hash, code_revision, _catalog_rows

OUT = Path(__file__).resolve().parent / 'evidence'


def rows_from_boxes(items):
    """Bottom-aligned shelf rows, using geometry only, no labels or catalog IDs."""
    rows = []
    for item in sorted(items, key=lambda x: x['box'][3]):
        b = item['box']
        candidates = [(abs(b[3] - np.median([x['box'][3] for x in row])), row) for row in rows]
        distance, row = min(candidates, key=lambda x: x[0]) if candidates else (float('inf'), [])
        if distance <= .25 * max(1, b[3] - b[1]):
            row.append(item)
        else:
            rows.append([item])
    return [sorted(row, key=lambda x: (x['box'][0] + x['box'][2]) / 2) for row in rows]


def features(crop, sift):
    gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
    kp, des = sift.detectAndCompute(gray, None)
    small = cv2.resize(crop, (32, 64), interpolation=cv2.INTER_AREA)
    lab = cv2.cvtColor(small, cv2.COLOR_BGR2LAB).astype(float) / 255
    edge = cv2.resize(cv2.Canny(gray, 70, 140), (16, 32)).astype(float).ravel() / 255
    return {'kp': kp, 'des': _root_sift(des), 'hist': _hue_histogram(crop), 'lab': lab,
            'edge': edge, 'shape': gray.shape, 'sharpness': float(cv2.Laplacian(gray, cv2.CV_64F).var())}


def pair_signals(a, b):
    result = {'hue': float(np.minimum(a['hist'], b['hist']).sum()),
              'pixels': float(max(0, 1 - np.abs(a['lab'] - b['lab']).mean() * 3)),
              'logo_pixels': float(max(0, 1 - np.abs(a['lab'][:32] - b['lab'][:32]).mean() * 3)),
              'edge': float(np.dot(a['edge'], b['edge']) / max(1e-6, np.linalg.norm(a['edge']) * np.linalg.norm(b['edge']))),
              'inliers': 0, 'match_fraction': 0., 'layout_cover': 0.}
    if a['des'] is not None and b['des'] is not None and len(b['des']) >= 2:
        pairs = cv2.BFMatcher(cv2.NORM_L2).knnMatch(a['des'], b['des'], k=2)
        good = [x[0] for x in pairs if len(x) == 2 and x[0].distance < .75 * x[1].distance]
        if len(good) >= 4:
            src = np.float32([a['kp'][m.queryIdx].pt for m in good])
            dst = np.float32([b['kp'][m.trainIdx].pt for m in good])
            _, mask = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=.02 * max(b['shape']))
            if mask is not None:
                yes = mask.ravel().astype(bool)
                result['inliers'] = int(yes.sum())
                result['match_fraction'] = result['inliers'] / max(1, min(len(a['kp']), len(b['kp'])))
                if yes.sum() >= 3:
                    result['layout_cover'] = min(float(cv2.contourArea(cv2.convexHull(src[yes])) / np.prod(a['shape'])), float(cv2.contourArea(cv2.convexHull(dst[yes])) / np.prod(b['shape'])))
    return result


def main():
    OUT.mkdir(parents=True, exist_ok=True)
    cv2.setNumThreads(1)
    cv2.setRNGSeed(20260929)
    label_path = ROOT / 'eval/labels/image6.json'
    labels = json.loads(label_path.read_text())
    source = cv2.imread(str(ROOT / labels['image']))
    stock = str(ROOT / 'stock-lopia-clean')
    local = LocalLane(stock)
    text = TextLane.from_catalog(_catalog_rows(stock), stock, build_ocr_engine('vision'))
    sift = cv2.SIFT_create(nfeatures=2000)
    records = [f for f in labels['facings'] if f['label'] != 'exclude']
    report = {'provenance': {'code': code_revision(), 'catalog': catalog_hash(stock), 'labels': file_hash(str(label_path)), 'image': file_hash(str(ROOT / labels['image'])), 'opencv_threads': 1, 'seed': 20260929}, 'signatures': text.signatures, 'resolutions': {}}
    # Contact sheet is visual annotation evidence, not an inference input.
    sheet = np.full((int(np.ceil(len(records)/12))*200, 12*110, 3), 245, np.uint8)
    for n, f in enumerate(records):
        x1,y1,x2,y2 = f['box']; crop = source[y1:y2,x1:x2]
        h,w = crop.shape[:2]; ratio = min(100/w, 170/h)
        tile = cv2.resize(crop, (max(1,int(w*ratio)), max(1,int(h*ratio))))
        x,y = n%12*110,n//12*200
        sheet[y+25:y+25+tile.shape[0],x:x+tile.shape[1]] = tile
        cv2.putText(sheet, str(f['index']), (x+4,y+18), cv2.FONT_HERSHEY_SIMPLEX,.5,(0,0,0),1)
    cv2.imwrite(str(OUT/'facings.jpg'), sheet)
    for resolution, scale in [('full',1.),('live',1280/max(source.shape[:2]))]:
        image = source if scale == 1 else cv2.resize(source, (round(source.shape[1]*scale),round(source.shape[0]*scale)), interpolation=cv2.INTER_AREA)
        obs, appearances = [], {}
        for f in records:
            box = [round(v*scale) for v in f['box']]; x1,y1,x2,y2=box
            crop = image[y1:y2,x1:x2]
            start=time.perf_counter(); ev=local.evidence(crop); local_ms=(time.perf_counter()-start)*1000
            ranked=sorted(ev.items(),key=lambda kv:local.rank_key(kv[1]),reverse=True)
            rank=[(sku,local.confidence(e,ranked[i+1][1] if i+1<len(ranked) else Evidence())) for i,(sku,e) in enumerate(ranked)]
            start=time.perf_counter(); read=text.read(crop); ocr_ms=(time.perf_counter()-start)*1000
            start=time.perf_counter(); appearance=features(crop,sift); features_ms=(time.perf_counter()-start)*1000
            appearances[f['index']]=appearance
            obs.append({'id':f['index'],'box':box,'truth':f['label'],'local':{sku:asdict(e) for sku,e in ev.items()},'rank':rank,'text':text.scores_from_text(read),'read':read,'local_ms':local_ms,'ocr_ms':ocr_ms,'pair_features_ms':features_ms,'sharpness':appearance['sharpness']})
        rows=rows_from_boxes(obs); pairs=[]
        for r,row in enumerate(rows):
            for a,b in zip(row,row[1:]):
                start=time.perf_counter(); signals=pair_signals(appearances[a['id']],appearances[b['id']]); elapsed=(time.perf_counter()-start)*1000
                ab,bb=a['box'],b['box']; aw,ah=ab[2]-ab[0],ab[3]-ab[1]; bw,bh=bb[2]-bb[0],bb[3]-bb[1]
                signals.update({'aspect':min(aw/ah,bw/bh)/max(aw/ah,bw/bh),'height':min(ah,bh)/max(ah,bh),'gap':(bb[0]-ab[2])/max(aw,bw),'bottom':abs(ab[3]-bb[3])/max(ah,bh)})
                from rebuild_catalog import bigrams
                at,bt=bigrams(a['read']),bigrams(b['read'])
                signals['ocr_jaccard']=len(at&bt)/max(1,len(at|bt))
                pairs.append({'a':a['id'],'b':b['id'],'row':r,'signals':signals,'pair_ms':elapsed})
        report['resolutions'][resolution]={'observations':obs,'rows':[[f['id'] for f in row] for row in rows],'pairs':pairs}
        print(resolution, len(obs),'facings', len(rows),'rows', 'local/ocr ms', round(sum(x['local_ms'] for x in obs)),round(sum(x['ocr_ms'] for x in obs)),flush=True)
    (OUT/'observations.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))

if __name__ == '__main__':
    main()