From 02c0fd1f4fc044020ced9529619b709c35f873ed Mon Sep 17 00:00:00 2001 From: kiyo9w Date: Thu, 24 Sep 2026 22:19:57 +0900 Subject: [PATCH] feat: see the tags we cannot read, so "no tag" means no tag Until now a tag reached the rules only if its text could be read. A tag that was there but unreadable was indistinguishable from a missing one, so a missing-tag claim rested on the reader's failures rather than on the shelf - the false violation that made an earlier run report 106 of them. Two measurements shaped the detector: - The tags are printed ON the rail: a red brick-patterned strip running the width of the shelf. The rail is therefore findable without reading anything, and it is where tags must be looked for. - On the rail, a tag is a run of columns carrying paper (bright, unsaturated) against the brick. Projecting onto x finds 27 tags on add/image6.jpg where connected components found 9, because tag text fragments a component at every printed line. The pipeline now merges the two views: a tag whose text was read keeps its fields, a tag that was only seen is reported unreadable with its box. Measured on add/image6.jpg: unreadable tags 8 -> 17, "no tag" 94 -> 85, and the per-row hints became specific - "tags detected for 8 of 16 facings in this row". That last number is also a domain fact worth keeping: this store prints one tag per product, not per facing, so facings are the wrong denominator for a missing-tag rule. The coverage bar keeps the rules silent until that is fixed, which is the honest behaviour in the meantime. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- recognization/code/tag_cards.py | 169 ++++++++++++++++++++++++++++++++ recognization/code/tags.py | 47 +++++++++ 2 files changed, 216 insertions(+) diff --git a/recognization/code/tag_cards.py b/recognization/code/tag_cards.py index 0a91ffd..3606d26 100644 --- a/recognization/code/tag_cards.py +++ b/recognization/code/tag_cards.py @@ -412,6 +412,175 @@ def card_from_lines(lines: Sequence[OcrLine]) -> TagCard: return TagCard(box=card_box(lines), fields=parse_tag_fields(lines), lines=list(lines)) +# --------------------------------------------------------------------------- finding unreadable tags + + +def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, int, int]]: + """Find the shelf's tag rails. + + In the stores in these images the price tags are printed *on* the rail: a + red brick-patterned strip that runs the width of the shelf, one tag per + product column. The rail is therefore the place to look for tags, and its + brick red is what makes it findable without reading anything. + """ + import cv2 + import numpy as np + + hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) + hue, sat, val = cv2.split(hsv) + brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8) + row_fraction = brick.mean(axis=1) + height, width = image.shape[:2] + rails: List[Tuple[int, int, int, int]] = [] + start: Optional[int] = None + for y, fraction in enumerate(row_fraction): + if fraction >= min_fraction and start is None: + start = y + elif fraction < min_fraction and start is not None: + if y - start >= 20: + rails.append((0, start, width, y)) + start = None + if start is not None and height - start >= 20: + rails.append((0, start, width, height)) + return rails + + +def find_tag_blocks_on_rail( + image: Any, + rail: Tuple[int, int, int, int], + min_height_ratio: float = 0.35, +) -> List[Tuple[int, int, int, int]]: + """Find the tags printed on a rail: bright text blocks against the brick. + + Every facing has a tag, so this is what separates "this product has no tag" + from "this tag could not be read" - the distinction the compliance rules need + before they accuse a shelf of anything. + """ + import cv2 + import numpy as np + + x1, y1, x2, y2 = rail + strip = image[y1:y2, x1:x2] + if strip.size == 0: + return [] + hsv = cv2.cvtColor(strip, cv2.COLOR_BGR2HSV) + hue, sat, val = cv2.split(hsv) + # The tag's text block is paper: brighter and much less saturated than the + # brick behind it. Tag text splits the paper into pieces, so the tag is found + # by projecting onto x (a tag is a run of columns that carry paper) rather + # than by connected components, which fragment at every printed line. + paper = ((val > 120) & (sat < 145)).astype(np.uint8) + column_fraction = paper.mean(axis=0) + rail_height = y2 - y1 + blocks: List[Tuple[int, int, int, int]] = [] + start: Optional[int] = None + for index, fraction in enumerate(column_fraction): + if fraction >= 0.18 and start is None: + start = index + elif fraction < 0.18 and start is not None: + blocks.append((start, index)) + start = None + if start is not None: + blocks.append((start, len(column_fraction))) + + tags: List[Tuple[int, int, int, int]] = [] + for start_x, end_x in blocks: + if end_x - start_x < 40: + continue + window = paper[:, start_x:end_x] + rows = np.where(window.mean(axis=1) >= 0.15)[0] + if len(rows) < min_height_ratio * rail_height: + continue + tags.append((x1 + start_x, y1 + int(rows.min()), x1 + end_x, y1 + int(rows.max()))) + return tags + + +def learn_header_template(image: Any, cards: Sequence[TagCard]) -> Optional[Any]: + """Learn what this store's tag header looks like, from a tag we could read. + + A tag whose text cannot be read still has to be *found*, or "no tag here" and + "I could not read this tag" become the same answer and the compliance rules + have nothing honest to say. The header is the one part of a tag that repeats + identically across the shelf, so a single readable tag teaches the detector + what to look for - no per-chain template to maintain. + """ + import cv2 + + for card in cards: + for line in card.lines: + if not HEADER_PATTERN.search(line.text or ""): + continue + x1, y1, x2, y2 = line.box + if x2 - x1 < 40 or y2 - y1 < 8: + continue + height, width = image.shape[:2] + # The header bar is wider than the text on it (measured: a 79 px text + # line sits on a ~196 px bar), so the template is padded out to the bar + # - a bare text template matches far fewer tags. + pad_x = max(6, int((x2 - x1) * 0.75)) + pad_y = max(2, int((y2 - y1) * 0.35)) + x1, x2 = max(0, x1 - pad_x), min(width, x2 + pad_x) + y1, y2 = max(0, y1 - pad_y), min(height, y2 + pad_y) + template = image[y1:y2, x1:x2] + if template.size: + return template + return None + + +def find_tags_by_template( + image: Any, + template: Any, + region: Optional[Tuple[int, int, int, int]] = None, + threshold: float = 0.55, + scales: Sequence[float] = (0.72, 0.82, 0.92, 1.0, 1.1, 1.2, 1.32), +) -> List[Tuple[int, int, int, int]]: + """Find tag headers across an image (or one region) by matching the learned header. + + Returns tag boxes: the header plus the text block beneath it, which is where + the name, the JAN and the price are printed. + """ + import cv2 + import numpy as np + + if template is None or not getattr(template, "size", 0): + return [] + height, width = image.shape[:2] + x1, y1, x2, y2 = region or (0, 0, width, height) + x1, y1 = max(0, x1), max(0, y1) + x2, y2 = min(width, x2), min(height, y2) + crop = image[y1:y2, x1:x2] + if crop.size == 0: + return [] + grey = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY) + hits: List[Tuple[float, int, int, int, int]] = [] + for scale in scales: + scaled = template if scale == 1.0 else cv2.resize( + template, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA + ) + if scaled.shape[0] >= grey.shape[0] or scaled.shape[1] >= grey.shape[1]: + continue + response = cv2.matchTemplate(grey, cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY), cv2.TM_CCOEFF_NORMED) + ys, xs = np.where(response >= threshold) + for x, y in zip(xs, ys): + hits.append((float(response[y, x]), int(x), int(y), scaled.shape[1], scaled.shape[0])) + hits.sort(reverse=True) + kept: List[Tuple[float, int, int, int, int]] = [] + for hit in hits: + _, x, y, w, h = hit + if all(abs(x - kx) > w * 0.5 or abs(y - ky) > h for _, kx, ky, _, _ in kept): + kept.append(hit) + boxes = [] + for _, x, y, w, h in kept: + # The tag continues below its header: the text block is about five header + # heights tall and a little wider than the header itself. + tag_x1 = x1 + x - 4 + tag_y1 = y1 + y - 4 + tag_x2 = min(width, tag_x1 + int(w * 1.30)) + tag_y2 = min(height, tag_y1 + int(h * 5.0)) + boxes.append((tag_x1, tag_y1, tag_x2, tag_y2)) + return boxes + + # --------------------------------------------------------------------------- reading diff --git a/recognization/code/tags.py b/recognization/code/tags.py index f4f2efd..4ec375b 100644 --- a/recognization/code/tags.py +++ b/recognization/code/tags.py @@ -37,6 +37,8 @@ from tag_cards import ( # noqa: E402 TagFields as TagFieldsModel, best_tag_rows, card_from_lines, + find_rails, + find_tag_blocks_on_rail, group_lines_into_cards, read_card_lines, ) @@ -472,6 +474,7 @@ class TagService: for line in result.lines ] candidates = self._card_candidates(image, lines, band, scale_bias) + candidates.extend(self._visual_candidates(image, band, candidates)) with self._lock: self._pass_id += 1 pass_id = self._pass_id @@ -529,6 +532,50 @@ class TagService: ) return cards + def _visual_candidates( + self, + image: Any, + band: RailBand, + known: Sequence[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """Tags that can be seen on the rail but whose text could not be read. + + Without this, "this product has no price tag" and "I could not read the + tag" are the same answer, and a missing-tag claim would be built on the + reader's failures rather than on the shelf. A tag found here is reported + with an empty reading, so the rules can tell the two apart. + """ + try: + rails = find_rails(image) + except Exception: # a vision helper must never break the frame loop + return [] + band_box = band.box + rail = None + best_overlap = 0 + for candidate in rails: + overlap = min(band_box[3], candidate[3]) - max(band_box[1], candidate[1]) + if overlap > best_overlap: + best_overlap, rail = overlap, candidate + if rail is None or best_overlap <= 0: + return [] + try: + blocks = find_tag_blocks_on_rail(image, rail) + except Exception: + return [] + extra: List[Dict[str, Any]] = [] + for block in blocks: + if any(_iou(block, item["box"]) > 0.25 for item in known): + continue + extra.append( + { + "box": list(block), + "lines": [], + "fields": TagFields(), + "band_index": band.row_index, + } + ) + return extra + def _band_settled(self, row_index: int) -> bool: row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index] if not row_tags: