feat: see the tags we cannot read, so "no tag" means no tag
Until now a tag reached the rules only if its text could be read. A tag that was there but unreadable was indistinguishable from a missing one, so a missing-tag claim rested on the reader's failures rather than on the shelf - the false violation that made an earlier run report 106 of them. Two measurements shaped the detector: - The tags are printed ON the rail: a red brick-patterned strip running the width of the shelf. The rail is therefore findable without reading anything, and it is where tags must be looked for. - On the rail, a tag is a run of columns carrying paper (bright, unsaturated) against the brick. Projecting onto x finds 27 tags on add/image6.jpg where connected components found 9, because tag text fragments a component at every printed line. The pipeline now merges the two views: a tag whose text was read keeps its fields, a tag that was only seen is reported unreadable with its box. Measured on add/image6.jpg: unreadable tags 8 -> 17, "no tag" 94 -> 85, and the per-row hints became specific - "tags detected for 8 of 16 facings in this row". That last number is also a domain fact worth keeping: this store prints one tag per product, not per facing, so facings are the wrong denominator for a missing-tag rule. The coverage bar keeps the rules silent until that is fixed, which is the honest behaviour in the meantime. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kiyo9w
parent
8e33b3ebdd
commit
02c0fd1f4f
2 changed files with 216 additions and 0 deletions
|
|
@ -412,6 +412,175 @@ def card_from_lines(lines: Sequence[OcrLine]) -> TagCard:
|
|||
return TagCard(box=card_box(lines), fields=parse_tag_fields(lines), lines=list(lines))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- finding unreadable tags
|
||||
|
||||
|
||||
def find_rails(image: Any, min_fraction: float = 0.25) -> List[Tuple[int, int, int, int]]:
|
||||
"""Find the shelf's tag rails.
|
||||
|
||||
In the stores in these images the price tags are printed *on* the rail: a
|
||||
red brick-patterned strip that runs the width of the shelf, one tag per
|
||||
product column. The rail is therefore the place to look for tags, and its
|
||||
brick red is what makes it findable without reading anything.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
|
||||
hue, sat, val = cv2.split(hsv)
|
||||
brick = (((hue < 14) | (hue > 166)) & (sat > 70) & (val > 70)).astype(np.uint8)
|
||||
row_fraction = brick.mean(axis=1)
|
||||
height, width = image.shape[:2]
|
||||
rails: List[Tuple[int, int, int, int]] = []
|
||||
start: Optional[int] = None
|
||||
for y, fraction in enumerate(row_fraction):
|
||||
if fraction >= min_fraction and start is None:
|
||||
start = y
|
||||
elif fraction < min_fraction and start is not None:
|
||||
if y - start >= 20:
|
||||
rails.append((0, start, width, y))
|
||||
start = None
|
||||
if start is not None and height - start >= 20:
|
||||
rails.append((0, start, width, height))
|
||||
return rails
|
||||
|
||||
|
||||
def find_tag_blocks_on_rail(
|
||||
image: Any,
|
||||
rail: Tuple[int, int, int, int],
|
||||
min_height_ratio: float = 0.35,
|
||||
) -> List[Tuple[int, int, int, int]]:
|
||||
"""Find the tags printed on a rail: bright text blocks against the brick.
|
||||
|
||||
Every facing has a tag, so this is what separates "this product has no tag"
|
||||
from "this tag could not be read" - the distinction the compliance rules need
|
||||
before they accuse a shelf of anything.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
x1, y1, x2, y2 = rail
|
||||
strip = image[y1:y2, x1:x2]
|
||||
if strip.size == 0:
|
||||
return []
|
||||
hsv = cv2.cvtColor(strip, cv2.COLOR_BGR2HSV)
|
||||
hue, sat, val = cv2.split(hsv)
|
||||
# The tag's text block is paper: brighter and much less saturated than the
|
||||
# brick behind it. Tag text splits the paper into pieces, so the tag is found
|
||||
# by projecting onto x (a tag is a run of columns that carry paper) rather
|
||||
# than by connected components, which fragment at every printed line.
|
||||
paper = ((val > 120) & (sat < 145)).astype(np.uint8)
|
||||
column_fraction = paper.mean(axis=0)
|
||||
rail_height = y2 - y1
|
||||
blocks: List[Tuple[int, int, int, int]] = []
|
||||
start: Optional[int] = None
|
||||
for index, fraction in enumerate(column_fraction):
|
||||
if fraction >= 0.18 and start is None:
|
||||
start = index
|
||||
elif fraction < 0.18 and start is not None:
|
||||
blocks.append((start, index))
|
||||
start = None
|
||||
if start is not None:
|
||||
blocks.append((start, len(column_fraction)))
|
||||
|
||||
tags: List[Tuple[int, int, int, int]] = []
|
||||
for start_x, end_x in blocks:
|
||||
if end_x - start_x < 40:
|
||||
continue
|
||||
window = paper[:, start_x:end_x]
|
||||
rows = np.where(window.mean(axis=1) >= 0.15)[0]
|
||||
if len(rows) < min_height_ratio * rail_height:
|
||||
continue
|
||||
tags.append((x1 + start_x, y1 + int(rows.min()), x1 + end_x, y1 + int(rows.max())))
|
||||
return tags
|
||||
|
||||
|
||||
def learn_header_template(image: Any, cards: Sequence[TagCard]) -> Optional[Any]:
|
||||
"""Learn what this store's tag header looks like, from a tag we could read.
|
||||
|
||||
A tag whose text cannot be read still has to be *found*, or "no tag here" and
|
||||
"I could not read this tag" become the same answer and the compliance rules
|
||||
have nothing honest to say. The header is the one part of a tag that repeats
|
||||
identically across the shelf, so a single readable tag teaches the detector
|
||||
what to look for - no per-chain template to maintain.
|
||||
"""
|
||||
import cv2
|
||||
|
||||
for card in cards:
|
||||
for line in card.lines:
|
||||
if not HEADER_PATTERN.search(line.text or ""):
|
||||
continue
|
||||
x1, y1, x2, y2 = line.box
|
||||
if x2 - x1 < 40 or y2 - y1 < 8:
|
||||
continue
|
||||
height, width = image.shape[:2]
|
||||
# The header bar is wider than the text on it (measured: a 79 px text
|
||||
# line sits on a ~196 px bar), so the template is padded out to the bar
|
||||
# - a bare text template matches far fewer tags.
|
||||
pad_x = max(6, int((x2 - x1) * 0.75))
|
||||
pad_y = max(2, int((y2 - y1) * 0.35))
|
||||
x1, x2 = max(0, x1 - pad_x), min(width, x2 + pad_x)
|
||||
y1, y2 = max(0, y1 - pad_y), min(height, y2 + pad_y)
|
||||
template = image[y1:y2, x1:x2]
|
||||
if template.size:
|
||||
return template
|
||||
return None
|
||||
|
||||
|
||||
def find_tags_by_template(
|
||||
image: Any,
|
||||
template: Any,
|
||||
region: Optional[Tuple[int, int, int, int]] = None,
|
||||
threshold: float = 0.55,
|
||||
scales: Sequence[float] = (0.72, 0.82, 0.92, 1.0, 1.1, 1.2, 1.32),
|
||||
) -> List[Tuple[int, int, int, int]]:
|
||||
"""Find tag headers across an image (or one region) by matching the learned header.
|
||||
|
||||
Returns tag boxes: the header plus the text block beneath it, which is where
|
||||
the name, the JAN and the price are printed.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
if template is None or not getattr(template, "size", 0):
|
||||
return []
|
||||
height, width = image.shape[:2]
|
||||
x1, y1, x2, y2 = region or (0, 0, width, height)
|
||||
x1, y1 = max(0, x1), max(0, y1)
|
||||
x2, y2 = min(width, x2), min(height, y2)
|
||||
crop = image[y1:y2, x1:x2]
|
||||
if crop.size == 0:
|
||||
return []
|
||||
grey = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
||||
hits: List[Tuple[float, int, int, int, int]] = []
|
||||
for scale in scales:
|
||||
scaled = template if scale == 1.0 else cv2.resize(
|
||||
template, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA
|
||||
)
|
||||
if scaled.shape[0] >= grey.shape[0] or scaled.shape[1] >= grey.shape[1]:
|
||||
continue
|
||||
response = cv2.matchTemplate(grey, cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY), cv2.TM_CCOEFF_NORMED)
|
||||
ys, xs = np.where(response >= threshold)
|
||||
for x, y in zip(xs, ys):
|
||||
hits.append((float(response[y, x]), int(x), int(y), scaled.shape[1], scaled.shape[0]))
|
||||
hits.sort(reverse=True)
|
||||
kept: List[Tuple[float, int, int, int, int]] = []
|
||||
for hit in hits:
|
||||
_, x, y, w, h = hit
|
||||
if all(abs(x - kx) > w * 0.5 or abs(y - ky) > h for _, kx, ky, _, _ in kept):
|
||||
kept.append(hit)
|
||||
boxes = []
|
||||
for _, x, y, w, h in kept:
|
||||
# The tag continues below its header: the text block is about five header
|
||||
# heights tall and a little wider than the header itself.
|
||||
tag_x1 = x1 + x - 4
|
||||
tag_y1 = y1 + y - 4
|
||||
tag_x2 = min(width, tag_x1 + int(w * 1.30))
|
||||
tag_y2 = min(height, tag_y1 + int(h * 5.0))
|
||||
boxes.append((tag_x1, tag_y1, tag_x2, tag_y2))
|
||||
return boxes
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- reading
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ from tag_cards import ( # noqa: E402
|
|||
TagFields as TagFieldsModel,
|
||||
best_tag_rows,
|
||||
card_from_lines,
|
||||
find_rails,
|
||||
find_tag_blocks_on_rail,
|
||||
group_lines_into_cards,
|
||||
read_card_lines,
|
||||
)
|
||||
|
|
@ -472,6 +474,7 @@ class TagService:
|
|||
for line in result.lines
|
||||
]
|
||||
candidates = self._card_candidates(image, lines, band, scale_bias)
|
||||
candidates.extend(self._visual_candidates(image, band, candidates))
|
||||
with self._lock:
|
||||
self._pass_id += 1
|
||||
pass_id = self._pass_id
|
||||
|
|
@ -529,6 +532,50 @@ class TagService:
|
|||
)
|
||||
return cards
|
||||
|
||||
def _visual_candidates(
|
||||
self,
|
||||
image: Any,
|
||||
band: RailBand,
|
||||
known: Sequence[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Tags that can be seen on the rail but whose text could not be read.
|
||||
|
||||
Without this, "this product has no price tag" and "I could not read the
|
||||
tag" are the same answer, and a missing-tag claim would be built on the
|
||||
reader's failures rather than on the shelf. A tag found here is reported
|
||||
with an empty reading, so the rules can tell the two apart.
|
||||
"""
|
||||
try:
|
||||
rails = find_rails(image)
|
||||
except Exception: # a vision helper must never break the frame loop
|
||||
return []
|
||||
band_box = band.box
|
||||
rail = None
|
||||
best_overlap = 0
|
||||
for candidate in rails:
|
||||
overlap = min(band_box[3], candidate[3]) - max(band_box[1], candidate[1])
|
||||
if overlap > best_overlap:
|
||||
best_overlap, rail = overlap, candidate
|
||||
if rail is None or best_overlap <= 0:
|
||||
return []
|
||||
try:
|
||||
blocks = find_tag_blocks_on_rail(image, rail)
|
||||
except Exception:
|
||||
return []
|
||||
extra: List[Dict[str, Any]] = []
|
||||
for block in blocks:
|
||||
if any(_iou(block, item["box"]) > 0.25 for item in known):
|
||||
continue
|
||||
extra.append(
|
||||
{
|
||||
"box": list(block),
|
||||
"lines": [],
|
||||
"fields": TagFields(),
|
||||
"band_index": band.row_index,
|
||||
}
|
||||
)
|
||||
return extra
|
||||
|
||||
def _band_settled(self, row_index: int) -> bool:
|
||||
row_tags = [tag for tag in self._tags.values() if tag.band_index == row_index]
|
||||
if not row_tags:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue