nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates on Image search using vectors

Browse Source
This commit is contained in:
sriram
2026-09-28 15:44:01 +05:30
parent 6628207810
commit c0489d89d6
16 changed files with 1184 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
app/services/image_match.py
View File

@@ -53,10 +53,12 @@ from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
from app.infrastructure.settings import (
IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
IMAGE_SEARCH_DEFAULT_MIN_SCORE,
IMAGE_SEARCH_DEFAULT_TOP_K,
IMAGE_SEARCH_MAX_FETCH_K,
IMAGE_SEARCH_MAX_TOP_K,
IMAGE_SEARCH_MIN_MARGIN,
)
from app.services.image_embedder import EMBED_DIM
@@ -90,6 +92,8 @@ class ImageSearchResult:
min_score: float = 0.0
query_text: Optional[str] = None
top_k: int = 0
match_confidence: str = "none" # "confirmed" | "low" | "none"
margin: Optional[float] = None # top score minus the best different photo's
# ---------------------------------------------------------------------------
@@ -226,16 +230,102 @@ def text_overlap(words: Set[str], sizes: Set[str], row: Dict[str, Any]) -> float
return _SIZE_WEIGHT * len(sizes & row_sizes) + _WORD_WEIGHT * len(words & row_words)
def size_in_name(sizes: Set[str], row: Dict[str, Any]) -> bool:
"""Whether a pack size the label printed is in the row's OWN name.
`text_overlap` also counts `size_variants`, and on some products every
pack-size row carries the SAME list (all Amul Cream rows say
["90g", "1kg"]), so a "1kg" label ties every sibling and the tie fell to
alphabetical order - "Amul Cream 125 ml" was shown, confirmed, for a
photo of the 1kg pack. The name is per row; this breaks that tie.
"""
if not sizes:
return False
name_sizes = tokens(" ".join([str(row.get("product_name") or ""), str(row.get("title") or "")]))[1]
return bool(sizes & name_sizes)
def rank_key(row: Dict[str, Any]) -> tuple:
"""Best first. Score rounded to 3 dp so siblings sharing a photo tie."""
return (
-round(float(row.get("score", 0.0)), 3),
-float(row.get("text_overlap", 0.0)),
-int(bool(row.get("size_in_name", False))),
str(row.get("product_name") or row.get("title") or "").lower(),
str(row.get("image_id") or ""),
)
# ---------------------------------------------------------------------------
# confidence
# ---------------------------------------------------------------------------
# Pack sizes that share one catalog photo score identically (to pgvector's
# float precision); anything further apart than this is a different picture.
_SAME_PHOTO_EPS = 5e-4
CONFIRMED = "confirmed"
LOW = "low"
NONE = "none"
def image_margin(rows: Sequence[Dict[str, Any]]) -> Optional[float]:
"""How far the best row leads the best row with a DIFFERENT photo.
`rows` are best first by score. Siblings sharing the winner's photo are
skipped: they are the same product in another size, which the label text
separates, not a rival. None when nothing else is in the list - no rival
was seen, which is not the same as a clear lead, so the caller treats it
as unconfirmable only when the score itself is low.
"""
if not rows:
return None
try:
top = float(rows[0].get("score", 0.0))
except (TypeError, ValueError):
return None
for row in rows[1:]:
try:
score = float(row.get("score", 0.0))
except (TypeError, ValueError):
continue
if top - score > _SAME_PHOTO_EPS:
return top - score
return None
def match_confidence(
rows: Sequence[Dict[str, Any]],
min_image_score: Optional[float] = None,
min_margin: Optional[float] = None,
) -> Tuple[str, Optional[float]]:
"""("confirmed" | "low" | "none", margin) for image rows, best first.
Confirmed needs BOTH a score at the identify floor and a lead over the
next different photo. A photo of a card on a phone screen scores ~0.5
against its own product, and a stranger can sit within a few hundredths
of it: shown as the answer, that is how "Dairy Milk Lickables" came back
as someone else's product. A client should offer the list, not pick one,
when this is not "confirmed". The thresholds default to this module's
settings, read at call time so scripts/eval_identify.py can vary them.
"""
if min_image_score is None:
min_image_score = IMAGE_IDENTIFY_MIN_IMAGE_SCORE
if min_margin is None:
min_margin = IMAGE_SEARCH_MIN_MARGIN
if not rows:
return NONE, None
margin = image_margin(rows)
try:
top = float(rows[0].get("score", 0.0))
except (TypeError, ValueError):
return LOW, margin
if top < min_image_score:
return LOW, margin
if margin is not None and margin < min_margin:
return LOW, margin
return CONFIRMED, margin
# ---------------------------------------------------------------------------
# the search
# ---------------------------------------------------------------------------
@@ -345,7 +435,11 @@ def search_by_vector(
words, sizes = tokens(label)
for row in rows:
row["text_overlap"] = text_overlap(words, sizes, row)
row["size_in_name"] = size_in_name(sizes, row)
rows.sort(key=rank_key)
# Judged on every candidate, not the top_k shown: with top_k=1 the rival
# would otherwise never be seen.
confidence, margin = match_confidence(rows)
winners = _hydrate(rows[:top_k])
return ImageSearchResult(
@@ -356,4 +450,6 @@ def search_by_vector(
min_score=min_score,
query_text=label,
top_k=top_k,
match_confidence=confidence,
margin=margin,
)

133
app/services/image_search_log.py Normal file
View File

@@ -0,0 +1,133 @@
"""One log line per search-by-photo request, and - when asked - the request itself.
WHY
---
A colleague photographed the "Cadbury Dairy Milk Lickables" card and got
"Milk Toned"; "Aachi Sambar Powder 100g" came back as Sakthi's "Sambar powder
50g". Replayed on the server, both photos rank the right product first, so
the difference is in what the app SENDS - the vector (is its on-device model
the same as ours?), whether any OCR `text` came with it, which route it hit.
None of that was visible. This module makes it visible:
* `record()` logs, for every /search/image*, /search/identify request: the
route, the text, brand, detected brand, a short hash and the norm of the
vector, the top three (brand, name, score) and the lead over the best
different photo. Always on; it is one INFO line.
* With IMAGE_SEARCH_CAPTURE_DIR set, the whole request - the 1024 floats,
the fields, the photo when there was one - is also written there, for
`python -m scripts.replay_image_query`. Off by default: these routes are
public. The newest IMAGE_SEARCH_CAPTURE_MAX requests are kept.
Never raises: a diagnostic must not fail a search.
"""
from __future__ import annotations
import hashlib
import json
import logging
import math
import struct
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence
from app.infrastructure import settings
logger = logging.getLogger(__name__)
_TEXT_LOG_CHARS = 120
def vector_fingerprint(vector: Optional[Sequence[float]]) -> Optional[str]:
"""10 hex characters identifying a vector: equal for the same float32 values."""
if vector is None:
return None
packed = struct.pack(f"<{len(vector)}f", *[float(v) for v in vector])
return hashlib.sha1(packed).hexdigest()[:10]
def _top(rows: Sequence[Dict[str, Any]], n: int = 3) -> List[Dict[str, Any]]:
out = []
for row in list(rows)[:n]:
try:
score = round(float(row.get("score", 0.0)), 4)
except (TypeError, ValueError):
score = None
out.append({
"brand": row.get("brand") or row.get("brand_table"),
"product_name": row.get("product_name") or row.get("title"),
"image_id": row.get("image_id"),
"score": score,
})
return out
def _prune(folder: Path, keep: int) -> None:
files = sorted(folder.glob("*.json"), key=lambda p: p.stat().st_mtime)
for stale in files[:max(0, len(files) - keep)]:
for sibling in folder.glob(stale.stem + ".*"):
sibling.unlink(missing_ok=True)
def record(
route: str,
*,
vector: Optional[Sequence[float]],
text: Optional[str],
brand: Optional[str],
category: Optional[str],
top_k: int,
rows: Sequence[Dict[str, Any]],
detected_brand: Optional[str] = None,
scoped_to_brand: bool = False,
match_confidence: Optional[str] = None,
margin: Optional[float] = None,
matched_by: Optional[str] = None,
fallback_reason: Optional[str] = None,
text_fallback: Optional[bool] = None,
photo: Optional[bytes] = None,
) -> None:
"""Log the request and its answer; save it when capture is on."""
try:
norm = math.sqrt(sum(float(v) * float(v) for v in vector)) if vector is not None else None
top = _top(rows)
logger.info(
"[IMAGE_SEARCH] route=%s text=%r brand=%r detected=%r scoped=%s vec=%s norm=%s "
"matched_by=%s reason=%s confidence=%s margin=%s top=%s",
route, (text or "")[:_TEXT_LOG_CHARS] or None, brand, detected_brand, scoped_to_brand,
vector_fingerprint(vector), None if norm is None else round(norm, 4),
matched_by, fallback_reason, match_confidence,
None if margin is None else round(margin, 4),
[(t["brand"], t["product_name"], t["score"]) for t in top],
)
folder_name = settings.IMAGE_SEARCH_CAPTURE_DIR
if not folder_name:
return
folder = Path(folder_name)
folder.mkdir(parents=True, exist_ok=True)
stem = time.strftime("%Y%m%dT%H%M%S") + "_" + uuid.uuid4().hex[:6]
payload = {
"route": route,
"received_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"request": {
"vector": None if vector is None else [float(v) for v in vector],
"text": text, "brand": brand, "category": category, "top_k": top_k,
"text_fallback": text_fallback,
},
"vector_fingerprint": vector_fingerprint(vector),
"answer": {
"matched_by": matched_by, "fallback_reason": fallback_reason,
"match_confidence": match_confidence, "margin": margin,
"detected_brand": detected_brand, "scoped_to_brand": scoped_to_brand, "top": top,
},
"photo": None,
}
if photo:
photo_path = folder / f"{stem}.img"
photo_path.write_bytes(photo)
payload["photo"] = photo_path.name
(folder / f"{stem}.json").write_text(json.dumps(payload), encoding="utf-8")
_prune(folder, max(1, settings.IMAGE_SEARCH_CAPTURE_MAX))
except Exception as exc: # noqa: BLE001 - a diagnostic must never fail the search
logger.warning("image search log skipped: %s", exc)

4
app/services/label_match.py
View File

@@ -59,7 +59,7 @@ from app.infrastructure.settings import (
IMAGE_SEARCH_MAX_TOP_K,
OCR_MAX_CHARS,
)
from app.services.image_match import _STOP, ImageSearchResult, _row_text, text_overlap, tokens
from app.services.image_match import _STOP, ImageSearchResult, _row_text, size_in_name, text_overlap, tokens
logger = logging.getLogger(__name__)
@@ -197,6 +197,7 @@ def label_rank_key(row: Dict[str, Any]) -> tuple:
"""
return (
-float(row.get("text_overlap", 0.0)),
-int(bool(row.get("size_in_name", False))), # the label's size in THIS row's name
int(row.get("text_extra", 0)),
-round(float(row.get("score", 0.0)), 3),
str(row.get("product_name") or row.get("title") or "").lower(),
@@ -336,6 +337,7 @@ def resolve_label(
for row in rows:
row["text_overlap"] = text_overlap(words, sizes, row)
row["text_extra"] = text_extra(words, row)
row["size_in_name"] = size_in_name(sizes, row)
rows.sort(key=label_rank_key)
return rows

11
app/services/product_identify.py
View File

@@ -19,6 +19,8 @@ THE LADDER
`fallback_reason` is the reason for the LAST step the ladder took:
image_below_threshold the image arm ran but did not clear the floor
image_ambiguous it cleared the floor, but another photo scored
within IMAGE_SEARCH_MIN_MARGIN of it
no_image_match the image arm ran and found nothing at all
image_embedder_unavailable no vector could be computed (no model here)
no_text no label to fall back to and no photo to read
@@ -46,6 +48,7 @@ from app.infrastructure.settings import (
IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
IMAGE_SEARCH_DEFAULT_MIN_SCORE,
IMAGE_SEARCH_DEFAULT_TOP_K,
IMAGE_SEARCH_MIN_MARGIN,
)
from app.services import ocr_service
from app.services.image_match import ImageSearchResult, search_by_vector
@@ -89,9 +92,11 @@ def _image_rung(
top = float(result.rows[0].get("score", 0.0))
except (TypeError, ValueError):
top = 0.0
if top >= min_image_score:
return result, top, None
return result, top, "image_below_threshold"
if top < min_image_score:
return result, top, "image_below_threshold"
if result.margin is not None and result.margin < IMAGE_SEARCH_MIN_MARGIN:
return result, top, "image_ambiguous"
return result, top, None
def identify_product(

9
app/services/vector_store.py
View File

@@ -1861,7 +1861,14 @@ def lexical_search(
finally:
conn.close()
results.sort(key=lambda r: (r.get("lex_tier", 9), r.get("distance", 9.0)))
# A row with a NULL `embedding` comes back with distance None (the key IS
# present), and one None among floats made this sort raise - which threw
# away every row of an unscoped search, the label resolver's lexical arm.
def _distance(r: Dict[str, Any]) -> float:
d = r.get("distance")
return 9.0 if d is None else float(d)
results.sort(key=lambda r: (r.get("lex_tier", 9), _distance(r)))
return results[:limit]