From c0489d89d6ec2d3559ca60b1cc173ea4f28425bc Mon Sep 17 00:00:00 2001 From: sriram Date: Mon, 28 Sep 2026 15:44:01 +0530 Subject: [PATCH] Updates on Image search using vectors --- .env.example | 7 + app/api/routers/search.py | 73 ++++++- app/api/schemas.py | 30 ++- app/infrastructure/settings.py | 15 ++ app/services/image_match.py | 96 +++++++++ app/services/image_search_log.py | 133 ++++++++++++ app/services/label_match.py | 4 +- app/services/product_identify.py | 11 +- app/services/vector_store.py | 9 +- docs/IMAGE_SEARCH_API.md | 51 ++++- scripts/eval_identify.py | 282 ++++++++++++++++++++++++++ scripts/replay_image_query.py | 173 ++++++++++++++++ tests/test_identify_api.py | 46 ++++- tests/test_image_confidence.py | 205 +++++++++++++++++++ tests/test_image_search_api.py | 3 +- tests/test_lexical_search_distance.py | 83 ++++++++ 16 files changed, 1184 insertions(+), 37 deletions(-) create mode 100644 app/services/image_search_log.py create mode 100644 scripts/eval_identify.py create mode 100644 scripts/replay_image_query.py create mode 100644 tests/test_image_confidence.py create mode 100644 tests/test_lexical_search_distance.py diff --git a/.env.example b/.env.example index 80ef243..d64c16a 100644 --- a/.env.example +++ b/.env.example @@ -338,6 +338,13 @@ ENABLE_SERVER_OCR=true #OCR_NUM_THREADS=2 #IMAGE_IDENTIFY_MIN_IMAGE_SCORE=0.70 #IMAGE_IDENTIFY_MIN_TEXT_SCORE=0.60 +# An image match is only "confirmed" (match_confidence) when it also leads the +# best different photo by this much. +#IMAGE_SEARCH_MIN_MARGIN=0.05 +# Diagnosis only: save every image-search request as JSON for +# `python -m scripts.replay_image_query`. Blank = off; keeps the newest N. +#IMAGE_SEARCH_CAPTURE_DIR=/app/data/image_search_requests +#IMAGE_SEARCH_CAPTURE_MAX=200 # Capture-to-catalog: when identify cannot confirm a photo, read the label and - # for a brand already in the catalog - add the product through the 11-stage diff --git a/app/api/routers/search.py b/app/api/routers/search.py index 211ad45..bfc11ee 100644 --- a/app/api/routers/search.py +++ b/app/api/routers/search.py @@ -27,7 +27,7 @@ from app.infrastructure.settings import ( SEARCH_DEFAULT_TOP_K, SEARCH_MAX_TOP_K, ) -from app.services import capture_discovery, image_embedder +from app.services import capture_discovery, image_embedder, image_search_log from app.services.catalog_search import search_catalog from app.services.image_match import ( ImageSearchResult, @@ -105,12 +105,28 @@ def _to_image_search_out(result: ImageSearchResult) -> ImageSearchOut: min_score=result.min_score, top_k=result.top_k, query_text=result.query_text, + match_confidence=result.match_confidence if matches else "none", + margin=None if result.margin is None else round(float(result.margin), 4), ) +def _identify_confidence(result: IdentifyResult) -> str: + """The ladder's verdict in match_confidence terms: confirmed exactly when + product_identify.py's client rule says so.""" + if not result.search.rows: + return "none" + if capture_discovery.is_confirmed(result.matched_by, result.fallback_reason): + return "confirmed" + return "low" + + def _to_identify_out(result: IdentifyResult) -> IdentifyOut: + fields = _to_image_search_out(result.search).model_dump() + fields["match_confidence"] = _identify_confidence(result) + if result.matched_by != "image_vector": + fields["margin"] = None # text rows: a MiniLM score, no image margin return IdentifyOut( - **_to_image_search_out(result.search).model_dump(), + **fields, matched_by=result.matched_by, ocr_text=result.ocr_text, ocr_source=result.ocr_source, @@ -119,6 +135,28 @@ def _to_identify_out(result: IdentifyResult) -> IdentifyOut: ) +def _log_search(route: str, result: ImageSearchResult, *, vector, text, brand, category, top_k, + photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None: + image_search_log.record( + route, vector=vector, text=text, brand=brand, category=category, top_k=top_k, + rows=result.rows, detected_brand=result.detected_brand, scoped_to_brand=result.scoped_to_brand, + match_confidence=result.match_confidence, margin=result.margin, + matched_by="image_vector" if result.rows else "none", photo=photo, text_fallback=text_fallback, + ) + + +def _log_identify(route: str, result: IdentifyResult, *, vector, text, brand, category, top_k, + photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None: + image_search_log.record( + route, vector=vector, text=text, brand=brand, category=category, top_k=top_k, + rows=result.search.rows, detected_brand=result.search.detected_brand, + scoped_to_brand=result.search.scoped_to_brand, match_confidence=_identify_confidence(result), + margin=result.search.margin if result.matched_by == "image_vector" else None, + matched_by=result.matched_by, fallback_reason=result.fallback_reason, + photo=photo, text_fallback=text_fallback, + ) + + @router.post("/search/image-vector", response_model=ImageSearchOut) def image_vector_search_endpoint(body: ImageVectorSearchRequest): """Products that look like the photo whose embedding is `vector`. @@ -131,20 +169,26 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest): filter; a brand recognised from `text` falls back to every brand when it finds nothing (`scope_fallback`). - With `text_fallback: true` the request runs the /search/identify ladder - instead: when the best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE - the label `text` is resolved against the catalogue, and the response is - an IdentifyOut (this response plus matched_by, fallback_reason, ...). - Off by default so the app's existing calls are answered exactly as before. + With `text_fallback` the request runs the /search/identify ladder + instead: when the image match cannot be confirmed (best score under + IMAGE_IDENTIFY_MIN_IMAGE_SCORE, or another photo within + IMAGE_SEARCH_MIN_MARGIN) the label `text` is resolved against the + catalogue, and the response is an IdentifyOut (this response plus + matched_by, fallback_reason, ...). Left out, it is on whenever `text` is + sent; `false` keeps the image-only ranking, where text only breaks ties. """ + ladder = body.text_fallback if body.text_fallback is not None else bool((body.text or "").strip()) + log_fields = dict(vector=body.vector, text=body.text, brand=body.brand, category=body.category, + top_k=body.top_k, text_fallback=body.text_fallback) try: - if body.text_fallback: + if ladder: identified = identify_product( vector=body.vector, image_bytes=None, text=body.text, brand=body.brand, category=body.category, top_k=body.top_k, min_score=body.min_score, ) + _log_identify("image-vector", identified, **log_fields) # A Response bypasses response_model, which is the point: the - # default path keeps its declared ImageSearchOut contract. + # image-only path keeps its declared ImageSearchOut contract. return JSONResponse(content=_to_identify_out(identified).model_dump(mode="json")) result = search_by_vector( body.vector, text=body.text, brand=body.brand, category=body.category, @@ -152,6 +196,7 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest): ) except InvalidVectorError as exc: raise HTTPException(status_code=422, detail=str(exc)) + _log_search("image-vector", result, **log_fields) return _to_image_search_out(result) @@ -185,6 +230,8 @@ def image_vector_search_get_endpoint( ) except InvalidVectorError as exc: raise HTTPException(status_code=422, detail=str(exc)) + _log_search("image-vector:get", result, vector=values, text=text, brand=brand, + category=category, top_k=top_k) return _to_image_search_out(result) @@ -230,6 +277,8 @@ async def image_search_endpoint( ) except InvalidVectorError as exc: # cannot happen for a model output, but the route must not 500 raise HTTPException(status_code=422, detail=str(exc)) + _log_search("image", result, vector=vector, text=text, brand=brand, category=category, + top_k=top_k, photo=content) return _to_image_search_out(result) @@ -295,6 +344,8 @@ async def identify_endpoint( "deployment. Send the label as `text`, or embed the photo client-side and POST " "the vector to /api/search/image-vector.", ) + _log_identify("identify", result, vector=vector, text=text, brand=brand, category=category, + top_k=top_k, photo=content) out = _to_identify_out(result) if settings.ENABLE_CAPTURE_DISCOVERY and not capture_discovery.is_confirmed( result.matched_by, result.fallback_reason @@ -327,10 +378,14 @@ def _apply_capture_outcome(out: IdentifyOut, outcome: capture_discovery.CaptureO out.results = [ImageMatchOut(**card.model_dump(), score=1.0, text_overlap=1.0)] out.total = 1 out.matched_by = capture_discovery.MATCHED_BY_LABEL_EXACT + out.match_confidence = "confirmed" + out.margin = None elif outcome.status == capture_discovery.PENDING: out.results = [] out.total = 0 out.matched_by = capture_discovery.MATCHED_BY_DISCOVERY + out.match_confidence = "none" + out.margin = None @router.get("/search/identify/jobs/{job_id}", response_model=CaptureJobOut) diff --git a/app/api/schemas.py b/app/api/schemas.py index 9b3e486..e14dad3 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -263,15 +263,19 @@ class ImageVectorSearchRequest(BaseModel): top_k: int = Field(IMAGE_SEARCH_DEFAULT_TOP_K, ge=1, le=IMAGE_SEARCH_MAX_TOP_K) min_score: float = Field(IMAGE_SEARCH_DEFAULT_MIN_SCORE, ge=-1.0, le=1.0, description="Drop matches with cosine similarity below this") - # Opt-in, so a client that never asked for it gets exactly the response - # it always did. With it, the route runs the identify ladder: when the - # best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE, `text` is - # resolved against the catalogue instead, and the response is an - # IdentifyOut (ImageSearchOut plus matched_by / fallback_reason / ...). - text_fallback: bool = Field( - False, - description="Fall back to resolving `text` when the image match is below the identify floor; " - "the response then carries the IdentifyOut fields", + # With it, the route runs the identify ladder: when the best image score + # is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE (or another photo is within + # IMAGE_SEARCH_MIN_MARGIN of it), `text` is resolved against the + # catalogue instead, and the response is an IdentifyOut (ImageSearchOut + # plus matched_by / fallback_reason / ...). Left out, it is ON whenever + # `text` is sent: a label that names the product must not lose to a 0.5 + # cosine, which it did when text only broke exact ties. `false` keeps the + # old image-only ranking. + text_fallback: Optional[bool] = Field( + None, + description="Resolve `text` when the image match cannot be confirmed; the response then " + "carries the IdentifyOut fields. Default: on when `text` is sent. " + "false = image-only ranking, text breaks ties only", ) @field_validator("vector") @@ -301,6 +305,14 @@ class ImageSearchOut(BaseModel): min_score: float = 0.0 top_k: int = 0 query_text: Optional[str] = None + # "confirmed" only when the best match clears IMAGE_IDENTIFY_MIN_IMAGE_SCORE + # AND leads the best different photo by IMAGE_SEARCH_MIN_MARGIN (on + # /identify: when the ladder confirmed it). "low": show the results as a + # list to pick from, never as the answer. "none": no results. + match_confidence: str = "none" + # Best score minus the best DIFFERENT photo's; None when there was no rival. + # Image space only - None when the rows came from the text rung. + margin: Optional[float] = None class IdentifyOut(ImageSearchOut): diff --git a/app/infrastructure/settings.py b/app/infrastructure/settings.py index d587214..6f0036f 100644 --- a/app/infrastructure/settings.py +++ b/app/infrastructure/settings.py @@ -432,6 +432,21 @@ IMAGE_SEARCH_DEFAULT_MIN_SCORE = float(os.getenv("IMAGE_SEARCH_DEFAULT_MIN_SCORE # product share an image and tie, so more than top_k must come back), and # the floor for hnsw.ef_search on that query so the index does not drop them. IMAGE_SEARCH_MAX_FETCH_K = int(os.getenv("IMAGE_SEARCH_MAX_FETCH_K", "100")) +# How far the best image match must lead the best DIFFERENT photo for the +# answer to count as confirmed (`match_confidence`). A phone photo of a card on +# a screen measured 0.507 against its own product and 0.419 against a +# stranger; with glare that gap closes, and a flipped order is a wrong +# product shown with confidence. Pack sizes sharing one photo tie exactly and +# are never each other's competitor. Unmeasured start: tune with +# scripts/eval_identify.py. +IMAGE_SEARCH_MIN_MARGIN = float(os.getenv("IMAGE_SEARCH_MIN_MARGIN", "0.05")) +# Diagnosis: when set, every image-search request (vector, text, brand and the +# top matches) is written here as one JSON file, for +# scripts/replay_image_query.py. Blank = off. At most IMAGE_SEARCH_CAPTURE_MAX +# files are kept; the oldest go first. The routes are public, so leave it off +# except while chasing a report. +IMAGE_SEARCH_CAPTURE_DIR = os.getenv("IMAGE_SEARCH_CAPTURE_DIR", "").strip() +IMAGE_SEARCH_CAPTURE_MAX = int(os.getenv("IMAGE_SEARCH_CAPTURE_MAX", "200")) # --------------------------------------------------------------------------- # Identify a product from a phone photo - POST /api/search/identify diff --git a/app/services/image_match.py b/app/services/image_match.py index 23acc65..3399dbe 100644 --- a/app/services/image_match.py +++ b/app/services/image_match.py @@ -53,10 +53,12 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Set, Tuple from app.infrastructure.settings import ( + IMAGE_IDENTIFY_MIN_IMAGE_SCORE, IMAGE_SEARCH_DEFAULT_MIN_SCORE, IMAGE_SEARCH_DEFAULT_TOP_K, IMAGE_SEARCH_MAX_FETCH_K, IMAGE_SEARCH_MAX_TOP_K, + IMAGE_SEARCH_MIN_MARGIN, ) from app.services.image_embedder import EMBED_DIM @@ -90,6 +92,8 @@ class ImageSearchResult: min_score: float = 0.0 query_text: Optional[str] = None top_k: int = 0 + match_confidence: str = "none" # "confirmed" | "low" | "none" + margin: Optional[float] = None # top score minus the best different photo's # --------------------------------------------------------------------------- @@ -226,16 +230,102 @@ def text_overlap(words: Set[str], sizes: Set[str], row: Dict[str, Any]) -> float return _SIZE_WEIGHT * len(sizes & row_sizes) + _WORD_WEIGHT * len(words & row_words) +def size_in_name(sizes: Set[str], row: Dict[str, Any]) -> bool: + """Whether a pack size the label printed is in the row's OWN name. + + `text_overlap` also counts `size_variants`, and on some products every + pack-size row carries the SAME list (all Amul Cream rows say + ["90g", "1kg"]), so a "1kg" label ties every sibling and the tie fell to + alphabetical order - "Amul Cream 125 ml" was shown, confirmed, for a + photo of the 1kg pack. The name is per row; this breaks that tie. + """ + if not sizes: + return False + name_sizes = tokens(" ".join([str(row.get("product_name") or ""), str(row.get("title") or "")]))[1] + return bool(sizes & name_sizes) + + def rank_key(row: Dict[str, Any]) -> tuple: """Best first. Score rounded to 3 dp so siblings sharing a photo tie.""" return ( -round(float(row.get("score", 0.0)), 3), -float(row.get("text_overlap", 0.0)), + -int(bool(row.get("size_in_name", False))), str(row.get("product_name") or row.get("title") or "").lower(), str(row.get("image_id") or ""), ) +# --------------------------------------------------------------------------- +# confidence +# --------------------------------------------------------------------------- +# Pack sizes that share one catalog photo score identically (to pgvector's +# float precision); anything further apart than this is a different picture. +_SAME_PHOTO_EPS = 5e-4 + +CONFIRMED = "confirmed" +LOW = "low" +NONE = "none" + + +def image_margin(rows: Sequence[Dict[str, Any]]) -> Optional[float]: + """How far the best row leads the best row with a DIFFERENT photo. + + `rows` are best first by score. Siblings sharing the winner's photo are + skipped: they are the same product in another size, which the label text + separates, not a rival. None when nothing else is in the list - no rival + was seen, which is not the same as a clear lead, so the caller treats it + as unconfirmable only when the score itself is low. + """ + if not rows: + return None + try: + top = float(rows[0].get("score", 0.0)) + except (TypeError, ValueError): + return None + for row in rows[1:]: + try: + score = float(row.get("score", 0.0)) + except (TypeError, ValueError): + continue + if top - score > _SAME_PHOTO_EPS: + return top - score + return None + + +def match_confidence( + rows: Sequence[Dict[str, Any]], + min_image_score: Optional[float] = None, + min_margin: Optional[float] = None, +) -> Tuple[str, Optional[float]]: + """("confirmed" | "low" | "none", margin) for image rows, best first. + + Confirmed needs BOTH a score at the identify floor and a lead over the + next different photo. A photo of a card on a phone screen scores ~0.5 + against its own product, and a stranger can sit within a few hundredths + of it: shown as the answer, that is how "Dairy Milk Lickables" came back + as someone else's product. A client should offer the list, not pick one, + when this is not "confirmed". The thresholds default to this module's + settings, read at call time so scripts/eval_identify.py can vary them. + """ + if min_image_score is None: + min_image_score = IMAGE_IDENTIFY_MIN_IMAGE_SCORE + if min_margin is None: + min_margin = IMAGE_SEARCH_MIN_MARGIN + if not rows: + return NONE, None + margin = image_margin(rows) + try: + top = float(rows[0].get("score", 0.0)) + except (TypeError, ValueError): + return LOW, margin + if top < min_image_score: + return LOW, margin + if margin is not None and margin < min_margin: + return LOW, margin + return CONFIRMED, margin + + # --------------------------------------------------------------------------- # the search # --------------------------------------------------------------------------- @@ -345,7 +435,11 @@ def search_by_vector( words, sizes = tokens(label) for row in rows: row["text_overlap"] = text_overlap(words, sizes, row) + row["size_in_name"] = size_in_name(sizes, row) rows.sort(key=rank_key) + # Judged on every candidate, not the top_k shown: with top_k=1 the rival + # would otherwise never be seen. + confidence, margin = match_confidence(rows) winners = _hydrate(rows[:top_k]) return ImageSearchResult( @@ -356,4 +450,6 @@ def search_by_vector( min_score=min_score, query_text=label, top_k=top_k, + match_confidence=confidence, + margin=margin, ) diff --git a/app/services/image_search_log.py b/app/services/image_search_log.py new file mode 100644 index 0000000..30bde8e --- /dev/null +++ b/app/services/image_search_log.py @@ -0,0 +1,133 @@ +"""One log line per search-by-photo request, and - when asked - the request itself. + +WHY +--- +A colleague photographed the "Cadbury Dairy Milk Lickables" card and got +"Milk Toned"; "Aachi Sambar Powder 100g" came back as Sakthi's "Sambar powder +50g". Replayed on the server, both photos rank the right product first, so +the difference is in what the app SENDS - the vector (is its on-device model +the same as ours?), whether any OCR `text` came with it, which route it hit. +None of that was visible. This module makes it visible: + +* `record()` logs, for every /search/image*, /search/identify request: the + route, the text, brand, detected brand, a short hash and the norm of the + vector, the top three (brand, name, score) and the lead over the best + different photo. Always on; it is one INFO line. +* With IMAGE_SEARCH_CAPTURE_DIR set, the whole request - the 1024 floats, + the fields, the photo when there was one - is also written there, for + `python -m scripts.replay_image_query`. Off by default: these routes are + public. The newest IMAGE_SEARCH_CAPTURE_MAX requests are kept. + +Never raises: a diagnostic must not fail a search. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import math +import struct +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from app.infrastructure import settings + +logger = logging.getLogger(__name__) + +_TEXT_LOG_CHARS = 120 + + +def vector_fingerprint(vector: Optional[Sequence[float]]) -> Optional[str]: + """10 hex characters identifying a vector: equal for the same float32 values.""" + if vector is None: + return None + packed = struct.pack(f"<{len(vector)}f", *[float(v) for v in vector]) + return hashlib.sha1(packed).hexdigest()[:10] + + +def _top(rows: Sequence[Dict[str, Any]], n: int = 3) -> List[Dict[str, Any]]: + out = [] + for row in list(rows)[:n]: + try: + score = round(float(row.get("score", 0.0)), 4) + except (TypeError, ValueError): + score = None + out.append({ + "brand": row.get("brand") or row.get("brand_table"), + "product_name": row.get("product_name") or row.get("title"), + "image_id": row.get("image_id"), + "score": score, + }) + return out + + +def _prune(folder: Path, keep: int) -> None: + files = sorted(folder.glob("*.json"), key=lambda p: p.stat().st_mtime) + for stale in files[:max(0, len(files) - keep)]: + for sibling in folder.glob(stale.stem + ".*"): + sibling.unlink(missing_ok=True) + + +def record( + route: str, + *, + vector: Optional[Sequence[float]], + text: Optional[str], + brand: Optional[str], + category: Optional[str], + top_k: int, + rows: Sequence[Dict[str, Any]], + detected_brand: Optional[str] = None, + scoped_to_brand: bool = False, + match_confidence: Optional[str] = None, + margin: Optional[float] = None, + matched_by: Optional[str] = None, + fallback_reason: Optional[str] = None, + text_fallback: Optional[bool] = None, + photo: Optional[bytes] = None, +) -> None: + """Log the request and its answer; save it when capture is on.""" + try: + norm = math.sqrt(sum(float(v) * float(v) for v in vector)) if vector is not None else None + top = _top(rows) + logger.info( + "[IMAGE_SEARCH] route=%s text=%r brand=%r detected=%r scoped=%s vec=%s norm=%s " + "matched_by=%s reason=%s confidence=%s margin=%s top=%s", + route, (text or "")[:_TEXT_LOG_CHARS] or None, brand, detected_brand, scoped_to_brand, + vector_fingerprint(vector), None if norm is None else round(norm, 4), + matched_by, fallback_reason, match_confidence, + None if margin is None else round(margin, 4), + [(t["brand"], t["product_name"], t["score"]) for t in top], + ) + folder_name = settings.IMAGE_SEARCH_CAPTURE_DIR + if not folder_name: + return + folder = Path(folder_name) + folder.mkdir(parents=True, exist_ok=True) + stem = time.strftime("%Y%m%dT%H%M%S") + "_" + uuid.uuid4().hex[:6] + payload = { + "route": route, + "received_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "request": { + "vector": None if vector is None else [float(v) for v in vector], + "text": text, "brand": brand, "category": category, "top_k": top_k, + "text_fallback": text_fallback, + }, + "vector_fingerprint": vector_fingerprint(vector), + "answer": { + "matched_by": matched_by, "fallback_reason": fallback_reason, + "match_confidence": match_confidence, "margin": margin, + "detected_brand": detected_brand, "scoped_to_brand": scoped_to_brand, "top": top, + }, + "photo": None, + } + if photo: + photo_path = folder / f"{stem}.img" + photo_path.write_bytes(photo) + payload["photo"] = photo_path.name + (folder / f"{stem}.json").write_text(json.dumps(payload), encoding="utf-8") + _prune(folder, max(1, settings.IMAGE_SEARCH_CAPTURE_MAX)) + except Exception as exc: # noqa: BLE001 - a diagnostic must never fail the search + logger.warning("image search log skipped: %s", exc) diff --git a/app/services/label_match.py b/app/services/label_match.py index 6c4b902..4c2d046 100644 --- a/app/services/label_match.py +++ b/app/services/label_match.py @@ -59,7 +59,7 @@ from app.infrastructure.settings import ( IMAGE_SEARCH_MAX_TOP_K, OCR_MAX_CHARS, ) -from app.services.image_match import _STOP, ImageSearchResult, _row_text, text_overlap, tokens +from app.services.image_match import _STOP, ImageSearchResult, _row_text, size_in_name, text_overlap, tokens logger = logging.getLogger(__name__) @@ -197,6 +197,7 @@ def label_rank_key(row: Dict[str, Any]) -> tuple: """ return ( -float(row.get("text_overlap", 0.0)), + -int(bool(row.get("size_in_name", False))), # the label's size in THIS row's name int(row.get("text_extra", 0)), -round(float(row.get("score", 0.0)), 3), str(row.get("product_name") or row.get("title") or "").lower(), @@ -336,6 +337,7 @@ def resolve_label( for row in rows: row["text_overlap"] = text_overlap(words, sizes, row) row["text_extra"] = text_extra(words, row) + row["size_in_name"] = size_in_name(sizes, row) rows.sort(key=label_rank_key) return rows diff --git a/app/services/product_identify.py b/app/services/product_identify.py index b2b57a1..02d7053 100644 --- a/app/services/product_identify.py +++ b/app/services/product_identify.py @@ -19,6 +19,8 @@ THE LADDER `fallback_reason` is the reason for the LAST step the ladder took: image_below_threshold the image arm ran but did not clear the floor + image_ambiguous it cleared the floor, but another photo scored + within IMAGE_SEARCH_MIN_MARGIN of it no_image_match the image arm ran and found nothing at all image_embedder_unavailable no vector could be computed (no model here) no_text no label to fall back to and no photo to read @@ -46,6 +48,7 @@ from app.infrastructure.settings import ( IMAGE_IDENTIFY_MIN_IMAGE_SCORE, IMAGE_SEARCH_DEFAULT_MIN_SCORE, IMAGE_SEARCH_DEFAULT_TOP_K, + IMAGE_SEARCH_MIN_MARGIN, ) from app.services import ocr_service from app.services.image_match import ImageSearchResult, search_by_vector @@ -89,9 +92,11 @@ def _image_rung( top = float(result.rows[0].get("score", 0.0)) except (TypeError, ValueError): top = 0.0 - if top >= min_image_score: - return result, top, None - return result, top, "image_below_threshold" + if top < min_image_score: + return result, top, "image_below_threshold" + if result.margin is not None and result.margin < IMAGE_SEARCH_MIN_MARGIN: + return result, top, "image_ambiguous" + return result, top, None def identify_product( diff --git a/app/services/vector_store.py b/app/services/vector_store.py index 2382b1a..1a73172 100644 --- a/app/services/vector_store.py +++ b/app/services/vector_store.py @@ -1861,7 +1861,14 @@ def lexical_search( finally: conn.close() - results.sort(key=lambda r: (r.get("lex_tier", 9), r.get("distance", 9.0))) + # A row with a NULL `embedding` comes back with distance None (the key IS + # present), and one None among floats made this sort raise - which threw + # away every row of an unscoped search, the label resolver's lexical arm. + def _distance(r: Dict[str, Any]) -> float: + d = r.get("distance") + return 9.0 if d is None else float(d) + + results.sort(key=lambda r: (r.get("lex_tier", 9), _distance(r))) return results[:limit] diff --git a/docs/IMAGE_SEARCH_API.md b/docs/IMAGE_SEARCH_API.md index a05c28d..ebc78af 100644 --- a/docs/IMAGE_SEARCH_API.md +++ b/docs/IMAGE_SEARCH_API.md @@ -9,6 +9,24 @@ Every catalogue row carries the same kind of vector in `img_vector` index). These two endpoints turn the app's vector - or a photo - into product cards. +## Recommended client flow (read this first) + +A colleague photographed the **Cadbury Dairy Milk Lickables** card and got +"Milk Toned"; **Aachi Sambar Powder 100g** came back as Sakthi's "Sambar +powder 50g". A photo of a card on a screen scores only ~0.5 against its own +product, and a stranger can sit a few hundredths behind it. So: + +1. **Send the photo to `POST /api/search/identify`** (preferred). The server + embeds it with the same model as the catalogue, reads the card/label text + with OCR, and lets the text decide when the image cannot. No model-parity + risk. *Or*, keeping the on-device vector, **always send the full OCR read + as `text`** to `/image-vector`: the label then decides (see `text_fallback`). +2. **Crop to the pack** (a framing guide in the camera UI). Measured: a card + photographed whole scored 0.51 for the right product; cropped to the pack + image, 0.94. +3. **Honour `match_confidence`.** Only `"confirmed"` is an answer. On `"low"` + show the results as a list to pick from - never auto-select the first. + ``` POST /api/search/image-vector JSON {vector[1024], text?, brand?, category?, top_k?, min_score?, text_fallback?} GET /api/search/image-vector query string: vector=&text=...&top_k=... (see "GET variant") @@ -55,6 +73,14 @@ a floor, and read `score` on each result. | `category` | both | – | `ILIKE` filter on the category column | | `top_k` | both | 10 | 1–50 | | `min_score` | both | 0.0 | −1…1; drop matches below it | +| `text_fallback` | `/image-vector` POST only | on when `text` is sent | run the identify ladder (below); `false` = image-only ranking, `text` breaks ties only | + +Every response also carries: + +| Field | Meaning | +|---|---| +| `match_confidence` | `"confirmed"`: the best match scored ≥ `IMAGE_IDENTIFY_MIN_IMAGE_SCORE` (0.70) **and** leads the best *different* photo by ≥ `IMAGE_SEARCH_MIN_MARGIN` (0.05) - or, on the identify shape, the ladder confirmed it. `"low"`: a list to pick from, not an answer. `"none"`: no results. | +| `margin` | best image score minus the best different photo's. Pack sizes sharing one photo are not rivals. `null` when there was no rival, or when the rows came from the label text. | ## Examples @@ -170,7 +196,8 @@ only ~0.63 against the catalogue's render of the same pack** (measured), so ladder and tells you which rung answered: ``` -photo ─► img_vector search ─► best score ≥ IMAGE_IDENTIFY_MIN_IMAGE_SCORE (0.70)? +photo ─► img_vector search ─► best score ≥ IMAGE_IDENTIFY_MIN_IMAGE_SCORE (0.70) + │ and ≥ IMAGE_SEARCH_MIN_MARGIN (0.05) ahead of the next different photo? │ yes → matched_by "image_vector", fallback_reason null (confirmed) ▼ no label text = your `text` (ocr_source "client") @@ -213,6 +240,7 @@ show it as such, or ask for another shot. |---|---| | `null` | the image match cleared the floor | | `image_below_threshold` | image ran, best score under the floor → the label decided (on a `"text"` answer) | +| `image_ambiguous` | best score cleared the floor, but a different photo was within `IMAGE_SEARCH_MIN_MARGIN` | | `no_image_match` | image ran and found nothing → the label decided | | `image_embedder_unavailable` | this deployment has no image model → text only | | `no_text` | `/image-vector` with `text_fallback` but no `text` | @@ -238,11 +266,22 @@ on the first photo that needs it (~2 s), then costs ~1–2 s per photo at the and loaded. `ENABLE_SERVER_OCR=false` turns it off; the route then relies on client `text`. -**The app's route.** `POST /api/search/image-vector` accepts -`"text_fallback": true`. Off (the default) the response is exactly what it -has always been. On, the same ladder runs with the app's vector and `text` -(no photo, so never server OCR), and the response is the identify shape -above. Not on the GET variant. +**The app's route.** `POST /api/search/image-vector` runs the same ladder +with the app's vector and `text` (no photo, so never server OCR) whenever +`text` is sent, and answers with the identify shape above. Before +2026-09-28 it did so only with `"text_fallback": true`, and otherwise the +label only broke exact ties - a label naming the product lost to any photo +that scored 0.01 higher. Send `"text_fallback": false` to keep that old +image-only ranking; without `text` nothing changes. Not on the GET variant. + +**Diagnosing a wrong match.** Every search-by-photo request logs one +`[IMAGE_SEARCH]` line (route, text, brand, a fingerprint of the vector, top 3 +with scores, margin). With `IMAGE_SEARCH_CAPTURE_DIR` set the full request - +and the photo, on `/image` and `/identify` - is saved for +`python -m scripts.replay_image_query [--photo shot.jpg]`, which +replays it and, given the photo, checks that the app's on-device vector +matches the server's (cosine ≥ 0.99). `python -m scripts.eval_identify` +measures accuracy and the thresholds on synthetic and real card photos. ## When the product is not in the catalogue (capture-to-catalog) diff --git a/scripts/eval_identify.py b/scripts/eval_identify.py new file mode 100644 index 0000000..cea603c --- /dev/null +++ b/scripts/eval_identify.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +How often does a photo of a product card come back as that product? + +WHY THIS EXISTS +--------------- +Colleagues photograph catalogue cards with the Nearle app, and two came back +as other brands' products. The thresholds that decide when an image match is +"confirmed" (IMAGE_IDENTIFY_MIN_IMAGE_SCORE, IMAGE_SEARCH_MIN_MARGIN) were +starting guesses. This measures them, on the database this process points at. + +WHAT IT MEASURES +---------------- +For N sampled catalogue products it downloads the image the card shows and +builds four synthetic "photos" of it: + + clean the catalogue image itself (the ceiling: should be 100%) + card the image on a white card with text lines under it + screen that card dimmed, desaturated, blurred, blue-shifted, JPEG'd + tight the screen shot cropped to the pack (a framing guide) + +and, with --photos DIR, real photos named `__.jpg` +(e.g. `brand_cadbury__cadbury_dairy_milk_lickables_20g.jpg`). + +Each photo is searched three ways: + + image search_by_vector, no text - what the app gets today + image+text the identify ladder with the product name as the label + (what the card's title reads as) + ladder+ocr (real photos only) the identify ladder with server OCR + +and scored: top-1 exact, top-1 "same photo" (a pack-size sibling sharing +the target's picture - right product, maybe wrong size), top-3 exact, and +the one that matters most - WRONG BUT CONFIRMED: a different product shown +with match_confidence "confirmed". Tune the thresholds until that is zero +without making everything "low". + +USAGE +----- + python -m scripts.eval_identify --n 40 + python -m scripts.eval_identify --n 40 --brands Cadbury,Aachi,Sakthi + python -m scripts.eval_identify --photos ~/card_photos --n 0 + python -m scripts.eval_identify --n 40 --min-margin 0.08 --min-image-score 0.65 + +Read-only. Downloads catalogue images (honours the downloader's pacing). +""" +from __future__ import annotations + +import argparse +import io +import random +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +Key = Tuple[str, str] # (brand_table, image_id) +MODES = ("clean", "card", "screen", "tight") + + +# --------------------------------------------------------------------------- +# catalogue +# --------------------------------------------------------------------------- + +def load_catalogue(brands: Optional[List[str]]) -> Dict[Key, Dict[str, Any]]: + """Every row with an image vector: (table, image_id) -> name and source URL.""" + from app.services import vector_store as vs + + conn = vs._connect() + if conn is None: + raise SystemExit("no database connection") + wanted = {vs._table_name(b) for b in brands} if brands else None + out: Dict[Key, Dict[str, Any]] = {} + try: + with conn.cursor() as cur: + cur.execute("SET TRANSACTION READ ONLY") + for suffix in vs._list_brand_table_suffixes(cur): + table = f"brand_{suffix}" + if wanted is not None and table not in wanted: + continue + try: + cur.execute(f"SELECT image_id, product_name, img_vector_src FROM {table} " + f"WHERE img_vector IS NOT NULL AND img_vector_src IS NOT NULL") + except Exception: # noqa: BLE001 - a table without the column + conn.rollback() + cur.execute("SET TRANSACTION READ ONLY") + continue + for image_id, name, src in cur.fetchall(): + out[(table, str(image_id))] = {"name": name, "src": src} + finally: + conn.rollback() + conn.close() + return out + + +# --------------------------------------------------------------------------- +# synthetic photos +# --------------------------------------------------------------------------- + +def synthetic_photo(raw: bytes, mode: str) -> Optional[bytes]: + import numpy as np + from PIL import Image, ImageDraw, ImageEnhance, ImageFilter + + try: + img = Image.open(io.BytesIO(raw)).convert("RGB") + except Exception: # noqa: BLE001 + return None + if mode != "clean": + w, h = img.size + side = max(w, h) + pack_h = int(h * side / w) + canvas = Image.new("RGB", (int(side * 1.3), int(side * 0.1) + pack_h + int(side * 0.9)), (255, 255, 255)) + canvas.paste(img.resize((side, pack_h)), (int(side * .15), int(side * .1))) + draw = ImageDraw.Draw(canvas) + for k in range(6): + y = int(side * .1) + pack_h + int(side * .1) + k * int(side * .1) + draw.rectangle([int(side * .15), y, int(side * (0.6 + .1 * (k % 3))), y + int(side * .04)], + fill=(40, 40, 40)) + img = canvas + if mode in ("screen", "tight"): + img = ImageEnhance.Brightness(img).enhance(0.85) + img = ImageEnhance.Color(img).enhance(0.8) + img = img.filter(ImageFilter.GaussianBlur(2)) + px = np.array(img).astype(np.float32) + px[..., 2] *= 1.08 + img = Image.fromarray(np.clip(px, 0, 255).astype(np.uint8)) + if mode == "tight": + img = img.crop((int(side * .15), int(side * .1), int(side * 1.15), int(side * .1) + pack_h)) + buf = io.BytesIO() + img.save(buf, "JPEG", quality=70) + return buf.getvalue() + + +# --------------------------------------------------------------------------- +# scoring +# --------------------------------------------------------------------------- + +class Tally: + def __init__(self) -> None: + self.n = 0 + self.top1 = 0 + self.top1_same_photo = 0 + self.top3 = 0 + self.confirmed = 0 + self.wrong_confirmed = 0 + self.misses: List[str] = [] + + def add(self, target: Key, rows: List[Dict[str, Any]], confidence: str, + catalogue: Dict[Key, Dict[str, Any]]) -> None: + self.n += 1 + keys = [(str(r.get("brand_table") or ""), str(r.get("image_id") or "")) for r in rows] + src = catalogue[target]["src"] + hit1 = bool(keys) and keys[0] == target + same_photo = bool(keys) and (hit1 or catalogue.get(keys[0], {}).get("src") == src) + self.top1 += hit1 + self.top1_same_photo += same_photo + self.top3 += target in keys[:3] + if confidence == "confirmed": + self.confirmed += 1 + if not same_photo: + self.wrong_confirmed += 1 + if not same_photo and len(self.misses) < 8: + got = rows[0].get("product_name") if rows else None + self.misses.append(f"{catalogue[target]['name']} -> {got} ({confidence})") + + def line(self, label: str) -> str: + if not self.n: + return f" {label:22s} (no photos)" + pct = lambda k: f"{100.0 * k / self.n:5.1f}%" # noqa: E731 + return (f" {label:22s} n={self.n:3d} top1 {pct(self.top1)} same-photo {pct(self.top1_same_photo)} " + f"top3 {pct(self.top3)} confirmed {pct(self.confirmed)} " + f"WRONG+confirmed {self.wrong_confirmed}") + + +def _ladder_confidence(result) -> str: + from app.services.capture_discovery import is_confirmed + + if not result.search.rows: + return "none" + return "confirmed" if is_confirmed(result.matched_by, result.fallback_reason) else "low" + + +def evaluate(photos: List[Tuple[str, Key, bytes, bool]], catalogue: Dict[Key, Dict[str, Any]], + top_k: int) -> Dict[Tuple[str, str], Tally]: + from app.services import image_embedder + from app.services.image_match import search_by_vector + from app.services import product_identify + from app.services.product_identify import identify_product + + floor = product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE + tallies: Dict[Tuple[str, str], Tally] = defaultdict(Tally) + for mode, target, data, real in photos: + vector = image_embedder.embedding_for_bytes(data) + if vector is None: + continue + image = search_by_vector(vector, top_k=top_k) + tallies[(mode, "image")].add(target, image.rows, image.match_confidence, catalogue) + + label = catalogue[target]["name"] + labelled = identify_product(vector=vector, image_bytes=None, text=label, top_k=top_k, + min_image_score=floor) + tallies[(mode, "image+text")].add(target, labelled.search.rows, _ladder_confidence(labelled), catalogue) + + if real: + ocr = identify_product(vector=vector, image_bytes=data, top_k=top_k, min_image_score=floor) + tallies[(mode, "ladder+ocr")].add(target, ocr.search.rows, _ladder_confidence(ocr), catalogue) + return tallies + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--n", type=int, default=30, help="catalogue products to sample (0 = none)") + parser.add_argument("--brands", help="comma-separated brands to sample from (default: all)") + parser.add_argument("--photos", type=Path, help="folder of real photos named __.jpg") + parser.add_argument("--modes", default=",".join(MODES), help=f"synthetic modes (default: {','.join(MODES)})") + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--top-k", type=int, default=5) + parser.add_argument("--min-margin", type=float, help="override IMAGE_SEARCH_MIN_MARGIN for this run") + parser.add_argument("--min-image-score", type=float, help="override IMAGE_IDENTIFY_MIN_IMAGE_SCORE") + args = parser.parse_args() + + from app.services import image_embedder, image_match, product_identify + from app.services.image_vector import download_image_bytes + + # Both modules read these at call time; identify_product's own floor is + # passed explicitly in evaluate(). + if args.min_margin is not None: + image_match.IMAGE_SEARCH_MIN_MARGIN = args.min_margin + product_identify.IMAGE_SEARCH_MIN_MARGIN = args.min_margin + if args.min_image_score is not None: + image_match.IMAGE_IDENTIFY_MIN_IMAGE_SCORE = args.min_image_score + product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE = args.min_image_score + if not image_embedder.available(): + raise SystemExit("the image embedding model is not available here") + + brands = [b.strip() for b in args.brands.split(",")] if args.brands else None + catalogue = load_catalogue(brands) + print(f"catalogue: {len(catalogue)} rows with an image vector" + f"{' in ' + ', '.join(brands) if brands else ''}") + + photos: List[Tuple[str, Key, bytes, bool]] = [] + modes = [m for m in args.modes.split(",") if m in MODES] + sample = random.Random(args.seed).sample(sorted(catalogue), min(args.n, len(catalogue))) + for target in sample: + raw = download_image_bytes(catalogue[target]["src"]) + if not raw: + print(f" skip (download failed): {catalogue[target]['name']}") + continue + for mode in modes: + data = synthetic_photo(raw, mode) + if data: + photos.append((mode, target, data, False)) + + if args.photos: + for path in sorted(args.photos.iterdir()): + if "__" not in path.stem: + continue + table, image_id = path.stem.split("__", 1) + if (table, image_id) not in catalogue: + print(f" skip (not in catalogue): {path.name}") + continue + photos.append(("real", (table, image_id), path.read_bytes(), True)) + + tallies = evaluate(photos, catalogue, args.top_k) + print(f"\nthresholds: min_image_score={product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE} " + f"min_margin={product_identify.IMAGE_SEARCH_MIN_MARGIN}\n") + for mode in modes + ["real"]: + for way in ("image", "image+text", "ladder+ocr"): + tally = tallies.get((mode, way)) + if tally and tally.n: + print(tally.line(f"{mode} / {way}")) + print("\nexamples of misses (target -> top-1):") + for (mode, way), tally in sorted(tallies.items()): + for miss in tally.misses[:3]: + print(f" [{mode} / {way}] {miss}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/replay_image_query.py b/scripts/replay_image_query.py new file mode 100644 index 0000000..3621f42 --- /dev/null +++ b/scripts/replay_image_query.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Replay a saved search-by-photo request, and check the app's model against ours. + +WHY THIS EXISTS +--------------- +A colleague photographs a product card and the app shows a different product +("Cadbury Dairy Milk Lickables" came back as "Milk Toned", "Aachi Sambar +Powder 100g" as Sakthi's "Sambar powder 50g"). On the server, synthetic +photos of those cards rank the right product first, so the question is what +the app sent. With IMAGE_SEARCH_CAPTURE_DIR set, every request is saved as +JSON (app/services/image_search_log.py); this script replays one. + +WHAT IT PRINTS +-------------- +1. The request: route, text, brand, the vector's fingerprint and norm, and + what the server answered at the time. +2. The image-only ranking (search_by_vector) and the identify ladder + (identify_product) for the same vector and text, against the database + this process points at. Both, whichever route was called, so you can see + whether sending text would have fixed it. +3. With --photo: the photo embedded HERE, with our model and preprocessing, + and the cosine between that and the app's vector. The same photo through + the same model gives >= 0.99. Well below that, the app's on-device model + or preprocessing is not ours, and every image score it sends is in a + different space: put the app's .tflite in app/services/models/mobilenet/ + and run `python -m scripts.backfill_image_vectors --all --force --apply`. + A saved /identify or /image request carries its photo; use --photo for + the vector route, with the picture the colleague took. + +USAGE +----- + python -m scripts.replay_image_query data/image_search_requests/20260928T101500_ab12cd.json + python -m scripts.replay_image_query saved.json --photo lickables_card.jpg + python -m scripts.replay_image_query --list data/image_search_requests + +Read-only: nothing here writes. +""" +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.services.image_search_log import vector_fingerprint # noqa: E402 + +PARITY_OK = 0.99 + + +def _cosine(a: Sequence[float], b: Sequence[float]) -> float: + dot = sum(float(x) * float(y) for x, y in zip(a, b)) + na = math.sqrt(sum(float(x) ** 2 for x in a)) + nb = math.sqrt(sum(float(y) ** 2 for y in b)) + return dot / (na * nb) if na and nb else 0.0 + + +def _print_rows(title: str, rows: List[Dict[str, Any]], limit: int = 5) -> None: + print(f" {title}") + if not rows: + print(" (nothing)") + for i, row in enumerate(rows[:limit], start=1): + score = row.get("score") + score_s = f"{float(score):.4f}" if score is not None else " - " + print(f" {i}. {score_s} {str(row.get('brand') or row.get('brand_table') or ''):22.22s} " + f"{row.get('product_name') or row.get('title')} (overlap {row.get('text_overlap', 0)})") + + +def _list(folder: Path) -> int: + for path in sorted(folder.glob("*.json")): + try: + saved = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + req, ans = saved.get("request", {}), saved.get("answer", {}) + top = (ans.get("top") or [{}])[0] + print(f"{path.name} {saved.get('route'):16s} text={str(req.get('text'))[:40]!r:44s} " + f"-> {top.get('product_name')} ({top.get('score')}) [{ans.get('match_confidence')}]") + return 0 + + +def replay(path: Path, photo: Optional[Path], top_k: int) -> int: + from app.services import image_embedder + from app.services.image_match import search_by_vector + from app.services.product_identify import identify_product + + saved = json.loads(path.read_text(encoding="utf-8")) + req = saved.get("request") or {} + answer = saved.get("answer") or {} + vector = req.get("vector") + text = req.get("text") + + print(f"== {path.name} route={saved.get('route')} received={saved.get('received_at')}") + print(f" text={text!r} brand={req.get('brand')!r} category={req.get('category')!r} " + f"text_fallback={req.get('text_fallback')!r}") + if vector is not None: + norm = math.sqrt(sum(v * v for v in vector)) + print(f" vector: {len(vector)} floats, norm {norm:.4f}, fingerprint {vector_fingerprint(vector)}") + else: + print(" vector: none (the server had no model when this was sent)") + print(f" answered then: matched_by={answer.get('matched_by')} reason={answer.get('fallback_reason')} " + f"confidence={answer.get('match_confidence')} margin={answer.get('margin')}") + _print_rows("top then:", answer.get("top") or []) + + photo_bytes: Optional[bytes] = None + if photo is not None: + photo_bytes = photo.read_bytes() + elif saved.get("photo"): + candidate = path.parent / saved["photo"] + if candidate.exists(): + photo_bytes = candidate.read_bytes() + + ours: Optional[List[float]] = None + if photo_bytes is not None: + if not image_embedder.available(): + print("\n (no embedding model here - cannot run the parity check)") + else: + ours = image_embedder.embedding_for_bytes(photo_bytes) + if ours is None: + print("\n (the photo could not be decoded)") + elif vector is not None: + cos = _cosine(vector, ours) + verdict = "SAME model and preprocessing" if cos >= PARITY_OK else ( + "DIFFERENT - the app's vectors are not comparable with img_vector") + print(f"\n model parity: cosine(app vector, server vector of the same photo) = {cos:.4f}" + f" -> {verdict}") + + query = vector if vector is not None else ours + if query is None: + print("\n nothing to replay: no vector in the request and none computed from a photo") + return 1 + + print("\n-- replayed now, against this process's database --") + image = search_by_vector(query, text=text, brand=req.get("brand"), category=req.get("category"), + top_k=top_k) + print(f" image only: confidence={image.match_confidence} margin={image.margin} " + f"detected_brand={image.detected_brand} scoped={image.scoped_to_brand}") + _print_rows("ranking:", image.rows) + + identified = identify_product(vector=query, image_bytes=photo_bytes, text=text, brand=req.get("brand"), + category=req.get("category"), top_k=top_k) + print(f"\n identify ladder: matched_by={identified.matched_by} reason={identified.fallback_reason} " + f"ocr_source={identified.ocr_source} ocr_text={identified.ocr_text!r}") + _print_rows("ranking:", identified.search.rows) + + if ours is not None and vector is not None and _cosine(vector, ours) < PARITY_OK: + print("\n and with the SERVER's vector of the same photo:") + _print_rows("ranking:", search_by_vector(ours, text=text, brand=req.get("brand"), + category=req.get("category"), top_k=top_k).rows) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("request", nargs="?", help="a saved request JSON") + parser.add_argument("--photo", type=Path, help="the photo the request was made from (parity check)") + parser.add_argument("--list", type=Path, metavar="DIR", help="summarise every saved request in DIR") + parser.add_argument("--top-k", type=int, default=5) + args = parser.parse_args() + + if args.list: + return _list(args.list) + if not args.request: + parser.error("give a saved request JSON, or --list DIR") + return replay(Path(args.request), args.photo, args.top_k) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_identify_api.py b/tests/test_identify_api.py index 640221b..65a8aff 100644 --- a/tests/test_identify_api.py +++ b/tests/test_identify_api.py @@ -4,8 +4,10 @@ The ladder's decisions are tested in tests/test_product_identify.py and the two arms in tests/test_image_match.py and tests/test_label_match.py. Here the assertions are about the HTTP contract: status codes, the guards the photo route shares with /search/image, the IdentifyOut fields, and - the -one that protects the app in the field - that /search/image-vector without -`text_fallback` answers exactly as it did before this route existed. +one that protects the app in the field - that /search/image-vector with +`text_fallback: false` answers exactly as it did before this route existed. +Left out, the flag is on whenever `text` is sent: a label naming the product +must not lose to a 0.5 cosine (the "Lickables came back as Milk Toned" report). """ from __future__ import annotations @@ -44,7 +46,7 @@ def _row(score: float = 0.631, image_id: str = "britannia_marie_gold_300g") -> D IMAGE_SEARCH_KEYS = {"results", "total", "detected_brand", "scoped_to_brand", "scope_fallback", - "min_score", "top_k", "query_text"} + "min_score", "top_k", "query_text", "match_confidence", "margin"} IDENTIFY_KEYS = IMAGE_SEARCH_KEYS | {"matched_by", "ocr_text", "ocr_source", "image_top_score", "fallback_reason"} # Capture-to-catalog fields: always present, None unless discovery ran (see # tests/test_capture_discovery.py). Additive - no existing key changed. @@ -248,21 +250,50 @@ def test_a_confident_image_never_touches_ocr(client, embedder_ok, ladder): # /search/image-vector - text_fallback # --------------------------------------------------------------------------- -def test_image_vector_without_the_flag_is_unchanged(client, ladder, fake_identify, monkeypatch): +@pytest.fixture +def image_only(monkeypatch): seen = [] def fake_search(vector, text=None, brand=None, category=None, top_k=10, min_score=0.0): seen.append(text) return ImageSearchResult(rows=[_row(0.63)], detected_brand="Britannia", scoped_to_brand=True, - min_score=min_score, query_text=text, top_k=top_k) + min_score=min_score, query_text=text, top_k=top_k, + match_confidence="low", margin=0.02) monkeypatch.setattr(search_router, "search_by_vector", fake_search) + return seen - res = client.post("/api/search/image-vector", json={"vector": _unit(), "text": "Marie Gold 300 g"}) + +def test_image_vector_with_the_flag_false_is_unchanged(client, ladder, fake_identify, image_only): + res = client.post("/api/search/image-vector", + json={"vector": _unit(), "text": "Marie Gold 300 g", "text_fallback": False}) + + assert res.status_code == 200, res.text + body = res.json() + assert set(body) == IMAGE_SEARCH_KEYS + assert body["match_confidence"] == "low" and body["margin"] == 0.02 + assert image_only == ["Marie Gold 300 g"] and fake_identify.calls == [] and ladder.text_calls == [] + + +def test_image_vector_without_text_or_flag_is_image_only(client, ladder, fake_identify, image_only): + res = client.post("/api/search/image-vector", json={"vector": _unit()}) assert res.status_code == 200, res.text assert set(res.json()) == IMAGE_SEARCH_KEYS - assert seen == ["Marie Gold 300 g"] and fake_identify.calls == [] and ladder.text_calls == [] + assert image_only == [None] and fake_identify.calls == [] + + +def test_image_vector_with_text_and_no_flag_lets_the_label_decide(client, ladder, image_only): + """The field report: the image arm's 0.63 must not outrank a label that + names the product.""" + res = client.post("/api/search/image-vector", json={"vector": _unit(), "text": "Marie Gold 300 g"}) + + body = res.json() + assert res.status_code == 200, res.text + assert set(body) == IDENTIFY_KEYS + assert body["matched_by"] == "text" and body["match_confidence"] == "confirmed" + assert body["results"][0]["image_id"] == "txt_marie_300" and body["margin"] is None + assert image_only == [] # the router's image-only path never ran def test_image_vector_with_the_flag_runs_the_ladder_and_adds_the_fields(client, ladder): @@ -285,6 +316,7 @@ def test_image_vector_with_the_flag_but_no_text_says_no_text(client, ladder): assert res.status_code == 200, res.text assert body["matched_by"] == "image_vector" and body["fallback_reason"] == "no_text" assert body["results"][0]["score"] == 0.63 + assert body["match_confidence"] == "low" # shown as a list to pick from def test_image_vector_with_the_flag_still_validates_the_vector(client, ladder): diff --git a/tests/test_image_confidence.py b/tests/test_image_confidence.py new file mode 100644 index 0000000..dc9dc0f --- /dev/null +++ b/tests/test_image_confidence.py @@ -0,0 +1,205 @@ +"""When a search-by-photo answer counts as confirmed, and what is logged. + +A photo of the "Cadbury Dairy Milk Lickables" card on a phone screen scored +0.507 against its own product and 0.419 against a stranger; the Aachi sambar +card 0.560 against 0.541. Those orders flip with a little glare, and the app +then showed a different product as THE answer. So an image match is +"confirmed" only with a score at the identify floor AND a lead over the best +different photo - pack sizes that share one photo never count as rivals. +""" +from __future__ import annotations + +import json +from typing import Any, Dict, List + +import pytest + +from app.infrastructure import settings +from app.services import image_match, image_search_log +from app.services import product_identify as pi +from app.services.image_match import ImageSearchResult, image_margin, match_confidence + + +def _rows(*scores: float) -> List[Dict[str, Any]]: + return [{"image_id": f"p{i}", "product_name": f"Product {i}", "brand": "Cadbury", "score": s} + for i, s in enumerate(scores)] + + +# --------------------------------------------------------------------------- +# image_margin / match_confidence +# --------------------------------------------------------------------------- + +def test_the_margin_skips_pack_sizes_that_share_the_winners_photo(): + # 100g and 200g share one photo and tie; the rival is the 0.761. + assert image_margin(_rows(0.960, 0.960, 0.9602, 0.761)) == pytest.approx(0.199) + + +def test_no_rival_means_no_margin(): + assert image_margin(_rows(0.93, 0.93)) is None + assert image_margin([]) is None + + +def test_a_high_score_with_a_clear_lead_is_confirmed(): + assert match_confidence(_rows(0.94, 0.58), min_image_score=0.70, min_margin=0.05)[0] == "confirmed" + + +def test_a_high_score_with_a_rival_close_behind_is_low(): + label, margin = match_confidence(_rows(0.74, 0.72), min_image_score=0.70, min_margin=0.05) + assert label == "low" and margin == pytest.approx(0.02) + + +def test_a_card_photographed_on_a_screen_is_low_even_when_it_ranks_first(): + assert match_confidence(_rows(0.507, 0.419), min_image_score=0.70, min_margin=0.05)[0] == "low" + + +def test_a_high_score_whose_only_neighbours_are_its_own_sizes_is_confirmed(): + assert match_confidence(_rows(0.96, 0.96), min_image_score=0.70, min_margin=0.05)[0] == "confirmed" + + +def test_no_rows_is_none(): + assert match_confidence([]) == ("none", None) + + +def test_search_by_vector_judges_confidence_on_every_candidate_not_just_top_k(monkeypatch): + candidates = [ + {"image_id": "lickables_20g", "product_name": "Cadbury Dairy Milk Lickables 20g", + "brand": "Cadbury", "brand_table": "brand_cadbury", "distance": 0.20}, + {"image_id": "milk_toned", "product_name": "Milk Toned", + "brand": "Own Products", "brand_table": "brand_own_products", "distance": 0.22}, + ] + monkeypatch.setattr("app.services.vector_store.image_vector_search", + lambda *a, **k: [dict(c) for c in candidates]) + monkeypatch.setattr(image_match, "_hydrate", lambda rows: rows) + + out = image_match.search_by_vector([1.0] + [0.0] * 1023, top_k=1) + + assert [r["image_id"] for r in out.rows] == ["lickables_20g"] + assert out.match_confidence == "low" and out.margin == pytest.approx(0.02) + + +# --------------------------------------------------------------------------- +# the identify ladder: an ambiguous image lets the label decide +# --------------------------------------------------------------------------- + +@pytest.fixture +def ladder(monkeypatch): + state: Dict[str, Any] = {"margin": None, "text_calls": 0} + + def image(vector, text=None, brand=None, category=None, top_k=10, min_score=0.0): + return ImageSearchResult(rows=_rows(0.74, 0.72), margin=state["margin"], top_k=top_k) + + def text(label, brand=None, category=None, top_k=10): + state["text_calls"] += 1 + return ImageSearchResult(rows=[{"image_id": "by_label", "score": 0.8, "text_overlap": 7.0}]) + + monkeypatch.setattr(pi, "search_by_vector", image) + monkeypatch.setattr(pi, "resolve_label", text) + monkeypatch.setattr(pi, "is_confident", lambda rows: bool(rows)) + monkeypatch.setattr(pi, "IMAGE_SEARCH_MIN_MARGIN", 0.05) + return state + + +def test_an_image_that_clears_the_floor_but_barely_leads_falls_to_the_label(ladder): + ladder["margin"] = 0.02 + + out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None, + text="Cadbury Dairy Milk Lickables 20g", min_image_score=0.70) + + assert out.matched_by == "text" and out.fallback_reason == "image_ambiguous" + assert out.search.rows[0]["image_id"] == "by_label" + + +def test_an_image_with_a_clear_lead_is_still_the_answer(ladder): + ladder["margin"] = 0.2 + + out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None, + text="Cadbury Dairy Milk Lickables 20g", min_image_score=0.70) + + assert out.matched_by == "image_vector" and out.fallback_reason is None + assert ladder["text_calls"] == 0 + + +def test_an_ambiguous_image_with_no_label_is_shown_unconfirmed(ladder): + ladder["margin"] = 0.02 + + out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None, min_image_score=0.70) + + assert out.matched_by == "image_vector" and out.fallback_reason == "no_text" + + +# --------------------------------------------------------------------------- +# image_search_log +# --------------------------------------------------------------------------- + +def test_the_fingerprint_is_stable_and_short(): + v = [0.5, -0.25] + [0.0] * 1022 + assert image_search_log.vector_fingerprint(v) == image_search_log.vector_fingerprint(list(v)) + assert len(image_search_log.vector_fingerprint(v)) == 10 + assert image_search_log.vector_fingerprint(None) is None + + +def test_capture_is_off_by_default(monkeypatch, tmp_path, caplog): + monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", "") + caplog.set_level("INFO", logger="app.services.image_search_log") + + image_search_log.record("image-vector", vector=[1.0] + [0.0] * 1023, text="Aachi Sambar Powder 100g", + brand=None, category=None, top_k=5, rows=_rows(0.56, 0.54)) + + assert "[IMAGE_SEARCH] route=image-vector" in caplog.text and "Aachi Sambar Powder 100g" in caplog.text + assert list(tmp_path.iterdir()) == [] + + +def test_capture_writes_a_replayable_request_and_keeps_only_the_newest(monkeypatch, tmp_path): + monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", str(tmp_path)) + monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_MAX", 2) + vec = [1.0] + [0.0] * 1023 + + for i in range(3): + image_search_log.record("identify", vector=vec, text=f"label {i}", brand=None, category=None, + top_k=5, rows=_rows(0.5), photo=b"jpegbytes") + + saved = sorted(tmp_path.glob("*.json")) + assert len(saved) == 2 and len(list(tmp_path.glob("*.img"))) == 2 + payload = json.loads(saved[-1].read_text(encoding="utf-8")) + assert payload["request"]["vector"] == vec and payload["route"] == "identify" + assert (tmp_path / payload["photo"]).read_bytes() == b"jpegbytes" + + +def test_a_capture_failure_never_raises(monkeypatch, tmp_path): + blocker = tmp_path / "not_a_dir" + blocker.write_text("x") + monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", str(blocker)) + + image_search_log.record("image", vector=None, text=None, brand=None, category=None, + top_k=5, rows=[]) + + +# --------------------------------------------------------------------------- +# size in the row's own name breaks a size_variants tie +# --------------------------------------------------------------------------- + +def _cream(name: str) -> Dict[str, Any]: + # Every Amul Cream row carries the SAME size_variants list in the catalogue. + return {"product_name": name, "size_variants": ["90g", "1kg"], "score": 0.651, + "image_id": name.lower().replace(" ", "_")} + + +def test_the_label_size_in_the_name_beats_the_same_size_in_shared_size_variants(): + from app.services.label_match import label_rank_key + from app.services.image_match import size_in_name, text_overlap, tokens + + words, sizes = tokens("Amul Cream 1kg") + rows = [_cream("Amul Cream 125 ml / 67 g"), _cream("Amul Cream 1kg"), _cream("Amul Cream 250 ml")] + for row in rows: + row["text_overlap"] = text_overlap(words, sizes, row) + row["size_in_name"] = size_in_name(sizes, row) + + assert len({r["text_overlap"] for r in rows}) == 1 # the tie this exists to break + assert sorted(rows, key=label_rank_key)[0]["product_name"] == "Amul Cream 1kg" + assert sorted(rows, key=image_match.rank_key)[0]["product_name"] == "Amul Cream 1kg" + + +def test_no_label_size_is_never_a_name_hit(): + from app.services.image_match import size_in_name + + assert size_in_name(set(), _cream("Amul Cream 1kg")) is False diff --git a/tests/test_image_search_api.py b/tests/test_image_search_api.py index baec36a..f945dc1 100644 --- a/tests/test_image_search_api.py +++ b/tests/test_image_search_api.py @@ -64,7 +64,8 @@ def fake_search(monkeypatch): def test_a_vector_returns_the_product_card_with_its_score(client, fake_search): res = client.post("/api/search/image-vector", - json={"vector": _unit(), "text": "Britannia Marie Gold 300 g", "top_k": 5}) + json={"vector": _unit(), "text": "Britannia Marie Gold 300 g", "top_k": 5, + "text_fallback": False}) assert res.status_code == 200, res.text body = res.json() diff --git a/tests/test_lexical_search_distance.py b/tests/test_lexical_search_distance.py new file mode 100644 index 0000000..9938d69 --- /dev/null +++ b/tests/test_lexical_search_distance.py @@ -0,0 +1,83 @@ +"""vector_store.lexical_search when a matching row has no text embedding. + +`embedding <=> q` is NULL for such a row, so its dict carries distance None - +the key is present, so `.get("distance", 9.0)` returned None, and sorting a +None among floats raised. The label resolver's lexical arm caught that and +lost EVERY row of the search: unscoped, one un-embedded product in any brand +table was enough to blank "Sambar Powder 100g". +""" +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +from app.services import vector_store + + +class _Cursor: + def __init__(self, tables: Dict[str, List[Tuple[str, float | None, int]]]): + self._tables = tables + self._rows: List[tuple] = [] + self.description = [("product_name",), ("distance",), ("lex_tier",)] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql: str, params=None) -> None: + table = sql.split(" FROM ", 1)[1].split()[0] + self._rows = list(self._tables[table]) + + def fetchall(self): + return self._rows + + +class _Conn: + def __init__(self, cursor: _Cursor): + self._cursor = cursor + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def cursor(self): + return self._cursor + + def close(self) -> None: + pass + + +def _patch(monkeypatch, tables): + cursor = _Cursor(tables) + monkeypatch.setattr(vector_store, "_connect", lambda: _Conn(cursor)) + monkeypatch.setattr(vector_store, "_list_brand_table_suffixes", + lambda cur: [t[len("brand_"):] for t in tables]) + monkeypatch.setattr(vector_store, "_table_exists", lambda cur, t: True) + monkeypatch.setattr(vector_store, "_product_columns", lambda cur, t: "product_name") + + +def test_a_row_without_an_embedding_sorts_last_in_its_tier_instead_of_raising(monkeypatch): + _patch(monkeypatch, { + "brand_aachi": [("Aachi Sambar Powder 100g", 0.41, 2)], + "brand_zzsmoketest": [("Zzsmoketest Sambar Powder 100g", None, 2)], + "brand_sakthi": [("Sambar powder 50g", 0.35, 2)], + }) + + rows = vector_store.lexical_search(["sambar", "powder"], query_embedding=[0.1] * 384) + + names: List[Any] = [r["product_name"] for r in rows] + assert names == ["Sambar powder 50g", "Aachi Sambar Powder 100g", "Zzsmoketest Sambar Powder 100g"] + + +def test_the_tier_still_outranks_the_distance(monkeypatch): + _patch(monkeypatch, { + "brand_a": [("exact", None, 0)], + "brand_b": [("contains", 0.01, 2)], + }) + + rows = vector_store.lexical_search(["x"], query_embedding=[0.1] * 384) + + assert [r["product_name"] for r in rows] == ["exact", "contains"]