Updates on Image search using vectors
This commit is contained in:
sriram
@@ -338,6 +338,13 @@ ENABLE_SERVER_OCR=true
|
||||
#OCR_NUM_THREADS=2
|
||||
#IMAGE_IDENTIFY_MIN_IMAGE_SCORE=0.70
|
||||
#IMAGE_IDENTIFY_MIN_TEXT_SCORE=0.60
|
||||
# An image match is only "confirmed" (match_confidence) when it also leads the
|
||||
# best different photo by this much.
|
||||
#IMAGE_SEARCH_MIN_MARGIN=0.05
|
||||
# Diagnosis only: save every image-search request as JSON for
|
||||
# `python -m scripts.replay_image_query`. Blank = off; keeps the newest N.
|
||||
#IMAGE_SEARCH_CAPTURE_DIR=/app/data/image_search_requests
|
||||
#IMAGE_SEARCH_CAPTURE_MAX=200
|
||||
|
||||
# Capture-to-catalog: when identify cannot confirm a photo, read the label and -
|
||||
# for a brand already in the catalog - add the product through the 11-stage
|
||||
|
||||
@@ -27,7 +27,7 @@ from app.infrastructure.settings import (
|
||||
SEARCH_DEFAULT_TOP_K,
|
||||
SEARCH_MAX_TOP_K,
|
||||
)
|
||||
from app.services import capture_discovery, image_embedder
|
||||
from app.services import capture_discovery, image_embedder, image_search_log
|
||||
from app.services.catalog_search import search_catalog
|
||||
from app.services.image_match import (
|
||||
ImageSearchResult,
|
||||
@@ -105,12 +105,28 @@ def _to_image_search_out(result: ImageSearchResult) -> ImageSearchOut:
|
||||
min_score=result.min_score,
|
||||
top_k=result.top_k,
|
||||
query_text=result.query_text,
|
||||
match_confidence=result.match_confidence if matches else "none",
|
||||
margin=None if result.margin is None else round(float(result.margin), 4),
|
||||
)
|
||||
|
||||
|
||||
def _identify_confidence(result: IdentifyResult) -> str:
|
||||
"""The ladder's verdict in match_confidence terms: confirmed exactly when
|
||||
product_identify.py's client rule says so."""
|
||||
if not result.search.rows:
|
||||
return "none"
|
||||
if capture_discovery.is_confirmed(result.matched_by, result.fallback_reason):
|
||||
return "confirmed"
|
||||
return "low"
|
||||
|
||||
|
||||
def _to_identify_out(result: IdentifyResult) -> IdentifyOut:
|
||||
fields = _to_image_search_out(result.search).model_dump()
|
||||
fields["match_confidence"] = _identify_confidence(result)
|
||||
if result.matched_by != "image_vector":
|
||||
fields["margin"] = None # text rows: a MiniLM score, no image margin
|
||||
return IdentifyOut(
|
||||
**_to_image_search_out(result.search).model_dump(),
|
||||
**fields,
|
||||
matched_by=result.matched_by,
|
||||
ocr_text=result.ocr_text,
|
||||
ocr_source=result.ocr_source,
|
||||
@@ -119,6 +135,28 @@ def _to_identify_out(result: IdentifyResult) -> IdentifyOut:
|
||||
)
|
||||
|
||||
|
||||
def _log_search(route: str, result: ImageSearchResult, *, vector, text, brand, category, top_k,
|
||||
photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None:
|
||||
image_search_log.record(
|
||||
route, vector=vector, text=text, brand=brand, category=category, top_k=top_k,
|
||||
rows=result.rows, detected_brand=result.detected_brand, scoped_to_brand=result.scoped_to_brand,
|
||||
match_confidence=result.match_confidence, margin=result.margin,
|
||||
matched_by="image_vector" if result.rows else "none", photo=photo, text_fallback=text_fallback,
|
||||
)
|
||||
|
||||
|
||||
def _log_identify(route: str, result: IdentifyResult, *, vector, text, brand, category, top_k,
|
||||
photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None:
|
||||
image_search_log.record(
|
||||
route, vector=vector, text=text, brand=brand, category=category, top_k=top_k,
|
||||
rows=result.search.rows, detected_brand=result.search.detected_brand,
|
||||
scoped_to_brand=result.search.scoped_to_brand, match_confidence=_identify_confidence(result),
|
||||
margin=result.search.margin if result.matched_by == "image_vector" else None,
|
||||
matched_by=result.matched_by, fallback_reason=result.fallback_reason,
|
||||
photo=photo, text_fallback=text_fallback,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/search/image-vector", response_model=ImageSearchOut)
|
||||
def image_vector_search_endpoint(body: ImageVectorSearchRequest):
|
||||
"""Products that look like the photo whose embedding is `vector`.
|
||||
@@ -131,20 +169,26 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest):
|
||||
filter; a brand recognised from `text` falls back to every brand when it
|
||||
finds nothing (`scope_fallback`).
|
||||
|
||||
With `text_fallback: true` the request runs the /search/identify ladder
|
||||
instead: when the best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE
|
||||
the label `text` is resolved against the catalogue, and the response is
|
||||
an IdentifyOut (this response plus matched_by, fallback_reason, ...).
|
||||
Off by default so the app's existing calls are answered exactly as before.
|
||||
With `text_fallback` the request runs the /search/identify ladder
|
||||
instead: when the image match cannot be confirmed (best score under
|
||||
IMAGE_IDENTIFY_MIN_IMAGE_SCORE, or another photo within
|
||||
IMAGE_SEARCH_MIN_MARGIN) the label `text` is resolved against the
|
||||
catalogue, and the response is an IdentifyOut (this response plus
|
||||
matched_by, fallback_reason, ...). Left out, it is on whenever `text` is
|
||||
sent; `false` keeps the image-only ranking, where text only breaks ties.
|
||||
"""
|
||||
ladder = body.text_fallback if body.text_fallback is not None else bool((body.text or "").strip())
|
||||
log_fields = dict(vector=body.vector, text=body.text, brand=body.brand, category=body.category,
|
||||
top_k=body.top_k, text_fallback=body.text_fallback)
|
||||
try:
|
||||
if body.text_fallback:
|
||||
if ladder:
|
||||
identified = identify_product(
|
||||
vector=body.vector, image_bytes=None, text=body.text, brand=body.brand,
|
||||
category=body.category, top_k=body.top_k, min_score=body.min_score,
|
||||
)
|
||||
_log_identify("image-vector", identified, **log_fields)
|
||||
# A Response bypasses response_model, which is the point: the
|
||||
# default path keeps its declared ImageSearchOut contract.
|
||||
# image-only path keeps its declared ImageSearchOut contract.
|
||||
return JSONResponse(content=_to_identify_out(identified).model_dump(mode="json"))
|
||||
result = search_by_vector(
|
||||
body.vector, text=body.text, brand=body.brand, category=body.category,
|
||||
@@ -152,6 +196,7 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest):
|
||||
)
|
||||
except InvalidVectorError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
_log_search("image-vector", result, **log_fields)
|
||||
return _to_image_search_out(result)
|
||||
|
||||
|
||||
@@ -185,6 +230,8 @@ def image_vector_search_get_endpoint(
|
||||
)
|
||||
except InvalidVectorError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
_log_search("image-vector:get", result, vector=values, text=text, brand=brand,
|
||||
category=category, top_k=top_k)
|
||||
return _to_image_search_out(result)
|
||||
|
||||
|
||||
@@ -230,6 +277,8 @@ async def image_search_endpoint(
|
||||
)
|
||||
except InvalidVectorError as exc: # cannot happen for a model output, but the route must not 500
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
_log_search("image", result, vector=vector, text=text, brand=brand, category=category,
|
||||
top_k=top_k, photo=content)
|
||||
return _to_image_search_out(result)
|
||||
|
||||
|
||||
@@ -295,6 +344,8 @@ async def identify_endpoint(
|
||||
"deployment. Send the label as `text`, or embed the photo client-side and POST "
|
||||
"the vector to /api/search/image-vector.",
|
||||
)
|
||||
_log_identify("identify", result, vector=vector, text=text, brand=brand, category=category,
|
||||
top_k=top_k, photo=content)
|
||||
out = _to_identify_out(result)
|
||||
if settings.ENABLE_CAPTURE_DISCOVERY and not capture_discovery.is_confirmed(
|
||||
result.matched_by, result.fallback_reason
|
||||
@@ -327,10 +378,14 @@ def _apply_capture_outcome(out: IdentifyOut, outcome: capture_discovery.CaptureO
|
||||
out.results = [ImageMatchOut(**card.model_dump(), score=1.0, text_overlap=1.0)]
|
||||
out.total = 1
|
||||
out.matched_by = capture_discovery.MATCHED_BY_LABEL_EXACT
|
||||
out.match_confidence = "confirmed"
|
||||
out.margin = None
|
||||
elif outcome.status == capture_discovery.PENDING:
|
||||
out.results = []
|
||||
out.total = 0
|
||||
out.matched_by = capture_discovery.MATCHED_BY_DISCOVERY
|
||||
out.match_confidence = "none"
|
||||
out.margin = None
|
||||
|
||||
|
||||
@router.get("/search/identify/jobs/{job_id}", response_model=CaptureJobOut)
|
||||
|
||||
@@ -263,15 +263,19 @@ class ImageVectorSearchRequest(BaseModel):
|
||||
top_k: int = Field(IMAGE_SEARCH_DEFAULT_TOP_K, ge=1, le=IMAGE_SEARCH_MAX_TOP_K)
|
||||
min_score: float = Field(IMAGE_SEARCH_DEFAULT_MIN_SCORE, ge=-1.0, le=1.0,
|
||||
description="Drop matches with cosine similarity below this")
|
||||
# Opt-in, so a client that never asked for it gets exactly the response
|
||||
# it always did. With it, the route runs the identify ladder: when the
|
||||
# best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE, `text` is
|
||||
# resolved against the catalogue instead, and the response is an
|
||||
# IdentifyOut (ImageSearchOut plus matched_by / fallback_reason / ...).
|
||||
text_fallback: bool = Field(
|
||||
False,
|
||||
description="Fall back to resolving `text` when the image match is below the identify floor; "
|
||||
"the response then carries the IdentifyOut fields",
|
||||
# With it, the route runs the identify ladder: when the best image score
|
||||
# is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE (or another photo is within
|
||||
# IMAGE_SEARCH_MIN_MARGIN of it), `text` is resolved against the
|
||||
# catalogue instead, and the response is an IdentifyOut (ImageSearchOut
|
||||
# plus matched_by / fallback_reason / ...). Left out, it is ON whenever
|
||||
# `text` is sent: a label that names the product must not lose to a 0.5
|
||||
# cosine, which it did when text only broke exact ties. `false` keeps the
|
||||
# old image-only ranking.
|
||||
text_fallback: Optional[bool] = Field(
|
||||
None,
|
||||
description="Resolve `text` when the image match cannot be confirmed; the response then "
|
||||
"carries the IdentifyOut fields. Default: on when `text` is sent. "
|
||||
"false = image-only ranking, text breaks ties only",
|
||||
)
|
||||
|
||||
@field_validator("vector")
|
||||
@@ -301,6 +305,14 @@ class ImageSearchOut(BaseModel):
|
||||
min_score: float = 0.0
|
||||
top_k: int = 0
|
||||
query_text: Optional[str] = None
|
||||
# "confirmed" only when the best match clears IMAGE_IDENTIFY_MIN_IMAGE_SCORE
|
||||
# AND leads the best different photo by IMAGE_SEARCH_MIN_MARGIN (on
|
||||
# /identify: when the ladder confirmed it). "low": show the results as a
|
||||
# list to pick from, never as the answer. "none": no results.
|
||||
match_confidence: str = "none"
|
||||
# Best score minus the best DIFFERENT photo's; None when there was no rival.
|
||||
# Image space only - None when the rows came from the text rung.
|
||||
margin: Optional[float] = None
|
||||
|
||||
|
||||
class IdentifyOut(ImageSearchOut):
|
||||
|
||||
@@ -432,6 +432,21 @@ IMAGE_SEARCH_DEFAULT_MIN_SCORE = float(os.getenv("IMAGE_SEARCH_DEFAULT_MIN_SCORE
|
||||
# product share an image and tie, so more than top_k must come back), and
|
||||
# the floor for hnsw.ef_search on that query so the index does not drop them.
|
||||
IMAGE_SEARCH_MAX_FETCH_K = int(os.getenv("IMAGE_SEARCH_MAX_FETCH_K", "100"))
|
||||
# How far the best image match must lead the best DIFFERENT photo for the
|
||||
# answer to count as confirmed (`match_confidence`). A phone photo of a card on
|
||||
# a screen measured 0.507 against its own product and 0.419 against a
|
||||
# stranger; with glare that gap closes, and a flipped order is a wrong
|
||||
# product shown with confidence. Pack sizes sharing one photo tie exactly and
|
||||
# are never each other's competitor. Unmeasured start: tune with
|
||||
# scripts/eval_identify.py.
|
||||
IMAGE_SEARCH_MIN_MARGIN = float(os.getenv("IMAGE_SEARCH_MIN_MARGIN", "0.05"))
|
||||
# Diagnosis: when set, every image-search request (vector, text, brand and the
|
||||
# top matches) is written here as one JSON file, for
|
||||
# scripts/replay_image_query.py. Blank = off. At most IMAGE_SEARCH_CAPTURE_MAX
|
||||
# files are kept; the oldest go first. The routes are public, so leave it off
|
||||
# except while chasing a report.
|
||||
IMAGE_SEARCH_CAPTURE_DIR = os.getenv("IMAGE_SEARCH_CAPTURE_DIR", "").strip()
|
||||
IMAGE_SEARCH_CAPTURE_MAX = int(os.getenv("IMAGE_SEARCH_CAPTURE_MAX", "200"))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Identify a product from a phone photo - POST /api/search/identify
|
||||
|
||||
@@ -53,10 +53,12 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
from app.infrastructure.settings import (
|
||||
IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
|
||||
IMAGE_SEARCH_DEFAULT_MIN_SCORE,
|
||||
IMAGE_SEARCH_DEFAULT_TOP_K,
|
||||
IMAGE_SEARCH_MAX_FETCH_K,
|
||||
IMAGE_SEARCH_MAX_TOP_K,
|
||||
IMAGE_SEARCH_MIN_MARGIN,
|
||||
)
|
||||
from app.services.image_embedder import EMBED_DIM
|
||||
|
||||
@@ -90,6 +92,8 @@ class ImageSearchResult:
|
||||
min_score: float = 0.0
|
||||
query_text: Optional[str] = None
|
||||
top_k: int = 0
|
||||
match_confidence: str = "none" # "confirmed" | "low" | "none"
|
||||
margin: Optional[float] = None # top score minus the best different photo's
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -226,16 +230,102 @@ def text_overlap(words: Set[str], sizes: Set[str], row: Dict[str, Any]) -> float
|
||||
return _SIZE_WEIGHT * len(sizes & row_sizes) + _WORD_WEIGHT * len(words & row_words)
|
||||
|
||||
|
||||
def size_in_name(sizes: Set[str], row: Dict[str, Any]) -> bool:
|
||||
"""Whether a pack size the label printed is in the row's OWN name.
|
||||
|
||||
`text_overlap` also counts `size_variants`, and on some products every
|
||||
pack-size row carries the SAME list (all Amul Cream rows say
|
||||
["90g", "1kg"]), so a "1kg" label ties every sibling and the tie fell to
|
||||
alphabetical order - "Amul Cream 125 ml" was shown, confirmed, for a
|
||||
photo of the 1kg pack. The name is per row; this breaks that tie.
|
||||
"""
|
||||
if not sizes:
|
||||
return False
|
||||
name_sizes = tokens(" ".join([str(row.get("product_name") or ""), str(row.get("title") or "")]))[1]
|
||||
return bool(sizes & name_sizes)
|
||||
|
||||
|
||||
def rank_key(row: Dict[str, Any]) -> tuple:
|
||||
"""Best first. Score rounded to 3 dp so siblings sharing a photo tie."""
|
||||
return (
|
||||
-round(float(row.get("score", 0.0)), 3),
|
||||
-float(row.get("text_overlap", 0.0)),
|
||||
-int(bool(row.get("size_in_name", False))),
|
||||
str(row.get("product_name") or row.get("title") or "").lower(),
|
||||
str(row.get("image_id") or ""),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# confidence
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pack sizes that share one catalog photo score identically (to pgvector's
|
||||
# float precision); anything further apart than this is a different picture.
|
||||
_SAME_PHOTO_EPS = 5e-4
|
||||
|
||||
CONFIRMED = "confirmed"
|
||||
LOW = "low"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def image_margin(rows: Sequence[Dict[str, Any]]) -> Optional[float]:
|
||||
"""How far the best row leads the best row with a DIFFERENT photo.
|
||||
|
||||
`rows` are best first by score. Siblings sharing the winner's photo are
|
||||
skipped: they are the same product in another size, which the label text
|
||||
separates, not a rival. None when nothing else is in the list - no rival
|
||||
was seen, which is not the same as a clear lead, so the caller treats it
|
||||
as unconfirmable only when the score itself is low.
|
||||
"""
|
||||
if not rows:
|
||||
return None
|
||||
try:
|
||||
top = float(rows[0].get("score", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for row in rows[1:]:
|
||||
try:
|
||||
score = float(row.get("score", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if top - score > _SAME_PHOTO_EPS:
|
||||
return top - score
|
||||
return None
|
||||
|
||||
|
||||
def match_confidence(
|
||||
rows: Sequence[Dict[str, Any]],
|
||||
min_image_score: Optional[float] = None,
|
||||
min_margin: Optional[float] = None,
|
||||
) -> Tuple[str, Optional[float]]:
|
||||
"""("confirmed" | "low" | "none", margin) for image rows, best first.
|
||||
|
||||
Confirmed needs BOTH a score at the identify floor and a lead over the
|
||||
next different photo. A photo of a card on a phone screen scores ~0.5
|
||||
against its own product, and a stranger can sit within a few hundredths
|
||||
of it: shown as the answer, that is how "Dairy Milk Lickables" came back
|
||||
as someone else's product. A client should offer the list, not pick one,
|
||||
when this is not "confirmed". The thresholds default to this module's
|
||||
settings, read at call time so scripts/eval_identify.py can vary them.
|
||||
"""
|
||||
if min_image_score is None:
|
||||
min_image_score = IMAGE_IDENTIFY_MIN_IMAGE_SCORE
|
||||
if min_margin is None:
|
||||
min_margin = IMAGE_SEARCH_MIN_MARGIN
|
||||
if not rows:
|
||||
return NONE, None
|
||||
margin = image_margin(rows)
|
||||
try:
|
||||
top = float(rows[0].get("score", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
return LOW, margin
|
||||
if top < min_image_score:
|
||||
return LOW, margin
|
||||
if margin is not None and margin < min_margin:
|
||||
return LOW, margin
|
||||
return CONFIRMED, margin
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# the search
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -345,7 +435,11 @@ def search_by_vector(
|
||||
words, sizes = tokens(label)
|
||||
for row in rows:
|
||||
row["text_overlap"] = text_overlap(words, sizes, row)
|
||||
row["size_in_name"] = size_in_name(sizes, row)
|
||||
rows.sort(key=rank_key)
|
||||
# Judged on every candidate, not the top_k shown: with top_k=1 the rival
|
||||
# would otherwise never be seen.
|
||||
confidence, margin = match_confidence(rows)
|
||||
winners = _hydrate(rows[:top_k])
|
||||
|
||||
return ImageSearchResult(
|
||||
@@ -356,4 +450,6 @@ def search_by_vector(
|
||||
min_score=min_score,
|
||||
query_text=label,
|
||||
top_k=top_k,
|
||||
match_confidence=confidence,
|
||||
margin=margin,
|
||||
)
|
||||
|
||||
133
app/services/image_search_log.py
Normal file
133
app/services/image_search_log.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""One log line per search-by-photo request, and - when asked - the request itself.
|
||||
|
||||
WHY
|
||||
---
|
||||
A colleague photographed the "Cadbury Dairy Milk Lickables" card and got
|
||||
"Milk Toned"; "Aachi Sambar Powder 100g" came back as Sakthi's "Sambar powder
|
||||
50g". Replayed on the server, both photos rank the right product first, so
|
||||
the difference is in what the app SENDS - the vector (is its on-device model
|
||||
the same as ours?), whether any OCR `text` came with it, which route it hit.
|
||||
None of that was visible. This module makes it visible:
|
||||
|
||||
* `record()` logs, for every /search/image*, /search/identify request: the
|
||||
route, the text, brand, detected brand, a short hash and the norm of the
|
||||
vector, the top three (brand, name, score) and the lead over the best
|
||||
different photo. Always on; it is one INFO line.
|
||||
* With IMAGE_SEARCH_CAPTURE_DIR set, the whole request - the 1024 floats,
|
||||
the fields, the photo when there was one - is also written there, for
|
||||
`python -m scripts.replay_image_query`. Off by default: these routes are
|
||||
public. The newest IMAGE_SEARCH_CAPTURE_MAX requests are kept.
|
||||
|
||||
Never raises: a diagnostic must not fail a search.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import struct
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from app.infrastructure import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TEXT_LOG_CHARS = 120
|
||||
|
||||
|
||||
def vector_fingerprint(vector: Optional[Sequence[float]]) -> Optional[str]:
|
||||
"""10 hex characters identifying a vector: equal for the same float32 values."""
|
||||
if vector is None:
|
||||
return None
|
||||
packed = struct.pack(f"<{len(vector)}f", *[float(v) for v in vector])
|
||||
return hashlib.sha1(packed).hexdigest()[:10]
|
||||
|
||||
|
||||
def _top(rows: Sequence[Dict[str, Any]], n: int = 3) -> List[Dict[str, Any]]:
|
||||
out = []
|
||||
for row in list(rows)[:n]:
|
||||
try:
|
||||
score = round(float(row.get("score", 0.0)), 4)
|
||||
except (TypeError, ValueError):
|
||||
score = None
|
||||
out.append({
|
||||
"brand": row.get("brand") or row.get("brand_table"),
|
||||
"product_name": row.get("product_name") or row.get("title"),
|
||||
"image_id": row.get("image_id"),
|
||||
"score": score,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _prune(folder: Path, keep: int) -> None:
|
||||
files = sorted(folder.glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
for stale in files[:max(0, len(files) - keep)]:
|
||||
for sibling in folder.glob(stale.stem + ".*"):
|
||||
sibling.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def record(
|
||||
route: str,
|
||||
*,
|
||||
vector: Optional[Sequence[float]],
|
||||
text: Optional[str],
|
||||
brand: Optional[str],
|
||||
category: Optional[str],
|
||||
top_k: int,
|
||||
rows: Sequence[Dict[str, Any]],
|
||||
detected_brand: Optional[str] = None,
|
||||
scoped_to_brand: bool = False,
|
||||
match_confidence: Optional[str] = None,
|
||||
margin: Optional[float] = None,
|
||||
matched_by: Optional[str] = None,
|
||||
fallback_reason: Optional[str] = None,
|
||||
text_fallback: Optional[bool] = None,
|
||||
photo: Optional[bytes] = None,
|
||||
) -> None:
|
||||
"""Log the request and its answer; save it when capture is on."""
|
||||
try:
|
||||
norm = math.sqrt(sum(float(v) * float(v) for v in vector)) if vector is not None else None
|
||||
top = _top(rows)
|
||||
logger.info(
|
||||
"[IMAGE_SEARCH] route=%s text=%r brand=%r detected=%r scoped=%s vec=%s norm=%s "
|
||||
"matched_by=%s reason=%s confidence=%s margin=%s top=%s",
|
||||
route, (text or "")[:_TEXT_LOG_CHARS] or None, brand, detected_brand, scoped_to_brand,
|
||||
vector_fingerprint(vector), None if norm is None else round(norm, 4),
|
||||
matched_by, fallback_reason, match_confidence,
|
||||
None if margin is None else round(margin, 4),
|
||||
[(t["brand"], t["product_name"], t["score"]) for t in top],
|
||||
)
|
||||
folder_name = settings.IMAGE_SEARCH_CAPTURE_DIR
|
||||
if not folder_name:
|
||||
return
|
||||
folder = Path(folder_name)
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
stem = time.strftime("%Y%m%dT%H%M%S") + "_" + uuid.uuid4().hex[:6]
|
||||
payload = {
|
||||
"route": route,
|
||||
"received_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"request": {
|
||||
"vector": None if vector is None else [float(v) for v in vector],
|
||||
"text": text, "brand": brand, "category": category, "top_k": top_k,
|
||||
"text_fallback": text_fallback,
|
||||
},
|
||||
"vector_fingerprint": vector_fingerprint(vector),
|
||||
"answer": {
|
||||
"matched_by": matched_by, "fallback_reason": fallback_reason,
|
||||
"match_confidence": match_confidence, "margin": margin,
|
||||
"detected_brand": detected_brand, "scoped_to_brand": scoped_to_brand, "top": top,
|
||||
},
|
||||
"photo": None,
|
||||
}
|
||||
if photo:
|
||||
photo_path = folder / f"{stem}.img"
|
||||
photo_path.write_bytes(photo)
|
||||
payload["photo"] = photo_path.name
|
||||
(folder / f"{stem}.json").write_text(json.dumps(payload), encoding="utf-8")
|
||||
_prune(folder, max(1, settings.IMAGE_SEARCH_CAPTURE_MAX))
|
||||
except Exception as exc: # noqa: BLE001 - a diagnostic must never fail the search
|
||||
logger.warning("image search log skipped: %s", exc)
|
||||
@@ -59,7 +59,7 @@ from app.infrastructure.settings import (
|
||||
IMAGE_SEARCH_MAX_TOP_K,
|
||||
OCR_MAX_CHARS,
|
||||
)
|
||||
from app.services.image_match import _STOP, ImageSearchResult, _row_text, text_overlap, tokens
|
||||
from app.services.image_match import _STOP, ImageSearchResult, _row_text, size_in_name, text_overlap, tokens
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -197,6 +197,7 @@ def label_rank_key(row: Dict[str, Any]) -> tuple:
|
||||
"""
|
||||
return (
|
||||
-float(row.get("text_overlap", 0.0)),
|
||||
-int(bool(row.get("size_in_name", False))), # the label's size in THIS row's name
|
||||
int(row.get("text_extra", 0)),
|
||||
-round(float(row.get("score", 0.0)), 3),
|
||||
str(row.get("product_name") or row.get("title") or "").lower(),
|
||||
@@ -336,6 +337,7 @@ def resolve_label(
|
||||
for row in rows:
|
||||
row["text_overlap"] = text_overlap(words, sizes, row)
|
||||
row["text_extra"] = text_extra(words, row)
|
||||
row["size_in_name"] = size_in_name(sizes, row)
|
||||
rows.sort(key=label_rank_key)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@ THE LADDER
|
||||
`fallback_reason` is the reason for the LAST step the ladder took:
|
||||
|
||||
image_below_threshold the image arm ran but did not clear the floor
|
||||
image_ambiguous it cleared the floor, but another photo scored
|
||||
within IMAGE_SEARCH_MIN_MARGIN of it
|
||||
no_image_match the image arm ran and found nothing at all
|
||||
image_embedder_unavailable no vector could be computed (no model here)
|
||||
no_text no label to fall back to and no photo to read
|
||||
@@ -46,6 +48,7 @@ from app.infrastructure.settings import (
|
||||
IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
|
||||
IMAGE_SEARCH_DEFAULT_MIN_SCORE,
|
||||
IMAGE_SEARCH_DEFAULT_TOP_K,
|
||||
IMAGE_SEARCH_MIN_MARGIN,
|
||||
)
|
||||
from app.services import ocr_service
|
||||
from app.services.image_match import ImageSearchResult, search_by_vector
|
||||
@@ -89,9 +92,11 @@ def _image_rung(
|
||||
top = float(result.rows[0].get("score", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
top = 0.0
|
||||
if top >= min_image_score:
|
||||
return result, top, None
|
||||
return result, top, "image_below_threshold"
|
||||
if top < min_image_score:
|
||||
return result, top, "image_below_threshold"
|
||||
if result.margin is not None and result.margin < IMAGE_SEARCH_MIN_MARGIN:
|
||||
return result, top, "image_ambiguous"
|
||||
return result, top, None
|
||||
|
||||
|
||||
def identify_product(
|
||||
|
||||
@@ -1861,7 +1861,14 @@ def lexical_search(
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
results.sort(key=lambda r: (r.get("lex_tier", 9), r.get("distance", 9.0)))
|
||||
# A row with a NULL `embedding` comes back with distance None (the key IS
|
||||
# present), and one None among floats made this sort raise - which threw
|
||||
# away every row of an unscoped search, the label resolver's lexical arm.
|
||||
def _distance(r: Dict[str, Any]) -> float:
|
||||
d = r.get("distance")
|
||||
return 9.0 if d is None else float(d)
|
||||
|
||||
results.sort(key=lambda r: (r.get("lex_tier", 9), _distance(r)))
|
||||
return results[:limit]
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,24 @@ Every catalogue row carries the same kind of vector in `img_vector`
|
||||
index). These two endpoints turn the app's vector - or a photo - into
|
||||
product cards.
|
||||
|
||||
## Recommended client flow (read this first)
|
||||
|
||||
A colleague photographed the **Cadbury Dairy Milk Lickables** card and got
|
||||
"Milk Toned"; **Aachi Sambar Powder 100g** came back as Sakthi's "Sambar
|
||||
powder 50g". A photo of a card on a screen scores only ~0.5 against its own
|
||||
product, and a stranger can sit a few hundredths behind it. So:
|
||||
|
||||
1. **Send the photo to `POST /api/search/identify`** (preferred). The server
|
||||
embeds it with the same model as the catalogue, reads the card/label text
|
||||
with OCR, and lets the text decide when the image cannot. No model-parity
|
||||
risk. *Or*, keeping the on-device vector, **always send the full OCR read
|
||||
as `text`** to `/image-vector`: the label then decides (see `text_fallback`).
|
||||
2. **Crop to the pack** (a framing guide in the camera UI). Measured: a card
|
||||
photographed whole scored 0.51 for the right product; cropped to the pack
|
||||
image, 0.94.
|
||||
3. **Honour `match_confidence`.** Only `"confirmed"` is an answer. On `"low"`
|
||||
show the results as a list to pick from - never auto-select the first.
|
||||
|
||||
```
|
||||
POST /api/search/image-vector JSON {vector[1024], text?, brand?, category?, top_k?, min_score?, text_fallback?}
|
||||
GET /api/search/image-vector query string: vector=<base64 or csv>&text=...&top_k=... (see "GET variant")
|
||||
@@ -55,6 +73,14 @@ a floor, and read `score` on each result.
|
||||
| `category` | both | – | `ILIKE` filter on the category column |
|
||||
| `top_k` | both | 10 | 1–50 |
|
||||
| `min_score` | both | 0.0 | −1…1; drop matches below it |
|
||||
| `text_fallback` | `/image-vector` POST only | on when `text` is sent | run the identify ladder (below); `false` = image-only ranking, `text` breaks ties only |
|
||||
|
||||
Every response also carries:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `match_confidence` | `"confirmed"`: the best match scored ≥ `IMAGE_IDENTIFY_MIN_IMAGE_SCORE` (0.70) **and** leads the best *different* photo by ≥ `IMAGE_SEARCH_MIN_MARGIN` (0.05) - or, on the identify shape, the ladder confirmed it. `"low"`: a list to pick from, not an answer. `"none"`: no results. |
|
||||
| `margin` | best image score minus the best different photo's. Pack sizes sharing one photo are not rivals. `null` when there was no rival, or when the rows came from the label text. |
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -170,7 +196,8 @@ only ~0.63 against the catalogue's render of the same pack** (measured), so
|
||||
ladder and tells you which rung answered:
|
||||
|
||||
```
|
||||
photo ─► img_vector search ─► best score ≥ IMAGE_IDENTIFY_MIN_IMAGE_SCORE (0.70)?
|
||||
photo ─► img_vector search ─► best score ≥ IMAGE_IDENTIFY_MIN_IMAGE_SCORE (0.70)
|
||||
│ and ≥ IMAGE_SEARCH_MIN_MARGIN (0.05) ahead of the next different photo?
|
||||
│ yes → matched_by "image_vector", fallback_reason null (confirmed)
|
||||
▼ no
|
||||
label text = your `text` (ocr_source "client")
|
||||
@@ -213,6 +240,7 @@ show it as such, or ask for another shot.
|
||||
|---|---|
|
||||
| `null` | the image match cleared the floor |
|
||||
| `image_below_threshold` | image ran, best score under the floor → the label decided (on a `"text"` answer) |
|
||||
| `image_ambiguous` | best score cleared the floor, but a different photo was within `IMAGE_SEARCH_MIN_MARGIN` |
|
||||
| `no_image_match` | image ran and found nothing → the label decided |
|
||||
| `image_embedder_unavailable` | this deployment has no image model → text only |
|
||||
| `no_text` | `/image-vector` with `text_fallback` but no `text` |
|
||||
@@ -238,11 +266,22 @@ on the first photo that needs it (~2 s), then costs ~1–2 s per photo at the
|
||||
and loaded. `ENABLE_SERVER_OCR=false` turns it off; the route then relies on
|
||||
client `text`.
|
||||
|
||||
**The app's route.** `POST /api/search/image-vector` accepts
|
||||
`"text_fallback": true`. Off (the default) the response is exactly what it
|
||||
has always been. On, the same ladder runs with the app's vector and `text`
|
||||
(no photo, so never server OCR), and the response is the identify shape
|
||||
above. Not on the GET variant.
|
||||
**The app's route.** `POST /api/search/image-vector` runs the same ladder
|
||||
with the app's vector and `text` (no photo, so never server OCR) whenever
|
||||
`text` is sent, and answers with the identify shape above. Before
|
||||
2026-09-28 it did so only with `"text_fallback": true`, and otherwise the
|
||||
label only broke exact ties - a label naming the product lost to any photo
|
||||
that scored 0.01 higher. Send `"text_fallback": false` to keep that old
|
||||
image-only ranking; without `text` nothing changes. Not on the GET variant.
|
||||
|
||||
**Diagnosing a wrong match.** Every search-by-photo request logs one
|
||||
`[IMAGE_SEARCH]` line (route, text, brand, a fingerprint of the vector, top 3
|
||||
with scores, margin). With `IMAGE_SEARCH_CAPTURE_DIR` set the full request -
|
||||
and the photo, on `/image` and `/identify` - is saved for
|
||||
`python -m scripts.replay_image_query <file> [--photo shot.jpg]`, which
|
||||
replays it and, given the photo, checks that the app's on-device vector
|
||||
matches the server's (cosine ≥ 0.99). `python -m scripts.eval_identify`
|
||||
measures accuracy and the thresholds on synthetic and real card photos.
|
||||
|
||||
## When the product is not in the catalogue (capture-to-catalog)
|
||||
|
||||
|
||||
282
scripts/eval_identify.py
Normal file
282
scripts/eval_identify.py
Normal file
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
How often does a photo of a product card come back as that product?
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
Colleagues photograph catalogue cards with the Nearle app, and two came back
|
||||
as other brands' products. The thresholds that decide when an image match is
|
||||
"confirmed" (IMAGE_IDENTIFY_MIN_IMAGE_SCORE, IMAGE_SEARCH_MIN_MARGIN) were
|
||||
starting guesses. This measures them, on the database this process points at.
|
||||
|
||||
WHAT IT MEASURES
|
||||
----------------
|
||||
For N sampled catalogue products it downloads the image the card shows and
|
||||
builds four synthetic "photos" of it:
|
||||
|
||||
clean the catalogue image itself (the ceiling: should be 100%)
|
||||
card the image on a white card with text lines under it
|
||||
screen that card dimmed, desaturated, blurred, blue-shifted, JPEG'd
|
||||
tight the screen shot cropped to the pack (a framing guide)
|
||||
|
||||
and, with --photos DIR, real photos named `<brand_table>__<image_id>.jpg`
|
||||
(e.g. `brand_cadbury__cadbury_dairy_milk_lickables_20g.jpg`).
|
||||
|
||||
Each photo is searched three ways:
|
||||
|
||||
image search_by_vector, no text - what the app gets today
|
||||
image+text the identify ladder with the product name as the label
|
||||
(what the card's title reads as)
|
||||
ladder+ocr (real photos only) the identify ladder with server OCR
|
||||
|
||||
and scored: top-1 exact, top-1 "same photo" (a pack-size sibling sharing
|
||||
the target's picture - right product, maybe wrong size), top-3 exact, and
|
||||
the one that matters most - WRONG BUT CONFIRMED: a different product shown
|
||||
with match_confidence "confirmed". Tune the thresholds until that is zero
|
||||
without making everything "low".
|
||||
|
||||
USAGE
|
||||
-----
|
||||
python -m scripts.eval_identify --n 40
|
||||
python -m scripts.eval_identify --n 40 --brands Cadbury,Aachi,Sakthi
|
||||
python -m scripts.eval_identify --photos ~/card_photos --n 0
|
||||
python -m scripts.eval_identify --n 40 --min-margin 0.08 --min-image-score 0.65
|
||||
|
||||
Read-only. Downloads catalogue images (honours the downloader's pacing).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import random
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
Key = Tuple[str, str] # (brand_table, image_id)
|
||||
MODES = ("clean", "card", "screen", "tight")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# catalogue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_catalogue(brands: Optional[List[str]]) -> Dict[Key, Dict[str, Any]]:
|
||||
"""Every row with an image vector: (table, image_id) -> name and source URL."""
|
||||
from app.services import vector_store as vs
|
||||
|
||||
conn = vs._connect()
|
||||
if conn is None:
|
||||
raise SystemExit("no database connection")
|
||||
wanted = {vs._table_name(b) for b in brands} if brands else None
|
||||
out: Dict[Key, Dict[str, Any]] = {}
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
for suffix in vs._list_brand_table_suffixes(cur):
|
||||
table = f"brand_{suffix}"
|
||||
if wanted is not None and table not in wanted:
|
||||
continue
|
||||
try:
|
||||
cur.execute(f"SELECT image_id, product_name, img_vector_src FROM {table} "
|
||||
f"WHERE img_vector IS NOT NULL AND img_vector_src IS NOT NULL")
|
||||
except Exception: # noqa: BLE001 - a table without the column
|
||||
conn.rollback()
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
continue
|
||||
for image_id, name, src in cur.fetchall():
|
||||
out[(table, str(image_id))] = {"name": name, "src": src}
|
||||
finally:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# synthetic photos
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def synthetic_photo(raw: bytes, mode: str) -> Optional[bytes]:
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageEnhance, ImageFilter
|
||||
|
||||
try:
|
||||
img = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if mode != "clean":
|
||||
w, h = img.size
|
||||
side = max(w, h)
|
||||
pack_h = int(h * side / w)
|
||||
canvas = Image.new("RGB", (int(side * 1.3), int(side * 0.1) + pack_h + int(side * 0.9)), (255, 255, 255))
|
||||
canvas.paste(img.resize((side, pack_h)), (int(side * .15), int(side * .1)))
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
for k in range(6):
|
||||
y = int(side * .1) + pack_h + int(side * .1) + k * int(side * .1)
|
||||
draw.rectangle([int(side * .15), y, int(side * (0.6 + .1 * (k % 3))), y + int(side * .04)],
|
||||
fill=(40, 40, 40))
|
||||
img = canvas
|
||||
if mode in ("screen", "tight"):
|
||||
img = ImageEnhance.Brightness(img).enhance(0.85)
|
||||
img = ImageEnhance.Color(img).enhance(0.8)
|
||||
img = img.filter(ImageFilter.GaussianBlur(2))
|
||||
px = np.array(img).astype(np.float32)
|
||||
px[..., 2] *= 1.08
|
||||
img = Image.fromarray(np.clip(px, 0, 255).astype(np.uint8))
|
||||
if mode == "tight":
|
||||
img = img.crop((int(side * .15), int(side * .1), int(side * 1.15), int(side * .1) + pack_h))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", quality=70)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Tally:
|
||||
def __init__(self) -> None:
|
||||
self.n = 0
|
||||
self.top1 = 0
|
||||
self.top1_same_photo = 0
|
||||
self.top3 = 0
|
||||
self.confirmed = 0
|
||||
self.wrong_confirmed = 0
|
||||
self.misses: List[str] = []
|
||||
|
||||
def add(self, target: Key, rows: List[Dict[str, Any]], confidence: str,
|
||||
catalogue: Dict[Key, Dict[str, Any]]) -> None:
|
||||
self.n += 1
|
||||
keys = [(str(r.get("brand_table") or ""), str(r.get("image_id") or "")) for r in rows]
|
||||
src = catalogue[target]["src"]
|
||||
hit1 = bool(keys) and keys[0] == target
|
||||
same_photo = bool(keys) and (hit1 or catalogue.get(keys[0], {}).get("src") == src)
|
||||
self.top1 += hit1
|
||||
self.top1_same_photo += same_photo
|
||||
self.top3 += target in keys[:3]
|
||||
if confidence == "confirmed":
|
||||
self.confirmed += 1
|
||||
if not same_photo:
|
||||
self.wrong_confirmed += 1
|
||||
if not same_photo and len(self.misses) < 8:
|
||||
got = rows[0].get("product_name") if rows else None
|
||||
self.misses.append(f"{catalogue[target]['name']} -> {got} ({confidence})")
|
||||
|
||||
def line(self, label: str) -> str:
|
||||
if not self.n:
|
||||
return f" {label:22s} (no photos)"
|
||||
pct = lambda k: f"{100.0 * k / self.n:5.1f}%" # noqa: E731
|
||||
return (f" {label:22s} n={self.n:3d} top1 {pct(self.top1)} same-photo {pct(self.top1_same_photo)} "
|
||||
f"top3 {pct(self.top3)} confirmed {pct(self.confirmed)} "
|
||||
f"WRONG+confirmed {self.wrong_confirmed}")
|
||||
|
||||
|
||||
def _ladder_confidence(result) -> str:
|
||||
from app.services.capture_discovery import is_confirmed
|
||||
|
||||
if not result.search.rows:
|
||||
return "none"
|
||||
return "confirmed" if is_confirmed(result.matched_by, result.fallback_reason) else "low"
|
||||
|
||||
|
||||
def evaluate(photos: List[Tuple[str, Key, bytes, bool]], catalogue: Dict[Key, Dict[str, Any]],
|
||||
top_k: int) -> Dict[Tuple[str, str], Tally]:
|
||||
from app.services import image_embedder
|
||||
from app.services.image_match import search_by_vector
|
||||
from app.services import product_identify
|
||||
from app.services.product_identify import identify_product
|
||||
|
||||
floor = product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE
|
||||
tallies: Dict[Tuple[str, str], Tally] = defaultdict(Tally)
|
||||
for mode, target, data, real in photos:
|
||||
vector = image_embedder.embedding_for_bytes(data)
|
||||
if vector is None:
|
||||
continue
|
||||
image = search_by_vector(vector, top_k=top_k)
|
||||
tallies[(mode, "image")].add(target, image.rows, image.match_confidence, catalogue)
|
||||
|
||||
label = catalogue[target]["name"]
|
||||
labelled = identify_product(vector=vector, image_bytes=None, text=label, top_k=top_k,
|
||||
min_image_score=floor)
|
||||
tallies[(mode, "image+text")].add(target, labelled.search.rows, _ladder_confidence(labelled), catalogue)
|
||||
|
||||
if real:
|
||||
ocr = identify_product(vector=vector, image_bytes=data, top_k=top_k, min_image_score=floor)
|
||||
tallies[(mode, "ladder+ocr")].add(target, ocr.search.rows, _ladder_confidence(ocr), catalogue)
|
||||
return tallies
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--n", type=int, default=30, help="catalogue products to sample (0 = none)")
|
||||
parser.add_argument("--brands", help="comma-separated brands to sample from (default: all)")
|
||||
parser.add_argument("--photos", type=Path, help="folder of real photos named <brand_table>__<image_id>.jpg")
|
||||
parser.add_argument("--modes", default=",".join(MODES), help=f"synthetic modes (default: {','.join(MODES)})")
|
||||
parser.add_argument("--seed", type=int, default=7)
|
||||
parser.add_argument("--top-k", type=int, default=5)
|
||||
parser.add_argument("--min-margin", type=float, help="override IMAGE_SEARCH_MIN_MARGIN for this run")
|
||||
parser.add_argument("--min-image-score", type=float, help="override IMAGE_IDENTIFY_MIN_IMAGE_SCORE")
|
||||
args = parser.parse_args()
|
||||
|
||||
from app.services import image_embedder, image_match, product_identify
|
||||
from app.services.image_vector import download_image_bytes
|
||||
|
||||
# Both modules read these at call time; identify_product's own floor is
|
||||
# passed explicitly in evaluate().
|
||||
if args.min_margin is not None:
|
||||
image_match.IMAGE_SEARCH_MIN_MARGIN = args.min_margin
|
||||
product_identify.IMAGE_SEARCH_MIN_MARGIN = args.min_margin
|
||||
if args.min_image_score is not None:
|
||||
image_match.IMAGE_IDENTIFY_MIN_IMAGE_SCORE = args.min_image_score
|
||||
product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE = args.min_image_score
|
||||
if not image_embedder.available():
|
||||
raise SystemExit("the image embedding model is not available here")
|
||||
|
||||
brands = [b.strip() for b in args.brands.split(",")] if args.brands else None
|
||||
catalogue = load_catalogue(brands)
|
||||
print(f"catalogue: {len(catalogue)} rows with an image vector"
|
||||
f"{' in ' + ', '.join(brands) if brands else ''}")
|
||||
|
||||
photos: List[Tuple[str, Key, bytes, bool]] = []
|
||||
modes = [m for m in args.modes.split(",") if m in MODES]
|
||||
sample = random.Random(args.seed).sample(sorted(catalogue), min(args.n, len(catalogue)))
|
||||
for target in sample:
|
||||
raw = download_image_bytes(catalogue[target]["src"])
|
||||
if not raw:
|
||||
print(f" skip (download failed): {catalogue[target]['name']}")
|
||||
continue
|
||||
for mode in modes:
|
||||
data = synthetic_photo(raw, mode)
|
||||
if data:
|
||||
photos.append((mode, target, data, False))
|
||||
|
||||
if args.photos:
|
||||
for path in sorted(args.photos.iterdir()):
|
||||
if "__" not in path.stem:
|
||||
continue
|
||||
table, image_id = path.stem.split("__", 1)
|
||||
if (table, image_id) not in catalogue:
|
||||
print(f" skip (not in catalogue): {path.name}")
|
||||
continue
|
||||
photos.append(("real", (table, image_id), path.read_bytes(), True))
|
||||
|
||||
tallies = evaluate(photos, catalogue, args.top_k)
|
||||
print(f"\nthresholds: min_image_score={product_identify.IMAGE_IDENTIFY_MIN_IMAGE_SCORE} "
|
||||
f"min_margin={product_identify.IMAGE_SEARCH_MIN_MARGIN}\n")
|
||||
for mode in modes + ["real"]:
|
||||
for way in ("image", "image+text", "ladder+ocr"):
|
||||
tally = tallies.get((mode, way))
|
||||
if tally and tally.n:
|
||||
print(tally.line(f"{mode} / {way}"))
|
||||
print("\nexamples of misses (target -> top-1):")
|
||||
for (mode, way), tally in sorted(tallies.items()):
|
||||
for miss in tally.misses[:3]:
|
||||
print(f" [{mode} / {way}] {miss}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
173
scripts/replay_image_query.py
Normal file
173
scripts/replay_image_query.py
Normal file
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Replay a saved search-by-photo request, and check the app's model against ours.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
A colleague photographs a product card and the app shows a different product
|
||||
("Cadbury Dairy Milk Lickables" came back as "Milk Toned", "Aachi Sambar
|
||||
Powder 100g" as Sakthi's "Sambar powder 50g"). On the server, synthetic
|
||||
photos of those cards rank the right product first, so the question is what
|
||||
the app sent. With IMAGE_SEARCH_CAPTURE_DIR set, every request is saved as
|
||||
JSON (app/services/image_search_log.py); this script replays one.
|
||||
|
||||
WHAT IT PRINTS
|
||||
--------------
|
||||
1. The request: route, text, brand, the vector's fingerprint and norm, and
|
||||
what the server answered at the time.
|
||||
2. The image-only ranking (search_by_vector) and the identify ladder
|
||||
(identify_product) for the same vector and text, against the database
|
||||
this process points at. Both, whichever route was called, so you can see
|
||||
whether sending text would have fixed it.
|
||||
3. With --photo: the photo embedded HERE, with our model and preprocessing,
|
||||
and the cosine between that and the app's vector. The same photo through
|
||||
the same model gives >= 0.99. Well below that, the app's on-device model
|
||||
or preprocessing is not ours, and every image score it sends is in a
|
||||
different space: put the app's .tflite in app/services/models/mobilenet/
|
||||
and run `python -m scripts.backfill_image_vectors --all --force --apply`.
|
||||
A saved /identify or /image request carries its photo; use --photo for
|
||||
the vector route, with the picture the colleague took.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
python -m scripts.replay_image_query data/image_search_requests/20260928T101500_ab12cd.json
|
||||
python -m scripts.replay_image_query saved.json --photo lickables_card.jpg
|
||||
python -m scripts.replay_image_query --list data/image_search_requests
|
||||
|
||||
Read-only: nothing here writes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.image_search_log import vector_fingerprint # noqa: E402
|
||||
|
||||
PARITY_OK = 0.99
|
||||
|
||||
|
||||
def _cosine(a: Sequence[float], b: Sequence[float]) -> float:
|
||||
dot = sum(float(x) * float(y) for x, y in zip(a, b))
|
||||
na = math.sqrt(sum(float(x) ** 2 for x in a))
|
||||
nb = math.sqrt(sum(float(y) ** 2 for y in b))
|
||||
return dot / (na * nb) if na and nb else 0.0
|
||||
|
||||
|
||||
def _print_rows(title: str, rows: List[Dict[str, Any]], limit: int = 5) -> None:
|
||||
print(f" {title}")
|
||||
if not rows:
|
||||
print(" (nothing)")
|
||||
for i, row in enumerate(rows[:limit], start=1):
|
||||
score = row.get("score")
|
||||
score_s = f"{float(score):.4f}" if score is not None else " - "
|
||||
print(f" {i}. {score_s} {str(row.get('brand') or row.get('brand_table') or ''):22.22s} "
|
||||
f"{row.get('product_name') or row.get('title')} (overlap {row.get('text_overlap', 0)})")
|
||||
|
||||
|
||||
def _list(folder: Path) -> int:
|
||||
for path in sorted(folder.glob("*.json")):
|
||||
try:
|
||||
saved = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
req, ans = saved.get("request", {}), saved.get("answer", {})
|
||||
top = (ans.get("top") or [{}])[0]
|
||||
print(f"{path.name} {saved.get('route'):16s} text={str(req.get('text'))[:40]!r:44s} "
|
||||
f"-> {top.get('product_name')} ({top.get('score')}) [{ans.get('match_confidence')}]")
|
||||
return 0
|
||||
|
||||
|
||||
def replay(path: Path, photo: Optional[Path], top_k: int) -> int:
|
||||
from app.services import image_embedder
|
||||
from app.services.image_match import search_by_vector
|
||||
from app.services.product_identify import identify_product
|
||||
|
||||
saved = json.loads(path.read_text(encoding="utf-8"))
|
||||
req = saved.get("request") or {}
|
||||
answer = saved.get("answer") or {}
|
||||
vector = req.get("vector")
|
||||
text = req.get("text")
|
||||
|
||||
print(f"== {path.name} route={saved.get('route')} received={saved.get('received_at')}")
|
||||
print(f" text={text!r} brand={req.get('brand')!r} category={req.get('category')!r} "
|
||||
f"text_fallback={req.get('text_fallback')!r}")
|
||||
if vector is not None:
|
||||
norm = math.sqrt(sum(v * v for v in vector))
|
||||
print(f" vector: {len(vector)} floats, norm {norm:.4f}, fingerprint {vector_fingerprint(vector)}")
|
||||
else:
|
||||
print(" vector: none (the server had no model when this was sent)")
|
||||
print(f" answered then: matched_by={answer.get('matched_by')} reason={answer.get('fallback_reason')} "
|
||||
f"confidence={answer.get('match_confidence')} margin={answer.get('margin')}")
|
||||
_print_rows("top then:", answer.get("top") or [])
|
||||
|
||||
photo_bytes: Optional[bytes] = None
|
||||
if photo is not None:
|
||||
photo_bytes = photo.read_bytes()
|
||||
elif saved.get("photo"):
|
||||
candidate = path.parent / saved["photo"]
|
||||
if candidate.exists():
|
||||
photo_bytes = candidate.read_bytes()
|
||||
|
||||
ours: Optional[List[float]] = None
|
||||
if photo_bytes is not None:
|
||||
if not image_embedder.available():
|
||||
print("\n (no embedding model here - cannot run the parity check)")
|
||||
else:
|
||||
ours = image_embedder.embedding_for_bytes(photo_bytes)
|
||||
if ours is None:
|
||||
print("\n (the photo could not be decoded)")
|
||||
elif vector is not None:
|
||||
cos = _cosine(vector, ours)
|
||||
verdict = "SAME model and preprocessing" if cos >= PARITY_OK else (
|
||||
"DIFFERENT - the app's vectors are not comparable with img_vector")
|
||||
print(f"\n model parity: cosine(app vector, server vector of the same photo) = {cos:.4f}"
|
||||
f" -> {verdict}")
|
||||
|
||||
query = vector if vector is not None else ours
|
||||
if query is None:
|
||||
print("\n nothing to replay: no vector in the request and none computed from a photo")
|
||||
return 1
|
||||
|
||||
print("\n-- replayed now, against this process's database --")
|
||||
image = search_by_vector(query, text=text, brand=req.get("brand"), category=req.get("category"),
|
||||
top_k=top_k)
|
||||
print(f" image only: confidence={image.match_confidence} margin={image.margin} "
|
||||
f"detected_brand={image.detected_brand} scoped={image.scoped_to_brand}")
|
||||
_print_rows("ranking:", image.rows)
|
||||
|
||||
identified = identify_product(vector=query, image_bytes=photo_bytes, text=text, brand=req.get("brand"),
|
||||
category=req.get("category"), top_k=top_k)
|
||||
print(f"\n identify ladder: matched_by={identified.matched_by} reason={identified.fallback_reason} "
|
||||
f"ocr_source={identified.ocr_source} ocr_text={identified.ocr_text!r}")
|
||||
_print_rows("ranking:", identified.search.rows)
|
||||
|
||||
if ours is not None and vector is not None and _cosine(vector, ours) < PARITY_OK:
|
||||
print("\n and with the SERVER's vector of the same photo:")
|
||||
_print_rows("ranking:", search_by_vector(ours, text=text, brand=req.get("brand"),
|
||||
category=req.get("category"), top_k=top_k).rows)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("request", nargs="?", help="a saved request JSON")
|
||||
parser.add_argument("--photo", type=Path, help="the photo the request was made from (parity check)")
|
||||
parser.add_argument("--list", type=Path, metavar="DIR", help="summarise every saved request in DIR")
|
||||
parser.add_argument("--top-k", type=int, default=5)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list:
|
||||
return _list(args.list)
|
||||
if not args.request:
|
||||
parser.error("give a saved request JSON, or --list DIR")
|
||||
return replay(Path(args.request), args.photo, args.top_k)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -4,8 +4,10 @@ The ladder's decisions are tested in tests/test_product_identify.py and the
|
||||
two arms in tests/test_image_match.py and tests/test_label_match.py. Here
|
||||
the assertions are about the HTTP contract: status codes, the guards the
|
||||
photo route shares with /search/image, the IdentifyOut fields, and - the
|
||||
one that protects the app in the field - that /search/image-vector without
|
||||
`text_fallback` answers exactly as it did before this route existed.
|
||||
one that protects the app in the field - that /search/image-vector with
|
||||
`text_fallback: false` answers exactly as it did before this route existed.
|
||||
Left out, the flag is on whenever `text` is sent: a label naming the product
|
||||
must not lose to a 0.5 cosine (the "Lickables came back as Milk Toned" report).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -44,7 +46,7 @@ def _row(score: float = 0.631, image_id: str = "britannia_marie_gold_300g") -> D
|
||||
|
||||
|
||||
IMAGE_SEARCH_KEYS = {"results", "total", "detected_brand", "scoped_to_brand", "scope_fallback",
|
||||
"min_score", "top_k", "query_text"}
|
||||
"min_score", "top_k", "query_text", "match_confidence", "margin"}
|
||||
IDENTIFY_KEYS = IMAGE_SEARCH_KEYS | {"matched_by", "ocr_text", "ocr_source", "image_top_score", "fallback_reason"}
|
||||
# Capture-to-catalog fields: always present, None unless discovery ran (see
|
||||
# tests/test_capture_discovery.py). Additive - no existing key changed.
|
||||
@@ -248,21 +250,50 @@ def test_a_confident_image_never_touches_ocr(client, embedder_ok, ladder):
|
||||
# /search/image-vector - text_fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_image_vector_without_the_flag_is_unchanged(client, ladder, fake_identify, monkeypatch):
|
||||
@pytest.fixture
|
||||
def image_only(monkeypatch):
|
||||
seen = []
|
||||
|
||||
def fake_search(vector, text=None, brand=None, category=None, top_k=10, min_score=0.0):
|
||||
seen.append(text)
|
||||
return ImageSearchResult(rows=[_row(0.63)], detected_brand="Britannia", scoped_to_brand=True,
|
||||
min_score=min_score, query_text=text, top_k=top_k)
|
||||
min_score=min_score, query_text=text, top_k=top_k,
|
||||
match_confidence="low", margin=0.02)
|
||||
|
||||
monkeypatch.setattr(search_router, "search_by_vector", fake_search)
|
||||
return seen
|
||||
|
||||
res = client.post("/api/search/image-vector", json={"vector": _unit(), "text": "Marie Gold 300 g"})
|
||||
|
||||
def test_image_vector_with_the_flag_false_is_unchanged(client, ladder, fake_identify, image_only):
|
||||
res = client.post("/api/search/image-vector",
|
||||
json={"vector": _unit(), "text": "Marie Gold 300 g", "text_fallback": False})
|
||||
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
assert set(body) == IMAGE_SEARCH_KEYS
|
||||
assert body["match_confidence"] == "low" and body["margin"] == 0.02
|
||||
assert image_only == ["Marie Gold 300 g"] and fake_identify.calls == [] and ladder.text_calls == []
|
||||
|
||||
|
||||
def test_image_vector_without_text_or_flag_is_image_only(client, ladder, fake_identify, image_only):
|
||||
res = client.post("/api/search/image-vector", json={"vector": _unit()})
|
||||
|
||||
assert res.status_code == 200, res.text
|
||||
assert set(res.json()) == IMAGE_SEARCH_KEYS
|
||||
assert seen == ["Marie Gold 300 g"] and fake_identify.calls == [] and ladder.text_calls == []
|
||||
assert image_only == [None] and fake_identify.calls == []
|
||||
|
||||
|
||||
def test_image_vector_with_text_and_no_flag_lets_the_label_decide(client, ladder, image_only):
|
||||
"""The field report: the image arm's 0.63 must not outrank a label that
|
||||
names the product."""
|
||||
res = client.post("/api/search/image-vector", json={"vector": _unit(), "text": "Marie Gold 300 g"})
|
||||
|
||||
body = res.json()
|
||||
assert res.status_code == 200, res.text
|
||||
assert set(body) == IDENTIFY_KEYS
|
||||
assert body["matched_by"] == "text" and body["match_confidence"] == "confirmed"
|
||||
assert body["results"][0]["image_id"] == "txt_marie_300" and body["margin"] is None
|
||||
assert image_only == [] # the router's image-only path never ran
|
||||
|
||||
|
||||
def test_image_vector_with_the_flag_runs_the_ladder_and_adds_the_fields(client, ladder):
|
||||
@@ -285,6 +316,7 @@ def test_image_vector_with_the_flag_but_no_text_says_no_text(client, ladder):
|
||||
assert res.status_code == 200, res.text
|
||||
assert body["matched_by"] == "image_vector" and body["fallback_reason"] == "no_text"
|
||||
assert body["results"][0]["score"] == 0.63
|
||||
assert body["match_confidence"] == "low" # shown as a list to pick from
|
||||
|
||||
|
||||
def test_image_vector_with_the_flag_still_validates_the_vector(client, ladder):
|
||||
|
||||
205
tests/test_image_confidence.py
Normal file
205
tests/test_image_confidence.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""When a search-by-photo answer counts as confirmed, and what is logged.
|
||||
|
||||
A photo of the "Cadbury Dairy Milk Lickables" card on a phone screen scored
|
||||
0.507 against its own product and 0.419 against a stranger; the Aachi sambar
|
||||
card 0.560 against 0.541. Those orders flip with a little glare, and the app
|
||||
then showed a different product as THE answer. So an image match is
|
||||
"confirmed" only with a score at the identify floor AND a lead over the best
|
||||
different photo - pack sizes that share one photo never count as rivals.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
from app.infrastructure import settings
|
||||
from app.services import image_match, image_search_log
|
||||
from app.services import product_identify as pi
|
||||
from app.services.image_match import ImageSearchResult, image_margin, match_confidence
|
||||
|
||||
|
||||
def _rows(*scores: float) -> List[Dict[str, Any]]:
|
||||
return [{"image_id": f"p{i}", "product_name": f"Product {i}", "brand": "Cadbury", "score": s}
|
||||
for i, s in enumerate(scores)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# image_margin / match_confidence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_margin_skips_pack_sizes_that_share_the_winners_photo():
|
||||
# 100g and 200g share one photo and tie; the rival is the 0.761.
|
||||
assert image_margin(_rows(0.960, 0.960, 0.9602, 0.761)) == pytest.approx(0.199)
|
||||
|
||||
|
||||
def test_no_rival_means_no_margin():
|
||||
assert image_margin(_rows(0.93, 0.93)) is None
|
||||
assert image_margin([]) is None
|
||||
|
||||
|
||||
def test_a_high_score_with_a_clear_lead_is_confirmed():
|
||||
assert match_confidence(_rows(0.94, 0.58), min_image_score=0.70, min_margin=0.05)[0] == "confirmed"
|
||||
|
||||
|
||||
def test_a_high_score_with_a_rival_close_behind_is_low():
|
||||
label, margin = match_confidence(_rows(0.74, 0.72), min_image_score=0.70, min_margin=0.05)
|
||||
assert label == "low" and margin == pytest.approx(0.02)
|
||||
|
||||
|
||||
def test_a_card_photographed_on_a_screen_is_low_even_when_it_ranks_first():
|
||||
assert match_confidence(_rows(0.507, 0.419), min_image_score=0.70, min_margin=0.05)[0] == "low"
|
||||
|
||||
|
||||
def test_a_high_score_whose_only_neighbours_are_its_own_sizes_is_confirmed():
|
||||
assert match_confidence(_rows(0.96, 0.96), min_image_score=0.70, min_margin=0.05)[0] == "confirmed"
|
||||
|
||||
|
||||
def test_no_rows_is_none():
|
||||
assert match_confidence([]) == ("none", None)
|
||||
|
||||
|
||||
def test_search_by_vector_judges_confidence_on_every_candidate_not_just_top_k(monkeypatch):
|
||||
candidates = [
|
||||
{"image_id": "lickables_20g", "product_name": "Cadbury Dairy Milk Lickables 20g",
|
||||
"brand": "Cadbury", "brand_table": "brand_cadbury", "distance": 0.20},
|
||||
{"image_id": "milk_toned", "product_name": "Milk Toned",
|
||||
"brand": "Own Products", "brand_table": "brand_own_products", "distance": 0.22},
|
||||
]
|
||||
monkeypatch.setattr("app.services.vector_store.image_vector_search",
|
||||
lambda *a, **k: [dict(c) for c in candidates])
|
||||
monkeypatch.setattr(image_match, "_hydrate", lambda rows: rows)
|
||||
|
||||
out = image_match.search_by_vector([1.0] + [0.0] * 1023, top_k=1)
|
||||
|
||||
assert [r["image_id"] for r in out.rows] == ["lickables_20g"]
|
||||
assert out.match_confidence == "low" and out.margin == pytest.approx(0.02)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# the identify ladder: an ambiguous image lets the label decide
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def ladder(monkeypatch):
|
||||
state: Dict[str, Any] = {"margin": None, "text_calls": 0}
|
||||
|
||||
def image(vector, text=None, brand=None, category=None, top_k=10, min_score=0.0):
|
||||
return ImageSearchResult(rows=_rows(0.74, 0.72), margin=state["margin"], top_k=top_k)
|
||||
|
||||
def text(label, brand=None, category=None, top_k=10):
|
||||
state["text_calls"] += 1
|
||||
return ImageSearchResult(rows=[{"image_id": "by_label", "score": 0.8, "text_overlap": 7.0}])
|
||||
|
||||
monkeypatch.setattr(pi, "search_by_vector", image)
|
||||
monkeypatch.setattr(pi, "resolve_label", text)
|
||||
monkeypatch.setattr(pi, "is_confident", lambda rows: bool(rows))
|
||||
monkeypatch.setattr(pi, "IMAGE_SEARCH_MIN_MARGIN", 0.05)
|
||||
return state
|
||||
|
||||
|
||||
def test_an_image_that_clears_the_floor_but_barely_leads_falls_to_the_label(ladder):
|
||||
ladder["margin"] = 0.02
|
||||
|
||||
out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None,
|
||||
text="Cadbury Dairy Milk Lickables 20g", min_image_score=0.70)
|
||||
|
||||
assert out.matched_by == "text" and out.fallback_reason == "image_ambiguous"
|
||||
assert out.search.rows[0]["image_id"] == "by_label"
|
||||
|
||||
|
||||
def test_an_image_with_a_clear_lead_is_still_the_answer(ladder):
|
||||
ladder["margin"] = 0.2
|
||||
|
||||
out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None,
|
||||
text="Cadbury Dairy Milk Lickables 20g", min_image_score=0.70)
|
||||
|
||||
assert out.matched_by == "image_vector" and out.fallback_reason is None
|
||||
assert ladder["text_calls"] == 0
|
||||
|
||||
|
||||
def test_an_ambiguous_image_with_no_label_is_shown_unconfirmed(ladder):
|
||||
ladder["margin"] = 0.02
|
||||
|
||||
out = pi.identify_product(vector=[1.0] + [0.0] * 1023, image_bytes=None, min_image_score=0.70)
|
||||
|
||||
assert out.matched_by == "image_vector" and out.fallback_reason == "no_text"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# image_search_log
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_fingerprint_is_stable_and_short():
|
||||
v = [0.5, -0.25] + [0.0] * 1022
|
||||
assert image_search_log.vector_fingerprint(v) == image_search_log.vector_fingerprint(list(v))
|
||||
assert len(image_search_log.vector_fingerprint(v)) == 10
|
||||
assert image_search_log.vector_fingerprint(None) is None
|
||||
|
||||
|
||||
def test_capture_is_off_by_default(monkeypatch, tmp_path, caplog):
|
||||
monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", "")
|
||||
caplog.set_level("INFO", logger="app.services.image_search_log")
|
||||
|
||||
image_search_log.record("image-vector", vector=[1.0] + [0.0] * 1023, text="Aachi Sambar Powder 100g",
|
||||
brand=None, category=None, top_k=5, rows=_rows(0.56, 0.54))
|
||||
|
||||
assert "[IMAGE_SEARCH] route=image-vector" in caplog.text and "Aachi Sambar Powder 100g" in caplog.text
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_capture_writes_a_replayable_request_and_keeps_only_the_newest(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_MAX", 2)
|
||||
vec = [1.0] + [0.0] * 1023
|
||||
|
||||
for i in range(3):
|
||||
image_search_log.record("identify", vector=vec, text=f"label {i}", brand=None, category=None,
|
||||
top_k=5, rows=_rows(0.5), photo=b"jpegbytes")
|
||||
|
||||
saved = sorted(tmp_path.glob("*.json"))
|
||||
assert len(saved) == 2 and len(list(tmp_path.glob("*.img"))) == 2
|
||||
payload = json.loads(saved[-1].read_text(encoding="utf-8"))
|
||||
assert payload["request"]["vector"] == vec and payload["route"] == "identify"
|
||||
assert (tmp_path / payload["photo"]).read_bytes() == b"jpegbytes"
|
||||
|
||||
|
||||
def test_a_capture_failure_never_raises(monkeypatch, tmp_path):
|
||||
blocker = tmp_path / "not_a_dir"
|
||||
blocker.write_text("x")
|
||||
monkeypatch.setattr(settings, "IMAGE_SEARCH_CAPTURE_DIR", str(blocker))
|
||||
|
||||
image_search_log.record("image", vector=None, text=None, brand=None, category=None,
|
||||
top_k=5, rows=[])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# size in the row's own name breaks a size_variants tie
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cream(name: str) -> Dict[str, Any]:
|
||||
# Every Amul Cream row carries the SAME size_variants list in the catalogue.
|
||||
return {"product_name": name, "size_variants": ["90g", "1kg"], "score": 0.651,
|
||||
"image_id": name.lower().replace(" ", "_")}
|
||||
|
||||
|
||||
def test_the_label_size_in_the_name_beats_the_same_size_in_shared_size_variants():
|
||||
from app.services.label_match import label_rank_key
|
||||
from app.services.image_match import size_in_name, text_overlap, tokens
|
||||
|
||||
words, sizes = tokens("Amul Cream 1kg")
|
||||
rows = [_cream("Amul Cream 125 ml / 67 g"), _cream("Amul Cream 1kg"), _cream("Amul Cream 250 ml")]
|
||||
for row in rows:
|
||||
row["text_overlap"] = text_overlap(words, sizes, row)
|
||||
row["size_in_name"] = size_in_name(sizes, row)
|
||||
|
||||
assert len({r["text_overlap"] for r in rows}) == 1 # the tie this exists to break
|
||||
assert sorted(rows, key=label_rank_key)[0]["product_name"] == "Amul Cream 1kg"
|
||||
assert sorted(rows, key=image_match.rank_key)[0]["product_name"] == "Amul Cream 1kg"
|
||||
|
||||
|
||||
def test_no_label_size_is_never_a_name_hit():
|
||||
from app.services.image_match import size_in_name
|
||||
|
||||
assert size_in_name(set(), _cream("Amul Cream 1kg")) is False
|
||||
@@ -64,7 +64,8 @@ def fake_search(monkeypatch):
|
||||
|
||||
def test_a_vector_returns_the_product_card_with_its_score(client, fake_search):
|
||||
res = client.post("/api/search/image-vector",
|
||||
json={"vector": _unit(), "text": "Britannia Marie Gold 300 g", "top_k": 5})
|
||||
json={"vector": _unit(), "text": "Britannia Marie Gold 300 g", "top_k": 5,
|
||||
"text_fallback": False})
|
||||
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
|
||||
83
tests/test_lexical_search_distance.py
Normal file
83
tests/test_lexical_search_distance.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""vector_store.lexical_search when a matching row has no text embedding.
|
||||
|
||||
`embedding <=> q` is NULL for such a row, so its dict carries distance None -
|
||||
the key is present, so `.get("distance", 9.0)` returned None, and sorting a
|
||||
None among floats raised. The label resolver's lexical arm caught that and
|
||||
lost EVERY row of the search: unscoped, one un-embedded product in any brand
|
||||
table was enough to blank "Sambar Powder 100g".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from app.services import vector_store
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self, tables: Dict[str, List[Tuple[str, float | None, int]]]):
|
||||
self._tables = tables
|
||||
self._rows: List[tuple] = []
|
||||
self.description = [("product_name",), ("distance",), ("lex_tier",)]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
table = sql.split(" FROM ", 1)[1].split()[0]
|
||||
self._rows = list(self._tables[table])
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
|
||||
class _Conn:
|
||||
def __init__(self, cursor: _Cursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self._cursor
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _patch(monkeypatch, tables):
|
||||
cursor = _Cursor(tables)
|
||||
monkeypatch.setattr(vector_store, "_connect", lambda: _Conn(cursor))
|
||||
monkeypatch.setattr(vector_store, "_list_brand_table_suffixes",
|
||||
lambda cur: [t[len("brand_"):] for t in tables])
|
||||
monkeypatch.setattr(vector_store, "_table_exists", lambda cur, t: True)
|
||||
monkeypatch.setattr(vector_store, "_product_columns", lambda cur, t: "product_name")
|
||||
|
||||
|
||||
def test_a_row_without_an_embedding_sorts_last_in_its_tier_instead_of_raising(monkeypatch):
|
||||
_patch(monkeypatch, {
|
||||
"brand_aachi": [("Aachi Sambar Powder 100g", 0.41, 2)],
|
||||
"brand_zzsmoketest": [("Zzsmoketest Sambar Powder 100g", None, 2)],
|
||||
"brand_sakthi": [("Sambar powder 50g", 0.35, 2)],
|
||||
})
|
||||
|
||||
rows = vector_store.lexical_search(["sambar", "powder"], query_embedding=[0.1] * 384)
|
||||
|
||||
names: List[Any] = [r["product_name"] for r in rows]
|
||||
assert names == ["Sambar powder 50g", "Aachi Sambar Powder 100g", "Zzsmoketest Sambar Powder 100g"]
|
||||
|
||||
|
||||
def test_the_tier_still_outranks_the_distance(monkeypatch):
|
||||
_patch(monkeypatch, {
|
||||
"brand_a": [("exact", None, 0)],
|
||||
"brand_b": [("contains", 0.01, 2)],
|
||||
})
|
||||
|
||||
rows = vector_store.lexical_search(["x"], query_embedding=[0.1] * 384)
|
||||
|
||||
assert [r["product_name"] for r in rows] == ["exact", "contains"]
|
||||
Reference in New Issue
Block a user