nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates on Image search using vectors

Browse Source
This commit is contained in:
sriram
2026-09-28 15:44:01 +05:30
parent 6628207810
commit c0489d89d6
16 changed files with 1184 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
app/api/routers/search.py
View File

@@ -27,7 +27,7 @@ from app.infrastructure.settings import (
SEARCH_DEFAULT_TOP_K,
SEARCH_MAX_TOP_K,
)
from app.services import capture_discovery, image_embedder
from app.services import capture_discovery, image_embedder, image_search_log
from app.services.catalog_search import search_catalog
from app.services.image_match import (
ImageSearchResult,
@@ -105,12 +105,28 @@ def _to_image_search_out(result: ImageSearchResult) -> ImageSearchOut:
min_score=result.min_score,
top_k=result.top_k,
query_text=result.query_text,
match_confidence=result.match_confidence if matches else "none",
margin=None if result.margin is None else round(float(result.margin), 4),
)
def _identify_confidence(result: IdentifyResult) -> str:
"""The ladder's verdict in match_confidence terms: confirmed exactly when
product_identify.py's client rule says so."""
if not result.search.rows:
return "none"
if capture_discovery.is_confirmed(result.matched_by, result.fallback_reason):
return "confirmed"
return "low"
def _to_identify_out(result: IdentifyResult) -> IdentifyOut:
fields = _to_image_search_out(result.search).model_dump()
fields["match_confidence"] = _identify_confidence(result)
if result.matched_by != "image_vector":
fields["margin"] = None # text rows: a MiniLM score, no image margin
return IdentifyOut(
**_to_image_search_out(result.search).model_dump(),
**fields,
matched_by=result.matched_by,
ocr_text=result.ocr_text,
ocr_source=result.ocr_source,
@@ -119,6 +135,28 @@ def _to_identify_out(result: IdentifyResult) -> IdentifyOut:
)
def _log_search(route: str, result: ImageSearchResult, *, vector, text, brand, category, top_k,
photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None:
image_search_log.record(
route, vector=vector, text=text, brand=brand, category=category, top_k=top_k,
rows=result.rows, detected_brand=result.detected_brand, scoped_to_brand=result.scoped_to_brand,
match_confidence=result.match_confidence, margin=result.margin,
matched_by="image_vector" if result.rows else "none", photo=photo, text_fallback=text_fallback,
)
def _log_identify(route: str, result: IdentifyResult, *, vector, text, brand, category, top_k,
photo: Optional[bytes] = None, text_fallback: Optional[bool] = None) -> None:
image_search_log.record(
route, vector=vector, text=text, brand=brand, category=category, top_k=top_k,
rows=result.search.rows, detected_brand=result.search.detected_brand,
scoped_to_brand=result.search.scoped_to_brand, match_confidence=_identify_confidence(result),
margin=result.search.margin if result.matched_by == "image_vector" else None,
matched_by=result.matched_by, fallback_reason=result.fallback_reason,
photo=photo, text_fallback=text_fallback,
)
@router.post("/search/image-vector", response_model=ImageSearchOut)
def image_vector_search_endpoint(body: ImageVectorSearchRequest):
"""Products that look like the photo whose embedding is `vector`.
@@ -131,20 +169,26 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest):
filter; a brand recognised from `text` falls back to every brand when it
finds nothing (`scope_fallback`).
With `text_fallback: true` the request runs the /search/identify ladder
instead: when the best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE
the label `text` is resolved against the catalogue, and the response is
an IdentifyOut (this response plus matched_by, fallback_reason, ...).
Off by default so the app's existing calls are answered exactly as before.
With `text_fallback` the request runs the /search/identify ladder
instead: when the image match cannot be confirmed (best score under
IMAGE_IDENTIFY_MIN_IMAGE_SCORE, or another photo within
IMAGE_SEARCH_MIN_MARGIN) the label `text` is resolved against the
catalogue, and the response is an IdentifyOut (this response plus
matched_by, fallback_reason, ...). Left out, it is on whenever `text` is
sent; `false` keeps the image-only ranking, where text only breaks ties.
"""
ladder = body.text_fallback if body.text_fallback is not None else bool((body.text or "").strip())
log_fields = dict(vector=body.vector, text=body.text, brand=body.brand, category=body.category,
top_k=body.top_k, text_fallback=body.text_fallback)
try:
if body.text_fallback:
if ladder:
identified = identify_product(
vector=body.vector, image_bytes=None, text=body.text, brand=body.brand,
category=body.category, top_k=body.top_k, min_score=body.min_score,
)
_log_identify("image-vector", identified, **log_fields)
# A Response bypasses response_model, which is the point: the
# default path keeps its declared ImageSearchOut contract.
# image-only path keeps its declared ImageSearchOut contract.
return JSONResponse(content=_to_identify_out(identified).model_dump(mode="json"))
result = search_by_vector(
body.vector, text=body.text, brand=body.brand, category=body.category,
@@ -152,6 +196,7 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest):
)
except InvalidVectorError as exc:
raise HTTPException(status_code=422, detail=str(exc))
_log_search("image-vector", result, **log_fields)
return _to_image_search_out(result)
@@ -185,6 +230,8 @@ def image_vector_search_get_endpoint(
)
except InvalidVectorError as exc:
raise HTTPException(status_code=422, detail=str(exc))
_log_search("image-vector:get", result, vector=values, text=text, brand=brand,
category=category, top_k=top_k)
return _to_image_search_out(result)
@@ -230,6 +277,8 @@ async def image_search_endpoint(
)
except InvalidVectorError as exc: # cannot happen for a model output, but the route must not 500
raise HTTPException(status_code=422, detail=str(exc))
_log_search("image", result, vector=vector, text=text, brand=brand, category=category,
top_k=top_k, photo=content)
return _to_image_search_out(result)
@@ -295,6 +344,8 @@ async def identify_endpoint(
"deployment. Send the label as `text`, or embed the photo client-side and POST "
"the vector to /api/search/image-vector.",
)
_log_identify("identify", result, vector=vector, text=text, brand=brand, category=category,
top_k=top_k, photo=content)
out = _to_identify_out(result)
if settings.ENABLE_CAPTURE_DISCOVERY and not capture_discovery.is_confirmed(
result.matched_by, result.fallback_reason
@@ -327,10 +378,14 @@ def _apply_capture_outcome(out: IdentifyOut, outcome: capture_discovery.CaptureO
out.results = [ImageMatchOut(**card.model_dump(), score=1.0, text_overlap=1.0)]
out.total = 1
out.matched_by = capture_discovery.MATCHED_BY_LABEL_EXACT
out.match_confidence = "confirmed"
out.margin = None
elif outcome.status == capture_discovery.PENDING:
out.results = []
out.total = 0
out.matched_by = capture_discovery.MATCHED_BY_DISCOVERY
out.match_confidence = "none"
out.margin = None
@router.get("/search/identify/jobs/{job_id}", response_model=CaptureJobOut)

30
app/api/schemas.py
View File

@@ -263,15 +263,19 @@ class ImageVectorSearchRequest(BaseModel):
top_k: int = Field(IMAGE_SEARCH_DEFAULT_TOP_K, ge=1, le=IMAGE_SEARCH_MAX_TOP_K)
min_score: float = Field(IMAGE_SEARCH_DEFAULT_MIN_SCORE, ge=-1.0, le=1.0,
description="Drop matches with cosine similarity below this")
# Opt-in, so a client that never asked for it gets exactly the response
# it always did. With it, the route runs the identify ladder: when the
# best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE, `text` is
# resolved against the catalogue instead, and the response is an
# IdentifyOut (ImageSearchOut plus matched_by / fallback_reason / ...).
text_fallback: bool = Field(
False,
description="Fall back to resolving `text` when the image match is below the identify floor; "
"the response then carries the IdentifyOut fields",
# With it, the route runs the identify ladder: when the best image score
# is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE (or another photo is within
# IMAGE_SEARCH_MIN_MARGIN of it), `text` is resolved against the
# catalogue instead, and the response is an IdentifyOut (ImageSearchOut
# plus matched_by / fallback_reason / ...). Left out, it is ON whenever
# `text` is sent: a label that names the product must not lose to a 0.5
# cosine, which it did when text only broke exact ties. `false` keeps the
# old image-only ranking.
text_fallback: Optional[bool] = Field(
None,
description="Resolve `text` when the image match cannot be confirmed; the response then "
"carries the IdentifyOut fields. Default: on when `text` is sent. "
"false = image-only ranking, text breaks ties only",
)
@field_validator("vector")
@@ -301,6 +305,14 @@ class ImageSearchOut(BaseModel):
min_score: float = 0.0
top_k: int = 0
query_text: Optional[str] = None
# "confirmed" only when the best match clears IMAGE_IDENTIFY_MIN_IMAGE_SCORE
# AND leads the best different photo by IMAGE_SEARCH_MIN_MARGIN (on
# /identify: when the ladder confirmed it). "low": show the results as a
# list to pick from, never as the answer. "none": no results.
match_confidence: str = "none"
# Best score minus the best DIFFERENT photo's; None when there was no rival.
# Image space only - None when the rows came from the text rung.
margin: Optional[float] = None
class IdentifyOut(ImageSearchOut):