nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Image vector to product details

Browse Source
This commit is contained in:
sriram
2026-09-19 15:39:53 +05:30
parent bc786b1c49
commit f933ea10a1
20 changed files with 2391 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/api/routers/health.py
View File

@@ -5,12 +5,12 @@ import logging
import requests
from fastapi import APIRouter
from app.api.schemas import AuthConfigOut, HealthOut, ImageVectorsOut
from app.api.schemas import AuthConfigOut, HealthOut, ImageVectorsOut, OcrOut
from app.infrastructure.security import auth_config_summary
from app.infrastructure.settings import (
ENABLE_IMAGE_VECTORS, OLLAMA_BASE_URL, OLLAMA_MODEL_NAME, EMBEDDINGS_MODEL,
)
from app.services import image_embedder
from app.services import image_embedder, ocr_service
from app.services.vector_store import _connect # internal, but handy for a connectivity probe
logger = logging.getLogger(__name__)
@@ -63,4 +63,7 @@ def health() -> HealthOut:
# usual cause (the model file is not in the image), and it must be
# visible from outside the container. status() never loads the model.
image_vectors=ImageVectorsOut(enabled=ENABLE_IMAGE_VECTORS, **image_embedder.status()),
# And for server-side OCR behind /search/identify: "ocr_unavailable"
# in a response has one of three causes, and this names it.
ocr=OcrOut(**ocr_service.status()),
)

94
app/api/routers/search.py
View File

@@ -3,10 +3,12 @@ from __future__ import annotations
from typing import Optional
from fastapi import APIRouter, File, Form, HTTPException, Query, UploadFile
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from app.api.routers.brands import _row_to_product_out
from app.api.schemas import (
IdentifyOut,
ImageMatchOut,
ImageSearchOut,
ImageVectorSearchRequest,
@@ -30,6 +32,7 @@ from app.services.image_match import (
parse_vector_param,
search_by_vector,
)
from app.services.product_identify import IdentifyResult, identify_product
router = APIRouter(tags=["search"])
@@ -101,8 +104,19 @@ def _to_image_search_out(result: ImageSearchResult) -> ImageSearchOut:
)
def _to_identify_out(result: IdentifyResult) -> IdentifyOut:
return IdentifyOut(
**_to_image_search_out(result.search).model_dump(),
matched_by=result.matched_by,
ocr_text=result.ocr_text,
ocr_source=result.ocr_source,
image_top_score=None if result.image_top_score is None else round(float(result.image_top_score), 4),
fallback_reason=result.fallback_reason,
)
@router.post("/search/image-vector", response_model=ImageSearchOut)
def image_vector_search_endpoint(body: ImageVectorSearchRequest) -> ImageSearchOut:
def image_vector_search_endpoint(body: ImageVectorSearchRequest):
"""Products that look like the photo whose embedding is `vector`.
`vector` is the 1024-float, L2-normalised MobileNetV3-Small embedding the
@@ -112,8 +126,22 @@ def image_vector_search_endpoint(body: ImageVectorSearchRequest) -> ImageSearchO
among products that share one photo. An explicit `brand` is a hard
filter; a brand recognised from `text` falls back to every brand when it
finds nothing (`scope_fallback`).
With `text_fallback: true` the request runs the /search/identify ladder
instead: when the best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE
the label `text` is resolved against the catalogue, and the response is
an IdentifyOut (this response plus matched_by, fallback_reason, ...).
Off by default so the app's existing calls are answered exactly as before.
"""
try:
if body.text_fallback:
identified = identify_product(
vector=body.vector, image_bytes=None, text=body.text, brand=body.brand,
category=body.category, top_k=body.top_k, min_score=body.min_score,
)
# A Response bypasses response_model, which is the point: the
# default path keeps its declared ImageSearchOut contract.
return JSONResponse(content=_to_identify_out(identified).model_dump(mode="json"))
result = search_by_vector(
body.vector, text=body.text, brand=body.brand, category=body.category,
top_k=body.top_k, min_score=body.min_score,
@@ -199,3 +227,67 @@ async def image_search_endpoint(
except InvalidVectorError as exc: # cannot happen for a model output, but the route must not 500
raise HTTPException(status_code=422, detail=str(exc))
return _to_image_search_out(result)
# ---------------------------------------------------------------------------
# Identify: image first, label text second
# ---------------------------------------------------------------------------
# A phone photo of a pack scores ~0.63 against the catalog's render of the
# same pack, so /search/image alone cannot confirm a product. This route
# runs the ladder in app/services/product_identify.py: the image match when
# it clears IMAGE_IDENTIFY_MIN_IMAGE_SCORE, otherwise the label text - the
# client's `text`, else what the server reads off the photo (ocr_service) -
# resolved through the catalogue's text embeddings and product names.
@router.post("/search/identify", response_model=IdentifyOut)
async def identify_endpoint(
file: UploadFile = File(..., description="The product photo (JPEG/PNG/WebP), ideally cropped to the pack"),
text: Optional[str] = Form(None, max_length=500,
description="OCR text read off the label; when absent the server reads it"),
brand: Optional[str] = Form(None, max_length=120),
category: Optional[str] = Form(None, max_length=120),
top_k: int = Form(IMAGE_SEARCH_DEFAULT_TOP_K, ge=1, le=IMAGE_SEARCH_MAX_TOP_K),
min_score: float = Form(IMAGE_SEARCH_DEFAULT_MIN_SCORE, ge=-1.0, le=1.0),
) -> IdentifyOut:
"""Which catalog product is in this photo.
`matched_by` says which rung answered and therefore which space each
result's `score` is in; `fallback_reason` is set whenever the answer is
a best effort rather than a confirmed match. Works text-only on a
deployment without the image model (fallback_reason
"image_embedder_unavailable"); 503 only when neither a vector nor server
OCR is possible and no `text` was sent - GET /api/health -> image_vectors
and ocr say which is missing.
"""
content = await file.read()
if not content:
raise HTTPException(status_code=400, detail="The uploaded image is empty.")
if len(content) > IMAGE_VECTOR_MAX_BYTES:
raise HTTPException(
status_code=413,
detail=f"Image is {len(content) / 1_048_576:.1f} MB; the limit is "
f"{IMAGE_VECTOR_MAX_BYTES // 1_048_576} MB. Crop or downscale it.",
)
vector = None
if await run_in_threadpool(image_embedder.available):
vector = await run_in_threadpool(image_embedder.embedding_for_bytes, content)
if vector is None:
raise HTTPException(
status_code=422,
detail="Could not decode the image (unsupported format, corrupt data, or too many pixels).",
)
try:
result = await run_in_threadpool(
identify_product, vector=vector, image_bytes=content, text=text, brand=brand,
category=category, top_k=top_k, min_score=min_score,
)
except InvalidVectorError as exc: # cannot happen for a model output, but the route must not 500
raise HTTPException(status_code=422, detail=str(exc))
if vector is None and result.fallback_reason == "ocr_unavailable":
raise HTTPException(
status_code=503,
detail="Neither the image embedding model nor server-side OCR is available on this "
"deployment. Send the label as `text`, or embed the photo client-side and POST "
"the vector to /api/search/image-vector.",
)
return _to_identify_out(result)

39
app/api/schemas.py
View File

@@ -138,6 +138,17 @@ class ImageVectorsOut(BaseModel):
state: str = "unknown"
class OcrOut(BaseModel):
"""Whether POST /api/search/identify can read a label off a photo itself.
Reported without loading the engine. `runtime_importable=false` after a
deploy means the rapidocr wheel was not installed (requirements-ocr.txt);
`onnxruntime_importable=false` means its engine was not."""
enabled: bool = True
runtime_importable: bool = False
onnxruntime_importable: bool = False
state: str = "unknown"
class HealthOut(BaseModel):
status: str
database: bool
@@ -148,6 +159,7 @@ class HealthOut(BaseModel):
# Defaulted so a client of this schema still validates against a
# deployment predating the field.
image_vectors: ImageVectorsOut = Field(default_factory=ImageVectorsOut)
ocr: OcrOut = Field(default_factory=OcrOut)
# ---------------------------------------------------------------------------
@@ -251,6 +263,16 @@ class ImageVectorSearchRequest(BaseModel):
top_k: int = Field(IMAGE_SEARCH_DEFAULT_TOP_K, ge=1, le=IMAGE_SEARCH_MAX_TOP_K)
min_score: float = Field(IMAGE_SEARCH_DEFAULT_MIN_SCORE, ge=-1.0, le=1.0,
description="Drop matches with cosine similarity below this")
# Opt-in, so a client that never asked for it gets exactly the response
# it always did. With it, the route runs the identify ladder: when the
# best image score is under IMAGE_IDENTIFY_MIN_IMAGE_SCORE, `text` is
# resolved against the catalogue instead, and the response is an
# IdentifyOut (ImageSearchOut plus matched_by / fallback_reason / ...).
text_fallback: bool = Field(
False,
description="Fall back to resolving `text` when the image match is below the identify floor; "
"the response then carries the IdentifyOut fields",
)
@field_validator("vector")
@classmethod
@@ -281,6 +303,23 @@ class ImageSearchOut(BaseModel):
query_text: Optional[str] = None
class IdentifyOut(ImageSearchOut):
"""POST /search/identify: an ImageSearchOut plus which rung answered.
`matched_by` names the space each result's `score` is in: "image_vector"
(cosine of the 1024-d photo embedding) or "text" (cosine of the 384-d
MiniLM embedding of the label). `image_top_score` always carries the
image side. The answer is confirmed when matched_by is "text", or
"image_vector" with no `fallback_reason`; otherwise `fallback_reason`
says why the best effort shown is unconfirmed (see product_identify.py).
"""
matched_by: str = "none" # "image_vector" | "text" | "none"
ocr_text: Optional[str] = None # the label text the ladder used
ocr_source: Optional[str] = None # "client" | "server"
image_top_score: Optional[float] = None
fallback_reason: Optional[str] = None
# ---------------------------------------------------------------------------
# RAG chat
# ---------------------------------------------------------------------------

36
app/infrastructure/settings.py
View File

@@ -433,6 +433,42 @@ IMAGE_SEARCH_DEFAULT_MIN_SCORE = float(os.getenv("IMAGE_SEARCH_DEFAULT_MIN_SCORE
# the floor for hnsw.ef_search on that query so the index does not drop them.
IMAGE_SEARCH_MAX_FETCH_K = int(os.getenv("IMAGE_SEARCH_MAX_FETCH_K", "100"))
# ---------------------------------------------------------------------------
# Identify a product from a phone photo - POST /api/search/identify
# (app/services/product_identify.py), with server-side OCR
# (app/services/ocr_service.py) and the label-text resolver
# (app/services/label_match.py). Public, read-only.
# ---------------------------------------------------------------------------
# A phone photo of a pack against the catalog's render of it scored 0.63 on
# img_vector (feasibility test), so the image alone cannot confirm a product.
# Below this floor the ladder falls through to the label text: whatever the
# client OCR'd, else what the server reads off the photo itself.
IMAGE_IDENTIFY_MIN_IMAGE_SCORE = float(os.getenv("IMAGE_IDENTIFY_MIN_IMAGE_SCORE", "0.70"))
# The text side is a MiniLM cosine (1 - embedding <=> q) in a different space
# from the image score. A nearest-neighbour query always returns SOMETHING, so
# a text match counts as found only when the label shares a name/size token
# with the row, or the cosine clears this floor.
IMAGE_IDENTIFY_MIN_TEXT_SCORE = float(os.getenv("IMAGE_IDENTIFY_MIN_TEXT_SCORE", "0.60"))
# Server-side OCR (rapidocr, PP-OCR models on onnxruntime CPU; models ship
# inside the wheel, nothing is downloaded). Loaded lazily on the first photo
# that needs it, never at boot; a missing wheel means "no server OCR" and
# GET /api/health -> ocr says so. tests/conftest.py pins it OFF.
ENABLE_SERVER_OCR = _bool("ENABLE_SERVER_OCR", "true")
# rapidocr's Global.text_score: recognised lines below this confidence are
# dropped before the label is assembled.
OCR_MIN_CONFIDENCE = float(os.getenv("OCR_MIN_CONFIDENCE", "0.5"))
# The photo is downscaled so its longer side is at most this before
# detection. The latency knob: a 12MP capture takes ~3x longer than 1280px
# and reads no better off a pack label.
OCR_MAX_SIDE_PX = int(os.getenv("OCR_MAX_SIDE_PX", "1280"))
# onnxruntime intra-op threads per session; the read itself is serialised by
# a lock (the engine is not thread-safe).
OCR_NUM_THREADS = int(os.getenv("OCR_NUM_THREADS", "2"))
# The assembled label is capped here, on a word boundary. 500 is the limit of
# the `text` field on the image-search routes, so client and server text are
# bounded alike.
OCR_MAX_CHARS = int(os.getenv("OCR_MAX_CHARS", "500"))
# ---------------------------------------------------------------------------
# USDA FoodData Central - nutrition for loose, unbranded commodities
# ---------------------------------------------------------------------------

57
app/services/image_embedder.py
View File

@@ -81,6 +81,43 @@ def _flatten_alpha_to_bgr(image_bytes: bytes) -> Optional[np.ndarray]:
return np.ascontiguousarray(np.asarray(rgb, dtype=np.uint8)[:, :, ::-1])
def decode_bgr(image_bytes: bytes) -> Optional[np.ndarray]:
"""`image_bytes` as a full-resolution uint8 BGR array, or None.
Step 1 of `preprocess()`, on its own because the OCR service
(app/services/ocr_service.py) needs the same decode - pixel-bomb guard,
EXIF orientation, transparency flattened onto white - but the WHOLE frame:
the 224px centre crop that follows here would throw away the label.
Never raises: an undecodable or oversized file is None (see preprocess).
"""
if not image_bytes:
return None
try:
import cv2
from PIL import Image
# Header only: the pixel-bomb guard and the transparency check both
# come from the file's metadata, no decode yet.
probe = Image.open(io.BytesIO(image_bytes))
if probe.width * probe.height > IMAGE_VECTOR_MAX_PIXELS:
logger.debug("image rejected: %dx%d exceeds pixel cap", probe.width, probe.height)
return None
has_alpha = "A" in probe.getbands() or "transparency" in probe.info
if has_alpha:
bgr = _flatten_alpha_to_bgr(image_bytes)
else:
bgr = cv2.imdecode(np.frombuffer(image_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
if bgr is None or bgr.ndim != 3 or bgr.shape[2] != 3:
logger.debug("image rejected: OpenCV could not decode it to BGR")
return None
return bgr
except Exception as exc: # noqa: BLE001 - see preprocess()
logger.debug("image undecodable: %s", exc)
return None
def preprocess(image_bytes: bytes) -> Optional[np.ndarray]:
"""Decode `image_bytes` into the model's input tensor, or None.
@@ -111,27 +148,11 @@ def preprocess(image_bytes: bytes) -> Optional[np.ndarray]:
Never raises - pytest runs warnings as errors, and a
DecompressionBombWarning is one of the things the broad except absorbs.
"""
if not image_bytes:
bgr = decode_bgr(image_bytes) # 1
if bgr is None:
return None
try:
import cv2
from PIL import Image
# Header only: the pixel-bomb guard and the transparency check both
# come from the file's metadata, no decode yet.
probe = Image.open(io.BytesIO(image_bytes))
if probe.width * probe.height > IMAGE_VECTOR_MAX_PIXELS:
logger.debug("image rejected: %dx%d exceeds pixel cap", probe.width, probe.height)
return None
has_alpha = "A" in probe.getbands() or "transparency" in probe.info
if has_alpha:
bgr = _flatten_alpha_to_bgr(image_bytes)
else:
bgr = cv2.imdecode(np.frombuffer(image_bytes, dtype=np.uint8), cv2.IMREAD_COLOR) # 1
if bgr is None or bgr.ndim != 3 or bgr.shape[2] != 3:
logger.debug("image rejected: OpenCV could not decode it to BGR")
return None
h, w = bgr.shape[:2] # 2
side = min(w, h)

360
app/services/label_match.py Normal file
View File

@@ -0,0 +1,360 @@
"""Find catalog products from the TEXT on a pack: OCR label -> `embedding` + names.
WHERE THIS SITS
---------------
The second rung of POST /api/search/identify (app/services/product_identify.py).
When the photo's img_vector cannot confirm a product - a phone photo scores
~0.63 against the catalog's render of the same pack - the label text takes
over: the client's OCR `text`, or what app/services/ocr_service.py read off
the photo. This module turns that text into ranked catalog rows.
WHY NOT catalog_search.search_catalog()
---------------------------------------
GET /api/search is tuned for a human typing in a search box, and two of its
habits are wrong for OCR output:
* it passes the FUZZY category detection through as a hard filter, and that
detector scores "taste" as Oral Care (matches "paste") and "Colgate" as
Chocolates - a misread word would delete the right product from the result;
* its lexical arm ANDs every term, so one smudged word means zero rows.
So this module goes to the store functions directly, never filters on a
category it inferred, and asks the lexical arm for only the two most
distinctive words.
HOW A MATCH IS RANKED
---------------------
Both arms run against the brand the label names (query_intent's
`extract_brand_mention`), else every active brand:
* semantic - `embed_texts([f"{brand} {label}"])` (the stored vector is of
"{brand} {name} {category} {description}", store_catalog_pipeline.py, so
the brand prefix keeps the query in-distribution) -> `semantic_search`;
* lexical - `lexical_search` on the two longest non-brand, non-size words,
which tolerates a misread elsewhere on the label.
Merged rows are ordered by `text_overlap` FIRST (image_match's scorer: 3 per
shared size token, 1 per shared word - the size is what separates the 89g,
300g and 1kg rows of one product), then by `text_extra` ascending (name
words the label never said - what separates "Dairy Milk" from "Dairy Milk
Fruit & Nut"), then by cosine. `score` on a text row is
`1 - (embedding <=> q)` in MiniLM's 384-d space: a different space from the
image score, and the response says which one it is (`matched_by`).
A nearest-neighbour query always returns something, so `is_confident()`
draws the line: the best row shares a token with the label, or its cosine
clears IMAGE_IDENTIFY_MIN_TEXT_SCORE. Below that the caller treats the text
rung as a miss. Read-only: nothing here writes.
"""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Set, Tuple
from app.infrastructure.settings import (
IMAGE_IDENTIFY_MIN_TEXT_SCORE,
IMAGE_SEARCH_DEFAULT_TOP_K,
IMAGE_SEARCH_MAX_FETCH_K,
IMAGE_SEARCH_MAX_TOP_K,
OCR_MAX_CHARS,
)
from app.services.image_match import _STOP, ImageSearchResult, _row_text, text_overlap, tokens
logger = logging.getLogger(__name__)
# Words a pack prints that name no product: the nutrition panel, the
# regulatory block, storage advice, the price line. Dropped before either
# arm sees the label so "ENERGY 480 kcal PROTEIN 7g" does not become the
# query. Lowercase; compared after image_match.tokens' own normalisation.
_NOISE: Set[str] = {
"nutrition", "nutritional", "information", "facts", "typical", "values", "value",
"energy", "kcal", "kj", "calories", "protein", "carbohydrate", "carbohydrates",
"carbs", "sugar", "sugars", "fat", "fats", "saturated", "trans", "sodium",
"cholesterol", "fibre", "fiber", "dietary", "serving", "servings", "serve",
"serves", "amount", "ingredients", "ingredient", "contains", "allergen",
"allergens", "allergy", "advice", "mfg", "mfd", "manufactured", "manufacturer",
"marketed", "packed", "packer", "lot", "batch", "best", "before", "expiry",
"exp", "use", "date", "fssai", "lic", "license", "licence", "mrp", "price",
"incl", "inclusive", "all", "taxes", "tax", "ltd", "pvt", "limited", "india",
"www", "com", "http", "https", "customer", "care", "helpline", "email", "call",
"store", "cool", "dry", "place", "keep", "away", "sunlight", "hygienic",
"net", "wt", "weight", "qty", "quantity", "approx", "veg", "vegetarian",
"non", "product", "code", "unit", "units",
}
# A nutrition-panel basis ("per 100 g", "per serve") carries a quantity that
# is NOT the pack size; left in, "100g" would score 3 points of overlap and
# hand the match to the 100g sibling. Removed as a span, before tokenising.
_PER_BASIS_RE = re.compile(
r"\bper\s*\d+(?:\.\d+)?\s*(?:kg|gms|gm|g|ml|ltr|litre|l)\b|\bper\s+serv\w*\b",
re.I,
)
# A nutrient with its value ("Protein 7 g", "Sugars: 20.5g", "Energy 480
# kcal") is also a quantity that is not the pack size. The name and the
# number go together, so the pair is removed as one span.
_NUTRIENT_VALUE_RE = re.compile(
r"\b(?:energy|calories|protein|carbohydrates?|carbs|sugars?|added\s+sugars?|"
r"total\s+fat|fat|saturated(?:\s+fat)?|trans(?:\s+fat)?|cholesterol|sodium|salt|"
r"dietary\s+fibre|dietary\s+fiber|fibre|fiber|calcium|iron|potassium|vitamin\s*\w*)"
r"\s*[:\-]?\s*(?:<\s*)?\d+(?:\.\d+)?\s*(?:kg|mg|mcg|g|kcal|kj|cal|ml|%)?\b",
re.I,
)
# What is left of a nutrition line once the names are gone: values in units
# no pack is sold in.
_NON_PACK_VALUE_RE = re.compile(r"\b\d+(?:\.\d+)?\s*(?:mg|mcg|kcal|kj|cal)\b|\d+(?:\.\d+)?\s*%", re.I)
_URL_RE = re.compile(r"(?:https?://|www\.)\S+|\b\S+\.(?:com|in|org|net)\b", re.I)
_PRICE_RE = re.compile(r"(?:₹|rs\.?|inr)\s*\d+(?:[.,]\d+)?", re.I)
_SIZE_RE = re.compile(r"^\d+(?:\.\d+)?(?:kg|gms|gm|g|ml|ltr|litre|l|pcs|pc|n)$", re.I)
_WORD_OR_SIZE_RE = re.compile(r"\d+(?:\.\d+)?\s*(?:kg|gms|gm|g|ml|ltr|litre|l|pcs|pc|n)\b|[a-z0-9]+(?:[-'&][a-z0-9]+)*", re.I)
_LEXICAL_TERMS = 2
_LONG_NUMBER = 6 # digits; an FSSAI licence is 14, a phone number 10, a barcode 8-13
# ---------------------------------------------------------------------------
# the label
# ---------------------------------------------------------------------------
def clean_label(text: Optional[str]) -> str:
"""The label as a query: boilerplate out, sizes kept, repeats folded.
Spans first (a per-100g basis, a URL, a price), then tokens: a size such
as "300 g" is kept whole (as "300 g" - image_match.tokens normalises it
later), a word in _NOISE or shorter than two characters is dropped, a
repeat keeps its first position. Capped at OCR_MAX_CHARS on a word
boundary like the OCR output it usually is.
"""
if not text:
return ""
lowered = " ".join(str(text).split())
lowered = _PER_BASIS_RE.sub(" ", lowered)
lowered = _NUTRIENT_VALUE_RE.sub(" ", lowered)
lowered = _NON_PACK_VALUE_RE.sub(" ", lowered)
lowered = _URL_RE.sub(" ", lowered)
lowered = _PRICE_RE.sub(" ", lowered)
kept: List[str] = []
seen: Set[str] = set()
for match in _WORD_OR_SIZE_RE.finditer(lowered):
piece = match.group(0)
key = piece.lower().replace(" ", "")
if _SIZE_RE.match(key):
pass # a size stays, whatever its length
elif len(key) < 2 or key in _NOISE or key in _STOP:
continue
elif key.isdigit() and len(key) >= _LONG_NUMBER:
continue # a licence, phone or barcode number names no product
if key in seen:
continue
seen.add(key)
kept.append(" ".join(piece.split()))
out = " ".join(kept)
if len(out) > OCR_MAX_CHARS:
cut = out.rfind(" ", 0, OCR_MAX_CHARS + 1)
out = out[:cut if cut > 0 else OCR_MAX_CHARS].rstrip()
return out
def _lexical_terms(words: Set[str], brand: Optional[str]) -> List[str]:
"""The two longest label words that are not the brand: length is a cheap
proxy for distinctiveness ("britannia" > "gold" > "go"), and two ANDed
terms survive one misread elsewhere on the label."""
brand_words = set(tokens(brand)[0]) if brand else set()
candidates = sorted((w for w in words if w not in brand_words and not w.isdigit()),
key=lambda w: (-len(w), w))
return candidates[:_LEXICAL_TERMS]
# ---------------------------------------------------------------------------
# ranking
# ---------------------------------------------------------------------------
def text_extra(words: Set[str], row: Dict[str, Any]) -> int:
"""How many words of the row's NAME the label does not account for.
`text_overlap` is recall - how much of the label the row explains - and
on its own it cannot tell "Dairy Milk 50g" from "Dairy Milk Fruit & Nut
50g": both explain every word of a "Dairy Milk 50 g" label. The variant
carries words the label never said, and this counts them, so the plain
product outranks its variants and a "sugar free" or "family pack" row
does not win on a label that mentions neither. Only the name and the
sizes are looked at, never the description.
"""
if not words:
return 0
row_words, _ = tokens(_row_text(row))
return len(row_words - words)
def label_rank_key(row: Dict[str, Any]) -> tuple:
"""Best first: what the label says, what the row adds, then the vector.
Overlap outranks cosine because the size token is the only thing that
tells the pack sizes of one product apart, and their descriptions - and
so their vectors - are near-identical. Unexplained name words come next
for the same reason (a variant's vector is as close as the original's).
"""
return (
-float(row.get("text_overlap", 0.0)),
int(row.get("text_extra", 0)),
-round(float(row.get("score", 0.0)), 3),
str(row.get("product_name") or row.get("title") or "").lower(),
str(row.get("image_id") or ""),
)
def is_confident(rows: List[Dict[str, Any]]) -> bool:
"""Whether the best text row is a find rather than the nearest stranger."""
if not rows:
return False
best = rows[0]
try:
overlap = float(best.get("text_overlap", 0.0))
score = float(best.get("score", 0.0))
except (TypeError, ValueError):
return False
return overlap >= 1.0 or score >= IMAGE_IDENTIFY_MIN_TEXT_SCORE
def _normalise_row(row: Dict[str, Any], scoped_table: Optional[str]) -> Dict[str, Any]:
"""Give a store row the keys image_match's rows have.
Brand tables carry no `brand` column, so semantic_search / lexical_search
fill `brand` with the caller's brand when scoped and with the TABLE
SUFFIX ("hindustan_unilever") when not; neither adds `brand_table`. The
identify response is built by the same card builder as the image rows,
so both get the display name and the table here.
"""
from app.services.vector_store import display_name_for_suffix
row = dict(row)
raw_brand = str(row.get("brand") or "")
if scoped_table:
row["brand_table"] = scoped_table
else:
row["brand_table"] = f"brand_{raw_brand}" if raw_brand else ""
row["brand"] = display_name_for_suffix(raw_brand) if raw_brand else raw_brand
distance = row.get("distance")
try:
row["score"] = 1.0 - float(distance) if distance is not None else 0.0
except (TypeError, ValueError):
row["score"] = 0.0
return row
def _candidates(
query_text: str,
embedding: Optional[List[float]],
words: Set[str],
brand: Optional[str],
category: Optional[str],
fetch_k: int,
) -> List[Dict[str, Any]]:
"""Both arms, merged on (brand_table, image_id); the semantic row wins a
tie because it carries the real distance for the full query."""
from app.services.vector_store import _table_name, lexical_search, semantic_search
scoped_table = _table_name(brand) if brand else None
merged: Dict[Tuple[str, str], Dict[str, Any]] = {}
if embedding is not None:
try:
for row in semantic_search(query_embedding=embedding, brand=brand,
top_k=fetch_k, category=category):
norm = _normalise_row(row, scoped_table)
merged.setdefault((norm["brand_table"], str(norm.get("image_id") or "")), norm)
except Exception as exc: # noqa: BLE001 - one arm failing must not lose the other
logger.warning("label semantic arm failed: %s", exc)
terms = _lexical_terms(words, brand)
for attempt in (terms, terms[:1]):
if not attempt:
break
try:
rows = lexical_search(attempt, brand=brand, limit=fetch_k, category=category,
query_embedding=embedding, exact_phrase=" ".join(attempt))
except Exception as exc: # noqa: BLE001
logger.warning("label lexical arm failed: %s", exc)
rows = []
for row in rows:
norm = _normalise_row(row, scoped_table)
merged.setdefault((norm["brand_table"], str(norm.get("image_id") or "")), norm)
if rows:
break
return list(merged.values())
# ---------------------------------------------------------------------------
# the search
# ---------------------------------------------------------------------------
def resolve_label(
text: Optional[str],
brand: Optional[str] = None,
category: Optional[str] = None,
top_k: int = IMAGE_SEARCH_DEFAULT_TOP_K,
) -> ImageSearchResult:
"""The catalog rows the label text names, best first, at most `top_k`.
Never raises: no text, no model and no database all give an empty result.
`query_text` on the result is the cleaned label the search actually used.
"""
cleaned = clean_label(text)
top_k = max(1, min(int(top_k), IMAGE_SEARCH_MAX_TOP_K))
if not cleaned:
return ImageSearchResult(query_text=None, top_k=top_k)
fetch_k = min(max(top_k * 3, 30), IMAGE_SEARCH_MAX_FETCH_K)
explicit = (brand or "").strip() or None
detected = explicit
if detected is None:
try:
from app.services.query_intent import extract_brand_mention
detected = extract_brand_mention(cleaned)
except Exception as exc: # noqa: BLE001 - brand detection is an optimisation
logger.debug("brand detection skipped: %s", exc)
detected = None
query = cleaned
if detected and detected.lower() not in cleaned.lower():
query = f"{detected} {cleaned}"
embedding: Optional[List[float]] = None
try:
from app.services.embeddings_service import embed_texts
vectors = embed_texts([query])
embedding = list(vectors[0]) if vectors else None
except Exception as exc: # noqa: BLE001 - lexical-only, like catalog_search does
logger.warning("label embedding failed: %s. Lexical-only.", exc)
embedding = None
words, sizes = tokens(cleaned)
def _run(scope: Optional[str]) -> List[Dict[str, Any]]:
rows = _candidates(query, embedding, words, scope, category, fetch_k)
for row in rows:
row["text_overlap"] = text_overlap(words, sizes, row)
row["text_extra"] = text_extra(words, row)
rows.sort(key=label_rank_key)
return rows
rows = _run(detected)
scoped = detected is not None
fallback = False
if detected and not explicit and not is_confident(rows):
# The OCR named a brand whose table does not hold this label - a
# misread, or a sub-brand mapped to the wrong parent. Try everywhere.
wider = _run(None)
if is_confident(wider) or not rows:
rows, scoped, fallback = wider, False, True
return ImageSearchResult(
rows=rows[:top_k],
detected_brand=detected,
scoped_to_brand=scoped,
scope_fallback=fallback,
min_score=0.0,
query_text=cleaned,
top_k=top_k,
)

245
app/services/ocr_service.py Normal file
View File

@@ -0,0 +1,245 @@
"""Read the label text off a product photo, server-side: rapidocr on onnxruntime.
WHY THIS EXISTS
---------------
POST /api/search/identify falls through to the label text when the image
vector cannot confirm a product (a phone photo of a pack scores ~0.63 against
the catalog's render of it). The Nearle app OCRs on-device and sends `text`;
a plain camera upload has no such text, so this module reads it here.
WHAT IT PRODUCES
----------------
One string: the recognised lines in detection order (top-to-bottom, which is
reading order for a pack front), each kept only above OCR_MIN_CONFIDENCE,
exact repeats dropped, capped at OCR_MAX_CHARS on a word boundary. It is
deliberately the same shape as the `text` field the routes already accept, so
app/services/label_match.py cannot tell the two apart.
THE ENGINE
----------
rapidocr (PP-OCR detection + classification + recognition, ONNX). The models
ship inside the wheel and are verified by checksum at construction, so the
first call makes no network request. Two things to know about it:
* Its metadata demands opencv-python (the GUI build), which would unpack over
this project's opencv-python-headless and fail on libGL in the slim image -
breaking image_embedder too. It is therefore installed with `--no-deps`
(requirements-ocr.txt) and its real dependencies are listed in
requirements.txt. Nothing here needs the GUI build.
* `RapidOCR.__call__` mutates the engine's own state, so one lock serialises
every read, exactly as image_embedder does for its interpreter.
FAILURE IS SILENT BY DESIGN
---------------------------
Imported lazily, on the first photo that needs it. A missing wheel, a missing
onnxruntime, a broken model: `available()` is False after ONE warning,
`read_text()` is None, and the identify route reports `ocr_unavailable`
instead of failing. GET /api/health -> ocr says which of those it was.
"""
from __future__ import annotations
import logging
import threading
import warnings
from typing import Any, List, Optional
import numpy as np
from app.infrastructure.settings import (
ENABLE_SERVER_OCR,
OCR_MAX_CHARS,
OCR_MAX_SIDE_PX,
OCR_MIN_CONFIDENCE,
OCR_NUM_THREADS,
)
from app.services import image_embedder
logger = logging.getLogger(__name__)
_lock = threading.Lock()
_engine: Any = None
_disabled_reason: Optional[str] = None
_warned = False
# ---------------------------------------------------------------------------
# the engine
# ---------------------------------------------------------------------------
def _disable(reason: str, *, warn: bool = True) -> None:
"""Record why OCR is off; say so once. Caller holds _lock."""
global _disabled_reason, _engine, _warned
_disabled_reason = reason
_engine = None
if warn and not _warned:
_warned = True
logger.warning("server OCR disabled: %s (identify falls back to client text)", reason)
def _load() -> None:
"""Construct the engine. Caller holds _lock. Never raises."""
global _engine
if not ENABLE_SERVER_OCR:
# A deliberate setting, not a fault: no warning line for it.
_disable("ENABLE_SERVER_OCR is false", warn=False)
return
try:
with warnings.catch_warnings():
# Import-time deprecation chatter from onnxruntime / omegaconf /
# shapely would be a hard error under pytest's filterwarnings=error.
warnings.simplefilter("ignore")
from rapidocr import RapidOCR
engine = RapidOCR(params={
"Global.text_score": float(OCR_MIN_CONFIDENCE),
# Its logger does not propagate and installs its own INFO
# handler; keep the per-call chatter out of the container log.
"Global.log_level": "warning",
"EngineConfig.onnxruntime.intra_op_num_threads": max(1, int(OCR_NUM_THREADS)),
})
except Exception as exc: # noqa: BLE001 - ImportError, a missing engine, a bad model, libGL
_disable(f"rapidocr is not usable ({exc})")
return
_engine = engine
logger.info("server OCR ready: rapidocr (%d thread(s))", OCR_NUM_THREADS)
def _ensure_loaded() -> bool:
"""Caller holds _lock."""
if _engine is None and _disabled_reason is None:
_load()
return _engine is not None
def available() -> bool:
"""True when the engine can be used. Loads it on first call."""
with _lock:
return _ensure_loaded()
def status() -> dict:
"""Diagnostics for /api/health - NEVER loads the engine.
Same idea as image_embedder.status(): the failure this module absorbs is
one log line at first use, so the same facts go on the health endpoint
where "why does identify say ocr_unavailable?" is one curl away.
"""
try:
import importlib.util
runtime = importlib.util.find_spec("rapidocr") is not None
onnx = importlib.util.find_spec("onnxruntime") is not None
except Exception: # noqa: BLE001 - a broken finder counts as "not importable"
runtime = onnx = False
with _lock:
if _engine is not None:
state = "ready"
elif _disabled_reason:
state = f"disabled: {_disabled_reason}"
elif not ENABLE_SERVER_OCR:
state = "disabled: ENABLE_SERVER_OCR is false"
else:
state = "not loaded yet (loads on first identify)"
return {
"enabled": bool(ENABLE_SERVER_OCR),
"runtime_importable": runtime,
"onnxruntime_importable": onnx,
"state": state,
}
# ---------------------------------------------------------------------------
# the read
# ---------------------------------------------------------------------------
def _downscale(bgr: np.ndarray, max_side: int) -> np.ndarray:
"""`bgr` with its longer side at most `max_side` (INTER_AREA), else as is."""
h, w = bgr.shape[:2]
longest = max(h, w)
if max_side <= 0 or longest <= max_side:
return bgr
import cv2
scale = max_side / float(longest)
size = (max(1, int(round(w * scale))), max(1, int(round(h * scale))))
return cv2.resize(bgr, size, interpolation=cv2.INTER_AREA)
def _join_lines(output: Any) -> Optional[str]:
"""The engine's lines as one label string, or None when it read nothing.
Detection order is kept (top-to-bottom is reading order on a pack front);
a line below OCR_MIN_CONFIDENCE is dropped (the engine already filters on
Global.text_score - this is belt and braces for a fake or a future
engine); an exact repeat is dropped (a pack often prints its name twice);
the whole is capped at OCR_MAX_CHARS on a word boundary so the server's
text is bounded exactly like the routes' `text` field.
"""
if output is None:
return None
txts = getattr(output, "txts", None)
scores = getattr(output, "scores", None)
if not txts:
return None
if scores is None or len(scores) != len(txts):
scores = [1.0] * len(txts)
lines: List[str] = []
seen = set()
for raw, score in zip(txts, scores):
try:
if float(score) < OCR_MIN_CONFIDENCE:
continue
except (TypeError, ValueError):
continue
line = " ".join(str(raw or "").split())
if not line:
continue
key = line.lower()
if key in seen:
continue
seen.add(key)
lines.append(line)
text = " ".join(lines)
if len(text) > OCR_MAX_CHARS:
cut = text.rfind(" ", 0, OCR_MAX_CHARS + 1)
text = text[:cut if cut > 0 else OCR_MAX_CHARS].rstrip()
return text or None
def read_text(image_bytes: bytes) -> Optional[str]:
"""The label text in `image_bytes`, or None (undecodable, nothing read, no engine).
The frame goes in as a BGR array - decoded by image_embedder.decode_bgr so
the pixel cap, EXIF orientation and alpha handling are the same as the
vector path's - and downscaled to OCR_MAX_SIDE_PX first. It is never
handed to the engine as raw bytes: that would bypass both guards.
"""
bgr = image_embedder.decode_bgr(image_bytes)
if bgr is None:
return None
try:
bgr = _downscale(bgr, OCR_MAX_SIDE_PX)
except Exception as exc: # noqa: BLE001 - a resize failure is "nothing read"
logger.debug("ocr downscale failed: %s", exc)
return None
with _lock:
if not _ensure_loaded():
return None
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
output = _engine(bgr)
except Exception as exc: # noqa: BLE001 - one bad photo must not poison the engine
logger.debug("ocr failed: %s", exc)
return None
return _join_lines(output)
def _reset() -> None:
"""Tests only: forget the engine and the disabled state."""
global _engine, _disabled_reason, _warned
with _lock:
_engine = None
_disabled_reason = None
_warned = False

162
app/services/product_identify.py Normal file
View File

@@ -0,0 +1,162 @@
"""Identify the product in a phone photo: image vector first, label text second.
THE LADDER
----------
photo ─► img_vector kNN (image_match.search_by_vector)
│ best score >= IMAGE_IDENTIFY_MIN_IMAGE_SCORE
│ -> matched_by="image_vector", fallback_reason=None
▼ else
label text: the client's `text` (ocr_source="client"),
else what ocr_service reads off the photo ("server")
▼
label_match.resolve_label
│ confident -> matched_by="text", fallback_reason=<why the image lost>
▼ else
the image rows as they were (they may still be right - the 0.63
in the feasibility test WAS the product), matched_by=
"image_vector" or "none", fallback_reason="text_no_match"
`fallback_reason` is the reason for the LAST step the ladder took:
image_below_threshold the image arm ran but did not clear the floor
no_image_match the image arm ran and found nothing at all
image_embedder_unavailable no vector could be computed (no model here)
no_text no label to fall back to and no photo to read
ocr_unavailable no client text, and server OCR is off / missing
ocr_empty server OCR ran and read nothing usable
text_no_match the label was resolved but nothing was confident
The rule for a client: the answer is confirmed when `matched_by == "text"`,
or `matched_by == "image_vector"` with no `fallback_reason`. Everything else
is a best effort the client should show as such.
The two `score`s are in different spaces (1024-d image, 384-d text);
`image_top_score` always carries the image side so the client can see both.
Server OCR is invoked only when it is needed - never when the image is
confident and never when the client sent text - because it costs a second.
Read-only: every function this calls only reads.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Optional, Sequence
from app.infrastructure.settings import (
IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
IMAGE_SEARCH_DEFAULT_MIN_SCORE,
IMAGE_SEARCH_DEFAULT_TOP_K,
)
from app.services import ocr_service
from app.services.image_match import ImageSearchResult, search_by_vector
from app.services.label_match import is_confident, resolve_label
logger = logging.getLogger(__name__)
MATCHED_BY_IMAGE = "image_vector"
MATCHED_BY_TEXT = "text"
MATCHED_BY_NONE = "none"
@dataclass
class IdentifyResult:
search: ImageSearchResult
matched_by: str = MATCHED_BY_NONE
ocr_text: Optional[str] = None
ocr_source: Optional[str] = None # "client" | "server"
image_top_score: Optional[float] = None
fallback_reason: Optional[str] = None
def _image_rung(
vector: Optional[Sequence[float]],
text: Optional[str],
brand: Optional[str],
category: Optional[str],
top_k: int,
min_score: float,
min_image_score: float,
) -> tuple[ImageSearchResult, Optional[float], Optional[str]]:
"""(image result, its best score, why it is not the answer - or None)."""
if vector is None:
empty = ImageSearchResult(min_score=min_score, top_k=top_k, query_text=(text or "").strip() or None)
return empty, None, "image_embedder_unavailable"
result = search_by_vector(vector, text=text, brand=brand, category=category,
top_k=top_k, min_score=min_score)
if not result.rows:
return result, None, "no_image_match"
try:
top = float(result.rows[0].get("score", 0.0))
except (TypeError, ValueError):
top = 0.0
if top >= min_image_score:
return result, top, None
return result, top, "image_below_threshold"
def identify_product(
*,
vector: Optional[Sequence[float]],
image_bytes: Optional[bytes],
text: Optional[str] = None,
brand: Optional[str] = None,
category: Optional[str] = None,
top_k: int = IMAGE_SEARCH_DEFAULT_TOP_K,
min_score: float = IMAGE_SEARCH_DEFAULT_MIN_SCORE,
min_image_score: float = IMAGE_IDENTIFY_MIN_IMAGE_SCORE,
) -> IdentifyResult:
"""Run the ladder. `vector` is the photo's embedding (None when this
deployment cannot compute one); `image_bytes` the photo for server OCR
(None on the vector-only route). Raises only InvalidVectorError, from
search_by_vector, for a vector that cannot be searched with.
"""
label = (text or "").strip() or None
image, top, reason = _image_rung(vector, label, brand, category, top_k, min_score, min_image_score)
if reason is None:
return IdentifyResult(search=image, matched_by=MATCHED_BY_IMAGE, image_top_score=top,
ocr_text=label, ocr_source="client" if label else None)
# --- the label ----------------------------------------------------------
ocr_source: Optional[str] = None
if label:
ocr_source = "client"
elif image_bytes:
if ocr_service.available():
label = ocr_service.read_text(image_bytes)
if label:
ocr_source = "server"
else:
reason = "ocr_empty"
else:
reason = "ocr_unavailable"
else:
reason = "no_text"
if not label:
return IdentifyResult(
search=image,
matched_by=MATCHED_BY_IMAGE if image.rows else MATCHED_BY_NONE,
image_top_score=top,
fallback_reason=reason,
)
resolved = resolve_label(label, brand=brand, category=category, top_k=top_k)
if is_confident(resolved.rows):
return IdentifyResult(
search=resolved,
matched_by=MATCHED_BY_TEXT,
ocr_text=label,
ocr_source=ocr_source,
image_top_score=top,
fallback_reason=reason, # why the image rung was not the answer
)
return IdentifyResult(
search=image,
matched_by=MATCHED_BY_IMAGE if image.rows else MATCHED_BY_NONE,
ocr_text=label,
ocr_source=ocr_source,
image_top_score=top,
fallback_reason="text_no_match",
)