nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Image vector embedding

Browse Source
This commit is contained in:
sriram
2026-09-16 16:33:39 +05:30
parent ce4fa70dee
commit 9c8dbf1759
10 changed files with 1493 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

417
app/services/image_vector.py Normal file
View File

@@ -0,0 +1,417 @@
"""A pixel thumbnail of every product's PRIMARY image, stored on its brand table.
WHAT `img_vector` IS
--------------------
The image the product card shows - `image_url`, else `image_urls[0]`, which is
exactly the order `frontend/src/components/ProductCard.jsx` walks - decoded,
EXIF-rotated, flattened onto white, resized to 32x32 RGB and unrolled into
3072 uint8 values. It is a literal downscaled picture, not a learned embedding:
two rows with the same photo have (near-)identical vectors, and a row whose
image changed has a different one. Stored as pgvector `vector(3072)` so that a
`<=>` comparison later needs no migration; 0..255 is exact in float4.
`img_vector_src` records the URL the vector was computed from. That is what
makes recomputation idempotent: a row is due when it has no vector, or when
its current primary URL differs from the one the vector came from.
WHY THIS MODULE, AND NOT THE UPSERT
-----------------------------------
`upsert_brand_products` never names these columns, for the reason documented
next to its INSERT: every product dict that reaches it comes from a
spreadsheet, the seed JSON or a SELECT round-trip, none of which can carry a
vector, so naming the column there would only ever NULL it. The writes here
are targeted UPDATEs by image_id, the same shape as nutrition_score_sync.
WHY A SINGLE BOUNDED WORKER
---------------------------
The upsert calls `schedule()`, which enqueues (brand, image_ids) and returns.
One daemon thread drains the queue, so at most one image is being downloaded
and decoded in the API process at any moment, a write never waits on a CDN,
and an overflowing queue simply leaves rows for
`scripts/backfill_image_vectors.py`. Same shape as app/core/batch_worker.py,
for the same reasons.
Pillow, not OpenCV: it is already a dependency, `Image.draft()` decodes a JPEG
at 1/8 scale for a target this small, and it honours EXIF orientation on
in-memory bytes. OpenCV would add ~130MB on disk and always decode at full
resolution, on a container that already holds torch.
"""
from __future__ import annotations
import io
import logging
import queue
import threading
import time
from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple
from urllib.parse import urlparse
from app.infrastructure.settings import (
ENABLE_IMAGE_VECTORS,
IMAGE_VECTOR_HOST_PAUSE_SECONDS,
IMAGE_VECTOR_MAX_BYTES,
IMAGE_VECTOR_MAX_PIXELS,
IMAGE_VECTOR_QUEUE_MAX,
IMAGE_VECTOR_TIMEOUT_SECONDS,
MIN_IMAGE_BYTES,
)
logger = logging.getLogger(__name__)
IMG_VECTOR_SIZE = 32
IMG_VECTOR_DIM = IMG_VECTOR_SIZE * IMG_VECTOR_SIZE * 3 # 3072
IMG_VECTOR_COLUMNS = ("img_vector", "img_vector_src")
_IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp")
_CHUNK = 64 * 1024
# ---------------------------------------------------------------------------
# Which image
# ---------------------------------------------------------------------------
def primary_image_url(row: Mapping[str, Any]) -> Optional[str]:
"""The URL the product card displays, or None if it shows nothing.
`image_url` first, else `image_urls[0]` - ProductCard.jsx builds its list in
that order. Normalised through `_usable_image_url` so blanks, bare paths and
hosts in `_DEAD_IMAGE_HOSTS` never cost a request. The string returned here
is what goes into `img_vector_src`, so staleness is always judged against
the same definition.
"""
from app.services.vector_store import _usable_image_url
candidate = _usable_image_url(row.get("image_url"))
if candidate:
return candidate
urls = row.get("image_urls") or []
if isinstance(urls, (list, tuple)) and urls:
return _usable_image_url(urls[0])
return None
# ---------------------------------------------------------------------------
# Bytes -> vector
# ---------------------------------------------------------------------------
def pixel_vector(image_bytes: bytes) -> Optional[List[int]]:
"""32x32 RGB pixels of `image_bytes` as 3072 ints in 0..255, or None.
None for anything Pillow cannot turn into a picture: corrupt or truncated
data, an unknown format, or a header claiming more than
IMAGE_VECTOR_MAX_PIXELS (checked before decoding, so a decompression bomb
costs a header read and nothing more). Never raises - pytest runs with
warnings as errors, and a DecompressionBombWarning is one of the things
the broad except is there to absorb.
"""
if not image_bytes:
return None
try:
from PIL import Image, ImageOps
im = Image.open(io.BytesIO(image_bytes))
if im.width * im.height > IMAGE_VECTOR_MAX_PIXELS:
logger.debug("image rejected: %dx%d exceeds pixel cap", im.width, im.height)
return None
# JPEG only: ask the decoder for a DCT-scaled decode no smaller than
# 4x the target. A no-op for every other format.
try:
im.draft("RGB", (IMG_VECTOR_SIZE * 4, IMG_VECTOR_SIZE * 4))
except Exception: # noqa: BLE001 - draft is an optimisation, not a requirement
pass
im = ImageOps.exif_transpose(im) or im
has_alpha = "A" in im.getbands() or "transparency" in im.info
if has_alpha:
rgba = im.convert("RGBA")
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
im = Image.alpha_composite(white, rgba)
im = im.convert("RGB")
im = im.resize((IMG_VECTOR_SIZE, IMG_VECTOR_SIZE), Image.Resampling.BILINEAR)
values = list(im.tobytes())
if len(values) != IMG_VECTOR_DIM:
logger.debug("image rejected: %d values, expected %d", len(values), IMG_VECTOR_DIM)
return None
return values
except Exception as exc: # noqa: BLE001 - see docstring
logger.debug("image undecodable: %s", exc)
return None
def to_pg(vector: Iterable[int]) -> str:
"""The text form pgvector accepts, identical to what the upsert uses for `embedding`."""
return "[" + ",".join(str(int(v)) for v in vector) + "]"
# ---------------------------------------------------------------------------
# URL -> bytes
# ---------------------------------------------------------------------------
def download_image_bytes(url: str, timeout: Optional[float] = None) -> Optional[bytes]:
"""Fetch `url` as image bytes, or None.
Same two-step Referer strategy as `image_search.validate_image_url_live`:
first with a same-site Referer (hotlink-protected CDNs), then with none
(hosts that reject a Referer pretending to be same-site). Streams in
chunks and gives up past IMAGE_VECTOR_MAX_BYTES, so a wrong URL to a video
cannot fill memory; anything under MIN_IMAGE_BYTES is a placeholder.
"""
if not url or not url.startswith(("http://", "https://")):
return None
import requests
from app.services.image_search import _BROWSER_UA
parsed = urlparse(url)
same_site = f"{parsed.scheme}://{parsed.netloc}/" if parsed.netloc else None
timeout = timeout or IMAGE_VECTOR_TIMEOUT_SECONDS
for referer in (same_site, None):
headers = {"User-Agent": _BROWSER_UA, "Accept": "image/*,*/*;q=0.8"}
if referer:
headers["Referer"] = referer
resp = None
try:
resp = requests.get(url, headers=headers, timeout=timeout, stream=True)
if resp.status_code != 200:
logger.debug("HTTP %s for %s", resp.status_code, url)
continue
content_type = (resp.headers.get("content-type") or "").lower()
if "image" not in content_type:
# A server that says text/html means it - typically a CDN that
# now redirects every dead path to a homepage (uat.amul.com does
# exactly this), and the .png in the URL is no evidence at all.
# Only an unlabelled body (octet-stream, blank) gets the benefit
# of the extension.
if content_type.startswith("text/") or not parsed.path.lower().endswith(_IMAGE_EXTENSIONS):
logger.debug("not an image (%s): %s", content_type, url)
return None
buf = bytearray()
for chunk in resp.iter_content(chunk_size=_CHUNK):
if not chunk:
continue
buf.extend(chunk)
if len(buf) > IMAGE_VECTOR_MAX_BYTES:
logger.debug("image over %d bytes, abandoned: %s", IMAGE_VECTOR_MAX_BYTES, url)
return None
if len(buf) < MIN_IMAGE_BYTES:
logger.debug("image too small (%d bytes): %s", len(buf), url)
return None
return bytes(buf)
except Exception as exc: # noqa: BLE001 - a bad host must not stop the batch
logger.debug("download failed for %s: %s", url, exc)
continue
finally:
if resp is not None:
try:
resp.close()
except Exception: # noqa: BLE001
pass
return None
def vector_for_url(url: str, timeout: Optional[float] = None) -> Tuple[Optional[List[int]], str]:
"""(vector or None, the URL it was computed from)."""
data = download_image_bytes(url, timeout=timeout)
if data is None:
return None, url
return pixel_vector(data), url
# ---------------------------------------------------------------------------
# Table access
# ---------------------------------------------------------------------------
def table_has_columns(cur, table_name: str) -> bool:
"""True when the table carries both img_vector columns.
Every writer checks this first, so a table the migration has not reached -
or one whose pgvector refused `vector(3072)` - is skipped, never errored.
"""
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = %s AND column_name = ANY(%s)",
(table_name, list(IMG_VECTOR_COLUMNS)),
)
present = {row[0] for row in cur.fetchall()}
return all(col in present for col in IMG_VECTOR_COLUMNS)
def rows_needing_vectors(
cur,
table_name: str,
image_ids: Optional[List[str]] = None,
*,
recompute_stale: bool = True,
limit: Optional[int] = None,
) -> List[Dict[str, Any]]:
"""Rows that have a primary image and no current vector for it.
"Current" means `img_vector_src` equals the primary URL; a row whose
`image_url` was rewritten by a later upsert is stale and is included when
`recompute_stale` is on. Rows with no usable primary are dropped here, so
the caller never spends a request on them.
"""
sql = (
f"SELECT image_id, product_name, image_url, image_urls, img_vector_src, "
f"(img_vector IS NULL) AS missing FROM {table_name} "
f"WHERE (img_vector IS NULL OR img_vector_src IS DISTINCT FROM "
f"COALESCE(NULLIF(image_url, ''), image_urls[1]))"
)
params: List[Any] = []
if image_ids is not None:
sql += " AND image_id = ANY(%s)"
params.append(list(image_ids))
sql += " ORDER BY updated_at DESC"
if limit is not None:
sql += " LIMIT %s"
params.append(int(limit))
cur.execute(sql, params)
colnames = [d[0] for d in cur.description]
out: List[Dict[str, Any]] = []
for raw in cur.fetchall():
row = dict(zip(colnames, raw))
primary = primary_image_url(row)
if not primary:
continue
if not row.get("missing") and not recompute_stale:
continue
if not row.get("missing") and row.get("img_vector_src") == primary:
continue
row["primary_url"] = primary
out.append(row)
return out
def store_vector(cur, table_name: str, image_id: str, vector: List[int], src: str) -> None:
"""Write one vector. Deliberately leaves `updated_at` alone: the catalog
listing orders on it, and a background enrichment must not reshuffle it."""
cur.execute(
f"UPDATE {table_name} SET img_vector = %s::vector, img_vector_src = %s WHERE image_id = %s",
(to_pg(vector), src, image_id),
)
def backfill_rows(
brand: str,
image_ids: Optional[List[str]] = None,
*,
recompute_stale: bool = True,
limit: Optional[int] = None,
dry_run: bool = False,
pause_seconds: Optional[float] = None,
) -> Dict[str, int]:
"""Compute and store vectors for one brand, sequentially. Never raises.
Returns counts: candidates, computed, failed, skipped (no columns yet).
`pause_seconds` is the gap between two requests to the same host.
"""
from app.services.vector_store import _connect, _table_name
totals = {"candidates": 0, "computed": 0, "failed": 0, "skipped": 0}
conn = _connect()
if conn is None:
logger.warning("image vectors: no database connection for %s", brand)
return totals
pause = IMAGE_VECTOR_HOST_PAUSE_SECONDS if pause_seconds is None else pause_seconds
table_name = _table_name(brand)
last_by_host: Dict[str, float] = {}
try:
with conn.cursor() as cur:
if not table_has_columns(cur, table_name):
logger.info("image vectors: %s has no img_vector columns yet - skipped", table_name)
totals["skipped"] = 1
return totals
rows = rows_needing_vectors(
cur, table_name, image_ids, recompute_stale=recompute_stale, limit=limit
)
totals["candidates"] = len(rows)
for row in rows:
url = row["primary_url"]
host = urlparse(url).netloc.lower()
wait = last_by_host.get(host, 0.0) + pause - time.monotonic()
if wait > 0:
time.sleep(wait)
last_by_host[host] = time.monotonic()
vector, src = vector_for_url(url)
if vector is None:
totals["failed"] += 1
continue
if not dry_run:
store_vector(cur, table_name, row["image_id"], vector, src)
totals["computed"] += 1
except Exception: # noqa: BLE001 - enrichment must never surface as a failure
logger.exception("image vectors: backfill failed for %s", brand)
finally:
conn.close()
if totals["candidates"]:
logger.info(
"image vectors: %s - %d candidate(s), %d computed, %d failed%s",
table_name, totals["candidates"], totals["computed"], totals["failed"],
" (dry run)" if dry_run else "",
)
return totals
# ---------------------------------------------------------------------------
# The hook the upsert calls
# ---------------------------------------------------------------------------
_queue: "queue.Queue[Tuple[str, List[str]]]" = queue.Queue(maxsize=max(1, IMAGE_VECTOR_QUEUE_MAX))
_worker: Optional[threading.Thread] = None
_lock = threading.Lock()
def queue_depth() -> int:
return _queue.qsize()
def is_running() -> bool:
return _worker is not None and _worker.is_alive()
def schedule(brand: str, image_ids: Iterable[str]) -> bool:
"""Queue the rows just written. Returns True if queued.
Off when ENABLE_IMAGE_VECTORS is false. `put_nowait`, never `put`: this
runs inside the write path and must not wait. A full queue is logged and
dropped - the backfill script picks those rows up, because they are the
ones with no vector.
"""
if not ENABLE_IMAGE_VECTORS:
return False
ids = [i for i in image_ids if i]
if not ids:
return False
try:
_queue.put_nowait((brand, ids))
except queue.Full:
logger.warning(
"image vectors: queue full (%d), %d row(s) for %s left for the backfill script",
_queue.maxsize, len(ids), brand,
)
return False
_ensure_worker()
return True
def _ensure_worker() -> None:
global _worker
with _lock:
if _worker is not None and _worker.is_alive():
return
_worker = threading.Thread(target=_loop, name="image-vector-worker", daemon=True)
_worker.start()
def _loop() -> None:
"""Drain forever; one bad batch must not strand the ones behind it."""
while True:
brand, ids = _queue.get()
try:
backfill_rows(brand, ids, recompute_stale=True)
except Exception: # noqa: BLE001 - see docstring
logger.exception("image vectors: worker failed on %s", brand)
finally:
_queue.task_done()

143
app/services/vector_store.py
View File

@@ -5,6 +5,7 @@ from typing import List, Optional, Dict, Any, Tuple
import logging
import os
import re
import threading
import time
from datetime import datetime
@@ -206,7 +207,16 @@ def get_brand_table_ddl(brand: str) -> str:
-- Vector embedding for search
embedding vector(384),
-- 32x32 RGB pixel thumbnail of the PRIMARY image (image_url, else
-- image_urls[0]) as 3072 uint8 values, and the URL it was computed
-- from. Written ONLY by app/services/image_vector.py via targeted
-- UPDATEs - never by upsert_brand_products, for the same reason as
-- nutrition_score above. Not indexed: pgvector caps vector indexes
-- at 2000 dims, and an exact scan is fine at this catalog's size.
img_vector vector(3072),
img_vector_src TEXT,
-- Timestamps
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -243,6 +253,11 @@ def _connect() -> Optional[psycopg.Connection]:
return None
# Columns whose ADD COLUMN may be refused by an older server and must not
# fail the write - see the loop in _ensure_columns.
_TYPE_OPTIONAL_COLUMNS = frozenset({"img_vector"})
def _ensure_columns(cur, table_name: str) -> None:
"""Add missing columns and relax legacy NOT NULL constraints for smooth schema migration."""
col_defs = {
@@ -300,6 +315,11 @@ def _ensure_columns(cur, table_name: str) -> None:
"nutrition_score": "NUMERIC",
"health_score": "NUMERIC",
"embedding": "vector(384)",
# Written only by app/services/image_vector.py - see get_brand_table_ddl.
# Plain string literals, always: scripts/migrate_brand_schema.py reads
# this dict back out of the source with ast.literal_eval.
"img_vector": "vector(3072)",
"img_vector_src": "TEXT",
}
cur.execute(
"SELECT column_name, is_nullable, column_default FROM information_schema.columns "
@@ -312,7 +332,22 @@ def _ensure_columns(cur, table_name: str) -> None:
# 1. Add missing columns
for col, col_type in col_defs.items():
if col not in existing:
cur.execute(f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {col} {col_type}")
if col in _TYPE_OPTIONAL_COLUMNS:
# vector(3072) needs pgvector >= 0.4 (the 16000-dim ceiling).
# The production server's version is not pinned anywhere, so
# a refusal here is logged and the write goes on without the
# column; image_vector.table_has_columns() skips the table.
# Safe to continue because _connect() is autocommit.
try:
cur.execute(f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {col} {col_type}")
except Exception as e: # noqa: BLE001 - optional column, see above
logger.warning(
f"Could not add {col} {col_type} to {table_name} "
f"(pgvector too old for this type?): {e}"
)
continue
else:
cur.execute(f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {col} {col_type}")
if col not in ("created_at", "updated_at"):
logger.info(f"Added missing column '{col}' to {table_name}")
@@ -327,7 +362,7 @@ def _ensure_columns(cur, table_name: str) -> None:
# `nutrients_per_100g` joins them: it is mirrored from nutrition_facts by
# the same sync, not written by the INSERT, and is likewise a current
# column rather than legacy debris.
inserted_cols = {"id", "product_name", "title", "description", "category", "image_id", "image_url", "image_urls", "price_range", "size_variants", "providers", "fssai_license", "product_sku", "sku_source", "hsn_code", "final_selling_price", "selling_price", "barcode", "barcode_type", "gtin", "ean13", "upc", "barcode_source", "barcode_verified", "barcode_lookup_status", "barcode_last_updated", "gst_percent", "tax_amount", "hsn_gst_needs_review", "highlights", "nutrients", "nutrients_per_100g", "field_sources", "search_query", "nutrition_score", "health_score", "embedding", "created_at", "updated_at"}
inserted_cols = {"id", "product_name", "title", "description", "category", "image_id", "image_url", "image_urls", "price_range", "size_variants", "providers", "fssai_license", "product_sku", "sku_source", "hsn_code", "final_selling_price", "selling_price", "barcode", "barcode_type", "gtin", "ean13", "upc", "barcode_source", "barcode_verified", "barcode_lookup_status", "barcode_last_updated", "gst_percent", "tax_amount", "hsn_gst_needs_review", "highlights", "nutrients", "nutrients_per_100g", "field_sources", "search_query", "nutrition_score", "health_score", "embedding", "img_vector", "img_vector_src", "created_at", "updated_at"}
for col, is_nullable, col_def in col_info:
if col not in inserted_cols and is_nullable == 'NO' and col_def is None:
cur.execute(f"ALTER TABLE {table_name} ALTER COLUMN {col} DROP NOT NULL")
@@ -343,6 +378,10 @@ def _ensure_columns(cur, table_name: str) -> None:
except Exception as e:
logger.warning(f"Unique index creation on {table_name}.image_id: {e}")
# The projected reads cache each table's column list; a column added here
# must be visible to the next SELECT.
_invalidate_product_columns_cache()
def ensure_brand_schema(brand: str) -> str:
"""Ensure brand-specific table exists and return table name"""
@@ -608,8 +647,13 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
#
# They are written only by app/services/nutrition_score_sync.py,
# which mirrors nutrition_insights. Extra score keys present in an
# incoming product dict (get_products_by_brand is SELECT *, so
# stage_11's merge carries them back in) are simply ignored here.
# incoming product dict (stage_11's merge carries prior rows back
# in) are simply ignored here.
#
# img_vector and img_vector_src are omitted for the same reason:
# no writer reaching this statement can carry a pixel vector, so
# naming them would only ever NULL them. app/services/image_vector.py
# owns them and is scheduled below, after the write.
cur.executemany(
f"""
INSERT INTO {table_name}
@@ -765,6 +809,15 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
except Exception: # noqa: BLE001 - cache invalidation must never fail a write
pass
# Queue the rows just written for their img_vector. Off the request path
# (one bounded worker thread), fill-only-missing-or-stale, and a failure
# here is a missing thumbnail, never a failed write.
try:
from app.services.image_vector import schedule as _schedule_image_vectors
_schedule_image_vectors(brand, expected_ids)
except Exception: # noqa: BLE001 - enrichment must never fail a write
logger.debug("image-vector scheduling skipped for %s", brand, exc_info=True)
return persisted
@@ -1127,7 +1180,7 @@ def get_products_by_brand(brand: str, limit: Optional[int] = None, offset: int =
logger.warning(f"Table {table_name} does not exist")
return []
sql = f"SELECT * FROM {table_name}"
sql = f"SELECT {_product_columns(cur, table_name)} FROM {table_name}"
params: List[Any] = []
if category:
sql += " WHERE category ILIKE %s"
@@ -1184,7 +1237,10 @@ def get_product_by_image_id(brand: str, image_id: str) -> Optional[Dict[str, Any
with conn, conn.cursor() as cur:
if not _table_exists(cur, table_name):
return None
cur.execute(f"SELECT * FROM {table_name} WHERE image_id = %s LIMIT 1", (image_id,))
cur.execute(
f"SELECT {_product_columns(cur, table_name)} FROM {table_name} WHERE image_id = %s LIMIT 1",
(image_id,),
)
row = cur.fetchone()
if not row:
return None
@@ -1319,8 +1375,8 @@ def semantic_search(
if not _table_exists(cur, table_name):
continue
sql = (
f"SELECT *, embedding <=> %s::vector AS distance FROM {table_name} "
f"WHERE embedding IS NOT NULL"
f"SELECT {_product_columns(cur, table_name)}, embedding <=> %s::vector AS distance "
f"FROM {table_name} WHERE embedding IS NOT NULL"
)
params: List[Any] = [embedding_str]
if category:
@@ -1389,7 +1445,7 @@ def text_search(
for brand_label, table_name in tables:
if not _table_exists(cur, table_name):
continue
sql = f"SELECT * FROM {table_name}"
sql = f"SELECT {_product_columns(cur, table_name)} FROM {table_name}"
where_clauses: List[str] = []
params: List[Any] = []
@@ -1567,7 +1623,7 @@ def lexical_search(
params.append(f"%{category}%")
sql = (
f"SELECT *{distance_select}, {tier_sql} FROM {table_name} "
f"SELECT {_product_columns(cur, table_name)}{distance_select}, {tier_sql} FROM {table_name} "
f"WHERE {' AND '.join(where_clauses)} "
f"ORDER BY lex_tier ASC, length(coalesce(product_name, title, '')) ASC, "
f"updated_at DESC LIMIT %s"
@@ -1621,6 +1677,71 @@ def _table_exists(cur, table_name: str) -> bool:
return bool(row and row[0])
# ---------------------------------------------------------------------------
# Projected reads
# ---------------------------------------------------------------------------
# Every product read used to be `SELECT *`, which dragged the vector(384)
# embedding over the wire on every listing: measured at 4.7KB of a 7.2KB row
# (see enrichment/catalog_consensus.py). img_vector is 12KB more per row, and
# the "all products" browse pulls the whole catalog in one call. Nothing that
# reads these dicts uses either column - the API mappers copy fields by name,
# and the one exporter that looked at `embedding` (brand_sync) was already
# getting a pgvector string it could not parse. So the read helpers select
# every column EXCEPT these, by name.
#
# Column lists are probed from information_schema (brand tables written under
# older DDLs have divergent columns) and cached for a short TTL; a table the
# cache does not know falls back to `*`, so a table created between refreshes
# cannot 500.
PRODUCT_READ_EXCLUDE = frozenset({"embedding", "img_vector"})
PRODUCT_COLUMNS_TTL_SECONDS = int(os.getenv("PRODUCT_COLUMNS_TTL_SECONDS", "60"))
_COLUMNS_CACHE: Dict[str, Any] = {"at": 0.0, "data": {}}
_COLUMNS_LOCK = threading.Lock()
def _invalidate_product_columns_cache() -> None:
with _COLUMNS_LOCK:
_COLUMNS_CACHE["at"] = 0.0
_COLUMNS_CACHE["data"] = {}
def _refresh_product_columns(cur) -> None:
cur.execute(
"""
SELECT table_name, column_name FROM information_schema.columns
WHERE table_schema = 'public' AND table_name LIKE 'brand\\_%'
ORDER BY table_name, ordinal_position
"""
)
data: Dict[str, List[str]] = {}
for table, col in cur.fetchall():
data.setdefault(table, []).append(col)
with _COLUMNS_LOCK:
_COLUMNS_CACHE["at"] = time.monotonic()
_COLUMNS_CACHE["data"] = data
def _product_columns(cur, table_name: str, exclude: frozenset = PRODUCT_READ_EXCLUDE) -> str:
"""The SELECT list for `table_name` without the vector columns, or `*`."""
with _COLUMNS_LOCK:
stale = time.monotonic() - _COLUMNS_CACHE["at"] > PRODUCT_COLUMNS_TTL_SECONDS
cols = None if stale else _COLUMNS_CACHE["data"].get(table_name)
if cols is None:
try:
_refresh_product_columns(cur)
except Exception as e: # noqa: BLE001 - a failed probe means SELECT *, not a failed read
logger.warning("Could not probe columns for %s: %s", table_name, e)
return "*"
with _COLUMNS_LOCK:
cols = _COLUMNS_CACHE["data"].get(table_name)
if not cols:
return "*"
kept = [c for c in cols if c not in exclude]
if not kept:
return "*"
return ", ".join(f'"{c}"' for c in kept)
def _list_brand_table_suffixes(cur, *, include_inactive: bool = False) -> List[str]:
"""Return brand-table suffixes (e.g. 'parle' from 'brand_parle') for every brand table.