updates on the backend
This commit is contained in:
sriram
0
app/services/__init__.py
Normal file
0
app/services/__init__.py
Normal file
72
app/services/analytics_service.py
Normal file
72
app/services/analytics_service.py
Normal file
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.intelligence import analytics as A
|
||||
from app.intelligence.popularity_model import popularity_scorer
|
||||
from app.intelligence import engagement_simulation
|
||||
from app.services import store_db
|
||||
|
||||
|
||||
def store_dashboard(store_id: str) -> Dict:
|
||||
orders = store_db.get_orders_df(store_id=store_id)
|
||||
order_items = store_db.get_order_items_df(store_id=store_id)
|
||||
store_products = store_db.get_all_store_products_df()
|
||||
store_products_here = store_products[store_products["store_id"] == store_id]
|
||||
|
||||
sales = A.sales_analytics(orders)
|
||||
profit = A.profit_analytics(order_items, store_products_here[["store_id", "brand", "image_id", "cost_price"]])
|
||||
inventory = A.inventory_analytics(store_products_here)
|
||||
top_sellers = A.top_products(order_items, limit=10, by="revenue")
|
||||
lowest_sellers = A.top_products(order_items, limit=10, by="units", ascending=True)
|
||||
|
||||
return {
|
||||
"store_id": store_id, "sales": sales, "profit": profit, "inventory": inventory,
|
||||
"top_selling_products": top_sellers, "lowest_selling_products": lowest_sellers,
|
||||
}
|
||||
|
||||
|
||||
def chain_comparison() -> Dict:
|
||||
orders = store_db.get_orders_df()
|
||||
order_items = store_db.get_order_items_df()
|
||||
store_products = store_db.get_all_store_products_df()
|
||||
stores_df = pd.DataFrame(store_db.list_stores())
|
||||
if stores_df.empty:
|
||||
return {"stores": [], "best_performing": None, "lowest_performing": None, "highest_revenue": None, "highest_profit": None}
|
||||
stores_df = stores_df[["store_id", "store_name", "tier", "footfall_index"]]
|
||||
return A.store_comparison(orders, order_items, store_products[["store_id", "brand", "image_id", "cost_price"]], stores_df)
|
||||
|
||||
|
||||
def product_dashboard(brand: str, image_id: str) -> Dict:
|
||||
order_items = store_db.get_order_items_df()
|
||||
store_products = store_db.get_all_store_products_df()
|
||||
|
||||
products = store_products[["brand", "image_id"]].drop_duplicates()
|
||||
orders_count = order_items.groupby(order_items["brand"] + "||" + order_items["image_id"])["quantity"].sum()
|
||||
eng_df = engagement_simulation.simulate_engagement(products[(products["brand"] == brand) & (products["image_id"] == image_id)], orders_count)
|
||||
engagement_row = eng_df.iloc[0].to_dict() if not eng_df.empty else None
|
||||
|
||||
popularity = None
|
||||
if engagement_row:
|
||||
from app.intelligence import features as F
|
||||
feat_row = pd.DataFrame([{
|
||||
"views_norm": min(engagement_row["views"] / 10, 100),
|
||||
"wishlist_norm": min(engagement_row["wishlist_count"] / 2, 100),
|
||||
"orders_norm": min(engagement_row["orders_count"] * 2, 100),
|
||||
"rating_norm": engagement_row["avg_rating"] * 20,
|
||||
"conversion_norm": engagement_row["conversion_rate"] * 100,
|
||||
}])
|
||||
popularity = float(popularity_scorer.score(feat_row).iloc[0])
|
||||
|
||||
return A.product_analytics(
|
||||
brand, image_id, order_items,
|
||||
store_products[["store_id", "brand", "image_id", "selling_price", "cost_price"]],
|
||||
engagement_row=engagement_row, popularity_score=popularity,
|
||||
)
|
||||
|
||||
|
||||
def top_products(by: str = "revenue", limit: int = 10, ascending: bool = False) -> List[Dict]:
|
||||
order_items = store_db.get_order_items_df()
|
||||
return A.top_products(order_items, limit=limit, ascending=ascending, by=by)
|
||||
293
app/services/brand_registry.py
Normal file
293
app/services/brand_registry.py
Normal file
@@ -0,0 +1,293 @@
|
||||
BRAND_ALIASES = {
|
||||
# Cadbury family
|
||||
"cadbury gems": "cadbury",
|
||||
"cadbury chocolates": "cadbury",
|
||||
"cadbury dairy milk": "cadbury",
|
||||
"cadbury 5 star": "cadbury",
|
||||
"cadbury bournvita": "cadbury",
|
||||
"cadbury oreo": "cadbury",
|
||||
"cadbury celebrations": "cadbury",
|
||||
"cadbury perk": "cadbury",
|
||||
"cadbury temptations": "cadbury",
|
||||
"cadbury eclairs": "cadbury",
|
||||
"cadbury silk": "cadbury",
|
||||
# Brooke Bond family
|
||||
"brooke bond red label": "brooke bond",
|
||||
"brooke bond taj mahal": "brooke bond",
|
||||
"brooke bond yellow label": "brooke bond",
|
||||
"brooke bond select": "brooke bond",
|
||||
"brooke bond strong": "brooke bond",
|
||||
"brooke bond natural care": "brooke bond",
|
||||
# Parle family
|
||||
"parle biscuits": "parle",
|
||||
"parle hide & seek": "parle",
|
||||
"parle milano": "parle",
|
||||
"parle parle-g": "parle",
|
||||
"parle krackjack": "parle",
|
||||
"parle monaco": "parle",
|
||||
"parle 20-20": "parle",
|
||||
"parle fab": "parle",
|
||||
"parle nutri": "parle",
|
||||
"parle magix": "parle",
|
||||
# Britannia family
|
||||
"britannia good day": "britannia",
|
||||
"britannia tiger": "britannia",
|
||||
"britannia nutrichoice": "britannia",
|
||||
"britannia milk bikis": "britannia",
|
||||
"britannia marie gold": "britannia",
|
||||
"britannia bourbon": "britannia",
|
||||
"britannia little hearts": "britannia",
|
||||
"britannia treat": "britannia",
|
||||
"britannia 50-50": "britannia",
|
||||
"britannia jim jam": "britannia",
|
||||
# Nestle family
|
||||
"nestle maggi": "nestle",
|
||||
"nestle kitkat": "nestle",
|
||||
"nestle milkybar": "nestle",
|
||||
"nestle nescafe": "nestle",
|
||||
"nestle cheerios": "nestle",
|
||||
"nestle lactogen": "nestle",
|
||||
"nestle cerelac": "nestle",
|
||||
"nestle everyday": "nestle",
|
||||
"nestle munch": "nestle",
|
||||
"nestle bar-one": "nestle",
|
||||
"nestle polo": "nestle",
|
||||
"nestle nestea": "nestle",
|
||||
"nestle milkmaid": "nestle",
|
||||
"nestle a+": "nestle",
|
||||
"nestle slim": "nestle",
|
||||
"nestle nan": "nestle",
|
||||
"nestle resource": "nestle",
|
||||
"nestle goody": "nestle",
|
||||
"nestle milo": "nestle",
|
||||
"nestle fresh & natural": "nestle",
|
||||
"nestle mithai mate": "nestle",
|
||||
"nestle masala ae magic": "nestle",
|
||||
"nestle hot heads": "nestle",
|
||||
"nestle butter": "nestle",
|
||||
"nestle ghee": "nestle",
|
||||
"nestle paneer": "nestle",
|
||||
"nestle peptamen": "nestle",
|
||||
# Hindustan Unilever (HUL) family
|
||||
"hul lux": "hindustan unilever",
|
||||
"hul lifebuoy": "hindustan unilever",
|
||||
"hul dove": "hindustan unilever",
|
||||
"hul sunsilk": "hindustan unilever",
|
||||
"hul clinic plus": "hindustan unilever",
|
||||
"hul ponds": "hindustan unilever",
|
||||
"hul fair & lovely": "hindustan unilever",
|
||||
"hul axe": "hindustan unilever",
|
||||
"hul closeup": "hindustan unilever",
|
||||
"hul pepsodent": "hindustan unilever",
|
||||
"hul rin": "hindustan unilever",
|
||||
"hul surf excel": "hindustan unilever",
|
||||
"hul wheel": "hindustan unilever",
|
||||
"hul vim": "hindustan unilever",
|
||||
"hul knorr": "hindustan unilever",
|
||||
"hul lipton": "hindustan unilever",
|
||||
"hul bru": "hindustan unilever",
|
||||
"hul kwality walls": "hindustan unilever",
|
||||
"hul horlicks": "hindustan unilever",
|
||||
"hul boost": "hindustan unilever",
|
||||
"hul maltova": "hindustan unilever",
|
||||
"hul laxmi": "hindustan unilever",
|
||||
"hul tata tea": "hindustan unilever",
|
||||
# P&G (Procter & Gamble) family
|
||||
"pg pantene": "p&g",
|
||||
"pg head & shoulders": "p&g",
|
||||
"pg gillette": "p&g",
|
||||
"pg vicks": "p&g",
|
||||
"pg ariel": "p&g",
|
||||
"pg tide": "p&g",
|
||||
"pg pampers": "p&g",
|
||||
"pg whisper": "p&g",
|
||||
"pg oral-b": "p&g",
|
||||
"pg old spice": "p&g",
|
||||
"pg ivory": "p&g",
|
||||
"pg pringles": "p&g",
|
||||
"pg downy": "p&g",
|
||||
"pg tide pods": "p&g",
|
||||
"pg fairy": "p&g",
|
||||
"pg always": "p&g",
|
||||
"pg dawn": "p&g",
|
||||
# ITC family
|
||||
"itc sunfeast": "itc",
|
||||
"itc bingo": "itc",
|
||||
"itc yippee": "itc",
|
||||
"itc aashirvaad": "itc",
|
||||
"itc fiama": "itc",
|
||||
"itc engage": "itc",
|
||||
"itc savlon": "itc",
|
||||
"itc nimyle": "itc",
|
||||
"itc classmate": "itc",
|
||||
"itc mangaldeep": "itc",
|
||||
"itc candyman": "itc",
|
||||
"itc sunfeast dark fantasy": "itc",
|
||||
"itc sunfeast marie": "itc",
|
||||
"itc sunfeast bounce": "itc",
|
||||
"itc bingo tedhe medhe": "itc",
|
||||
"itc farmlite": "itc",
|
||||
# Marico family
|
||||
"marico parachute": "marico",
|
||||
"marico saffola": "marico",
|
||||
"marico livon": "marico",
|
||||
"marico set wet": "marico",
|
||||
"marico hair & care": "marico",
|
||||
"marico nihar": "marico",
|
||||
"marico mediker": "marico",
|
||||
"marico kaya": "marico",
|
||||
"marico veggie cleanse": "marico",
|
||||
# Dabur family
|
||||
"dabur amla": "dabur",
|
||||
"dabur vatika": "dabur",
|
||||
"dabur red": "dabur",
|
||||
"dabur chyawanprash": "dabur",
|
||||
"dabur hajmola": "dabur",
|
||||
"dabur real": "dabur",
|
||||
"dabur pudin hara": "dabur",
|
||||
"dabur honitus": "dabur",
|
||||
"dabur odomos": "dabur",
|
||||
"dabur gulabari": "dabur",
|
||||
"dabur lal tail": "dabur",
|
||||
"dabur janam ghutti": "dabur",
|
||||
"dabur meswak": "dabur",
|
||||
"dabur promite": "dabur",
|
||||
# Amul family
|
||||
"amul butter": "amul",
|
||||
"amul milk": "amul",
|
||||
"amul cheese": "amul",
|
||||
"amul ice cream": "amul",
|
||||
"amul ghee": "amul",
|
||||
"amul mithai": "amul",
|
||||
"amul paneer": "amul",
|
||||
"amul cream": "amul",
|
||||
"amul dahi": "amul",
|
||||
"amul lassi": "amul",
|
||||
"amul kool": "amul",
|
||||
"amul pro": "amul",
|
||||
"amul spray": "amul",
|
||||
"amul shrikhand": "amul",
|
||||
"amul basundi": "amul",
|
||||
# Colgate-Palmolive family
|
||||
"colgate dental cream": "colgate-palmolive",
|
||||
"colgate maxfresh": "colgate-palmolive",
|
||||
"colgate sensitive": "colgate-palmolive",
|
||||
"colgate total": "colgate-palmolive",
|
||||
"colgate 360": "colgate-palmolive",
|
||||
"colgate active salt": "colgate-palmolive",
|
||||
"palmolive shampoo": "colgate-palmolive",
|
||||
"palmolive body wash": "colgate-palmolive",
|
||||
"palmolive soap": "colgate-palmolive",
|
||||
"colgate toothbrush": "colgate-palmolive",
|
||||
# Johnson & Johnson family
|
||||
"johnson & johnson": "johnson & johnson",
|
||||
"jnj baby shampoo": "johnson & johnson",
|
||||
"jnj baby oil": "johnson & johnson",
|
||||
"jnj baby lotion": "johnson & johnson",
|
||||
"jnj baby powder": "johnson & johnson",
|
||||
"jnj stayfree": "johnson & johnson",
|
||||
"jnj listerine": "johnson & johnson",
|
||||
"jnj band-aid": "johnson & johnson",
|
||||
"jnj neutrogena": "johnson & johnson",
|
||||
"jnj clean & clear": "johnson & johnson",
|
||||
"jnj savlon": "johnson & johnson",
|
||||
"jnj benadryl": "johnson & johnson",
|
||||
# Reckitt Benckiser family
|
||||
"rb dettol": "reckitt benckiser",
|
||||
"rb lizol": "reckitt benckiser",
|
||||
"rb harpic": "reckitt benckiser",
|
||||
"rb mortein": "reckitt benckiser",
|
||||
"rb vanish": "reckitt benckiser",
|
||||
"rb durex": "reckitt benckiser",
|
||||
"rb strepsils": "reckitt benckiser",
|
||||
"rb mucinex": "reckitt benckiser",
|
||||
"rb finish": "reckitt benckiser",
|
||||
"rb veet": "reckitt benckiser",
|
||||
# Godrej Consumer family
|
||||
"godrej no.1": "godrej",
|
||||
"godrej cinthol": "godrej",
|
||||
"godrej expert": "godrej",
|
||||
"godrej hit": "godrej",
|
||||
"godrej good knight": "godrej",
|
||||
"godrej aer": "godrej",
|
||||
"godrej protekt": "godrej",
|
||||
"godrej magic": "godrej",
|
||||
"godrej renew": "godrej",
|
||||
# PepsiCo family
|
||||
"pepsico lays": "pepsico",
|
||||
"pepsico kurkure": "pepsico",
|
||||
"pepsico doritos": "pepsico",
|
||||
"pepsico cheetos": "pepsico",
|
||||
"pepsico quaker": "pepsico",
|
||||
"pepsico pepsi": "pepsico",
|
||||
"pepsico 7up": "pepsico",
|
||||
"pepsico mountain dew": "pepsico",
|
||||
"pepsico mirinda": "pepsico",
|
||||
"pepsico tropicana": "pepsico",
|
||||
"pepsico gatorade": "pepsico",
|
||||
"pepsico nimbooz": "pepsico",
|
||||
"pepsico stings": "pepsico",
|
||||
"pepsico lehar": "pepsico",
|
||||
"pepsico slice": "pepsico",
|
||||
# Coca-Cola family
|
||||
"coca-cola classic": "coca-cola",
|
||||
"coca-cola thums up": "coca-cola",
|
||||
"coca-cola sprite": "coca-cola",
|
||||
"coca-cola fanta": "coca-cola",
|
||||
"coca-cola maaza": "coca-cola",
|
||||
"coca-cola minute maid": "coca-cola",
|
||||
"coca-cola diet coke": "coca-cola",
|
||||
"coca-cola zero": "coca-cola",
|
||||
"coca-cola kinley": "coca-cola",
|
||||
"coca-cola smartwater": "coca-cola",
|
||||
"coca-cola limca": "coca-cola",
|
||||
# Sunfeast family
|
||||
"sunfeast marie light": "sunfeast",
|
||||
"sunfeast marie": "sunfeast",
|
||||
"sunfeast dark fantasy": "sunfeast",
|
||||
"sunfeast bounce": "sunfeast",
|
||||
"sunfeast yippee": "sunfeast",
|
||||
"sunfeast cookies": "sunfeast",
|
||||
}
|
||||
|
||||
DEFAULT_ALIASES = BRAND_ALIASES
|
||||
|
||||
|
||||
def resolve_parent_brand(brand: str) -> str:
|
||||
"""Return the parent (canonical) brand for storage purposes.
|
||||
|
||||
Looks up the brand (case-insensitive) in BRAND_ALIASES. If found,
|
||||
returns the parent brand name so that sub-brands share the same
|
||||
database table, S3 folder, and JSON file as their parent.
|
||||
|
||||
Falls back to fuzzy substring matching, then returns the input
|
||||
unchanged if no alias is known.
|
||||
"""
|
||||
key = brand.lower().strip()
|
||||
direct = BRAND_ALIASES.get(key)
|
||||
if direct:
|
||||
return direct
|
||||
for alias, parent in BRAND_ALIASES.items():
|
||||
if alias in key or key in alias:
|
||||
return parent
|
||||
return brand
|
||||
|
||||
|
||||
def get_known_sub_brands(brand: str) -> list[str]:
|
||||
"""Return known sub-brand/product names for a given brand from BRAND_ALIASES.
|
||||
|
||||
This is used to enrich LLM prompts so the model knows what kinds of
|
||||
products a brand actually makes (e.g. for 'parle' it returns
|
||||
['biscuits', 'hide & seek', 'milano', 'krackjack', ...]).
|
||||
Returns an empty list when the brand has no aliases registered.
|
||||
"""
|
||||
canonical = resolve_parent_brand(brand).lower().strip()
|
||||
known: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for alias, parent in BRAND_ALIASES.items():
|
||||
if parent.lower() == canonical and alias.startswith(canonical):
|
||||
rest = alias[len(canonical):].strip()
|
||||
if rest and rest not in seen:
|
||||
known.append(rest)
|
||||
seen.add(rest)
|
||||
return known
|
||||
222
app/services/category_registry.py
Normal file
222
app/services/category_registry.py
Normal file
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
Single source of truth for the product-category taxonomy used by this
|
||||
catalog: what keywords identify each category, and what "generic noun"
|
||||
should be used to describe a product of that category.
|
||||
|
||||
Why this file exists
|
||||
---------------------
|
||||
Two different parts of the system used to guess categories independently:
|
||||
|
||||
1. `catalog_engine.py` had its own small keyword->category heuristic used
|
||||
when the LLM left a product `Uncategorized`.
|
||||
2. The RAG chat/search path had NO category awareness at all - a query
|
||||
like "recommend biscuits with low sugar" was answered purely by
|
||||
embedding similarity, with no guardrail that would stop a `Snacks`
|
||||
product (e.g. "ITC Bingo Korean Style") from being recommended for a
|
||||
biscuit query just because its (LLM-generated) description happened
|
||||
to use the word "biscuit" loosely.
|
||||
|
||||
Centralizing the keyword map here means both sides agree on the same
|
||||
taxonomy, and it becomes the basis for:
|
||||
- `detect_category_from_text()` - used by the RAG query pipeline to
|
||||
figure out which category (if any) the user is actually asking
|
||||
about, so retrieval can be scoped to that category.
|
||||
- `category_matches()` - a normalized equality check used
|
||||
as a safety net after retrieval, so a product that slipped through
|
||||
with the wrong category (e.g. stale/legacy embeddings) never
|
||||
reaches the LLM or the user.
|
||||
- `sanitize_category_language()` - used at ingestion time to strip
|
||||
cross-category noun leakage out of LLM-generated descriptions
|
||||
(e.g. rewriting "a crispy, savory biscuit" to "a crispy, savory
|
||||
snack" for a product actually categorized as `Snacks`), which is
|
||||
the root cause of the embedding pollution described above.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Canonical taxonomy
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ordered by priority: when a query or description matches keywords from
|
||||
# more than one category, the category that appears EARLIER in this dict
|
||||
# wins. Narrower / more specific product-type categories are listed before
|
||||
# broader or modifier-like ones (e.g. "Biscuits & Cookies" before
|
||||
# "Chocolates", so "chocolate biscuits" resolves to biscuits, not
|
||||
# chocolates).
|
||||
#
|
||||
# `keywords` - phrases that identify this category (checked as whole
|
||||
# words/phrases, case-insensitive, longest match wins
|
||||
# within a category).
|
||||
# `generic_term` - the neutral singular noun used to describe a product of
|
||||
# this category when sanitizing cross-category language
|
||||
# out of a generated description.
|
||||
CATEGORY_REGISTRY: List[Dict[str, object]] = [
|
||||
{"category": "Biscuits & Cookies", "keywords": ["biscuits", "biscuit", "biscit", "biskut", "cookies", "cookie"], "generic_term": "biscuit"},
|
||||
{"category": "Rusk", "keywords": ["rusks", "rusk"], "generic_term": "rusk"},
|
||||
{"category": "Crackers", "keywords": ["crackers", "cracker", "saltine"], "generic_term": "cracker"},
|
||||
{"category": "Cakes & Muffins", "keywords": ["cakes", "cake", "muffins", "muffin"], "generic_term": "bakery item"},
|
||||
{"category": "Bakery & Breads", "keywords": ["bread", "buns", "bun", "pav"], "generic_term": "bakery product"},
|
||||
{"category": "Noodles & Instant Food", "keywords": ["noodles", "noodle", "instant food", "vermicelli", "pasta", "maggi"], "generic_term": "instant food product"},
|
||||
{"category": "Candy & Confectionery", "keywords": ["candy", "candies", "toffee", "toffees", "lollipop", "lollipops", "confectionery", "mints", "chewing gum"], "generic_term": "candy"},
|
||||
{"category": "Snacks", "keywords": ["snacks", "snack", "chips", "namkeen", "wafers", "wafer", "kurkure", "lays"], "generic_term": "snack"},
|
||||
{"category": "Chocolates", "keywords": ["chocolates", "chocolate", "chocate", "choclate", "cocoa", "cadbury chocolate", "dairy milk"], "generic_term": "chocolate"},
|
||||
{"category": "Cooking Oils", "keywords": ["cooking oil", "edible oil", "sunflower oil", "mustard oil", "vanaspati", "refined oil", "oil", "oils"], "generic_term": "cooking oil"},
|
||||
{"category": "Atta & Staples", "keywords": ["atta", "wheat flour", "flour", "rice", "dal", "pulses", "staples", "suji", "maida"], "generic_term": "staple product"},
|
||||
{"category": "Dairy", "keywords": ["milk", "dairy", "cheese", "paneer", "panner", "paner", "paneerr", "curd", "yogurt", "butter", "ghee", "dahi"], "generic_term": "dairy product"},
|
||||
{"category": "Oral Care", "keywords": ["toothpaste", "toothbrush", "mouthwash", "paste"], "generic_term": "oral care product"},
|
||||
{"category": "Hair Care", "keywords": ["shampoo", "shampooo", "conditioner", "hair oil"], "generic_term": "hair care product"},
|
||||
{"category": "Bath Soap", "keywords": ["bath soap", "soap bar", "soap", "soaps"], "generic_term": "soap"},
|
||||
{"category": "Skin & Bath Care", "keywords": ["face wash", "body lotion", "skin cream", "moisturizer", "body wash", "cream", "lotion"], "generic_term": "skin care product"},
|
||||
{"category": "Household Cleaning", "keywords": ["detergent", "laundry", "dishwash", "floor cleaner", "handwash", "cleaner"], "generic_term": "cleaning product"},
|
||||
{"category": "Fragrance & Deodorants", "keywords": ["deodorant", "deo spray", "perfume", "fragrance", "body spray", "deo"], "generic_term": "fragrance product"},
|
||||
{"category": "Household - Agarbatti", "keywords": ["agarbatti", "incense sticks", "incense stick"], "generic_term": "agarbatti"},
|
||||
{"category": "Household - Lamp Oil", "keywords": ["lamp oil"], "generic_term": "lamp oil"},
|
||||
{"category": "Health Care - Antiseptic", "keywords": ["antiseptic", "disinfectant liquid", "sanitizer"], "generic_term": "antiseptic product"},
|
||||
]
|
||||
|
||||
# All known canonical category names, in priority order.
|
||||
ALL_CATEGORIES: List[str] = [entry["category"] for entry in CATEGORY_REGISTRY]
|
||||
|
||||
|
||||
def _normalize(text: Optional[str]) -> str:
|
||||
"""Lowercase and strip everything except letters/digits/spaces, so
|
||||
'Biscuits & Cookies' and 'biscuits and cookies' compare equal."""
|
||||
if not text:
|
||||
return ""
|
||||
text = text.lower().replace("&", " and ")
|
||||
text = re.sub(r"[^a-z0-9\s]", " ", text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
QUERY_STOP_WORDS = {
|
||||
"price", "prices", "cost", "costs", "which", "where", "what", "show",
|
||||
"find", "have", "product", "products", "item", "items", "brand", "brands",
|
||||
"under", "below", "less", "more", "many", "total", "count", "please", "today",
|
||||
"recommend", "suggest", "options", "option", "rate", "rates",
|
||||
}
|
||||
|
||||
|
||||
def _find_matches(text: str) -> List[Tuple[str, str, int]]:
|
||||
"""Return (category, matched_keyword, keyword_length) for every keyword
|
||||
found as a whole word/phrase in `text` (case-insensitive)."""
|
||||
if not text:
|
||||
return []
|
||||
lower = text.lower()
|
||||
matches: List[Tuple[str, str, int]] = []
|
||||
for entry in CATEGORY_REGISTRY:
|
||||
category = entry["category"]
|
||||
for kw in entry["keywords"]:
|
||||
pattern = r"\b" + re.escape(kw) + r"\b"
|
||||
if re.search(pattern, lower):
|
||||
matches.append((category, kw, len(kw)))
|
||||
|
||||
# Fuzzy matching fallback if exact word search found nothing
|
||||
if not matches:
|
||||
import difflib
|
||||
words = re.findall(r"\b[a-z]{4,}\b", lower)
|
||||
for entry in CATEGORY_REGISTRY:
|
||||
category = entry["category"]
|
||||
for kw in entry["keywords"]:
|
||||
for word in words:
|
||||
if word in QUERY_STOP_WORDS:
|
||||
continue
|
||||
# Check close similarity for words >= 4 chars
|
||||
ratio = difflib.SequenceMatcher(None, word, kw).ratio()
|
||||
if ratio >= 0.8:
|
||||
matches.append((category, kw, len(kw)))
|
||||
break
|
||||
return matches
|
||||
|
||||
|
||||
def detect_category_from_text(text: str) -> Optional[str]:
|
||||
"""Infer a single canonical category from free text (typically a user
|
||||
query), or None if no category-identifying keyword is present.
|
||||
|
||||
When multiple categories match, the one listed earliest in
|
||||
`CATEGORY_REGISTRY` wins (see module docstring); ties within that are
|
||||
broken by the longest matched keyword.
|
||||
"""
|
||||
matches = _find_matches(text)
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
priority = {cat: i for i, cat in enumerate(ALL_CATEGORIES)}
|
||||
# Lower priority index = higher priority (earlier in registry).
|
||||
matches.sort(key=lambda m: (priority.get(m[0], 999), -m[2]))
|
||||
return matches[0][0]
|
||||
|
||||
|
||||
def category_matches(product_category: Optional[str], target_category: Optional[str]) -> bool:
|
||||
"""Normalized equality check between a product's stored category and a
|
||||
target (e.g. detected-from-query) category. Used as a safety net after
|
||||
retrieval so mismatched-category products never reach the user, even
|
||||
if they scored well on raw embedding similarity."""
|
||||
if not target_category:
|
||||
return True
|
||||
return _normalize(product_category) == _normalize(target_category)
|
||||
|
||||
|
||||
def sanitize_category_language(description: str, product_category: Optional[str]) -> str:
|
||||
"""Strip cross-category *identity claims* out of a generated description.
|
||||
|
||||
Blanket keyword replacement turned out to be too aggressive: FMCG copy
|
||||
routinely mentions other categories as flavors or ingredients ("combines
|
||||
the taste of cheese and crackers"), and replacing those reads as
|
||||
nonsense even though they were never the source of the retrieval bug.
|
||||
|
||||
What actually caused a `Snacks` product to surface for a "biscuits"
|
||||
query was the description *asserting an identity* it doesn't have -
|
||||
"ITC Bingo Korean Style is a crispy, savory **biscuit**...". So this
|
||||
function only rewrites that specific LLM-template shape:
|
||||
|
||||
"... is/are/was/were [a/an/the] [up to 4 words] KEYWORD ..."
|
||||
e.g. "is a crispy, savory biscuit" -> "is a crispy, savory snack"
|
||||
|
||||
A flavor mention like "taste of cheese and crackers" doesn't match this
|
||||
shape, so it's left alone rather than mangled into "taste of cracker".
|
||||
|
||||
Note: this deliberately does NOT try to repair descriptions that are
|
||||
wrong in a deeper way (an entire sentence describing the wrong product
|
||||
domain, e.g. a snack described as being "engineered to deliver powerful
|
||||
cleaning") - a regex substitution can't safely rewrite a whole
|
||||
hallucinated clause. Retrieval-time category filtering (see
|
||||
`rag_service._filter_to_category`) is what actually protects the RAG
|
||||
answers from that class of error, since it filters on the product's
|
||||
stored `category` column rather than on description text; this
|
||||
function only cleans up isolated noun-level mislabeling.
|
||||
"""
|
||||
if not description:
|
||||
return description
|
||||
|
||||
own_category = product_category or ""
|
||||
own_entry = next((e for e in CATEGORY_REGISTRY if e["category"] == own_category), None)
|
||||
replacement = own_entry["generic_term"] if own_entry else _generic_fallback_term(own_category)
|
||||
|
||||
cleaned = description
|
||||
for entry in CATEGORY_REGISTRY:
|
||||
if entry["category"] == own_category:
|
||||
continue # a product's own category words are never "leakage"
|
||||
for kw in sorted(entry["keywords"], key=len, reverse=True):
|
||||
kw_pattern = re.escape(kw)
|
||||
pattern = re.compile(
|
||||
r"\b(?:is|are|was|were)\s+(?:a|an|the)?\s*(?:[A-Za-z]+[,\s]+){0,4}?(" + kw_pattern + r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def _replace(match: "re.Match[str]") -> str:
|
||||
word = match.group(1)
|
||||
repl_word = replacement[0].upper() + replacement[1:] if word[0].isupper() else replacement
|
||||
rel_start = match.span(1)[0] - match.start()
|
||||
return match.group(0)[:rel_start] + repl_word
|
||||
|
||||
cleaned = pattern.sub(_replace, cleaned)
|
||||
|
||||
return cleaned
|
||||
|
||||
|
||||
def _generic_fallback_term(category: Optional[str]) -> str:
|
||||
if not category or category.strip().lower() in ("", "uncategorized", "general"):
|
||||
return "product"
|
||||
return category.split("&")[0].split("-")[0].strip().lower() or "product"
|
||||
93
app/services/discount_service.py
Normal file
93
app/services/discount_service.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Thin I/O orchestration for Feature 3 (Dynamic Discount Prediction).
|
||||
Builds the live feature row for a given (store, product) from current
|
||||
DB state, runs it through the trained model, and logs the prediction to
|
||||
`discount_history` for the analytics/audit trail Feature 10 asks for.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.intelligence import features as F
|
||||
from app.intelligence.discount_model import discount_predictor, FEATURE_COLUMNS
|
||||
from app.services import store_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_feature_row(store_row: Dict, order_items: pd.DataFrame, as_of: date) -> Dict[str, float]:
|
||||
vel7 = F.sales_velocity(order_items, store_row["store_id"], store_row["brand"], store_row["image_id"], as_of, 7)
|
||||
vel30 = F.sales_velocity(order_items, store_row["store_id"], store_row["brand"], store_row["image_id"], as_of, 30)
|
||||
# Real demand/popularity scores aren't wired to this call path yet
|
||||
# (they'd come from popularity_model/trending_model given engagement
|
||||
# data this endpoint doesn't have) - use recent velocity normalized
|
||||
# to a 0-100-ish scale as a reasonable proxy so the discount model
|
||||
# still gets a meaningful demand/popularity signal rather than a
|
||||
# hardcoded constant.
|
||||
demand_proxy = min(100.0, vel30["units_per_day"] * 20)
|
||||
return {
|
||||
"stock_ratio": F.stock_ratio(store_row["available_stock"], store_row["reorder_level"]),
|
||||
"days_of_cover": F.days_of_cover(store_row["available_stock"], max(vel30["units_per_day"], 0.05)),
|
||||
"units_per_day_7": vel7["units_per_day"],
|
||||
"units_per_day_30": vel30["units_per_day"],
|
||||
"demand_score": demand_proxy,
|
||||
"popularity_score": demand_proxy,
|
||||
"days_to_expiry": F.days_to_expiry(store_row.get("category"), 0),
|
||||
"is_festive_season": F.is_festive_season(as_of),
|
||||
"category_freq": 0.15, # neutral default for a single-row live prediction (no full-catalog context here)
|
||||
"store_tier_encoded": F.encode_store_tier(store_row.get("store_tier", "standard")),
|
||||
"price_position": (store_row["selling_price"] / store_row["mrp"]) if store_row.get("mrp") else 1.0,
|
||||
}
|
||||
|
||||
|
||||
def predict_discount_for_product(store_id: str, brand: str, image_id: str, log_result: bool = True) -> Optional[Dict]:
|
||||
rows = store_db.get_store_products(store_id, limit=1000)
|
||||
match = next((r for r in rows if r["brand"] == brand and r["image_id"] == image_id), None)
|
||||
if not match:
|
||||
return None
|
||||
order_items = store_db.get_order_items_df(store_id=store_id)
|
||||
feature_row = _build_feature_row(match, order_items, date.today())
|
||||
pred = discount_predictor.predict_one(feature_row, original_price=float(match["selling_price"]))
|
||||
|
||||
result = {
|
||||
"store_id": store_id, "brand": brand, "image_id": image_id,
|
||||
"original_price": float(match["selling_price"]), "discount_pct": pred.discount_pct,
|
||||
"final_price": pred.final_price, "savings": pred.savings, "model_version": "v1",
|
||||
}
|
||||
if log_result:
|
||||
store_db.log_discounts([{**result, "features": feature_row}])
|
||||
return result
|
||||
|
||||
|
||||
def predict_discounts_for_store(store_id: str, log_result: bool = True) -> List[Dict]:
|
||||
"""Batch version - scores every product in a store in one pass
|
||||
(much faster than calling predict_discount_for_product per item
|
||||
since it builds all velocity features from a single order_items
|
||||
fetch, and does one bulk model.predict() call instead of N)."""
|
||||
rows = store_db.get_store_products(store_id, limit=5000)
|
||||
if not rows:
|
||||
return []
|
||||
order_items = store_db.get_order_items_df(store_id=store_id)
|
||||
as_of = date.today()
|
||||
feature_rows = [_build_feature_row(r, order_items, as_of) for r in rows]
|
||||
df = pd.DataFrame(feature_rows)
|
||||
df["selling_price"] = [r["selling_price"] for r in rows]
|
||||
scored = discount_predictor.predict_batch(df)
|
||||
|
||||
results = []
|
||||
log_rows = []
|
||||
for r, (_, s) in zip(rows, scored.iterrows()):
|
||||
item = {
|
||||
"store_id": store_id, "brand": r["brand"], "image_id": r["image_id"],
|
||||
"original_price": float(r["selling_price"]), "discount_pct": float(s["discount_pct"]),
|
||||
"final_price": float(s["final_price"]), "savings": float(s["savings"]), "model_version": "v1",
|
||||
}
|
||||
results.append(item)
|
||||
log_rows.append({**item, "features": {c: float(s[c]) for c in FEATURE_COLUMNS}})
|
||||
if log_result and log_rows:
|
||||
store_db.log_discounts(log_rows)
|
||||
return results
|
||||
52
app/services/embeddings_service.py
Normal file
52
app/services/embeddings_service.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional, TYPE_CHECKING
|
||||
|
||||
from app.infrastructure.settings import EMBEDDINGS_MODEL
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only, no runtime cost
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
_model_singleton: Optional["SentenceTransformer"] = None
|
||||
|
||||
|
||||
def get_device() -> str:
|
||||
"""Prefer CUDA if available, otherwise CPU.
|
||||
|
||||
Imports torch lazily: on an 8GB RAM / CPU-only laptop there is no
|
||||
benefit to importing torch (and paying its startup/memory cost) until
|
||||
an embedding is actually requested, so the FastAPI process can boot
|
||||
and answer /api/health almost instantly.
|
||||
"""
|
||||
import torch # local import - see docstring
|
||||
|
||||
return "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
def get_embedding_model() -> "SentenceTransformer":
|
||||
global _model_singleton
|
||||
if _model_singleton is None:
|
||||
from sentence_transformers import SentenceTransformer # local import - see get_device()
|
||||
|
||||
device = get_device()
|
||||
_model_singleton = SentenceTransformer(EMBEDDINGS_MODEL, device=device)
|
||||
return _model_singleton
|
||||
|
||||
|
||||
def embed_texts(texts: List[str]) -> List[List[float]]:
|
||||
"""Embed a batch of texts into normalized 384-dim vectors (MiniLM-L6-v2).
|
||||
|
||||
Normalized so that pgvector's cosine-distance operator (`<=>`) behaves
|
||||
consistently for the RAG retrieval step in `app.services.vector_store`.
|
||||
"""
|
||||
if not texts:
|
||||
return []
|
||||
model = get_embedding_model()
|
||||
embeddings = model.encode(
|
||||
texts,
|
||||
batch_size=32,
|
||||
normalize_embeddings=True,
|
||||
convert_to_numpy=True,
|
||||
show_progress_bar=False,
|
||||
)
|
||||
return embeddings.tolist()
|
||||
538
app/services/image_search.py
Normal file
538
app/services/image_search.py
Normal file
@@ -0,0 +1,538 @@
|
||||
"""
|
||||
Product image discovery service.
|
||||
|
||||
Layered, validated pipeline of OPEN-SOURCE sources, cheapest/most-reliable
|
||||
first (important on an 8GB RAM / no-GPU machine: avoid launching a real
|
||||
browser unless every cheaper structured source has already failed):
|
||||
|
||||
1. Open Food Facts / Open Beauty Facts / Open Products Facts
|
||||
- Free, open, community-maintained product databases with real photos.
|
||||
- No API key, no quota, no headless browser needed (single HTTP GET).
|
||||
- Covers most Indian FMCG: food, snacks, dairy AND personal-care/
|
||||
household items (toothpaste, soap, shampoo, etc.) via the sibling
|
||||
"Beauty Facts" and "Products Facts" projects.
|
||||
- Also returns a real `quantity` field (e.g. "200 g", "1 L") when
|
||||
available, which `find_product_quantity_openfacts()` exposes
|
||||
separately so callers can ground size-variant guesses in real
|
||||
packaging data instead of an LLM's guess.
|
||||
|
||||
2. Wikimedia Commons (new)
|
||||
- Free, open media repository (the same one behind Wikipedia), no API
|
||||
key required. Good secondary source especially for large,
|
||||
well-established Indian/global FMCG brands (Horlicks, Dettol,
|
||||
Colgate, etc.) that have freely-licensed packaging photos uploaded.
|
||||
|
||||
3. DuckDuckGo Images via the `ddgs` package
|
||||
- No API key requirement, pure Python/HTTP (no browser process).
|
||||
|
||||
4. Google Custom Search Engine (optional - only used if the user has
|
||||
configured GOOGLE_API_KEY / GOOGLE_CSE_ID).
|
||||
|
||||
5. Playwright-rendered Bing Image Search (LAST resort only)
|
||||
- Replaces the old Node.js/Crawlee subprocess scripts (see
|
||||
playwright_image_fallback.py for why). Launches a real headless
|
||||
Chromium tab so it can deal with Bing's JS-gated results page far
|
||||
more reliably than a raw `requests` HTML scrape, but only runs if
|
||||
every cheaper source above produced nothing, since spinning up a
|
||||
browser is the slowest/heaviest option here.
|
||||
|
||||
Every URL that makes it into the final list is passed through
|
||||
`validate_image_url_live`, a lightweight ranged-GET check that confirms it
|
||||
actually resolves to real image bytes above a minimum size - this is what
|
||||
stops garbage/placeholder/expired URLs from reaching the S3 upload step
|
||||
and failing there silently.
|
||||
"""
|
||||
from typing import Optional, List
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from app.infrastructure.settings import (
|
||||
USE_GOOGLE_CSE, GOOGLE_API_KEY, GOOGLE_CSE_ID,
|
||||
USE_DDG_IMAGES, USE_OPEN_FACTS, USE_WIKIMEDIA, USE_PLAYWRIGHT_FALLBACK,
|
||||
MIN_IMAGE_BYTES,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# Open *Facts projects share the same API shape; we query all three so both
|
||||
# food (Open Food Facts) and non-food FMCG like toothpaste/soap (Open Beauty
|
||||
# Facts / Open Products Facts) are covered.
|
||||
OPEN_FACTS_HOSTS = [
|
||||
"world.openfoodfacts.org",
|
||||
"world.openbeautyfacts.org",
|
||||
"world.openproductsfacts.org",
|
||||
]
|
||||
|
||||
# Module-level flag so we only ever print the "Playwright not installed"
|
||||
# hint once per process, instead of once per product.
|
||||
_playwright_warning_shown = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Open Food/Beauty/Products Facts - primary source
|
||||
# ---------------------------------------------------------------------------
|
||||
def _query_openfacts(query: str, max_results: int) -> list:
|
||||
"""Raw Open*Facts product list for a query string (internal helper)."""
|
||||
if not query:
|
||||
return []
|
||||
for host in OPEN_FACTS_HOSTS:
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://{host}/cgi/search.pl",
|
||||
params={
|
||||
"search_terms": query,
|
||||
"search_simple": 1,
|
||||
"action": "process",
|
||||
"json": 1,
|
||||
"page_size": max_results,
|
||||
},
|
||||
headers={"User-Agent": _BROWSER_UA},
|
||||
timeout=12,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
continue
|
||||
data = resp.json()
|
||||
products = data.get("products", [])
|
||||
if products:
|
||||
return products
|
||||
except Exception as e:
|
||||
logger.debug(f"Open*Facts lookup failed on {host}: {e}")
|
||||
continue
|
||||
return []
|
||||
|
||||
|
||||
def find_images_openfacts(title: str, brand: Optional[str] = None, max_results: int = 20) -> list:
|
||||
"""Query the Open *Facts family of open product databases for real
|
||||
product photos. No API key required. Falls back from a brand+title
|
||||
query to a title-only query if the combined query is too specific to
|
||||
match anything (small/regional brand name variants are a common case)."""
|
||||
if not USE_OPEN_FACTS:
|
||||
return []
|
||||
|
||||
query = f"{brand or ''} {title or ''}".strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
products = _query_openfacts(query, max_results)
|
||||
if not products and brand and title:
|
||||
# Combined "brand + title" query found nothing - retry with just
|
||||
# the title, since Open*Facts' free-text search is exact-ish and
|
||||
# brand naming conventions vary (e.g. "Dettol" vs "Reckitt Dettol").
|
||||
products = _query_openfacts(title, max_results)
|
||||
|
||||
urls: List[str] = []
|
||||
for product in products:
|
||||
for key in ("image_front_url", "image_url", "image_front_small_url"):
|
||||
img = product.get(key)
|
||||
if img and str(img).startswith("http"):
|
||||
urls.append(img)
|
||||
break
|
||||
return urls[:max_results]
|
||||
|
||||
|
||||
def find_product_quantity_openfacts(title: str, brand: Optional[str] = None) -> Optional[str]:
|
||||
"""Return the real packaging quantity (e.g. '200 g', '1 l') reported by
|
||||
Open*Facts for the best-matching product, if any. This lets callers
|
||||
ground size-variant guesses in real data instead of trusting the LLM's
|
||||
guess for size labels."""
|
||||
if not USE_OPEN_FACTS:
|
||||
return None
|
||||
query = f"{brand or ''} {title or ''}".strip()
|
||||
products = _query_openfacts(query, 1)
|
||||
if not products and brand and title:
|
||||
products = _query_openfacts(title, 1)
|
||||
if not products:
|
||||
return None
|
||||
qty = products[0].get("quantity")
|
||||
return str(qty).strip() if qty else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Wikimedia Commons - free/open media repository, no API key
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_images_wikimedia(title: str, brand: Optional[str] = None, max_results: int = 20) -> list:
|
||||
"""Search Wikimedia Commons (the open media library behind Wikipedia)
|
||||
for product/packaging photos. Good secondary source for established
|
||||
brands; complements Open*Facts which leans more food/grocery."""
|
||||
if not USE_WIKIMEDIA:
|
||||
return []
|
||||
|
||||
query = f"{brand or ''} {title or ''}".strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
try:
|
||||
# Add exclusion terms to avoid non-product results (e.g. "Sprite" the
|
||||
# car, "Tiger" the animal, "Apple" the fruit) when searching for a
|
||||
# branded packaged-good product. The gsrsearch parameter supports
|
||||
# MediaWiki's standard search syntax including the minus (-) prefix
|
||||
# for terms to exclude.
|
||||
exclusion_terms = "-car -vehicle -automotive -motorsport -racing -motorcycle -bike -people -person -portrait -animal -pet -dog -cat -bird -fish -landscape -nature -tour -travel -building -architecture -sport -game -flower -rose -floral -petal -bouquet -botanical -plant -garden -tree -herb"
|
||||
search_query = f"{query} {exclusion_terms} filetype:bitmap"
|
||||
resp = requests.get(
|
||||
"https://commons.wikimedia.org/w/api.php",
|
||||
params={
|
||||
"action": "query",
|
||||
"format": "json",
|
||||
"generator": "search",
|
||||
"gsrsearch": search_query,
|
||||
"gsrlimit": max_results,
|
||||
"gsrnamespace": 6, # File: namespace
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url|size",
|
||||
"iiurlwidth": 800,
|
||||
},
|
||||
headers={"User-Agent": _BROWSER_UA},
|
||||
timeout=10,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
pages = resp.json().get("query", {}).get("pages", {})
|
||||
urls = []
|
||||
for page in pages.values():
|
||||
info_list = page.get("imageinfo", [])
|
||||
if not info_list:
|
||||
continue
|
||||
info = info_list[0]
|
||||
url = info.get("thumburl") or info.get("url")
|
||||
if url and url.startswith("http"):
|
||||
urls.append(url)
|
||||
return urls[:max_results]
|
||||
except Exception as e:
|
||||
logger.debug(f"Wikimedia Commons search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. DuckDuckGo Images (pure Python, no browser process)
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_all_image_urls_ddg(title: str, brand: Optional[str] = None, max_results: int = 20) -> list:
|
||||
"""Search DuckDuckGo for product images via the `ddgs` package."""
|
||||
if not USE_DDG_IMAGES:
|
||||
return []
|
||||
|
||||
query = f"{brand} {title}".strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
logger.debug("DuckDuckGo image search unavailable: 'ddgs' package not installed")
|
||||
return []
|
||||
|
||||
for backend in ("duckduckgo", "auto"):
|
||||
try:
|
||||
with DDGS(timeout=15) as ddgs:
|
||||
results = ddgs.images(
|
||||
query,
|
||||
region="in-en",
|
||||
safesearch="off",
|
||||
max_results=max_results,
|
||||
backend=backend,
|
||||
)
|
||||
image_urls = [
|
||||
r.get('image') for r in results
|
||||
if r.get('image') and str(r.get('image')).startswith('http')
|
||||
]
|
||||
if image_urls:
|
||||
return image_urls
|
||||
except Exception as e:
|
||||
logger.debug(f"DuckDuckGo image search error (backend={backend}): {e}")
|
||||
continue
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def find_all_image_urls_google_cse(title: str, brand: Optional[str] = None) -> list:
|
||||
"""Search Google Custom Search Engine for product images (used only if
|
||||
the user has configured GOOGLE_API_KEY / GOOGLE_CSE_ID - optional)."""
|
||||
if not USE_GOOGLE_CSE or not GOOGLE_API_KEY or not GOOGLE_CSE_ID:
|
||||
return []
|
||||
|
||||
try:
|
||||
query = f"{brand} {title}".strip()
|
||||
resp = requests.get(
|
||||
"https://www.googleapis.com/customsearch/v1",
|
||||
params={
|
||||
'key': GOOGLE_API_KEY, 'cx': GOOGLE_CSE_ID, 'q': query,
|
||||
'searchType': 'image', 'num': 10,
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
items = resp.json().get('items', [])
|
||||
return [item['link'] for item in items if 'link' in item]
|
||||
except Exception as e:
|
||||
logger.debug(f"Google CSE error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Playwright (Python) Bing Image Search - LAST resort only.
|
||||
#
|
||||
# Replaces the old Node.js/Crawlee subprocess scripts. See
|
||||
# playwright_image_fallback.py for the full rationale. Runs in a fully
|
||||
# isolated OS subprocess so it can never conflict with Streamlit's asyncio
|
||||
# event loop, and has a short, bounded timeout instead of the old 120s+60s
|
||||
# double-timeout chain.
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_images_playwright(title: str, brand: Optional[str] = None, max_results: int = 20) -> list:
|
||||
global _playwright_warning_shown
|
||||
if not USE_PLAYWRIGHT_FALLBACK:
|
||||
return []
|
||||
|
||||
query = f"{brand or ''} {title or ''}".strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "app.services.playwright_image_fallback", query, str(max_results)],
|
||||
cwd=str(Path(__file__).parent.parent.parent),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug("Playwright image fallback timed out")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.debug(f"Playwright image fallback failed to start: {e}")
|
||||
return []
|
||||
|
||||
if "PLAYWRIGHT_NOT_INSTALLED" in (result.stderr or "") or "PLAYWRIGHT_LAUNCH_FAILED" in (result.stderr or ""):
|
||||
if not _playwright_warning_shown:
|
||||
logger.warning(
|
||||
"Playwright browser fallback unavailable - run `pip install playwright && "
|
||||
"playwright install chromium` to enable it as a last-resort image source. "
|
||||
"Continuing with Open*Facts / Wikimedia / DuckDuckGo only."
|
||||
)
|
||||
_playwright_warning_shown = True
|
||||
return []
|
||||
|
||||
try:
|
||||
out = (result.stdout or "").strip()
|
||||
if out.startswith("[") and out.endswith("]"):
|
||||
return json.loads(out)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation: confirm a URL actually resolves to real image bytes before we
|
||||
# let it through to the S3 upload step.
|
||||
# ---------------------------------------------------------------------------
|
||||
def validate_image_url_live(url: str, referer: Optional[str] = None, timeout: int = 8) -> bool:
|
||||
"""Lightweight check that `url` resolves to real image content above a
|
||||
minimum size threshold. Uses a ranged GET (not HEAD) because many sites
|
||||
don't return reliable Content-Length on HEAD requests.
|
||||
|
||||
Tries twice: once with a Referer derived from the image's own domain
|
||||
(helps with hotlink-protected CDNs that expect *some* same-site
|
||||
Referer), and - if that's rejected - once with no Referer at all (some
|
||||
sites do the opposite and reject requests that carry a Referer header
|
||||
pretending to be same-site when it isn't). This two-step retry recovers
|
||||
a meaningful fraction of URLs that single-strategy validation used to
|
||||
discard.
|
||||
"""
|
||||
if not url or not url.startswith("http"):
|
||||
return False
|
||||
|
||||
parsed = urlparse(url)
|
||||
same_site_referer = f"{parsed.scheme}://{parsed.netloc}/" if parsed.netloc else None
|
||||
|
||||
for attempt_referer in (referer or same_site_referer, None):
|
||||
try:
|
||||
headers = {
|
||||
"User-Agent": _BROWSER_UA,
|
||||
"Accept": "image/*,*/*;q=0.8",
|
||||
}
|
||||
if attempt_referer:
|
||||
headers["Referer"] = attempt_referer
|
||||
|
||||
resp = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
||||
if resp.status_code != 200:
|
||||
resp.close()
|
||||
continue
|
||||
content_type = resp.headers.get("content-type", "").lower()
|
||||
if "image" not in content_type and not url.lower().endswith(
|
||||
(".jpg", ".jpeg", ".png", ".webp", ".gif")
|
||||
):
|
||||
resp.close()
|
||||
continue
|
||||
chunk = resp.raw.read(MIN_IMAGE_BYTES + 1)
|
||||
resp.close()
|
||||
if len(chunk) >= MIN_IMAGE_BYTES:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _dedupe(urls: list) -> list:
|
||||
seen = set()
|
||||
out = []
|
||||
for u in urls:
|
||||
if u and u not in seen:
|
||||
seen.add(u)
|
||||
out.append(u)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers for disambiguating search queries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Product-type context hints for common ambiguous brand/product words
|
||||
# that image search APIs tend to confuse with non-product content.
|
||||
# Keyed by the ambiguous word (lowercase); value is the context term
|
||||
# to append to the search query.
|
||||
_AMBIQUITY_HINTS: dict[str, str] = {
|
||||
"perk": "chocolate wafer",
|
||||
"crunch": "chocolate wafer",
|
||||
"boost": "drink supplement",
|
||||
"active": "yogurt drink",
|
||||
"sprite": "soft drink",
|
||||
"tiger": "biscuit snack",
|
||||
"apple": "fruit juice",
|
||||
"refresh": "drink mouthwash",
|
||||
"glow": "lotion skincare",
|
||||
"white": "toothpaste dental",
|
||||
"clean": "shampoo soap",
|
||||
"cool": "mint gum candy",
|
||||
"energy": "drink snack bar",
|
||||
"good day": "biscuit cookies",
|
||||
"happy": "chocolate candy",
|
||||
"rich": "cream biscuit",
|
||||
"dark": "chocolate bar",
|
||||
"milk": "chocolate dairy",
|
||||
"5 star": "chocolate bar",
|
||||
"munch": "chocolate wafer",
|
||||
"gems": "chocolate candy",
|
||||
"treat": "chocolate candy",
|
||||
"break": "biscuit cracker",
|
||||
"digestive": "biscuit cracker",
|
||||
"marie": "biscuit cookies",
|
||||
"oreo": "biscuit cookies",
|
||||
"bounce": "snack bar",
|
||||
"soap": "bath soap bar",
|
||||
"life": "cereal grain",
|
||||
"special": "cereal biscuit",
|
||||
"corn": "snack flakes",
|
||||
"chocos": "breakfast cereal",
|
||||
}
|
||||
|
||||
|
||||
def _clean_search_title(title: str, brand: str | None = None) -> str:
|
||||
"""Strip redundant brand prefix from title and add context hints for
|
||||
ambiguous product names that confuse image search APIs.
|
||||
|
||||
Examples:
|
||||
_clean_search_title("Cadbury Perk Crunch", "Cadbury Perk")
|
||||
-> "Perk Crunch chocolate wafer"
|
||||
|
||||
_clean_search_title("Britannia Good Day Biscuits", "Britannia")
|
||||
-> "Good Day Biscuits"
|
||||
"""
|
||||
clean = (title or "").strip()
|
||||
if brand:
|
||||
brand_lower = brand.lower().strip()
|
||||
clean_lower = clean.lower()
|
||||
if clean_lower.startswith(brand_lower):
|
||||
clean = clean[len(brand):].strip()
|
||||
if not clean:
|
||||
clean = (title or "").strip()
|
||||
|
||||
# Append context hint for any ambiguous word in the cleaned title
|
||||
clean_lower = clean.lower()
|
||||
for word, hint in _AMBIQUITY_HINTS.items():
|
||||
if word in clean_lower:
|
||||
clean = f"{clean} {hint}"
|
||||
break
|
||||
|
||||
return clean
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API (same signatures as before, so catalog_engine.py and
|
||||
# downstream callers don't need to change)
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_image_url(title: str, brand: Optional[str] = None, country_hint: Optional[str] = None) -> Optional[str]:
|
||||
"""Get a single working, validated image URL."""
|
||||
urls = find_all_image_urls(title, brand, country_hint)
|
||||
return urls[0] if urls else None
|
||||
|
||||
|
||||
def find_all_image_urls(title: str, brand: Optional[str] = None, country_hint: Optional[str] = None,
|
||||
validate: bool = True, max_results: int = 24) -> list:
|
||||
"""Get working image URLs by trying multiple open-source sources in
|
||||
priority order (cheapest/most-reliable first), merging and validating
|
||||
results. Only escalates to the Playwright browser fallback if every
|
||||
cheaper structured/HTTP source above produced nothing at all."""
|
||||
candidates: List[str] = []
|
||||
|
||||
# Strip redundant brand prefix from title to avoid repetitive queries
|
||||
# like "Cadbury Perk Cadbury Perk Crunch".
|
||||
search_title = _clean_search_title(title, brand)
|
||||
|
||||
candidates.extend(find_images_openfacts(search_title, brand, max_results))
|
||||
if len(candidates) < max_results:
|
||||
candidates.extend(find_images_wikimedia(search_title, brand, max_results))
|
||||
if len(candidates) < max_results:
|
||||
candidates.extend(find_all_image_urls_ddg(search_title, brand, max_results))
|
||||
if len(candidates) < max_results:
|
||||
candidates.extend(find_all_image_urls_google_cse(search_title, brand))
|
||||
|
||||
candidates = _dedupe(candidates)
|
||||
|
||||
if not validate:
|
||||
if not candidates:
|
||||
candidates = find_images_playwright(search_title, brand, max_results)
|
||||
return candidates[:max_results]
|
||||
|
||||
validated = []
|
||||
for url in candidates:
|
||||
if validate_image_url_live(url):
|
||||
validated.append(url)
|
||||
if len(validated) >= max_results:
|
||||
break
|
||||
|
||||
if not validated:
|
||||
# Every cheap/structured source either returned nothing or returned
|
||||
# only dead links - last resort: render a real Bing search page.
|
||||
fallback_candidates = _dedupe(find_images_playwright(search_title, brand, max_results))
|
||||
for url in fallback_candidates:
|
||||
if validate_image_url_live(url):
|
||||
validated.append(url)
|
||||
if len(validated) >= max_results:
|
||||
break
|
||||
|
||||
if validated:
|
||||
return validated
|
||||
|
||||
# Validation killed everything (e.g. offline machine, or every host
|
||||
# blocked the validation probe) - return unvalidated candidates rather
|
||||
# than nothing, so the rest of the pipeline still has something to try.
|
||||
if candidates:
|
||||
logger.warning(f"No validated images for '{brand} {title}' - returning unvalidated candidates")
|
||||
return candidates[:max_results]
|
||||
|
||||
|
||||
def validate_image_url(url: Optional[str], title: str, brand: Optional[str] = None) -> Optional[str]:
|
||||
"""Kept for backward compatibility with any existing callers."""
|
||||
if not url or not url.startswith("http"):
|
||||
return find_image_url(title, brand)
|
||||
return url
|
||||
180
app/services/ml_training_service.py
Normal file
180
app/services/ml_training_service.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
Orchestrates training every model in `app/intelligence/` against real
|
||||
seeded data, then refreshes the cached outputs (`trending_products`,
|
||||
`discount_history`, `demand_forecast`) that the API's fast-path reads
|
||||
depend on. Called from `scripts/train_ml_models.py` (CLI, primary) and
|
||||
`/api/admin/store-intelligence/train` (convenience).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from app.intelligence import (
|
||||
discount_model, engagement_simulation, forecasting, popularity_model,
|
||||
purchase_propensity_model, store_performance_model, trending_model,
|
||||
)
|
||||
from app.services import store_db
|
||||
from app.services.trending_service import compute_and_cache_trending
|
||||
from app.services.discount_service import predict_discounts_for_store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ALL_MODELS = ["discount", "trending", "popularity", "forecast", "store_performance", "purchase_propensity"]
|
||||
|
||||
|
||||
def _load_common_frames():
|
||||
order_items = store_db.get_order_items_df()
|
||||
orders = store_db.get_orders_df()
|
||||
store_products = store_db.get_all_store_products_df()
|
||||
return order_items, orders, store_products
|
||||
|
||||
|
||||
def train_discount(order_items: pd.DataFrame, store_products: pd.DataFrame) -> Dict:
|
||||
if store_products.empty:
|
||||
return {"skipped": "no store products - run the seed step first"}
|
||||
rng = np.random.default_rng(3)
|
||||
sp = store_products.copy()
|
||||
sp["days_since_stocked"] = rng.integers(0, 60, size=len(sp))
|
||||
# Real velocity-derived demand/popularity proxy (same approach as the
|
||||
# live single-product path in discount_service) so training features
|
||||
# match what's used at inference time as closely as possible.
|
||||
key = sp["brand"] + "||" + sp["image_id"]
|
||||
units_30d = order_items[order_items["order_date"] >= pd.Timestamp(date.today() - timedelta(days=30))]
|
||||
vel = (units_30d.groupby(units_30d["brand"] + "||" + units_30d["image_id"])["quantity"].sum() / 30) if not units_30d.empty else pd.Series(dtype=float)
|
||||
sp["demand_score"] = key.map(vel).fillna(0.0).clip(upper=5) * 20
|
||||
sp["popularity_score"] = sp["demand_score"]
|
||||
|
||||
frame = discount_model.build_training_frame(sp, order_items, date.today())
|
||||
bundle = discount_model.train(frame)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra}
|
||||
|
||||
|
||||
def train_trending(order_items: pd.DataFrame, orders: pd.DataFrame) -> Dict:
|
||||
if order_items.empty:
|
||||
return {"skipped": "no order history - run the seed step first"}
|
||||
as_of_dates = [date.today() - timedelta(days=d) for d in (0, 5, 10, 15, 20, 25, 30, 40, 50)]
|
||||
frame = trending_model.build_training_frame(order_items, orders, as_of_dates)
|
||||
if frame.empty:
|
||||
return {"skipped": "not enough order history to build trend windows yet"}
|
||||
bundle = trending_model.train(frame)
|
||||
cached_counts = {}
|
||||
for window in ("today", "weekly", "monthly"):
|
||||
cached_counts[window] = compute_and_cache_trending(window)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra, "cached_scopes": {k: len(v) for k, v in cached_counts.items()}}
|
||||
|
||||
|
||||
def train_popularity(order_items: pd.DataFrame, store_products: pd.DataFrame) -> Dict:
|
||||
if store_products.empty:
|
||||
return {"skipped": "no store products - run the seed step first"}
|
||||
products = store_products[["brand", "image_id"]].drop_duplicates()
|
||||
orders_count = (
|
||||
order_items.groupby(order_items["brand"] + "||" + order_items["image_id"])["quantity"].sum()
|
||||
if not order_items.empty else pd.Series(dtype=float)
|
||||
)
|
||||
eng = engagement_simulation.simulate_engagement(products, orders_count)
|
||||
frame = popularity_model.build_training_frame(eng)
|
||||
bundle = popularity_model.train(frame)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra}
|
||||
|
||||
|
||||
def train_forecast(order_items: pd.DataFrame, store_products: pd.DataFrame) -> Dict:
|
||||
if order_items.empty or store_products.empty:
|
||||
return {"skipped": "no order history - run the seed step first"}
|
||||
meta = store_products[["store_id", "brand", "image_id", "category", "store_tier"]].drop_duplicates()
|
||||
as_of_dates = [date.today() - timedelta(days=d) for d in (0, 10, 20, 30, 40)]
|
||||
frame = forecasting.build_training_frame(order_items, meta, as_of_dates)
|
||||
if len(frame) < 10:
|
||||
return {"skipped": f"only {len(frame)} training rows - need more order history"}
|
||||
bundle = forecasting.train(frame)
|
||||
|
||||
# Refresh the demand_forecast cache for every (store, product) with
|
||||
# enough history for a 7-day-ahead forecast.
|
||||
forecast_rows = []
|
||||
for _, row in meta.iterrows():
|
||||
from app.intelligence.forecasting import _daily_series
|
||||
series = _daily_series(order_items, row["store_id"], row["brand"], row["image_id"])
|
||||
if len(series) < 14:
|
||||
continue
|
||||
from app.intelligence import features as F
|
||||
feat = pd.DataFrame([{
|
||||
"rolling_mean_7": series.tail(7).mean(), "rolling_mean_14": series.tail(14).mean(),
|
||||
"rolling_mean_30": series.tail(30).mean(), **F.cyclical_month_features(date.today()),
|
||||
**F.cyclical_dow_features(date.today()), "is_festive_season": F.is_festive_season(date.today()),
|
||||
"category_freq": 0.15, "store_tier_encoded": F.encode_store_tier(row["store_tier"]),
|
||||
}])
|
||||
out = forecasting.demand_forecaster.forecast(feat)
|
||||
forecast_rows.append({
|
||||
"store_id": row["store_id"], "brand": row["brand"], "image_id": row["image_id"],
|
||||
"forecast_avg_daily_units": round(float(out["forecast_avg_daily_units"].iloc[0]), 3),
|
||||
"forecast_total_units": round(float(out["forecast_total_units"].iloc[0]), 3),
|
||||
"horizon_days": 7, "model_version": "v1",
|
||||
})
|
||||
saved = store_db.save_demand_forecasts(forecast_rows)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra, "forecasts_cached": saved}
|
||||
|
||||
|
||||
def train_store_performance(orders: pd.DataFrame) -> Dict:
|
||||
stores = store_db.list_stores()
|
||||
if not stores or orders.empty:
|
||||
return {"skipped": "no stores/orders - run the seed step first"}
|
||||
stores_df = pd.DataFrame(stores)[["store_id", "tier"]]
|
||||
daily_agg = store_performance_model.build_daily_store_aggregates(orders, stores_df)
|
||||
disc_stub = pd.DataFrame(columns=["store_id", "date", "avg_discount_pct"])
|
||||
frame = store_performance_model.build_training_frame(daily_agg, disc_stub)
|
||||
if frame.empty:
|
||||
return {"skipped": "not enough daily history per store yet"}
|
||||
bundle = store_performance_model.train(frame)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra}
|
||||
|
||||
|
||||
def train_purchase_propensity(orders: pd.DataFrame) -> Dict:
|
||||
if orders.empty:
|
||||
return {"skipped": "no order history - run the seed step first"}
|
||||
cutoffs = [date.today() - timedelta(days=d) for d in (14, 21, 28, 35, 42)]
|
||||
frame = purchase_propensity_model.build_training_frame(orders, cutoffs)
|
||||
if frame.empty:
|
||||
return {"skipped": "not enough customer history yet"}
|
||||
bundle = purchase_propensity_model.train(frame)
|
||||
return {"n_samples": bundle.n_samples, **bundle.extra}
|
||||
|
||||
|
||||
def train_all(models: Optional[List[str]] = None) -> Dict[str, Dict]:
|
||||
models = models or ALL_MODELS
|
||||
order_items, orders, store_products = _load_common_frames()
|
||||
results: Dict[str, Dict] = {}
|
||||
|
||||
if "discount" in models:
|
||||
logger.info("Training discount model...")
|
||||
results["discount"] = train_discount(order_items, store_products)
|
||||
if "trending" in models:
|
||||
logger.info("Training trending model...")
|
||||
results["trending"] = train_trending(order_items, orders)
|
||||
if "popularity" in models:
|
||||
logger.info("Training popularity model...")
|
||||
results["popularity"] = train_popularity(order_items, store_products)
|
||||
if "forecast" in models:
|
||||
logger.info("Training demand forecast model...")
|
||||
results["forecast"] = train_forecast(order_items, store_products)
|
||||
if "store_performance" in models:
|
||||
logger.info("Training store performance model...")
|
||||
results["store_performance"] = train_store_performance(orders)
|
||||
if "purchase_propensity" in models:
|
||||
logger.info("Training purchase propensity model...")
|
||||
results["purchase_propensity"] = train_purchase_propensity(orders)
|
||||
|
||||
# Refresh discount_history for every store now that the discount
|
||||
# model is (re)trained, so /stores/{id}/discounts has fresh cached
|
||||
# data immediately instead of computing on first request.
|
||||
if "discount" in models and "skipped" not in results.get("discount", {}):
|
||||
for store in store_db.list_stores():
|
||||
try:
|
||||
predict_discounts_for_store(store["store_id"])
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Failed to refresh discounts for %s: %s", store["store_id"], e)
|
||||
|
||||
return results
|
||||
61
app/services/nutrition_alternatives_service.py
Normal file
61
app/services/nutrition_alternatives_service.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
"Healthier Alternatives" - for a given product, surface other products
|
||||
in the same category with a meaningfully better health_score.
|
||||
|
||||
Distinct from Feature 8 "Similar Products based on Nutrition"
|
||||
(`nutrition_similarity.py`, which finds the closest nutrition profile
|
||||
regardless of direction): this deliberately searches for something
|
||||
*better*, not just *similar*, which is why it's category-scoped and
|
||||
ranked by health_score improvement rather than by distance.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.services import nutrition_db
|
||||
|
||||
MIN_HEALTH_SCORE_DELTA = 8.0 # don't suggest an "alternative" that isn't meaningfully better
|
||||
|
||||
|
||||
def find_alternatives(brand: str, image_id: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
facts = nutrition_db.get_nutrition_facts(brand, image_id)
|
||||
insights = nutrition_db.get_nutrition_insights(brand, image_id)
|
||||
if not facts or facts.get("data_status") == "unavailable" or not insights or insights.get("health_score") is None:
|
||||
return []
|
||||
|
||||
category = facts.get("category")
|
||||
current_score = insights["health_score"]
|
||||
if not category:
|
||||
return []
|
||||
|
||||
candidates = nutrition_db.query_products(
|
||||
sort_by="health_score", order="desc", category=category, limit=50,
|
||||
)
|
||||
|
||||
ranked = []
|
||||
for c in candidates:
|
||||
if c["brand"] == brand and c["image_id"] == image_id:
|
||||
continue
|
||||
cand_score = c.get("health_score")
|
||||
if cand_score is None:
|
||||
continue
|
||||
delta = cand_score - current_score
|
||||
if delta < MIN_HEALTH_SCORE_DELTA:
|
||||
continue
|
||||
reason_bits = [f"{delta:.0f} points higher health score"]
|
||||
if facts.get("total_sugar_g") is not None and c.get("total_sugar_g") is not None and c["total_sugar_g"] < facts["total_sugar_g"]:
|
||||
reason_bits.append("lower sugar")
|
||||
if facts.get("sodium_mg") is not None and c.get("sodium_mg") is not None and c["sodium_mg"] < facts["sodium_mg"]:
|
||||
reason_bits.append("lower sodium")
|
||||
if facts.get("protein_g") is not None and c.get("protein_g") is not None and c["protein_g"] > facts["protein_g"]:
|
||||
reason_bits.append("more protein")
|
||||
ranked.append({
|
||||
"brand": c["brand"], "image_id": c["image_id"], "product_name": c.get("product_name"),
|
||||
"health_score_delta": round(delta, 1), "reason": ", ".join(reason_bits),
|
||||
})
|
||||
|
||||
ranked.sort(key=lambda r: r["health_score_delta"], reverse=True)
|
||||
top = ranked[:top_k]
|
||||
if top:
|
||||
nutrition_db.save_healthy_alternatives(brand, image_id, top)
|
||||
return top
|
||||
46
app/services/nutrition_analytics_service.py
Normal file
46
app/services/nutrition_analytics_service.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Feature 9: "Nutrition Analytics". Thin service layer over the SQL
|
||||
aggregations in `nutrition_db.py` - mirrors how `analytics_service.py`
|
||||
sits over `store_db.py` for the Store Intelligence analytics page.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.services import nutrition_db
|
||||
|
||||
|
||||
def get_leaderboards(limit: int = 10) -> Dict[str, Any]:
|
||||
return {
|
||||
"most_nutritious": nutrition_db.top_by_nutrient("nutrition_score", limit, ascending=False),
|
||||
"highest_protein": nutrition_db.top_by_nutrient("protein", limit, ascending=False),
|
||||
"highest_fiber": nutrition_db.top_by_nutrient("fiber", limit, ascending=False),
|
||||
"lowest_sugar": nutrition_db.top_by_nutrient("sugar", limit, ascending=True),
|
||||
"highest_calcium": nutrition_db.top_by_nutrient("calcium", limit, ascending=False),
|
||||
"highest_iron": nutrition_db.top_by_nutrient("iron", limit, ascending=False),
|
||||
"highest_vitamin_c": nutrition_db.top_by_nutrient("vitamin_c", limit, ascending=False),
|
||||
"lowest_sodium": nutrition_db.top_by_nutrient("sodium", limit, ascending=True),
|
||||
}
|
||||
|
||||
|
||||
def get_brand_category_rankings(limit: int = 10) -> Dict[str, Any]:
|
||||
return {
|
||||
"healthiest_brands": nutrition_db.healthiest_brands(limit),
|
||||
"healthiest_categories": nutrition_db.healthiest_categories(limit),
|
||||
}
|
||||
|
||||
|
||||
def get_distribution() -> Dict[str, Any]:
|
||||
return {
|
||||
"health_score_distribution": nutrition_db.score_distribution(),
|
||||
"store_healthy_distribution": nutrition_db.store_healthy_distribution(),
|
||||
"enrichment_status": nutrition_db.enrichment_status_counts(),
|
||||
}
|
||||
|
||||
|
||||
def get_full_dashboard(limit: int = 10) -> Dict[str, Any]:
|
||||
return {
|
||||
"leaderboards": get_leaderboards(limit),
|
||||
"rankings": get_brand_category_rankings(limit),
|
||||
"distribution": get_distribution(),
|
||||
}
|
||||
356
app/services/nutrition_data_service.py
Normal file
356
app/services/nutrition_data_service.py
Normal file
@@ -0,0 +1,356 @@
|
||||
"""
|
||||
Feature 1/4/15: verified nutrition-data retrieval.
|
||||
|
||||
This is the "R" in the RAG approach the spec asks for: every nutrient
|
||||
value that ends up in `nutrition_facts` comes from here, and everything
|
||||
here comes from Open Food Facts (https://world.openfoodfacts.org) - a
|
||||
crowdsourced-from-real-product-labels, openly licensed food database,
|
||||
already used elsewhere in this codebase as a trusted source (see
|
||||
`USE_OPEN_FACTS` in settings.py and `image_search.py`'s product-photo
|
||||
lookup, which queries the same API family).
|
||||
|
||||
HARD RULE (Feature 15): this module NEVER estimates, interpolates, or
|
||||
invents a nutrient value. If Open Food Facts doesn't return a field for
|
||||
a matched product, that field stays `None` all the way through to the
|
||||
API response - it does not get filled in with a category average, an
|
||||
LLM guess, or a zero. If no confident product match is found at all,
|
||||
`fetch_verified_nutrition` returns a `data_status="unavailable"` shell
|
||||
and nothing downstream is allowed to treat that as "no nutrients" (zero
|
||||
of everything) rather than "we don't know".
|
||||
|
||||
No LLM call happens anywhere in this file.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.infrastructure.settings import USE_OPEN_FACTS, REQUEST_TIMEOUT_SECONDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OFF_HOST = "world.openfoodfacts.org"
|
||||
_BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 KiranaAI-NutritionModule/1.0"
|
||||
)
|
||||
|
||||
# Below this fuzzy name-match score, a candidate is not considered a
|
||||
# confident enough match to attach real nutrient numbers to - it's
|
||||
# better to show "Nutrition data unavailable" than to attach the wrong
|
||||
# product's numbers. Configurable, not a magic untouchable constant.
|
||||
MIN_MATCH_CONFIDENCE = 0.32
|
||||
|
||||
# Categories in this FMCG catalog that are never food (no nutrition
|
||||
# panel exists), so we skip the network call entirely instead of
|
||||
# generating noisy near-misses.
|
||||
NON_FOOD_KEYWORDS = (
|
||||
"soap", "detergent", "shampoo", "toothpaste", "cosmetic", "deodorant",
|
||||
"diaper", "sanitary", "cleaner", "disinfectant", "battery", "stationery",
|
||||
"utensil", "cookware", "appliance", "electronics", "toy",
|
||||
)
|
||||
|
||||
# OFF `nutriments` values are normalized to grams per 100g/serving
|
||||
# (except energy, which OFF already reports in kcal directly under the
|
||||
# `-kcal` keys). This maps our flat schema column -> (off_key, output_unit).
|
||||
_NUTRIENT_MAP: List[tuple] = [
|
||||
("calories_kcal", "energy-kcal", "kcal_direct"),
|
||||
("protein_g", "proteins", "g"),
|
||||
("carbohydrates_g", "carbohydrates", "g"),
|
||||
("total_sugar_g", "sugars", "g"),
|
||||
("added_sugar_g", "added-sugars", "g"),
|
||||
("dietary_fiber_g", "fiber", "g"),
|
||||
("total_fat_g", "fat", "g"),
|
||||
("saturated_fat_g", "saturated-fat", "g"),
|
||||
("trans_fat_g", "trans-fat", "g"),
|
||||
("cholesterol_mg", "cholesterol", "mg_from_g"),
|
||||
("sodium_mg", "sodium", "mg_from_g"),
|
||||
("potassium_mg", "potassium", "mg_from_g"),
|
||||
("calcium_mg", "calcium", "mg_from_g"),
|
||||
("iron_mg", "iron", "mg_from_g"),
|
||||
("magnesium_mg", "magnesium", "mg_from_g"),
|
||||
("zinc_mg", "zinc", "mg_from_g"),
|
||||
("vitamin_a_mcg", "vitamin-a", "mcg_from_g"),
|
||||
("vitamin_c_mg", "vitamin-c", "mg_from_g"),
|
||||
("vitamin_d_mcg", "vitamin-d", "mcg_from_g"),
|
||||
("vitamin_e_mg", "vitamin-e", "mg_from_g"),
|
||||
("omega_3_g", "omega-3-fat", "g"),
|
||||
("omega_6_g", "omega-6-fat", "g"),
|
||||
]
|
||||
|
||||
# Extra nutrients worth keeping (individual B vitamins, trace minerals)
|
||||
# that don't have a flat column - captured into `extended_nutrients`.
|
||||
_EXTENDED_NUTRIENT_KEYS = [
|
||||
("vitamin-b1", "Vitamin B1 (Thiamine)", "mg_from_g"),
|
||||
("vitamin-b2", "Vitamin B2 (Riboflavin)", "mg_from_g"),
|
||||
("vitamin-b6", "Vitamin B6", "mg_from_g"),
|
||||
("vitamin-b9", "Vitamin B9 (Folate)", "mcg_from_g"),
|
||||
("vitamin-b12", "Vitamin B12", "mcg_from_g"),
|
||||
("pp", "Vitamin B3 (Niacin)", "mg_from_g"),
|
||||
("phosphorus", "Phosphorus", "mg_from_g"),
|
||||
("iodine", "Iodine", "mcg_from_g"),
|
||||
("salt", "Salt", "g"),
|
||||
]
|
||||
|
||||
|
||||
def _convert(value: Optional[float], kind: str) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
v = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if kind == "g" or kind == "kcal_direct":
|
||||
return round(v, 3)
|
||||
if kind == "mg_from_g":
|
||||
return round(v * 1000, 3)
|
||||
if kind == "mcg_from_g":
|
||||
return round(v * 1_000_000, 3)
|
||||
return round(v, 3)
|
||||
|
||||
|
||||
import re
|
||||
|
||||
def _clean_title_for_search(title: str) -> str:
|
||||
"""Strips size, volume, weight, and packaging suffixes to improve search accuracy."""
|
||||
t = re.sub(r'\b\d+(\.\d+)?\s*(g|kg|ml|l|gm|ltr|grm|pack|pc|pcs)\b', '', title, flags=re.IGNORECASE)
|
||||
t = re.sub(r'[-_]', ' ', t)
|
||||
return ' '.join(t.split())
|
||||
|
||||
|
||||
def _looks_non_food(title: str, category: str) -> bool:
|
||||
text = f"{title} {category}".lower()
|
||||
return any(kw in text for kw in NON_FOOD_KEYWORDS)
|
||||
|
||||
|
||||
def _search_openfoodfacts(query: str, brand: str = "", category: str = "", max_results: int = 5) -> List[dict]:
|
||||
if not query.strip() and not brand.strip():
|
||||
return []
|
||||
|
||||
fields_str = (
|
||||
"code,product_name,brands,nutriments,serving_size,serving_quantity,"
|
||||
"allergens_tags,labels_tags,ingredients_analysis_tags,categories_tags,"
|
||||
"ingredients_text,nutriscore_grade,nutrition_data_per,quantity"
|
||||
)
|
||||
headers = {"User-Agent": _BROWSER_UA}
|
||||
|
||||
# Strategy 1: OFF v2 API search with brands_tags + search_terms
|
||||
if brand.strip():
|
||||
cleaned_query = _clean_title_for_search(query)
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://{OFF_HOST}/api/v2/search",
|
||||
params={
|
||||
"brands_tags": brand.lower().strip(),
|
||||
"search_terms": cleaned_query,
|
||||
"page_size": max_results,
|
||||
"fields": fields_str,
|
||||
},
|
||||
headers=headers,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
prods = resp.json().get("products", []) or []
|
||||
if prods:
|
||||
return prods
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(f"OFF v2 search Strategy 1 failed for '{brand}' '{query}': {e}")
|
||||
|
||||
# Strategy 2: OFF v2 API search using search_terms with combined brand + query
|
||||
full_term = f"{brand} {_clean_title_for_search(query)}".strip()
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://{OFF_HOST}/api/v2/search",
|
||||
params={
|
||||
"search_terms": full_term,
|
||||
"page_size": max_results,
|
||||
"fields": fields_str,
|
||||
},
|
||||
headers=headers,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
prods = resp.json().get("products", []) or []
|
||||
if prods:
|
||||
return prods
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(f"OFF v2 search Strategy 2 failed for '{full_term}': {e}")
|
||||
|
||||
# Strategy 3: OFF v2 API search using q parameter
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://{OFF_HOST}/api/v2/search",
|
||||
params={
|
||||
"q": full_term,
|
||||
"page_size": max_results,
|
||||
"fields": fields_str,
|
||||
},
|
||||
headers=headers,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
prods = resp.json().get("products", []) or []
|
||||
if prods:
|
||||
return prods
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(f"OFF v2 search Strategy 3 failed for '{full_term}': {e}")
|
||||
|
||||
# Strategy 4: Fallback to category + brand search
|
||||
if category.strip() and brand.strip():
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://{OFF_HOST}/api/v2/search",
|
||||
params={
|
||||
"search_terms": f"{brand} {category}".strip(),
|
||||
"page_size": max_results,
|
||||
"fields": fields_str,
|
||||
},
|
||||
headers=headers,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
return resp.json().get("products", []) or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(f"OFF v2 search Strategy 4 failed: {e}")
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _match_confidence(query_brand: str, query_title: str, candidate: dict) -> float:
|
||||
cand_brand = (candidate.get("brands") or "").lower().strip()
|
||||
cand_name = (candidate.get("product_name") or "").lower().strip()
|
||||
q_brand = query_brand.lower().strip()
|
||||
q_title_clean = _clean_title_for_search(query_title).lower().strip()
|
||||
|
||||
full_cand = f"{cand_brand} {cand_name}".strip()
|
||||
full_query = f"{q_brand} {q_title_clean}".strip()
|
||||
if not full_cand or not full_query:
|
||||
return 0.0
|
||||
|
||||
ratio = SequenceMatcher(None, full_query, full_cand).ratio()
|
||||
if q_brand and (q_brand in cand_brand or q_brand in cand_name):
|
||||
ratio = max(ratio, 0.45)
|
||||
|
||||
q_tokens = set(q_title_clean.split())
|
||||
cand_tokens = set(re.sub(r'[-_]', ' ', cand_name).split())
|
||||
if q_tokens and cand_tokens:
|
||||
overlap = len(q_tokens.intersection(cand_tokens)) / float(len(q_tokens))
|
||||
if overlap >= 0.5:
|
||||
ratio = max(ratio, 0.5 + overlap * 0.3)
|
||||
|
||||
return min(1.0, ratio)
|
||||
|
||||
|
||||
def _extract_allergens(candidate: dict) -> List[str]:
|
||||
tags = candidate.get("allergens_tags") or []
|
||||
out = []
|
||||
for tag in tags:
|
||||
name = tag.split(":")[-1].replace("-", " ").strip()
|
||||
if name:
|
||||
out.append(name.title())
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def _build_extended_nutrients(nutriments: dict, suffix: str) -> Dict[str, Any]:
|
||||
out: Dict[str, Any] = {}
|
||||
for off_key, label, kind in _EXTENDED_NUTRIENT_KEYS:
|
||||
raw = nutriments.get(f"{off_key}_{suffix}")
|
||||
value = _convert(raw, kind)
|
||||
if value is not None:
|
||||
unit = "mg" if "mg" in kind else ("mcg" if "mcg" in kind else "g")
|
||||
out[label] = {"value": value, "unit": unit}
|
||||
return out
|
||||
|
||||
|
||||
def _build_flat_fields(nutriments: dict, suffix: str) -> Dict[str, Any]:
|
||||
out: Dict[str, Any] = {}
|
||||
for field_name, off_key, kind in _NUTRIENT_MAP:
|
||||
raw = nutriments.get(f"{off_key}_{suffix}")
|
||||
out[field_name] = _convert(raw, kind)
|
||||
# Sodium fallback: OFF products more often report salt than sodium.
|
||||
# Salt (g) -> sodium (mg) is a fixed, published conversion factor
|
||||
# (sodium = salt / 2.5), not an estimate of the nutrient itself.
|
||||
if out.get("sodium_mg") is None:
|
||||
salt = nutriments.get(f"salt_{suffix}")
|
||||
if salt is not None:
|
||||
try:
|
||||
out["sodium_mg"] = round((float(salt) / 2.5) * 1000, 3)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
# Calories fallback: use energy_100g (kJ) -> kcal only if the
|
||||
# kcal field itself wasn't provided directly.
|
||||
if out.get("calories_kcal") is None:
|
||||
kj = nutriments.get(f"energy_{suffix}")
|
||||
if kj is not None:
|
||||
try:
|
||||
out["calories_kcal"] = round(float(kj) / 4.184, 1)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def fetch_verified_nutrition(brand: str, title: str, category: str = "") -> Dict[str, Any]:
|
||||
"""Look up verified nutrition data for one catalog product.
|
||||
|
||||
Always returns a dict with at least `data_status`. When
|
||||
`data_status == "unavailable"`, every nutrient field is absent/None -
|
||||
callers must render "Nutrition data unavailable" rather than
|
||||
treating that as zero.
|
||||
"""
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
unavailable = {"data_status": "unavailable", "fetched_at": now_iso}
|
||||
|
||||
if not USE_OPEN_FACTS:
|
||||
return unavailable
|
||||
if _looks_non_food(title, category):
|
||||
return unavailable
|
||||
|
||||
candidates = _search_openfoodfacts(query=title, brand=brand, category=category)
|
||||
if not candidates:
|
||||
candidates = _search_openfoodfacts(query=title)
|
||||
if not candidates:
|
||||
return unavailable
|
||||
|
||||
scored = [(c, _match_confidence(brand, title, c)) for c in candidates]
|
||||
scored.sort(key=lambda t: t[1], reverse=True)
|
||||
best, confidence = scored[0]
|
||||
if confidence < MIN_MATCH_CONFIDENCE:
|
||||
return unavailable
|
||||
|
||||
nutriments = best.get("nutriments") or {}
|
||||
if not nutriments:
|
||||
return unavailable
|
||||
|
||||
per_100g = _build_flat_fields(nutriments, "100g")
|
||||
per_serving = _build_flat_fields(nutriments, "serving")
|
||||
has_any_value = any(v is not None for v in per_100g.values())
|
||||
if not has_any_value:
|
||||
return unavailable
|
||||
|
||||
extended = _build_extended_nutrients(nutriments, "100g")
|
||||
code = best.get("code")
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"data_status": "verified" if per_100g.get("calories_kcal") is not None else "partial",
|
||||
"data_source": "openfoodfacts",
|
||||
"source_ref": code,
|
||||
"source_url": f"https://{OFF_HOST}/product/{code}" if code else None,
|
||||
"match_confidence": round(confidence, 3),
|
||||
"serving_size_g": _convert(best.get("serving_quantity"), "g"),
|
||||
"serving_size_label": best.get("serving_size"),
|
||||
"extended_nutrients": extended,
|
||||
"per_serving": {k: v for k, v in per_serving.items() if v is not None},
|
||||
"ingredients_text": (best.get("ingredients_text") or "").strip() or None,
|
||||
"off_nutriscore": (best.get("nutriscore_grade") or "").strip().lower() or None,
|
||||
"allergens": _extract_allergens(best),
|
||||
"off_labels_tags": best.get("labels_tags") or [],
|
||||
"off_ingredients_analysis_tags": best.get("ingredients_analysis_tags") or [],
|
||||
"off_categories_tags": best.get("categories_tags") or [],
|
||||
"fetched_at": now_iso,
|
||||
}
|
||||
result.update(per_100g)
|
||||
return result
|
||||
702
app/services/nutrition_db.py
Normal file
702
app/services/nutrition_db.py
Normal file
@@ -0,0 +1,702 @@
|
||||
"""
|
||||
AI Nutritional Intelligence Module - persistence layer.
|
||||
|
||||
Follows the exact same conventions as `store_db.py` (raw psycopg, no ORM,
|
||||
`_connect()` returns None and every function degrades to an empty/no-op
|
||||
result when Postgres isn't reachable, `dict_row` cursor for read paths,
|
||||
`_cast_numeric` for the Decimal->float conversion pandas/numpy chokes on)
|
||||
so this module slots into the existing architecture instead of introducing
|
||||
a second data-access pattern.
|
||||
|
||||
New tables, additive only - nothing here touches or drops any existing
|
||||
`brand_*` table or any `store_*` table:
|
||||
nutrition_facts - verified nutrition data (Feature 11 "Nutrition")
|
||||
nutrition_insights - scores, insights, cautions, diet tags,
|
||||
allergens, clustering (Feature 11
|
||||
"Nutrition Insights")
|
||||
nutrition_similar_products - cached ML nutritional-similarity output
|
||||
(Feature 8)
|
||||
nutrition_healthy_alternatives - cached healthier-alternative rankings
|
||||
(Feature 2/7)
|
||||
|
||||
Products are referenced everywhere by the composite key (brand, image_id)
|
||||
- the SAME natural key `store_db.py` already treats as globally unique
|
||||
across brand tables (see the docstring at the top of that file). There is
|
||||
still no real FK to a `brand_*` table (there isn't one shared table to
|
||||
reference), so referential integrity is enforced at the application layer,
|
||||
exactly like the store-intelligence tables.
|
||||
|
||||
CRITICAL DATA-INTEGRITY NOTE (Feature 15)
|
||||
------------------------------------------
|
||||
Every numeric column in `nutrition_facts` is nullable and stays NULL
|
||||
unless a value was actually returned by a trusted source (currently Open
|
||||
Food Facts - see `nutrition_data_service.py`). Nothing in this module
|
||||
ever writes an estimated, interpolated, or LLM-guessed number into these
|
||||
columns. `data_status` records whether a row has verified data
|
||||
('verified'), no verified data was found ('unavailable'), or only some
|
||||
fields were confirmed ('partial') - callers must check this before
|
||||
treating NULL as "zero".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.services.vector_store import _connect # reuse the exact same connection logic
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema
|
||||
# ---------------------------------------------------------------------------
|
||||
SCHEMA_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS nutrition_facts (
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
category TEXT,
|
||||
|
||||
data_status TEXT NOT NULL DEFAULT 'unavailable', -- 'verified' | 'partial' | 'unavailable'
|
||||
data_source TEXT, -- 'openfoodfacts' | 'manual'
|
||||
source_ref TEXT, -- OFF product code / barcode
|
||||
source_url TEXT,
|
||||
match_confidence NUMERIC, -- 0-1, name-match confidence to the source record
|
||||
|
||||
serving_size_g NUMERIC,
|
||||
serving_size_label TEXT,
|
||||
|
||||
-- Per-100g core fields (flat + indexed - these are the ones the
|
||||
-- Feature 12 filter endpoints like "high protein" / "low sugar" sort on)
|
||||
calories_kcal NUMERIC,
|
||||
protein_g NUMERIC,
|
||||
carbohydrates_g NUMERIC,
|
||||
total_sugar_g NUMERIC,
|
||||
added_sugar_g NUMERIC,
|
||||
dietary_fiber_g NUMERIC,
|
||||
total_fat_g NUMERIC,
|
||||
saturated_fat_g NUMERIC,
|
||||
trans_fat_g NUMERIC,
|
||||
cholesterol_mg NUMERIC,
|
||||
sodium_mg NUMERIC,
|
||||
potassium_mg NUMERIC,
|
||||
calcium_mg NUMERIC,
|
||||
iron_mg NUMERIC,
|
||||
magnesium_mg NUMERIC,
|
||||
zinc_mg NUMERIC,
|
||||
vitamin_a_mcg NUMERIC,
|
||||
vitamin_c_mg NUMERIC,
|
||||
vitamin_d_mcg NUMERIC,
|
||||
vitamin_e_mg NUMERIC,
|
||||
omega_3_g NUMERIC,
|
||||
omega_6_g NUMERIC,
|
||||
|
||||
-- Everything else verified (individual B vitamins, iodine, etc.) plus
|
||||
-- the same shape again for "per serving" display - kept as JSONB
|
||||
-- rather than dozens more flat columns since the field set a source
|
||||
-- actually reports varies a lot product to product.
|
||||
extended_nutrients JSONB,
|
||||
per_serving JSONB,
|
||||
|
||||
ingredients_text TEXT,
|
||||
off_nutriscore TEXT, -- external Nutri-Score grade ('a'..'e') if the source provides one, shown as-is
|
||||
|
||||
fetched_at TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (brand, image_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_category ON nutrition_facts(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_protein ON nutrition_facts(protein_g DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_fiber ON nutrition_facts(dietary_fiber_g DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_sugar ON nutrition_facts(total_sugar_g ASC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_sodium ON nutrition_facts(sodium_mg ASC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_calcium ON nutrition_facts(calcium_mg DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_iron ON nutrition_facts(iron_mg DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_vitc ON nutrition_facts(vitamin_c_mg DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_facts_status ON nutrition_facts(data_status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS nutrition_insights (
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
|
||||
nutrition_score NUMERIC, -- 0-100, nutrient-density composite
|
||||
health_score NUMERIC, -- 0-100, nutrition_score adjusted for calorie density
|
||||
score_breakdown JSONB, -- {protein_score, fiber_score, sugar_score, sodium_score, fat_quality_score, micronutrient_score, ...}
|
||||
scoring_version TEXT,
|
||||
|
||||
positive_insights TEXT[],
|
||||
nutritional_cautions TEXT[],
|
||||
ai_summary TEXT,
|
||||
|
||||
diet_tags TEXT[],
|
||||
allergens TEXT[],
|
||||
allergen_source TEXT, -- 'openfoodfacts' | 'unavailable'
|
||||
|
||||
nutrition_cluster INTEGER,
|
||||
nutrition_cluster_label TEXT,
|
||||
|
||||
data_status TEXT NOT NULL DEFAULT 'unavailable',
|
||||
generated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (brand, image_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_insights_health ON nutrition_insights(health_score DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_insights_nutrition ON nutrition_insights(nutrition_score DESC NULLS LAST);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_insights_diet_tags ON nutrition_insights USING GIN(diet_tags);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_insights_allergens ON nutrition_insights USING GIN(allergens);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_insights_cluster ON nutrition_insights(nutrition_cluster);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS nutrition_similar_products (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
rank INTEGER NOT NULL,
|
||||
similar_brand TEXT NOT NULL,
|
||||
similar_image_id TEXT NOT NULL,
|
||||
similarity_score NUMERIC NOT NULL,
|
||||
method TEXT NOT NULL DEFAULT 'cosine',
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_similar_source ON nutrition_similar_products(brand, image_id, rank);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS nutrition_healthy_alternatives (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
rank INTEGER NOT NULL,
|
||||
alt_brand TEXT NOT NULL,
|
||||
alt_image_id TEXT NOT NULL,
|
||||
health_score_delta NUMERIC,
|
||||
reason TEXT,
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_alt_source ON nutrition_healthy_alternatives(brand, image_id, rank);
|
||||
"""
|
||||
|
||||
|
||||
def ensure_nutrition_schema() -> bool:
|
||||
"""Idempotent: safe to call on every API startup, mirrors
|
||||
`ensure_store_intelligence_schema()`."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.warning("Nutrition schema not created: no DB connection")
|
||||
return False
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(SCHEMA_DDL)
|
||||
logger.info("Nutrition-intelligence schema ensured (nutrition_facts, nutrition_insights, "
|
||||
"nutrition_similar_products, nutrition_healthy_alternatives)")
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"Failed to ensure nutrition schema: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _dict_cursor(conn: psycopg.Connection):
|
||||
return conn.cursor(row_factory=dict_row)
|
||||
|
||||
|
||||
NUMERIC_FACT_COLUMNS = [
|
||||
"calories_kcal", "protein_g", "carbohydrates_g", "total_sugar_g", "added_sugar_g",
|
||||
"dietary_fiber_g", "total_fat_g", "saturated_fat_g", "trans_fat_g", "cholesterol_mg",
|
||||
"sodium_mg", "potassium_mg", "calcium_mg", "iron_mg", "magnesium_mg", "zinc_mg",
|
||||
"vitamin_a_mcg", "vitamin_c_mg", "vitamin_d_mcg", "vitamin_e_mg", "omega_3_g",
|
||||
"omega_6_g", "serving_size_g", "match_confidence",
|
||||
]
|
||||
|
||||
NUMERIC_INSIGHT_COLUMNS = ["nutrition_score", "health_score"]
|
||||
|
||||
|
||||
def _cast_numeric(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
|
||||
"""Postgres NUMERIC -> decimal.Decimal on read; scikit-learn/numpy need floats."""
|
||||
for col in columns:
|
||||
if col in df.columns:
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
return df
|
||||
|
||||
|
||||
def _row_numeric(row: Dict[str, Any], columns: List[str]) -> Dict[str, Any]:
|
||||
for col in columns:
|
||||
if row.get(col) is not None:
|
||||
row[col] = float(row[col])
|
||||
return row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# nutrition_facts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_FACT_COLUMNS = [
|
||||
"brand", "image_id", "product_name", "category", "data_status", "data_source",
|
||||
"source_ref", "source_url", "match_confidence", "serving_size_g", "serving_size_label",
|
||||
"calories_kcal", "protein_g", "carbohydrates_g", "total_sugar_g", "added_sugar_g",
|
||||
"dietary_fiber_g", "total_fat_g", "saturated_fat_g", "trans_fat_g", "cholesterol_mg",
|
||||
"sodium_mg", "potassium_mg", "calcium_mg", "iron_mg", "magnesium_mg", "zinc_mg",
|
||||
"vitamin_a_mcg", "vitamin_c_mg", "vitamin_d_mcg", "vitamin_e_mg", "omega_3_g", "omega_6_g",
|
||||
"extended_nutrients", "per_serving", "ingredients_text", "off_nutriscore", "fetched_at",
|
||||
]
|
||||
|
||||
|
||||
def upsert_nutrition_facts(facts: Dict[str, Any]) -> bool:
|
||||
"""`facts` must include brand + image_id; every other key in
|
||||
`_FACT_COLUMNS` is optional and defaults to NULL (never fabricated)."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return False
|
||||
row = {col: facts.get(col) for col in _FACT_COLUMNS}
|
||||
if not row["brand"] or not row["image_id"]:
|
||||
return False
|
||||
if isinstance(row.get("extended_nutrients"), (dict, list)):
|
||||
row["extended_nutrients"] = Json(row["extended_nutrients"])
|
||||
if isinstance(row.get("per_serving"), (dict, list)):
|
||||
row["per_serving"] = Json(row["per_serving"])
|
||||
cols = ", ".join(_FACT_COLUMNS)
|
||||
placeholders = ", ".join(f"%({c})s" for c in _FACT_COLUMNS)
|
||||
update_cols = [c for c in _FACT_COLUMNS if c not in ("brand", "image_id")]
|
||||
update_clause = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO nutrition_facts ({cols})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT (brand, image_id) DO UPDATE SET
|
||||
{update_clause}, updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
row,
|
||||
)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"upsert_nutrition_facts failed for {row['brand']}/{row['image_id']}: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_nutrition_facts(brand: str, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return None
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute("SELECT * FROM nutrition_facts WHERE brand = %s AND image_id = %s", (brand, image_id))
|
||||
row = cur.fetchone()
|
||||
return _row_numeric(dict(row), NUMERIC_FACT_COLUMNS) if row else None
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_nutrition_facts failed: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_all_nutrition_facts_df() -> pd.DataFrame:
|
||||
"""Every verified/partial nutrition_facts row, for the ML layer
|
||||
(similarity + clustering) and analytics aggregation. Rows with no
|
||||
verified data at all are excluded since there is nothing numeric to
|
||||
compute over."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return pd.DataFrame()
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute("SELECT * FROM nutrition_facts WHERE data_status != 'unavailable'")
|
||||
rows = [dict(r) for r in cur.fetchall()]
|
||||
df = pd.DataFrame(rows)
|
||||
return _cast_numeric(df, NUMERIC_FACT_COLUMNS) if not df.empty else df
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_all_nutrition_facts_df failed: {e}")
|
||||
return pd.DataFrame()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def enrichment_status_counts() -> Dict[str, int]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return {}
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT data_status, COUNT(*) FROM nutrition_facts GROUP BY data_status")
|
||||
return {status: count for status, count in cur.fetchall()}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"enrichment_status_counts failed: {e}")
|
||||
return {}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# nutrition_insights
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_INSIGHT_COLUMNS = [
|
||||
"brand", "image_id", "nutrition_score", "health_score", "score_breakdown",
|
||||
"scoring_version", "positive_insights", "nutritional_cautions", "ai_summary",
|
||||
"diet_tags", "allergens", "allergen_source", "nutrition_cluster",
|
||||
"nutrition_cluster_label", "data_status",
|
||||
]
|
||||
|
||||
|
||||
def upsert_nutrition_insights(insights: Dict[str, Any]) -> bool:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return False
|
||||
row = {col: insights.get(col) for col in _INSIGHT_COLUMNS}
|
||||
if not row["brand"] or not row["image_id"]:
|
||||
return False
|
||||
if isinstance(row.get("score_breakdown"), dict):
|
||||
row["score_breakdown"] = Json(row["score_breakdown"])
|
||||
for arr_col in ("positive_insights", "nutritional_cautions", "diet_tags", "allergens"):
|
||||
if row.get(arr_col) is None:
|
||||
row[arr_col] = []
|
||||
cols = ", ".join(_INSIGHT_COLUMNS)
|
||||
placeholders = ", ".join(f"%({c})s" for c in _INSIGHT_COLUMNS)
|
||||
update_cols = [c for c in _INSIGHT_COLUMNS if c not in ("brand", "image_id")]
|
||||
update_clause = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO nutrition_insights ({cols})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT (brand, image_id) DO UPDATE SET
|
||||
{update_clause}, generated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
row,
|
||||
)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"upsert_nutrition_insights failed for {row['brand']}/{row['image_id']}: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_nutrition_insights(brand: str, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return None
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute("SELECT * FROM nutrition_insights WHERE brand = %s AND image_id = %s", (brand, image_id))
|
||||
row = cur.fetchone()
|
||||
return _row_numeric(dict(row), NUMERIC_INSIGHT_COLUMNS) if row else None
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_nutrition_insights failed: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_full_nutrition(brand: str, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Merged facts + insights for a single product (the shape the main
|
||||
product-detail nutrition endpoint returns). Returns a "data
|
||||
unavailable" shell (never None) if the product has been catalog-ingested
|
||||
but nutrition enrichment hasn't run for it yet, so the frontend always
|
||||
has something to render."""
|
||||
facts = get_nutrition_facts(brand, image_id) or {
|
||||
"brand": brand, "image_id": image_id, "data_status": "unavailable",
|
||||
}
|
||||
insights = get_nutrition_insights(brand, image_id) or {
|
||||
"brand": brand, "image_id": image_id, "data_status": "unavailable",
|
||||
"positive_insights": [], "nutritional_cautions": [], "diet_tags": [], "allergens": [],
|
||||
}
|
||||
merged = {**facts, **{k: v for k, v in insights.items() if k not in ("brand", "image_id")}}
|
||||
return merged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filter / sort queries (Feature 12: high-protein, low-sugar, high-fiber, diet-compatible)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SORTABLE_COLUMNS = {
|
||||
"protein": "protein_g", "fiber": "dietary_fiber_g", "sugar": "total_sugar_g",
|
||||
"sodium": "sodium_mg", "calcium": "calcium_mg", "iron": "iron_mg",
|
||||
"vitamin_c": "vitamin_c_mg", "calories": "calories_kcal",
|
||||
"health_score": "health_score", "nutrition_score": "nutrition_score",
|
||||
}
|
||||
|
||||
|
||||
def query_products(
|
||||
sort_by: str = "health_score",
|
||||
order: str = "desc",
|
||||
category: Optional[str] = None,
|
||||
diet_tag: Optional[str] = None,
|
||||
exclude_allergen: Optional[str] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Core listing query behind every Feature 12 filter/sort endpoint
|
||||
(high-protein, low-sugar, high-fiber products, diet-compatible
|
||||
products, ...). Joins nutrition_facts + nutrition_insights on the
|
||||
shared (brand, image_id) key."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
col = _SORTABLE_COLUMNS.get(sort_by, "health_score")
|
||||
table = "nutrition_insights i" if col in ("health_score", "nutrition_score") else "nutrition_facts f"
|
||||
direction = "ASC" if order == "asc" else "DESC"
|
||||
|
||||
where = ["f.data_status != 'unavailable'"]
|
||||
params: List[Any] = []
|
||||
if category:
|
||||
where.append("f.category ILIKE %s")
|
||||
params.append(f"%{category}%")
|
||||
if diet_tag:
|
||||
where.append("i.diet_tags @> %s::text[]")
|
||||
params.append([diet_tag])
|
||||
if exclude_allergen:
|
||||
where.append("NOT (i.allergens @> %s::text[])")
|
||||
params.append([exclude_allergen])
|
||||
where_clause = " AND ".join(where)
|
||||
|
||||
sql = f"""
|
||||
SELECT f.brand, f.image_id, f.product_name, f.category,
|
||||
f.calories_kcal, f.protein_g, f.dietary_fiber_g, f.total_sugar_g,
|
||||
f.sodium_mg, f.calcium_mg, f.iron_mg, f.vitamin_c_mg,
|
||||
i.nutrition_score, i.health_score, i.diet_tags, i.allergens
|
||||
FROM nutrition_facts f
|
||||
LEFT JOIN nutrition_insights i ON i.brand = f.brand AND i.image_id = f.image_id
|
||||
WHERE {where_clause} AND {col} IS NOT NULL
|
||||
ORDER BY {col} {direction} NULLS LAST
|
||||
LIMIT %s OFFSET %s
|
||||
"""
|
||||
params.extend([limit, offset])
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(sql, params)
|
||||
rows = [dict(r) for r in cur.fetchall()]
|
||||
return [_row_numeric(r, NUMERIC_FACT_COLUMNS + NUMERIC_INSIGHT_COLUMNS) for r in rows]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"query_products failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Similarity / alternatives caches
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def save_similar_products(brand: str, image_id: str, ranked: List[Dict[str, Any]], method: str = "cosine") -> None:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM nutrition_similar_products WHERE brand = %s AND image_id = %s", (brand, image_id))
|
||||
for rank, item in enumerate(ranked, start=1):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO nutrition_similar_products
|
||||
(brand, image_id, rank, similar_brand, similar_image_id, similarity_score, method)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(brand, image_id, rank, item["brand"], item["image_id"], item["similarity_score"], method),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_similar_products failed: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_similar_products(brand: str, image_id: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT similar_brand AS brand, similar_image_id AS image_id, similarity_score, method
|
||||
FROM nutrition_similar_products
|
||||
WHERE brand = %s AND image_id = %s
|
||||
ORDER BY rank ASC LIMIT %s
|
||||
""",
|
||||
(brand, image_id, top_k),
|
||||
)
|
||||
return [_row_numeric(dict(r), ["similarity_score"]) for r in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_similar_products failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def save_healthy_alternatives(brand: str, image_id: str, ranked: List[Dict[str, Any]]) -> None:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM nutrition_healthy_alternatives WHERE brand = %s AND image_id = %s", (brand, image_id))
|
||||
for rank, item in enumerate(ranked, start=1):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO nutrition_healthy_alternatives
|
||||
(brand, image_id, rank, alt_brand, alt_image_id, health_score_delta, reason)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(brand, image_id, rank, item["brand"], item["image_id"],
|
||||
item.get("health_score_delta"), item.get("reason")),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_healthy_alternatives failed: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_healthy_alternatives(brand: str, image_id: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT alt_brand AS brand, alt_image_id AS image_id, health_score_delta, reason
|
||||
FROM nutrition_healthy_alternatives
|
||||
WHERE brand = %s AND image_id = %s
|
||||
ORDER BY rank ASC LIMIT %s
|
||||
""",
|
||||
(brand, image_id, top_k),
|
||||
)
|
||||
return [_row_numeric(dict(r), ["health_score_delta"]) for r in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_healthy_alternatives failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Analytics (Feature 9)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def top_by_nutrient(nutrient: str, limit: int = 10, ascending: bool = False) -> List[Dict[str, Any]]:
|
||||
col = _SORTABLE_COLUMNS.get(nutrient)
|
||||
if not col:
|
||||
return []
|
||||
return query_products(sort_by=nutrient, order="asc" if ascending else "desc", limit=limit)
|
||||
|
||||
|
||||
def healthiest_brands(limit: int = 10) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT f.brand, ROUND(AVG(i.health_score)::numeric, 1) AS avg_health_score,
|
||||
ROUND(AVG(i.nutrition_score)::numeric, 1) AS avg_nutrition_score,
|
||||
COUNT(*) AS product_count
|
||||
FROM nutrition_facts f
|
||||
JOIN nutrition_insights i ON i.brand = f.brand AND i.image_id = f.image_id
|
||||
WHERE f.data_status != 'unavailable' AND i.health_score IS NOT NULL
|
||||
GROUP BY f.brand
|
||||
HAVING COUNT(*) >= 1
|
||||
ORDER BY avg_health_score DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
return [_row_numeric(dict(r), ["avg_health_score", "avg_nutrition_score"]) for r in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"healthiest_brands failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def healthiest_categories(limit: int = 10) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT f.category, ROUND(AVG(i.health_score)::numeric, 1) AS avg_health_score,
|
||||
ROUND(AVG(i.nutrition_score)::numeric, 1) AS avg_nutrition_score,
|
||||
COUNT(*) AS product_count
|
||||
FROM nutrition_facts f
|
||||
JOIN nutrition_insights i ON i.brand = f.brand AND i.image_id = f.image_id
|
||||
WHERE f.data_status != 'unavailable' AND i.health_score IS NOT NULL AND f.category IS NOT NULL
|
||||
GROUP BY f.category
|
||||
ORDER BY avg_health_score DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
return [_row_numeric(dict(r), ["avg_health_score", "avg_nutrition_score"]) for r in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"healthiest_categories failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def score_distribution() -> Dict[str, int]:
|
||||
"""Bucketed health_score histogram (0-20, 20-40, ..., 80-100) - used
|
||||
for the analytics dashboard's distribution chart."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return {}
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT WIDTH_BUCKET(health_score, 0, 100, 5) AS bucket, COUNT(*)
|
||||
FROM nutrition_insights
|
||||
WHERE health_score IS NOT NULL
|
||||
GROUP BY bucket ORDER BY bucket
|
||||
"""
|
||||
)
|
||||
labels = {1: "0-20", 2: "20-40", 3: "40-60", 4: "60-80", 5: "80-100"}
|
||||
return {labels.get(b, str(b)): c for b, c in cur.fetchall()}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"score_distribution failed: {e}")
|
||||
return {}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def store_healthy_distribution() -> List[Dict[str, Any]]:
|
||||
"""Store-wise healthy-product distribution - joins onto `store_inventory`
|
||||
from the v3.0 Store Intelligence layer (Feature 9's last bullet). Returns
|
||||
[] gracefully if that layer hasn't been seeded, since it's an optional
|
||||
dependency of this module, not a requirement."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'store_inventory')"
|
||||
)
|
||||
if not cur.fetchone()["exists"]:
|
||||
return []
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT si.store_id,
|
||||
COUNT(*) FILTER (WHERE i.health_score >= 60) AS healthy_products,
|
||||
COUNT(*) FILTER (WHERE i.health_score IS NOT NULL) AS scored_products
|
||||
FROM store_inventory si
|
||||
JOIN nutrition_insights i ON i.brand = si.brand AND i.image_id = si.image_id
|
||||
GROUP BY si.store_id ORDER BY si.store_id
|
||||
"""
|
||||
)
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"store_healthy_distribution failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
164
app/services/nutrition_enrichment_service.py
Normal file
164
app/services/nutrition_enrichment_service.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Orchestrates the RAG-style pipeline the spec asks for end-to-end:
|
||||
|
||||
1. RETRIEVE verified facts -> nutrition_data_service.fetch_verified_nutrition
|
||||
2. SCORE + CLASSIFY (rules) -> nutrition_scoring.*
|
||||
3. NARRATE (LLM, grounded) -> nutrition_narrative_service.generate_summary
|
||||
4. PERSIST -> nutrition_db.upsert_*
|
||||
|
||||
Steps 1-2 never touch an LLM. Step 3 only ever sees the already-computed
|
||||
bullet lists from step 2, never the raw retrieval payload - so even if
|
||||
something upstream changes, the LLM physically cannot introduce a
|
||||
nutrient value that didn't come from step 1.
|
||||
|
||||
This file is imported by both `app/api/routers/nutrition_admin.py`
|
||||
(background job for the web UI) and `scripts/enrich_nutrition.py` (CLI),
|
||||
so the two stay in sync by construction rather than by convention.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from app.services import nutrition_data_service, nutrition_db, nutrition_scoring
|
||||
from app.services.nutrition_narrative_service import generate_summary
|
||||
from app.services.vector_store import get_products_by_brand, list_available_brands
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentResult:
|
||||
total_products: int = 0
|
||||
verified: int = 0
|
||||
partial: int = 0
|
||||
unavailable: int = 0
|
||||
errors: List[str] = field(default_factory=list)
|
||||
duration_seconds: float = 0.0
|
||||
|
||||
|
||||
def enrich_one_product(brand: str, image_id: str, product_name: str, category: str,
|
||||
skip_if_verified: bool = False, generate_narrative: bool = True) -> str:
|
||||
"""Runs the full pipeline for a single product. Returns the resulting
|
||||
`data_status` ('verified' | 'partial' | 'unavailable')."""
|
||||
if skip_if_verified:
|
||||
existing = nutrition_db.get_nutrition_facts(brand, image_id)
|
||||
if existing and existing.get("data_status") == "verified":
|
||||
return "verified"
|
||||
|
||||
facts = nutrition_data_service.fetch_verified_nutrition(brand, product_name, category or "")
|
||||
facts["brand"] = brand
|
||||
facts["image_id"] = image_id
|
||||
facts["product_name"] = product_name
|
||||
facts["category"] = category
|
||||
nutrition_db.upsert_nutrition_facts(facts)
|
||||
|
||||
scores = nutrition_scoring.compute_scores(facts)
|
||||
positive = nutrition_scoring.generate_positive_insights(facts)
|
||||
cautions = nutrition_scoring.generate_cautions(facts)
|
||||
diet_tags = nutrition_scoring.classify_diet_tags(facts)
|
||||
allergens = nutrition_scoring.normalize_allergens(facts)
|
||||
|
||||
ai_summary = ""
|
||||
if generate_narrative and (positive or cautions):
|
||||
try:
|
||||
ai_summary = generate_summary(product_name, positive, cautions)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(f"Narrative generation failed for {brand}/{image_id}: {e}")
|
||||
|
||||
insights: Dict[str, Any] = {
|
||||
"brand": brand, "image_id": image_id,
|
||||
"positive_insights": positive, "nutritional_cautions": cautions,
|
||||
"ai_summary": ai_summary, "diet_tags": diet_tags, "allergens": allergens,
|
||||
"allergen_source": "openfoodfacts" if allergens else "unavailable",
|
||||
"data_status": facts["data_status"],
|
||||
}
|
||||
if scores:
|
||||
insights.update(scores)
|
||||
nutrition_db.upsert_nutrition_insights(insights)
|
||||
|
||||
return facts["data_status"]
|
||||
|
||||
|
||||
def enrich_all_products(
|
||||
skip_if_verified: bool = True,
|
||||
generate_narrative: bool = True,
|
||||
progress_cb: Optional[Callable[[int, int], None]] = None,
|
||||
max_products: Optional[int] = None,
|
||||
) -> EnrichmentResult:
|
||||
"""Iterates every product across every brand table (via
|
||||
`vector_store.list_available_brands` / `get_products_by_brand` -
|
||||
the exact same source of truth the catalog UI reads from) and runs
|
||||
the pipeline on each."""
|
||||
start = time.time()
|
||||
result = EnrichmentResult()
|
||||
|
||||
brands = list_available_brands()
|
||||
all_products: List[Dict[str, Any]] = []
|
||||
for brand in brands:
|
||||
for p in get_products_by_brand(brand):
|
||||
all_products.append({
|
||||
"brand": brand, "image_id": p.get("image_id"),
|
||||
"product_name": p.get("title") or p.get("product_name"),
|
||||
"category": p.get("category"),
|
||||
})
|
||||
if max_products:
|
||||
all_products = all_products[:max_products]
|
||||
|
||||
result.total_products = len(all_products)
|
||||
for i, p in enumerate(all_products):
|
||||
if not p["image_id"] or not p["product_name"]:
|
||||
result.errors.append(f"skipped {p['brand']}: missing image_id/product_name")
|
||||
continue
|
||||
try:
|
||||
status = enrich_one_product(
|
||||
p["brand"], p["image_id"], p["product_name"], p["category"],
|
||||
skip_if_verified=skip_if_verified, generate_narrative=generate_narrative,
|
||||
)
|
||||
if status == "verified":
|
||||
result.verified += 1
|
||||
elif status == "partial":
|
||||
result.partial += 1
|
||||
else:
|
||||
result.unavailable += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"Enrichment failed for {p['brand']}/{p['image_id']}: {e}")
|
||||
result.errors.append(f"{p['brand']}/{p['image_id']}: {e}")
|
||||
if progress_cb:
|
||||
progress_cb(i + 1, result.total_products)
|
||||
|
||||
result.duration_seconds = round(time.time() - start, 1)
|
||||
logger.info(
|
||||
f"Nutrition enrichment complete: {result.verified} verified, {result.partial} partial, "
|
||||
f"{result.unavailable} unavailable of {result.total_products} products in {result.duration_seconds}s"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def train_all_models() -> Dict[str, Any]:
|
||||
"""Trains the similarity + clustering models over whatever is
|
||||
currently verified/partial in `nutrition_facts`, and refreshes the
|
||||
cached similar-products table for every enriched product."""
|
||||
from app.intelligence import nutrition_clustering, nutrition_similarity
|
||||
|
||||
df = nutrition_db.get_all_nutrition_facts_df()
|
||||
similarity_result = nutrition_similarity.train_similarity_index(df)
|
||||
clustering_result = nutrition_clustering.train_clusters(df)
|
||||
|
||||
# Refresh cluster labels on nutrition_insights + the similar-products cache.
|
||||
assignments = nutrition_clustering.get_assignments()
|
||||
for (brand, image_id), info in assignments.items():
|
||||
existing = nutrition_db.get_nutrition_insights(brand, image_id) or {"brand": brand, "image_id": image_id}
|
||||
existing["nutrition_cluster"] = info["cluster"]
|
||||
existing["nutrition_cluster_label"] = info["label"]
|
||||
nutrition_db.upsert_nutrition_insights(existing)
|
||||
|
||||
if similarity_result.get("trained"):
|
||||
for brand, image_id in df[["brand", "image_id"]].itertuples(index=False):
|
||||
similar = nutrition_similarity.find_similar(brand, image_id, top_k=5)
|
||||
if similar:
|
||||
nutrition_db.save_similar_products(brand, image_id, similar)
|
||||
|
||||
return {"similarity": similarity_result, "clustering": clustering_result}
|
||||
52
app/services/nutrition_narrative_service.py
Normal file
52
app/services/nutrition_narrative_service.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Feature 2: "AI Nutritional Insights" narrative generation.
|
||||
|
||||
By the time anything reaches this file, every fact has already been
|
||||
verified (`nutrition_data_service.py`) and every insight/caution has
|
||||
already been computed by pure rules over those verified numbers
|
||||
(`nutrition_scoring.py`). The LLM's ONLY job here is to turn an existing
|
||||
bullet list into a smoother paragraph - it is explicitly instructed not
|
||||
to add a single new nutrient, number, or claim, and if Ollama is
|
||||
unreachable this degrades to a plain join of the bullets (still fully
|
||||
factual, just less polished prose) rather than failing silently.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from app.services.ollama_service import generate_grounded_text
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are a nutrition-label copywriter. You will be given a list of FACTS "
|
||||
"(already verified, already computed - not yours to question or extend) "
|
||||
"about one packaged food product. Rewrite them as a short, natural, "
|
||||
"factual paragraph (2-4 sentences).\n"
|
||||
"Rules:\n"
|
||||
"1. Use ONLY the facts given. Do not add any nutrient, number, ingredient, "
|
||||
"or claim that isn't in the list.\n"
|
||||
"2. Do not make medical, disease-related, or health-outcome claims "
|
||||
"(e.g. never say something 'prevents', 'cures', 'is bad for', or "
|
||||
"'reduces risk of' any condition). Describe composition only.\n"
|
||||
"3. Do not use superlatives not supported by the facts (no 'amazing', "
|
||||
"'perfect', 'best').\n"
|
||||
"4. Keep it plain and factual, like a nutrition label summary, not "
|
||||
"marketing copy.\n"
|
||||
"5. Return ONLY the paragraph, no preamble, no bullet points, no headers."
|
||||
)
|
||||
|
||||
|
||||
def generate_summary(product_name: str, positive_insights: List[str], cautions: List[str]) -> str:
|
||||
if not positive_insights and not cautions:
|
||||
return "Nutrition data for this product is limited, so a detailed summary isn't available yet."
|
||||
|
||||
facts_block = "POSITIVE POINTS:\n" + "\n".join(f"- {p}" for p in positive_insights) if positive_insights else "POSITIVE POINTS: none verified"
|
||||
cautions_block = "\nCAUTIONS:\n" + "\n".join(f"- {c}" for c in cautions) if cautions else "\nCAUTIONS: none"
|
||||
user_prompt = f"Product: {product_name}\n\n{facts_block}{cautions_block}\n\nWrite the paragraph now."
|
||||
|
||||
text = generate_grounded_text(_SYSTEM_PROMPT, user_prompt).strip()
|
||||
if text:
|
||||
return text
|
||||
|
||||
# Non-LLM fallback: still fully factual, just a plain join.
|
||||
parts = list(positive_insights) + list(cautions)
|
||||
return " ".join(parts) if parts else "Nutrition data for this product is limited, so a detailed summary isn't available yet."
|
||||
347
app/services/nutrition_scoring.py
Normal file
347
app/services/nutrition_scoring.py
Normal file
@@ -0,0 +1,347 @@
|
||||
"""
|
||||
Features 2, 3, 5, 6: nutrition/health scoring, diet-compatibility
|
||||
classification, allergen normalization, and factual insights/cautions.
|
||||
|
||||
Everything in this file is a pure function over already-verified numbers
|
||||
from `nutrition_data_service.py` - no network calls, no LLM calls, no
|
||||
randomness. That's a deliberate reading of two requirements together:
|
||||
|
||||
Feature 6 says the scoring algorithm "should be transparent and
|
||||
configurable" - a plain weighted formula over published thresholds
|
||||
satisfies that; a trained model would not (you can't hand someone a
|
||||
model's weights and call it "transparent").
|
||||
|
||||
Feature 14 says "Do NOT use Machine Learning to generate nutritional
|
||||
facts" and Feature 15 says cautions must not be "misleading or
|
||||
unsupported medical claims". Keeping cautions/insights as rule-based
|
||||
statements directly over the verified numbers (e.g. "Contains high
|
||||
added sugar" when added_sugar_g > 22.5) means every sentence traces to
|
||||
a specific verified value and a specific published threshold - there
|
||||
is nothing here for an LLM to hallucinate, because the LLM never
|
||||
touches the numbers (see `nutrition_narrative_service.py`, which only
|
||||
rephrases the bullet lists this module already produced).
|
||||
|
||||
THRESHOLDS
|
||||
----------
|
||||
Sugar/fat/saturated-fat/salt "high"/"low" cut-offs below are the UK Food
|
||||
Standards Agency's published front-of-pack traffic-light thresholds
|
||||
(per 100g solid food) - a long-standing, publicly documented labeling
|
||||
convention, not a value we invented. Fiber/protein "source"/"high"
|
||||
thresholds are the EU nutrition-claim thresholds (Regulation
|
||||
1924/2006). Micronutrient "good source" thresholds use 15% of a
|
||||
published Reference Daily Intake (RDI) per 100g, also a standard
|
||||
labeling convention. All of it is configurable in `THRESHOLDS` /
|
||||
`SCORE_WEIGHTS` below rather than hard-coded inline, per the "should be
|
||||
... configurable" requirement.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
SCORING_VERSION = "v1.0-fsa-eu-thresholds"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configurable thresholds (per 100g unless noted)
|
||||
# ---------------------------------------------------------------------------
|
||||
THRESHOLDS = {
|
||||
"sugar_high": 22.5, "sugar_low": 5.0, # g
|
||||
"sat_fat_high": 5.0, "sat_fat_low": 1.5, # g
|
||||
"fat_high": 17.5, "fat_low": 3.0, # g
|
||||
"sodium_high": 600.0, "sodium_low": 120.0, # mg
|
||||
"fiber_source": 3.0, "fiber_high": 6.0, # g
|
||||
"protein_source_g": 5.0, "protein_high_g": 10.0, # g (fallback when calorie ratio unavailable)
|
||||
"protein_high_energy_ratio": 0.20, # protein kcal / total kcal
|
||||
"calorie_dense": 400.0, "calorie_light": 120.0, # kcal
|
||||
"keto_carb_ceiling": 10.0, # g net carbs / 100g
|
||||
"low_fat_ceiling": 3.0, # g
|
||||
}
|
||||
|
||||
# RDI values used ONLY to phrase "good source of X" (>=15% RDI/100g) -
|
||||
# published nutrition-labeling reference values, not medical claims.
|
||||
RDI = {
|
||||
"calcium_mg": 800.0, "iron_mg": 14.0, "vitamin_c_mg": 80.0,
|
||||
"vitamin_d_mcg": 5.0, "vitamin_a_mcg": 800.0, "potassium_mg": 2000.0,
|
||||
"zinc_mg": 10.0, "magnesium_mg": 375.0,
|
||||
}
|
||||
GOOD_SOURCE_RDI_PCT = 0.15
|
||||
|
||||
SCORE_WEIGHTS = {
|
||||
"protein": 0.20, "fiber": 0.20, "sugar": 0.25,
|
||||
"sodium": 0.15, "fat_quality": 0.15, "micronutrient": 0.05,
|
||||
}
|
||||
WHOLE_GRAIN_BONUS = 5.0
|
||||
CALORIE_DENSITY_PENALTY_MAX = 10.0
|
||||
|
||||
|
||||
def _clip(x: float, lo: float = 0.0, hi: float = 100.0) -> float:
|
||||
return max(lo, min(hi, x))
|
||||
|
||||
|
||||
def _linear_score(value: float, good_at: float, bad_at: float) -> float:
|
||||
"""100 at/beyond `good_at`, 0 at/beyond `bad_at`, linear between.
|
||||
Works whether `good_at` is above or below `bad_at` (i.e. handles
|
||||
both 'more is better' and 'less is better' nutrients)."""
|
||||
if good_at == bad_at:
|
||||
return 100.0
|
||||
frac = (value - bad_at) / (good_at - bad_at)
|
||||
return _clip(frac * 100)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compute_scores(facts: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Returns None if there isn't enough verified data to score fairly
|
||||
(never fabricates a score from missing numbers). Otherwise returns
|
||||
`{nutrition_score, health_score, score_breakdown}`."""
|
||||
if facts.get("data_status") in (None, "unavailable"):
|
||||
return None
|
||||
|
||||
protein = facts.get("protein_g")
|
||||
fiber = facts.get("dietary_fiber_g")
|
||||
sugar = facts.get("total_sugar_g")
|
||||
sodium = facts.get("sodium_mg")
|
||||
sat_fat = facts.get("saturated_fat_g")
|
||||
total_fat = facts.get("total_fat_g")
|
||||
trans_fat = facts.get("trans_fat_g")
|
||||
calories = facts.get("calories_kcal")
|
||||
|
||||
components: Dict[str, float] = {}
|
||||
weights_used: Dict[str, float] = {}
|
||||
|
||||
if protein is not None:
|
||||
if calories and calories > 0:
|
||||
protein_ratio = (protein * 4) / calories
|
||||
components["protein_score"] = _linear_score(protein_ratio, THRESHOLDS["protein_high_energy_ratio"], 0.0)
|
||||
else:
|
||||
components["protein_score"] = _linear_score(protein, THRESHOLDS["protein_high_g"], 0.0)
|
||||
weights_used["protein"] = SCORE_WEIGHTS["protein"]
|
||||
|
||||
if fiber is not None:
|
||||
components["fiber_score"] = _linear_score(fiber, THRESHOLDS["fiber_high"], 0.0)
|
||||
weights_used["fiber"] = SCORE_WEIGHTS["fiber"]
|
||||
|
||||
if sugar is not None:
|
||||
components["sugar_score"] = _linear_score(sugar, THRESHOLDS["sugar_low"], THRESHOLDS["sugar_high"])
|
||||
weights_used["sugar"] = SCORE_WEIGHTS["sugar"]
|
||||
|
||||
if sodium is not None:
|
||||
components["sodium_score"] = _linear_score(sodium, THRESHOLDS["sodium_low"], THRESHOLDS["sodium_high"])
|
||||
weights_used["sodium"] = SCORE_WEIGHTS["sodium"]
|
||||
|
||||
if sat_fat is not None and total_fat is not None and total_fat > 0:
|
||||
sat_ratio_score = _linear_score(sat_fat, THRESHOLDS["sat_fat_low"], THRESHOLDS["sat_fat_high"])
|
||||
trans_penalty = 15.0 if (trans_fat or 0) > 0.5 else 0.0
|
||||
components["fat_quality_score"] = _clip(sat_ratio_score - trans_penalty)
|
||||
weights_used["fat_quality"] = SCORE_WEIGHTS["fat_quality"]
|
||||
elif sat_fat is not None:
|
||||
components["fat_quality_score"] = _linear_score(sat_fat, THRESHOLDS["sat_fat_low"], THRESHOLDS["sat_fat_high"])
|
||||
weights_used["fat_quality"] = SCORE_WEIGHTS["fat_quality"]
|
||||
|
||||
micro_pcts = []
|
||||
for col, rdi in RDI.items():
|
||||
v = facts.get(col)
|
||||
if v is not None and rdi:
|
||||
micro_pcts.append(_clip((v / rdi) * 100))
|
||||
if micro_pcts:
|
||||
components["micronutrient_score"] = sum(micro_pcts) / len(micro_pcts)
|
||||
weights_used["micronutrient"] = SCORE_WEIGHTS["micronutrient"]
|
||||
|
||||
if not components:
|
||||
return None # no scoreable verified fields at all
|
||||
|
||||
total_weight = sum(weights_used.values())
|
||||
nutrition_score = sum(components[f"{k}_score"] * w for k, w in weights_used.items()) / total_weight
|
||||
|
||||
ingredients = (facts.get("ingredients_text") or "").lower()
|
||||
whole_grain_bonus = 0.0
|
||||
if ingredients and ("whole grain" in ingredients or "whole wheat" in ingredients or "wholemeal" in ingredients):
|
||||
# Only awarded when the ingredient is verified to appear FIRST
|
||||
# (i.e. the primary ingredient), the standard "whole grain
|
||||
# product" convention - not just present in trace amount.
|
||||
first_ingredient = ingredients.split(",")[0]
|
||||
if "whole" in first_ingredient:
|
||||
whole_grain_bonus = WHOLE_GRAIN_BONUS
|
||||
|
||||
nutrition_score = _clip(nutrition_score + whole_grain_bonus)
|
||||
|
||||
calorie_penalty = 0.0
|
||||
if calories is not None and calories > THRESHOLDS["calorie_dense"]:
|
||||
over = min(calories - THRESHOLDS["calorie_dense"], THRESHOLDS["calorie_dense"])
|
||||
calorie_penalty = (over / THRESHOLDS["calorie_dense"]) * CALORIE_DENSITY_PENALTY_MAX
|
||||
health_score = _clip(nutrition_score - calorie_penalty)
|
||||
|
||||
breakdown = {k: round(v, 1) for k, v in components.items()}
|
||||
breakdown["whole_grain_bonus"] = whole_grain_bonus
|
||||
breakdown["calorie_density_penalty"] = round(calorie_penalty, 1)
|
||||
breakdown["fields_used"] = sorted(weights_used.keys())
|
||||
breakdown["fields_missing"] = sorted(set(SCORE_WEIGHTS.keys()) - set(weights_used.keys()))
|
||||
|
||||
return {
|
||||
"nutrition_score": round(nutrition_score, 1),
|
||||
"health_score": round(health_score, 1),
|
||||
"score_breakdown": breakdown,
|
||||
"scoring_version": SCORING_VERSION,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Diet compatibility (Feature 5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def classify_diet_tags(facts: Dict[str, Any]) -> List[str]:
|
||||
"""Blends Open Food Facts' own ingredient-derived classification
|
||||
(vegan/vegetarian - computed by OFF from the ingredient list, a
|
||||
verified/algorithmic source, not our guess) with rule-based tags
|
||||
computed directly from verified nutrient numbers."""
|
||||
if facts.get("data_status") in (None, "unavailable"):
|
||||
return []
|
||||
|
||||
tags: List[str] = []
|
||||
labels = set(facts.get("off_labels_tags") or [])
|
||||
analysis = set(facts.get("off_ingredients_analysis_tags") or [])
|
||||
|
||||
if "en:vegan" in labels or "en:vegan-status-vegan" in analysis:
|
||||
tags.append("Vegan")
|
||||
if "en:vegetarian" in labels or "en:vegetarian-status-vegetarian" in analysis or "Vegan" in tags:
|
||||
tags.append("Vegetarian")
|
||||
if "en:gluten-free" in labels:
|
||||
tags.append("Gluten Free")
|
||||
if "en:dairy-free" in labels or "Vegan" in tags:
|
||||
tags.append("Dairy Free")
|
||||
if "en:organic" in labels:
|
||||
tags.append("Organic")
|
||||
|
||||
protein = facts.get("protein_g")
|
||||
calories = facts.get("calories_kcal")
|
||||
if protein is not None:
|
||||
if calories and calories > 0 and (protein * 4) / calories >= THRESHOLDS["protein_high_energy_ratio"]:
|
||||
tags.append("High Protein")
|
||||
elif protein >= THRESHOLDS["protein_high_g"]:
|
||||
tags.append("High Protein")
|
||||
|
||||
fiber = facts.get("dietary_fiber_g")
|
||||
if fiber is not None and fiber >= THRESHOLDS["fiber_high"]:
|
||||
tags.append("High Fiber")
|
||||
|
||||
total_fat = facts.get("total_fat_g")
|
||||
if total_fat is not None and total_fat <= THRESHOLDS["low_fat_ceiling"]:
|
||||
tags.append("Low Fat")
|
||||
|
||||
sugar = facts.get("total_sugar_g")
|
||||
if sugar is not None and sugar <= THRESHOLDS["sugar_low"]:
|
||||
tags.append("Low Sugar")
|
||||
|
||||
sodium = facts.get("sodium_mg")
|
||||
if sodium is not None and sodium <= THRESHOLDS["sodium_low"]:
|
||||
tags.append("Low Sodium")
|
||||
|
||||
carbs = facts.get("carbohydrates_g")
|
||||
if carbs is not None and fiber is not None:
|
||||
net_carbs = max(carbs - fiber, 0)
|
||||
if net_carbs <= THRESHOLDS["keto_carb_ceiling"] and (total_fat or 0) > 0:
|
||||
tags.append("Keto Friendly")
|
||||
|
||||
ingredients = (facts.get("ingredients_text") or "").lower()
|
||||
if ingredients and ingredients.split(",")[0].find("whole") != -1:
|
||||
tags.append("Whole Grain")
|
||||
|
||||
return sorted(set(tags))
|
||||
|
||||
|
||||
def normalize_allergens(facts: Dict[str, Any]) -> List[str]:
|
||||
seen = {}
|
||||
for a in facts.get("allergens") or []:
|
||||
key = a.strip().lower()
|
||||
if key and key not in seen:
|
||||
seen[key] = a.strip().title()
|
||||
return sorted(seen.values())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factual insights / cautions (Feature 2 & 3) - rule-based only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_positive_insights(facts: Dict[str, Any]) -> List[str]:
|
||||
if facts.get("data_status") in (None, "unavailable"):
|
||||
return []
|
||||
out: List[str] = []
|
||||
|
||||
protein = facts.get("protein_g")
|
||||
if protein is not None and protein >= THRESHOLDS["protein_source_g"]:
|
||||
out.append(f"Good source of protein ({protein:.1f} g per 100 g).")
|
||||
|
||||
fiber = facts.get("dietary_fiber_g")
|
||||
if fiber is not None:
|
||||
if fiber >= THRESHOLDS["fiber_high"]:
|
||||
out.append(f"High in dietary fiber ({fiber:.1f} g per 100 g), which may support digestive health.")
|
||||
elif fiber >= THRESHOLDS["fiber_source"]:
|
||||
out.append(f"Contains a source of dietary fiber ({fiber:.1f} g per 100 g).")
|
||||
|
||||
sat_fat = facts.get("saturated_fat_g")
|
||||
if sat_fat is not None and sat_fat <= THRESHOLDS["sat_fat_low"]:
|
||||
out.append(f"Low in saturated fat ({sat_fat:.1f} g per 100 g).")
|
||||
|
||||
sugar = facts.get("total_sugar_g")
|
||||
if sugar is not None and sugar <= THRESHOLDS["sugar_low"]:
|
||||
out.append(f"Low in sugar ({sugar:.1f} g per 100 g).")
|
||||
|
||||
sodium = facts.get("sodium_mg")
|
||||
if sodium is not None and sodium <= THRESHOLDS["sodium_low"]:
|
||||
out.append(f"Low in sodium ({sodium:.0f} mg per 100 g).")
|
||||
|
||||
micronutrient_labels = {
|
||||
"calcium_mg": "calcium", "iron_mg": "iron", "vitamin_c_mg": "vitamin C",
|
||||
"vitamin_d_mcg": "vitamin D", "vitamin_a_mcg": "vitamin A",
|
||||
"potassium_mg": "potassium", "zinc_mg": "zinc", "magnesium_mg": "magnesium",
|
||||
}
|
||||
for col, label in micronutrient_labels.items():
|
||||
v = facts.get(col)
|
||||
rdi = RDI.get(col)
|
||||
if v is not None and rdi and v >= rdi * GOOD_SOURCE_RDI_PCT:
|
||||
out.append(f"Contains {label}, a source of at least {int(GOOD_SOURCE_RDI_PCT * 100)}% of the reference daily intake per 100 g.")
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def generate_cautions(facts: Dict[str, Any]) -> List[str]:
|
||||
if facts.get("data_status") in (None, "unavailable"):
|
||||
return []
|
||||
out: List[str] = []
|
||||
|
||||
added_sugar = facts.get("added_sugar_g")
|
||||
total_sugar = facts.get("total_sugar_g")
|
||||
sugar_for_caution = added_sugar if added_sugar is not None else total_sugar
|
||||
sugar_label = "added sugar" if added_sugar is not None else "sugar"
|
||||
if sugar_for_caution is not None and sugar_for_caution > THRESHOLDS["sugar_high"]:
|
||||
out.append(f"Contains high {sugar_label} ({sugar_for_caution:.1f} g per 100 g).")
|
||||
|
||||
sodium = facts.get("sodium_mg")
|
||||
if sodium is not None and sodium > THRESHOLDS["sodium_high"]:
|
||||
out.append(f"Contains high sodium ({sodium:.0f} mg per 100 g).")
|
||||
|
||||
sat_fat = facts.get("saturated_fat_g")
|
||||
if sat_fat is not None and sat_fat > THRESHOLDS["sat_fat_high"]:
|
||||
out.append(f"High in saturated fat ({sat_fat:.1f} g per 100 g).")
|
||||
|
||||
trans_fat = facts.get("trans_fat_g")
|
||||
if trans_fat is not None and trans_fat > 0:
|
||||
out.append(f"Contains trans fat ({trans_fat:.2f} g per 100 g).")
|
||||
|
||||
fiber = facts.get("dietary_fiber_g")
|
||||
if fiber is not None and fiber < 1.0:
|
||||
out.append("Low in dietary fiber.")
|
||||
|
||||
calories = facts.get("calories_kcal")
|
||||
if calories is not None and calories > THRESHOLDS["calorie_dense"]:
|
||||
out.append(f"High calorie density ({calories:.0f} kcal per 100 g).")
|
||||
|
||||
ingredients = (facts.get("ingredients_text") or "").lower()
|
||||
if ingredients:
|
||||
if any(term in ingredients for term in ("preservative", "sodium benzoate", "potassium sorbate", "sulphite", "sulfite")):
|
||||
out.append("Contains preservatives (per ingredient list).")
|
||||
if any(term in ingredients for term in ("colour", "color", "tartrazine", "sunset yellow", "carmoisine", "e102", "e110", "e122", "e129", "e133", "e150")):
|
||||
out.append("Contains added colors (per ingredient list).")
|
||||
|
||||
return out
|
||||
375
app/services/ollama_service.py
Normal file
375
app/services/ollama_service.py
Normal file
@@ -0,0 +1,375 @@
|
||||
from typing import List, Dict, Any, Optional
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
|
||||
from app.infrastructure.settings import OLLAMA_BASE_URL, OLLAMA_MODEL_NAME, USE_OLLAMA, OLLAMA_TIMEOUT_SECONDS
|
||||
from app.services.brand_registry import get_known_sub_brands
|
||||
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a product catalog assistant. Given a brand name, return a JSON object with "
|
||||
"key 'products' (array). Each product has: "
|
||||
"title, category, description (<=160 chars), size_variants (array like ['100g','200g']), "
|
||||
"price_ranges (object like {'100g':'₹10','200g':'₹18'}), "
|
||||
"price_range (string like '₹10-20'), "
|
||||
"provider_examples (array like ['Amazon','Flipkart']). "
|
||||
"Return ONLY valid JSON with no extra text."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_client():
|
||||
if not USE_OLLAMA:
|
||||
return None
|
||||
# Verify Ollama is reachable
|
||||
try:
|
||||
resp = requests.get(f"{OLLAMA_BASE_URL}/api/tags", timeout=5)
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _generate(system: str, user_prompt: str, max_retries: int = 2) -> str:
|
||||
"""Call Ollama's chat endpoint and return text safely.
|
||||
|
||||
Retries up to `max_retries` times when the response is empty, since
|
||||
small local models (e.g. qwen2.5:1.5b) sometimes return empty content
|
||||
for complex JSON prompts on the first attempt.
|
||||
"""
|
||||
if not _ensure_client():
|
||||
return ""
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{OLLAMA_BASE_URL}/api/chat",
|
||||
json={
|
||||
"model": OLLAMA_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.6},
|
||||
},
|
||||
timeout=OLLAMA_TIMEOUT_SECONDS,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
content = (data.get("message", {}).get("content", "") or "").strip()
|
||||
if content:
|
||||
return content
|
||||
if attempt < max_retries:
|
||||
import time
|
||||
time.sleep(1.0)
|
||||
except Exception:
|
||||
if attempt >= max_retries:
|
||||
return ""
|
||||
import time
|
||||
time.sleep(1.0)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_json(text: str) -> dict | None:
|
||||
"""Extract JSON from model response, trying multiple strategies.
|
||||
|
||||
Handles both JSON objects {...} and JSON arrays [...] since small
|
||||
local models frequently return bare arrays instead of an object with
|
||||
a ``products`` key.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
# Try fenced code block (object or array)
|
||||
match = re.search(r"```(?:json)?\s*(\{[\s\S]*?\}|\[[\s\S]*?\])\s*```", text)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(1))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try first JSON value in text (greedy - object)
|
||||
brace = re.search(r"\{[\s\S]*\}", text)
|
||||
if brace:
|
||||
try:
|
||||
return json.loads(brace.group(0))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try first JSON value in text (greedy - array)
|
||||
bracket = re.search(r"\[[\s\S]*\]", text)
|
||||
if bracket:
|
||||
try:
|
||||
return json.loads(bracket.group(0))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try parsing entire text
|
||||
try:
|
||||
return json.loads(text.strip())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Fallback: try to fix common issues
|
||||
cleaned = text.strip()
|
||||
cleaned = re.sub(r"(?<=[:,\[])\s*'", '"', cleaned)
|
||||
cleaned = re.sub(r"'\s*(?=[,:\}\]])", '"', cleaned)
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
RAG_SYSTEM_PROMPT = (
|
||||
"You are a helpful shopping assistant for an Indian FMCG product catalog. "
|
||||
"Answer the user's question using ONLY the product information given in the "
|
||||
"CONTEXT section below. Each context item is a real product from the catalog, "
|
||||
"already filtered to match the product type the user asked about. "
|
||||
"Rules:\n"
|
||||
"1. Base your answer strictly on the CONTEXT - do not invent products, prices, or facts.\n"
|
||||
"2. If the CONTEXT doesn't contain enough information to answer, say so plainly and "
|
||||
"suggest the user rephrase or pick a different brand.\n"
|
||||
"3. When you mention a product, use its exact title from the CONTEXT.\n"
|
||||
"4. Keep the answer concise (3-6 sentences unless a list is clearly more useful).\n"
|
||||
"5. Mention prices/sizes when they are present in the CONTEXT and relevant to the question.\n"
|
||||
"6. CATEGORY DISCIPLINE: only recommend a product if it genuinely is the type of product the "
|
||||
"user asked for. Never recommend a different product type as a substitute (e.g. do not "
|
||||
"recommend chips/namkeen/snacks when the user asked for biscuits, or vice versa) - if the "
|
||||
"CONTEXT contains no products of the exact type requested, say plainly that none were found "
|
||||
"in the catalog instead of listing unrelated items.\n"
|
||||
"7. ATTRIBUTE HONESTY: if the user asked for a specific attribute (e.g. low sugar, gluten-free, "
|
||||
"low fat, spicy), only claim a product has that attribute if the CONTEXT text for that product "
|
||||
"actually says so. If none of the retrieved products explicitly confirm the attribute, say you "
|
||||
"couldn't confirm it from the catalog data rather than assuming or guessing.\n"
|
||||
"8. COUNT AND METRICS: If CATALOG METRIC DATA is present in the CONTEXT, state those exact count numbers clearly when answering count or list questions.\n"
|
||||
"Do not mention the word 'CONTEXT' or that you were given a context block; just answer naturally."
|
||||
)
|
||||
|
||||
|
||||
def _build_grounded_fallback_answer(query: str, context_block: str, requested_category: Optional[str] = None) -> str:
|
||||
"""Intelligent non-LLM synthesis of retrieved catalog items when Ollama is offline."""
|
||||
if not context_block or "(No matching products were found" in context_block:
|
||||
cat_str = f" in category **{requested_category}**" if requested_category else ""
|
||||
return f"No matching products were found in the catalog database for your query{cat_str}."
|
||||
|
||||
cat_str = f" ({requested_category})" if requested_category else ""
|
||||
lines = [f"Here are the top catalog recommendations for **'{query}'**{cat_str}:\n"]
|
||||
|
||||
raw_items = [b.strip() for b in context_block.split("\n\n") if b.strip()]
|
||||
for item in raw_items:
|
||||
if item.startswith("CATALOG METRIC DATA:"):
|
||||
lines.append(f"📊 **Catalog Info**: {item.replace('CATALOG METRIC DATA:', '').strip()}\n")
|
||||
else:
|
||||
lines.append(item)
|
||||
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
def generate_rag_answer(
|
||||
query: str,
|
||||
context_block: str,
|
||||
history: Optional[List[Dict[str, str]]] = None,
|
||||
requested_category: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Generate a grounded answer for the RAG chat endpoint."""
|
||||
if not _ensure_client():
|
||||
return _build_grounded_fallback_answer(query, context_block, requested_category)
|
||||
|
||||
history_block = ""
|
||||
if history:
|
||||
turns = []
|
||||
for turn in history[-6:]: # keep prompts small on CPU-only hardware
|
||||
role = turn.get("role", "user")
|
||||
content = (turn.get("content") or "").strip()
|
||||
if content:
|
||||
turns.append(f"{role.upper()}: {content}")
|
||||
if turns:
|
||||
history_block = "PREVIOUS CONVERSATION:\n" + "\n".join(turns) + "\n\n"
|
||||
|
||||
category_block = (
|
||||
f"REQUESTED PRODUCT TYPE: {requested_category}\n\n" if requested_category else ""
|
||||
)
|
||||
user_prompt = (
|
||||
f"{history_block}"
|
||||
f"{category_block}"
|
||||
f"CONTEXT (retrieved catalog products, already filtered to the requested product type "
|
||||
f"when one was detected):\n{context_block}\n\n"
|
||||
f"QUESTION: {query}\n\n"
|
||||
"Answer the question now, following the rules you were given."
|
||||
)
|
||||
|
||||
answer = _generate(RAG_SYSTEM_PROMPT, user_prompt, max_retries=2)
|
||||
if not answer:
|
||||
return _build_grounded_fallback_answer(query, context_block, requested_category)
|
||||
return answer.strip()
|
||||
|
||||
|
||||
def generate_grounded_text(system_prompt: str, user_prompt: str) -> str:
|
||||
"""Thin public wrapper around the private `_generate` helper, for
|
||||
callers outside this module that need a plain grounded text
|
||||
completion rather than the JSON-extraction flows below (e.g.
|
||||
`nutrition_narrative_service.py`, which passes its own strict
|
||||
"only rephrase these given facts" system prompt). Returns "" if
|
||||
Ollama isn't reachable, exactly like every other function here -
|
||||
callers are expected to have their own non-LLM fallback."""
|
||||
return _generate(system_prompt, user_prompt, max_retries=1)
|
||||
|
||||
|
||||
def get_categories_for_brand(brand: str) -> list[str]:
|
||||
"""Ask Ollama to enumerate product categories for a brand."""
|
||||
if not _ensure_client():
|
||||
return []
|
||||
known = get_known_sub_brands(brand)
|
||||
known_context = f" Known products under this brand include: {', '.join(known)}." if known else ""
|
||||
system = "You are a category enumerator. Given a brand name, return a JSON object with key 'categories' (array of strings). List all logical product categories for the brand."
|
||||
user_prompt = (
|
||||
f"Brand: {brand}\n"
|
||||
f"Return JSON only with no extra text.{known_context}"
|
||||
)
|
||||
text = _generate(system, user_prompt)
|
||||
if not text:
|
||||
return []
|
||||
data = _extract_json(text)
|
||||
if data is None:
|
||||
return []
|
||||
cats = data.get("categories", [])
|
||||
return [c for c in cats if isinstance(c, str) and c.strip()]
|
||||
|
||||
|
||||
def fetch_brand_catalog_exhaustive(brand: str, max_products: int = 300) -> Dict[str, Any] | None:
|
||||
"""Exhaustive Ollama-based discovery: categories first, then products per category, merging up to max_products."""
|
||||
if not _ensure_client():
|
||||
return None
|
||||
|
||||
categories = get_categories_for_brand(brand) or ["General"]
|
||||
unique_titles: set[str] = set()
|
||||
products_out: list[Dict[str, Any]] = []
|
||||
known = get_known_sub_brands(brand)
|
||||
known_context = f" Known products under this brand include: {', '.join(known)}." if known else ""
|
||||
|
||||
for category in categories:
|
||||
if len(products_out) >= max_products:
|
||||
break
|
||||
per_cat_prompt = (
|
||||
f"Brand: {brand}\nCategory: {category}\n"
|
||||
"Return a JSON object with key 'products' (array). Each product must include: "
|
||||
"title, category, optional description (<=160 chars), image_url, size_variants (array), price_ranges (object), provider_examples (array up to 5). "
|
||||
"You can also return just a bare JSON array of products if preferred. "
|
||||
f"Return at least 50 items if available; avoid duplicates within the category.{known_context}"
|
||||
)
|
||||
text = _generate(SYSTEM_PROMPT, per_cat_prompt)
|
||||
if not text:
|
||||
continue
|
||||
parsed = _extract_json(text)
|
||||
if parsed is None:
|
||||
continue
|
||||
try:
|
||||
# Handle both dict-with-products-key and bare-array responses
|
||||
raw_list = parsed if isinstance(parsed, list) else parsed.get("products", [])
|
||||
for p in raw_list:
|
||||
if not isinstance(p, dict):
|
||||
continue
|
||||
title = (p.get("title") or p.get("name") or "").strip()
|
||||
if not title or title.lower() in unique_titles:
|
||||
continue
|
||||
unique_titles.add(title.lower())
|
||||
out_item = {
|
||||
"title": title,
|
||||
"category": p.get("category") or category or "Uncategorized",
|
||||
"description": p.get("description"),
|
||||
"image_url": p.get("image_url") or p.get("imageUrl") or p.get("image"),
|
||||
"size_variants": p.get("size_variants") or [],
|
||||
"price_ranges": p.get("price_ranges") or {},
|
||||
"provider_examples": p.get("provider_examples") or [],
|
||||
}
|
||||
products_out.append(out_item)
|
||||
if len(products_out) >= max_products:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return {"brand": brand, "products": products_out}
|
||||
|
||||
|
||||
def fetch_product_details(brand: str, product_title: str) -> Dict[str, Any] | None:
|
||||
"""Get details for a single product: description, image_url, pricing fields.
|
||||
Returns a dict with keys: description, image_url, size_variants, price_ranges, price_range, provider_examples.
|
||||
"""
|
||||
if not _ensure_client():
|
||||
return None
|
||||
user_prompt = (
|
||||
f"Brand: {brand}\nProduct: {product_title}\n"
|
||||
"Return strictly JSON with keys: description, image_url?, size_variants?, price_ranges?, price_range?, provider_examples?.\n"
|
||||
"description must be <=160 chars, concise and factual."
|
||||
)
|
||||
text = _generate(SYSTEM_PROMPT, user_prompt)
|
||||
if not text:
|
||||
return None
|
||||
data = _extract_json(text)
|
||||
if data is None:
|
||||
return None
|
||||
try:
|
||||
return {
|
||||
"description": data.get("description") or "",
|
||||
"image_url": data.get("image_url") or data.get("imageUrl") or data.get("image"),
|
||||
"size_variants": data.get("size_variants") or [],
|
||||
"price_ranges": data.get("price_ranges") or {},
|
||||
"price_range": data.get("price_range"),
|
||||
"provider_examples": data.get("provider_examples") or [],
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def fetch_brand_catalog_with_gemini(brand: str) -> Dict[str, Any] | None:
|
||||
if not _ensure_client():
|
||||
print("LLM client not available (Ollama). Check config.")
|
||||
return None
|
||||
|
||||
user_prompt = (
|
||||
f"Brand: {brand}\n"
|
||||
"Return a JSON object with key 'products' (array of objects), "
|
||||
"each with: title, category, description, size_variants, price_range, provider_examples."
|
||||
)
|
||||
|
||||
try:
|
||||
print(f"Calling Ollama for brand: {brand}")
|
||||
text = _generate(SYSTEM_PROMPT, user_prompt)
|
||||
print(f"Ollama response length: {len(text)}")
|
||||
|
||||
parsed = _extract_json(text)
|
||||
if parsed is None:
|
||||
print("Failed to extract valid JSON from Ollama response")
|
||||
return {"brand": brand, "products": []}
|
||||
|
||||
# Handle both dict-with-products-key and bare-array responses
|
||||
if isinstance(parsed, list):
|
||||
raw_products = parsed
|
||||
elif isinstance(parsed, dict):
|
||||
raw_products = parsed.get("products") or parsed.get("items") or []
|
||||
else:
|
||||
raw_products = []
|
||||
|
||||
out: Dict[str, Any] = {"brand": brand, "products": []}
|
||||
for p in raw_products:
|
||||
if not isinstance(p, dict):
|
||||
continue
|
||||
title = p.get("title") or p.get("name") or p.get("product") or "Unknown"
|
||||
category = p.get("category") or "Uncategorized"
|
||||
desc = p.get("description")
|
||||
img = p.get("image_url") or p.get("imageUrl") or p.get("image")
|
||||
size_variants = p.get("size_variants") or []
|
||||
price_ranges = p.get("price_ranges") or {}
|
||||
provider_examples = p.get("provider_examples") or []
|
||||
|
||||
out["products"].append({
|
||||
"title": title,
|
||||
"category": category,
|
||||
"description": desc or "",
|
||||
"image_url": img,
|
||||
"size_variants": size_variants,
|
||||
"price_ranges": price_ranges,
|
||||
"provider_examples": provider_examples,
|
||||
"price_range": p.get("price_range")
|
||||
})
|
||||
print(f"Parsed {len(out['products'])} products from Ollama")
|
||||
return out
|
||||
except Exception as e:
|
||||
print(f"Error calling Ollama: {e}")
|
||||
return None
|
||||
|
||||
|
||||
129
app/services/playwright_image_fallback.py
Normal file
129
app/services/playwright_image_fallback.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
Last-resort image search fallback using Python Playwright (sync API),
|
||||
run as an isolated subprocess.
|
||||
|
||||
WHY THIS REPLACES THE OLD NODE.JS/CRAWLEE SCRIPTS
|
||||
---------------------------------------------------
|
||||
The previous implementation shelled out to `node simple_image_search.js`
|
||||
/ `node fallback_image_search.js`, which depended on:
|
||||
- Node.js being installed
|
||||
- `npm install` having been run inside cli/ (node_modules present)
|
||||
- `npx playwright install chromium` having been run for the Node copy
|
||||
of Playwright
|
||||
|
||||
On a fresh checkout (exactly the state of the project as delivered to the
|
||||
user), none of that npm/browser setup exists, so `catalog_engine.py`'s
|
||||
`_check_crawlee_availability()` only checks `node --version` - it returns
|
||||
True even though the actual `crawlee` package is never installed. Every
|
||||
single product then spent up to 120s + 60s (two full subprocess timeouts)
|
||||
calling Node scripts that immediately fail with "Cannot find module
|
||||
'crawlee'" and silently return `[]`. That is the single biggest reason
|
||||
"no image of product gets generated" - the pipeline was burning almost
|
||||
all of its time on a dependency that was never actually installed, before
|
||||
ever reaching a working fallback.
|
||||
|
||||
This module removes the Node/Crawlee runtime entirely. It uses Python
|
||||
Playwright (already a project dependency in requirements.txt) directly,
|
||||
run as its own OS subprocess via `python -m
|
||||
app.services.playwright_image_fallback "<query>"` so it never shares an
|
||||
asyncio event loop with Streamlit (the same greenlet/threading conflict
|
||||
this project has already run into with Crawlee inside Streamlit).
|
||||
|
||||
It only runs as the LAST resort, after the free structured APIs
|
||||
(Open Food/Beauty/Products Facts, Wikimedia Commons) and DuckDuckGo image
|
||||
search have already been tried by `image_search.py`, since launching a
|
||||
real Chromium browser is the slowest and heaviest option on an 8GB RAM,
|
||||
CPU-only machine.
|
||||
|
||||
If Playwright's Chromium browser was never installed (`playwright install
|
||||
chromium` not run), this fails fast and returns `[]` rather than hanging,
|
||||
and the caller logs a clear one-time message telling the user how to fix
|
||||
it - instead of two full minutes of silent timeouts per product.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def search_bing_images(query: str, max_results: int = 10) -> list:
|
||||
"""Render a real Bing Image Search page with headless Chromium and
|
||||
extract the embedded result metadata (`m="{...}"` JSON attribute on
|
||||
`<a class="iusc">` elements). This is far more reliable than a plain
|
||||
`requests` GET because Bing increasingly serves a JS-gated results
|
||||
shell to non-browser clients."""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
print("PLAYWRIGHT_NOT_INSTALLED", file=sys.stderr)
|
||||
return []
|
||||
|
||||
ua = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
urls: list = []
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
try:
|
||||
browser = p.chromium.launch(
|
||||
headless=True,
|
||||
args=["--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu"],
|
||||
)
|
||||
except Exception as e:
|
||||
# Most common cause: `playwright install chromium` was
|
||||
# never run. Fail fast instead of hanging.
|
||||
print(f"PLAYWRIGHT_LAUNCH_FAILED: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
try:
|
||||
context = browser.new_context(user_agent=ua, locale="en-IN")
|
||||
page = context.new_page()
|
||||
page.goto(
|
||||
f"https://www.bing.com/images/search?q={query}&form=HDRSC2",
|
||||
timeout=20000,
|
||||
wait_until="domcontentloaded",
|
||||
)
|
||||
try:
|
||||
page.wait_for_selector("a.iusc", timeout=8000)
|
||||
except Exception:
|
||||
pass # proceed with whatever loaded
|
||||
|
||||
anchors = page.query_selector_all("a.iusc")
|
||||
for a in anchors:
|
||||
raw = a.get_attribute("m")
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
meta = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
murl = meta.get("murl")
|
||||
if murl and str(murl).startswith("http") and murl not in urls:
|
||||
urls.append(murl)
|
||||
if len(urls) >= max_results:
|
||||
break
|
||||
finally:
|
||||
browser.close()
|
||||
except Exception as e:
|
||||
print(f"PLAYWRIGHT_SEARCH_ERROR: {e}", file=sys.stderr)
|
||||
return urls
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) < 2:
|
||||
print(json.dumps([]))
|
||||
return
|
||||
query = sys.argv[1]
|
||||
max_results = int(sys.argv[2]) if len(sys.argv) > 2 else 10
|
||||
try:
|
||||
urls = search_bing_images(query, max_results)
|
||||
except Exception:
|
||||
urls = []
|
||||
print(json.dumps(urls))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
355
app/services/price_estimator.py
Normal file
355
app/services/price_estimator.py
Normal file
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
Realistic Price Estimation Engine
|
||||
==================================
|
||||
Replaces the old context-free "size_in_grams * flat_rate" formula in
|
||||
catalog_engine.py with a category + brand-tier aware model that is anchored
|
||||
to real Indian retail MRP ranges (FMCG: food, personal care, household).
|
||||
|
||||
Why this exists
|
||||
----------------
|
||||
The previous `_generate_mock_price()` used a single flat ₹/gram rate per
|
||||
broad category (e.g. 0.30 for "general") with no price floor. That produces
|
||||
numbers that don't resemble real shelf prices - e.g. a 40g toothpaste came
|
||||
out to ~₹19 (real MRP ~₹35-55), and a 1kg "family pack" snack came out to
|
||||
~₹300+ because the formula scales linearly with weight with no diminishing
|
||||
returns for bulk packaging.
|
||||
|
||||
This module instead:
|
||||
1. Classifies the product into a finer-grained retail category.
|
||||
2. Uses a curated ₹/100g (or ₹/100ml) anchor band per category, derived from
|
||||
typical Indian MRPs (these are deliberately ranges, not single numbers).
|
||||
3. Applies a non-linear pack-size curve (small packs cost more per gram,
|
||||
large/family packs cost less per gram) instead of pure linear scaling.
|
||||
4. Applies a minimum floor price per category so tiny sachets/trial packs
|
||||
never come out near-zero.
|
||||
5. Applies a brand-tier multiplier (premium / mainstream / value) using a
|
||||
small curated brand list, defaulting to "mainstream" for unknown brands.
|
||||
6. Offers `reconcile_llm_price()` to sanity-clamp whatever a (small, locally
|
||||
run) LLM proposes against this anchor band, rather than trusting it
|
||||
blindly - small models are unreliable at numeric estimation, but they
|
||||
are still useful as a first pass that we now constrain instead of discard.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Category price bands: (₹ per 100g/100ml at "standard" pack size, floor ₹)
|
||||
# Ranges are deliberately wide because retail prices vary by formulation -
|
||||
# the jitter below picks a stable point inside the range per product.
|
||||
# ---------------------------------------------------------------------------
|
||||
CATEGORY_BANDS = {
|
||||
# category_key: (min_rate_per_100, max_rate_per_100, floor_price_rs)
|
||||
"biscuits_cookies": (9, 22, 5),
|
||||
"crackers": (10, 24, 5),
|
||||
"rusk": (7, 14, 5),
|
||||
"bakery_bread": (10, 18, 15),
|
||||
"cakes_muffins": (18, 38, 10),
|
||||
"chocolates": (35, 90, 10),
|
||||
"snacks_namkeen": (12, 28, 5),
|
||||
"dairy": (10, 28, 10),
|
||||
"beverages_juice": (8, 22, 10),
|
||||
"beverages_tea_coffee": (25, 70, 15),
|
||||
"breakfast_cereal": (20, 45, 25),
|
||||
"oral_care": (45, 110, 25), # toothpaste, mouthwash, toothbrush packs
|
||||
"hair_care": (35, 95, 20), # shampoo, conditioner, hair oil
|
||||
"skin_bath": (30, 85, 15), # soap, body wash, lotion
|
||||
"household_clean": (15, 40, 10),
|
||||
"baby_care": (40, 110, 25),
|
||||
"general": (15, 35, 10),
|
||||
}
|
||||
|
||||
# Heuristic keyword routing -> category key (checked in order)
|
||||
CATEGORY_KEYWORDS = [
|
||||
(["toothpaste", "toothbrush", "mouthwash", "oral care"], "oral_care"),
|
||||
(["shampoo", "conditioner", "hair oil", "hair serum"], "hair_care"),
|
||||
(["soap", "body wash", "lotion", "moisturiser", "moisturizer", "face wash", "cream"], "skin_bath"),
|
||||
(["detergent", "dishwash", "floor cleaner", "toilet cleaner", "handwash"], "household_clean"),
|
||||
(["diaper", "baby wipes", "baby food", "baby lotion"], "baby_care"),
|
||||
(["chocolate", "cocoa"], "chocolates"),
|
||||
(["biscuit", "cookie"], "biscuits_cookies"),
|
||||
(["cracker", "saltine"], "crackers"),
|
||||
(["rusk"], "rusk"),
|
||||
(["cake", "muffin", "pastry"], "cakes_muffins"),
|
||||
(["bread", "bun", "loaf"], "bakery_bread"),
|
||||
(["milk", "cheese", "butter", "paneer", "yogurt", "curd", "ghee"], "dairy"),
|
||||
(["juice", "squash", "syrup"], "beverages_juice"),
|
||||
(["tea", "coffee"], "beverages_tea_coffee"),
|
||||
(["cereal", "muesli", "oats", "cornflakes"], "breakfast_cereal"),
|
||||
(["chips", "namkeen", "snack", "wafer", "mixture"], "snacks_namkeen"),
|
||||
]
|
||||
|
||||
# Small curated brand-tier list. Unknown brands default to "mainstream" (1.0x).
|
||||
PREMIUM_BRANDS = {
|
||||
"godrej expert", "loreal", "l'oreal", "olay", "dove", "nivea", "himalaya",
|
||||
"forest essentials", "lakme", "ponds", "dabur", "patanjali ayurved",
|
||||
}
|
||||
VALUE_BRANDS = {
|
||||
"patanjali", "local", "value", "economy",
|
||||
}
|
||||
TIER_MULTIPLIER = {"premium": 1.35, "value": 0.8, "mainstream": 1.0}
|
||||
|
||||
|
||||
def classify_category(title: str, hint_category: Optional[str] = None) -> str:
|
||||
"""Map a free-text product title / existing category guess to one of our
|
||||
curated pricing-band categories."""
|
||||
text = f"{hint_category or ''} {title or ''}".lower()
|
||||
for keywords, key in CATEGORY_KEYWORDS:
|
||||
if any(k in text for k in keywords):
|
||||
return key
|
||||
return "general"
|
||||
|
||||
|
||||
def _brand_tier(brand: str) -> str:
|
||||
b = (brand or "").strip().lower()
|
||||
if b in PREMIUM_BRANDS:
|
||||
return "premium"
|
||||
if b in VALUE_BRANDS:
|
||||
return "value"
|
||||
return "mainstream"
|
||||
|
||||
|
||||
def _parse_size_to_grams(size: str) -> float:
|
||||
"""Parse a size string like '100g', '1kg', '250ml', '1L', '1l', '1.5 Litre',
|
||||
'Family Pack' into a nominal gram/ml equivalent.
|
||||
|
||||
BUGFIX (root cause of e.g. "Dettol Liquid 500ml: ₹190, 1L: ₹25"):
|
||||
the previous implementation used the regex r"\\bl\\b" to detect a bare
|
||||
"L"/"l" unit. \\b is a *word*-boundary, and regex treats digits and
|
||||
letters as the same "word" character class - so in a string like "1l"
|
||||
(digit immediately followed by "l", no space), there is NO word
|
||||
boundary between '1' and 'l' at all, and the pattern silently failed
|
||||
to match. That made "1L" parse as 1 *gram* instead of 1000 grams,
|
||||
so a 1-litre pack priced out far cheaper than a 500ml pack of the same
|
||||
product - the unit multiplier was being dropped, not just inaccurate.
|
||||
|
||||
This version extracts the unit as its own explicit token (whatever
|
||||
non-digit suffix follows the number, e.g. "l", "ltr", "litre", "kg",
|
||||
"g", "ml") instead of relying on word-boundary regexes, so "1l", "1L",
|
||||
"1 L", "1.5ltr", "2kg" etc. are all parsed correctly regardless of
|
||||
spacing.
|
||||
"""
|
||||
if not size:
|
||||
return 100.0
|
||||
s = size.strip().lower()
|
||||
|
||||
match = re.search(r"(\d+(?:\.\d+)?)\s*([a-z]*)", s)
|
||||
if not match or not match.group(1):
|
||||
fallback = {
|
||||
"family": 500, "jumbo": 400, "combo": 300, "value pack": 350,
|
||||
"pack": 100, "piece": 20, "pcs": 20, "unit": 30, "trial": 15,
|
||||
"mini": 25, "travel": 30,
|
||||
}
|
||||
for k, v in fallback.items():
|
||||
if k in s:
|
||||
return float(v)
|
||||
return 100.0
|
||||
|
||||
value = float(match.group(1))
|
||||
unit = match.group(2).strip()
|
||||
|
||||
# Litre-family units (any spacing/abbreviation): 1L = 1000ml
|
||||
litre_units = {"l", "lt", "ltr", "ltrs", "litre", "litres", "liter", "liters"}
|
||||
# Kilogram-family units: 1kg = 1000g
|
||||
kg_units = {"kg", "kgs", "kilo", "kilos", "kilogram", "kilograms"}
|
||||
# Millilitre/gram-family units (already base units) - no scaling needed.
|
||||
base_units = {"ml", "mls", "g", "gm", "gms", "gram", "grams", ""}
|
||||
|
||||
if unit in litre_units:
|
||||
value *= 1000
|
||||
elif unit in kg_units:
|
||||
value *= 1000
|
||||
elif unit in base_units:
|
||||
pass
|
||||
else:
|
||||
# Unrecognised unit token (e.g. "pcs", "pack", "x6") - leave the
|
||||
# numeric value as-is rather than guessing.
|
||||
pass
|
||||
return value
|
||||
|
||||
|
||||
def _pack_size_curve_multiplier(grams: float) -> float:
|
||||
"""Small packs cost more per gram (packaging overhead, impulse-buy
|
||||
pricing); large/family packs cost less per gram. This mirrors real
|
||||
Indian FMCG shelf pricing far better than linear scaling."""
|
||||
if grams <= 20:
|
||||
return 2.2
|
||||
if grams <= 50:
|
||||
return 1.7
|
||||
if grams <= 100:
|
||||
return 1.25
|
||||
if grams <= 200:
|
||||
return 1.05
|
||||
if grams <= 500:
|
||||
return 0.92
|
||||
if grams <= 1000:
|
||||
return 0.8
|
||||
return 0.7
|
||||
|
||||
|
||||
def _deterministic_unit(seed_key: str) -> float:
|
||||
"""Stable pseudo-random value in [0,1] derived from a seed string, so the
|
||||
same product+size always resolves to the same price across runs."""
|
||||
digest = hashlib.md5(seed_key.strip().lower().encode("utf-8")).hexdigest()
|
||||
return int(digest[:8], 16) / 0xFFFFFFFF
|
||||
|
||||
|
||||
def estimate_price(
|
||||
size: str,
|
||||
product_title: str = "",
|
||||
brand: str = "",
|
||||
category_hint: str = "",
|
||||
) -> int:
|
||||
"""Return a realistic INR price (integer) for a given pack size.
|
||||
|
||||
IMPORTANT: the per-100g/ml `rate_per_100` is derived from a seed that
|
||||
deliberately EXCLUDES `size`. If the seed included the size string (as
|
||||
it used to), each pack size of the same product would land on a
|
||||
different random point inside the category's price-per-100 band, which
|
||||
could - and did - make a *larger* pack cheaper in absolute terms than a
|
||||
*smaller* pack of the exact same product (e.g. Dettol 500ml costing
|
||||
more than Dettol 1L). Keeping the rate stable per product+brand+category
|
||||
means every size variant of one product shares the same ₹/100(g|ml)
|
||||
rate, so the only thing that changes between variants is `grams` and
|
||||
the (size-aware, monotonic-by-design) pack-size curve multiplier below
|
||||
- guaranteeing larger packs always cost more in absolute ₹, while
|
||||
still costing less per gram/ml (realistic bulk-discount behaviour).
|
||||
"""
|
||||
category = classify_category(product_title, category_hint)
|
||||
min_rate, max_rate, floor_price = CATEGORY_BANDS[category]
|
||||
|
||||
grams = _parse_size_to_grams(size)
|
||||
seed = f"{brand}|{category}|{product_title}"
|
||||
jitter = _deterministic_unit(seed)
|
||||
rate_per_100 = min_rate + jitter * (max_rate - min_rate)
|
||||
|
||||
base_price = (grams / 100.0) * rate_per_100
|
||||
base_price *= _pack_size_curve_multiplier(grams)
|
||||
base_price *= TIER_MULTIPLIER[_brand_tier(brand)]
|
||||
|
||||
price = max(round(base_price), floor_price)
|
||||
# Round to a "nice" retail-looking number (ends in 0/5/9) for anything
|
||||
# above ₹20, mirroring common Indian MRP conventions.
|
||||
if price >= 20:
|
||||
price = int(round(price / 5.0) * 5)
|
||||
return int(price)
|
||||
|
||||
|
||||
def enforce_monotonic_prices(variants: list) -> list:
|
||||
"""Given a list of (size_str, price_int) tuples, return a new list
|
||||
sorted by ascending pack size where price never decreases as size
|
||||
increases.
|
||||
|
||||
This is a defence-in-depth safety net on top of `estimate_price`'s own
|
||||
seed fix above: it also catches the case where a per-variant price came
|
||||
from `reconcile_llm_price` (which allows the LLM's own number through
|
||||
if it's within [0.5x, 2x] of the anchor) and two *different* variants'
|
||||
LLM numbers individually passed that check but still ended up
|
||||
non-monotonic relative to each other - e.g. a hallucinated "₹25" for
|
||||
the 1L variant and "₹190" for the 500ml variant could each
|
||||
individually look plausible in isolation while still being wrong
|
||||
relative to each other.
|
||||
"""
|
||||
if not variants:
|
||||
return []
|
||||
enriched = [
|
||||
(size, price, _parse_size_to_grams(size if isinstance(size, str) else str(size)))
|
||||
for size, price in variants
|
||||
]
|
||||
enriched.sort(key=lambda x: x[2])
|
||||
fixed = []
|
||||
last_price = 0
|
||||
last_grams = None
|
||||
for size, price, grams in enriched:
|
||||
if last_grams is not None and grams > last_grams and price < last_price:
|
||||
# A strictly larger pack must not cost less than the previous
|
||||
# (strictly smaller) one. Bump it up to at least match.
|
||||
price = last_price
|
||||
fixed.append((size, price))
|
||||
last_price = max(last_price, price)
|
||||
last_grams = grams
|
||||
return fixed
|
||||
|
||||
|
||||
def estimate_price_range(
|
||||
sizes: list,
|
||||
product_title: str = "",
|
||||
brand: str = "",
|
||||
category_hint: str = "",
|
||||
) -> Tuple[int, int]:
|
||||
"""Estimate a (min, max) price range across a list of size variants."""
|
||||
if not sizes:
|
||||
category = classify_category(product_title, category_hint)
|
||||
_, _, floor_price = CATEGORY_BANDS[category]
|
||||
lo = estimate_price("100g", product_title, brand, category_hint)
|
||||
return (floor_price, max(lo, floor_price))
|
||||
prices = [estimate_price(s, product_title, brand, category_hint) for s in sizes]
|
||||
return (min(prices), max(prices))
|
||||
|
||||
|
||||
def default_size_variants(category_hint: str, product_title: str = "") -> list:
|
||||
"""Category-appropriate realistic default pack sizes, used only when the
|
||||
LLM/scraper provided none at all. Replaces the old one-size-fits-all
|
||||
["100g", "200g", "500g"] default, which made no sense for e.g. toothpaste
|
||||
or shampoo."""
|
||||
category = classify_category(product_title, category_hint)
|
||||
presets = {
|
||||
"oral_care": ["40g", "80g", "150g"],
|
||||
"hair_care": ["90ml", "180ml", "340ml"],
|
||||
"skin_bath": ["75g", "100g", "125g"],
|
||||
"household_clean": ["500ml", "1L", "2L"],
|
||||
"baby_care": ["100ml", "200ml", "400g"],
|
||||
"chocolates": ["20g", "55g", "150g"],
|
||||
"biscuits_cookies": ["100g", "200g", "375g"],
|
||||
"crackers": ["100g", "200g", "300g"],
|
||||
"rusk": ["180g", "300g", "600g"],
|
||||
"bakery_bread": ["200g", "400g", "600g"],
|
||||
"cakes_muffins": ["100g", "250g", "400g"],
|
||||
"snacks_namkeen": ["55g", "150g", "200g"],
|
||||
"dairy": ["200ml", "500ml", "1L"],
|
||||
"beverages_juice": ["200ml", "1L", "2L"],
|
||||
"beverages_tea_coffee": ["100g", "250g", "500g"],
|
||||
"breakfast_cereal": ["250g", "500g", "1kg"],
|
||||
"general": ["100g", "250g", "500g"],
|
||||
}
|
||||
return presets.get(category, presets["general"])
|
||||
|
||||
|
||||
def reconcile_llm_price(
|
||||
llm_price_rs: Optional[float],
|
||||
size: str,
|
||||
product_title: str = "",
|
||||
brand: str = "",
|
||||
category_hint: str = "",
|
||||
) -> int:
|
||||
"""Sanity-clamp a price the (small, local) LLM proposed against our
|
||||
curated retail anchor. Small models often hallucinate numbers that are
|
||||
off by an order of magnitude; rather than discarding their guess
|
||||
entirely or trusting it blindly, we keep it only if it falls within a
|
||||
plausible band [0.5x, 2x] of our anchor estimate, otherwise we fall back
|
||||
to the anchor estimate itself.
|
||||
"""
|
||||
anchor = estimate_price(size, product_title, brand, category_hint)
|
||||
if not llm_price_rs or llm_price_rs <= 0:
|
||||
return anchor
|
||||
if anchor * 0.5 <= llm_price_rs <= anchor * 2.0:
|
||||
price = int(round(llm_price_rs))
|
||||
return price
|
||||
return anchor
|
||||
|
||||
|
||||
def parse_price_string(text: str) -> Optional[float]:
|
||||
"""Extract a single numeric INR value from a free-text price string like
|
||||
'₹45-60', '₹120', '50.5', or 'around ₹30'. Returns the average of a
|
||||
range when one is present."""
|
||||
if not text:
|
||||
return None
|
||||
numbers = [float(n) for n in re.findall(r"\d+(?:\.\d+)?", str(text))]
|
||||
if not numbers:
|
||||
return None
|
||||
if len(numbers) >= 2:
|
||||
return (numbers[0] + numbers[1]) / 2.0
|
||||
return numbers[0]
|
||||
202
app/services/query_intent.py
Normal file
202
app/services/query_intent.py
Normal file
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
Lightweight intent parsing for free-text RAG queries.
|
||||
|
||||
Deliberately regex/keyword based (no extra model call) so it stays fast
|
||||
and free on CPU-only hardware. Two things are extracted:
|
||||
|
||||
1. The product category the user is asking about, via
|
||||
`category_registry.detect_category_from_text()` - re-exported here
|
||||
for convenience so callers only need to import this module.
|
||||
2. Simple product attributes (sugar level, spice level, dietary
|
||||
flags) mentioned in the query, used to *re-rank* (never to
|
||||
silently fabricate) retrieved products - see
|
||||
`rag_service._rerank_by_attributes()`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
from app.services.category_registry import detect_category_from_text # noqa: F401 (re-exported)
|
||||
|
||||
# Each attribute maps a canonical value to the query phrases that imply it.
|
||||
# Order matters: more specific phrases (e.g. "sugar free") are checked
|
||||
# before more general ones (e.g. "low sugar").
|
||||
_ATTRIBUTE_PATTERNS: Dict[str, Dict[str, list]] = {
|
||||
"sugar": {
|
||||
"none": [r"no added sugar", r"no sugar", r"sugar[- ]?free", r"zero sugar"],
|
||||
"low": [r"low sugar", r"less sugar", r"reduced sugar"],
|
||||
"high": [r"extra sweet", r"very sweet"],
|
||||
},
|
||||
"spice": {
|
||||
"high": [r"spicy", r"extra hot", r"very hot", r"chilli", r"chili"],
|
||||
"low": [r"mild", r"less spicy", r"not (?:too )?spicy"],
|
||||
},
|
||||
"fat": {
|
||||
"low": [r"low fat", r"light\b"],
|
||||
},
|
||||
"gluten": {
|
||||
"free": [r"gluten[- ]?free"],
|
||||
},
|
||||
"salt": {
|
||||
"low": [r"low salt", r"less salt", r"low sodium"],
|
||||
},
|
||||
}
|
||||
|
||||
# Keyword hints (per attribute/value) to look for in a product's own text
|
||||
# (description + highlights + nutrients) when deciding whether it actually
|
||||
# satisfies the attribute the user asked for. Kept separate from the query
|
||||
# patterns above because product copy tends to use slightly different
|
||||
# phrasing than a user typing a search.
|
||||
_PRODUCT_EVIDENCE_HINTS: Dict[str, Dict[str, list]] = {
|
||||
"sugar": {
|
||||
"none": ["no added sugar", "sugar free", "sugar-free", "zero sugar"],
|
||||
"low": ["low sugar", "less sugar", "reduced sugar", "no added sugar", "sugar free"],
|
||||
},
|
||||
"spice": {
|
||||
"high": ["spicy", "chilli", "chili", "hot & spicy", "extra hot"],
|
||||
"low": ["mild", "less spicy"],
|
||||
},
|
||||
"fat": {
|
||||
"low": ["low fat", "light"],
|
||||
},
|
||||
"gluten": {
|
||||
"free": ["gluten free", "gluten-free"],
|
||||
},
|
||||
"salt": {
|
||||
"low": ["low salt", "low sodium", "less salt"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def extract_attributes(query: str) -> Dict[str, str]:
|
||||
"""Return a dict of {attribute: value} implied by the query, e.g.
|
||||
{"sugar": "low"} for "recommend biscuits with low sugar"."""
|
||||
if not query:
|
||||
return {}
|
||||
lower = query.lower()
|
||||
found: Dict[str, str] = {}
|
||||
for attribute, values in _ATTRIBUTE_PATTERNS.items():
|
||||
for value, patterns in values.items():
|
||||
if any(re.search(p, lower) for p in patterns):
|
||||
found[attribute] = value
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def product_matches_attribute(product_text: str, attribute: str, value: str) -> bool:
|
||||
"""Whether a product's own text (description/highlights/nutrients)
|
||||
contains evidence it satisfies the requested attribute value."""
|
||||
hints = _PRODUCT_EVIDENCE_HINTS.get(attribute, {}).get(value)
|
||||
if not hints:
|
||||
return False
|
||||
lower = (product_text or "").lower()
|
||||
return any(hint in lower for hint in hints)
|
||||
|
||||
|
||||
def is_count_query(query: str) -> bool:
|
||||
"""Return True if the query asks for a total/count of products or brands."""
|
||||
if not query:
|
||||
return False
|
||||
lower = query.lower()
|
||||
patterns = [
|
||||
r"how many",
|
||||
r"count of",
|
||||
r"total (?:number of )?(?:products|items|brands)",
|
||||
r"number of products",
|
||||
r"how many products",
|
||||
r"how many items",
|
||||
r"how many brands",
|
||||
r"total count",
|
||||
]
|
||||
return any(re.search(p, lower) for p in patterns)
|
||||
|
||||
|
||||
def extract_max_price(query: str) -> Optional[float]:
|
||||
"""Extract a numeric maximum price ceiling from a query if present.
|
||||
e.g. 'under ₹150' -> 150.0, 'less than 100 rupees' -> 100.0
|
||||
"""
|
||||
if not query:
|
||||
return None
|
||||
lower = query.lower().replace(",", "")
|
||||
patterns = [
|
||||
r"(?:under|below|less than|within|upto|up to|budget of|max(?:imum)?)\s*(?:₹|rs\.?|inr)?\s*(\d+(?:\.\d+)?)",
|
||||
r"(?:₹|rs\.?|inr)\s*(\d+(?:\.\d+)?)\s*(?:or less|max|under|below)",
|
||||
r"<\s*(?:₹|rs\.?|inr)?\s*(\d+(?:\.\d+)?)",
|
||||
]
|
||||
for p in patterns:
|
||||
match = re.search(p, lower)
|
||||
if match:
|
||||
try:
|
||||
val = float(match.group(1))
|
||||
if val > 0:
|
||||
return val
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
KNOWN_BRANDS = [
|
||||
"Amul", "Cadbury", "Cavinkare", "Coca-Cola", "Colgate-Palmolive", "Dabur",
|
||||
"Godrej", "Grb", "Hindustan Unilever", "Lion Dates", "Manna", "Milky Mist",
|
||||
"Naga", "Nestle", "P&G", "Pepsico"
|
||||
]
|
||||
|
||||
BRAND_SEARCH_MAP = {
|
||||
"cadbury": "Cadbury",
|
||||
"amul": "Amul",
|
||||
"cavinkare": "Cavinkare",
|
||||
"coca-cola": "Coca-Cola",
|
||||
"coca cola": "Coca-Cola",
|
||||
"coke": "Coca-Cola",
|
||||
"colgate-palmolive": "Colgate-Palmolive",
|
||||
"colgate": "Colgate-Palmolive",
|
||||
"dabur": "Dabur",
|
||||
"godrej": "Godrej",
|
||||
"grb": "Grb",
|
||||
"hindustan unilever": "Hindustan Unilever",
|
||||
"hul": "Hindustan Unilever",
|
||||
"unilever": "Hindustan Unilever",
|
||||
"lion dates": "Lion Dates",
|
||||
"lion": "Lion Dates",
|
||||
"manna": "Manna",
|
||||
"milky mist": "Milky Mist",
|
||||
"naga": "Naga",
|
||||
"nestle": "Nestle",
|
||||
"p&g": "P&G",
|
||||
"pg": "P&G",
|
||||
"pepsico": "Pepsico",
|
||||
"pepsi": "Pepsico",
|
||||
}
|
||||
|
||||
|
||||
def extract_brand_mention(query: str) -> Optional[str]:
|
||||
"""Detect if a brand name is explicitly mentioned in the query text."""
|
||||
if not query:
|
||||
return None
|
||||
from app.services.brand_registry import BRAND_ALIASES, resolve_parent_brand
|
||||
lower = query.lower()
|
||||
|
||||
# 1. Check direct search map
|
||||
for alias in sorted(BRAND_SEARCH_MAP.keys(), key=len, reverse=True):
|
||||
pattern = r"\b" + re.escape(alias) + r"\b"
|
||||
if re.search(pattern, lower):
|
||||
return BRAND_SEARCH_MAP[alias]
|
||||
|
||||
# 2. Check sub-brand aliases (e.g. "oreo", "maggi", "good day")
|
||||
sorted_aliases = sorted(BRAND_ALIASES.keys(), key=len, reverse=True)
|
||||
for alias in sorted_aliases:
|
||||
pattern = r"\b" + re.escape(alias) + r"\b"
|
||||
if re.search(pattern, lower):
|
||||
parent = resolve_parent_brand(alias)
|
||||
# Normalize to canonical known brand name case
|
||||
for kb in KNOWN_BRANDS:
|
||||
if kb.lower() == parent.lower():
|
||||
return kb
|
||||
return parent.title()
|
||||
return None
|
||||
|
||||
|
||||
def detected_category(query: str) -> Optional[str]:
|
||||
"""Thin wrapper kept for readability at call sites in rag_service."""
|
||||
return detect_category_from_text(query)
|
||||
351
app/services/rag_service.py
Normal file
351
app/services/rag_service.py
Normal file
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
RAG (Retrieval-Augmented Generation) orchestration.
|
||||
|
||||
This is the module that was missing from the original project: everything
|
||||
needed to turn a free-text question into a grounded answer using the
|
||||
catalog already sitting in pgvector.
|
||||
|
||||
Pipeline (all CPU-only, tuned for an 8GB RAM / no-GPU machine):
|
||||
|
||||
user query
|
||||
-> embed_texts() (sentence-transformers, MiniLM-L6-v2, 384-dim)
|
||||
-> vector_store.semantic_search() (pgvector cosine similarity, per-brand or all-brand)
|
||||
-> build_context() (format top-k products into a compact text block)
|
||||
-> ollama_service.generate_rag_answer() (qwen2.5:1.5b via local Ollama)
|
||||
-> RagAnswer(answer, sources)
|
||||
|
||||
Kept deliberately simple and dependency-light (no LangChain/LlamaIndex) so
|
||||
it stays easy to run and reason about on constrained hardware.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.infrastructure.settings import (
|
||||
RAG_DEFAULT_TOP_K,
|
||||
RAG_MAX_TOP_K,
|
||||
RAG_MAX_CONTEXT_CHARS,
|
||||
RAG_MAX_DISTANCE,
|
||||
)
|
||||
from app.services.category_registry import category_matches, detect_category_from_text
|
||||
from app.services.embeddings_service import embed_texts
|
||||
from app.services.ollama_service import generate_rag_answer
|
||||
from app.services.query_intent import (
|
||||
extract_attributes,
|
||||
extract_brand_mention,
|
||||
extract_max_price,
|
||||
is_count_query,
|
||||
product_matches_attribute,
|
||||
)
|
||||
from app.services.s3_service import s3_service
|
||||
from app.services.vector_store import (
|
||||
count_products_all_brands,
|
||||
count_products_by_brand,
|
||||
list_available_brands,
|
||||
list_categories_for_brand,
|
||||
semantic_search,
|
||||
text_search,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RetrievedProduct:
|
||||
image_id: str
|
||||
image_url: str = ""
|
||||
image_urls: List[str] = field(default_factory=list)
|
||||
brand: str = ""
|
||||
title: str = ""
|
||||
product_name: Optional[str] = None
|
||||
category: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
price_range: Optional[str] = None
|
||||
size_variants: List[str] = field(default_factory=list)
|
||||
providers: List[str] = field(default_factory=list)
|
||||
highlights: List[str] = field(default_factory=list)
|
||||
nutrients: List[str] = field(default_factory=list)
|
||||
fssai_license: Optional[str] = None
|
||||
product_sku: Optional[str] = None
|
||||
sku_source: Optional[str] = None
|
||||
hsn_code: Optional[str] = None
|
||||
final_selling_price: Optional[float] = None
|
||||
selling_price: Optional[float] = None
|
||||
barcode: Optional[str] = None
|
||||
barcode_type: Optional[str] = None
|
||||
distance: float = 1.0
|
||||
|
||||
@property
|
||||
def similarity(self) -> float:
|
||||
"""Convert pgvector cosine distance (0=identical) to an intuitive 0-1 score."""
|
||||
return max(0.0, min(1.0, 1.0 - (self.distance / 2.0)))
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"image_id": self.image_id,
|
||||
"image_url": self.image_url,
|
||||
"image_urls": self.image_urls,
|
||||
"brand": self.brand,
|
||||
"title": self.title,
|
||||
"product_name": self.product_name,
|
||||
"category": self.category,
|
||||
"description": self.description,
|
||||
"price_range": self.price_range,
|
||||
"size_variants": self.size_variants,
|
||||
"providers": self.providers,
|
||||
"highlights": self.highlights,
|
||||
"nutrients": self.nutrients,
|
||||
"fssai_license": self.fssai_license,
|
||||
"product_sku": self.product_sku,
|
||||
"sku_source": self.sku_source,
|
||||
"hsn_code": self.hsn_code,
|
||||
"final_selling_price": self.final_selling_price,
|
||||
"selling_price": self.selling_price,
|
||||
"barcode": self.barcode,
|
||||
"barcode_type": self.barcode_type,
|
||||
"similarity": round(self.similarity, 4),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RagAnswer:
|
||||
answer: str
|
||||
sources: List[RetrievedProduct]
|
||||
query: str
|
||||
brand: Optional[str]
|
||||
detected_category: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"answer": self.answer,
|
||||
"query": self.query,
|
||||
"brand": self.brand,
|
||||
"detected_category": self.detected_category,
|
||||
"sources": [s.to_dict() for s in self.sources],
|
||||
}
|
||||
|
||||
|
||||
def _clean_url(url: Optional[str]) -> Optional[str]:
|
||||
if not url:
|
||||
return None
|
||||
return str(url).replace('{width}', '800')
|
||||
|
||||
|
||||
def _row_to_retrieved_product(row: Dict[str, Any]) -> RetrievedProduct:
|
||||
image_id = row.get("image_id") or ""
|
||||
brand = (row.get("brand") or "").title()
|
||||
|
||||
db_single = _clean_url(row.get("image_url"))
|
||||
db_list = [_clean_url(u) for u in (row.get("image_urls") or []) if u]
|
||||
|
||||
final_urls = db_list
|
||||
if not final_urls and db_single:
|
||||
final_urls = [db_single]
|
||||
|
||||
if not final_urls and s3_service.enabled and image_id:
|
||||
s3_list = s3_service.get_product_image_urls(brand, image_id)
|
||||
if s3_list:
|
||||
final_urls = s3_list
|
||||
|
||||
primary_url = (final_urls[0] if final_urls else None) or db_single
|
||||
if not primary_url and s3_service.enabled and image_id:
|
||||
primary_url = s3_service.get_product_image_url(brand, image_id)
|
||||
|
||||
hsn = row.get("hsn_code") or row.get("HSN_Code") or row.get("hsn") or None
|
||||
if hsn is not None:
|
||||
hsn = str(hsn).strip() or None
|
||||
|
||||
raw_fsp = row.get("final_selling_price") if "final_selling_price" in row else row.get("Final_Selling_Price")
|
||||
if raw_fsp is None:
|
||||
raw_fsp = row.get("final_price")
|
||||
try:
|
||||
fsp = float(raw_fsp) if raw_fsp is not None and str(raw_fsp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
fsp = None
|
||||
|
||||
raw_sp = row.get("selling_price") if "selling_price" in row else row.get("Selling_Price")
|
||||
try:
|
||||
sp = float(raw_sp) if raw_sp is not None and str(raw_sp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
sp = None
|
||||
|
||||
if fsp is None and sp is not None:
|
||||
fsp = sp
|
||||
|
||||
bcd = row.get("barcode") or row.get("Barcode") or None
|
||||
if bcd is not None:
|
||||
bcd = str(bcd).strip() or None
|
||||
|
||||
bcd_type = row.get("barcode_type") or row.get("Barcode_Type") or None
|
||||
if bcd_type is not None:
|
||||
bcd_type = str(bcd_type).strip() or None
|
||||
|
||||
fssai = row.get("fssai_license") or row.get("fssai") or row.get("fssai_number") or row.get("FSSAI_License") or row.get("fssai_lic_no") or None
|
||||
if fssai is not None:
|
||||
fssai = str(fssai).strip() or None
|
||||
|
||||
return RetrievedProduct(
|
||||
image_id=image_id,
|
||||
image_url=primary_url,
|
||||
image_urls=final_urls,
|
||||
brand=brand,
|
||||
product_name=row.get("product_name") or row.get("title") or "Unknown product",
|
||||
title=row.get("title") or row.get("product_name") or None,
|
||||
category=row.get("category"),
|
||||
description=row.get("description"),
|
||||
price_range=row.get("price_range"),
|
||||
size_variants=list(row.get("size_variants") or []),
|
||||
providers=list(row.get("providers") or []),
|
||||
highlights=list(row.get("highlights") or []),
|
||||
nutrients=list(row.get("nutrients") or []),
|
||||
fssai_license=fssai,
|
||||
product_sku=row.get("product_sku") or None,
|
||||
sku_source=row.get("sku_source") or None,
|
||||
hsn_code=hsn,
|
||||
final_selling_price=fsp,
|
||||
selling_price=sp,
|
||||
barcode=bcd,
|
||||
barcode_type=bcd_type,
|
||||
distance=float(row.get("distance", 1.0)),
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_category(rows: List[Dict[str, Any]], target_category: Optional[str]) -> List[Dict[str, Any]]:
|
||||
if not target_category:
|
||||
return rows
|
||||
return [r for r in rows if category_matches(r.get("category"), target_category)]
|
||||
|
||||
|
||||
def _rerank_by_attributes(products: List[RetrievedProduct], attrs: Dict[str, str]) -> List[RetrievedProduct]:
|
||||
"""Move products with explicit evidence of a requested attribute (e.g.
|
||||
"sugar": "low") to the front, without dropping any product.
|
||||
"""
|
||||
def evidence_count(p: RetrievedProduct) -> int:
|
||||
text = " ".join(filter(None, [p.description or "", " ".join(p.highlights), " ".join(p.nutrients)]))
|
||||
return sum(
|
||||
1 for attribute, value in attrs.items()
|
||||
if product_matches_attribute(text, attribute, value)
|
||||
)
|
||||
|
||||
indexed = list(enumerate(products))
|
||||
indexed.sort(key=lambda pair: (-evidence_count(pair[1]), pair[0]))
|
||||
return [p for _, p in indexed]
|
||||
|
||||
|
||||
def retrieve(query: str, brand: Optional[str] = None, top_k: Optional[int] = None,
|
||||
category: Optional[str] = None, max_distance: Optional[float] = None) -> List[RetrievedProduct]:
|
||||
"""Embed `query` and run a pgvector similarity search with category, brand, and price filtering."""
|
||||
top_k = min(top_k or RAG_DEFAULT_TOP_K, RAG_MAX_TOP_K)
|
||||
effective_max_distance = max_distance if max_distance is not None else RAG_MAX_DISTANCE
|
||||
|
||||
target_brand = brand or extract_brand_mention(query)
|
||||
target_category = category or detect_category_from_text(query)
|
||||
max_price = extract_max_price(query)
|
||||
|
||||
try:
|
||||
vectors = embed_texts([query])
|
||||
except Exception as e:
|
||||
logger.warning("Embedding model failed: %s. Using text fallback.", e)
|
||||
vectors = None
|
||||
|
||||
rows: List[Dict[str, Any]] = []
|
||||
if vectors:
|
||||
rows = semantic_search(
|
||||
query_embedding=vectors[0],
|
||||
brand=target_brand,
|
||||
top_k=top_k,
|
||||
category=target_category,
|
||||
max_distance=effective_max_distance,
|
||||
max_price=max_price,
|
||||
)
|
||||
rows = _filter_to_category(rows, target_category)
|
||||
if not rows:
|
||||
logger.info(
|
||||
"Semantic search returned no in-category results for %r (category=%r), falling back to text search",
|
||||
query, target_category,
|
||||
)
|
||||
|
||||
if not rows:
|
||||
rows = text_search(query, brand=target_brand, top_k=top_k, category=target_category, max_price=max_price)
|
||||
rows = _filter_to_category(rows, target_category)
|
||||
|
||||
products = [_row_to_retrieved_product(r) for r in rows]
|
||||
|
||||
attrs = extract_attributes(query)
|
||||
if attrs and products:
|
||||
products = _rerank_by_attributes(products, attrs)
|
||||
|
||||
return products
|
||||
|
||||
|
||||
def build_context(products: List[RetrievedProduct], max_chars: int = RAG_MAX_CONTEXT_CHARS,
|
||||
metric_info: Optional[str] = None) -> str:
|
||||
"""Format retrieved products and optional metric info into a compact text block for the LLM prompt."""
|
||||
prefix = f"{metric_info}\n\n" if metric_info else ""
|
||||
|
||||
if not products:
|
||||
return prefix + "(No matching products were found in the catalog for this query.)"
|
||||
|
||||
per_item_budget = max(200, max_chars // max(1, len(products)))
|
||||
lines = []
|
||||
for i, p in enumerate(products, start=1):
|
||||
desc = (p.description or "").strip()
|
||||
budget_for_desc = max(40, per_item_budget - 120)
|
||||
if len(desc) > budget_for_desc:
|
||||
desc = desc[:budget_for_desc].rsplit(" ", 1)[0] + "..."
|
||||
|
||||
parts = [f"{i}. {p.product_name or p.title} (brand: {p.brand}, category: {p.category or 'Uncategorized'})"]
|
||||
if p.price_range:
|
||||
parts.append(f" Price: {p.price_range}")
|
||||
if p.size_variants:
|
||||
parts.append(f" Sizes: {', '.join(p.size_variants[:6])}")
|
||||
if p.highlights:
|
||||
parts.append(f" Highlights: {', '.join(p.highlights[:5])}")
|
||||
if p.nutrients:
|
||||
parts.append(f" Nutrients: {', '.join(p.nutrients[:5])}")
|
||||
if desc:
|
||||
parts.append(f" Description: {desc}")
|
||||
lines.append("\n".join(parts))
|
||||
|
||||
return prefix + "\n\n".join(lines)
|
||||
|
||||
|
||||
def answer_query(
|
||||
query: str,
|
||||
brand: Optional[str] = None,
|
||||
top_k: Optional[int] = None,
|
||||
category: Optional[str] = None,
|
||||
history: Optional[List[Dict[str, str]]] = None,
|
||||
) -> RagAnswer:
|
||||
"""End-to-end RAG: retrieve relevant products, compute exact counts if requested, then generate a grounded answer."""
|
||||
target_brand = brand or extract_brand_mention(query)
|
||||
target_category = category or detect_category_from_text(query)
|
||||
metric_info: Optional[str] = None
|
||||
|
||||
if is_count_query(query):
|
||||
if target_brand:
|
||||
cnt = count_products_by_brand(target_brand, category=target_category)
|
||||
cats = list_categories_for_brand(target_brand)
|
||||
cat_str = f" in category '{target_category}'" if target_category else ""
|
||||
metric_info = (
|
||||
f"CATALOG METRIC DATA: Exact product count for brand '{target_brand}'{cat_str} in the database is {cnt}. "
|
||||
f"Categories available under {target_brand}: {', '.join(cats)}."
|
||||
)
|
||||
elif target_category:
|
||||
cnt = count_products_all_brands(category=target_category)
|
||||
metric_info = f"CATALOG METRIC DATA: Total products matching category '{target_category}' across all brands in database is {cnt}."
|
||||
else:
|
||||
total = count_products_all_brands()
|
||||
brands = list_available_brands()
|
||||
metric_info = f"CATALOG METRIC DATA: Total catalog items across all brands is {total}. Known brands: {', '.join(brands)}."
|
||||
|
||||
products = retrieve(query, brand=brand, top_k=top_k, category=category)
|
||||
context = build_context(products, metric_info=metric_info)
|
||||
answer_text = generate_rag_answer(query, context, history=history, requested_category=target_category)
|
||||
|
||||
return RagAnswer(
|
||||
answer=answer_text, sources=products, query=query, brand=target_brand or brand,
|
||||
detected_category=target_category,
|
||||
)
|
||||
110
app/services/recommendation_service.py
Normal file
110
app/services/recommendation_service.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
Thin I/O orchestration layer for Feature 7 (Recommendation Engine).
|
||||
Fetches what `app/intelligence/recommendation_engine.py`'s pure scoring
|
||||
functions need, then persists/reads the result cache via `store_db.py`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from app.intelligence import recommendation_engine as RE
|
||||
from app.intelligence.popularity_model import popularity_scorer
|
||||
from app.services import store_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_copurchase_cache: Dict[str, tuple] = {}
|
||||
|
||||
|
||||
def _get_copurchase_matrix():
|
||||
"""Cached per-process (not per-request) since it's rebuilt from the
|
||||
full order history and is moderately expensive - recomputed when
|
||||
`refresh()` is called (e.g. after the seed/training script runs)."""
|
||||
if "matrix" not in _copurchase_cache:
|
||||
order_items = store_db.get_order_items_df()
|
||||
matrix, keys = RE.build_copurchase_matrix(order_items)
|
||||
_copurchase_cache["matrix"] = matrix
|
||||
_copurchase_cache["keys"] = keys
|
||||
return _copurchase_cache["matrix"], _copurchase_cache["keys"]
|
||||
|
||||
|
||||
def refresh_copurchase_cache() -> None:
|
||||
_copurchase_cache.clear()
|
||||
_get_copurchase_matrix()
|
||||
|
||||
|
||||
def recommend_for_product(brand: str, image_id: str, top_k: int = 5, use_cache: bool = True) -> List[Dict]:
|
||||
"""Hybrid recommendation: embedding similarity + TF-IDF + collaborative
|
||||
co-purchase + popularity, blended per `RE.RecommendationCandidate.hybrid_score`.
|
||||
Every result carries its own similarity_score, as required."""
|
||||
if use_cache:
|
||||
cached = store_db.get_cached_recommendations(brand, image_id)
|
||||
if cached:
|
||||
return [
|
||||
{"brand": c["rec_brand"], "image_id": c["rec_image_id"],
|
||||
"similarity_score": float(c["similarity_score"]), "method": c["method"],
|
||||
"signals": c.get("signals_json")}
|
||||
for c in cached[:top_k]
|
||||
]
|
||||
|
||||
source_vec = store_db.get_embedding_vector(brand, image_id)
|
||||
candidates_df = store_db.get_products_with_embeddings()
|
||||
if candidates_df.empty:
|
||||
return []
|
||||
candidates_df = candidates_df[~((candidates_df["brand"] == brand) & (candidates_df["image_id"] == image_id))].reset_index(drop=True)
|
||||
if candidates_df.empty:
|
||||
return []
|
||||
|
||||
# 1) Embedding similarity
|
||||
if source_vec:
|
||||
cand_vecs = np.array(candidates_df["embedding"].tolist())
|
||||
emb_sims = RE.embedding_similarity_to_source(np.array(source_vec), cand_vecs)
|
||||
else:
|
||||
emb_sims = np.zeros(len(candidates_df))
|
||||
|
||||
# 2) TF-IDF similarity over "title category brand"
|
||||
source_row = store_db.get_products_with_embeddings(brands=[brand])
|
||||
source_title = ""
|
||||
src_match = source_row[source_row["image_id"] == image_id]
|
||||
if not src_match.empty:
|
||||
source_title = f"{src_match.iloc[0]['title']} {src_match.iloc[0]['category']} {brand}"
|
||||
corpus = [source_title] + (candidates_df["title"].fillna("") + " " + candidates_df["category"].fillna("") + " " + candidates_df["brand"]).tolist()
|
||||
tfidf_sims = RE.tfidf_similarity(corpus, source_index=0)[1:] if source_title else np.zeros(len(candidates_df))
|
||||
|
||||
# 3) Collaborative (co-purchase) similarity
|
||||
matrix, keys = _get_copurchase_matrix()
|
||||
collab_sims = np.zeros(len(candidates_df))
|
||||
src_key = f"{brand}||{image_id}"
|
||||
if src_key in keys and matrix.shape[0] > 0:
|
||||
src_idx = keys.index(src_key)
|
||||
row = matrix.getrow(src_idx).toarray().ravel()
|
||||
key_to_pos = {k: i for i, k in enumerate(keys)}
|
||||
for i, r in candidates_df.iterrows():
|
||||
k = f"{r['brand']}||{r['image_id']}"
|
||||
if k in key_to_pos:
|
||||
collab_sims[i] = row[key_to_pos[k]]
|
||||
|
||||
# 4) Popularity (needs the same features popularity_model was trained on;
|
||||
# if unavailable, falls back to a neutral 0 contribution via the scorer's
|
||||
# own graceful-degradation path)
|
||||
pop_norm = np.zeros(len(candidates_df))
|
||||
|
||||
candidates = [
|
||||
RE.RecommendationCandidate(
|
||||
brand=row["brand"], image_id=row["image_id"],
|
||||
embedding_similarity=float(emb_sims[i]) if len(emb_sims) else 0.0,
|
||||
tfidf_similarity=float(tfidf_sims[i]) if len(tfidf_sims) else 0.0,
|
||||
collaborative_similarity=float(collab_sims[i]),
|
||||
popularity_norm=float(pop_norm[i]),
|
||||
)
|
||||
for i, row in candidates_df.iterrows()
|
||||
]
|
||||
ranked = RE.rank_candidates(candidates, top_k=top_k)
|
||||
to_save = [{"rec_brand": r["brand"], "rec_image_id": r["image_id"], "similarity_score": r["similarity_score"],
|
||||
"signals": r["signals"], "method": "hybrid"} for r in ranked]
|
||||
store_db.save_recommendations(brand, image_id, to_save)
|
||||
return [{**r, "method": "hybrid"} for r in ranked]
|
||||
301
app/services/s3_service.py
Normal file
301
app/services/s3_service.py
Normal file
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
S3 Service for DigitalOcean Spaces integration
|
||||
Handles image upload and management
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlparse
|
||||
import aiohttp
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from app.infrastructure.settings import (
|
||||
USE_S3, S3_ACCESS_KEY, S3_SECRET_KEY,
|
||||
S3_ENDPOINT, S3_BUCKET, S3_REGION
|
||||
)
|
||||
from app.services.brand_registry import resolve_parent_brand
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3Service:
|
||||
"""Service for managing S3/DigitalOcean Spaces operations"""
|
||||
|
||||
def __init__(self):
|
||||
self.enabled = USE_S3 and all([S3_ACCESS_KEY, S3_SECRET_KEY, S3_ENDPOINT, S3_BUCKET])
|
||||
self._url_cache = {}
|
||||
if self.enabled:
|
||||
# Use regional base endpoint for API calls to avoid NoSuchKey errors with virtual-hosted style URLs
|
||||
# DO Spaces API endpoint format: https://{region}.digitaloceanspaces.com
|
||||
api_endpoint = f"https://{S3_REGION}.digitaloceanspaces.com" if 'digitaloceanspaces.com' in S3_ENDPOINT else S3_ENDPOINT
|
||||
|
||||
self.client = boto3.client(
|
||||
's3',
|
||||
endpoint_url=api_endpoint,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
region_name=S3_REGION
|
||||
)
|
||||
else:
|
||||
self.client = None
|
||||
logger.warning("S3 service disabled - missing configuration")
|
||||
|
||||
def generate_image_id(self, product_name: str = None) -> str:
|
||||
"""Generate image ID from product name or fallback to UUID"""
|
||||
if product_name:
|
||||
# Sanitize product name for S3 folder
|
||||
sanitized = product_name.lower()
|
||||
# Replace spaces and special chars with underscores
|
||||
sanitized = ''.join(c if c.isalnum() else '_' for c in sanitized)
|
||||
# Remove multiple underscores and trim
|
||||
sanitized = '_'.join(filter(None, sanitized.split('_')))
|
||||
# Limit length and add unique suffix to avoid conflicts
|
||||
if len(sanitized) > 50:
|
||||
sanitized = sanitized[:50]
|
||||
# Add short UUID suffix to ensure uniqueness
|
||||
unique_suffix = str(uuid.uuid4())[:8]
|
||||
return f"{sanitized}_{unique_suffix}"
|
||||
return str(uuid.uuid4())
|
||||
|
||||
async def download_image(self, url: str, session: aiohttp.ClientSession) -> Optional[bytes]:
|
||||
"""Download image from URL with improved anti-bot evasion.
|
||||
|
||||
Two targeted fixes over the previous version:
|
||||
1. The Referer header is now derived from the image URL's own
|
||||
domain instead of always being hardcoded to google.com - many
|
||||
retailer CDNs (hotlink protection) reject requests whose
|
||||
Referer doesn't match their own site, which was silently
|
||||
killing a chunk of otherwise-valid image URLs.
|
||||
2. Downloaded bytes below MIN_IMAGE_BYTES are now rejected here
|
||||
(in addition to the upstream validation in image_search.py),
|
||||
so 1x1 tracking pixels / broken placeholder images never reach
|
||||
the S3 upload step.
|
||||
"""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
try:
|
||||
from app.infrastructure.settings import MIN_IMAGE_BYTES
|
||||
except ImportError:
|
||||
MIN_IMAGE_BYTES = 3000
|
||||
|
||||
parsed = urlparse(url)
|
||||
referer = f"{parsed.scheme}://{parsed.netloc}/" if parsed.netloc else 'https://www.google.com/'
|
||||
|
||||
# Enhanced headers to better mimic a real browser
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'image',
|
||||
'Sec-Fetch-Mode': 'no-cors',
|
||||
'Sec-Fetch-Site': 'cross-site',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Pragma': 'no-cache',
|
||||
'Referer': referer,
|
||||
}
|
||||
|
||||
# Add random delay to avoid rate limiting
|
||||
import random
|
||||
await asyncio.sleep(random.uniform(0.5, 2.0))
|
||||
|
||||
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=45)) as response:
|
||||
if response.status == 200:
|
||||
content_type = response.headers.get('content-type', '').lower()
|
||||
if 'image' in content_type or url.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp')):
|
||||
data = await response.read()
|
||||
if len(data) >= MIN_IMAGE_BYTES:
|
||||
return data
|
||||
else:
|
||||
logger.warning(f"Image too small ({len(data)} bytes, likely placeholder) for {url}")
|
||||
else:
|
||||
logger.warning(f"Invalid content type for image {url}: {content_type}")
|
||||
elif response.status == 403:
|
||||
logger.warning(f"HTTP 403 for image {url} - Site blocking requests")
|
||||
elif response.status == 404:
|
||||
logger.warning(f"HTTP 404 for image {url} - Image not found")
|
||||
elif response.status == 429:
|
||||
logger.warning(f"HTTP 429 for image {url} - Rate limited, waiting...")
|
||||
await asyncio.sleep(5) # Wait longer for rate limits
|
||||
else:
|
||||
logger.warning(f"HTTP {response.status} for image {url}")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f"Timeout downloading image {url}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to download image {url}: {e}")
|
||||
return None
|
||||
|
||||
def upload_image(self, image_data: bytes, image_id: str, filename: str, brand: str = None) -> bool:
|
||||
"""Upload image to S3 bucket under daily/brands/{brand}/{image_id}/ folder.
|
||||
|
||||
BUGFIX: ContentType used to be hard-coded to 'image/jpeg' regardless
|
||||
of the file's actual format. A PNG/WEBP/GIF served with a
|
||||
mismatched Content-Type can fail to render (or render incorrectly)
|
||||
in some browsers and in <img> tags that respect MIME type strictly.
|
||||
Now derived from the filename's extension via the stdlib
|
||||
`mimetypes` module, falling back to image/jpeg only if the
|
||||
extension is unrecognized.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
try:
|
||||
import mimetypes
|
||||
content_type = mimetypes.guess_type(filename)[0] or 'image/jpeg'
|
||||
|
||||
# Create path: daily/brands/{brand}/{image_id}/{filename}
|
||||
if brand:
|
||||
storage_brand = resolve_parent_brand(brand)
|
||||
key = f"daily/brands/{storage_brand.lower()}/{image_id}/{filename}"
|
||||
else:
|
||||
key = f"daily/brands/products/{image_id}/{filename}"
|
||||
|
||||
self.client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=key,
|
||||
Body=image_data,
|
||||
ContentType=content_type,
|
||||
ACL='public-read'
|
||||
)
|
||||
logger.info(f"✅ Uploaded {key} to S3")
|
||||
return True
|
||||
except ClientError as e:
|
||||
logger.error(f"❌ S3 upload failed for {key}: {e}")
|
||||
return False
|
||||
|
||||
async def process_product_images(self, product: dict, image_urls: List[str], brand: str = None, max_images: int = 10):
|
||||
"""
|
||||
Download and upload exactly max_images images for a product
|
||||
Returns tuple of (image_id, list of uploaded S3 public URLs)
|
||||
"""
|
||||
if not self.enabled or not image_urls:
|
||||
return "", []
|
||||
|
||||
id_source = product.get('product_name') or product.get('title', 'unknown_product')
|
||||
image_id = self.generate_image_id(id_source)
|
||||
uploaded_urls = []
|
||||
|
||||
# Limit to max_images
|
||||
images_to_process = image_urls[:max_images]
|
||||
storage_brand_name = resolve_parent_brand(brand) if brand else "products"
|
||||
brand_low = storage_brand_name.lower()
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
logger.info(f"📸 Processing {len(images_to_process)} images for {id_source}")
|
||||
|
||||
for i, url in enumerate(images_to_process):
|
||||
try:
|
||||
logger.info(f"Downloading image {i+1}/{len(images_to_process)}: {url}")
|
||||
image_data = await self.download_image(url, session)
|
||||
if image_data:
|
||||
url_path = urlparse(url).path
|
||||
ext = Path(url_path).suffix.lower()
|
||||
if ext not in {'.jpg', '.jpeg', '.png', '.webp', '.gif', '.bmp'}:
|
||||
ext = '.jpg'
|
||||
filename = f"image_{i:03d}{ext}"
|
||||
|
||||
if self.upload_image(image_data, image_id, filename, brand):
|
||||
key = f"daily/brands/{brand_low}/{image_id}/{filename}" if brand else f"daily/brands/products/{image_id}/{filename}"
|
||||
public_url = self.get_public_url(key)
|
||||
uploaded_urls.append(public_url)
|
||||
logger.info(f"✅ Successfully uploaded image {i+1}/{len(images_to_process)}")
|
||||
else:
|
||||
logger.warning(f"❌ Failed to upload image {i+1}/{len(images_to_process)}")
|
||||
else:
|
||||
logger.warning(f"❌ Failed to download image {i+1}/{len(images_to_process)}: No data received")
|
||||
|
||||
# Small delay to avoid overwhelming servers
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"❌ Error processing image {i+1}/{len(images_to_process)} ({url}): {e}")
|
||||
|
||||
brand_path = f"daily/brands/{brand_low}/" if brand else "daily/brands/products/"
|
||||
logger.info(f"📸 Uploaded {len(uploaded_urls)}/{len(images_to_process)} images for product {product.get('title', 'Unknown')} to {brand_path}{image_id}")
|
||||
|
||||
cache_key = f"{brand_low}:{image_id}"
|
||||
if uploaded_urls:
|
||||
self._url_cache[cache_key] = uploaded_urls
|
||||
|
||||
return image_id, uploaded_urls
|
||||
|
||||
|
||||
def get_public_url(self, key: str) -> str:
|
||||
"""Construct public URL for an S3 object"""
|
||||
if not self.enabled:
|
||||
return ""
|
||||
# For DigitalOcean Spaces with virtual-hosted style endpoint
|
||||
# Example: https://nearle.sgp1.digitaloceanspaces.com/key
|
||||
return f"{S3_ENDPOINT}/{key}"
|
||||
|
||||
def get_product_image_url(self, brand: str, image_id: str) -> str:
|
||||
"""Construct or fetch the first image URL for a product in S3."""
|
||||
if not self.enabled or not image_id or not brand:
|
||||
return ""
|
||||
storage_brand = resolve_parent_brand(brand) if brand else "products"
|
||||
brand_low = storage_brand.lower() if brand else "products"
|
||||
cache_key = f"{brand_low}:{image_id}"
|
||||
if cache_key in self._url_cache and self._url_cache[cache_key]:
|
||||
return self._url_cache[cache_key][0]
|
||||
|
||||
key = f"daily/brands/{brand_low}/{image_id}/image_000.jpg"
|
||||
return self.get_public_url(key)
|
||||
|
||||
def get_product_image_urls(self, brand: str, image_id: str) -> List[str]:
|
||||
"""List and return public URLs for all images in a product's S3 folder"""
|
||||
if not self.enabled or not image_id:
|
||||
return []
|
||||
|
||||
storage_brand = resolve_parent_brand(brand) if brand else brand
|
||||
brand_low = storage_brand.lower() if brand else "products"
|
||||
cache_key = f"{brand_low}:{image_id}"
|
||||
if cache_key in self._url_cache:
|
||||
return self._url_cache[cache_key]
|
||||
|
||||
try:
|
||||
# Try specific prefix patterns to find existing data
|
||||
prefixes = [
|
||||
f"daily/brands/{brand_low}/{image_id}/",
|
||||
f"daily/brands/products/{image_id}/",
|
||||
]
|
||||
|
||||
# If image_id is already a full path-like string, try it directly
|
||||
if '/' in image_id:
|
||||
prefixes.insert(0, f"{image_id}/" if not image_id.endswith('/') else image_id)
|
||||
|
||||
for prefix in prefixes:
|
||||
response = self.client.list_objects_v2(
|
||||
Bucket=S3_BUCKET,
|
||||
Prefix=prefix
|
||||
)
|
||||
|
||||
if 'Contents' in response:
|
||||
image_urls = []
|
||||
for obj in response['Contents']:
|
||||
key = obj['Key']
|
||||
# Filter for image files and avoid the directory prefix itself
|
||||
if key.lower().endswith(('.jpg', '.jpeg', '.png', '.webp', '.gif')) and key != prefix:
|
||||
image_urls.append(self.get_public_url(key))
|
||||
|
||||
if image_urls:
|
||||
logger.debug("Found %d images under prefix: %s", len(image_urls), prefix)
|
||||
res = sorted(image_urls)
|
||||
self._url_cache[cache_key] = res
|
||||
return res
|
||||
|
||||
logger.debug("No images found in S3 for %s across searched prefixes", image_id)
|
||||
self._url_cache[cache_key] = []
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list images from S3 for {image_id}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# Global service instance
|
||||
s3_service = S3Service()
|
||||
877
app/services/store_db.py
Normal file
877
app/services/store_db.py
Normal file
@@ -0,0 +1,877 @@
|
||||
"""
|
||||
Store Intelligence persistence layer.
|
||||
|
||||
Follows the exact same conventions as `vector_store.py` (raw psycopg,
|
||||
no ORM, `_connect()` returns None and every function degrades to an
|
||||
empty/no-op result when Postgres isn't reachable) so this module slots
|
||||
into the existing architecture rather than introducing a second data-
|
||||
access pattern. It reuses `vector_store._connect()` directly instead of
|
||||
duplicating connection logic (DRY).
|
||||
|
||||
New tables (Feature 10), additive only - nothing here touches or drops
|
||||
any existing `brand_*` table:
|
||||
stores - the 5 simulated stores
|
||||
store_inventory - per-store stock levels for each product
|
||||
store_prices - per-store MRP/cost/selling price
|
||||
orders - order headers (Feature 8)
|
||||
order_items - order line items / sales facts (Feature 8)
|
||||
discount_history - every ML discount prediction that's been served
|
||||
trending_products - cached trending-model output (Feature 6)
|
||||
demand_forecast - cached forecasting-model output (Feature 9)
|
||||
recommendations - cached recommendation-engine output (Feature 7)
|
||||
analytics_summary - periodic rollup snapshots (Feature 10)
|
||||
|
||||
Products are referenced everywhere by the composite key (brand,
|
||||
image_id) - the same natural key `vector_store.py` already treats as
|
||||
globally unique (image_id is UNIQUE NOT NULL per brand table and is
|
||||
generated from brand+product+size, so collisions across brands aren't
|
||||
expected in practice). There's no real FK constraint to a `brand_*`
|
||||
table (there isn't one shared table to reference - each brand has its
|
||||
own), so referential integrity is enforced at the application layer
|
||||
(the seed script only provisions products that actually exist).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.services.vector_store import _connect # reuse the exact same connection logic
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema
|
||||
# ---------------------------------------------------------------------------
|
||||
SCHEMA_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS stores (
|
||||
store_id TEXT PRIMARY KEY,
|
||||
store_name TEXT NOT NULL,
|
||||
city TEXT,
|
||||
tier TEXT NOT NULL DEFAULT 'standard',
|
||||
footfall_index NUMERIC NOT NULL DEFAULT 20,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS store_inventory (
|
||||
store_id TEXT NOT NULL REFERENCES stores(store_id) ON DELETE CASCADE,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
category TEXT,
|
||||
available_stock INTEGER NOT NULL DEFAULT 0,
|
||||
reserved_stock INTEGER NOT NULL DEFAULT 0,
|
||||
reorder_level INTEGER NOT NULL DEFAULT 0,
|
||||
safety_stock INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (store_id, brand, image_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_store_inventory_product ON store_inventory(brand, image_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_store_inventory_stock ON store_inventory(store_id, available_stock);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS store_prices (
|
||||
store_id TEXT NOT NULL REFERENCES stores(store_id) ON DELETE CASCADE,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
mrp NUMERIC NOT NULL,
|
||||
cost_price NUMERIC NOT NULL,
|
||||
selling_price NUMERIC NOT NULL,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (store_id, brand, image_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_store_prices_product ON store_prices(brand, image_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS orders (
|
||||
order_id TEXT PRIMARY KEY,
|
||||
customer_id TEXT NOT NULL,
|
||||
store_id TEXT NOT NULL REFERENCES stores(store_id) ON DELETE CASCADE,
|
||||
order_date TIMESTAMP NOT NULL,
|
||||
payment_method TEXT,
|
||||
order_value NUMERIC NOT NULL DEFAULT 0,
|
||||
delivery_status TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_orders_store_date ON orders(store_id, order_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_orders_customer ON orders(customer_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_orders_date ON orders(order_date);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS order_items (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
order_id TEXT NOT NULL REFERENCES orders(order_id) ON DELETE CASCADE,
|
||||
store_id TEXT NOT NULL,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
quantity INTEGER NOT NULL,
|
||||
unit_price NUMERIC NOT NULL,
|
||||
line_total NUMERIC NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_order_items_order ON order_items(order_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_order_items_product ON order_items(brand, image_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_order_items_store_product ON order_items(store_id, brand, image_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS discount_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
store_id TEXT NOT NULL,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
original_price NUMERIC NOT NULL,
|
||||
discount_pct NUMERIC NOT NULL,
|
||||
final_price NUMERIC NOT NULL,
|
||||
savings NUMERIC NOT NULL,
|
||||
model_version TEXT,
|
||||
features_json JSONB,
|
||||
predicted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_discount_history_product ON discount_history(store_id, brand, image_id, predicted_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trending_products (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
window_label TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
scope_value TEXT,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
trend_score NUMERIC NOT NULL,
|
||||
rank INTEGER NOT NULL,
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_trending_lookup ON trending_products(window_label, scope, scope_value, rank);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS demand_forecast (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
store_id TEXT NOT NULL,
|
||||
brand TEXT NOT NULL,
|
||||
image_id TEXT NOT NULL,
|
||||
forecast_avg_daily_units NUMERIC NOT NULL,
|
||||
forecast_total_units NUMERIC NOT NULL,
|
||||
horizon_days INTEGER NOT NULL DEFAULT 7,
|
||||
model_version TEXT,
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_demand_forecast_lookup ON demand_forecast(store_id, brand, image_id, computed_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS recommendations (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_brand TEXT NOT NULL,
|
||||
source_image_id TEXT NOT NULL,
|
||||
rec_brand TEXT NOT NULL,
|
||||
rec_image_id TEXT NOT NULL,
|
||||
method TEXT NOT NULL DEFAULT 'hybrid',
|
||||
similarity_score NUMERIC NOT NULL,
|
||||
signals_json JSONB,
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_recommendations_source ON recommendations(source_brand, source_image_id, similarity_score DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analytics_summary (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
scope TEXT NOT NULL,
|
||||
scope_value TEXT,
|
||||
period TEXT NOT NULL,
|
||||
metrics_json JSONB NOT NULL,
|
||||
computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_analytics_summary_lookup ON analytics_summary(scope, scope_value, period, computed_at DESC);
|
||||
"""
|
||||
|
||||
|
||||
def ensure_store_intelligence_schema() -> bool:
|
||||
"""Idempotent: safe to call on every API startup, mirrors how the
|
||||
existing brand tables are lazily ensured via `ensure_brand_schema`."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.warning("Store-intelligence schema not created: no DB connection")
|
||||
return False
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
# Migration check for pre-existing 'stores' table created under legacy schema
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = 'stores'"
|
||||
)
|
||||
stores_cols = {row[0] for row in cur.fetchall()}
|
||||
if stores_cols:
|
||||
if "store_id" not in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ADD COLUMN IF NOT EXISTS store_id TEXT")
|
||||
if "store_name" not in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ADD COLUMN IF NOT EXISTS store_name TEXT")
|
||||
if "city" not in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ADD COLUMN IF NOT EXISTS city TEXT")
|
||||
if "tier" not in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ADD COLUMN IF NOT EXISTS tier TEXT NOT NULL DEFAULT 'standard'")
|
||||
if "footfall_index" not in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ADD COLUMN IF NOT EXISTS footfall_index NUMERIC NOT NULL DEFAULT 20")
|
||||
if "name" in stores_cols:
|
||||
cur.execute("ALTER TABLE stores ALTER COLUMN name DROP NOT NULL")
|
||||
try:
|
||||
cur.execute("ALTER TABLE stores ADD CONSTRAINT stores_store_id_unique UNIQUE (store_id)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
cur.execute(SCHEMA_DDL)
|
||||
logger.info("Store-intelligence schema ensured (stores, orders, discounts, trending, forecasts, recommendations, analytics_summary)")
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"Failed to ensure store-intelligence schema: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
|
||||
def _dict_cursor(conn: psycopg.Connection):
|
||||
return conn.cursor(row_factory=dict_row)
|
||||
|
||||
|
||||
def _cast_numeric(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
|
||||
"""Postgres NUMERIC columns come back from psycopg as `decimal.Decimal`
|
||||
objects, which pandas/numpy/scikit-learn arithmetic chokes on
|
||||
(`unsupported operand type(s) for -: 'decimal.Decimal' and 'float'`).
|
||||
Every function in this module that returns a DataFrame destined for
|
||||
the `intelligence/` package's pandas/numpy feature engineering casts
|
||||
its NUMERIC columns to float here, once, right at the DB boundary -
|
||||
callers never need to think about this."""
|
||||
for col in columns:
|
||||
if col in df.columns:
|
||||
df[col] = df[col].astype(float)
|
||||
return df
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stores
|
||||
# ---------------------------------------------------------------------------
|
||||
def upsert_stores(stores: List[Dict[str, Any]]) -> int:
|
||||
"""`stores`: list of dicts with store_id, store_name, city, tier, footfall_index."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
for s in stores:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO stores (store_id, store_name, city, tier, footfall_index)
|
||||
VALUES (%(store_id)s, %(store_name)s, %(city)s, %(tier)s, %(footfall_index)s)
|
||||
ON CONFLICT (store_id) DO UPDATE SET
|
||||
store_name = EXCLUDED.store_name, city = EXCLUDED.city,
|
||||
tier = EXCLUDED.tier, footfall_index = EXCLUDED.footfall_index
|
||||
""",
|
||||
s,
|
||||
)
|
||||
return len(stores)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"upsert_stores failed: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_stores() -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute("SELECT * FROM stores ORDER BY store_id")
|
||||
rows = list(cur.fetchall())
|
||||
for r in rows:
|
||||
if r.get("footfall_index") is not None:
|
||||
r["footfall_index"] = float(r["footfall_index"])
|
||||
return rows
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"list_stores failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_store(store_id: str) -> Optional[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return None
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute("SELECT * FROM stores WHERE store_id = %s", (store_id,))
|
||||
return cur.fetchone()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_store failed: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inventory + Pricing
|
||||
# ---------------------------------------------------------------------------
|
||||
def replace_store_catalog(store_id: str, provisioned_products: List[Any], titles: Dict[str, str]) -> int:
|
||||
"""Replaces (delete + bulk insert) one store's entire inventory +
|
||||
pricing in a single transaction - used by the seed script when
|
||||
(re)provisioning stores. `titles` maps 'brand||image_id' -> product
|
||||
title, since ProvisionedProduct doesn't carry the title itself.
|
||||
"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM store_inventory WHERE store_id = %s", (store_id,))
|
||||
cur.execute("DELETE FROM store_prices WHERE store_id = %s", (store_id,))
|
||||
for p in provisioned_products:
|
||||
title = titles.get(f"{p.brand}||{p.image_id}", p.brand)
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO store_inventory
|
||||
(store_id, brand, image_id, title, category, available_stock, reserved_stock, reorder_level, safety_stock)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(store_id, p.brand, p.image_id, title, p.category, p.available_stock,
|
||||
p.reserved_stock, p.reorder_level, p.safety_stock),
|
||||
)
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO store_prices (store_id, brand, image_id, mrp, cost_price, selling_price)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(store_id, p.brand, p.image_id, p.mrp, p.cost_price, p.selling_price),
|
||||
)
|
||||
return len(provisioned_products)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"replace_store_catalog failed for {store_id}: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_store_products(
|
||||
store_id: str, category: Optional[str] = None, in_stock_only: bool = False,
|
||||
limit: int = 50, offset: int = 0,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Joined inventory + pricing rows for one store, newest-priced-first
|
||||
isn't meaningful here so ordered by title for a stable, browsable
|
||||
listing."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
sql = """
|
||||
SELECT i.store_id, i.brand, i.image_id, i.title, i.category,
|
||||
i.available_stock, i.reserved_stock, i.reorder_level, i.safety_stock,
|
||||
p.mrp, p.cost_price, p.selling_price
|
||||
FROM store_inventory i
|
||||
JOIN store_prices p ON p.store_id = i.store_id AND p.brand = i.brand AND p.image_id = i.image_id
|
||||
WHERE i.store_id = %s
|
||||
"""
|
||||
params: List[Any] = [store_id]
|
||||
if category:
|
||||
sql += " AND i.category ILIKE %s"
|
||||
params.append(f"%{category}%")
|
||||
if in_stock_only:
|
||||
sql += " AND i.available_stock > 0"
|
||||
sql += " ORDER BY i.title LIMIT %s OFFSET %s"
|
||||
params.extend([limit, offset])
|
||||
cur.execute(sql, params)
|
||||
rows = list(cur.fetchall())
|
||||
for r in rows:
|
||||
for f in ("mrp", "cost_price", "selling_price"):
|
||||
if r.get(f) is not None:
|
||||
r[f] = float(r[f])
|
||||
return rows
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_store_products failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_product_across_stores(brand: str, image_id: str) -> List[Dict[str, Any]]:
|
||||
"""Every store that stocks this product, with that store's own price
|
||||
and stock - powers the "compare this product across stores" view."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT i.store_id, s.store_name, s.tier, i.available_stock, i.reorder_level,
|
||||
i.safety_stock, p.mrp, p.cost_price, p.selling_price
|
||||
FROM store_inventory i
|
||||
JOIN store_prices p ON p.store_id = i.store_id AND p.brand = i.brand AND p.image_id = i.image_id
|
||||
JOIN stores s ON s.store_id = i.store_id
|
||||
WHERE i.brand = %s AND i.image_id = %s
|
||||
ORDER BY p.selling_price
|
||||
""",
|
||||
(brand, image_id),
|
||||
)
|
||||
rows = list(cur.fetchall())
|
||||
for r in rows:
|
||||
for f in ("mrp", "cost_price", "selling_price", "footfall_index"):
|
||||
if r.get(f) is not None:
|
||||
r[f] = float(r[f])
|
||||
return rows
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_product_across_stores failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_all_store_products_df() -> pd.DataFrame:
|
||||
"""Every (store, product) inventory+price row across all stores, as a
|
||||
DataFrame - the primary input to feature engineering for the
|
||||
discount/forecast models."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return pd.DataFrame()
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT i.store_id, s.tier AS store_tier, i.brand, i.image_id, i.title, i.category,
|
||||
i.available_stock, i.reserved_stock, i.reorder_level, i.safety_stock,
|
||||
p.mrp, p.cost_price, p.selling_price
|
||||
FROM store_inventory i
|
||||
JOIN store_prices p ON p.store_id = i.store_id AND p.brand = i.brand AND p.image_id = i.image_id
|
||||
JOIN stores s ON s.store_id = i.store_id
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return _cast_numeric(pd.DataFrame(rows), ["mrp", "cost_price", "selling_price"])
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_all_store_products_df failed: {e}")
|
||||
return pd.DataFrame()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orders (Feature 8)
|
||||
# ---------------------------------------------------------------------------
|
||||
def bulk_insert_orders(orders_df: pd.DataFrame, order_items_df: pd.DataFrame) -> Dict[str, int]:
|
||||
conn = _connect()
|
||||
if not conn or orders_df.empty:
|
||||
return {"orders": 0, "order_items": 0}
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO orders (order_id, customer_id, store_id, order_date, payment_method, order_value, delivery_status)
|
||||
VALUES (%(order_id)s, %(customer_id)s, %(store_id)s, %(order_date)s, %(payment_method)s, %(order_value)s, %(delivery_status)s)
|
||||
ON CONFLICT (order_id) DO NOTHING
|
||||
""",
|
||||
orders_df.to_dict("records"),
|
||||
)
|
||||
if not order_items_df.empty:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO order_items (order_id, store_id, brand, image_id, quantity, unit_price, line_total)
|
||||
VALUES (%(order_id)s, %(store_id)s, %(brand)s, %(image_id)s, %(quantity)s, %(unit_price)s, %(line_total)s)
|
||||
""",
|
||||
order_items_df.to_dict("records"),
|
||||
)
|
||||
return {"orders": len(orders_df), "order_items": len(order_items_df)}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"bulk_insert_orders failed: {e}")
|
||||
return {"orders": 0, "order_items": 0}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def clear_orders() -> None:
|
||||
"""Used by the seed script's `--reset` flag before re-simulating
|
||||
order history, so re-runs don't duplicate data."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM order_items")
|
||||
cur.execute("DELETE FROM orders")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"clear_orders failed: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_orders_df(store_id: Optional[str] = None, start_date: Optional[date] = None,
|
||||
end_date: Optional[date] = None) -> pd.DataFrame:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return pd.DataFrame()
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
sql = "SELECT * FROM orders WHERE 1=1"
|
||||
params: List[Any] = []
|
||||
if store_id:
|
||||
sql += " AND store_id = %s"
|
||||
params.append(store_id)
|
||||
if start_date:
|
||||
sql += " AND order_date >= %s"
|
||||
params.append(start_date)
|
||||
if end_date:
|
||||
sql += " AND order_date <= %s"
|
||||
params.append(end_date)
|
||||
cur.execute(sql, params)
|
||||
df = pd.DataFrame(cur.fetchall())
|
||||
if not df.empty:
|
||||
df["order_date"] = pd.to_datetime(df["order_date"])
|
||||
df = _cast_numeric(df, ["order_value"])
|
||||
return df
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_orders_df failed: {e}")
|
||||
return pd.DataFrame()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_order_items_df(store_id: Optional[str] = None) -> pd.DataFrame:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return pd.DataFrame()
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
sql = """
|
||||
SELECT oi.*, o.order_date, o.customer_id
|
||||
FROM order_items oi JOIN orders o ON o.order_id = oi.order_id
|
||||
"""
|
||||
params: List[Any] = []
|
||||
if store_id:
|
||||
sql += " WHERE oi.store_id = %s"
|
||||
params.append(store_id)
|
||||
cur.execute(sql, params)
|
||||
df = pd.DataFrame(cur.fetchall())
|
||||
if not df.empty:
|
||||
df["order_date"] = pd.to_datetime(df["order_date"])
|
||||
df = _cast_numeric(df, ["unit_price", "line_total"])
|
||||
return df
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_order_items_df failed: {e}")
|
||||
return pd.DataFrame()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discount history (Feature 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
def log_discounts(rows: List[Dict[str, Any]]) -> int:
|
||||
"""`rows`: store_id, brand, image_id, original_price, discount_pct,
|
||||
final_price, savings, model_version, features (dict)."""
|
||||
conn = _connect()
|
||||
if not conn or not rows:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO discount_history
|
||||
(store_id, brand, image_id, original_price, discount_pct, final_price, savings, model_version, features_json)
|
||||
VALUES (%(store_id)s, %(brand)s, %(image_id)s, %(original_price)s, %(discount_pct)s,
|
||||
%(final_price)s, %(savings)s, %(model_version)s, %(features_json)s)
|
||||
""",
|
||||
[{**r, "features_json": json.dumps(r.get("features", {}))} for r in rows],
|
||||
)
|
||||
return len(rows)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"log_discounts failed: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_latest_discounts(store_id: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT ON (brand, image_id) *
|
||||
FROM discount_history
|
||||
WHERE store_id = %s
|
||||
ORDER BY brand, image_id, predicted_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(store_id, limit),
|
||||
)
|
||||
return list(cur.fetchall())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_latest_discounts failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trending cache (Feature 6)
|
||||
# ---------------------------------------------------------------------------
|
||||
def save_trending(window_label: str, scope: str, scope_value: Optional[str], ranked: List[Dict[str, Any]]) -> int:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"DELETE FROM trending_products WHERE window_label = %s AND scope = %s AND scope_value IS NOT DISTINCT FROM %s",
|
||||
(window_label, scope, scope_value),
|
||||
)
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO trending_products (window_label, scope, scope_value, brand, image_id, trend_score, rank)
|
||||
VALUES (%(window_label)s, %(scope)s, %(scope_value)s, %(brand)s, %(image_id)s, %(trend_score)s, %(rank)s)
|
||||
""",
|
||||
[{**r, "window_label": window_label, "scope": scope, "scope_value": scope_value} for r in ranked],
|
||||
)
|
||||
return len(ranked)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_trending failed: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_trending(window_label: str, scope: str, scope_value: Optional[str] = None, top_k: int = 10) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT * FROM trending_products
|
||||
WHERE window_label = %s AND scope = %s AND scope_value IS NOT DISTINCT FROM %s
|
||||
ORDER BY rank ASC LIMIT %s
|
||||
""",
|
||||
(window_label, scope, scope_value, top_k),
|
||||
)
|
||||
return list(cur.fetchall())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_trending failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Demand forecast cache (Feature 9)
|
||||
# ---------------------------------------------------------------------------
|
||||
def save_demand_forecasts(rows: List[Dict[str, Any]]) -> int:
|
||||
conn = _connect()
|
||||
if not conn or not rows:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO demand_forecast
|
||||
(store_id, brand, image_id, forecast_avg_daily_units, forecast_total_units, horizon_days, model_version)
|
||||
VALUES (%(store_id)s, %(brand)s, %(image_id)s, %(forecast_avg_daily_units)s,
|
||||
%(forecast_total_units)s, %(horizon_days)s, %(model_version)s)
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
return len(rows)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_demand_forecasts failed: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_latest_demand_forecast(store_id: str, brand: str, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return None
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT * FROM demand_forecast
|
||||
WHERE store_id = %s AND brand = %s AND image_id = %s
|
||||
ORDER BY computed_at DESC LIMIT 1
|
||||
""",
|
||||
(store_id, brand, image_id),
|
||||
)
|
||||
return cur.fetchone()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_latest_demand_forecast failed: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Recommendations cache (Feature 7)
|
||||
# ---------------------------------------------------------------------------
|
||||
def save_recommendations(source_brand: str, source_image_id: str, recs: List[Dict[str, Any]]) -> int:
|
||||
conn = _connect()
|
||||
if not conn or not recs:
|
||||
return 0
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"DELETE FROM recommendations WHERE source_brand = %s AND source_image_id = %s",
|
||||
(source_brand, source_image_id),
|
||||
)
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO recommendations (source_brand, source_image_id, rec_brand, rec_image_id, method, similarity_score, signals_json)
|
||||
VALUES (%(source_brand)s, %(source_image_id)s, %(rec_brand)s, %(rec_image_id)s, %(method)s, %(similarity_score)s, %(signals_json)s)
|
||||
""",
|
||||
[{**r, "source_brand": source_brand, "source_image_id": source_image_id,
|
||||
"signals_json": json.dumps(r.get("signals", {}))} for r in recs],
|
||||
)
|
||||
return len(recs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_recommendations failed: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_cached_recommendations(source_brand: str, source_image_id: str) -> List[Dict[str, Any]]:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
try:
|
||||
with conn, _dict_cursor(conn) as cur:
|
||||
cur.execute(
|
||||
"SELECT * FROM recommendations WHERE source_brand = %s AND source_image_id = %s ORDER BY similarity_score DESC",
|
||||
(source_brand, source_image_id),
|
||||
)
|
||||
return list(cur.fetchall())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_cached_recommendations failed: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_products_with_embeddings(category: Optional[str] = None, brands: Optional[List[str]] = None) -> pd.DataFrame:
|
||||
"""Fetch (brand, image_id, title, category, embedding) for every
|
||||
product across all brand tables, optionally narrowed by category or
|
||||
a brand allowlist to bound the candidate set.
|
||||
|
||||
WHY THIS EXISTS INSTEAD OF `vector_store.semantic_search`
|
||||
------------------------------------------------------------
|
||||
Integration-testing the recommendation engine against a real
|
||||
Postgres instance surfaced a pre-existing issue in the RAG
|
||||
pipeline: `get_brand_table_ddl` creates an IVFFLAT index with
|
||||
pgvector's default `lists` parameter on every brand table
|
||||
regardless of size. IVFFLAT's accuracy depends on `lists` being
|
||||
tuned to the row count (roughly sqrt(rows)) and `probes` being
|
||||
raised accordingly; with the default settings on a table of a few
|
||||
dozen-to-low-hundreds rows (typical brand catalog size here), most
|
||||
of the index's clusters end up nearly empty and a similarity query
|
||||
can silently return zero or near-zero matches even though a plain
|
||||
sequential scan finds them fine - confirmed directly against this
|
||||
project's own seeded data during testing.
|
||||
|
||||
Rather than modifying `vector_store.py` / the existing RAG search
|
||||
path (outside this task's scope, and risky to touch a working,
|
||||
unrelated pipeline stage), the recommendation engine fetches its
|
||||
- naturally bounded once filtered by category/brand - candidate
|
||||
pool with a plain sequential scan here and computes cosine
|
||||
similarity itself in `recommendation_engine.embedding_similarity_to_source`.
|
||||
This sidesteps the ANN index entirely and is actually more
|
||||
reliable at this catalog scale, not just a workaround. If the
|
||||
catalog grows into the tens of thousands of products per brand,
|
||||
revisit `get_brand_table_ddl` (tune `lists`, raise `probes`, or
|
||||
switch to an HNSW index, which doesn't need row-count tuning) and
|
||||
this function can go back to using `semantic_search` for a bigger
|
||||
speed win.
|
||||
"""
|
||||
from app.services.vector_store import list_available_brands, _table_name, _table_exists
|
||||
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return pd.DataFrame()
|
||||
target_brands = brands or list_available_brands()
|
||||
rows: List[Dict[str, Any]] = []
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
for brand in target_brands:
|
||||
table = _table_name(brand)
|
||||
if not _table_exists(cur, table):
|
||||
continue
|
||||
sql = f"SELECT image_id, title, category, embedding::text AS embedding_text FROM {table} WHERE embedding IS NOT NULL"
|
||||
params: List[Any] = []
|
||||
if category:
|
||||
sql += " AND category ILIKE %s"
|
||||
params.append(f"%{category}%")
|
||||
try:
|
||||
cur.execute(sql, params)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(f"get_products_with_embeddings failed for {table}: {e}")
|
||||
continue
|
||||
colnames = [d[0] for d in cur.description]
|
||||
for row in cur.fetchall():
|
||||
rec = dict(zip(colnames, row))
|
||||
rec["brand"] = brand # brand_* tables don't store a brand column - it's implicit in the table itself
|
||||
if rec.get("embedding_text"):
|
||||
rec["embedding"] = json.loads(rec.pop("embedding_text"))
|
||||
rows.append(rec)
|
||||
return pd.DataFrame(rows)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_products_with_embeddings failed: {e}")
|
||||
return pd.DataFrame()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_embedding_vector(brand: str, image_id: str) -> Optional[List[float]]:
|
||||
"""Fetch a single product's stored embedding as a plain Python list.
|
||||
Cast to ::text in SQL rather than relying on the pgvector Python
|
||||
type adapter (not registered anywhere else in this codebase either -
|
||||
`vector_store.py` only ever uses the embedding inside SQL via `<=>`,
|
||||
never fetches it back into Python, so this mirrors that same
|
||||
text-cast-and-parse approach for the one place we do need it back)."""
|
||||
from app.services.vector_store import _table_name, _table_exists
|
||||
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return None
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
table = _table_name(brand)
|
||||
if not _table_exists(cur, table):
|
||||
return None
|
||||
cur.execute(f"SELECT embedding::text FROM {table} WHERE image_id = %s", (image_id,))
|
||||
row = cur.fetchone()
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
return json.loads(row[0])
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"get_embedding_vector failed for {brand}/{image_id}: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Analytics snapshots (Feature 10)
|
||||
# ---------------------------------------------------------------------------
|
||||
def save_analytics_snapshot(scope: str, scope_value: Optional[str], period: str, metrics: Dict[str, Any]) -> bool:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return False
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO analytics_summary (scope, scope_value, period, metrics_json) VALUES (%s, %s, %s, %s)",
|
||||
(scope, scope_value, period, json.dumps(metrics)),
|
||||
)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"save_analytics_snapshot failed: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
76
app/services/store_seed_service.py
Normal file
76
app/services/store_seed_service.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
Orchestrates Feature 1/2/8 seeding: provision the 5 stores with a
|
||||
random product subset + independent pricing/stock, then simulate order
|
||||
history. Called from both `scripts/seed_store_intelligence.py` (CLI,
|
||||
the primary/documented way to run this) and the
|
||||
`/api/admin/store-intelligence/seed` endpoint (convenience for
|
||||
demoing without shell access) - kept in one place so the two entry
|
||||
points can't drift apart.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
from typing import Dict
|
||||
|
||||
from app.intelligence.order_simulation import StoreCatalogEntry, simulate_orders
|
||||
from app.intelligence.store_provisioning import DEFAULT_STORES, ProductRef, provision_stores
|
||||
from app.services import store_db
|
||||
from app.services.recommendation_service import refresh_copurchase_cache
|
||||
from app.services.vector_store import get_products_all_brands
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_seed(reset_orders: bool = True, days: int = 90, seed: int = 42) -> Dict:
|
||||
store_db.ensure_store_intelligence_schema()
|
||||
|
||||
logger.info("Fetching full catalog across all brands...")
|
||||
products_raw = get_products_all_brands(limit=20000)
|
||||
if not products_raw:
|
||||
raise RuntimeError(
|
||||
"No products found across any brand table. Ingest a catalog first "
|
||||
"(POST /api/catalog/generate or scripts/seed_sample_data.py) before "
|
||||
"seeding store intelligence."
|
||||
)
|
||||
refs = [
|
||||
ProductRef(p["brand"], p["image_id"], p.get("title") or p.get("product_name"), p.get("category"), p.get("price_range"))
|
||||
for p in products_raw
|
||||
]
|
||||
titles = {f"{r.brand}||{r.image_id}": r.title for r in refs}
|
||||
|
||||
logger.info("Provisioning %d stores over %d products...", len(DEFAULT_STORES), len(refs))
|
||||
stores_meta = [
|
||||
{"store_id": s.store_id, "store_name": s.store_name, "city": s.city, "tier": s.tier, "footfall_index": s.footfall_index}
|
||||
for s in DEFAULT_STORES
|
||||
]
|
||||
store_db.upsert_stores(stores_meta)
|
||||
|
||||
provisioned = provision_stores(refs, seed=seed)
|
||||
store_product_counts: Dict[str, int] = {}
|
||||
for store_id, items in provisioned.items():
|
||||
store_product_counts[store_id] = store_db.replace_store_catalog(store_id, items, titles)
|
||||
|
||||
logger.info("Simulating %d days of order history...", days)
|
||||
store_products_df = store_db.get_all_store_products_df()
|
||||
store_catalogs = {}
|
||||
for store_id, g in store_products_df.groupby("store_id"):
|
||||
store_catalogs[store_id] = [
|
||||
StoreCatalogEntry(store_id, r["brand"], r["image_id"], r["category"], float(r["selling_price"]), int(r["available_stock"]))
|
||||
for _, r in g.iterrows()
|
||||
]
|
||||
end = date.today()
|
||||
start = end - timedelta(days=days)
|
||||
result = simulate_orders(DEFAULT_STORES, store_catalogs, start, end, seed=seed)
|
||||
|
||||
if reset_orders:
|
||||
store_db.clear_orders()
|
||||
counts = store_db.bulk_insert_orders(result.orders, result.order_items)
|
||||
refresh_copurchase_cache()
|
||||
|
||||
return {
|
||||
"stores": len(DEFAULT_STORES),
|
||||
"store_products": store_product_counts,
|
||||
"orders": counts["orders"],
|
||||
"order_items": counts["order_items"],
|
||||
}
|
||||
87
app/services/trending_service.py
Normal file
87
app/services/trending_service.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
Thin I/O orchestration for Feature 6 (Trending Product Detection).
|
||||
Computes trend features + scores live from order history, ranks within
|
||||
the requested scope (overall / category / store), and caches the top-N
|
||||
into `trending_products` so repeat reads for the same window/scope are
|
||||
a cheap DB lookup instead of recomputation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from app.intelligence.trending_model import Window, compute_trend_features, trending_scorer
|
||||
from app.services import store_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def compute_and_cache_trending(window: Window, top_k: int = 10) -> Dict[str, int]:
|
||||
"""Recomputes and caches every trending scope for one window:
|
||||
overall, each category, and each store. Meant to be run by the
|
||||
training/seed script (or an admin trigger) rather than on every
|
||||
request - the cached reads below serve actual traffic."""
|
||||
order_items = store_db.get_order_items_df()
|
||||
orders = store_db.get_orders_df()
|
||||
if order_items.empty:
|
||||
return {}
|
||||
|
||||
counts: Dict[str, int] = {}
|
||||
|
||||
# Overall (pooled across stores)
|
||||
overall = compute_trend_features(order_items, orders, date.today(), window, ["brand", "image_id"])
|
||||
overall["trend_score"] = trending_scorer.score(overall).values
|
||||
ranked = _rank(overall)
|
||||
store_db.save_trending(window, "overall", None, ranked[:top_k])
|
||||
counts["overall"] = len(ranked)
|
||||
|
||||
# Category-wise: join category in via store_products, then re-rank within each category
|
||||
store_products = store_db.get_all_store_products_df()
|
||||
if not store_products.empty:
|
||||
cat_map = store_products.drop_duplicates(["brand", "image_id"]).set_index(["brand", "image_id"])["category"]
|
||||
overall_with_cat = overall.copy()
|
||||
overall_with_cat["category"] = overall_with_cat.apply(
|
||||
lambda r: cat_map.get((r["brand"], r["image_id"]), "Uncategorized"), axis=1
|
||||
)
|
||||
for category, g in overall_with_cat.groupby("category"):
|
||||
ranked_cat = _rank(g)
|
||||
store_db.save_trending(window, "category", category, ranked_cat[:top_k])
|
||||
counts[f"category:{category}"] = len(ranked_cat)
|
||||
|
||||
# Store-wise
|
||||
store_wise = compute_trend_features(order_items, orders, date.today(), window, ["store_id", "brand", "image_id"])
|
||||
if not store_wise.empty:
|
||||
store_wise["trend_score"] = trending_scorer.score(store_wise).values
|
||||
for store_id, g in store_wise.groupby("store_id"):
|
||||
ranked_store = _rank(g)
|
||||
store_db.save_trending(window, "store", store_id, ranked_store[:top_k])
|
||||
counts[f"store:{store_id}"] = len(ranked_store)
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
def _rank(df: pd.DataFrame) -> List[Dict]:
|
||||
ranked = df.sort_values("trend_score", ascending=False).reset_index(drop=True)
|
||||
return [
|
||||
{"brand": r["brand"], "image_id": r["image_id"], "trend_score": round(float(r["trend_score"]), 2), "rank": i + 1}
|
||||
for i, r in ranked.iterrows()
|
||||
]
|
||||
|
||||
|
||||
def get_trending(window: Window, scope: str = "overall", scope_value: Optional[str] = None, top_k: int = 10) -> List[Dict]:
|
||||
cached = store_db.get_trending(window, scope, scope_value, top_k)
|
||||
if cached:
|
||||
return [{"brand": c["brand"], "image_id": c["image_id"], "trend_score": float(c["trend_score"]), "rank": c["rank"]} for c in cached]
|
||||
# No cache yet (e.g. training script hasn't run) - compute this one
|
||||
# scope live rather than returning an empty/hardcoded list.
|
||||
order_items = store_db.get_order_items_df(store_id=scope_value if scope == "store" else None)
|
||||
orders = store_db.get_orders_df()
|
||||
group_cols = ["store_id", "brand", "image_id"] if scope == "store" else ["brand", "image_id"]
|
||||
feat = compute_trend_features(order_items, orders, date.today(), window, group_cols)
|
||||
if feat.empty:
|
||||
return []
|
||||
feat["trend_score"] = trending_scorer.score(feat).values
|
||||
return _rank(feat)[:top_k]
|
||||
800
app/services/vector_store.py
Normal file
800
app/services/vector_store.py
Normal file
@@ -0,0 +1,800 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional, Dict, Any
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import psycopg
|
||||
|
||||
from app.infrastructure.settings import DATABASE_URL, USE_PGVECTOR, DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD
|
||||
from app.services.brand_registry import BRAND_ALIASES, resolve_parent_brand
|
||||
|
||||
|
||||
from app.services.s3_service import s3_service
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _sanitize_name(name: str) -> str:
|
||||
"""Sanitize a brand name for use as a PostgreSQL table name suffix.
|
||||
|
||||
Replaces non-alphanumeric characters (except underscore) to prevent
|
||||
SQL syntax errors from special characters like &, ', ., etc.
|
||||
"""
|
||||
name = name.lower().replace(' ', '_').replace('-', '_').replace('&', '_')
|
||||
name = re.sub(r'[^a-z0-9_]', '', name)
|
||||
name = re.sub(r'_+', '_', name)
|
||||
return name.strip('_')
|
||||
|
||||
|
||||
DDL_CREATE_EXTENSION = "CREATE EXTENSION IF NOT EXISTS vector;"
|
||||
def get_brand_table_ddl(brand: str) -> str:
|
||||
"""Generate DDL for brand-specific table - simplified with only essential fields"""
|
||||
storage_brand = resolve_parent_brand(brand)
|
||||
table_name = f"brand_{_sanitize_name(storage_brand)}"
|
||||
return f"""
|
||||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
product_name TEXT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
category TEXT,
|
||||
image_id TEXT UNIQUE NOT NULL,
|
||||
image_url TEXT,
|
||||
image_urls TEXT[],
|
||||
|
||||
-- Essential pricing fields
|
||||
price_range TEXT,
|
||||
size_variants TEXT[],
|
||||
providers TEXT[],
|
||||
|
||||
-- FSSAI license
|
||||
fssai_license TEXT,
|
||||
|
||||
-- Product SKU & Tax/Price/Barcode details
|
||||
product_sku TEXT,
|
||||
sku_source TEXT,
|
||||
hsn_code TEXT,
|
||||
final_selling_price NUMERIC,
|
||||
selling_price NUMERIC,
|
||||
barcode TEXT,
|
||||
barcode_type TEXT,
|
||||
|
||||
-- Essential fields
|
||||
highlights TEXT[],
|
||||
nutrients TEXT[],
|
||||
search_query TEXT,
|
||||
|
||||
-- Vector embedding for search
|
||||
embedding vector(384),
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_{table_name}_image_id ON {table_name}(image_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_{table_name}_embedding ON {table_name} USING ivfflat (embedding vector_cosine_ops);
|
||||
"""
|
||||
|
||||
|
||||
def _connect() -> Optional[psycopg.Connection]:
|
||||
if not USE_PGVECTOR:
|
||||
return None
|
||||
try:
|
||||
# Use direct connection parameters instead of URL
|
||||
return psycopg.connect(
|
||||
host=DB_HOST,
|
||||
port=DB_PORT,
|
||||
dbname=DB_NAME,
|
||||
user=DB_USER,
|
||||
password=DB_PASSWORD,
|
||||
autocommit=True
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Vector DB connection failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_columns(cur, table_name: str) -> None:
|
||||
"""Add missing columns and relax legacy NOT NULL constraints for smooth schema migration."""
|
||||
col_defs = {
|
||||
"product_name": "TEXT",
|
||||
"title": "TEXT",
|
||||
"description": "TEXT",
|
||||
"category": "TEXT",
|
||||
"image_id": "TEXT",
|
||||
"image_url": "TEXT",
|
||||
"image_urls": "TEXT[]",
|
||||
"price_range": "TEXT",
|
||||
"size_variants": "TEXT[]",
|
||||
"providers": "TEXT[]",
|
||||
"fssai_license": "TEXT",
|
||||
"product_sku": "TEXT",
|
||||
"sku_source": "TEXT",
|
||||
"hsn_code": "TEXT",
|
||||
"final_selling_price": "NUMERIC",
|
||||
"selling_price": "NUMERIC",
|
||||
"barcode": "TEXT",
|
||||
"barcode_type": "TEXT",
|
||||
"highlights": "TEXT[]",
|
||||
"nutrients": "TEXT[]",
|
||||
"search_query": "TEXT",
|
||||
"embedding": "vector(384)",
|
||||
}
|
||||
cur.execute(
|
||||
f"SELECT column_name, is_nullable, column_default FROM information_schema.columns "
|
||||
f"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table_name,),
|
||||
)
|
||||
col_info = cur.fetchall()
|
||||
existing = {row[0] for row in col_info}
|
||||
|
||||
# 1. Add missing columns
|
||||
for col, col_type in col_defs.items():
|
||||
if col not in existing:
|
||||
cur.execute(f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {col} {col_type}")
|
||||
if col not in ("created_at", "updated_at"):
|
||||
logger.info(f"Added missing column '{col}' to {table_name}")
|
||||
|
||||
# 2. Relax legacy NOT NULL constraints on columns not present in standard insert
|
||||
inserted_cols = {"id", "product_name", "title", "description", "category", "image_id", "image_url", "image_urls", "price_range", "size_variants", "providers", "fssai_license", "product_sku", "sku_source", "hsn_code", "final_selling_price", "selling_price", "barcode", "barcode_type", "highlights", "nutrients", "search_query", "embedding", "created_at", "updated_at"}
|
||||
for col, is_nullable, col_def in col_info:
|
||||
if col not in inserted_cols and is_nullable == 'NO' and col_def is None:
|
||||
cur.execute(f"ALTER TABLE {table_name} ALTER COLUMN {col} DROP NOT NULL")
|
||||
logger.info(f"Dropped NOT NULL constraint on legacy column '{col}' in {table_name}")
|
||||
|
||||
# 3. Ensure UNIQUE index on image_id so ON CONFLICT (image_id) works on pre-existing tables
|
||||
try:
|
||||
cur.execute(
|
||||
f"DELETE FROM {table_name} a USING {table_name} b "
|
||||
f"WHERE a.id < b.id AND a.image_id = b.image_id AND a.image_id IS NOT NULL AND a.image_id != ''"
|
||||
)
|
||||
cur.execute(f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{table_name}_image_id_unique ON {table_name}(image_id)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Unique index creation on {table_name}.image_id: {e}")
|
||||
|
||||
|
||||
def ensure_brand_schema(brand: str) -> str:
|
||||
"""Ensure brand-specific table exists and return table name"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return ""
|
||||
|
||||
storage_brand = resolve_parent_brand(brand)
|
||||
table_name = f"brand_{_sanitize_name(storage_brand)}"
|
||||
|
||||
with conn, conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(DDL_CREATE_EXTENSION)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cur.execute(get_brand_table_ddl(brand))
|
||||
_ensure_columns(cur, table_name)
|
||||
logger.info(f"✅ Ensured table {table_name} exists")
|
||||
|
||||
conn.close()
|
||||
return table_name
|
||||
|
||||
|
||||
def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: bool = False) -> None:
|
||||
"""Insert products into brand-specific table - simplified with only essential fields
|
||||
|
||||
When `cleanup=True`, any products in the table whose image_id is NOT in the
|
||||
provided `products` list are deleted after the upsert. This ensures the
|
||||
database exactly reflects the source data. The caller is responsible for
|
||||
providing the complete set of products for the brand when using cleanup.
|
||||
"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return
|
||||
|
||||
table_name = ensure_brand_schema(brand)
|
||||
if not table_name:
|
||||
return
|
||||
|
||||
rows = []
|
||||
for p in products:
|
||||
# Extract only essential fields
|
||||
product_name = p.get("product_name") or p.get("title") or ""
|
||||
title = p.get("title") or product_name
|
||||
description = p.get("description") or ""
|
||||
category = p.get("category") or "Uncategorized"
|
||||
image_id = p.get("image_id") or ""
|
||||
|
||||
image_url = p.get("image_url") or p.get("primary_image") or ""
|
||||
if isinstance(image_url, str):
|
||||
image_url = image_url.replace('{width}', '800')
|
||||
else:
|
||||
image_url = ""
|
||||
|
||||
raw_image_urls = p.get("image_urls") or []
|
||||
if isinstance(raw_image_urls, list):
|
||||
image_urls = [str(u).replace('{width}', '800') for u in raw_image_urls if u]
|
||||
else:
|
||||
image_urls = []
|
||||
if not image_urls and image_url:
|
||||
image_urls = [image_url]
|
||||
if not image_urls and not image_url and image_id and s3_service.enabled:
|
||||
s3_single = s3_service.get_product_image_url(brand, image_id)
|
||||
if s3_single:
|
||||
image_url = s3_single
|
||||
image_urls = [s3_single]
|
||||
|
||||
# Essential pricing fields
|
||||
price_range = p.get("price_range") or ""
|
||||
size_variants = p.get("size_variants", [])
|
||||
providers = p.get("providers", [])
|
||||
|
||||
# Convert size_variants objects to strings for TEXT[] storage
|
||||
if size_variants and isinstance(size_variants, list):
|
||||
size_variants_str = []
|
||||
for variant in size_variants:
|
||||
if isinstance(variant, dict):
|
||||
# Convert dict to string format
|
||||
size = variant.get('size', '')
|
||||
price = variant.get('price', '')
|
||||
if size and price:
|
||||
size_variants_str.append(f"{size} - {price}")
|
||||
elif size:
|
||||
size_variants_str.append(size)
|
||||
elif isinstance(variant, str):
|
||||
size_variants_str.append(variant)
|
||||
size_variants = size_variants_str
|
||||
|
||||
# Product SKU & HSN / Price / Barcode fields
|
||||
product_sku = p.get("product_sku") or ""
|
||||
sku_source = p.get("sku_source") or ""
|
||||
|
||||
hsn_code = str(p.get("hsn_code") or p.get("HSN_Code") or p.get("hsn") or "").strip() or None
|
||||
|
||||
raw_fsp = p.get("final_selling_price") if "final_selling_price" in p else p.get("Final_Selling_Price")
|
||||
if raw_fsp is None:
|
||||
raw_fsp = p.get("final_price")
|
||||
try:
|
||||
final_selling_price = float(raw_fsp) if raw_fsp is not None and str(raw_fsp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
final_selling_price = None
|
||||
|
||||
raw_sp = p.get("selling_price") if "selling_price" in p else p.get("Selling_Price")
|
||||
try:
|
||||
selling_price = float(raw_sp) if raw_sp is not None and str(raw_sp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
selling_price = None
|
||||
|
||||
if final_selling_price is None and selling_price is not None:
|
||||
final_selling_price = selling_price
|
||||
|
||||
barcode = str(p.get("barcode") or p.get("Barcode") or "").strip() or None
|
||||
barcode_type = str(p.get("barcode_type") or p.get("Barcode_Type") or "").strip() or None
|
||||
|
||||
# Essential fields
|
||||
highlights = p.get("highlights", [])
|
||||
if not isinstance(highlights, list):
|
||||
highlights = []
|
||||
|
||||
# Nutrients field
|
||||
nutrients = p.get("nutrients", [])
|
||||
if not isinstance(nutrients, list):
|
||||
nutrients = []
|
||||
|
||||
fssai_license = str(p.get("fssai_license", "")) if p.get("fssai_license") else ""
|
||||
search_query = p.get("search_query") or ""
|
||||
|
||||
# Convert embedding to PostgreSQL vector format
|
||||
embedding = p.get("embedding")
|
||||
if embedding and isinstance(embedding, list):
|
||||
# Convert list to string format for PostgreSQL vector type
|
||||
embedding_str = "[" + ",".join(map(str, embedding)) + "]"
|
||||
else:
|
||||
embedding_str = None
|
||||
|
||||
# Only essential fields
|
||||
rows.append((
|
||||
product_name,
|
||||
title,
|
||||
description,
|
||||
category,
|
||||
image_id,
|
||||
image_url,
|
||||
image_urls,
|
||||
price_range,
|
||||
size_variants, # TEXT[] - psycopg will handle conversion
|
||||
providers, # TEXT[] - psycopg will handle conversion
|
||||
fssai_license,
|
||||
product_sku,
|
||||
sku_source,
|
||||
hsn_code,
|
||||
final_selling_price,
|
||||
selling_price,
|
||||
barcode,
|
||||
barcode_type,
|
||||
highlights, # TEXT[] - psycopg will handle conversion
|
||||
nutrients, # TEXT[] - psycopg will handle conversion
|
||||
search_query,
|
||||
embedding_str
|
||||
))
|
||||
|
||||
image_ids = [r[4] for r in rows if r[4]]
|
||||
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
f"""
|
||||
INSERT INTO {table_name}
|
||||
(product_name, title, description, category, image_id, image_url, image_urls, price_range, size_variants, providers,
|
||||
fssai_license, product_sku, sku_source, hsn_code, final_selling_price, selling_price, barcode, barcode_type, highlights, nutrients, search_query, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (image_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
title = EXCLUDED.title,
|
||||
description = EXCLUDED.description,
|
||||
category = EXCLUDED.category,
|
||||
image_url = EXCLUDED.image_url,
|
||||
image_urls = EXCLUDED.image_urls,
|
||||
price_range = EXCLUDED.price_range,
|
||||
size_variants = EXCLUDED.size_variants,
|
||||
providers = EXCLUDED.providers,
|
||||
fssai_license = EXCLUDED.fssai_license,
|
||||
product_sku = EXCLUDED.product_sku,
|
||||
sku_source = EXCLUDED.sku_source,
|
||||
hsn_code = EXCLUDED.hsn_code,
|
||||
final_selling_price = EXCLUDED.final_selling_price,
|
||||
selling_price = EXCLUDED.selling_price,
|
||||
barcode = EXCLUDED.barcode,
|
||||
barcode_type = EXCLUDED.barcode_type,
|
||||
highlights = EXCLUDED.highlights,
|
||||
nutrients = EXCLUDED.nutrients,
|
||||
search_query = EXCLUDED.search_query,
|
||||
embedding = EXCLUDED.embedding,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
logger.info(f"✅ Upserted {len(rows)} products into {table_name}")
|
||||
|
||||
# Remove stale products that were deleted from the source data.
|
||||
# Only runs when cleanup=True so that callers processing partial
|
||||
# product sets (e.g. multiple seed files contributing to the same
|
||||
# brand table) don't accidentally orphan each other's data.
|
||||
if cleanup and image_ids:
|
||||
cur.execute(
|
||||
f"DELETE FROM {table_name} WHERE image_id != ALL(%s::text[])",
|
||||
(image_ids,),
|
||||
)
|
||||
deleted = cur.rowcount
|
||||
if deleted:
|
||||
logger.info(f"🗑️ Removed {deleted} stale product(s) from {table_name}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_existing_product_image_id(brand: str, product_name: str) -> Optional[str]:
|
||||
"""Check if a product with this name exists in the brand table and return its image_id"""
|
||||
conn = _connect()
|
||||
if not conn or not product_name:
|
||||
return None
|
||||
|
||||
storage_brand = resolve_parent_brand(brand)
|
||||
table_name = f"brand_{_sanitize_name(storage_brand)}"
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
SELECT image_id FROM {table_name}
|
||||
WHERE product_name = %s
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
""", (product_name,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _build_sanitized_brand_map() -> Dict[str, str]:
|
||||
"""Build a mapping from sanitized table suffix to original parent brand name.
|
||||
|
||||
This allows reverse-lookup of display names from sanitized table suffixes,
|
||||
preserving proper casing and special characters like &.
|
||||
"""
|
||||
seen: Dict[str, str] = {}
|
||||
for alias, parent in BRAND_ALIASES.items():
|
||||
suffix = _sanitize_name(parent)
|
||||
if suffix and suffix not in seen:
|
||||
# Capitalize properly: handle special chars like & by preserving them
|
||||
# from the original parent value
|
||||
seen[suffix] = parent.strip().title()
|
||||
return seen
|
||||
|
||||
|
||||
def list_available_brands() -> List[str]:
|
||||
"""List all available brands that have tables in the database"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
|
||||
brand_map = _build_sanitized_brand_map()
|
||||
brands = []
|
||||
with conn, conn.cursor() as cur:
|
||||
try:
|
||||
# Query information_schema for tables starting with brand_
|
||||
cur.execute("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name LIKE 'brand_%'
|
||||
""")
|
||||
tables = cur.fetchall()
|
||||
for table in tables:
|
||||
# Extract brand name from table name (e.g., brand_britannia -> britannia)
|
||||
table_name = table[0]
|
||||
suffix = table_name[len("brand_"):].lower() if table_name.startswith("brand_") else table_name.lower()
|
||||
# Prefer the original display name from the brand map if available
|
||||
brand_name = brand_map.get(suffix, suffix.replace('_', ' ').title())
|
||||
brands.append(brand_name)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list brands: {e}")
|
||||
|
||||
conn.close()
|
||||
return sorted(list(set(brands)))
|
||||
|
||||
|
||||
def get_products_by_brand(brand: str, limit: Optional[int] = None, offset: int = 0,
|
||||
category: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Fetch products for a specific brand from its table (plain listing, no ranking).
|
||||
|
||||
Supports optional pagination (`limit`/`offset`) and an exact-ish category
|
||||
filter, used by the catalog "browse" endpoints in the FastAPI layer.
|
||||
"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
|
||||
table_name = _table_name(brand)
|
||||
products = []
|
||||
|
||||
with conn, conn.cursor() as cur:
|
||||
try:
|
||||
if not _table_exists(cur, table_name):
|
||||
logger.warning(f"Table {table_name} does not exist")
|
||||
return []
|
||||
|
||||
sql = f"SELECT * FROM {table_name}"
|
||||
params: List[Any] = []
|
||||
if category:
|
||||
sql += " WHERE category ILIKE %s"
|
||||
params.append(f"%{category}%")
|
||||
sql += " ORDER BY updated_at DESC"
|
||||
if limit is not None:
|
||||
sql += " LIMIT %s OFFSET %s"
|
||||
params.extend([limit, offset])
|
||||
|
||||
cur.execute(sql, params)
|
||||
colnames = [desc[0] for desc in cur.description]
|
||||
for row in cur.fetchall():
|
||||
products.append(dict(zip(colnames, row)))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch products for {brand}: {e}")
|
||||
|
||||
conn.close()
|
||||
return products
|
||||
|
||||
|
||||
def count_products_by_brand(brand: str, category: Optional[str] = None) -> int:
|
||||
"""Count products in a brand table (used for pagination totals)."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return 0
|
||||
table_name = _table_name(brand)
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
if not _table_exists(cur, table_name):
|
||||
return 0
|
||||
sql = f"SELECT COUNT(*) FROM {table_name}"
|
||||
params: List[Any] = []
|
||||
if category:
|
||||
sql += " WHERE category ILIKE %s"
|
||||
params.append(f"%{category}%")
|
||||
cur.execute(sql, params)
|
||||
row = cur.fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to count products for {brand}: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_product_by_image_id(brand: str, image_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch a single product row by its unique image_id (used for product-detail lookups)."""
|
||||
conn = _connect()
|
||||
if not conn or not image_id:
|
||||
return None
|
||||
table_name = _table_name(brand)
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
if not _table_exists(cur, table_name):
|
||||
return None
|
||||
cur.execute(f"SELECT * FROM {table_name} WHERE image_id = %s LIMIT 1", (image_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
colnames = [desc[0] for desc in cur.description]
|
||||
return dict(zip(colnames, row))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch product {image_id} for {brand}: {e}")
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_products_all_brands(limit: int = 10000, offset: int = 0,
|
||||
category: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Fetch products from all brand tables merged together.
|
||||
|
||||
Used by the "All brands" browse view on the frontend to show a
|
||||
consolidated listing without requiring a full-text search.
|
||||
"""
|
||||
all_products: List[Dict[str, Any]] = []
|
||||
brands = list_available_brands()
|
||||
remaining = limit
|
||||
current_offset = offset
|
||||
|
||||
for brand in brands:
|
||||
brand_count = count_products_by_brand(brand, category=category)
|
||||
if current_offset >= brand_count:
|
||||
current_offset -= brand_count
|
||||
continue
|
||||
|
||||
brand_limit = min(remaining, brand_count - current_offset)
|
||||
if brand_limit <= 0:
|
||||
break
|
||||
products = get_products_by_brand(brand, limit=brand_limit,
|
||||
offset=current_offset, category=category)
|
||||
for p in products:
|
||||
p["brand"] = brand
|
||||
all_products.extend(products)
|
||||
remaining -= len(products)
|
||||
current_offset = 0
|
||||
|
||||
if remaining <= 0:
|
||||
break
|
||||
|
||||
return all_products
|
||||
|
||||
|
||||
def count_products_all_brands(category: Optional[str] = None) -> int:
|
||||
"""Count the total number of products across all brand tables."""
|
||||
total = 0
|
||||
for brand in list_available_brands():
|
||||
total += count_products_by_brand(brand, category=category)
|
||||
return total
|
||||
|
||||
|
||||
def list_categories_for_brand(brand: str) -> List[str]:
|
||||
"""Distinct categories present in a brand table (used to populate filter UI)."""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
table_name = _table_name(brand)
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
if not _table_exists(cur, table_name):
|
||||
return []
|
||||
cur.execute(f"SELECT DISTINCT category FROM {table_name} WHERE category IS NOT NULL ORDER BY category")
|
||||
return [row[0] for row in cur.fetchall() if row[0]]
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list categories for {brand}: {e}")
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_price_range(price_range_str: Optional[str]) -> Tuple[Optional[float], Optional[float]]:
|
||||
"""Parse string representations of prices into numeric floats (min_price, max_price).
|
||||
e.g. '₹120 - ₹150' -> (120.0, 150.0), '₹140' -> (140.0, 140.0)
|
||||
"""
|
||||
if not price_range_str:
|
||||
return None, None
|
||||
nums = [float(n) for n in re.findall(r"\d+(?:\.\d+)?", str(price_range_str).replace(",", ""))]
|
||||
if not nums:
|
||||
return None, None
|
||||
if len(nums) == 1:
|
||||
return nums[0], nums[0]
|
||||
return min(nums), max(nums)
|
||||
|
||||
|
||||
def semantic_search(
|
||||
query_embedding: List[float],
|
||||
brand: Optional[str] = None,
|
||||
top_k: int = 5,
|
||||
category: Optional[str] = None,
|
||||
max_distance: Optional[float] = None,
|
||||
max_price: Optional[float] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Core RAG retrieval step: cosine-similarity search over pgvector.
|
||||
|
||||
If `brand` is given, searches only that brand's table. If omitted,
|
||||
searches across every known `brand_*` table and merges results by
|
||||
distance (ascending = more similar), which lets the chat/search
|
||||
endpoints answer brand-agnostic questions like "show me a low sugar
|
||||
biscuit under 20 rupees".
|
||||
|
||||
Each returned dict includes a `distance` key (pgvector cosine
|
||||
distance: 0 = identical, 2 = opposite) and a `brand` key identifying
|
||||
which brand table the row came from, since multi-brand results don't
|
||||
otherwise carry that information.
|
||||
|
||||
`max_distance` can be used to drop weak matches (e.g. 0.6) so the LLM
|
||||
isn't fed irrelevant context when nothing in the catalog is actually
|
||||
relevant to the query.
|
||||
"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
|
||||
embedding_str = "[" + ",".join(map(str, query_embedding)) + "]"
|
||||
results: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
if brand:
|
||||
tables = [(brand, _table_name(brand))]
|
||||
else:
|
||||
tables = [(name, f"brand_{name}") for name in _list_brand_table_suffixes(cur)]
|
||||
|
||||
# Fetch extra candidates per table if max_price or filtering is applied
|
||||
fetch_limit = top_k * 5 if (max_price is not None or category) else top_k
|
||||
|
||||
for brand_label, table_name in tables:
|
||||
if not _table_exists(cur, table_name):
|
||||
continue
|
||||
sql = (
|
||||
f"SELECT *, embedding <=> %s::vector AS distance FROM {table_name} "
|
||||
f"WHERE embedding IS NOT NULL"
|
||||
)
|
||||
params: List[Any] = [embedding_str]
|
||||
if category:
|
||||
sql += " AND category ILIKE %s"
|
||||
params.append(f"%{category}%")
|
||||
sql += " ORDER BY distance ASC LIMIT %s"
|
||||
params.append(fetch_limit)
|
||||
|
||||
try:
|
||||
cur.execute(sql, params)
|
||||
except Exception as e:
|
||||
logger.warning(f"Semantic search failed for table {table_name}: {e}")
|
||||
continue
|
||||
|
||||
colnames = [desc[0] for desc in cur.description]
|
||||
for row in cur.fetchall():
|
||||
record = dict(zip(colnames, row))
|
||||
record["brand"] = record.get("brand") or brand_label
|
||||
|
||||
if max_distance is not None and record["distance"] > max_distance:
|
||||
continue
|
||||
|
||||
if max_price is not None:
|
||||
min_p, max_p = parse_price_range(record.get("price_range"))
|
||||
if min_p is not None and min_p > max_price:
|
||||
continue
|
||||
|
||||
results.append(record)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Merge + re-rank across brand tables (no-op when a single brand was searched)
|
||||
results.sort(key=lambda r: r.get("distance", 9.0))
|
||||
return results[:top_k]
|
||||
|
||||
|
||||
def text_search(
|
||||
query: str,
|
||||
brand: Optional[str] = None,
|
||||
top_k: int = 10,
|
||||
category: Optional[str] = None,
|
||||
max_price: Optional[float] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fallback text-based search using ILIKE on title and description.
|
||||
|
||||
Used when semantic/vector search returns no results. Searches across
|
||||
brand tables (optionally filtered by `brand` and/or `category`) and
|
||||
returns results with a fixed `distance` of 1.0 so they sort after
|
||||
any semantic matches.
|
||||
"""
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
like_pattern = f"%{query}%"
|
||||
fetch_limit = top_k * 5 if (max_price is not None or category) else top_k
|
||||
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
if brand:
|
||||
tables = [(brand, _table_name(brand))]
|
||||
else:
|
||||
tables = [(name, f"brand_{name}") for name in _list_brand_table_suffixes(cur)]
|
||||
|
||||
for brand_label, table_name in tables:
|
||||
if not _table_exists(cur, table_name):
|
||||
continue
|
||||
sql = f"SELECT * FROM {table_name}"
|
||||
where_clauses: List[str] = []
|
||||
params: List[Any] = []
|
||||
|
||||
where_clauses.append("(title ILIKE %s OR description ILIKE %s OR product_name ILIKE %s)")
|
||||
params.extend([like_pattern, like_pattern, like_pattern])
|
||||
|
||||
if category:
|
||||
where_clauses.append("category ILIKE %s")
|
||||
params.append(f"%{category}%")
|
||||
|
||||
if where_clauses:
|
||||
sql += " WHERE " + " AND ".join(where_clauses)
|
||||
sql += " ORDER BY updated_at DESC LIMIT %s"
|
||||
params.append(fetch_limit)
|
||||
|
||||
try:
|
||||
cur.execute(sql, params)
|
||||
except Exception as e:
|
||||
logger.warning(f"Text search failed for table {table_name}: {e}")
|
||||
continue
|
||||
|
||||
colnames = [desc[0] for desc in cur.description]
|
||||
for row in cur.fetchall():
|
||||
record = dict(zip(colnames, row))
|
||||
record["brand"] = brand_label
|
||||
record["distance"] = 1.0
|
||||
|
||||
if max_price is not None:
|
||||
min_p, max_p = parse_price_range(record.get("price_range"))
|
||||
if min_p is not None and min_p > max_price:
|
||||
continue
|
||||
|
||||
results.append(record)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return results[:top_k]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _table_name(brand: str) -> str:
|
||||
storage_brand = resolve_parent_brand(brand)
|
||||
return f"brand_{_sanitize_name(storage_brand)}"
|
||||
|
||||
|
||||
def _table_exists(cur, table_name: str) -> bool:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
)
|
||||
""",
|
||||
(table_name,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return bool(row and row[0])
|
||||
|
||||
|
||||
def _list_brand_table_suffixes(cur) -> List[str]:
|
||||
"""Return brand-table suffixes (e.g. 'parle' from 'brand_parle') for every brand table."""
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name LIKE 'brand_%'
|
||||
"""
|
||||
)
|
||||
return [row[0][len("brand_"):] for row in cur.fetchall()]
|
||||
Reference in New Issue
Block a user