Initial commit
This commit is contained in:
59
backend/scripts/enrich_nutrition.py
Normal file
59
backend/scripts/enrich_nutrition.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
CLI entry point for Feature 1/4/15's retrieval pipeline - fetches
|
||||
verified nutrition data (Open Food Facts) for every product in the
|
||||
catalog, computes transparent scores/insights, and persists them.
|
||||
|
||||
Usage:
|
||||
python scripts/enrich_nutrition.py
|
||||
python scripts/enrich_nutrition.py --max-products 200 --no-narrative
|
||||
python scripts/enrich_nutrition.py --force # re-fetch even already-verified products
|
||||
|
||||
Equivalent to POST /api/admin/nutrition-intelligence/enrich.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.nutrition_db import ensure_nutrition_schema # noqa: E402
|
||||
from app.services.nutrition_enrichment_service import enrich_all_products # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _progress(done: int, total: int) -> None:
|
||||
if total and (done % 25 == 0 or done == total):
|
||||
logger.info(f"Enrichment progress: {done}/{total}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Enrich the catalog with verified nutrition data")
|
||||
parser.add_argument("--force", action="store_true", help="Re-fetch even products already marked 'verified'")
|
||||
parser.add_argument("--no-narrative", action="store_true", help="Skip the LLM narrative step (facts/scores/tags only)")
|
||||
parser.add_argument("--max-products", type=int, default=None, help="Cap the number of products processed (for a quick test run)")
|
||||
args = parser.parse_args()
|
||||
|
||||
ensure_nutrition_schema()
|
||||
result = enrich_all_products(
|
||||
skip_if_verified=not args.force,
|
||||
generate_narrative=not args.no_narrative,
|
||||
progress_cb=_progress,
|
||||
max_products=args.max_products,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Done in {result.duration_seconds}s - "
|
||||
f"{result.verified} verified, {result.partial} partial, {result.unavailable} unavailable "
|
||||
f"of {result.total_products} total products"
|
||||
)
|
||||
if result.errors:
|
||||
logger.warning(f"{len(result.errors)} errors (showing up to 10): {result.errors[:10]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
144
backend/scripts/export_seed_data.py
Normal file
144
backend/scripts/export_seed_data.py
Normal file
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Export data FROM pgvector brand tables BACK INTO seed_catalog JSON files.
|
||||
|
||||
This is the REVERSE of seed_sample_data.py. Use it when the database has
|
||||
been populated with better/real data (via direct ingestion or manual fixes)
|
||||
and you want to snapshot that state back into the seed files so that future
|
||||
runs of `seed_sample_data.py` don't regress to stale AI-generated content.
|
||||
|
||||
Usage:
|
||||
python scripts/export_seed_data.py
|
||||
python scripts/export_seed_data.py --only-having-products
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.vector_store import _connect, _sanitize_name
|
||||
from app.infrastructure.settings import DB_HOST, DB_PORT, DB_NAME
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def export_all(only_having_products: bool = False) -> None:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.error("Cannot connect to database. Is pgvector running?")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name LIKE 'brand_%'
|
||||
ORDER BY table_name
|
||||
""")
|
||||
tables = cur.fetchall()
|
||||
|
||||
if not tables:
|
||||
logger.warning("No brand_* tables found in the database.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
exported_count = 0
|
||||
for (table_name,) in tables:
|
||||
suffix = table_name[len("brand_"):]
|
||||
display_brand = suffix.replace("_", " ").title()
|
||||
|
||||
# Build a friendly filename (strip problematic chars)
|
||||
safe_name = suffix.replace(" ", "_").replace("-", "_").replace("&", "_")
|
||||
safe_name = safe_name.strip("_")
|
||||
filepath = SEED_DIR / f"brand_catalog_{safe_name}.json"
|
||||
|
||||
products = _read_products(conn, table_name)
|
||||
|
||||
if only_having_products and not products:
|
||||
logger.info("Skipping %s (0 products)", table_name)
|
||||
continue
|
||||
|
||||
catalog = {
|
||||
"brand": display_brand,
|
||||
"search_query": None,
|
||||
"generation_timestamp": f"Exported from {DB_HOST}:{DB_PORT}/{DB_NAME} on {datetime.now().isoformat()}",
|
||||
"total_products": len(products),
|
||||
"total_images": sum(len(p.get("image_urls") or []) for p in products),
|
||||
"engine_info": {
|
||||
"gemini_enabled": False,
|
||||
"architecture": "Exported from pgvector database",
|
||||
},
|
||||
"products": products,
|
||||
}
|
||||
|
||||
SEED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
filepath.write_text(
|
||||
json.dumps(catalog, indent=2, ensure_ascii=False, default=str),
|
||||
encoding="utf-8",
|
||||
)
|
||||
logger.info("Exported %d products -> %s", len(products), filepath.name)
|
||||
exported_count += 1
|
||||
|
||||
print(f"\nDone. Exported {exported_count} brand table(s) to {SEED_DIR}")
|
||||
print("Run `python scripts/seed_sample_data.py` to re-import them if needed.")
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _read_products(conn, table_name: str) -> List[Dict[str, Any]]:
|
||||
"""Read all rows from a brand table and format them as seed-catalog products."""
|
||||
with conn, conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(f"SELECT * FROM {table_name} ORDER BY updated_at DESC")
|
||||
colnames = [desc[0] for desc in cur.description]
|
||||
rows = cur.fetchall()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to read %s: %s", table_name, e)
|
||||
return []
|
||||
|
||||
products = []
|
||||
for row in rows:
|
||||
record = dict(zip(colnames, row))
|
||||
product = {
|
||||
"title": record.get("title") or record.get("product_name") or "Unknown",
|
||||
"category": record.get("category") or "Uncategorized",
|
||||
"description": record.get("description") or "",
|
||||
"image_id": record.get("image_id") or "",
|
||||
"price_range": record.get("price_range") or "",
|
||||
"size_variants": list(record.get("size_variants") or []),
|
||||
"providers": list(record.get("providers") or []),
|
||||
"highlights": list(record.get("highlights") or []),
|
||||
"nutrients": list(record.get("nutrients") or []),
|
||||
"search_query": record.get("search_query") or "",
|
||||
}
|
||||
|
||||
if record.get("embedding") is not None:
|
||||
emb = record["embedding"]
|
||||
if hasattr(emb, "__iter__"):
|
||||
product["embedding"] = [float(x) for x in emb]
|
||||
|
||||
products.append(product)
|
||||
|
||||
return products
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Export pgvector brand tables to seed catalog JSON files")
|
||||
parser.add_argument(
|
||||
"--only-having-products", action="store_true",
|
||||
help="Skip brand tables with zero products",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
export_all(only_having_products=args.only_having_products)
|
||||
103
backend/scripts/fix_duplicate_image_ids.py
Normal file
103
backend/scripts/fix_duplicate_image_ids.py
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fix duplicate image_ids in seed catalog JSON files.
|
||||
|
||||
Products sharing the same `title` (e.g. "Nestle Kitkat" with sizes 30g, 50g, 70g)
|
||||
previously all got the same `image_id` (generated from `title` only). Since the
|
||||
database uses `ON CONFLICT (image_id) DO UPDATE`, only one variant per title
|
||||
survived the upsert.
|
||||
|
||||
This script re-generates `image_id` from `product_name` (or `title` + `size`)
|
||||
so every product variant gets a unique, deterministic image_id.
|
||||
|
||||
Usage:
|
||||
python scripts/fix_duplicate_image_ids.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def make_image_id(source: str) -> str:
|
||||
"""Deterministic image_id: sanitize source name + short hash suffix."""
|
||||
sanitized = source.lower()
|
||||
sanitized = ''.join(c if c.isalnum() else '_' for c in sanitized)
|
||||
sanitized = '_'.join(filter(None, sanitized.split('_')))
|
||||
if len(sanitized) > 50:
|
||||
sanitized = sanitized[:50]
|
||||
# Use a hash of the source string so the same source always produces the same ID
|
||||
short_hash = str(uuid.uuid5(uuid.NAMESPACE_DNS, source))[:8]
|
||||
return f"{sanitized}_{short_hash}"
|
||||
|
||||
|
||||
def fix_seed_file(path: Path) -> int:
|
||||
with open(path, encoding="utf-8-sig") as f:
|
||||
data = json.load(f)
|
||||
|
||||
products = data.get("products", [])
|
||||
fixed = 0
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
for product in products:
|
||||
# Determine the source for the image_id: prefer product_name, fall back to title+size, then title
|
||||
source = product.get("product_name") or ""
|
||||
if not source:
|
||||
title = product.get("title", "")
|
||||
size = product.get("size", "")
|
||||
source = f"{title}_{size}" if size else title
|
||||
|
||||
if not source:
|
||||
continue
|
||||
|
||||
new_id = make_image_id(source)
|
||||
|
||||
# Ensure no collisions across all products
|
||||
while new_id in seen_ids:
|
||||
new_id = make_image_id(source + "_" + str(uuid.uuid4())[:4])
|
||||
|
||||
old_id = product.get("image_id", "")
|
||||
if old_id != new_id:
|
||||
logger.debug(f" {old_id} -> {new_id} (source={source!r})")
|
||||
product["image_id"] = new_id
|
||||
seen_ids.add(new_id)
|
||||
fixed += 1
|
||||
|
||||
data["total_products"] = len(products)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"Fixed {fixed} products in {path.name}")
|
||||
return fixed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not SEED_DIR.exists():
|
||||
logger.error("Seed directory not found: %s", SEED_DIR)
|
||||
sys.exit(1)
|
||||
|
||||
files = sorted(SEED_DIR.glob("*.json"))
|
||||
if not files:
|
||||
logger.warning("No JSON files found in %s", SEED_DIR)
|
||||
return
|
||||
|
||||
total = 0
|
||||
for f in files:
|
||||
total += fix_seed_file(f)
|
||||
|
||||
print(f"\nDone. Fixed {total} total products across {len(files)} file(s).")
|
||||
print("Now re-run: python scripts/seed_sample_data.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
147
backend/scripts/resanitize_catalog.py
Normal file
147
backend/scripts/resanitize_catalog.py
Normal file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Retroactively fix cross-category language in already-generated product
|
||||
descriptions, and recompute embeddings from the cleaned text.
|
||||
|
||||
Why this is needed
|
||||
-------------------
|
||||
This is a one-time (or occasional) *data* fix to go with the *code* fix in
|
||||
`app/services/category_registry.py` / `app/core/catalog_engine.py`. New
|
||||
catalog generations already sanitize descriptions before embedding them,
|
||||
but products ingested BEFORE that change (including the bundled
|
||||
`data/seed_catalogs/*.json` samples) may still have descriptions like:
|
||||
|
||||
"ITC Bingo Korean Style is a crispy, savory biscuit that adds a
|
||||
touch of spice..."
|
||||
|
||||
for a product whose actual category is `Snacks`, not `Biscuits & Cookies`.
|
||||
That leftover word is exactly what let a query like "recommend biscuits
|
||||
with low sugar" surface Bingo snack products - the description text (and
|
||||
therefore its embedding) said "biscuit" even though the product isn't one.
|
||||
|
||||
What this script does
|
||||
----------------------
|
||||
For every product in every `data/seed_catalogs/*.json` file:
|
||||
1. Re-run `sanitize_category_language(description, category)` to strip
|
||||
out any other category's keywords.
|
||||
2. If the description changed, recompute its embedding the same way
|
||||
`catalog_engine.py` does at ingestion time.
|
||||
3. Write the updated JSON back to disk.
|
||||
|
||||
Pass `--apply-to-db` to also push the corrected rows straight into
|
||||
pgvector (equivalent to re-running `seed_sample_data.py` afterwards).
|
||||
|
||||
Usage:
|
||||
python scripts/resanitize_catalog.py # dry-run, prints a diff summary
|
||||
python scripts/resanitize_catalog.py --write # updates the JSON files in place
|
||||
python scripts/resanitize_catalog.py --write --apply-to-db
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.category_registry import sanitize_category_language # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def _process_file(path: Path, write: bool) -> List[Dict[str, Any]]:
|
||||
"""Returns the list of products whose description changed (for reporting)."""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
products = data.get("products", [])
|
||||
changed: List[Dict[str, Any]] = []
|
||||
needs_reembed: List[int] = []
|
||||
|
||||
for i, p in enumerate(products):
|
||||
original_desc = p.get("description") or ""
|
||||
category = p.get("category") or "General"
|
||||
cleaned_desc = sanitize_category_language(original_desc, category)
|
||||
if cleaned_desc != original_desc:
|
||||
changed.append({
|
||||
"title": p.get("title") or p.get("product_name"),
|
||||
"category": category,
|
||||
"before": original_desc[:160],
|
||||
"after": cleaned_desc[:160],
|
||||
})
|
||||
p["description"] = cleaned_desc
|
||||
needs_reembed.append(i)
|
||||
|
||||
if needs_reembed and write:
|
||||
# Local import: only pay the torch/sentence-transformers startup
|
||||
# cost when there's actually something to re-embed.
|
||||
from app.services.embeddings_service import embed_texts
|
||||
|
||||
texts = []
|
||||
for i in needs_reembed:
|
||||
p = products[i]
|
||||
title = p.get("title") or ""
|
||||
cat = p.get("category") or ""
|
||||
desc = p.get("description") or ""
|
||||
texts.append(f"{title} [CAT={cat}] :: {desc}")
|
||||
vectors = embed_texts(texts)
|
||||
for i, vec in zip(needs_reembed, vectors):
|
||||
products[i]["embedding"] = vec
|
||||
logger.info("Recomputed %d embedding(s) for %s", len(needs_reembed), path.name)
|
||||
|
||||
if write and changed:
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
logger.info("Wrote %d description fix(es) to %s", len(changed), path.name)
|
||||
|
||||
return changed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--write", action="store_true", help="Write changes to the JSON files (default: dry-run report only)")
|
||||
parser.add_argument("--apply-to-db", action="store_true", help="Also upsert the corrected rows into pgvector (implies --write)")
|
||||
args = parser.parse_args()
|
||||
write = args.write or args.apply_to_db
|
||||
|
||||
if not SEED_DIR.exists():
|
||||
logger.error("Seed directory not found: %s", SEED_DIR)
|
||||
sys.exit(1)
|
||||
|
||||
total_changed = 0
|
||||
for path in sorted(SEED_DIR.glob("*.json")):
|
||||
changed = _process_file(path, write=write)
|
||||
total_changed += len(changed)
|
||||
for c in changed:
|
||||
logger.info(" [%s] %r\n before: %s...\n after: %s...",
|
||||
c["category"], c["title"], c["before"], c["after"])
|
||||
|
||||
if not write:
|
||||
logger.info(
|
||||
"%d description(s) would be changed across %d file(s). "
|
||||
"Re-run with --write to apply (and --apply-to-db to also update pgvector).",
|
||||
total_changed, len(list(SEED_DIR.glob('*.json'))),
|
||||
)
|
||||
return
|
||||
|
||||
logger.info("%d description(s) fixed.", total_changed)
|
||||
|
||||
if args.apply_to_db:
|
||||
from app.services.vector_store import ensure_brand_schema, upsert_brand_products
|
||||
|
||||
for path in sorted(SEED_DIR.glob("*.json")):
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
brand = data.get("brand")
|
||||
products = data.get("products", [])
|
||||
if not brand or not products:
|
||||
continue
|
||||
ensure_brand_schema(brand)
|
||||
upsert_brand_products(brand, products, cleanup=True)
|
||||
logger.info("Upserted %d product(s) for brand %r into pgvector", len(products), brand)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
105
backend/scripts/seed_sample_data.py
Normal file
105
backend/scripts/seed_sample_data.py
Normal file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seed pgvector with the sample catalogs bundled in `data/seed_catalogs/`.
|
||||
|
||||
These JSON files are catalogs produced by an earlier ingestion run
|
||||
(Parle, Cadbury, Brooke Bond, Sunfeast, ...) and already include
|
||||
precomputed 384-dim embeddings, so this script does NOT call Ollama or
|
||||
the embeddings model at all - it just upserts the existing rows straight
|
||||
into pgvector. This is the fastest way to get the React UI + RAG chat
|
||||
showing real results without waiting on web scraping or a local LLM.
|
||||
|
||||
For brands not covered by the bundled samples, use
|
||||
`cli/ingest_brand.py "<brand>"` instead, which runs the full
|
||||
discovery -> images -> embeddings pipeline.
|
||||
|
||||
Usage:
|
||||
python scripts/seed_sample_data.py
|
||||
python scripts/seed_sample_data.py --only Parle Cadbury
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.vector_store import ensure_brand_schema, upsert_brand_products, _connect, _sanitize_name # noqa: E402
|
||||
from app.services.brand_registry import resolve_parent_brand # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Seed pgvector with bundled sample catalogs")
|
||||
parser.add_argument(
|
||||
"--only", nargs="*", default=None,
|
||||
help="Optional list of brand names (case-insensitive substring match on filename) to limit seeding to",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not SEED_DIR.exists():
|
||||
logger.error("Seed directory not found: %s", SEED_DIR)
|
||||
sys.exit(1)
|
||||
|
||||
files = sorted(SEED_DIR.glob("*.json"))
|
||||
if args.only:
|
||||
wanted = [w.lower() for w in args.only]
|
||||
files = [f for f in files if any(w in f.name.lower() for w in wanted)]
|
||||
|
||||
if not files:
|
||||
logger.warning("No matching seed files found in %s", SEED_DIR)
|
||||
return
|
||||
|
||||
# Aggregate products by resolved (canonical) brand so that multiple
|
||||
# seed files contributing to the same brand table are upserted as a
|
||||
# single batch. This ensures the stale-product cleanup (cleanup=True)
|
||||
# doesn't orphan products from a sibling file.
|
||||
brand_products: dict[str, list[dict]] = defaultdict(list)
|
||||
file_brand_map: dict[str, str] = {}
|
||||
|
||||
for f in files:
|
||||
data = json.loads(f.read_text(encoding="utf-8-sig"))
|
||||
products = data.get("products", [])
|
||||
brand = data.get("brand") or (products[0].get("brand_name") if products else None)
|
||||
if not brand or not products:
|
||||
logger.warning("Skipping %s - no brand/products found", f.name)
|
||||
continue
|
||||
|
||||
resolved = resolve_parent_brand(brand)
|
||||
brand_products[resolved].extend(products)
|
||||
file_brand_map[f.name] = resolved
|
||||
logger.info("Read %d products from %s -> resolved brand '%s'", len(products), f.name, resolved)
|
||||
|
||||
if not brand_products:
|
||||
logger.warning("No valid seed data found in %s", SEED_DIR)
|
||||
return
|
||||
|
||||
# Do not drop existing brand tables to preserve user database state.
|
||||
# Stale table cleanup disabled per environment requirements.
|
||||
|
||||
|
||||
total = 0
|
||||
for resolved_brand, all_products in brand_products.items():
|
||||
logger.info("Seeding %d product(s) for brand '%s'", len(all_products), resolved_brand)
|
||||
table = ensure_brand_schema(resolved_brand)
|
||||
if not table:
|
||||
logger.error("Could not create/verify table for brand '%s' - is pgvector reachable?", resolved_brand)
|
||||
continue
|
||||
upsert_brand_products(resolved_brand, all_products, cleanup=True)
|
||||
logger.info("Seeded %d products for brand '%s' (table=%s)", len(all_products), resolved_brand, table)
|
||||
total += len(all_products)
|
||||
|
||||
print(f"\nDone. Seeded {total} products across {len(brand_products)} brand(s) into pgvector.")
|
||||
print("Start the API with `uvicorn app.main:app --reload` and try a search/chat query.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
53
backend/scripts/seed_store_intelligence.py
Normal file
53
backend/scripts/seed_store_intelligence.py
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seeds the Multi-Store Intelligence layer (Features 1, 2, 8):
|
||||
- Creates the 5 stores (Store-A .. Store-E)
|
||||
- Assigns each store a random-but-reproducible subset of the existing
|
||||
product catalog, with independent pricing and stock
|
||||
- Simulates realistic order history over the requested date range
|
||||
|
||||
Run from `backend/`:
|
||||
python scripts/seed_store_intelligence.py
|
||||
python scripts/seed_store_intelligence.py --days 120 --seed 7
|
||||
python scripts/seed_store_intelligence.py --no-reset-orders # append instead of replacing order history
|
||||
|
||||
Requires at least one brand already ingested via the existing catalog
|
||||
pipeline (`python scripts/seed_sample_data.py` or POST /api/catalog/generate) -
|
||||
this script only distributes/prices/stocks/simulates orders for
|
||||
products that already exist; it never generates new products itself.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger("seed_store_intelligence")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--days", type=int, default=90, help="Days of order history to simulate (default: 90)")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducible store/order generation")
|
||||
parser.add_argument("--no-reset-orders", action="store_true", help="Append to existing order history instead of clearing it first")
|
||||
args = parser.parse_args()
|
||||
|
||||
from app.services.store_seed_service import run_seed
|
||||
|
||||
logger.info("Seeding store intelligence (days=%d, seed=%d, reset_orders=%s)...", args.days, args.seed, not args.no_reset_orders)
|
||||
result = run_seed(reset_orders=not args.no_reset_orders, days=args.days, seed=args.seed)
|
||||
|
||||
logger.info("Done.")
|
||||
logger.info(" Stores: %d", result["stores"])
|
||||
for store_id, count in result["store_products"].items():
|
||||
logger.info(" %s: %d products", store_id, count)
|
||||
logger.info(" Orders simulated: %d (%d line items)", result["orders"], result["order_items"])
|
||||
logger.info("Next: python scripts/train_ml_models.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
50
backend/scripts/train_ml_models.py
Normal file
50
backend/scripts/train_ml_models.py
Normal file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Trains every ML model in `app/intelligence/` against the seeded store/
|
||||
order data (Feature 9) and refreshes cached outputs (trending
|
||||
snapshots, demand forecasts, discount history).
|
||||
|
||||
Run from `backend/` AFTER scripts/seed_store_intelligence.py:
|
||||
python scripts/train_ml_models.py
|
||||
python scripts/train_ml_models.py --models discount trending # train a subset
|
||||
|
||||
Models trained: discount, trending, popularity, forecast (demand +
|
||||
inventory), store_performance, purchase_propensity. Every trained
|
||||
model is saved to app/intelligence/artifacts/*.joblib and loaded lazily
|
||||
by the API on first use - restart the API process after retraining to
|
||||
pick up new artifacts (see docs/CHANGES.md, "Retraining").
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger("train_ml_models")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument(
|
||||
"--models", nargs="+", default=None,
|
||||
choices=["discount", "trending", "popularity", "forecast", "store_performance", "purchase_propensity"],
|
||||
help="Subset of models to train (default: all)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
from app.services.ml_training_service import train_all
|
||||
|
||||
logger.info("Training models: %s", args.models or "all")
|
||||
results = train_all(models=args.models)
|
||||
|
||||
logger.info("Training summary:")
|
||||
for name, info in results.items():
|
||||
logger.info(" %s: %s", name, info)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
34
backend/scripts/train_nutrition_models.py
Normal file
34
backend/scripts/train_nutrition_models.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
CLI entry point for Feature 8/14's ML layer - trains the nutritional-
|
||||
similarity (cosine/KNN) and nutrition-based clustering (KMeans) models
|
||||
over whatever is currently enriched in `nutrition_facts`.
|
||||
|
||||
Run after scripts/enrich_nutrition.py has populated some data.
|
||||
|
||||
Usage:
|
||||
python scripts/train_nutrition_models.py
|
||||
|
||||
Equivalent to POST /api/admin/nutrition-intelligence/train.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.nutrition_enrichment_service import train_all_models # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result = train_all_models()
|
||||
logger.info(f"Nutrition similarity index: {result['similarity']}")
|
||||
logger.info(f"Nutrition clustering: {result['clustering']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user