update userpage catalog
This commit is contained in:
sriram
93
backend/scripts/backfill_barcodes.py
Normal file
93
backend/scripts/backfill_barcodes.py
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill PostgreSQL brand tables with `barcode` and `barcode_type` from seed catalogs.
|
||||
This ensures existing products with barcode details in seed files or DB rows are populated and served to the API and UI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.vector_store import _connect, list_available_brands, _sanitize_name, resolve_parent_brand, ensure_brand_schema
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def backfill_barcodes() -> None:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.error("Could not connect to PostgreSQL database.")
|
||||
return
|
||||
|
||||
brands = list_available_brands()
|
||||
logger.info("Ensuring schema for %d brand(s)...", len(brands))
|
||||
for brand in brands:
|
||||
ensure_brand_schema(brand)
|
||||
|
||||
seed_files = sorted(SEED_DIR.glob("*.json")) if SEED_DIR.exists() else []
|
||||
logger.info("Found %d seed file(s) in %s", len(seed_files), SEED_DIR)
|
||||
|
||||
updated_count = 0
|
||||
with conn.cursor() as cur:
|
||||
for seed_file in seed_files:
|
||||
try:
|
||||
data = json.loads(seed_file.read_text(encoding="utf-8-sig"))
|
||||
except Exception as e:
|
||||
logger.warning("Could not read seed file %s: %s", seed_file.name, e)
|
||||
continue
|
||||
|
||||
products = data.get("products", [])
|
||||
brand = data.get("brand") or (products[0].get("brand_name") if products else None)
|
||||
if not brand or not products:
|
||||
continue
|
||||
|
||||
table_name = f"brand_{_sanitize_name(resolve_parent_brand(brand))}"
|
||||
|
||||
for p in products:
|
||||
image_id = p.get("image_id") or p.get("sku") or ""
|
||||
product_name = p.get("product_name") or p.get("title") or ""
|
||||
barcode = str(p.get("barcode") or p.get("Barcode") or "").strip() or None
|
||||
barcode_type = str(p.get("barcode_type") or p.get("Barcode_Type") or "").strip() or None
|
||||
|
||||
if not barcode and not barcode_type:
|
||||
continue
|
||||
|
||||
if image_id:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE {table_name}
|
||||
SET barcode = COALESCE(%s, barcode),
|
||||
barcode_type = COALESCE(%s, barcode_type)
|
||||
WHERE image_id = %s
|
||||
""",
|
||||
(barcode, barcode_type, image_id)
|
||||
)
|
||||
if cur.rowcount > 0:
|
||||
updated_count += cur.rowcount
|
||||
elif product_name:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE {table_name}
|
||||
SET barcode = COALESCE(%s, barcode),
|
||||
barcode_type = COALESCE(%s, barcode_type)
|
||||
WHERE product_name = %s
|
||||
""",
|
||||
(barcode, barcode_type, product_name)
|
||||
)
|
||||
if cur.rowcount > 0:
|
||||
updated_count += cur.rowcount
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("🎉 Barcode backfill complete! Updated %d product row(s).", updated_count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
backfill_barcodes()
|
||||
111
backend/scripts/backfill_hsn_prices.py
Normal file
111
backend/scripts/backfill_hsn_prices.py
Normal file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill PostgreSQL brand tables with `hsn_code`, `final_selling_price`, and `selling_price` from seed catalogs.
|
||||
This ensures existing products with these details in seed files or DB rows are populated and served to the API and UI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.vector_store import _connect, list_available_brands, _sanitize_name, resolve_parent_brand, ensure_brand_schema
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEED_DIR = Path(__file__).resolve().parents[1] / "data" / "seed_catalogs"
|
||||
|
||||
|
||||
def backfill_hsn_prices() -> None:
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.error("Could not connect to PostgreSQL database.")
|
||||
return
|
||||
|
||||
brands = list_available_brands()
|
||||
logger.info("Ensuring schema for %d brand(s)...", len(brands))
|
||||
for brand in brands:
|
||||
ensure_brand_schema(brand)
|
||||
|
||||
seed_files = sorted(SEED_DIR.glob("*.json")) if SEED_DIR.exists() else []
|
||||
logger.info("Found %d seed file(s) in %s", len(seed_files), SEED_DIR)
|
||||
|
||||
updated_count = 0
|
||||
with conn.cursor() as cur:
|
||||
for seed_file in seed_files:
|
||||
try:
|
||||
data = json.loads(seed_file.read_text(encoding="utf-8-sig"))
|
||||
except Exception as e:
|
||||
logger.warning("Could not read seed file %s: %s", seed_file.name, e)
|
||||
continue
|
||||
|
||||
products = data.get("products", [])
|
||||
brand = data.get("brand") or (products[0].get("brand_name") if products else None)
|
||||
if not brand or not products:
|
||||
continue
|
||||
|
||||
table_name = f"brand_{_sanitize_name(resolve_parent_brand(brand))}"
|
||||
|
||||
for p in products:
|
||||
image_id = p.get("image_id") or p.get("sku") or ""
|
||||
product_name = p.get("product_name") or p.get("title") or ""
|
||||
hsn = str(p.get("hsn_code") or p.get("HSN_Code") or p.get("hsn") or "").strip() or None
|
||||
|
||||
raw_fsp = p.get("final_selling_price") if "final_selling_price" in p else p.get("Final_Selling_Price")
|
||||
if raw_fsp is None:
|
||||
raw_fsp = p.get("final_price")
|
||||
try:
|
||||
fsp = float(raw_fsp) if raw_fsp is not None and str(raw_fsp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
fsp = None
|
||||
|
||||
raw_sp = p.get("selling_price") if "selling_price" in p else p.get("Selling_Price")
|
||||
try:
|
||||
sp = float(raw_sp) if raw_sp is not None and str(raw_sp).strip() != "" else None
|
||||
except (ValueError, TypeError):
|
||||
sp = None
|
||||
|
||||
if fsp is None and sp is not None:
|
||||
fsp = sp
|
||||
|
||||
if not hsn and fsp is None and sp is None:
|
||||
continue
|
||||
|
||||
if image_id:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE {table_name}
|
||||
SET hsn_code = COALESCE(%s, hsn_code),
|
||||
final_selling_price = COALESCE(%s, final_selling_price),
|
||||
selling_price = COALESCE(%s, selling_price)
|
||||
WHERE image_id = %s
|
||||
""",
|
||||
(hsn, fsp, sp, image_id)
|
||||
)
|
||||
if cur.rowcount > 0:
|
||||
updated_count += cur.rowcount
|
||||
elif product_name:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE {table_name}
|
||||
SET hsn_code = COALESCE(%s, hsn_code),
|
||||
final_selling_price = COALESCE(%s, final_selling_price),
|
||||
selling_price = COALESCE(%s, selling_price)
|
||||
WHERE product_name = %s
|
||||
""",
|
||||
(hsn, fsp, sp, product_name)
|
||||
)
|
||||
if cur.rowcount > 0:
|
||||
updated_count += cur.rowcount
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("🎉 HSN and Price backfill complete! Updated %d product row(s).", updated_count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
backfill_hsn_prices()
|
||||
80
backend/scripts/backfill_s3_urls.py
Normal file
80
backend/scripts/backfill_s3_urls.py
Normal file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill PostgreSQL product rows that have `image_id` set but empty/null `image_url` or `image_urls`.
|
||||
This updates products in-place in pgvector so that API requests and RAG queries serve image URLs directly from DB
|
||||
without making S3 list_objects_v2 network calls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.services.vector_store import _connect, list_available_brands, _sanitize_name, resolve_parent_brand, ensure_brand_schema
|
||||
from app.services.s3_service import s3_service
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def backfill_brand_images() -> None:
|
||||
if not s3_service.enabled:
|
||||
logger.warning("S3 service is not enabled. Skipping backfill.")
|
||||
return
|
||||
|
||||
brands = list_available_brands()
|
||||
logger.info("Found %d brands in PostgreSQL database", len(brands))
|
||||
|
||||
for brand in brands:
|
||||
ensure_brand_schema(brand)
|
||||
|
||||
conn = _connect()
|
||||
if not conn:
|
||||
logger.error("Could not connect to PostgreSQL database.")
|
||||
return
|
||||
|
||||
brands = list_available_brands()
|
||||
logger.info("Found %d brands in PostgreSQL database", len(brands))
|
||||
|
||||
updated_total = 0
|
||||
with conn.cursor() as cur:
|
||||
for brand in brands:
|
||||
table_name = f"brand_{_sanitize_name(resolve_parent_brand(brand))}"
|
||||
try:
|
||||
cur.execute(f"SELECT id, image_id, image_url, image_urls FROM {table_name}")
|
||||
rows = cur.fetchall()
|
||||
except Exception as e:
|
||||
logger.warning("Could not read table %s: %s", table_name, e)
|
||||
conn.rollback()
|
||||
continue
|
||||
|
||||
for row in rows:
|
||||
row_id, image_id, image_url, image_urls = row
|
||||
if not image_id:
|
||||
continue
|
||||
|
||||
db_has_images = bool(image_url or (image_urls and any(image_urls)))
|
||||
if not db_has_images:
|
||||
# Fetch from S3 (or construct primary S3 URL)
|
||||
s3_urls = s3_service.get_product_image_urls(brand, image_id)
|
||||
primary_url = s3_urls[0] if s3_urls else s3_service.get_product_image_url(brand, image_id)
|
||||
if not s3_urls and primary_url:
|
||||
s3_urls = [primary_url]
|
||||
|
||||
if primary_url or s3_urls:
|
||||
cur.execute(
|
||||
f"UPDATE {table_name} SET image_url = %s, image_urls = %s WHERE id = %s",
|
||||
(primary_url, s3_urls, row_id)
|
||||
)
|
||||
updated_total += 1
|
||||
logger.info("Updated product ID %s in %s (image_id=%s) with %d image(s)", row_id, table_name, image_id, len(s3_urls))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("🎉 Backfill complete! Updated %d product(s) in database.", updated_total)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
backfill_brand_images()
|
||||
Reference in New Issue
Block a user