nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend stores data file enrichment pipeline

Browse Source
This commit is contained in:
sriram
2026-08-18 16:58:36 +05:30
parent 691efef880
commit 7a4583372f
36 changed files with 4599 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
app/infrastructure/settings.py
View File

@@ -185,6 +185,49 @@ USE_PLAYWRIGHT_FALLBACK = _bool("USE_PLAYWRIGHT_FALLBACK", "true")
# out 1x1 tracking pixels / broken placeholder images)
MIN_IMAGE_BYTES = int(os.getenv("MIN_IMAGE_BYTES", "3000"))
# ---------------------------------------------------------------------------
# Store-catalog enrichment pipeline (app/core/store_catalog_pipeline.py)
# ---------------------------------------------------------------------------
# Stages 7-10 of the store-Excel ingestion pipeline. Ported from the sibling
# Universal_Catalog_Barcode_Enrichment project along with the code that reads
# them; the names are kept identical so the ported modules need no edits.
#
# The three network-touching flags default to FALSE here, unlike in the
# sibling. This backend serves an interactive API on a shared 8GB host, and a
# 2000-row upload with web lookups on would fire thousands of outbound
# requests. Turn them on deliberately, per environment.
ENABLE_SKU_WEB_LOOKUP = _bool("ENABLE_SKU_WEB_LOOKUP", "false")
ENABLE_BARCODE_LOOKUP = _bool("ENABLE_BARCODE_LOOKUP", "false")
ENABLE_MANUFACTURER_SITE_LOOKUP = _bool("ENABLE_MANUFACTURER_SITE_LOOKUP", "false")
# Offline/deterministic stages - safe to leave on.
ENABLE_HSN_GST_ENRICHMENT = _bool("ENABLE_HSN_GST_ENRICHMENT", "true")
ENABLE_PRODUCT_VALIDATION = _bool("ENABLE_PRODUCT_VALIDATION", "true")
# Validation gate thresholds: below REJECT the row is dropped, below REVIEW it
# is stored but flagged `validation_status="review"`.
VALIDATION_REJECT_THRESHOLD = float(os.getenv("VALIDATION_REJECT_THRESHOLD", "0.35"))
VALIDATION_REVIEW_THRESHOLD = float(os.getenv("VALIDATION_REVIEW_THRESHOLD", "0.70"))
# Pack-size explosion: how many size rows one uploaded product may become.
MAX_VARIANTS_PER_PRODUCT = int(os.getenv("MAX_VARIANTS_PER_PRODUCT", "6"))
ENABLE_PER_VARIANT_IMAGES = _bool("ENABLE_PER_VARIANT_IMAGES", "false")
PER_VARIANT_IMAGE_MAX_RESULTS = int(os.getenv("PER_VARIANT_IMAGE_MAX_RESULTS", "10"))
# Barcode lookup tuning. The cache TTL is long (30 days) because a GTIN for a
# given pack size does not change, and negative results are cached too.
BARCODE_LOOKUP_TIMEOUT_SECONDS = float(os.getenv("BARCODE_LOOKUP_TIMEOUT_SECONDS", "10"))
BARCODE_LOOKUP_CACHE_TTL_SECONDS = float(os.getenv("BARCODE_LOOKUP_CACHE_TTL_SECONDS", str(30 * 24 * 3600)))
BARCODE_LOOKUP_MAX_CONCURRENCY = int(os.getenv("BARCODE_LOOKUP_MAX_CONCURRENCY", "5"))
BARCODE_COUNTRY_TAG = os.getenv("BARCODE_COUNTRY_TAG", "india")
# Optional barcode source credentials. Each source disables itself when its
# key is blank, so leaving these unset simply narrows the lookup cascade.
GS1_INDIA_API_BASE_URL = os.getenv("GS1_INDIA_API_BASE_URL", "")
GS1_INDIA_API_KEY = os.getenv("GS1_INDIA_API_KEY", "")
UPC_DATABASE_API_KEY = os.getenv("UPC_DATABASE_API_KEY", "")
# ---------------------------------------------------------------------------
# HTTP client defaults
# ---------------------------------------------------------------------------