Catalog feature updates on column fields
This commit is contained in:
22
.env.example
22
.env.example
@@ -340,10 +340,24 @@ ENABLE_PER_VARIANT_IMAGES=true
|
||||
PER_VARIANT_IMAGE_MAX_RESULTS=10
|
||||
|
||||
# Barcode Retrieval & Product Enrichment (see
|
||||
# app/services/enrichment/barcode/ and docs/BARCODE_ENRICHMENT.md). Set to
|
||||
# false to disable barcode lookup entirely (rows are stored with
|
||||
# barcode=NULL, barcode_lookup_status="disabled").
|
||||
ENABLE_BARCODE_LOOKUP=true
|
||||
# app/services/enrichment/barcode/ and docs/BARCODE_ENRICHMENT.md).
|
||||
#
|
||||
# INLINE, per product, during ingestion. Defaults FALSE in settings.py and this
|
||||
# file used to ship `true`, which disagreed with the code for as long as both
|
||||
# existed - anyone copying .env.example got a very different pipeline from
|
||||
# anyone relying on the defaults. It is false here now to match.
|
||||
#
|
||||
# Leave it false unless you know the upload is small: it costs one search
|
||||
# request per product against an endpoint capped at 10 requests/minute, so a
|
||||
# 200-row sheet is twenty minutes of held request. ENRICH_BARCODES_ON_UPLOAD
|
||||
# below is the cheap path and is on by default.
|
||||
ENABLE_BARCODE_LOOKUP=false
|
||||
|
||||
# BULK, per brand, after the upload settles. Fetches each brand's whole Open
|
||||
# Food Facts catalogue (~5 requests) and matches offline, on the enrichment
|
||||
# job's own thread. Runs before the nutrition phase, because a barcode turns a
|
||||
# 0.32-confidence name lookup into a 0.95-confidence exact one.
|
||||
ENRICH_BARCODES_ON_UPLOAD=true
|
||||
BARCODE_LOOKUP_TIMEOUT_SECONDS=10
|
||||
# 30 days, in seconds
|
||||
BARCODE_LOOKUP_CACHE_TTL_SECONDS=2592000
|
||||
|
||||
@@ -53,6 +53,11 @@ from app.services.vector_store import (
|
||||
get_products_by_brand,
|
||||
)
|
||||
from app.services.brand_sync import upsert_products_into_catalog_file
|
||||
from app.services.enrichment.catalog_consensus import (
|
||||
consensus_rows,
|
||||
consensus_value,
|
||||
fssai_for_brand,
|
||||
)
|
||||
from app.services.embeddings_service import embed_texts
|
||||
from app.services.s3_service import s3_service
|
||||
|
||||
@@ -409,12 +414,20 @@ class _PersistOutcome:
|
||||
unavailable: Optional[str] = None
|
||||
|
||||
|
||||
# How many of a brand's rows to read when working out what they agree on.
|
||||
# Bounded because this is a nicety, not the write: the largest brand table here
|
||||
# holds ~250 rows, so this reads all of them for every real brand while still
|
||||
# refusing to degenerate into the full `SELECT *` that once ran per uploaded
|
||||
# row. One query per brand, not per row - that part is load-bearing.
|
||||
_CONSENSUS_SAMPLE_LIMIT = 300
|
||||
|
||||
|
||||
def _brand_sample(brand_parent: str) -> Dict[str, Any]:
|
||||
"""The most recently updated product for a brand, used to inherit defaults.
|
||||
|
||||
`limit=1` matters: this used to be a full `SELECT *` of the brand table,
|
||||
executed once per uploaded row. A 200-row file against a brand with a few
|
||||
thousand products meant 200 full table reads before a single insert.
|
||||
Kept for the fields where one arbitrary sibling is a defensible default
|
||||
(category, price band, size). For `fssai_license` and `providers` it is
|
||||
NOT defensible - see `_brand_defaults`.
|
||||
"""
|
||||
try:
|
||||
existing = get_products_by_brand(brand_parent, limit=1)
|
||||
@@ -424,8 +437,51 @@ def _brand_sample(brand_parent: str) -> Dict[str, Any]:
|
||||
return existing[0] if existing else {}
|
||||
|
||||
|
||||
def _brand_defaults(brand_parent: str) -> Dict[str, Any]:
|
||||
"""One arbitrary sibling row, plus what the brand's rows actually AGREE on.
|
||||
|
||||
The distinction matters for exactly two fields.
|
||||
|
||||
`fssai_license` identifies the food business legally answerable for the
|
||||
product. Inheriting it from one arbitrary sibling is already weak; the code
|
||||
this replaces was worse - it fell back to the bare constant
|
||||
"10012042000244", which is Lion Dates' real registered licence, and stamped
|
||||
it onto any brand with no sample row. `scripts/merge_haldiram.py` exists
|
||||
because that reached production.
|
||||
|
||||
`providers` is a claim about where a product can be bought. The replaced
|
||||
default asserted all six of Amazon/Flipkart/BigBasket/Jiomart/Blinkit/Zepto
|
||||
for every product nobody had checked.
|
||||
|
||||
Consensus over the brand's own rows is the honest version of both, and it
|
||||
declines to answer when the rows disagree.
|
||||
"""
|
||||
# Two reads on purpose, and the split matters on a memory-capped host.
|
||||
#
|
||||
# `_brand_sample` is SELECT * limit 1 - one row, embedding and all, because
|
||||
# the fields it seeds (category, price band, size) need the whole row.
|
||||
#
|
||||
# The consensus read is 300 rows, so it takes only the two columns it
|
||||
# actually inspects. Measured: SELECT * over 244 rows costs 3.0 MB, of
|
||||
# which 4.7 KB per row is an embedding string nothing here reads. Two named
|
||||
# columns is roughly 50 KB for the same rows.
|
||||
rows = consensus_rows(brand_parent, ["fssai_license", "providers"],
|
||||
limit=_CONSENSUS_SAMPLE_LIMIT)
|
||||
|
||||
fssai, fssai_source = fssai_for_brand(brand_parent, rows)
|
||||
providers, _why = consensus_value("providers", rows)
|
||||
|
||||
return {
|
||||
"sample": _brand_sample(brand_parent),
|
||||
"fssai_license": fssai,
|
||||
"fssai_source": fssai_source,
|
||||
"providers": providers,
|
||||
}
|
||||
|
||||
|
||||
def _build_product_dict(req: AddProductRequest, brand_parent: str,
|
||||
sample_existing: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sample_existing: Dict[str, Any],
|
||||
defaults: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""Fill in everything the catalog needs that the user did not supply.
|
||||
|
||||
Pure apart from the optional S3 image lookup - no database access and no
|
||||
@@ -438,8 +494,22 @@ def _build_product_dict(req: AddProductRequest, brand_parent: str,
|
||||
raise ValueError(f"product name {product_name!r} has no letters or digits to identify it by")
|
||||
image_id = f"{brand_slug}_{product_slug}"
|
||||
|
||||
defaults = defaults or {}
|
||||
category = req.category or sample_existing.get("category") or "Health Foods"
|
||||
fssai_license = req.fssai_license or sample_existing.get("fssai_license") or "10012042000244"
|
||||
|
||||
# NO CONSTANT FALLBACK HERE, EVER.
|
||||
#
|
||||
# This line used to end `or "10012042000244"` - Lion Dates' real registered
|
||||
# FSSAI licence - so any brand without a sample row was silently attributed
|
||||
# to a food business that had never heard of the product. An FSSAI number
|
||||
# is who is legally answerable for what is in the packet; inventing one is
|
||||
# not a cosmetic default.
|
||||
#
|
||||
# The order now is: what the uploader supplied, then the curated brand
|
||||
# registry, then what the brand's own rows unambiguously agree on, then
|
||||
# NOTHING. A blank licence is a gap someone can fill; a confidently wrong
|
||||
# one is a liability nobody knows to look for.
|
||||
fssai_license = req.fssai_license or defaults.get("fssai_license") or None
|
||||
|
||||
description = req.description or (
|
||||
f"Introducing {product_name} from the trusted {brand_parent} brand. "
|
||||
@@ -465,7 +535,11 @@ def _build_product_dict(req: AddProductRequest, brand_parent: str,
|
||||
else:
|
||||
price_range = "₹100-250"
|
||||
|
||||
providers = req.providers or list(sample_existing.get("providers") or ["Amazon", "Flipkart", "BigBasket", "Jiomart", "Blinkit", "Zepto"])
|
||||
# Likewise no invented marketplace list. Claiming a product is stocked by
|
||||
# Amazon, Flipkart, BigBasket, Jiomart, Blinkit AND Zepto because nobody
|
||||
# checked is a false availability claim on every row it touches. Consensus
|
||||
# across the brand's own rows, or empty.
|
||||
providers = req.providers or list(defaults.get("providers") or [])
|
||||
highlights = req.highlights or list(sample_existing.get("highlights") or ["100% Quality Assurance", "Authentic Brand Product"])
|
||||
nutrients = req.nutrients or list(sample_existing.get("nutrients") or ["Energy - High", "Protein - Good Source"])
|
||||
|
||||
@@ -582,11 +656,13 @@ def _persist_products(items: List[Tuple[Optional[int], AddProductRequest]]) -> _
|
||||
# row) and one embedding call for the whole upload.
|
||||
built: "OrderedDict[str, List[Tuple[Optional[int], AddProductRequest, Dict[str, Any]]]]" = OrderedDict()
|
||||
for brand_parent, rows in groups.items():
|
||||
sample = _brand_sample(brand_parent)
|
||||
defaults = _brand_defaults(brand_parent)
|
||||
sample = defaults["sample"]
|
||||
prepared = []
|
||||
for row_number, req in rows:
|
||||
try:
|
||||
prepared.append((row_number, req, _build_product_dict(req, brand_parent, sample)))
|
||||
prepared.append((row_number, req,
|
||||
_build_product_dict(req, brand_parent, sample, defaults)))
|
||||
except Exception as exc: # noqa: BLE001 - one bad row, not the file
|
||||
outcome.failures.append({
|
||||
"row": row_number,
|
||||
|
||||
@@ -91,6 +91,8 @@ from app.services.generic_products import (
|
||||
)
|
||||
from app.services.embeddings_service import embed_texts
|
||||
from app.services.enrichment.barcode.stage import BarcodeEnrichmentStage
|
||||
from app.services.enrichment.barcode.identity_stage import BarcodeIdentityStage
|
||||
from app.services.enrichment.content.stage import ContentEnrichmentStage
|
||||
from app.services.enrichment.hsn_gst.stage import HsnGstEnrichmentStage
|
||||
from app.services.enrichment.pipeline import EnrichmentPipeline
|
||||
from app.services.product_validator import validate_catalog
|
||||
@@ -658,8 +660,21 @@ async def stages_8_9_enrichment(rows: List[Dict[str, Any]], brand: str) -> List[
|
||||
stages = []
|
||||
if ENABLE_BARCODE_LOOKUP:
|
||||
stages.append(BarcodeEnrichmentStage())
|
||||
# Unconditional, and deliberately not behind ENABLE_BARCODE_LOOKUP: it
|
||||
# performs no lookup. It validates whatever barcode the row already has -
|
||||
# which for a sheet-supplied one is the first check it ever gets - and
|
||||
# derives barcode_type/gtin/ean13/upc from those digits offline. Measured
|
||||
# before it existed: upc 0.0%, ean13 6.4%, gtin 8.7%, all computable from
|
||||
# the barcode sitting in the same row.
|
||||
stages.append(BarcodeIdentityStage())
|
||||
if ENABLE_HSN_GST_ENRICHMENT:
|
||||
stages.append(HsnGstEnrichmentStage())
|
||||
# Also unconditional and also offline. `highlights` and `nutrients` land
|
||||
# empty on EVERY upload today because this pipeline has no stage that
|
||||
# generates them - only the older brand-discovery path calls the
|
||||
# generators. Fills blanks only, and refuses to put nutrients on a
|
||||
# non-consumable.
|
||||
stages.append(ContentEnrichmentStage())
|
||||
if not stages:
|
||||
return rows
|
||||
return await EnrichmentPipeline(stages).run(rows, brand)
|
||||
@@ -736,9 +751,32 @@ def _to_storage_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"selling_price": row.get("selling_price"),
|
||||
"barcode": row.get("barcode"),
|
||||
"barcode_type": row.get("barcode_type"),
|
||||
# THIS PROJECTION IS THE WHOLE POINT OF THIS FUNCTION. A key the
|
||||
# enrichment stages computed but that is not named here never reaches
|
||||
# the database, however correct the stage was and however many columns
|
||||
# exist to hold it.
|
||||
#
|
||||
# That is exactly what happened to the seven fields below and to the
|
||||
# HSN/GST figures: stage 8 and stage 9 computed them on every run and
|
||||
# this dict silently dropped them. Measured before the fix - upc 0.0%,
|
||||
# ean13 6.4%, gst_percent and tax_amount 0% of upload rows.
|
||||
#
|
||||
# Anything added to an enrichment stage from here on has to be added
|
||||
# here AND to vector_store's INSERT, or it goes nowhere.
|
||||
"gtin": row.get("gtin"),
|
||||
"ean13": row.get("ean13"),
|
||||
"upc": row.get("upc"),
|
||||
"barcode_source": row.get("barcode_source"),
|
||||
"barcode_verified": row.get("barcode_verified"),
|
||||
"barcode_lookup_status": row.get("barcode_lookup_status"),
|
||||
"barcode_last_updated": row.get("barcode_last_updated"),
|
||||
"gst_percent": row.get("gst_percent"),
|
||||
"tax_amount": row.get("tax_amount"),
|
||||
"hsn_gst_needs_review": row.get("hsn_gst_needs_review"),
|
||||
"highlights": list(row.get("highlights") or []),
|
||||
"nutrients": list(row.get("nutrients") or []),
|
||||
"search_query": search_query,
|
||||
"field_sources": dict(row.get("field_sources") or {}),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -425,6 +425,22 @@ ENABLE_SKU_WEB_LOOKUP = _bool("ENABLE_SKU_WEB_LOOKUP", "false")
|
||||
ENABLE_BARCODE_LOOKUP = _bool("ENABLE_BARCODE_LOOKUP", "false")
|
||||
ENABLE_MANUFACTURER_SITE_LOOKUP = _bool("ENABLE_MANUFACTURER_SITE_LOOKUP", "false")
|
||||
|
||||
# Barcodes AFTER the upload settles, in bulk, on the enrichment job's own
|
||||
# thread. Default TRUE where ENABLE_BARCODE_LOOKUP above is false, and the
|
||||
# difference is cost, not appetite for risk:
|
||||
#
|
||||
# ENABLE_BARCODE_LOOKUP = one search request PER PRODUCT, inline, against an
|
||||
# endpoint capped at 10 requests/minute. A 200-row
|
||||
# upload is twenty minutes of a held request.
|
||||
# ENRICH_BARCODES_ON_UPLOAD = one corpus fetch PER BRAND (~5 requests total),
|
||||
# matched offline, after the uploader has their
|
||||
# result. Cost is per brand, not per row.
|
||||
#
|
||||
# It also has to run before the nutrition phase rather than beside it: a
|
||||
# barcode makes the nutrition lookup exact (0.95) instead of fuzzy (0.32), and
|
||||
# skip_if_verified means whichever lands first wins permanently.
|
||||
ENRICH_BARCODES_ON_UPLOAD = _bool("ENRICH_BARCODES_ON_UPLOAD", "true")
|
||||
|
||||
# Offline/deterministic stages - safe to leave on.
|
||||
ENABLE_HSN_GST_ENRICHMENT = _bool("ENABLE_HSN_GST_ENRICHMENT", "true")
|
||||
ENABLE_PRODUCT_VALIDATION = _bool("ENABLE_PRODUCT_VALIDATION", "true")
|
||||
|
||||
@@ -24,6 +24,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
@@ -83,13 +84,27 @@ def seed_catalog_paths(seed_dir: Path = SEED_DIR) -> List[Path]:
|
||||
# carries them, but they do NOT round-trip back into the database - re-seeding
|
||||
# ignores them, and app/services/nutrition_score_sync.py is what restores them
|
||||
# from nutrition_insights, which is their source of truth.
|
||||
#
|
||||
# `nutrients_per_100g` is in the same category as the two scores: mirrored from
|
||||
# nutrition_facts by nutrition_score_sync, exported for readers, never
|
||||
# round-tripped back in.
|
||||
#
|
||||
# The barcode identity/provenance columns and the HSN/GST figures below ARE
|
||||
# round-tripped. They were absent from this tuple for as long as they were
|
||||
# absent from the INSERT, which is why scripts/backfill_barcodes_from_off.py
|
||||
# refuses to call export_brand_to_seed_file() - exporting used to silently
|
||||
# strip the nine barcode keys off every product. Listing them here is what
|
||||
# makes that helper safe to use again.
|
||||
EXPORT_COLUMNS = (
|
||||
"product_name", "title", "description", "category", "image_id",
|
||||
"image_url", "image_urls", "price_range", "size_variants", "providers",
|
||||
"fssai_license", "product_sku", "sku_source", "hsn_code",
|
||||
"final_selling_price", "selling_price", "barcode", "barcode_type",
|
||||
"highlights", "nutrients", "search_query",
|
||||
"nutrition_score", "health_score",
|
||||
"gtin", "ean13", "upc", "barcode_source", "barcode_verified",
|
||||
"barcode_lookup_status", "barcode_last_updated",
|
||||
"gst_percent", "tax_amount", "hsn_gst_needs_review",
|
||||
"highlights", "nutrients", "search_query", "field_sources",
|
||||
"nutrition_score", "health_score", "nutrients_per_100g",
|
||||
)
|
||||
|
||||
|
||||
@@ -249,8 +264,18 @@ def _jsonable(value: Any) -> Any:
|
||||
"""Coerce a psycopg row value into something json.dumps accepts."""
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
# `barcode_last_updated` is a TIMESTAMP column, so psycopg hands back a
|
||||
# datetime, which json.dumps refuses. Emitted as an ISO-8601 string rather
|
||||
# than an epoch float so the seed file stays human-readable; the DB write
|
||||
# path accepts either (see vector_store._epoch_to_timestamp).
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_jsonable(v) for v in value]
|
||||
# JSONB (field_sources, nutrients_per_100g) arrives as a dict; recurse so
|
||||
# a Decimal nested inside a nutrient block does not break the dump.
|
||||
if isinstance(value, dict):
|
||||
return {k: _jsonable(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
|
||||
117
app/services/enrichment/barcode/identity_stage.py
Normal file
117
app/services/enrichment/barcode/identity_stage.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""Derives the rest of a product's barcode identity from the barcode itself.
|
||||
|
||||
WHY THIS IS A SEPARATE STAGE FROM BarcodeEnrichmentStage
|
||||
--------------------------------------------------------
|
||||
That stage FINDS a barcode, needs the network, and is off by default
|
||||
(`ENABLE_BARCODE_LOOKUP`, see settings.py:420-423 for why). This one FINDS
|
||||
NOTHING. It takes a barcode the row already has - typed into the merchant's
|
||||
spreadsheet, seeded from a catalog, or just located by the cascade - and fills
|
||||
in the fields that are pure arithmetic on those digits:
|
||||
|
||||
barcode_type from the length (classify_barcode_type)
|
||||
gtin the validated digits (a GTIN is what a barcode encodes)
|
||||
ean13 zero-padded UPC-A (to_ean13)
|
||||
upc the digits, for UPC-A only
|
||||
|
||||
There is no lookup, no host, no rate limit and no failure mode beyond "these
|
||||
digits are not a valid GTIN", so it needs no settings flag and costs nothing.
|
||||
|
||||
THE FAILURE IT ADDRESSES
|
||||
Measured against production on 2026-09-08: `upc` was 0.0% filled, `ean13`
|
||||
6.4%, `gtin` 8.7% - against `barcode` at 18.4%. Every one of those could
|
||||
have been computed from the barcode already sitting in the same row. They
|
||||
were not, because the only code that produced them was inside the disabled
|
||||
network cascade, and the writer dropped them anyway.
|
||||
|
||||
WHY IT RUNS AFTER THE LOOKUP STAGE
|
||||
So it also normalises whatever the cascade just found. The cascade already
|
||||
validates, but a sheet-supplied barcode never passes through
|
||||
`validate_barcode` at all today - it goes straight from the spreadsheet to
|
||||
the database. This stage is the first thing that checks those digits.
|
||||
|
||||
WHAT IT WILL NOT DO
|
||||
It will not correct, reformat or delete `barcode`. If the digits fail
|
||||
checksum validation the stage returns NOTHING, leaving the merchant's value
|
||||
exactly as typed - `enrichment/base.py`'s merge guard would refuse to blank
|
||||
it anyway, and silently "fixing" a barcode a shop supplied would be worse
|
||||
than leaving it visibly wrong. The failure is recorded in `field_sources`
|
||||
so the coverage report can surface it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.services.enrichment.base import EnrichmentStage, StageOutcome
|
||||
from app.services.enrichment.barcode.models import BarcodeType
|
||||
from app.services.enrichment.barcode.validators import (
|
||||
classify_barcode_type,
|
||||
normalize_barcode,
|
||||
to_ean13,
|
||||
validate_barcode,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BarcodeIdentityStage(EnrichmentStage):
|
||||
"""Offline, deterministic, additive. Never raises, never erases."""
|
||||
|
||||
name = "barcode_identity"
|
||||
|
||||
async def enrich_one(self, product: Dict[str, Any], brand: str) -> StageOutcome:
|
||||
raw = product.get("barcode")
|
||||
if not str(raw or "").strip():
|
||||
return StageOutcome(stage_name=self.name, fields={})
|
||||
|
||||
code = validate_barcode(raw)
|
||||
if not code:
|
||||
# Not a GTIN. Say so in the provenance rather than in the data, and
|
||||
# leave `barcode` untouched.
|
||||
digits = normalize_barcode(raw)
|
||||
reason = (f"{len(digits)} digits is not a GTIN-8/12/13/14 length"
|
||||
if digits else "no digits in the value")
|
||||
return StageOutcome(
|
||||
stage_name=self.name,
|
||||
fields={"field_sources": {"barcode": {
|
||||
"method": "unvalidated",
|
||||
"source": product.get("barcode_source") or "sheet",
|
||||
"note": f"failed checksum/format validation: {reason}",
|
||||
}}},
|
||||
error=f"barcode {raw!r} failed validation: {reason}",
|
||||
)
|
||||
|
||||
barcode_type = classify_barcode_type(code)
|
||||
fields: Dict[str, Any] = {
|
||||
"barcode": code, # normalised digits, same value
|
||||
"barcode_type": barcode_type.value,
|
||||
"gtin": code,
|
||||
"ean13": to_ean13(code), # None for GTIN-8, which is not a short EAN-13
|
||||
"upc": code if barcode_type is BarcodeType.UPC_A else None,
|
||||
}
|
||||
|
||||
# Only claim provenance we can stand behind. A barcode that arrived on
|
||||
# the sheet is the merchant's assertion, not ours, and is emphatically
|
||||
# not "verified" - that word is reserved for the cascade's
|
||||
# brand+size+name-matched result.
|
||||
if not str(product.get("barcode_source") or "").strip():
|
||||
fields["barcode_source"] = "sheet"
|
||||
fields["barcode_lookup_status"] = "sheet_validated"
|
||||
fields["barcode_verified"] = False
|
||||
fields["barcode_last_updated"] = time.time()
|
||||
|
||||
fields["field_sources"] = {
|
||||
"barcode": {
|
||||
"method": "sourced" if product.get("barcode_verified") else "asserted",
|
||||
"source": product.get("barcode_source") or "sheet",
|
||||
},
|
||||
# These four are arithmetic on the barcode, never a lookup. Calling
|
||||
# them "sourced" would overstate them.
|
||||
"gtin": {"method": "derived", "source": "validators.validate_barcode"},
|
||||
"ean13": {"method": "derived", "source": "validators.to_ean13"},
|
||||
"upc": {"method": "derived", "source": "validators.classify_barcode_type"},
|
||||
"barcode_type": {"method": "derived", "source": "validators.classify_barcode_type"},
|
||||
}
|
||||
|
||||
return StageOutcome(stage_name=self.name, fields=fields)
|
||||
@@ -114,9 +114,74 @@ def name_similarity(candidate_title: str, target_title: str) -> float:
|
||||
return round((overlap * 0.6) + (seq_ratio * 0.4), 3)
|
||||
|
||||
|
||||
def name_is_contained(candidate_title: str, target_title: str,
|
||||
target_brand: str = "") -> bool:
|
||||
"""True when the candidate's name is our name with only brand/size removed.
|
||||
|
||||
WHY THIS EXISTS - measured, not theoretical
|
||||
Open Food Facts stores short product names. We store long ones. Running
|
||||
`backfill_nutrition_from_barcodes` over the catalog on 2026-09-08, 149
|
||||
of 300 barcoded rows were rejected as "found, wrong product" when the
|
||||
barcode had resolved perfectly:
|
||||
|
||||
"Nestle Munch 8.9g" -> OFF "Munch" similarity 0.332
|
||||
"Coca-Cola Maaza 750ml" -> OFF "Maaza" similarity 0.304
|
||||
"Cadbury Perk 22 g" -> OFF "Perk" similarity 0.302
|
||||
|
||||
`name_similarity` divides the token overlap by the TARGET's token
|
||||
count, so a one-token candidate against a three-token target cannot
|
||||
exceed ~0.33 however right it is.
|
||||
|
||||
WHY NOT JUST LOWER THE THRESHOLD
|
||||
Because the same run also correctly rejected:
|
||||
|
||||
"Pepsico Lays 1kg" -> OFF "Spanish tomato tango" 0.133
|
||||
"Coca-Cola Fanta 750ml" -> OFF "Orange" 0.089
|
||||
"Lion Dates Powder 100g" -> OFF "PEPER NOTEN" 0.097
|
||||
|
||||
Those sit BELOW the containment cases but a threshold low enough to
|
||||
admit 0.30 also admits them. The measured yield table at
|
||||
settings.py:449-477 raised this floor to 0.78 for exactly that reason.
|
||||
Containment separates the two groups on structure rather than on a
|
||||
number: "Munch" is every token of our name minus brand and size;
|
||||
"Orange" is not a subset of "Coca-Cola Fanta 750ml" at all.
|
||||
|
||||
THE RULE
|
||||
Every token of the candidate's name must appear in the target's, once
|
||||
brand tokens and size tokens are discounted, and the candidate must
|
||||
carry at least one token that is not the brand. A bare brand name
|
||||
("Colgate", "godrej" - both real OFF titles) therefore does NOT match,
|
||||
which matters because those would otherwise attach to every product of
|
||||
that brand.
|
||||
"""
|
||||
cand_tokens = _tokens(candidate_title)
|
||||
target_tokens = _tokens(target_title)
|
||||
if not cand_tokens or not target_tokens:
|
||||
return False
|
||||
|
||||
brand_tokens = _tokens(target_brand)
|
||||
# A candidate that is only the brand identifies a brand, not a product.
|
||||
if not (cand_tokens - brand_tokens):
|
||||
return False
|
||||
|
||||
# Size tokens are not identity: our title carries the pack size, OFF's
|
||||
# usually does not, and `size_matches` has already checked the size
|
||||
# separately by the time this is consulted.
|
||||
def _meaningful(tokens):
|
||||
return {t for t in tokens if not _SIZE_TOKEN_RE.fullmatch(t)}
|
||||
|
||||
return _meaningful(cand_tokens) <= _meaningful(target_tokens | brand_tokens)
|
||||
|
||||
|
||||
# A token that is purely a quantity ("750ml", "8", "9g", "1kg"). Size is
|
||||
# compared by `size_matches`, so it must not also decide name identity.
|
||||
_SIZE_TOKEN_RE = re.compile(r"\d+(?:\.\d+)?(?:g|kg|ml|l|mg|cl|oz|gm|ltr|pcs|n)?", re.I)
|
||||
|
||||
|
||||
def is_match(candidate: BarcodeCandidate, target_brand: str, target_title: str, target_size: str,
|
||||
brand_aliases: Optional[Iterable[str]] = None,
|
||||
min_name_similarity: float = 0.45) -> tuple[bool, float]:
|
||||
min_name_similarity: float = 0.45,
|
||||
barcode_is_identity: bool = False) -> tuple[bool, float]:
|
||||
"""The combined gate a candidate must pass to be accepted:
|
||||
1. Brand matches (or overlaps a known alias).
|
||||
2. Pack size matches within a tight tolerance.
|
||||
@@ -126,16 +191,52 @@ def is_match(candidate: BarcodeCandidate, target_brand: str, target_title: str,
|
||||
product line from the same brand.
|
||||
Returns (matched, confidence) - confidence is diagnostic only, stored
|
||||
on the result for audit/QA but never used to override rule 1-3.
|
||||
|
||||
`barcode_is_identity` says the caller already knows WHICH product this is,
|
||||
because it looked the candidate up BY its GTIN rather than by searching.
|
||||
That changes what rules 2 and 4 are for: they stop being evidence of
|
||||
identity and become sanity checks against our barcode being on the wrong
|
||||
row. A sanity check cannot fail on information the source does not have, so
|
||||
under this flag:
|
||||
|
||||
* rule 2 (size) - a BLANK candidate size no longer vetoes. Open Food
|
||||
Facts leaves `quantity` null on a large share of records (57 of 146
|
||||
Amul hits), and `size_matches` returns False whenever either side is
|
||||
blank. A record with no quantity does not disagree with our pack size;
|
||||
it says nothing about it. A quantity that is PRESENT and different
|
||||
still vetoes - that is our barcode pointing at the wrong pack.
|
||||
* rule 4 (name) - see `name_is_contained`.
|
||||
|
||||
Rules 1 and 3 are unaffected: a different brand, or a "sugar free" the
|
||||
target does not have, still means a different product.
|
||||
|
||||
It defaults to False because every relaxation here is unsafe on the SEARCH
|
||||
path, where many candidates compete and name and size are the only things
|
||||
telling them apart - "Munch" with no size would match every Nestle product
|
||||
containing that word. Pass True only where a single candidate was fetched
|
||||
by barcode. Today that is `fetch_verified_nutrition_by_barcode` and
|
||||
`scripts/backfill_nutrition_from_barcodes`, and nothing else.
|
||||
|
||||
Measured on 2026-09-08: of 300 barcoded catalog rows, 149 were refused as
|
||||
"found, wrong product" with the barcode resolving perfectly. The name gate
|
||||
was the visible symptom, but the SIZE gate rejected most of them first.
|
||||
"""
|
||||
if not brand_matches(candidate.candidate_brand, target_brand, brand_aliases):
|
||||
return False, 0.0
|
||||
if not size_matches(candidate.candidate_size, target_size):
|
||||
# A blank candidate size is missing information, not a disagreement - but
|
||||
# only when the barcode already established identity. On the search path a
|
||||
# sizeless candidate is genuinely unidentifiable and must still be refused.
|
||||
size_unknown = barcode_is_identity and not str(candidate.candidate_size or "").strip()
|
||||
if not size_unknown and not size_matches(candidate.candidate_size, target_size):
|
||||
return False, 0.0
|
||||
if has_conflicting_variant_terms(candidate.candidate_title, target_title):
|
||||
return False, 0.0
|
||||
|
||||
similarity = name_similarity(candidate.candidate_title, target_title)
|
||||
if similarity < min_name_similarity:
|
||||
if barcode_is_identity and name_is_contained(
|
||||
candidate.candidate_title, target_title, target_brand):
|
||||
return True, similarity
|
||||
return False, similarity
|
||||
|
||||
return True, similarity
|
||||
|
||||
@@ -88,6 +88,25 @@ class EnrichmentStage(ABC):
|
||||
# HsnGstEnrichmentStage and BarcodeEnrichmentStage.
|
||||
if outcome.fields:
|
||||
for key, value in outcome.fields.items():
|
||||
# `field_sources` ACCUMULATES; every other key is assigned.
|
||||
#
|
||||
# It is a map keyed by column name, and each stage knows the
|
||||
# provenance of only the columns it filled. Assigning it like
|
||||
# anything else would mean the last stage to run erases what
|
||||
# every earlier stage recorded - so the barcode stage's
|
||||
# provenance would vanish the moment the HSN stage ran, and
|
||||
# the coverage report would show values with no origin.
|
||||
#
|
||||
# A shallow merge is the right depth: each key's value is one
|
||||
# flat record about one column. This mirrors the `||` in
|
||||
# vector_store's ON CONFLICT clause, so the in-memory merge
|
||||
# and the database merge agree.
|
||||
if key == "field_sources" and isinstance(value, dict):
|
||||
merged = dict(product.get("field_sources") or {})
|
||||
merged.update(value)
|
||||
product["field_sources"] = merged
|
||||
continue
|
||||
|
||||
blank_incoming = value is None or (isinstance(value, str) and not value.strip())
|
||||
existing = product.get(key)
|
||||
held = existing is not None and not (isinstance(existing, str) and not existing.strip())
|
||||
|
||||
142
app/services/enrichment/catalog_consensus.py
Normal file
142
app/services/enrichment/catalog_consensus.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Fills a blank field from what the brand's OWN rows already agree on.
|
||||
|
||||
WHY THIS EXISTS
|
||||
`fssai_license` was 69.3% filled on 2026-09-08, sourced entirely from a
|
||||
hardcoded 34-brand map (`brand_registry.FSSAI_LICENSES`). A brand outside
|
||||
that map got nothing - except on one path, which got something far worse.
|
||||
|
||||
`user_products._build_product_dict` read:
|
||||
|
||||
fssai_license = req.fssai_license or sample_existing.get(...) or "10012042000244"
|
||||
|
||||
That constant is LION DATES' real, registered FSSAI licence. Any brand with
|
||||
no sample row was stamped with it. This is not a cosmetic default: an FSSAI
|
||||
number identifies the food business legally answerable for the product, and
|
||||
inventing one attributes a stranger's regulatory liability to a product they
|
||||
never made. `scripts/merge_haldiram.py:36` exists because this already
|
||||
reached production once, on `brand_haldirams`.
|
||||
|
||||
The honest source for a blank licence is the brand's own catalog: 400
|
||||
Britannia rows carrying one licence is good evidence for the 401st. That is
|
||||
what this module reads.
|
||||
|
||||
THE RULE IT ENFORCES
|
||||
Propagate only from UNAMBIGUOUS agreement. If a brand's rows carry two
|
||||
different licences, one of them is already wrong and this module returns
|
||||
None rather than picking. A blank field is a gap; a confidently wrong
|
||||
regulatory identifier is a liability.
|
||||
|
||||
Nothing here invents a value. Every result is a value already present on a
|
||||
row of the same brand, which is why the provenance method is
|
||||
`catalog_consensus` and never `sourced`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A single dissenting row should not veto 400 agreeing ones, but a genuine
|
||||
# split must. Set so that "399 of 400 agree" propagates and "60/40" does not.
|
||||
_MIN_AGREEMENT = 0.85
|
||||
|
||||
# Below this many populated rows there is no consensus to speak of, only a
|
||||
# coincidence. Two rows agreeing proves nothing about a third.
|
||||
_MIN_ROWS = 3
|
||||
|
||||
|
||||
def _modal(values: List[Any], min_agreement: float = _MIN_AGREEMENT,
|
||||
min_rows: int = _MIN_ROWS) -> Tuple[Optional[Any], Dict[str, Any]]:
|
||||
"""The one value the population agrees on, or None with the reason why."""
|
||||
populated = [v for v in values if v not in (None, "", [], {})]
|
||||
if len(populated) < min_rows:
|
||||
return None, {"reason": "too few populated rows", "rows": len(populated)}
|
||||
|
||||
# Lists (providers) are unhashable; compare them as ordered tuples.
|
||||
keyed = [tuple(v) if isinstance(v, list) else v for v in populated]
|
||||
counts = Counter(keyed)
|
||||
winner, hits = counts.most_common(1)[0]
|
||||
agreement = hits / len(keyed)
|
||||
if agreement < min_agreement:
|
||||
return None, {"reason": "no clear majority", "agreement": round(agreement, 3),
|
||||
"distinct": len(counts)}
|
||||
|
||||
return (list(winner) if isinstance(winner, tuple) else winner), {
|
||||
"agreement": round(agreement, 3), "rows": len(keyed)}
|
||||
|
||||
|
||||
def consensus_value(column: str, rows: List[Dict[str, Any]],
|
||||
min_agreement: float = _MIN_AGREEMENT
|
||||
) -> Tuple[Optional[Any], Dict[str, Any]]:
|
||||
"""The agreed value of `column` across `rows`, plus why it was or was not
|
||||
reached. Never raises: an unreadable row set yields (None, reason)."""
|
||||
try:
|
||||
return _modal([r.get(column) for r in rows], min_agreement=min_agreement)
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
logger.debug("consensus for %s failed: %s", column, e)
|
||||
return None, {"reason": f"error: {e}"}
|
||||
|
||||
|
||||
def consensus_rows(brand: str, columns: List[str], limit: int = 300) -> List[Dict[str, Any]]:
|
||||
"""Read only the columns consensus needs, for a brand's rows.
|
||||
|
||||
Deliberately NOT `get_products_by_brand`, which is `SELECT *` and therefore
|
||||
carries the 384-dimension embedding on every row. Measured against
|
||||
production: 244 Hindustan Unilever rows cost 3.0 MB that way, 4.7 KB of the
|
||||
7.2 KB per row being an embedding string nothing here looks at.
|
||||
|
||||
Two named columns bring the same read down to roughly 50 KB. On a backend
|
||||
container capped at 2560 MB that difference is not dangerous either way -
|
||||
it is just the difference between reading what is needed and reading
|
||||
everything, once per brand per upload.
|
||||
|
||||
Probes `information_schema` first, because the column set genuinely differs
|
||||
between brand tables and a missing column would otherwise raise.
|
||||
"""
|
||||
from app.services.vector_store import _connect, _sanitize_name
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
return []
|
||||
table = f"brand_{_sanitize_name(brand)}"
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table,),
|
||||
)
|
||||
present = {r[0] for r in cur.fetchall()}
|
||||
wanted = [c for c in columns if c in present]
|
||||
if not wanted:
|
||||
return []
|
||||
select = ", ".join(f'"{c}"' for c in wanted)
|
||||
cur.execute(f'SELECT {select} FROM "{table}" LIMIT %s', (limit,))
|
||||
return [dict(zip(wanted, row)) for row in cur.fetchall()]
|
||||
except Exception as e: # noqa: BLE001 - defaults are a nicety, not the write
|
||||
logger.debug("consensus read failed for %s: %s", brand, e)
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def fssai_for_brand(brand: str, rows: Optional[List[Dict[str, Any]]] = None) -> Tuple[Optional[str], str]:
|
||||
"""The licence to use for a new product of `brand`, and where it came from.
|
||||
|
||||
Order: the curated registry map, then the brand's own rows. Never a
|
||||
constant, never another brand's number.
|
||||
"""
|
||||
from app.services.brand_registry import get_fssai_license
|
||||
|
||||
mapped = get_fssai_license(brand)
|
||||
if mapped:
|
||||
return mapped, "brand_registry"
|
||||
|
||||
if rows:
|
||||
value, _why = consensus_value("fssai_license", rows)
|
||||
if value:
|
||||
return str(value), "catalog_consensus"
|
||||
|
||||
return None, "unknown"
|
||||
1
app/services/enrichment/content/__init__.py
Normal file
1
app/services/enrichment/content/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Offline content enrichment - the display columns the store pipeline left blank."""
|
||||
112
app/services/enrichment/content/stage.py
Normal file
112
app/services/enrichment/content/stage.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Fills `highlights` and `nutrients` for rows the store pipeline leaves empty.
|
||||
|
||||
THE FAILURE THIS ADDRESSES
|
||||
`catalog_engine.generate_product_highlights` and `generate_nutrients_info`
|
||||
have existed for a long time and `brand_discovery._build_product` calls
|
||||
both. The store-catalog pipeline never did: `_to_storage_row` simply passed
|
||||
whatever the sheet had through, so a colleague's upload - which carries
|
||||
neither column - landed `highlights=[]` and `nutrients=[]` on every row.
|
||||
|
||||
That is the whole reason those two columns look healthy in aggregate
|
||||
(95.3% / 69.7% on 2026-09-08) while being empty for exactly the rows this
|
||||
work is about.
|
||||
|
||||
WHAT IT WRITES, AND HOW HONESTLY
|
||||
`highlights` is marketing copy derived from fields we already hold - the
|
||||
category, the pack size, the brand. It is `derived`, never `sourced`.
|
||||
|
||||
`nutrients` is the display list. Where real per-100g figures exist,
|
||||
`nutrition_score_sync.sync_nutrients_to_brand_tables` renders them from
|
||||
`nutrition_facts` and overwrites whatever this stage wrote - that mirror is
|
||||
the better source and runs later. This stage only supplies the
|
||||
category-keyword fallback, flagged `estimated`, so a row is not blank while
|
||||
it waits for a nutrition lookup that may never succeed.
|
||||
|
||||
THE CONSUMABILITY GATE
|
||||
`generate_nutrients_info` works off category keywords, so a Hair Care row
|
||||
whose category or description happens to contain a matching word acquires
|
||||
entries like "Vitamin B Complex - Energy". Shampoo has no nutrients. This
|
||||
stage refuses to write the column at all for a non-consumable, which is the
|
||||
same gate `nutrition_data_service` applies on the lookup path and the same
|
||||
reason `purge_non_consumable_nutrition.py` had to exist.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.services.enrichment.base import EnrichmentStage, StageOutcome
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ContentEnrichmentStage(EnrichmentStage):
|
||||
"""Offline, deterministic, fills blanks only. Never raises, never erases."""
|
||||
|
||||
name = "content"
|
||||
|
||||
async def enrich_one(self, product: Dict[str, Any], brand: str) -> StageOutcome:
|
||||
# Imported lazily: catalog_engine pulls in the image and LLM services,
|
||||
# and this stage runs inside ingestion where those are already loaded
|
||||
# but the enrichment package on its own should not require them.
|
||||
from app.core.catalog_engine import (
|
||||
generate_nutrients_info,
|
||||
generate_product_highlights,
|
||||
)
|
||||
from app.services.consumability import is_non_consumable
|
||||
|
||||
fields: Dict[str, Any] = {}
|
||||
sources: Dict[str, Any] = {}
|
||||
|
||||
title = product.get("title") or product.get("product_name") or ""
|
||||
category = product.get("category") or ""
|
||||
|
||||
if not _has_entries(product.get("highlights")):
|
||||
try:
|
||||
highlights = generate_product_highlights(product, brand)
|
||||
except Exception as e: # never abort a row
|
||||
logger.debug("highlight generation failed for %r: %s", title, e)
|
||||
highlights = []
|
||||
if highlights:
|
||||
fields["highlights"] = highlights
|
||||
sources["highlights"] = {"method": "derived",
|
||||
"source": "catalog_engine.generate_product_highlights"}
|
||||
|
||||
if not _has_entries(product.get("nutrients")):
|
||||
if is_non_consumable(category, title):
|
||||
# Not a gap - a column that cannot apply. Recording it stops
|
||||
# the coverage report counting shampoo as missing nutrition
|
||||
# forever, which is what makes someone eventually fabricate it.
|
||||
sources["nutrients"] = {"method": "not_applicable",
|
||||
"source": "non_consumable_product"}
|
||||
else:
|
||||
try:
|
||||
nutrients = generate_nutrients_info(product, brand)
|
||||
except Exception as e:
|
||||
logger.debug("nutrient generation failed for %r: %s", title, e)
|
||||
nutrients = []
|
||||
if nutrients:
|
||||
fields["nutrients"] = nutrients
|
||||
sources["nutrients"] = {
|
||||
"method": "estimated",
|
||||
"source": "catalog_engine.generate_nutrients_info",
|
||||
"note": "category keywords; replaced by real per-100g "
|
||||
"figures when a nutrition lookup succeeds",
|
||||
}
|
||||
|
||||
if sources:
|
||||
fields["field_sources"] = sources
|
||||
|
||||
return StageOutcome(stage_name=self.name, fields=fields)
|
||||
|
||||
|
||||
def _has_entries(value: Any) -> bool:
|
||||
"""True when the column already carries something worth keeping.
|
||||
|
||||
A list of empty strings counts as empty: the spreadsheet parser produces
|
||||
those from a column that exists but has no value in it, and treating one as
|
||||
"already filled" is how a row keeps `['']` forever.
|
||||
"""
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return bool(value)
|
||||
return any(str(v).strip() for v in value)
|
||||
@@ -60,6 +60,17 @@ def _build_default_stages() -> List[EnrichmentStage]:
|
||||
except Exception as e:
|
||||
logger.error(f"Barcode enrichment stage unavailable: {e}")
|
||||
|
||||
# Runs AFTER the lookup so it normalises whatever that found, and runs at
|
||||
# all even when the lookup is disabled - which is the point. It derives
|
||||
# barcode_type/gtin/ean13/upc from a barcode the row already has, offline
|
||||
# and for free, so a sheet-supplied barcode finally gets validated and
|
||||
# expanded instead of going straight to the database unchecked.
|
||||
try:
|
||||
from app.services.enrichment.barcode.identity_stage import BarcodeIdentityStage
|
||||
stages.append(BarcodeIdentityStage())
|
||||
except Exception as e:
|
||||
logger.error(f"Barcode identity stage unavailable: {e}")
|
||||
|
||||
# HSN / GST & pricing enrichment (see app/services/enrichment/hsn_gst/) -
|
||||
# deterministic, offline, pure-additive. Runs AFTER the barcode stage so
|
||||
# every stored/exported row carries both sets of fields; a failure here
|
||||
@@ -70,9 +81,15 @@ def _build_default_stages() -> List[EnrichmentStage]:
|
||||
except Exception as e:
|
||||
logger.error(f"HSN/GST enrichment stage unavailable: {e}")
|
||||
|
||||
# Future stages register here, e.g.:
|
||||
# from app.services.enrichment.nutrition.stage import NutritionEnrichmentStage
|
||||
# stages.append(NutritionEnrichmentStage())
|
||||
# Offline display columns. Registered last so the nutrients fallback it
|
||||
# writes is the lowest-priority source: the real per-100g figures mirrored
|
||||
# by nutrition_score_sync overwrite it whenever a lookup succeeds.
|
||||
try:
|
||||
from app.services.enrichment.content.stage import ContentEnrichmentStage
|
||||
stages.append(ContentEnrichmentStage())
|
||||
except Exception as e:
|
||||
logger.error(f"Content enrichment stage unavailable: {e}")
|
||||
|
||||
return stages
|
||||
|
||||
|
||||
|
||||
254
app/services/enrichment/post_ingest_barcodes.py
Normal file
254
app/services/enrichment/post_ingest_barcodes.py
Normal file
@@ -0,0 +1,254 @@
|
||||
"""Finds barcodes for freshly-ingested rows, in bulk, before nutrition runs.
|
||||
|
||||
WHY THIS RUNS BEFORE THE NUTRITION JOB, NOT ALONGSIDE IT
|
||||
--------------------------------------------------------
|
||||
Ordering here is a correctness property, not a preference.
|
||||
|
||||
`fetch_verified_nutrition_by_barcode` matches on the GTIN and returns at
|
||||
confidence 0.95. The name search it falls back to accepts at a minimum of 0.32.
|
||||
The nutrition job runs with `skip_if_verified=True`, so whichever path lands
|
||||
first WINS PERMANENTLY - a 0.32 name match blocks the 0.95 barcode match from
|
||||
ever being attempted. Two jobs racing would produce exactly that, silently, and
|
||||
the catalog would end up with the worse of two available answers.
|
||||
|
||||
So this is a phase inside the same job, ahead of the nutrition phases.
|
||||
|
||||
WHY BULK, NOT THE PER-PRODUCT CASCADE
|
||||
-------------------------------------
|
||||
The per-product search endpoint Open Food Facts exposes is capped at 10
|
||||
requests per minute. A 200-row upload is twenty minutes of waiting, which is
|
||||
why `ENABLE_BARCODE_LOOKUP` defaults false (settings.py:420-423) and why the
|
||||
inline stage stays off.
|
||||
|
||||
`off_bulk.fetch_brand_corpus` fetches a brand's ENTIRE Open Food Facts
|
||||
catalogue in about five requests and matches offline against it. A brand is a
|
||||
brand whether it has 3 rows or 300, so the cost is per brand, not per product.
|
||||
That is what makes barcode enrichment affordable on the shared host at all.
|
||||
|
||||
It also uses `off_bulk.score_candidates` rather than the live matcher, because
|
||||
that scorer already fixes two measured flaws: `matching.name_similarity` is
|
||||
asymmetric ("Butter milk amul" vs "Amul Butter" scores 0.882 one way and 0.418
|
||||
the other), and `matching.size_matches` vetoes any candidate with a blank size
|
||||
when 57 of 146 Amul OFF records have `quantity: null`.
|
||||
|
||||
THE THRESHOLD IS 0.88, NOT 0.78
|
||||
-------------------------------
|
||||
`BARCODE_MIN_NAME_SIMILARITY` (0.78) is the floor for the REVERSE direction,
|
||||
where a barcode has already established identity and the name is a sanity
|
||||
check. This is the forward direction: many candidates compete and the name
|
||||
carries the whole decision. `scripts/backfill_barcodes_from_off.py` measured
|
||||
0.88 as the safe auto-apply point and 0.70-0.88 as review-only, and this reuses
|
||||
that number rather than inventing one.
|
||||
|
||||
WHAT IT WRITES
|
||||
barcode, barcode_type, gtin, ean13, barcode_source, barcode_verified=False,
|
||||
barcode_lookup_status='name_matched', barcode_last_updated, and the
|
||||
field_sources record - via targeted UPDATEs that pin the row's current
|
||||
value, never via upsert_brand_products.
|
||||
|
||||
WHY NOT THE UPSERT
|
||||
`get_products_by_brand` is `SELECT *`, so a row's `embedding` comes back as
|
||||
a pgvector string, and `upsert_brand_products` only accepts a list - it
|
||||
would write NULL and destroy the embedding. Targeted UPDATEs also cannot
|
||||
clobber a concurrent write.
|
||||
|
||||
WHAT IT WILL NOT DO
|
||||
It never overwrites a barcode a row already holds. A merchant typing one in
|
||||
is holding the pack; nothing found by name similarity outranks that.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.services.enrichment.barcode.validators import (
|
||||
classify_barcode_type,
|
||||
to_ean13,
|
||||
validate_barcode,
|
||||
)
|
||||
from app.services.vector_store import _connect, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Matches scripts/backfill_barcodes_from_off.py, which measured it.
|
||||
DEFAULT_MIN_SIMILARITY = 0.88
|
||||
|
||||
BARCODE_SOURCE = "openfoodfacts_bulk (search.openfoodfacts.org)"
|
||||
|
||||
|
||||
def _rows_needing_a_barcode(cur, table: str) -> List[Dict[str, Any]]:
|
||||
"""Rows with no usable barcode. Probes the column list first, because a
|
||||
table written before the schema migration may still lack the newer ones."""
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table,),
|
||||
)
|
||||
present = {r[0] for r in cur.fetchall()}
|
||||
if not {"id", "product_name", "barcode"} <= present:
|
||||
return []
|
||||
|
||||
size = "size" if "size" in present else "NULL AS size"
|
||||
cur.execute(
|
||||
f'SELECT id, product_name, title, {size}, category, barcode '
|
||||
f'FROM "{table}" '
|
||||
f"WHERE barcode IS NULL OR btrim(barcode) = ''"
|
||||
)
|
||||
return [{"id": r[0], "product_name": r[1], "title": r[2], "size": r[3],
|
||||
"category": r[4], "barcode": r[5]} for r in cur.fetchall()]
|
||||
|
||||
|
||||
def enrich_brand_barcodes(brand: str, *, min_similarity: float = DEFAULT_MIN_SIMILARITY,
|
||||
dry_run: bool = False,
|
||||
progress_cb=None) -> Dict[str, int]:
|
||||
"""Fill blank barcodes for one brand from its Open Food Facts corpus.
|
||||
|
||||
Never raises: a brand whose corpus cannot be fetched reports zero and the
|
||||
caller moves to the next one. Enrichment is best-effort by contract.
|
||||
"""
|
||||
from app.services.enrichment.barcode.sources.off_bulk import (
|
||||
brand_tokens,
|
||||
fetch_brand_corpus,
|
||||
score_candidates,
|
||||
)
|
||||
|
||||
stats = {"candidates": 0, "matched": 0, "written": 0, "rejected": 0}
|
||||
|
||||
# The same refusal `store_catalog_pipeline.stages_8_9_enrichment` makes for
|
||||
# the inline stages, for the same reason: a barcode identifies a
|
||||
# manufactured article and the Own Products bucket is loose produce - an
|
||||
# apple, a bunch of coriander. There is no GTIN to find, and a name match
|
||||
# against some packaged product's corpus could only attach the wrong one.
|
||||
from app.services.generic_products import OWN_PRODUCTS_BRAND
|
||||
if brand == OWN_PRODUCTS_BRAND:
|
||||
return stats
|
||||
|
||||
table = f"brand_{_sanitize_name(brand)}"
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
return stats
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
rows = _rows_needing_a_barcode(cur, table)
|
||||
if not rows:
|
||||
return stats
|
||||
stats["candidates"] = len(rows)
|
||||
|
||||
try:
|
||||
# Returns the hit LIST directly (the on-disk cache file wraps it in
|
||||
# a "hits" key; the function unwraps it). About five requests for a
|
||||
# whole brand, then served from disk on later runs.
|
||||
hits = fetch_brand_corpus(brand) or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("OFF corpus unavailable for %s: %s", brand, e)
|
||||
return stats
|
||||
|
||||
if not hits:
|
||||
logger.info("Open Food Facts holds no India catalogue for %s", brand)
|
||||
return stats
|
||||
|
||||
# Stripped from both sides before names are compared, so "Hindustan
|
||||
# Unilever Hul Lux" reduces to "lux" on our side and matches OFF's
|
||||
# "Lux". Computed once per brand, not once per row.
|
||||
drop = brand_tokens(brand)
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
if progress_cb:
|
||||
progress_cb(index, len(rows))
|
||||
|
||||
title = row.get("title") or row.get("product_name") or ""
|
||||
try:
|
||||
scored = score_candidates(
|
||||
hits, title, [row.get("size") or ""], drop,
|
||||
review_min=min_similarity,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("scoring failed for %r: %s", title, e)
|
||||
continue
|
||||
|
||||
# score_candidates already drops anything under review_min, so the
|
||||
# first entry is the best acceptable one. The explicit re-check is
|
||||
# kept because the ordering contract is "best first", not "all
|
||||
# above the floor" - relying on the filter alone would silently
|
||||
# break if that ever changed.
|
||||
best = scored[0] if scored else None
|
||||
if not best or best.score < min_similarity:
|
||||
stats["rejected"] += 1
|
||||
continue
|
||||
|
||||
code = validate_barcode(getattr(best, "barcode", None))
|
||||
if not code:
|
||||
stats["rejected"] += 1
|
||||
continue
|
||||
|
||||
stats["matched"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
kind = classify_barcode_type(code)
|
||||
sources = {
|
||||
"barcode": {"method": "sourced", "source": BARCODE_SOURCE,
|
||||
"confidence": round(float(best.score), 3),
|
||||
"note": "matched on name against the brand's OFF "
|
||||
"catalogue; not verified against the pack"},
|
||||
"gtin": {"method": "derived", "source": "validators.validate_barcode"},
|
||||
"ean13": {"method": "derived", "source": "validators.to_ean13"},
|
||||
}
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f'UPDATE "{table}" SET barcode = %s, barcode_type = %s, '
|
||||
f"gtin = %s, ean13 = %s, barcode_source = %s, "
|
||||
f"barcode_verified = FALSE, barcode_lookup_status = %s, "
|
||||
f"barcode_last_updated = NOW(), "
|
||||
f"field_sources = COALESCE(field_sources, '{{}}'::jsonb) "
|
||||
f" || %s::jsonb, "
|
||||
f"updated_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id = %s "
|
||||
f" AND (barcode IS NULL OR btrim(barcode) = '')",
|
||||
(code, kind.value, code, to_ean13(code), BARCODE_SOURCE,
|
||||
"name_matched", Json(sources), row["id"]),
|
||||
)
|
||||
stats["written"] += cur.rowcount
|
||||
conn.commit()
|
||||
except Exception as e: # noqa: BLE001
|
||||
conn.rollback()
|
||||
logger.warning("barcode write failed for %s id=%s: %s",
|
||||
table, row["id"], e)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def enrich_barcodes_for_brands(brands: Iterable[str], *,
|
||||
min_similarity: float = DEFAULT_MIN_SIMILARITY,
|
||||
dry_run: bool = False,
|
||||
progress_cb=None) -> Dict[str, Dict[str, int]]:
|
||||
"""Run `enrich_brand_barcodes` over several brands, one at a time.
|
||||
|
||||
Deliberately sequential. `EnrichmentPipeline`'s five-way concurrency is for
|
||||
per-row work against a local corpus; firing five brand-corpus fetches at
|
||||
Open Food Facts at once is how a shared host earns a rate limit.
|
||||
"""
|
||||
out: Dict[str, Dict[str, int]] = {}
|
||||
names = [b.strip() for b in brands if b and b.strip()]
|
||||
for i, brand in enumerate(names):
|
||||
if progress_cb:
|
||||
progress_cb(i, len(names))
|
||||
try:
|
||||
stats = enrich_brand_barcodes(brand, min_similarity=min_similarity,
|
||||
dry_run=dry_run)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("barcode enrichment failed for %s: %s", brand, e)
|
||||
continue
|
||||
if stats.get("candidates"):
|
||||
out[brand] = stats
|
||||
logger.info("%s: %d without a barcode, %d matched, %d written",
|
||||
brand, stats["candidates"], stats["matched"], stats["written"])
|
||||
return out
|
||||
@@ -52,6 +52,34 @@ def run_enrich_job(job_id: str, *, label: str = "", **enrich_kwargs: Any) -> Non
|
||||
def progress_cb(done: int, total: int) -> None:
|
||||
nutrition_job_store.update(job_id, processed=done, total=total)
|
||||
|
||||
# PHASE 1 - BARCODES, BEFORE ANY NUTRITION LOOKUP.
|
||||
#
|
||||
# The ordering is a correctness property. `fetch_verified_nutrition_by_barcode`
|
||||
# matches on the GTIN and returns confidence 0.95; the name search it falls
|
||||
# back to accepts at 0.32. Because the nutrition phase runs with
|
||||
# skip_if_verified=True, whichever lands first wins permanently - so a name
|
||||
# match obtained before the barcode exists blocks the far better barcode
|
||||
# match from ever being tried.
|
||||
#
|
||||
# It is also cheap: one brand corpus (~5 requests) instead of one search per
|
||||
# product against an endpoint capped at 10 requests/minute.
|
||||
#
|
||||
# Guarded separately and never fatal: a barcode phase that fails must still
|
||||
# leave the nutrition phase to do what it always did.
|
||||
brands_for_barcodes = enrich_kwargs.get("brands") or []
|
||||
if brands_for_barcodes and settings.ENRICH_BARCODES_ON_UPLOAD:
|
||||
try:
|
||||
nutrition_job_store.update(job_id, detail="Finding barcodes")
|
||||
from app.services.enrichment.post_ingest_barcodes import (
|
||||
enrich_barcodes_for_brands,
|
||||
)
|
||||
found = enrich_barcodes_for_brands(brands_for_barcodes)
|
||||
written = sum(s.get("written", 0) for s in found.values())
|
||||
if written:
|
||||
logger.info("Barcode phase wrote %d barcode(s) before scoring", written)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Barcode phase failed (nutrition still runs): %s", e)
|
||||
|
||||
try:
|
||||
result = nutrition_enrichment_service.enrich_all_products(
|
||||
progress_cb=progress_cb, **enrich_kwargs)
|
||||
|
||||
@@ -545,9 +545,17 @@ def fetch_verified_nutrition_by_barcode(
|
||||
candidate_size=product.get("quantity") or "",
|
||||
candidate_countries=",".join(product.get("countries_tags") or []),
|
||||
)
|
||||
# barcode_is_identity=True is safe HERE and nowhere else. The barcode has
|
||||
# already established which single product this is - there are no competing
|
||||
# candidates to disambiguate - so the name check is a guard against our
|
||||
# barcode being wrong, not the evidence for identity. Open Food Facts
|
||||
# stores short names ("Munch", "Maaza") where we store long ones, and
|
||||
# without this 149 of 300 barcoded rows were rejected as "wrong product"
|
||||
# when the barcode had resolved perfectly. See matching.name_is_contained.
|
||||
matched, similarity = is_match(
|
||||
candidate, brand, title, size,
|
||||
min_name_similarity=min_name_similarity,
|
||||
barcode_is_identity=True,
|
||||
)
|
||||
if not matched:
|
||||
logger.info(
|
||||
|
||||
@@ -49,6 +49,53 @@ class EnrichmentResult:
|
||||
duration_seconds: float = 0.0
|
||||
|
||||
|
||||
def _score_from_existing_facts(brand: str, image_id: str,
|
||||
facts: Dict[str, Any]) -> bool:
|
||||
"""Compute and store insights from facts already held. No network.
|
||||
|
||||
Exists because the two ways a `nutrition_facts` row can be written disagree
|
||||
about whose job scoring is. `enrich_one_product` fetches and scores in one
|
||||
pass; `scripts/backfill_nutrition_from_barcodes` and its siblings write
|
||||
facts only, on purpose, because they are narrow repair tools that should
|
||||
not also be recomputing narratives. Nothing then closed the gap.
|
||||
|
||||
Returns True when it wrote something. A row that already carries a score is
|
||||
left alone, so this is idempotent and safe to call on every skip.
|
||||
"""
|
||||
try:
|
||||
existing = nutrition_db.get_nutrition_insights(brand, image_id)
|
||||
if existing and existing.get("nutrition_score") is not None:
|
||||
return False
|
||||
|
||||
scores = nutrition_scoring.compute_scores(facts)
|
||||
if not scores:
|
||||
# `compute_scores` returns None rather than fabricating when there
|
||||
# is nothing scoreable. Respect that - do not write a zero.
|
||||
return False
|
||||
|
||||
positive = nutrition_scoring.generate_positive_insights(facts)
|
||||
cautions = nutrition_scoring.generate_cautions(facts)
|
||||
allergens = nutrition_scoring.normalize_allergens(facts)
|
||||
insights: Dict[str, Any] = {
|
||||
"brand": brand, "image_id": image_id,
|
||||
"positive_insights": positive, "nutritional_cautions": cautions,
|
||||
# No narrative: generating one is an LLM call, and this path exists
|
||||
# precisely to avoid doing expensive work for a row that already
|
||||
# has its data. The admin /enrich run fills narratives in later.
|
||||
"ai_summary": (existing or {}).get("ai_summary", ""),
|
||||
"diet_tags": nutrition_scoring.classify_diet_tags(facts),
|
||||
"allergens": allergens,
|
||||
"allergen_source": (facts.get("data_source") or "unavailable") if allergens else "unavailable",
|
||||
"data_status": facts.get("data_status"),
|
||||
}
|
||||
insights.update(scores)
|
||||
nutrition_db.upsert_nutrition_insights(insights)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001 - scoring is a nicety, not the write
|
||||
logger.warning("Could not score existing facts for %s/%s: %s", brand, image_id, e)
|
||||
return False
|
||||
|
||||
|
||||
def enrich_one_product(brand: str, image_id: str, product_name: str, category: str,
|
||||
skip_if_verified: bool = False, generate_narrative: bool = True) -> str:
|
||||
"""Runs the full pipeline for a single product. Returns the resulting
|
||||
@@ -66,6 +113,22 @@ def enrich_one_product(brand: str, image_id: str, product_name: str, category: s
|
||||
if skip_if_verified:
|
||||
existing = nutrition_db.get_nutrition_facts(brand, image_id)
|
||||
if existing and existing.get("data_status") == "verified":
|
||||
# SKIPPING THE FETCH MUST NOT ALSO SKIP THE SCORING.
|
||||
#
|
||||
# This used to `return "verified"` outright, which meant a product
|
||||
# whose facts arrived from a backfill script could never acquire a
|
||||
# score. Those scripts write `nutrition_facts` and deliberately do
|
||||
# not touch `nutrition_insights` - so the row looks verified here,
|
||||
# gets skipped, and its health score stays NULL forever. Measured
|
||||
# on 2026-09-08: 157 rows held usable facts with no score, which is
|
||||
# most of the gap between `nutrients_per_100g` at 65.6% and
|
||||
# `health_score` at 54.8%.
|
||||
#
|
||||
# Scoring is pure computation over facts already in hand - no
|
||||
# network, no LLM - so doing it here costs one cheap read and
|
||||
# nothing else. It is skipped when a score already exists, so a
|
||||
# re-run is a no-op.
|
||||
_score_from_existing_facts(brand, image_id, existing)
|
||||
return "verified"
|
||||
|
||||
facts = nutrition_data_service.fetch_verified_nutrition(brand, product_name, category or "")
|
||||
|
||||
@@ -57,6 +57,8 @@ from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.services.vector_store import (
|
||||
_connect,
|
||||
_list_brand_table_suffixes,
|
||||
@@ -216,6 +218,175 @@ def sync_scores_to_brand_tables(brands: Optional[List[str]] = None,
|
||||
return changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# per-100g nutrients
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The flat per-100g columns on `nutrition_facts`, with the unit each is stored
|
||||
# in. Order is the order a label reads, not alphabetical, because it is also
|
||||
# the order the rendered display list comes out in.
|
||||
_NUTRIENT_COLUMNS: Tuple[Tuple[str, str, str], ...] = (
|
||||
("calories_kcal", "Energy", "kcal"),
|
||||
("protein_g", "Protein", "g"),
|
||||
("carbohydrates_g", "Carbohydrates", "g"),
|
||||
("total_sugar_g", "Total sugar", "g"),
|
||||
("added_sugar_g", "Added sugar", "g"),
|
||||
("dietary_fiber_g", "Dietary fibre", "g"),
|
||||
("total_fat_g", "Total fat", "g"),
|
||||
("saturated_fat_g", "Saturated fat", "g"),
|
||||
("trans_fat_g", "Trans fat", "g"),
|
||||
("cholesterol_mg", "Cholesterol", "mg"),
|
||||
("sodium_mg", "Sodium", "mg"),
|
||||
("potassium_mg", "Potassium", "mg"),
|
||||
("calcium_mg", "Calcium", "mg"),
|
||||
("iron_mg", "Iron", "mg"),
|
||||
("magnesium_mg", "Magnesium", "mg"),
|
||||
("zinc_mg", "Zinc", "mg"),
|
||||
("vitamin_a_mcg", "Vitamin A", "mcg"),
|
||||
("vitamin_c_mg", "Vitamin C", "mg"),
|
||||
("vitamin_d_mcg", "Vitamin D", "mcg"),
|
||||
("vitamin_e_mg", "Vitamin E", "mg"),
|
||||
("omega_3_g", "Omega-3", "g"),
|
||||
("omega_6_g", "Omega-6", "g"),
|
||||
)
|
||||
|
||||
# How many rendered lines the display column carries. The old keyword
|
||||
# generator capped at 8 and the UI is laid out for roughly that.
|
||||
_DISPLAY_LIMIT = 8
|
||||
|
||||
|
||||
def render_nutrient_lines(facts: Dict[str, Any], limit: int = _DISPLAY_LIMIT) -> List[str]:
|
||||
"""The `nutrients TEXT[]` display list, built from real measured numbers.
|
||||
|
||||
WHY THIS REPLACES WHAT WAS THERE
|
||||
`catalog_engine.generate_nutrients_info` produced this column from
|
||||
category keywords - "Energy - High", "Protein - Good Source" - with no
|
||||
connection to `nutrition_facts` at all. A product could show
|
||||
"Vitamin B Complex - Energy" because its category name matched a word.
|
||||
Where real per-100g figures exist they are strictly better, and they
|
||||
are what the merchant console is actually being asked for.
|
||||
|
||||
Values are emitted per 100 g because that is the basis `nutrition_facts`
|
||||
stores and the basis Indian FSSAI labelling uses. A missing nutrient is
|
||||
omitted rather than rendered as zero - "Trans fat 0 g" is a claim, and an
|
||||
absent measurement is not evidence of absence.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
for column, label, unit in _NUTRIENT_COLUMNS:
|
||||
value = facts.get(column)
|
||||
if value is None:
|
||||
continue
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
# Trim a trailing .0 so "Protein 12 g" does not read as "12.0".
|
||||
text = f"{number:g}"
|
||||
lines.append(f"{label} {text} {unit} per 100 g")
|
||||
if len(lines) >= limit:
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def sync_nutrients_to_brand_tables(brands: Optional[List[str]] = None,
|
||||
*, dry_run: bool = False) -> Dict[str, int]:
|
||||
"""Mirror `nutrition_facts` per-100g figures onto the brand tables.
|
||||
|
||||
Writes two columns, for two different readers:
|
||||
|
||||
* `nutrients_per_100g` (JSONB) - the numbers, for anything computing on
|
||||
them. This is a denormalised copy so a single product read needs no
|
||||
join, exactly as `nutrition_score`/`health_score` already are.
|
||||
* `nutrients` (TEXT[]) - the same numbers rendered for display, replacing
|
||||
the category-keyword strings.
|
||||
|
||||
`nutrition_facts` remains the source of truth; nothing here writes back to
|
||||
it. Like the score mirror this is a MIRROR, not an accumulator: a row whose
|
||||
facts have gone (the non-consumable purge removes them) has its copy
|
||||
cleared rather than left stale.
|
||||
|
||||
A row whose facts exist but are `unavailable` is left alone rather than
|
||||
blanked - `data_status='unavailable'` means "we looked and found nothing",
|
||||
which is not a reason to destroy a display list the catalog already had.
|
||||
"""
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
logger.warning("nutrient sync: no database connection")
|
||||
return {}
|
||||
|
||||
columns = ", ".join(c for c, _, _ in _NUTRIENT_COLUMNS)
|
||||
changed: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
targets = _target_suffixes(cur, brands)
|
||||
|
||||
for suffix, display in targets:
|
||||
table = f"brand_{suffix}"
|
||||
if not dry_run:
|
||||
ensure_brand_schema(display)
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT image_id, {columns} FROM nutrition_facts "
|
||||
f"WHERE lower(brand) = lower(%s) AND data_status <> 'unavailable'",
|
||||
(display,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
continue
|
||||
|
||||
names = [c for c, _, _ in _NUTRIENT_COLUMNS]
|
||||
updates = 0
|
||||
for row in rows:
|
||||
image_id = row[0]
|
||||
facts = {n: v for n, v in zip(names, row[1:]) if v is not None}
|
||||
if not facts:
|
||||
continue
|
||||
block = {n: float(v) for n, v in facts.items()}
|
||||
lines = render_nutrient_lines(facts)
|
||||
if dry_run:
|
||||
cur.execute(
|
||||
f'SELECT 1 FROM "{table}" WHERE image_id = %s '
|
||||
f"AND nutrients_per_100g IS DISTINCT FROM %s::jsonb",
|
||||
(image_id, Json(block)),
|
||||
)
|
||||
updates += 1 if cur.fetchone() else 0
|
||||
continue
|
||||
cur.execute(
|
||||
f'UPDATE "{table}" SET nutrients_per_100g = %s::jsonb, '
|
||||
f" nutrients = %s, "
|
||||
f" field_sources = COALESCE(field_sources, '{{}}'::jsonb) || %s::jsonb, "
|
||||
f" updated_at = CURRENT_TIMESTAMP "
|
||||
f" WHERE image_id = %s "
|
||||
f" AND nutrients_per_100g IS DISTINCT FROM %s::jsonb",
|
||||
(Json(block), lines,
|
||||
Json({"nutrients": {"method": "sourced",
|
||||
"source": "nutrition_facts"},
|
||||
"nutrients_per_100g": {"method": "sourced",
|
||||
"source": "nutrition_facts"}}),
|
||||
image_id, Json(block)),
|
||||
)
|
||||
updates += cur.rowcount
|
||||
|
||||
if not dry_run:
|
||||
conn.commit()
|
||||
except Exception as e: # noqa: BLE001
|
||||
conn.rollback()
|
||||
logger.warning("nutrient sync failed for %s: %s", table, e)
|
||||
continue
|
||||
|
||||
if updates:
|
||||
changed[display] = updates
|
||||
logger.info("%s %d row(s) in %s",
|
||||
"would update" if dry_run else "updated", updates, table)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# seed catalog JSON
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -356,3 +527,15 @@ def sync_after_write(brands: Optional[List[str]] = None) -> None:
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("score mirror failed (scores are still in "
|
||||
"nutrition_insights): %s", e)
|
||||
|
||||
# Separately guarded: a failure mirroring the numbers must not lose the
|
||||
# scores that were just mirrored successfully, and vice versa. Both are
|
||||
# copies of data that is safe in nutrition_facts / nutrition_insights.
|
||||
try:
|
||||
changed = sync_nutrients_to_brand_tables(brands)
|
||||
if changed:
|
||||
logger.info("mirrored nutrients onto %d brand table(s): %s",
|
||||
len(changed), ", ".join(sorted(changed)))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("nutrient mirror failed (figures are still in "
|
||||
"nutrition_facts): %s", e)
|
||||
|
||||
@@ -6,7 +6,10 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import psycopg
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.infrastructure.settings import (
|
||||
USE_PGVECTOR, DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD,
|
||||
@@ -56,6 +59,56 @@ def _sanitize_name(name: str) -> str:
|
||||
return name.strip('_')
|
||||
|
||||
|
||||
def _epoch_to_timestamp(value: Any) -> Optional[datetime]:
|
||||
"""Convert BarcodeResult.barcode_last_updated to what the column holds.
|
||||
|
||||
`BarcodeResult` defaults this field to `time.time()` - a float epoch - but
|
||||
the seven brand tables that already carry `barcode_last_updated` were
|
||||
migrated out-of-band as TIMESTAMP. Handing psycopg a bare float for a
|
||||
timestamp column raises, so the conversion happens here rather than being
|
||||
pushed onto every caller. A datetime is passed through untouched, and
|
||||
anything unparseable degrades to None.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
# A seed catalog round-trips this as ISO-8601 (brand_sync._jsonable), while
|
||||
# BarcodeResult and the older JSON files carry a float epoch. Both have to
|
||||
# load, or re-seeding a brand would drop the timestamp it just exported.
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return datetime.fromtimestamp(float(value))
|
||||
except (ValueError, TypeError, OSError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _to_numeric_or_none(value: Any) -> Optional[float]:
|
||||
"""Coerce an enrichment stage's numeric output for a NUMERIC column.
|
||||
|
||||
Returns None for anything unparseable rather than raising, because a
|
||||
malformed tax figure must degrade to "no tax figure stored" and never
|
||||
abort a whole batch's write. Strips a leading currency symbol and commas,
|
||||
which is how these arrive when a sheet supplied them as text.
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
if isinstance(value, str):
|
||||
cleaned = re.sub(r"[^\d.\-]", "", value)
|
||||
return float(cleaned) if cleaned not in ("", "-", ".", "-.") else None
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
DDL_CREATE_EXTENSION = "CREATE EXTENSION IF NOT EXISTS vector;"
|
||||
def get_brand_table_ddl(brand: str) -> str:
|
||||
"""Generate DDL for brand-specific table - simplified with only essential fields"""
|
||||
@@ -88,12 +141,52 @@ def get_brand_table_ddl(brand: str) -> str:
|
||||
selling_price NUMERIC,
|
||||
barcode TEXT,
|
||||
barcode_type TEXT,
|
||||
|
||||
|
||||
-- The rest of what BarcodeResult.as_product_fields() produces. Until
|
||||
-- these existed the barcode stage returned nine fields and the INSERT
|
||||
-- named two, so seven were computed and then dropped on the floor -
|
||||
-- including the provenance needed to tell a verified GTIN from a
|
||||
-- name-matched guess.
|
||||
--
|
||||
-- TYPES ARE ADOPTED, NOT CHOSEN. Seven brand tables already carry
|
||||
-- these columns, added out-of-band before any code created them.
|
||||
-- ADD COLUMN IF NOT EXISTS does not reconcile a type difference, so
|
||||
-- picking a "better" type here would leave 7 tables disagreeing with
|
||||
-- 49 forever. Verified against information_schema on 2026-09-08:
|
||||
-- barcode_last_updated is TIMESTAMP (not the float epoch
|
||||
-- BarcodeResult carries - see _epoch_to_timestamp), and the tax
|
||||
-- columns are REAL (not NUMERIC).
|
||||
gtin TEXT,
|
||||
ean13 TEXT,
|
||||
upc TEXT,
|
||||
barcode_source TEXT,
|
||||
barcode_verified BOOLEAN,
|
||||
barcode_lookup_status TEXT,
|
||||
barcode_last_updated TIMESTAMP,
|
||||
|
||||
-- Computed by the HSN/GST stage, likewise discarded before this.
|
||||
gst_percent REAL,
|
||||
tax_amount REAL,
|
||||
hsn_gst_needs_review BOOLEAN,
|
||||
|
||||
-- Essential fields
|
||||
highlights TEXT[],
|
||||
nutrients TEXT[],
|
||||
-- Real per-100g figures mirrored from nutrition_facts by
|
||||
-- nutrition_score_sync. `nutrients` above stays the human-readable
|
||||
-- marketing list; these are the numbers.
|
||||
nutrients_per_100g JSONB,
|
||||
search_query TEXT,
|
||||
|
||||
-- Per-field provenance, keyed by column name; each value records how
|
||||
-- that column's value was arrived at. One JSONB map rather than ~60
|
||||
-- scalar columns, because every scalar column would have to be named
|
||||
-- explicitly in the INSERT below and in brand_sync.EXPORT_COLUMNS or
|
||||
-- it gets silently dropped - the exact failure the barcode columns
|
||||
-- above are here to fix.
|
||||
-- `method` is one of: sourced | estimated | derived | not_applicable.
|
||||
field_sources JSONB DEFAULT '{{}}'::jsonb,
|
||||
|
||||
-- Nutrition scores, mirrored from nutrition_insights by
|
||||
-- nutrition_score_sync. Deliberately NOT written by
|
||||
-- upsert_brand_products - see the comment on its INSERT.
|
||||
@@ -160,9 +253,30 @@ def _ensure_columns(cur, table_name: str) -> None:
|
||||
"selling_price": "NUMERIC",
|
||||
"barcode": "TEXT",
|
||||
"barcode_type": "TEXT",
|
||||
# The other seven fields BarcodeResult.as_product_fields() returns.
|
||||
# Adding them here is what puts them on all 56 existing brand tables -
|
||||
# before this only 7 tables had them, added out-of-band, and even those
|
||||
# never received a value because the INSERT did not name them.
|
||||
# Types adopted from the 7 tables that already have them, NOT chosen -
|
||||
# see get_brand_table_ddl. TIMESTAMP and REAL are what is on disk.
|
||||
"gtin": "TEXT",
|
||||
"ean13": "TEXT",
|
||||
"upc": "TEXT",
|
||||
"barcode_source": "TEXT",
|
||||
"barcode_verified": "BOOLEAN",
|
||||
"barcode_lookup_status": "TEXT",
|
||||
"barcode_last_updated": "TIMESTAMP",
|
||||
# Computed by the HSN/GST stage and discarded before this.
|
||||
"gst_percent": "REAL",
|
||||
"tax_amount": "REAL",
|
||||
"hsn_gst_needs_review": "BOOLEAN",
|
||||
"highlights": "TEXT[]",
|
||||
"nutrients": "TEXT[]",
|
||||
"nutrients_per_100g": "JSONB",
|
||||
"search_query": "TEXT",
|
||||
# Per-field provenance map - see get_brand_table_ddl for why this is
|
||||
# one JSONB column and not sixty scalar ones.
|
||||
"field_sources": "JSONB",
|
||||
# Mirrored from nutrition_insights by nutrition_score_sync, never by
|
||||
# the INSERT below. This dict is the only migration mechanism there
|
||||
# is (no Alembic), so listing them here is what creates them on every
|
||||
@@ -193,7 +307,11 @@ def _ensure_columns(cur, table_name: str) -> None:
|
||||
# nutrition_score_sync, not legacy debris from an older DDL, and this sweep
|
||||
# must never claim them. (Both are nullable, so it would be a no-op today -
|
||||
# the entry is here so it stays a no-op if that ever changes.)
|
||||
inserted_cols = {"id", "product_name", "title", "description", "category", "image_id", "image_url", "image_urls", "price_range", "size_variants", "providers", "fssai_license", "product_sku", "sku_source", "hsn_code", "final_selling_price", "selling_price", "barcode", "barcode_type", "highlights", "nutrients", "search_query", "nutrition_score", "health_score", "embedding", "created_at", "updated_at"}
|
||||
#
|
||||
# `nutrients_per_100g` joins them: it is mirrored from nutrition_facts by
|
||||
# the same sync, not written by the INSERT, and is likewise a current
|
||||
# column rather than legacy debris.
|
||||
inserted_cols = {"id", "product_name", "title", "description", "category", "image_id", "image_url", "image_urls", "price_range", "size_variants", "providers", "fssai_license", "product_sku", "sku_source", "hsn_code", "final_selling_price", "selling_price", "barcode", "barcode_type", "gtin", "ean13", "upc", "barcode_source", "barcode_verified", "barcode_lookup_status", "barcode_last_updated", "gst_percent", "tax_amount", "hsn_gst_needs_review", "highlights", "nutrients", "nutrients_per_100g", "field_sources", "search_query", "nutrition_score", "health_score", "embedding", "created_at", "updated_at"}
|
||||
for col, is_nullable, col_def in col_info:
|
||||
if col not in inserted_cols and is_nullable == 'NO' and col_def is None:
|
||||
cur.execute(f"ALTER TABLE {table_name} ALTER COLUMN {col} DROP NOT NULL")
|
||||
@@ -346,6 +464,30 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
|
||||
barcode = str(p.get("barcode") or p.get("Barcode") or "").strip() or None
|
||||
barcode_type = str(p.get("barcode_type") or p.get("Barcode_Type") or "").strip() or None
|
||||
|
||||
# The remaining barcode fields. A product dict that never went through
|
||||
# the barcode stage simply has none of these, so they arrive as None
|
||||
# and the COALESCE in DO UPDATE SET below keeps whatever is already
|
||||
# stored rather than blanking it.
|
||||
gtin = str(p.get("gtin") or "").strip() or None
|
||||
ean13 = str(p.get("ean13") or "").strip() or None
|
||||
upc = str(p.get("upc") or "").strip() or None
|
||||
barcode_source = str(p.get("barcode_source") or "").strip() or None
|
||||
barcode_lookup_status = str(p.get("barcode_lookup_status") or "").strip() or None
|
||||
barcode_verified = p.get("barcode_verified")
|
||||
barcode_verified = bool(barcode_verified) if barcode_verified is not None else None
|
||||
barcode_last_updated = _epoch_to_timestamp(p.get("barcode_last_updated"))
|
||||
|
||||
# HSN/GST stage output beyond hsn_code itself.
|
||||
gst_percent = _to_numeric_or_none(p.get("gst_percent"))
|
||||
tax_amount = _to_numeric_or_none(p.get("tax_amount"))
|
||||
hsn_gst_needs_review = p.get("hsn_gst_needs_review")
|
||||
hsn_gst_needs_review = bool(hsn_gst_needs_review) if hsn_gst_needs_review is not None else None
|
||||
|
||||
# Per-field provenance. Must be a JSON object; anything else is
|
||||
# dropped rather than stored as a shape readers cannot index into.
|
||||
field_sources = p.get("field_sources")
|
||||
field_sources = Json(field_sources) if isinstance(field_sources, dict) and field_sources else None
|
||||
|
||||
# Essential fields
|
||||
highlights = p.get("highlights", [])
|
||||
if not isinstance(highlights, list):
|
||||
@@ -387,9 +529,20 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
|
||||
selling_price,
|
||||
barcode,
|
||||
barcode_type,
|
||||
gtin,
|
||||
ean13,
|
||||
upc,
|
||||
barcode_source,
|
||||
barcode_verified,
|
||||
barcode_lookup_status,
|
||||
barcode_last_updated,
|
||||
gst_percent,
|
||||
tax_amount,
|
||||
hsn_gst_needs_review,
|
||||
highlights, # TEXT[] - psycopg will handle conversion
|
||||
nutrients, # TEXT[] - psycopg will handle conversion
|
||||
search_query,
|
||||
field_sources,
|
||||
embedding_str
|
||||
))
|
||||
|
||||
@@ -430,8 +583,11 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
|
||||
f"""
|
||||
INSERT INTO {table_name}
|
||||
(product_name, title, description, category, image_id, image_url, image_urls, price_range, size_variants, providers,
|
||||
fssai_license, product_sku, sku_source, hsn_code, final_selling_price, selling_price, barcode, barcode_type, highlights, nutrients, search_query, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
fssai_license, product_sku, sku_source, hsn_code, final_selling_price, selling_price, barcode, barcode_type,
|
||||
gtin, ean13, upc, barcode_source, barcode_verified, barcode_lookup_status, barcode_last_updated,
|
||||
gst_percent, tax_amount, hsn_gst_needs_review,
|
||||
highlights, nutrients, search_query, field_sources, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (image_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
title = EXCLUDED.title,
|
||||
@@ -448,12 +604,63 @@ def upsert_brand_products(brand: str, products: List[Dict[str, Any]], cleanup: b
|
||||
hsn_code = EXCLUDED.hsn_code,
|
||||
final_selling_price = EXCLUDED.final_selling_price,
|
||||
selling_price = EXCLUDED.selling_price,
|
||||
barcode = EXCLUDED.barcode,
|
||||
barcode_type = EXCLUDED.barcode_type,
|
||||
-- barcode/barcode_type are COALESCEd with the seven
|
||||
-- columns below rather than assigned like their legacy
|
||||
-- neighbours, because they are the same identity group.
|
||||
-- Proven against brand_zzsmoketest on 2026-09-08: a bare
|
||||
-- re-seed (a product dict with no barcode keys, which is
|
||||
-- what the seed loader and user_products build) set
|
||||
-- barcode to NULL while COALESCE kept gtin and ean13 -
|
||||
-- leaving a row claiming a GTIN with no barcode. That
|
||||
-- half-erased state is worse than either whole one.
|
||||
--
|
||||
-- It also aligns this statement with the invariant the
|
||||
-- rest of the barcode code already enforces: stage.py:38
|
||||
-- skips a row that has a barcode, and base.py:76-100
|
||||
-- refuses to blank a held value. The upsert was the one
|
||||
-- place that still could.
|
||||
barcode = COALESCE(EXCLUDED.barcode, {table_name}.barcode),
|
||||
barcode_type = COALESCE(EXCLUDED.barcode_type, {table_name}.barcode_type),
|
||||
highlights = EXCLUDED.highlights,
|
||||
nutrients = EXCLUDED.nutrients,
|
||||
search_query = EXCLUDED.search_query,
|
||||
embedding = EXCLUDED.embedding,
|
||||
-- COALESCE, NOT PLAIN EXCLUDED, FOR EVERY COLUMN BELOW.
|
||||
--
|
||||
-- These are enrichment outputs. Most writers that reach
|
||||
-- this statement never carry them: the seed loader,
|
||||
-- user_products._build_product_dict and brand_sync's
|
||||
-- re-seed all build a product dict from a spreadsheet,
|
||||
-- so their EXCLUDED values are NULL. A plain assignment
|
||||
-- would therefore wipe a hard-won barcode and its
|
||||
-- provenance on the next re-seed of the brand - the same
|
||||
-- failure the score-column comment above describes, which
|
||||
-- is why those two columns are omitted entirely.
|
||||
-- (Naming them here, even inside a comment, trips the
|
||||
-- substring guard in tests/test_brand_table_scores.py.
|
||||
-- That guard is crude on purpose; leave it that way.)
|
||||
--
|
||||
-- COALESCE keeps the stored value when nothing new
|
||||
-- arrives, while still letting a real incoming value
|
||||
-- correct a stored one. That matches the enrichment
|
||||
-- invariant in enrichment/base.py: a stage may fill a gap
|
||||
-- or correct a value, it may not erase one.
|
||||
gtin = COALESCE(EXCLUDED.gtin, {table_name}.gtin),
|
||||
ean13 = COALESCE(EXCLUDED.ean13, {table_name}.ean13),
|
||||
upc = COALESCE(EXCLUDED.upc, {table_name}.upc),
|
||||
barcode_source = COALESCE(EXCLUDED.barcode_source, {table_name}.barcode_source),
|
||||
barcode_verified = COALESCE(EXCLUDED.barcode_verified, {table_name}.barcode_verified),
|
||||
barcode_lookup_status = COALESCE(EXCLUDED.barcode_lookup_status, {table_name}.barcode_lookup_status),
|
||||
barcode_last_updated = COALESCE(EXCLUDED.barcode_last_updated, {table_name}.barcode_last_updated),
|
||||
gst_percent = COALESCE(EXCLUDED.gst_percent, {table_name}.gst_percent),
|
||||
tax_amount = COALESCE(EXCLUDED.tax_amount, {table_name}.tax_amount),
|
||||
hsn_gst_needs_review = COALESCE(EXCLUDED.hsn_gst_needs_review, {table_name}.hsn_gst_needs_review),
|
||||
-- Merged, not replaced: a run that learns the provenance
|
||||
-- of one field must not drop what is known about the
|
||||
-- others. `||` is a shallow merge, which is the right
|
||||
-- depth here - each key's value is one flat record.
|
||||
field_sources = COALESCE({table_name}.field_sources, '{{}}'::jsonb)
|
||||
|| COALESCE(EXCLUDED.field_sources, '{{}}'::jsonb),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
rows,
|
||||
|
||||
7
data/cache/off_brand_corpus/24_mantra.json
vendored
Normal file
7
data/cache/off_brand_corpus/24_mantra.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "24 mantra",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852019.7252567,
|
||||
"fetched_at_human": "2026-09-08 12:50:19",
|
||||
"hits": []
|
||||
}
|
||||
354
data/cache/off_brand_corpus/balaji.json
vendored
Normal file
354
data/cache/off_brand_corpus/balaji.json
vendored
Normal file
@@ -0,0 +1,354 @@
|
||||
{
|
||||
"brand": "balaji",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852012.4404967,
|
||||
"fetched_at_human": "2026-09-08 12:50:12",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906010503987",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "100000",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Soft Imli",
|
||||
"product_name_en": "Soft Imli"
|
||||
},
|
||||
{
|
||||
"code": "8906010500016",
|
||||
"brands": [
|
||||
"balaji"
|
||||
],
|
||||
"quantity": "40",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "POTATO WAFERS",
|
||||
"product_name_en": "POTATO WAFERS"
|
||||
},
|
||||
{
|
||||
"code": "8906010501228",
|
||||
"brands": [
|
||||
"Balaji Wafers"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Scoopitos Masala Flavour",
|
||||
"product_name_en": "Scoopitos Masala Flavour"
|
||||
},
|
||||
{
|
||||
"code": "8906010502782",
|
||||
"brands": [
|
||||
"Balaji Wafer"
|
||||
],
|
||||
"quantity": "60g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Stack Up - Simply Salted",
|
||||
"product_name_en": "Stack Up - Simply Salted"
|
||||
},
|
||||
{
|
||||
"code": "8906010500269",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "22 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Poprings Masala flavour",
|
||||
"product_name_en": "Poprings Masala flavour"
|
||||
},
|
||||
{
|
||||
"code": "8906010502171",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "300g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Gathiya",
|
||||
"product_name_en": "Gathiya"
|
||||
},
|
||||
{
|
||||
"code": "8906010503611",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "135g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHRUNCHEX",
|
||||
"product_name_en": "CHRUNCHEX"
|
||||
},
|
||||
{
|
||||
"code": "8906010500559",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "25 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Mung Dal",
|
||||
"product_name_en": "Mung Dal"
|
||||
},
|
||||
{
|
||||
"code": "8906010501419",
|
||||
"brands": [
|
||||
"balaji"
|
||||
],
|
||||
"quantity": "30g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "poprings",
|
||||
"product_name_en": "poprings"
|
||||
},
|
||||
{
|
||||
"code": "8906010505431",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pop Rings",
|
||||
"product_name_en": "Pop Rings"
|
||||
},
|
||||
{
|
||||
"code": "8906010500139",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "35 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers",
|
||||
"product_name_en": "Balaji Wafers"
|
||||
},
|
||||
{
|
||||
"code": "8906010501952",
|
||||
"brands": [
|
||||
"Balaji",
|
||||
"Balaji Wafers",
|
||||
"Rumbles"
|
||||
],
|
||||
"quantity": "40 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers Rumbles Potato Chips Pudina Twist flavour",
|
||||
"product_name_en": "Balaji Wafers Rumbles Potato Chips Pudina Twist flavour"
|
||||
},
|
||||
{
|
||||
"code": "8906010500900",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "22 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "wheels",
|
||||
"product_name_en": "wheels"
|
||||
},
|
||||
{
|
||||
"code": "8906010502591",
|
||||
"brands": [
|
||||
"Balaji Wafers"
|
||||
],
|
||||
"quantity": "72g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Crunchem",
|
||||
"product_name_en": "Crunchem"
|
||||
},
|
||||
{
|
||||
"code": "8906010501822",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "gippi",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "GIPPI",
|
||||
"product_name_en": "GIPPI"
|
||||
},
|
||||
{
|
||||
"code": "8906010503581",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "420g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "GIPPI masala noodles",
|
||||
"product_name_en": "GIPPI masala noodles"
|
||||
},
|
||||
{
|
||||
"code": "8906010500047",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "15g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "masala masti",
|
||||
"product_name_en": "masala masti"
|
||||
},
|
||||
{
|
||||
"code": "8906010500535",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "35 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers Chaat Chaska",
|
||||
"product_name_en": "Balaji Wafers Chaat Chaska"
|
||||
},
|
||||
{
|
||||
"code": "8906010500023",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "POTATO WAFERS",
|
||||
"product_name_en": "POTATO WAFERS"
|
||||
},
|
||||
{
|
||||
"code": "8906010500221",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers Mung Dal",
|
||||
"product_name_en": "Balaji Wafers Mung Dal"
|
||||
},
|
||||
{
|
||||
"code": "0710859306108",
|
||||
"brands": [
|
||||
"Vittal Balaji"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CASHEWS",
|
||||
"product_name_en": "CASHEWS"
|
||||
},
|
||||
{
|
||||
"code": "8906010500368",
|
||||
"brands": [
|
||||
"Balaji Namkeen"
|
||||
],
|
||||
"quantity": "30 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Tikha Mitha Mix",
|
||||
"product_name_en": "Tikha Mitha Mix"
|
||||
},
|
||||
{
|
||||
"code": "8906010500276",
|
||||
"brands": [
|
||||
"Balaji Wafers"
|
||||
],
|
||||
"quantity": "25g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers Shing Bhujia",
|
||||
"product_name_en": "Balaji Wafers Shing Bhujia"
|
||||
},
|
||||
{
|
||||
"code": "8906010500764",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "25g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "balaji wafers chataka pataka",
|
||||
"product_name_en": "balaji wafers chataka pataka"
|
||||
},
|
||||
{
|
||||
"code": "8906010500290",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "210g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Shing Bhujia",
|
||||
"product_name_en": "Shing Bhujia"
|
||||
},
|
||||
{
|
||||
"code": "8906010502508",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "35g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Crunchem Peri Peri Flavour",
|
||||
"product_name_en": "Crunchem Peri Peri Flavour"
|
||||
},
|
||||
{
|
||||
"code": "8906010503529",
|
||||
"brands": [
|
||||
"Balaji"
|
||||
],
|
||||
"quantity": "150",
|
||||
"countries_tags": [
|
||||
"en:ghana",
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHRUNCHEM Masala Masti",
|
||||
"product_name_en": "CHRUNCHEM Masala Masti"
|
||||
},
|
||||
{
|
||||
"code": "26246246",
|
||||
"brands": [
|
||||
"balaji"
|
||||
],
|
||||
"quantity": "25g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "aloo sev",
|
||||
"product_name_en": "aloo sev"
|
||||
},
|
||||
{
|
||||
"code": "8906010500092",
|
||||
"brands": [
|
||||
"Balaji Wafers"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Balaji Wafers Crunchex",
|
||||
"product_name_en": "Balaji Wafers Crunchex"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/brooke_bond.json
vendored
Normal file
7
data/cache/off_brand_corpus/brooke_bond.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "brooke bond",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851994.807076,
|
||||
"fetched_at_human": "2026-09-08 12:49:54",
|
||||
"hits": []
|
||||
}
|
||||
343
data/cache/off_brand_corpus/catch.json
vendored
Normal file
343
data/cache/off_brand_corpus/catch.json
vendored
Normal file
@@ -0,0 +1,343 @@
|
||||
{
|
||||
"brand": "catch",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852025.2826416,
|
||||
"fetched_at_human": "2026-09-08 12:50:25",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901192204011",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHATPATA CHAT MASALA catch",
|
||||
"product_name_en": "CHATPATA CHAT MASALA catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192217110",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "JALJEERA",
|
||||
"product_name_en": "JALJEERA"
|
||||
},
|
||||
{
|
||||
"code": "8901192101013",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "BLACK PEPPER POWDER Catch",
|
||||
"product_name_en": "BLACK PEPPER POWDER Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192208019",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "KITCHEN KING catch",
|
||||
"product_name_en": "KITCHEN KING catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192214010",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50 gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHICKEN MASALA Catch",
|
||||
"product_name_en": "CHICKEN MASALA Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192000101",
|
||||
"brands": [
|
||||
"catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "catch sprinklers Table Salt",
|
||||
"product_name_en": "catch sprinklers Table Salt"
|
||||
},
|
||||
{
|
||||
"code": "8901192205100",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "catch chat masala",
|
||||
"product_name_en": "catch chat masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192226150",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Hing Asafoetida",
|
||||
"product_name_en": "Hing Asafoetida"
|
||||
},
|
||||
{
|
||||
"code": "8901192227003",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Dahi Masala",
|
||||
"product_name_en": "Dahi Masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192002105",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100 Grams",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Catch Pink Rock Salt(Sendha Namak)",
|
||||
"product_name_en": "Catch Pink Rock Salt(Sendha Namak)"
|
||||
},
|
||||
{
|
||||
"code": "8901192204110",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Catch Chatpata Chat Masala",
|
||||
"product_name_en": "Catch Chatpata Chat Masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192205261",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "200 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Ginger Garlic - Culinary Paste",
|
||||
"product_name_en": "Ginger Garlic - Culinary Paste"
|
||||
},
|
||||
{
|
||||
"code": "8901192215017",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sabzi masala",
|
||||
"product_name_en": "Sabzi masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192216014",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50 gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Garam masala",
|
||||
"product_name_en": "Garam masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192226051",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Hing",
|
||||
"product_name_en": "Hing"
|
||||
},
|
||||
{
|
||||
"code": "8901192103116",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kashmiri mirch power",
|
||||
"product_name_en": "Kashmiri mirch power"
|
||||
},
|
||||
{
|
||||
"code": "8901192105110",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Amchur power",
|
||||
"product_name_en": "Amchur power"
|
||||
},
|
||||
{
|
||||
"code": "8901192223111",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Rajma masala",
|
||||
"product_name_en": "Rajma masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192213112",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "MEAT MASALA Catch",
|
||||
"product_name_en": "MEAT MASALA Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192219220",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Catch Super Garam Masala",
|
||||
"product_name_en": "Catch Super Garam Masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192107626",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "500gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Haldi powder",
|
||||
"product_name_en": "Haldi powder"
|
||||
},
|
||||
{
|
||||
"code": "8901192105011",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "AMCHUR POWDER Catch",
|
||||
"product_name_en": "AMCHUR POWDER Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192221018",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHHOLE MASALA Catch",
|
||||
"product_name_en": "CHHOLE MASALA Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192214119",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "CHICKEN MASALA Catch",
|
||||
"product_name_en": "CHICKEN MASALA Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192202116",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "SAMBHAR MASALA Catch",
|
||||
"product_name_en": "SAMBHAR MASALA Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192208118",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "KITCHEN KING Catch",
|
||||
"product_name_en": "KITCHEN KING Catch"
|
||||
},
|
||||
{
|
||||
"code": "8901192216113",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "100gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Garam masala",
|
||||
"product_name_en": "Garam masala"
|
||||
},
|
||||
{
|
||||
"code": "8901192114006",
|
||||
"brands": [
|
||||
"Catch"
|
||||
],
|
||||
"quantity": "50 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sprinkler Black Pepper",
|
||||
"product_name_en": "Sprinkler Black Pepper"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/clinic_all_clear.json
vendored
Normal file
7
data/cache/off_brand_corpus/clinic_all_clear.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "clinic all clear",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852017.1407056,
|
||||
"fetched_at_human": "2026-09-08 12:50:17",
|
||||
"hits": []
|
||||
}
|
||||
7
data/cache/off_brand_corpus/colin.json
vendored
Normal file
7
data/cache/off_brand_corpus/colin.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "colin",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852016.0197957,
|
||||
"fetched_at_human": "2026-09-08 12:50:16",
|
||||
"hits": []
|
||||
}
|
||||
56
data/cache/off_brand_corpus/complan.json
vendored
Normal file
56
data/cache/off_brand_corpus/complan.json
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"brand": "complan",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852010.1317456,
|
||||
"fetched_at_human": "2026-09-08 12:50:10",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901542002298",
|
||||
"brands": [
|
||||
"Complan"
|
||||
],
|
||||
"quantity": "18g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Complan",
|
||||
"product_name_en": "Complan"
|
||||
},
|
||||
{
|
||||
"code": "8901542001796",
|
||||
"brands": [
|
||||
"Complan"
|
||||
],
|
||||
"quantity": "500 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Explore Knowledge world",
|
||||
"product_name_en": "Explore Knowledge world"
|
||||
},
|
||||
{
|
||||
"code": "8901542003158",
|
||||
"brands": [
|
||||
"Complan"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Complan Rich Chocolate flavor",
|
||||
"product_name_en": "Complan Rich Chocolate flavor"
|
||||
},
|
||||
{
|
||||
"code": "8901542000959",
|
||||
"brands": [
|
||||
"Complan"
|
||||
],
|
||||
"quantity": "500",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Complan royale chocolate flavour",
|
||||
"product_name_en": "Complan royale chocolate flavour"
|
||||
}
|
||||
]
|
||||
}
|
||||
102
data/cache/off_brand_corpus/daawat.json
vendored
Normal file
102
data/cache/off_brand_corpus/daawat.json
vendored
Normal file
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"brand": "daawat",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852002.8657465,
|
||||
"fetched_at_human": "2026-09-08 12:50:02",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901537079595",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "White basmati rice",
|
||||
"product_name_en": "White basmati rice"
|
||||
},
|
||||
{
|
||||
"code": "8901537077485",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Daawat Cuppa Rice Rajma Chawal",
|
||||
"product_name_en": "Daawat Cuppa Rice Rajma Chawal"
|
||||
},
|
||||
{
|
||||
"code": "8901537074415",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "5 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sona Masoori Rice",
|
||||
"product_name_en": "Sona Masoori Rice"
|
||||
},
|
||||
{
|
||||
"code": "2000000150588",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "5 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sona Masoori Rice",
|
||||
"product_name_en": "Sona Masoori Rice"
|
||||
},
|
||||
{
|
||||
"code": "4901135025127",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Dawat super basmati rice 1 kg",
|
||||
"product_name_en": "Dawat super basmati rice 1 kg"
|
||||
},
|
||||
{
|
||||
"code": "8901537074231",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Daawat pulav basmati rice 1kg",
|
||||
"product_name_en": "Daawat pulav basmati rice 1kg"
|
||||
},
|
||||
{
|
||||
"code": "8901537007116",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "5 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Daawat Rozana Basmati Rice",
|
||||
"product_name_en": "Daawat Rozana Basmati Rice"
|
||||
},
|
||||
{
|
||||
"code": "8901537078314",
|
||||
"brands": [
|
||||
"Daawat"
|
||||
],
|
||||
"quantity": "334 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Biryani Kit Lucknowi",
|
||||
"product_name_en": "Biryani Kit Lucknowi"
|
||||
}
|
||||
]
|
||||
}
|
||||
43
data/cache/off_brand_corpus/dhara.json
vendored
Normal file
43
data/cache/off_brand_corpus/dhara.json
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"brand": "dhara",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852026.4211528,
|
||||
"fetched_at_human": "2026-09-08 12:50:26",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906004620751",
|
||||
"brands": [
|
||||
"Dhara Life"
|
||||
],
|
||||
"quantity": "1l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Refined Ricebran Oil",
|
||||
"product_name_en": "Refined Ricebran Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906004620256",
|
||||
"brands": [
|
||||
"Dhara"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kachi Ghani Mustard Oil",
|
||||
"product_name_en": "Kachi Ghani Mustard Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906004620485",
|
||||
"brands": [
|
||||
"Dhara"
|
||||
],
|
||||
"quantity": "5l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Dhara Refined Sunflower oil 5l",
|
||||
"product_name_en": "Dhara Refined Sunflower oil 5l"
|
||||
}
|
||||
]
|
||||
}
|
||||
296
data/cache/off_brand_corpus/fortune.json
vendored
Normal file
296
data/cache/off_brand_corpus/fortune.json
vendored
Normal file
@@ -0,0 +1,296 @@
|
||||
{
|
||||
"brand": "fortune",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851992.080097,
|
||||
"fetched_at_human": "2026-09-08 12:49:52",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906007280969",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1 L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Premium kachi ghani pure mustard oil",
|
||||
"product_name_en": "Premium kachi ghani pure mustard oil"
|
||||
},
|
||||
{
|
||||
"code": "8906007285018",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "500 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Chana besan",
|
||||
"product_name_en": "Chana besan"
|
||||
},
|
||||
{
|
||||
"code": "8906008811766",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "60g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Punjabi Khichdi",
|
||||
"product_name_en": "Punjabi Khichdi"
|
||||
},
|
||||
{
|
||||
"code": "8906007288408",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune dubar basmati rice",
|
||||
"product_name_en": "Fortune dubar basmati rice"
|
||||
},
|
||||
{
|
||||
"code": "8906008811902",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pro Immunity Multi-source Edible Oil",
|
||||
"product_name_en": "Pro Immunity Multi-source Edible Oil"
|
||||
},
|
||||
{
|
||||
"code": "8146007283090",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "200",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune mini soya chunks",
|
||||
"product_name_en": "Fortune mini soya chunks"
|
||||
},
|
||||
{
|
||||
"code": "8906007280693",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Rice Bran Oil",
|
||||
"product_name_en": "Rice Bran Oil"
|
||||
},
|
||||
{
|
||||
"code": "02867633",
|
||||
"brands": [
|
||||
"fortune"
|
||||
],
|
||||
"quantity": "1 l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "refined soya bean oil",
|
||||
"product_name_en": "refined soya bean oil"
|
||||
},
|
||||
{
|
||||
"code": "8906007280259",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Physically Refined Rice Bran Oil",
|
||||
"product_name_en": "Fortune Physically Refined Rice Bran Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906007286039",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Chana Dal",
|
||||
"product_name_en": "Fortune Chana Dal"
|
||||
},
|
||||
{
|
||||
"code": "8906008811131",
|
||||
"brands": [
|
||||
"fortune"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "suji semolina",
|
||||
"product_name_en": "suji semolina"
|
||||
},
|
||||
{
|
||||
"code": "8906008810844",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Gujarati Khichdi",
|
||||
"product_name_en": "Gujarati Khichdi"
|
||||
},
|
||||
{
|
||||
"code": "8906007280280",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "5L (4.550 Kg)",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Refined Sunflower Oil",
|
||||
"product_name_en": "Fortune Refined Sunflower Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906007280105",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "15kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Refined Soyabean Oil",
|
||||
"product_name_en": "Fortune Refined Soyabean Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906007280655",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "5l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune cotton seed oil",
|
||||
"product_name_en": "Fortune cotton seed oil"
|
||||
},
|
||||
{
|
||||
"code": "8906008812817",
|
||||
"brands": [
|
||||
"fortune"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "poha (thick)",
|
||||
"product_name_en": "poha (thick)"
|
||||
},
|
||||
{
|
||||
"code": "8906007280945",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "500ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Premium kachi gani pre mustard oil",
|
||||
"product_name_en": "Premium kachi gani pre mustard oil"
|
||||
},
|
||||
{
|
||||
"code": "8906008812435",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Achaari Khichdi",
|
||||
"product_name_en": "Achaari Khichdi"
|
||||
},
|
||||
{
|
||||
"code": "4916534287045",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Besan",
|
||||
"product_name_en": "Besan"
|
||||
},
|
||||
{
|
||||
"code": "8906007289139",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Atta",
|
||||
"product_name_en": "Fortune Atta"
|
||||
},
|
||||
{
|
||||
"code": "8906007280037",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1 l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Soya Health Refined Oil",
|
||||
"product_name_en": "Fortune Soya Health Refined Oil"
|
||||
},
|
||||
{
|
||||
"code": "8906008811933",
|
||||
"brands": [
|
||||
"Fortune"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fortune Xpert",
|
||||
"product_name_en": "Fortune Xpert"
|
||||
},
|
||||
{
|
||||
"code": "8906008811117",
|
||||
"brands": [
|
||||
"fortune"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "maida refined wheat flour",
|
||||
"product_name_en": "maida refined wheat flour"
|
||||
},
|
||||
{
|
||||
"code": "8906007280242",
|
||||
"brands": [
|
||||
"fortune"
|
||||
],
|
||||
"quantity": "1 l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "refined sunflower oil",
|
||||
"product_name_en": "refined sunflower oil"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/india_gate.json
vendored
Normal file
7
data/cache/off_brand_corpus/india_gate.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "india gate",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851993.6795115,
|
||||
"fetched_at_human": "2026-09-08 12:49:53",
|
||||
"hits": []
|
||||
}
|
||||
466
data/cache/off_brand_corpus/itc.json
vendored
Normal file
466
data/cache/off_brand_corpus/itc.json
vendored
Normal file
@@ -0,0 +1,466 @@
|
||||
{
|
||||
"brand": "itc",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851979.9834924,
|
||||
"fetched_at_human": "2026-09-08 12:49:39",
|
||||
"hits": [
|
||||
{
|
||||
"code": "7524341327808",
|
||||
"brands": [
|
||||
"itc"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "veda digestive",
|
||||
"product_name_en": "veda digestive"
|
||||
},
|
||||
{
|
||||
"code": "8901725181420",
|
||||
"brands": [
|
||||
"ITC Sunfeast"
|
||||
],
|
||||
"quantity": "70 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast Yippee Noodles",
|
||||
"product_name_en": "Sunfeast Yippee Noodles"
|
||||
},
|
||||
{
|
||||
"code": "8901725006358",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "11",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast Smoothies Strawberry with Chia Seeds 300ml",
|
||||
"product_name_en": "Sunfeast Smoothies Strawberry with Chia Seeds 300ml"
|
||||
},
|
||||
{
|
||||
"code": "8901725007102",
|
||||
"brands": [
|
||||
"Itc"
|
||||
],
|
||||
"quantity": "21g g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Bingo",
|
||||
"product_name_en": "Bingo"
|
||||
},
|
||||
{
|
||||
"code": "8901725100025",
|
||||
"brands": [
|
||||
"ITC Limied"
|
||||
],
|
||||
"quantity": "12",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "B Natural Mixed fruit 1ltr tpk",
|
||||
"product_name_en": "B Natural Mixed fruit 1ltr tpk"
|
||||
},
|
||||
{
|
||||
"code": "8901725004682",
|
||||
"brands": [
|
||||
"Sunfeast",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "150 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "farmlite oats & chocolate Cookies",
|
||||
"product_name_en": "farmlite oats & chocolate Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8901725121754",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aashirvaad",
|
||||
"product_name_en": "Aashirvaad"
|
||||
},
|
||||
{
|
||||
"code": "8906016579993",
|
||||
"brands": [
|
||||
"WIMCO",
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Homelites",
|
||||
"product_name_en": "Homelites"
|
||||
},
|
||||
{
|
||||
"code": "8901725181529",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "74.5 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Yippe noodles"
|
||||
},
|
||||
{
|
||||
"code": "8901725109868",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "36.5g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Bingo Mad Angles PIZZA..AAAAH",
|
||||
"product_name_en": "Bingo Mad Angles PIZZA..AAAAH"
|
||||
},
|
||||
{
|
||||
"code": "8901725119195",
|
||||
"brands": [
|
||||
"Sunfeast",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "70 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Yippee! noodles mood masala",
|
||||
"product_name_en": "Yippee! noodles mood masala"
|
||||
},
|
||||
{
|
||||
"code": "8901725015275",
|
||||
"brands": [
|
||||
"ITC sanfeast"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Dark Fantasy",
|
||||
"product_name_en": "Dark Fantasy"
|
||||
},
|
||||
{
|
||||
"code": "8901725116729",
|
||||
"brands": [
|
||||
"ITC",
|
||||
"Sunfeast"
|
||||
],
|
||||
"quantity": "405 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast Yippee! noodles Magic Masala",
|
||||
"product_name_en": "Sunfeast Yippee! noodles Magic Masala"
|
||||
},
|
||||
{
|
||||
"code": "8901725006341",
|
||||
"brands": [
|
||||
"Sunfeast Smoothies",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "11",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast smoothies Litchi with fruit chunks 300ml",
|
||||
"product_name_en": "Sunfeast smoothies Litchi with fruit chunks 300ml"
|
||||
},
|
||||
{
|
||||
"code": "8901725121723",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "5kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aashirvaad Superior MP Atta",
|
||||
"product_name_en": "Aashirvaad Superior MP Atta"
|
||||
},
|
||||
{
|
||||
"code": "8901725001308",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "52.0 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Bingo Potato Chips Cream & Onion",
|
||||
"product_name_en": "Bingo Potato Chips Cream & Onion"
|
||||
},
|
||||
{
|
||||
"code": "8906065168599",
|
||||
"brands": [
|
||||
"Itc"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aloo Tikki",
|
||||
"product_name_en": "Aloo Tikki"
|
||||
},
|
||||
{
|
||||
"code": "8901725000851",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast all rounder chatpata",
|
||||
"product_name_en": "Sunfeast all rounder chatpata"
|
||||
},
|
||||
{
|
||||
"code": "8901725113186",
|
||||
"brands": [
|
||||
"ITC Master Chef"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Paneer Makhni",
|
||||
"product_name_en": "Paneer Makhni"
|
||||
},
|
||||
{
|
||||
"code": "8906065166045",
|
||||
"brands": [
|
||||
"Farmland ITC"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Frozen Green Peas",
|
||||
"product_name_en": "Frozen Green Peas"
|
||||
},
|
||||
{
|
||||
"code": "8901725001070",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aashirvaad Double Roasted Suji Rava",
|
||||
"product_name_en": "Aashirvaad Double Roasted Suji Rava"
|
||||
},
|
||||
{
|
||||
"code": "8901725135218",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "65g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast Yippee! Pasta Treat",
|
||||
"product_name_en": "Sunfeast Yippee! Pasta Treat"
|
||||
},
|
||||
{
|
||||
"code": "8901781000536",
|
||||
"brands": [
|
||||
"Sunrise",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "50g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aloo dum",
|
||||
"product_name_en": "Aloo dum"
|
||||
},
|
||||
{
|
||||
"code": "8906065169374",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "ITC Master Chef Classic Aloo Tikki",
|
||||
"product_name_en": "ITC Master Chef Classic Aloo Tikki"
|
||||
},
|
||||
{
|
||||
"code": "8909081001536",
|
||||
"brands": [
|
||||
"Sunfeast",
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast SUPER EGG & MILK",
|
||||
"product_name_en": "Sunfeast SUPER EGG & MILK"
|
||||
},
|
||||
{
|
||||
"code": "8901725004996",
|
||||
"brands": [
|
||||
"Sunfeast",
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sunfeast Thin Arrowroot",
|
||||
"product_name_en": "Sunfeast Thin Arrowroot"
|
||||
},
|
||||
{
|
||||
"code": "10012031000312",
|
||||
"brands": [
|
||||
"Candyman",
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "FruiteeFun 3 in 1",
|
||||
"product_name_en": "FruiteeFun 3 in 1"
|
||||
},
|
||||
{
|
||||
"code": "8901725007508",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "90g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Bingo! Potato Chips Chilli Sprinkled",
|
||||
"product_name_en": "Bingo! Potato Chips Chilli Sprinkled"
|
||||
},
|
||||
{
|
||||
"code": "8901725006969",
|
||||
"brands": [
|
||||
"ITC",
|
||||
"Sunfeast"
|
||||
],
|
||||
"quantity": "282",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "sunfeast all rounder chatpata",
|
||||
"product_name_en": "sunfeast all rounder chatpata"
|
||||
},
|
||||
{
|
||||
"code": "8901725013660",
|
||||
"brands": [
|
||||
"ITC Limited"
|
||||
],
|
||||
"quantity": "15g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "bingo mad angles",
|
||||
"product_name_en": "bingo mad angles"
|
||||
},
|
||||
{
|
||||
"code": "8901725121747",
|
||||
"brands": [
|
||||
"Aashirvaad",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Superior MP Atta",
|
||||
"product_name_en": "Superior MP Atta"
|
||||
},
|
||||
{
|
||||
"code": "8901725121129",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "5 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Aashirvaad Shudh Chakki Atta",
|
||||
"product_name_en": "Aashirvaad Shudh Chakki Atta"
|
||||
},
|
||||
{
|
||||
"code": "8901725105181",
|
||||
"brands": [
|
||||
"B Natural",
|
||||
"ITC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "B Natural Litchi Juice",
|
||||
"product_name_en": "B Natural Litchi Juice"
|
||||
},
|
||||
{
|
||||
"code": "8901725007492",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "90g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Bingo",
|
||||
"product_name_en": "Bingo"
|
||||
},
|
||||
{
|
||||
"code": "8906065169749",
|
||||
"brands": [
|
||||
"Itc"
|
||||
],
|
||||
"quantity": "420g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Itc master chef",
|
||||
"product_name_en": "Itc master chef"
|
||||
},
|
||||
{
|
||||
"code": "8901725003319",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "60G",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Yippe noodles",
|
||||
"product_name_en": "Yippe noodles"
|
||||
},
|
||||
{
|
||||
"code": "8901725100131",
|
||||
"brands": [
|
||||
"B Natural",
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "12",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "B Natural Orange Juice 1ltr tpk",
|
||||
"product_name_en": "B Natural Orange Juice 1ltr tpk"
|
||||
},
|
||||
{
|
||||
"code": "8901725198336",
|
||||
"brands": [
|
||||
"ITC"
|
||||
],
|
||||
"quantity": "80g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "MAD ANGLES",
|
||||
"product_name_en": "MAD ANGLES"
|
||||
}
|
||||
]
|
||||
}
|
||||
241
data/cache/off_brand_corpus/kissan.json
vendored
Normal file
241
data/cache/off_brand_corpus/kissan.json
vendored
Normal file
@@ -0,0 +1,241 @@
|
||||
{
|
||||
"brand": "kissan",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851999.3589082,
|
||||
"fetched_at_human": "2026-09-08 12:49:59",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901030532719",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "500 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fresh 🍅 500g",
|
||||
"product_name_en": "Fresh 🍅 500g"
|
||||
},
|
||||
{
|
||||
"code": "8901030922787",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "90g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Mixed Fruit Jam",
|
||||
"product_name_en": "Mixed Fruit Jam"
|
||||
},
|
||||
{
|
||||
"code": "8901030926457",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Orange marmalade",
|
||||
"product_name_en": "Orange marmalade"
|
||||
},
|
||||
{
|
||||
"code": "4901034831706",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Mixed Fruit Jam",
|
||||
"product_name_en": "Mixed Fruit Jam"
|
||||
},
|
||||
{
|
||||
"code": "8901030667756",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "950 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fresh Tomato Ketchup",
|
||||
"product_name_en": "Fresh Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030959776",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "350g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan Peanut Butter Crunchy",
|
||||
"product_name_en": "Kissan Peanut Butter Crunchy"
|
||||
},
|
||||
{
|
||||
"code": "8901030921797",
|
||||
"brands": [
|
||||
"Kissan",
|
||||
"Unilever"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "kissan fresh tomato",
|
||||
"product_name_en": "kissan fresh tomato"
|
||||
},
|
||||
{
|
||||
"code": "8901030699085",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fresh tomato ketchup",
|
||||
"product_name_en": "Fresh tomato ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030926549",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "350g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Hazelnut Choco Peanut Spread",
|
||||
"product_name_en": "Hazelnut Choco Peanut Spread"
|
||||
},
|
||||
{
|
||||
"code": "8901030634475",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan Orange Squash",
|
||||
"product_name_en": "Kissan Orange Squash"
|
||||
},
|
||||
{
|
||||
"code": "8901030743047",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "12",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fresh Tomato Ketchup",
|
||||
"product_name_en": "Fresh Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030922565",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "425 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "No Onion No Garlic Tomato Sauce",
|
||||
"product_name_en": "No Onion No Garlic Tomato Sauce"
|
||||
},
|
||||
{
|
||||
"code": "8901030857973",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "350g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan peanut butter",
|
||||
"product_name_en": "Kissan peanut butter"
|
||||
},
|
||||
{
|
||||
"code": "8901030653407",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "950gm",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan Fresh Tomato Ketchup",
|
||||
"product_name_en": "Kissan Fresh Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030897542",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"quantity": "90g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan Fresh Tomato Ketchup",
|
||||
"product_name_en": "Kissan Fresh Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030534898",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kissan Fresh Tomato Ketchup",
|
||||
"product_name_en": "Kissan Fresh Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030902932",
|
||||
"brands": [
|
||||
"kissan"
|
||||
],
|
||||
"quantity": "850 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "tomato ketchup",
|
||||
"product_name_en": "tomato ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030865237",
|
||||
"brands": [
|
||||
"kissan"
|
||||
],
|
||||
"quantity": "weight: 1.2 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Tomato Ketchup"
|
||||
},
|
||||
{
|
||||
"code": "8901030831713",
|
||||
"brands": [
|
||||
"kissan"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "fruit jam",
|
||||
"product_name_en": "fruit jam"
|
||||
},
|
||||
{
|
||||
"code": "8901030532832",
|
||||
"brands": [
|
||||
"Kissan"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fresh Tomato Ketchup"
|
||||
}
|
||||
]
|
||||
}
|
||||
45
data/cache/off_brand_corpus/kohinoor.json
vendored
Normal file
45
data/cache/off_brand_corpus/kohinoor.json
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"brand": "kohinoor",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852007.7260816,
|
||||
"fetched_at_human": "2026-09-08 12:50:07",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906009320281",
|
||||
"brands": [
|
||||
"Kohinoor"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kohinoor Packaged Drinking Water",
|
||||
"product_name_en": "Kohinoor Packaged Drinking Water"
|
||||
},
|
||||
{
|
||||
"code": "8904250701521",
|
||||
"brands": [
|
||||
"Kohinoor"
|
||||
],
|
||||
"quantity": "300 g",
|
||||
"countries_tags": [
|
||||
"en:brazil",
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Palak Paneer"
|
||||
},
|
||||
{
|
||||
"code": "8901047816208",
|
||||
"brands": [
|
||||
"Kohinoor"
|
||||
],
|
||||
"quantity": "300g",
|
||||
"countries_tags": [
|
||||
"en:india",
|
||||
"en:united-kingdom"
|
||||
],
|
||||
"product_name": "Dal makhani",
|
||||
"product_name_en": "Dal makhani"
|
||||
}
|
||||
]
|
||||
}
|
||||
46
data/cache/off_brand_corpus/marico.json
vendored
Normal file
46
data/cache/off_brand_corpus/marico.json
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"brand": "marico",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851989.84839,
|
||||
"fetched_at_human": "2026-09-08 12:49:49",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901088206280",
|
||||
"brands": [
|
||||
"Marico"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Marico Revive",
|
||||
"product_name_en": "Marico Revive"
|
||||
},
|
||||
{
|
||||
"code": "8901088213608",
|
||||
"brands": [
|
||||
"Saffola",
|
||||
"Marico"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Saffola Oats",
|
||||
"product_name_en": "Saffola Oats"
|
||||
},
|
||||
{
|
||||
"code": "8901088068772",
|
||||
"brands": [
|
||||
"Saffola",
|
||||
"Marico"
|
||||
],
|
||||
"quantity": "38g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Masala oats",
|
||||
"product_name_en": "Masala oats"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/mother_dairy.json
vendored
Normal file
7
data/cache/off_brand_corpus/mother_dairy.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "mother dairy",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852018.2923563,
|
||||
"fetched_at_human": "2026-09-08 12:50:18",
|
||||
"hits": []
|
||||
}
|
||||
289
data/cache/off_brand_corpus/nandini.json
vendored
Normal file
289
data/cache/off_brand_corpus/nandini.json
vendored
Normal file
@@ -0,0 +1,289 @@
|
||||
{
|
||||
"brand": "nandini",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851996.9550085,
|
||||
"fetched_at_human": "2026-09-08 12:49:56",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906036673596",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "500ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini GoodLife Toned Milk",
|
||||
"product_name_en": "Nandini GoodLife Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036675248",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Double Toned Milk",
|
||||
"product_name_en": "Double Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036674265",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "500ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Desi Cow Milk",
|
||||
"product_name_en": "Desi Cow Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036673947",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "180ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini Good Life",
|
||||
"product_name_en": "Nandini Good Life"
|
||||
},
|
||||
{
|
||||
"code": "8906036670977",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "200",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini Paneer",
|
||||
"product_name_en": "Nandini Paneer"
|
||||
},
|
||||
{
|
||||
"code": "8906036676290",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Santhrupthi Milk",
|
||||
"product_name_en": "Santhrupthi Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036670205",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini Curd",
|
||||
"product_name_en": "Nandini Curd"
|
||||
},
|
||||
{
|
||||
"code": "8906036677907",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Butterscotch ice cream",
|
||||
"product_name_en": "Butterscotch ice cream"
|
||||
},
|
||||
{
|
||||
"code": "8906036670885",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "1 l (905 g)",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pure Ghee",
|
||||
"product_name_en": "Pure Ghee"
|
||||
},
|
||||
{
|
||||
"code": "8906036673602",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "1litre",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini GoodLife Toned Milk",
|
||||
"product_name_en": "Nandini GoodLife Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036672995",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "500 ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Slim UHT Sterilised Skimmed Milk",
|
||||
"product_name_en": "Slim UHT Sterilised Skimmed Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036676665",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini goodlife pista flavouring milk",
|
||||
"product_name_en": "GoodLife Pista Flavoured Milk"
|
||||
},
|
||||
{
|
||||
"code": "15486196",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "500ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Toned Milk",
|
||||
"product_name_en": "Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036670830",
|
||||
"brands": [
|
||||
"nandini"
|
||||
],
|
||||
"quantity": "500",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "butter",
|
||||
"product_name_en": "butter"
|
||||
},
|
||||
{
|
||||
"code": "8906036670014",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "500 ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pasteurised Toned Milk",
|
||||
"product_name_en": "Pasteurised Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036670090",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Shubham",
|
||||
"product_name_en": "Shubham"
|
||||
},
|
||||
{
|
||||
"code": "8906036670441",
|
||||
"brands": [
|
||||
"nandini"
|
||||
],
|
||||
"quantity": "1l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "goodlife slim",
|
||||
"product_name_en": "goodlife slim"
|
||||
},
|
||||
{
|
||||
"code": "8906036672292",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "200 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pasteurised Table Butter Unsalted",
|
||||
"product_name_en": "Pasteurised Table Butter Unsalted"
|
||||
},
|
||||
{
|
||||
"code": "8906001025559",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "750ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Govardhan ghee",
|
||||
"product_name_en": "Govardhan ghee"
|
||||
},
|
||||
{
|
||||
"code": "8906036674760",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini Godhi Ladoo",
|
||||
"product_name_en": "Nandini Godhi Ladoo"
|
||||
},
|
||||
{
|
||||
"code": "8906036675163",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "250g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Milk Peda",
|
||||
"product_name_en": "Milk Peda"
|
||||
},
|
||||
{
|
||||
"code": "8906036674739",
|
||||
"brands": [
|
||||
"nandini"
|
||||
],
|
||||
"quantity": "200ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Special Toned Milk",
|
||||
"product_name_en": "Special Toned Milk"
|
||||
},
|
||||
{
|
||||
"code": "8906036674173",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "200ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini Creamy delight",
|
||||
"product_name_en": "Nandini Creamy delight"
|
||||
},
|
||||
{
|
||||
"code": "8906036675750",
|
||||
"brands": [
|
||||
"Nandini"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nandini GoodLife Plus Toned Milk",
|
||||
"product_name_en": "Nandini GoodLife Plus Toned Milk"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/nature_fresh.json
vendored
Normal file
7
data/cache/off_brand_corpus/nature_fresh.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "nature fresh",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852003.9524193,
|
||||
"fetched_at_human": "2026-09-08 12:50:03",
|
||||
"hits": []
|
||||
}
|
||||
152
data/cache/off_brand_corpus/nivea.json
vendored
Normal file
152
data/cache/off_brand_corpus/nivea.json
vendored
Normal file
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"brand": "nivea",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852006.482772,
|
||||
"fetched_at_human": "2026-09-08 12:50:06",
|
||||
"hits": [
|
||||
{
|
||||
"code": "42204169",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "50ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Soft Light Moisturising Cream",
|
||||
"product_name_en": "Nivea Soft Light Moisturising Cream"
|
||||
},
|
||||
{
|
||||
"code": "42419976",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "50ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Soft Cream",
|
||||
"product_name_en": "Nivea Soft Cream"
|
||||
},
|
||||
{
|
||||
"code": "8904256005166",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "100ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea",
|
||||
"product_name_en": "Nivea"
|
||||
},
|
||||
{
|
||||
"code": "8904256000956",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "250ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea shower gel",
|
||||
"product_name_en": "Nivea shower gel"
|
||||
},
|
||||
{
|
||||
"code": "8904256009812",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "600ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Body lotion Aloe Hydration",
|
||||
"product_name_en": "Nivea Body lotion Aloe Hydration"
|
||||
},
|
||||
{
|
||||
"code": "42204268",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea 200 ML",
|
||||
"product_name_en": "Nivea 200 ML"
|
||||
},
|
||||
{
|
||||
"code": "42204091",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "100ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Soft Light Moisturising Cream",
|
||||
"product_name_en": "Nivea Soft Light Moisturising Cream"
|
||||
},
|
||||
{
|
||||
"code": "8904256010139",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "120ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Body Lotion Aloe Hydration",
|
||||
"product_name_en": "Nivea Body Lotion Aloe Hydration"
|
||||
},
|
||||
{
|
||||
"code": "42420002",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "100ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Soft Cream",
|
||||
"product_name_en": "Nivea Soft Cream"
|
||||
},
|
||||
{
|
||||
"code": "8904256020473",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "4.8g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Ruby Red tinted Cherry Shine",
|
||||
"product_name_en": "Nivea Ruby Red tinted Cherry Shine"
|
||||
},
|
||||
{
|
||||
"code": "4005808516261",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "75ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Sun Protect & Moisture",
|
||||
"product_name_en": "Nivea Sun Protect & Moisture"
|
||||
},
|
||||
{
|
||||
"code": "42316718",
|
||||
"brands": [
|
||||
"Nivea"
|
||||
],
|
||||
"quantity": "120ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Nivea Nourishing Lotion Body Milk",
|
||||
"product_name_en": "Nivea Nourishing Lotion Body Milk"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/paper_boat.json
vendored
Normal file
7
data/cache/off_brand_corpus/paper_boat.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "paper boat",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852011.324778,
|
||||
"fetched_at_human": "2026-09-08 12:50:11",
|
||||
"hits": []
|
||||
}
|
||||
1724
data/cache/off_brand_corpus/parle.json
vendored
Normal file
1724
data/cache/off_brand_corpus/parle.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
482
data/cache/off_brand_corpus/patanjali.json
vendored
Normal file
482
data/cache/off_brand_corpus/patanjali.json
vendored
Normal file
@@ -0,0 +1,482 @@
|
||||
{
|
||||
"brand": "patanjali",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851998.149914,
|
||||
"fetched_at_human": "2026-09-08 12:49:58",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906032019282",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Rolled Oats",
|
||||
"product_name_en": "Rolled Oats"
|
||||
},
|
||||
{
|
||||
"code": "8904109465826",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "500 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Chyawanprash with Saffron",
|
||||
"product_name_en": "Chyawanprash with Saffron"
|
||||
},
|
||||
{
|
||||
"code": "8904100018816",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "5.5ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kesh kanti Hair Cleanser",
|
||||
"product_name_en": "Kesh kanti Hair Cleanser"
|
||||
},
|
||||
{
|
||||
"code": "8904422700963",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 litre",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Kacchi Ghani Mustard Oil",
|
||||
"product_name_en": "Patanjali Kacchi Ghani Mustard Oil"
|
||||
},
|
||||
{
|
||||
"code": "00709358",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:germany",
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Chyawanprash",
|
||||
"product_name_en": "Patanjali Chyawanprash"
|
||||
},
|
||||
{
|
||||
"code": "8906032018506",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Doodh Biscuits"
|
||||
},
|
||||
{
|
||||
"code": "8904109470073",
|
||||
"brands": [
|
||||
"patanjali"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "बादाम पाक"
|
||||
},
|
||||
{
|
||||
"code": "8904109449239",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Honey",
|
||||
"product_name_en": "Patanjali Honey"
|
||||
},
|
||||
{
|
||||
"code": "8906154300220",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Navratan Mixture",
|
||||
"product_name_en": "Patanjali Navratan Mixture"
|
||||
},
|
||||
{
|
||||
"code": "8904422702233",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 ltr",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali cow ghee 1 ltr",
|
||||
"product_name_en": "Patanjali cow ghee 1 ltr"
|
||||
},
|
||||
{
|
||||
"code": "8904109443107",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Moong Dal",
|
||||
"product_name_en": "Patanjali Moong Dal"
|
||||
},
|
||||
{
|
||||
"code": "8904109490545",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "500 ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali COW'S GHEE",
|
||||
"product_name_en": "Patanjali COW'S GHEE"
|
||||
},
|
||||
{
|
||||
"code": "8904109463549",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "250g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Amla Chatpata Candy",
|
||||
"product_name_en": "Amla Chatpata Candy"
|
||||
},
|
||||
{
|
||||
"code": "8904109420092",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "60N",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Tulsi Ghanvati",
|
||||
"product_name_en": "Tulsi Ghanvati"
|
||||
},
|
||||
{
|
||||
"code": "8904109470196",
|
||||
"brands": [
|
||||
"patanjali"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "chyawanprash",
|
||||
"product_name_en": "chyawanprash"
|
||||
},
|
||||
{
|
||||
"code": "8904109400049",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "100ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Body Lotion",
|
||||
"product_name_en": "Patanjali Body Lotion"
|
||||
},
|
||||
{
|
||||
"code": "8904109463082",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "300ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Kesh Kanti Hair Oil",
|
||||
"product_name_en": "Patanjali Kesh Kanti Hair Oil"
|
||||
},
|
||||
{
|
||||
"code": "8904109465239",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Shodhit Harad",
|
||||
"product_name_en": "Patanjali Shodhit Harad"
|
||||
},
|
||||
{
|
||||
"code": "8906032019787",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "250g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Marie biscut",
|
||||
"product_name_en": "Marie biscut"
|
||||
},
|
||||
{
|
||||
"code": "8908004906002",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1L",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Gonyle floor cleaner",
|
||||
"product_name_en": "Gonyle floor cleaner"
|
||||
},
|
||||
{
|
||||
"code": "8904109445309",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Pineapple Jam",
|
||||
"product_name_en": "Patanjali Pineapple Jam"
|
||||
},
|
||||
{
|
||||
"code": "8904109474088",
|
||||
"brands": [
|
||||
"Patanjali Ayurved Ltd."
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unpolished Pulses (Mix Pulses)",
|
||||
"product_name_en": "Unpolished Pulses (Mix Pulses)"
|
||||
},
|
||||
{
|
||||
"code": "8904109471490",
|
||||
"brands": [
|
||||
"patanjali"
|
||||
],
|
||||
"quantity": "750 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Chyawanprash",
|
||||
"product_name_en": "Chyawanprash"
|
||||
},
|
||||
{
|
||||
"code": "8904422701083",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Iodised Salt",
|
||||
"product_name_en": "Iodised Salt"
|
||||
},
|
||||
{
|
||||
"code": "8904422702998",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "500 ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Giloy Juice",
|
||||
"product_name_en": "Giloy Juice"
|
||||
},
|
||||
{
|
||||
"code": "8904109470264",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "500ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Mango Drink",
|
||||
"product_name_en": "Patanjali Mango Drink"
|
||||
},
|
||||
{
|
||||
"code": "8904109402012",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 l",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Divya Jal",
|
||||
"product_name_en": "Patanjali Divya Jal"
|
||||
},
|
||||
{
|
||||
"code": "8904109451560",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "100g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Dantkanti",
|
||||
"product_name_en": "Dantkanti"
|
||||
},
|
||||
{
|
||||
"code": "8904109463174",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "250ml",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Super Dishwash Gel",
|
||||
"product_name_en": "Patanjali Super Dishwash Gel"
|
||||
},
|
||||
{
|
||||
"code": "8904422700710",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "10 kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Atta 10kg",
|
||||
"product_name_en": "Patanjali Atta 10kg"
|
||||
},
|
||||
{
|
||||
"code": "8904109449291",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Drishti Eye Drop",
|
||||
"product_name_en": "Patanjali Drishti Eye Drop"
|
||||
},
|
||||
{
|
||||
"code": "8904422700826",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "250 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Honey",
|
||||
"product_name_en": "Honey"
|
||||
},
|
||||
{
|
||||
"code": "8904422704992",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "200 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Ragi Digestive Cookies",
|
||||
"product_name_en": "Ragi Digestive Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8904109400773",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Soundarya Shower Gel",
|
||||
"product_name_en": "Patanjali Soundarya Shower Gel"
|
||||
},
|
||||
{
|
||||
"code": "8904109400667",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Soundarya sabun",
|
||||
"product_name_en": "Soundarya sabun"
|
||||
},
|
||||
{
|
||||
"code": "8904422702417",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "250g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Amla Chatpata Candy",
|
||||
"product_name_en": "Amla Chatpata Candy"
|
||||
},
|
||||
{
|
||||
"code": "709358",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1 kg",
|
||||
"countries_tags": [
|
||||
"en:germany",
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Chyawanprash",
|
||||
"product_name_en": "Patanjali Chyawanprash"
|
||||
},
|
||||
{
|
||||
"code": "8904109410093",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "84 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali cream feast ( milk vanilla )",
|
||||
"product_name_en": "Patanjali cream feast ( milk vanilla )"
|
||||
},
|
||||
{
|
||||
"code": "8904109401268",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "80g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Toothbrush",
|
||||
"product_name_en": "Toothbrush"
|
||||
},
|
||||
{
|
||||
"code": "8904422701151",
|
||||
"brands": [
|
||||
"Patanjali"
|
||||
],
|
||||
"quantity": "1kg",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Patanjali Rock Salt",
|
||||
"product_name_en": "Patanjali Rock Salt"
|
||||
}
|
||||
]
|
||||
}
|
||||
69
data/cache/off_brand_corpus/priyagold.json
vendored
Normal file
69
data/cache/off_brand_corpus/priyagold.json
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"brand": "priyagold",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852013.6838248,
|
||||
"fetched_at_human": "2026-09-08 12:50:13",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906029790033",
|
||||
"brands": [
|
||||
"Priyagold"
|
||||
],
|
||||
"quantity": "24 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Snakker",
|
||||
"product_name_en": "Snakker"
|
||||
},
|
||||
{
|
||||
"code": "8901652140897",
|
||||
"brands": [
|
||||
"Priyagold"
|
||||
],
|
||||
"quantity": "40g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Marie lite biscuit",
|
||||
"product_name_en": "Marie lite biscuit"
|
||||
},
|
||||
{
|
||||
"code": "8901652140620",
|
||||
"brands": [
|
||||
"PRIYAGOLD"
|
||||
],
|
||||
"quantity": "40g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Puff",
|
||||
"product_name_en": "Puff"
|
||||
},
|
||||
{
|
||||
"code": "8901652142136",
|
||||
"brands": [
|
||||
"priyagold"
|
||||
],
|
||||
"quantity": "35 g",
|
||||
"countries_tags": [
|
||||
"en:australia",
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Butter Delite",
|
||||
"product_name_en": "Butter Delite"
|
||||
},
|
||||
{
|
||||
"code": "8901652140583",
|
||||
"brands": [
|
||||
"Priyagold"
|
||||
],
|
||||
"quantity": "40g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Puff choco vanilla flavoured sandwich biscuits",
|
||||
"product_name_en": "Puff choco vanilla flavoured sandwich biscuits"
|
||||
}
|
||||
]
|
||||
}
|
||||
20
data/cache/off_brand_corpus/rajdhani.json
vendored
Normal file
20
data/cache/off_brand_corpus/rajdhani.json
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"brand": "rajdhani",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852000.6414518,
|
||||
"fetched_at_human": "2026-09-08 12:50:00",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906002348169",
|
||||
"brands": [
|
||||
"Rajdhani"
|
||||
],
|
||||
"quantity": "500g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Poha Mota",
|
||||
"product_name_en": "Poha Mota"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/reckitt_benckiser.json
vendored
Normal file
7
data/cache/off_brand_corpus/reckitt_benckiser.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "reckitt benckiser",
|
||||
"country": "india",
|
||||
"fetched_at": 1788851995.867008,
|
||||
"fetched_at_human": "2026-09-08 12:49:55",
|
||||
"hits": []
|
||||
}
|
||||
44
data/cache/off_brand_corpus/society.json
vendored
Normal file
44
data/cache/off_brand_corpus/society.json
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"brand": "society",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852008.9179409,
|
||||
"fetched_at_human": "2026-09-08 12:50:08",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8901095001465",
|
||||
"brands": [
|
||||
"Society Daily"
|
||||
],
|
||||
"quantity": "14 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Masala Flavour Instant Tea Premix",
|
||||
"product_name_en": "Masala Flavour Instant Tea Premix"
|
||||
},
|
||||
{
|
||||
"code": "8901095900089",
|
||||
"brands": [
|
||||
"society Indian leaf tea masala"
|
||||
],
|
||||
"quantity": "250 grams",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "tea masala",
|
||||
"product_name_en": "tea masala"
|
||||
},
|
||||
{
|
||||
"code": "8901095000482",
|
||||
"brands": [
|
||||
"society"
|
||||
],
|
||||
"quantity": "40g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "society Sweet spicy mango pickle",
|
||||
"product_name_en": "society Sweet spicy mango pickle"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/cache/off_brand_corpus/too_yumm.json
vendored
Normal file
7
data/cache/off_brand_corpus/too_yumm.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"brand": "too yumm",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852014.7936301,
|
||||
"fetched_at_human": "2026-09-08 12:50:14",
|
||||
"hits": []
|
||||
}
|
||||
399
data/cache/off_brand_corpus/unibic.json
vendored
Normal file
399
data/cache/off_brand_corpus/unibic.json
vendored
Normal file
@@ -0,0 +1,399 @@
|
||||
{
|
||||
"brand": "unibic",
|
||||
"country": "india",
|
||||
"fetched_at": 1788852005.1305947,
|
||||
"fetched_at_human": "2026-09-08 12:50:05",
|
||||
"hits": [
|
||||
{
|
||||
"code": "8906009070902",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Honey Oatmeal Cookies",
|
||||
"product_name_en": "Unibic Honey Oatmeal Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009075600",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Swaadesi shahi kaju katli",
|
||||
"product_name_en": "Swaadesi shahi kaju katli"
|
||||
},
|
||||
{
|
||||
"code": "8906009079288",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "200g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Kesar Cashew Badam Cookies",
|
||||
"product_name_en": "Kesar Cashew Badam Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009075068",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "495 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Atta Marie Thinz",
|
||||
"product_name_en": "Atta Marie Thinz"
|
||||
},
|
||||
{
|
||||
"code": "9900409072640",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"quantity": "300g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "UNIBIC Snappers Potato Crackers",
|
||||
"product_name_en": "UNIBIC Snappers Potato Crackers"
|
||||
},
|
||||
{
|
||||
"code": "8906009077420",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pineapple cookies (sugar free)",
|
||||
"product_name_en": "Pineapple cookies (sugar free)"
|
||||
},
|
||||
{
|
||||
"code": "8906009077802",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Shortbread"
|
||||
},
|
||||
{
|
||||
"code": "8906009072906",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "UNIBIC Cashew Cookies",
|
||||
"product_name_en": "UNIBIC Cashew Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009072678",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Choco nut Cookies",
|
||||
"product_name_en": "Unibic Choco nut Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009071183",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "20g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Chocokiss",
|
||||
"product_name_en": "Chocokiss"
|
||||
},
|
||||
{
|
||||
"code": "8906009075204",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Dates & Carrot cake 140g",
|
||||
"product_name_en": "Unibic Dates & Carrot cake 140g"
|
||||
},
|
||||
{
|
||||
"code": "8906009072012",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "75",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Pista badam",
|
||||
"product_name_en": "Pista badam"
|
||||
},
|
||||
{
|
||||
"code": "8906009077277",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Butter cookies",
|
||||
"product_name_en": "Butter Cookies Sugar Free75g"
|
||||
},
|
||||
{
|
||||
"code": "19066554",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"quantity": "150 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Oats Digestive",
|
||||
"product_name_en": "Oats Digestive"
|
||||
},
|
||||
{
|
||||
"code": "8906009072784",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "250g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Biscott",
|
||||
"product_name_en": "Biscott"
|
||||
},
|
||||
{
|
||||
"code": "8906009078014",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Butter cookies",
|
||||
"product_name_en": "Butter cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009077017",
|
||||
"brands": [
|
||||
"unibic"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "fruit & nut cookies",
|
||||
"product_name_en": "fruit & nut cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009079844",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"quantity": "30g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "UNIBIC Cashew Badam Cookies",
|
||||
"product_name_en": "UNIBIC Cashew Badam Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009078021",
|
||||
"brands": [
|
||||
"UNIBIC"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "UNIBIC Cashew Cookies",
|
||||
"product_name_en": "UNIBIC Cashew Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009073958",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Coconut Cookies",
|
||||
"product_name_en": "Unibic Coconut Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009077543",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Sugar Free Multigrain Cookies",
|
||||
"product_name_en": "Sugar Free Multigrain Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009078762",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic oats 50",
|
||||
"product_name_en": "Unibic oats 50"
|
||||
},
|
||||
{
|
||||
"code": "4906409077291",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Sugar Free Buscits",
|
||||
"product_name_en": "Unibic Sugar Free Buscits"
|
||||
},
|
||||
{
|
||||
"code": "8906009073729",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "300 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Big & Bold Fruit Blast",
|
||||
"product_name_en": "Big & Bold Fruit Blast"
|
||||
},
|
||||
{
|
||||
"code": "8906009072449",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Snappers Cream n'Onion",
|
||||
"product_name_en": "Snappers Cream n'Onion"
|
||||
},
|
||||
{
|
||||
"code": "8906009078670",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "150 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Frunit and Nut cookies",
|
||||
"product_name_en": "Unibic Frunit and Nut cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009077314",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "150 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Honey Oatmeal Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009074788",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "58 g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Oats Marie Thinz",
|
||||
"product_name_en": "Oats Marie Thinz"
|
||||
},
|
||||
{
|
||||
"code": "8906009079363",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "37.5g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Fruit & nut cookis",
|
||||
"product_name_en": "Fruit & nut cookis"
|
||||
},
|
||||
{
|
||||
"code": "7906048075181",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Fruit N Nut 140g",
|
||||
"product_name_en": "Unibic Fruit N Nut 140g"
|
||||
},
|
||||
{
|
||||
"code": "8906009075167",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Unibic Royal Vanilla cake 140g",
|
||||
"product_name_en": "Unibic Royal Vanilla cake 140g"
|
||||
},
|
||||
{
|
||||
"code": "8906009073163",
|
||||
"brands": [
|
||||
"swaadesi",
|
||||
"unibic"
|
||||
],
|
||||
"quantity": "180g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "swaadesi ghee besan laddoo",
|
||||
"product_name_en": "swaadesi ghee besan laddoo"
|
||||
},
|
||||
{
|
||||
"code": "8906009077581",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"quantity": "75g",
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Choco Kiss Cookies",
|
||||
"product_name_en": "Choco Kiss Cookies"
|
||||
},
|
||||
{
|
||||
"code": "8906009077291",
|
||||
"brands": [
|
||||
"Unibic"
|
||||
],
|
||||
"countries_tags": [
|
||||
"en:india"
|
||||
],
|
||||
"product_name": "Oatmeal cookies Sugar free"
|
||||
}
|
||||
]
|
||||
}
|
||||
198
docs/BARCODE_ENRICHMENT.md
Normal file
198
docs/BARCODE_ENRICHMENT.md
Normal file
@@ -0,0 +1,198 @@
|
||||
# Barcode enrichment
|
||||
|
||||
`app/services/enrichment/barcode/` — how a catalog row acquires a barcode, what
|
||||
the barcode is then allowed to claim, and which knob to turn.
|
||||
|
||||
Referenced from `.env.example` and `app/services/enrichment/pipeline.py`, both
|
||||
of which pointed at this file for a long time before it existed.
|
||||
|
||||
---
|
||||
|
||||
## The four ways a row gets a barcode
|
||||
|
||||
| # | Path | When | Cost | Confidence |
|
||||
|---|---|---|---|---|
|
||||
| 1 | **The sheet** | The merchant typed it | free | highest — they are holding the pack |
|
||||
| 2 | **Identity derivation** | Always, inline | free, offline | derives `gtin`/`ean13`/`upc`/`barcode_type` from a barcode already present |
|
||||
| 3 | **Bulk brand corpus** | After upload, per brand | ~5 requests **per brand** | name-matched at ≥ 0.88 |
|
||||
| 4 | **Per-product cascade** | Only if enabled | 1 request **per product**, 10/min cap | brand + size + name matched |
|
||||
|
||||
Paths 1 and 2 are always on. Path 3 is on by default. Path 4 is off by default.
|
||||
|
||||
### Why path 4 is off and path 3 is on
|
||||
|
||||
They differ in cost, not in appetite for risk.
|
||||
|
||||
The Open Food Facts per-product search endpoint is capped at **10 requests per
|
||||
minute**. A 200-row upload on path 4 is twenty minutes of a held HTTP request,
|
||||
on a shared 8 GB host — which is what `settings.py:420-423` is about.
|
||||
|
||||
Path 3 fetches a brand's *entire* India catalogue in about five requests, caches
|
||||
it to `data/cache/off_brand_corpus/<slug>.json`, and matches offline. A brand
|
||||
costs the same whether it has 3 rows or 300.
|
||||
|
||||
```
|
||||
ENABLE_BARCODE_LOOKUP=false # path 4: per product, inline
|
||||
ENRICH_BARCODES_ON_UPLOAD=true # path 3: per brand, after the upload settles
|
||||
```
|
||||
|
||||
> `.env.example` shipped `ENABLE_BARCODE_LOOKUP=true` while `settings.py`
|
||||
> defaulted it `false`, for as long as both existed. Anyone copying the example
|
||||
> got a materially different pipeline from anyone relying on defaults. Fixed;
|
||||
> if you see the two disagree again, `settings.py` is authoritative.
|
||||
|
||||
---
|
||||
|
||||
## Two thresholds, and why they are different numbers
|
||||
|
||||
| Constant | Value | Direction | Used by |
|
||||
|---|---|---|---|
|
||||
| `BARCODE_MIN_NAME_SIMILARITY` | 0.78 | reverse — barcode known, name is a sanity check | `fetch_verified_nutrition_by_barcode` |
|
||||
| `OFF bulk review_min` | 0.88 | forward — name carries the whole decision | `post_ingest_barcodes`, `backfill_barcodes_from_off` |
|
||||
|
||||
`settings.py:449-477` records the measurement behind 0.78: at 0.45 the cascade
|
||||
accepted 15 candidates of which 8 were wrong; at 0.78 it accepted 2 and none
|
||||
were wrong. **Do not lower it to fix a coverage complaint.**
|
||||
|
||||
### The containment rule
|
||||
|
||||
Open Food Facts stores short names. We store long ones. Measured over the
|
||||
catalog on 2026-09-08, 149 of 300 barcoded rows were refused as "found, wrong
|
||||
product" when the barcode had resolved perfectly:
|
||||
|
||||
```
|
||||
ours "Nestle Munch 8.9g" OFF "Munch" similarity 0.332
|
||||
ours "Coca-Cola Maaza 750ml" OFF "Maaza" similarity 0.304
|
||||
```
|
||||
|
||||
`name_similarity` divides overlap by *our* token count, so a one-token candidate
|
||||
cannot exceed ~0.33 however right it is. But the same run correctly refused:
|
||||
|
||||
```
|
||||
ours "Pepsico Lays 1kg" OFF "Spanish tomato tango" 0.133
|
||||
ours "Coca-Cola Fanta 750ml" OFF "Orange" 0.089
|
||||
```
|
||||
|
||||
A threshold low enough to admit the first group admits the second. So
|
||||
`matching.name_is_contained` separates them **structurally**: every candidate
|
||||
token must be one of ours once brand and size tokens are discounted, and a bare
|
||||
brand name ("Colgate", "godrej" — both real OFF titles) never matches.
|
||||
|
||||
### `barcode_is_identity` — and the gate that actually blocked most of them
|
||||
|
||||
The name gate was the *visible* symptom. When the fix was measured it moved only
|
||||
3 rows to 8, and the reason is that `is_match` applies its rules in order and the
|
||||
**size** gate rejects first:
|
||||
|
||||
```
|
||||
Nestle Munch 38.5 g <- OFF "Munch" blocked by SIZE (off quantity = None)
|
||||
Coca-Cola Maaza 750ml <- OFF "Maaza" blocked by SIZE (off quantity = None)
|
||||
Cadbury Perk 22 g <- OFF "Perk" blocked by SIZE (off quantity = None)
|
||||
```
|
||||
|
||||
`size_matches` returns False whenever *either* side is blank, and Open Food
|
||||
Facts leaves `quantity` null on a large share of records — 57 of 146 Amul hits.
|
||||
`off_bulk` had already documented this and worked around it by treating size as
|
||||
a ranking bonus rather than a veto.
|
||||
|
||||
So the flag is `barcode_is_identity`, not `allow_containment`, because it
|
||||
describes the precondition rather than one of its consequences: **the caller
|
||||
already knows which product this is, because it fetched by GTIN.** Under it,
|
||||
name and size stop being evidence of identity and become sanity checks against
|
||||
our barcode being on the wrong row — and a sanity check cannot fail on
|
||||
information the source does not have:
|
||||
|
||||
| Rule | Normally | Under `barcode_is_identity` |
|
||||
|---|---|---|
|
||||
| 1. brand | must match | unchanged |
|
||||
| 2. size | blank ⇒ reject | **blank ⇒ no information, allowed.** Present-and-different still rejects |
|
||||
| 3. variant terms | must not conflict | unchanged |
|
||||
| 4. name similarity | ≥ floor | floor, **or** containment |
|
||||
|
||||
**Defaults off.** On the search path many candidates compete and name and size
|
||||
are the only things telling them apart — "Munch" with no size would match every
|
||||
Nestle product containing that word. Pass it only where a single candidate was
|
||||
fetched by barcode: `fetch_verified_nutrition_by_barcode` and
|
||||
`scripts/backfill_nutrition_from_barcodes`, and nothing else today.
|
||||
|
||||
---
|
||||
|
||||
## What a barcode is allowed to claim
|
||||
|
||||
`barcode_verified = true` means brand, size **and** name were matched against a
|
||||
source record. It is not a synonym for "we have a barcode".
|
||||
|
||||
| `barcode_lookup_status` | Meaning |
|
||||
|---|---|
|
||||
| `verified` | the cascade matched brand + size + name |
|
||||
| `name_matched` | bulk corpus match ≥ 0.88; the pack is **not** confirmed |
|
||||
| `sheet_validated` | the merchant supplied it and it passes the GTIN checksum |
|
||||
| `not_found` / `error` / `disabled` | no barcode stored |
|
||||
|
||||
A `name_matched` code is a real GS1 barcode for *that brand*, written to every
|
||||
size variant of a title. Good enough for catalog matching, dedup and nutrition
|
||||
lookups. **Not** good enough for logistics, invoicing, or anything a scanner
|
||||
drives. Filter on `barcode_verified = false` to select, correct or revert them.
|
||||
|
||||
---
|
||||
|
||||
## Validation is not optional and not per-source
|
||||
|
||||
`validators.validate_barcode` is the single gate: digits only → a legal GTIN
|
||||
length (8/12/13/14) → recomputed check digit. A source saying "this is the
|
||||
barcode" is never sufficient. Trust affects the *order* sources are tried, never
|
||||
whether validation runs.
|
||||
|
||||
Derived fields follow from the digits alone:
|
||||
|
||||
- `gtin` — the validated code
|
||||
- `ean13` — a UPC-A zero-padded to 13. **GTIN-8 is not padded**: an 8-digit GTIN
|
||||
is its own symbology, not a truncated EAN-13.
|
||||
- `upc` — 12-digit codes only
|
||||
|
||||
> **`upc` will always be 0% in this catalog, and that is correct.** Every code
|
||||
> here is GS1 India (prefix `890`), which issues EAN-13 and GTIN-8. UPC-A is
|
||||
> North American. Measured: 0 of 300. The coverage report counts it *not
|
||||
> applicable*, not missing.
|
||||
|
||||
---
|
||||
|
||||
## Where the fields go
|
||||
|
||||
A field computed by a stage reaches Postgres only if it is named in **all three**
|
||||
of these. Two of them were missing the barcode fields for a long time, which is
|
||||
why `upc` sat at 0% and `gtin` at 8.7% while the code that produced them ran on
|
||||
every ingestion:
|
||||
|
||||
1. `store_catalog_pipeline._to_storage_row` — the projection
|
||||
2. `vector_store.upsert_brand_products` — the INSERT column list
|
||||
3. `vector_store._ensure_columns` — the only migration mechanism (no Alembic)
|
||||
|
||||
Plus `brand_sync.EXPORT_COLUMNS` for the seed-file round trip.
|
||||
|
||||
Every enrichment column uses `COALESCE(EXCLUDED.x, table.x)` in the
|
||||
`ON CONFLICT` clause, including `barcode` itself. Without it a bare re-seed —
|
||||
which carries no enrichment keys — sets them to NULL. Proven against a live
|
||||
table: it kept `gtin` and blanked `barcode`, leaving a row claiming a GTIN with
|
||||
no barcode.
|
||||
|
||||
---
|
||||
|
||||
## Running it
|
||||
|
||||
```bash
|
||||
# What is filled, and where each value came from
|
||||
python -m scripts.catalog_coverage --by-provenance
|
||||
|
||||
# Derive gtin/ean13/upc/barcode_type from barcodes already stored (offline)
|
||||
python -m scripts.backfill_barcode_identity --apply
|
||||
|
||||
# Bulk-match barcodes from the OFF brand corpora
|
||||
python -m scripts.backfill_barcodes_from_off --apply
|
||||
|
||||
# Upgrade nutrition from a name match to an exact barcode match
|
||||
python -m scripts.backfill_nutrition_from_barcodes --apply
|
||||
```
|
||||
|
||||
Every script is **dry-run by default** and prints its target database on
|
||||
startup. `backend/.env` points at production.
|
||||
@@ -507,6 +507,52 @@ But that stability is per `image_id`: change the product name and you get a new
|
||||
10. Deterministic Product Validation Gate
|
||||
11. Vector Embedding & Storage
|
||||
|
||||
Stages 8 and 9 each run several enrichment steps internally; the stage count and
|
||||
numbering are unchanged.
|
||||
|
||||
### What gets filled for a branded row, and how far to trust it
|
||||
|
||||
The rules above for `Own Products` still hold — an unbranded row is stored as
|
||||
you sent it. A **branded** row is gap-filled, and every filled value records
|
||||
*how* it was arrived at in a `field_sources` map on the row, so nothing has to
|
||||
be taken on trust:
|
||||
|
||||
| `method` | Means | Example |
|
||||
| --- | --- | --- |
|
||||
| `sourced` | a real value from a real external source | nutrients measured per 100 g, from Open Food Facts |
|
||||
| `derived` | computed from another field on the same row | `ean13` from `barcode`; `tax_amount` from your price |
|
||||
| `estimated` | a category-level or brand-level inference | the keyword nutrient list before a lookup succeeds |
|
||||
| `not_applicable` | cannot exist for this product, and should not | nutrition on a shampoo; an FSSAI *food* licence on a detergent |
|
||||
| `unknown` | we have not got it and have no source for it | an FSSAI licence for a brand not in the registry |
|
||||
|
||||
Two consequences worth reading twice:
|
||||
|
||||
- **`not_applicable` is not a gap.** Roughly 30% of a general FMCG catalog is
|
||||
soap, shampoo and detergent. Those rows will never have nutrients or a health
|
||||
score. Coverage percentages are reported against the *applicable* rows, so
|
||||
they describe work remaining rather than work impossible.
|
||||
- **`barcode_verified: false` means the pack is not confirmed.** A barcode found
|
||||
by name match against a brand's catalogue is a real GS1 code for that brand,
|
||||
written to every size variant of the title. Good enough for catalog matching,
|
||||
dedup and nutrition lookups; **not** good enough for logistics, invoicing, or
|
||||
anything a scanner drives. Filter on it before relying on a barcode
|
||||
commercially.
|
||||
|
||||
Nothing is ever invented. There is no LLM anywhere in the nutrient, barcode,
|
||||
FSSAI or tax path, and a value that cannot be sourced or safely derived is left
|
||||
empty and labelled rather than guessed.
|
||||
|
||||
### Barcodes arrive after the upload finishes
|
||||
|
||||
Barcode and nutrition enrichment need the network, so they do **not** hold up
|
||||
your response. Ingestion returns as soon as the rows are stored; a background
|
||||
job then fetches each brand's catalogue and fills barcodes, nutrition and health
|
||||
scores over the following minutes. Its id is on the batch as `nutrition_job_id`
|
||||
and its progress is at `GET /api/admin/nutrition-intelligence/jobs/{job_id}`.
|
||||
|
||||
So a row read immediately after a `200` may have no barcode yet and have one a
|
||||
few minutes later. That is expected, and re-reading is the only action needed.
|
||||
|
||||
### Limits
|
||||
|
||||
| Limit | Value | Env var | Exceeded |
|
||||
|
||||
216
docs/off_barcodes.csv
Normal file
216
docs/off_barcodes.csv
Normal file
@@ -0,0 +1,216 @@
|
||||
brand_slug,brand_name,catalog_file,catalog_state,product_name,title,size,category,variant_key,product_sku,barcode_raw,barcode,barcode_type,computed_type,gtin,ean13,upc,barcode_source,barcode_verified,barcode_lookup_status,barcode_last_updated,checksum_valid,db_barcode,db_status
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Biryani Masala 160g,Aachi Biryani Masala,,Spices & Masalas,,AACHI-BIR-160-001,8906021120272,8906021120272,EAN-13,EAN-13,8906021120272,8906021120272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.8244982,True,8906021120272,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Biryani Masala 1kg,Aachi Biryani Masala,,Spices & Masalas,,B08TC3SNH1,8906021120272,8906021120272,EAN-13,EAN-13,8906021120272,8906021120272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.8244982,True,8906021120272,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Biryani Masala 200g,Aachi Biryani Masala,,Spices & Masalas,,B00ZGT1CEI,8906021120272,8906021120272,EAN-13,EAN-13,8906021120272,8906021120272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.8244982,True,8906021120272,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Biryani Masala 250g,Aachi Biryani Masala,,Spices & Masalas,,AACHI-BIR-250-002,8906021120272,8906021120272,EAN-13,EAN-13,8906021120272,8906021120272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.8244982,True,8906021120272,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Biryani Masala 500g,Aachi Biryani Masala,,Spices & Masalas,,B09V6MM1SF,8906021120272,8906021120272,EAN-13,EAN-13,8906021120272,8906021120272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.8244982,True,8906021120272,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Chicken Masala 500g,Aachi Chicken Masala,,Spices & Masalas,,AACHI-CHI-500-001,8906021120470,8906021120470,EAN-13,EAN-13,8906021120470,8906021120470,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.7751327,True,8906021120470,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Chilli Powder 200g,Aachi Chilli Powder,,Spices & Masalas,,100285629,8904209304087,8904209304087,EAN-13,EAN-13,8904209304087,8904209304087,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.826589,True,8904209304087,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Chilli Powder 500g,Aachi Chilli Powder,,Spices & Masalas,,B073V9GMB2,8904209304087,8904209304087,EAN-13,EAN-13,8904209304087,8904209304087,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.826589,True,8904209304087,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Mutton Masala 200g,Aachi Mutton Masala,,Spices & Masalas,,B08C7ZD99P,8906021122290,8906021122290,EAN-13,EAN-13,8906021122290,8906021122290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.7772982,True,8906021122290,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Mutton Masala 500g,Aachi Mutton Masala,,Spices & Masalas,,B073VB4QV2,8906021122290,8906021122290,EAN-13,EAN-13,8906021122290,8906021122290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.7772982,True,8906021122290,in_sync
|
||||
aachi,Aachi,data/seed_catalogs/archive/brand_catalog_aachi.json,archived,Aachi Mutton Masala 50g,Aachi Mutton Masala,,Spices & Masalas,,100286167,8906021122290,8906021122290,EAN-13,EAN-13,8906021122290,8906021122290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773804.7772982,True,8906021122290,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Dahi 1kg,Amul Dahi,1kg,Dairy,amul_amul_dahi_1kg,B0D7W14TS5,8901262200271,8901262200271,EAN-13,EAN-13,8901262200271,8901262200271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.0632794,True,8901262200271,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Dahi 200 g,Amul Dahi,200 g,Dairy,amul_amul_dahi_200_g,30000356,8901262200271,8901262200271,EAN-13,EAN-13,8901262200271,8901262200271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.0632794,True,8901262200271,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Dahi 400g,Amul Dahi,400g,Dairy,amul_amul_dahi_400g,104851,8901262200271,8901262200271,EAN-13,EAN-13,8901262200271,8901262200271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.0632794,True,8901262200271,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Dahi 90g,Amul Dahi,90g,Dairy,amul_amul_dahi_90g,45533,8901262200271,8901262200271,EAN-13,EAN-13,8901262200271,8901262200271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.0632794,True,8901262200271,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Fresh Cream 1L,Amul Fresh Cream,1L,Cheese,amul_amul_fresh_cream_1l,162,8901262150118,8901262150118,EAN-13,EAN-13,8901262150118,8901262150118,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2152038,True,8901262150118,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Fresh Cream 250 ml,Amul Fresh Cream,250 ml,Cheese,amul_amul_fresh_cream_250_ml,B0758LVKLL,8901262150118,8901262150118,EAN-13,EAN-13,8901262150118,8901262150118,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2152038,True,8901262150118,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Fresh Cream 90ml,Amul Fresh Cream,90ml,Cheese,amul_amul_fresh_cream_90ml,162,8901262150118,8901262150118,EAN-13,EAN-13,8901262150118,8901262150118,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2152038,True,8901262150118,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ghee 1L,Amul Ghee,1L,Dairy,amul_amul_ghee_1l,40096994,8901262031059,8901262031059,EAN-13,EAN-13,8901262031059,8901262031059,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.015503,True,8901262031059,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ghee 200ml,Amul Ghee,200ml,Dairy,amul_amul_ghee_200ml,40166276,8901262031059,8901262031059,EAN-13,EAN-13,8901262031059,8901262031059,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.015503,True,8901262031059,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ghee 500ml,Amul Ghee,500ml,Dairy,amul_amul_ghee_500ml,40050541,8901262031059,8901262031059,EAN-13,EAN-13,8901262031059,8901262031059,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.015503,True,8901262031059,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ghee 90ml,Amul Ghee,90ml,Dairy,amul_amul_ghee_90ml,40166277,8901262031059,8901262031059,EAN-13,EAN-13,8901262031059,8901262031059,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.015503,True,8901262031059,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ice Cream 1L,Amul Ice Cream,1L,Ice Cream,amul_amul_ice_cream_1l,40003798,8901262172363,8901262172363,EAN-13,EAN-13,8901262172363,8901262172363,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.1036828,True,8901262172363,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ice Cream 250g,Amul Ice Cream,250g,Ice Cream,amul_amul_ice_cream_250g,40003801,8901262172363,8901262172363,EAN-13,EAN-13,8901262172363,8901262172363,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.1036828,True,8901262172363,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Ice Cream 90ml,Amul Ice Cream,90ml,Ice Cream,amul_amul_ice_cream_90ml,40344404,8901262172363,8901262172363,EAN-13,EAN-13,8901262172363,8901262172363,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.1036828,True,8901262172363,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Kool 100ml,Amul Kool,100ml,Beverages,amul_amul_kool_100ml,B00ZCLCCWG,8901262153355,8901262153355,EAN-13,EAN-13,8901262153355,8901262153355,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.123552,True,8901262153355,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Kool 180ml,Amul Kool,180ml,Beverages,amul_amul_kool_180ml,B00NTU7YOS,8901262153355,8901262153355,EAN-13,EAN-13,8901262153355,8901262153355,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.123552,True,8901262153355,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Kool 250ml,Amul Kool,250ml,Beverages,amul_amul_kool_250ml,69633,8901262153355,8901262153355,EAN-13,EAN-13,8901262153355,8901262153355,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.123552,True,8901262153355,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lassi 100ml,Amul Lassi,100ml,Beverages,amul_amul_lassi_100ml,656965,8901262200189,8901262200189,EAN-13,EAN-13,8901262200189,8901262200189,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.072889,True,8901262200189,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lassi 1kg,Amul Lassi,1kg,Beverages,amul_amul_lassi_1kg,68553,8901262200189,8901262200189,EAN-13,EAN-13,8901262200189,8901262200189,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.072889,True,8901262200189,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lassi 200g,Amul Lassi,200g,Beverages,amul_amul_lassi_200g,178319,8901262200189,8901262200189,EAN-13,EAN-13,8901262200189,8901262200189,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.072889,True,8901262200189,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lassi 250ml,Amul Lassi,250ml,Beverages,amul_amul_lassi_250ml,40026295,8901262200189,8901262200189,EAN-13,EAN-13,8901262200189,8901262200189,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.072889,True,8901262200189,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lite Bread Spread 1kg,Amul Lite Bread Spread,1kg,Bakery & Breads,amul_amul_lite_bread_spread_1kg,255187,8901262140065,8901262140065,EAN-13,EAN-13,8901262140065,8901262140065,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.3098247,True,8901262140065,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lite Bread Spread 500 g,Amul Lite Bread Spread,500 g,Bakery & Breads,amul_amul_lite_bread_spread_500_g,257195,8901262140065,8901262140065,EAN-13,EAN-13,8901262140065,8901262140065,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.3098247,True,8901262140065,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Lite Bread Spread 90g,Amul Lite Bread Spread,90g,Bakery & Breads,amul_amul_lite_bread_spread_90g,255187,8901262140065,8901262140065,EAN-13,EAN-13,8901262140065,8901262140065,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.3098247,True,8901262140065,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Malai Paneer 1L,Amul Malai Paneer,1L,Dairy,amul_amul_malai_paneer_1l,40096747,8901262180016,8901262180016,EAN-13,EAN-13,8901262180016,8901262180016,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2311263,True,8901262180016,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Malai Paneer 200ml,Amul Malai Paneer,200ml,Dairy,amul_amul_malai_paneer_200ml,40096747,8901262180016,8901262180016,EAN-13,EAN-13,8901262180016,8901262180016,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2311263,True,8901262180016,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Malai Paneer 425ml,Amul Malai Paneer,425ml,Dairy,amul_amul_malai_paneer_425ml,40096747,8901262180016,8901262180016,EAN-13,EAN-13,8901262180016,8901262180016,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2311263,True,8901262180016,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Malai Paneer 90ml,Amul Malai Paneer,90ml,Dairy,amul_amul_malai_paneer_90ml,40096747,8901262180016,8901262180016,EAN-13,EAN-13,8901262180016,8901262180016,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2311263,True,8901262180016,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Masti Dahi 1L,Amul Masti Dahi,1L,Cheese,amul_amul_masti_dahi_1l,40323755,8901262200677,8901262200677,EAN-13,EAN-13,8901262200677,8901262200677,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2026575,True,8901262200677,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Masti Dahi 200ml,Amul Masti Dahi,200ml,Cheese,amul_amul_masti_dahi_200ml,30000356,8901262200677,8901262200677,EAN-13,EAN-13,8901262200677,8901262200677,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2026575,True,8901262200677,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Masti Dahi 5L,Amul Masti Dahi,5L,Cheese,amul_amul_masti_dahi_5l,AMUL-MAS-5-001,8901262200677,8901262200677,EAN-13,EAN-13,8901262200677,8901262200677,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2026575,True,8901262200677,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Masti Dahi 90ml,Amul Masti Dahi,90ml,Cheese,amul_amul_masti_dahi_90ml,45533,8901262200677,8901262200677,EAN-13,EAN-13,8901262200677,8901262200677,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.2026575,True,8901262200677,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Shrikhand 1kg,Amul Shrikhand,1kg,Dairy - Desserts,amul_amul_shrikhand_1kg,AMUL-SHR-1-001,8901262040051,8901262040051,EAN-13,EAN-13,8901262040051,8901262040051,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.152311,True,8901262040051,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Shrikhand 500g,Amul Shrikhand,500g,Dairy - Desserts,amul_amul_shrikhand_500g,104833,8901262040051,8901262040051,EAN-13,EAN-13,8901262040051,8901262040051,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.152311,True,8901262040051,in_sync
|
||||
amul,amul,data/seed_catalogs/brand_catalog_amul.json,active,Amul Shrikhand 90g,Amul Shrikhand,90g,Dairy - Desserts,amul_amul_shrikhand_90g,SAMEZ3BURAXPCXGQ,8901262040051,8901262040051,EAN-13,EAN-13,8901262040051,8901262040051,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768769.152311,True,8901262040051,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 100g,Cadbury 5 Star,100g,Chocolates,cadbury_cadbury_5_star_100g,B0758Q2W7D,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 18g,Cadbury 5 Star,18g,Chocolates,cadbury_cadbury_5_star_18g,B0H6GKBR83,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 200g,Cadbury 5 Star,200g,Chocolates,cadbury_cadbury_5_star_200g,B00XYALG1K,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 21 g,Cadbury 5 Star,21 g,Chocolates,cadbury_cadbury_5_star_21_g,40325909,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 5 gm,Cadbury 5 Star,5 gm,Chocolates,cadbury_cadbury_5_star_5_gm,B0BYN4RN4T,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury 5 Star 9.8 g,Cadbury 5 Star,9.8 g,Chocolates,cadbury_cadbury_5_star_9_8_g,900457475,8901233020273,8901233020273,EAN-13,EAN-13,8901233020273,8901233020273,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8735542,True,8901233020273,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Bournvita 14.4g,Cadbury Bournvita,14.4g,Health Drinks,cadbury_cadbury_bournvita_14_4g,B00LIVCED6,8901233018362,8901233018362,EAN-13,EAN-13,8901233018362,8901233018362,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8771605,True,8901233018362,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Bournvita 1kg,Cadbury Bournvita,1kg,Health Drinks,cadbury_cadbury_bournvita_1kg,B08N5PTZC7,8901233018362,8901233018362,EAN-13,EAN-13,8901233018362,8901233018362,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8771605,True,8901233018362,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Bournvita 2 kg,Cadbury Bournvita,2 kg,Health Drinks,cadbury_cadbury_bournvita_2_kg,1214685,8901233018362,8901233018362,EAN-13,EAN-13,8901233018362,8901233018362,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8771605,True,8901233018362,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Bournvita 25g,Cadbury Bournvita,25g,Health Drinks,cadbury_cadbury_bournvita_25g,400825,8901233018362,8901233018362,EAN-13,EAN-13,8901233018362,8901233018362,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8771605,True,8901233018362,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Bournvita 500g,Cadbury Bournvita,500g,Health Drinks,cadbury_cadbury_bournvita_500g,B06XSB6RB2,8901233018362,8901233018362,EAN-13,EAN-13,8901233018362,8901233018362,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8771605,True,8901233018362,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Dairy Milk 100g,Cadbury Dairy Milk,100g,Chocolates,cadbury_cadbury_dairy_milk_100g,B079TQSV9S,8901233028361,8901233028361,EAN-13,EAN-13,8901233028361,8901233028361,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8806314,True,8901233028361,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Dairy Milk 500g,Cadbury Dairy Milk,500g,Chocolates,cadbury_cadbury_dairy_milk_500g,CADBUR-DAI-500-001,8901233028361,8901233028361,EAN-13,EAN-13,8901233028361,8901233028361,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.8806314,True,8901233028361,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Fuse 100g,Cadbury Fuse,100g,Chocolates,cadbury_cadbury_fuse_100g,B07FNZRTZD,8901233023687,8901233023687,EAN-13,EAN-13,8901233023687,8901233023687,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.924282,True,8901233023687,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Fuse 25 g,Cadbury Fuse,25 g,Chocolates,cadbury_cadbury_fuse_25_g,B0BRXV8R1C,8901233023687,8901233023687,EAN-13,EAN-13,8901233023687,8901233023687,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.924282,True,8901233023687,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Fuse 50g,Cadbury Fuse,50g,Chocolates,cadbury_cadbury_fuse_50g,B07F6B4R1W,8901233023687,8901233023687,EAN-13,EAN-13,8901233023687,8901233023687,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.924282,True,8901233023687,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Nutties 100g,Cadbury Nutties,100g,Chocolates,cadbury_cadbury_nutties_100g,34422,8901233021492,8901233021492,EAN-13,EAN-13,8901233021492,8901233021492,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.9583745,True,8901233021492,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Nutties 20g,Cadbury Nutties,20g,Chocolates,cadbury_cadbury_nutties_20g,B01IHCPDJA,8901233021492,8901233021492,EAN-13,EAN-13,8901233021492,8901233021492,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.9583745,True,8901233021492,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Nutties 30g,Cadbury Nutties,30g,Chocolates,cadbury_cadbury_nutties_30g,B01IHCPDJA,8901233021492,8901233021492,EAN-13,EAN-13,8901233021492,8901233021492,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.9583745,True,8901233021492,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Nutties 55g,Cadbury Nutties,55g,Chocolates,cadbury_cadbury_nutties_55g,B0721MLS76,8901233021492,8901233021492,EAN-13,EAN-13,8901233021492,8901233021492,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.9583745,True,8901233021492,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Perk 11g,Cadbury Perk,11g,Chocolates,cadbury_cadbury_perk_11g,20005969,8901233030272,8901233030272,EAN-13,EAN-13,8901233030272,8901233030272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.886849,True,8901233030272,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Perk 14.3 g,Cadbury Perk,14.3 g,Chocolates,cadbury_cadbury_perk_14_3_g,CADBUR-PER-143-001,8901233030272,8901233030272,EAN-13,EAN-13,8901233030272,8901233030272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.886849,True,8901233030272,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Perk 1kg,Cadbury Perk,1kg,Chocolates,cadbury_cadbury_perk_1kg,B01B5ZXNLQ,8901233030272,8901233030272,EAN-13,EAN-13,8901233030272,8901233030272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.886849,True,8901233030272,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Perk 22 g,Cadbury Perk,22 g,Chocolates,cadbury_cadbury_perk_22_g,CHCFWY5YHXYBGDYH,8901233030272,8901233030272,EAN-13,EAN-13,8901233030272,8901233030272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.886849,True,8901233030272,in_sync
|
||||
cadbury,cadbury,data/seed_catalogs/brand_catalog_cadbury.json,active,Cadbury Perk 90g,Cadbury Perk,90g,Chocolates,cadbury_cadbury_perk_90g,225506,8901233030272,8901233030272,EAN-13,EAN-13,8901233030272,8901233030272,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768771.886849,True,8901233030272,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Fanta 350ml,Coca-Cola Fanta,350ml,Beverages,coca_cola_coca_cola_fanta_350ml,COCACO-FAN-350-001,8906000379134,8906000379134,EAN-13,EAN-13,8906000379134,8906000379134,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.368387,True,8906000379134,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Fanta 750ml,Coca-Cola Fanta,750ml,Beverages,coca_cola_coca_cola_fanta_750ml,B0752S51ZL,8906000379134,8906000379134,EAN-13,EAN-13,8906000379134,8906000379134,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.368387,True,8906000379134,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Limca 1L,Coca-Cola Limca,1L,Beverages,coca_cola_coca_cola_limca_1l,COCACO-LIM-1-001,89000601,89000601,GTIN-8,GTIN-8,89000601,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3936577,True,89000601,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Limca 750g,Coca-Cola Limca,750g,Beverages,coca_cola_coca_cola_limca_750g,427683,89000601,89000601,GTIN-8,GTIN-8,89000601,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3936577,True,89000601,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 1.2L,Coca-Cola Maaza,1.2L,Beverages,coca_cola_coca_cola_maaza_1_2l,B00GX9TS6O,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 125ml,Coca-Cola Maaza,125ml,Beverages,coca_cola_coca_cola_maaza_125ml,COCACO-MAA-125-001,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 250ml,Coca-Cola Maaza,250ml,Beverages,coca_cola_coca_cola_maaza_250ml,COCACO-MAA-250-001,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 350ml,Coca-Cola Maaza,350ml,Beverages,coca_cola_coca_cola_maaza_350ml,COCACO-MAA-350-001,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 600 ml,Coca-Cola Maaza,600 ml,Beverages,coca_cola_coca_cola_maaza_600_ml,COCACO-MAA-600-001,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Maaza 750ml,Coca-Cola Maaza,750ml,Beverages,coca_cola_coca_cola_maaza_750ml,B004ZXK6FC,8901764175015,8901764175015,EAN-13,EAN-13,8901764175015,8901764175015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3713913,True,8901764175015,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Minute Maid Apple 150 ml,Coca-Cola Minute Maid Apple,150 ml,Beverages,coca_cola_coca_cola_minute_maid_apple_150_ml,COCACO-MIN-150-002,8901764385155,8901764385155,EAN-13,EAN-13,8901764385155,8901764385155,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.4120922,True,8901764385155,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Minute Maid Apple 350ml,Coca-Cola Minute Maid Apple,350ml,Beverages,coca_cola_coca_cola_minute_maid_apple_350ml,COCACO-MIN-350-004,8901764385155,8901764385155,EAN-13,EAN-13,8901764385155,8901764385155,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.4120922,True,8901764385155,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Minute Maid Apple 750ml,Coca-Cola Minute Maid Apple,750ml,Beverages,coca_cola_coca_cola_minute_maid_apple_750ml,COCACO-MIN-750-004,8901764385155,8901764385155,EAN-13,EAN-13,8901764385155,8901764385155,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.4120922,True,8901764385155,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Sprite 350ml,Coca-Cola Sprite,350ml,Beverages,coca_cola_coca_cola_sprite_350ml,COCACO-SPR-350-001,8901764032271,8901764032271,EAN-13,EAN-13,8901764032271,8901764032271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3654883,True,8901764032271,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Sprite 750ml,Coca-Cola Sprite,750ml,Beverages,coca_cola_coca_cola_sprite_750ml,COCACO-SPR-750-001,8901764032271,8901764032271,EAN-13,EAN-13,8901764032271,8901764032271,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3654883,True,8901764032271,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Thums Up 350ml,Coca-Cola Thums Up,350ml,Beverages,coca_cola_coca_cola_thums_up_350ml,COCACO-THU-350-001,8901764042300,8901764042300,EAN-13,EAN-13,8901764042300,8901764042300,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3626227,True,8901764042300,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Thums Up 750ml,Coca-Cola Thums Up,750ml,Beverages,coca_cola_coca_cola_thums_up_750ml,251014,8901764042300,8901764042300,EAN-13,EAN-13,8901764042300,8901764042300,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3626227,True,8901764042300,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,"Coca-Cola Zero 1,25 L e",Coca-Cola Zero,"1,25 L e",Beverages,coca_cola_coca_cola_zero_1_25_l_e,COCACO-ZER-1-001,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Zero 1.5l,Coca-Cola Zero,1.5l,Beverages,coca_cola_coca_cola_zero_1_5l,COCACO-ZER-15-001,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Zero 1L,Coca-Cola Zero,1L,Beverages,coca_cola_coca_cola_zero_1l,COCACO-ZER-1-002,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Zero 250 ml,Coca-Cola Zero,250 ml,Beverages,coca_cola_coca_cola_zero_250_ml,COCACO-ZER-250-001,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Zero 330 ml,Coca-Cola Zero,330 ml,Beverages,coca_cola_coca_cola_zero_330_ml,COCACO-ZER-330-001,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
coca_cola,coca-cola,data/seed_catalogs/archive/brand_catalog_coca_cola.json,archived,Coca-Cola Zero 500ml,Coca-Cola Zero,500ml,Beverages,coca_cola_coca_cola_zero_500ml,COCACO-ZER-500-001,8901764112706,8901764112706,EAN-13,EAN-13,8901764112706,8901764112706,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773807.3840604,True,8901764112706,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Dental Cream 120g,Colgate-Palmolive Colgate Dental Cream,120g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_dental_cream_120g,COLGAT-COL-120-001,8901314765352,8901314765352,EAN-13,EAN-13,8901314765352,8901314765352,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5714066,True,8901314765352,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Dental Cream 50g,Colgate-Palmolive Colgate Dental Cream,50g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_dental_cream_50g,COLGAT-COL-50-001,8901314765352,8901314765352,EAN-13,EAN-13,8901314765352,8901314765352,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5714066,True,8901314765352,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Dental Cream 90g,Colgate-Palmolive Colgate Dental Cream,90g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_dental_cream_90g,COLGAT-COL-90-001,8901314765352,8901314765352,EAN-13,EAN-13,8901314765352,8901314765352,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5714066,True,8901314765352,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Maxfresh 10g,Colgate-Palmolive Colgate Maxfresh,10g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_maxfresh_10g,B079RXNHHT,8901314543653,8901314543653,EAN-13,EAN-13,8901314543653,8901314543653,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5746639,True,8901314543653,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Maxfresh 19g,Colgate-Palmolive Colgate Maxfresh,19g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_maxfresh_19g,B079RXNHHT,8901314543653,8901314543653,EAN-13,EAN-13,8901314543653,8901314543653,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5746639,True,8901314543653,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Maxfresh 8g,Colgate-Palmolive Colgate Maxfresh,8g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_maxfresh_8g,COLGAT-COL-8-001,8901314543653,8901314543653,EAN-13,EAN-13,8901314543653,8901314543653,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5746639,True,8901314543653,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Sensitive 20g,Colgate-Palmolive Colgate Sensitive,20g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_sensitive_20g,COLGAT-COL-20-001,8901314311832,8901314311832,EAN-13,EAN-13,8901314311832,8901314311832,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5771685,True,8901314311832,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Sensitive 30g,Colgate-Palmolive Colgate Sensitive,30g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_sensitive_30g,COLGAT-COL-30-001,8901314311832,8901314311832,EAN-13,EAN-13,8901314311832,8901314311832,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5771685,True,8901314311832,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Sensitive 50g,Colgate-Palmolive Colgate Sensitive,50g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_sensitive_50g,COLGAT-COL-50-002,8901314311832,8901314311832,EAN-13,EAN-13,8901314311832,8901314311832,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5771685,True,8901314311832,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Sensitive 8g,Colgate-Palmolive Colgate Sensitive,8g,Oral Care,colgate_palmolive_colgate_palmolive_colgate_sensitive_8g,COLGAT-COL-8-002,8901314311832,8901314311832,EAN-13,EAN-13,8901314311832,8901314311832,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5771685,True,8901314311832,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Visible White 100g,Colgate-Palmolive Colgate Visible White,100g,General,colgate_palmolive_colgate_palmolive_colgate_visible_white_100g,COLGAT-COL-100-001,8901314011183,8901314011183,EAN-13,EAN-13,8901314011183,8901314011183,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5915623,True,8901314011183,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Visible White 150ml,Colgate-Palmolive Colgate Visible White,150ml,General,colgate_palmolive_colgate_palmolive_colgate_visible_white_150ml,B09QSBPKTF,8901314011183,8901314011183,EAN-13,EAN-13,8901314011183,8901314011183,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5915623,True,8901314011183,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Visible White 200ml,Colgate-Palmolive Colgate Visible White,200ml,General,colgate_palmolive_colgate_palmolive_colgate_visible_white_200ml,COLGAT-COL-200-001,8901314011183,8901314011183,EAN-13,EAN-13,8901314011183,8901314011183,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5915623,True,8901314011183,in_sync
|
||||
colgate_palmolive,colgate-palmolive,data/seed_catalogs/archive/brand_catalog_colgate_palmolive.json,archived,Colgate-Palmolive Colgate Visible White 50g,Colgate-Palmolive Colgate Visible White,50g,General,colgate_palmolive_colgate_palmolive_colgate_visible_white_50g,B09QSBPKTF,8901314011183,8901314011183,EAN-13,EAN-13,8901314011183,8901314011183,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773808.5915623,True,8901314011183,in_sync
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Chyawanprash 10g,Chyawanprash,10g,Health Care - Ayurvedic,dabur_chyawanprash_10g,DABUR-CHY-10-001,8901207036989,8901207036989,EAN-13,EAN-13,8901207036989,8901207036989,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.447363,True,,row_not_in_db
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Chyawanprash 20g,Chyawanprash,20g,Health Care - Ayurvedic,dabur_chyawanprash_20g,DABUR-CHY-20-001,8901207036989,8901207036989,EAN-13,EAN-13,8901207036989,8901207036989,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.447363,True,8901207036989,in_sync
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Chyawanprash 5g,Chyawanprash,5g,Health Care - Ayurvedic,dabur_chyawanprash_5g,30009463,8901207036989,8901207036989,EAN-13,EAN-13,8901207036989,8901207036989,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.447363,True,,row_not_in_db
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Gulabari 100g,Dabur Gulabari,100g,Skin Care,dabur_dabur_gulabari_100g,B0BDRRJNLC,89005590,89005590,GTIN-8,GTIN-8,89005590,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.5249188,True,89005590,in_sync
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Gulabari 59g,Dabur Gulabari,59g,Skin Care,dabur_dabur_gulabari_59g,DABUR-GUL-59-001,89005590,89005590,GTIN-8,GTIN-8,89005590,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.5249188,True,,row_not_in_db
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Gulabari 75g,Dabur Gulabari,75g,Skin Care,dabur_dabur_gulabari_75g,DABUR-GUL-75-001,89005590,89005590,GTIN-8,GTIN-8,89005590,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.5249188,True,,row_not_in_db
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Odomos 100g,Dabur Odomos,100g,Personal Care - Mosquito Repellent,dabur_dabur_odomos_100g,B00HVSSZY2,8901207500053,8901207500053,EAN-13,EAN-13,8901207500053,8901207500053,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.519291,True,8901207500053,in_sync
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Odomos 20g,Dabur Odomos,20g,Personal Care - Mosquito Repellent,dabur_dabur_odomos_20g,B00AXX608K,8901207500053,8901207500053,EAN-13,EAN-13,8901207500053,8901207500053,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.519291,True,,row_not_in_db
|
||||
dabur,dabur,data/seed_catalogs/archive/brand_catalog_dabur.json,archived,Dabur Odomos 50g,Dabur Odomos,50g,Personal Care - Mosquito Repellent,dabur_dabur_odomos_50g,DABUR-ODO-50-001,8901207500053,8901207500053,EAN-13,EAN-13,8901207500053,8901207500053,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.519291,True,,row_not_in_db
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Cinthol 100g,Godrej Cinthol,100g,Bath Soap,godrej_godrej_cinthol_100g,GODREJ-CIN-100-001,8901023020353,8901023020353,EAN-13,EAN-13,8901023020353,8901023020353,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.976005,True,8901023020353,in_sync
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Cinthol 50g,Godrej Cinthol,50g,Bath Soap,godrej_godrej_cinthol_50g,B0739RXZT8,8901023020353,8901023020353,EAN-13,EAN-13,8901023020353,8901023020353,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.976005,True,,row_not_in_db
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Cinthol 75g,Godrej Cinthol,75g,Bath Soap,godrej_godrej_cinthol_75g,B01MZWIZA9,8901023020353,8901023020353,EAN-13,EAN-13,8901023020353,8901023020353,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.976005,True,,row_not_in_db
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Nupur Henna 100ml,Godrej Nupur Henna,100ml,Hair Care,godrej_godrej_nupur_henna_100ml,438469,8901023018602,8901023018602,EAN-13,EAN-13,8901023018602,8901023018602,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.9603631,True,,row_not_in_db
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Nupur Henna 250ml,Godrej Nupur Henna,250ml,Hair Care,godrej_godrej_nupur_henna_250ml,B08D8Z9JNL,8901023018602,8901023018602,EAN-13,EAN-13,8901023018602,8901023018602,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.9603631,True,8901023018602,in_sync
|
||||
godrej,godrej,data/seed_catalogs/archive/brand_catalog_godrej.json,archived,Godrej Nupur Henna 90ml,Godrej Nupur Henna,90ml,Hair Care,godrej_godrej_nupur_henna_90ml,B005ZLCIU4,8901023018602,8901023018602,EAN-13,EAN-13,8901023018602,8901023018602,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773809.9603631,True,,row_not_in_db
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Health Mix 100g,Manna Health Mix 50g,100g,Health Foods,manna_manna_health_mix_50g_100g,B074778SPY,8906008350852,8906008350852,EAN-13,EAN-13,8906008350852,8906008350852,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4010751,True,8906008350852,in_sync
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Health Mix 25g,Manna Health Mix 50g,25g,Health Foods,manna_manna_health_mix_50g_25g,MDMFYWX4SC4NRTFZ,8906008350852,8906008350852,EAN-13,EAN-13,8906008350852,8906008350852,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4010751,True,,row_not_in_db
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Health Mix 50g,Manna Health Mix 50g,50g,Health Foods,manna_manna_health_mix_50g_50g,B074778SPY,8906008350852,8906008350852,EAN-13,EAN-13,8906008350852,8906008350852,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4010751,True,,row_not_in_db
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Ragi Malt 100g,Manna Ragi Malt 50g,100g,Health Foods,manna_manna_ragi_malt_50g_100g,B07D755GSF,8906008350388,8906008350388,EAN-13,EAN-13,8906008350388,8906008350388,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4015913,True,8906008350388,in_sync
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Ragi Malt 25g,Manna Ragi Malt 50g,25g,Health Foods,manna_manna_ragi_malt_50g_25g,B07D755GSF,8906008350388,8906008350388,EAN-13,EAN-13,8906008350388,8906008350388,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4015913,True,,row_not_in_db
|
||||
manna,Manna,data/seed_catalogs/archive/brand_catalog_manna.json,archived,Manna Ragi Malt 50g,Manna Ragi Malt 50g,50g,Health Foods,manna_manna_ragi_malt_50g_50g,B00DRE5614,8906008350388,8906008350388,EAN-13,EAN-13,8906008350388,8906008350388,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.4015913,True,,row_not_in_db
|
||||
mtr,Mtr,data/seed_catalogs/archive/brand_catalog_mtr.json,archived,MTR Sambar Powder 1.5kg,MTR Sambar Powder,,Spices & Masalas,,SCMETEMHEY5Z3VMX,8901042954721,8901042954721,EAN-13,EAN-13,8901042954721,8901042954721,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.924069,True,8901042954721,in_sync
|
||||
mtr,Mtr,data/seed_catalogs/archive/brand_catalog_mtr.json,archived,MTR Sambar Powder 200g,MTR Sambar Powder,,Spices & Masalas,,B009LL92VC,8901042954721,8901042954721,EAN-13,EAN-13,8901042954721,8901042954721,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.924069,True,8901042954721,in_sync
|
||||
mtr,Mtr,data/seed_catalogs/archive/brand_catalog_mtr.json,archived,MTR Sambar Powder 500g,MTR Sambar Powder,,Spices & Masalas,,40185042,8901042954721,8901042954721,EAN-13,EAN-13,8901042954721,8901042954721,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.924069,True,8901042954721,in_sync
|
||||
mtr,Mtr,data/seed_catalogs/archive/brand_catalog_mtr.json,archived,MTR Sambar Powder 90g,MTR Sambar Powder,,Spices & Masalas,,SCMETEMHEY5Z3VMX,8901042954721,8901042954721,EAN-13,EAN-13,8901042954721,8901042954721,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773810.924069,True,8901042954721,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Cerelac 125g,Nestle Cerelac,125g,Baby Care,nestle_nestle_cerelac_125g,B004ZKZMAE,8901058844627,8901058844627,EAN-13,EAN-13,8901058844627,8901058844627,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5886683,True,8901058844627,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Cerelac 250g,Nestle Cerelac,250g,Baby Care,nestle_nestle_cerelac_250g,NESTLE-CER-250-001,8901058844627,8901058844627,EAN-13,EAN-13,8901058844627,8901058844627,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5886683,True,8901058844627,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Cerelac 300g,Nestle Cerelac,300g,Baby Care,nestle_nestle_cerelac_300g,25012,8901058844627,8901058844627,EAN-13,EAN-13,8901058844627,8901058844627,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5886683,True,8901058844627,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Cerelac 400g,Nestle Cerelac,400g,Baby Care,nestle_nestle_cerelac_400g,NESTLE-CER-400-001,8901058844627,8901058844627,EAN-13,EAN-13,8901058844627,8901058844627,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5886683,True,8901058844627,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Cerelac 90g,Nestle Cerelac,90g,Baby Care,nestle_nestle_cerelac_90g,NESTLE-CER-90-001,8901058844627,8901058844627,EAN-13,EAN-13,8901058844627,8901058844627,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5886683,True,8901058844627,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Kitkat 120g,Nestle Kitkat,120g,Chocolates,nestle_nestle_kitkat_120g,NESTLE-KIT-120-001,8901058857245,8901058857245,EAN-13,EAN-13,8901058857245,8901058857245,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5304384,True,8901058857245,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Kitkat 170g,Nestle Kitkat,170g,Chocolates,nestle_nestle_kitkat_170g,40018531,8901058857245,8901058857245,EAN-13,EAN-13,8901058857245,8901058857245,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5304384,True,8901058857245,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Kitkat 50g,Nestle Kitkat,50g,Chocolates,nestle_nestle_kitkat_50g,NESTLE-KIT-50-001,8901058857245,8901058857245,EAN-13,EAN-13,8901058857245,8901058857245,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5304384,True,8901058857245,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 10ml,Nestle Milkybar,10ml,Chocolates,nestle_nestle_milkybar_10ml,40090019,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 1kg,Nestle Milkybar,1kg,Chocolates,nestle_nestle_milkybar_1kg,B01ILWLMLE,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 24.5ml,Nestle Milkybar,24.5ml,Chocolates,nestle_nestle_milkybar_24_5ml,B08P5Y1GPF,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 30ml,Nestle Milkybar,30ml,Chocolates,nestle_nestle_milkybar_30ml,B08S55766X,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 38ml,Nestle Milkybar,38ml,Chocolates,nestle_nestle_milkybar_38ml,NESTLE-MIL-38-001,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milkybar 90g,Nestle Milkybar,90g,Chocolates,nestle_nestle_milkybar_90g,B005GLIBLI,89008478,89008478,GTIN-8,GTIN-8,89008478,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5370526,True,89008478,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 100g,Nestle Milo,100g,Health Drinks,nestle_nestle_milo_100g,B00RBMP37A,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 165g,Nestle Milo,165g,Health Drinks,nestle_nestle_milo_165g,B00RBMP37A,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 200 g,Nestle Milo,200 g,Health Drinks,nestle_nestle_milo_200_g,NESTLE-MIL-200-001,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 20g,Nestle Milo,20g,Health Drinks,nestle_nestle_milo_20g,40184472,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 25 g,Nestle Milo,25 g,Health Drinks,nestle_nestle_milo_25_g,B00RBMP37A,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Milo 30g,Nestle Milo,30g,Health Drinks,nestle_nestle_milo_30g,NESTLE-MIL-30-001,8901058904017,8901058904017,EAN-13,EAN-13,8901058904017,8901058904017,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6294699,True,8901058904017,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 150g,Nestle Munch,150g,Chocolates,nestle_nestle_munch_150g,B01MRFIF28,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 20g,Nestle Munch,20g,Chocolates,nestle_nestle_munch_20g,496297,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 38.5 g,Nestle Munch,38.5 g,Chocolates,nestle_nestle_munch_38_5_g,40269268,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 55g,Nestle Munch,55g,Chocolates,nestle_nestle_munch_55g,496297,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 6 x 90 g,Nestle Munch,6 x 90 g,Chocolates,nestle_nestle_munch_6_x_90_g,127096,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Munch 8.9g,Nestle Munch,8.9g,Chocolates,nestle_nestle_munch_8_9g,B01MQEA436,89009802,89009802,GTIN-8,GTIN-8,89009802,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5439541,True,89009802,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nescafe Sunrise 1kg,Nestle Nescafe Sunrise,1kg,Tea & Coffee,nestle_nestle_nescafe_sunrise_1kg,B079H34CLY,8901058902938,8901058902938,EAN-13,EAN-13,8901058902938,8901058902938,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6650498,True,8901058902938,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nescafe Sunrise 200g,Nestle Nescafe Sunrise,200g,Tea & Coffee,nestle_nestle_nescafe_sunrise_200g,B079H34CLY,8901058902938,8901058902938,EAN-13,EAN-13,8901058902938,8901058902938,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6650498,True,8901058902938,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nescafe Sunrise 250g,Nestle Nescafe Sunrise,250g,Tea & Coffee,nestle_nestle_nescafe_sunrise_250g,B079H34CLY,8901058902938,8901058902938,EAN-13,EAN-13,8901058902938,8901058902938,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6650498,True,8901058902938,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nescafe Sunrise 5 g,Nestle Nescafe Sunrise,5 g,Tea & Coffee,nestle_nestle_nescafe_sunrise_5_g,B0971VNDPW,8901058902938,8901058902938,EAN-13,EAN-13,8901058902938,8901058902938,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6650498,True,8901058902938,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nescafe Sunrise 90 g,Nestle Nescafe Sunrise,90 g,Tea & Coffee,nestle_nestle_nescafe_sunrise_90_g,B0971VNDPW,8901058902938,8901058902938,EAN-13,EAN-13,8901058902938,8901058902938,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6650498,True,8901058902938,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 200g,Nestle Nestea,200g,Beverages,nestle_nestle_nestea_200g,402001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 240g,Nestle Nestea,240g,Beverages,nestle_nestle_nestea_240g,NESTLE-NES-240-001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 25g,Nestle Nestea,25g,Beverages,nestle_nestle_nestea_25g,NESTLE-NES-25-001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 33 g,Nestle Nestea,33 g,Beverages,nestle_nestle_nestea_33_g,NESTLE-NES-33-001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 350ml,Nestle Nestea,350ml,Beverages,nestle_nestle_nestea_350ml,NESTLE-NES-350-001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Nestea 750ml,Nestle Nestea,750ml,Beverages,nestle_nestle_nestea_750ml,NESTLE-NES-750-001,8901058869293,8901058869293,EAN-13,EAN-13,8901058869293,8901058869293,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.6030266,True,8901058869293,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 100g,Nestle Polo,100g,Candy & Confectionery,nestle_nestle_polo_100g,B000Q6POKY,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 12g,Nestle Polo,12g,Candy & Confectionery,nestle_nestle_polo_12g,B01FRZ3AGI,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 15g,Nestle Polo,15g,Candy & Confectionery,nestle_nestle_polo_15g,B01FRZ3AGI,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 20g,Nestle Polo,20g,Candy & Confectionery,nestle_nestle_polo_20g,B007C53VSO,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 30g,Nestle Polo,30g,Candy & Confectionery,nestle_nestle_polo_30g,B079TJK8Y3,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
nestle,Nestle,data/seed_catalogs/archive/brand_catalog_nestle.json,archived,Nestle Polo 50g,Nestle Polo,50g,Candy & Confectionery,nestle_nestle_polo_50g,B000Q6POKY,89009871,89009871,GTIN-8,GTIN-8,89009871,,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773811.5981793,True,89009871,in_sync
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico 7Up 1kg,Pepsico 7Up,1kg,Beverages,pepsico_pepsico_7up_1kg,PEPSIC-7UP-1-001,8902080002290,8902080002290,EAN-13,EAN-13,8902080002290,8902080002290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5105531,True,8902080002290,in_sync
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico 7Up 200g,Pepsico 7Up,200g,Beverages,pepsico_pepsico_7up_200g,PEPSIC-7UP-200-001,8902080002290,8902080002290,EAN-13,EAN-13,8902080002290,8902080002290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5105531,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico 7Up 500g,Pepsico 7Up,500g,Beverages,pepsico_pepsico_7up_500g,40211516,8902080002290,8902080002290,EAN-13,EAN-13,8902080002290,8902080002290,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5105531,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Lays 1kg,Pepsico Lays,1kg,Snacks,pepsico_pepsico_lays_1kg,PEPSIC-LAY-1-001,8901491502047,8901491502047,EAN-13,EAN-13,8901491502047,8901491502047,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5016623,True,8901491502047,in_sync
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Lays 200g,Pepsico Lays,200g,Snacks,pepsico_pepsico_lays_200g,PEPSIC-LAY-200-002,8901491502047,8901491502047,EAN-13,EAN-13,8901491502047,8901491502047,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5016623,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Lays 500g,Pepsico Lays,500g,Snacks,pepsico_pepsico_lays_500g,PEPSIC-LAY-500-001,8901491502047,8901491502047,EAN-13,EAN-13,8901491502047,8901491502047,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5016623,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mirinda 100g,Pepsico Mirinda,100g,Beverages,pepsico_pepsico_mirinda_100g,PEPSIC-MIR-100-001,8902080204021,8902080204021,EAN-13,EAN-13,8902080204021,8902080204021,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.517687,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mirinda 1L,Pepsico Mirinda,1L,Beverages,pepsico_pepsico_mirinda_1l,PEPSIC-MIR-1-001,8902080204021,8902080204021,EAN-13,EAN-13,8902080204021,8902080204021,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.517687,True,8902080204021,in_sync
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mirinda 250ml,Pepsico Mirinda,250ml,Beverages,pepsico_pepsico_mirinda_250ml,PEPSIC-MIR-250-001,8902080204021,8902080204021,EAN-13,EAN-13,8902080204021,8902080204021,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.517687,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mountain Dew 100g,Pepsico Mountain Dew,100g,Beverages,pepsico_pepsico_mountain_dew_100g,PEPSIC-MOU-100-002,8902080364022,8902080364022,EAN-13,EAN-13,8902080364022,8902080364022,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5143652,True,,row_not_in_db
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mountain Dew 1L,Pepsico Mountain Dew,1L,Beverages,pepsico_pepsico_mountain_dew_1l,B01LWK1TYZ,8902080364022,8902080364022,EAN-13,EAN-13,8902080364022,8902080364022,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5143652,True,8902080364022,in_sync
|
||||
pepsico,pepsico,data/seed_catalogs/archive/brand_catalog_pepsico.json,archived,Pepsico Mountain Dew 250ml,Pepsico Mountain Dew,250ml,Beverages,pepsico_pepsico_mountain_dew_250ml,B01N2NSWV8,8902080364022,8902080364022,EAN-13,EAN-13,8902080364022,8902080364022,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788773813.5143652,True,,row_not_in_db
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Coffee Classic 2g,Tata Coffee Classic,2g,Tea & Coffee,tata_tata_coffee_classic_2g,TATA-COF-2-002,8901090328109,8901090328109,EAN-13,EAN-13,8901090328109,8901090328109,,Open Food Facts,True,verified,1786092482.6336787,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Coffee Gold 90g,Tata Coffee Gold,90g,Tea & Coffee,tata_tata_coffee_gold_90g,488028,8901090223749,8901090223749,EAN-13,EAN-13,8901090223749,8901090223749,,Open Food Facts,True,verified,1786092504.5926466,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Coffee Grand 180g,Tata Coffee Grand,180g,Tea & Coffee,tata_tata_coffee_grand_180g,TATA-COF-180-001,8903754000826,8903754000826,EAN-13,EAN-13,8903754000826,8903754000826,,Open Food Facts,True,verified,1786092477.0314271,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Coffee Grand 90g,Tata Coffee Grand,90g,Tea & Coffee,tata_tata_coffee_grand_90g,298829,8901090328802,8901090328802,EAN-13,EAN-13,8901090328802,8901090328802,,Open Food Facts,True,verified,1786092476.2067864,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Salt 1 kg,Tata Salt,1 kg,Salt & Staples,tata_tata_salt_1_kg,B07575FPC3,8904043901015,8904043901015,EAN-13,EAN-13,8904043901015,8904043901015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1030667,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Salt 100g,Tata Salt,100g,Salt & Staples,tata_tata_salt_100g,105,8904043901015,8904043901015,EAN-13,EAN-13,8904043901015,8904043901015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1030667,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Salt 250g,Tata Salt,250g,Salt & Staples,tata_tata_salt_250g,TATA-SAL-250-001,8904043901015,8904043901015,EAN-13,EAN-13,8904043901015,8904043901015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1030667,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Salt 500g,Tata Salt,500g,Salt & Staples,tata_tata_salt_500g,105,8904043901015,8904043901015,EAN-13,EAN-13,8904043901015,8904043901015,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1030667,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Chana Dal 1kg,Tata Sampann Chana Dal,1kg,"Pulses, Grains & Spices",tata_tata_sampann_chana_dal_1kg,B07532J31B,8904043926643,8904043926643,EAN-13,EAN-13,8904043926643,8904043926643,,Open Food Facts,True,verified,1786092640.9217572,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Chana Dal 500g,Tata Sampann Chana Dal,500g,"Pulses, Grains & Spices",tata_tata_sampann_chana_dal_500g,B077X8G5DK,8904043926629,8904043926629,EAN-13,EAN-13,8904043926629,8904043926629,,Open Food Facts,True,verified,1786092639.7288995,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Chilli 100g,Tata Sampann Chilli,100g,Spices & Masalas,tata_tata_sampann_chilli_100g,185991,8904043927152,8904043927152,EAN-13,EAN-13,8904043927152,8904043927152,,Open Food Facts,True,verified,1786092578.5348673,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Garam Masala 100 grams,Tata Sampann Garam Masala,100 grams,Spices & Masalas,tata_tata_sampann_garam_masala_100_grams,B079H113LK,8904043927015,8904043927015,EAN-13,EAN-13,8904043927015,8904043927015,,Open Food Facts,True,verified,1786092605.4605205,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Moong Dal 1kg,Tata Sampann Moong Dal,1kg,"Pulses, Grains & Spices",tata_tata_sampann_moong_dal_1kg,B01L1LVGDQ,8904043926315,8904043926315,EAN-13,EAN-13,8904043926315,8904043926315,,Open Food Facts,True,verified,1786092636.9080367,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Poha 1 kg,Tata Sampann Poha,1 kg,"Pulses, Grains & Spices",tata_tata_sampann_poha_1_kg,B09G6JQWL7,8904043904061,8904043904061,EAN-13,EAN-13,8904043904061,8904043904061,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1124144,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Poha 10g,Tata Sampann Poha,10g,"Pulses, Grains & Spices",tata_tata_sampann_poha_10g,480044,8904043904061,8904043904061,EAN-13,EAN-13,8904043904061,8904043904061,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1124144,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Poha 25g,Tata Sampann Poha,25g,"Pulses, Grains & Spices",tata_tata_sampann_poha_25g,B07V3CM9L8,8904043904061,8904043904061,EAN-13,EAN-13,8904043904061,8904043904061,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1124144,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Poha 500g,Tata Sampann Poha,500g,"Pulses, Grains & Spices",tata_tata_sampann_poha_500g,B07V3CM9L8,8904043904061,8904043904061,EAN-13,EAN-13,8904043904061,8904043904061,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1124144,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Poha 50g,Tata Sampann Poha,50g,"Pulses, Grains & Spices",tata_tata_sampann_poha_50g,B07V3CM9L8,8904043904061,8904043904061,EAN-13,EAN-13,8904043904061,8904043904061,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1124144,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Spices 200g,Tata Sampann Spices,200g,"Pulses, Grains & Spices",tata_tata_sampann_spices_200g,40334093,8904043927299,8904043927299,EAN-13,EAN-13,8904043927299,8904043927299,,Open Food Facts,True,verified,1786092404.598989,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Sampann Toor Dal 1kg,Tata Sampann Toor Dal,1kg,"Pulses, Grains & Spices",tata_tata_sampann_toor_dal_1kg,B074N7VHV4,8904043926216,8904043926216,EAN-13,EAN-13,8904043926216,8904043926216,,Open Food Facts,True,verified,1786092638.5300956,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Chakra Gold 250g,Tata Tea Chakra Gold,250g,Tea & Coffee,tata_tata_tea_chakra_gold_250g,297575,8901052005604,8901052005604,EAN-13,EAN-13,8901052005604,8901052005604,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.0995708,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Chakra Gold 6g,Tata Tea Chakra Gold,6g,Tea & Coffee,tata_tata_tea_chakra_gold_6g,57894,8901052005604,8901052005604,EAN-13,EAN-13,8901052005604,8901052005604,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.0995708,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Gold 250g,Tata Tea Gold,250g,Tea & Coffee,tata_tata_tea_gold_250g,254,8901052006243,8901052006243,EAN-13,EAN-13,8901052006243,8901052006243,,Open Food Facts,True,verified,1786092525.6062307,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Gold 500g,Tata Tea Gold,500g,Tea & Coffee,tata_tata_tea_gold_500g,B00XW5HH6U,8901052005161,8901052005161,EAN-13,EAN-13,8901052005161,8901052005161,,Open Food Facts,True,verified,1786092527.1051967,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Premium 1kg,Tata Tea Premium,1kg,Tea & Coffee,tata_tata_tea_premium_1kg,B08DY62Z87,8901052010318,8901052010318,EAN-13,EAN-13,8901052010318,8901052010318,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1740298,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Premium 450g,Tata Tea Premium,450g,Tea & Coffee,tata_tata_tea_premium_450g,B0058PHQYU,8901052010318,8901052010318,EAN-13,EAN-13,8901052010318,8901052010318,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1740298,True,,no_table
|
||||
tata,tata,data/seed_catalogs/brand_catalog_tata.json,active,Tata Tea Premium 50g,Tata Tea Premium,50g,Tea & Coffee,tata_tata_tea_premium_50g,B00AI87X0O,8901052010318,8901052010318,EAN-13,EAN-13,8901052010318,8901052010318,,openfoodfacts_bulk (search.openfoodfacts.org),False,name_matched,1788768773.1740298,True,,no_table
|
||||
|
248
scripts/backfill_barcode_identity.py
Normal file
248
scripts/backfill_barcode_identity.py
Normal file
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fill gtin / ean13 / upc / barcode_type from barcodes the catalog already holds.
|
||||
|
||||
WHY
|
||||
Measured against production on 2026-09-08:
|
||||
|
||||
barcode 300 rows (18.4%)
|
||||
gtin 30 rows (8.7% of the tables that had the column)
|
||||
ean13 22 rows (6.4%)
|
||||
upc 0 rows (0.0%)
|
||||
|
||||
Every one of the missing values is arithmetic on digits already sitting in
|
||||
the same row. Nothing needs to be looked up, matched or fetched. They were
|
||||
empty because the only code that computed them lived inside the network
|
||||
cascade (`ENABLE_BARCODE_LOOKUP`, false in production) and because
|
||||
`upsert_brand_products` dropped the fields before they reached Postgres.
|
||||
|
||||
`BarcodeIdentityStage` now does this for every NEW upload. This script does
|
||||
it once for the rows already stored.
|
||||
|
||||
WHAT IT TOUCHES
|
||||
brand_<slug>.gtin, .ean13, .upc, .barcode_type - and ONLY where they are
|
||||
currently empty. Every UPDATE pins the row's own current barcode in its
|
||||
WHERE clause, so a concurrent write is never lost.
|
||||
|
||||
It also records provenance in `field_sources` under the derived keys, so
|
||||
the coverage report can tell a derived value from a looked-up one.
|
||||
|
||||
WHAT IT WILL NOT DO
|
||||
* It will not change, reformat or delete `barcode`. A barcode that fails
|
||||
checksum validation is REPORTED and skipped - the 38 rows holding the
|
||||
placeholder `8900000000000.0` are found this way, not repaired. Repairing
|
||||
them needs a real source, which is a different job.
|
||||
* It will not overwrite a gtin/ean13/upc that already has a value, even if
|
||||
it disagrees with the barcode. A disagreement is reported instead: it
|
||||
means one of the two is wrong and a script should not pick.
|
||||
* It makes no network request of any kind.
|
||||
|
||||
USAGE
|
||||
python -m scripts.backfill_barcode_identity # dry run
|
||||
python -m scripts.backfill_barcode_identity --brand amul # repeatable
|
||||
python -m scripts.backfill_barcode_identity --apply
|
||||
python -m scripts.backfill_barcode_identity --json
|
||||
|
||||
`--dry-run` is the default and `--apply` must be explicit: backend/.env points
|
||||
at the PRODUCTION database. The target host is printed on startup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.infrastructure.settings import DB_HOST, DB_NAME
|
||||
from app.services.enrichment.barcode.models import BarcodeType
|
||||
from app.services.enrichment.barcode.validators import (
|
||||
classify_barcode_type,
|
||||
to_ean13,
|
||||
validate_barcode,
|
||||
)
|
||||
from app.services.vector_store import _connect
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("backfill_barcode_identity")
|
||||
|
||||
DERIVED = ("gtin", "ean13", "upc", "barcode_type")
|
||||
|
||||
|
||||
def brand_tables(cur, only: Optional[List[str]] = None) -> List[str]:
|
||||
cur.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema = 'public' AND table_name LIKE 'brand\\_%' "
|
||||
"ORDER BY table_name"
|
||||
)
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
if only:
|
||||
wanted = {f"brand_{s.strip().lower().replace(' ', '_').replace('-', '_')}"
|
||||
for s in only}
|
||||
tables = [t for t in tables if t in wanted]
|
||||
return tables
|
||||
|
||||
|
||||
def has_columns(cur, table: str) -> bool:
|
||||
"""Every script here probes information_schema before selecting, because
|
||||
the column set genuinely differed per table until very recently."""
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table,),
|
||||
)
|
||||
present = {r[0] for r in cur.fetchall()}
|
||||
return {"barcode", *DERIVED, "field_sources"} <= present
|
||||
|
||||
|
||||
def derive(barcode: str) -> Optional[Dict[str, Any]]:
|
||||
code = validate_barcode(barcode)
|
||||
if not code:
|
||||
return None
|
||||
kind = classify_barcode_type(code)
|
||||
return {
|
||||
"gtin": code,
|
||||
"ean13": to_ean13(code),
|
||||
"upc": code if kind is BarcodeType.UPC_A else None,
|
||||
"barcode_type": kind.value,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--brand", action="append", dest="brands")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--dry-run", action="store_true", default=False)
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
apply = args.apply and not args.dry_run
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
logger.error("Database unreachable - nothing to do.")
|
||||
return 2
|
||||
|
||||
logger.info("database : %s / %s", DB_HOST, DB_NAME)
|
||||
logger.info("mode : %s", "APPLY (writing)" if apply else "dry run (no writes)")
|
||||
logger.info("")
|
||||
|
||||
tally = Counter()
|
||||
invalid: List[Dict[str, str]] = []
|
||||
conflicts: List[Dict[str, Any]] = []
|
||||
per_brand: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for table in brand_tables(cur, args.brands):
|
||||
if not has_columns(cur, table):
|
||||
tally["tables_skipped_missing_columns"] += 1
|
||||
continue
|
||||
|
||||
cur.execute(
|
||||
f'SELECT id, product_name, barcode, gtin, ean13, upc, barcode_type, '
|
||||
f'field_sources FROM "{table}" '
|
||||
f"WHERE barcode IS NOT NULL AND btrim(barcode) <> ''"
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
written = 0
|
||||
|
||||
for rid, name, barcode, gtin, ean13, upc, btype, sources in rows:
|
||||
tally["barcoded_rows"] += 1
|
||||
derived = derive(barcode)
|
||||
if derived is None:
|
||||
tally["invalid_barcode"] += 1
|
||||
invalid.append({"table": table, "product": name, "barcode": barcode})
|
||||
continue
|
||||
|
||||
current = {"gtin": gtin, "ean13": ean13, "upc": upc, "barcode_type": btype}
|
||||
# Only fill blanks; report a populated value that disagrees.
|
||||
updates = {}
|
||||
for col, want in derived.items():
|
||||
have = current.get(col)
|
||||
if have is None or str(have).strip() == "":
|
||||
if want is not None:
|
||||
updates[col] = want
|
||||
elif str(have).strip() != str(want or "").strip():
|
||||
conflicts.append({"table": table, "product": name, "column": col,
|
||||
"stored": have, "derived": want})
|
||||
if not updates:
|
||||
tally["already_complete"] += 1
|
||||
continue
|
||||
|
||||
merged = dict(sources or {})
|
||||
for col in updates:
|
||||
merged[col] = {"method": "derived",
|
||||
"source": "validators.validate_barcode"}
|
||||
|
||||
tally["rows_to_update"] += 1
|
||||
for col in updates:
|
||||
tally[f"fill_{col}"] += 1
|
||||
written += 1
|
||||
|
||||
if apply:
|
||||
assignments = ", ".join(f"{c} = %s" for c in updates)
|
||||
cur.execute(
|
||||
f'UPDATE "{table}" SET {assignments}, field_sources = %s, '
|
||||
f"updated_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id = %s AND barcode = %s",
|
||||
(*updates.values(), Json(merged), rid, barcode),
|
||||
)
|
||||
|
||||
if written:
|
||||
per_brand[table] = written
|
||||
|
||||
if apply:
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({"applied": apply, "tally": dict(tally),
|
||||
"per_brand": per_brand, "invalid": invalid,
|
||||
"conflicts": conflicts}, indent=2))
|
||||
return 0
|
||||
|
||||
for table, n in sorted(per_brand.items(), key=lambda kv: -kv[1]):
|
||||
logger.info(" %-32s %d row(s)", table, n)
|
||||
logger.info("")
|
||||
logger.info("barcoded rows %d", tally["barcoded_rows"])
|
||||
logger.info(" not a valid GTIN %d", tally["invalid_barcode"])
|
||||
logger.info(" already complete %d", tally["already_complete"])
|
||||
logger.info(" %s %d",
|
||||
"updated" if apply else "to update", tally["rows_to_update"])
|
||||
for col in DERIVED:
|
||||
logger.info(" %-14s %d", col, tally[f"fill_{col}"])
|
||||
|
||||
if invalid:
|
||||
logger.info("")
|
||||
logger.info("%d row(s) hold something that is not a barcode (left untouched):",
|
||||
len(invalid))
|
||||
for item in invalid[:10]:
|
||||
logger.info(" %-28s %s", item["product"][:28], item["barcode"])
|
||||
if len(invalid) > 10:
|
||||
logger.info(" ... and %d more", len(invalid) - 10)
|
||||
|
||||
if conflicts:
|
||||
logger.info("")
|
||||
logger.info("%d stored value(s) DISAGREE with the barcode (left untouched, "
|
||||
"one of the two is wrong):", len(conflicts))
|
||||
for c in conflicts[:10]:
|
||||
logger.info(" %-28s %s stored=%s derived=%s",
|
||||
c["product"][:28], c["column"], c["stored"], c["derived"])
|
||||
|
||||
if not apply and tally["rows_to_update"]:
|
||||
logger.info("")
|
||||
logger.info("Dry run - nothing written. Re-run with --apply to commit.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -218,8 +218,16 @@ def main() -> int:
|
||||
candidate_brand=product.get("brands") or "",
|
||||
candidate_size=product.get("quantity") or "",
|
||||
)
|
||||
# barcode_is_identity mirrors what fetch_verified_nutrition_by_barcode
|
||||
# passes, and it has to: this gate runs FIRST, so without it the row is
|
||||
# rejected here and the service's relaxed check is never reached. That
|
||||
# is exactly what happened - a run on 2026-09-08 reported 149 of 300
|
||||
# rows as "found, wrong product" where the barcode had resolved
|
||||
# perfectly and OFF simply stores the short name ("Munch" for our
|
||||
# "Nestle Munch 8.9g"). See matching.name_is_contained.
|
||||
matched, _sim = is_match(candidate, row["brand"], row["product_name"],
|
||||
row["size"], min_name_similarity=args.min_similarity)
|
||||
row["size"], min_name_similarity=args.min_similarity,
|
||||
barcode_is_identity=True)
|
||||
if not matched:
|
||||
rejected.append(line)
|
||||
time.sleep(PAUSE_SECONDS)
|
||||
|
||||
212
scripts/backfill_offline_fields.py
Normal file
212
scripts/backfill_offline_fields.py
Normal file
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fill the fields that need no network: FSSAI licences, and not-applicable marks.
|
||||
|
||||
WHY
|
||||
Two separate gaps, both closable from data already on this machine.
|
||||
|
||||
1. FSSAI licences on brands that HAVE one.
|
||||
Measured 2026-09-08: 264 rows have no `fssai_license`. Of those, 185
|
||||
belong to brands with a curated licence in
|
||||
`brand_registry.FSSAI_LICENSES` - Amul 10, HUL 82, MTR 39, Dabur 17,
|
||||
CavinKare 17, Nestle 16, Kaleesuwari 4. The map knows the answer; the
|
||||
rows were written before stage 1 filled it, or by a path that skipped it.
|
||||
|
||||
Consensus over a brand's own rows was the other candidate mechanism and
|
||||
it fills ZERO of these - every brand with blanks either already has a
|
||||
mapping or has no populated row to learn from. It is left in place for
|
||||
future uploads into an established brand, but it is not what closes this.
|
||||
|
||||
2. Marking what cannot apply.
|
||||
Roughly 30% of the catalog is shampoo, soap, detergent and toothpaste.
|
||||
Those rows will never have nutrients, a health score or an FSSAI FOOD
|
||||
licence, and a coverage report that counts them as "missing" shows a
|
||||
permanent red number - which is exactly the pressure that eventually
|
||||
gets it "fixed" by inventing values. This writes an explicit
|
||||
`not_applicable` into `field_sources` so the report can exclude them
|
||||
honestly.
|
||||
|
||||
WHAT IT TOUCHES
|
||||
* brand_<slug>.fssai_license - ONLY where blank, and ONLY from
|
||||
`FSSAI_LICENSES`. Never from another brand, never a constant.
|
||||
* brand_<slug>.field_sources - the provenance record for both operations.
|
||||
|
||||
WHAT IT WILL NOT DO
|
||||
* It will not invent an FSSAI number. A brand absent from the curated map
|
||||
gets nothing. `10012042000244` is Lion Dates' real licence and the reason
|
||||
this rule is written down - see tests/test_no_fabricated_identifiers.py.
|
||||
* It will not overwrite an existing licence, even one that disagrees with
|
||||
the map. A disagreement is reported instead.
|
||||
* It makes no network request.
|
||||
|
||||
USAGE
|
||||
python -m scripts.backfill_offline_fields # dry run
|
||||
python -m scripts.backfill_offline_fields --apply
|
||||
python -m scripts.backfill_offline_fields --json
|
||||
|
||||
`--dry-run` is the default; backend/.env points at PRODUCTION.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from app.infrastructure.settings import DB_HOST, DB_NAME
|
||||
from app.services.brand_registry import get_fssai_license
|
||||
from app.services.consumability import is_non_consumable
|
||||
from app.services.vector_store import _connect
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("backfill_offline_fields")
|
||||
|
||||
# Columns a non-consumable row can never have a value for.
|
||||
NON_FOOD_NA = ("nutrients", "nutrients_per_100g", "nutrition_score",
|
||||
"health_score", "fssai_license")
|
||||
|
||||
|
||||
def brand_tables(cur, only: Optional[List[str]] = None) -> List[str]:
|
||||
cur.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema = 'public' AND table_name LIKE 'brand\\_%' "
|
||||
"AND table_name <> 'brand_zzsmoketest' ORDER BY table_name"
|
||||
)
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
if only:
|
||||
wanted = {f"brand_{s.strip().lower().replace(' ', '_').replace('-', '_')}"
|
||||
for s in only}
|
||||
tables = [t for t in tables if t in wanted]
|
||||
return tables
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--brand", action="append", dest="brands")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--dry-run", action="store_true", default=False)
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
apply = args.apply and not args.dry_run
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
logger.error("Database unreachable.")
|
||||
return 2
|
||||
|
||||
logger.info("database : %s / %s", DB_HOST, DB_NAME)
|
||||
logger.info("mode : %s", "APPLY (writing)" if apply else "dry run (no writes)")
|
||||
logger.info("")
|
||||
|
||||
tally = Counter()
|
||||
per_brand: Dict[str, Dict[str, int]] = {}
|
||||
disagreements: List[Dict[str, str]] = []
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for table in brand_tables(cur, args.brands):
|
||||
display = table[len("brand_"):].replace("_", " ")
|
||||
mapped = get_fssai_license(display)
|
||||
|
||||
cur.execute(
|
||||
f'SELECT id, product_name, category, fssai_license, field_sources '
|
||||
f'FROM "{table}"'
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
counts = Counter()
|
||||
|
||||
for rid, name, category, licence, sources in rows:
|
||||
merged = dict(sources or {})
|
||||
updates: Dict[str, Any] = {}
|
||||
|
||||
non_food = is_non_consumable(category or "", name or "")
|
||||
|
||||
if non_food:
|
||||
for column in NON_FOOD_NA:
|
||||
if merged.get(column, {}).get("method") != "not_applicable":
|
||||
merged[column] = {"method": "not_applicable",
|
||||
"source": "non_consumable_product"}
|
||||
counts["marked_not_applicable"] += 1
|
||||
elif not (licence or "").strip():
|
||||
if mapped:
|
||||
updates["fssai_license"] = mapped
|
||||
merged["fssai_license"] = {"method": "sourced",
|
||||
"source": "brand_registry"}
|
||||
counts["fssai_filled"] += 1
|
||||
else:
|
||||
merged["fssai_license"] = {"method": "unknown",
|
||||
"source": "no_registry_entry"}
|
||||
counts["fssai_unknown"] += 1
|
||||
elif mapped and licence.strip() != mapped:
|
||||
disagreements.append({"table": table, "product": name,
|
||||
"stored": licence, "registry": mapped})
|
||||
counts["fssai_disagrees"] += 1
|
||||
|
||||
if merged == (sources or {}) and not updates:
|
||||
continue
|
||||
|
||||
if apply:
|
||||
if updates:
|
||||
cur.execute(
|
||||
f'UPDATE "{table}" SET fssai_license = %s, '
|
||||
f"field_sources = %s, updated_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id = %s AND (fssai_license IS NULL "
|
||||
f" OR btrim(fssai_license) = '')",
|
||||
(updates["fssai_license"], Json(merged), rid),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
f'UPDATE "{table}" SET field_sources = %s, '
|
||||
f"updated_at = CURRENT_TIMESTAMP WHERE id = %s",
|
||||
(Json(merged), rid),
|
||||
)
|
||||
|
||||
if counts:
|
||||
per_brand[table] = dict(counts)
|
||||
tally.update(counts)
|
||||
|
||||
if apply:
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({"applied": apply, "tally": dict(tally),
|
||||
"per_brand": per_brand,
|
||||
"disagreements": disagreements}, indent=2))
|
||||
return 0
|
||||
|
||||
for table, counts in sorted(per_brand.items(),
|
||||
key=lambda kv: -sum(kv[1].values())):
|
||||
parts = ", ".join(f"{k.replace('_', ' ')} {v}" for k, v in sorted(counts.items()))
|
||||
logger.info(" %-30s %s", table, parts)
|
||||
|
||||
logger.info("")
|
||||
logger.info("fssai filled from the brand registry %d", tally["fssai_filled"])
|
||||
logger.info("fssai left blank, no registry entry %d", tally["fssai_unknown"])
|
||||
logger.info("rows marked not-applicable (non-food) %d", tally["marked_not_applicable"])
|
||||
if disagreements:
|
||||
logger.info("")
|
||||
logger.info("%d row(s) hold a licence that DISAGREES with the registry "
|
||||
"(left untouched):", len(disagreements))
|
||||
for d in disagreements[:10]:
|
||||
logger.info(" %-28s stored=%s registry=%s",
|
||||
d["product"][:28], d["stored"], d["registry"])
|
||||
|
||||
if not apply and sum(tally.values()):
|
||||
logger.info("")
|
||||
logger.info("Dry run - nothing written. Re-run with --apply to commit.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
279
scripts/catalog_coverage.py
Normal file
279
scripts/catalog_coverage.py
Normal file
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
How completely is the catalog filled, and where did each value come from?
|
||||
|
||||
WHY
|
||||
"Fill every column" is not actually the goal, and a report that treats it
|
||||
as one produces a permanent, unfixable red number that somebody eventually
|
||||
"fixes" by inventing data. Three of these columns can never be filled for
|
||||
large parts of the catalog, and that is correct:
|
||||
|
||||
* `upc` - every barcode here is GS1 India (prefix 890), which issues
|
||||
EAN-13 and GTIN-8. UPC-A is a North American symbology. Measured: 0 of
|
||||
300 barcodes are UPC-A, and none ever will be.
|
||||
* `nutrients` / `health_score` - roughly 30% of rows are shampoo, soap,
|
||||
detergent and toothpaste. Soap has no protein content.
|
||||
* `fssai_license` - an FSSAI licence covers a FOOD business. P&G,
|
||||
Colgate-Palmolive and Reckitt Benckiser should not carry one.
|
||||
|
||||
So this report counts four states, not two:
|
||||
|
||||
sourced a real value from a real source
|
||||
derived computed from another field we hold (gtin from barcode)
|
||||
estimated a category-level or consensus guess, flagged as such
|
||||
not applicable cannot exist for this row, and should not
|
||||
MISSING we have not got it yet - the only number worth chasing
|
||||
|
||||
Coverage percentages are taken against the APPLICABLE denominator, so the
|
||||
numbers describe work remaining rather than work impossible.
|
||||
|
||||
WHAT IT TOUCHES
|
||||
Nothing. Every statement is a SELECT. There is no --apply because there is
|
||||
nothing to apply.
|
||||
|
||||
USAGE
|
||||
python -m scripts.catalog_coverage
|
||||
python -m scripts.catalog_coverage --brand amul --brand cadbury
|
||||
python -m scripts.catalog_coverage --column barcode --column nutrients
|
||||
python -m scripts.catalog_coverage --json > coverage.json
|
||||
python -m scripts.catalog_coverage --by-provenance
|
||||
|
||||
Run it before and after any enrichment change: the diff of two --json runs is
|
||||
the evidence that the change did what it claimed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.infrastructure.settings import DB_HOST, DB_NAME
|
||||
from app.services.vector_store import _connect
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("catalog_coverage")
|
||||
|
||||
|
||||
# Columns worth reporting on, in the order a reader wants them.
|
||||
TRACKED = [
|
||||
"product_name", "title", "description", "category", "image_url",
|
||||
"price_range", "size_variants", "providers", "highlights",
|
||||
"fssai_license", "product_sku", "hsn_code", "gst_percent", "tax_amount",
|
||||
"selling_price", "final_selling_price",
|
||||
"barcode", "barcode_type", "gtin", "ean13", "upc",
|
||||
"nutrients", "nutrients_per_100g", "nutrition_score", "health_score",
|
||||
]
|
||||
|
||||
# Columns that simply cannot apply to some rows, and the rule for which.
|
||||
#
|
||||
# food_only - meaningless for a non-consumable product
|
||||
# india_only - UPC-A does not occur in a GS1 India catalog
|
||||
# barcoded - derived from a barcode, so absent when the barcode is
|
||||
NOT_APPLICABLE_RULES = {
|
||||
"nutrients": "food_only",
|
||||
"nutrients_per_100g": "food_only",
|
||||
"nutrition_score": "food_only",
|
||||
"health_score": "food_only",
|
||||
"fssai_license": "food_only",
|
||||
"upc": "india_only",
|
||||
"gtin": "barcoded",
|
||||
"ean13": "barcoded",
|
||||
"barcode_type": "barcoded",
|
||||
}
|
||||
|
||||
# A value that is present but means "nothing here".
|
||||
EMPTY_LITERALS = ("", "[]", "{}", "null", "0", "Uncategorized")
|
||||
|
||||
|
||||
def brand_tables(cur, only: Optional[List[str]] = None) -> List[str]:
|
||||
cur.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema = 'public' AND table_name LIKE 'brand\\_%' "
|
||||
"AND table_name <> 'brand_zzsmoketest' ORDER BY table_name"
|
||||
)
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
if only:
|
||||
wanted = {f"brand_{s.strip().lower().replace(' ', '_').replace('-', '_')}"
|
||||
for s in only}
|
||||
tables = [t for t in tables if t in wanted]
|
||||
return tables
|
||||
|
||||
|
||||
def columns_of(cur, table: str) -> Set[str]:
|
||||
"""Probed per table rather than assumed. The column set genuinely differed
|
||||
per table until the schema migration, and a script that assumes otherwise
|
||||
dies on the first old table it meets."""
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table,),
|
||||
)
|
||||
return {r[0] for r in cur.fetchall()}
|
||||
|
||||
|
||||
def _is_non_food(category: str) -> bool:
|
||||
from app.services.consumability import is_non_consumable
|
||||
try:
|
||||
return bool(is_non_consumable(category or "", ""))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def scan(cur, table: str, wanted: List[str]) -> Dict[str, Dict[str, int]]:
|
||||
present = columns_of(cur, table)
|
||||
cols = [c for c in wanted if c in present]
|
||||
if not cols:
|
||||
return {}
|
||||
|
||||
select = ", ".join(f'"{c}"' for c in cols)
|
||||
extra = ', "category"' if "category" in present else ""
|
||||
barcode_idx = cols.index("barcode") if "barcode" in cols else None
|
||||
|
||||
cur.execute(f'SELECT {select}{extra} FROM "{table}"')
|
||||
rows = cur.fetchall()
|
||||
|
||||
stats: Dict[str, Dict[str, int]] = {
|
||||
c: {"rows": 0, "filled": 0, "not_applicable": 0} for c in cols
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
category = row[len(cols)] if extra else ""
|
||||
non_food = _is_non_food(category)
|
||||
has_barcode = bool(barcode_idx is not None and row[barcode_idx])
|
||||
|
||||
for i, col in enumerate(cols):
|
||||
s = stats[col]
|
||||
s["rows"] += 1
|
||||
|
||||
rule = NOT_APPLICABLE_RULES.get(col)
|
||||
if ((rule == "food_only" and non_food)
|
||||
or (rule == "india_only")
|
||||
or (rule == "barcoded" and not has_barcode)):
|
||||
s["not_applicable"] += 1
|
||||
continue
|
||||
|
||||
value = row[i]
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, (list, tuple, dict)) and not value:
|
||||
continue
|
||||
if isinstance(value, str) and value.strip() in EMPTY_LITERALS:
|
||||
continue
|
||||
s["filled"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def provenance(cur, table: str) -> Dict[str, Dict[str, int]]:
|
||||
"""How each filled value was arrived at, read from `field_sources`."""
|
||||
if "field_sources" not in columns_of(cur, table):
|
||||
return {}
|
||||
cur.execute(f'SELECT field_sources FROM "{table}" '
|
||||
f"WHERE field_sources IS NOT NULL AND field_sources <> '{{}}'::jsonb")
|
||||
out: Dict[str, Dict[str, int]] = {}
|
||||
for (blob,) in cur.fetchall():
|
||||
for column, record in (blob or {}).items():
|
||||
method = (record or {}).get("method", "unspecified")
|
||||
out.setdefault(column, {}).setdefault(method, 0)
|
||||
out[column][method] += 1
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--brand", action="append", dest="brands")
|
||||
ap.add_argument("--column", action="append", dest="columns")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
ap.add_argument("--by-provenance", action="store_true",
|
||||
help="also break filled values down by how they were obtained")
|
||||
args = ap.parse_args()
|
||||
|
||||
wanted = args.columns or TRACKED
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
logger.error("Database unreachable.")
|
||||
return 2
|
||||
|
||||
totals: Dict[str, Dict[str, int]] = {c: {"rows": 0, "filled": 0, "not_applicable": 0}
|
||||
for c in wanted}
|
||||
per_brand: Dict[str, Any] = {}
|
||||
prov_totals: Dict[str, Dict[str, int]] = {}
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
tables = brand_tables(cur, args.brands)
|
||||
for table in tables:
|
||||
stats = scan(cur, table, wanted)
|
||||
if not stats:
|
||||
continue
|
||||
per_brand[table] = stats
|
||||
for col, s in stats.items():
|
||||
for k in ("rows", "filled", "not_applicable"):
|
||||
totals[col][k] += s[k]
|
||||
|
||||
if args.by_provenance:
|
||||
for col, methods in provenance(cur, table).items():
|
||||
for method, n in methods.items():
|
||||
prov_totals.setdefault(col, {}).setdefault(method, 0)
|
||||
prov_totals[col][method] += n
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def applicable(s):
|
||||
return s["rows"] - s["not_applicable"]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({
|
||||
"database": f"{DB_HOST}/{DB_NAME}",
|
||||
"totals": {c: {**s, "applicable": applicable(s),
|
||||
"pct": round(100 * s["filled"] / applicable(s), 1)
|
||||
if applicable(s) else None}
|
||||
for c, s in totals.items() if s["rows"]},
|
||||
"provenance": prov_totals,
|
||||
"per_brand": per_brand,
|
||||
}, indent=2))
|
||||
return 0
|
||||
|
||||
row_count = max((s["rows"] for s in totals.values()), default=0)
|
||||
logger.info("database : %s / %s", DB_HOST, DB_NAME)
|
||||
logger.info("%d product rows across %d brand tables", row_count, len(per_brand))
|
||||
logger.info("")
|
||||
logger.info("%-22s %8s %8s %7s %s", "column", "filled", "of", "pct", "not applicable")
|
||||
logger.info("%s", "-" * 72)
|
||||
|
||||
for col in wanted:
|
||||
s = totals.get(col)
|
||||
if not s or not s["rows"]:
|
||||
continue
|
||||
app_n = applicable(s)
|
||||
pct = f'{100 * s["filled"] / app_n:5.1f}%' if app_n else " -"
|
||||
na = f'{s["not_applicable"]:d}' if s["not_applicable"] else ""
|
||||
flag = ""
|
||||
if app_n and s["filled"] < app_n:
|
||||
flag = f' <- {app_n - s["filled"]} missing'
|
||||
logger.info("%-22s %8d %8d %7s %-6s%s", col, s["filled"], app_n, pct, na, flag)
|
||||
|
||||
if args.by_provenance and prov_totals:
|
||||
logger.info("")
|
||||
logger.info("provenance of filled values (from field_sources)")
|
||||
logger.info("%s", "-" * 72)
|
||||
for col in sorted(prov_totals):
|
||||
methods = ", ".join(f"{m} {n}" for m, n in
|
||||
sorted(prov_totals[col].items(), key=lambda kv: -kv[1]))
|
||||
logger.info("%-22s %s", col, methods)
|
||||
elif args.by_provenance:
|
||||
logger.info("")
|
||||
logger.info("No field_sources recorded yet - run an enrichment pass first.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
257
scripts/migrate_brand_schema.py
Normal file
257
scripts/migrate_brand_schema.py
Normal file
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bring every brand table up to the current schema.
|
||||
|
||||
WHY
|
||||
`vector_store._ensure_columns` is the only migration mechanism in this
|
||||
repo - there is no Alembic and no migrations directory - and it runs only
|
||||
as a side effect of `ensure_brand_schema`, i.e. on the next WRITE to a
|
||||
table. A brand nobody uploads to therefore never gets a new column.
|
||||
|
||||
Measured on 2026-09-08, before the enrichment columns landed: seven of
|
||||
fifty-six brand tables carried `gtin` / `ean13` / `upc` /
|
||||
`barcode_source` / `barcode_verified` / `barcode_lookup_status` /
|
||||
`barcode_last_updated`, all added out-of-band by hand. The other
|
||||
forty-nine did not, and no code path would ever have added them. This
|
||||
script closes that gap deliberately instead of waiting for a write that
|
||||
may never come.
|
||||
|
||||
WHAT IT TOUCHES
|
||||
* `ALTER TABLE brand_<slug> ADD COLUMN IF NOT EXISTS ...` for every column
|
||||
in `_ensure_columns`' `col_defs` that the table does not already have.
|
||||
* The retroactive UNIQUE index on `image_id`, and the DROP NOT NULL sweep
|
||||
over legacy columns - both are part of `_ensure_columns` and cannot be
|
||||
run separately.
|
||||
|
||||
Nothing else. No row is read, updated or deleted by this script.
|
||||
|
||||
WHAT IT WILL NOT DO
|
||||
* It will not change the type of a column that already exists.
|
||||
`ADD COLUMN IF NOT EXISTS` skips a column that is present, whatever its
|
||||
type. This is deliberate: the seven hand-migrated tables define the
|
||||
types the rest must match, which is why `col_defs` says TIMESTAMP for
|
||||
`barcode_last_updated` and REAL for the tax figures rather than the
|
||||
types those values look like they want. Type drift is REPORTED here,
|
||||
never silently "fixed".
|
||||
* It will not create a brand table that does not exist.
|
||||
* It will not touch `nutrition_facts` or any non-brand table.
|
||||
|
||||
WHY IT IS SAFE ON A LIVE DATABASE
|
||||
`ADD COLUMN` with no DEFAULT and no NOT NULL is a catalogue-only change in
|
||||
PostgreSQL 11+: no table rewrite, no full-table lock, no time proportional
|
||||
to row count. On 1 630 rows across 56 tables this is milliseconds. The one
|
||||
exception is `field_sources`, which does carry a DEFAULT - and since
|
||||
PostgreSQL 11 a non-volatile default is also metadata-only.
|
||||
|
||||
USAGE
|
||||
python -m scripts.migrate_brand_schema # dry run, all brands
|
||||
python -m scripts.migrate_brand_schema --brand amul # repeatable
|
||||
python -m scripts.migrate_brand_schema --apply
|
||||
python -m scripts.migrate_brand_schema --json
|
||||
|
||||
`--dry-run` is the default and `--apply` must be explicit: backend/.env points
|
||||
at the PRODUCTION database, so an accidental run must not be able to write.
|
||||
The target host is printed on startup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.infrastructure.settings import DB_HOST, DB_NAME
|
||||
|
||||
from app.services.vector_store import _connect, _ensure_columns
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("migrate_brand_schema")
|
||||
|
||||
|
||||
def brand_tables(cur, only: Optional[List[str]] = None) -> List[str]:
|
||||
"""Every brand table actually present, in name order."""
|
||||
cur.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema = 'public' AND table_name LIKE 'brand\\_%' "
|
||||
"ORDER BY table_name"
|
||||
)
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
if only:
|
||||
wanted = {f"brand_{s.strip().lower().replace(' ', '_').replace('-', '_')}"
|
||||
for s in only}
|
||||
tables = [t for t in tables if t in wanted]
|
||||
return tables
|
||||
|
||||
|
||||
def existing_columns(cur, table: str) -> Dict[str, str]:
|
||||
cur.execute(
|
||||
"SELECT column_name, data_type FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = %s",
|
||||
(table,),
|
||||
)
|
||||
return {r[0]: r[1] for r in cur.fetchall()}
|
||||
|
||||
|
||||
# What `information_schema.data_type` reports for each col_defs type, so a
|
||||
# type-drift check does not raise false alarms on spelling differences.
|
||||
_TYPE_ALIASES = {
|
||||
"TEXT": {"text"},
|
||||
"TEXT[]": {"ARRAY"},
|
||||
"NUMERIC": {"numeric"},
|
||||
"REAL": {"real"},
|
||||
"BOOLEAN": {"boolean"},
|
||||
"TIMESTAMP": {"timestamp without time zone"},
|
||||
"JSONB": {"jsonb"},
|
||||
"DOUBLE PRECISION": {"double precision"},
|
||||
"vector(384)": {"USER-DEFINED"},
|
||||
}
|
||||
|
||||
|
||||
class RecordingCursor:
|
||||
"""Wraps a real cursor so a dry run can see the statements without
|
||||
executing them. Reads are passed through - the whole point is to compute
|
||||
the diff against what is really on the table."""
|
||||
|
||||
def __init__(self, inner):
|
||||
self._inner = inner
|
||||
self.statements: List[str] = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
text = " ".join(str(sql).split())
|
||||
upper = text.upper()
|
||||
if upper.startswith("SELECT"):
|
||||
return self._inner.execute(sql, params)
|
||||
self.statements.append(text)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return self._inner.fetchall()
|
||||
|
||||
def fetchone(self):
|
||||
return self._inner.fetchone()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--brand", action="append", dest="brands",
|
||||
help="brand table suffix; repeatable. Default: every brand table.")
|
||||
ap.add_argument("--apply", action="store_true", help="actually run the ALTERs")
|
||||
ap.add_argument("--dry-run", action="store_true", default=False,
|
||||
help="report only (the default)")
|
||||
ap.add_argument("--json", action="store_true", help="machine-readable output")
|
||||
args = ap.parse_args()
|
||||
|
||||
apply = args.apply and not args.dry_run
|
||||
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
logger.error("Database unreachable - nothing to do.")
|
||||
return 2
|
||||
|
||||
logger.info("database : %s / %s", DB_HOST, DB_NAME)
|
||||
logger.info("mode : %s", "APPLY (writing)" if apply else "dry run (no writes)")
|
||||
logger.info("")
|
||||
|
||||
declared_types = _declared_types()
|
||||
if not declared_types:
|
||||
logger.error("Could not read col_defs out of _ensure_columns - refusing to "
|
||||
"guess at the schema. Has that function been restructured?")
|
||||
return 2
|
||||
|
||||
report: List[Dict[str, Any]] = []
|
||||
total_missing = 0
|
||||
total_drift = 0
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
tables = brand_tables(cur, args.brands)
|
||||
if not tables:
|
||||
logger.error("No brand tables matched.")
|
||||
return 1
|
||||
|
||||
for table in tables:
|
||||
before = existing_columns(cur, table)
|
||||
|
||||
recorder = RecordingCursor(cur)
|
||||
_ensure_columns(recorder, table)
|
||||
adds = [s for s in recorder.statements if "ADD COLUMN" in s]
|
||||
|
||||
# Type drift: a column that exists but whose type is not what
|
||||
# col_defs would have created. Reported, never altered.
|
||||
drift = []
|
||||
for col, declared in declared_types.items():
|
||||
actual = before.get(col)
|
||||
if actual is None:
|
||||
continue
|
||||
allowed = _TYPE_ALIASES.get(declared.upper(), set())
|
||||
if allowed and actual not in allowed:
|
||||
drift.append({"column": col, "declared": declared, "actual": actual})
|
||||
|
||||
entry = {
|
||||
"table": table,
|
||||
"missing_columns": [s.split("ADD COLUMN IF NOT EXISTS ")[1] for s in adds],
|
||||
"type_drift": drift,
|
||||
}
|
||||
report.append(entry)
|
||||
total_missing += len(adds)
|
||||
total_drift += len(drift)
|
||||
|
||||
if apply and adds:
|
||||
_ensure_columns(cur, table)
|
||||
|
||||
if apply:
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({"applied": apply, "tables": report}, indent=2))
|
||||
return 0
|
||||
|
||||
width = max(len(e["table"]) for e in report)
|
||||
changed = [e for e in report if e["missing_columns"] or e["type_drift"]]
|
||||
for entry in sorted(changed, key=lambda e: -len(e["missing_columns"])):
|
||||
logger.info("%-*s %d column(s) missing", width, entry["table"],
|
||||
len(entry["missing_columns"]))
|
||||
for col in entry["missing_columns"]:
|
||||
logger.info("%-*s + %s", width, "", col)
|
||||
for d in entry["type_drift"]:
|
||||
logger.info("%-*s ! %s is %s, col_defs declares %s (NOT changed)",
|
||||
width, "", d["column"], d["actual"], d["declared"])
|
||||
|
||||
logger.info("")
|
||||
logger.info("%d table(s) scanned, %d already current",
|
||||
len(report), len(report) - len(changed))
|
||||
logger.info("%d column(s) %s, %d type mismatch(es) reported",
|
||||
total_missing, "added" if apply else "would be added", total_drift)
|
||||
if not apply and total_missing:
|
||||
logger.info("")
|
||||
logger.info("Re-run with --apply to write these changes.")
|
||||
return 0
|
||||
|
||||
|
||||
def _declared_types() -> Dict[str, str]:
|
||||
"""The col_defs dict, read back out of the function that owns it.
|
||||
|
||||
Parsed from source rather than duplicated here, so this script cannot
|
||||
drift from the single migration mechanism it exists to drive.
|
||||
"""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(_ensure_columns)))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign) and getattr(node.targets[0], "id", "") == "col_defs":
|
||||
return {ast.literal_eval(k): ast.literal_eval(v)
|
||||
for k, v in zip(node.value.keys, node.value.values)}
|
||||
return {}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
198
tests/test_barcode_identity_stage.py
Normal file
198
tests/test_barcode_identity_stage.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""The offline stage that expands a barcode into the rest of its identity.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
Measured against the production database on 2026-09-08:
|
||||
|
||||
barcode 18.4% filled
|
||||
gtin 8.7%
|
||||
ean13 6.4%
|
||||
upc 0.0%
|
||||
|
||||
Every one of those three could have been computed from the barcode already
|
||||
sitting in the same row - they are arithmetic on the digits, not a lookup. They
|
||||
were empty because the only code that produced them lived inside
|
||||
`BarcodeEnrichmentStage`, which is off by default (`ENABLE_BARCODE_LOOKUP`,
|
||||
false in production), and because the writer dropped the fields anyway.
|
||||
|
||||
`BarcodeIdentityStage` closes that. It performs NO lookup, so it needs no
|
||||
settings flag and costs nothing, and it runs on every ingestion.
|
||||
|
||||
The three properties that matter, each pinned below:
|
||||
|
||||
1. It expands a valid barcode into barcode_type / gtin / ean13 / upc.
|
||||
2. It NEVER touches a barcode that fails validation. A sheet-supplied barcode
|
||||
is the merchant's assertion; silently "correcting" or deleting one would be
|
||||
worse than leaving it visibly wrong. The failure goes into `field_sources`,
|
||||
not into the data.
|
||||
3. It never claims a value is "verified". That word is reserved for the
|
||||
cascade's brand+size+name-matched result, and a barcode typed into a
|
||||
spreadsheet has passed no such check.
|
||||
|
||||
No network and no database: the stage has neither.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.enrichment.barcode.identity_stage import BarcodeIdentityStage
|
||||
|
||||
|
||||
def run(product, brand="Cadbury"):
|
||||
"""Apply the stage the way EnrichmentPipeline does, returning the row."""
|
||||
stage = BarcodeIdentityStage()
|
||||
return asyncio.run(stage.apply(dict(product), brand))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Expansion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_an_ean13_expands_into_gtin_and_ean13():
|
||||
"""8901233018362 is Cadbury Bournvita's real barcode - the one that had to
|
||||
be repaired in production by hand on 2026-09-08."""
|
||||
row = run({"product_name": "Cadbury Bournvita 500g", "barcode": "8901233018362"})
|
||||
|
||||
assert row["barcode"] == "8901233018362"
|
||||
assert row["barcode_type"] == "EAN-13"
|
||||
assert row["gtin"] == "8901233018362"
|
||||
assert row["ean13"] == "8901233018362"
|
||||
assert row["upc"] is None # a 13-digit code is not a UPC-A
|
||||
|
||||
|
||||
def test_a_upc_a_expands_into_both_upc_and_a_padded_ean13():
|
||||
"""UPC-A is numerically a GTIN-13 with a leading zero, so both fields are
|
||||
real for the same pack - the zero-padded form is what an EAN-13 scanner
|
||||
reports."""
|
||||
row = run({"product_name": "Imported Bar 50g", "barcode": "036000291452"})
|
||||
|
||||
assert row["barcode_type"] == "UPC-A"
|
||||
assert row["upc"] == "036000291452"
|
||||
assert row["ean13"] == "0036000291452"
|
||||
assert row["gtin"] == "036000291452"
|
||||
|
||||
|
||||
def test_a_gtin8_is_not_padded_into_an_ean13():
|
||||
"""An 8-digit GTIN is its own symbology, not a truncated EAN-13. Padding it
|
||||
would invent a code that identifies nothing. 89009802 is the real GTIN-8
|
||||
Open Food Facts holds for Nestle Munch."""
|
||||
row = run({"product_name": "Nestle Munch 8.9g", "barcode": "89009802"})
|
||||
|
||||
assert row["barcode_type"] == "GTIN-8"
|
||||
assert row["gtin"] == "89009802"
|
||||
assert row["ean13"] is None
|
||||
assert row["upc"] is None
|
||||
|
||||
|
||||
def test_separators_are_stripped_but_the_value_is_not_otherwise_changed():
|
||||
row = run({"product_name": "Amul Butter 100g", "barcode": " 8901262-010016 "})
|
||||
|
||||
assert row["barcode"] == "8901262010016"
|
||||
assert row["gtin"] == "8901262010016"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. It never damages what the merchant supplied
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_an_invalid_barcode_is_left_exactly_as_typed():
|
||||
"""The placeholder `8900000000000.0` sat in 38 production rows. It is not a
|
||||
barcode, but it is also not this stage's to delete - a value visibly wrong
|
||||
is findable, a value silently blanked is not."""
|
||||
row = run({"product_name": "Cadbury 5 Star 24g", "barcode": "8900000000000.0"})
|
||||
|
||||
assert row["barcode"] == "8900000000000.0"
|
||||
assert row.get("gtin") is None
|
||||
assert row.get("ean13") is None
|
||||
assert row.get("barcode_type") is None
|
||||
|
||||
|
||||
def test_a_failed_checksum_is_recorded_in_provenance_not_in_the_data():
|
||||
"""13 digits of the right length but the wrong check digit."""
|
||||
row = run({"product_name": "Probe", "barcode": "8901233018363"})
|
||||
|
||||
assert row["barcode"] == "8901233018363"
|
||||
assert row["field_sources"]["barcode"]["method"] == "unvalidated"
|
||||
assert "failed checksum" in row["field_sources"]["barcode"]["note"]
|
||||
|
||||
|
||||
def test_a_row_with_no_barcode_is_untouched():
|
||||
row = run({"product_name": "Amul Butter 100g", "category": "Dairy"})
|
||||
|
||||
assert "gtin" not in row
|
||||
assert "field_sources" not in row
|
||||
|
||||
|
||||
def test_it_never_invents_a_barcode():
|
||||
"""The stage has no source and no network. If the row has no barcode, it
|
||||
cannot acquire one here - that is BarcodeEnrichmentStage's job."""
|
||||
row = run({"product_name": "Unknown Product 1kg", "barcode": ""})
|
||||
|
||||
assert not row.get("barcode")
|
||||
assert not row.get("gtin")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. It does not overstate what it knows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_a_sheet_barcode_is_never_marked_verified():
|
||||
row = run({"product_name": "Probe 100g", "barcode": "8901233018362"})
|
||||
|
||||
assert row["barcode_verified"] is False
|
||||
assert row["barcode_lookup_status"] == "sheet_validated"
|
||||
assert row["barcode_source"] == "sheet"
|
||||
|
||||
|
||||
def test_the_derived_fields_are_flagged_derived_not_sourced():
|
||||
"""gtin/ean13/upc are arithmetic on the barcode. Recording them as
|
||||
`sourced` would claim a lookup confirmed them, which is the exact
|
||||
overstatement the provenance map exists to prevent."""
|
||||
row = run({"product_name": "Probe 100g", "barcode": "8901233018362"})
|
||||
|
||||
for field in ("gtin", "ean13", "upc", "barcode_type"):
|
||||
assert row["field_sources"][field]["method"] == "derived", field
|
||||
|
||||
|
||||
def test_an_existing_source_is_not_overwritten_by_sheet():
|
||||
"""When the cascade found the barcode, its provenance is the real one and
|
||||
must survive this stage running afterwards."""
|
||||
row = run({
|
||||
"product_name": "Probe 100g",
|
||||
"barcode": "8901233018362",
|
||||
"barcode_source": "Open Food Facts",
|
||||
"barcode_verified": True,
|
||||
"barcode_lookup_status": "verified",
|
||||
})
|
||||
|
||||
assert row["barcode_source"] == "Open Food Facts"
|
||||
assert row["barcode_verified"] is True
|
||||
assert row["field_sources"]["barcode"]["method"] == "sourced"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Provenance accumulates across stages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_field_sources_from_an_earlier_stage_is_merged_not_replaced():
|
||||
"""`base.apply()` assigns every key except this one. Assigning it would
|
||||
mean the last stage to run erases what every earlier stage recorded, so a
|
||||
value would end up in the database with no origin."""
|
||||
row = run({
|
||||
"product_name": "Probe 100g",
|
||||
"barcode": "8901233018362",
|
||||
"field_sources": {"hsn_code": {"method": "estimated", "source": "category"}},
|
||||
})
|
||||
|
||||
assert row["field_sources"]["hsn_code"]["method"] == "estimated"
|
||||
assert row["field_sources"]["gtin"]["method"] == "derived"
|
||||
|
||||
|
||||
def test_the_stage_never_raises_on_a_malformed_row():
|
||||
"""The EnrichmentStage contract: a stage bug degrades to "no fields added",
|
||||
never an aborted catalog row."""
|
||||
for barcode in (None, "", "abc", 12345, [], {"nested": 1}, 8901233018362):
|
||||
row = run({"product_name": "Probe", "barcode": barcode})
|
||||
assert isinstance(row, dict)
|
||||
236
tests/test_barcode_name_containment.py
Normal file
236
tests/test_barcode_name_containment.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""Accepting an Open Food Facts record whose name is shorter than ours.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
Running `scripts/backfill_nutrition_from_barcodes` over the production catalog
|
||||
on 2026-09-08 reported, of 300 barcoded rows:
|
||||
|
||||
not a valid GTIN 37
|
||||
not in Open Food Facts 100
|
||||
matched but empty 11
|
||||
found, WRONG PRODUCT 149 <-- this file
|
||||
would write 3
|
||||
|
||||
The 149 were not wrong products. The barcode resolved perfectly; Open Food
|
||||
Facts simply stores a short name where we store a long one:
|
||||
|
||||
ours "Nestle Munch 8.9g" OFF "Munch" similarity 0.332
|
||||
ours "Coca-Cola Maaza 750ml" OFF "Maaza" similarity 0.304
|
||||
ours "Cadbury Perk 22 g" OFF "Perk" similarity 0.302
|
||||
|
||||
`name_similarity` divides token overlap by the TARGET's token count, so a
|
||||
one-token candidate against a three-token target cannot exceed about 0.33 no
|
||||
matter how correct it is.
|
||||
|
||||
WHY THE FIX IS NOT A LOWER THRESHOLD
|
||||
The same run correctly rejected these, which sit BELOW the containment cases
|
||||
but not far enough below to be separable by a number:
|
||||
|
||||
ours "Pepsico Lays 1kg" OFF "Spanish tomato tango" 0.133
|
||||
ours "Coca-Cola Fanta 750ml" OFF "Orange" 0.089
|
||||
|
||||
and `settings.py:449-477` records the measurement that raised this floor to
|
||||
0.78 in the first place (at 0.45: 15 accepted / 8 wrong; at 0.78: 2 / 0).
|
||||
Lowering it re-admits exactly what it was raised to exclude.
|
||||
|
||||
Containment separates the groups structurally instead. It also has to reject
|
||||
two cases a naive substring check would wave through, both of which are real
|
||||
Open Food Facts titles: the bare brand name ("Colgate", "godrej"), which would
|
||||
otherwise attach to every product of that brand, and a same-brand sibling
|
||||
("Dairy Milk Silk" against our "Cadbury Dairy Milk").
|
||||
|
||||
The relaxation is OFF by default and enabled on exactly one call site -
|
||||
`fetch_verified_nutrition_by_barcode` - because there the barcode has already
|
||||
established identity and there are no competing candidates. On the search path,
|
||||
where many candidates compete and the name is the only discriminator, "Munch"
|
||||
would match every Nestle product containing that word.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.enrichment.barcode.matching import is_match, name_is_contained
|
||||
from app.services.enrichment.barcode.models import BarcodeCandidate
|
||||
|
||||
|
||||
# (our stored title, what OFF calls it, brand) - all measured on 2026-09-08
|
||||
CONTAINED = [
|
||||
("Nestle Munch 8.9g", "Munch", "Nestle"),
|
||||
("Coca-Cola Maaza 750ml", "Maaza", "Coca-Cola"),
|
||||
("Cadbury Perk 22 g", "Perk", "Cadbury"),
|
||||
("Nestle Milo 25 g", "MILO", "Nestle"),
|
||||
("Cadbury Fuse 25 g", "FUSE", "Cadbury"),
|
||||
("Coca-Cola Limca 750g", "limca", "Coca-Cola"),
|
||||
("Nestle Milkybar 25g", "Milkybar", "Nestle"),
|
||||
]
|
||||
|
||||
NOT_CONTAINED = [
|
||||
("Pepsico Lays 1kg", "Spanish tomato tango", "Pepsico"),
|
||||
("Lion Dates Powder 100g", "PEPER NOTEN", "Lion Dates"),
|
||||
("Coca-Cola Fanta 750ml", "Orange", "Coca-Cola"),
|
||||
# Bare brand names. Both are real OFF product_name values.
|
||||
("Colgate Total Toothpaste 150g", "Colgate", "Colgate"),
|
||||
("Godrej No1 Soap 100g", "godrej", "Godrej"),
|
||||
# Same brand, different product - the case the name gate exists for.
|
||||
("Amul Butter 100g", "Amul Cheese", "Amul"),
|
||||
("Cadbury Dairy Milk 50g", "Dairy Milk Silk", "Cadbury"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ours,theirs,brand", CONTAINED)
|
||||
def test_a_short_off_name_is_recognised_as_ours(ours, theirs, brand):
|
||||
assert name_is_contained(theirs, ours, brand) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ours,theirs,brand", NOT_CONTAINED)
|
||||
def test_a_different_product_is_still_refused(ours, theirs, brand):
|
||||
assert name_is_contained(theirs, ours, brand) is False
|
||||
|
||||
|
||||
def test_a_bare_brand_name_never_matches():
|
||||
""""Colgate" as a product name identifies a brand, not a product. Accepting
|
||||
it would attach one arbitrary pack's nutrition to every Colgate row."""
|
||||
assert name_is_contained("Colgate", "Colgate MaxFresh Toothpaste 150g", "Colgate") is False
|
||||
|
||||
|
||||
def test_size_tokens_do_not_decide_identity():
|
||||
"""`size_matches` has already compared the pack size by the time this is
|
||||
consulted, so a size token in our title must not make the names differ."""
|
||||
assert name_is_contained("Munch", "Nestle Munch 8.9g", "Nestle") is True
|
||||
assert name_is_contained("Munch", "Nestle Munch 38.5 g", "Nestle") is True
|
||||
|
||||
|
||||
def test_an_extra_token_in_the_candidate_breaks_containment():
|
||||
"""Containment is one-directional on purpose: every candidate token must be
|
||||
ours. "Dairy Milk Silk" carries "silk", which our "Cadbury Dairy Milk" does
|
||||
not, so it is a different product."""
|
||||
assert name_is_contained("Dairy Milk Silk", "Cadbury Dairy Milk 50g", "Cadbury") is False
|
||||
|
||||
|
||||
def test_empty_names_are_refused_rather_than_treated_as_contained():
|
||||
"""The empty set is a subset of everything - the one case where the maths
|
||||
says yes and the answer is obviously no."""
|
||||
assert name_is_contained("", "Nestle Munch 8.9g", "Nestle") is False
|
||||
assert name_is_contained("Munch", "", "Nestle") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The gate as a whole
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _candidate(title, brand, size=""):
|
||||
return BarcodeCandidate(barcode="8901058857245", source_name="Open Food Facts",
|
||||
candidate_title=title, candidate_brand=brand,
|
||||
candidate_size=size)
|
||||
|
||||
|
||||
def test_containment_is_off_by_default():
|
||||
"""The search path must not get this relaxation: there, many candidates
|
||||
compete and "Munch" would match every Nestle product containing it."""
|
||||
matched, _ = is_match(_candidate("Munch", "Nestle", "8.9g"),
|
||||
"Nestle", "Nestle Munch 8.9g", "8.9g",
|
||||
min_name_similarity=0.78)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_containment_accepts_when_explicitly_enabled():
|
||||
matched, similarity = is_match(_candidate("Munch", "Nestle", "8.9g"),
|
||||
"Nestle", "Nestle Munch 8.9g", "8.9g",
|
||||
min_name_similarity=0.78,
|
||||
barcode_is_identity=True)
|
||||
|
||||
assert matched is True
|
||||
# The reported confidence is still the honest similarity, not 1.0 - it is
|
||||
# stored on the row for audit and must not be inflated by the relaxation.
|
||||
assert similarity < 0.5
|
||||
|
||||
|
||||
def test_containment_does_not_bypass_the_brand_gate():
|
||||
"""Rules 1-3 still apply. A containment name match with the wrong brand is
|
||||
still a wrong product."""
|
||||
matched, _ = is_match(_candidate("Munch", "Britannia", "8.9g"),
|
||||
"Nestle", "Nestle Munch 8.9g", "8.9g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_a_conflicting_size_is_still_refused():
|
||||
"""A quantity that is PRESENT and different means our barcode is on the
|
||||
wrong row. That is exactly what the sanity check is for."""
|
||||
matched, _ = is_match(_candidate("Munch", "Nestle", "500g"),
|
||||
"Nestle", "Nestle Munch 8.9g", "8.9g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The gate that actually blocked most of the 149
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_a_blank_candidate_size_no_longer_vetoes_under_a_barcode():
|
||||
"""The real blocker, found only after measuring the containment fix.
|
||||
|
||||
`size_matches` returns False whenever EITHER side is blank, and Open Food
|
||||
Facts leaves `quantity` null on a large share of records - 57 of 146 Amul
|
||||
hits. Because `is_match` applies its rules in order, that rejected these
|
||||
rows before the name rule was ever consulted, so fixing the name gate alone
|
||||
moved the measured result from 3 rows to 8 rather than to ~149.
|
||||
|
||||
A record with no quantity does not disagree with our pack size. It says
|
||||
nothing about it, and the barcode has already established identity.
|
||||
"""
|
||||
matched, _ = is_match(_candidate("Munch", "Nestle", ""),
|
||||
"Nestle", "Nestle Munch 38.5 g", "38.5 g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is True
|
||||
|
||||
|
||||
def test_a_blank_candidate_size_still_vetoes_on_the_search_path():
|
||||
"""Without a barcode a sizeless candidate is genuinely unidentifiable: it
|
||||
could be any pack of that product, and a GTIN belongs to exactly one."""
|
||||
matched, _ = is_match(_candidate("Nestle Munch", "Nestle", ""),
|
||||
"Nestle", "Nestle Munch 38.5 g", "38.5 g",
|
||||
min_name_similarity=0.45)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_the_size_relaxation_does_not_also_relax_the_brand_gate():
|
||||
matched, _ = is_match(_candidate("Munch", "Britannia", ""),
|
||||
"Nestle", "Nestle Munch 38.5 g", "38.5 g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_the_size_relaxation_does_not_also_relax_variant_conflicts():
|
||||
matched, _ = is_match(_candidate("Munch sugar free", "Nestle", ""),
|
||||
"Nestle", "Nestle Munch 38.5 g", "38.5 g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_containment_does_not_bypass_the_variant_conflict_gate():
|
||||
""""sugar free" on the candidate but not on ours is a different product
|
||||
however well the rest of the name contains."""
|
||||
matched, _ = is_match(_candidate("Munch sugar free", "Nestle", "8.9g"),
|
||||
"Nestle", "Nestle Munch 8.9g", "8.9g",
|
||||
min_name_similarity=0.78, barcode_is_identity=True)
|
||||
|
||||
assert matched is False
|
||||
|
||||
|
||||
def test_a_normal_high_similarity_match_is_unaffected():
|
||||
"""The relaxation is only consulted when the similarity floor fails, so it
|
||||
cannot change any decision the existing gate already made."""
|
||||
matched, similarity = is_match(_candidate("Nestle Munch", "Nestle", "8.9g"),
|
||||
"Nestle", "Nestle Munch", "8.9g",
|
||||
min_name_similarity=0.78)
|
||||
|
||||
assert matched is True
|
||||
assert similarity >= 0.78
|
||||
276
tests/test_brand_table_enrichment_columns.py
Normal file
276
tests/test_brand_table_enrichment_columns.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""The enrichment columns on the brand tables, and the write path that fills them.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
The barcode stage returns nine fields - `BarcodeResult.as_product_fields()` -
|
||||
and `upsert_brand_products` named two of them. The other seven were computed on
|
||||
every run and then dropped on the floor by the writer. Measured against the
|
||||
production database on 2026-09-08, before this landed:
|
||||
|
||||
upc 0.0% (column existed on 7 of 56 tables, never written)
|
||||
ean13 6.4% (added out-of-band, never written by any code)
|
||||
gtin 8.7%
|
||||
barcode 18.4%
|
||||
|
||||
The same was true of the HSN/GST stage: it computes `gst_percent`, `tax_amount`
|
||||
and `hsn_gst_needs_review`, and `_to_storage_row` projected none of them.
|
||||
|
||||
Three distinct things have to hold for a value to survive, and each of them
|
||||
broke independently at some point, so each gets a test here:
|
||||
|
||||
1. The column has to EXIST. `_ensure_columns` is the only migration mechanism
|
||||
in the repo - there is no Alembic and no migrations directory - so a column
|
||||
missing from its `col_defs` dict never appears on the 56 brand tables that
|
||||
already exist.
|
||||
|
||||
2. The INSERT has to NAME it. Adding the column is not enough; that is exactly
|
||||
how seven tables ended up carrying `gtin` and `ean13` columns that no code
|
||||
ever wrote a value into.
|
||||
|
||||
3. A later re-seed must not BLANK it. `ON CONFLICT DO UPDATE SET x =
|
||||
EXCLUDED.x` overwrites with whatever arrived, and the seed loader,
|
||||
`user_products._build_product_dict` and `brand_sync`'s re-seed all build a
|
||||
product dict from a spreadsheet with no enrichment keys in it - so their
|
||||
EXCLUDED values are NULL. This is the same trap `test_brand_table_scores`
|
||||
documents for the score columns, which is why those are omitted from the
|
||||
statement entirely. These columns cannot be omitted (the pipeline is what
|
||||
writes them), so they use COALESCE instead.
|
||||
|
||||
`barcode` and `barcode_type` are COALESCEd alongside the seven, though they are
|
||||
older columns. Proven against a live table: a bare re-seed set `barcode` to
|
||||
NULL while COALESCE kept `gtin` and `ean13`, leaving a row that claimed a GTIN
|
||||
with no barcode. A half-erased identity is worse than either whole state, and
|
||||
the rest of the barcode package already promises never to erase one -
|
||||
`stage.py` skips a row that has a barcode and `enrichment/base.py` refuses to
|
||||
blank a held value. The upsert was the one place that still could.
|
||||
|
||||
No database is involved: the cursor is a recorder, so the assertions are about
|
||||
the exact SQL sent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.services import brand_sync, vector_store
|
||||
|
||||
|
||||
# The columns this change added, with the type each MUST be created as.
|
||||
#
|
||||
# THE TYPES ARE ADOPTED, NOT CHOSEN. Seven brand tables already carried these
|
||||
# columns before any code created them, and `ADD COLUMN IF NOT EXISTS` does not
|
||||
# reconcile a type difference - it silently leaves the old table alone. Picking
|
||||
# a "better" type here (NUMERIC for the tax figures, DOUBLE PRECISION for the
|
||||
# epoch) would leave 7 tables permanently disagreeing with 49. These are what
|
||||
# `information_schema` reported for those 7 tables on 2026-09-08.
|
||||
PIPELINE_OWNED = {
|
||||
"gtin": "TEXT",
|
||||
"ean13": "TEXT",
|
||||
"upc": "TEXT",
|
||||
"barcode_source": "TEXT",
|
||||
"barcode_verified": "BOOLEAN",
|
||||
"barcode_lookup_status": "TEXT",
|
||||
"barcode_last_updated": "TIMESTAMP",
|
||||
"gst_percent": "REAL",
|
||||
"tax_amount": "REAL",
|
||||
"hsn_gst_needs_review": "BOOLEAN",
|
||||
"field_sources": "JSONB",
|
||||
}
|
||||
|
||||
# Written only by nutrition_score_sync, never by the INSERT - the same
|
||||
# category as nutrition_score / health_score.
|
||||
MIRROR_OWNED = {"nutrients_per_100g": "JSONB"}
|
||||
|
||||
|
||||
class MigrationCursor:
|
||||
"""A brand table that exists but has none of the current columns."""
|
||||
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.statements.append(" ".join(str(sql).split()))
|
||||
|
||||
def fetchall(self):
|
||||
return [] # no existing columns -> every column is missing
|
||||
|
||||
|
||||
def _insert_statement() -> str:
|
||||
"""The INSERT ... ON CONFLICT text, whitespace-normalised."""
|
||||
source = vector_store.upsert_brand_products.__doc__ or ""
|
||||
# The statement is built inline, so read it off the module source rather
|
||||
# than reaching into a closure.
|
||||
import inspect
|
||||
body = inspect.getsource(vector_store.upsert_brand_products)
|
||||
match = re.search(r"INSERT INTO \{table_name\}.*?updated_at = CURRENT_TIMESTAMP",
|
||||
body, re.S)
|
||||
assert match, "could not locate the INSERT statement in upsert_brand_products"
|
||||
return " ".join(match.group(0).split())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. The columns exist
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_migration_adds_every_enrichment_column():
|
||||
cur = MigrationCursor()
|
||||
|
||||
vector_store._ensure_columns(cur, "brand_cadbury")
|
||||
|
||||
for col, col_type in {**PIPELINE_OWNED, **MIRROR_OWNED}.items():
|
||||
assert (f"ALTER TABLE brand_cadbury ADD COLUMN IF NOT EXISTS "
|
||||
f"{col} {col_type}") in cur.statements, col
|
||||
|
||||
|
||||
def test_the_types_match_the_tables_that_already_had_these_columns():
|
||||
"""A wrong type here is invisible until a write fails on one of the seven
|
||||
pre-existing tables, because ADD COLUMN IF NOT EXISTS skips them silently.
|
||||
|
||||
`barcode_last_updated` is the one most likely to be "corrected" by a future
|
||||
reader: `BarcodeResult` carries a float epoch, so DOUBLE PRECISION looks
|
||||
right. The column on disk is TIMESTAMP, and vector_store._epoch_to_timestamp
|
||||
is what bridges the two.
|
||||
"""
|
||||
assert vector_store._ensure_columns.__doc__ is not None
|
||||
import inspect
|
||||
body = inspect.getsource(vector_store._ensure_columns)
|
||||
|
||||
assert '"barcode_last_updated": "TIMESTAMP"' in body
|
||||
assert '"gst_percent": "REAL"' in body
|
||||
assert '"tax_amount": "REAL"' in body
|
||||
|
||||
|
||||
def test_the_create_table_ddl_carries_them_too():
|
||||
"""`col_defs` migrates existing tables; the DDL is what a brand table
|
||||
created from scratch gets. A column in one but not the other means a new
|
||||
brand's table differs from every other brand's."""
|
||||
ddl = vector_store.get_brand_table_ddl("Cadbury")
|
||||
|
||||
for col in {**PIPELINE_OWNED, **MIRROR_OWNED}:
|
||||
assert re.search(rf"^\s*{col}\s", ddl, re.M), col
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. The INSERT names them - and does not name the mirror-owned ones
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_insert_names_every_pipeline_owned_column():
|
||||
statement = _insert_statement()
|
||||
column_list = statement.split("VALUES")[0]
|
||||
|
||||
for col in PIPELINE_OWNED:
|
||||
assert re.search(rf"[(,] ?{col}[,)]", column_list), col
|
||||
|
||||
|
||||
def test_the_insert_does_not_name_the_mirror_owned_columns():
|
||||
"""Same rule as nutrition_score / health_score: a column this statement
|
||||
never names is a column it cannot damage."""
|
||||
statement = _insert_statement()
|
||||
|
||||
for col in MIRROR_OWNED:
|
||||
assert col not in statement, col
|
||||
|
||||
|
||||
def test_the_placeholder_count_matches_the_column_count():
|
||||
"""An arity mismatch here is a runtime error on every single write, so it
|
||||
is worth catching at import time rather than on the next upload."""
|
||||
statement = _insert_statement()
|
||||
match = re.search(r"INSERT INTO \{table_name\} \((.*?)\) VALUES \((.*?)\)", statement)
|
||||
columns = [c.strip() for c in match.group(1).split(",") if c.strip()]
|
||||
|
||||
assert len(columns) == match.group(2).count("%s")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. A re-seed cannot blank them
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_every_enrichment_column_is_coalesced_on_conflict():
|
||||
"""This is the guard that makes "a re-seed wipes the enrichment"
|
||||
structurally impossible rather than merely unlikely."""
|
||||
statement = _insert_statement()
|
||||
do_update = statement.split("DO UPDATE SET", 1)[1]
|
||||
|
||||
for col in PIPELINE_OWNED:
|
||||
if col == "field_sources":
|
||||
continue # merged, asserted separately below
|
||||
assert (f"{col} = COALESCE(EXCLUDED.{col}, {{table_name}}.{col})"
|
||||
in do_update), col
|
||||
|
||||
|
||||
def test_the_barcode_pair_is_coalesced_with_its_identity_group():
|
||||
"""`barcode` and `barcode_type` predate this change but belong to the same
|
||||
identity group as gtin/ean13/upc. Leaving them on plain EXCLUDED produced a
|
||||
row with a GTIN and no barcode after a bare re-seed."""
|
||||
do_update = _insert_statement().split("DO UPDATE SET", 1)[1]
|
||||
|
||||
for col in ("barcode", "barcode_type"):
|
||||
assert (f"{col} = COALESCE(EXCLUDED.{col}, {{table_name}}.{col})"
|
||||
in do_update), col
|
||||
|
||||
|
||||
def test_field_sources_is_merged_rather_than_replaced():
|
||||
"""A run that learns the provenance of one field must not drop what is
|
||||
already known about the others, so this one is `||`, not COALESCE."""
|
||||
do_update = _insert_statement().split("DO UPDATE SET", 1)[1]
|
||||
|
||||
# Read off the module source, so the f-string's escaped braces are still
|
||||
# doubled here - `'{{}}'` is what renders as the SQL literal `'{}'`.
|
||||
assert "field_sources = COALESCE({table_name}.field_sources, '{{}}'::jsonb) " \
|
||||
"|| COALESCE(EXCLUDED.field_sources, '{{}}'::jsonb)" in do_update
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. The seed export carries them
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_seed_export_carries_every_new_column():
|
||||
"""`export_brand_to_seed_file` rebuilds a product from EXPORT_COLUMNS and
|
||||
replaces the whole dict. A column missing from that tuple is stripped out
|
||||
of the catalog file on export - which is precisely why
|
||||
scripts/backfill_barcodes_from_off.py refuses to call that helper today."""
|
||||
for col in {**PIPELINE_OWNED, **MIRROR_OWNED}:
|
||||
assert col in brand_sync.EXPORT_COLUMNS, col
|
||||
|
||||
|
||||
def test_the_export_coerces_values_json_dumps_would_refuse():
|
||||
"""`barcode_last_updated` comes back from psycopg as a datetime and
|
||||
`field_sources` as a dict. json.dumps refuses the first outright and chokes
|
||||
on a Decimal nested in the second."""
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
assert brand_sync._jsonable(datetime(2026, 9, 8, 10, 51, 42)) == "2026-09-08T10:51:42"
|
||||
assert brand_sync._jsonable({"barcode": {"confidence": Decimal("0.91")}}) == {
|
||||
"barcode": {"confidence": 0.91}
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. The epoch/timestamp bridge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_the_timestamp_bridge_accepts_every_shape_that_reaches_it():
|
||||
"""Three writers feed this column and they disagree about the type:
|
||||
BarcodeResult emits a float epoch, a re-seeded catalog file carries the
|
||||
ISO-8601 string brand_sync exported, and a DB read hands back a datetime.
|
||||
All three have to load or a round-trip drops the value it just wrote."""
|
||||
from datetime import datetime
|
||||
|
||||
assert vector_store._epoch_to_timestamp(1788773811.0) == datetime.fromtimestamp(1788773811.0)
|
||||
assert vector_store._epoch_to_timestamp("2026-09-08T10:51:42") == datetime(2026, 9, 8, 10, 51, 42)
|
||||
assert vector_store._epoch_to_timestamp(datetime(2026, 1, 1)) == datetime(2026, 1, 1)
|
||||
assert vector_store._epoch_to_timestamp("not a date") is None
|
||||
assert vector_store._epoch_to_timestamp("") is None
|
||||
assert vector_store._epoch_to_timestamp(None) is None
|
||||
|
||||
|
||||
def test_the_numeric_coercion_refuses_rather_than_raises():
|
||||
"""A malformed tax figure must degrade to "no figure stored" and never
|
||||
abort a whole batch's write."""
|
||||
assert vector_store._to_numeric_or_none("18%") == 18.0
|
||||
assert vector_store._to_numeric_or_none("₹1,250.50") == 1250.50
|
||||
assert vector_store._to_numeric_or_none(12.5) == 12.5
|
||||
assert vector_store._to_numeric_or_none("not a number") is None
|
||||
assert vector_store._to_numeric_or_none(None) is None
|
||||
# bool is an int subclass; True must not become 1.0 in a NUMERIC column
|
||||
assert vector_store._to_numeric_or_none(True) is None
|
||||
159
tests/test_content_enrichment_stage.py
Normal file
159
tests/test_content_enrichment_stage.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""The stage that fills `highlights` and `nutrients` on an uploaded row.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
`catalog_engine.generate_product_highlights` and `generate_nutrients_info` have
|
||||
existed for a long time, and `brand_discovery._build_product` calls both. The
|
||||
store-catalog pipeline never did - `_to_storage_row` passed through whatever
|
||||
the sheet carried, and a colleague's sheet carries neither column. So every
|
||||
single uploaded row landed `highlights=[]` and `nutrients=[]`.
|
||||
|
||||
That is why those columns look healthy in aggregate (95.3% / 69.7% measured on
|
||||
2026-09-08) while being empty for exactly the rows this work is about: the
|
||||
percentages come from the older brand-discovery path.
|
||||
|
||||
The second thing this file pins is the consumability gate.
|
||||
`generate_nutrients_info` matches category keywords, so without a gate a Hair
|
||||
Care row can acquire "Vitamin B Complex - Energy". Shampoo has no nutrients.
|
||||
`scripts/purge_non_consumable_nutrition.py` exists because this already
|
||||
happened once at the `nutrition_facts` level; the display column needs the same
|
||||
refusal, and it must record `not_applicable` rather than leave a silent blank -
|
||||
a permanent unexplained gap is what eventually gets "fixed" by fabricating.
|
||||
|
||||
No network, no database, no LLM: the generators are keyword functions over the
|
||||
row dict.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.enrichment.content.stage import ContentEnrichmentStage, _has_entries
|
||||
|
||||
|
||||
def run(product, brand="Cadbury"):
|
||||
return asyncio.run(ContentEnrichmentStage().apply(dict(product), brand))
|
||||
|
||||
|
||||
FOOD_ROW = {
|
||||
"product_name": "Cadbury Dairy Milk 50g",
|
||||
"title": "Cadbury Dairy Milk",
|
||||
"category": "Chocolates",
|
||||
"size": "50g",
|
||||
"description": "Smooth milk chocolate bar.",
|
||||
}
|
||||
|
||||
SHAMPOO_ROW = {
|
||||
"product_name": "Dove Daily Shine Shampoo 340ml",
|
||||
"title": "Dove Daily Shine Shampoo",
|
||||
"category": "Hair Care",
|
||||
"size": "340ml",
|
||||
"description": "Nourishing shampoo for daily use.",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# It fills what the pipeline used to leave empty
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_an_uploaded_food_row_gets_highlights():
|
||||
row = run(FOOD_ROW)
|
||||
|
||||
assert row["highlights"], "every upload landed highlights=[] before this stage"
|
||||
assert all(isinstance(h, str) and h.strip() for h in row["highlights"])
|
||||
|
||||
|
||||
def test_an_uploaded_food_row_gets_nutrients():
|
||||
row = run(FOOD_ROW)
|
||||
|
||||
assert row["nutrients"]
|
||||
|
||||
|
||||
def test_highlights_are_flagged_derived_not_sourced():
|
||||
"""They are marketing copy computed from fields we already hold. Calling
|
||||
them sourced would claim something confirmed them."""
|
||||
row = run(FOOD_ROW)
|
||||
|
||||
assert row["field_sources"]["highlights"]["method"] == "derived"
|
||||
|
||||
|
||||
def test_the_keyword_nutrients_are_flagged_estimated():
|
||||
"""They are category guesses standing in until a real lookup succeeds, and
|
||||
the mirror from nutrition_facts overwrites them when one does."""
|
||||
row = run(FOOD_ROW)
|
||||
|
||||
assert row["field_sources"]["nutrients"]["method"] == "estimated"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The consumability gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_a_shampoo_gets_no_nutrients():
|
||||
"""The real defect: keyword matching gave personal-care rows entries like
|
||||
"Vitamin B Complex - Energy"."""
|
||||
row = run(SHAMPOO_ROW, brand="Dove")
|
||||
|
||||
assert not row.get("nutrients")
|
||||
|
||||
|
||||
def test_a_shampoo_records_not_applicable_rather_than_a_silent_blank():
|
||||
"""A gap nobody can explain is the one somebody eventually fills with
|
||||
invented data. The coverage report reads this to exclude the row from the
|
||||
nutrition denominator instead of reporting it missing forever."""
|
||||
row = run(SHAMPOO_ROW, brand="Dove")
|
||||
|
||||
assert row["field_sources"]["nutrients"]["method"] == "not_applicable"
|
||||
|
||||
|
||||
def test_a_shampoo_still_gets_highlights():
|
||||
"""Non-consumable rules out nutrition, not description. A shampoo has
|
||||
perfectly good highlights."""
|
||||
row = run(SHAMPOO_ROW, brand="Dove")
|
||||
|
||||
assert row["highlights"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# It fills blanks only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_values_the_sheet_supplied_are_never_overwritten():
|
||||
"""The pipeline's first rule: the store's own data is authoritative."""
|
||||
row = run({**FOOD_ROW,
|
||||
"highlights": ["Fairtrade cocoa"],
|
||||
"nutrients": ["Protein 7.3 g per 100 g"]})
|
||||
|
||||
assert row["highlights"] == ["Fairtrade cocoa"]
|
||||
assert row["nutrients"] == ["Protein 7.3 g per 100 g"]
|
||||
|
||||
|
||||
def test_a_column_of_empty_strings_counts_as_blank():
|
||||
"""The spreadsheet parser produces [''] from a column that exists with no
|
||||
value in it. Treating that as "already filled" keeps the row [''] forever."""
|
||||
assert _has_entries([""]) is False
|
||||
assert _has_entries(["", " "]) is False
|
||||
assert _has_entries(["Real"]) is True
|
||||
assert _has_entries([]) is False
|
||||
|
||||
row = run({**FOOD_ROW, "highlights": [""]})
|
||||
assert row["highlights"] != [""]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The stage contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_provenance_from_an_earlier_stage_survives():
|
||||
row = run({**FOOD_ROW,
|
||||
"field_sources": {"gtin": {"method": "derived"}}})
|
||||
|
||||
assert row["field_sources"]["gtin"]["method"] == "derived"
|
||||
assert row["field_sources"]["highlights"]["method"] == "derived"
|
||||
|
||||
|
||||
def test_the_stage_never_raises_on_a_malformed_row():
|
||||
for product in ({}, {"product_name": None}, {"category": 123},
|
||||
{"title": "", "category": None, "highlights": "not a list"}):
|
||||
assert isinstance(run(product), dict)
|
||||
177
tests/test_no_fabricated_identifiers.py
Normal file
177
tests/test_no_fabricated_identifiers.py
Normal file
@@ -0,0 +1,177 @@
|
||||
"""Defaults that invent a regulatory identifier or a commercial claim.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
`user_products._build_product_dict` filled two blanks with constants:
|
||||
|
||||
fssai_license = req.fssai_license or sample.get("fssai_license") or "10012042000244"
|
||||
providers = req.providers or list(sample.get("providers") or
|
||||
["Amazon", "Flipkart", "BigBasket", "Jiomart", "Blinkit", "Zepto"])
|
||||
|
||||
The first constant is not a placeholder. `10012042000244` is Lion Dates' real,
|
||||
registered FSSAI licence - it is still in `brand_registry.FSSAI_LICENSES` under
|
||||
that brand. Every product uploaded for a brand with no existing row was stamped
|
||||
with it, which attributes legal responsibility for that food to a business that
|
||||
never made it. It reached production at least once:
|
||||
`scripts/merge_haldiram.py` exists specifically to strip it back off
|
||||
`brand_haldirams`.
|
||||
|
||||
The second asserts a product is stocked by six named marketplaces on the basis
|
||||
of nothing at all.
|
||||
|
||||
Both are now sourced from the brand's own rows, or left empty. The tests below
|
||||
pin three things:
|
||||
|
||||
1. The literal constants are gone from the module.
|
||||
2. An unknown brand gets NO licence rather than someone else's.
|
||||
3. Consensus refuses to answer when a brand's own rows disagree - because at
|
||||
that point one of them is already wrong and a tie-break would just be
|
||||
picking which product to mislabel.
|
||||
|
||||
No database: `consensus_value` and `fssai_for_brand` take the rows as an
|
||||
argument, which is what makes them testable at all.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
|
||||
from app.api.routers import user_products
|
||||
from app.services.enrichment.catalog_consensus import consensus_value, fssai_for_brand
|
||||
|
||||
|
||||
LION_DATES_LICENCE = "10012042000244"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. The constants are gone
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _executable_string_literals(module) -> list:
|
||||
"""Every string constant the module can actually evaluate.
|
||||
|
||||
Docstrings and comments are excluded deliberately: the fix's own comment
|
||||
has to be free to name the constant it removed, or the explanation of why
|
||||
the bug mattered cannot be written down next to the code that had it.
|
||||
"""
|
||||
tree = ast.parse(inspect.getsource(module))
|
||||
|
||||
docstrings = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
body = getattr(node, "body", [])
|
||||
if (body and isinstance(body[0], ast.Expr)
|
||||
and isinstance(body[0].value, ast.Constant)
|
||||
and isinstance(body[0].value.value, str)):
|
||||
docstrings.add(id(body[0].value))
|
||||
|
||||
return [n.value for n in ast.walk(tree)
|
||||
if isinstance(n, ast.Constant) and isinstance(n.value, str)
|
||||
and id(n) not in docstrings]
|
||||
|
||||
|
||||
def test_lion_dates_licence_is_not_a_fallback_anywhere_in_the_upload_path():
|
||||
literals = _executable_string_literals(user_products)
|
||||
|
||||
assert LION_DATES_LICENCE not in literals, (
|
||||
"A real registered FSSAI licence must never appear as a default. "
|
||||
"It belongs to Lion Dates and to no other brand."
|
||||
)
|
||||
|
||||
|
||||
def test_the_six_marketplace_default_is_gone():
|
||||
literals = _executable_string_literals(user_products)
|
||||
|
||||
for marketplace in ("Blinkit", "BigBasket", "Jiomart", "Zepto"):
|
||||
assert marketplace not in literals, (
|
||||
f"{marketplace} appears as an evaluable literal. Listing "
|
||||
f"marketplaces nobody verified is a false availability claim."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. An unknown brand gets nothing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_an_unknown_brand_with_no_rows_gets_no_licence():
|
||||
value, source = fssai_for_brand("Entirely Unknown Brand", rows=[])
|
||||
|
||||
assert value is None
|
||||
assert source == "unknown"
|
||||
|
||||
|
||||
def test_a_registry_brand_still_gets_its_mapped_licence():
|
||||
"""The curated map remains the first and best source."""
|
||||
value, source = fssai_for_brand("Lion Dates", rows=[])
|
||||
|
||||
assert value == LION_DATES_LICENCE
|
||||
assert source == "brand_registry"
|
||||
|
||||
|
||||
def test_an_unmapped_brand_inherits_from_its_own_agreeing_rows():
|
||||
"""400 Britannia rows carrying one licence is good evidence for the 401st -
|
||||
unlike a constant, this value genuinely belongs to the brand."""
|
||||
rows = [{"fssai_license": "11223344556677"} for _ in range(20)]
|
||||
|
||||
value, source = fssai_for_brand("Some Unmapped Brand", rows=rows)
|
||||
|
||||
assert value == "11223344556677"
|
||||
assert source == "catalog_consensus"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Consensus declines rather than guesses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_conflicting_licences_propagate_nothing():
|
||||
"""Two different licences on one brand means one is already wrong. Picking
|
||||
the more common one would just spread whichever error is ahead."""
|
||||
rows = ([{"fssai_license": "11111111111111"}] * 10 +
|
||||
[{"fssai_license": "22222222222222"}] * 8)
|
||||
|
||||
value, source = fssai_for_brand("Conflicted Brand", rows=rows)
|
||||
|
||||
assert value is None
|
||||
assert source == "unknown"
|
||||
|
||||
|
||||
def test_a_single_dissenting_row_does_not_veto_a_clear_majority():
|
||||
"""One bad row among many should not block the other 19 from being useful."""
|
||||
rows = [{"fssai_license": "11111111111111"}] * 19 + [{"fssai_license": "99999999999999"}]
|
||||
|
||||
value, _ = consensus_value("fssai_license", rows)
|
||||
|
||||
assert value == "11111111111111"
|
||||
|
||||
|
||||
def test_too_few_rows_is_not_a_consensus():
|
||||
"""Two rows agreeing proves nothing about a third."""
|
||||
value, why = consensus_value("fssai_license", [{"fssai_license": "1"}, {"fssai_license": "1"}])
|
||||
|
||||
assert value is None
|
||||
assert why["reason"] == "too few populated rows"
|
||||
|
||||
|
||||
def test_blank_values_are_not_counted_as_agreement():
|
||||
"""A column that is empty on every row must not come back as a consensus of
|
||||
empties - that would read as "the brand agrees there is no licence"."""
|
||||
value, _ = consensus_value("fssai_license", [{"fssai_license": None}] * 20)
|
||||
|
||||
assert value is None
|
||||
|
||||
|
||||
def test_list_valued_columns_reach_consensus_too():
|
||||
"""`providers` is a TEXT[]; lists are unhashable, so the modal calculation
|
||||
has to key them as tuples or it raises."""
|
||||
rows = [{"providers": ["Amazon", "Flipkart"]}] * 10
|
||||
|
||||
value, _ = consensus_value("providers", rows)
|
||||
|
||||
assert value == ["Amazon", "Flipkart"]
|
||||
|
||||
|
||||
def test_consensus_never_raises_on_unusable_rows():
|
||||
value, why = consensus_value("providers", [{"providers": {"unhashable": ["dict"]}}] * 5)
|
||||
|
||||
assert value is None
|
||||
assert "reason" in why
|
||||
@@ -184,15 +184,36 @@ def test_a_matching_record_with_no_usable_values_is_unavailable(off_returns):
|
||||
assert facts["data_status"] == "unavailable"
|
||||
|
||||
|
||||
def test_the_floor_can_be_relaxed_per_call(off_returns):
|
||||
"""The threshold is a judgement, not a constant, so it is a parameter.
|
||||
def test_a_shorter_off_name_no_longer_needs_the_floor_relaxed(off_returns):
|
||||
""""Aachi Biryani Masala" against our "Aachi Biryani Masala 50 g" is plainly
|
||||
the same product and scores 0.716 - below the conservative 0.78 default.
|
||||
|
||||
"Aachi Biryani Masala" against our "Aachi Biryani Masala 50 g" is plainly
|
||||
the same product and scores 0.716 - below the conservative default. The
|
||||
backfill script exposes this as --min-similarity for exactly this reason.
|
||||
It used to need `min_name_similarity=0.70` to get through. It no longer
|
||||
does: this path passes `barcode_is_identity=True`, and the candidate's name
|
||||
is ours minus the pack size, so the containment rule accepts it while the
|
||||
floor stays where the measured yield table put it. See
|
||||
tests/test_barcode_name_containment.py - 149 of 300 barcoded rows were
|
||||
being refused this way.
|
||||
"""
|
||||
off_returns["body"] = {"status": 1, "product": _product(
|
||||
"Aachi Biryani Masala", "Aachi", "50 g")}
|
||||
|
||||
facts = nds.fetch_verified_nutrition_by_barcode(
|
||||
"8906021120272", "Aachi", "Aachi Biryani Masala 50 g", "50 g")
|
||||
|
||||
assert facts["data_status"] == "verified"
|
||||
|
||||
|
||||
def test_the_floor_can_still_be_relaxed_per_call(off_returns):
|
||||
"""The threshold is a judgement, not a constant, so it is a parameter, and
|
||||
the backfill script exposes it as --min-similarity.
|
||||
|
||||
"Aachi Biryani Masala Mix" carries a token ours does not, so containment
|
||||
does NOT rescue it - it is exactly the shape the floor exists to judge. It
|
||||
scores 0.703: refused at the 0.78 default, accepted at 0.70.
|
||||
"""
|
||||
off_returns["body"] = {"status": 1, "product": _product(
|
||||
"Aachi Biryani Masala Mix", "Aachi", "50 g")}
|
||||
args = ("8906021120272", "Aachi", "Aachi Biryani Masala 50 g", "50 g")
|
||||
|
||||
assert nds.fetch_verified_nutrition_by_barcode(*args)["data_status"] == "unavailable"
|
||||
|
||||
Reference in New Issue
Block a user