Backend catalog recent updates
This commit is contained in:
sriram
295
scripts/backfill_nutrition_from_barcodes.py
Normal file
295
scripts/backfill_nutrition_from_barcodes.py
Normal file
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Upgrade nutrition data from a fuzzy name match to an exact barcode match.
|
||||
|
||||
WHY
|
||||
---
|
||||
Every Open Food Facts call in this project searches by brand and product name
|
||||
and then scores whatever comes back - `nutrition_data_service._search_openfoodfacts`,
|
||||
the barcode cascade's own `OpenFoodFactsSource.search`, and `image_search`. That
|
||||
is a guess, and the stored evidence says so: OFF-sourced rows in
|
||||
`nutrition_facts` carry match confidences as low as 0.32, which is exactly
|
||||
`MIN_MATCH_CONFIDENCE`.
|
||||
|
||||
For the products that carry a barcode we can do better. The barcode is the
|
||||
identifier printed on the pack, so `/api/v2/product/{code}` returns that product
|
||||
or nothing. This walks the catalogue's barcoded rows and rewrites their
|
||||
nutrition from the exact record.
|
||||
|
||||
WHAT IT WILL AND WILL NOT TOUCH
|
||||
-------------------------------
|
||||
It writes to `nutrition_facts` and to NOTHING else. No product name, price,
|
||||
image, category or barcode in any brand table is modified. The invalid barcodes
|
||||
it finds are reported, not repaired.
|
||||
|
||||
It also refuses to overwrite a human's work: `upsert_nutrition_facts` is an
|
||||
ON CONFLICT ... DO UPDATE, so a row whose existing `data_source` is `manual` or
|
||||
`excel_upload` is skipped. Replacing a 0.32 name-match with an exact barcode
|
||||
match is the point of this script; replacing something a person typed is not.
|
||||
|
||||
THE THING TO UNDERSTAND BEFORE READING THE OUTPUT
|
||||
-------------------------------------------------
|
||||
The LOOKUP is exact. The STORED BARCODE is not. Measured across the catalogue,
|
||||
a quarter of the records found this way described a different product, because
|
||||
the barcode on our row was wrong:
|
||||
|
||||
Aachi Chicken Masala 50g -> OFF "Chicken Kabab/65 Masala"
|
||||
Tata Tea Gold 500g -> OFF "Tata Tea Gold Care"
|
||||
Lion Dates Powder 100g -> OFF "PEPER NOTEN"
|
||||
|
||||
So every record still passes `matching.is_match`, and the report below prints
|
||||
the name-similarity score for every candidate - accepted or not - because no
|
||||
threshold cleanly separates the two groups (two products tie at 0.773 with
|
||||
opposite verdicts). Read the SKIPPED list: it is a list of barcodes that are
|
||||
probably wrong in OUR catalogue.
|
||||
|
||||
Usage:
|
||||
|
||||
python -m scripts.backfill_nutrition_from_barcodes # dry run
|
||||
python -m scripts.backfill_nutrition_from_barcodes --apply
|
||||
python -m scripts.backfill_nutrition_from_barcodes --min-similarity 0.7
|
||||
|
||||
`--dry-run` is the default and `--apply` must be explicit: this writes to
|
||||
whatever database `backend/.env` points at, which is production. The target host
|
||||
is printed on startup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.infrastructure.settings import DB_HOST, DB_NAME
|
||||
from app.services.enrichment.barcode.matching import is_match, name_similarity
|
||||
from app.services.enrichment.barcode.models import BarcodeCandidate
|
||||
from app.services.enrichment.barcode.sources.open_food_facts import (
|
||||
fetch_product_by_barcode,
|
||||
)
|
||||
from app.services.nutrition_data_service import (
|
||||
BARCODE_MIN_NAME_SIMILARITY,
|
||||
fetch_verified_nutrition_by_barcode,
|
||||
)
|
||||
from app.services.nutrition_db import get_nutrition_facts, upsert_nutrition_facts
|
||||
from app.services.vector_store import _connect, display_name_for_suffix
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("backfill_nutrition")
|
||||
|
||||
# Courtesy gap between calls to a free community API.
|
||||
PAUSE_SECONDS = 0.35
|
||||
|
||||
# data_source values that mean "a person put this here". Never overwritten.
|
||||
HUMAN_SOURCES = {"manual", "excel_upload"}
|
||||
|
||||
# A GTIN is 8, 12, 13 or 14 digits. Anything else in the barcode column is not a
|
||||
# barcode - the catalogue holds "8900000000000.0" (a float that survived an
|
||||
# Excel import) and several 8-digit codes attached to three different pack sizes
|
||||
# at once. Reported rather than looked up; a bad code cannot match anything.
|
||||
_GTIN = re.compile(r"^\d{8}$|^\d{12,14}$")
|
||||
|
||||
_SIZE_IN_NAME = re.compile(r"(\d+(?:[.,]\d+)?\s*(?:kg|g|gm|gms|ml|l|ltr))\b", re.I)
|
||||
|
||||
|
||||
def _catalogue_rows() -> List[Dict[str, Any]]:
|
||||
"""Every catalogue row that carries a barcode.
|
||||
|
||||
Columns are read defensively: `_ensure_columns` adds them lazily, so an
|
||||
older brand table can be missing `size_variants` entirely - which is a
|
||||
crash, not a warning, if you SELECT it blindly.
|
||||
"""
|
||||
conn = _connect()
|
||||
if conn is None:
|
||||
return []
|
||||
rows: List[Dict[str, Any]] = []
|
||||
with conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema='public' AND table_name LIKE 'brand_%' "
|
||||
"ORDER BY table_name"
|
||||
)
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
for table in tables:
|
||||
cur.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_name=%s", (table,))
|
||||
cols = {r[0] for r in cur.fetchall()}
|
||||
if "barcode" not in cols:
|
||||
continue
|
||||
select = "product_name,barcode,image_id,category"
|
||||
if "size_variants" in cols:
|
||||
select += ",size_variants"
|
||||
cur.execute(
|
||||
f"SELECT {select} FROM {table} "
|
||||
f"WHERE barcode IS NOT NULL AND barcode <> ''")
|
||||
for record in cur.fetchall():
|
||||
name, barcode, image_id, category = record[:4]
|
||||
sizes = record[4] if len(record) > 4 else None
|
||||
match = _SIZE_IN_NAME.search(name or "")
|
||||
size = (sizes[0] if sizes else "") or (match.group(1) if match else "")
|
||||
rows.append({
|
||||
"table": table,
|
||||
"brand": display_name_for_suffix(table[len("brand_"):]),
|
||||
"product_name": name,
|
||||
"barcode": str(barcode).strip(),
|
||||
"image_id": image_id,
|
||||
"category": category or "",
|
||||
"size": size,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--apply", action="store_true",
|
||||
help="commit the changes (default is a dry run)")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="explicit no-op; this is already the default")
|
||||
parser.add_argument("--min-similarity", type=float,
|
||||
default=BARCODE_MIN_NAME_SIMILARITY,
|
||||
help=f"name-similarity floor (default {BARCODE_MIN_NAME_SIMILARITY})")
|
||||
args = parser.parse_args()
|
||||
apply = args.apply and not args.dry_run
|
||||
|
||||
logger.info("Target database: %s / %s", DB_HOST, DB_NAME)
|
||||
logger.info("Mode: %s", "APPLY - this writes to nutrition_facts"
|
||||
if apply else "DRY RUN - nothing is written")
|
||||
logger.info("Name-similarity floor: %.2f", args.min_similarity)
|
||||
|
||||
rows = _catalogue_rows()
|
||||
if not rows:
|
||||
logger.error("No barcoded rows found (or no database connection).")
|
||||
return 1
|
||||
logger.info("")
|
||||
logger.info("%d catalogue row(s) carry a barcode.", len(rows))
|
||||
|
||||
invalid: List[Dict] = []
|
||||
lookups: List[Dict] = []
|
||||
for row in rows:
|
||||
(invalid if not _GTIN.match(row["barcode"]) else lookups).append(row)
|
||||
|
||||
if invalid:
|
||||
logger.info("")
|
||||
logger.info("NOT A VALID GTIN - skipped, and wrong in the catalogue "
|
||||
"rather than wrong here (%d):", len(invalid))
|
||||
for row in invalid:
|
||||
logger.info(" %-18s %s", row["barcode"], row["product_name"][:48])
|
||||
|
||||
written = skipped_human = not_in_off = no_nutriments = 0
|
||||
accepted: List[str] = []
|
||||
rejected: List[str] = []
|
||||
thin: List[str] = []
|
||||
|
||||
logger.info("")
|
||||
logger.info("Looking up %d barcode(s) ...", len(lookups))
|
||||
for i, row in enumerate(lookups, start=1):
|
||||
existing = get_nutrition_facts(row["brand"], row["image_id"]) or {}
|
||||
if (existing.get("data_source") or "") in HUMAN_SOURCES:
|
||||
skipped_human += 1
|
||||
continue
|
||||
|
||||
product = fetch_product_by_barcode(row["barcode"])
|
||||
if not product:
|
||||
not_in_off += 1
|
||||
time.sleep(PAUSE_SECONDS)
|
||||
continue
|
||||
|
||||
off_name = product.get("product_name") or ""
|
||||
similarity = name_similarity(off_name, row["product_name"])
|
||||
line = (f"{similarity:5.3f} {row['product_name'][:36]:38} "
|
||||
f"-> {off_name[:36]}")
|
||||
|
||||
# The gate is run HERE, separately, so the report can tell two very
|
||||
# different outcomes apart. Deciding it from the service's
|
||||
# "unavailable" alone conflated them, and the first version of this
|
||||
# report accused a dozen perfectly good barcodes of being wrong when
|
||||
# the real answer was that Open Food Facts holds a near-empty record
|
||||
# for them. One is our data to fix; the other is nobody's fault.
|
||||
candidate = BarcodeCandidate(
|
||||
barcode=row["barcode"], source_name="Open Food Facts",
|
||||
candidate_title=off_name,
|
||||
candidate_brand=product.get("brands") or "",
|
||||
candidate_size=product.get("quantity") or "",
|
||||
)
|
||||
matched, _sim = is_match(candidate, row["brand"], row["product_name"],
|
||||
row["size"], min_name_similarity=args.min_similarity)
|
||||
if not matched:
|
||||
rejected.append(line)
|
||||
time.sleep(PAUSE_SECONDS)
|
||||
continue
|
||||
|
||||
facts = fetch_verified_nutrition_by_barcode(
|
||||
row["barcode"], row["brand"], row["product_name"],
|
||||
row["size"], row["category"],
|
||||
min_name_similarity=args.min_similarity,
|
||||
)
|
||||
if facts.get("data_status") == "unavailable":
|
||||
# Gate passed, so this IS our product - OFF simply has no usable
|
||||
# numbers for it. Nothing to fix on either side.
|
||||
no_nutriments += 1
|
||||
thin.append(line)
|
||||
else:
|
||||
accepted.append(line)
|
||||
if apply:
|
||||
facts.update({
|
||||
"brand": row["brand"],
|
||||
"image_id": row["image_id"],
|
||||
"product_name": row["product_name"],
|
||||
"category": row["category"],
|
||||
})
|
||||
if upsert_nutrition_facts(facts):
|
||||
written += 1
|
||||
else:
|
||||
written += 1
|
||||
|
||||
if i % 25 == 0:
|
||||
logger.info(" %d/%d", i, len(lookups))
|
||||
time.sleep(PAUSE_SECONDS)
|
||||
|
||||
# ---- the report --------------------------------------------------------
|
||||
logger.info("")
|
||||
logger.info("ACCEPTED (%d) - barcode found AND the record is our product:",
|
||||
len(accepted))
|
||||
for line in sorted(accepted, reverse=True):
|
||||
logger.info(" %s", line)
|
||||
|
||||
if thin:
|
||||
logger.info("")
|
||||
logger.info("MATCHED BUT EMPTY (%d) - the right product, but Open Food "
|
||||
"Facts holds no usable nutrient values. Nothing wrong with "
|
||||
"our barcode:", len(thin))
|
||||
for line in sorted(thin, reverse=True):
|
||||
logger.info(" %s", line)
|
||||
|
||||
logger.info("")
|
||||
logger.info("SKIPPED (%d) - OFF knows the barcode, but as a different "
|
||||
"product. OUR barcode is the suspect one:", len(rejected))
|
||||
for line in sorted(rejected, reverse=True):
|
||||
logger.info(" %s", line)
|
||||
|
||||
logger.info("")
|
||||
logger.info(" barcoded rows %d", len(rows))
|
||||
logger.info(" not a valid GTIN %d", len(invalid))
|
||||
logger.info(" human-entered, kept %d", skipped_human)
|
||||
logger.info(" not in Open Food Facts %d", not_in_off)
|
||||
logger.info(" matched but empty %d", no_nutriments)
|
||||
logger.info(" found, wrong product %d", len(rejected))
|
||||
logger.info(" %s %d", "WRITTEN " if apply else "would write ", written)
|
||||
|
||||
logger.info("")
|
||||
if apply:
|
||||
logger.info("Done. nutrition_facts updated; no brand table was touched.")
|
||||
else:
|
||||
logger.info("Dry run - nothing written. Re-run with --apply to commit.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user