nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dagster Orchestration

Browse Source
This commit is contained in:
sriram
2026-08-29 14:49:45 +05:30
parent 27d53fa957
commit 998df898db
28 changed files with 1455 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
app/api/batch_common.py
View File

@@ -28,7 +28,7 @@ from dataclasses import dataclass
from typing import List, Optional, Tuple
from fastapi import HTTPException, UploadFile
from pydantic import BaseModel
from pydantic import BaseModel, Field
from app.api.batch_job_store import batch_job_store
from app.core import batch_ingest, batch_worker
@@ -182,6 +182,22 @@ def parse_all(read: List[Tuple[str, bytes]], limits: UploadLimits):
# ---------------------------------------------------------------------------
# Response shape
# ---------------------------------------------------------------------------
class StageOut(BaseModel):
"""One pipeline stage as it happened to one file.
Kept even in slim list responses: eleven of these per file is a few hundred
bytes, unlike the `products` manifest slim exists to drop.
"""
index: int
name: str
rows_done: int = 0
rows_total: int = 0
started_at: Optional[float] = None
# None while the stage is still running.
finished_at: Optional[float] = None
class BatchFileOut(BaseModel):
index: int
filename: str
@@ -196,7 +212,12 @@ class BatchFileOut(BaseModel):
total_stages: int = pipeline.TOTAL_STAGES
rows_done: int = 0
rows_total: int = 0
# The stages this file has been through, so a FINISHED file can still show
# its timeline. The scalars above only ever say where it is right now.
stages: List[StageOut] = Field(default_factory=list)
size_bytes: int = 0
started_at: Optional[float] = None
finished_at: Optional[float] = None
result: Optional[dict] = None
@@ -213,6 +234,14 @@ class BatchOut(BaseModel):
current_file: Optional[str] = None
use_llm: bool
fetch_images: bool
# Which executor owns this batch - "inprocess" or "dagster". A dagster batch
# sits queued until the orchestrator picks it up, which the UI has to be
# able to say out loud rather than showing a run that looks stuck.
runner: str = batch_ingest.RUNNER_INPROCESS
# The 11 stage names, in order, so a client can draw the whole pipeline
# before a file has entered any of it. Served rather than duplicated in the
# frontend so the two cannot drift when a stage is added.
stage_names: List[str] = Field(default_factory=lambda: list(pipeline.STAGE_NAMES))
totals: dict
brands: List[str]
files: List[BatchFileOut]
@@ -287,6 +316,49 @@ def stage_and_queue(
return manifest, True
def stage_for_orchestrator(
valid: List[Tuple[str, bytes, int]],
invalid: List[Tuple[str, str]],
*,
use_llm: bool,
fetch_images: bool,
submitted_by: Optional[str] = None,
) -> batch_ingest.BatchManifest:
"""Publish a batch for Dagster to claim, and hand it to no one else.
Same staging as `stage_and_queue`, minus the `batch_worker.submit`. The
batch is left `queued` and stamped `runner="dagster"`, which is what
`_pick_batch_id` and `batch_upload_sensor` filter on; the in-process worker
never scans for work, so leaving it unsubmitted is enough to keep the two
executors off each other's batches.
A third function rather than a `runner=` argument on `stage_and_queue`, for
the reason given in `stage_pending`: whether work starts here or somewhere
else is not the kind of decision that should hang off a boolean anyone can
flip later.
The batch sits queued until an orchestrator actually runs - which, if
`dagster dev` is not up, is never. Callers are expected to say so rather
than present it as a run in progress.
"""
manifest = batch_ingest.stage_batch(
[(name, contents) for name, contents, _n in valid],
use_llm=use_llm,
fetch_images=fetch_images,
invalid=invalid,
submitted_by=submitted_by,
)
for entry, (_name, _contents, rows) in zip(manifest.files, valid):
entry.rows_total = rows
manifest.runner = batch_ingest.RUNNER_DAGSTER
manifest.detail = (
"Waiting for the Dagster orchestrator to pick this batch up."
)
batch_ingest.write_manifest(manifest)
batch_job_store.put(manifest)
return manifest
def stage_pending(
valid: List[Tuple[str, bytes, int]],
invalid: List[Tuple[str, str]],

65
app/api/routers/batch_catalog.py
View File

@@ -202,7 +202,13 @@ def get_catalog_batch(batch_id: str) -> BatchOut:
@router.post("/batches/{batch_id}/resume", dependencies=[Depends(require_admin)])
def resume_catalog_batch(batch_id: str) -> BatchOut:
"""Re-queue a batch a restart cut short, or one that was queued behind a full queue."""
"""Re-queue a batch a restart cut short, or one that was queued behind a full queue.
Also the way out of a batch staged for Dagster that no orchestrator ever
came for - the "run it here instead" button. Because this hands the batch to
THIS container's worker, it also takes ownership: the runner is flipped to
`inprocess` so Dagster will not claim a batch that is already running here.
"""
manifest = batch_ingest.read_manifest(batch_id)
if not manifest:
raise HTTPException(status_code=404, detail="Batch not found")
@@ -217,6 +223,7 @@ def resume_catalog_batch(batch_id: str) -> BatchOut:
batch_job_store.clear_cancel(batch_id)
manifest.status = batch_ingest.QUEUED
manifest.detail = None
manifest.runner = batch_ingest.RUNNER_INPROCESS
batch_ingest.write_manifest(manifest)
batch_job_store.put(manifest)
@@ -314,6 +321,10 @@ class InboxStartRequest(InboxSelection):
# belongs to the person who can see what the machine is already doing.
use_llm: bool = False
fetch_images: bool = False
# Who runs it. "inprocess" is this container's worker thread and is the
# default, so an existing client that never sends the field is unaffected.
# "dagster" stages the batch and leaves it for the orchestrator to claim.
runner: str = batch_ingest.RUNNER_INPROCESS
class InboxDismissOut(BaseModel):
@@ -405,6 +416,14 @@ def start_batch_from_inbox(request: InboxStartRequest) -> BatchOut:
The originals are removed afterwards, so the same sheet cannot be started
twice from a stale checkbox in another tab.
"""
if request.runner not in batch_ingest.RUNNERS:
raise HTTPException(
status_code=400,
detail="Unknown runner {!r}. Expected one of: {}.".format(
request.runner, ", ".join(sorted(batch_ingest.RUNNERS))
),
)
grouped = _parse_file_ids(request.file_ids)
if not grouped:
raise HTTPException(status_code=400, detail="No files were selected.")
@@ -447,23 +466,35 @@ def start_batch_from_inbox(request: InboxStartRequest) -> BatchOut:
unique_senders = sorted(set(senders))
submitted_by = ", ".join(unique_senders)[:120] if unique_senders else None
manifest, started = batch_common.stage_and_queue(
picked,
[],
use_llm=request.use_llm,
fetch_images=request.fetch_images,
submitted_by=submitted_by,
)
if not started:
raise HTTPException(
status_code=429,
detail=(
"Too many batches are already queued. These files have been "
"taken out of the inbox and saved as batch "
f"{manifest.batch_id} - press Resume on it once the current "
"batch finishes."
),
if request.runner == batch_ingest.RUNNER_DAGSTER:
# Staged and left alone: Dagster claims it on its next sensor tick, or
# from the Launchpad. Nothing here waits on that, and the batch is
# durable either way.
manifest = batch_common.stage_for_orchestrator(
picked,
[],
use_llm=request.use_llm,
fetch_images=request.fetch_images,
submitted_by=submitted_by,
)
else:
manifest, started = batch_common.stage_and_queue(
picked,
[],
use_llm=request.use_llm,
fetch_images=request.fetch_images,
submitted_by=submitted_by,
)
if not started:
raise HTTPException(
status_code=429,
detail=(
"Too many batches are already queued. These files have been "
"taken out of the inbox and saved as batch "
f"{manifest.batch_id} - press Resume on it once the current "
"batch finishes."
),
)
# Only now, once the bytes are safely staged under a new id. Retiring them
# first would lose the files outright if staging then failed.

58
app/api/routers/user_products.py
View File

@@ -173,7 +173,13 @@ _KEYWORD_RULES: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
("description", ("description", "desc", "detail")),
("category", ("category", "segment")),
("brand", ("brand", "manufacturer", "company")),
("size_variants", ("size", "pack", "weight", "volume", "net qty", "quantity")),
# "net qty" / "net quantity" is the Indian labelling term for a pack size.
# A bare "Quantity" column is not: in a store sheet it is how many units the
# shop has or is ordering, and mapping it here made a case-pack count of 72
# into the pack size, which then got appended to the product name. Worse,
# mapping is first-wins by column position, so a leading "Quantity" column
# also shut out the sheet's real "Pack Size" column.
("size_variants", ("size", "pack", "weight", "volume", "net qty", "net quantity")),
("providers", ("provider", "platform", "marketplace", "available at")),
("highlights", ("highlight", "feature", "benefit")),
("nutrients", ("nutrient", "nutrition")),
@@ -186,12 +192,38 @@ _KEYWORD_RULES: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
_BLANK_VALUES = frozenset({"", "nan", "none", "null", "na", "n/a", "-", "--", "#n/a"})
# Fields whose value is a NAME, and which therefore must not be fed from a
# sheet's id/code column for the same concept. The keyword rules match on
# substring, so "categoryid" satisfies the "category" rule and a column of
# 1/2/3 ends up stored as the product's category. Only these two fields need
# the guard: "hsn code" and "barcode" ARE identifier fields and must keep
# matching their rules.
_NAME_ONLY_FIELDS = frozenset({"category", "brand"})
_IDENTIFIER_SUFFIXES = ("id", "ids", "code", "codes", "no", "num", "number")
def _is_identifier_header(normalized: str) -> bool:
"""True when a header names an id/code column rather than a name column,
covering both "categoryid" and "category id" (the normalizer strips the
underscore in "category_id" to a space, and nothing at all from
"categoryid")."""
return any(
normalized.endswith(suffix) and normalized[: -len(suffix)].strip()
for suffix in _IDENTIFIER_SUFFIXES
)
def _canonical_field(normalized: str) -> Optional[str]:
exact = _EXACT_HEADERS.get(normalized)
if exact:
return exact
for canonical, keywords in _KEYWORD_RULES:
if any(keyword in normalized for keyword in keywords):
if canonical in _NAME_ONLY_FIELDS and _is_identifier_header(normalized):
# An id column for a name field: claim nothing, so the real
# name column (if the sheet has one) is still free to match and
# the id is reported back under `unrecognised` in the preview.
return None
return canonical
return None
@@ -453,14 +485,24 @@ def _build_product_dict(req: AddProductRequest, brand_parent: str,
if s3_urls:
final_image_urls = list(s3_urls)
# 2. Inherit from brand sample
if not final_image_urls and sample_existing.get("image_urls"):
final_image_urls = list(sample_existing.get("image_urls"))
# 3. Canonical S3 fallback URL
# A product's images are NOT inheritable from its brand.
#
# This used to fall back to `sample_existing["image_urls"]` - the
# `_brand_sample()` row, i.e. one arbitrary product of the brand,
# resolved once and reused for every row in the upload. It copied that
# product's photographs verbatim onto every image-less sibling, which is
# why Marie Gold and Milk Bikis both shipped carrying four
# `britannia_..._good_day_cashew_cookies_200g/` URLs while their own
# image_ids were perfectly correct. Another product's photo is never a
# defensible default for this one, so there is no fallback here.
#
# Nor is a URL invented. The old canonical fallback guessed
# `https://nearledaily.s3.ap-south-1.amazonaws.com/...` while the
# configured bucket is DigitalOcean Spaces (see S3_ENDPOINT), so it
# produced a guaranteed 404 that merely looked like an image. Leaving
# the list empty lets ProductCard render its real "no image" state.
if not final_image_urls:
canonical_s3 = f"https://nearledaily.s3.ap-south-1.amazonaws.com/daily/brands/{brand_slug}/{image_id}/image_000.jpg"
final_image_urls = [canonical_s3]
logger.info("No image found for '%s' (%s)", product_name, image_id)
primary_image_url = final_image_urls[0] if final_image_urls else None
search_text = f"{brand_parent} {product_name} {category} {description} {price_range}"

124
app/core/batch_ingest.py
View File

@@ -92,6 +92,11 @@ RETIRED = "retired"
TERMINAL_BATCH_STATES = {DONE, FAILED, PARTIAL, CANCELLED, RETIRED}
# Who runs a batch. See `BatchManifest.runner`.
RUNNER_INPROCESS = "inprocess"
RUNNER_DAGSTER = "dagster"
RUNNERS = {RUNNER_INPROCESS, RUNNER_DAGSTER}
# `..`, separators and drive letters all stripped. UploadFile.filename is
# attacker-controlled in the general case, and it is used to build a path.
_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
@@ -110,6 +115,31 @@ def _safe_name(filename: str) -> str:
return base[:120]
@dataclass
class StageRecord:
"""One of the 11 pipeline stages, as it happened to one file.
The scalar `stage_index`/`stage_name` fields below say where a file is *now*
and are overwritten on every tick, so once a file finishes there is no trace
of what it went through. This keeps that trace: a completed file can still
show its whole timeline, which is the point of the orchestration view.
One record per stage INDEX, not per callback. Stages 8-11 run once per brand
(`store_catalog_pipeline.run_pipeline` loops `for brand, rows in
by_brand.items()`), so a three-brand sheet reports 8,9,10,11 three times
over. Those fold into the same record - earliest start, latest finish,
largest row counts - so a file always has at most eleven of these however
many brands its rows land in.
"""
index: int # 1-based, matching STAGE_NAMES
name: str
rows_done: int = 0
rows_total: int = 0
started_at: Optional[float] = None
finished_at: Optional[float] = None
@dataclass
class BatchFile:
"""One spreadsheet inside a batch, and how far it got."""
@@ -129,6 +159,9 @@ class BatchFile:
total_stages: int = pipeline.TOTAL_STAGES
rows_done: int = 0
rows_total: int = 0
# The stages this file has entered so far, in the order it entered them.
# Empty while queued; eleven entries once the pipeline has run through.
stages: List[StageRecord] = field(default_factory=list)
result: Optional[Dict[str, Any]] = None
started_at: Optional[float] = None
finished_at: Optional[float] = None
@@ -149,6 +182,18 @@ class BatchManifest:
# batch an admin uploaded directly. Carried so the Batch tab can say where
# a run came from instead of leaving it to be guessed from filenames.
submitted_by: Optional[str] = None
# Which executor owns this batch: the API's worker thread, or Dagster.
#
# Both watch the same directory and both can run the same `run_batch`, and
# until this field existed nothing arbitrated between them - enabling
# `batch_upload_sensor` beside a running API meant both claimed every queued
# batch and ingested it twice. The worker only ever runs what is explicitly
# submitted to it, so the field is really a claim check for the Dagster
# side: `_pick_batch_id` and the sensor ignore anything not marked "dagster".
#
# Defaults to INPROCESS so every manifest written before this existed, and
# every batch an admin uploads directly, keeps behaving exactly as it did.
runner: str = RUNNER_INPROCESS
files: List[BatchFile] = field(default_factory=list)
# -- derived, recomputed rather than stored, so they cannot drift ---------
@@ -222,6 +267,7 @@ class BatchManifest:
"updated_at": self.updated_at,
"detail": self.detail,
"submitted_by": self.submitted_by,
"runner": self.runner,
"files_total": self.files_total,
"files_done": self.files_done,
"files_failed": self.files_failed,
@@ -234,10 +280,18 @@ class BatchManifest:
@classmethod
def from_dict(cls, raw: Dict[str, Any]) -> "BatchManifest":
allowed = set(BatchFile.__dataclass_fields__)
files = [
BatchFile(**{k: v for k, v in entry.items() if k in allowed})
for entry in (raw.get("files") or [])
]
stage_fields = set(StageRecord.__dataclass_fields__)
files = []
for entry in raw.get("files") or []:
fields_ = {k: v for k, v in entry.items() if k in allowed}
# `asdict` flattened these to plain dicts on the way out; rebuild
# them so callers get StageRecords whichever direction the manifest
# came from (live object, or re-read off disk after a restart).
fields_["stages"] = [
StageRecord(**{k: v for k, v in stage.items() if k in stage_fields})
for stage in (entry.get("stages") or [])
]
files.append(BatchFile(**fields_))
return cls(
batch_id=raw["batch_id"],
status=raw.get("status", QUEUED),
@@ -247,6 +301,7 @@ class BatchManifest:
updated_at=float(raw.get("updated_at") or time.time()),
detail=raw.get("detail"),
submitted_by=raw.get("submitted_by"),
runner=raw.get("runner") or RUNNER_INPROCESS,
files=files,
)
@@ -457,6 +512,53 @@ def _noop_change(manifest: "BatchManifest") -> None:
_PROGRESS_FLUSH_SECONDS = 5.0
def _record_stage(entry: "BatchFile", index: int, name: str,
done: int, total: int, now: float) -> None:
"""Fold one progress tick into `entry.stages`.
Keyed by stage INDEX rather than appended, because the pipeline visits
stages 8-11 once per brand in the sheet: a three-brand file reports
8,9,10,11 three times over, with `rows_total` reset to that brand's group
size each pass. Appending would produce twenty-three entries for eleven
stages and a UI that appears to run backwards. Folding keeps the first
`started_at`, extends `finished_at`, and takes the high-water mark of both
row counts, so the record reads as "this stage, across the whole file".
Entering a stage closes every record before it. That is deliberate rather
than closing only the immediately-previous one: stage 1 emits a single tick
and stages 8-11 interleave, so "everything with a lower index is done" is
the only rule that leaves no record permanently open.
"""
if index <= 0:
return
for stage in entry.stages:
if stage.index < index and stage.finished_at is None:
stage.finished_at = now
for stage in entry.stages:
if stage.index == index:
stage.rows_done = max(stage.rows_done, done)
stage.rows_total = max(stage.rows_total, total)
stage.finished_at = None if done < total else now
return
entry.stages.append(StageRecord(
index=index,
name=name,
rows_done=done,
rows_total=total,
started_at=now,
finished_at=now if total and done >= total else None,
))
def _close_stages(entry: "BatchFile", now: float) -> None:
"""Mark whatever is still open as finished, once the file itself is done."""
for stage in entry.stages:
if stage.finished_at is None:
stage.finished_at = now
def run_batch(
batch_id: str,
*,
@@ -495,6 +597,9 @@ def run_batch(
entry.started_at = time.time()
entry.stage_index = 0
entry.stage_name = ""
# A re-run of an interrupted file starts its timeline over rather than
# appending to the one from the attempt that died.
entry.stages = []
write_manifest(manifest)
on_change(manifest)
@@ -502,13 +607,14 @@ def run_batch(
def progress(stage_index: int, stage_name: str, done: int, total: int,
_entry: BatchFile = entry) -> None:
now = time.time()
_record_stage(_entry, stage_index, stage_name, done, total, now)
_entry.stage_index = stage_index
_entry.stage_name = stage_name
_entry.rows_done = done
_entry.rows_total = total
manifest.updated_at = time.time()
manifest.updated_at = now
on_change(manifest)
now = time.time()
if now - last_flush[0] >= _PROGRESS_FLUSH_SECONDS:
last_flush[0] = now
write_manifest(manifest)
@@ -548,6 +654,9 @@ def run_batch(
entry.detail = str(exc)
entry.finished_at = time.time()
# The last stage never sees a "next stage" tick to close it, and a file
# that raised leaves whichever stage it died in open.
_close_stages(entry, entry.finished_at)
write_manifest(manifest)
on_change(manifest)
@@ -586,6 +695,9 @@ def scan_interrupted() -> List[str]:
entry.stage_index = 0
entry.stage_name = ""
entry.rows_done = 0
# The timeline described an attempt that no longer counts; the
# re-run builds a fresh one.
entry.stages = []
entry.detail = "Interrupted by a restart; queued again."
manifest.status = INTERRUPTED
manifest.detail = "Interrupted by a restart. Press Resume to continue."

79
app/core/catalog_engine.py
View File

@@ -8,6 +8,7 @@ been removed - see app/services/image_search.py and
app/services/playwright_image_fallback.py for details.
"""
import json
import re
import sys
from pathlib import Path
from typing import Dict, List, Any
@@ -376,10 +377,41 @@ class ProductCatalogEngine:
"""Select the best images from a list of URLs based on quality and relevance"""
if not image_urls:
return []
# Words that distinguish THIS product from its brand-mates. The brand
# itself is removed on purpose: every Britannia URL contains
# "britannia", so it separates nothing - what tells Marie Gold from Good
# Day is "marie"/"gold" vs "good"/"day". Pack sizes and short filler
# words are dropped for the same reason.
_brand_words = {w for w in re.split(r"[^a-z0-9]+", (brand or "").lower()) if w}
_distinctive: List[str] = []
for word in re.split(r"[^a-z0-9]+", (product_title or "").lower()):
if (
len(word) > 2
and word not in _brand_words
and word not in _distinctive
and not re.fullmatch(r"\d+(?:kg|g|gm|gms|ml|l|ltr|pcs|n)?", word)
):
_distinctive.append(word)
# Earlier words identify a product more strongly than later ones: a
# title runs brand -> sub-brand -> variant -> size, so in "Coca-Cola
# Sprite Lemon 750ml" the word that separates this product from its
# brand-mates is "sprite", and "lemon" is only a modifier. Without this
# weighting a combo listing that merely shares the modifier
# ("...coca-cola-750-ml-limca-soft-drink-lemon-lime...") ties with the
# real Sprite image and then wins on domain reputation.
_weights = {word: len(_distinctive) - i for i, word in enumerate(_distinctive)}
def _mentions_product(url_lower: str) -> int:
"""How strongly the URL names THIS product, not merely its brand."""
if not _distinctive:
return 1 # nothing to distinguish by; don't penalise anything
return sum(weight for word, weight in _weights.items() if word in url_lower)
# Filter and score images
scored_images = []
for url in image_urls:
if not url or not url.startswith('http'):
continue
@@ -476,14 +508,41 @@ class ProductCatalogEngine:
# Avoid problematic URLs
if any(bad in url_lower for bad in ['encrypted-tbn', 'googleusercontent', 'data:', 'placeholder']):
score -= 5
scored_images.append((score, url))
# Sort by score (highest first) and take top images
scored_images.sort(key=lambda x: x[0], reverse=True)
best_images = [url for score, url in scored_images[:max_images]]
logger.info(f"📸 Selected {len(best_images)} best images for {product_title}")
# Multi-product listings picture several products at once, so even
# when they name this one the photo is not of it alone.
if any(kw in url_lower for kw in ['combo', 'multipack', 'multi-pack', 'pack-of', 'packof', 'assorted', 'variety-pack']):
score -= 8
scored_images.append((_mentions_product(url_lower), score, url))
# Sort by (does the URL name this product, then score).
#
# Relevance has to be a GATE, not another additive term. Domain
# reputation is worth up to +20 here while a matching product word is
# worth +1, so a BigBasket photo of a Limca combo (15 + 2 + 2 + 2 = 21)
# outranked the correct Sprite image on a lesser domain (8 + 2 + 1 + 1 =
# 12) - and index 0 is what becomes the product's `image_url`. That is
# the "top image is not this product" bug. Ranking every URL that names
# the product above every URL that doesn't makes the primary image
# correct, while the existing scoring still orders each group.
#
# Non-matching URLs are kept as a tail rather than dropped: a product
# whose distinctive words never appear in any URL (common for
# CDN-hashed filenames) would otherwise end up with no images at all.
scored_images.sort(key=lambda x: (x[0], x[1]), reverse=True)
best_images = [url for _relevant, _score, url in scored_images[:max_images]]
matched = sum(1 for relevant, _s, _u in scored_images[:max_images] if relevant)
logger.info(
f"📸 Selected {len(best_images)} best images for {product_title} "
f"({matched} naming the product)"
)
if best_images and not matched:
logger.warning(
f"📸 No candidate image URL names {product_title!r} - the primary "
f"image may not be this product"
)
return best_images
def search_with_python(self, query: str, brand: str) -> List[str]:

79
app/core/store_catalog_pipeline.py
View File

@@ -79,7 +79,7 @@ from app.services.category_registry import (
detect_category_from_text,
sanitize_category_language,
)
from app.services.category_units import fix_or_reject_size
from app.services.category_units import fix_or_reject_size, parse_unit
from app.services.embeddings_service import embed_texts
from app.services.enrichment.barcode.stage import BarcodeEnrichmentStage
from app.services.enrichment.hsn_gst.stage import HsnGstEnrichmentStage
@@ -274,10 +274,33 @@ def stage_2_row_intake(row: Dict[str, Any], *, use_llm: bool = True) -> Dict[str
# ---------------------------------------------------------------------------
# Stage 3 - Title & category consistency
# ---------------------------------------------------------------------------
def _is_category_code(category: Any) -> bool:
"""True when `category` is an opaque code rather than a category name.
A sheet with a `categoryid` column (values 1, 2, 3) has that column claimed
by the `category` keyword rule - "category" is a substring of "categoryid" -
so the id lands in the category field. Stored as-is it reaches the user
("Coca-Cola . 1" on the product card, "1" in the sidebar filter) and it
silently disables three downstream systems that are all keyed by category
NAME: the pack-size unit rulebook, HSN/GST enrichment, and category-scoped
search. A number is never a category name, so treat it as not supplied and
let the keyword detector resolve it from the product name instead.
"""
text = str(category or "").strip()
return bool(text) and text.replace(".", "", 1).isdigit()
def stage_3_title_category(row: Dict[str, Any]) -> Dict[str, Any]:
title = row.get("title") or row.get("product_name") or ""
category = row.get("category")
if _is_category_code(category):
row.setdefault("_notes", []).append(
f"category {str(category).strip()!r} is an id, not a name; detecting from the product name"
)
category = None
row["category"] = None
if _blank(category):
category = detect_category_from_text(f"{title} {row.get('description') or ''}")
row["_category_deterministic"] = bool(category)
@@ -311,10 +334,40 @@ def stage_3_title_category(row: Dict[str, Any]) -> Dict[str, Any]:
# ---------------------------------------------------------------------------
# Stage 4 - Pack-size explosion & unit safety
# ---------------------------------------------------------------------------
def _is_unitless_number(size: str) -> bool:
"""True for a bare quantity like "72" - a number carrying no unit.
A store sheet's `Quantity` / `Case Pack` / `Units Per Pack` column is claimed
by the `size_variants` keyword rule in `map_spreadsheet_columns`, so its value
arrives here dressed as a pack size. It is not one: a pack size needs a unit
to mean anything, and keeping the bare number costs twice over. It is
appended to the product name ("Coca-Cola 750ml" becomes "Coca-Cola 750ml 72"),
and it is folded into `image_id`, so the next upload with a different
quantity inserts a duplicate product instead of updating this one.
`fix_or_reject_size` deliberately passes bare numbers through - a number
without a unit contradicts no category - so the check has to happen here,
before the size ever reaches it.
Only a parsed number with no unit token qualifies. A word-only label
("Standard", "Family Pack") parses as (None, None) and is left alone.
"""
value, unit = parse_unit(size)
return value is not None and not unit
def _sizes_for(row: Dict[str, Any]) -> List[str]:
sizes = [str(s).strip() for s in (row.get("size_variants") or []) if str(s).strip()]
declared = [str(s).strip() for s in (row.get("size_variants") or []) if str(s).strip()]
sizes = [s for s in declared if not _is_unitless_number(s)]
for ignored in declared:
if _is_unitless_number(ignored):
row.setdefault("_notes", []).append(
f"ignored pack size {ignored!r}: a number with no unit is a quantity, not a size"
)
if sizes:
return sizes
# Every declared size was a bare quantity (or none were given). Fall through
# to the name, which for a store sheet usually carries the real size already.
match = _SIZE_IN_TITLE.search(row.get("product_name") or "")
if match:
return [match.group(0).strip()]
@@ -391,7 +444,14 @@ def stage_6_images(row: Dict[str, Any], *, enabled: bool = True) -> Dict[str, An
row["image_urls"] = list(best)
row["image_url"] = best[0]
except Exception as exc: # noqa: BLE001 - an image is not worth the row
logger.debug("Image search skipped for %r: %s", row.get("product_name"), exc)
# `warning`, not `debug`: at the default log level a debug line is
# invisible, so a row that silently lost its images looked identical to
# one that never wanted any. The row still survives - an image is not
# worth failing it - but the operator gets told.
logger.warning("Image search failed for %r: %s", row.get("product_name"), exc)
row.setdefault("_notes", []).append(f"image search failed: {exc}")
if _blank(row.get("image_urls")):
row.setdefault("_notes", []).append("no image found for this product")
return row
@@ -696,6 +756,13 @@ def run_pipeline(
progress(4, STAGE_NAMES[3], len(exploded), len(exploded))
result.products_built = len(exploded)
# Stage 4 copies the parent row into each variant, notes included, and the
# parent's notes have just been reported. Clear them so that the collection
# after stage 7 reports only what stages 5-7 add, once per variant, rather
# than repeating stages 1-4 once per pack size.
for row in exploded:
row["_notes"] = []
# ---- stages 5-7, per exploded row --------------------------------------
for index, row in enumerate(exploded, start=1):
stage_5_pricing(row)
@@ -707,6 +774,12 @@ def run_pipeline(
stage_7_sku(row)
progress(7, STAGE_NAMES[6], index, len(exploded))
# Whatever stages 5-7 recorded per variant - most usefully, that a product
# ended up with no image.
for row in exploded:
for note in row.get("_notes") or []:
result.warnings.append(f"row {row.get('_row')}: {note}")
# ---- stages 8-11, grouped by destination brand -------------------------
by_brand: Dict[str, List[Dict[str, Any]]] = {}
for row in exploded:

14
app/services/category_registry.py
View File

@@ -53,7 +53,12 @@ from typing import Dict, List, Optional, Tuple
# this category when sanitizing cross-category language
# out of a generated description.
CATEGORY_REGISTRY: List[Dict[str, object]] = [
{"category": "Biscuits & Cookies", "keywords": ["biscuits", "biscuit", "biscit", "biskut", "cookies", "cookie"], "generic_term": "biscuit"},
# "bikis" is here so that "Britannia Milk Bikis" resolves as the biscuit it
# is. Without it the only keyword in that name is Dairy's "milk", which not
# only mislabels the product but hands stage 4 a volume unit rulebook - the
# exact "Britannia Milk Bikis - 200ml, 500ml, 1L" defect category_units.py
# was written to stop.
{"category": "Biscuits & Cookies", "keywords": ["biscuits", "biscuit", "biscit", "biskut", "cookies", "cookie", "bikis"], "generic_term": "biscuit"},
{"category": "Rusk", "keywords": ["rusks", "rusk"], "generic_term": "rusk"},
{"category": "Crackers", "keywords": ["crackers", "cracker", "saltine"], "generic_term": "cracker"},
{"category": "Cakes & Muffins", "keywords": ["cakes", "cake", "muffins", "muffin"], "generic_term": "bakery item"},
@@ -62,6 +67,13 @@ CATEGORY_REGISTRY: List[Dict[str, object]] = [
{"category": "Candy & Confectionery", "keywords": ["candy", "candies", "toffee", "toffees", "lollipop", "lollipops", "confectionery", "mints", "chewing gum"], "generic_term": "candy"},
{"category": "Snacks", "keywords": ["snacks", "snack", "chips", "namkeen", "wafers", "wafer", "kurkure", "lays"], "generic_term": "snack"},
{"category": "Chocolates", "keywords": ["chocolates", "chocolate", "chocate", "choclate", "cocoa", "cadbury chocolate", "dairy milk"], "generic_term": "chocolate"},
# Listed ahead of "Cooking Oils" so a drink is resolved before that entry's
# very broad bare "oil" keyword gets a chance, and ahead of "Dairy" only in
# keywords it does not share - "milk", "lassi" and "buttermilk" are
# deliberately left to Dairy. Kept free of "soda" (baking soda is a staple,
# not a drink) and of "tea"/"coffee" on their own (those are sold as leaves
# and grounds far more often than as a drink).
{"category": "Beverages", "keywords": ["beverages", "beverage", "soft drink", "soft drinks", "cold drink", "cold drinks", "carbonated", "aerated drink", "cola", "coke", "juice", "juices", "squash", "sharbat", "energy drink", "sports drink", "mineral water", "packaged drinking water", "lemonade", "iced tea", "thums up", "sprite", "fanta", "limca", "maaza", "pepsi", "mirinda"], "generic_term": "beverage"},
{"category": "Cooking Oils", "keywords": ["cooking oil", "edible oil", "sunflower oil", "mustard oil", "vanaspati", "refined oil", "oil", "oils"], "generic_term": "cooking oil"},
{"category": "Atta & Staples", "keywords": ["atta", "wheat flour", "flour", "rice", "dal", "pulses", "staples", "suji", "maida"], "generic_term": "staple product"},
{"category": "Dairy", "keywords": ["milk", "dairy", "cheese", "paneer", "panner", "paner", "paneerr", "curd", "yogurt", "butter", "ghee", "dahi"], "generic_term": "dairy product"},

4
data/sku_sequences/Aachi.json
View File

@@ -1,6 +1,6 @@
{
"_sku_sequences": {
"AACHI-SAM-100": 2,
"AACHI-SAM-200": 2
"AACHI-SAM-100": 8,
"AACHI-SAM-200": 8
}
}

2
data/sku_sequences/Bikaji.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"BIKAJI-ALO-200": 2
"BIKAJI-ALO-200": 8
}
}

2
data/sku_sequences/Idhayam.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"IDHAYA-SES-500": 2
"IDHAYA-SES-500": 8
}
}

2
data/sku_sequences/amul.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"AMUL-BUT-500": 2
"AMUL-BUT-500": 8
}
}

19
data/sku_sequences/britannia.json
View File

@@ -1,7 +1,20 @@
{
"_sku_sequences": {
"BRITAN-GOO-100": 2,
"BRITAN-GOO-200": 2,
"BRITAN-GOO-500": 2
"BRITAN-GOO-100": 10,
"BRITAN-GOO-200": 10,
"BRITAN-GOO-500": 9,
"BRITAN-GOO-250": 1,
"BRITAN-MAR-100": 2,
"BRITAN-MAR-250": 2,
"BRITAN-MAR-500": 1,
"BRITAN-MIL-200": 2,
"BRITAN-MIL-500": 1,
"BRITAN-MIL-1": 1,
"BRITAN-GOO-375": 1,
"BRITAN-MAR-200": 1,
"BRITAN-MAR-375": 1,
"BRITAN-MIL-100": 1,
"BRITAN-MIL-375": 1,
"BRITAN-MIL-150": 1
}
}

2
data/sku_sequences/cadbury.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"CADBUR-DAI-150": 2
"CADBUR-DAI-150": 8
}
}

5
data/sku_sequences/coca_cola.json Normal file
View File

@@ -0,0 +1,5 @@
{
"_sku_sequences": {
"COCACO-750-750": 3
}
}

2
data/sku_sequences/itc.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"ITC-AAS-1": 2
"ITC-AAS-1": 8
}
}

6
data/sku_sequences/nestle.json
View File

@@ -1,7 +1,7 @@
{
"_sku_sequences": {
"NESTLE-MUN-20": 2,
"NESTLE-MUN-55": 2,
"NESTLE-MUN-150": 2
"NESTLE-MUN-20": 8,
"NESTLE-MUN-55": 8,
"NESTLE-MUN-150": 8
}
}

2
data/sku_sequences/parle.json
View File

@@ -1,5 +1,5 @@
{
"_sku_sequences": {
"PARLE-G-250": 2
"PARLE-G-250": 8
}
}

12
orchestration/README.md
View File

@@ -260,8 +260,16 @@ process - see `app/core/batch_worker.py` for why one worker rather than a
thread per upload. The job here exists so the graph is inspectable and a batch
can be re-run from the Launchpad on a development machine.
Do not turn `batch_upload_sensor` on next to a running API: both would pick up
the same staged batch. That is why it ships STOPPED like the rest.
`batch_upload_sensor` is safe to run next to a live API. Each batch carries a
`runner` field naming its owner, and the sensor (and `_pick_batch_id`) claim
only `runner == "dagster"` — the batches an admin sent here from **Admin →
Dagster Orchestration**. Anything staged for the API's own worker is left
alone, so the two no longer race for the same files. It still ships STOPPED
like the rest: a development machine should not begin ingesting just because a
directory has something in it.
Passing an explicit `{"batch_id": "..."}` in the Launchpad bypasses the runner
filter — that is a person naming a batch, not a poll.
Port 3030, not Dagster's default 3000 - `serve.py` binds `PORTS=3000,8000` and
the frontend nginx also listens on 3000.

60
orchestration/assets/batch_catalog.py
View File

@@ -65,18 +65,30 @@ def _pick_batch_id(config: BatchConfig) -> str:
if config.batch_id:
return config.batch_id
# `runner` is the claim check. The API's worker thread and this asset both
# read the same directory and both call the same run_batch, so without it
# every queued batch was fair game to both and enabling batch_upload_sensor
# beside a running API ingested each batch twice. An admin choosing
# "Dagster" in the orchestration tab is what stamps a batch for this side.
#
# An explicit config.batch_id above bypasses this on purpose: that is a
# person naming a batch in the Launchpad, which is an instruction, not a
# poll.
waiting = [
m for m in batch_ingest.list_manifests()
if m.status in (batch_ingest.QUEUED, batch_ingest.INTERRUPTED)
and m.runner == batch_ingest.RUNNER_DAGSTER
]
if not waiting:
raise Failure(
description=(
"No batch_id given and no staged batch is waiting to run.\n\n"
"Stage one through POST /api/admin/catalog-batch/ingest (or "
"POST /api/uploads/catalog), or pass {\"batch_id\": \"...\"} in "
"the Launchpad. Staged batches live under BATCH_UPLOAD_DIR "
f"({batch_ingest.batch_root()})."
"No batch_id given and no staged batch is waiting for Dagster.\n\n"
"Stage one from the Admin -> Dagster Orchestration tab (tick "
"files, Start selected), or pass {\"batch_id\": \"...\"} in the "
"Launchpad to run a specific batch regardless of its runner. "
"Batches staged for this container's own worker are "
"deliberately not picked up here. Staged batches live under "
f"BATCH_UPLOAD_DIR ({batch_ingest.batch_root()})."
),
metadata={"staged_batches": len(batch_ingest.list_manifests())},
)
@@ -208,9 +220,16 @@ def batch_catalog_rows(
the write has already happened - checking there would be checking after the
fact. backend/.env points at production; see orchestration/config.py.
Per-file progress is logged rather than pushed anywhere. In production the
same callback feeds the polling endpoint; in Dagster the run log IS the
progress view, and writing to both would be two sources of truth.
Stage progress is logged here, one line per stage per file, so the run log
is a readable progress view rather than a wall of silence between "Ingesting
x.xlsx" and the final metadata.
It is not pushed anywhere from this callback, and does not need to be:
`run_batch` already flushes the manifest to disk every few seconds, and the
API reads batches from that same manifest. So the Admin orchestration tab
follows a Dagster run through the ordinary batch endpoints, with no channel
between the two processes beyond the file both already use - which is what
keeps that tab working in production, where Dagster does not exist.
"""
from app.core import batch_ingest
@@ -225,13 +244,32 @@ def batch_catalog_rows(
manifest.fetch_images = bool(config.fetch_images or manifest.fetch_images)
batch_ingest.write_manifest(manifest)
seen = {"file": None}
# (filename, stage_index) of the last line written, so a callback that
# fires many times a second produces one line per stage rather than per row.
seen = {"at": None}
def on_change(current):
name = current.current_file
if name and name != seen["file"]:
seen["file"] = name
if not name:
return
entry = next(
(f for f in current.files if f.filename == name and f.status == "running"),
None,
)
if entry is None:
return
at = (name, entry.stage_index)
if at == seen["at"]:
return
seen["at"] = at
if entry.stage_index <= 0:
context.log.info("Ingesting %s", name)
else:
context.log.info(
"%s - stage %d/%d %s (%d/%d rows)",
name, entry.stage_index, entry.total_stages, entry.stage_name,
entry.rows_done, entry.rows_total,
)
result = batch_ingest.run_batch(batch_id, on_change=on_change)
totals = result.totals()

14
orchestration/schedules.py
View File

@@ -172,8 +172,13 @@ def batch_upload_sensor(context: SensorEvaluationContext):
NOTE ON THE NORMAL PRODUCTION PATH: nothing here is involved. The API runs
a staged batch itself, on a bounded worker thread, and this sensor exists
for the development machine where Dagster is the thing driving the work.
Turning it on alongside a running API would mean both trying to ingest the
same batch, which is why it ships STOPPED.
It is now safe to run alongside a live API. Both executors read the same
directory, but a batch carries a `runner` naming which one owns it, and
this sensor claims only `runner == "dagster"` - the batches an admin sent
here from the orchestration tab. It still ships STOPPED, because a
development machine should not start ingesting because a directory
happened to have something in it.
"""
from app.core import batch_ingest
@@ -181,12 +186,15 @@ def batch_upload_sensor(context: SensorEvaluationContext):
if not root.exists():
return SkipReason("Batch upload directory {} does not exist.".format(root))
# Only batches an admin explicitly handed to Dagster. See the same filter,
# and the reason for it, in assets/batch_catalog.py:_pick_batch_id.
waiting = [
m for m in batch_ingest.list_manifests()
if m.status in (batch_ingest.QUEUED, batch_ingest.INTERRUPTED)
and m.runner == batch_ingest.RUNNER_DAGSTER
]
if not waiting:
return SkipReason("No staged batch is waiting to run.")
return SkipReason("No staged batch is waiting for Dagster.")
already = set(filter(None, (context.cursor or "").split(",")))
fresh = [m for m in waiting if m.batch_id not in already]

261
scripts/repair_catalog.py Normal file
View File

@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""
Repair catalog rows already stored by a buggy ingestion run.
Fixing the pipeline only helps the NEXT upload. Rows written before the fix keep
their corrupted values, so this walks the brand tables and repairs them in
place:
* **name** - strips a trailing bare number that a mis-mapped `Quantity` /
`Case Pack` column contributed ("Coca-Cola 750ml 72").
* **category** - re-resolves a value that is a numeric id ("1", "2") or is
otherwise absent from the canonical taxonomy, using the same
`detect_category_from_text` the pipeline uses.
* **images** - clears `image_url` / `image_urls` that point at a *different*
product's image folder. That is the brand-sample bleed which
put Good Day photographs on Marie Gold and Milk Bikis. Cleared
rather than re-pointed, because there is nothing correct to
point them at; `--refetch-images` can then fill them back in.
Two things it deliberately never does:
* It never recomputes `image_id`. That column is the UNIQUE upsert key AND the
S3 folder name, so changing it would orphan the uploaded images and make the
next ingest insert a duplicate instead of updating the row. Only display
fields are rewritten.
* It never calls `upsert_brand_products(..., cleanup=True)`, which deletes
every row not in the batch it was handed.
Usage:
python -m scripts.repair_catalog --brands coca-cola,britannia # dry run
python -m scripts.repair_catalog --brands coca-cola,britannia --apply
python -m scripts.repair_catalog --all --apply --refetch-images
`--dry-run` is the default and `--apply` must be given explicitly: this rewrites
whatever database `backend/.env` points at, which may well be production. The
target host is printed on startup so it can be checked before committing.
"""
from __future__ import annotations
import argparse
import logging
import re
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.infrastructure.settings import DB_HOST, DB_NAME
from app.services.category_registry import (
ALL_CATEGORIES,
detect_category_from_text,
)
from app.services.category_units import parse_unit
from app.services.vector_store import (
_connect,
_sanitize_name,
list_available_brands,
resolve_parent_brand,
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
_CANONICAL = {c.strip().lower() for c in ALL_CATEGORIES}
# "General" is what the pipeline stores when detection genuinely found nothing.
# It is a real decision, not corruption, so it is left alone.
_ACCEPTED_CATEGORIES = _CANONICAL | {"general"}
# ---------------------------------------------------------------------------
# Row-level repairs (pure, and unit-testable without a database)
# ---------------------------------------------------------------------------
def repair_product_name(product_name: str) -> Optional[str]:
"""Strip a trailing bare number that is not a pack size.
Returns the corrected name, or None when nothing needed changing. Only a
trailing token is considered, and only one: the pipeline appended exactly
one size, and stripping deeper would start eating real product names
("Britannia 50-50", "Sprite 7Up 250ml").
"""
name = (product_name or "").strip()
if not name:
return None
head, _, last = name.rpartition(" ")
if not head:
return None
value, unit = parse_unit(last)
# A trailing token that parses as a number with NO unit is the bug. A real
# pack size ("750ml") has a unit; a word ("Pack") parses as (None, None).
if value is None or unit:
return None
# Guard against a name that is genuinely numeric at the end, e.g. "5 Star"
# reversed, or a variant number the store means ("Maggi 2"). Only strip when
# the remaining name still carries letters.
if not re.search(r"[a-zA-Z]", head):
return None
return head.strip()
def repair_category(category: Optional[str], product_name: str,
description: str = "") -> Optional[str]:
"""Re-resolve a category that is an id or is not a known category name."""
current = (category or "").strip()
if current.lower() in _ACCEPTED_CATEGORIES:
return None
detected = detect_category_from_text(f"{product_name} {description}".strip())
resolved = detected or "General"
return resolved if resolved != current else None
def _folder_of(url: str) -> str:
"""The `{image_id}` path segment of a stored image URL, if it has one."""
match = re.search(r"/brands/[^/]+/([^/]+)/", url or "")
return match.group(1) if match else ""
def repair_images(image_id: str, image_url: Optional[str],
image_urls: Optional[List[str]]) -> Optional[Tuple[None, list]]:
"""Detect images belonging to a different product and clear them.
Only URLs that carry an explicit `{image_id}` folder can be judged, so a
plain CDN link is left alone - it may well be correct and there is no
evidence either way.
"""
urls = list(image_urls or [])
candidates = [u for u in ([image_url] if image_url else []) + urls if u]
folders = {_folder_of(u) for u in candidates}
folders.discard("")
if not folders:
return None
if folders == {image_id}:
return None
# At least one URL is filed under another product's image_id.
return (None, [])
# ---------------------------------------------------------------------------
# Database walk
# ---------------------------------------------------------------------------
def repair_brand(conn, brand: str, *, apply: bool, refetch: bool) -> Dict[str, int]:
table = f"brand_{_sanitize_name(resolve_parent_brand(brand) or brand)}"
counts = {"scanned": 0, "name": 0, "category": 0, "images": 0, "refetched": 0}
with conn.cursor() as cur:
try:
cur.execute(
f"SELECT image_id, product_name, title, category, description, "
f"image_url, image_urls FROM {table}"
)
except Exception as exc: # noqa: BLE001 - an absent table is not fatal
logger.warning("Skipping %s: %s", table, exc)
return counts
rows = cur.fetchall()
for image_id, product_name, title, category, description, image_url, image_urls in rows:
counts["scanned"] += 1
updates: Dict[str, Any] = {}
fixed_name = repair_product_name(product_name)
if fixed_name:
updates["product_name"] = fixed_name
counts["name"] += 1
logger.info("%s: name %r -> %r", table, product_name, fixed_name)
effective_name = fixed_name or product_name
fixed_category = repair_category(category, effective_name, description or "")
if fixed_category:
updates["category"] = fixed_category
counts["category"] += 1
logger.info("%s: category %r -> %r (%s)", table, category, fixed_category, effective_name)
cleared = repair_images(image_id, image_url, image_urls)
if cleared:
updates["image_url"], updates["image_urls"] = cleared
counts["images"] += 1
logger.info("%s: clearing images filed under another product (%s)", table, image_id)
if refetch and (cleared or not (image_urls or image_url)):
found = _refetch_images(brand, effective_name)
if found:
updates["image_url"], updates["image_urls"] = found[0], found
counts["refetched"] += 1
logger.info("%s: refetched %d image(s) for %s", table, len(found), effective_name)
if updates and apply:
assignments = ", ".join(f"{col} = %s" for col in updates)
with conn.cursor() as cur:
cur.execute(
f"UPDATE {table} SET {assignments}, updated_at = CURRENT_TIMESTAMP "
f"WHERE image_id = %s",
list(updates.values()) + [image_id],
)
return counts
def _refetch_images(brand: str, product_name: str) -> List[str]:
"""Re-run the normal image path for one product. Network, hence opt-in."""
try:
from app.core.catalog_engine import catalog_engine
from app.services.image_search import find_all_image_urls
candidates = find_all_image_urls(product_name, brand=brand, max_results=24)
if not candidates:
return []
return list(catalog_engine._select_best_images(candidates, product_name, brand, max_images=10))
except Exception as exc: # noqa: BLE001 - an image is not worth failing the row
logger.warning("Image refetch failed for %r: %s", product_name, exc)
return []
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--brands", help="comma-separated brands, e.g. coca-cola,britannia")
group.add_argument("--all", action="store_true", help="every brand table present")
parser.add_argument("--apply", action="store_true",
help="write the changes (default is a dry run)")
parser.add_argument("--refetch-images", action="store_true",
help="re-run image search for rows left without images (slow, network)")
args = parser.parse_args()
logger.info("Target database: %s/%s", DB_HOST, DB_NAME)
logger.info("Mode: %s", "APPLY - rows will be rewritten" if args.apply else "DRY RUN - no writes")
conn = _connect()
if not conn:
logger.error("Could not connect to PostgreSQL (USE_PGVECTOR off, or DB unreachable).")
return 1
brands = list_available_brands() if args.all else [
b.strip() for b in args.brands.split(",") if b.strip()
]
if not brands:
logger.error("No brands to process.")
return 1
logger.info("Brands: %s", ", ".join(brands))
totals = {"scanned": 0, "name": 0, "category": 0, "images": 0, "refetched": 0}
try:
for brand in brands:
counts = repair_brand(conn, brand, apply=args.apply, refetch=args.refetch_images)
for key, value in counts.items():
totals[key] += value
finally:
conn.close()
logger.info(
"Scanned %d row(s): %d name(s), %d category(ies), %d image set(s) to fix, %d refetched.",
totals["scanned"], totals["name"], totals["category"], totals["images"], totals["refetched"],
)
if not args.apply and any(totals[k] for k in ("name", "category", "images")):
logger.info("Dry run - nothing written. Re-run with --apply to commit these changes.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

177
tests/test_batch_stage_timeline.py Normal file
View File

@@ -0,0 +1,177 @@
"""Tests for the per-file stage timeline and for batch runner ownership.
Two features, tested together because they exist for the same screen: the Admin
"Dagster Orchestration" tab, which shows every file's progress through the 11
named pipeline stages and has to say which executor owns the batch it is
watching.
Fixture conventions follow test_batch_catalog_ingest.py: BATCH_UPLOAD_DIR points
at tmp_path, the storage and embedding boundary is patched on the pipeline
module object, and nothing here touches the network or a real database.
"""
from __future__ import annotations
import io
import pytest
from app.core import batch_ingest
from app.core import store_catalog_pipeline as pipeline
openpyxl = pytest.importorskip("openpyxl")
# Three brands on purpose: stages 8-11 run once per brand, so this is the sheet
# shape that used to produce a jittering stage index and duplicate records.
THREE_BRANDS = [
["Amul Butter 100g", "Dairy", "Amul"],
["Britannia Marie Gold 250g", "Biscuits", "Britannia"],
["Cadbury Dairy Milk 150g", "Chocolate", "Cadbury"],
]
def _sheet(rows=THREE_BRANDS) -> bytes:
wb = openpyxl.Workbook()
ws = wb.active
ws.append(["Product Name", "Category", "Brand"])
for row in rows:
ws.append(row)
buf = io.BytesIO()
wb.save(buf)
return buf.getvalue()
@pytest.fixture(autouse=True)
def _isolate_sku_counter(tmp_path, monkeypatch):
from app.services import sku_service
monkeypatch.setattr(sku_service, "_data_dir", tmp_path / "sku_sequences")
@pytest.fixture(autouse=True)
def batch_root(tmp_path, monkeypatch):
root = tmp_path / "batch_uploads"
monkeypatch.setattr(batch_ingest, "BATCH_UPLOAD_DIR", root)
return root
@pytest.fixture(autouse=True)
def store(monkeypatch):
table: dict = {}
def fake_upsert(brand, rows, cleanup=False):
assert cleanup is False, "cleanup=True would delete the brand's existing catalog"
for row in rows:
table[row["image_id"]] = dict(row)
return len(rows)
monkeypatch.setattr(pipeline, "upsert_brand_products", fake_upsert)
monkeypatch.setattr(pipeline, "get_products_by_brand", lambda b, **kw: list(table.values()))
monkeypatch.setattr(pipeline, "embed_texts", lambda texts: [[0.0] * 384 for _ in texts])
return table
def _run_one_file() -> batch_ingest.BatchFile:
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
result = batch_ingest.run_batch(manifest.batch_id)
return result.files[0]
# ---------------------------------------------------------------------------
# The stage timeline
# ---------------------------------------------------------------------------
def test_a_finished_file_records_every_stage_exactly_once():
"""Stages 8-11 run once per brand. Three brands must still yield eleven
records, not twenty-three."""
entry = _run_one_file()
assert entry.status == batch_ingest.DONE
assert [s.index for s in entry.stages] == list(range(1, pipeline.TOTAL_STAGES + 1))
def test_the_timeline_names_match_the_pipeline():
entry = _run_one_file()
assert [s.name for s in entry.stages] == list(pipeline.STAGE_NAMES)
def test_every_stage_is_closed_once_the_file_finishes():
"""The last stage never sees a following tick to close it, and a file that
raised leaves whichever stage it died in open."""
entry = _run_one_file()
assert all(s.finished_at is not None for s in entry.stages)
assert all(s.started_at <= s.finished_at for s in entry.stages)
def test_the_timeline_survives_a_reread_from_disk():
"""A Dagster run is a different process; the API only ever sees the
manifest that run flushed to disk."""
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
batch_ingest.run_batch(manifest.batch_id)
reloaded = batch_ingest.read_manifest(manifest.batch_id)
stages = reloaded.files[0].stages
assert len(stages) == pipeline.TOTAL_STAGES
assert isinstance(stages[0], batch_ingest.StageRecord)
def test_a_queued_file_has_no_timeline_yet():
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
assert manifest.files[0].stages == []
def test_a_restart_clears_a_half_finished_timeline():
"""It described an attempt that no longer counts; the re-run builds a new
one rather than appending to it."""
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
entry = manifest.files[0]
entry.status = batch_ingest.RUNNING
entry.stages = [batch_ingest.StageRecord(index=1, name="Brand Resolution", started_at=1.0)]
batch_ingest.write_manifest(manifest)
batch_ingest.scan_interrupted()
after = batch_ingest.read_manifest(manifest.batch_id).files[0]
assert after.status == batch_ingest.QUEUED
assert after.stages == []
def test_repeated_ticks_for_one_stage_fold_into_one_record():
"""Directly, so the folding rule is pinned independently of a real run."""
entry = batch_ingest.BatchFile(index=0, filename="x.xlsx", stored_name="x")
batch_ingest._record_stage(entry, 8, "Barcode", 0, 5, 100.0)
batch_ingest._record_stage(entry, 9, "HSN", 5, 5, 101.0)
batch_ingest._record_stage(entry, 8, "Barcode", 0, 9, 102.0) # next brand
batch_ingest._record_stage(entry, 9, "HSN", 9, 9, 103.0)
assert [s.index for s in entry.stages] == [8, 9]
# High-water mark across brands, and the first sighting keeps its start.
assert entry.stages[0].rows_total == 9
assert entry.stages[0].started_at == 100.0
# ---------------------------------------------------------------------------
# Runner ownership
# ---------------------------------------------------------------------------
def test_a_batch_defaults_to_the_in_process_runner():
"""Every manifest written before `runner` existed must keep behaving as it
did, which means the default has to be the API's own worker."""
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
assert manifest.runner == batch_ingest.RUNNER_INPROCESS
def test_the_runner_survives_a_reread_from_disk():
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
manifest.runner = batch_ingest.RUNNER_DAGSTER
batch_ingest.write_manifest(manifest)
assert batch_ingest.read_manifest(manifest.batch_id).runner == batch_ingest.RUNNER_DAGSTER
def test_a_manifest_without_a_runner_field_reads_as_in_process():
"""Manifests staged before this field existed are still on disk."""
manifest = batch_ingest.stage_batch([("catalog.xlsx", _sheet())])
raw = manifest.to_dict()
raw.pop("runner")
assert batch_ingest.BatchManifest.from_dict(raw).runner == batch_ingest.RUNNER_INPROCESS

96
tests/test_image_selection.py Normal file
View File

@@ -0,0 +1,96 @@
"""Tests for image selection - which candidate becomes a product's primary image.
`_select_best_images` returns a ranked list and index 0 becomes the row's
`image_url`, which is what a ProductCard shows. Its scoring is built almost
entirely from URL-string heuristics, and domain reputation is worth up to +20
while a matching product word used to be worth +1. That let a well-known shop's
photo of a *different* product outrank the correct image, which is how a
Coca-Cola product ended up fronted by a Limca combo shot and how every Britannia
biscuit ended up showing Good Day.
These tests pin the ordering rule: a URL that names THIS product outranks one
that only names its brand, however reputable the brand URL's domain.
"""
from __future__ import annotations
from app.core.catalog_engine import catalog_engine
def _top(urls, title, brand):
return catalog_engine._select_best_images(urls, title, brand, max_images=10)[0]
def test_a_product_matching_url_beats_a_higher_domain_score():
"""media.britannia.co.in scores +20 as an official brand domain; the
correct image sits on an anonymous CDN worth +8. Relevance must still win."""
good_day_on_official_domain = "https://media.britannia.co.in/good_day_cashew.jpg"
marie_gold_on_a_cdn = "https://cdn.example.com/marie_gold_biscuit.jpg"
assert _top(
[good_day_on_official_domain, marie_gold_on_a_cdn],
"Britannia Marie Gold 250g",
"Britannia",
) == marie_gold_on_a_cdn
def test_britannia_siblings_each_get_their_own_image():
"""The reported bug: Good Day, Marie Gold and Milk Bikis all showed Good Day."""
candidates = [
"https://media.britannia.co.in/britannia_good_day_cashew_cookies.jpg",
"https://media.britannia.co.in/britannia_marie_gold_biscuits.jpg",
"https://media.britannia.co.in/britannia_milk_bikis_pack.jpg",
]
assert "good_day" in _top(candidates, "Britannia Good Day Cashew Cookies 200g", "Britannia")
assert "marie_gold" in _top(candidates, "Britannia Marie Gold 250g", "Britannia")
assert "milk_bikis" in _top(candidates, "Britannia Milk Bikis 150g", "Britannia")
def test_the_earlier_words_of_a_title_identify_it_more_strongly():
"""A title runs brand -> sub-brand -> variant -> size. A combo listing that
shares only the trailing modifier ("lemon") must not outrank the image that
names the product itself ("sprite")."""
combo = (
"https://www.bigbasket.com/media/uploads/p/xl/1212306_1-bb-combo-coca-cola-"
"soft-drink-original-taste-750-ml-limca-soft-drink-lemon-lime-750-ml.jpg"
)
sprite = "https://cdn.example.com/750ml-sprite-soft-drink-1000x1000.jpg"
assert _top([combo, sprite], "Coca-Cola Sprite Lemon 750ml", "Coca-Cola") == sprite
def test_a_brand_only_url_is_kept_rather_than_discarded():
"""Relevance is a ranking gate, not a filter: a product whose words appear
in no URL (CDN hashes, opaque filenames) must still end up with images."""
opaque = "https://cdn.example.com/a1b2c3d4e5.jpg"
assert catalog_engine._select_best_images(
[opaque], "Britannia Tiger 100g", "Britannia"
) == [opaque]
def test_the_brand_name_alone_does_not_count_as_relevance():
"""Every Britannia URL contains "britannia", so it separates nothing."""
brand_only = "https://media.britannia.co.in/britannia.jpg"
named = "https://cdn.example.com/tiger_glucose.jpg"
assert _top([brand_only, named], "Britannia Tiger Glucose 100g", "Britannia") == named
def test_a_pack_size_alone_does_not_count_as_relevance():
""""750ml" is shared by every drink in the range."""
wrong_product = "https://media.britannia.co.in/thums-up-750ml.jpg"
right_product = "https://cdn.example.com/fanta-orange.jpg"
assert _top(
[wrong_product, right_product], "Coca-Cola Fanta 750ml", "Coca-Cola"
) == right_product
# ---------------------------------------------------------------------------
# Category detection feeding the same products
# ---------------------------------------------------------------------------
def test_milk_bikis_is_a_biscuit_not_a_dairy_product():
""""milk" was the only keyword in the name, so it resolved to Dairy - which
then licenses ml/L pack sizes for a biscuit."""
from app.services.category_registry import detect_category_from_text
assert detect_category_from_text("Britannia Milk Bikis 150g") == "Biscuits & Cookies"
# ...without stealing genuinely dairy products from Dairy.
assert detect_category_from_text("Amul Milk 500ml") == "Dairy"
assert detect_category_from_text("Amul Butter Pasteurised") == "Dairy"

55
tests/test_orchestration_defs.py
View File

@@ -249,3 +249,58 @@ def test_resolve_brands_prefers_explicit_run_config(monkeypatch):
from orchestration.config import resolve_brands
assert resolve_brands(["Britannia"]) == ["Britannia"]
# ---------------------------------------------------------------------------
# Runner ownership - which batches this side is allowed to claim
# ---------------------------------------------------------------------------
@pytest.fixture
def staged(tmp_path, monkeypatch):
"""A batch directory holding one manifest per runner."""
from app.core import batch_ingest
monkeypatch.setattr(batch_ingest, "BATCH_UPLOAD_DIR", tmp_path / "batch_uploads")
def stage(runner):
manifest = batch_ingest.stage_batch([("catalog.csv", b"Product Name\nAmul Butter 100g\n")])
manifest.runner = runner
manifest.status = batch_ingest.QUEUED
batch_ingest.write_manifest(manifest)
return manifest.batch_id
return stage
def test_dagster_ignores_a_batch_owned_by_the_api_worker(staged):
"""Both executors watch the same directory. Before `runner` existed, running
the sensor beside a live API meant both claimed every queued batch and
ingested it twice."""
from dagster import Failure
from orchestration.assets.batch_catalog import _pick_batch_id
from orchestration.config import BatchConfig
staged("inprocess")
with pytest.raises(Failure):
_pick_batch_id(BatchConfig())
def test_dagster_claims_a_batch_staged_for_it(staged):
from orchestration.assets.batch_catalog import _pick_batch_id
from orchestration.config import BatchConfig
staged("inprocess")
mine = staged("dagster")
assert _pick_batch_id(BatchConfig()) == mine
def test_an_explicit_batch_id_bypasses_the_runner_filter(staged):
"""A person naming a batch in the Launchpad is an instruction, not a poll."""
from orchestration.assets.batch_catalog import _pick_batch_id
from orchestration.config import BatchConfig
theirs = staged("inprocess")
assert _pick_batch_id(BatchConfig(batch_id=theirs)) == theirs

95
tests/test_repair_catalog.py Normal file
View File

@@ -0,0 +1,95 @@
"""Tests for the row-level repairs in scripts/repair_catalog.py.
The three repair functions are pure, so they are tested here without a database.
What matters most is what they DON'T touch: this script rewrites live catalog
rows, so a false positive silently destroys a correct product name or a correct
set of images.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from scripts.repair_catalog import repair_category, repair_images, repair_product_name
# ---------------------------------------------------------------------------
# Names
# ---------------------------------------------------------------------------
def test_a_trailing_bare_number_is_stripped():
assert repair_product_name("Coca-Cola 750ml 72") == "Coca-Cola 750ml"
def test_a_real_pack_size_suffix_is_kept():
"""The size suffix is correct behaviour for a multi-pack-size row."""
assert repair_product_name("Britannia Good Day Cashew Cookies 200g") is None
def test_a_clean_name_is_left_alone():
assert repair_product_name("Coca-Cola 750ml") is None
assert repair_product_name("Britannia Marie Gold") is None
def test_a_word_suffix_is_not_a_number():
assert repair_product_name("Britannia 50-50 Family Pack") is None
def test_a_name_that_is_only_a_number_is_not_gutted():
"""Nothing with letters would remain, so there is no safe repair."""
assert repair_product_name("750 72") is None
def test_a_single_token_name_is_untouched():
assert repair_product_name("72") is None
# ---------------------------------------------------------------------------
# Categories
# ---------------------------------------------------------------------------
def test_a_numeric_category_is_resolved_from_the_name():
assert repair_category("1", "Coca-Cola 750ml") == "Beverages"
def test_a_known_category_is_left_alone():
assert repair_category("Biscuits & Cookies", "Britannia Marie Gold") is None
def test_general_is_a_decision_not_corruption():
"""`General` is what the pipeline stores when detection genuinely failed."""
assert repair_category("General", "Something Unclassifiable Xyz") is None
def test_an_unresolvable_numeric_category_becomes_general():
assert repair_category("3", "Mystery Item Xyz") == "General"
# ---------------------------------------------------------------------------
# Images
# ---------------------------------------------------------------------------
_MINE = "https://cdn.example.com/daily/brands/britannia/britannia_marie_gold_250g/image_000.jpg"
_THEIRS = "https://cdn.example.com/daily/brands/britannia/britannia_good_day_200g/image_000.jpg"
def test_images_filed_under_another_product_are_cleared():
assert repair_images("britannia_marie_gold_250g", _THEIRS, [_THEIRS]) == (None, [])
def test_a_products_own_images_are_kept():
assert repair_images("britannia_marie_gold_250g", _MINE, [_MINE]) is None
def test_a_plain_cdn_url_is_not_judged():
"""No image_id folder in the path means no evidence either way."""
plain = "https://cdn.example.com/some/photo.jpg"
assert repair_images("britannia_marie_gold_250g", plain, [plain]) is None
def test_a_row_with_no_images_needs_no_repair():
assert repair_images("britannia_marie_gold_250g", None, []) is None
def test_one_foreign_url_among_several_condemns_the_set():
"""A bled set is not partially trustworthy - it was copied wholesale."""
assert repair_images("britannia_marie_gold_250g", _MINE, [_MINE, _THEIRS]) == (None, [])

71
tests/test_review_inbox.py
View File

@@ -345,3 +345,74 @@ def test_the_inbox_is_closed_to_anonymous(client):
assert client.get(INBOX).status_code == 401
assert client.post(FROM_INBOX, json={"file_ids": []}).status_code == 401
assert client.post(DISMISS, json={"file_ids": []}).status_code == 401
# ---------------------------------------------------------------------------
# Choosing who runs the batch
# ---------------------------------------------------------------------------
def test_starting_a_file_defaults_to_this_containers_worker(client, admin_headers, submitted):
"""An existing client that never sends `runner` must be unaffected."""
batch_id = _drop(client, "priya", "catalog.csv")
started = client.post(FROM_INBOX, json={"file_ids": [f"{batch_id}:0"]},
headers=admin_headers)
assert started.json()["runner"] == batch_ingest.RUNNER_INPROCESS
assert submitted == [started.json()["batch_id"]]
def test_a_dagster_batch_is_staged_but_handed_to_no_worker(client, admin_headers, submitted):
"""The whole point of the runner field: Dagster claims this one, so the
in-process worker must never be given it."""
batch_id = _drop(client, "priya", "catalog.csv")
started = client.post(
FROM_INBOX,
json={"file_ids": [f"{batch_id}:0"], "runner": "dagster"},
headers=admin_headers,
)
assert started.status_code == 202, started.text
run = started.json()
assert run["runner"] == batch_ingest.RUNNER_DAGSTER
assert run["status"] == batch_ingest.QUEUED
assert submitted == [], "a dagster batch must not reach the in-process worker"
# And it still left the inbox, so it cannot be started twice.
assert client.get(INBOX, headers=admin_headers).json()["pending_count"] == 0
def test_an_unknown_runner_is_refused(client, admin_headers, submitted):
batch_id = _drop(client, "priya", "catalog.csv")
started = client.post(
FROM_INBOX,
json={"file_ids": [f"{batch_id}:0"], "runner": "kubernetes"},
headers=admin_headers,
)
assert started.status_code == 400
assert "kubernetes" in started.json()["detail"]
assert submitted == []
# Refused before anything was taken out of the inbox.
assert client.get(INBOX, headers=admin_headers).json()["pending_count"] == 1
def test_running_a_stranded_dagster_batch_here_takes_ownership(client, admin_headers, submitted):
"""Resume is the "run it here instead" button for a batch no orchestrator
came for. It must also claim the batch, or Dagster could still pick up
something already running in this process."""
batch_id = _drop(client, "priya", "catalog.csv")
run = client.post(
FROM_INBOX,
json={"file_ids": [f"{batch_id}:0"], "runner": "dagster"},
headers=admin_headers,
).json()
resumed = client.post(
f"/api/admin/catalog-batch/batches/{run['batch_id']}/resume",
headers=admin_headers,
)
assert resumed.status_code == 200, resumed.text
assert resumed.json()["runner"] == batch_ingest.RUNNER_INPROCESS
assert submitted == [run["batch_id"]]

90
tests/test_store_catalog_pipeline.py
View File

@@ -143,6 +143,43 @@ def test_an_uncategorised_row_gets_no_hsn_code(store):
assert any("category could not be detected" in w for w in result.warnings)
def test_a_categoryid_column_is_not_read_as_the_category(store):
""""category" is a substring of "categoryid", so the keyword rule used to
claim the id column and store a bare 1/2/3 as the product's category."""
content = _sheet(["Item Name", "categoryid"], [["Coca-Cola 750ml", "1"]])
result = _run(content)
assert "category" not in result.recognised_columns
assert "categoryid" in result.unrecognised_columns
def test_a_numeric_category_is_resolved_from_the_product_name(store):
"""Defence in depth: even when a numeric code reaches the category field
it must not be stored as one - a number is never a category name."""
content = _sheet(["Item Name", "Category"], [["Coca-Cola 750ml", "1"]])
result = _run(content)
row = next(iter(store.values()))
assert row["category"] == "Beverages"
assert any("is an id, not a name" in w for w in result.warnings)
def test_a_soft_drink_resolves_to_beverages(store):
"""`Beverages` was missing from the canonical taxonomy entirely, so a cola
fell through to General while category_units and the HSN table both knew
the name."""
content = _sheet(["Item Name"], [["Coca-Cola 750ml"]])
_run(content)
assert next(iter(store.values()))["category"] == "Beverages"
def test_a_real_category_name_is_still_trusted(store):
"""Only numeric codes are re-detected; a name the store supplied wins."""
content = _sheet(["Item Name", "Segment"], [["Britannia 50-50", "Biscuits"]])
_run(content)
assert next(iter(store.values()))["category"] == "Biscuits"
def test_non_food_brands_get_no_fssai_licence(store):
"""A miss means 'not a food brand', not an error - the column stays empty."""
assert pipeline.get_fssai_license("Colgate") is None
@@ -178,6 +215,59 @@ def test_a_pack_size_with_a_nonsense_unit_is_dropped_with_a_reason(store):
assert any("15cm" in w for w in result.warnings)
def test_a_quantity_column_never_reaches_the_product_name(store):
"""The reported bug: a case-pack count of 72 became part of the name
("Coca-Cola 750ml 72") and was folded into the image_id, so the next upload
inserted a duplicate instead of updating the row."""
content = _sheet(["Item Name", "Quantity"], [["Coca-Cola 750ml", "72"]])
result = _run(content)
stored = list(store.values())
assert [r["product_name"] for r in stored] == ["Coca-Cola 750ml"]
assert "72" not in stored[0]["image_id"]
assert "Quantity" in result.unrecognised_columns
def test_a_unitless_number_in_a_size_column_is_rejected(store):
"""Defence in depth for the headers that legitimately DO map to
size_variants: "Net Weight" is a pack size column, but a bare 72 in it is
still not a pack size."""
content = _sheet(["Item Name", "Net Weight"], [["Coca-Cola 750ml", "72"]])
result = _run(content)
stored = list(store.values())
assert [r["product_name"] for r in stored] == ["Coca-Cola 750ml"]
assert any("72" in w for w in result.warnings)
def test_the_size_in_the_name_is_used_when_the_sheet_supplies_no_real_one(store):
"""Having rejected the bare number, the pipeline falls back to the name."""
content = _sheet(["Item Name", "Net Weight"], [["Coca-Cola 750ml", "72"]])
_run(content)
assert [r["size_variants"] for r in store.values()] == [["750ml"]]
def test_a_quantity_column_does_not_shut_out_the_real_pack_size(store):
"""Column mapping is first-wins by position, so a leading `Quantity` column
used to claim size_variants and discard the sheet's actual `Pack Size`."""
content = _sheet(
["Item Name", "Quantity", "Pack Size"],
[["Britannia Marie Gold Biscuits", "24", "250g"]],
)
result = _run(content)
assert result.recognised_columns["size_variants"] == "Pack Size"
assert "Quantity" in result.unrecognised_columns
assert [r["size_variants"] for r in store.values()] == [["250g"]]
def test_a_word_only_pack_size_is_still_accepted(store):
"""The unitless-number filter must not swallow "Family Pack"."""
content = _sheet(["Item Name", "Pack Size"], [["Britannia 50-50", "Family Pack"]])
_run(content)
assert [r["size_variants"] for r in store.values()] == [["Family Pack"]]
# ---------------------------------------------------------------------------
# Stage 11 - storage semantics
# ---------------------------------------------------------------------------

61
tests/test_user_products_upload.py
View File

@@ -368,3 +368,64 @@ def test_the_seed_catalog_is_only_written_for_rows_the_database_took(
assert resp.status_code == 503
assert catalog_writes == [], "the JSON catalog must not gain products the database refused"
# ---------------------------------------------------------------------------
# Brand-level defaults must not include images
# ---------------------------------------------------------------------------
def test_a_product_does_not_inherit_another_products_images(monkeypatch):
"""The reported bug: every image-less Britannia product showed Good Day.
`_brand_sample()` returns ONE arbitrary product of the brand and is resolved
once per upload, so copying its `image_urls` onto each image-less sibling
stamped that product's photographs across the whole brand. A photograph is
never a brand-level default.
"""
monkeypatch.setattr(user_products.s3_service, "enabled", False)
good_day_images = [
"https://cdn.example.com/britannia_good_day_cashew_cookies_200g/image_000.jpg",
"https://cdn.example.com/britannia_good_day_cashew_cookies_200g/image_001.jpg",
]
sample = {"image_urls": good_day_images, "category": "Biscuits & Cookies"}
built = user_products._build_product_dict(
user_products.AddProductRequest(product_name="Britannia Marie Gold 250g", brand="Britannia"),
"Britannia",
sample,
)
assert built["image_urls"] == []
assert built["image_url"] is None
def test_a_product_with_no_image_gets_no_invented_url(monkeypatch):
"""The old fallback guessed an S3 URL on a bucket that is not the configured
one, so it 404'd while still looking like an image to the frontend."""
monkeypatch.setattr(user_products.s3_service, "enabled", False)
built = user_products._build_product_dict(
user_products.AddProductRequest(product_name="Britannia Tiger 100g", brand="Britannia"),
"Britannia",
{},
)
assert built["image_urls"] == []
assert built["image_url"] is None
def test_an_explicit_image_url_is_still_honoured(monkeypatch):
"""Removing the fallbacks must not drop an image the user actually gave."""
monkeypatch.setattr(user_products.s3_service, "enabled", False)
built = user_products._build_product_dict(
user_products.AddProductRequest(
product_name="Britannia Tiger 100g",
brand="Britannia",
image_url="https://cdn.example.com/tiger.jpg",
),
"Britannia",
{"image_urls": ["https://cdn.example.com/good_day.jpg"]},
)
assert built["image_url"] == "https://cdn.example.com/tiger.jpg"