backend stores data file enrichment pipeline

tests/test_store_catalog_pipeline.py

@@ -0,0 +1,283 @@
+"""Tests for the store-spreadsheet -> 11-stage pipeline -> brand table flow.
+
+Follows the pattern established by test_user_products_upload.py: monkeypatch
+the storage/network boundary *on the pipeline module object* (it imports those
+names directly), build real .xlsx fixtures in memory, and assert on what was
+captured rather than on the HTTP status alone.
+
+Every test runs with use_llm=False and fetch_images=False so nothing here
+touches Ollama or the open web.
+"""
+from __future__ import annotations
+
+import io
+
+import pytest
+
+from app.core import store_catalog_pipeline as pipeline
+
+openpyxl = pytest.importorskip("openpyxl")
+
+
+def _sheet(headers, rows) -> bytes:
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws.append(headers)
+    for row in rows:
+        ws.append(row)
+    buf = io.BytesIO()
+    wb.save(buf)
+    return buf.getvalue()
+
+
+@pytest.fixture
+def store(monkeypatch):
+    """A fake brand table. Returns the dict of image_id -> stored row."""
+    table: dict = {}
+
+    def fake_upsert(brand, rows, cleanup=False):
+        assert cleanup is False, "cleanup=True would delete the brand's existing catalog"
+        for row in rows:
+            table[row["image_id"]] = dict(row)
+        return len(rows)
+
+    monkeypatch.setattr(pipeline, "upsert_brand_products", fake_upsert)
+    monkeypatch.setattr(pipeline, "get_products_by_brand", lambda b, **kw: list(table.values()))
+    monkeypatch.setattr(pipeline, "embed_texts", lambda texts: [[0.0] * 384 for _ in texts])
+    return table
+
+
+def _run(content, **kw):
+    return pipeline.run_pipeline("store.xlsx", content, use_llm=False, fetch_images=False, **kw)
+
+
+# ---------------------------------------------------------------------------
+# Column mapping / brand resolution
+# ---------------------------------------------------------------------------
+def test_messy_headers_map_onto_catalog_fields(store):
+    content = _sheet(
+        ["Item Name", "Product Description", "Segment", "Net Weight", "Random Column"],
+        [["Britannia 50-50", "Salty biscuit", "Biscuits", "200g", "junk"]],
+    )
+    result = _run(content)
+    assert result.recognised_columns["product_name"] == "Item Name"
+    assert result.recognised_columns["description"] == "Product Description"
+    assert result.recognised_columns["category"] == "Segment"
+    assert result.recognised_columns["size_variants"] == "Net Weight"
+    assert result.unrecognised_columns == ["Random Column"]
+
+
+def test_brand_is_inferred_when_the_sheet_has_no_brand_column(store):
+    """Store files routinely omit the brand; it lives in the product name."""
+    content = _sheet(["Item Name"], [["Britannia 50-50"], ["Cadbury Dairy Milk 100g"]])
+    result = _run(content)
+    assert result.errors == []
+    assert result.brands == ["Britannia", "Cadbury"]
+
+
+def test_longest_alias_wins_when_inferring_a_brand():
+    """'cadbury dairy milk' must beat the shorter 'cadbury' substring."""
+    assert pipeline.infer_brand("Cadbury Dairy Milk Silk 100g") == "cadbury dairy milk"
+
+
+def test_rows_land_in_the_brand_table_the_product_belongs_to(store):
+    content = _sheet(
+        ["Item Name", "Net Weight"],
+        [["Britannia 50-50", "200g"], ["Aachi Sambar Powder", "100g"]],
+    )
+    _run(content)
+    assert sorted(store) == ["aachi_aachi_sambar_powder_100g", "britannia_britannia_50_50_200g"]
+
+
+# ---------------------------------------------------------------------------
+# Gap filling
+# ---------------------------------------------------------------------------
+def test_missing_fields_are_filled_by_the_pipeline(store):
+    """A sheet with only a product name still produces a complete row."""
+    content = _sheet(["Item Name"], [["Britannia Good Day Biscuits 200g"]])
+    _run(content)
+
+    row = next(iter(store.values()))
+    assert row["fssai_license"] == "10012022000103"      # stage 1
+    assert row["category"]                                # stage 3
+    assert row["price_range"].startswith("₹")       # stage 5
+    assert row["product_sku"]                             # stage 7
+    assert row["hsn_code"]                                # stage 9
+    assert row["search_query"]                            # stage 11
+
+
+def test_values_supplied_by_the_store_are_never_overwritten(store):
+    content = _sheet(
+        ["Item Name", "Net Weight", "MRP Range", "HSN Code"],
+        [["Britannia 50-50", "200g", "₹111-222", "9999"]],
+    )
+    _run(content)
+    row = next(iter(store.values()))
+    assert row["price_range"] == "₹111-222"
+    assert row["hsn_code"] == "9999"
+
+
+def test_an_uncategorised_row_gets_no_hsn_code(store):
+    """HSN/GST is a regulatory value keyed on category. When the category
+    cannot be determined the column is deliberately left empty rather than
+    guessed - a wrong tax code is worse than a missing one."""
+    content = _sheet(["Item Name"], [["Britannia 50-50"]])
+    result = _run(content)
+
+    row = next(iter(store.values()))
+    assert row["category"] == "General"
+    assert row["hsn_code"] is None
+    assert any("category could not be detected" in w for w in result.warnings)
+
+
+def test_non_food_brands_get_no_fssai_licence(store):
+    """A miss means 'not a food brand', not an error - the column stays empty."""
+    assert pipeline.get_fssai_license("Colgate") is None
+
+
+# ---------------------------------------------------------------------------
+# Stage 4 - pack-size explosion and unit safety
+# ---------------------------------------------------------------------------
+def test_one_row_explodes_into_one_row_per_pack_size(store):
+    content = _sheet(["Item Name", "Pack Size"], [["Britannia 50-50", "100g, 200g, 500g"]])
+    _run(content)
+    assert len(store) == 3
+    assert {r["size_variants"][0] for r in store.values()} == {"100g", "200g", "500g"}
+
+
+def test_each_pack_size_gets_its_own_image_id(store):
+    """Without the size in the id every variant collapses onto one row under
+    ON CONFLICT (image_id)."""
+    content = _sheet(["Item Name", "Pack Size"], [["Britannia 50-50", "100g, 200g"]])
+    _run(content)
+    assert len(set(store)) == 2
+
+
+def test_the_size_is_not_duplicated_when_already_in_the_product_name():
+    assert pipeline.build_image_id("Cadbury", "Cadbury Dairy Milk 100g", "100g") == \
+        "cadbury_cadbury_dairy_milk_100g"
+
+
+def test_a_pack_size_with_a_nonsense_unit_is_dropped_with_a_reason(store):
+    """A biscuit measured in centimetres is a data error, not a pack size."""
+    content = _sheet(["Item Name", "Segment", "Pack Size"], [["Britannia 50-50", "Biscuits", "15cm"]])
+    result = _run(content)
+    assert any("15cm" in w for w in result.warnings)
+
+
+# ---------------------------------------------------------------------------
+# Stage 11 - storage semantics
+# ---------------------------------------------------------------------------
+def test_reingesting_the_same_file_changes_nothing(store):
+    content = _sheet(["Item Name", "Net Weight"], [["Britannia 50-50", "200g"]])
+    first = _run(content)
+    assert (first.inserted, first.skipped_existing) == (1, 0)
+
+    second = _run(content)
+    assert (second.inserted, second.backfilled, second.skipped_existing) == (0, 0, 1)
+    assert len(store) == 1
+
+
+def test_an_existing_row_has_only_its_blank_columns_backfilled(store):
+    content = _sheet(["Item Name", "Net Weight"], [["Britannia Good Day Biscuits", "200g"]])
+    _run(content)
+
+    key = next(iter(store))
+    store[key]["hsn_code"] = None
+    store[key]["price_range"] = "₹999-1000"      # a value already held
+
+    result = _run(content)
+    assert result.backfilled == 1
+    assert store[key]["hsn_code"] == "1905"            # blank -> filled
+    assert store[key]["price_range"] == "₹999-1000"  # held value untouched
+
+
+def test_a_storage_failure_is_reported_rather_than_counted_as_success(store, monkeypatch):
+    def boom(brand, rows, cleanup=False):
+        raise RuntimeError("pgvector is down")
+
+    monkeypatch.setattr(pipeline, "upsert_brand_products", boom)
+    result = _run(_sheet(["Item Name"], [["Britannia 50-50 200g"]]))
+    assert result.storage_error and "pgvector is down" in result.storage_error
+    assert result.inserted == 0
+
+
+# ---------------------------------------------------------------------------
+# Row-level error handling
+# ---------------------------------------------------------------------------
+def test_a_row_with_no_usable_product_name_is_reported_not_fatal(store):
+    content = _sheet(["Item Name", "Net Weight"], [["", "200g"], ["Britannia 50-50", "200g"]])
+    result = _run(content)
+    assert len(result.errors) == 1
+    assert result.errors[0].row == 2          # header is row 1
+    assert result.inserted == 1               # the good row still landed
+
+
+def test_the_same_pack_listed_twice_is_written_once(store):
+    content = _sheet(
+        ["Item Name", "Net Weight"],
+        [["Britannia 50-50", "200g"], ["Britannia 50-50", "200g"]],
+    )
+    _run(content)
+    assert len(store) == 1
+
+
+# ---------------------------------------------------------------------------
+# HTTP surface
+# ---------------------------------------------------------------------------
+def test_preview_reports_how_the_columns_were_understood(client, admin_headers):
+    content = _sheet(["Item Name", "Segment", "Mystery"], [["Britannia 50-50", "Biscuits", "?"]])
+    resp = client.post(
+        "/api/admin/store-catalog/preview",
+        files={"file": ("store.xlsx", content)},
+        headers=admin_headers,
+    )
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["recognised_columns"]["product_name"] == "Item Name"
+    assert body["unrecognised_columns"] == ["Mystery"]
+    assert body["brand_column_present"] is False
+    assert len(body["stages"]) == 11
+
+
+def test_preview_requires_admin(client):
+    content = _sheet(["Item Name"], [["Britannia 50-50"]])
+    resp = client.post("/api/admin/store-catalog/preview", files={"file": ("s.xlsx", content)})
+    assert resp.status_code == 401
+
+
+def test_an_unparseable_file_is_rejected_before_a_job_is_created(client, admin_headers):
+    resp = client.post(
+        "/api/admin/store-catalog/ingest",
+        files={"file": ("notes.txt", b"this is not a spreadsheet")},
+        headers=admin_headers,
+    )
+    assert resp.status_code == 400
+
+
+def test_ingest_returns_a_job_id_that_can_be_polled(client, admin_headers, monkeypatch):
+    # Keep the worker off the network and out of the database.
+    monkeypatch.setattr(pipeline, "upsert_brand_products", lambda b, r, cleanup=False: len(r))
+    monkeypatch.setattr(pipeline, "get_products_by_brand", lambda b, **kw: [])
+    monkeypatch.setattr(pipeline, "embed_texts", lambda texts: [[0.0] * 384 for _ in texts])
+
+    content = _sheet(["Item Name", "Segment"], [["Britannia 50-50", "Biscuits"]])
+    resp = client.post(
+        "/api/admin/store-catalog/ingest",
+        files={"file": ("store.xlsx", content)},
+        headers=admin_headers,
+    )
+    assert resp.status_code == 202
+    job_id = resp.json()["job_id"]
+    assert resp.json()["rows_total"] == 1
+
+    poll = client.get(f"/api/admin/store-catalog/jobs/{job_id}", headers=admin_headers)
+    assert poll.status_code == 200
+    body = poll.json()
+    assert body["status"] in {"pending", "running", "done", "failed"}
+    assert body["total_stages"] == 11
+
+
+def test_polling_an_unknown_job_is_a_404(client, admin_headers):
+    resp = client.get("/api/admin/store-catalog/jobs/does-not-exist", headers=admin_headers)
+    assert resp.status_code == 404