Health score updates in backend

2026-09-04 12:06:40 +05:30
parent 1300d4c678
commit 0c3e23fac4
33 changed files with 59703 additions and 94 deletions
--- a/tests/test_consumability.py
+++ b/tests/test_consumability.py
@@ -0,0 +1,271 @@
+"""
+Walls off the defect where non-food products carried a health score.
+
+Measured against the live catalogue on 2026-09-03, these rows had one:
+
+    Colgate-Palmolive Palmolive Naturals   General                    soap
+    Cavinkare Nyle, Cavinkare Nature's     Hair Care                  shampoo
+    P&G Pantene                            Hair Care                  shampoo
+    Godrej Hit Spray                       Personal Care - Mosquito   291 kcal
+
+Every category string in LIVE_CATEGORIES below was read out of the production
+brand tables, with its row count, so this file tests the taxonomy that exists
+rather than the one `CATEGORY_REGISTRY` describes - the two have drifted, which
+is the whole reason `consumability` is not just a field on the registry.
+
+Pure unit tests: no database, no network, no LLM.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from app.services import nutrition_data_service as nds  # noqa: E402
+from app.services.consumability import (  # noqa: E402
+    CATEGORY_VERDICTS,
+    Edibility,
+    classify_edibility,
+    is_consumable,
+    is_junk_category,
+    is_non_consumable,
+)
+from app.services.enrichment.hsn_gst.models import HSN_GST_TABLE  # noqa: E402
+
+# (category string, live row count, expected verdict). Read from the production
+# database; the counts are kept so a future reader can tell a 189-row decision
+# from a 1-row one.
+LIVE_CATEGORIES = [
+    ("Dairy", 189, Edibility.CONSUMABLE),
+    ("Beverages", 174, Edibility.CONSUMABLE),
+    ("Spices & Masalas", 136, Edibility.CONSUMABLE),
+    ("Chocolates", 119, Edibility.CONSUMABLE),
+    ("Fruits & Vegetables", 95, Edibility.CONSUMABLE),
+    ("Food - Mixes", 83, Edibility.CONSUMABLE),
+    ("Tea & Coffee", 76, Edibility.CONSUMABLE),
+    ("Hair Care", 51, Edibility.NON_CONSUMABLE),
+    ("Oral Care", 45, Edibility.NON_CONSUMABLE),
+    ("Snacks", 42, Edibility.CONSUMABLE),
+    ("Cooking Oils", 42, Edibility.CONSUMABLE),
+    ("Detergents & Fabric Care", 38, Edibility.NON_CONSUMABLE),
+    ("Pulses, Grains & Spices", 35, Edibility.CONSUMABLE),
+    ("Health Drinks", 26, Edibility.CONSUMABLE),
+    ("Pasta & Noodles", 24, Edibility.CONSUMABLE),
+    ("Pickles & Chutneys", 24, Edibility.CONSUMABLE),
+    ("Skin Care", 19, Edibility.NON_CONSUMABLE),
+    ("Bath Soap", 17, Edibility.NON_CONSUMABLE),
+    ("Fresh Herbs & Greens", 17, Edibility.CONSUMABLE),
+    ("Atta & Staples", 16, Edibility.CONSUMABLE),
+    ("Fish & Seafood", 15, Edibility.CONSUMABLE),
+    ("Flowers", 14, Edibility.NON_CONSUMABLE),
+    ("Candy & Confectionery", 13, Edibility.CONSUMABLE),
+    ("Dairy - Desserts", 13, Edibility.CONSUMABLE),
+    ("Biscuits & Cookies", 12, Edibility.CONSUMABLE),
+    ("Dishwash", 12, Edibility.NON_CONSUMABLE),
+    ("Health Foods", 12, Edibility.CONSUMABLE),
+    ("Food - Spreads", 11, Edibility.CONSUMABLE),
+    ("Skin & Bath Care", 10, Edibility.NON_CONSUMABLE),
+    ("Noodles & Instant Food", 10, Edibility.CONSUMABLE),
+    ("Cheese", 10, Edibility.CONSUMABLE),
+    ("Salt & Staples", 9, Edibility.CONSUMABLE),
+    ("Breakfast Cereal", 8, Edibility.CONSUMABLE),
+    ("Feminine Hygiene", 6, Edibility.NON_CONSUMABLE),
+    ("Ready to Eat", 6, Edibility.CONSUMABLE),
+    ("Dry Fruits & Nuts", 6, Edibility.CONSUMABLE),
+    ("Eggs", 5, Edibility.CONSUMABLE),
+    ("Bakery & Breads", 5, Edibility.CONSUMABLE),
+    ("Ice Cream", 5, Edibility.CONSUMABLE),
+    ("Health Care - Cold & Cough", 5, Edibility.NON_CONSUMABLE),
+    ("Personal Care - Mosquito Repellent", 5, Edibility.NON_CONSUMABLE),
+    ("Food & Beverages", 4, Edibility.CONSUMABLE),
+    ("Men's Grooming", 4, Edibility.NON_CONSUMABLE),
+    ("Household - Lamp Oil", 4, Edibility.NON_CONSUMABLE),
+    ("Fragrance & Deodorants", 3, Edibility.NON_CONSUMABLE),
+    ("Household Cleaning", 2, Edibility.NON_CONSUMABLE),
+    ("Biscuits", 2, Edibility.CONSUMABLE),
+    ("Food - Soups & Sauces", 2, Edibility.CONSUMABLE),
+    ("Crackers", 2, Edibility.CONSUMABLE),
+    ("Health Care - Antiseptic", 2, Edibility.NON_CONSUMABLE),
+    ("Namkeen", 1, Edibility.CONSUMABLE),
+    ("Household - Air Freshener", 1, Edibility.NON_CONSUMABLE),
+    ("Flour & Grains", 1, Edibility.CONSUMABLE),
+    ("Staples", 1, Edibility.CONSUMABLE),
+    ("Personal Care", 1, Edibility.NON_CONSUMABLE),
+    ("Health Care - Digestive", 1, Edibility.NON_CONSUMABLE),
+]
+
+# The two category values that genuinely carry no signal. "General" holds 40
+# rows of mixed food and non-food; "1".."5" are import damage on 17 rows.
+# Both are meant to reach the title, so they are excluded here rather than
+# given a verdict this file would have to invent.
+UNDECIDABLE_CATEGORIES = ("General", "1", "2", "3", "4", "5")
+
+# Categories holding both food and non-food, so they carry no verdict of their
+# own and are excluded from the table above. Found by running the purge audit
+# before deleting anything - see test_baby_food_is_food.
+MIXED_CATEGORIES = ("Baby Care", "Health Care - Ayurvedic")
+
+
+@pytest.mark.parametrize("category,count,expected", LIVE_CATEGORIES)
+def test_every_live_category_has_the_expected_verdict(category, count, expected):
+    verdict = classify_edibility(category)
+    assert verdict.edibility is expected, (
+        f"{category!r} ({count} live rows) classified {verdict.edibility.value}: "
+        f"{verdict.reason}"
+    )
+
+
+@pytest.mark.parametrize(
+    "category",
+    [c for c in sorted(HSN_GST_TABLE) if c not in MIXED_CATEGORIES],
+)
+def test_every_hsn_category_resolves(category):
+    """Drift guard. Someone adding a category to the tax table and not here
+    would otherwise get a silent UNKNOWN, and UNKNOWN means no health score.
+
+    The mixed categories are excluded because their UNKNOWN is the intended
+    answer - they are decided by the title, not by the category.
+    """
+    assert classify_edibility(category).edibility is not Edibility.UNKNOWN
+
+
+def test_lamp_oil_is_not_a_cooking_oil():
+    """These must not collapse into each other.
+
+    `resolve_hsn_gst`'s keyword fallbacks contain a greedy ("oil", "1517")
+    entry, which is chapter 15 - an edible fat. Reading HSN_GST_TABLE directly
+    instead of calling that resolver is what keeps lamp oil inedible.
+    """
+    assert is_consumable("Cooking Oils", "Idhayam Sesame Oil 500ml") is True
+    assert is_non_consumable("Household - Lamp Oil", "Idhayam Lamp Oil 500ml") is True
+
+
+def test_flowers_are_not_food():
+    """HSN 0603 is chapter 06, which sits inside the naive 01-24 "food" range.
+    Flowers reach the Own Products table through the same produce lexicon as the
+    vegetables, so nothing upstream separates them - this is the separation."""
+    assert is_non_consumable("Flowers", "Jasmine Garland") is True
+    assert is_consumable("Flowers", "Jasmine Garland") is False
+
+
+def test_medicines_are_not_scored():
+    """Ingested, but they publish dosage rather than a nutrition panel."""
+    for category in ("Health Care - Cold & Cough", "Health Care - Digestive",
+                     "Health Care - Antiseptic"):
+        assert is_non_consumable(category) is True, category
+    assert is_non_consumable("Health Care - Cold & Cough", "Dabur Honitus 1L") is True
+
+
+def test_baby_food_is_food():
+    """The error this classifier made on its first audit run, caught before a
+    single row was deleted.
+
+    "Baby Care" held 21 live rows and a blanket non-consumable verdict. They are
+    Nestle Cerelac, Nan Pro and Lactogen - infant formula and baby cereal, among
+    the most heavily nutrition-labelled products sold in India - sitting beside a
+    bottle of baby oil. One verdict cannot cover both, so the category defers to
+    the title.
+    """
+    assert is_consumable("Baby Care", "Nestle Cerelac 125g") is True
+    assert is_consumable("Baby Care", "Nestle Nan Pro 400 g") is True
+    assert is_consumable("Baby Care", "Nestle Lactogen 400 g") is True
+    assert is_non_consumable("Baby Care", "Johnson Baby Soap 100g") is True
+    assert is_non_consumable("Baby Care", "Pampers Diaper Large 20s") is True
+
+
+def test_chyawanprash_is_food_but_cough_syrup_is_not():
+    """Same shape as Baby Care: "Health Care - Ayurvedic" mixes a spoonable food
+    supplement with medicine, so it too defers to the title."""
+    assert is_consumable("Health Care - Ayurvedic",
+                         "Chyawanprash 100% Natural 70g") is True
+
+
+@pytest.mark.parametrize("category", MIXED_CATEGORIES)
+def test_a_mixed_category_carries_no_verdict_of_its_own(category):
+    """With no title to go on there is nothing to decide, and UNKNOWN neither
+    enriches nor deletes."""
+    assert classify_edibility(category).edibility is Edibility.UNKNOWN
+
+
+def test_health_drinks_are_food_despite_the_name():
+    assert is_consumable("Health Drinks", "Horlicks Classic Malt 500g") is True
+    assert is_consumable("Health Foods", "Manna Health Mix 500g") is True
+
+
+def test_unknown_is_neither_consumable_nor_non_consumable():
+    """The asymmetry the purge script depends on: an unrecognised product is
+    never enriched AND never deleted. If these two ever become inverses, a
+    classifier miss turns into data loss."""
+    verdict = classify_edibility("Zzz Unheard Of Category", "Mystery Item 1kg")
+    assert verdict.edibility is Edibility.UNKNOWN
+    assert is_consumable("Zzz Unheard Of Category", "Mystery Item 1kg") is False
+    assert is_non_consumable("Zzz Unheard Of Category", "Mystery Item 1kg") is False
+
+
+def test_a_missing_category_falls_back_to_the_title():
+    assert is_consumable("", "Amul Butter 500g") is True
+    assert is_consumable(None, "Toor Dhal 1kg") is True
+    assert is_non_consumable("", "Lifebuoy Soap 100g") is True
+    assert classify_edibility(None, "").edibility is Edibility.UNKNOWN
+
+
+@pytest.mark.parametrize("junk", UNDECIDABLE_CATEGORIES)
+def test_an_uninformative_category_defers_to_the_title(junk):
+    assert is_consumable(junk, "Aachi Sambar Powder 100g") is True
+    assert is_non_consumable(junk, "Lifebuoy Soap 100g") is True
+
+
+def test_numeric_categories_are_reported_as_junk():
+    """17 live rows carry a bare digit as their category. They are recognised so
+    the purge audit can report them for catalogue repair, rather than being
+    quietly accommodated."""
+    assert is_junk_category("3") is True
+    assert is_junk_category("Dairy") is False
+
+
+def test_a_toothpaste_brand_is_not_a_chocolate():
+    """The regression that forced `exact_only` into `detect_category_from_text`.
+
+    Its fuzzy fallback scores "colgate" at >=0.8 against the misspelling keyword
+    "choclate", so "Colgate-Palmolive Palmolive Naturals" - whose stored
+    category is the uninformative "General" - resolved to Chocolates and would
+    have been handed a health score.
+    """
+    verdict = classify_edibility("General", "Colgate-Palmolive Palmolive Naturals")
+    assert verdict.edibility is not Edibility.CONSUMABLE
+
+
+def test_soapnut_is_not_a_soap():
+    """The old gate matched NON_FOOD_KEYWORDS by substring, so "soap" hit
+    *soapnut* (reetha). Matching here is whole-word."""
+    assert is_non_consumable("Fruits & Vegetables", "Soapnut Reetha 500g") is False
+
+
+@pytest.mark.parametrize("category,title", [
+    ("Bath Soap", "Colgate-Palmolive Palmolive Naturals"),
+    ("Hair Care", "Cavinkare Nyle"),
+    ("Hair Care", "Cavinkare Nature's"),
+    ("Hair Care", "P G Pg Pantene"),
+    ("Personal Care - Mosquito Repellent", "Godrej Hit Spray"),
+])
+def test_the_products_that_are_wrongly_scored_today_all_classify_non_food(category, title):
+    """Named individually because each one is a live row with a health score."""
+    assert is_non_consumable(category, title) is True
+
+
+def test_the_legacy_gate_still_answers():
+    """`_looks_non_food` is kept as a delegate; nothing outside its module
+    imports it, but silently changing a public-looking name is worse than
+    keeping it honest."""
+    assert nds._looks_non_food("Lifebuoy Soap", "Bath Soap") is True
+    assert nds._looks_non_food("Amul Butter", "Dairy") is False
+
+
+def test_the_verdict_map_has_no_contradictions():
+    """A category listed as both consumable and non-consumable would resolve by
+    dict-ordering luck."""
+    assert len(CATEGORY_VERDICTS) == len(set(CATEGORY_VERDICTS))
+    assert all(isinstance(v, Edibility) for v in CATEGORY_VERDICTS.values())