nearle_daily/catalogue_backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Dagster orchestration and reduce active brands in backend

Browse Source
This commit is contained in:
sriram
2026-08-20 16:39:54 +05:30
parent fbb1356e47
commit 7bf8dc6922
66 changed files with 2664 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
orchestration/assets/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from orchestration.assets import catalog, ml, nutrition
__all__ = ["catalog", "nutrition", "ml"]

515
orchestration/assets/catalog.py Normal file
View File

@@ -0,0 +1,515 @@
"""Catalog ingestion assets: brand selection -> validated -> enriched -> stored.
Every asset delegates to a function that already exists and is already tested.
What Dagster adds is the lineage between them, per-brand partitioning, retries
on the flaky steps, and a run history that survives a restart - none of which
the daemon-thread + in-memory job store could give.
The functions being wrapped are `app.core.store_catalog_pipeline`'s eleven
stages and the services they call, which the store-spreadsheet upload path
already uses. Wrapping rather than reimplementing is what stops the two paths
drifting: a fix to a stage fixes both.
"""
# NOTE: deliberately no `from __future__ import annotations` here.
# Dagster resolves the decorated function signatures at definition time to
# validate the `context` parameter and to infer asset input types. Under
# PEP 563/649 the annotations arrive as strings and that validation fails
# with "Cannot annotate `context` parameter with type AssetExecutionContext".
# Local Python is 3.14, which defers annotations by default, so this is not
# hypothetical.
import time
from typing import Any, Dict, List
from dagster import (
AssetExecutionContext,
Backoff,
Failure,
Jitter,
MetadataValue,
RetryPolicy,
asset,
)
from orchestration.config import BrandConfig, database_target, require_local_database
from orchestration.partitions import brand_partitions
# Only the assets that reach the network get retried. A failed DB write is
# rarely transient here (bad schema, bad row), so retrying it just delays a red
# run that a human has to look at anyway.
NETWORK_RETRY = RetryPolicy(
max_retries=2, delay=5, backoff=Backoff.EXPONENTIAL, jitter=Jitter.PLUS_MINUS
)
GROUP = "catalog"
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
description="The brand this partition builds, resolved through the alias table.",
)
def active_brand(context: AssetExecutionContext) -> str:
"""Root of the lineage: one partition per brand in ACTIVE_BRANDS.
Resolving here rather than inside each downstream asset means the UI shows
which storage table a partition actually targets. "Tata" landing in
brand_hindustan_unilever is a real case in this data set, and it should be
visible up front instead of discovered halfway through a run.
"""
from app.services.brand_registry import resolve_parent_brand
from app.services.vector_store import _sanitize_name
brand = context.partition_key
parent = resolve_parent_brand(brand)
table = "brand_" + _sanitize_name(parent)
context.add_output_metadata(
{
"brand": brand,
"resolved_parent": parent,
"storage_table": table,
"database": database_target(),
}
)
return brand
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
retry_policy=NETWORK_RETRY,
description="Products for this brand, read from its seed catalog(s).",
)
def raw_products(
context: AssetExecutionContext, config: BrandConfig, active_brand: str
) -> List[Dict[str, Any]]:
"""Source rows via `brand_sync.load_seed_catalogs(only=...)`.
Seed files are the default source rather than live discovery because
`catalog_engine.generate_catalog` drives Ollama plus a few hundred image
searches - minutes of CPU and a lot of outbound traffic per brand. Seed
files make the whole DAG runnable offline in seconds, which is what makes
it usable as a development tool. Live discovery stays reachable through the
existing POST /api/catalog/generate and cli/ingest_brand.py.
`load_brand_products` resolves the brand through the seed-file index rather
than matching the brand name against file names. That matters: the obvious
`load_seed_catalogs(only=[brand])` matches substrings of the FILE NAME, so
"Hindustan Unilever" (a space) never matches
`brand_catalog_hindustan_unilever.json` (an underscore) and the partition
silently ingests nothing. It also picks up every file feeding the brand's
table, including `brand_catalog_tata.json`'s 121 HUL products, and reaches
archived catalogs so an explicitly requested brand is always found.
"""
from app.services.brand_sync import load_brand_products
products = load_brand_products(active_brand)
available = len(products)
if config.max_products_per_brand:
products = products[: config.max_products_per_brand]
if not products:
# Loud, not a warning. A partition that ingests zero rows and reports
# success is the worst outcome here: everything downstream materializes
# green over an empty list, and the brand looks processed when nothing
# happened. This is the exact failure the name-matching bug above
# produced before it was found.
raise Failure(
description=(
f"No seed products found for '{active_brand}'. Expected a "
f"catalog feeding table "
f"'brand_{_slug(active_brand)}' in data/seed_catalogs/ or its "
f"archive/ subdirectory."
),
metadata={"brand": active_brand, "products": 0},
)
context.add_output_metadata(
{
"brand": active_brand,
"products": len(products),
"products_available": available,
"preview": MetadataValue.md(_preview(products)),
}
)
return products
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
description="Rows that passed the title, pack-size and confidence gates.",
)
def validated_products(
context: AssetExecutionContext,
active_brand: str,
raw_products: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Stages 3, 4 and 10 of the existing pipeline, in that order.
`validate_catalog` is the same deterministic gate the upload path runs, so
a row rejected here would have been rejected there too. Rejections are
recorded as metadata rather than raised: a brand where 3 of 120 rows fail
is a data problem worth seeing, not a reason to fail the run and block the
other 117.
"""
from app.services.category_units import fix_or_reject_size
from app.services.product_validator import validate_catalog
from app.services.title_validator import validate_and_fix_title
started = time.time()
prepared: List[Dict[str, Any]] = []
title_fixes = 0
size_rejects = 0
for row in raw_products:
row = dict(row)
category = row.get("category") or ""
title = row.get("title") or row.get("product_name") or ""
fixed_title, changed, _removed = validate_and_fix_title(
title, category, brand=active_brand
)
if changed:
title_fixes += 1
row["title"] = fixed_title
size = row.get("size") or ""
if size:
fixed_size, rejected_size, reason = fix_or_reject_size(
size, category, row.get("product_name") or title
)
if rejected_size:
size_rejects += 1
context.log.debug("Dropped %s (%s): %s", title, size, reason)
continue
row["size"] = fixed_size
prepared.append(row)
kept, rejected, summary = validate_catalog(
prepared, active_brand, images_checked=False
)
context.add_output_metadata(
{
"brand": active_brand,
"rows_in": len(raw_products),
"rows_out": len(kept),
"rejected": len(rejected),
"title_fixes": title_fixes,
"size_rejects": size_rejects,
"validator_summary": MetadataValue.json(_jsonable(summary)),
"duration_s": round(time.time() - started, 2),
}
)
return kept
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
retry_policy=NETWORK_RETRY,
description="SKU, barcode, HSN/GST and pricing filled in - blanks only.",
)
def enriched_products(
context: AssetExecutionContext,
active_brand: str,
validated_products: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Stages 5 and 7-9, through the enrichment framework that already exists.
FILL-ONLY-BLANKS is the contract, inherited from the upload pipeline: a
stage writes a field only when it is empty, so re-running over the same
input is a no-op and a brand's own data is never overwritten by a guess.
The barcode and manufacturer-site lookups stay off unless a run turns them
on - see orchestration/.env.orchestration for why.
"""
import asyncio
from app.services.enrichment.pipeline import run_default_pipeline
from app.services.price_estimator import estimate_price_range_for_size
from app.services.sku_service import resolve_product_sku
started = time.time()
rows = [dict(r) for r in validated_products]
skus_added = 0
prices_added = 0
for row in rows:
title = row.get("title") or row.get("product_name") or ""
size = row.get("size") or ""
category = row.get("category") or ""
if not row.get("product_sku"):
row.update(resolve_product_sku(active_brand, title, size))
skus_added += 1
if not row.get("price_range") and size:
try:
low, high = estimate_price_range_for_size(
size, title, active_brand, category
)
row["price_range"] = "₹{}-{}".format(low, high)
prices_added += 1
except Exception as exc: # noqa: BLE001 - enrichment is best effort
context.log.debug("Price estimate failed for %s: %s", title, exc)
# Barcode + HSN/GST. Each individual stage is already never-raising; this
# only guards against the pipeline itself being unavailable, and degrades
# to "the rows we enriched so far" rather than failing the partition.
try:
rows = asyncio.run(run_default_pipeline(rows, active_brand))
except Exception as exc: # noqa: BLE001
context.log.warning(
"Enrichment pipeline degraded for %s (%s) - continuing with the "
"rows enriched so far.",
active_brand,
exc,
)
context.add_output_metadata(
{
"brand": active_brand,
"rows": len(rows),
"skus_assigned": skus_added,
"price_bands_estimated": prices_added,
"rows_with_barcode": sum(1 for r in rows if r.get("barcode")),
"rows_with_hsn": sum(1 for r in rows if r.get("hsn_code")),
"duration_s": round(time.time() - started, 2),
}
)
return rows
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
description="Rows upserted into the brand's Postgres table.",
)
def catalog_database(
context: AssetExecutionContext,
active_brand: str,
enriched_products: List[Dict[str, Any]],
) -> int:
"""Stage 11's persistence half: `upsert_brand_products(..., cleanup=False)`.
TWO THINGS HERE ARE LOAD-BEARING.
`cleanup=False`: cleanup=True deletes every row in the table that is not in
the batch being written, so a partial or filtered batch would wipe the rest
of the brand's catalog. A test already asserts this for the upload path;
the same reasoning applies with more force here, where a run can be
retried or cancelled halfway through.
The guard: this asset writes, so it refuses a non-local database. backend/.env
points at production. See orchestration/config.py.
"""
from app.services.vector_store import ensure_brand_schema, upsert_brand_products
target = require_local_database("catalog_database")
if not enriched_products:
context.log.warning("Nothing to store for %s.", active_brand)
context.add_output_metadata({"brand": active_brand, "rows_written": 0})
return 0
table = ensure_brand_schema(active_brand)
written = upsert_brand_products(active_brand, enriched_products, cleanup=False)
context.add_output_metadata(
{
"brand": active_brand,
"table": table or "?",
"rows_offered": len(enriched_products),
"rows_written": written,
"database": target,
"cleanup": "False (never deletes rows outside this batch)",
}
)
return written
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
retry_policy=NETWORK_RETRY,
description="384-dim vectors generated for rows that do not have one.",
)
def product_embeddings(
context: AssetExecutionContext,
config: BrandConfig,
active_brand: str,
catalog_database: int,
) -> int:
"""Embed only the rows missing a vector, in batches.
Re-embedding an entire brand on every run would be the most expensive thing
this DAG does and would change nothing. Selecting on `embedding IS NULL`
makes the asset idempotent and makes a re-run nearly free, which is what
allows the refresh schedule to be daily.
"""
from app.services.embeddings_service import embed_texts
from app.services.vector_store import _connect, _table_name
require_local_database("product_embeddings")
table = _table_name(active_brand)
batch = max(1, config.embed_batch_size)
started = time.time()
conn = _connect()
if conn is None:
raise Failure(description="Database unreachable at " + database_target())
embedded = 0
try:
with conn, conn.cursor() as cur:
cur.execute(
"SELECT image_id, title, product_name, description, category "
"FROM {} WHERE embedding IS NULL".format(table)
)
pending = cur.fetchall()
for start in range(0, len(pending), batch):
chunk = pending[start : start + batch]
texts = [
" ".join(
str(part)
for part in (row[1], row[2], row[4], row[3])
if part
)[:2000]
for row in chunk
]
vectors = embed_texts(texts)
for row, vector in zip(chunk, vectors):
cur.execute(
"UPDATE {} SET embedding = %s, "
"updated_at = CURRENT_TIMESTAMP WHERE image_id = %s".format(
table
),
(str(list(vector)), row[0]),
)
embedded += len(chunk)
context.log.info(
"Embedded %d/%d rows for %s", embedded, len(pending), active_brand
)
finally:
conn.close()
context.add_output_metadata(
{
"brand": active_brand,
"table": table,
"rows_embedded": embedded,
"batch_size": batch,
"model": _embedding_model_name(),
"duration_s": round(time.time() - started, 2),
}
)
return embedded
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
description="Verification: the brand is fully searchable in pgvector.",
)
def vector_index(
context: AssetExecutionContext, active_brand: str, product_embeddings: int
) -> Dict[str, Any]:
"""Read-back check, deliberately a separate node from the write.
"The upsert reported success" and "the table can answer a similarity query"
are different claims. Making the second one its own asset means the UI can
show a green `catalog_database` beside a red `vector_index` when rows land
but never become searchable - precisely the failure that used to present
to users as "the catalog looks empty".
"""
from app.services.vector_store import _connect, _table_name
table = _table_name(active_brand)
conn = _connect()
if conn is None:
raise Failure(description="Database unreachable at " + database_target())
try:
with conn, conn.cursor() as cur:
cur.execute(
"SELECT count(*), count(embedding), count(DISTINCT category) "
"FROM {}".format(table)
)
rows, embedded, categories = cur.fetchone()
finally:
conn.close()
missing = (rows or 0) - (embedded or 0)
if rows and missing:
context.log.warning(
"%s has %d row(s) without an embedding - they will not appear in "
"semantic search.",
table,
missing,
)
context.add_output_metadata(
{
"table": table,
"rows": rows or 0,
"embedded": embedded or 0,
"missing_embeddings": missing,
"categories": categories or 0,
"searchable": missing == 0,
}
)
return {
"table": table,
"rows": rows or 0,
"embedded": embedded or 0,
"missing_embeddings": missing,
"categories": categories or 0,
}
# --- small helpers ----------------------------------------------------------
def _slug(brand: str) -> str:
from app.services.brand_registry import resolve_parent_brand
from app.services.vector_store import _sanitize_name
return _sanitize_name(resolve_parent_brand(brand))
def _embedding_model_name() -> str:
from app.infrastructure.settings import EMBEDDINGS_MODEL
return EMBEDDINGS_MODEL
def _preview(products: List[Dict[str, Any]], limit: int = 5) -> str:
if not products:
return "_no rows_"
lines = ["| title | category | size |", "|---|---|---|"]
for row in products[:limit]:
lines.append(
"| {} | {} | {} |".format(
str(row.get("title") or row.get("product_name") or "")[:60],
row.get("category") or "",
row.get("size") or "",
)
)
return "\n".join(lines)
def _jsonable(value: Any) -> Any:
if isinstance(value, dict):
return {str(k): _jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_jsonable(v) for v in value]
if isinstance(value, (str, int, float, bool)) or value is None:
return value
return str(value)

187
orchestration/assets/ml.py Normal file
View File

@@ -0,0 +1,187 @@
"""Store-intelligence ML: dataset -> training -> evaluation.
Wraps `store_seed_service` and `ml_training_service`. These are the only ML
workflows worth orchestrating: the nutrition models live in the nutrition
group next to the data they fit, and the experimental models (forecast,
store_performance, purchase_propensity) are excluded on purpose - nothing
serves their output, so scheduling them would burn CPU to write artifacts no
request reads. They remain trainable by name.
"""
# NOTE: deliberately no `from __future__ import annotations` here.
# Dagster resolves the decorated function signatures at definition time to
# validate the `context` parameter and to infer asset input types. Under
# PEP 563/649 the annotations arrive as strings and that validation fails
# with "Cannot annotate `context` parameter with type AssetExecutionContext".
# Local Python is 3.14, which defers annotations by default, so this is not
# hypothetical.
import time
from typing import Any, Dict, List, Optional
from dagster import AssetExecutionContext, Config, Failure, MetadataValue, asset
from orchestration.config import require_local_database
GROUP = "ml"
class TrainingConfig(Config):
"""Which models to fit.
`None` means `ml_training_service.PRODUCTION_MODELS` - the three that are
actually served. Name any of ALL_MODELS to train more.
"""
models: Optional[List[str]] = None
class SeedConfig(Config):
"""Synthetic store/order history parameters.
`reset_orders` defaults to False so a scheduled run cannot silently discard
an order history someone is mid-analysis on. Turn it on per-run to rebuild
from scratch.
"""
days: int = 90
seed: int = 42
reset_orders: bool = False
@asset(
group_name=GROUP,
description="Stores, inventory, prices and simulated order history.",
)
def training_dataset(context: AssetExecutionContext, config: SeedConfig) -> Dict[str, Any]:
"""`store_seed_service.run_seed()` - the ML models' training data.
The store intelligence models learn from order history, and this project
has no real orders, so the history is deterministically simulated
(`intelligence/order_simulation.py`, fixed seed). That is documented as
cold-start bootstrapping; the important part is that INFERENCE goes through
the trained model, not the generating formula.
Made an explicit asset so the lineage answers "what were these models
actually fitted on" instead of leaving it implicit.
"""
from app.services.store_seed_service import run_seed
require_local_database("training_dataset")
started = time.time()
result = run_seed(
reset_orders=config.reset_orders, days=config.days, seed=config.seed
)
context.add_output_metadata(
{
"days_simulated": config.days,
"random_seed": config.seed,
"reset_orders": config.reset_orders,
"result": MetadataValue.json(_jsonable(result)),
"duration_s": round(time.time() - started, 2),
}
)
return _jsonable(result)
@asset(
group_name=GROUP,
description="Fitted discount / trending / popularity models (*.joblib).",
)
def trained_models(
context: AssetExecutionContext,
config: TrainingConfig,
training_dataset: Dict[str, Any],
) -> Dict[str, Any]:
"""`ml_training_service.train_all()`.
Depends on `training_dataset` because training against an empty orders
table produces a model that fits nothing and reports success - a silent
failure the dependency turns into an ordering guarantee.
"""
from app.services.ml_training_service import PRODUCTION_MODELS, train_all
require_local_database("trained_models")
models = config.models or list(PRODUCTION_MODELS)
started = time.time()
results = train_all(models=models) or {}
trained = [name for name, r in results.items() if "skipped" not in (r or {})]
skipped = {
name: (r or {}).get("skipped") for name, r in results.items() if "skipped" in (r or {})
}
if not trained:
# Every model skipping means there was no usable training data. That is
# a failed run, not a successful no-op - the artifacts on disk are now
# stale and nothing says so.
raise Failure(
description=(
"No model trained. Every requested model reported 'skipped', "
"which means the training frames were empty - materialize "
"training_dataset first."
),
metadata={"requested": MetadataValue.json(models),
"skipped": MetadataValue.json(_jsonable(skipped))},
)
context.add_output_metadata(
{
"requested": MetadataValue.json(models),
"trained": MetadataValue.json(trained),
"skipped": MetadataValue.json(_jsonable(skipped)),
"results": MetadataValue.json(_jsonable(results)),
"duration_s": round(time.time() - started, 2),
"note": "Restart the API process to pick up new artifacts.",
}
)
return _jsonable(results)
@asset(
group_name=GROUP,
description="Reports each production artifact's metrics and sample count.",
)
def model_evaluation(
context: AssetExecutionContext, trained_models: Dict[str, Any]
) -> Dict[str, Any]:
"""Read the bundles back and report what is actually on disk.
Separate from training for the same reason `vector_index` is separate from
`catalog_database`: "training returned a metrics dict" and "a loadable
artifact exists at MODEL_ARTIFACTS_DIR" are different claims, and only the
second is what the API will serve.
"""
from app.intelligence.model_utils import artifact_path, load_bundle
report: Dict[str, Any] = {}
for name in ("discount_model", "trending_model", "popularity_model"):
path = artifact_path(name)
if not path.exists():
report[name] = {"present": False}
context.log.warning("Artifact missing: %s", path)
continue
bundle = load_bundle(name)
report[name] = {
"present": True,
"size_bytes": path.stat().st_size,
"n_samples": getattr(bundle, "n_samples", None),
"trained_at": getattr(bundle, "trained_at", None),
"features": len(getattr(bundle, "feature_columns", []) or []),
"metrics": _jsonable(getattr(bundle, "extra", {}) or {}),
}
context.add_output_metadata({"report": MetadataValue.json(_jsonable(report))})
return _jsonable(report)
def _jsonable(value: Any) -> Any:
if isinstance(value, dict):
return {str(k): _jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_jsonable(v) for v in value]
if isinstance(value, (str, int, float, bool)) or value is None:
return value
return str(value)

162
orchestration/assets/nutrition.py Normal file
View File

@@ -0,0 +1,162 @@
"""Nutrition enrichment and the two nutrition models.
Wraps `nutrition_enrichment_service`, which already owns the Open Food Facts
lookup, the scoring, the diet-tag classification and the persistence. Dagster
contributes the dependency on `catalog_database` (you cannot enrich products
that are not stored yet), a bounded retry around a third-party API, and a
visible record of how many products were enriched versus skipped.
"""
# NOTE: deliberately no `from __future__ import annotations` here.
# Dagster resolves the decorated function signatures at definition time to
# validate the `context` parameter and to infer asset input types. Under
# PEP 563/649 the annotations arrive as strings and that validation fails
# with "Cannot annotate `context` parameter with type AssetExecutionContext".
# Local Python is 3.14, which defers annotations by default, so this is not
# hypothetical.
import time
from typing import Any, Dict
from dagster import (
AssetExecutionContext,
Backoff,
Config,
Jitter,
MetadataValue,
RetryPolicy,
asset,
)
from orchestration.config import require_local_database
from orchestration.partitions import brand_partitions
GROUP = "nutrition"
# Open Food Facts is a free public API and does rate-limit. Two retries with
# exponential backoff and jitter is enough for a transient 429/timeout without
# turning a bad afternoon into a retry storm.
OFF_RETRY = RetryPolicy(
max_retries=2, delay=10, backoff=Backoff.EXPONENTIAL, jitter=Jitter.PLUS_MINUS
)
class NutritionConfig(Config):
"""Per-run knobs.
`max_products` exists because a first enrichment of a large brand is one
outbound request per product. Capping it makes the asset safe to try
interactively before committing to a full run.
"""
skip_if_verified: bool = True
generate_narrative: bool = False
max_products: int = 50
@asset(
group_name=GROUP,
partitions_def=brand_partitions(),
retry_policy=OFF_RETRY,
deps=["catalog_database"],
description="Nutrition facts, scores and diet tags for this brand's products.",
)
def nutrition_data(
context: AssetExecutionContext, config: NutritionConfig
) -> Dict[str, Any]:
"""Per-product enrichment via `enrich_one_product`.
The single-product entry point is used rather than `enrich_all_products`
because the latter walks every brand in the database. Here the brand comes
from the partition, which is what keeps a partitioned run doing one brand's
worth of work.
`generate_narrative` defaults to False: the narrative is written by the
local Ollama model, and on an 8GB CPU-only machine that is by far the
slowest part of enrichment. The nutrition panel renders without it.
"""
from app.services.nutrition_enrichment_service import enrich_one_product
from app.services.vector_store import get_products_by_brand
require_local_database("nutrition_data")
brand = context.partition_key
started = time.time()
products = get_products_by_brand(brand, limit=config.max_products) or []
counts: Dict[str, int] = {}
for product in products:
image_id = product.get("image_id")
if not image_id:
continue
try:
status = enrich_one_product(
brand,
image_id,
product.get("product_name") or product.get("title") or "",
product.get("category") or "",
skip_if_verified=config.skip_if_verified,
generate_narrative=config.generate_narrative,
)
except Exception as exc: # noqa: BLE001 - one bad product must not
# end the partition; the count of errors is the signal.
context.log.debug("Enrichment failed for %s: %s", image_id, exc)
status = "error"
counts[status] = counts.get(status, 0) + 1
result = {
"brand": brand,
"products_considered": len(products),
"by_status": counts,
"duration_s": round(time.time() - started, 2),
}
context.add_output_metadata(
{
"brand": brand,
"products_considered": len(products),
"by_status": MetadataValue.json(counts),
"narrative_generated": config.generate_narrative,
"duration_s": result["duration_s"],
}
)
return result
@asset(
group_name=GROUP,
deps=["nutrition_data"],
description="KNN similarity index and KMeans clusters over nutrition_facts.",
)
def nutrition_models(context: AssetExecutionContext) -> Dict[str, Any]:
"""`nutrition_enrichment_service.train_all_models()`, unpartitioned.
Deliberately NOT partitioned by brand: both models fit across the whole
nutrition_facts table, and "find me a healthier alternative" is only useful
if it can cross brands. Partitioning would have produced per-brand indexes
that answer a narrower question than the endpoint asks.
"""
from app.services.nutrition_enrichment_service import train_all_models
require_local_database("nutrition_models")
started = time.time()
result = train_all_models() or {}
context.add_output_metadata(
{
"result": MetadataValue.json(_jsonable(result)),
"duration_s": round(time.time() - started, 2),
"artifacts": "nutrition_similarity.joblib, nutrition_clustering.joblib",
}
)
return _jsonable(result)
def _jsonable(value: Any) -> Any:
if isinstance(value, dict):
return {str(k): _jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_jsonable(v) for v in value]
if isinstance(value, (str, int, float, bool)) or value is None:
return value
return str(value)