Brand Discovery-LLM Updates

app/api/routers/brand_discovery.py

@@ -33,6 +33,7 @@ from app.infrastructure.settings import (
     BATCH_MAX_TOTAL_ROWS,
     BRAND_DISCOVERY_DEADLINE_SECONDS,
     BRAND_DISCOVERY_MAX_PRODUCTS,
+    WEB_DISCOVERY_ENABLED,
 )
 from app.services import active_brands, brand_discovery
 
@@ -72,6 +73,11 @@ class DiscoveryPreviewRequest(BaseModel):
     deadline_seconds: float = Field(default=90.0, ge=0.0,
                                     le=BRAND_DISCOVERY_DEADLINE_SECONDS)
     refresh_corpus: bool = False
+    # Web & retail listings. Reads a FINISHED web-discovery job (start one at
+    # POST /web-jobs first); off by default, and off means the preview is
+    # exactly what it was before this source existed.
+    use_web: bool = False
+    web_job_id: Optional[str] = None
 
 
 class DiscoveredProductIn(BaseModel):
@@ -93,6 +99,19 @@ class DiscoveredProductIn(BaseModel):
     fssai_license: Optional[str] = None
     barcode: Optional[str] = None
     image_url: Optional[str] = None
+    # Web-found rows only: the listings that justified the product. Not
+    # written by the pipeline; recorded afterwards into field_sources by
+    # web_discovery.provenance, which also fills a blank price_range from them.
+    listings: List[Dict[str, Any]] = Field(default_factory=list)
+    price_range: Optional[str] = None
+    retailer_count: int = 0
+
+
+class WebJobRequest(BaseModel):
+    brand: str
+    # Re-search even when a finished job for this brand is less than a day old.
+    # Answers already cached are still reused, so this is cheap.
+    refresh: bool = False
 
 
 class DiscoveryIngestRequest(BaseModel):
@@ -131,6 +150,8 @@ async def preview_brand_discovery(payload: DiscoveryPreviewRequest) -> Dict[str,
             use_llm=payload.use_llm,
             require_evidence=payload.require_evidence,
             refresh_corpus=payload.refresh_corpus,
+            use_web=payload.use_web and WEB_DISCOVERY_ENABLED,
+            web_job_id=payload.web_job_id,
         )
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
@@ -220,9 +241,51 @@ async def ingest_brand_discovery(payload: DiscoveryIngestRequest) -> batch_commo
                 "press Resume on it once the current batch finishes."
             ),
         )
+    # Web-found rows: record their retailer listings once the batch has stored
+    # them. Keyed by CSV position, which is how stage 11 reports source rows.
+    web_entries = {
+        index: {"listings": item.listings, "price_range": item.price_range or "",
+                "retailer_count": item.retailer_count}
+        for index, item in enumerate(payload.products) if item.listings
+    }
+    if web_entries:
+        from app.services.web_discovery import provenance
+        provenance.watch(manifest.batch_id, brand, web_entries)
     return batch_common.to_out(manifest)
 
 
+# ---------------------------------------------------------------------------
+# Web & retail listings - a background search the panel polls
+# ---------------------------------------------------------------------------
+def _require_web() -> None:
+    if not WEB_DISCOVERY_ENABLED:
+        raise HTTPException(status_code=404, detail="Web discovery is switched off (WEB_DISCOVERY_ENABLED).")
+
+
+@router.post("/web-jobs", status_code=status.HTTP_202_ACCEPTED,
+             dependencies=[Depends(require_admin)])
+def start_web_job(payload: WebJobRequest) -> Dict[str, Any]:
+    """Start searching retailer listings for a brand, or return the job that is
+    already running or finished within the last day. Poll GET /web-jobs/{id}."""
+    _require_web()
+    from app.services.web_discovery import jobs as web_jobs
+    try:
+        job = web_jobs.start_job(payload.brand, refresh=payload.refresh)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return job.summary()
+
+
+@router.get("/web-jobs/{job_id}", dependencies=[Depends(require_admin)])
+def get_web_job(job_id: str) -> Dict[str, Any]:
+    _require_web()
+    from app.services.web_discovery import jobs as web_jobs
+    job = web_jobs.get_job(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="No web discovery job with that id.")
+    return job.summary()
+
+
 # ---------------------------------------------------------------------------
 # The ACTIVE_BRANDS message, in one place
 # ---------------------------------------------------------------------------