frontend enrichment pipeline
This commit is contained in:
sriram
@@ -284,6 +284,19 @@ export const api = {
|
||||
body: JSON.stringify({ store_id, rules }),
|
||||
}),
|
||||
|
||||
// --- Admin: store-catalog ingestion (Excel -> 11-stage pipeline -> brand tables) ---
|
||||
// See app/api/routers/store_catalog.py. `preview` parses only, so the
|
||||
// operator can check the column mapping before paying for a full scrape.
|
||||
previewStoreCatalog: (file) => upload('/api/admin/store-catalog/preview', file),
|
||||
|
||||
// The two flags are query params, not form fields: the endpoint takes them
|
||||
// as plain query arguments alongside the multipart body.
|
||||
ingestStoreCatalog: (file, { use_llm = true, fetch_images = true } = {}) =>
|
||||
upload(`/api/admin/store-catalog/ingest${qs({ use_llm, fetch_images })}`, file),
|
||||
|
||||
getStoreCatalogJob: (jobId) =>
|
||||
request(`/api/admin/store-catalog/jobs/${encodeURIComponent(jobId)}`),
|
||||
|
||||
// --- User workspace: single + batch product entry ---
|
||||
// Guarded by `add_product` / `upload_batch_products` - see
|
||||
// app/api/routers/user_products.py.
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { Wrench, Database } from 'lucide-react';
|
||||
import { Wrench, Database, UploadCloud } from 'lucide-react';
|
||||
import { api } from '../api/client';
|
||||
import { NavigationHeader } from '../components/NavigationHeader';
|
||||
import { StoreCatalogPanel } from './StoreCatalogPanel';
|
||||
|
||||
/*
|
||||
* The admin panel is the project overview and nothing else.
|
||||
* The admin panel carries two views: the read-only project overview, and the
|
||||
* store-catalog ingestion workspace.
|
||||
*
|
||||
* It used to carry three more sub-tabs behind a local `activeTab` switch -
|
||||
* train/test dataset upload, a stock-remaining discount allocator, and a
|
||||
* trigger for the vector ingestion pipeline. They were retired from the UI;
|
||||
* with one view left there is nothing to switch between, so the tab bar and
|
||||
* its state went too.
|
||||
*
|
||||
* Only the UI was removed. The endpoints behind those panels are still
|
||||
* registered and still work - ingestion in particular is reachable from
|
||||
* It briefly carried neither tab bar nor second view - three earlier sub-tabs
|
||||
* (train/test dataset upload, stock-remaining discount allocator, and a
|
||||
* trigger for the brand ingestion pipeline) were retired from the UI, leaving
|
||||
* only the overview. Their endpoints are all still registered and still work;
|
||||
* brand ingestion in particular is reachable from
|
||||
* `python cli/ingest_brand.py "<brand>"` and from POST /api/catalog/generate.
|
||||
*/
|
||||
const TABS = [
|
||||
{ id: 'project', label: 'Project Overview Details', icon: Database },
|
||||
{ id: 'ingest', label: 'Store Catalog Ingestion', icon: UploadCloud },
|
||||
];
|
||||
|
||||
export function AdminPage() {
|
||||
const [activeTab, setActiveTab] = useState('project');
|
||||
const [projectDetails, setProjectDetails] = useState(null);
|
||||
const [loadingProject, setLoadingProject] = useState(true);
|
||||
|
||||
@@ -40,11 +45,30 @@ export function AdminPage() {
|
||||
<div className="flex h-screen w-full flex-col overflow-y-auto bg-paper-100 text-ink-950 font-sans">
|
||||
<NavigationHeader
|
||||
title="Admin Intelligence Panel"
|
||||
subtitle="Brand catalogs, vector database tables & S3 storage overview"
|
||||
subtitle="Brand catalogs, vector tables & store-spreadsheet ingestion"
|
||||
icon={Wrench}
|
||||
/>
|
||||
|
||||
<div className="mx-auto w-full max-w-6xl px-6 py-8 space-y-6">
|
||||
<div className="flex items-center gap-2 border-b border-ink-900/10 pb-3 overflow-x-auto no-scrollbar">
|
||||
{TABS.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
onClick={() => setActiveTab(id)}
|
||||
className={`flex items-center gap-2 px-4 py-2 rounded-xl text-xs font-bold transition whitespace-nowrap cursor-pointer ${
|
||||
activeTab === id
|
||||
? 'bg-amber-500 text-slate-950 shadow-sm'
|
||||
: 'bg-paper-50 text-slate-600 hover:text-ink-950 border border-ink-900/10'
|
||||
}`}
|
||||
>
|
||||
<Icon className="h-4 w-4" /> {label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{activeTab === 'ingest' && <StoreCatalogPanel />}
|
||||
|
||||
{activeTab === 'project' && (
|
||||
<div className="space-y-6">
|
||||
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
|
||||
<h2 className="font-display text-lg font-bold text-ink-950 flex items-center gap-2 mb-2">
|
||||
@@ -99,6 +123,7 @@ export function AdminPage() {
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
265
src/pages/StoreCatalogPanel.jsx
Normal file
265
src/pages/StoreCatalogPanel.jsx
Normal file
@@ -0,0 +1,265 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import {
|
||||
UploadCloud, FileSpreadsheet, PlayCircle, Loader2, CheckCircle2,
|
||||
XCircle, AlertTriangle, ArrowRight, Table2,
|
||||
} from 'lucide-react';
|
||||
import { api } from '../api/client';
|
||||
|
||||
/*
|
||||
* Store spreadsheet -> 11-stage scraping pipeline -> brand tables.
|
||||
*
|
||||
* Two deliberate steps. "Preview" only parses the file and shows how each
|
||||
* column was understood; the mapping from a store's own headers onto catalog
|
||||
* fields is a guess, and discovering that "Item" was read as the description
|
||||
* after a 2000-row scrape is expensive. "Start Ingestion" is the committing
|
||||
* action, and from then on this polls the job for stage + row progress.
|
||||
*/
|
||||
|
||||
const POLL_MS = 3000;
|
||||
const TERMINAL = new Set(['done', 'failed']);
|
||||
|
||||
export function StoreCatalogPanel() {
|
||||
const [file, setFile] = useState(null);
|
||||
const [preview, setPreview] = useState(null);
|
||||
const [previewing, setPreviewing] = useState(false);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
const [job, setJob] = useState(null);
|
||||
|
||||
const pollRef = useRef(null);
|
||||
const jobIdRef = useRef(null);
|
||||
|
||||
// One interval for the component's lifetime. It reads the job id from a ref
|
||||
// rather than closing over it, so starting a second ingestion does not leave
|
||||
// an older interval polling the previous job.
|
||||
useEffect(() => {
|
||||
pollRef.current = setInterval(async () => {
|
||||
const id = jobIdRef.current;
|
||||
if (!id) return;
|
||||
try {
|
||||
const next = await api.getStoreCatalogJob(id);
|
||||
setJob(next);
|
||||
if (TERMINAL.has(next.status)) jobIdRef.current = null;
|
||||
} catch {
|
||||
/* a dropped poll is not worth surfacing; the next tick retries */
|
||||
}
|
||||
}, POLL_MS);
|
||||
return () => clearInterval(pollRef.current);
|
||||
}, []);
|
||||
|
||||
const onPick = (e) => {
|
||||
setFile(e.target.files?.[0] || null);
|
||||
setPreview(null);
|
||||
setJob(null);
|
||||
setError('');
|
||||
};
|
||||
|
||||
const handlePreview = async () => {
|
||||
if (!file) return;
|
||||
setPreviewing(true);
|
||||
setError('');
|
||||
try {
|
||||
setPreview(await api.previewStoreCatalog(file));
|
||||
} catch (err) {
|
||||
setError(err?.message || 'Could not read that file.');
|
||||
} finally {
|
||||
setPreviewing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleIngest = async () => {
|
||||
if (!file) return;
|
||||
setStarting(true);
|
||||
setError('');
|
||||
try {
|
||||
const started = await api.ingestStoreCatalog(file);
|
||||
jobIdRef.current = started.job_id;
|
||||
setJob(started);
|
||||
} catch (err) {
|
||||
setError(err?.message || 'Could not start the ingestion job.');
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const running = job && !TERMINAL.has(job.status);
|
||||
const result = job?.result;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* ---- 1. File selection --------------------------------------- */}
|
||||
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
|
||||
<h2 className="font-display text-lg font-bold text-ink-950 flex items-center gap-2 mb-2">
|
||||
<UploadCloud className="h-5 w-5 text-amber-500" /> Store Catalog Ingestion
|
||||
</h2>
|
||||
<p className="text-xs text-slate-500 leading-relaxed max-w-3xl">
|
||||
Upload a store's product list (.xlsx, .xls, .csv or .tsv). Every row is run through
|
||||
the 11-stage scraping pipeline to fill in whatever the sheet does not supply —
|
||||
images, price bands, pack sizes, SKU, barcode, HSN/GST and FSSAI — then validated
|
||||
and written into the matching brand table. Columns the sheet already fills are never
|
||||
overwritten.
|
||||
</p>
|
||||
|
||||
<div className="mt-5 flex flex-wrap items-center gap-3">
|
||||
<label className="flex items-center gap-2 px-4 py-2 rounded-xl border border-ink-900/15 bg-white text-xs font-bold text-ink-950 cursor-pointer hover:border-amber-500/60 transition">
|
||||
<FileSpreadsheet className="h-4 w-4 text-amber-600" />
|
||||
{file ? file.name : 'Choose spreadsheet'}
|
||||
<input type="file" accept=".xlsx,.xls,.csv,.tsv" onChange={onPick} className="hidden" />
|
||||
</label>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={handlePreview}
|
||||
disabled={!file || previewing}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-xl bg-ink-900 text-white text-xs font-bold hover:bg-ink-950 disabled:opacity-40 transition cursor-pointer"
|
||||
>
|
||||
{previewing ? <Loader2 className="h-4 w-4 animate-spin" /> : <Table2 className="h-4 w-4" />}
|
||||
Preview columns
|
||||
</button>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleIngest}
|
||||
disabled={!file || starting || running}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-xl bg-amber-500 text-slate-950 text-xs font-bold hover:bg-amber-600 disabled:opacity-40 transition cursor-pointer"
|
||||
>
|
||||
{starting ? <Loader2 className="h-4 w-4 animate-spin" /> : <PlayCircle className="h-4 w-4" />}
|
||||
Start ingestion <ArrowRight className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="mt-4 rounded-lg bg-maroon-100 border border-maroon-500/30 p-3 text-xs text-maroon-600 flex items-start gap-2">
|
||||
<XCircle className="h-4 w-4 shrink-0 mt-0.5" /> {error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* ---- 2. Column mapping preview ------------------------------- */}
|
||||
{preview && (
|
||||
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
|
||||
<h3 className="font-display text-sm font-bold text-ink-950 mb-1">
|
||||
Column mapping — {preview.rows_total} row(s)
|
||||
</h3>
|
||||
<p className="text-[11px] text-slate-500 mb-4">
|
||||
{preview.brand_column_present
|
||||
? 'Brand column found.'
|
||||
: 'No brand column; the brand will be inferred from each product name.'}
|
||||
</p>
|
||||
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-2">
|
||||
{Object.entries(preview.recognised_columns).map(([field, column]) => (
|
||||
<div key={field} className="flex items-center justify-between gap-2 p-2.5 rounded-xl border border-ink-900/10 bg-white">
|
||||
<span className="text-[11px] text-slate-500 truncate">{column}</span>
|
||||
<ArrowRight className="h-3 w-3 text-slate-400 shrink-0" />
|
||||
<span className="text-[11px] font-bold text-ink-950 truncate">{field}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{preview.unrecognised_columns?.length > 0 && (
|
||||
<p className="mt-4 text-[11px] text-amber-700 flex items-start gap-1.5">
|
||||
<AlertTriangle className="h-3.5 w-3.5 shrink-0 mt-0.5" />
|
||||
Ignored (no matching catalog field): {preview.unrecognised_columns.join(', ')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ---- 3. Live stage tracker ----------------------------------- */}
|
||||
{job && (
|
||||
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
|
||||
<div className="flex items-center justify-between gap-3 mb-4">
|
||||
<h3 className="font-display text-sm font-bold text-ink-950 flex items-center gap-2">
|
||||
{job.status === 'done' && <CheckCircle2 className="h-4 w-4 text-leaf-600" />}
|
||||
{job.status === 'failed' && <XCircle className="h-4 w-4 text-maroon-600" />}
|
||||
{running && <Loader2 className="h-4 w-4 animate-spin text-amber-600" />}
|
||||
Pipeline — {job.filename}
|
||||
</h3>
|
||||
<span className="font-mono text-[11px] uppercase text-slate-400">{job.status}</span>
|
||||
</div>
|
||||
|
||||
{job.stage_index > 0 && (
|
||||
<>
|
||||
<div className="flex items-center justify-between text-[11px] text-slate-500 mb-1.5">
|
||||
<span className="font-bold text-ink-950">
|
||||
Stage {job.stage_index}/{job.total_stages} — {job.stage_name}
|
||||
</span>
|
||||
<span className="font-mono">
|
||||
{job.rows_done}/{job.rows_total} rows
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-2 w-full rounded-full bg-ink-100 overflow-hidden">
|
||||
<div
|
||||
className="h-full rounded-full bg-amber-500 transition-all"
|
||||
style={{ width: `${Math.round((job.stage_index / job.total_stages) * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{job.detail && <p className="mt-3 text-xs text-slate-600">{job.detail}</p>}
|
||||
|
||||
{result && (
|
||||
<>
|
||||
<div className="mt-5 grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-5 gap-3">
|
||||
<Stat label="Inserted" value={result.inserted} tone="emerald" />
|
||||
<Stat label="Backfilled" value={result.backfilled} tone="blue" />
|
||||
<Stat label="Unchanged" value={result.skipped_existing} tone="slate" />
|
||||
<Stat label="Rejected" value={result.rejected} tone="maroon" />
|
||||
<Stat label="Row errors" value={result.error_count} tone="amber" />
|
||||
</div>
|
||||
|
||||
{result.brands?.length > 0 && (
|
||||
<p className="mt-4 text-[11px] text-slate-500">
|
||||
Brands touched: <span className="font-bold text-ink-950">{result.brands.join(', ')}</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<IssueList title="Rejected by the validation gate" items={
|
||||
(result.rejections || []).map((r) => `${r.product_name} ${r.size || ''} — ${r.reason}`)
|
||||
} />
|
||||
<IssueList title="Row errors" items={
|
||||
(result.errors || []).map((e) => `row ${e.row}: ${e.product_name || '(unnamed)'} — ${e.error}`)
|
||||
} />
|
||||
<IssueList title="Warnings" items={result.warnings || []} />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const TONES = {
|
||||
emerald: 'border-emerald-500/20 bg-emerald-500/5 text-emerald-700',
|
||||
blue: 'border-blue-500/20 bg-blue-500/5 text-blue-700',
|
||||
slate: 'border-ink-900/10 bg-white text-slate-600',
|
||||
maroon: 'border-maroon-500/20 bg-maroon-100/40 text-maroon-600',
|
||||
amber: 'border-amber-500/20 bg-amber-500/5 text-amber-700',
|
||||
};
|
||||
|
||||
function Stat({ label, value, tone }) {
|
||||
return (
|
||||
<div className={`rounded-xl border p-3 ${TONES[tone] || TONES.slate}`}>
|
||||
<p className="text-[10px] font-semibold uppercase">{label}</p>
|
||||
<p className="font-mono text-2xl font-extrabold text-ink-950 mt-0.5">{value ?? 0}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function IssueList({ title, items }) {
|
||||
if (!items || items.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-4">
|
||||
<h4 className="text-[11px] font-bold text-ink-950 mb-1.5">{title} ({items.length})</h4>
|
||||
<ul className="max-h-40 overflow-y-auto space-y-1 pr-1">
|
||||
{items.map((text, idx) => (
|
||||
<li key={idx} className="text-[11px] text-slate-600 bg-white border border-ink-900/10 rounded-lg px-2.5 py-1.5">
|
||||
{text}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user