From e073d68d6005caeb76f60723d4517c766b0d6330 Mon Sep 17 00:00:00 2001 From: sriram Date: Fri, 28 Aug 2026 07:47:44 +0530 Subject: [PATCH] Feature insertion- File importing --- src/api/client.js | 76 +++++ src/pages/AdminPage.jsx | 8 +- src/pages/BatchCatalogPanel.jsx | 501 ++++++++++++++++++++++++++++++++ 3 files changed, 583 insertions(+), 2 deletions(-) create mode 100644 src/pages/BatchCatalogPanel.jsx diff --git a/src/api/client.js b/src/api/client.js index 9f7a61e..1899f0f 100644 --- a/src/api/client.js +++ b/src/api/client.js @@ -131,6 +131,52 @@ async function upload(path, file, { fieldName = 'file' } = {}) { return res.json(); } +/** + * Post several files under one field name. + * + * Separate from `upload` rather than a flag on it: `upload` is used by six + * call sites that send exactly one file, and widening its signature would put + * an array check in front of all of them for the benefit of one. + * + * FastAPI reads `files: List[UploadFile]` from repeated parts sharing a name, + * which is why every file is appended under the same key. As with `upload`, + * Content-Type is deliberately left unset so the browser writes the multipart + * boundary itself. + */ +async function uploadMany(path, files, { fieldName = 'files' } = {}) { + const formData = new FormData(); + Array.from(files || []).forEach((file) => formData.append(fieldName, file)); + + let res; + try { + res = await fetch(`${BASE}${path}`, { + method: 'POST', + headers: authHeaders(), + body: formData, + }); + } catch { + throw new ApiError( + `Could not reach the API at ${BASE || '(same origin)'}${path}. Is the backend running ` + + `and reachable?`, + 0 + ); + } + + if (!res.ok) { + let detail = `Upload failed (${res.status})`; + try { + const body = await res.json(); + detail = body.detail || JSON.stringify(body); + } catch { + /* ignore parse errors, keep generic message */ + } + if (res.status === 401 && onUnauthorized) onUnauthorized(detail); + throw new ApiError(detail, res.status); + } + + return res.json(); +} + function qs(params = {}) { const usp = new URLSearchParams(); Object.entries(params).forEach(([k, v]) => { @@ -304,6 +350,36 @@ export const api = { getStoreCatalogJob: (jobId) => request(`/api/admin/store-catalog/jobs/${encodeURIComponent(jobId)}`), + // --- Admin: batch catalog ingestion (many spreadsheets -> one batch) --- + // The multi-file sibling of the block above. See + // app/api/routers/batch_catalog.py. + // + // NOTE the flag defaults are the OPPOSITE of ingestStoreCatalog's. One file + // with image search on is a considered trade; twenty files is thousands of + // outbound requests and a Playwright subprocess per row, on a single-vCPU + // container that is also serving the API. A batch opts in. + previewCatalogBatch: (files) => + uploadMany('/api/admin/catalog-batch/preview', files), + + ingestCatalogBatch: (files, { use_llm = false, fetch_images = false } = {}) => + uploadMany(`/api/admin/catalog-batch/ingest${qs({ use_llm, fetch_images })}`, files), + + getCatalogBatch: (batchId) => + request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}`), + + listCatalogBatches: (limit = 20) => + request(`/api/admin/catalog-batch/batches${qs({ limit })}`), + + resumeCatalogBatch: (batchId) => + request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}/resume`, { + method: 'POST', + }), + + cancelCatalogBatch: (batchId) => + request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}/cancel`, { + method: 'POST', + }), + // --- User workspace: single + batch product entry --- // Guarded by `add_product` / `upload_batch_products` - see // app/api/routers/user_products.py. diff --git a/src/pages/AdminPage.jsx b/src/pages/AdminPage.jsx index 0a3d21a..eeecff1 100644 --- a/src/pages/AdminPage.jsx +++ b/src/pages/AdminPage.jsx @@ -1,8 +1,9 @@ import React, { useEffect, useState } from 'react'; -import { Wrench, Database, UploadCloud } from 'lucide-react'; +import { Wrench, Database, UploadCloud, Layers } from 'lucide-react'; import { api } from '../api/client'; import { NavigationHeader } from '../components/NavigationHeader'; import { StoreCatalogPanel } from './StoreCatalogPanel'; +import { BatchCatalogPanel } from './BatchCatalogPanel'; /* * The admin panel carries two views: the read-only project overview, and the @@ -18,6 +19,7 @@ import { StoreCatalogPanel } from './StoreCatalogPanel'; const TABS = [ { id: 'project', label: 'Project Overview Details', icon: Database }, { id: 'ingest', label: 'Store Catalog Ingestion', icon: UploadCloud }, + { id: 'batch', label: 'Batch Catalog Ingestion', icon: Layers }, ]; export function AdminPage() { @@ -45,7 +47,7 @@ export function AdminPage() {
@@ -68,6 +70,8 @@ export function AdminPage() { {activeTab === 'ingest' && } + {activeTab === 'batch' && } + {activeTab === 'project' && (
diff --git a/src/pages/BatchCatalogPanel.jsx b/src/pages/BatchCatalogPanel.jsx new file mode 100644 index 0000000..2eaf668 --- /dev/null +++ b/src/pages/BatchCatalogPanel.jsx @@ -0,0 +1,501 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { + FileSpreadsheet, PlayCircle, Loader2, CheckCircle2, + XCircle, AlertTriangle, ArrowRight, Table2, Layers, X, RotateCcw, Ban, + Clock, MinusCircle, +} from 'lucide-react'; +import { api } from '../api/client'; + +/* + * Several store spreadsheets -> one batch -> the same 11-stage pipeline. + * + * The multi-file sibling of StoreCatalogPanel, which is left exactly as it + * was. What is different here is the unit of work: five files are ONE batch + * with one id, so the question a colleague actually asks - "did the drop + * land?" - has one answer instead of five. + * + * Two things are deliberately not like the single-file panel: + * + * - the enrichment toggles default OFF. At one file, image search is a + * considered trade; at twenty it is thousands of outbound requests on a + * box that is also serving the API. + * - a file that fails does not fail the batch. The per-file table is the + * point of the screen, and a `partial` batch is rendered as its own state + * so four-of-five landing never reads as plain success. + */ + +const POLL_MS = 3000; +const TERMINAL = new Set(['done', 'failed', 'partial', 'cancelled']); + +const FILE_TONE = { + done: 'bg-emerald-500/10 text-emerald-700 border-emerald-500/20', + failed: 'bg-maroon-100 text-maroon-600 border-maroon-500/30', + running: 'bg-amber-500/10 text-amber-700 border-amber-500/20', + queued: 'bg-white text-slate-500 border-ink-900/10', + cancelled: 'bg-ink-100 text-slate-500 border-ink-900/10', +}; + +const FILE_ICON = { + done: CheckCircle2, + failed: XCircle, + running: Loader2, + queued: Clock, + cancelled: MinusCircle, +}; + +function fmtBytes(n) { + if (!n) return ''; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(0)} KB`; + return `${(n / (1024 * 1024)).toFixed(1)} MB`; +} + +export function BatchCatalogPanel() { + const [files, setFiles] = useState([]); + const [preview, setPreview] = useState(null); + const [previewing, setPreviewing] = useState(false); + const [starting, setStarting] = useState(false); + const [error, setError] = useState(''); + const [batch, setBatch] = useState(null); + const [useLlm, setUseLlm] = useState(false); + const [fetchImages, setFetchImages] = useState(false); + + const pollRef = useRef(null); + const batchIdRef = useRef(null); + + // One interval for the component's lifetime, reading the id from a ref + // rather than closing over it. Same reasoning as StoreCatalogPanel: starting + // a second batch must not leave an older interval polling the previous one. + useEffect(() => { + pollRef.current = setInterval(async () => { + const id = batchIdRef.current; + if (!id) return; + try { + const next = await api.getCatalogBatch(id); + setBatch(next); + // Stop polling once there is nothing left to watch, so an admin tab + // left open overnight is not hitting the API forever. + if (TERMINAL.has(next.status)) batchIdRef.current = null; + } catch { + /* a dropped poll is not worth surfacing; the next tick retries */ + } + }, POLL_MS); + return () => clearInterval(pollRef.current); + }, []); + + const onPick = (e) => { + setFiles(Array.from(e.target.files || [])); + setPreview(null); + setBatch(null); + setError(''); + // Clearing the input means re-picking the same files fires onChange again. + e.target.value = ''; + }; + + const removeAt = (index) => { + setFiles((current) => current.filter((_, i) => i !== index)); + setPreview(null); + }; + + const handlePreview = async () => { + if (!files.length) return; + setPreviewing(true); + setError(''); + try { + setPreview(await api.previewCatalogBatch(files)); + } catch (err) { + setError(err?.message || 'Could not read those files.'); + } finally { + setPreviewing(false); + } + }; + + const handleIngest = async () => { + if (!files.length) return; + setStarting(true); + setError(''); + try { + const started = await api.ingestCatalogBatch(files, { + use_llm: useLlm, + fetch_images: fetchImages, + }); + batchIdRef.current = started.batch_id; + setBatch(started); + } catch (err) { + setError(err?.message || 'Could not start the batch.'); + } finally { + setStarting(false); + } + }; + + const handleResume = async () => { + if (!batch) return; + setError(''); + try { + const resumed = await api.resumeCatalogBatch(batch.batch_id); + batchIdRef.current = resumed.batch_id; + setBatch(resumed); + } catch (err) { + setError(err?.message || 'Could not resume the batch.'); + } + }; + + const handleCancel = async () => { + if (!batch) return; + setError(''); + try { + setBatch(await api.cancelCatalogBatch(batch.batch_id)); + } catch (err) { + setError(err?.message || 'Could not cancel the batch.'); + } + }; + + const running = batch && !TERMINAL.has(batch.status); + const totals = batch?.totals; + + return ( +
+ {/* ---- 1. File selection --------------------------------------- */} +
+

+ Batch Catalog Ingestion +

+

+ Upload one or many store product lists at once (.xlsx, .xls, .csv or .tsv). They run + as a single batch through the same 11-stage pipeline — images, price bands, pack + sizes, SKU, barcode, HSN/GST and FSSAI — then validated and written into the + matching brand tables. Files are processed one at a time; a file that fails is + reported and the rest carry on. Columns a sheet already fills are never overwritten. +

+ +
+ + + + + +
+ + {/* Enrichment toggles. Off by default, and the cost is stated rather + than left for someone to discover on a 20-file run. */} +
+ + +
+ + {files.length > 0 && ( +
    + {files.map((file, index) => ( +
  • + {file.name} + + {fmtBytes(file.size)} + + +
    • + ))} +
+ )} + + {error && ( +
+ {error} +
+ )} +
+ + {/* ---- 2. Per-file column mapping preview ---------------------- */} + {preview && ( +
+

+ Column mapping — {preview.files_ok}/{preview.files_total} file(s) readable, + {' '}{preview.rows_total} row(s) total +

+

+ Nothing has been ingested yet. Check the mapping, then start the batch. +

+ +
+ {preview.files.map((file) => ( +
+
+ {file.filename} + {file.ok ? ( + + {file.rows_total} rows + + ) : ( + + will be skipped + + )} +
+ + {!file.ok && ( +

+ {file.error} +

+ )} + + {file.ok && ( + <> +

+ {file.brand_column_present + ? 'Brand column found.' + : 'No brand column; the brand will be inferred from each product name.'} +

+
+ {Object.entries(file.recognised_columns).map(([field, column]) => ( +
+ {column} + + {field} +
+ ))} +
+ {file.unrecognised_columns?.length > 0 && ( +

+ + Ignored: {file.unrecognised_columns.join(', ')} +

+ )} + + )} +
+ ))} +
+
+ )} + + {/* ---- 3. Live batch tracker ----------------------------------- */} + {batch && ( +
+
+

+ {batch.status === 'done' && } + {batch.status === 'partial' && } + {batch.status === 'failed' && } + {batch.status === 'interrupted' && } + {running && } + Batch — {batch.files_done}/{batch.files_total} file(s) done +

+
+ {batch.status === 'interrupted' && ( + + )} + {running && ( + + )} + {batch.status} +
+
+ +

{batch.batch_id}

+ + {batch.status === 'partial' && ( +
+ + {batch.files_done} file(s) landed and {batch.files_failed} failed. Check the failed + rows below — the rest of the batch was written. +
+ )} + +
+
+
+ + {batch.detail &&

{batch.detail}

} + + {/* Per-file steps */} +
    + {batch.files.map((file) => { + const Icon = FILE_ICON[file.status] || Clock; + return ( +
  • +
    + + + {file.filename} + + {file.status} +
    + + {file.stage_index > 0 && file.status === 'running' && ( +

    + Stage {file.stage_index}/{file.total_stages} — {file.stage_name} + {' · '}{file.rows_done}/{file.rows_total} rows +

    + )} + + {file.detail && ( +

    {file.detail}

    + )} +
    • + ); + })} +
+ + {/* Batch totals, once anything has landed */} + {totals && (batch.files_done > 0 || batch.files_failed > 0) && ( + <> +
+ + + + + +
+ + {batch.brands?.length > 0 && ( +

+ Brands touched:{' '} + {batch.brands.join(', ')} +

+ )} + + + (f.result?.rejections || []).map( + (r) => `${f.filename} · ${r.product_name} ${r.size || ''} — ${r.reason}` + ) + )} + /> + + (f.result?.errors || []).map( + (e) => `${f.filename} · row ${e.row}: ${e.product_name || '(unnamed)'} — ${e.error}` + ) + )} + /> + + (f.result?.warnings || []).map((w) => `${f.filename} · ${w}`) + )} + /> + + )} +
+ )} +
+ ); +} + +const TONES = { + emerald: 'border-emerald-500/20 bg-emerald-500/5 text-emerald-700', + blue: 'border-blue-500/20 bg-blue-500/5 text-blue-700', + slate: 'border-ink-900/10 bg-white text-slate-600', + maroon: 'border-maroon-500/20 bg-maroon-100/40 text-maroon-600', + amber: 'border-amber-500/20 bg-amber-500/5 text-amber-700', +}; + +function Stat({ label, value, tone }) { + return ( +
+

{label}

+

{value ?? 0}

+
+ ); +} + +function IssueList({ title, items }) { + if (!items || items.length === 0) return null; + return ( +
+

{title} ({items.length})

+
    + {items.map((text, idx) => ( +
  • + {text} +
    • + ))} +
+
+ ); +}