Feature insertion- File importing

This commit is contained in:
sriram
2026-08-28 07:47:44 +05:30
parent e1247ce1b2
commit e073d68d60
3 changed files with 583 additions and 2 deletions

View File

@@ -131,6 +131,52 @@ async function upload(path, file, { fieldName = 'file' } = {}) {
return res.json();
}
/**
* Post several files under one field name.
*
* Separate from `upload` rather than a flag on it: `upload` is used by six
* call sites that send exactly one file, and widening its signature would put
* an array check in front of all of them for the benefit of one.
*
* FastAPI reads `files: List[UploadFile]` from repeated parts sharing a name,
* which is why every file is appended under the same key. As with `upload`,
* Content-Type is deliberately left unset so the browser writes the multipart
* boundary itself.
*/
async function uploadMany(path, files, { fieldName = 'files' } = {}) {
const formData = new FormData();
Array.from(files || []).forEach((file) => formData.append(fieldName, file));
let res;
try {
res = await fetch(`${BASE}${path}`, {
method: 'POST',
headers: authHeaders(),
body: formData,
});
} catch {
throw new ApiError(
`Could not reach the API at ${BASE || '(same origin)'}${path}. Is the backend running ` +
`and reachable?`,
0
);
}
if (!res.ok) {
let detail = `Upload failed (${res.status})`;
try {
const body = await res.json();
detail = body.detail || JSON.stringify(body);
} catch {
/* ignore parse errors, keep generic message */
}
if (res.status === 401 && onUnauthorized) onUnauthorized(detail);
throw new ApiError(detail, res.status);
}
return res.json();
}
function qs(params = {}) {
const usp = new URLSearchParams();
Object.entries(params).forEach(([k, v]) => {
@@ -304,6 +350,36 @@ export const api = {
getStoreCatalogJob: (jobId) =>
request(`/api/admin/store-catalog/jobs/${encodeURIComponent(jobId)}`),
// --- Admin: batch catalog ingestion (many spreadsheets -> one batch) ---
// The multi-file sibling of the block above. See
// app/api/routers/batch_catalog.py.
//
// NOTE the flag defaults are the OPPOSITE of ingestStoreCatalog's. One file
// with image search on is a considered trade; twenty files is thousands of
// outbound requests and a Playwright subprocess per row, on a single-vCPU
// container that is also serving the API. A batch opts in.
previewCatalogBatch: (files) =>
uploadMany('/api/admin/catalog-batch/preview', files),
ingestCatalogBatch: (files, { use_llm = false, fetch_images = false } = {}) =>
uploadMany(`/api/admin/catalog-batch/ingest${qs({ use_llm, fetch_images })}`, files),
getCatalogBatch: (batchId) =>
request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}`),
listCatalogBatches: (limit = 20) =>
request(`/api/admin/catalog-batch/batches${qs({ limit })}`),
resumeCatalogBatch: (batchId) =>
request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}/resume`, {
method: 'POST',
}),
cancelCatalogBatch: (batchId) =>
request(`/api/admin/catalog-batch/batches/${encodeURIComponent(batchId)}/cancel`, {
method: 'POST',
}),
// --- User workspace: single + batch product entry ---
// Guarded by `add_product` / `upload_batch_products` - see
// app/api/routers/user_products.py.

View File

@@ -1,8 +1,9 @@
import React, { useEffect, useState } from 'react';
import { Wrench, Database, UploadCloud } from 'lucide-react';
import { Wrench, Database, UploadCloud, Layers } from 'lucide-react';
import { api } from '../api/client';
import { NavigationHeader } from '../components/NavigationHeader';
import { StoreCatalogPanel } from './StoreCatalogPanel';
import { BatchCatalogPanel } from './BatchCatalogPanel';
/*
* The admin panel carries two views: the read-only project overview, and the
@@ -18,6 +19,7 @@ import { StoreCatalogPanel } from './StoreCatalogPanel';
const TABS = [
{ id: 'project', label: 'Project Overview Details', icon: Database },
{ id: 'ingest', label: 'Store Catalog Ingestion', icon: UploadCloud },
{ id: 'batch', label: 'Batch Catalog Ingestion', icon: Layers },
];
export function AdminPage() {
@@ -45,7 +47,7 @@ export function AdminPage() {
<div className="flex h-screen w-full flex-col overflow-y-auto bg-paper-100 text-ink-950 font-sans">
<NavigationHeader
title="Admin Intelligence Panel"
subtitle="Brand catalogs, vector tables & store-spreadsheet ingestion"
subtitle="Brand catalogs, vector tables & single or batch spreadsheet ingestion"
icon={Wrench}
/>
@@ -68,6 +70,8 @@ export function AdminPage() {
{activeTab === 'ingest' && <StoreCatalogPanel />}
{activeTab === 'batch' && <BatchCatalogPanel />}
{activeTab === 'project' && (
<div className="space-y-6">
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">

View File

@@ -0,0 +1,501 @@
import React, { useEffect, useRef, useState } from 'react';
import {
FileSpreadsheet, PlayCircle, Loader2, CheckCircle2,
XCircle, AlertTriangle, ArrowRight, Table2, Layers, X, RotateCcw, Ban,
Clock, MinusCircle,
} from 'lucide-react';
import { api } from '../api/client';
/*
* Several store spreadsheets -> one batch -> the same 11-stage pipeline.
*
* The multi-file sibling of StoreCatalogPanel, which is left exactly as it
* was. What is different here is the unit of work: five files are ONE batch
* with one id, so the question a colleague actually asks - "did the drop
* land?" - has one answer instead of five.
*
* Two things are deliberately not like the single-file panel:
*
* - the enrichment toggles default OFF. At one file, image search is a
* considered trade; at twenty it is thousands of outbound requests on a
* box that is also serving the API.
* - a file that fails does not fail the batch. The per-file table is the
* point of the screen, and a `partial` batch is rendered as its own state
* so four-of-five landing never reads as plain success.
*/
const POLL_MS = 3000;
const TERMINAL = new Set(['done', 'failed', 'partial', 'cancelled']);
const FILE_TONE = {
done: 'bg-emerald-500/10 text-emerald-700 border-emerald-500/20',
failed: 'bg-maroon-100 text-maroon-600 border-maroon-500/30',
running: 'bg-amber-500/10 text-amber-700 border-amber-500/20',
queued: 'bg-white text-slate-500 border-ink-900/10',
cancelled: 'bg-ink-100 text-slate-500 border-ink-900/10',
};
const FILE_ICON = {
done: CheckCircle2,
failed: XCircle,
running: Loader2,
queued: Clock,
cancelled: MinusCircle,
};
function fmtBytes(n) {
if (!n) return '';
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(0)} KB`;
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
}
export function BatchCatalogPanel() {
const [files, setFiles] = useState([]);
const [preview, setPreview] = useState(null);
const [previewing, setPreviewing] = useState(false);
const [starting, setStarting] = useState(false);
const [error, setError] = useState('');
const [batch, setBatch] = useState(null);
const [useLlm, setUseLlm] = useState(false);
const [fetchImages, setFetchImages] = useState(false);
const pollRef = useRef(null);
const batchIdRef = useRef(null);
// One interval for the component's lifetime, reading the id from a ref
// rather than closing over it. Same reasoning as StoreCatalogPanel: starting
// a second batch must not leave an older interval polling the previous one.
useEffect(() => {
pollRef.current = setInterval(async () => {
const id = batchIdRef.current;
if (!id) return;
try {
const next = await api.getCatalogBatch(id);
setBatch(next);
// Stop polling once there is nothing left to watch, so an admin tab
// left open overnight is not hitting the API forever.
if (TERMINAL.has(next.status)) batchIdRef.current = null;
} catch {
/* a dropped poll is not worth surfacing; the next tick retries */
}
}, POLL_MS);
return () => clearInterval(pollRef.current);
}, []);
const onPick = (e) => {
setFiles(Array.from(e.target.files || []));
setPreview(null);
setBatch(null);
setError('');
// Clearing the input means re-picking the same files fires onChange again.
e.target.value = '';
};
const removeAt = (index) => {
setFiles((current) => current.filter((_, i) => i !== index));
setPreview(null);
};
const handlePreview = async () => {
if (!files.length) return;
setPreviewing(true);
setError('');
try {
setPreview(await api.previewCatalogBatch(files));
} catch (err) {
setError(err?.message || 'Could not read those files.');
} finally {
setPreviewing(false);
}
};
const handleIngest = async () => {
if (!files.length) return;
setStarting(true);
setError('');
try {
const started = await api.ingestCatalogBatch(files, {
use_llm: useLlm,
fetch_images: fetchImages,
});
batchIdRef.current = started.batch_id;
setBatch(started);
} catch (err) {
setError(err?.message || 'Could not start the batch.');
} finally {
setStarting(false);
}
};
const handleResume = async () => {
if (!batch) return;
setError('');
try {
const resumed = await api.resumeCatalogBatch(batch.batch_id);
batchIdRef.current = resumed.batch_id;
setBatch(resumed);
} catch (err) {
setError(err?.message || 'Could not resume the batch.');
}
};
const handleCancel = async () => {
if (!batch) return;
setError('');
try {
setBatch(await api.cancelCatalogBatch(batch.batch_id));
} catch (err) {
setError(err?.message || 'Could not cancel the batch.');
}
};
const running = batch && !TERMINAL.has(batch.status);
const totals = batch?.totals;
return (
<div className="space-y-6">
{/* ---- 1. File selection --------------------------------------- */}
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
<h2 className="font-display text-lg font-bold text-ink-950 flex items-center gap-2 mb-2">
<Layers className="h-5 w-5 text-amber-500" /> Batch Catalog Ingestion
</h2>
<p className="text-xs text-slate-500 leading-relaxed max-w-3xl">
Upload one or many store product lists at once (.xlsx, .xls, .csv or .tsv). They run
as a single batch through the same 11-stage pipeline &mdash; images, price bands, pack
sizes, SKU, barcode, HSN/GST and FSSAI &mdash; then validated and written into the
matching brand tables. Files are processed one at a time; a file that fails is
reported and the rest carry on. Columns a sheet already fills are never overwritten.
</p>
<div className="mt-5 flex flex-wrap items-center gap-3">
<label className="flex items-center gap-2 px-4 py-2 rounded-xl border border-ink-900/15 bg-white text-xs font-bold text-ink-950 cursor-pointer hover:border-amber-500/60 transition">
<FileSpreadsheet className="h-4 w-4 text-amber-600" />
{files.length ? `${files.length} file${files.length === 1 ? '' : 's'} selected` : 'Choose spreadsheets'}
<input
type="file"
multiple
accept=".xlsx,.xls,.csv,.tsv"
onChange={onPick}
className="hidden"
/>
</label>
<button
type="button"
onClick={handlePreview}
disabled={!files.length || previewing}
className="flex items-center gap-2 px-4 py-2 rounded-xl bg-ink-900 text-white text-xs font-bold hover:bg-ink-950 disabled:opacity-40 transition cursor-pointer"
>
{previewing ? <Loader2 className="h-4 w-4 animate-spin" /> : <Table2 className="h-4 w-4" />}
Preview all
</button>
<button
type="button"
onClick={handleIngest}
disabled={!files.length || starting || running}
className="flex items-center gap-2 px-4 py-2 rounded-xl bg-amber-500 text-slate-950 text-xs font-bold hover:bg-amber-600 disabled:opacity-40 transition cursor-pointer"
>
{starting ? <Loader2 className="h-4 w-4 animate-spin" /> : <PlayCircle className="h-4 w-4" />}
Start batch ingestion <ArrowRight className="h-3.5 w-3.5" />
</button>
</div>
{/* Enrichment toggles. Off by default, and the cost is stated rather
than left for someone to discover on a 20-file run. */}
<div className="mt-4 flex flex-col gap-2">
<label className="flex items-start gap-2 text-[11px] text-slate-600 cursor-pointer">
<input
type="checkbox"
checked={fetchImages}
onChange={(e) => setFetchImages(e.target.checked)}
className="mt-0.5 accent-amber-500 cursor-pointer"
/>
<span>
<span className="font-bold text-ink-950">Search for product images</span>
{' '}&mdash; much slower, and makes outbound requests for every row. Leave off
for a large batch.
</span>
</label>
<label className="flex items-start gap-2 text-[11px] text-slate-600 cursor-pointer">
<input
type="checkbox"
checked={useLlm}
onChange={(e) => setUseLlm(e.target.checked)}
className="mt-0.5 accent-amber-500 cursor-pointer"
/>
<span>
<span className="font-bold text-ink-950">Use the LLM for missing descriptions</span>
{' '}&mdash; no effect unless Ollama is reachable, which it is not in production.
</span>
</label>
</div>
{files.length > 0 && (
<ul className="mt-4 space-y-1.5">
{files.map((file, index) => (
<li
key={`${file.name}-${index}`}
className="flex items-center justify-between gap-2 px-3 py-2 rounded-xl border border-ink-900/10 bg-white"
>
<span className="text-[11px] font-bold text-ink-950 truncate">{file.name}</span>
<span className="flex items-center gap-2 shrink-0">
<span className="font-mono text-[10px] text-slate-400">{fmtBytes(file.size)}</span>
<button
type="button"
onClick={() => removeAt(index)}
aria-label={`Remove ${file.name}`}
className="text-slate-400 hover:text-maroon-600 transition cursor-pointer"
>
<X className="h-3.5 w-3.5" />
</button>
</span>
</li>
))}
</ul>
)}
{error && (
<div className="mt-4 rounded-lg bg-maroon-100 border border-maroon-500/30 p-3 text-xs text-maroon-600 flex items-start gap-2">
<XCircle className="h-4 w-4 shrink-0 mt-0.5" /> {error}
</div>
)}
</div>
{/* ---- 2. Per-file column mapping preview ---------------------- */}
{preview && (
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
<h3 className="font-display text-sm font-bold text-ink-950 mb-1">
Column mapping &mdash; {preview.files_ok}/{preview.files_total} file(s) readable,
{' '}{preview.rows_total} row(s) total
</h3>
<p className="text-[11px] text-slate-500 mb-4">
Nothing has been ingested yet. Check the mapping, then start the batch.
</p>
<div className="space-y-4">
{preview.files.map((file) => (
<div key={file.filename} className="rounded-xl border border-ink-900/10 bg-white p-4">
<div className="flex items-center justify-between gap-2 mb-2">
<span className="text-xs font-bold text-ink-950 truncate">{file.filename}</span>
{file.ok ? (
<span className="font-mono text-[10px] text-slate-400 shrink-0">
{file.rows_total} rows
</span>
) : (
<span className="text-[10px] font-bold text-maroon-600 shrink-0">
will be skipped
</span>
)}
</div>
{!file.ok && (
<p className="text-[11px] text-maroon-600 flex items-start gap-1.5">
<XCircle className="h-3.5 w-3.5 shrink-0 mt-0.5" /> {file.error}
</p>
)}
{file.ok && (
<>
<p className="text-[11px] text-slate-500 mb-2">
{file.brand_column_present
? 'Brand column found.'
: 'No brand column; the brand will be inferred from each product name.'}
</p>
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-2">
{Object.entries(file.recognised_columns).map(([field, column]) => (
<div
key={field}
className="flex items-center justify-between gap-2 p-2 rounded-lg border border-ink-900/10 bg-paper-50"
>
<span className="text-[10px] text-slate-500 truncate">{column}</span>
<ArrowRight className="h-3 w-3 text-slate-400 shrink-0" />
<span className="text-[10px] font-bold text-ink-950 truncate">{field}</span>
</div>
))}
</div>
{file.unrecognised_columns?.length > 0 && (
<p className="mt-3 text-[11px] text-amber-700 flex items-start gap-1.5">
<AlertTriangle className="h-3.5 w-3.5 shrink-0 mt-0.5" />
Ignored: {file.unrecognised_columns.join(', ')}
</p>
)}
</>
)}
</div>
))}
</div>
</div>
)}
{/* ---- 3. Live batch tracker ----------------------------------- */}
{batch && (
<div className="rounded-2xl border border-ink-900/10 bg-paper-50 p-6 shadow-xs">
<div className="flex items-center justify-between gap-3 mb-1">
<h3 className="font-display text-sm font-bold text-ink-950 flex items-center gap-2">
{batch.status === 'done' && <CheckCircle2 className="h-4 w-4 text-leaf-600" />}
{batch.status === 'partial' && <AlertTriangle className="h-4 w-4 text-amber-600" />}
{batch.status === 'failed' && <XCircle className="h-4 w-4 text-maroon-600" />}
{batch.status === 'interrupted' && <RotateCcw className="h-4 w-4 text-amber-600" />}
{running && <Loader2 className="h-4 w-4 animate-spin text-amber-600" />}
Batch &mdash; {batch.files_done}/{batch.files_total} file(s) done
</h3>
<div className="flex items-center gap-2 shrink-0">
{batch.status === 'interrupted' && (
<button
type="button"
onClick={handleResume}
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-amber-500 text-slate-950 text-[11px] font-bold hover:bg-amber-600 transition cursor-pointer"
>
<RotateCcw className="h-3.5 w-3.5" /> Resume
</button>
)}
{running && (
<button
type="button"
onClick={handleCancel}
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg border border-ink-900/15 bg-white text-[11px] font-bold text-slate-600 hover:text-maroon-600 transition cursor-pointer"
>
<Ban className="h-3.5 w-3.5" /> Cancel
</button>
)}
<span className="font-mono text-[11px] uppercase text-slate-400">{batch.status}</span>
</div>
</div>
<p className="font-mono text-[10px] text-slate-400 mb-4">{batch.batch_id}</p>
{batch.status === 'partial' && (
<div className="mb-4 rounded-lg bg-amber-500/5 border border-amber-500/20 p-3 text-[11px] text-amber-700 flex items-start gap-2">
<AlertTriangle className="h-3.5 w-3.5 shrink-0 mt-0.5" />
{batch.files_done} file(s) landed and {batch.files_failed} failed. Check the failed
rows below &mdash; the rest of the batch was written.
</div>
)}
<div className="h-2 w-full rounded-full bg-ink-100 overflow-hidden mb-4">
<div
className="h-full rounded-full bg-amber-500 transition-all"
style={{
width: `${batch.files_total
? Math.round(((batch.files_done + batch.files_failed) / batch.files_total) * 100)
: 0}%`,
}}
/>
</div>
{batch.detail && <p className="mb-4 text-xs text-slate-600">{batch.detail}</p>}
{/* Per-file steps */}
<ul className="space-y-2">
{batch.files.map((file) => {
const Icon = FILE_ICON[file.status] || Clock;
return (
<li
key={`${file.index}-${file.filename}`}
className={`rounded-xl border p-3 ${FILE_TONE[file.status] || FILE_TONE.queued}`}
>
<div className="flex items-center justify-between gap-2">
<span className="flex items-center gap-2 min-w-0">
<Icon className={`h-3.5 w-3.5 shrink-0 ${file.status === 'running' ? 'animate-spin' : ''}`} />
<span className="text-[11px] font-bold truncate">{file.filename}</span>
</span>
<span className="font-mono text-[10px] uppercase shrink-0">{file.status}</span>
</div>
{file.stage_index > 0 && file.status === 'running' && (
<p className="mt-1.5 text-[10px] font-mono">
Stage {file.stage_index}/{file.total_stages} &mdash; {file.stage_name}
{' · '}{file.rows_done}/{file.rows_total} rows
</p>
)}
{file.detail && (
<p className="mt-1.5 text-[10px] leading-relaxed">{file.detail}</p>
)}
</li>
);
})}
</ul>
{/* Batch totals, once anything has landed */}
{totals && (batch.files_done > 0 || batch.files_failed > 0) && (
<>
<div className="mt-5 grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-5 gap-3">
<Stat label="Inserted" value={totals.inserted} tone="emerald" />
<Stat label="Backfilled" value={totals.backfilled} tone="blue" />
<Stat label="Unchanged" value={totals.skipped_existing} tone="slate" />
<Stat label="Rejected" value={totals.rejected} tone="maroon" />
<Stat label="Row errors" value={totals.error_count} tone="amber" />
</div>
{batch.brands?.length > 0 && (
<p className="mt-4 text-[11px] text-slate-500">
Brands touched:{' '}
<span className="font-bold text-ink-950">{batch.brands.join(', ')}</span>
</p>
)}
<IssueList
title="Rejected by the validation gate"
items={batch.files.flatMap((f) =>
(f.result?.rejections || []).map(
(r) => `${f.filename} · ${r.product_name} ${r.size || ''} — ${r.reason}`
)
)}
/>
<IssueList
title="Row errors"
items={batch.files.flatMap((f) =>
(f.result?.errors || []).map(
(e) => `${f.filename} · row ${e.row}: ${e.product_name || '(unnamed)'} — ${e.error}`
)
)}
/>
<IssueList
title="Warnings"
items={batch.files.flatMap((f) =>
(f.result?.warnings || []).map((w) => `${f.filename} · ${w}`)
)}
/>
</>
)}
</div>
)}
</div>
);
}
const TONES = {
emerald: 'border-emerald-500/20 bg-emerald-500/5 text-emerald-700',
blue: 'border-blue-500/20 bg-blue-500/5 text-blue-700',
slate: 'border-ink-900/10 bg-white text-slate-600',
maroon: 'border-maroon-500/20 bg-maroon-100/40 text-maroon-600',
amber: 'border-amber-500/20 bg-amber-500/5 text-amber-700',
};
function Stat({ label, value, tone }) {
return (
<div className={`rounded-xl border p-3 ${TONES[tone] || TONES.slate}`}>
<p className="text-[10px] font-semibold uppercase">{label}</p>
<p className="font-mono text-2xl font-extrabold text-ink-950 mt-0.5">{value ?? 0}</p>
</div>
);
}
function IssueList({ title, items }) {
if (!items || items.length === 0) return null;
return (
<div className="mt-4">
<h4 className="text-[11px] font-bold text-ink-950 mb-1.5">{title} ({items.length})</h4>
<ul className="max-h-40 overflow-y-auto space-y-1 pr-1">
{items.map((text, idx) => (
<li key={idx} className="text-[11px] text-slate-600 bg-white border border-ink-900/10 rounded-lg px-2.5 py-1.5">
{text}
</li>
))}
</ul>
</div>
);
}