nearle_daily/daily_console_web
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upload sheet according to the tenant

Browse Source
This commit is contained in:
abhishek
2026-08-29 12:08:38 +05:30
parent 32c3bef3ad
commit 29613e25ee
8 changed files with 1227 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/api/catalogue.ts
View File

@@ -36,6 +36,30 @@ export const catalogueApi = {
/** Brands with product counts, for the filter chip row. Never hardcode this list. */
brands: () => api.list<CatalogueBrand>(`${WEB}/catalogue/getbrands`),
/**
* Every `image_id` → catalogue row id for one brand, in as few calls as the
* page size allows.
*
* Built for reconciling an ingest manifest. Resolving those one at a time is a
* request per product — a 500-row sheet would open 500 connections from a
* shop's browser — while a brand is at most a few hundred rows and comes back
* in one or two pages.
*
* `pagesize` is deliberately large but bounded, and paging stops on a short
* page rather than trusting a total the list endpoint does not return.
*/
idsByImageId: async (brand: string, pageSize = 500): Promise<Map<string, number>> => {
const out = new Map<string, number>();
for (let page = 0; page < 20; page += 1) {
const rows = await catalogueApi.products({ brand, pageno: page, pagesize: pageSize });
for (const row of rows) {
if (row.image_id && typeof row.id === 'number') out.set(row.image_id, row.id);
}
if (rows.length < pageSize) break;
}
return out;
},
/** Requires a brand — the backend reads categories from one brand's table. */
categories: (brand: string) =>
api.list<string>(`${WEB}/catalogue/getcategories`, { brand }),
@@ -50,6 +74,24 @@ export const catalogueApi = {
product: (brand: string, sku: string) =>
api.get<CatalogueProduct | null>(`${WEB}/catalogue/getproduct`, { brand, sku }),
/**
* One catalogue row by the id the ingest pipeline treats as canonical.
*
* An ingest run reports what it wrote as a manifest of `image_id` values, and
* `importcatalogueproduct` addresses products by `catalogueid` — the row id.
* This is the only bridge between the two, and without it a manifest could
* only be matched on the product NAME, which the owning team warns silently
* creates duplicates rather than updating.
*
* Prefer `productsByBrand` below when resolving more than a handful: this is
* one request per product.
*/
productByImageId: (brand: string, imageId: string) =>
api.get<CatalogueProduct | null>(`${WEB}/catalogue/getproductbyimageid`, {
brand,
image_id: imageId,
}),
/**
* The `(brand, catalogueid)` pairs this tenant has already imported, for
* badging "Imported" in the browser. Called without `brand` because the list

163
src/api/ingest.test.ts Normal file
View File

@@ -0,0 +1,163 @@
/**
* The review inbox, and the status that nearly slipped through as success.
*
* The fixture is the live response to an anonymous upload on 28 Aug 2026 —
* `status: "pending"`, every total zero, the file still queued.
*/
import assert from 'node:assert/strict';
import { test } from 'node:test';
import {
isAwaitingReview,
isDismissed,
isSettled,
productsOf,
releasedRunId,
summarise,
type IngestBatch,
} from './ingest';
const held = {
batch_id: '2ee38d06b583454ea0278a7f6de2c87f',
status: 'pending',
detail: 'Waiting for review. Nothing runs until an admin starts it.',
submitted_by: 'anonymous',
files_total: 1,
files_done: 0,
files_failed: 0,
totals: {
rows_total: 0,
products_built: 0,
inserted: 0,
backfilled: 0,
skipped_existing: 0,
rejected: 0,
},
brands: [],
files: [{ index: 0, filename: 'qa.csv', status: 'queued' as const }],
} satisfies IngestBatch;
// The bug this guards: `isSettled` used to mean "not queued and not running",
// so `pending` counted as finished and the panel rendered a completed import of
// zero products for a batch that had not started.
test('a batch held for review is not treated as finished', () => {
assert.equal(isSettled(held), false, 'a held batch must not read as settled');
assert.equal(isAwaitingReview(held), true);
});
test('the summary says it is waiting, not that nothing imported', () => {
const line = summarise(held);
assert.match(line, /review/i);
assert.doesNotMatch(line, /0 added/, 'must not report an import that never ran');
});
test('a real result is still settled', () => {
for (const status of ['done', 'partial', 'failed', 'interrupted', 'cancelled'] as const) {
assert.equal(isSettled({ ...held, status }), true, `${status} should be settled`);
}
});
test('queued and running are still in flight', () => {
for (const status of ['queued', 'running'] as const) {
assert.equal(isSettled({ ...held, status }), false, `${status} should not be settled`);
}
});
// isSettled is written as a positive list precisely so a status nobody
// anticipated stalls a spinner rather than fabricating a completed import.
test('an unknown future status does not read as finished', () => {
const unknown = { ...held, status: 'quarantined' as unknown as IngestBatch['status'] };
assert.equal(isSettled(unknown), false);
});
/* ── The drop lifecycle ───────────────────────────────────────────────────── */
const released = {
...held,
batch_id: '9f088d949aa9',
status: 'pending' as const,
files: [{ index: 0, filename: 'qa.csv', status: 'released' as const, released_to: '8dcef8a2ad94' }],
} satisfies IngestBatch;
const dismissed = {
...held,
files: [{ index: 0, filename: 'qa.csv', status: 'dismissed' as const, released_to: null }],
} satisfies IngestBatch;
// A released drop is not still waiting — the run is one hop away, and treating
// it as held would leave the screen saying "queued for review" forever.
test('a released drop is no longer awaiting review', () => {
assert.equal(releasedRunId(released), '8dcef8a2ad94');
assert.equal(isAwaitingReview(released), false);
assert.equal(isAwaitingReview(held), true, 'an unreleased drop is still waiting');
});
// Declined is terminal. Polling on is waiting for something that cannot happen.
test('a dismissed drop is recognised and reported as declined', () => {
assert.equal(isDismissed(dismissed), true);
assert.equal(isDismissed(held), false);
assert.match(summarise(dismissed), /declined/i);
assert.doesNotMatch(summarise(dismissed), /0 added/);
});
test('the manifest is collected across files', () => {
const done = {
...held,
status: 'done' as const,
files: [
{
index: 0,
filename: 'a.csv',
status: 'done' as const,
result: {
products: [
{
image_id: 'amul_amul_butter_100g',
brand: 'amul',
product_name: 'Amul Butter 100g',
product_sku: 'ACME-BUT-100',
sku_source: 'sheet',
disposition: 'inserted' as const,
},
],
},
},
],
} satisfies IngestBatch;
const products = productsOf(done);
assert.equal(products.length, 1);
// image_id is the join key; matching on name creates duplicates instead of
// updating, which is why it is asserted rather than the name.
assert.equal(products[0]?.image_id, 'amul_amul_butter_100g');
assert.equal(products[0]?.disposition, 'inserted');
});
/* ── retired: the drop is spent, the answer is on the files ───────────────── */
// A drop released into a run reads `retired`, and the run is elsewhere. Calling
// it finished would report an import that is running right now as a completed
// import of zero products.
test('a retired drop that was released is not finished', () => {
const retired = {
...held,
status: 'retired' as const,
files: [
{ index: 0, filename: 'qa.csv', status: 'released' as const, released_to: '8dcef8a2ad94' },
],
} satisfies IngestBatch;
assert.equal(isSettled(retired), false, 'the run still has to be followed');
assert.equal(releasedRunId(retired), '8dcef8a2ad94');
});
// Retired with nothing to follow is genuinely over — otherwise the panel spins
// on a drop that no longer exists.
test('a retired drop with nowhere to follow is finished', () => {
const retired = {
...held,
status: 'retired' as const,
files: [{ index: 0, filename: 'qa.csv', status: 'dismissed' as const, released_to: null }],
} satisfies IngestBatch;
assert.equal(isSettled(retired), true);
});

303
src/api/ingest.ts
View File

@@ -1,44 +1,50 @@
/**
* The catalogue ingest service — `mcp.nearle.ai.in`.
*
* A workbook goes up, the service runs its eleven-stage pipeline over every
* row, and the products land in the global catalogue's per-brand tables. From
* there Fiesta already sees them: `/web/catalogue/getbrands` and
* `/web/catalogue/getproducts` read the SAME database this service writes to,
* which is why an upload here shows up in the console's catalogue with nothing
* in between to build or synchronise.
* A spreadsheet goes up, an admin reviews it, and once released the eleven-stage
* pipeline writes the products into the global catalogue. From there Fiesta
* already sees them: `/web/catalogue/getbrands` and `/web/catalogue/getproducts`
* read the SAME database the pipeline writes to, so an upload appears in the
* console with nothing in between to build or synchronise.
*
* ── Batches, not jobs ────────────────────────────────────────────────────────
* ── A drop is not a run ──────────────────────────────────────────────────────
*
* This module used to call `/api/admin/store-catalog/*` — one file, one
* `job_id`, jobs held in memory and lost on restart. That API is still
* deployed, but the owning team's guidance is to integrate against
* `/api/uploads/catalog`, which takes up to twenty files at once and answers
* with a `batch_id` that survives a restart.
* `POST /api/uploads/catalog` creates a DROP, and nothing runs on arrival. The
* files wait in an admin review inbox; only when someone selects them and
* presses Start does a RUN begin, under a different id. The drop id stays valid
* for the whole lifecycle and its per-file status is how you follow it:
*
* `POST` answers 202 the moment the files are staged; the outcome arrives from
* `GET /api/uploads/catalog/{batch_id}`. Two things about that shape earn their
* own handling below:
* queued — still in the inbox, nobody has looked
* released — accepted; `released_to` is the run, and the results are there
* dismissed — declined; nothing further is coming
*
* - **A file that could not be read stays in the batch** as a failed member
* rather than being dropped, so a sender who submitted two files and sees
* one knows what became of the other.
* - **`partial` is not `done`.** Some files landed and some did not, and four
* of five succeeding must never render as flat success.
* `resolveBatch` below makes that hop automatically, so callers poll one id and
* get whichever record actually has the answer.
*
* ── The credential never reaches this file ───────────────────────────────────
* ── No credential ────────────────────────────────────────────────────────────
*
* `X-API-Key`, attached by the Vite proxy in development and by nginx in
* production, both from `INGEST_TOKEN`. It stays server-side because the bundle
* is served to anyone who opens the console.
* The drop endpoint takes none, and that is safe precisely because of the review
* gate: an unwanted drop costs disk until somebody declines it, never products
* in the live catalogue.
*
* `INGEST_TOKEN` is the SECRET ALONE — the 43-character value. The service
* holds `name:role:secret` triples in its own `API_KEYS` and looks keys up by
* the secret, so pasting the whole triple fails as "Invalid API key." rather
* than as something that names the real mistake.
* So `INGEST_TOKEN` should be left EMPTY. nginx omits an empty header, and a
* WRONG key is a 401 rather than a downgrade to anonymous — verified against the
* live service. A stale token in the environment would therefore break every
* upload while looking like a service fault.
*
* Only the LIST read (`GET /api/uploads/catalog`) still wants a credential;
* reading one batch by its id does not, because the id is itself the proof of
* having sent it.
*/
const INGEST_BASE = import.meta.env['VITE_INGEST_BASE'] ?? '/ingest';
/**
* Optional-chained because `import.meta.env` is Vite's, and it is undefined
* anywhere Vite is not — the `node --test` runner included. Without the `?.`
* this line throws on import, so every test that so much as names this module
* fails before it runs, with a TypeError that points here rather than at the
* test. Cheap insurance for a value that already has a fallback.
*/
const INGEST_BASE = import.meta.env?.['VITE_INGEST_BASE'] ?? '/ingest';
const ROOT = '/api/uploads/catalog';
@@ -70,6 +76,28 @@ export const ACCEPTED_EXTENSIONS = ['.xlsx', '.xls', '.csv', '.tsv'];
* waiting to be continued.
*/
export type BatchStatus =
/**
* Accepted and staged, but NOTHING RUNS until an admin releases it.
*
* A review inbox now sits in front of the pipeline — the service answers
* `"Waiting for review. Nothing runs until an admin starts it."` — and this
* status was not in the contract we were given. It matters far more than an
* extra enum member: `isSettled` originally read "not queued and not
* running", so `pending` counted as FINISHED and the panel rendered a
* completed batch reporting nothing imported. An upload that had not yet
* begun would have been shown as a successful import of zero products.
*/
| 'pending'
/**
* Every file in this DROP has been released or dismissed — the drop is spent.
*
* Not an outcome of its own: the answer is on the files. A released file
* carries `released_to`, which is where the run actually is; a dismissed one
* carries nothing because nothing will come. Treating `retired` as finished
* would report a drop that was accepted and is running right now as a
* completed import of zero products.
*/
| 'retired'
| 'queued'
| 'running'
| 'done'
@@ -78,7 +106,44 @@ export type BatchStatus =
| 'interrupted'
| 'cancelled';
export type BatchFileStatus = 'queued' | 'running' | 'done' | 'failed';
/**
* A file inside a drop.
*
* `released` and `dismissed` are the review inbox's two outcomes and neither is
* a result: released means an admin accepted it and the RUN is somewhere else —
* follow `released_to` — while dismissed means they declined it and nothing will
* ever come. Reading either as a finished import reports products that were
* never written.
*/
export type BatchFileStatus =
| 'queued'
| 'running'
| 'done'
| 'failed'
| 'released'
| 'dismissed';
/**
* One product the pipeline wrote, from the run's manifest.
*
* `image_id` is the join key and the only safe one. The owning team calls it
* "the primary key every other product is deduplicated on", and warns that a
* product name differing by one character is a different product — so matching
* a manifest on NAME silently creates duplicates instead of updating.
*
* `unchanged` rows are included on purpose: re-sending a sheet writes nothing,
* and omitting them would make a completely successful upload return an empty
* list that reads as total failure.
*/
export interface IngestProduct {
image_id: string;
brand: string;
product_name: string;
product_sku?: string;
/** `sheet` when the sheet supplied it, `Internal` when the pipeline minted one. */
sku_source?: string;
disposition: 'inserted' | 'backfilled' | 'unchanged';
}
/** What the pipeline made of one file, once it has finished. */
export interface BatchFileResult {
@@ -97,6 +162,14 @@ export interface BatchFileResult {
unrecognised_columns?: string[];
/** Non-null means rows were built but never stored. */
storage_error?: string | null;
/**
* What the run actually wrote, product by product. Returned on the
* single-batch read only — the list endpoints omit it, because twenty runs of
* thousands of rows is not a list payload.
*/
products?: IngestProduct[];
/** True when the manifest was capped at 5,000 rows for this file. */
products_truncated?: boolean;
}
export interface BatchFile {
@@ -105,6 +178,15 @@ export interface BatchFile {
status: BatchFileStatus;
/** Present on a file the service refused to read, and the reason it gives. */
detail?: string | null;
/**
* The RUN this file became once an admin released it.
*
* Null while it waits and after it is dismissed. The drop id stays valid for
* the whole lifecycle — an earlier build deleted the drop on release and the
* poll started 404ing, which made running, declined and lost look identical
* from outside.
*/
released_to?: string | null;
size_bytes?: number;
rows_total?: number;
/** Progress through the eleven stages, while it runs. */
@@ -163,17 +245,14 @@ export class IngestError extends Error {
export interface SubmitOptions {
files: File[];
/**
* Default FALSE. Stage 6 spawns a Playwright subprocess and searches for an
* image per row — minutes per batch on one vCPU. Worth turning on
* deliberately, never by default.
* A label for the review inbox, so the admin can see who sent what.
*
* Free text, trimmed to 60 characters by the service, and defaulting to
* "anonymous" when omitted. It is worth sending: the drop endpoint takes no
* credential, so without this every submission in the inbox is indistinguishable
* and an admin approving one cannot tell whose it is.
*/
fetchImages?: boolean;
/**
* Default FALSE, and not caution: production runs with `USE_OLLAMA=false`, so
* asking for it is a documented no-op. Left as an option only so the flag is
* not silently unavailable the day that changes.
*/
useLlm?: boolean;
sender?: string;
signal?: AbortSignal;
}
@@ -233,20 +312,26 @@ function guardFiles(files: File[]): void {
* at all.
*/
export async function submitBatch(options: SubmitOptions): Promise<IngestBatch> {
const { files, fetchImages = false, useLlm = false, signal } = options;
const { files, sender = 'nearle-console', signal } = options;
guardFiles(files);
const form = new FormData();
for (const file of files) form.append('files', file, file.name);
// Labels the drop in the review inbox. The endpoint takes no credential, so
// without this every submission arrives as "anonymous" and the admin deciding
// whether to run it cannot tell ours from anyone else's.
form.append('sender', sender);
const query = new URLSearchParams({
fetch_images: String(fetchImages),
use_llm: String(useLlm),
});
// `use_llm` and `fetch_images` are no longer sent, and passing them is inert.
//
// They decide how a run behaves and commit the host to outbound work — image
// search is minutes per batch on one vCPU — so the choice belongs to the admin
// pressing Start, not to whoever dropped the file. Keeping them in the request
// would have read like control we do not have.
let response: Response;
try {
response = await fetch(`${INGEST_BASE}${ROOT}?${query}`, {
response = await fetch(`${INGEST_BASE}${ROOT}`, {
method: 'POST',
body: form,
// Content-Type is deliberately unset: the browser adds it WITH the
@@ -281,9 +366,95 @@ export async function fetchBatch(batchId: string, signal?: AbortSignal): Promise
return readResponse<IngestBatch>(response);
}
/** True once the batch has stopped moving, whatever the outcome. */
/**
* True when the batch is sitting in the review inbox, untouched.
*
* Not a failure and not a result — it is waiting for a person. The distinction
* has to be explicit, because the two obvious ways to classify it are both
* wrong: called finished, the screen reports an import of zero products that
* never ran; called in-progress, the browser polls indefinitely for something
* only an admin can move.
*/
export function isAwaitingReview(batch: IngestBatch): boolean {
return batch.status === 'pending' && !releasedRunId(batch);
}
/**
* The run a released drop became, if an admin has accepted it.
*
* A drop is a submission, not a run. Releasing it starts a separate batch and
* records its id on the file as `released_to`; the drop id keeps working and
* keeps saying `released`, so the results are one hop away rather than at the
* id you already hold.
*
* Read off the files rather than the drop, because that is where the service
* puts it — a drop of several files can in principle be released in parts.
*/
export function releasedRunId(batch: IngestBatch): string | null {
for (const file of batch.files ?? []) {
if (file.released_to) return file.released_to;
}
return null;
}
/**
* True when an admin declined the drop. Nothing further will ever arrive, so a
* client that keeps polling is waiting for something that cannot happen.
*/
export function isDismissed(batch: IngestBatch): boolean {
const files = batch.files ?? [];
return files.length > 0 && files.every((file) => file.status === 'dismissed');
}
/**
* Follows a drop to its run, once, and returns whichever is the real answer.
*
* The caller polls a drop id. If it is released, the numbers it wants are on
* the RUN — so this hops and returns that instead. Everything else comes back
* unchanged, so a caller never has to know a drop and a run are different
* things.
*/
export async function resolveBatch(batch: IngestBatch, signal?: AbortSignal): Promise<IngestBatch> {
const runId = releasedRunId(batch);
if (!runId || runId === batch.batch_id) return batch;
try {
return await fetchBatch(runId, signal);
} catch {
// The drop is still the honest answer if the run cannot be read — better a
// stale "released" than an error for something that did succeed.
return batch;
}
}
/** Every product a finished batch wrote, across its files. */
export function productsOf(batch: IngestBatch): IngestProduct[] {
return (batch.files ?? []).flatMap((file) => file.result?.products ?? []);
}
/**
* True once the batch has stopped moving, whatever the outcome.
*
* Listed positively rather than as "not queued and not running". The negative
* form silently absorbed every status added later — which is exactly how
* `pending` came to read as a completed import the day the review inbox
* appeared. A new status now shows up as "not settled" and stalls a spinner,
* which is visible, rather than as "done" and fabricates a result.
*/
export function isSettled(batch: IngestBatch): boolean {
return batch.status !== 'queued' && batch.status !== 'running';
// A retired drop whose files went nowhere we can follow is over. Normally
// resolveBatch has already hopped to the run, or isDismissed has caught a
// decline — this is the remainder, and leaving it unsettled would spin a
// progress bar on a drop that no longer exists.
if (batch.status === 'retired') {
return !releasedRunId(batch);
}
return (
batch.status === 'done' ||
batch.status === 'partial' ||
batch.status === 'failed' ||
batch.status === 'interrupted' ||
batch.status === 'cancelled'
);
}
/**
@@ -302,9 +473,16 @@ export async function pollBatch(
for (;;) {
if (signal?.aborted) throw new IngestError('Cancelled.', 0);
const batch = await fetchBatch(batchId, signal);
// Follows a released drop to the run it became, so the caller polls the
// thing that actually has progress on it rather than a record that will say
// "released" forever.
const batch = await resolveBatch(await fetchBatch(batchId, signal), signal);
onTick(batch);
if (isSettled(batch)) return batch;
// Stops on a review hold and on a dismissal as well as on a result. Waiting
// for an admin is not progress, a declined drop will never produce one, and
// a browser tab cannot outlast either — the drop id is what the operator
// comes back with.
if (isSettled(batch) || isAwaitingReview(batch) || isDismissed(batch)) return batch;
await new Promise((resolve) => setTimeout(resolve, 2000));
}
@@ -395,7 +573,7 @@ function describe(status: number, payload: unknown, text: string): IngestError {
if (status === 429) {
return new IngestError(
detail ??
'Four batches are already queued. This upload was staged rather than lost — wait a moment and send it again.',
'The review inbox is full, so NOTHING was stored — this upload was not merely delayed. An admin has to clear it before you resend.',
status,
body,
);
@@ -428,6 +606,29 @@ export function isIncomplete(batch: IngestBatch): boolean {
export function summarise(batch: IngestBatch): string {
const { totals } = batch;
if (isDismissed(batch)) {
// A refusal, not a failure, and nothing further is coming.
//
// The DROP-level detail is deliberately not used here. It still reads
// "Waiting for review. Nothing runs until an admin starts it." on a drop
// that has since been declined — the sentence was written when the file was
// accepted and nothing rewrites it. Rendering it would tell the operator to
// keep waiting for a decision that has already been made against them.
//
// A reason attached to the FILE is the admin's own and is worth showing.
const reason = (batch.files ?? []).map((file) => file.detail).find(Boolean);
return reason
? `An admin declined this upload: ${reason}`
: 'An admin declined this upload. Nothing was imported.';
}
if (isAwaitingReview(batch)) {
// The service's own sentence when it has one — it is clearer than anything
// invented here, and it changes if their review policy does.
return (
batch.detail ??
'Waiting for review. Nothing runs until an admin on the ingest service starts it.'
);
}
if (batch.status === 'failed') {
return batch.detail ?? 'No file could be ingested.';
}