agent conformation

This commit is contained in:
2026-08-31 12:32:09 +05:30
parent 7e3571f90e
commit bd133e7cc6
14 changed files with 1480 additions and 7 deletions

View File

@@ -39,7 +39,15 @@ import type { FiestaEnvelope } from './types';
* Override per machine with `.env.local`, which is gitignored — set it to
* `/fiesta` to route through the dev proxy or nginx instead.
*/
const configuredBase = (import.meta.env['VITE_API_BASE'] ?? '').trim();
/**
* Optional-chained for the same reason `ingest.ts` is: `import.meta.env` is
* Vite's, and it is undefined anywhere Vite is not — the test runner included.
* Without the `?.` this line throws on import, so every test that so much as
* names a module reaching this one fails before it runs, with a TypeError
* pointing here rather than at the test. The value already has a fallback; this
* only stops the read itself from being fatal.
*/
const configuredBase = (import.meta.env?.['VITE_API_BASE'] ?? '').trim();
export const API_BASE = (configuredBase || 'https://fiesta.nearle.app').replace(
/\/+$/,

View File

@@ -10,6 +10,8 @@ import {
isAwaitingReview,
isDismissed,
isSettled,
currentStage,
isStuckOnMissingRunner,
productsOf,
releasedRunId,
summarise,
@@ -203,3 +205,132 @@ test('only our own file contributes products', () => {
// and every caller that prices products must make it.
assert.equal(productsOf(run).length, 2);
});
/*
The stage timeline, and the runner that silently isn't there.
Both arrived with the ingest team's 31 Aug documentation update. The timeline is
what lets the console draw the real eleven stages instead of a file-count bar;
the runner is a trap, and the more important of the two.
*/
const runningFile = {
index: 0,
filename: 'catalog.csv',
status: 'running' as const,
stage_index: 6,
stage_name: 'Image Search & Contamination Filtering',
total_stages: 11,
rows_done: 120,
rows_total: 400,
stages: [
{
index: 1,
name: 'Brand Resolution & FSSAI Licence Mapping',
rows_done: 400,
rows_total: 400,
started_at: 1756612800.1,
finished_at: 1756612801.4,
},
{
index: 6,
name: 'Image Search & Contamination Filtering',
rows_done: 120,
rows_total: 400,
started_at: 1756612809.7,
finished_at: null,
},
],
};
// `finished_at: null` is the marker, not the last array entry and not
// `stage_index`. Reading the position any other way breaks the moment a stage
// completes out of order or the array carries a trailing finished entry.
test('the running stage is the one with no finish time', () => {
const stage = currentStage(runningFile);
assert.equal(stage?.index, 6);
assert.equal(stage?.rows_done, 120);
});
// A finished file keeps its history, which is the whole reason the timeline
// exists — the scalars only ever describe the present moment, and for a
// finished file that moment is over.
test('a finished file still reports its last stage', () => {
const done = {
...runningFile,
status: 'done' as const,
stages: runningFile.stages.map((s) => ({ ...s, finished_at: s.finished_at ?? 1756612900.0 })),
};
assert.equal(currentStage(done)?.index, 6);
});
// A service build that predates the timeline still has to render. The scalars
// are the fallback, not the source of truth.
test('a response without a timeline falls back to the scalars', () => {
const { stages: _stages, ...noTimeline } = runningFile;
const stage = currentStage(noTimeline);
assert.equal(stage?.index, 6);
assert.equal(stage?.name, 'Image Search & Contamination Filtering');
});
test('a file that has not started reports no stage at all', () => {
assert.equal(currentStage({ index: 0, filename: 'a.csv', status: 'queued' }), null);
});
/*
`runner: "dagster"` never runs in production — Dagster is a development tool,
absent from the deployed image — so the batch waits for a worker that will never
claim it. Every visible signal is identical to a batch merely waiting its turn,
which is exactly why it has to be named rather than rendered as progress.
*/
test('a batch staged for the absent orchestrator is called out', () => {
assert.equal(isStuckOnMissingRunner({ ...held, status: 'queued', runner: 'dagster' }), true);
});
test('the in-process runner is not a stall', () => {
assert.equal(isStuckOnMissingRunner({ ...held, status: 'queued', runner: 'inprocess' }), false);
});
// A batch that reached `running` plainly found an executor, whatever it was
// staged for. Warning then would contradict the progress on screen.
test('a batch already running is not stuck, whatever it was staged for', () => {
assert.equal(isStuckOnMissingRunner({ ...held, status: 'running', runner: 'dagster' }), false);
});
/*
A drop is not a run, and the difference is easy to lose.
`released_to` lives on the DROP's files. Once an admin releases it, following
that pointer lands on the run — and the run carries no `released_to` of its own,
because nothing released it. So a caller who resolves first and asks for the run
id second gets null, and the only pointer from the id they hold to the id with
the results is never recorded.
This cost a real bug in both directions: the Uploads page never saved a run id,
and the import panel keyed the shelving write on `batch.batch_id` — which by
then was the run — updating a receipt row that does not exist, silently.
*/
test('the run id is on the drop, and gone from the run it points to', () => {
const drop = {
...held,
batch_id: 'drop-1',
status: 'retired' as const,
files: [
{ index: 0, filename: 'catalog.csv', status: 'released' as const, released_to: 'run-1' },
],
};
assert.equal(releasedRunId(drop), 'run-1');
// The same question asked of the run answers null. Read the drop first.
const run = {
...held,
batch_id: 'run-1',
status: 'done' as const,
files: [{ index: 0, filename: 'catalog.csv', status: 'done' as const }],
};
assert.equal(releasedRunId(run), null);
// And the two ids differ, which is exactly why a receipt keyed on the drop
// cannot be written using the run's.
assert.notEqual(drop.batch_id, run.batch_id);
});

View File

@@ -172,6 +172,25 @@ export interface BatchFileResult {
products_truncated?: boolean;
}
/**
* One stage a file has entered, from the run's own timeline.
*
* `finished_at: null` marks the stage running RIGHT NOW — that is how the
* current position is found, not by trusting `stage_index` alone. The array
* persists after the run ends, so a finished file can still show its whole
* history; the scalars on the file only ever describe the present moment, which
* is why they are not enough on their own.
*/
export interface BatchStage {
index: number;
name: string;
rows_done?: number;
rows_total?: number;
/** Epoch SECONDS as a float, like every other timestamp here. */
started_at?: number;
finished_at?: number | null;
}
export interface BatchFile {
index: number;
filename: string;
@@ -194,6 +213,8 @@ export interface BatchFile {
stage_name?: string;
total_stages?: number;
rows_done?: number;
/** The stages this file has entered, oldest first. */
stages?: BatchStage[];
result?: BatchFileResult | null;
}
@@ -220,6 +241,26 @@ export interface IngestBatch {
current_file?: string | null;
use_llm?: boolean;
fetch_images?: boolean;
/**
* All eleven stage names, in order.
*
* Served rather than left for us to hardcode, deliberately — draw the
* pipeline from this and the console cannot drift out of step when a stage is
* added or renamed on their side.
*/
stage_names?: string[];
/**
* Who is executing the batch.
*
* `"dagster"` is a silent failure in production and has to be surfaced rather
* than rendered as progress. Dagster is a local development orchestrator — it
* is absent from the deployed image, which never copies `orchestration/` — so
* a batch staged for it is handed to nobody and parks at `queued` forever
* saying "Waiting for the Dagster orchestrator to pick this batch up". From
* outside that is indistinguishable from a hang, and the fix is not to wait:
* an admin resumes it onto the in-process worker.
*/
runner?: 'inprocess' | 'dagster' | string;
totals: BatchTotals;
/** The brands this batch touched — the way back into the catalogue view. */
brands: string[];
@@ -674,3 +715,49 @@ export function summarise(batch: IngestBatch): string {
export function progressOf(batch: IngestBatch): { done: number; total: number } {
return { done: batch.files_done + batch.files_failed, total: batch.files_total };
}
/**
* True when the batch was handed to an orchestrator that is not there.
*
* `runner: "dagster"` never runs in production: Dagster is a development tool,
* absent from the deployed image, so the batch waits for a worker that will
* never claim it and sits at `queued` indefinitely. It has to be named, because
* every visible signal — a queued status, a stage index of 0, a progress bar at
* nothing — is identical to a batch that is merely waiting its turn.
*
* Only meaningful while it is still waiting. A batch that reached `running`
* plainly found an executor whatever it was staged for.
*/
export function isStuckOnMissingRunner(batch: IngestBatch): boolean {
return batch.runner === 'dagster' && (batch.status === 'queued' || batch.status === 'pending');
}
/**
* Where a file is in the eleven stages, read from the timeline rather than the
* scalars.
*
* `stages[]` is the authority: the entry with `finished_at: null` is the stage
* running now. `stage_index`/`stage_name` describe the same moment and are used
* as a fallback for a service build that does not send the timeline, but they
* cannot show a finished file's history and the timeline can.
*
* Returns null when there is nothing to draw — a file that has not started, or
* one from a response carrying neither.
*/
export function currentStage(file: BatchFile): BatchStage | null {
const running = (file.stages ?? []).find((stage) => stage.finished_at == null);
if (running) return running;
// Finished, or a build without the timeline. The last entered stage is the
// most useful thing to show for a file that has stopped moving.
const last = (file.stages ?? []).at(-1);
if (last) return last;
if (!file.stage_index) return null;
return {
index: file.stage_index,
name: file.stage_name ?? `Stage ${file.stage_index}`,
...(file.rows_done === undefined ? {} : { rows_done: file.rows_done }),
...(file.rows_total === undefined ? {} : { rows_total: file.rows_total }),
};
}

100
src/api/uploads.test.ts Normal file
View File

@@ -0,0 +1,100 @@
/**
* The receipt, and the two things it has to get right.
*
* A receipt exists because the ingest service's batch id is the only credential
* for reading a result back, it is handed out once, and an unreviewed drop is
* deleted after seven days. So the tests that matter are about not losing that
* window, and about the label their admin reads when deciding whether to
* approve the file.
*/
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { buildSender, daysUntilExpiry, DROP_RETENTION_DAYS, type UploadReceipt } from './uploads';
function receipt(overrides: Partial<UploadReceipt> = {}): UploadReceipt {
return {
uploadid: 1,
tenantid: 1141,
locationid: 1180,
categoryid: 2,
batchid: '49a82536866a483a9189954d3c749243',
runid: '',
filename: 'kmart-opening.xlsx',
sender: 'Kmart · Peelamedu · abhishek',
uploadedby: 1475,
uploadedname: 'abhishek',
rowcount: 20,
laststatus: 'pending',
inserted: 0,
backfilled: 0,
skipped: 0,
rejected: 0,
shelvedcount: 0,
skippedcount: 0,
shelvedat: null,
created: new Date().toISOString(),
updated: new Date().toISOString(),
...overrides,
};
}
const daysAgo = (n: number) => new Date(Date.now() - n * 86_400_000).toISOString();
test('a drop uploaded today has the full window left', () => {
assert.equal(daysUntilExpiry(receipt()), DROP_RETENTION_DAYS);
});
test('the window closes as the drop sits unreviewed', () => {
assert.equal(daysUntilExpiry(receipt({ created: daysAgo(5) })), 2);
});
// Never negative. A drop past the deadline is gone, and "-3 days left" would
// read as a countdown that is still running.
test('an expired drop reports zero, not a negative', () => {
assert.equal(daysUntilExpiry(receipt({ created: daysAgo(30) })), 0);
});
/*
Retention applies to a drop nobody acted on. Once an admin releases it the run
is the record and the drop's own expiry is irrelevant — showing a countdown on
a released upload would push someone to chase a deadline that has already been
met.
*/
test('a released drop has no expiry to report', () => {
assert.equal(daysUntilExpiry(receipt({ runid: '8dcef8a2ad94', laststatus: 'retired' })), null);
assert.equal(daysUntilExpiry(receipt({ laststatus: 'done' })), null);
});
test('a receipt with an unreadable date reports nothing rather than guessing', () => {
assert.equal(daysUntilExpiry(receipt({ created: 'not a date' })), null);
});
/*
The sender label. Free text on their side, capped at 60 characters, and shown to
the admin who decides whether to run the file — so it has to identify the shop,
not the console.
*/
test('the sender names the merchant, the branch and the person', () => {
assert.equal(
buildSender({ tenantname: 'Kmart', locationname: 'Peelamedu', username: 'abhishek' }),
'Kmart · Peelamedu · abhishek',
);
});
// The tenant name is truncated, not the whole label. Cutting the tail would
// drop the branch and the person — the two parts that say WHICH shelf and WHO —
// and leave only a long restaurant name that identifies neither.
test('a long merchant name is trimmed so the branch and person survive', () => {
const sender = buildSender({
tenantname: 'Ninhao The New Age Chinese Restaurant',
locationname: 'Race Course',
username: 'abhishek',
});
assert.ok(sender.length <= 60, `sender was ${sender.length} chars: ${sender}`);
assert.ok(sender.includes('Race Course'), sender);
assert.ok(sender.includes('abhishek'), sender);
});
test('a sender with nothing to say still labels the console', () => {
assert.equal(buildSender({}), 'nearle-console');
});

191
src/api/uploads.ts Normal file
View File

@@ -0,0 +1,191 @@
/**
* Receipts for spreadsheets sent to the catalogue ingest service.
*
* This is OUR record, in Fiesta — not the ingest service's. The two are read
* together and neither is redundant:
*
* - **Fiesta** knows the batch id, which shop the sheet was for, who sent it,
* and whether the products reached that shop's shelf. None of which the
* ingest service has any concept of — it writes the shared global
* catalogue and has no tenant and no branch.
* - **The ingest service** knows what became of the drop, and is the only
* authority on that.
*
* Fiesta's status columns are a CACHE of the second, written by whichever
* browser last polled. They exist so a list of twenty receipts renders without
* twenty network calls to a host that spends minutes per batch; the live read
* is what any single receipt is judged by.
*
* ── Why the receipt has to exist at all ──────────────────────────────────────
*
* Three facts from the ingest service's own documentation, and any one of them
* would be enough:
*
* 1. **The batch id is the credential.** `GET /api/uploads/catalog/{id}` is
* anonymous by design — holding the id is the proof of having sent the
* drop. Handed out once, to one browser. Lose it and the result is
* unreadable by anyone, including whoever uploaded the file.
* 2. **An unreviewed drop is deleted after seven days.** Nothing runs on
* arrival; a drop waits for an admin to press Start. If nobody does, the
* evidence expires.
* 3. **We cannot list our own drops.** `GET /api/uploads/catalog` is scoped to
* the credential that sent them, production has no API keys configured at
* all, and the only account that could read it is a superuser over their
* entire application.
*/
import { api, WEB } from './client';
/**
* One upload, as Fiesta stores it.
*
* The two count groups are deliberately not merged. `inserted` and its
* neighbours are the ingest service's — products in the GLOBAL catalogue, which
* every merchant shares and which therefore carries no price and no stock.
* `shelvedcount` is ours: priced, on a branch's shelf, with opening stock
* recorded. A product can be in the first and not the second, and reporting it
* as "added" would tell a shopkeeper they can sell something nobody can buy.
*/
export interface UploadReceipt {
uploadid: number;
tenantid: number;
locationid: number;
categoryid: number;
/** The drop id. The only field here that cannot be reconstructed. */
batchid: string;
/** The run an admin released the drop into, once they have. */
runid: string;
filename: string;
/** The label their review inbox shows. */
sender: string;
uploadedby: number;
uploadedname: string;
/** Rows we parsed before sending — independent of anything the service says. */
rowcount: number;
/* ── Cached from the ingest service ──────────────────────────────────── */
laststatus: string;
inserted: number;
backfilled: number;
skipped: number;
rejected: number;
/* ── Ours ────────────────────────────────────────────────────────────── */
shelvedcount: number;
skippedcount: number;
shelvedat: string | null;
created: string;
updated: string;
/** Joined for display; a receipt outlives the page that made it. */
tenantname?: string;
locationname?: string;
}
export interface RecordUploadBody {
tenantid: number;
locationid: number;
categoryid: number;
batchid: string;
filename: string;
sender: string;
uploadedby: number;
uploadedname: string;
rowcount: number;
laststatus?: string;
}
export interface UploadQuery {
/** 0 or omitted means every tenant — how a Nearle Admin sees the platform. */
tenantid?: number;
locationid?: number;
pageno?: number;
pagesize?: number;
}
/**
* How long the ingest service keeps a drop nobody has acted on.
*
* `BATCH_RETENTION_DAYS` on their side. Worth showing rather than discovering:
* a drop that expires unreviewed leaves no trace at either end, and the only
* remedy — asking an admin to release it — has to happen before the deadline.
*/
export const DROP_RETENTION_DAYS = 7;
/** Days left before an unreviewed drop is deleted; null once it has run. */
export function daysUntilExpiry(receipt: UploadReceipt): number | null {
// Only a drop still sitting in the review inbox expires. Once released, the
// run is the record and retention no longer applies to it.
if (receipt.runid || receipt.laststatus !== 'pending') return null;
const created = Date.parse(receipt.created);
if (Number.isNaN(created)) return null;
const elapsedDays = (Date.now() - created) / 86_400_000;
return Math.max(0, Math.ceil(DROP_RETENTION_DAYS - elapsedDays));
}
/**
* The label the ingest service's admin sees in their review inbox.
*
* Their field is free text capped at 60 characters, and until now every upload
* from this console arrived as the same constant — so an admin deciding what to
* approve could not tell one merchant's sheet from another's.
*
* Safe to make specific precisely because we never send a credential. Their
* ownership filter matches `sender` EXACTLY, and a run an admin assembles from
* several drops carries a joined list ("alice, bob") — so a credentialed caller
* gets a 404 on a run containing their own file. We read anonymously, holding
* the id, which is what their documentation tells integrators to do.
*
* The tenant name is truncated rather than the whole label, so the branch and
* the person survive: "Ninhao The New Age Chinese Restaurant" is 36 characters
* on its own and would otherwise push everything identifying off the end.
*/
export function buildSender(parts: {
tenantname?: string;
locationname?: string;
username?: string;
}): string {
const tenant = (parts.tenantname ?? '').trim().slice(0, 24);
const label = [tenant, (parts.locationname ?? '').trim(), (parts.username ?? '').trim()]
.filter(Boolean)
.join(' · ');
return (label || 'nearle-console').slice(0, 60);
}
export const uploadsApi = {
/**
* Store the receipt. Called the instant the drop is accepted, before polling.
*
* That timing is the whole point: it is the one moment the batch id is
* guaranteed to exist and guaranteed not to have been lost to a closed tab.
* Idempotent on `batchid` server-side, so a retry or a second tab is safe.
*/
record: (body: RecordUploadBody) => api.post<UploadReceipt>(`${WEB}/uploads/record`, body),
list: (query: UploadQuery = {}) =>
api.list<UploadReceipt>(`${WEB}/uploads/list`, {
tenantid: query.tenantid ?? 0,
locationid: query.locationid ?? 0,
pageno: query.pageno ?? 1,
pagesize: query.pagesize ?? 50,
}),
/** Cache what the ingest service last reported, so the next reader need not wait. */
updateStatus: (body: {
batchid: string;
laststatus: string;
runid?: string;
inserted?: number;
backfilled?: number;
skipped?: number;
rejected?: number;
}) => api.put<unknown>(`${WEB}/uploads/update`, body),
/** Record the other half: priced, shelved and stocked at a branch. */
markShelved: (body: { batchid: string; shelved: number; skipped: number }) =>
api.put<unknown>(`${WEB}/uploads/shelved`, body),
};