lat and long
This commit is contained in:
397
src/api/ingest.ts
Normal file
397
src/api/ingest.ts
Normal file
@@ -0,0 +1,397 @@
|
||||
/**
|
||||
* The catalogue ingest service — `mcp.nearle.ai.in`, pipeline v3.2.0.
|
||||
*
|
||||
* A workbook goes up, the service parses and enriches it, and the rows land in
|
||||
* the global catalogue's per-brand tables. It replaces the row-by-row create
|
||||
* loop the console used to run in the browser.
|
||||
*
|
||||
* Everything here is written against the contract the owning team supplied, not
|
||||
* against a guess. Where a decision looks arbitrary it usually is not — the
|
||||
* reason is in the comment.
|
||||
*
|
||||
* ── Submit, then poll ────────────────────────────────────────────────────────
|
||||
*
|
||||
* `POST /ingest` answers 202 with a job id and hands off to a background
|
||||
* thread; the outcome arrives from `GET /jobs/{id}`. Two operational details
|
||||
* from the owning team shape `pollJob` below, and both are the kind of thing
|
||||
* that silently produces a wrong screen if ignored:
|
||||
*
|
||||
* - **Jobs live in memory.** A backend restart loses them and polling returns
|
||||
* 404. That is "we no longer know", NOT "it failed" — the rows may well have
|
||||
* been written. Reporting a failure there would send someone re-uploading a
|
||||
* sheet that already landed.
|
||||
* - **`products_built > 0` is not success.** A job whose rows were built but
|
||||
* could not be stored is marked `failed` with the reason in
|
||||
* `result.storage_error`. The counts are populated either way, so reading
|
||||
* them without checking `status` reports an import that never happened.
|
||||
*
|
||||
* ── The credential never reaches this file ───────────────────────────────────
|
||||
*
|
||||
* `X-API-Key`, attached by the Vite proxy from `INGEST_TOKEN` in `.env.local`.
|
||||
* The key carries `require_admin`, which on that service is a superuser — the
|
||||
* same key reaches `/api/catalog/generate`, `/api/system/init` and the ML
|
||||
* training endpoints. A key in the browser bundle is a key published to every
|
||||
* visitor, so it stays server-side and this module never sees one.
|
||||
*
|
||||
* That is why production is not solved here. The console's origin is not in
|
||||
* their `API_CORS_ORIGINS` and should not be added: the owning team's own
|
||||
* recommendation is server-to-server, which means Fiesta relays the call. Until
|
||||
* that exists, this path works in development only.
|
||||
*/
|
||||
|
||||
const INGEST_BASE = import.meta.env['VITE_INGEST_BASE'] ?? '/ingest';
|
||||
|
||||
const ROOT = '/api/admin/store-catalog';
|
||||
|
||||
/**
|
||||
* Client-side limits, mirroring the service's own.
|
||||
*
|
||||
* Checked here so a 12 MB workbook is refused in the browser instead of being
|
||||
* uploaded over a shop's connection to earn a 413. The service remains the
|
||||
* authority; this is politeness, not validation.
|
||||
*/
|
||||
export const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
export const MAX_ROWS = 2000;
|
||||
|
||||
/** Everything the service parses. `.txt`/`.tab` included because it takes them. */
|
||||
export const ACCEPTED_EXTENSIONS = ['.xlsx', '.xlsm', '.xls', '.csv', '.tsv', '.txt', '.tab'];
|
||||
|
||||
/* ── Response types, from the owning team's real output ───────────────────── */
|
||||
|
||||
export type JobStatus = 'pending' | 'running' | 'done' | 'failed';
|
||||
|
||||
/** A row that could not be turned into a product at all. */
|
||||
export interface IngestRowError {
|
||||
/** 1-based spreadsheet row. The header is row 1, so this matches Excel. */
|
||||
row: number;
|
||||
product_name: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
/** A row that was built, then failed the deterministic validation gate. */
|
||||
export interface IngestRejection {
|
||||
product_name: string;
|
||||
size: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface IngestJobResult {
|
||||
rows_total: number;
|
||||
/** Can exceed `rows_total`: "100g, 200g, 500g" in one cell is three products. */
|
||||
products_built: number;
|
||||
inserted: number;
|
||||
/** Existing rows whose blank columns this upload filled in. */
|
||||
backfilled: number;
|
||||
/** Already present and already complete — nothing to do. */
|
||||
skipped_existing: number;
|
||||
rejected: number;
|
||||
brands: string[];
|
||||
/** Capped at 50 by the service; `rejected` carries the true total. */
|
||||
rejections: IngestRejection[];
|
||||
/** Capped at 50 by the service; `error_count` carries the true total. */
|
||||
errors: IngestRowError[];
|
||||
error_count: number;
|
||||
/** Non-fatal corrections that were applied anyway, row-numbered. */
|
||||
warnings: string[];
|
||||
/** Sheet header → the field it was read as. */
|
||||
recognised_columns: Record<string, string>;
|
||||
/** Headers that matched nothing and were silently dropped. */
|
||||
unrecognised_columns: string[];
|
||||
/** Non-null means the rows were built but never stored. */
|
||||
storage_error: string | null;
|
||||
}
|
||||
|
||||
export interface IngestJob {
|
||||
job_id: string;
|
||||
filename: string;
|
||||
status: JobStatus;
|
||||
detail: string | null;
|
||||
stage_index: number;
|
||||
stage_name: string;
|
||||
total_stages: number;
|
||||
rows_done: number;
|
||||
rows_total: number;
|
||||
result: IngestJobResult | null;
|
||||
}
|
||||
|
||||
/** What `/preview` answers — a dry run that writes nothing. */
|
||||
export interface IngestPreview {
|
||||
recognised_columns: Record<string, string>;
|
||||
unrecognised_columns: string[];
|
||||
rows: Record<string, unknown>[];
|
||||
rows_total?: number;
|
||||
}
|
||||
|
||||
export class IngestError extends Error {
|
||||
readonly status: number;
|
||||
readonly body: string;
|
||||
|
||||
constructor(message: string, status: number, body = '') {
|
||||
super(message);
|
||||
this.name = 'IngestError';
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
/** True when a poll found no such job — see the note about in-memory jobs. */
|
||||
export class JobVanishedError extends IngestError {
|
||||
constructor(jobId: string) {
|
||||
super(
|
||||
'The service no longer knows about this job — it restarts with jobs held in memory. The products may well have been written; check the catalogue before uploading again.',
|
||||
404,
|
||||
);
|
||||
this.name = 'JobVanishedError';
|
||||
this.jobId = jobId;
|
||||
}
|
||||
readonly jobId: string;
|
||||
}
|
||||
|
||||
/* ── Requests ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
export interface SubmitOptions {
|
||||
file: File;
|
||||
/**
|
||||
* Default FALSE, and that is not caution — production reports
|
||||
* `"ollama": false`, so the LLM path is not available there. Asking for it
|
||||
* buys nothing and the enrichment that matters (HSN, price band, FSSAI, SKU)
|
||||
* is deterministic lookup rather than generation.
|
||||
*/
|
||||
useLlm?: boolean;
|
||||
/**
|
||||
* Default FALSE. Image fetching is a network round trip per row evaluating up
|
||||
* to 24 candidates, and it is the stage that turns ten seconds into minutes.
|
||||
* Worth turning on deliberately, not by default.
|
||||
*/
|
||||
fetchImages?: boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function guardFile(file: File): void {
|
||||
if (file.size > MAX_FILE_BYTES) {
|
||||
throw new IngestError(
|
||||
`That file is ${(file.size / 1024 / 1024).toFixed(1)} MB. The service accepts up to 10 MB.`,
|
||||
413,
|
||||
);
|
||||
}
|
||||
if (file.size === 0) {
|
||||
throw new IngestError('That file is empty.', 400);
|
||||
}
|
||||
}
|
||||
|
||||
async function send<T>(path: string, file: File, query: URLSearchParams, signal?: AbortSignal) {
|
||||
guardFile(file);
|
||||
|
||||
const form = new FormData();
|
||||
// `file`, confirmed: the handler signature is `file: UploadFile = File(...)`.
|
||||
form.append('file', file, file.name);
|
||||
|
||||
// NOTE: no tenantid/locationid. This endpoint writes the GLOBAL catalogue and
|
||||
// has no concept of an outlet — making a product sellable at a shop is a
|
||||
// separate call to `/api/upload/stores`, keyed on `image_id`. Sending them
|
||||
// here achieved nothing and implied a link that does not exist.
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(`${INGEST_BASE}${ROOT}${path}?${query}`, {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
// Content-Type is deliberately unset: the browser adds it WITH the
|
||||
// multipart boundary. Setting it by hand omits the boundary and the
|
||||
// server parses nothing.
|
||||
headers: { Accept: 'application/json' },
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new IngestError(
|
||||
cause instanceof DOMException && cause.name === 'AbortError'
|
||||
? 'Cancelled.'
|
||||
: 'Could not reach the ingest service.',
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
return readResponse<T>(response);
|
||||
}
|
||||
|
||||
async function readResponse<T>(response: Response): Promise<T> {
|
||||
const text = await response.text();
|
||||
let payload: unknown = null;
|
||||
try {
|
||||
payload = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
payload = text;
|
||||
}
|
||||
|
||||
if (!response.ok) throw describe(response.status, payload, text);
|
||||
return payload as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* The service's failures, in words that name the fix.
|
||||
*
|
||||
* Each of these has one cause and one remedy, and a generic "request failed"
|
||||
* sends people to look at their spreadsheet for a problem that is in the
|
||||
* deployment.
|
||||
*/
|
||||
function describe(status: number, payload: unknown, text: string): IngestError {
|
||||
const detail =
|
||||
(payload !== null &&
|
||||
typeof payload === 'object' &&
|
||||
typeof (payload as { detail?: unknown }).detail === 'string' &&
|
||||
(payload as { detail: string }).detail) ||
|
||||
undefined;
|
||||
|
||||
if (status === 401 || status === 403) {
|
||||
return new IngestError(
|
||||
detail ??
|
||||
'The ingest service rejected the credential. Set INGEST_TOKEN in .env.local and restart the dev server — and note the key only works once their backend is rebuilt with it, since API_KEYS is baked in at build time.',
|
||||
status,
|
||||
text.slice(0, 2000),
|
||||
);
|
||||
}
|
||||
if (status === 413) {
|
||||
return new IngestError(
|
||||
detail ?? 'Too large for the service — the limits are 10 MB and 2000 rows.',
|
||||
status,
|
||||
text.slice(0, 2000),
|
||||
);
|
||||
}
|
||||
if (status === 400) {
|
||||
return new IngestError(
|
||||
detail ?? 'The service could not read that file — it may be empty or have no data rows.',
|
||||
status,
|
||||
text.slice(0, 2000),
|
||||
);
|
||||
}
|
||||
return new IngestError(detail ?? `The ingest service returned HTTP ${status}.`, status, text.slice(0, 2000));
|
||||
}
|
||||
|
||||
/**
|
||||
* A true dry run. Parses, reports the column mapping and the first rows, and
|
||||
* writes nothing at all.
|
||||
*
|
||||
* Run before every ingest. It is the only way to see `unrecognised_columns`
|
||||
* before the fact, and a header that matched nothing is dropped SILENTLY — a
|
||||
* price column the service never saw looks exactly like a successful import
|
||||
* until someone opens the catalogue.
|
||||
*/
|
||||
export function previewSheet(file: File, signal?: AbortSignal): Promise<IngestPreview> {
|
||||
return send<IngestPreview>('/preview', file, new URLSearchParams(), signal);
|
||||
}
|
||||
|
||||
/** Submits the sheet. Answers 202 with a job to poll — it does not wait. */
|
||||
export function submitIngest(options: SubmitOptions): Promise<IngestJob> {
|
||||
const { file, useLlm = false, fetchImages = false, signal } = options;
|
||||
const query = new URLSearchParams({
|
||||
use_llm: String(useLlm),
|
||||
fetch_images: String(fetchImages),
|
||||
});
|
||||
return send<IngestJob>('/ingest', file, query, signal);
|
||||
}
|
||||
|
||||
/** One poll. Throws `JobVanishedError` on 404 — see the note at the top. */
|
||||
export async function fetchJob(jobId: string, signal?: AbortSignal): Promise<IngestJob> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(`${INGEST_BASE}${ROOT}/jobs/${encodeURIComponent(jobId)}`, {
|
||||
headers: { Accept: 'application/json' },
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
} catch {
|
||||
throw new IngestError('Lost contact with the ingest service while waiting.', 0);
|
||||
}
|
||||
if (response.status === 404) throw new JobVanishedError(jobId);
|
||||
return readResponse<IngestJob>(response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls until the job settles.
|
||||
*
|
||||
* Every second. The service does no rate limiting on this and a person is
|
||||
* watching a progress bar, so a slower cadence buys nothing but a screen that
|
||||
* looks stuck. `onTick` fires on each reading so the caller can render
|
||||
* `stage_name` and `rows_done` as they move.
|
||||
*/
|
||||
export async function pollJob(
|
||||
jobId: string,
|
||||
onTick: (job: IngestJob) => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<IngestJob> {
|
||||
for (;;) {
|
||||
if (signal?.aborted) throw new IngestError('Cancelled.', 0);
|
||||
|
||||
const job = await fetchJob(jobId, signal);
|
||||
onTick(job);
|
||||
if (job.status === 'done' || job.status === 'failed') return job;
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Deriving what the service does not return ────────────────────────────── */
|
||||
|
||||
/**
|
||||
* The catalogue's primary key, computed locally.
|
||||
*
|
||||
* The ingest returns counts, not ids — but the key is deterministic, so the
|
||||
* rows can be addressed without being told. That matters for the step after
|
||||
* this one: `/api/upload/stores` joins on exactly this value to put a product
|
||||
* on a shop's shelf.
|
||||
*
|
||||
* image_id = sanitize(brand) + "_" + slugify(name [+ " " + size])
|
||||
*
|
||||
* The size is appended ONLY when its slug is not already inside the name's —
|
||||
* "Good Day Cashew Cookies 100g" with size "100g" must not become
|
||||
* `..._100g_100g`.
|
||||
*
|
||||
* Verified against real output: `britannia_britannia_good_day_cashew_cookies_100g`.
|
||||
*/
|
||||
export function imageId(brand: string, productName: string, size?: string): string {
|
||||
const nameSlug = slugify(productName);
|
||||
const sizeSlug = size ? slugify(size) : '';
|
||||
const tail = sizeSlug && !nameSlug.includes(sizeSlug) ? `${nameSlug}_${sizeSlug}` : nameSlug;
|
||||
return `${sanitize(brand)}_${tail}`;
|
||||
}
|
||||
|
||||
/** lowercase · space, hyphen and & become `_` · drop the rest · collapse runs. */
|
||||
function sanitize(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[\s\-&]/g, '_')
|
||||
.replace(/[^a-z0-9_]/g, '')
|
||||
.replace(/_{2,}/g, '_')
|
||||
.replace(/^_+|_+$/g, '');
|
||||
}
|
||||
|
||||
/** lowercase · any run of non-alphanumerics becomes one `_` · trim. */
|
||||
function slugify(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '_')
|
||||
.replace(/^_+|_+$/g, '');
|
||||
}
|
||||
|
||||
/* ── Reading a finished job ───────────────────────────────────────────────── */
|
||||
|
||||
/** True when the job ended without the rows reaching the database. */
|
||||
export function isStorageFailure(job: IngestJob): boolean {
|
||||
return job.status === 'failed' || Boolean(job.result?.storage_error);
|
||||
}
|
||||
|
||||
/** One line for the top of the result panel. */
|
||||
export function summarise(job: IngestJob): string {
|
||||
const result = job.result;
|
||||
if (!result) return job.detail ?? 'The service returned no result.';
|
||||
|
||||
if (isStorageFailure(job)) {
|
||||
return result.storage_error
|
||||
? `Built ${result.products_built} products but could not store them: ${result.storage_error}`
|
||||
: (job.detail ?? 'The job failed.');
|
||||
}
|
||||
|
||||
const parts = [`${result.inserted} added`];
|
||||
if (result.backfilled > 0) parts.push(`${result.backfilled} filled in`);
|
||||
if (result.skipped_existing > 0) parts.push(`${result.skipped_existing} already there`);
|
||||
return parts.join(' · ');
|
||||
}
|
||||
@@ -180,6 +180,16 @@ export interface SheetImportOptions {
|
||||
* batch create, and firing 500 concurrent writes at a single-instance Go
|
||||
* service to save a few seconds is a poor trade against a half-imported tenant.
|
||||
*/
|
||||
/**
|
||||
* NO LONGER WIRED TO ANY SCREEN.
|
||||
*
|
||||
* The Upload sheet panel now hands the workbook to the ingest service
|
||||
* (`api/ingest.ts`) instead of running this loop from the browser. Kept, not
|
||||
* deleted, because the ingest contract is still unconfirmed and this is the
|
||||
* known-working path back if that service turns out not to fit. Delete it once
|
||||
* the ingest has run against real data and been signed off — a second import
|
||||
* path that nobody calls is a thing that rots.
|
||||
*/
|
||||
export async function importSheetProducts(
|
||||
options: SheetImportOptions,
|
||||
): Promise<SheetImportResult> {
|
||||
|
||||
Reference in New Issue
Block a user