Electronics Catalog: API, MCP server, frontend and deployment
Verified catalogue of mobiles and laptops sold in India, collected from real retail listings (FastAPI backend, React frontend, Postgres/pgvector). - REST API under /api/elec (read-only catalogue; admin endpoints need login) - MCP server (FastMCP) at /mcp/ with list_categories, search_products, get_product and price_history tools - Real ratings and reviews read from product pages and search results - Production Dockerfile (requirements-api.txt, no PyTorch) and .env.production.example; remote database only via an explicit ELEC_ALLOW_REMOTE_DB host/name allowlist - docs/API.md: endpoint and MCP reference with live examples Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
sriram
4
backend/app/electronics/db/migrations/0001_schema.sql
Normal file
4
backend/app/electronics/db/migrations/0001_schema.sql
Normal file
@@ -0,0 +1,4 @@
|
||||
-- Extensions and the dedicated schema. Everything this project owns lives in
|
||||
-- schema `elec` of database `electronics_catalog`.
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
CREATE SCHEMA IF NOT EXISTS elec;
|
||||
57
backend/app/electronics/db/migrations/0002_reference.sql
Normal file
57
backend/app/electronics/db/migrations/0002_reference.sql
Normal file
@@ -0,0 +1,57 @@
|
||||
-- Reference data: brands, categories, retail sites. Seeded from
|
||||
-- app/electronics/reference/*.yaml by `elec seed-reference`.
|
||||
|
||||
CREATE TABLE elec.brand (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
parent_brand_id INT REFERENCES elec.brand(id),
|
||||
is_popular BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
official_domains TEXT[] NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Every spelling that resolves to a brand. Sub-brands (Redmi, iQOO, Pixel)
|
||||
-- resolve to their parent and are remembered as the product family.
|
||||
CREATE TABLE elec.brand_alias (
|
||||
alias TEXT PRIMARY KEY CHECK (alias = lower(alias)),
|
||||
brand_id INT NOT NULL REFERENCES elec.brand(id) ON DELETE CASCADE,
|
||||
is_sub_brand BOOLEAN NOT NULL DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE TABLE elec.category (
|
||||
id SERIAL PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE elec.brand_category (
|
||||
brand_id INT NOT NULL REFERENCES elec.brand(id) ON DELETE CASCADE,
|
||||
category_id INT NOT NULL REFERENCES elec.category(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (brand_id, category_id)
|
||||
);
|
||||
|
||||
-- A retail platform or a brand's own site, with the outcome of its probe.
|
||||
-- probe_outcome A = fetchable with structured product data (scraped)
|
||||
-- B = fetchable, product data from page HTML/state (scraped)
|
||||
-- C = not fetched: serp_only policy, robots.txt disallow,
|
||||
-- block/CAPTCHA, or unreachable -> web search only
|
||||
CREATE TABLE elec.site (
|
||||
id SERIAL PRIMARY KEY,
|
||||
domain TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('marketplace','national_chain','tn_regional','brand_official')),
|
||||
region TEXT NOT NULL CHECK (region IN ('national','TN')),
|
||||
policy TEXT NOT NULL CHECK (policy IN ('probe','serp_only')),
|
||||
brand_id INT REFERENCES elec.brand(id),
|
||||
product_url TEXT,
|
||||
pincode_param TEXT,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
probe_outcome CHAR(1) CHECK (probe_outcome IN ('A','B','C')),
|
||||
robots_allowed BOOLEAN,
|
||||
probe_evidence JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
probed_at TIMESTAMPTZ,
|
||||
breaker_until TIMESTAMPTZ,
|
||||
breaker_reason TEXT,
|
||||
CHECK (kind <> 'brand_official' OR brand_id IS NOT NULL)
|
||||
);
|
||||
160
backend/app/electronics/db/migrations/0003_observations.sql
Normal file
160
backend/app/electronics/db/migrations/0003_observations.sql
Normal file
@@ -0,0 +1,160 @@
|
||||
-- Runs, fetch audit trail, search cache, listings, prices, canonical products.
|
||||
|
||||
CREATE TABLE elec.crawl_run (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
params JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running','done','failed')),
|
||||
stats JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
error TEXT,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
ended_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Every HTTP request made to a retail or brand site. Evidence that the
|
||||
-- crawler obeyed robots.txt and its rate limits.
|
||||
CREATE TABLE elec.fetch_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_run_id BIGINT REFERENCES elec.crawl_run(id) ON DELETE SET NULL,
|
||||
url TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
status INT,
|
||||
bytes INT,
|
||||
outcome TEXT NOT NULL,
|
||||
robots_allowed BOOLEAN,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX fetch_log_host_time ON elec.fetch_log (host, fetched_at DESC);
|
||||
|
||||
CREATE TABLE elec.search_cache (
|
||||
provider TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('text','images')),
|
||||
query TEXT NOT NULL,
|
||||
results JSONB NOT NULL,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (provider, kind, query)
|
||||
);
|
||||
|
||||
-- One canonical product = one real-world variant (model + RAM + storage).
|
||||
-- verification_status becomes 'verified' only when the product has a brand
|
||||
-- official page, or listings on at least two different sites.
|
||||
CREATE TABLE elec.product (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
brand_id INT NOT NULL REFERENCES elec.brand(id),
|
||||
category_id INT NOT NULL REFERENCES elec.category(id),
|
||||
family TEXT,
|
||||
model TEXT NOT NULL,
|
||||
model_norm TEXT NOT NULL,
|
||||
variant_key TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
ram_gb NUMERIC(6,1),
|
||||
storage_gb NUMERIC(7,1),
|
||||
processor TEXT,
|
||||
mpn TEXT,
|
||||
gtin TEXT,
|
||||
canonical_specs JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
spec_sources JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
verification_status TEXT NOT NULL DEFAULT 'unverified'
|
||||
CHECK (verification_status IN ('verified','unverified','rejected')),
|
||||
evidence_count INT NOT NULL DEFAULT 0,
|
||||
embedding vector(384),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX product_brand_cat ON elec.product (brand_id, category_id);
|
||||
CREATE INDEX product_specs_gin ON elec.product USING GIN (canonical_specs);
|
||||
CREATE INDEX product_embedding_hnsw ON elec.product USING hnsw (embedding vector_cosine_ops);
|
||||
|
||||
-- The latest state of one product page on one site, or of one search result
|
||||
-- that points at such a page. Nothing is stored without the URL it came from
|
||||
-- and the text that the values were read from.
|
||||
CREATE TABLE elec.source_listing (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
site_id INT NOT NULL REFERENCES elec.site(id),
|
||||
source_sku TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL CHECK (source_url ~ '^https?://'),
|
||||
source_type TEXT NOT NULL CHECK (source_type IN ('scraped_page','search_snippet','brand_official')),
|
||||
brand_id INT NOT NULL REFERENCES elec.brand(id),
|
||||
category_id INT NOT NULL REFERENCES elec.category(id),
|
||||
family TEXT,
|
||||
title TEXT NOT NULL,
|
||||
model TEXT,
|
||||
model_number TEXT,
|
||||
ram_gb NUMERIC(6,1),
|
||||
storage_gb NUMERIC(7,1),
|
||||
colour TEXT,
|
||||
price NUMERIC(12,2) CHECK (price IS NULL OR price BETWEEN 500 AND 1000000),
|
||||
mrp NUMERIC(12,2) CHECK (mrp IS NULL OR mrp BETWEEN 500 AND 1000000),
|
||||
currency TEXT NOT NULL DEFAULT 'INR' CHECK (currency = 'INR'),
|
||||
availability TEXT,
|
||||
in_stock BOOLEAN,
|
||||
pincode TEXT,
|
||||
pincode_applied BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
rating NUMERIC(3,2) CHECK (rating IS NULL OR rating BETWEEN 0 AND 5),
|
||||
review_count INT,
|
||||
gtin TEXT,
|
||||
image_urls TEXT[] NOT NULL DEFAULT '{}',
|
||||
specs_raw JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
specs JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
evidence_text TEXT NOT NULL CHECK (length(evidence_text) > 0),
|
||||
search_query TEXT,
|
||||
confidence NUMERIC(3,2) NOT NULL CHECK (confidence BETWEEN 0 AND 1),
|
||||
parser TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
crawl_run_id BIGINT REFERENCES elec.crawl_run(id) ON DELETE SET NULL,
|
||||
UNIQUE (site_id, source_sku),
|
||||
CHECK (pincode_applied = FALSE OR pincode IS NOT NULL)
|
||||
);
|
||||
CREATE INDEX listing_brand_cat ON elec.source_listing (brand_id, category_id);
|
||||
|
||||
-- Append-only price observations. UPDATE is refused by a trigger.
|
||||
CREATE TABLE elec.price_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
listing_id BIGINT NOT NULL REFERENCES elec.source_listing(id) ON DELETE CASCADE,
|
||||
price NUMERIC(12,2) CHECK (price IS NULL OR price BETWEEN 500 AND 1000000),
|
||||
mrp NUMERIC(12,2),
|
||||
availability TEXT,
|
||||
in_stock BOOLEAN,
|
||||
source_type TEXT NOT NULL,
|
||||
pincode TEXT,
|
||||
pincode_applied BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
evidence_text TEXT NOT NULL CHECK (length(evidence_text) > 0),
|
||||
observed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
crawl_run_id BIGINT REFERENCES elec.crawl_run(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX price_history_listing_time ON elec.price_history (listing_id, observed_at DESC);
|
||||
|
||||
CREATE FUNCTION elec.refuse_update() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'elec.price_history is append-only';
|
||||
END $$;
|
||||
CREATE TRIGGER price_history_append_only BEFORE UPDATE ON elec.price_history
|
||||
FOR EACH ROW EXECUTE FUNCTION elec.refuse_update();
|
||||
|
||||
-- Which canonical product a listing belongs to, and how sure we are.
|
||||
-- Only 'auto' and 'approved' links count as evidence or appear in views.
|
||||
CREATE TABLE elec.product_listing_map (
|
||||
listing_id BIGINT PRIMARY KEY REFERENCES elec.source_listing(id) ON DELETE CASCADE,
|
||||
product_id BIGINT NOT NULL REFERENCES elec.product(id) ON DELETE CASCADE,
|
||||
method TEXT NOT NULL CHECK (method IN ('gtin','mpn','variant_key','fuzzy','manual')),
|
||||
confidence NUMERIC(3,2) NOT NULL CHECK (confidence BETWEEN 0 AND 1),
|
||||
review_status TEXT NOT NULL CHECK (review_status IN ('auto','pending','approved','rejected')),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
reviewed_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX map_product ON elec.product_listing_map (product_id);
|
||||
|
||||
-- Images are URLs only (never downloaded), each tied to the listing it was
|
||||
-- found on and checked live.
|
||||
CREATE TABLE elec.product_image (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
product_id BIGINT NOT NULL REFERENCES elec.product(id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL CHECK (url ~ '^https?://'),
|
||||
source_listing_id BIGINT NOT NULL REFERENCES elec.source_listing(id) ON DELETE CASCADE,
|
||||
source_type TEXT NOT NULL,
|
||||
rank INT NOT NULL DEFAULT 100,
|
||||
validated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (product_id, url)
|
||||
);
|
||||
71
backend/app/electronics/db/migrations/0004_views.sql
Normal file
71
backend/app/electronics/db/migrations/0004_views.sql
Normal file
@@ -0,0 +1,71 @@
|
||||
-- Read-side views. Public views only ever show VERIFIED products and links
|
||||
-- that are 'auto' or 'approved'.
|
||||
|
||||
CREATE VIEW elec.v_product_availability AS
|
||||
SELECT p.id AS product_id,
|
||||
b.name AS brand,
|
||||
c.slug AS category,
|
||||
p.display_name,
|
||||
s.id AS site_id,
|
||||
s.name AS site,
|
||||
s.domain,
|
||||
s.kind AS site_kind,
|
||||
s.region AS site_region,
|
||||
l.id AS listing_id,
|
||||
l.source_url,
|
||||
l.source_type,
|
||||
l.title AS listing_title,
|
||||
l.colour,
|
||||
l.price,
|
||||
l.mrp,
|
||||
l.in_stock,
|
||||
l.availability,
|
||||
l.pincode,
|
||||
l.pincode_applied,
|
||||
l.confidence,
|
||||
l.last_seen_at AS observed_at
|
||||
FROM elec.product p
|
||||
JOIN elec.brand b ON b.id = p.brand_id
|
||||
JOIN elec.category c ON c.id = p.category_id
|
||||
JOIN elec.product_listing_map m ON m.product_id = p.id AND m.review_status IN ('auto','approved')
|
||||
JOIN elec.source_listing l ON l.id = m.listing_id
|
||||
JOIN elec.site s ON s.id = l.site_id
|
||||
WHERE p.verification_status = 'verified';
|
||||
|
||||
-- Cheapest known price per product. Scraped prices are preferred over search
|
||||
-- snippet prices; a listing known to be out of stock is skipped.
|
||||
CREATE VIEW elec.v_best_price AS
|
||||
SELECT DISTINCT ON (product_id)
|
||||
product_id, site, domain, source_url, source_type, price, mrp, in_stock, observed_at
|
||||
FROM elec.v_product_availability
|
||||
WHERE price IS NOT NULL AND in_stock IS DISTINCT FROM FALSE
|
||||
ORDER BY product_id, (source_type = 'search_snippet'), price, observed_at DESC;
|
||||
|
||||
CREATE VIEW elec.v_brand_catalog AS
|
||||
SELECT p.id AS product_id, b.name AS brand, b.slug AS brand_slug, c.slug AS category,
|
||||
p.family, p.display_name, p.model, p.ram_gb, p.storage_gb, p.processor,
|
||||
p.canonical_specs,
|
||||
bp.price AS best_price,
|
||||
bp.site AS best_price_site,
|
||||
bp.source_type AS best_price_source_type,
|
||||
(SELECT count(DISTINCT a.site_id) FROM elec.v_product_availability a
|
||||
WHERE a.product_id = p.id) AS platform_count,
|
||||
(SELECT coalesce(bool_or(a.site_region = 'TN'), FALSE) FROM elec.v_product_availability a
|
||||
WHERE a.product_id = p.id) AS sold_by_tn_retailer,
|
||||
(SELECT i.url FROM elec.product_image i WHERE i.product_id = p.id
|
||||
ORDER BY i.rank, i.id LIMIT 1) AS image_url,
|
||||
p.updated_at
|
||||
FROM elec.product p
|
||||
JOIN elec.brand b ON b.id = p.brand_id
|
||||
JOIN elec.category c ON c.id = p.category_id
|
||||
LEFT JOIN elec.v_best_price bp ON bp.product_id = p.id
|
||||
WHERE p.verification_status = 'verified';
|
||||
|
||||
CREATE VIEW elec.v_brand_summary AS
|
||||
SELECT brand, brand_slug, category,
|
||||
count(*) AS product_count,
|
||||
min(best_price) AS min_price,
|
||||
max(best_price) AS max_price,
|
||||
max(platform_count) AS max_platforms
|
||||
FROM elec.v_brand_catalog
|
||||
GROUP BY brand, brand_slug, category;
|
||||
@@ -0,0 +1,44 @@
|
||||
-- Search results carry cached, sometimes seller-specific prices. A price that
|
||||
-- disagrees sharply with the product-page price for the same product (or is
|
||||
-- below what the category can cost) is kept with its evidence but flagged, and
|
||||
-- is never used as the "best price". Set by repository.flag_price_outliers().
|
||||
ALTER TABLE elec.source_listing ADD COLUMN price_outlier BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
CREATE OR REPLACE VIEW elec.v_product_availability AS
|
||||
SELECT p.id AS product_id,
|
||||
b.name AS brand,
|
||||
c.slug AS category,
|
||||
p.display_name,
|
||||
s.id AS site_id,
|
||||
s.name AS site,
|
||||
s.domain,
|
||||
s.kind AS site_kind,
|
||||
s.region AS site_region,
|
||||
l.id AS listing_id,
|
||||
l.source_url,
|
||||
l.source_type,
|
||||
l.title AS listing_title,
|
||||
l.colour,
|
||||
l.price,
|
||||
l.mrp,
|
||||
l.in_stock,
|
||||
l.availability,
|
||||
l.pincode,
|
||||
l.pincode_applied,
|
||||
l.confidence,
|
||||
l.last_seen_at AS observed_at,
|
||||
l.price_outlier
|
||||
FROM elec.product p
|
||||
JOIN elec.brand b ON b.id = p.brand_id
|
||||
JOIN elec.category c ON c.id = p.category_id
|
||||
JOIN elec.product_listing_map m ON m.product_id = p.id AND m.review_status IN ('auto','approved')
|
||||
JOIN elec.source_listing l ON l.id = m.listing_id
|
||||
JOIN elec.site s ON s.id = l.site_id
|
||||
WHERE p.verification_status = 'verified';
|
||||
|
||||
CREATE OR REPLACE VIEW elec.v_best_price AS
|
||||
SELECT DISTINCT ON (product_id)
|
||||
product_id, site, domain, source_url, source_type, price, mrp, in_stock, observed_at
|
||||
FROM elec.v_product_availability
|
||||
WHERE price IS NOT NULL AND NOT price_outlier AND in_stock IS DISTINCT FROM FALSE
|
||||
ORDER BY product_id, (source_type = 'search_snippet'), price, observed_at DESC;
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Best price: a product whose every priced listing is out of stock still has a
|
||||
-- price worth showing. In-stock (or unknown-stock) prices still win; an
|
||||
-- out-of-stock price is used only when nothing else is priced. Same columns
|
||||
-- as 0005, so v_brand_catalog keeps working unchanged.
|
||||
CREATE OR REPLACE VIEW elec.v_best_price AS
|
||||
SELECT DISTINCT ON (product_id)
|
||||
product_id, site, domain, source_url, source_type, price, mrp, in_stock, observed_at
|
||||
FROM elec.v_product_availability
|
||||
WHERE price IS NOT NULL AND NOT price_outlier
|
||||
ORDER BY product_id, (in_stock IS FALSE), (source_type = 'search_snippet'), price, observed_at DESC;
|
||||
|
||||
-- Individual customer reviews, exactly as a product page publishes them in its
|
||||
-- schema.org JSON-LD. Nothing here is generated: every row is a review the
|
||||
-- listing's own page stated. Sentiment is derived only from the reviewer's
|
||||
-- own star rating (>=4 positive, >=3 neutral, <3 negative); NULL when the
|
||||
-- review states no rating.
|
||||
CREATE TABLE elec.listing_review (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
listing_id BIGINT NOT NULL REFERENCES elec.source_listing(id) ON DELETE CASCADE,
|
||||
author TEXT,
|
||||
rating NUMERIC(2,1) CHECK (rating IS NULL OR rating BETWEEN 0 AND 5),
|
||||
title TEXT,
|
||||
body TEXT NOT NULL,
|
||||
review_date TEXT,
|
||||
sentiment TEXT CHECK (sentiment IS NULL OR sentiment IN ('positive','neutral','negative')),
|
||||
content_hash TEXT NOT NULL,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (listing_id, content_hash)
|
||||
);
|
||||
CREATE INDEX listing_review_listing_idx ON elec.listing_review (listing_id);
|
||||
Reference in New Issue
Block a user