Electronics Catalog: API, MCP server, frontend and deployment
Verified catalogue of mobiles and laptops sold in India, collected from real retail listings (FastAPI backend, React frontend, Postgres/pgvector). - REST API under /api/elec (read-only catalogue; admin endpoints need login) - MCP server (FastMCP) at /mcp/ with list_categories, search_products, get_product and price_history tools - Real ratings and reviews read from product pages and search results - Production Dockerfile (requirements-api.txt, no PyTorch) and .env.production.example; remote database only via an explicit ELEC_ALLOW_REMOTE_DB host/name allowlist - docs/API.md: endpoint and MCP reference with live examples Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
149
backend/tests/conftest.py
Normal file
149
backend/tests/conftest.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""
|
||||
Shared test setup.
|
||||
|
||||
Every environment variable here must be set BEFORE `app.main` is imported,
|
||||
because app/infrastructure/settings.py reads the environment once at import
|
||||
time.
|
||||
|
||||
Hermetic by construction:
|
||||
* no web search, no LLM calls (USE_DDG_SEARCH / USE_GOOGLE_CSE / USE_OLLAMA off)
|
||||
* database tests use their OWN database, electronics_catalog_test, on the
|
||||
local Docker server - never the real electronics_catalog data, and the
|
||||
settings guard makes any non-local host impossible anyway. They are
|
||||
skipped when that server is not running.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
BACKEND = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(BACKEND))
|
||||
|
||||
TEST_ADMIN_PASSWORD = "test-admin-password"
|
||||
TEST_USER_PASSWORD = "test-user-password"
|
||||
TEST_API_KEY = "test-api-key-value-not-a-real-secret"
|
||||
TEST_DB_NAME = "electronics_catalog_test"
|
||||
|
||||
|
||||
def _hash(password: str, iterations: int = 20_000) -> str:
|
||||
"""Byte-compatible with security.hash_password, at a low iteration count."""
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, iterations)
|
||||
return "$".join(("pbkdf2_sha256", str(iterations), base64.b64encode(salt).decode(),
|
||||
base64.b64encode(digest).decode()))
|
||||
|
||||
|
||||
# Local DB connection details come from backend/.env when it exists (the
|
||||
# password is generated per machine); the database NAME is always the test one.
|
||||
try:
|
||||
from dotenv import dotenv_values
|
||||
|
||||
_local_env = dotenv_values(BACKEND / ".env")
|
||||
except Exception: # noqa: BLE001
|
||||
_local_env = {}
|
||||
os.environ["DB_HOST"] = "127.0.0.1"
|
||||
os.environ["DB_PORT"] = _local_env.get("DB_PORT") or "5433"
|
||||
os.environ["DB_USER"] = _local_env.get("DB_USER") or "postgres"
|
||||
os.environ["DB_PASSWORD"] = _local_env.get("DB_PASSWORD") or "test-password-not-real"
|
||||
os.environ["DB_NAME"] = TEST_DB_NAME
|
||||
os.environ["DB_CONNECT_TIMEOUT_SECONDS"] = "3"
|
||||
|
||||
os.environ["USE_OLLAMA"] = "false"
|
||||
os.environ["ELEC_USE_LLM"] = "false"
|
||||
os.environ["USE_DDG_SEARCH"] = "false"
|
||||
os.environ["USE_GOOGLE_CSE"] = "false"
|
||||
os.environ["GOOGLE_API_KEY"] = ""
|
||||
os.environ["GOOGLE_CSE_ID"] = ""
|
||||
|
||||
# Auth is set unconditionally: the suite asserts on the real guards.
|
||||
os.environ["AUTH_ENABLED"] = "true"
|
||||
os.environ["AUTH_ALLOW_ANY_LOGIN"] = "false"
|
||||
os.environ["AUTH_SECRET_KEY"] = "test-secret-key-not-for-production-use-at-all"
|
||||
os.environ["AUTH_ADMIN_USERNAME"] = "admin"
|
||||
os.environ["AUTH_ADMIN_PASSWORD_HASH"] = _hash(TEST_ADMIN_PASSWORD)
|
||||
os.environ["AUTH_USER_USERNAME"] = "user"
|
||||
os.environ["AUTH_USER_PASSWORD_HASH"] = _hash(TEST_USER_PASSWORD)
|
||||
os.environ["API_KEYS"] = f"test-machine:user:{TEST_API_KEY}"
|
||||
os.environ["AUTH_MAX_LOGIN_ATTEMPTS"] = "3"
|
||||
os.environ["AUTH_LOCKOUT_SECONDS"] = "60"
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from app.main import app # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_login_throttle():
|
||||
from app.api.routers import auth as auth_router
|
||||
|
||||
with auth_router._failures_lock:
|
||||
auth_router._failures.clear()
|
||||
yield
|
||||
with auth_router._failures_lock:
|
||||
auth_router._failures.clear()
|
||||
|
||||
|
||||
def _token(client: TestClient, username: str, password: str) -> str:
|
||||
resp = client.post("/api/auth/login", json={"username": username, "password": password})
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["access_token"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_headers(client: TestClient) -> dict:
|
||||
return {"Authorization": f"Bearer {_token(client, 'admin', TEST_ADMIN_PASSWORD)}"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_headers(client: TestClient) -> dict:
|
||||
return {"Authorization": f"Bearer {_token(client, 'user', TEST_USER_PASSWORD)}"}
|
||||
|
||||
|
||||
_DATA_TABLES = ("product_image, product_listing_map, price_history, source_listing, product, "
|
||||
"fetch_log, search_cache, crawl_run")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def test_database():
|
||||
"""Create/migrate/seed electronics_catalog_test once per session, or skip."""
|
||||
import psycopg
|
||||
|
||||
try:
|
||||
with psycopg.connect(host="127.0.0.1", port=os.environ["DB_PORT"], dbname="postgres",
|
||||
user=os.environ["DB_USER"], password=os.environ["DB_PASSWORD"],
|
||||
connect_timeout=3, autocommit=True) as admin:
|
||||
exists = admin.execute("SELECT 1 FROM pg_database WHERE datname = %s", (TEST_DB_NAME,)).fetchone()
|
||||
if not exists:
|
||||
admin.execute(f'CREATE DATABASE "{TEST_DB_NAME}"')
|
||||
except Exception as exc: # noqa: BLE001
|
||||
pytest.skip(f"local Postgres not reachable ({exc}); run `docker compose up -d`")
|
||||
from app.electronics.db import repository as repo
|
||||
from app.electronics.db.migrate import run_migrations
|
||||
from app.electronics.reference import load_reference
|
||||
|
||||
run_migrations()
|
||||
repo.seed_reference(load_reference())
|
||||
return TEST_DB_NAME
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(test_database):
|
||||
"""A clean test database for one test (reference data kept)."""
|
||||
from app.electronics.db.connection import connect
|
||||
|
||||
with connect(autocommit=True) as conn:
|
||||
conn.execute(f"TRUNCATE {', '.join('elec.' + t.strip() for t in _DATA_TABLES.split(','))} CASCADE")
|
||||
conn.execute("UPDATE elec.site SET probe_outcome = NULL, breaker_until = NULL, breaker_reason = NULL")
|
||||
yield test_database
|
||||
39
backend/tests/test_api.py
Normal file
39
backend/tests/test_api.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Smoke tests for the FastAPI layer (no database needed)."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_root(client) -> None:
|
||||
resp = client.get("/")
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
def test_health_reports_database_and_search(client) -> None:
|
||||
resp = client.get("/api/health")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["status"] in {"ok", "degraded"}
|
||||
assert body["database_name"] == "electronics_catalog_test"
|
||||
assert body["search"] == {"ddg": False, "google_cse": False}
|
||||
|
||||
|
||||
def test_openapi_lists_only_electronics_routes(client) -> None:
|
||||
paths = client.get("/openapi.json").json()["paths"]
|
||||
for expected in ("/api/health", "/api/auth/login", "/api/elec/categories", "/api/elec/brands",
|
||||
"/api/elec/products", "/api/elec/products/{product_id}", "/api/elec/sites",
|
||||
"/api/elec/admin/runs"):
|
||||
assert expected in paths, f"missing route: {expected}"
|
||||
grocery = [p for p in paths if any(w in p for w in ("nutrition", "catalog/generate", "stores", "discounts",
|
||||
"trending", "uploads", "brand-discovery"))]
|
||||
assert grocery == []
|
||||
|
||||
|
||||
def test_admin_run_requires_admin(client, user_headers) -> None:
|
||||
assert client.post("/api/elec/admin/runs", json={"category": "mobiles"}).status_code == 401
|
||||
assert client.post("/api/elec/admin/runs", json={"category": "mobiles"},
|
||||
headers=user_headers).status_code == 403
|
||||
|
||||
|
||||
def test_admin_run_rejects_brand_outside_allow_list(client, admin_headers) -> None:
|
||||
resp = client.post("/api/elec/admin/runs", json={"category": "mobiles", "brands": ["nokia"]},
|
||||
headers=admin_headers)
|
||||
assert resp.status_code == 400
|
||||
405
backend/tests/test_auth.py
Normal file
405
backend/tests/test_auth.py
Normal file
@@ -0,0 +1,405 @@
|
||||
"""
|
||||
Tests for authentication and the endpoint guards.
|
||||
|
||||
The behaviours asserted here are the ones the previous implementation got
|
||||
wrong, so each has a comment saying what it prevents rather than just what it
|
||||
checks. They need no database or Ollama: a request that is rejected at the
|
||||
guard never reaches a service.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
|
||||
from tests.conftest import TEST_ADMIN_PASSWORD, TEST_API_KEY, TEST_USER_PASSWORD
|
||||
|
||||
# Every write/compute endpoint, with a request body valid enough that a 422
|
||||
# would prove the guard let the request through to validation.
|
||||
WRITE_ENDPOINTS = [
|
||||
("/api/elec/admin/runs", {"json": {"category": "mobiles"}}),
|
||||
("/api/elec/admin/review/1", {"json": {"approve": True}}),
|
||||
("/api/elec/admin/sites/croma.com/probe", {}),
|
||||
]
|
||||
|
||||
ADMIN_ONLY_ENDPOINTS = [
|
||||
("/api/elec/admin/runs", {"category": "mobiles"}),
|
||||
("/api/elec/admin/review/1", {"approve": True}),
|
||||
("/api/elec/admin/sites/croma.com/probe", None),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Guards
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize("path,kwargs", WRITE_ENDPOINTS)
|
||||
def test_write_endpoints_reject_anonymous_callers(client, path, kwargs):
|
||||
"""Starting a crawl or probing a site must never be open to anyone who can
|
||||
reach the port."""
|
||||
resp = client.post(path, **kwargs)
|
||||
assert resp.status_code == 401, f"{path} answered {resp.status_code}, expected 401"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path,body", ADMIN_ONLY_ENDPOINTS)
|
||||
def test_admin_endpoints_reject_the_user_role(client, user_headers, path, body):
|
||||
"""403, not 401: the caller is authenticated, just not allowed."""
|
||||
resp = client.post(path, json=body, headers=user_headers)
|
||||
assert resp.status_code == 403, f"{path} answered {resp.status_code}, expected 403"
|
||||
|
||||
|
||||
def test_admin_passes_an_admin_only_endpoint(client, admin_headers):
|
||||
"""400 (a brand outside the allow-list) proves the guard let admin through."""
|
||||
resp = client.post("/api/elec/admin/runs", json={"category": "mobiles", "brands": ["nokia"]},
|
||||
headers=admin_headers)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/api/health", "/api/auth/roles", "/openapi.json"])
|
||||
def test_read_endpoints_stay_public(client, path):
|
||||
"""Guarding writes must not have closed off what the app browses."""
|
||||
assert client.get(path).status_code == 200
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Login
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_login_succeeds_and_returns_a_token(client):
|
||||
resp = client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["token_type"] == "bearer"
|
||||
assert body["access_token"]
|
||||
assert body["expires_in"] > 0
|
||||
assert body["user"]["role"] == "admin"
|
||||
|
||||
|
||||
def test_login_is_case_insensitive_on_username_only(client):
|
||||
"""Usernames are normalised; passwords are not. The old version lowercased
|
||||
the password before comparing, which quietly shrank the keyspace."""
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "ADMIN", "password": TEST_ADMIN_PASSWORD}
|
||||
).status_code == 200
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD.upper()}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
def test_login_rejects_an_empty_password(client):
|
||||
"""The old implementation treated an empty password as valid for any known
|
||||
username (`if pwd in passwords or pwd == ""`)."""
|
||||
resp = client.post("/api/auth/login", json={"username": "admin", "password": ""})
|
||||
assert resp.status_code == 422 # min_length=1 on the schema
|
||||
|
||||
|
||||
def test_login_rejects_an_unknown_username(client):
|
||||
"""The old fallback granted a profile to ANY username, and `admin` to any
|
||||
username that also asked for role='admin'."""
|
||||
resp = client.post(
|
||||
"/api/auth/login", json={"username": "somebody-new", "password": "whatever"}
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
def test_login_cannot_be_talked_into_a_role(client):
|
||||
"""A `role` field in the body is not part of the schema and must not be
|
||||
honoured - the role comes from the account the password belongs to."""
|
||||
resp = client.post(
|
||||
"/api/auth/login",
|
||||
json={"username": "user", "password": TEST_USER_PASSWORD, "role": "admin"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["user"]["role"] == "user"
|
||||
|
||||
|
||||
def test_failed_logins_are_throttled(client):
|
||||
"""An exposed login endpoint must not be an unlimited password oracle."""
|
||||
for _ in range(3): # AUTH_MAX_LOGIN_ATTEMPTS in conftest
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": "wrong"}
|
||||
).status_code == 401
|
||||
|
||||
resp = client.post("/api/auth/login", json={"username": "admin", "password": "wrong"})
|
||||
assert resp.status_code == 429
|
||||
assert "Retry-After" in resp.headers
|
||||
|
||||
# The lockout must also hold against the CORRECT password, or it is trivial
|
||||
# to sidestep by guessing until you land on it.
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD}
|
||||
).status_code == 429
|
||||
|
||||
|
||||
def test_roles_endpoint_no_longer_publishes_working_passwords(client):
|
||||
"""It used to return demo_username/demo_password for both accounts."""
|
||||
body = client.get("/api/auth/roles").json()
|
||||
assert "demo_password" not in client.get("/api/auth/roles").text
|
||||
assert {r["id"] for r in body["roles"]} == {"admin", "user"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokens
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_me_returns_the_signed_in_profile(client, admin_headers):
|
||||
resp = client.get("/api/auth/me", headers=admin_headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["username"] == "admin"
|
||||
|
||||
|
||||
def test_me_requires_a_token(client):
|
||||
assert client.get("/api/auth/me").status_code == 401
|
||||
|
||||
|
||||
def test_expired_token_is_rejected(client):
|
||||
from app.infrastructure.security import create_access_token
|
||||
|
||||
token, _ = create_access_token("admin", "admin", [], ttl_minutes=-1)
|
||||
resp = client.get("/api/auth/me", headers={"Authorization": f"Bearer {token}"})
|
||||
assert resp.status_code == 401
|
||||
assert "expired" in resp.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_unsigned_alg_none_token_is_rejected(client):
|
||||
"""
|
||||
The classic JWT bypass: present a token with `alg: none` and no signature.
|
||||
decode_access_token pins algorithms to ["HS256"] instead of trusting the
|
||||
header, which is what closes it.
|
||||
"""
|
||||
forged = jwt.encode(
|
||||
{
|
||||
"sub": "admin",
|
||||
"role": "admin",
|
||||
"perms": [],
|
||||
"iss": "brand-catalog-rag",
|
||||
"iat": int(time.time()),
|
||||
"exp": int(time.time()) + 3600,
|
||||
},
|
||||
key="",
|
||||
algorithm="none",
|
||||
)
|
||||
assert client.get(
|
||||
"/api/auth/me", headers={"Authorization": f"Bearer {forged}"}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
def test_token_signed_with_the_wrong_key_is_rejected(client):
|
||||
forged = jwt.encode(
|
||||
{
|
||||
"sub": "admin",
|
||||
"role": "admin",
|
||||
"perms": [],
|
||||
"iss": "brand-catalog-rag",
|
||||
"iat": int(time.time()),
|
||||
"exp": int(time.time()) + 3600,
|
||||
},
|
||||
# At least 32 bytes: PyJWT warns about shorter HMAC keys (RFC 7518
|
||||
# §3.2), and a warning raised from a test asserting a rejection is
|
||||
# noise that hides real ones.
|
||||
key="a-wrong-key-that-is-long-enough-to-not-warn",
|
||||
algorithm="HS256",
|
||||
)
|
||||
assert client.get(
|
||||
"/api/auth/me", headers={"Authorization": f"Bearer {forged}"}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
def test_token_naming_an_unknown_role_is_rejected(client):
|
||||
"""A validly signed token still cannot invent a role."""
|
||||
from app.infrastructure.settings import AUTH_SECRET_KEY
|
||||
|
||||
token = jwt.encode(
|
||||
{
|
||||
"sub": "admin",
|
||||
"role": "superuser",
|
||||
"perms": ["everything"],
|
||||
"iss": "brand-catalog-rag",
|
||||
"iat": int(time.time()),
|
||||
"exp": int(time.time()) + 3600,
|
||||
},
|
||||
key=AUTH_SECRET_KEY,
|
||||
algorithm="HS256",
|
||||
)
|
||||
assert client.get(
|
||||
"/api/auth/me", headers={"Authorization": f"Bearer {token}"}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
def test_garbage_bearer_token_is_rejected(client):
|
||||
assert client.get(
|
||||
"/api/auth/me", headers={"Authorization": "Bearer not-even-a-jwt"}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API keys (machine consumers)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_valid_api_key_authenticates(client):
|
||||
resp = client.get("/api/auth/me", headers={"X-API-Key": TEST_API_KEY})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["role"] == "user"
|
||||
|
||||
|
||||
def test_invalid_api_key_is_rejected(client):
|
||||
assert client.get(
|
||||
"/api/auth/me", headers={"X-API-Key": "wrong-key"}
|
||||
).status_code == 401
|
||||
|
||||
|
||||
def test_api_key_is_bound_to_its_configured_role(client):
|
||||
"""The test key is a `user`, so admin-only endpoints must still refuse it."""
|
||||
resp = client.post(
|
||||
"/api/elec/admin/runs",
|
||||
json={"category": "mobiles"},
|
||||
headers={"X-API-Key": TEST_API_KEY},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Password hashing
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_password_round_trip():
|
||||
from app.infrastructure.security import hash_password, verify_password
|
||||
|
||||
encoded = hash_password("correct horse battery staple", iterations=1000)
|
||||
assert verify_password("correct horse battery staple", encoded)
|
||||
assert not verify_password("wrong", encoded)
|
||||
|
||||
|
||||
def test_hashes_are_salted():
|
||||
"""Two hashes of the same password must differ, or the digest leaks that
|
||||
two accounts share a password."""
|
||||
from app.infrastructure.security import hash_password
|
||||
|
||||
assert hash_password("same", iterations=1000) != hash_password("same", iterations=1000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["", "not-a-hash", "pbkdf2_sha256$notanint$a$b", "a$b$c$d"])
|
||||
def test_malformed_hash_fails_closed(bad):
|
||||
"""A typo in AUTH_ADMIN_PASSWORD_HASH must fail the login, not 500 the
|
||||
endpoint and hand the caller a stack trace of the credential store."""
|
||||
from app.infrastructure.security import verify_password
|
||||
|
||||
assert verify_password("anything", bad) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Why a sign-in failed
|
||||
# ---------------------------------------------------------------------------
|
||||
# The caller is told the same thing whatever went wrong - that is deliberate and
|
||||
# is pinned below. The operator is not: an account whose configured hash is
|
||||
# stale or corrupt needs a different repair from a mistyped password, and
|
||||
# collapsing the two is how a production sign-in outage stayed unexplained for a
|
||||
# day. These tests hold both halves at once: three reasons in the log, one
|
||||
# response on the wire.
|
||||
#
|
||||
# Throttle budget: conftest sets AUTH_MAX_LOGIN_ATTEMPTS=3 per (username, IP),
|
||||
# so each test below keeps `admin` to at most two attempts. Exceeding it turns a
|
||||
# 401 assertion into a 429 and reads like a code bug.
|
||||
import logging
|
||||
|
||||
from app.api.routers import auth as auth_router
|
||||
from app.infrastructure.security import hash_is_wellformed
|
||||
|
||||
_AUTH_LOGGER = "app.api.routers.auth"
|
||||
|
||||
|
||||
def test_an_unknown_username_is_logged_as_such(client, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger=_AUTH_LOGGER):
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "nobody", "password": "whatever"}
|
||||
).status_code == 401
|
||||
|
||||
assert "reason=unknown-username" in caplog.text
|
||||
# Names the setting to look at, since that is the actual repair.
|
||||
assert "AUTH_ADMIN_USERNAME" in caplog.text
|
||||
|
||||
|
||||
def test_a_wrong_password_is_logged_as_such(client, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger=_AUTH_LOGGER):
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": "not-the-password"}
|
||||
).status_code == 401
|
||||
|
||||
assert "reason=bad-password" in caplog.text
|
||||
|
||||
|
||||
def test_a_malformed_configured_hash_is_logged_as_an_error(client, caplog, monkeypatch):
|
||||
"""Not a WARNING: no password can match an unparseable digest, so this is a
|
||||
broken deployment rather than a failed guess. `_accounts()` re-reads this
|
||||
module global on every call, which is what makes it patchable here."""
|
||||
monkeypatch.setattr(auth_router, "AUTH_ADMIN_PASSWORD_HASH", "not-a-hash")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=_AUTH_LOGGER):
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD}
|
||||
).status_code == 401
|
||||
|
||||
assert "reason=malformed-hash" in caplog.text
|
||||
assert any(
|
||||
r.levelno == logging.ERROR and "malformed-hash" in r.getMessage()
|
||||
for r in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_every_failure_reason_returns_an_identical_response(client, monkeypatch):
|
||||
"""The log distinguishes them; the wire must not. If any of these three
|
||||
responses differed - by status, body, or headers - the endpoint would
|
||||
enumerate valid usernames and report its own misconfiguration to anyone."""
|
||||
unknown = client.post(
|
||||
"/api/auth/login", json={"username": "nobody", "password": "x"}
|
||||
)
|
||||
wrong = client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": "not-the-password"}
|
||||
)
|
||||
monkeypatch.setattr(auth_router, "AUTH_ADMIN_PASSWORD_HASH", "not-a-hash")
|
||||
broken = client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD}
|
||||
)
|
||||
|
||||
responses = [unknown, wrong, broken]
|
||||
assert {r.status_code for r in responses} == {401}
|
||||
assert len({r.text for r in responses}) == 1
|
||||
assert all(r.json() == {"detail": "Invalid username or password."} for r in responses)
|
||||
for r in responses:
|
||||
joined = r.text + " ".join(f"{k}:{v}" for k, v in r.headers.items())
|
||||
for leak in ("unknown-username", "bad-password", "malformed-hash", "reason"):
|
||||
assert leak not in joined
|
||||
|
||||
|
||||
def test_the_failure_log_never_carries_the_hash_or_the_password(client, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger=_AUTH_LOGGER):
|
||||
client.post(
|
||||
"/api/auth/login",
|
||||
json={"username": "admin", "password": "some-guessed-password"},
|
||||
)
|
||||
|
||||
assert "some-guessed-password" not in caplog.text
|
||||
assert TEST_ADMIN_PASSWORD not in caplog.text
|
||||
assert "pbkdf2_sha256$" not in caplog.text
|
||||
|
||||
|
||||
def test_a_malformed_hash_still_costs_a_full_password_check(client, monkeypatch):
|
||||
"""verify_password returns from an unparseable digest without doing any
|
||||
PBKDF2 work - measured at 0.16ms against 439ms for a real one. Left alone,
|
||||
an account with a corrupt hash would answer ~2700x faster than every other
|
||||
username and announce itself to anyone with a stopwatch, inverting the
|
||||
property _DUMMY_HASH exists to provide. So the work must still be paid."""
|
||||
checked = []
|
||||
real_verify = auth_router.verify_password
|
||||
|
||||
def spy(password, encoded):
|
||||
checked.append(encoded)
|
||||
return real_verify(password, encoded)
|
||||
|
||||
monkeypatch.setattr(auth_router, "AUTH_ADMIN_PASSWORD_HASH", "not-a-hash")
|
||||
monkeypatch.setattr(auth_router, "verify_password", spy)
|
||||
|
||||
assert client.post(
|
||||
"/api/auth/login", json={"username": "admin", "password": TEST_ADMIN_PASSWORD}
|
||||
).status_code == 401
|
||||
|
||||
assert len(checked) == 1, "exactly one verification per attempt"
|
||||
assert hash_is_wellformed(checked[0]), "the broken hash must not short-circuit it"
|
||||
329
backend/tests/test_auth_diagnostics.py
Normal file
329
backend/tests/test_auth_diagnostics.py
Normal file
@@ -0,0 +1,329 @@
|
||||
"""
|
||||
The credential-diagnostics surface: hash fingerprints, config provenance, and
|
||||
the `auth` block on /api/health.
|
||||
|
||||
These exist because of a real incident. Production rejected the correct admin
|
||||
password while localhost accepted it, and every observable said the app was
|
||||
healthy: /api/health was 200, CORS passed, the route table was current, and the
|
||||
only log line was `Failed sign-in for 'admin'` - which is what a user with caps
|
||||
lock on produces too. Nothing distinguished "wrong password" from "this image
|
||||
was built from a different .env.production", so there was no way to tell which
|
||||
of them it was without a shell on the box.
|
||||
|
||||
What is pinned here is therefore not a feature so much as the ability to answer
|
||||
one question from outside a container: *is this deployment running the
|
||||
credential I think it is?* The fingerprint is the answer, and these tests hold
|
||||
it to the two properties that make it usable - it identifies a hash, and it
|
||||
discloses nothing about the password behind it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.infrastructure.security import (
|
||||
api_key_fingerprint,
|
||||
auth_config_summary,
|
||||
describe_api_keys,
|
||||
describe_password_hash,
|
||||
hash_is_wellformed,
|
||||
hash_password,
|
||||
password_hash_fingerprint,
|
||||
)
|
||||
from app.infrastructure.settings import API_KEY_MIN_LENGTH, _parse_api_keys, config_source
|
||||
from tests.conftest import TEST_ADMIN_PASSWORD
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fingerprint
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_fingerprint_is_stable_for_a_given_hash():
|
||||
"""Comparing prod against local is the whole point, so the same input must
|
||||
give the same answer on both machines and across runs."""
|
||||
encoded = hash_password("whatever", iterations=1000)
|
||||
assert password_hash_fingerprint(encoded) == password_hash_fingerprint(encoded)
|
||||
|
||||
|
||||
def test_fingerprint_differs_when_the_hash_does():
|
||||
"""Including for the same password: two deployments that hashed the same
|
||||
password separately are NOT running the same credential, and a fingerprint
|
||||
that hid that would defeat the comparison."""
|
||||
a = hash_password("same-password", iterations=1000)
|
||||
b = hash_password("same-password", iterations=1000)
|
||||
assert a != b, "salts must differ"
|
||||
assert password_hash_fingerprint(a) != password_hash_fingerprint(b)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapper", ['"{}"', "'{}'", " {} ", "{}\r", "\n{}\n"])
|
||||
def test_fingerprint_ignores_quotes_and_whitespace(wrapper):
|
||||
"""A hash pasted into a platform's Environment tab arrives wrapped. It is
|
||||
the same credential, so it must fingerprint the same - otherwise the
|
||||
comparison reports a spurious mismatch in exactly the case it exists for."""
|
||||
encoded = hash_password("p", iterations=1000)
|
||||
assert password_hash_fingerprint(wrapper.format(encoded)) == password_hash_fingerprint(
|
||||
encoded
|
||||
)
|
||||
|
||||
|
||||
def test_fingerprint_discloses_no_part_of_the_hash():
|
||||
"""It is served unauthenticated, so it must be a digest OF the credential
|
||||
and not a piece of it."""
|
||||
encoded = hash_password("p", iterations=1000)
|
||||
fp = password_hash_fingerprint(encoded)
|
||||
|
||||
assert len(fp) == 12
|
||||
assert all(c in "0123456789abcdef" for c in fp)
|
||||
assert fp not in encoded
|
||||
# Nor any run of it long enough to be a foothold into salt or digest.
|
||||
for start in range(len(fp) - 5):
|
||||
assert fp[start : start + 6] not in encoded
|
||||
|
||||
|
||||
def test_absent_hash_fingerprints_as_empty():
|
||||
assert password_hash_fingerprint("") == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# describe_password_hash / hash_is_wellformed
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize(
|
||||
"bad", ["", "not-a-hash", "pbkdf2_sha256$notanint$a$b", "a$b$c$d", "bcrypt$1$a$b"]
|
||||
)
|
||||
def test_a_malformed_hash_is_reported_invalid(bad):
|
||||
"""Same inputs as test_malformed_hash_fails_closed, held against the shared
|
||||
parser - the two must agree on what 'unusable' means, since one decides the
|
||||
login and the other decides what the log calls it."""
|
||||
assert hash_is_wellformed(bad) is False
|
||||
assert describe_password_hash(bad)["valid"] is False
|
||||
|
||||
|
||||
def test_a_real_hash_is_reported_valid_with_its_iteration_count():
|
||||
described = describe_password_hash(hash_password("p", iterations=4321))
|
||||
assert described["valid"] is True
|
||||
assert described["iterations"] == 4321
|
||||
assert described["algorithm"] == "pbkdf2_sha256"
|
||||
|
||||
|
||||
def test_describe_never_returns_the_hash_itself():
|
||||
encoded = hash_password("p", iterations=1000)
|
||||
assert encoded not in str(describe_password_hash(encoded))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config provenance
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_config_source_reports_process_env_for_harness_supplied_values():
|
||||
"""conftest writes the AUTH_* values into os.environ before app.main is
|
||||
imported - which is structurally the same thing a deployment platform's
|
||||
Environment tab does. That this reads back as 'process-env' is the
|
||||
executable proof that an override is detectable at all."""
|
||||
assert config_source("AUTH_ADMIN_PASSWORD_HASH") == "process-env"
|
||||
assert config_source("AUTH_ADMIN_USERNAME") == "process-env"
|
||||
|
||||
|
||||
def test_config_source_reports_default_for_something_never_set():
|
||||
assert config_source("AUTH_NOT_A_REAL_SETTING_XYZ") == "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api/health
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_health_reports_the_effective_auth_configuration(client):
|
||||
auth = client.get("/api/health").json()["auth"]
|
||||
|
||||
assert auth["enabled"] is True
|
||||
assert auth["allow_any_login"] is False
|
||||
assert auth["admin_username"] == "admin"
|
||||
assert auth["password_hash_valid"] is True
|
||||
assert auth["password_hash_iterations"] == 20_000 # conftest._hash
|
||||
assert auth["password_hash_fingerprint"] == auth_config_summary()[
|
||||
"password_hash_fingerprint"
|
||||
]
|
||||
assert auth["password_hash_source"] == "process-env"
|
||||
|
||||
|
||||
def test_health_never_exposes_a_hash_or_a_password(client):
|
||||
"""The leak canary on an unauthenticated endpoint. A configured digest
|
||||
always contains '$' separators; a password would appear verbatim."""
|
||||
body = client.get("/api/health").text
|
||||
|
||||
assert TEST_ADMIN_PASSWORD not in body
|
||||
assert "pbkdf2_sha256$" not in body
|
||||
assert "$" not in body
|
||||
|
||||
|
||||
def test_health_stays_ok_shaped_when_auth_is_misconfigured(client, monkeypatch):
|
||||
"""An unusable credential must NOT flip `status` to degraded: the container
|
||||
healthcheck and the frontend's connectivity banner both read that field, so
|
||||
doing so would turn a login problem into an outage and a misleading "database
|
||||
unreachable" banner. The signal belongs in auth.password_hash_valid."""
|
||||
from app.api.routers import health as health_router
|
||||
|
||||
monkeypatch.setattr(
|
||||
health_router, "auth_config_summary", lambda: {**auth_config_summary(),
|
||||
"password_hash_valid": False}
|
||||
)
|
||||
body = client.get("/api/health").json()
|
||||
|
||||
assert body["auth"]["password_hash_valid"] is False
|
||||
assert body["status"] in {"ok", "degraded"} # decided by db/ollama only
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API keys
|
||||
# ---------------------------------------------------------------------------
|
||||
# Same incident, one layer out. A key added to .env.production and then merely
|
||||
# restarted into a running container is absent from the process, because the
|
||||
# Dockerfile copies that file in at BUILD time - and from outside, an undeployed
|
||||
# key and a wrong key are both just a 401. These pin the ability to tell them
|
||||
# apart without anyone sending the secret to find out.
|
||||
|
||||
_GOOD_SECRET = "cs3JwvApS5Je_Qfe1sNYq6YtUBDDeqp4OEgy2_41sQg"
|
||||
|
||||
|
||||
def test_api_key_fingerprint_is_stable_and_hex():
|
||||
fp = api_key_fingerprint("partner", _GOOD_SECRET)
|
||||
|
||||
assert fp == api_key_fingerprint("partner", _GOOD_SECRET)
|
||||
assert len(fp) == 12
|
||||
assert all(c in "0123456789abcdef" for c in fp)
|
||||
|
||||
|
||||
def test_api_key_fingerprint_differs_when_the_secret_does():
|
||||
assert api_key_fingerprint("partner", _GOOD_SECRET) != api_key_fingerprint(
|
||||
"partner", _GOOD_SECRET[:-1] + "X"
|
||||
)
|
||||
|
||||
|
||||
def test_api_key_fingerprint_separates_consumers_sharing_a_secret():
|
||||
"""The name is mixed in, so two consumers mistakenly issued the same secret
|
||||
do not report the same fingerprint - which would hide the mistake behind the
|
||||
very field meant to reveal it."""
|
||||
assert api_key_fingerprint("console-a", _GOOD_SECRET) != api_key_fingerprint(
|
||||
"console-b", _GOOD_SECRET
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapper", ["{}", "'{}'", '"{}"', " {} "])
|
||||
def test_api_key_fingerprint_ignores_quotes_and_whitespace(wrapper):
|
||||
"""A value pasted into a deployment platform's Environment tab arrives
|
||||
wrapped often enough that settings strips it; the fingerprint must agree,
|
||||
or comparing two ends reports a mismatch that is not real."""
|
||||
assert api_key_fingerprint("partner", wrapper.format(_GOOD_SECRET)) == (
|
||||
api_key_fingerprint("partner", _GOOD_SECRET)
|
||||
)
|
||||
|
||||
|
||||
def test_api_key_fingerprint_discloses_no_part_of_the_secret():
|
||||
"""Served unauthenticated, so it must be a digest OF the key, not a piece."""
|
||||
fp = api_key_fingerprint("partner", _GOOD_SECRET)
|
||||
|
||||
assert fp not in _GOOD_SECRET
|
||||
for start in range(len(fp) - 5):
|
||||
assert fp[start : start + 6] not in _GOOD_SECRET
|
||||
|
||||
|
||||
def test_absent_secret_fingerprints_as_empty():
|
||||
assert api_key_fingerprint("partner", "") == ""
|
||||
|
||||
|
||||
def test_describe_api_keys_is_sorted_by_name(monkeypatch):
|
||||
"""API_KEYS is keyed by secret, whose order says nothing. Sorting is what
|
||||
lets two deployments' output be diffed line for line."""
|
||||
from app.infrastructure import security
|
||||
|
||||
monkeypatch.setattr(
|
||||
security, "API_KEYS",
|
||||
{_GOOD_SECRET: ("zulu", "user"), _GOOD_SECRET[::-1]: ("alpha", "admin")},
|
||||
)
|
||||
|
||||
assert [k["name"] for k in describe_api_keys()] == ["alpha", "zulu"]
|
||||
assert [k["role"] for k in describe_api_keys()] == ["admin", "user"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API keys on /api/health
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_health_reports_the_keys_this_deployment_actually_loaded(client):
|
||||
"""Against the harness's own API_KEYS, not a monkeypatched one - this is the
|
||||
end-to-end wiring from settings through the summary to the response body."""
|
||||
from tests.conftest import TEST_API_KEY
|
||||
|
||||
auth = client.get("/api/health").json()["auth"]
|
||||
|
||||
assert auth["api_keys_count"] == 1
|
||||
assert auth["api_keys"] == [{
|
||||
"name": "test-machine",
|
||||
"role": "user",
|
||||
"fingerprint": api_key_fingerprint("test-machine", TEST_API_KEY),
|
||||
}]
|
||||
|
||||
|
||||
def test_health_reports_an_empty_list_when_no_keys_are_configured(client, monkeypatch):
|
||||
"""The state production was in while the colleague's console got 401s: auth
|
||||
enabled, admin login working, and not one machine consumer deployed."""
|
||||
from app.infrastructure import security
|
||||
|
||||
monkeypatch.setattr(security, "API_KEYS", {})
|
||||
auth = client.get("/api/health").json()["auth"]
|
||||
|
||||
assert auth["api_keys_count"] == 0
|
||||
assert auth["api_keys"] == []
|
||||
|
||||
|
||||
def test_health_names_configured_keys_and_fingerprints_them(client, monkeypatch):
|
||||
from app.infrastructure import security
|
||||
|
||||
monkeypatch.setattr(security, "API_KEYS", {_GOOD_SECRET: ("colleague-console", "admin")})
|
||||
auth = client.get("/api/health").json()["auth"]
|
||||
|
||||
assert auth["api_keys_count"] == 1
|
||||
assert auth["api_keys"] == [{
|
||||
"name": "colleague-console",
|
||||
"role": "admin",
|
||||
"fingerprint": api_key_fingerprint("colleague-console", _GOOD_SECRET),
|
||||
}]
|
||||
|
||||
|
||||
def test_health_never_exposes_an_api_key_secret(client, monkeypatch):
|
||||
"""The leak canary, extended to machine credentials."""
|
||||
from app.infrastructure import security
|
||||
|
||||
monkeypatch.setattr(security, "API_KEYS", {_GOOD_SECRET: ("colleague-console", "admin")})
|
||||
body = client.get("/api/health").text
|
||||
|
||||
assert _GOOD_SECRET not in body
|
||||
for start in range(0, len(_GOOD_SECRET) - 7):
|
||||
assert _GOOD_SECRET[start : start + 8] not in body
|
||||
|
||||
|
||||
def test_health_reports_where_the_keys_came_from(client):
|
||||
"""Which of the two config sources won. Unlike the admin hash - where
|
||||
"process-env" flags a stale Environment tab shadowing the image - API_KEYS is
|
||||
deliberately supplied by that tab, so "process-env" is the expected value in
|
||||
production and "env-file" would mean the tab entry has gone missing."""
|
||||
auth = client.get("/api/health").json()["auth"]
|
||||
|
||||
assert auth["api_keys_source"] in {"process-env", "env-file", "default"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_parse_api_keys_accepts_a_generated_secret():
|
||||
parsed = _parse_api_keys(f"partner:admin:{_GOOD_SECRET}")
|
||||
|
||||
assert parsed == {_GOOD_SECRET: ("partner", "admin")}
|
||||
|
||||
|
||||
def test_parse_api_keys_rejects_a_secret_too_short_to_fingerprint_safely():
|
||||
"""A raw key carries no salt, so publishing its digest is only safe while the
|
||||
key itself is unguessable offline. A hand-picked one must be refused at
|
||||
startup rather than quietly fingerprinted onto a public endpoint."""
|
||||
with pytest.raises(RuntimeError, match="at least"):
|
||||
_parse_api_keys("partner:admin:changeme")
|
||||
|
||||
|
||||
def test_parse_api_keys_length_limit_admits_the_documented_generator():
|
||||
import secrets as _secrets
|
||||
|
||||
assert len(_secrets.token_urlsafe(32)) >= API_KEY_MIN_LENGTH
|
||||
129
backend/tests/test_cors_origins.py
Normal file
129
backend/tests/test_cors_origins.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""The browser is the only client that enforces CORS, and no test here is one.
|
||||
|
||||
THE FAILURE THIS FILE EXISTS FOR
|
||||
--------------------------------
|
||||
The merchant console called `https://mcp.nearle.ai.in/api/...` from Chrome and got
|
||||
`TypeError: Failed to fetch` for a whole day. The same URL under curl returned 200
|
||||
with the right body, so every server-side check - ours and theirs - passed. The
|
||||
response carried `Access-Control-Allow-Credentials: true` and no
|
||||
`Access-Control-Allow-Origin`, and the browser discarded it before any code saw it.
|
||||
|
||||
The cause was not missing middleware. `CORSMiddleware` was installed and correct;
|
||||
the console's origin was simply not in `API_CORS_ORIGINS`, so Starlette declined to
|
||||
echo the header. Nothing in the server log looked wrong: a healthy 200, every time.
|
||||
|
||||
That is why the deployed allowlist is asserted as DATA below. A unit test cannot
|
||||
fail the way a browser fails, so the next best thing is to pin the one value whose
|
||||
absence produces a silent, total outage for a first-party client.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ENV_PRODUCTION = Path(__file__).resolve().parents[1] / ".env.production"
|
||||
|
||||
# Every first-party browser client of this API. A new one added to the console
|
||||
# and not added here is the bug above, repeated.
|
||||
REQUIRED_ORIGINS = (
|
||||
"https://app.nearledaily.com", # merchant console
|
||||
"http://localhost:3100", # merchant console, local development
|
||||
"https://catalogue.nearle.ai.in", # catalogue frontend
|
||||
)
|
||||
|
||||
|
||||
def _deployed_origins():
|
||||
"""The allowlist the image ships with.
|
||||
|
||||
Read from the file rather than from `settings`, because the test process has
|
||||
its own environment - importing the setting would assert on the developer's
|
||||
machine instead of on what gets deployed.
|
||||
"""
|
||||
if not ENV_PRODUCTION.exists():
|
||||
pytest.skip(".env.production is not present in this checkout")
|
||||
for line in ENV_PRODUCTION.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("API_CORS_ORIGINS="):
|
||||
value = line.split("=", 1)[1]
|
||||
return [o.strip() for o in value.split(",") if o.strip()]
|
||||
return []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("origin", REQUIRED_ORIGINS)
|
||||
def test_every_first_party_browser_client_is_allowed(origin):
|
||||
assert origin in _deployed_origins(), (
|
||||
"%s is missing from API_CORS_ORIGINS in .env.production. Browser calls "
|
||||
"from it fail as an opaque 'Failed to fetch' while curl still returns "
|
||||
"200, so nothing server-side will catch this." % origin
|
||||
)
|
||||
|
||||
|
||||
def test_the_allowlist_is_not_a_wildcard():
|
||||
"""A wildcard would disable `allow_credentials` (see the guard in main.py),
|
||||
silently breaking any authenticated browser call to this API."""
|
||||
assert "*" not in _deployed_origins()
|
||||
|
||||
|
||||
def test_every_origin_is_a_bare_scheme_and_host():
|
||||
"""An Origin header is scheme + host + port, never a path and never a
|
||||
trailing slash. `https://app.nearledaily.com/` does not match and fails
|
||||
exactly as if it were absent."""
|
||||
for origin in _deployed_origins():
|
||||
assert re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:\d+)?", origin), origin
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The middleware itself
|
||||
# ---------------------------------------------------------------------------
|
||||
# These run against whatever origins the TEST environment carries, so they pin
|
||||
# the behaviour rather than the deployed list: an allowed origin is echoed, an
|
||||
# unknown one is not, and a preflight from an unknown origin is refused.
|
||||
|
||||
def _allowed_origin():
|
||||
from app.infrastructure.settings import API_CORS_ORIGINS
|
||||
if not API_CORS_ORIGINS:
|
||||
pytest.skip("no CORS origins configured in the test environment")
|
||||
return API_CORS_ORIGINS[0]
|
||||
|
||||
|
||||
def test_an_allowed_origin_is_echoed_back(client):
|
||||
origin = _allowed_origin()
|
||||
|
||||
response = client.get("/api/health", headers={"Origin": origin})
|
||||
|
||||
assert response.headers.get("access-control-allow-origin") == origin
|
||||
|
||||
|
||||
def test_an_unknown_origin_gets_no_header_at_all(client):
|
||||
"""The response still returns 200 with a correct body - which is why this is
|
||||
invisible everywhere except a browser."""
|
||||
response = client.get("/api/health", headers={"Origin": "https://not-listed.example"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "access-control-allow-origin" not in response.headers
|
||||
|
||||
|
||||
def test_a_preflight_from_an_allowed_origin_succeeds(client):
|
||||
response = client.options(
|
||||
"/api/health",
|
||||
headers={"Origin": _allowed_origin(),
|
||||
"Access-Control-Request-Method": "GET",
|
||||
"Access-Control-Request-Headers": "content-type"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("access-control-allow-origin") == _allowed_origin()
|
||||
|
||||
|
||||
def test_a_preflight_from_an_unknown_origin_is_refused(client):
|
||||
"""Starlette answers 400 here. It reads as a malformed request in the log,
|
||||
which is how this was mistaken for a second, unrelated bug."""
|
||||
response = client.options(
|
||||
"/api/health",
|
||||
headers={"Origin": "https://not-listed.example",
|
||||
"Access-Control-Request-Method": "GET"},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
239
backend/tests/test_elec_database.py
Normal file
239
backend/tests/test_elec_database.py
Normal file
@@ -0,0 +1,239 @@
|
||||
"""Database tests against the local electronics_catalog_test database:
|
||||
constraints, append-only history, verification rule, views and the API.
|
||||
Skipped when the local Postgres container is not running."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
|
||||
from app.electronics.collector import Collector, RunOptions
|
||||
from app.electronics.db import repository as repo
|
||||
from app.electronics.db.connection import connect
|
||||
from app.electronics.models import Listing
|
||||
from app.electronics.normalise.title_parser import parse_title, variant_key
|
||||
|
||||
|
||||
def _listing(site: str, sku: str, title: str, *, price=None, source_type="search_snippet",
|
||||
evidence=None) -> Listing:
|
||||
p = parse_title(title, "mobiles")
|
||||
l = Listing(site_domain=site, source_sku=sku, source_url=f"https://www.{site}/p/{sku}",
|
||||
source_type=source_type, brand_slug=p.brand.brand_slug, category="mobiles", title=title,
|
||||
evidence_text=evidence or f"{title} ₹{price}", confidence=0.5, parser="test",
|
||||
model=p.model, ram_gb=p.ram_gb, storage_gb=p.storage_gb, price=price)
|
||||
l.model_norm, l.variant_key = p.model_norm, variant_key(p, "mobiles")
|
||||
return l
|
||||
|
||||
|
||||
def test_settings_guard_refuses_remote_database():
|
||||
from app.infrastructure.settings import _guard_local_database
|
||||
|
||||
with pytest.raises(RuntimeError, match="not a local host"):
|
||||
_guard_local_database("31.97.228.132", "electronics_catalog")
|
||||
with pytest.raises(RuntimeError, match="expected 'electronics_catalog'"):
|
||||
_guard_local_database("localhost", "pgvector")
|
||||
_guard_local_database("localhost", "electronics_catalog")
|
||||
|
||||
|
||||
def test_settings_guard_remote_opt_in_is_exact():
|
||||
from app.infrastructure.settings import _guard_local_database
|
||||
|
||||
prod = dict(remote_hosts=frozenset({"31.97.228.132"}), remote_names=frozenset({"loyalycatalogue"}))
|
||||
# Listed host + name, with the flag on: allowed.
|
||||
_guard_local_database("31.97.228.132", "loyalycatalogue", allow_remote=True, **prod)
|
||||
# Same host without the flag: still refused.
|
||||
with pytest.raises(RuntimeError, match="not a local host"):
|
||||
_guard_local_database("31.97.228.132", "loyalycatalogue", **prod)
|
||||
# Flag on but another host or another database: refused.
|
||||
with pytest.raises(RuntimeError, match="not a local host"):
|
||||
_guard_local_database("10.0.0.9", "loyalycatalogue", allow_remote=True, **prod)
|
||||
with pytest.raises(RuntimeError, match="not in ELEC_REMOTE_DB_NAMES"):
|
||||
_guard_local_database("31.97.228.132", "pgvector", allow_remote=True, **prod)
|
||||
|
||||
|
||||
def test_constraints_reject_fabricated_rows(db):
|
||||
ids = repo.id_maps()
|
||||
with connect() as conn:
|
||||
base = dict(site=ids["site"]["croma.com"], brand=ids["brand"]["samsung"], cat=ids["category"]["mobiles"])
|
||||
bad_rows = [
|
||||
("no URL", "INSERT INTO elec.source_listing (site_id, source_sku, source_url, source_type, brand_id, "
|
||||
"category_id, title, evidence_text, confidence, parser) VALUES (%(site)s,'x','not-a-url',"
|
||||
"'search_snippet',%(brand)s,%(cat)s,'t','e',0.5,'t')"),
|
||||
("no evidence", "INSERT INTO elec.source_listing (site_id, source_sku, source_url, source_type, brand_id, "
|
||||
"category_id, title, evidence_text, confidence, parser) VALUES (%(site)s,'x','https://a.in/x',"
|
||||
"'search_snippet',%(brand)s,%(cat)s,'t','',0.5,'t')"),
|
||||
("absurd price", "INSERT INTO elec.source_listing (site_id, source_sku, source_url, source_type, brand_id, "
|
||||
"category_id, title, evidence_text, confidence, parser, price) VALUES (%(site)s,'x','https://a.in/x',"
|
||||
"'search_snippet',%(brand)s,%(cat)s,'t','e',0.5,'t', 5)"),
|
||||
("non-INR", "INSERT INTO elec.source_listing (site_id, source_sku, source_url, source_type, brand_id, "
|
||||
"category_id, title, evidence_text, confidence, parser, currency) VALUES (%(site)s,'x','https://a.in/x',"
|
||||
"'search_snippet',%(brand)s,%(cat)s,'t','e',0.5,'t','USD')"),
|
||||
("pincode claim", "INSERT INTO elec.source_listing (site_id, source_sku, source_url, source_type, brand_id, "
|
||||
"category_id, title, evidence_text, confidence, parser, pincode_applied) VALUES (%(site)s,'x',"
|
||||
"'https://a.in/x','search_snippet',%(brand)s,%(cat)s,'t','e',0.5,'t',TRUE)"),
|
||||
("bad grade", "UPDATE elec.site SET probe_outcome = 'D' WHERE id = %(site)s"),
|
||||
]
|
||||
for label, sql in bad_rows:
|
||||
with pytest.raises(psycopg.errors.CheckViolation):
|
||||
with conn.transaction():
|
||||
conn.execute(sql, base)
|
||||
pytest.fail(label)
|
||||
|
||||
|
||||
def test_listing_validation_rejects_missing_evidence():
|
||||
l = _listing("croma.com", "1", "Samsung Galaxy S24 5G (8GB RAM, 256GB)", price=Decimal(74999))
|
||||
l.evidence_text = " "
|
||||
with pytest.raises(ValueError):
|
||||
l.validate()
|
||||
|
||||
|
||||
def test_price_history_is_append_only(db):
|
||||
ids = repo.id_maps()
|
||||
lid = repo.upsert_listing(_listing("croma.com", "1", "Samsung Galaxy S24 5G (8GB RAM, 256GB)",
|
||||
price=Decimal(74999)), ids, None)
|
||||
with connect() as conn:
|
||||
with pytest.raises(psycopg.errors.RaiseException):
|
||||
conn.execute("UPDATE elec.price_history SET price = 1000 WHERE listing_id = %s", (lid,))
|
||||
|
||||
|
||||
def test_verification_needs_two_sites_including_a_retailer(db):
|
||||
c = Collector.__new__(Collector) # use store() without network setup
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids = repo.id_maps()
|
||||
c.run_id = None
|
||||
c._touched_products = {}
|
||||
from app.electronics.collector import RunStats
|
||||
c.stats = RunStats()
|
||||
|
||||
title = "Samsung Galaxy S24 5G (Onyx Black, 8GB RAM, 256GB Storage)"
|
||||
c.store(_listing("amazon.in", "B0CS5XW6TN", title, price=Decimal(74999)))
|
||||
assert repo.refresh_verification() == {"unverified": 1}
|
||||
with connect() as conn:
|
||||
assert conn.execute("SELECT count(*) n FROM elec.v_brand_catalog").fetchone()["n"] == 0
|
||||
|
||||
# A second, different platform listing the same variant (written its own way).
|
||||
c.store(_listing("flipkart.com", "itm1", "SAMSUNG Galaxy S24 5G (Onyx Black, 256 GB) (8 GB RAM)",
|
||||
price=Decimal(72999)))
|
||||
assert repo.refresh_verification() == {"verified": 1}
|
||||
with connect() as conn:
|
||||
row = conn.execute("SELECT * FROM elec.v_brand_catalog").fetchone()
|
||||
offers = conn.execute("SELECT site FROM elec.v_product_availability ORDER BY price").fetchall()
|
||||
assert row["platform_count"] == 2 and row["best_price"] == Decimal("72999.00")
|
||||
assert [o["site"] for o in offers] == ["Flipkart", "Amazon.in"]
|
||||
|
||||
|
||||
def test_scraped_listing_is_not_downgraded_by_a_snippet(db):
|
||||
ids = repo.id_maps()
|
||||
title = "Samsung Galaxy S24 5G (8GB RAM, 256GB)"
|
||||
scraped = _listing("croma.com", "303838", title, price=Decimal(74999), source_type="scraped_page",
|
||||
evidence='{"price": "74999"}')
|
||||
lid = repo.upsert_listing(scraped, ids, None)
|
||||
snippet = _listing("croma.com", "303838", title, price=Decimal(69999))
|
||||
assert repo.upsert_listing(snippet, ids, None) == lid
|
||||
with connect() as conn:
|
||||
row = conn.execute("SELECT price, source_type FROM elec.source_listing WHERE id = %s", (lid,)).fetchone()
|
||||
assert (row["price"], row["source_type"]) == (Decimal("74999.00"), "scraped_page")
|
||||
|
||||
|
||||
def test_catalogue_api_serves_verified_products(db, client):
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids, c.run_id, c._touched_products = repo.id_maps(), None, {}
|
||||
from app.electronics.collector import RunStats
|
||||
c.stats = RunStats()
|
||||
c.store(_listing("amazon.in", "B0CS5XW6TN", "Samsung Galaxy S24 5G (8GB RAM, 256GB)", price=Decimal(74999)))
|
||||
c.store(_listing("poorvika.com", "samsung-galaxy-s24", "Samsung Galaxy S24 5G (8GB RAM, 256GB)",
|
||||
price=Decimal(73999)))
|
||||
repo.refresh_verification()
|
||||
|
||||
brands = client.get("/api/elec/brands", params={"category": "mobiles"}).json()
|
||||
assert brands[0]["brand_slug"] == "samsung" and brands[0]["product_count"] == 1
|
||||
listing = client.get("/api/elec/products", params={"category": "mobiles", "tn_only": True}).json()
|
||||
assert listing["total"] == 1
|
||||
product = listing["products"][0]
|
||||
assert product["best_price"] == "73999.00" and product["sold_by_tn_retailer"] is True
|
||||
detail = client.get(f"/api/elec/products/{product['product_id']}").json()
|
||||
assert {o["site"] for o in detail["offers"]} == {"Amazon.in", "Poorvika"}
|
||||
assert all(o["source_url"].startswith("https://") for o in detail["offers"])
|
||||
assert client.get("/api/elec/products/999999").status_code == 404
|
||||
|
||||
|
||||
def test_implausible_snippet_prices_are_flagged(db):
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids, c.run_id, c._touched_products = repo.id_maps(), None, {}
|
||||
from app.electronics.collector import RunStats
|
||||
c.stats = RunStats()
|
||||
title = "Samsung Galaxy S24 5G (8GB RAM, 256GB)"
|
||||
c.store(_listing("poorvika.com", "s24", title, price=Decimal(74999), source_type="scraped_page",
|
||||
evidence='{"price": "74999"}'))
|
||||
c.store(_listing("flipkart.com", "itm1", title, price=Decimal(129999))) # 73% above the page price
|
||||
c.store(_listing("amazon.in", "B0X", title, price=Decimal(72999))) # plausible
|
||||
repo.refresh_verification()
|
||||
with connect() as conn:
|
||||
flagged = {r["site"]: r["price_outlier"] for r in conn.execute(
|
||||
"SELECT site, price_outlier FROM elec.v_product_availability")}
|
||||
best = conn.execute("SELECT price FROM elec.v_best_price").fetchone()["price"]
|
||||
assert flagged == {"Poorvika": False, "Flipkart": True, "Amazon.in": False}
|
||||
assert best == Decimal("74999.00") # page price preferred; the outlier never wins
|
||||
|
||||
|
||||
def test_google_price_lookup_only_trusts_the_same_page(db, monkeypatch):
|
||||
from app.electronics import price_lookup
|
||||
from app.electronics.search.providers import SearchHit
|
||||
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids, c.run_id, c._touched_products = repo.id_maps(), None, {}
|
||||
from app.electronics.collector import RunStats
|
||||
c.stats = RunStats()
|
||||
title = "Samsung Galaxy S24 5G (8GB RAM, 256GB)"
|
||||
c.store(_listing("amazon.in", "B0CS5XW6TN", title)) # no price yet
|
||||
c.store(_listing("poorvika.com", "s24", title, price=Decimal(74999), source_type="scraped_page",
|
||||
evidence='{"price": "74999"}'))
|
||||
offer = {"price": "72999", "currency": "INR", "availability": "InStock", "raw": {"price": "72999"}}
|
||||
hits = [
|
||||
# Another product page on the same site, with a price: must be ignored.
|
||||
SearchHit("https://www.amazon.in/other/dp/B0OTHER123", "Samsung Galaxy S24 Ultra", "", "google", 0,
|
||||
offer={**offer, "price": "129999"}),
|
||||
SearchHit("https://www.amazon.in/Samsung-Galaxy/dp/B0CS5XW6TN/ref=x", title, "", "google", 1, offer=offer),
|
||||
]
|
||||
|
||||
class FakeGoogle:
|
||||
enabled, error = True, None
|
||||
|
||||
class FakeEngine:
|
||||
def __init__(self, budget):
|
||||
self.google = FakeGoogle()
|
||||
|
||||
def text(self, query, max_results=10, providers="default"):
|
||||
assert providers == "google"
|
||||
return hits
|
||||
|
||||
monkeypatch.setattr(price_lookup, "SearchEngine", FakeEngine)
|
||||
stats = price_lookup.lookup_prices(limit=5)
|
||||
assert stats["priced"] == 1
|
||||
with connect() as conn:
|
||||
row = conn.execute("SELECT price, parser, evidence_text FROM elec.source_listing WHERE source_sku = 'B0CS5XW6TN'").fetchone()
|
||||
assert row["price"] == Decimal("72999.00") and row["parser"].endswith("pagemap") and "72999" in row["evidence_text"]
|
||||
|
||||
|
||||
def test_google_disables_itself_on_a_rejected_key(monkeypatch):
|
||||
import app.electronics.search.providers as providers
|
||||
|
||||
calls = []
|
||||
|
||||
class Resp:
|
||||
status_code = 403
|
||||
text = "forbidden"
|
||||
|
||||
def json(self):
|
||||
return {"error": {"message": "This project does not have the access to Custom Search JSON API."}}
|
||||
|
||||
monkeypatch.setattr(providers, "USE_GOOGLE_CSE", True)
|
||||
monkeypatch.setattr(providers.requests, "get", lambda *a, **k: calls.append(1) or Resp())
|
||||
g = providers.GoogleCseProvider(quota_left=lambda: 100)
|
||||
g._pacer.interval = 0
|
||||
assert g.text("q") is None and g.text("q2") is None
|
||||
assert calls == [1] and not g.enabled and "Custom Search JSON API" in g.error
|
||||
308
backend/tests/test_elec_extract_and_net.py
Normal file
308
backend/tests/test_elec_extract_and_net.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""Offline tests: JSON-LD/HTML extraction, the polite HTTP client (robots.txt,
|
||||
pacing, circuit breaker, CAPTCHA detection), the site probe grading and the
|
||||
matcher. Network access is replaced with httpx.MockTransport."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
|
||||
from app.electronics.extract.html_fallback import extract_page
|
||||
from app.electronics.extract.jsonld import extract_products
|
||||
from app.electronics.match.matcher import decide
|
||||
from app.electronics.net.breaker import CircuitBreaker
|
||||
from app.electronics.net.polite_client import PoliteClient
|
||||
from app.electronics.probe.site_probe import grade_page
|
||||
from app.electronics.reference import load_reference, site_for_url
|
||||
from app.electronics.collector import is_product_url, source_sku
|
||||
|
||||
PRODUCT_PAGE = """
|
||||
<html><head>
|
||||
<meta property="og:title" content="Samsung Galaxy S24 5G (8GB RAM, 256GB, Onyx Black)">
|
||||
<meta property="og:image" content="https://media.example-cdn.in/s24.jpg">
|
||||
<script type="application/ld+json">
|
||||
{"@context":"https://schema.org","@type":"Product","name":"Samsung Galaxy S24 5G (8GB RAM, 256GB, Onyx Black)",
|
||||
"sku":"303838","mpn":"SM-S921BZKCINS","gtin13":"8806095467245","brand":{"@type":"Brand","name":"Samsung"},
|
||||
"image":["https://media.example-cdn.in/s24-1.jpg","https://media.example-cdn.in/s24-2.jpg"],
|
||||
"aggregateRating":{"@type":"AggregateRating","ratingValue":"4.4","reviewCount":"120"},
|
||||
"additionalProperty":[{"@type":"PropertyValue","name":"Battery Capacity","value":"4000 mAh"}],
|
||||
"offers":{"@type":"Offer","price":"74999.00","priceCurrency":"INR","availability":"https://schema.org/InStock"}}
|
||||
</script></head><body><h1>Samsung Galaxy S24</h1>
|
||||
<table><tr><th>RAM</th><td>8 GB</td></tr><tr><th>Internal Storage</th><td>256 GB</td></tr></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
HTML_ONLY_PAGE = """
|
||||
<html><head><meta property="og:title" content="Apple iPhone 15 (128 GB) - Black">
|
||||
<meta property="product:price:amount" content="69900"><meta property="product:price:currency" content="INR">
|
||||
</head><body><p>No cost EMI from ₹3,300/month</p>
|
||||
<table><tr><td>Display Size</td><td>6.1 inch</td></tr></table></body></html>
|
||||
"""
|
||||
|
||||
|
||||
def test_jsonld_product():
|
||||
[p] = extract_products(PRODUCT_PAGE)
|
||||
assert p["price"] == Decimal("74999.00") and p["currency"] == "INR"
|
||||
assert p["in_stock"] is True and p["availability"] == "InStock"
|
||||
assert p["gtin"] == "8806095467245" and p["mpn"] == "SM-S921BZKCINS"
|
||||
assert p["images"][0].endswith("s24-1.jpg")
|
||||
assert p["properties"] == {"Battery Capacity": "4000 mAh"}
|
||||
assert "74999" in p["evidence"]
|
||||
|
||||
|
||||
def test_jsonld_aggregate_offer_and_graph():
|
||||
html = """<script type="application/ld+json">{"@graph":[{"@type":"WebPage"},
|
||||
{"@type":["Product"],"name":"Lenovo IdeaPad Slim 3","offers":{"@type":"AggregateOffer",
|
||||
"lowPrice":"48990","highPrice":"52990","priceCurrency":"INR"}}]}</script>"""
|
||||
[p] = extract_products(html)
|
||||
assert p["price"] == Decimal("48990")
|
||||
|
||||
|
||||
def test_html_meta_price_only_from_markup():
|
||||
page = extract_page(HTML_ONLY_PAGE)
|
||||
assert page["price"] == Decimal("69900")
|
||||
assert page["properties"] == {"Display Size": "6.1 inch"}
|
||||
# The EMI amount in body text is never read as a price.
|
||||
assert "3300" not in page["evidence"].replace(",", "")
|
||||
|
||||
|
||||
def test_grade_page():
|
||||
assert grade_page(PRODUCT_PAGE)["jsonld_priced"] == 1
|
||||
g = grade_page(HTML_ONLY_PAGE)
|
||||
assert g["jsonld_priced"] == 0 and g["meta_price"] and g["has_title"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Polite client
|
||||
# ---------------------------------------------------------------------------
|
||||
def _client(handler, **kw) -> PoliteClient:
|
||||
kw.setdefault("sleep", lambda s: None)
|
||||
return PoliteClient(transport=httpx.MockTransport(handler), min_interval=kw.pop("min_interval", 0), **kw)
|
||||
|
||||
|
||||
def test_robots_disallow_is_obeyed():
|
||||
requested = []
|
||||
|
||||
def handler(req: httpx.Request):
|
||||
requested.append(req.url.path)
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /p/\n")
|
||||
return httpx.Response(200, text=PRODUCT_PAGE, headers={"content-type": "text/html"})
|
||||
|
||||
with _client(handler) as c:
|
||||
res = c.get("https://shop.example.in/p/123")
|
||||
assert res.outcome == "robots_disallowed"
|
||||
assert requested == ["/robots.txt"] # the product page itself was never requested
|
||||
|
||||
|
||||
def test_unreadable_robots_means_disallowed():
|
||||
def handler(req):
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(500)
|
||||
return httpx.Response(200, text="ok", headers={"content-type": "text/html"})
|
||||
|
||||
with _client(handler) as c:
|
||||
assert c.get("https://shop.example.in/p/1").outcome == "robots_disallowed"
|
||||
|
||||
|
||||
def test_missing_robots_allows():
|
||||
def handler(req):
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, text=PRODUCT_PAGE, headers={"content-type": "text/html"})
|
||||
|
||||
with _client(handler) as c:
|
||||
res = c.get("https://shop.example.in/p/1")
|
||||
assert res.ok and "Galaxy S24" in res.text
|
||||
|
||||
|
||||
def test_429_trips_breaker_and_stops_further_requests():
|
||||
calls = []
|
||||
|
||||
def handler(req):
|
||||
calls.append(req.url.path)
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(429)
|
||||
|
||||
trips = []
|
||||
breaker = CircuitBreaker(on_trip=lambda host, reason, until: trips.append(host))
|
||||
with _client(handler, breaker=breaker) as c:
|
||||
assert c.get("https://shop.example.in/p/1").outcome == "blocked"
|
||||
assert c.get("https://shop.example.in/p/2").outcome == "breaker_open"
|
||||
assert trips == ["shop.example.in"]
|
||||
assert calls == ["/robots.txt", "/p/1"] # nothing after the block
|
||||
|
||||
|
||||
def test_captcha_page_trips_breaker():
|
||||
def handler(req):
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, text="<html>Robot Check. Enter the characters you see</html>",
|
||||
headers={"content-type": "text/html"})
|
||||
|
||||
with _client(handler) as c:
|
||||
assert c.get("https://shop.example.in/p/1").outcome == "captcha"
|
||||
assert c.breaker.is_open("shop.example.in")
|
||||
|
||||
|
||||
def test_requests_to_one_host_are_paced():
|
||||
now = [0.0]
|
||||
slept = []
|
||||
|
||||
def sleep(s):
|
||||
slept.append(round(s, 2))
|
||||
now[0] += s
|
||||
|
||||
def handler(req):
|
||||
now[0] += 0.5 # each request takes 0.5 s
|
||||
if req.url.path == "/robots.txt":
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, text="<html>x</html>", headers={"content-type": "text/html"})
|
||||
|
||||
c = PoliteClient(transport=httpx.MockTransport(handler), min_interval=3.0, sleep=sleep, clock=lambda: now[0])
|
||||
for i in range(3):
|
||||
c.get(f"https://shop.example.in/p/{i}")
|
||||
c.close()
|
||||
# robots.txt, then 3 pages: every gap after the first is topped up to 3 s.
|
||||
assert slept == [2.5, 2.5, 2.5]
|
||||
|
||||
|
||||
def test_user_agent_is_honest():
|
||||
seen = {}
|
||||
|
||||
def handler(req):
|
||||
seen["ua"] = req.headers["user-agent"]
|
||||
return httpx.Response(404)
|
||||
|
||||
with _client(handler) as c:
|
||||
c.get("https://shop.example.in/p/1")
|
||||
assert seen["ua"].startswith("ElectronicsCatalogBot/") and "mailto:" in seen["ua"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sites, URLs, matching
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_marketplaces_are_never_fetched_directly():
|
||||
ref = load_reference()
|
||||
assert ref.sites["amazon.in"].policy == "serp_only"
|
||||
assert ref.sites["flipkart.com"].policy == "serp_only"
|
||||
assert {s.name for s in ref.sites.values() if s.region == "TN"} >= {"Poorvika", "Sangeetha Mobiles"}
|
||||
|
||||
|
||||
def test_product_urls_and_skus():
|
||||
amazon = site_for_url("https://www.amazon.in/Samsung-Galaxy-Storage/dp/B0CS5XW6TN/ref=sr_1_1")
|
||||
assert amazon.domain == "amazon.in"
|
||||
assert source_sku(amazon, "https://www.amazon.in/Samsung-Galaxy-Storage/dp/B0CS5XW6TN/ref=sr_1_1") == "B0CS5XW6TN"
|
||||
assert not is_product_url(amazon, "https://www.amazon.in/s?k=samsung+galaxy")
|
||||
fk = site_for_url("https://www.flipkart.com/samsung-galaxy-s24/p/itm123abc?pid=MOBGX")
|
||||
assert source_sku(fk, "https://www.flipkart.com/samsung-galaxy-s24/p/itm123abc?pid=MOBGX") == "itm123abc"
|
||||
croma = site_for_url("https://www.croma.com/samsung-galaxy-s24/p/303838")
|
||||
assert is_product_url(croma, "https://www.croma.com/samsung-galaxy-s24/p/303838")
|
||||
assert not is_product_url(croma, "https://www.croma.com/phones-wearables/c/1")
|
||||
samsung = site_for_url("https://www.samsung.com/in/smartphones/galaxy-s24/buy/")
|
||||
assert samsung.kind == "brand_official"
|
||||
assert is_product_url(samsung, "https://www.samsung.com/in/smartphones/galaxy-s24/buy/")
|
||||
assert not is_product_url(samsung, "https://us.samsung.com/smartphones/galaxy-s24/buy/")
|
||||
assert not is_product_url(samsung, "https://www.samsung.com/uk/smartphones/galaxy-s24/buy/")
|
||||
assert not is_product_url(samsung, "https://news.samsung.com/in/galaxy-s24-launch")
|
||||
oneplus = site_for_url("https://www.oneplus.in/nord-ce4-lite")
|
||||
assert is_product_url(oneplus, "https://www.oneplus.in/nord-ce4-lite/specs")
|
||||
assert site_for_url("https://example.com/whatever") is None
|
||||
|
||||
|
||||
def _listing(**kw):
|
||||
base = dict(variant_key=None, model_norm=None, ram_gb=None, storage_gb=None, gtin=None,
|
||||
model_number=None, category="mobiles", processor=None)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def _cand(pid, key, norm, ram, storage, **kw):
|
||||
return dict(id=pid, variant_key=key, model_norm=norm, ram_gb=ram, storage_gb=storage,
|
||||
processor=kw.get("processor"), mpn=kw.get("mpn"), gtin=kw.get("gtin"))
|
||||
|
||||
|
||||
def test_matcher():
|
||||
cands = [
|
||||
_cand(1, "samsung|mobiles|galaxy s24|8|256", "galaxy s24", Decimal(8), Decimal(256)),
|
||||
_cand(2, "samsung|mobiles|galaxy s24 ultra|12|256", "galaxy s24 ultra", Decimal(12), Decimal(256)),
|
||||
]
|
||||
exact = decide(_listing(variant_key="samsung|mobiles|galaxy s24|8|256", model_norm="galaxy s24",
|
||||
ram_gb=Decimal(8), storage_gb=Decimal(256)), cands)
|
||||
assert (exact.product_id, exact.review_status) == (1, "auto")
|
||||
# "galaxy s24" must never be merged into "galaxy s24 ultra".
|
||||
other = decide(_listing(variant_key="samsung|mobiles|galaxy s24 plus|12|256", model_norm="galaxy s24 plus",
|
||||
ram_gb=Decimal(12), storage_gb=Decimal(256)), cands)
|
||||
assert other.product_id is None
|
||||
# Unknown RAM, one candidate with the same model+storage -> same variant.
|
||||
no_ram = decide(_listing(variant_key="samsung|mobiles|galaxy s24|na|256", model_norm="galaxy s24",
|
||||
storage_gb=Decimal(256)), cands)
|
||||
assert (no_ram.product_id, no_ram.review_status) == (1, "auto")
|
||||
assert decide(_listing(), cands) is None # nothing to identify a variant by
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search-engine offer data (Google CSE pagemap)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_pagemap_offer_inr_only():
|
||||
from app.electronics.search.providers import pagemap_offer
|
||||
|
||||
assert pagemap_offer({"offer": [{"price": "74999", "pricecurrency": "INR",
|
||||
"availability": "https://schema.org/InStock"}]})["price"] == "74999"
|
||||
assert pagemap_offer({"metatags": [{"product:price:amount": "69900", "product:price:currency": "INR"}]})
|
||||
assert pagemap_offer({"offer": [{"price": "799", "pricecurrency": "USD"}]}) is None
|
||||
assert pagemap_offer({}) is None
|
||||
|
||||
|
||||
def test_listing_from_search_uses_engine_offer():
|
||||
from app.electronics.collector import Collector, RunOptions, RunStats
|
||||
from app.electronics.search.providers import SearchHit
|
||||
from app.electronics.normalise.title_parser import parse_title
|
||||
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt, c.stats = RunOptions(category="mobiles", brands=["samsung"]), RunStats()
|
||||
title = "Samsung Galaxy A56 5G (Awesome Olive, 256 GB) (8 GB RAM) - Flipkart"
|
||||
hit = SearchHit("https://www.flipkart.com/samsung-galaxy-a56/p/itmabc", title,
|
||||
"Buy Samsung Galaxy A56 5G online at best price", "google", 0,
|
||||
offer={"price": "42999", "currency": "INR", "availability": "InStock",
|
||||
"raw": {"price": "42999", "pricecurrency": "INR"}})
|
||||
site = site_for_url(hit.url)
|
||||
listing = c.listing_from_search(hit, site, parse_title(title, "mobiles"), "q")
|
||||
assert listing.price == Decimal("42999") and listing.in_stock is True
|
||||
assert "42999" in listing.evidence_text and listing.parser.endswith("pagemap")
|
||||
assert listing.source_type == "search_snippet" # still never fetched from Flipkart
|
||||
|
||||
|
||||
def test_matcher_never_merges_different_model_numbers():
|
||||
for existing, incoming in (("galaxy s25 ultra", "galaxy s26 ultra"), ("galaxy s25 fe", "galaxy s26 fe"),
|
||||
("galaxy a37", "galaxy a27"), ("iphone 15", "iphone 16")):
|
||||
cands = [_cand(1, f"samsung|mobiles|{existing}|12|256", existing, Decimal(12), Decimal(256))]
|
||||
d = decide(_listing(variant_key=f"samsung|mobiles|{incoming}|12|256", model_norm=incoming,
|
||||
ram_gb=Decimal(12), storage_gb=Decimal(256)), cands)
|
||||
assert d.product_id is None, (existing, incoming)
|
||||
|
||||
|
||||
def test_laptop_matching_by_configuration():
|
||||
cands = [_cand(1, "lenovo|laptops|ideapad slim 3|i5-13420h|16|512", "ideapad slim 3 15irh10",
|
||||
Decimal(16), Decimal(512), processor="i5-13420h")]
|
||||
same = decide(_listing(category="laptops", variant_key="lenovo|laptops|ideapad slim 3|i5-13420h|16|512x",
|
||||
model_norm="ideapad slim 3", processor="i5-13420h",
|
||||
ram_gb=Decimal(16), storage_gb=Decimal(512)), cands)
|
||||
assert (same.product_id, same.review_status) == (1, "auto")
|
||||
# A different line (Slim 5) or a different CPU is a different laptop.
|
||||
for norm, cpu in (("ideapad slim 5", "i5-13420h"), ("ideapad slim 3", "i5-1235u")):
|
||||
d = decide(_listing(category="laptops", variant_key="x", model_norm=norm, processor=cpu,
|
||||
ram_gb=Decimal(16), storage_gb=Decimal(512)), cands)
|
||||
assert d.product_id is None, (norm, cpu)
|
||||
|
||||
|
||||
def test_vague_laptop_line_goes_to_review():
|
||||
cands = [
|
||||
_cand(1, "hp|laptops|pavilion 14|i5-1240p|8|512", "pavilion 14", Decimal(8), Decimal(512), processor="i5-1240p"),
|
||||
_cand(2, "hp|laptops|pavilion 15|i5-1240p|8|512", "pavilion 15", Decimal(8), Decimal(512), processor="i5-1240p"),
|
||||
]
|
||||
d = decide(_listing(category="laptops", variant_key="hp|laptops|pavilion|i5-1240p|8|512", model_norm="pavilion",
|
||||
processor="i5-1240p", ram_gb=Decimal(8), storage_gb=Decimal(512)), cands)
|
||||
assert d.review_status == "pending"
|
||||
330
backend/tests/test_elec_parsers.py
Normal file
330
backend/tests/test_elec_parsers.py
Normal file
@@ -0,0 +1,330 @@
|
||||
"""Offline tests: title parsing, SERP price reading, spec normalisation,
|
||||
grounding and the LLM guard. No network, no database."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from app.electronics.extract.serp_parser import clean_result_title, read_price, read_stock
|
||||
from app.electronics.normalise.brand_alias import looks_like_device_title, resolve_brand
|
||||
from app.electronics.normalise.grounding import value_in_source
|
||||
from app.electronics.normalise.llm_fill import fill_missing
|
||||
from app.electronics.normalise.spec_normaliser import normalise_specs
|
||||
from app.electronics.normalise.title_parser import parse_title, variant_key
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Titles: the same variant written the way different sites write it must give
|
||||
# the same key; different variants must not.
|
||||
# ---------------------------------------------------------------------------
|
||||
SAME_VARIANT = [
|
||||
("mobiles", "samsung|mobiles|galaxy s24|8|256", [
|
||||
"Samsung Galaxy S24 5G (Onyx Black, 8GB RAM, 256GB Storage)",
|
||||
"SAMSUNG Galaxy S24 5G (Onyx Black, 256 GB) (8 GB RAM)",
|
||||
"Samsung Galaxy S24 5G (8GB RAM, 256GB, Onyx Black)",
|
||||
"Samsung Galaxy S24 8GB 256GB Onyx Black",
|
||||
]),
|
||||
("mobiles", "xiaomi|mobiles|redmi note 13 pro|8|256", [
|
||||
"Redmi Note 13 Pro 5G (8GB + 256GB)",
|
||||
"Xiaomi Redmi Note 13 Pro 5G (Arctic White, 8GB RAM, 256GB Storage)",
|
||||
"REDMI Note 13 Pro 5G (Arctic White, 256 GB) (8 GB RAM)",
|
||||
]),
|
||||
("mobiles", "apple|mobiles|iphone 15|na|128", [
|
||||
"Apple iPhone 15 (128 GB) - Black",
|
||||
"Apple iPhone 15 128GB Black",
|
||||
"iPhone 15 128 GB: 5G Smartphone with Dynamic Island",
|
||||
]),
|
||||
("mobiles", "nothing|mobiles|2a|8|128", [
|
||||
"Nothing Phone (2a) 5G (Black, 8GB RAM, 128GB)",
|
||||
"Nothing Phone (2a) 5G (Black, 128 GB) (8 GB RAM)",
|
||||
]),
|
||||
("laptops", "hp|laptops|15s|i5-1334u|16|512", [
|
||||
"HP 15s, 13th Gen Intel Core i5-1334U, 16GB DDR4, 512GB SSD, (Win 11, Office 21, Silver, 1.69kg), "
|
||||
"15.6-inch(39.6 cm) FHD, Intel Iris Xe Graphics, fd0112TU",
|
||||
"HP 15s Laptop fd0112TU Intel Core i5 1334U 16GB 512GB SSD",
|
||||
]),
|
||||
("laptops", "dell|laptops|inspiron 3520|i5-1235u|16|512", [
|
||||
"Dell Inspiron 3520 Laptop, Intel Core i5-1235U, 16GB, 512GB SSD, 15.6\" FHD",
|
||||
"Dell Inspiron 3520 Intel Core i5-1235U 16GB RAM 512GB SSD 15.6 inch",
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("category,key,titles", SAME_VARIANT)
|
||||
def test_same_variant_same_key(category, key, titles):
|
||||
for title in titles:
|
||||
assert variant_key(parse_title(title, category), category) == key, title
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a,b", [
|
||||
("Samsung Galaxy S24 5G (8GB RAM, 256GB)", "Samsung Galaxy S24 Ultra 5G (12GB RAM, 256GB)"),
|
||||
("Samsung Galaxy S24 5G (8GB RAM, 128GB)", "Samsung Galaxy S24 5G (8GB RAM, 256GB)"),
|
||||
("Redmi Note 13 5G (8GB + 256GB)", "Redmi Note 13 Pro 5G (8GB + 256GB)"),
|
||||
("Apple iPhone 15 (128 GB)", "Apple iPhone 15 Plus (128 GB)"),
|
||||
])
|
||||
def test_different_variants_differ(a, b):
|
||||
assert variant_key(parse_title(a, "mobiles"), "mobiles") != variant_key(parse_title(b, "mobiles"), "mobiles")
|
||||
|
||||
|
||||
def test_title_without_storage_has_no_variant():
|
||||
# A title that does not say which variant it is cannot be linked to one.
|
||||
assert variant_key(parse_title("Samsung Galaxy S24 5G price in India", "mobiles"), "mobiles") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("title,brand,family", [
|
||||
("Google Pixel 8a (Obsidian, 128GB)", "google", "Pixel"),
|
||||
("iQOO Z9 5G (Brushed Green, 8GB RAM, 128GB Storage)", "vivo", "iQOO"),
|
||||
("POCO X6 Pro 5G (Racing Grey, 12GB RAM, 512GB)", "xiaomi", "POCO"),
|
||||
("Lenovo IdeaPad Slim 3 Intel Core i5", "lenovo", "IdeaPad"),
|
||||
])
|
||||
def test_sub_brands_resolve_to_parent(title, brand, family):
|
||||
m = resolve_brand(title)
|
||||
assert m.brand_slug == brand and m.family == family
|
||||
|
||||
|
||||
def test_brand_outside_allow_list_is_rejected():
|
||||
assert resolve_brand("Nokia G42 5G (6GB RAM, 128GB)") is None
|
||||
# The brand must lead the title - an accessory "for Samsung" is not Samsung.
|
||||
assert resolve_brand("Spigen case for Samsung Galaxy S24") is None
|
||||
|
||||
|
||||
def test_accessories_are_not_devices():
|
||||
assert not looks_like_device_title("Samsung Galaxy S24 Back Cover Case")
|
||||
assert not looks_like_device_title("Samsung Galaxy S24 vs iPhone 15 comparison")
|
||||
assert looks_like_device_title("Samsung Galaxy S24 5G (8GB RAM, 256GB)")
|
||||
|
||||
|
||||
def test_laptop_fields():
|
||||
p = parse_title("ASUS Vivobook 15, Intel Core i3-1215U 12th Gen, 8GB RAM, 512GB SSD, 15.6\" FHD, "
|
||||
"Windows 11, X1504ZA-NJ321WS", "laptops")
|
||||
assert (p.model, p.processor, p.ram_gb, p.storage_gb, p.mpn) == (
|
||||
"Vivobook 15", "i3-1215u", Decimal(8), Decimal(512), "X1504ZA-NJ321WS")
|
||||
mac = parse_title("Apple MacBook Air Laptop: Apple M2 chip, 13.6-inch Liquid Retina Display, "
|
||||
"8GB Unified Memory, 256GB SSD Storage - Midnight", "laptops")
|
||||
assert (mac.model, mac.processor, mac.ram_gb, mac.storage_gb) == ("MacBook Air", "m2", Decimal(8), Decimal(256))
|
||||
|
||||
|
||||
def test_terabyte_storage():
|
||||
p = parse_title("Samsung Galaxy S24 Ultra 5G (Titanium Black, 12GB RAM, 1TB Storage)", "mobiles")
|
||||
assert p.storage_gb == Decimal(1024)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SERP snippet prices: only an unambiguous selling price is read.
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize("text,price", [
|
||||
("Buy Samsung Galaxy S24 5G for ₹74,999 online", Decimal("74999")),
|
||||
("Price: ₹1,29,999.00 Free delivery", Decimal("129999.00")),
|
||||
("Rs. 12,999 only", Decimal("12999")),
|
||||
("Rs.12999", Decimal("12999")),
|
||||
("INR 45,490", Decimal("45490")),
|
||||
("₹64,999 M.R.P: ₹79,999 (19% off)", Decimal("64999")),
|
||||
("Deal price ₹15,499. No Cost EMI from ₹2,583/month.", Decimal("15499")),
|
||||
("₹18,999 Save ₹4,000 with bank offers", Decimal("18999")),
|
||||
("Get ₹3,000 off. Now at ₹21,999", Decimal("21999")),
|
||||
("Price ₹74,999 . Up to ₹5,000 cashback", Decimal("74999")),
|
||||
("₹74999 ₹74,999 in stock", Decimal("74999")), # same price twice is still one price
|
||||
])
|
||||
def test_snippet_price_read(text, price):
|
||||
assert read_price(text).price == price
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"No Cost EMI starting from ₹2,583/month",
|
||||
"Save ₹4,000 on exchange",
|
||||
"Exchange offer up to ₹12,000",
|
||||
"Price range ₹10,999 - ₹12,999",
|
||||
"Starting from ₹9,999",
|
||||
"Get extra ₹2,000 off with HDFC Bank cards",
|
||||
"Samsung Galaxy S24 5G 8GB 256GB", # no amount at all
|
||||
"₹299 screen guard", # outside plausible device range
|
||||
"8GB+128GB ₹17,999 | 8GB+256GB ₹19,999", # two variants, two prices: ambiguous
|
||||
"₹2,583 per month EMI",
|
||||
"Flat ₹3,000 discount",
|
||||
"Delivery charges ₹49",
|
||||
])
|
||||
def test_snippet_non_prices_rejected(text):
|
||||
assert read_price(text).price is None
|
||||
|
||||
|
||||
def test_mrp_is_separate_and_never_below_price():
|
||||
r = read_price("₹64,999 MRP ₹79,999")
|
||||
assert (r.price, r.mrp) == (Decimal("64999"), Decimal("79999"))
|
||||
assert read_price("₹64,999 MRP ₹59,999").mrp is None
|
||||
|
||||
|
||||
def test_stock_phrases():
|
||||
assert read_stock("Currently unavailable. We don't know when") is False
|
||||
assert read_stock("Out of Stock") is False
|
||||
assert read_stock("In stock. Delivery by tomorrow") is True
|
||||
assert read_stock("Samsung Galaxy S24 8GB 256GB") is None
|
||||
|
||||
|
||||
def test_clean_result_title():
|
||||
assert clean_result_title("Samsung Galaxy S24 5G (Onyx Black, 8GB RAM, 256GB) : Amazon.in: Electronics") \
|
||||
== "Samsung Galaxy S24 5G (Onyx Black, 8GB RAM, 256GB)"
|
||||
assert clean_result_title("Buy Apple iPhone 15 (128 GB) - Black Online at Best Price | Croma") \
|
||||
== "Apple iPhone 15 (128 GB) - Black"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Specs and grounding
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_spec_normalisation_units_and_ranges():
|
||||
specs, sources = normalise_specs("mobiles", {
|
||||
"Battery Capacity": "5000 mAh",
|
||||
"Display Size": "15.49 cm (6.1 inch)",
|
||||
"RAM": "8 GB",
|
||||
"Internal Storage": "1 TB",
|
||||
"Primary Camera": "50MP + 12MP + 10MP",
|
||||
"Network Type": "5G, 4G VOLTE",
|
||||
"Operating System": "Android 14",
|
||||
"Refresh Rate": "120 Hz",
|
||||
"Unrelated Label": "whatever",
|
||||
"Screen Size": "600 inch", # later duplicate key; also out of range
|
||||
})
|
||||
assert specs == {"battery_mah": 5000, "display_inch": 6.1, "ram_gb": 8, "storage_gb": 1024,
|
||||
"rear_camera_mp": 50, "network": "5G", "os": "Android", "refresh_hz": 120}
|
||||
assert sources["battery_mah"].startswith("Battery Capacity")
|
||||
|
||||
|
||||
def test_spec_out_of_range_dropped():
|
||||
specs, _ = normalise_specs("mobiles", {"Battery Capacity": "50000 mAh", "RAM": "512 GB"})
|
||||
assert specs == {}
|
||||
|
||||
|
||||
def test_laptop_weight_in_grams():
|
||||
specs, _ = normalise_specs("laptops", {"Weight": "1690 g"})
|
||||
assert specs == {"weight_kg": 1.69}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value,source,ok", [
|
||||
("5000 mAh", "Battery: 5,000mAh Li-ion", True),
|
||||
(5000, "Battery 5000 mAh", True),
|
||||
("6000 mAh", "Battery 5000 mAh", False),
|
||||
("Snapdragon 8 Gen 3", "Processor: Qualcomm Snapdragon 8 Gen 3 for Galaxy", True),
|
||||
("Snapdragon 8 Gen 2", "Processor: Qualcomm Snapdragon 8 Gen 3 for Galaxy", False),
|
||||
("AMOLED", "6.2-inch Dynamic AMOLED 2X display", True),
|
||||
("OLED", "6.2-inch LCD display", False),
|
||||
(None, "anything", False),
|
||||
])
|
||||
def test_value_in_source(value, source, ok):
|
||||
assert value_in_source(value, source) is ok
|
||||
|
||||
|
||||
def test_llm_values_not_in_source_are_discarded(monkeypatch):
|
||||
"""The guard that stops a small model's guess becoming a stored fact."""
|
||||
import app.electronics.normalise.llm_fill as llm_fill
|
||||
|
||||
monkeypatch.setattr(llm_fill, "ELEC_USE_LLM", True)
|
||||
source = "Display: 6.2 inch Dynamic AMOLED 2X, 120Hz. Battery 4000 mAh. Processor: Exynos 2400"
|
||||
|
||||
def fake_generate(system, prompt):
|
||||
return {
|
||||
"battery_mah": "4000 mAh", # stated -> kept
|
||||
"refresh_hz": "120Hz", # stated -> kept
|
||||
"rear_camera_mp": "200 MP", # invented -> dropped
|
||||
"processor": "Snapdragon 8 Gen 3", # invented -> dropped
|
||||
"os": "Android", # not in the text -> dropped
|
||||
}
|
||||
|
||||
specs, sources = fill_missing("mobiles", source,
|
||||
["battery_mah", "refresh_hz", "rear_camera_mp", "processor", "os"],
|
||||
generate=fake_generate)
|
||||
assert specs == {"battery_mah": 4000, "refresh_hz": 120}
|
||||
assert all(v.startswith("llm-extracted") for v in sources.values())
|
||||
|
||||
|
||||
def test_llm_disabled_returns_nothing():
|
||||
called = []
|
||||
specs, _ = fill_missing("mobiles", "Battery 5000 mAh", ["battery_mah"],
|
||||
generate=lambda *a: called.append(1) or {"battery_mah": "5000 mAh"})
|
||||
assert specs == {} and called == [] # ELEC_USE_LLM=false in the test environment
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Laptops: real titles from the pilot run that previously failed to group.
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize("title,cpu", [
|
||||
("HP 15 (2024) AMD Ryzen 3 Quad Core 7320U - (8 GB/512 GB SSD", "ryzen 3 7320u"),
|
||||
("HP 15 AMD Ryzen R3 7320U Windows 11 Home Laptop, 15-fc0500AU ( Natural Silver, 8GB-512GB )", "ryzen 3 7320u"),
|
||||
("HP 15s AMD Ryzen 5 Hexa Core 5500U - (16 GB/512 GB SSD", "ryzen 5 5500u"),
|
||||
("HP 15 (2026), AMD Athlon Dual Core 7120U - (8 GB DDR5/512 GB", "athlon 7120u"),
|
||||
("HP 15 Laptop, Intel Core 5-120U, 16GB DDR4 RAM, 512GB SSD", "core 5 120u"),
|
||||
("Lenovo IdeaPad Slim 3 Intel Core i5 13th Gen 13420H - (16 GB/512 GB SSD", "i5-13420h"),
|
||||
("Lenovo IdeaPad Slim 3 15IPH11 Intel Core Ultra 5 125H (16GB RAM/ 512GB SSD)", "core ultra 5 125h"),
|
||||
])
|
||||
def test_laptop_processors(title, cpu):
|
||||
assert parse_title(title, "laptops").processor == cpu
|
||||
|
||||
|
||||
def test_cpu_slash_is_not_a_ram_storage_pair():
|
||||
p = parse_title("Lenovo IdeaPad Slim 3 15IPH11 Laptop (Intel Core Ultra 5/ 16GB RAM/ 512GB SSD/ 15.3 Inch)",
|
||||
"laptops")
|
||||
assert (p.ram_gb, p.storage_gb) == (Decimal(16), Decimal(512))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("category,key,titles", [
|
||||
("laptops", "hp|laptops|15|ryzen 3 7320u|8|512", [
|
||||
"HP 15 (2024) AMD Ryzen 3 Quad Core 7320U - (8 GB/512 GB SSD/Windows 11 Home)",
|
||||
"HP 15 AMD Ryzen R3 7320U Windows 11 Home Laptop, 15-fc0500AU ( Natural Silver, 8GB-512GB )",
|
||||
"Buy HP 15-FC0500AU AMD Ryzen 3 7320U 8GB RAM 512GB SSD Windows 11 Home Silver Laptop",
|
||||
]),
|
||||
("laptops", "lenovo|laptops|ideapad slim 3|i5-13420h|16|512", [
|
||||
"Lenovo IdeaPad Slim 3 Intel Core i5 13th Gen 13420H - (16 GB/512 GB SSD/Windows 11 Home)",
|
||||
"Lenovo IdeaPad Slim 3 15IRH10 Intel Core i5-13420H 16GB RAM 512GB SSD 15.3 inch",
|
||||
]),
|
||||
])
|
||||
def test_laptop_configuration_keys(category, key, titles):
|
||||
for title in titles:
|
||||
assert variant_key(parse_title(title, category), category) == key, title
|
||||
|
||||
|
||||
def test_laptop_mpn_is_the_fallback_key():
|
||||
# No CPU model number stated: the part number still identifies it.
|
||||
p = parse_title("Lenovo IdeaPad Slim 3 Intel Core i3 13th Gen Laptop, 82X700HMIN (Arctic Grey, 8GB-512GB)",
|
||||
"laptops")
|
||||
assert variant_key(p, "laptops") == "lenovo|laptops|mpn:82x700hmin"
|
||||
|
||||
|
||||
def test_truncated_title_filled_from_snippet():
|
||||
from app.electronics.normalise.title_parser import fill_from_context
|
||||
|
||||
p = parse_title("Lenovo IdeaPad Slim 5 Intel Core i7 13th Gen 13700H - (16 GB", "laptops")
|
||||
assert variant_key(p, "laptops") is None
|
||||
fill_from_context(p, "laptops", snippet="Lenovo IdeaPad Slim 5 Intel Core i7 13th Gen 13700H - (16 GB/512 GB SSD/"
|
||||
"Windows 11 Home) 14 inch WUXGA OLED")
|
||||
assert variant_key(p, "laptops") == "lenovo|laptops|ideapad slim 5|i7-13700h|16|512"
|
||||
|
||||
|
||||
def test_ambiguous_snippet_is_not_used():
|
||||
from app.electronics.normalise.title_parser import fill_from_context
|
||||
|
||||
p = parse_title("HP Pavilion Intel Core i5 12th Gen 1240P - (8 GB", "laptops")
|
||||
fill_from_context(p, "laptops", snippet="Available in 8 GB/256 GB, 16 GB/512 GB and 16 GB/1 TB SSD")
|
||||
assert p.storage_gb is None
|
||||
|
||||
|
||||
def test_cpu_filled_from_spec_table():
|
||||
from app.electronics.normalise.title_parser import fill_from_context
|
||||
|
||||
p = parse_title("Lenovo IdeaPad Slim 3 Laptop (13th Gen Intel Core i7/ 16GB RAM/ 512GB SSD)", "laptops")
|
||||
fill_from_context(p, "laptops", spec_texts=("13th Gen Intel Core i7-13620H",))
|
||||
assert p.processor == "i7-13620h"
|
||||
|
||||
|
||||
def test_merged_search_titles_are_cut():
|
||||
assert clean_result_title("HP 15 (2026), AMD Athlon Dual Core 7120U - (8 GB DDR5/512 GB ...HP 15 (2026), "
|
||||
"AMD Ryzen 5 Hexa Core 7535U") == "HP 15 (2026), AMD Athlon Dual Core 7120U - (8 GB DDR5/512 GB"
|
||||
|
||||
|
||||
def test_merged_snippet_never_supplies_a_cpu():
|
||||
from app.electronics.normalise.title_parser import fill_from_context
|
||||
|
||||
p = parse_title("ASUS Vivobook 16, Snapdragon X, 16GB RAM, 512GB SSD, FHD+ 16", "laptops")
|
||||
fill_from_context(p, "laptops", snippet="ASUS Vivobook 16 (2026),Intel Core Ultra 5 225H ... ASUS Vivobook 16, "
|
||||
"Intel Core i5-13420H Processor")
|
||||
assert p.processor != "core ultra 5 225h"
|
||||
q = parse_title("HP 15 Laptop (16 GB", "laptops")
|
||||
fill_from_context(q, "laptops", snippet="HP 15 Intel Core i5-1334U or AMD Ryzen 5 7530U variants")
|
||||
assert q.processor is None # two CPUs named: ambiguous
|
||||
144
backend/tests/test_elec_reviews.py
Normal file
144
backend/tests/test_elec_reviews.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""Ratings and reviews: read only what a page or search result states, and pick
|
||||
the review mix by the product's rating. Offline tests first; the database
|
||||
tests are skipped when the local Postgres container is not running."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from app.electronics.extract.jsonld import extract_products
|
||||
from app.electronics.extract.serp_parser import read_rating
|
||||
from app.electronics.reviews import select_reviews, sentiment_for
|
||||
from app.electronics.search.providers import SearchHit, pagemap_rating
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search-result ratings
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_read_rating_accepts_explicit_statements():
|
||||
r = read_rating("Samsung Galaxy S24 5G ... 4.3 out of 5 stars 1,234 ratings. ₹74,999")
|
||||
assert r.rating == Decimal("4.3") and r.review_count == 1234
|
||||
assert read_rating("Rating: 4.1/5 based on reviews").rating == Decimal("4.1")
|
||||
r = read_rating("4.4★ (12,345 ratings)")
|
||||
assert r.rating == Decimal("4.4") and r.review_count == 12345
|
||||
|
||||
|
||||
def test_read_rating_refuses_guesses():
|
||||
assert read_rating("Galaxy S24 8GB 256GB ₹74,999").rating is None
|
||||
assert read_rating("1/5 inch sensor, 50MP").rating is None # a fraction, not a rating
|
||||
assert read_rating("5/5G phone").rating is None
|
||||
assert read_rating("4.2 out of 5 ... 3.9 out of 5").rating is None # two products: ambiguous
|
||||
assert read_rating("7 out of 5").rating is None
|
||||
|
||||
|
||||
def test_pagemap_rating_and_cached_hits_without_rating():
|
||||
got = pagemap_rating({"aggregaterating": [{"ratingvalue": "4.5", "reviewcount": "2,310", "bestrating": "5"}]})
|
||||
assert got["rating"] == 4.5 and got["review_count"] == 2310
|
||||
assert pagemap_rating({"aggregaterating": [{"ratingvalue": "9", "bestrating": "10"}]}) is None
|
||||
# Search results cached before the rating field existed still load.
|
||||
hit = SearchHit.from_dict({"url": "https://a.in/p", "title": "t", "snippet": "s", "provider": "ddg", "rank": 0})
|
||||
assert hit.rating is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page reviews (schema.org JSON-LD)
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_jsonld_reviews_are_read_verbatim():
|
||||
ld = {
|
||||
"@context": "https://schema.org", "@type": "Product", "name": "Samsung Galaxy S24",
|
||||
"aggregateRating": {"ratingValue": "4.4", "reviewCount": "120"},
|
||||
"review": [
|
||||
{"@type": "Review", "author": {"@type": "Person", "name": "Arun"}, "name": "Great phone",
|
||||
"reviewBody": "Battery lasts all day.", "datePublished": "2026-05-01",
|
||||
"reviewRating": {"ratingValue": "5", "bestRating": "5"}},
|
||||
{"@type": "Review", "author": "Priya", "reviewBody": "Heats up while gaming.",
|
||||
"reviewRating": {"ratingValue": "4", "bestRating": "10"}},
|
||||
{"@type": "Review", "author": "No words", "reviewRating": {"ratingValue": "1"}},
|
||||
],
|
||||
}
|
||||
html = f'<script type="application/ld+json">{json.dumps(ld)}</script>'
|
||||
p = extract_products(html)[0]
|
||||
assert p["rating"] == Decimal("4.4") and p["review_count"] == 120
|
||||
assert [r["body"] for r in p["reviews"]] == ["Battery lasts all day.", "Heats up while gaming."]
|
||||
assert p["reviews"][0]["author"] == "Arun" and p["reviews"][0]["title"] == "Great phone"
|
||||
assert p["reviews"][1]["rating"] == Decimal("2.0") # 4 out of 10, rescaled
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Review mix
|
||||
# ---------------------------------------------------------------------------
|
||||
def _pool(pos: int, neu: int, neg: int) -> list:
|
||||
out = []
|
||||
for label, n, stars in (("p", pos, 5), ("u", neu, 3), ("n", neg, 1)):
|
||||
out += [{"body": f"{label}{i}", "rating": stars} for i in range(n)]
|
||||
return out
|
||||
|
||||
|
||||
def _counts(picked: list) -> tuple:
|
||||
return tuple(sum(1 for r in picked if r["sentiment"] == s) for s in ("positive", "neutral", "negative"))
|
||||
|
||||
|
||||
def test_sentiment_is_the_reviewers_own_stars():
|
||||
assert [sentiment_for(x) for x in (5, 4, 3.5, 3, 2.9, 1, None)] == [
|
||||
"positive", "positive", "neutral", "neutral", "negative", "negative", None]
|
||||
|
||||
|
||||
def test_high_rating_shows_mostly_positive():
|
||||
assert _counts(select_reviews(4.7, _pool(20, 20, 20))) == (6, 3, 1)
|
||||
|
||||
|
||||
def test_middling_rating_shows_mostly_neutral():
|
||||
assert _counts(select_reviews(3.6, _pool(20, 20, 20))) == (3, 5, 2)
|
||||
|
||||
|
||||
def test_low_rating_shows_mostly_negative():
|
||||
assert _counts(select_reviews(2.5, _pool(20, 20, 20))) == (2, 2, 6)
|
||||
|
||||
|
||||
def test_short_groups_hand_slots_on_and_nothing_is_padded():
|
||||
picked = select_reviews(4.8, _pool(3, 20, 0))
|
||||
assert len(picked) == 10 and _counts(picked) == (3, 7, 0)
|
||||
assert len(select_reviews(4.8, _pool(1, 1, 1))) == 3
|
||||
assert select_reviews(4.8, [{"body": "no stars", "rating": None}]) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database + API
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_api_serves_ratings_reviews_and_out_of_stock_price(db, client):
|
||||
from app.electronics.collector import Collector, RunOptions, RunStats
|
||||
from app.electronics.db import repository as repo
|
||||
from app.electronics.models import Listing
|
||||
from app.electronics.normalise.title_parser import parse_title, variant_key
|
||||
|
||||
def _listing(site, sku, title, *, price, source_type="search_snippet", evidence=None):
|
||||
p = parse_title(title, "mobiles")
|
||||
l = Listing(site_domain=site, source_sku=sku, source_url=f"https://www.{site}/p/{sku}",
|
||||
source_type=source_type, brand_slug=p.brand.brand_slug, category="mobiles", title=title,
|
||||
evidence_text=evidence or f"{title} ₹{price}", confidence=0.5, parser="test",
|
||||
model=p.model, ram_gb=p.ram_gb, storage_gb=p.storage_gb, price=price)
|
||||
l.model_norm, l.variant_key = p.model_norm, variant_key(p, "mobiles")
|
||||
return l
|
||||
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids, c.run_id, c._touched_products, c.stats = repo.id_maps(), None, {}, RunStats()
|
||||
title = "Samsung Galaxy S24 5G (8GB RAM, 256GB)"
|
||||
a = _listing("amazon.in", "B0CS5XW6TN", title, price=Decimal(74999))
|
||||
a.in_stock, a.rating, a.review_count = False, Decimal("4.6"), 300
|
||||
b = _listing("croma.com", "303838", title, price=Decimal(73999), source_type="scraped_page",
|
||||
evidence='{"price": "73999"}')
|
||||
b.in_stock, b.rating, b.review_count = False, Decimal("4.0"), 100
|
||||
b.reviews = [{"author": "Arun", "rating": Decimal(5), "title": "Great", "body": "Battery lasts all day."},
|
||||
{"author": "Priya", "rating": Decimal(2), "body": "Heats up."}]
|
||||
c.store(a)
|
||||
c.store(b)
|
||||
repo.refresh_verification()
|
||||
|
||||
product = client.get("/api/elec/products", params={"category": "mobiles"}).json()["products"][0]
|
||||
assert product["best_price"] == "73999.00" # every listing out of stock, price still shown
|
||||
detail = client.get(f"/api/elec/products/{product['product_id']}").json()
|
||||
assert detail["rating"]["value"] == 4.5 and detail["rating"]["count"] == 400
|
||||
assert {s["site"] for s in detail["rating"]["sources"]} == {"Amazon.in", "Croma"}
|
||||
assert [r["body"] for r in detail["reviews"]] == ["Battery lasts all day.", "Heats up."]
|
||||
assert all(r["source_url"].startswith("https://") for r in detail["reviews"])
|
||||
93
backend/tests/test_mcp.py
Normal file
93
backend/tests/test_mcp.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""MCP endpoint (app/mcp_server.py): the tool surface, and that tools return the
|
||||
same catalogue data as the REST API. Database tests are skipped when the local
|
||||
Postgres container is not running."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
import anyio
|
||||
from fastmcp import Client
|
||||
|
||||
from app.mcp_server import mcp
|
||||
|
||||
|
||||
def _call(name: str, args: dict):
|
||||
async def go():
|
||||
async with Client(mcp) as c:
|
||||
return (await c.call_tool(name, args)).data
|
||||
return anyio.run(go)
|
||||
|
||||
|
||||
def test_only_read_only_catalogue_tools_are_exposed():
|
||||
async def go():
|
||||
async with Client(mcp) as c:
|
||||
return {t.name: set(t.input_schema.get("properties", {})) for t in await c.list_tools()}
|
||||
tools = anyio.run(go)
|
||||
assert set(tools) == {"list_categories", "search_products", "get_product", "price_history"}
|
||||
assert tools["search_products"] == {"query", "category", "brand", "max_price", "min_price", "limit"}
|
||||
# Nothing that can start a run, log in, or change data.
|
||||
assert not any(w in name for name in tools for w in ("admin", "run", "login", "probe", "review"))
|
||||
|
||||
|
||||
def test_mcp_endpoint_answers_an_initialize_handshake():
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
body = {"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {"protocolVersion": "2025-06-18", "capabilities": {},
|
||||
"clientInfo": {"name": "test", "version": "1"}}}
|
||||
headers = {"Accept": "application/json, text/event-stream"}
|
||||
with TestClient(app) as c: # `with` runs the lifespan that starts MCP sessions
|
||||
r = c.post("/mcp/", json=body, headers=headers)
|
||||
assert r.status_code == 200, r.text
|
||||
assert "Electronics Catalog" in r.text # server name in the initialize result
|
||||
|
||||
|
||||
def test_tools_return_the_catalogue(db):
|
||||
from app.electronics.collector import Collector, RunOptions, RunStats
|
||||
from app.electronics.db import repository as repo
|
||||
from app.electronics.models import Listing
|
||||
from app.electronics.normalise.title_parser import parse_title, variant_key
|
||||
|
||||
def listing(site, sku, price):
|
||||
title = "Samsung Galaxy S24 5G (8GB RAM, 256GB)"
|
||||
p = parse_title(title, "mobiles")
|
||||
l = Listing(site_domain=site, source_sku=sku, source_url=f"https://www.{site}/p/{sku}",
|
||||
source_type="search_snippet", brand_slug="samsung", category="mobiles", title=title,
|
||||
evidence_text=f"{title} ₹{price}", confidence=0.5, parser="test",
|
||||
model=p.model, ram_gb=p.ram_gb, storage_gb=p.storage_gb, price=Decimal(price))
|
||||
l.model_norm, l.variant_key = p.model_norm, variant_key(p, "mobiles")
|
||||
return l
|
||||
|
||||
c = Collector.__new__(Collector)
|
||||
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
||||
c.ids, c.run_id, c._touched_products, c.stats = repo.id_maps(), None, {}, RunStats()
|
||||
c.store(listing("amazon.in", "B0CS5XW6TN", 74999))
|
||||
c.store(listing("poorvika.com", "samsung-galaxy-s24", 73999))
|
||||
repo.refresh_verification()
|
||||
|
||||
cats = {x["slug"]: x["product_count"] for x in _call("list_categories", {})}
|
||||
assert cats["mobiles"] == 1
|
||||
|
||||
found = _call("search_products", {"query": "galaxy", "category": "mobiles", "max_price": 80000})
|
||||
assert found["total"] == 1
|
||||
hit = found["products"][0]
|
||||
assert hit["best_price"] == "73999.00" and hit["best_price_site"] == "Poorvika"
|
||||
assert _call("search_products", {"max_price": 1000})["total"] == 0
|
||||
|
||||
detail = _call("get_product", {"product_id": hit["product_id"]})
|
||||
assert {o["site"] for o in detail["offers"]} == {"Amazon.in", "Poorvika"}
|
||||
assert all(o["source_url"].startswith("https://") for o in detail["offers"])
|
||||
assert detail["image_urls"] == [] and detail["rating"] is None
|
||||
|
||||
history = _call("price_history", {"product_id": hit["product_id"]})
|
||||
assert {h["price"] for h in history} == {"74999.00", "73999.00"}
|
||||
|
||||
|
||||
def test_unknown_product_is_a_tool_error(db):
|
||||
import pytest
|
||||
from fastmcp.exceptions import ToolError
|
||||
|
||||
with pytest.raises(ToolError, match="not found"):
|
||||
_call("get_product", {"product_id": 999999})
|
||||
108
backend/tests/test_ollama_reachability.py
Normal file
108
backend/tests/test_ollama_reachability.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""The reachability probe in front of every Ollama call.
|
||||
|
||||
WHY THIS FILE EXISTS
|
||||
--------------------
|
||||
`_ensure_client()` asks Ollama for `/api/tags` with a 5-second timeout, and
|
||||
`stage_2_row_intake` calls it once per ROW through `fetch_product_details`.
|
||||
Uncached, a 2000-row sheet ingested with `use_llm` on, against a configured but
|
||||
unreachable Ollama, spends up to ~2.8 hours doing nothing but timing out - and
|
||||
shows as a batch that has hung, not one that has failed.
|
||||
|
||||
That was survivable only while `use_llm` defaulted to false everywhere. It no
|
||||
longer does: `UPLOAD_AUTORUN_USE_LLM` is true, so every auto-started upload now
|
||||
takes this path. The cache is what makes that default safe, and the first test
|
||||
below is the one that stops it being quietly removed in a later refactor.
|
||||
|
||||
`/api/health` calls the same function, so a down Ollama also stops adding five
|
||||
seconds to every health request.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services import ollama_service
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_probe_cache():
|
||||
"""The cache is a module global and outlives a test."""
|
||||
ollama_service.reset_reachability_cache()
|
||||
yield
|
||||
ollama_service.reset_reachability_cache()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def probe_calls(monkeypatch):
|
||||
"""Count the HTTP probes, and make every one of them fail.
|
||||
|
||||
Failure is the case that matters: a reachable Ollama answers in
|
||||
milliseconds, an unreachable one costs the full timeout, and it is the
|
||||
second that used to be paid per row.
|
||||
"""
|
||||
calls: list = []
|
||||
|
||||
def boom(url, **kwargs):
|
||||
calls.append(url)
|
||||
raise OSError("connection refused")
|
||||
|
||||
monkeypatch.setattr(ollama_service, "USE_OLLAMA", True)
|
||||
monkeypatch.setattr(ollama_service.requests, "get", boom)
|
||||
return calls
|
||||
|
||||
|
||||
def test_an_unreachable_ollama_is_probed_once_not_once_per_call(probe_calls):
|
||||
"""The whole point. Ten rows must not be ten timeouts."""
|
||||
for _ in range(10):
|
||||
assert ollama_service._ensure_client() is False
|
||||
|
||||
assert len(probe_calls) == 1, (
|
||||
f"{len(probe_calls)} probes for 10 calls - the cache is not holding, and "
|
||||
f"an ingest will pay the 5s timeout per row"
|
||||
)
|
||||
|
||||
|
||||
def test_the_cache_expires_so_a_late_start_is_noticed(probe_calls, monkeypatch):
|
||||
"""A permanent memo would mean an Ollama started after the API is never
|
||||
seen, and /api/health reports it down until someone redeploys."""
|
||||
clock = [1000.0]
|
||||
monkeypatch.setattr(ollama_service.time, "monotonic", lambda: clock[0])
|
||||
|
||||
ollama_service._ensure_client()
|
||||
assert len(probe_calls) == 1
|
||||
|
||||
clock[0] += ollama_service._PROBE_TTL_SECONDS + 1
|
||||
ollama_service._ensure_client()
|
||||
assert len(probe_calls) == 2, "the probe never expired"
|
||||
|
||||
|
||||
def test_a_reachable_ollama_is_also_cached(monkeypatch):
|
||||
"""Both outcomes are cached. Caching only the failure would leave the happy
|
||||
path paying an HTTP round trip per row - cheap, but per row and pointless."""
|
||||
calls: list = []
|
||||
|
||||
class Ok:
|
||||
status_code = 200
|
||||
|
||||
def ok(url, **kwargs):
|
||||
calls.append(url)
|
||||
return Ok()
|
||||
|
||||
monkeypatch.setattr(ollama_service, "USE_OLLAMA", True)
|
||||
monkeypatch.setattr(ollama_service.requests, "get", ok)
|
||||
|
||||
assert [ollama_service._ensure_client() for _ in range(5)] == [True] * 5
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_disabled_stays_none_and_never_touches_the_network(monkeypatch):
|
||||
"""Three return values, not two: `system.py` tells "switched off" from
|
||||
"configured but down", and /api/health's `ollama` field means different
|
||||
things in each case. Collapsing this to a bool would break that.
|
||||
"""
|
||||
def never(*_args, **_kwargs):
|
||||
raise AssertionError("USE_OLLAMA is false - nothing may be requested")
|
||||
|
||||
monkeypatch.setattr(ollama_service, "USE_OLLAMA", False)
|
||||
monkeypatch.setattr(ollama_service.requests, "get", never)
|
||||
|
||||
assert ollama_service._ensure_client() is None
|
||||
Reference in New Issue
Block a user