From f698720ee27e39c1178334caa2e819dc1c1debb6 Mon Sep 17 00:00:00 2001 From: sriram Date: Thu, 27 Aug 2026 16:41:25 +0530 Subject: [PATCH] excel file update --- .env.production | 22 ++++- .env.production.bak.20260827 | 128 +++++++++++++++++++++++++ app/api/schemas.py | 25 ++++- app/infrastructure/security.py | 44 +++++++++ app/infrastructure/settings.py | 23 +++++ app/main.py | 8 +- tests/test_auth_diagnostics.py | 165 ++++++++++++++++++++++++++++++++- 7 files changed, 411 insertions(+), 4 deletions(-) create mode 100644 .env.production.bak.20260827 diff --git a/.env.production b/.env.production index 746b92a..e54b702 100644 --- a/.env.production +++ b/.env.production @@ -55,7 +55,27 @@ AUTH_LOCKOUT_SECONDS=300 # anyone who finds mcp.nearle.ai.in signs in as admin by typing anything at all. AUTH_ALLOW_ANY_LOGIN=false -# Machine consumers. Empty: MCP clients authenticate with a login token instead. +# Machine consumers. `name:role:secret` triples, comma-separated; keyed by the +# secret, so deleting one entry revokes exactly one caller and leaves the rest +# working. Role MUST be `admin` for the store-catalog / catalog-generate / +# training routes: those guard with require_admin, which is a ROLE check, and +# `admin` is a superuser - a key here unlocks every admin endpoint, not just +# the one it was issued for. +# +# DELIBERATELY EMPTY HERE. The real value lives in the Dokploy Environment tab: +# +# 1. This file is committed. It already carries the DB password, S3 keys and +# the token-signing secret; a per-consumer API key is the one credential +# that gets issued and revoked often, and it does not belong in git. +# 2. Dockerfile does `COPY .env.production .env`, so a value here is baked at +# BUILD time - issuing or revoking a key would mean rebuilding an image +# that installs CPU torch, which has already failed once on disk space. +# settings.py calls load_dotenv() WITHOUT override=True, so the tab's value +# wins and takes effect on a plain restart. +# +# Consequence: /api/health reports api_keys_source "process-env" for this one, +# and that is correct here, not a warning. A value set below would be silently +# ignored while the tab is populated - so leave it empty. API_KEYS= # --- Postgres / pgvector --------------------------------------------------- diff --git a/.env.production.bak.20260827 b/.env.production.bak.20260827 new file mode 100644 index 0000000..746b92a --- /dev/null +++ b/.env.production.bak.20260827 @@ -0,0 +1,128 @@ +# Deployment configuration for mcp.nearle.ai.in. +# +# Committed at the repo owner's instruction so the deploy does not depend on +# re-entering config in the Dokploy UI. Everything needed to boot is here; no +# environment variables are required in Dokploy any more. +# +# A real environment variable still overrides anything set here - settings.py +# calls load_dotenv() without override=True, so the process environment wins. +# That is the escape hatch for changing a value without a commit. +# +# WHAT IS IN THIS FILE: live database, S3 and Google credentials, and the key +# that signs every access token. Anyone with read access to this repository has +# all of it, and git history keeps it after any rotation. + +# --- Ports ----------------------------------------------------------------- +# Dokploy routes the domain to 3000; 8000 is kept for the vite dev proxy and +# docker-compose. serve.py binds both. +PORTS=3000,8000 + +# --- CORS ------------------------------------------------------------------ +# The FRONTEND's origin, not this API's. Wrong value = the browser blocks every +# response while the server logs healthy 200s - which is exactly what happened +# here: this was set to catalogue.nearle.ai.in, but the domain Traefik actually +# serves is spelled "catalouge". That host does not even resolve, so nothing +# pointed at the mistake except a silently failing UI. +# +# Both spellings are listed so this keeps working if the typo is ever corrected +# in Dokploy. Exact origins, never a wildcard: the app sends an Authorization +# header, and browsers reject credentialed requests to a wildcard origin. +API_CORS_ORIGINS=https://catalouge.nearle.ai.in,https://catalogue.nearle.ai.in + +# --- Authentication -------------------------------------------------------- +AUTH_ENABLED=true + +# One interactive account: admin. The `user` account is disabled here by +# leaving AUTH_USER_PASSWORD_HASH unset - auth.py omits any account whose +# hash is empty, so only admin can sign in. +# +# AUTH_SECRET_KEY stays as generated for this deployment; rotating it would +# invalidate every token already issued. +# Sign-in password for the hash below: admin / admin123. +AUTH_SECRET_KEY=4Kmyr4Cjf_kdUIq_4EGxo5vFHfCT5_uKVR3eouszB8Le6F0n45m7eDY94_KJoqSz +AUTH_ADMIN_USERNAME=admin +AUTH_ADMIN_PASSWORD_HASH=pbkdf2_sha256$600000$S28AccXqQnNNElilb0JFsg==$IGPLr56iqwkwbrM5skVoXBMDFEfpZIKUA7NiaM74cmk= +# AUTH_USER_USERNAME=user +# AUTH_USER_PASSWORD_HASH= (unset: the `user` account is disabled) + +AUTH_TOKEN_TTL_MINUTES=720 +AUTH_MAX_LOGIN_ATTEMPTS=10 +AUTH_LOCKOUT_SECONDS=300 + +# MUST stay false here. The development .env has this true, where it is a +# convenience: it skips the password check entirely, so any username signs in +# and `admin` gets the admin pages. On a host published to the internet it means +# anyone who finds mcp.nearle.ai.in signs in as admin by typing anything at all. +AUTH_ALLOW_ANY_LOGIN=false + +# Machine consumers. Empty: MCP clients authenticate with a login token instead. +API_KEYS= + +# --- Postgres / pgvector --------------------------------------------------- +# DB_NAME is not set in the development .env, so it falls back to settings.py's +# default. Stated explicitly here so the deployment does not depend on that +# default staying the same. +USE_PGVECTOR=true +DB_HOST=31.97.228.132 +DB_PORT=6054 +DB_NAME=pgvector +DB_USER=admin +# The single quotes are PART OF THE PASSWORD, not shell/dotenv syntax. The outer +# double quotes are what dotenv strips, leaving 'Package@321#' including quotes. +# Writing it bare as Package@321# is what made every connection fail with +# "password authentication failed for user admin", which surfaces as +# /api/health reporting "database": false and an empty catalog on every page - +# the API looks healthy and the database looks empty. Do not "tidy" the quotes. +DB_PASSWORD="'Package@321#'" + +# --- Embeddings ------------------------------------------------------------ +USE_EMBEDDINGS=true +EMBEDDINGS_MODEL=sentence-transformers/all-MiniLM-L6-v2 +EMBEDDINGS_DIM=384 + +# --- Ollama (local LLM, powers /api/chat) ---------------------------------- +# Off, because the development value (http://localhost:11434) cannot work from +# inside a container: there, localhost is the container itself, not the VPS +# host. Left on with nothing listening, /api/chat fails AND every healthcheck +# takes ~3s longer, because the health handler probes Ollama with a 3s timeout. +# +# To enable: set USE_OLLAMA=true and point OLLAMA_BASE_URL at something the +# container can actually reach - http://host.docker.internal:11434 with a +# host-gateway mapping, the VPS's LAN IP, or an ollama service name. +USE_OLLAMA=false +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_MODEL_NAME=qwen2.5:1.5b +OLLAMA_TIMEOUT_SECONDS=120 + +# --- DigitalOcean Spaces (product image storage) --------------------------- +USE_S3=true +S3_ACCESS_KEY=DO801G8Q8JAZKF49U3WJ +S3_SECRET_KEY=lBQExYfkVqH+ybmGVmQH5MkThBbrIohA/VQLgcPUvug +S3_ENDPOINT=https://nearle.sgp1.digitaloceanspaces.com +S3_BUCKET=nearle +S3_REGION=sgp1 + +# --- Google Custom Search (optional image source) -------------------------- +USE_GOOGLE_CSE=true +GOOGLE_API_KEY=AIzaSyBY4pIO_Fp5FCMqeVxDNcfalzdWNHJWVn0 +GOOGLE_CSE_ID=9745cbd96dd164562 + +# --- Open-source image sources (no key needed) ----------------------------- +USE_DDG_IMAGES=true +USE_OPEN_FACTS=true +USE_WIKIMEDIA=true +# The Playwright browser binary is NOT installed in the image (see Dockerfile), +# so this tier is skipped at runtime regardless. false stops it being attempted. +USE_PLAYWRIGHT_FALLBACK=false + +MIN_IMAGE_BYTES=3000 + +# --- Product validation ---------------------------------------------------- +ENABLE_PRODUCT_VALIDATION=true +VALIDATION_REJECT_THRESHOLD=0.35 +VALIDATION_REVIEW_THRESHOLD=0.70 + +# --- RAG ------------------------------------------------------------------- +RAG_DEFAULT_TOP_K=5 +RAG_MAX_TOP_K=15 +RAG_MAX_CONTEXT_CHARS=4000 diff --git a/app/api/schemas.py b/app/api/schemas.py index 31f8f59..f2a78c4 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -63,6 +63,20 @@ class SourceProductOut(BaseModel): # Health # --------------------------------------------------------------------------- +class ApiKeyInfoOut(BaseModel): + """One configured machine consumer, named but never quoted. + + `fingerprint` is a truncated digest of name+secret, not the secret. It exists + so a caller who was issued a key can confirm THAT key is the one this + deployment loaded - the question a 401 cannot answer, since an undeployed key + and a wrong key fail identically. + """ + + name: str + role: str + fingerprint: str + + class AuthConfigOut(BaseModel): """ The effective auth configuration, reported by /api/health. @@ -73,7 +87,9 @@ class AuthConfigOut(BaseModel): already the documented one, allow_any_login=true is a fact an operator urgently needs (and an attacker discovers with a single login attempt anyway), and the fingerprint is a truncated hash of a salted digest, not a - password. What it buys is a one-command answer to "is this deployment + password. The API key block follows the same rule: it names which consumers + are configured and fingerprints their keys, so a caller can tell an + undeployed key from a rejected one, but it never renders a secret. What it buys is a one-command answer to "is this deployment running the config I think it is?" - compare the fingerprint here against the one printed by scripts/make_auth_secrets.py --fingerprint. """ @@ -87,6 +103,13 @@ class AuthConfigOut(BaseModel): # "process-env" | "env-file" | "default" - which one actually won. admin_username_source: str password_hash_source: str + # Machine consumers. Names and fingerprints only - the secrets themselves are + # never rendered here, and _parse_api_keys enforces enough entropy that the + # fingerprints do not give them away. Defaulted so a client of this schema + # still validates against a deployment predating these fields. + api_keys_count: int = 0 + api_keys: List[ApiKeyInfoOut] = Field(default_factory=list) + api_keys_source: str = "default" class HealthOut(BaseModel): diff --git a/app/infrastructure/security.py b/app/infrastructure/security.py index f152c4c..1b2e01f 100644 --- a/app/infrastructure/security.py +++ b/app/infrastructure/security.py @@ -186,6 +186,40 @@ def password_hash_fingerprint(encoded: str) -> str: return hashlib.sha256(encoded.strip().strip("'\"").encode("utf-8")).hexdigest()[:12] +def api_key_fingerprint(name: str, secret: str) -> str: + """ + A short, non-reversible identifier for a configured API key. + + Same purpose as password_hash_fingerprint - say *which* credential is loaded + without moving the credential - but the safety argument is different and + worth stating. That function digests an encoded hash which already embeds a + 16-byte random salt. An API key has no salt, so the name is mixed in here to + keep two consumers that were mistakenly issued the same secret from + fingerprinting identically, and settings._parse_api_keys enforces a minimum + secret length so the digest cannot be walked back with a wordlist. + """ + if not secret: + return "" + cleaned = secret.strip().strip("'\"") + material = f"{name}:{cleaned}" + return hashlib.sha256(material.encode("utf-8")).hexdigest()[:12] + + +def describe_api_keys() -> List[Dict[str, object]]: + """Every configured key as {name, role, fingerprint}, sorted by name. + + Sorted so two deployments' /api/health output can be diffed line for line; + API_KEYS is keyed by secret, whose iteration order says nothing useful. + """ + return sorted( + ( + {"name": name, "role": role, "fingerprint": api_key_fingerprint(name, secret)} + for secret, (name, role) in API_KEYS.items() + ), + key=lambda entry: entry["name"], + ) + + def describe_password_hash(encoded: str) -> Dict[str, object]: """A loggable/publishable summary of a configured digest. Never its bytes.""" parsed = _parse_encoded_hash(encoded) @@ -211,6 +245,13 @@ def auth_config_summary() -> Dict[str, object]: means the container's own environment supplied it and the .env file baked into the image was ignored - which is invisible from anywhere else, and is precisely how a corrected credential can keep failing after a redeploy. + + The same argument is why the API keys are summarised here. backend/Dockerfile + copies .env.production in at BUILD time, so a key added to that file and then + merely restarted is not present in the running process - and from outside, + an undeployed key is indistinguishable from a wrong one, because both are + just a 401. Publishing the names and fingerprints answers "is my key on this + deployment?" without anyone having to send the secret to find out. """ described = describe_password_hash(AUTH_ADMIN_PASSWORD_HASH) return { @@ -222,6 +263,9 @@ def auth_config_summary() -> Dict[str, object]: "password_hash_fingerprint": described["fingerprint"], "admin_username_source": config_source("AUTH_ADMIN_USERNAME"), "password_hash_source": config_source("AUTH_ADMIN_PASSWORD_HASH"), + "api_keys_count": len(API_KEYS), + "api_keys": describe_api_keys(), + "api_keys_source": config_source("API_KEYS"), } diff --git a/app/infrastructure/settings.py b/app/infrastructure/settings.py index 422f4e1..fd8a0f6 100644 --- a/app/infrastructure/settings.py +++ b/app/infrastructure/settings.py @@ -386,6 +386,12 @@ AUTH_LOCKOUT_SECONDS = int(os.getenv("AUTH_LOCKOUT_SECONDS", "300")) AUTH_ALLOW_ANY_LOGIN = _bool("AUTH_ALLOW_ANY_LOGIN", "false") +# Shortest acceptable API key secret. token_urlsafe(32) yields 43 characters, so +# this rejects hand-typed values without rejecting anything the documented +# generator produces. +API_KEY_MIN_LENGTH = 32 + + def _parse_api_keys(raw: str) -> dict: """ Parse ``API_KEYS`` - ``name:role:secret`` triples, comma-separated. @@ -393,6 +399,16 @@ def _parse_api_keys(raw: str) -> dict: Keyed by secret because that is what an inbound request presents. One entry per consumer is the point: a shared key cannot be revoked for one caller without breaking all of them. + + Secrets must be at least API_KEY_MIN_LENGTH characters. That is not about + guessing the key over the network - the lockout and the network itself make + online brute force impractical - but about what /api/health publishes. It + reports a truncated digest of every configured key so a deployment can be + checked against the config it was built from, and a digest of a *raw* secret + is only safe when the secret is unguessable offline. An admin password hash + embeds a random salt, so its fingerprint discloses nothing; an API key has no + salt, and a hand-picked "changeme" would fall to a wordlist in seconds. + Generate one with: python -c "import secrets; print(secrets.token_urlsafe(32))" """ parsed: dict = {} for entry in raw.split(","): @@ -412,6 +428,13 @@ def _parse_api_keys(raw: str) -> dict: ) if not secret: raise RuntimeError(f"API_KEYS entry {name!r} has an empty secret.") + if len(secret) < API_KEY_MIN_LENGTH: + raise RuntimeError( + f"API_KEYS entry {name!r} has a {len(secret)}-character secret; at least " + f"{API_KEY_MIN_LENGTH} are required, because /api/health publishes a digest " + f"of it. Generate one with: " + f"python -c \"import secrets; print(secrets.token_urlsafe(32))\"" + ) parsed[secret] = (name, role) return parsed diff --git a/app/main.py b/app/main.py index 53ceb0c..0db6465 100644 --- a/app/main.py +++ b/app/main.py @@ -205,7 +205,8 @@ if API_CORS_ORIGINS and all( _auth_cfg = auth_config_summary() logger.info( "Auth config: enabled=%s allow_any_login=%s admin_username=%r " - "hash=%s/%s fingerprint=%s source=%s (username source=%s)", + "hash=%s/%s fingerprint=%s source=%s (username source=%s) " + "api_keys=%d/%s %s", _auth_cfg["enabled"], _auth_cfg["allow_any_login"], _auth_cfg["admin_username"], @@ -214,6 +215,11 @@ logger.info( _auth_cfg["password_hash_fingerprint"] or "(none)", _auth_cfg["password_hash_source"], _auth_cfg["admin_username_source"], + _auth_cfg["api_keys_count"], + _auth_cfg["api_keys_source"], + # Names, not secrets. A key added to .env.production but only restarted into + # a running container never appears here - which is the whole point. + [k["name"] for k in _auth_cfg["api_keys"]] or "(none)", ) if cleaned_env_names(): logger.warning( diff --git a/tests/test_auth_diagnostics.py b/tests/test_auth_diagnostics.py index b88e094..23e4863 100644 --- a/tests/test_auth_diagnostics.py +++ b/tests/test_auth_diagnostics.py @@ -21,13 +21,15 @@ from __future__ import annotations import pytest from app.infrastructure.security import ( + api_key_fingerprint, auth_config_summary, + describe_api_keys, describe_password_hash, hash_is_wellformed, hash_password, password_hash_fingerprint, ) -from app.infrastructure.settings import config_source +from app.infrastructure.settings import API_KEY_MIN_LENGTH, _parse_api_keys, config_source from tests.conftest import TEST_ADMIN_PASSWORD @@ -164,3 +166,164 @@ def test_health_stays_ok_shaped_when_auth_is_misconfigured(client, monkeypatch): assert body["auth"]["password_hash_valid"] is False assert body["status"] in {"ok", "degraded"} # decided by db/ollama only + + +# --------------------------------------------------------------------------- +# API keys +# --------------------------------------------------------------------------- +# Same incident, one layer out. A key added to .env.production and then merely +# restarted into a running container is absent from the process, because the +# Dockerfile copies that file in at BUILD time - and from outside, an undeployed +# key and a wrong key are both just a 401. These pin the ability to tell them +# apart without anyone sending the secret to find out. + +_GOOD_SECRET = "cs3JwvApS5Je_Qfe1sNYq6YtUBDDeqp4OEgy2_41sQg" + + +def test_api_key_fingerprint_is_stable_and_hex(): + fp = api_key_fingerprint("partner", _GOOD_SECRET) + + assert fp == api_key_fingerprint("partner", _GOOD_SECRET) + assert len(fp) == 12 + assert all(c in "0123456789abcdef" for c in fp) + + +def test_api_key_fingerprint_differs_when_the_secret_does(): + assert api_key_fingerprint("partner", _GOOD_SECRET) != api_key_fingerprint( + "partner", _GOOD_SECRET[:-1] + "X" + ) + + +def test_api_key_fingerprint_separates_consumers_sharing_a_secret(): + """The name is mixed in, so two consumers mistakenly issued the same secret + do not report the same fingerprint - which would hide the mistake behind the + very field meant to reveal it.""" + assert api_key_fingerprint("console-a", _GOOD_SECRET) != api_key_fingerprint( + "console-b", _GOOD_SECRET + ) + + +@pytest.mark.parametrize("wrapper", ["{}", "'{}'", '"{}"', " {} "]) +def test_api_key_fingerprint_ignores_quotes_and_whitespace(wrapper): + """A value pasted into a deployment platform's Environment tab arrives + wrapped often enough that settings strips it; the fingerprint must agree, + or comparing two ends reports a mismatch that is not real.""" + assert api_key_fingerprint("partner", wrapper.format(_GOOD_SECRET)) == ( + api_key_fingerprint("partner", _GOOD_SECRET) + ) + + +def test_api_key_fingerprint_discloses_no_part_of_the_secret(): + """Served unauthenticated, so it must be a digest OF the key, not a piece.""" + fp = api_key_fingerprint("partner", _GOOD_SECRET) + + assert fp not in _GOOD_SECRET + for start in range(len(fp) - 5): + assert fp[start : start + 6] not in _GOOD_SECRET + + +def test_absent_secret_fingerprints_as_empty(): + assert api_key_fingerprint("partner", "") == "" + + +def test_describe_api_keys_is_sorted_by_name(monkeypatch): + """API_KEYS is keyed by secret, whose order says nothing. Sorting is what + lets two deployments' output be diffed line for line.""" + from app.infrastructure import security + + monkeypatch.setattr( + security, "API_KEYS", + {_GOOD_SECRET: ("zulu", "user"), _GOOD_SECRET[::-1]: ("alpha", "admin")}, + ) + + assert [k["name"] for k in describe_api_keys()] == ["alpha", "zulu"] + assert [k["role"] for k in describe_api_keys()] == ["admin", "user"] + + +# --------------------------------------------------------------------------- +# API keys on /api/health +# --------------------------------------------------------------------------- +def test_health_reports_the_keys_this_deployment_actually_loaded(client): + """Against the harness's own API_KEYS, not a monkeypatched one - this is the + end-to-end wiring from settings through the summary to the response body.""" + from tests.conftest import TEST_API_KEY + + auth = client.get("/api/health").json()["auth"] + + assert auth["api_keys_count"] == 1 + assert auth["api_keys"] == [{ + "name": "test-machine", + "role": "user", + "fingerprint": api_key_fingerprint("test-machine", TEST_API_KEY), + }] + + +def test_health_reports_an_empty_list_when_no_keys_are_configured(client, monkeypatch): + """The state production was in while the colleague's console got 401s: auth + enabled, admin login working, and not one machine consumer deployed.""" + from app.infrastructure import security + + monkeypatch.setattr(security, "API_KEYS", {}) + auth = client.get("/api/health").json()["auth"] + + assert auth["api_keys_count"] == 0 + assert auth["api_keys"] == [] + + +def test_health_names_configured_keys_and_fingerprints_them(client, monkeypatch): + from app.infrastructure import security + + monkeypatch.setattr(security, "API_KEYS", {_GOOD_SECRET: ("colleague-console", "admin")}) + auth = client.get("/api/health").json()["auth"] + + assert auth["api_keys_count"] == 1 + assert auth["api_keys"] == [{ + "name": "colleague-console", + "role": "admin", + "fingerprint": api_key_fingerprint("colleague-console", _GOOD_SECRET), + }] + + +def test_health_never_exposes_an_api_key_secret(client, monkeypatch): + """The leak canary, extended to machine credentials.""" + from app.infrastructure import security + + monkeypatch.setattr(security, "API_KEYS", {_GOOD_SECRET: ("colleague-console", "admin")}) + body = client.get("/api/health").text + + assert _GOOD_SECRET not in body + for start in range(0, len(_GOOD_SECRET) - 7): + assert _GOOD_SECRET[start : start + 8] not in body + + +def test_health_reports_where_the_keys_came_from(client): + """Which of the two config sources won. Unlike the admin hash - where + "process-env" flags a stale Environment tab shadowing the image - API_KEYS is + deliberately supplied by that tab, so "process-env" is the expected value in + production and "env-file" would mean the tab entry has gone missing.""" + auth = client.get("/api/health").json()["auth"] + + assert auth["api_keys_source"] in {"process-env", "env-file", "default"} + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- +def test_parse_api_keys_accepts_a_generated_secret(): + parsed = _parse_api_keys(f"partner:admin:{_GOOD_SECRET}") + + assert parsed == {_GOOD_SECRET: ("partner", "admin")} + + +def test_parse_api_keys_rejects_a_secret_too_short_to_fingerprint_safely(): + """A raw key carries no salt, so publishing its digest is only safe while the + key itself is unguessable offline. A hand-picked one must be refused at + startup rather than quietly fingerprinted onto a public endpoint.""" + with pytest.raises(RuntimeError, match="at least"): + _parse_api_keys("partner:admin:changeme") + + +def test_parse_api_keys_length_limit_admits_the_documented_generator(): + import secrets as _secrets + + assert len(_secrets.token_urlsafe(32)) >= API_KEY_MIN_LENGTH