#!/usr/bin/env bash # Post-deploy smoke test. # # ./scripts/check_deploy.sh # https://mcp.nearle.ai.in # ./scripts/check_deploy.sh http://localhost:3000 # a local container # # ADMIN_PASSWORD=... ./scripts/check_deploy.sh # also prove sign-in WORKS # # ADMIN_PASSWORD is read from the environment and never stored here: a committed # smoke test must not carry a live credential. Without it the positive sign-in # check is skipped with a WARN rather than failing. # # Separates the three failures that all look alike from a browser: # 502 everywhere - the container is not running (it exited at startup) # 200 + database:false - the API is fine, Postgres is not # 200 + database:true - working set -uo pipefail BASE="${1:-https://mcp.nearle.ai.in}" pass=0; fail=0 hit() { curl -sS -m 20 -o /tmp/_body -w '%{http_code}' "$BASE$1" 2>/dev/null || echo 000; } check() { # path expected label local code; code=$(hit "$1") if [ "$code" = "$2" ]; then printf ' \033[32mPASS\033[0m %-16s %s\n' "$1" "$3"; pass=$((pass+1)) else printf ' \033[31mFAIL\033[0m %-16s expected %s, got %s\n' "$1" "$2" "$code"; fail=$((fail+1)); fi } echo "Checking $BASE" echo echo "Is the container running at all?" code=$(hit /) if [ "$code" = "502" ] || [ "$code" = "000" ]; then echo " FAIL / -> $code" echo echo " The container is not serving. It almost certainly exited at startup." echo " Read the Dokploy logs; settings.py names the missing value explicitly." echo " Confirm the image actually contains /app/.env - the build log must show" echo " a 'COPY .env .' step, and .dockerignore must not list .env." exit 1 fi printf ' \033[32mPASS\033[0m %-16s container is up\n' "/" pass=$((pass+1)) echo echo "Routes that must not depend on the database:" check /docs 200 "interactive API docs" check /openapi.json 200 "OpenAPI schema" check /api/health 200 "health endpoint answers" echo echo "Dependencies (reported in the body; 'false' does not mean the API is broken):" health=$(curl -sS -m 20 "$BASE/api/health" 2>/dev/null) db=$(printf '%s' "$health" | sed -n 's/.*"database":\([a-z]*\).*/\1/p') ol=$(printf '%s' "$health" | sed -n 's/.*"ollama":\([a-z]*\).*/\1/p') if [ "$db" = "true" ]; then printf ' \033[32mPASS\033[0m database connected\n'; pass=$((pass+1)) else printf ' \033[33mWARN\033[0m database NOT connected\n' echo " The API works; catalog pages will be empty. Check DB_HOST/DB_USER/" echo " DB_PASSWORD/DB_NAME. A wrong password logs 'password authentication" echo " failed' rather than a timeout." fi [ "$ol" = "true" ] \ && printf ' \033[32mPASS\033[0m ollama connected\n' \ || printf ' \033[33mWARN\033[0m ollama not connected (/api/chat unavailable; expected if USE_OLLAMA=false)\n' echo echo "Auth:" code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' -X POST "$BASE/api/auth/login" \ -H 'Content-Type: application/json' -d '{"username":"admin","password":"definitely-not-the-password"}' 2>/dev/null) if [ "$code" = "401" ]; then printf ' \033[32mPASS\033[0m wrong password rejected (401)\n'; pass=$((pass+1)) elif [ "$code" = "200" ]; then printf ' \033[31mFAIL\033[0m a WRONG PASSWORD WAS ACCEPTED - AUTH_ALLOW_ANY_LOGIN is true.\n' echo " Anyone who finds this host can sign in as admin. Set it to false." fail=$((fail+1)) else printf ' \033[31mFAIL\033[0m login endpoint returned %s\n' "$code"; fail=$((fail+1)); fi # The complementary half, and the one that actually catches a bad deploy. The # check above passes just as happily when NOBODY can sign in - which is exactly # the outage this script failed to notice: prod rejected the correct password # while every check here stayed green. Note the negative check runs FIRST on # purpose, since a successful login calls _clear_failures() and would otherwise # hand the wrong-password probe a fresh throttle bucket. if [ -n "${ADMIN_PASSWORD:-}" ]; then code=$(curl -sS -m 20 -o /tmp/_login -w '%{http_code}' -X POST "$BASE/api/auth/login" \ -H 'Content-Type: application/json' \ -d "{\"username\":\"${ADMIN_USERNAME:-admin}\",\"password\":\"$ADMIN_PASSWORD\"}" 2>/dev/null) if [ "$code" = "200" ] && grep -q access_token /tmp/_login 2>/dev/null; then printf ' \033[32mPASS\033[0m correct password accepted (200 + token)\n'; pass=$((pass+1)) elif [ "$code" = "429" ]; then printf ' \033[33mWARN\033[0m throttled (429) by the earlier failed attempt, not a\n' echo " credential problem. Wait AUTH_LOCKOUT_SECONDS and rerun." elif [ "$code" = "401" ]; then printf ' \033[31mFAIL\033[0m the CORRECT password was rejected (401).\n' echo " This deployment is not carrying the credential you think it is." echo " The fingerprint check below says which way; the server log names" echo " the reason - grep it for 'reason=' and for 'Auth config:'." fail=$((fail+1)) else printf ' \033[31mFAIL\033[0m login with the correct password returned %s\n' "$code"; fail=$((fail+1)) fi rm -f /tmp/_login else printf ' \033[33mWARN\033[0m sign-in not proven to WORK - set ADMIN_PASSWORD to check that.\n' echo " Rejecting a wrong password is only half the test; an image with a" echo " stale hash passes that half while nobody can sign in." fi # Which credential is this deployment actually running? The fingerprint is a # digest of the configured hash, so comparing it against the local file settles # "is my config live?" without either side revealing a secret. fp=$(printf '%s' "$health" | sed -n 's/.*"password_hash_fingerprint":"\([a-z0-9]*\)".*/\1/p') hash_ok=$(printf '%s' "$health" | sed -n 's/.*"password_hash_valid":\([a-z]*\).*/\1/p') hsrc=$(printf '%s' "$health" | sed -n 's/.*"password_hash_source":"\([a-z-]*\)".*/\1/p') if [ -z "$fp" ]; then printf ' \033[33mWARN\033[0m no auth diagnostics in /api/health - this deployment predates\n' echo " them and needs a rebuild before it can be compared." else printf ' \033[32mPASS\033[0m credential fingerprint %s (from %s)\n' "$fp" "$hsrc"; pass=$((pass+1)) if [ "$hash_ok" = "false" ]; then printf ' \033[31mFAIL\033[0m the configured password hash does not parse. NO password can\n' echo " match it. Regenerate: python scripts/make_auth_secrets.py" fail=$((fail+1)) fi local_env="$(dirname "$0")/../.env.production" # Probe interpreters by RUNNING one, not with `command -v`: on Windows, # /c/.../WindowsApps/python3 is an App Store stub that resolves fine, prints a # "Python was not found" notice and exits 0. Trusting the lookup made this # comparison skip silently - which reads as "checked, matches", the exact kind # of quiet pass this script exists to eliminate. py="" for candidate in python3 python py; do if [ "$("$candidate" -c 'print(1)' 2>/dev/null)" = "1" ]; then py="$candidate"; break; fi done if [ ! -f "$local_env" ]; then : elif [ -z "$py" ]; then printf ' \033[33mWARN\033[0m no working python on PATH - cannot compare the deployment\n' echo " against local .env.production." else local_fp=$("$py" "$(dirname "$0")/make_auth_secrets.py" --fingerprint "$local_env" 2>/dev/null | sed -n 's/^fingerprint *: *//p') if [ -z "$local_fp" ]; then printf ' \033[33mWARN\033[0m could not read a fingerprint out of %s\n' "$local_env" elif [ "$local_fp" = "$fp" ]; then printf ' \033[32mPASS\033[0m matches local .env.production\n'; pass=$((pass+1)) else printf ' \033[31mFAIL\033[0m local .env.production is %s, deployment is %s\n' "$local_fp" "$fp" if [ "$hsrc" = "process-env" ]; then echo " It came from the process environment, which OVERRIDES the .env" echo " baked into the image. Clear AUTH_ADMIN_PASSWORD_HASH from the" echo " deployment platform's Environment tab." else echo " It came from the image's own .env, so that image was built from a" echo " different .env.production. This needs a REBUILD - the Dockerfile" echo " copies the file at BUILD time, so a restart will not pick it up." fi fail=$((fail+1)) fi fi fi echo echo "MCP:" code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/mcp/info" 2>/dev/null) [ "$code" = "401" ] \ && { printf ' \033[32mPASS\033[0m /api/mcp/info guarded (401 without a token)\n'; pass=$((pass+1)); } \ || printf ' \033[33mWARN\033[0m /api/mcp/info returned %s (expected 401)\n' "$code" echo echo "-------- $pass passed, $fail failed --------" [ "$fail" -eq 0 ]