173 lines
8.5 KiB
Bash
Executable File
173 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Post-deploy smoke test.
|
|
#
|
|
# ./scripts/check_deploy.sh # https://mcp.nearle.ai.in
|
|
# ./scripts/check_deploy.sh http://localhost:3000 # a local container
|
|
#
|
|
# ADMIN_PASSWORD=... ./scripts/check_deploy.sh # also prove sign-in WORKS
|
|
#
|
|
# ADMIN_PASSWORD is read from the environment and never stored here: a committed
|
|
# smoke test must not carry a live credential. Without it the positive sign-in
|
|
# check is skipped with a WARN rather than failing.
|
|
#
|
|
# Separates the three failures that all look alike from a browser:
|
|
# 502 everywhere - the container is not running (it exited at startup)
|
|
# 200 + database:false - the API is fine, Postgres is not
|
|
# 200 + database:true - working
|
|
set -uo pipefail
|
|
|
|
BASE="${1:-https://mcp.nearle.ai.in}"
|
|
pass=0; fail=0
|
|
|
|
hit() { curl -sS -m 20 -o /tmp/_body -w '%{http_code}' "$BASE$1" 2>/dev/null || echo 000; }
|
|
|
|
check() { # path expected label
|
|
local code; code=$(hit "$1")
|
|
if [ "$code" = "$2" ]; then printf ' \033[32mPASS\033[0m %-16s %s\n' "$1" "$3"; pass=$((pass+1))
|
|
else printf ' \033[31mFAIL\033[0m %-16s expected %s, got %s\n' "$1" "$2" "$code"; fail=$((fail+1)); fi
|
|
}
|
|
|
|
echo "Checking $BASE"
|
|
echo
|
|
echo "Is the container running at all?"
|
|
code=$(hit /)
|
|
if [ "$code" = "502" ] || [ "$code" = "000" ]; then
|
|
echo " FAIL / -> $code"
|
|
echo
|
|
echo " The container is not serving. It almost certainly exited at startup."
|
|
echo " Read the Dokploy logs; settings.py names the missing value explicitly."
|
|
echo " Confirm the image actually contains /app/.env - the build log must show"
|
|
echo " a 'COPY .env .' step, and .dockerignore must not list .env."
|
|
exit 1
|
|
fi
|
|
printf ' \033[32mPASS\033[0m %-16s container is up\n' "/"
|
|
pass=$((pass+1))
|
|
|
|
echo
|
|
echo "Routes that must not depend on the database:"
|
|
check /docs 200 "interactive API docs"
|
|
check /openapi.json 200 "OpenAPI schema"
|
|
check /api/health 200 "health endpoint answers"
|
|
|
|
echo
|
|
echo "Dependencies (reported in the body; 'false' does not mean the API is broken):"
|
|
health=$(curl -sS -m 20 "$BASE/api/health" 2>/dev/null)
|
|
db=$(printf '%s' "$health" | sed -n 's/.*"database":\([a-z]*\).*/\1/p')
|
|
ol=$(printf '%s' "$health" | sed -n 's/.*"ollama":\([a-z]*\).*/\1/p')
|
|
if [ "$db" = "true" ]; then printf ' \033[32mPASS\033[0m database connected\n'; pass=$((pass+1))
|
|
else
|
|
printf ' \033[33mWARN\033[0m database NOT connected\n'
|
|
echo " The API works; catalog pages will be empty. Check DB_HOST/DB_USER/"
|
|
echo " DB_PASSWORD/DB_NAME. A wrong password logs 'password authentication"
|
|
echo " failed' rather than a timeout."
|
|
fi
|
|
[ "$ol" = "true" ] \
|
|
&& printf ' \033[32mPASS\033[0m ollama connected\n' \
|
|
|| printf ' \033[33mWARN\033[0m ollama not connected (/api/chat unavailable; expected if USE_OLLAMA=false)\n'
|
|
|
|
echo
|
|
echo "Auth:"
|
|
code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' -X POST "$BASE/api/auth/login" \
|
|
-H 'Content-Type: application/json' -d '{"username":"admin","password":"definitely-not-the-password"}' 2>/dev/null)
|
|
if [ "$code" = "401" ]; then printf ' \033[32mPASS\033[0m wrong password rejected (401)\n'; pass=$((pass+1))
|
|
elif [ "$code" = "200" ]; then
|
|
printf ' \033[31mFAIL\033[0m a WRONG PASSWORD WAS ACCEPTED - AUTH_ALLOW_ANY_LOGIN is true.\n'
|
|
echo " Anyone who finds this host can sign in as admin. Set it to false."
|
|
fail=$((fail+1))
|
|
else printf ' \033[31mFAIL\033[0m login endpoint returned %s\n' "$code"; fail=$((fail+1)); fi
|
|
|
|
# The complementary half, and the one that actually catches a bad deploy. The
|
|
# check above passes just as happily when NOBODY can sign in - which is exactly
|
|
# the outage this script failed to notice: prod rejected the correct password
|
|
# while every check here stayed green. Note the negative check runs FIRST on
|
|
# purpose, since a successful login calls _clear_failures() and would otherwise
|
|
# hand the wrong-password probe a fresh throttle bucket.
|
|
if [ -n "${ADMIN_PASSWORD:-}" ]; then
|
|
code=$(curl -sS -m 20 -o /tmp/_login -w '%{http_code}' -X POST "$BASE/api/auth/login" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"username\":\"${ADMIN_USERNAME:-admin}\",\"password\":\"$ADMIN_PASSWORD\"}" 2>/dev/null)
|
|
if [ "$code" = "200" ] && grep -q access_token /tmp/_login 2>/dev/null; then
|
|
printf ' \033[32mPASS\033[0m correct password accepted (200 + token)\n'; pass=$((pass+1))
|
|
elif [ "$code" = "429" ]; then
|
|
printf ' \033[33mWARN\033[0m throttled (429) by the earlier failed attempt, not a\n'
|
|
echo " credential problem. Wait AUTH_LOCKOUT_SECONDS and rerun."
|
|
elif [ "$code" = "401" ]; then
|
|
printf ' \033[31mFAIL\033[0m the CORRECT password was rejected (401).\n'
|
|
echo " This deployment is not carrying the credential you think it is."
|
|
echo " The fingerprint check below says which way; the server log names"
|
|
echo " the reason - grep it for 'reason=' and for 'Auth config:'."
|
|
fail=$((fail+1))
|
|
else
|
|
printf ' \033[31mFAIL\033[0m login with the correct password returned %s\n' "$code"; fail=$((fail+1))
|
|
fi
|
|
rm -f /tmp/_login
|
|
else
|
|
printf ' \033[33mWARN\033[0m sign-in not proven to WORK - set ADMIN_PASSWORD to check that.\n'
|
|
echo " Rejecting a wrong password is only half the test; an image with a"
|
|
echo " stale hash passes that half while nobody can sign in."
|
|
fi
|
|
|
|
# Which credential is this deployment actually running? The fingerprint is a
|
|
# digest of the configured hash, so comparing it against the local file settles
|
|
# "is my config live?" without either side revealing a secret.
|
|
fp=$(printf '%s' "$health" | sed -n 's/.*"password_hash_fingerprint":"\([a-z0-9]*\)".*/\1/p')
|
|
hash_ok=$(printf '%s' "$health" | sed -n 's/.*"password_hash_valid":\([a-z]*\).*/\1/p')
|
|
hsrc=$(printf '%s' "$health" | sed -n 's/.*"password_hash_source":"\([a-z-]*\)".*/\1/p')
|
|
if [ -z "$fp" ]; then
|
|
printf ' \033[33mWARN\033[0m no auth diagnostics in /api/health - this deployment predates\n'
|
|
echo " them and needs a rebuild before it can be compared."
|
|
else
|
|
printf ' \033[32mPASS\033[0m credential fingerprint %s (from %s)\n' "$fp" "$hsrc"; pass=$((pass+1))
|
|
if [ "$hash_ok" = "false" ]; then
|
|
printf ' \033[31mFAIL\033[0m the configured password hash does not parse. NO password can\n'
|
|
echo " match it. Regenerate: python scripts/make_auth_secrets.py"
|
|
fail=$((fail+1))
|
|
fi
|
|
local_env="$(dirname "$0")/../.env.production"
|
|
# Probe interpreters by RUNNING one, not with `command -v`: on Windows,
|
|
# /c/.../WindowsApps/python3 is an App Store stub that resolves fine, prints a
|
|
# "Python was not found" notice and exits 0. Trusting the lookup made this
|
|
# comparison skip silently - which reads as "checked, matches", the exact kind
|
|
# of quiet pass this script exists to eliminate.
|
|
py=""
|
|
for candidate in python3 python py; do
|
|
if [ "$("$candidate" -c 'print(1)' 2>/dev/null)" = "1" ]; then py="$candidate"; break; fi
|
|
done
|
|
if [ ! -f "$local_env" ]; then
|
|
:
|
|
elif [ -z "$py" ]; then
|
|
printf ' \033[33mWARN\033[0m no working python on PATH - cannot compare the deployment\n'
|
|
echo " against local .env.production."
|
|
else
|
|
local_fp=$("$py" "$(dirname "$0")/make_auth_secrets.py" --fingerprint "$local_env" 2>/dev/null | sed -n 's/^fingerprint *: *//p')
|
|
if [ -z "$local_fp" ]; then
|
|
printf ' \033[33mWARN\033[0m could not read a fingerprint out of %s\n' "$local_env"
|
|
elif [ "$local_fp" = "$fp" ]; then
|
|
printf ' \033[32mPASS\033[0m matches local .env.production\n'; pass=$((pass+1))
|
|
else
|
|
printf ' \033[31mFAIL\033[0m local .env.production is %s, deployment is %s\n' "$local_fp" "$fp"
|
|
if [ "$hsrc" = "process-env" ]; then
|
|
echo " It came from the process environment, which OVERRIDES the .env"
|
|
echo " baked into the image. Clear AUTH_ADMIN_PASSWORD_HASH from the"
|
|
echo " deployment platform's Environment tab."
|
|
else
|
|
echo " It came from the image's own .env, so that image was built from a"
|
|
echo " different .env.production. This needs a REBUILD - the Dockerfile"
|
|
echo " copies the file at BUILD time, so a restart will not pick it up."
|
|
fi
|
|
fail=$((fail+1))
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "MCP:"
|
|
code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/mcp/info" 2>/dev/null)
|
|
[ "$code" = "401" ] \
|
|
&& { printf ' \033[32mPASS\033[0m /api/mcp/info guarded (401 without a token)\n'; pass=$((pass+1)); } \
|
|
|| printf ' \033[33mWARN\033[0m /api/mcp/info returned %s (expected 401)\n' "$code"
|
|
|
|
echo
|
|
echo "-------- $pass passed, $fail failed --------"
|
|
[ "$fail" -eq 0 ]
|