diff --git a/backend/command-api/src/worker-app.js b/backend/command-api/src/worker-app.js new file mode 100644 index 00000000..8ccd96ed --- /dev/null +++ b/backend/command-api/src/worker-app.js @@ -0,0 +1,46 @@ +import express from 'express'; +import pino from 'pino'; +import pinoHttp from 'pino-http'; + +const logger = pino({ level: process.env.LOG_LEVEL || 'info' }); + +export function createWorkerApp({ dispatch = async () => ({}) } = {}) { + const app = express(); + + app.use( + pinoHttp({ + logger, + }) + ); + app.use(express.json({ limit: '256kb' })); + + app.get('/health', (_req, res) => { + res.status(200).json({ ok: true, service: 'notification-worker-v2' }); + }); + + app.get('/readyz', (_req, res) => { + res.status(200).json({ ok: true, service: 'notification-worker-v2' }); + }); + + app.post('/tasks/dispatch-notifications', async (req, res) => { + try { + const summary = await dispatch(); + res.status(200).json({ ok: true, summary }); + } catch (error) { + req.log?.error?.({ err: error }, 'notification dispatch failed'); + res.status(500).json({ + ok: false, + error: error?.message || String(error), + }); + } + }); + + app.use((_req, res) => { + res.status(404).json({ + code: 'NOT_FOUND', + message: 'Route not found', + }); + }); + + return app; +} diff --git a/backend/command-api/src/worker-server.js b/backend/command-api/src/worker-server.js new file mode 100644 index 00000000..fbff2ec4 --- /dev/null +++ b/backend/command-api/src/worker-server.js @@ -0,0 +1,12 @@ +import { createWorkerApp } from './worker-app.js'; +import { dispatchPendingNotifications } from './services/notification-dispatcher.js'; + +const port = Number(process.env.PORT || 8080); +const app = createWorkerApp({ + dispatch: () => dispatchPendingNotifications(), +}); + +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`krow-notification-worker listening on port ${port}`); +}); diff --git a/backend/command-api/test/notification-worker.test.js b/backend/command-api/test/notification-worker.test.js new file mode 100644 index 00000000..a4865b55 --- /dev/null +++ b/backend/command-api/test/notification-worker.test.js @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import request from 'supertest'; +import { createWorkerApp } from '../src/worker-app.js'; + +test('GET /readyz returns healthy response', async () => { + const app = createWorkerApp(); + const res = await request(app).get('/readyz'); + + assert.equal(res.status, 200); + assert.equal(res.body.ok, true); + assert.equal(res.body.service, 'notification-worker-v2'); +}); + +test('POST /tasks/dispatch-notifications returns dispatch summary', async () => { + const app = createWorkerApp({ + dispatch: async () => ({ + claimed: 2, + sent: 2, + }), + }); + + const res = await request(app) + .post('/tasks/dispatch-notifications') + .send({}); + + assert.equal(res.status, 200); + assert.equal(res.body.ok, true); + assert.equal(res.body.summary.claimed, 2); + assert.equal(res.body.summary.sent, 2); +}); + +test('POST /tasks/dispatch-notifications returns 500 on dispatch error', async () => { + const app = createWorkerApp({ + dispatch: async () => { + throw new Error('dispatch exploded'); + }, + }); + + const res = await request(app) + .post('/tasks/dispatch-notifications') + .send({}); + + assert.equal(res.status, 500); + assert.equal(res.body.ok, false); + assert.match(res.body.error, /dispatch exploded/); +}); diff --git a/docs/BACKEND/API_GUIDES/V2/README.md b/docs/BACKEND/API_GUIDES/V2/README.md index 4099f3e4..c9964d51 100644 --- a/docs/BACKEND/API_GUIDES/V2/README.md +++ b/docs/BACKEND/API_GUIDES/V2/README.md @@ -107,7 +107,7 @@ Important operational rules: - background location streams are stored as raw batch payloads in the private v2 bucket and summarized in SQL for query speed - incident review lives on `GET /client/coverage/incidents` - confirmed late-worker recovery is exposed on `POST /client/coverage/late-workers/:assignmentId/cancel` -- queued alerts are written to `notification_outbox`, dispatched by Cloud Run job `krow-notification-dispatcher-v2`, and recorded in `notification_deliveries` +- queued alerts are written to `notification_outbox`, dispatched by the private Cloud Run worker service `krow-notification-worker-v2`, and recorded in `notification_deliveries` ## 5) Route model diff --git a/docs/BACKEND/API_GUIDES/V2/unified-api.md b/docs/BACKEND/API_GUIDES/V2/unified-api.md index 70844f0b..079ec8ab 100644 --- a/docs/BACKEND/API_GUIDES/V2/unified-api.md +++ b/docs/BACKEND/API_GUIDES/V2/unified-api.md @@ -185,7 +185,8 @@ These are exposed as direct unified aliases even though they are backed by `core - SQL token registry in `device_push_tokens` - durable queue in `notification_outbox` - per-attempt delivery records in `notification_deliveries` - - Cloud Run job `krow-notification-dispatcher-v2` + - private Cloud Run worker service `krow-notification-worker-v2` + - Cloud Scheduler job `krow-notification-dispatch-v2` ### Push token request example diff --git a/makefiles/backend.mk b/makefiles/backend.mk index e0723813..e940d293 100644 --- a/makefiles/backend.mk +++ b/makefiles/backend.mk @@ -42,8 +42,12 @@ BACKEND_V2_COMMAND_SERVICE_NAME ?= krow-command-api-v2 BACKEND_V2_QUERY_SERVICE_NAME ?= krow-query-api-v2 BACKEND_V2_UNIFIED_SERVICE_NAME ?= krow-api-v2 BACKEND_V2_NOTIFICATION_JOB_NAME ?= krow-notification-dispatcher-v2 +BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME ?= krow-notification-worker-v2 +BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME ?= krow-notification-dispatch-v2 BACKEND_V2_RUNTIME_SA_NAME ?= krow-backend-v2-runtime BACKEND_V2_RUNTIME_SA_EMAIL := $(BACKEND_V2_RUNTIME_SA_NAME)@$(GCP_PROJECT_ID).iam.gserviceaccount.com +BACKEND_V2_SCHEDULER_SA_NAME ?= krow-backend-v2-scheduler +BACKEND_V2_SCHEDULER_SA_EMAIL := $(BACKEND_V2_SCHEDULER_SA_NAME)@$(GCP_PROJECT_ID).iam.gserviceaccount.com BACKEND_V2_CORE_DIR ?= backend/core-api BACKEND_V2_COMMAND_DIR ?= backend/command-api @@ -82,12 +86,14 @@ BACKEND_V2_PUSH_DELIVERY_MODE ?= live BACKEND_V2_SHIFT_REMINDERS_ENABLED ?= true BACKEND_V2_SHIFT_REMINDER_LEAD_MINUTES ?= 60,15 BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES ?= 5 +BACKEND_V2_NOTIFICATION_SCHEDULE ?= * * * * * +BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE ?= UTC BACKEND_V2_NFC_ENFORCE_PROOF_NONCE ?= false BACKEND_V2_NFC_ENFORCE_DEVICE_ID ?= false BACKEND_V2_NFC_ENFORCE_ATTESTATION ?= false BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS ?= 120 -.PHONY: backend-help backend-enable-apis backend-bootstrap-dev backend-migrate-idempotency backend-deploy-core backend-deploy-commands backend-deploy-workers backend-smoke-core backend-smoke-commands backend-logs-core backend-bootstrap-v2-dev backend-deploy-core-v2 backend-deploy-commands-v2 backend-deploy-query-v2 backend-deploy-unified-v2 backend-deploy-notification-job-v2 backend-run-notification-job-v2 backend-smoke-core-v2 backend-smoke-commands-v2 backend-smoke-query-v2 backend-smoke-unified-v2 backend-logs-core-v2 backend-v2-migrate-idempotency backend-v2-migrate-schema +.PHONY: backend-help backend-enable-apis backend-bootstrap-dev backend-migrate-idempotency backend-deploy-core backend-deploy-commands backend-deploy-workers backend-smoke-core backend-smoke-commands backend-logs-core backend-bootstrap-v2-dev backend-deploy-core-v2 backend-deploy-commands-v2 backend-deploy-query-v2 backend-deploy-unified-v2 backend-deploy-notification-worker-v2 backend-configure-notification-scheduler-v2 backend-run-notification-worker-v2 backend-smoke-notification-worker-v2 backend-deploy-notification-job-v2 backend-run-notification-job-v2 backend-smoke-core-v2 backend-smoke-commands-v2 backend-smoke-query-v2 backend-smoke-unified-v2 backend-logs-core-v2 backend-v2-migrate-idempotency backend-v2-migrate-schema backend-help: @echo "--> Backend Foundation Commands" @@ -107,8 +113,10 @@ backend-help: @echo " make backend-deploy-commands-v2 [ENV=dev] Build + deploy command API v2 service" @echo " make backend-deploy-query-v2 [ENV=dev] Build + deploy query API v2 service" @echo " make backend-deploy-unified-v2 [ENV=dev] Build + deploy unified API v2 gateway" - @echo " make backend-deploy-notification-job-v2 Deploy notification dispatcher v2 job" - @echo " make backend-run-notification-job-v2 Run notification dispatcher v2 job once" + @echo " make backend-deploy-notification-worker-v2 Deploy private notification worker v2 service" + @echo " make backend-configure-notification-scheduler-v2 Configure Cloud Scheduler for notification worker" + @echo " make backend-run-notification-worker-v2 Invoke notification worker v2 once" + @echo " make backend-smoke-notification-worker-v2 Smoke test private notification worker v2" @echo " make backend-v2-migrate-schema Apply v2 domain schema against krow-sql-v2" @echo " make backend-v2-migrate-idempotency Apply command idempotency migration against v2 DB" @echo " make backend-smoke-core-v2 [ENV=dev] Smoke test core API v2 /health" @@ -131,7 +139,8 @@ backend-enable-apis: iam.googleapis.com \ iamcredentials.googleapis.com \ serviceusage.googleapis.com \ - firebase.googleapis.com; do \ + firebase.googleapis.com \ + cloudscheduler.googleapis.com; do \ echo " - $$api"; \ gcloud services enable $$api --project=$(GCP_PROJECT_ID); \ done @@ -278,6 +287,14 @@ backend-bootstrap-v2-dev: backend-enable-apis else \ echo " - Runtime service account already exists."; \ fi + @echo "--> Ensuring v2 scheduler service account [$(BACKEND_V2_SCHEDULER_SA_NAME)] exists..." + @if ! gcloud iam service-accounts describe $(BACKEND_V2_SCHEDULER_SA_EMAIL) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + gcloud iam service-accounts create $(BACKEND_V2_SCHEDULER_SA_NAME) \ + --display-name="KROW Backend Scheduler V2" \ + --project=$(GCP_PROJECT_ID); \ + else \ + echo " - Scheduler service account already exists."; \ + fi @echo "--> Ensuring v2 runtime service account IAM roles..." @gcloud projects add-iam-policy-binding $(GCP_PROJECT_ID) \ --member="serviceAccount:$(BACKEND_V2_RUNTIME_SA_EMAIL)" \ @@ -394,34 +411,95 @@ backend-deploy-commands-v2: $(BACKEND_V2_RUN_AUTH_FLAG) @echo "✅ Command backend v2 service deployed." -backend-deploy-notification-job-v2: - @echo "--> Deploying notification dispatcher v2 job [$(BACKEND_V2_NOTIFICATION_JOB_NAME)]..." +backend-deploy-notification-worker-v2: + @echo "--> Deploying private notification worker v2 service [$(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)]..." @test -d $(BACKEND_V2_COMMAND_DIR) || (echo "❌ Missing directory: $(BACKEND_V2_COMMAND_DIR)" && exit 1) @test -f $(BACKEND_V2_COMMAND_DIR)/Dockerfile || (echo "❌ Missing Dockerfile: $(BACKEND_V2_COMMAND_DIR)/Dockerfile" && exit 1) @gcloud builds submit $(BACKEND_V2_COMMAND_DIR) --tag $(BACKEND_V2_COMMAND_IMAGE) --project=$(GCP_PROJECT_ID) - @gcloud run jobs deploy $(BACKEND_V2_NOTIFICATION_JOB_NAME) \ + @EXTRA_ENV="APP_ENV=$(ENV),APP_STACK=v2,GCP_PROJECT_ID=$(GCP_PROJECT_ID),PUBLIC_BUCKET=$(BACKEND_V2_PUBLIC_BUCKET),PRIVATE_BUCKET=$(BACKEND_V2_PRIVATE_BUCKET),INSTANCE_CONNECTION_NAME=$(BACKEND_V2_SQL_CONNECTION_NAME),DB_NAME=$(BACKEND_V2_SQL_DATABASE),DB_USER=$(BACKEND_V2_SQL_APP_USER),NOTIFICATION_BATCH_LIMIT=$(BACKEND_V2_NOTIFICATION_BATCH_LIMIT),PUSH_DELIVERY_MODE=$(BACKEND_V2_PUSH_DELIVERY_MODE),SHIFT_REMINDERS_ENABLED=$(BACKEND_V2_SHIFT_REMINDERS_ENABLED),SHIFT_REMINDER_WINDOW_MINUTES=$(BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES),NFC_ENFORCE_PROOF_NONCE=$(BACKEND_V2_NFC_ENFORCE_PROOF_NONCE),NFC_ENFORCE_DEVICE_ID=$(BACKEND_V2_NFC_ENFORCE_DEVICE_ID),NFC_ENFORCE_ATTESTATION=$(BACKEND_V2_NFC_ENFORCE_ATTESTATION),NFC_PROOF_MAX_AGE_SECONDS=$(BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS)"; \ + gcloud run deploy $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) \ --image=$(BACKEND_V2_COMMAND_IMAGE) \ --region=$(BACKEND_REGION) \ --project=$(GCP_PROJECT_ID) \ --service-account=$(BACKEND_V2_RUNTIME_SA_EMAIL) \ --command=node \ - --args=scripts/dispatch-notifications.mjs \ - --set-env-vars=APP_ENV=$(ENV),APP_STACK=v2,GCP_PROJECT_ID=$(GCP_PROJECT_ID),PUBLIC_BUCKET=$(BACKEND_V2_PUBLIC_BUCKET),PRIVATE_BUCKET=$(BACKEND_V2_PRIVATE_BUCKET),INSTANCE_CONNECTION_NAME=$(BACKEND_V2_SQL_CONNECTION_NAME),DB_NAME=$(BACKEND_V2_SQL_DATABASE),DB_USER=$(BACKEND_V2_SQL_APP_USER),NOTIFICATION_BATCH_LIMIT=$(BACKEND_V2_NOTIFICATION_BATCH_LIMIT),PUSH_DELIVERY_MODE=$(BACKEND_V2_PUSH_DELIVERY_MODE),SHIFT_REMINDERS_ENABLED=$(BACKEND_V2_SHIFT_REMINDERS_ENABLED),SHIFT_REMINDER_WINDOW_MINUTES=$(BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES),NFC_ENFORCE_PROOF_NONCE=$(BACKEND_V2_NFC_ENFORCE_PROOF_NONCE),NFC_ENFORCE_DEVICE_ID=$(BACKEND_V2_NFC_ENFORCE_DEVICE_ID),NFC_ENFORCE_ATTESTATION=$(BACKEND_V2_NFC_ENFORCE_ATTESTATION),NFC_PROOF_MAX_AGE_SECONDS=$(BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS) \ + --args=src/worker-server.js \ + --set-env-vars=$$EXTRA_ENV \ --set-secrets=DB_PASSWORD=$(BACKEND_V2_SQL_PASSWORD_SECRET):latest \ --set-cloudsql-instances=$(BACKEND_V2_SQL_CONNECTION_NAME) \ - --tasks=1 \ - --parallelism=1 \ - --max-retries=1 \ - --task-timeout=10m - @echo "✅ Notification dispatcher v2 job deployed." - -backend-run-notification-job-v2: - @echo "--> Running notification dispatcher v2 job [$(BACKEND_V2_NOTIFICATION_JOB_NAME)]..." - @gcloud run jobs execute $(BACKEND_V2_NOTIFICATION_JOB_NAME) \ + --concurrency=1 \ + --max-instances=1 \ + --no-allow-unauthenticated + @if ! gcloud iam service-accounts describe $(BACKEND_V2_SCHEDULER_SA_EMAIL) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + gcloud iam service-accounts create $(BACKEND_V2_SCHEDULER_SA_NAME) \ + --display-name="KROW Backend Scheduler V2" \ + --project=$(GCP_PROJECT_ID); \ + fi + @gcloud run services add-iam-policy-binding $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) \ --region=$(BACKEND_REGION) \ --project=$(GCP_PROJECT_ID) \ - --wait - @echo "✅ Notification dispatcher v2 job completed." + --member="serviceAccount:$(BACKEND_V2_SCHEDULER_SA_EMAIL)" \ + --role="roles/run.invoker" \ + --quiet >/dev/null + @echo "✅ Notification worker v2 service deployed." + +backend-configure-notification-scheduler-v2: + @echo "--> Configuring notification scheduler [$(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME)]..." + @gcloud services enable cloudscheduler.googleapis.com --project=$(GCP_PROJECT_ID) >/dev/null + @URL=$$(gcloud run services describe $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) --region=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) --format='value(status.url)'); \ + if [ -z "$$URL" ]; then \ + echo "❌ Could not resolve URL for service $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)"; \ + exit 1; \ + fi; \ + if gcloud scheduler jobs describe $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) --location=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + gcloud scheduler jobs update http $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \ + --location=$(BACKEND_REGION) \ + --project=$(GCP_PROJECT_ID) \ + --schedule='$(BACKEND_V2_NOTIFICATION_SCHEDULE)' \ + --time-zone='$(BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE)' \ + --uri="$$URL/tasks/dispatch-notifications" \ + --http-method=POST \ + --headers=Content-Type=application/json \ + --message-body='{}' \ + --oidc-service-account-email=$(BACKEND_V2_SCHEDULER_SA_EMAIL) \ + --oidc-token-audience="$$URL"; \ + else \ + gcloud scheduler jobs create http $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \ + --location=$(BACKEND_REGION) \ + --project=$(GCP_PROJECT_ID) \ + --schedule='$(BACKEND_V2_NOTIFICATION_SCHEDULE)' \ + --time-zone='$(BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE)' \ + --uri="$$URL/tasks/dispatch-notifications" \ + --http-method=POST \ + --headers=Content-Type=application/json \ + --message-body='{}' \ + --oidc-service-account-email=$(BACKEND_V2_SCHEDULER_SA_EMAIL) \ + --oidc-token-audience="$$URL"; \ + fi + @echo "✅ Notification scheduler configured." + +backend-smoke-notification-worker-v2: + @echo "--> Running notification worker smoke check..." + @URL=$$(gcloud run services describe $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) --region=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) --format='value(status.url)'); \ + if [ -z "$$URL" ]; then \ + echo "❌ Could not resolve URL for service $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)"; \ + exit 1; \ + fi; \ + gcloud scheduler jobs describe $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) --location=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) >/dev/null && \ + echo "✅ Notification worker smoke check passed: $$URL (scheduler wired)" + +backend-run-notification-worker-v2: + @echo "--> Triggering notification worker via scheduler job [$(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME)]..." + @gcloud scheduler jobs run $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \ + --location=$(BACKEND_REGION) \ + --project=$(GCP_PROJECT_ID) >/dev/null + @echo "✅ Notification worker v2 invocation requested through Cloud Scheduler." + +backend-deploy-notification-job-v2: backend-deploy-notification-worker-v2 + @echo "⚠️ Cloud Run Job dispatcher is deprecated. Using private worker service instead." + +backend-run-notification-job-v2: backend-run-notification-worker-v2 + @echo "⚠️ Cloud Run Job dispatcher is deprecated. Using private worker service instead." backend-deploy-query-v2: @echo "--> Deploying query backend v2 service [$(BACKEND_V2_QUERY_SERVICE_NAME)] to [$(ENV)]..."