fix(notifications): replace Cloud Run job with worker service

backend/command-api/src/worker-app.js

@@ -0,0 +1,46 @@
+import express from 'express';
+import pino from 'pino';
+import pinoHttp from 'pino-http';
+
+const logger = pino({ level: process.env.LOG_LEVEL || 'info' });
+
+export function createWorkerApp({ dispatch = async () => ({}) } = {}) {
+  const app = express();
+
+  app.use(
+    pinoHttp({
+      logger,
+    })
+  );
+  app.use(express.json({ limit: '256kb' }));
+
+  app.get('/health', (_req, res) => {
+    res.status(200).json({ ok: true, service: 'notification-worker-v2' });
+  });
+
+  app.get('/readyz', (_req, res) => {
+    res.status(200).json({ ok: true, service: 'notification-worker-v2' });
+  });
+
+  app.post('/tasks/dispatch-notifications', async (req, res) => {
+    try {
+      const summary = await dispatch();
+      res.status(200).json({ ok: true, summary });
+    } catch (error) {
+      req.log?.error?.({ err: error }, 'notification dispatch failed');
+      res.status(500).json({
+        ok: false,
+        error: error?.message || String(error),
+      });
+    }
+  });
+
+  app.use((_req, res) => {
+    res.status(404).json({
+      code: 'NOT_FOUND',
+      message: 'Route not found',
+    });
+  });
+
+  return app;
+}

backend/command-api/src/worker-server.js

@@ -0,0 +1,12 @@
+import { createWorkerApp } from './worker-app.js';
+import { dispatchPendingNotifications } from './services/notification-dispatcher.js';
+
+const port = Number(process.env.PORT || 8080);
+const app = createWorkerApp({
+  dispatch: () => dispatchPendingNotifications(),
+});
+
+app.listen(port, () => {
+  // eslint-disable-next-line no-console
+  console.log(`krow-notification-worker listening on port ${port}`);
+});

backend/command-api/test/notification-worker.test.js

@@ -0,0 +1,47 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import request from 'supertest';
+import { createWorkerApp } from '../src/worker-app.js';
+
+test('GET /readyz returns healthy response', async () => {
+  const app = createWorkerApp();
+  const res = await request(app).get('/readyz');
+
+  assert.equal(res.status, 200);
+  assert.equal(res.body.ok, true);
+  assert.equal(res.body.service, 'notification-worker-v2');
+});
+
+test('POST /tasks/dispatch-notifications returns dispatch summary', async () => {
+  const app = createWorkerApp({
+    dispatch: async () => ({
+      claimed: 2,
+      sent: 2,
+    }),
+  });
+
+  const res = await request(app)
+    .post('/tasks/dispatch-notifications')
+    .send({});
+
+  assert.equal(res.status, 200);
+  assert.equal(res.body.ok, true);
+  assert.equal(res.body.summary.claimed, 2);
+  assert.equal(res.body.summary.sent, 2);
+});
+
+test('POST /tasks/dispatch-notifications returns 500 on dispatch error', async () => {
+  const app = createWorkerApp({
+    dispatch: async () => {
+      throw new Error('dispatch exploded');
+    },
+  });
+
+  const res = await request(app)
+    .post('/tasks/dispatch-notifications')
+    .send({});
+
+  assert.equal(res.status, 500);
+  assert.equal(res.body.ok, false);
+  assert.match(res.body.error, /dispatch exploded/);
+});

docs/BACKEND/API_GUIDES/V2/README.md

@@ -107,7 +107,7 @@ Important operational rules:
 - background location streams are stored as raw batch payloads in the private v2 bucket and summarized in SQL for query speed
 - incident review lives on `GET /client/coverage/incidents`
 - confirmed late-worker recovery is exposed on `POST /client/coverage/late-workers/:assignmentId/cancel`
-- queued alerts are written to `notification_outbox`, dispatched by Cloud Run job `krow-notification-dispatcher-v2`, and recorded in `notification_deliveries`
+- queued alerts are written to `notification_outbox`, dispatched by the private Cloud Run worker service `krow-notification-worker-v2`, and recorded in `notification_deliveries`
 
 ## 5) Route model
 

docs/BACKEND/API_GUIDES/V2/unified-api.md

@@ -185,7 +185,8 @@ These are exposed as direct unified aliases even though they are backed by `core
   - SQL token registry in `device_push_tokens`
   - durable queue in `notification_outbox`
   - per-attempt delivery records in `notification_deliveries`
-  - Cloud Run job `krow-notification-dispatcher-v2`
+  - private Cloud Run worker service `krow-notification-worker-v2`
+  - Cloud Scheduler job `krow-notification-dispatch-v2`
 
 ### Push token request example
 

makefiles/backend.mk

@@ -42,8 +42,12 @@ BACKEND_V2_COMMAND_SERVICE_NAME ?= krow-command-api-v2
 BACKEND_V2_QUERY_SERVICE_NAME ?= krow-query-api-v2
 BACKEND_V2_UNIFIED_SERVICE_NAME ?= krow-api-v2
 BACKEND_V2_NOTIFICATION_JOB_NAME ?= krow-notification-dispatcher-v2
+BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME ?= krow-notification-worker-v2
+BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME ?= krow-notification-dispatch-v2
 BACKEND_V2_RUNTIME_SA_NAME ?= krow-backend-v2-runtime
 BACKEND_V2_RUNTIME_SA_EMAIL := $(BACKEND_V2_RUNTIME_SA_NAME)@$(GCP_PROJECT_ID).iam.gserviceaccount.com
+BACKEND_V2_SCHEDULER_SA_NAME ?= krow-backend-v2-scheduler
+BACKEND_V2_SCHEDULER_SA_EMAIL := $(BACKEND_V2_SCHEDULER_SA_NAME)@$(GCP_PROJECT_ID).iam.gserviceaccount.com
 
 BACKEND_V2_CORE_DIR ?= backend/core-api
 BACKEND_V2_COMMAND_DIR ?= backend/command-api
@@ -82,12 +86,14 @@ BACKEND_V2_PUSH_DELIVERY_MODE ?= live
 BACKEND_V2_SHIFT_REMINDERS_ENABLED ?= true
 BACKEND_V2_SHIFT_REMINDER_LEAD_MINUTES ?= 60,15
 BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES ?= 5
+BACKEND_V2_NOTIFICATION_SCHEDULE ?= * * * * *
+BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE ?= UTC
 BACKEND_V2_NFC_ENFORCE_PROOF_NONCE ?= false
 BACKEND_V2_NFC_ENFORCE_DEVICE_ID ?= false
 BACKEND_V2_NFC_ENFORCE_ATTESTATION ?= false
 BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS ?= 120
 
-.PHONY: backend-help backend-enable-apis backend-bootstrap-dev backend-migrate-idempotency backend-deploy-core backend-deploy-commands backend-deploy-workers backend-smoke-core backend-smoke-commands backend-logs-core backend-bootstrap-v2-dev backend-deploy-core-v2 backend-deploy-commands-v2 backend-deploy-query-v2 backend-deploy-unified-v2 backend-deploy-notification-job-v2 backend-run-notification-job-v2 backend-smoke-core-v2 backend-smoke-commands-v2 backend-smoke-query-v2 backend-smoke-unified-v2 backend-logs-core-v2 backend-v2-migrate-idempotency backend-v2-migrate-schema
+.PHONY: backend-help backend-enable-apis backend-bootstrap-dev backend-migrate-idempotency backend-deploy-core backend-deploy-commands backend-deploy-workers backend-smoke-core backend-smoke-commands backend-logs-core backend-bootstrap-v2-dev backend-deploy-core-v2 backend-deploy-commands-v2 backend-deploy-query-v2 backend-deploy-unified-v2 backend-deploy-notification-worker-v2 backend-configure-notification-scheduler-v2 backend-run-notification-worker-v2 backend-smoke-notification-worker-v2 backend-deploy-notification-job-v2 backend-run-notification-job-v2 backend-smoke-core-v2 backend-smoke-commands-v2 backend-smoke-query-v2 backend-smoke-unified-v2 backend-logs-core-v2 backend-v2-migrate-idempotency backend-v2-migrate-schema
 
 backend-help:
 	@echo "--> Backend Foundation Commands"
@@ -107,8 +113,10 @@ backend-help:
 	@echo "  make backend-deploy-commands-v2 [ENV=dev]    Build + deploy command API v2 service"
 	@echo "  make backend-deploy-query-v2 [ENV=dev]       Build + deploy query API v2 service"
 	@echo "  make backend-deploy-unified-v2 [ENV=dev]    Build + deploy unified API v2 gateway"
-	@echo "  make backend-deploy-notification-job-v2      Deploy notification dispatcher v2 job"
+	@echo "  make backend-deploy-notification-worker-v2   Deploy private notification worker v2 service"
-	@echo "  make backend-run-notification-job-v2         Run notification dispatcher v2 job once"
+	@echo "  make backend-configure-notification-scheduler-v2 Configure Cloud Scheduler for notification worker"
+	@echo "  make backend-run-notification-worker-v2      Invoke notification worker v2 once"
+	@echo "  make backend-smoke-notification-worker-v2    Smoke test private notification worker v2"
 	@echo "  make backend-v2-migrate-schema               Apply v2 domain schema against krow-sql-v2"
 	@echo "  make backend-v2-migrate-idempotency          Apply command idempotency migration against v2 DB"
 	@echo "  make backend-smoke-core-v2 [ENV=dev]         Smoke test core API v2 /health"
@@ -131,7 +139,8 @@ backend-enable-apis:
 		iam.googleapis.com \
 		iamcredentials.googleapis.com \
 		serviceusage.googleapis.com \
-		firebase.googleapis.com; do \
+		firebase.googleapis.com \
+		cloudscheduler.googleapis.com; do \
 		echo "   - $$api"; \
 		gcloud services enable $$api --project=$(GCP_PROJECT_ID); \
 	done
@@ -278,6 +287,14 @@ backend-bootstrap-v2-dev: backend-enable-apis
 	else \
 		echo "   - Runtime service account already exists."; \
 	fi
+	@echo "--> Ensuring v2 scheduler service account [$(BACKEND_V2_SCHEDULER_SA_NAME)] exists..."
+	@if ! gcloud iam service-accounts describe $(BACKEND_V2_SCHEDULER_SA_EMAIL) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
+		gcloud iam service-accounts create $(BACKEND_V2_SCHEDULER_SA_NAME) \
+			--display-name="KROW Backend Scheduler V2" \
+			--project=$(GCP_PROJECT_ID); \
+	else \
+		echo "   - Scheduler service account already exists."; \
+	fi
 	@echo "--> Ensuring v2 runtime service account IAM roles..."
 	@gcloud projects add-iam-policy-binding $(GCP_PROJECT_ID) \
 		--member="serviceAccount:$(BACKEND_V2_RUNTIME_SA_EMAIL)" \
@@ -394,34 +411,95 @@ backend-deploy-commands-v2:
 		$(BACKEND_V2_RUN_AUTH_FLAG)
 	@echo "✅ Command backend v2 service deployed."
 
-backend-deploy-notification-job-v2:
+backend-deploy-notification-worker-v2:
-	@echo "--> Deploying notification dispatcher v2 job [$(BACKEND_V2_NOTIFICATION_JOB_NAME)]..."
+	@echo "--> Deploying private notification worker v2 service [$(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)]..."
 	@test -d $(BACKEND_V2_COMMAND_DIR) || (echo "❌ Missing directory: $(BACKEND_V2_COMMAND_DIR)" && exit 1)
 	@test -f $(BACKEND_V2_COMMAND_DIR)/Dockerfile || (echo "❌ Missing Dockerfile: $(BACKEND_V2_COMMAND_DIR)/Dockerfile" && exit 1)
 	@gcloud builds submit $(BACKEND_V2_COMMAND_DIR) --tag $(BACKEND_V2_COMMAND_IMAGE) --project=$(GCP_PROJECT_ID)
-	@gcloud run jobs deploy $(BACKEND_V2_NOTIFICATION_JOB_NAME) \
+	@EXTRA_ENV="APP_ENV=$(ENV),APP_STACK=v2,GCP_PROJECT_ID=$(GCP_PROJECT_ID),PUBLIC_BUCKET=$(BACKEND_V2_PUBLIC_BUCKET),PRIVATE_BUCKET=$(BACKEND_V2_PRIVATE_BUCKET),INSTANCE_CONNECTION_NAME=$(BACKEND_V2_SQL_CONNECTION_NAME),DB_NAME=$(BACKEND_V2_SQL_DATABASE),DB_USER=$(BACKEND_V2_SQL_APP_USER),NOTIFICATION_BATCH_LIMIT=$(BACKEND_V2_NOTIFICATION_BATCH_LIMIT),PUSH_DELIVERY_MODE=$(BACKEND_V2_PUSH_DELIVERY_MODE),SHIFT_REMINDERS_ENABLED=$(BACKEND_V2_SHIFT_REMINDERS_ENABLED),SHIFT_REMINDER_WINDOW_MINUTES=$(BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES),NFC_ENFORCE_PROOF_NONCE=$(BACKEND_V2_NFC_ENFORCE_PROOF_NONCE),NFC_ENFORCE_DEVICE_ID=$(BACKEND_V2_NFC_ENFORCE_DEVICE_ID),NFC_ENFORCE_ATTESTATION=$(BACKEND_V2_NFC_ENFORCE_ATTESTATION),NFC_PROOF_MAX_AGE_SECONDS=$(BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS)"; \
+	gcloud run deploy $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) \
 		--image=$(BACKEND_V2_COMMAND_IMAGE) \
 		--region=$(BACKEND_REGION) \
 		--project=$(GCP_PROJECT_ID) \
 		--service-account=$(BACKEND_V2_RUNTIME_SA_EMAIL) \
 		--command=node \
-		--args=scripts/dispatch-notifications.mjs \
+		--args=src/worker-server.js \
-		--set-env-vars=APP_ENV=$(ENV),APP_STACK=v2,GCP_PROJECT_ID=$(GCP_PROJECT_ID),PUBLIC_BUCKET=$(BACKEND_V2_PUBLIC_BUCKET),PRIVATE_BUCKET=$(BACKEND_V2_PRIVATE_BUCKET),INSTANCE_CONNECTION_NAME=$(BACKEND_V2_SQL_CONNECTION_NAME),DB_NAME=$(BACKEND_V2_SQL_DATABASE),DB_USER=$(BACKEND_V2_SQL_APP_USER),NOTIFICATION_BATCH_LIMIT=$(BACKEND_V2_NOTIFICATION_BATCH_LIMIT),PUSH_DELIVERY_MODE=$(BACKEND_V2_PUSH_DELIVERY_MODE),SHIFT_REMINDERS_ENABLED=$(BACKEND_V2_SHIFT_REMINDERS_ENABLED),SHIFT_REMINDER_WINDOW_MINUTES=$(BACKEND_V2_SHIFT_REMINDER_WINDOW_MINUTES),NFC_ENFORCE_PROOF_NONCE=$(BACKEND_V2_NFC_ENFORCE_PROOF_NONCE),NFC_ENFORCE_DEVICE_ID=$(BACKEND_V2_NFC_ENFORCE_DEVICE_ID),NFC_ENFORCE_ATTESTATION=$(BACKEND_V2_NFC_ENFORCE_ATTESTATION),NFC_PROOF_MAX_AGE_SECONDS=$(BACKEND_V2_NFC_PROOF_MAX_AGE_SECONDS) \
+		--set-env-vars=$$EXTRA_ENV \
 		--set-secrets=DB_PASSWORD=$(BACKEND_V2_SQL_PASSWORD_SECRET):latest \
 		--set-cloudsql-instances=$(BACKEND_V2_SQL_CONNECTION_NAME) \
-		--tasks=1 \
+		--concurrency=1 \
-		--parallelism=1 \
+		--max-instances=1 \
-		--max-retries=1 \
+		--no-allow-unauthenticated
-		--task-timeout=10m
+	@if ! gcloud iam service-accounts describe $(BACKEND_V2_SCHEDULER_SA_EMAIL) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
-	@echo "✅ Notification dispatcher v2 job deployed."
+		gcloud iam service-accounts create $(BACKEND_V2_SCHEDULER_SA_NAME) \
-
+			--display-name="KROW Backend Scheduler V2" \
-backend-run-notification-job-v2:
+			--project=$(GCP_PROJECT_ID); \
-	@echo "--> Running notification dispatcher v2 job [$(BACKEND_V2_NOTIFICATION_JOB_NAME)]..."
+	fi
-	@gcloud run jobs execute $(BACKEND_V2_NOTIFICATION_JOB_NAME) \
+	@gcloud run services add-iam-policy-binding $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) \
 		--region=$(BACKEND_REGION) \
 		--project=$(GCP_PROJECT_ID) \
-		--wait
+		--member="serviceAccount:$(BACKEND_V2_SCHEDULER_SA_EMAIL)" \
-	@echo "✅ Notification dispatcher v2 job completed."
+		--role="roles/run.invoker" \
+		--quiet >/dev/null
+	@echo "✅ Notification worker v2 service deployed."
+
+backend-configure-notification-scheduler-v2:
+	@echo "--> Configuring notification scheduler [$(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME)]..."
+	@gcloud services enable cloudscheduler.googleapis.com --project=$(GCP_PROJECT_ID) >/dev/null
+	@URL=$$(gcloud run services describe $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) --region=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) --format='value(status.url)'); \
+	if [ -z "$$URL" ]; then \
+		echo "❌ Could not resolve URL for service $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)"; \
+		exit 1; \
+	fi; \
+	if gcloud scheduler jobs describe $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) --location=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
+		gcloud scheduler jobs update http $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \
+			--location=$(BACKEND_REGION) \
+			--project=$(GCP_PROJECT_ID) \
+			--schedule='$(BACKEND_V2_NOTIFICATION_SCHEDULE)' \
+			--time-zone='$(BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE)' \
+			--uri="$$URL/tasks/dispatch-notifications" \
+			--http-method=POST \
+			--headers=Content-Type=application/json \
+			--message-body='{}' \
+			--oidc-service-account-email=$(BACKEND_V2_SCHEDULER_SA_EMAIL) \
+			--oidc-token-audience="$$URL"; \
+	else \
+		gcloud scheduler jobs create http $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \
+			--location=$(BACKEND_REGION) \
+			--project=$(GCP_PROJECT_ID) \
+			--schedule='$(BACKEND_V2_NOTIFICATION_SCHEDULE)' \
+			--time-zone='$(BACKEND_V2_NOTIFICATION_SCHEDULER_TIME_ZONE)' \
+			--uri="$$URL/tasks/dispatch-notifications" \
+			--http-method=POST \
+			--headers=Content-Type=application/json \
+			--message-body='{}' \
+			--oidc-service-account-email=$(BACKEND_V2_SCHEDULER_SA_EMAIL) \
+			--oidc-token-audience="$$URL"; \
+	fi
+	@echo "✅ Notification scheduler configured."
+
+backend-smoke-notification-worker-v2:
+	@echo "--> Running notification worker smoke check..."
+	@URL=$$(gcloud run services describe $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME) --region=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) --format='value(status.url)'); \
+	if [ -z "$$URL" ]; then \
+		echo "❌ Could not resolve URL for service $(BACKEND_V2_NOTIFICATION_WORKER_SERVICE_NAME)"; \
+		exit 1; \
+	fi; \
+	gcloud scheduler jobs describe $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) --location=$(BACKEND_REGION) --project=$(GCP_PROJECT_ID) >/dev/null && \
+	echo "✅ Notification worker smoke check passed: $$URL (scheduler wired)"
+
+backend-run-notification-worker-v2:
+	@echo "--> Triggering notification worker via scheduler job [$(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME)]..."
+	@gcloud scheduler jobs run $(BACKEND_V2_NOTIFICATION_SCHEDULER_JOB_NAME) \
+		--location=$(BACKEND_REGION) \
+		--project=$(GCP_PROJECT_ID) >/dev/null
+	@echo "✅ Notification worker v2 invocation requested through Cloud Scheduler."
+
+backend-deploy-notification-job-v2: backend-deploy-notification-worker-v2
+	@echo "⚠️  Cloud Run Job dispatcher is deprecated. Using private worker service instead."
+
+backend-run-notification-job-v2: backend-run-notification-worker-v2
+	@echo "⚠️  Cloud Run Job dispatcher is deprecated. Using private worker service instead."
 
 backend-deploy-query-v2:
 	@echo "--> Deploying query backend v2 service [$(BACKEND_V2_QUERY_SERVICE_NAME)] to [$(ENV)]..."