diff --git a/backend/dataconnect/dataconnect.dev.yaml b/backend/dataconnect/dataconnect.dev.yaml new file mode 100644 index 00000000..39e01fdb --- /dev/null +++ b/backend/dataconnect/dataconnect.dev.yaml @@ -0,0 +1,13 @@ +specVersion: "v1" +serviceId: "krow-workforce-db" +location: "us-central1" +schema: + source: "./schema" + datasource: + postgresql: + database: "krow_db" + cloudSql: + instanceId: "krow-sql" + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +connectorDirs: ["./connector"] diff --git a/backend/dataconnect/dataconnect.validation.yaml b/backend/dataconnect/dataconnect.validation.yaml new file mode 100644 index 00000000..9e1775d6 --- /dev/null +++ b/backend/dataconnect/dataconnect.validation.yaml @@ -0,0 +1,13 @@ +specVersion: "v1" +serviceId: "krow-workforce-db-validation" +location: "us-central1" +schema: + source: "./schema" + datasource: + postgresql: + database: "krow_db" + cloudSql: + instanceId: "krow-sql-validation" + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +connectorDirs: ["./connector"] diff --git a/makefiles/dataconnect.mk b/makefiles/dataconnect.mk index ba7298ef..c2db1fd6 100644 --- a/makefiles/dataconnect.mk +++ b/makefiles/dataconnect.mk @@ -1,6 +1,47 @@ # --- Data Connect / Backend --- -.PHONY: dataconnect-enable-apis dataconnect-init dataconnect-deploy dataconnect-sql-migrate dataconnect-generate-sdk dataconnect-sync dataconnect-bootstrap-db check-gcloud-beta dataconnect-clean +# Usage examples: +# make dataconnect-sync DC_ENV=dev +# make dataconnect-seed DC_ENV=validation +# make dataconnect-clean DC_ENV=validation +# make dataconnect-generate-sdk DC_ENV=dev +# +DC_ENV ?= dev + +DC_SERVICE_DEV := krow-workforce-db +DC_SERVICE_VALIDATION := krow-workforce-db-validation + +ifeq ($(DC_ENV),dev) + DC_SERVICE := $(DC_SERVICE_DEV) +else ifeq ($(DC_ENV),validation) + DC_SERVICE := $(DC_SERVICE_VALIDATION) +else + $(error Invalid DC_ENV '$(DC_ENV)'. Use DC_ENV=dev or DC_ENV=validation) +endif + +.PHONY: dataconnect-enable-apis dataconnect-init dataconnect-deploy dataconnect-sql-migrate dataconnect-generate-sdk dataconnect-sync dataconnect-bootstrap-db check-gcloud-beta dataconnect-clean dataconnect-bootstrap-validation-db dataconnect-file dataconnect-file-validation dataconnect-file-dev dataconnect-seed dataconnect-test + +#creation dataconnect file +dataconnect-file: + @echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..." + @test -f backend/dataconnect/dataconnect.$(DC_ENV).yaml || (echo "❌ Missing backend/dataconnect/dataconnect.$(DC_ENV).yaml" && exit 1) + @cp backend/dataconnect/dataconnect.$(DC_ENV).yaml backend/dataconnect/dataconnect.yaml + @echo "✅ Creation Data Connect file completed." + +#creation dev dataconnect file +dataconnect-file-dev: + @echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..." + @test -f backend/dataconnect/dataconnect.dev.yaml || (echo "❌ Missing backend/dataconnect/dataconnect.dev.yaml" && exit 1) + @cp backend/dataconnect/dataconnect.dev.yaml backend/dataconnect/dataconnect.yaml + @echo "✅ Creation Data Connect file completed." + + +#creation validation dataconnect file +dataconnect-file-validation: + @echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..." + @test -f backend/dataconnect/dataconnect.validation.yaml || (echo "❌ Missing backend/dataconnect/dataconnect.validation.yaml" && exit 1) + @cp backend/dataconnect/dataconnect.validation.yaml backend/dataconnect/dataconnect.yaml + @echo "✅ Creation Data Connect file completed." # Enable all required APIs for Firebase Data Connect + Cloud SQL dataconnect-enable-apis: @@ -14,63 +55,94 @@ dataconnect-enable-apis: @echo "✅ APIs enabled for project [$(GCP_PROJECT_ID)]." # Initialize Firebase Data Connect (interactive wizard). +# use only once per project dataconnect-init: @echo "--> Initializing Firebase Data Connect for alias [$(FIREBASE_ALIAS)] (project: $(GCP_PROJECT_ID))..." @firebase init dataconnect --project $(FIREBASE_ALIAS) @echo "✅ Data Connect initialization command executed. Follow the interactive steps in the CLI." # Deploy Data Connect schemas (GraphQL → Cloud SQL) -dataconnect-deploy: - @echo "--> Deploying Firebase Data Connect schemas to [$(ENV)] (project: $(FIREBASE_ALIAS))..." - @firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS) +dataconnect-deploy: dataconnect-file + @echo "--> Deploying Firebase Data Connect schemas to [$(ENV)] (service: $(DC_SERVICE)) (project: $(FIREBASE_ALIAS))..." + @firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS) @echo "✅ Data Connect deployment completed for [$(ENV)]." # Apply pending SQL migrations for Firebase Data Connect -dataconnect-sql-migrate: - @echo "--> Applying Firebase Data Connect SQL migrations to [$(ENV)] (project: $(FIREBASE_ALIAS))..." +dataconnect-sql-migrate: dataconnect-file + @echo "--> Applying Firebase Data Connect SQL migrations to [$(ENV)] (service: $(DC_SERVICE)) (project: $(FIREBASE_ALIAS))..." @firebase dataconnect:sql:migrate --project=$(FIREBASE_ALIAS) @echo "✅ Data Connect SQL migration completed for [$(ENV)]." # Generate Data Connect client SDK for frontend-web and internal-api-harness -dataconnect-generate-sdk: +dataconnect-generate-sdk: dataconnect-file @echo "--> Generating Firebase Data Connect SDK for web frontend and API harness..." - @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) + @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) @echo "✅ Data Connect SDK generation completed for [$(ENV)]." # Unified backend schema update workflow (schema -> deploy -> SDK) -dataconnect-sync: - @echo "--> [1/3] Deploying Data Connect..." - @firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS) - @echo "--> [2/3] Applying SQL migrations..." - @firebase dataconnect:sql:migrate --project=$(FIREBASE_ALIAS) - @echo "--> [3/3] Regenerating SDK..." +dataconnect-sync: dataconnect-file + @echo "--> [1/3] Deploying Data Connect [$(DC_SERVICE)]..." + @firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS) + @echo "--> [2/3] Applying SQL migrations [$(DC_SERVICE)]..." + @firebase dataconnect:sql:migrate $(DC_SERVICE) --project=$(FIREBASE_ALIAS) + @echo "--> [3/3] Regenerating SDK [$(DC_SERVICE)]..." @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) - @echo "✅ Data Connect SQL, deploy, and SDK generation completed for [$(ENV)]." + @echo "✅ Data Connect SQL, deploy, and SDK generation [$(ENV)]." # Execute seed in Firebase Data Connect -dataconnect-seed: - @echo "--> Exec seed in Firebase Data Connect..." - @firebase dataconnect:execute backend/dataconnect/functions/seed.gql --project=$(FIREBASE_ALIAS) +dataconnect-seed: dataconnect-file + @echo "--> Exec seed in Firebase Data Connect (service: $(DC_SERVICE))..." + @firebase dataconnect:execute backend/dataconnect/functions/seed.gql --project=$(FIREBASE_ALIAS) @echo "✅ Seed executed successfully." # Execute clean, to delete all the data in Firebase Data Connect -dataconnect-clean: - @echo "--> Exec clean all the data in Firebase Data Connect..." - @firebase dataconnect:execute backend/dataconnect/functions/clean.gql --project=$(FIREBASE_ALIAS) +dataconnect-clean: dataconnect-file + @echo "--> Exec clean all the data in Firebase Data Connect (service: $(DC_SERVICE))..." + @firebase dataconnect:execute backend/dataconnect/functions/clean.gql --project=$(FIREBASE_ALIAS) @echo "✅ Clean information executed successfully." # Run tests for Data Connect deployment and migrations -dataconnect-test: - @echo "--> Running Data Connect tests..." - @echo "--> [1/3] Deploying Data Connect..." - @firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS) --dry-run - @echo "--> [2/3] Applying SQL migrations..." +dataconnect-test: dataconnect-file + @echo "--> Running Data Connect tests (service: $(DC_SERVICE))..." + @echo "--> [1/2] Deploying Data Connect..." + @firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS) --dry-run + @echo "--> [2/2] Applying SQL migrations..." @firebase dataconnect:sql:diff --project=$(FIREBASE_ALIAS) @echo "✅ Data Connect tests completed." +dataconnect-backup-dev-to-validation: + @echo "🔍 Validating instances exist in [$(GCP_PROJECT_ID)]..." + @if ! gcloud sql instances describe krow-sql --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + echo "❌ Dev instance 'krow-sql' not found in project $(GCP_PROJECT_ID)."; \ + exit 1; \ + fi + @if ! gcloud sql instances describe krow-sql-validation --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + echo "❌ Validation instance 'krow-sql-validation' not found in project $(GCP_PROJECT_ID)."; \ + exit 1; \ + fi + @echo "✅ Instances found." + + @echo "🧰 Creating a backup on dev (krow-sql)..." + @gcloud sql backups create --instance=krow-sql --project=$(GCP_PROJECT_ID) >/dev/null + + @echo "🔎 Fetching latest backup ID..." + @LATEST_BACKUP_ID="$$(gcloud sql backups list --instance=krow-sql --project=$(GCP_PROJECT_ID) --sort-by=~endTime --limit=1 --format='value(id)')"; \ + if [ -z "$$LATEST_BACKUP_ID" ]; then \ + echo "❌ Could not find any backup ID for krow-sql."; \ + exit 1; \ + fi; \ + echo "✅ Latest backup ID: $$LATEST_BACKUP_ID"; \ + echo "⚠️ Restoring into validation..."; \ + gcloud sql backups restore $$LATEST_BACKUP_ID \ + --restore-instance=krow-sql-validation \ + --backup-instance=krow-sql \ + --project=$(GCP_PROJECT_ID) + + @echo "🎉 Done." + # ------------------------------------------------------------------- -# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT +# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT DEV # ------------------------------------------------------------------- # Check if gcloud and beta group are available @@ -85,7 +157,7 @@ check-gcloud-beta: } @echo "✅ gcloud CLI and 'gcloud beta' are available." -dataconnect-bootstrap-db: check-gcloud-beta +dataconnect-bootstrap-db: dataconnect-file-dev check-gcloud-beta @echo "🔍 Checking if Cloud SQL instance krow-sql already exists in [$(GCP_PROJECT_ID)]..." @if gcloud sql instances describe krow-sql --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ echo "⚠️ Cloud SQL instance 'krow-sql' already exists in project $(GCP_PROJECT_ID)."; \ @@ -112,7 +184,7 @@ dataconnect-bootstrap-db: check-gcloud-beta @echo "⚠️ Creating Firebase Data Connect service identity..." gcloud beta services identity create \ --service=firebasedataconnect.googleapis.com \ - --project=$(GCP_PROJECT_ID) + --project=$(GCP_PROJECT_ID) @echo "⚠️ Enabling IAM authentication on Cloud SQL instance krow-sql..." gcloud sql instances patch krow-sql \ @@ -124,9 +196,77 @@ dataconnect-bootstrap-db: check-gcloud-beta firebase dataconnect:sql:setup krow-workforce-db --project=$(FIREBASE_ALIAS) @echo "⚠️ Deploying initial Data Connect configuration..." - @firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS) + @firebase deploy --only dataconnect:$(DC_SERVICE_DEV) --project=$(FIREBASE_ALIAS) @echo "⚠️ Generating initial Data Connect SDK..." - @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) + @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) @echo "🎉 Cloud SQL + Data Connect bootstrap completed successfully!" + + +# ------------------------------------------------------------------- +# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT VALIDATEION +# ------------------------------------------------------------------- + +# Creates: krow-sql-validation + krow_db (inside it) + links service krow-workforce-db-validation +# Then clones data from krow-sql -> krow-sql-validation via backup/restore + +dataconnect-bootstrap-validation-database: dataconnect-file-validation + @echo "🔍 Checking if Cloud SQL instance krow-sql-validation already exists in [$(GCP_PROJECT_ID)]..." + @if gcloud sql instances describe krow-sql-validation --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \ + echo "⚠️ Cloud SQL instance 'krow-sql-validation' already exists in project $(GCP_PROJECT_ID)."; \ + echo " If you need to recreate it, delete the instance manually first."; \ + exit 1; \ + fi + + @echo "⚠️ Creating Cloud SQL instance krow-sql-validation (tier: $(SQL_TIER))..." + gcloud sql instances create krow-sql-validation \ + --database-version=POSTGRES_15 \ + --tier=$(SQL_TIER) \ + --region=us-central1 \ + --storage-size=10 \ + --storage-auto-increase \ + --availability-type=zonal \ + --backup-start-time=03:00 \ + --project=$(GCP_PROJECT_ID) + + @echo "⚠️ Creating Cloud SQL database krow_db on krow-sql-validation..." + gcloud sql databases create krow_db \ + --instance=krow-sql-validation \ + --project=$(GCP_PROJECT_ID) + + @echo "⚠️ Enabling IAM authentication on Cloud SQL instance krow-sql-validation..." + gcloud sql instances patch krow-sql-validation \ + --project=$(GCP_PROJECT_ID) \ + --database-flags=cloudsql.iam_authentication=on \ + --quiet + + @echo "🔁 Creating a backup on dev instance (krow-sql) to clone data into validation..." + @echo " (Prereq: krow-sql must already exist and be stable.)" + gcloud sql backups create --instance=krow-sql --project=$(GCP_PROJECT_ID) + echo "✅ Backup creation started. Operation: $$BACKUP_OP" + + @echo "🔎 Fetching latest backup ID from krow-sql..." + @BACKUP_ID="$$(gcloud sql backups list --instance=krow-sql --project=$(GCP_PROJECT_ID) --limit=1 --sort-by=~endTime --format='value(id)')"; \ + if [ -z "$$BACKUP_ID" ]; then \ + echo "❌ Could not find a backup ID for krow-sql."; \ + exit 1; \ + fi; \ + echo "✅ Latest backup ID: $$BACKUP_ID"; \ + echo "⚠️ Restoring backup into krow-sql-validation..."; \ + gcloud sql backups restore $$BACKUP_ID \ + --restore-instance=krow-sql-validation \ + --backup-instance=krow-sql \ + --project=$(GCP_PROJECT_ID) + + @echo "⚠️ Linking Data Connect service (krow-workforce-db-validation) with Cloud SQL..." + @echo " When prompted, select instance: krow-sql-validation and database: krow_db" + firebase dataconnect:sql:setup krow-workforce-db-validation --project=$(FIREBASE_ALIAS) + + @echo "⚠️ Deploying Data Connect configuration ($(DC_SERVICE_VALIDATION))..." + @firebase deploy --only dataconnect:$(DC_SERVICE_VALIDATION) --project=$(FIREBASE_ALIAS) + + @echo "⚠️ Generating Data Connect SDK ($(DC_SERVICE))..." + @firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS) + + @echo "🎉 Validation Cloud SQL + Data Connect bootstrap completed successfully!"