new instance for db validation dataconnect
This commit is contained in:
José Salazar
13
backend/dataconnect/dataconnect.dev.yaml
Normal file
13
backend/dataconnect/dataconnect.dev.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
specVersion: "v1"
|
||||
serviceId: "krow-workforce-db"
|
||||
location: "us-central1"
|
||||
schema:
|
||||
source: "./schema"
|
||||
datasource:
|
||||
postgresql:
|
||||
database: "krow_db"
|
||||
cloudSql:
|
||||
instanceId: "krow-sql"
|
||||
# schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly.
|
||||
# schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect.
|
||||
connectorDirs: ["./connector"]
|
||||
13
backend/dataconnect/dataconnect.validation.yaml
Normal file
13
backend/dataconnect/dataconnect.validation.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
specVersion: "v1"
|
||||
serviceId: "krow-workforce-db-validation"
|
||||
location: "us-central1"
|
||||
schema:
|
||||
source: "./schema"
|
||||
datasource:
|
||||
postgresql:
|
||||
database: "krow_db"
|
||||
cloudSql:
|
||||
instanceId: "krow-sql-validation"
|
||||
# schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly.
|
||||
# schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect.
|
||||
connectorDirs: ["./connector"]
|
||||
@@ -1,6 +1,47 @@
|
||||
# --- Data Connect / Backend ---
|
||||
|
||||
.PHONY: dataconnect-enable-apis dataconnect-init dataconnect-deploy dataconnect-sql-migrate dataconnect-generate-sdk dataconnect-sync dataconnect-bootstrap-db check-gcloud-beta dataconnect-clean
|
||||
# Usage examples:
|
||||
# make dataconnect-sync DC_ENV=dev
|
||||
# make dataconnect-seed DC_ENV=validation
|
||||
# make dataconnect-clean DC_ENV=validation
|
||||
# make dataconnect-generate-sdk DC_ENV=dev
|
||||
#
|
||||
DC_ENV ?= dev
|
||||
|
||||
DC_SERVICE_DEV := krow-workforce-db
|
||||
DC_SERVICE_VALIDATION := krow-workforce-db-validation
|
||||
|
||||
ifeq ($(DC_ENV),dev)
|
||||
DC_SERVICE := $(DC_SERVICE_DEV)
|
||||
else ifeq ($(DC_ENV),validation)
|
||||
DC_SERVICE := $(DC_SERVICE_VALIDATION)
|
||||
else
|
||||
$(error Invalid DC_ENV '$(DC_ENV)'. Use DC_ENV=dev or DC_ENV=validation)
|
||||
endif
|
||||
|
||||
.PHONY: dataconnect-enable-apis dataconnect-init dataconnect-deploy dataconnect-sql-migrate dataconnect-generate-sdk dataconnect-sync dataconnect-bootstrap-db check-gcloud-beta dataconnect-clean dataconnect-bootstrap-validation-db dataconnect-file dataconnect-file-validation dataconnect-file-dev dataconnect-seed dataconnect-test
|
||||
|
||||
#creation dataconnect file
|
||||
dataconnect-file:
|
||||
@echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..."
|
||||
@test -f backend/dataconnect/dataconnect.$(DC_ENV).yaml || (echo "❌ Missing backend/dataconnect/dataconnect.$(DC_ENV).yaml" && exit 1)
|
||||
@cp backend/dataconnect/dataconnect.$(DC_ENV).yaml backend/dataconnect/dataconnect.yaml
|
||||
@echo "✅ Creation Data Connect file completed."
|
||||
|
||||
#creation dev dataconnect file
|
||||
dataconnect-file-dev:
|
||||
@echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..."
|
||||
@test -f backend/dataconnect/dataconnect.dev.yaml || (echo "❌ Missing backend/dataconnect/dataconnect.dev.yaml" && exit 1)
|
||||
@cp backend/dataconnect/dataconnect.dev.yaml backend/dataconnect/dataconnect.yaml
|
||||
@echo "✅ Creation Data Connect file completed."
|
||||
|
||||
|
||||
#creation validation dataconnect file
|
||||
dataconnect-file-validation:
|
||||
@echo "--> Starting creation Firebase Data Connect schema file for service [$(DC_SERVICE)]..."
|
||||
@test -f backend/dataconnect/dataconnect.validation.yaml || (echo "❌ Missing backend/dataconnect/dataconnect.validation.yaml" && exit 1)
|
||||
@cp backend/dataconnect/dataconnect.validation.yaml backend/dataconnect/dataconnect.yaml
|
||||
@echo "✅ Creation Data Connect file completed."
|
||||
|
||||
# Enable all required APIs for Firebase Data Connect + Cloud SQL
|
||||
dataconnect-enable-apis:
|
||||
@@ -14,63 +55,94 @@ dataconnect-enable-apis:
|
||||
@echo "✅ APIs enabled for project [$(GCP_PROJECT_ID)]."
|
||||
|
||||
# Initialize Firebase Data Connect (interactive wizard).
|
||||
# use only once per project
|
||||
dataconnect-init:
|
||||
@echo "--> Initializing Firebase Data Connect for alias [$(FIREBASE_ALIAS)] (project: $(GCP_PROJECT_ID))..."
|
||||
@firebase init dataconnect --project $(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect initialization command executed. Follow the interactive steps in the CLI."
|
||||
|
||||
# Deploy Data Connect schemas (GraphQL → Cloud SQL)
|
||||
dataconnect-deploy:
|
||||
@echo "--> Deploying Firebase Data Connect schemas to [$(ENV)] (project: $(FIREBASE_ALIAS))..."
|
||||
@firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS)
|
||||
dataconnect-deploy: dataconnect-file
|
||||
@echo "--> Deploying Firebase Data Connect schemas to [$(ENV)] (service: $(DC_SERVICE)) (project: $(FIREBASE_ALIAS))..."
|
||||
@firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect deployment completed for [$(ENV)]."
|
||||
|
||||
# Apply pending SQL migrations for Firebase Data Connect
|
||||
dataconnect-sql-migrate:
|
||||
@echo "--> Applying Firebase Data Connect SQL migrations to [$(ENV)] (project: $(FIREBASE_ALIAS))..."
|
||||
dataconnect-sql-migrate: dataconnect-file
|
||||
@echo "--> Applying Firebase Data Connect SQL migrations to [$(ENV)] (service: $(DC_SERVICE)) (project: $(FIREBASE_ALIAS))..."
|
||||
@firebase dataconnect:sql:migrate --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect SQL migration completed for [$(ENV)]."
|
||||
|
||||
# Generate Data Connect client SDK for frontend-web and internal-api-harness
|
||||
dataconnect-generate-sdk:
|
||||
dataconnect-generate-sdk: dataconnect-file
|
||||
@echo "--> Generating Firebase Data Connect SDK for web frontend and API harness..."
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect SDK generation completed for [$(ENV)]."
|
||||
|
||||
# Unified backend schema update workflow (schema -> deploy -> SDK)
|
||||
dataconnect-sync:
|
||||
@echo "--> [1/3] Deploying Data Connect..."
|
||||
@firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS)
|
||||
@echo "--> [2/3] Applying SQL migrations..."
|
||||
@firebase dataconnect:sql:migrate --project=$(FIREBASE_ALIAS)
|
||||
@echo "--> [3/3] Regenerating SDK..."
|
||||
dataconnect-sync: dataconnect-file
|
||||
@echo "--> [1/3] Deploying Data Connect [$(DC_SERVICE)]..."
|
||||
@firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS)
|
||||
@echo "--> [2/3] Applying SQL migrations [$(DC_SERVICE)]..."
|
||||
@firebase dataconnect:sql:migrate $(DC_SERVICE) --project=$(FIREBASE_ALIAS)
|
||||
@echo "--> [3/3] Regenerating SDK [$(DC_SERVICE)]..."
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect SQL, deploy, and SDK generation completed for [$(ENV)]."
|
||||
@echo "✅ Data Connect SQL, deploy, and SDK generation [$(ENV)]."
|
||||
|
||||
# Execute seed in Firebase Data Connect
|
||||
dataconnect-seed:
|
||||
@echo "--> Exec seed in Firebase Data Connect..."
|
||||
@firebase dataconnect:execute backend/dataconnect/functions/seed.gql --project=$(FIREBASE_ALIAS)
|
||||
dataconnect-seed: dataconnect-file
|
||||
@echo "--> Exec seed in Firebase Data Connect (service: $(DC_SERVICE))..."
|
||||
@firebase dataconnect:execute backend/dataconnect/functions/seed.gql --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Seed executed successfully."
|
||||
|
||||
|
||||
# Execute clean, to delete all the data in Firebase Data Connect
|
||||
dataconnect-clean:
|
||||
@echo "--> Exec clean all the data in Firebase Data Connect..."
|
||||
@firebase dataconnect:execute backend/dataconnect/functions/clean.gql --project=$(FIREBASE_ALIAS)
|
||||
dataconnect-clean: dataconnect-file
|
||||
@echo "--> Exec clean all the data in Firebase Data Connect (service: $(DC_SERVICE))..."
|
||||
@firebase dataconnect:execute backend/dataconnect/functions/clean.gql --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Clean information executed successfully."
|
||||
|
||||
# Run tests for Data Connect deployment and migrations
|
||||
dataconnect-test:
|
||||
@echo "--> Running Data Connect tests..."
|
||||
@echo "--> [1/3] Deploying Data Connect..."
|
||||
@firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS) --dry-run
|
||||
@echo "--> [2/3] Applying SQL migrations..."
|
||||
dataconnect-test: dataconnect-file
|
||||
@echo "--> Running Data Connect tests (service: $(DC_SERVICE))..."
|
||||
@echo "--> [1/2] Deploying Data Connect..."
|
||||
@firebase deploy --only dataconnect:$(DC_SERVICE) --project=$(FIREBASE_ALIAS) --dry-run
|
||||
@echo "--> [2/2] Applying SQL migrations..."
|
||||
@firebase dataconnect:sql:diff --project=$(FIREBASE_ALIAS)
|
||||
@echo "✅ Data Connect tests completed."
|
||||
|
||||
dataconnect-backup-dev-to-validation:
|
||||
@echo "🔍 Validating instances exist in [$(GCP_PROJECT_ID)]..."
|
||||
@if ! gcloud sql instances describe krow-sql --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
|
||||
echo "❌ Dev instance 'krow-sql' not found in project $(GCP_PROJECT_ID)."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if ! gcloud sql instances describe krow-sql-validation --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
|
||||
echo "❌ Validation instance 'krow-sql-validation' not found in project $(GCP_PROJECT_ID)."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "✅ Instances found."
|
||||
|
||||
@echo "🧰 Creating a backup on dev (krow-sql)..."
|
||||
@gcloud sql backups create --instance=krow-sql --project=$(GCP_PROJECT_ID) >/dev/null
|
||||
|
||||
@echo "🔎 Fetching latest backup ID..."
|
||||
@LATEST_BACKUP_ID="$$(gcloud sql backups list --instance=krow-sql --project=$(GCP_PROJECT_ID) --sort-by=~endTime --limit=1 --format='value(id)')"; \
|
||||
if [ -z "$$LATEST_BACKUP_ID" ]; then \
|
||||
echo "❌ Could not find any backup ID for krow-sql."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
echo "✅ Latest backup ID: $$LATEST_BACKUP_ID"; \
|
||||
echo "⚠️ Restoring into validation..."; \
|
||||
gcloud sql backups restore $$LATEST_BACKUP_ID \
|
||||
--restore-instance=krow-sql-validation \
|
||||
--backup-instance=krow-sql \
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
|
||||
@echo "🎉 Done."
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT
|
||||
# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT DEV
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
# Check if gcloud and beta group are available
|
||||
@@ -85,7 +157,7 @@ check-gcloud-beta:
|
||||
}
|
||||
@echo "✅ gcloud CLI and 'gcloud beta' are available."
|
||||
|
||||
dataconnect-bootstrap-db: check-gcloud-beta
|
||||
dataconnect-bootstrap-db: dataconnect-file-dev check-gcloud-beta
|
||||
@echo "🔍 Checking if Cloud SQL instance krow-sql already exists in [$(GCP_PROJECT_ID)]..."
|
||||
@if gcloud sql instances describe krow-sql --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
|
||||
echo "⚠️ Cloud SQL instance 'krow-sql' already exists in project $(GCP_PROJECT_ID)."; \
|
||||
@@ -112,7 +184,7 @@ dataconnect-bootstrap-db: check-gcloud-beta
|
||||
@echo "⚠️ Creating Firebase Data Connect service identity..."
|
||||
gcloud beta services identity create \
|
||||
--service=firebasedataconnect.googleapis.com \
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
|
||||
@echo "⚠️ Enabling IAM authentication on Cloud SQL instance krow-sql..."
|
||||
gcloud sql instances patch krow-sql \
|
||||
@@ -124,9 +196,77 @@ dataconnect-bootstrap-db: check-gcloud-beta
|
||||
firebase dataconnect:sql:setup krow-workforce-db --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "⚠️ Deploying initial Data Connect configuration..."
|
||||
@firebase deploy --only dataconnect --project=$(FIREBASE_ALIAS)
|
||||
@firebase deploy --only dataconnect:$(DC_SERVICE_DEV) --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "⚠️ Generating initial Data Connect SDK..."
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "🎉 Cloud SQL + Data Connect bootstrap completed successfully!"
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ONE-TIME FULL SETUP FOR CLOUD SQL + DATA CONNECT VALIDATEION
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
# Creates: krow-sql-validation + krow_db (inside it) + links service krow-workforce-db-validation
|
||||
# Then clones data from krow-sql -> krow-sql-validation via backup/restore
|
||||
|
||||
dataconnect-bootstrap-validation-database: dataconnect-file-validation
|
||||
@echo "🔍 Checking if Cloud SQL instance krow-sql-validation already exists in [$(GCP_PROJECT_ID)]..."
|
||||
@if gcloud sql instances describe krow-sql-validation --project=$(GCP_PROJECT_ID) >/dev/null 2>&1; then \
|
||||
echo "⚠️ Cloud SQL instance 'krow-sql-validation' already exists in project $(GCP_PROJECT_ID)."; \
|
||||
echo " If you need to recreate it, delete the instance manually first."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
@echo "⚠️ Creating Cloud SQL instance krow-sql-validation (tier: $(SQL_TIER))..."
|
||||
gcloud sql instances create krow-sql-validation \
|
||||
--database-version=POSTGRES_15 \
|
||||
--tier=$(SQL_TIER) \
|
||||
--region=us-central1 \
|
||||
--storage-size=10 \
|
||||
--storage-auto-increase \
|
||||
--availability-type=zonal \
|
||||
--backup-start-time=03:00 \
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
|
||||
@echo "⚠️ Creating Cloud SQL database krow_db on krow-sql-validation..."
|
||||
gcloud sql databases create krow_db \
|
||||
--instance=krow-sql-validation \
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
|
||||
@echo "⚠️ Enabling IAM authentication on Cloud SQL instance krow-sql-validation..."
|
||||
gcloud sql instances patch krow-sql-validation \
|
||||
--project=$(GCP_PROJECT_ID) \
|
||||
--database-flags=cloudsql.iam_authentication=on \
|
||||
--quiet
|
||||
|
||||
@echo "🔁 Creating a backup on dev instance (krow-sql) to clone data into validation..."
|
||||
@echo " (Prereq: krow-sql must already exist and be stable.)"
|
||||
gcloud sql backups create --instance=krow-sql --project=$(GCP_PROJECT_ID)
|
||||
echo "✅ Backup creation started. Operation: $$BACKUP_OP"
|
||||
|
||||
@echo "🔎 Fetching latest backup ID from krow-sql..."
|
||||
@BACKUP_ID="$$(gcloud sql backups list --instance=krow-sql --project=$(GCP_PROJECT_ID) --limit=1 --sort-by=~endTime --format='value(id)')"; \
|
||||
if [ -z "$$BACKUP_ID" ]; then \
|
||||
echo "❌ Could not find a backup ID for krow-sql."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
echo "✅ Latest backup ID: $$BACKUP_ID"; \
|
||||
echo "⚠️ Restoring backup into krow-sql-validation..."; \
|
||||
gcloud sql backups restore $$BACKUP_ID \
|
||||
--restore-instance=krow-sql-validation \
|
||||
--backup-instance=krow-sql \
|
||||
--project=$(GCP_PROJECT_ID)
|
||||
|
||||
@echo "⚠️ Linking Data Connect service (krow-workforce-db-validation) with Cloud SQL..."
|
||||
@echo " When prompted, select instance: krow-sql-validation and database: krow_db"
|
||||
firebase dataconnect:sql:setup krow-workforce-db-validation --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "⚠️ Deploying Data Connect configuration ($(DC_SERVICE_VALIDATION))..."
|
||||
@firebase deploy --only dataconnect:$(DC_SERVICE_VALIDATION) --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "⚠️ Generating Data Connect SDK ($(DC_SERVICE))..."
|
||||
@firebase dataconnect:sdk:generate --project=$(FIREBASE_ALIAS)
|
||||
|
||||
@echo "🎉 Validation Cloud SQL + Data Connect bootstrap completed successfully!"
|
||||
|
||||
Reference in New Issue
Block a user