package migrations import ( "doormile/internal/ai/registry" "doormile/models" "doormile/utils" "gorm.io/gorm" ) func Migrate(db *gorm.DB) error { utils.Info("Starting database auto-migrations for 21 logistics tables...") err := db.AutoMigrate( &models.PartnerInfo{}, &models.Hub{}, &models.Vehicle{}, &models.AppUser{}, &models.MilerProfile{}, &models.AppCustomer{}, &models.AppCustomerLocation{}, &models.Consignment{}, &models.PickupBooking{}, &models.BookingParcel{}, &models.BookingServiceOption{}, &models.BookingPayment{}, &models.BookingAssignment{}, &models.BookingVehicleRequirement{}, &models.Tripsheet{}, &models.TripsheetItem{}, &models.DeliveryProof{}, &models.Pricing{}, &models.ConsignmentHistory{}, &models.ConsignmentException{}, &models.TenantLocation{}, &models.AppLocation{}, &models.Tenant{}, &models.Customer{}, &models.CustomerLocation{}, &models.DoormileClient{}, &models.DoormileAuth{}, &models.CompetitorBranch{}, &models.CarrierPricing{}, &models.DoormilePricing{}, &models.AgentDecision{}, &models.AISkillFinding{}, &models.DemandForecast{}, &models.HubStaffAccount{}, &models.HubConversation{}, &models.HubMessage{}, &models.MilerDutyLog{}, &models.MilerBreakLog{}, &models.MilerSupportTicket{}, // Customer app (doormile_cx). All additive: new tables plus new // nullable/zero-default columns on pickupbookings, so an existing row // and every console-created booking stay valid with no backfill. &models.ServiceableState{}, &models.ServiceableDistrict{}, &models.PickupSlotTemplate{}, &models.CustomerBookingLimit{}, &models.BookingDestination{}, &models.BookingParcelPhoto{}, &models.BookingStageEvent{}, &models.CustomerRefreshToken{}, &models.CustomerDevice{}, // AI agent registry (agent-platform-plan Phase 1). Five new tables, // nothing existing touched. &models.AIAgent{}, &models.AITool{}, &models.AISkill{}, &models.AISkillTool{}, &models.AIRegistryAudit{}, // Agent runs recorded from AI_engine telemetry (plan Phase 4). One new // append-only table, pruned after 30 days. &models.AIAgentRun{}, ) if err != nil { utils.Error("❌ Database migration failed", "error", err) return err } utils.Info("✅ Database migration completed successfully!") // Upsert the code-defined agent registry. A failure is logged, not fatal: // the registry is read by Agent Studio and AI_engine, and neither is on a // booking's path, so it must not stop the API from serving orders. if err := registry.Seed(db); err != nil { utils.Error("⚠️ AI registry seed failed", "error", err.Error()) } // pgvector must exist before the vector column can be declared. This was // missing: the ALTER TABLE below has always run against a database where // the extension was installed by hand, and its failure is logged // non-fatally — so on any database where it was not, the column and its // index silently never existed and every similarity query 500s. // // Creating an extension needs rights a plain application role may not // have. A failure here is logged and not fatal for the same reason the // column add is not: the registry and decision log are not on a booking's // path, and the API must keep serving orders. if res := db.Exec(`CREATE EXTENSION IF NOT EXISTS vector`); res.Error != nil { utils.Error("⚠️ pgvector extension unavailable — decision memory and similarity search are disabled", "error", res.Error) } else { utils.Info("✅ pgvector extension ready") } if res := db.Exec(`ALTER TABLE agent_decisions ADD COLUMN IF NOT EXISTS context_embedding vector(1536)`); res.Error != nil { utils.Error("❌ Failed to add context_embedding column", "error", res.Error) } else { utils.Info("✅ context_embedding vector column ready") } if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_agent_decisions_embedding ON agent_decisions USING ivfflat (context_embedding vector_cosine_ops) WITH (lists = 100)`); res.Error != nil { utils.Error("❌ Failed to create ivfflat index on context_embedding", "error", res.Error) } else { utils.Info("✅ ivfflat index on context_embedding ready") } // The tripsheets.status CHECK constraint predates this codebase (not derived // from any gorm tag) and only allowed Draft/Dispatched/Arrived/Cancelled. // Hub batches need an intermediate "Ready" stage before dispatch. db.Exec(`ALTER TABLE tripsheets DROP CONSTRAINT IF EXISTS tripsheets_status_check`) if res := db.Exec(`ALTER TABLE tripsheets ADD CONSTRAINT tripsheets_status_check CHECK (status::text = ANY (ARRAY['Draft','Ready','Dispatched','Arrived','Cancelled']::text[]))`); res.Error != nil { utils.Error("❌ Failed to widen tripsheets_status_check constraint", "error", res.Error) } else { utils.Info("✅ tripsheets_status_check constraint includes Ready") } // The consignments.status CHECK constraint also predates this codebase (not // derived from a gorm tag) and was missing two values the code actually // writes: Collected_By_Miler — the hyperlocal two-step intermediate state, // which a pickup-complete writes when MILER_COLLECTED_STATE_ENABLED is on — // and Cancelled, which the admin console sets via PUT /consignments/:id/status. // Without them the INSERT/UPDATE fails with a 23514 check violation. Widen it // to the full set of constants.Consignment* statuses plus Cancelled. db.Exec(`ALTER TABLE consignments DROP CONSTRAINT IF EXISTS consignments_status_check`) if res := db.Exec(`ALTER TABLE consignments ADD CONSTRAINT consignments_status_check CHECK (status::text = ANY (ARRAY['Created','Inwarded_at_Hub','Collected_By_Miler','Tripsheet_Loaded','In_Transit','Out_for_Delivery','Delivered','RTO_Initiated','Returned_to_Sender','Missing','Damaged','Cancelled']::text[]))`); res.Error != nil { utils.Error("❌ Failed to widen consignments_status_check constraint", "error", res.Error) } else { utils.Info("✅ consignments_status_check constraint includes Collected_By_Miler") } // Human-facing identifiers for the customer app: DM-###### for a pickup // booking, DMX######## for an order. Sequence-backed rather than random, // because both columns are UNIQUE and a random short id collides long // before a short id runs out — a collision here is a failed booking at the // moment of payment, not a retry. // // No CYCLE and no MAXVALUE on purpose: past 999999 the format simply grows // a digit (DM-1000000) instead of wrapping onto an id that already exists. // Existing rows keep their old DM-BK-/DM-TRK- strings; nothing parses // either format, so the two coexist safely. if res := db.Exec(`CREATE SEQUENCE IF NOT EXISTS cx_booking_reference_seq START 100000 INCREMENT 1`); res.Error != nil { utils.Error("❌ Failed to create cx_booking_reference_seq", "error", res.Error) } else { utils.Info("✅ cx_booking_reference_seq ready") } if res := db.Exec(`CREATE SEQUENCE IF NOT EXISTS cx_tracking_seq START 10000000 INCREMENT 1`); res.Error != nil { utils.Error("❌ Failed to create cx_tracking_seq", "error", res.Error) } else { utils.Info("✅ cx_tracking_seq ready") } // One booking must not hold two destinations at the same position: the // customer addresses a destination by index in // PATCH /customer/bookings/{ref}/destinations/{index}, so a duplicate seq // makes that route ambiguous and would let an edit land on the wrong // address. if res := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_bookingdestinations_booking_seq ON bookingdestinations (bookingid, seq)`); res.Error != nil { utils.Error("❌ Failed to create bookingdestinations (bookingid, seq) unique index", "error", res.Error) } else { utils.Info("✅ bookingdestinations (bookingid, seq) unique index ready") } // The timeline is read as "every event for this booking, oldest first" on // every tracking poll, which is the hottest customer read there is. if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_bookingstageevents_booking_time ON bookingstageevents (bookingid, occurredat)`); res.Error != nil { utils.Error("❌ Failed to create bookingstageevents (bookingid, occurredat) index", "error", res.Error) } else { utils.Info("✅ bookingstageevents (bookingid, occurredat) index ready") } // Reverse logistics for clients onboarded before the field existed. // // They have no delivery category and were all using returns, so they must // keep them — new information must not withdraw a working capability. Done // as a one-off backfill rather than a column default: a default makes GORM // omit an explicit `false` on every future INSERT, which is how Food // clients were silently created with returns enabled. // // Guarded on deliverycategory being empty, so it only ever touches rows // that predate the field and can never re-enable returns for a client an // operator has deliberately switched off. if res := db.Exec(`UPDATE tenants SET reverselogisticsenabled = true WHERE COALESCE(deliverycategory, '') = ''`); res.Error != nil { utils.Error("❌ Failed to backfill tenants.reverselogisticsenabled", "error", res.Error) } else if res.RowsAffected > 0 { utils.Info("✅ reverse logistics kept on for pre-existing clients", "rows", res.RowsAffected) } // ── ETA calibration (docs/prediction-plan.md rung 1.1) ────────────────── // // consignment_booking resolves a consignment to its booking. There is no // consignments.bookingid column, and the link runs two ways: through // bookingdestinations for multi-destination customer pickups, and through // the legacy pickupbookings.consignmentid for console/express bookings and // anything written before the fan-out existed. That legacy column names // only the FIRST order of a multi-destination pickup, which is why the // bookingdestinations path is tried first and the fallback is guarded. // // Every query that needs a consignment's booking goes through this view. // Joining consignments to bookings by hand attaches features to the wrong // parcel for multi-destination bookings, silently — the defect CLAUDE.md // §8.5 describes and docs/prediction-plan.md records as hazard H4. if res := db.Exec(`CREATE OR REPLACE VIEW consignment_booking AS SELECT c.consignmentid, COALESCE(bd.bookingid, pb.bookingid) AS bookingid, bd.bookingdestinationid FROM consignments c LEFT JOIN bookingdestinations bd ON bd.consignmentid = c.consignmentid LEFT JOIN pickupbookings pb ON pb.consignmentid = c.consignmentid AND bd.bookingid IS NULL`); res.Error != nil { utils.Error("❌ Failed to create consignment_booking view", "error", res.Error) } else { utils.Info("✅ consignment_booking view ready") } // etacalibration holds the grouped p80 of actual-over-routed duration that // internal/prediction multiplies a routed ETA by. Written only by the // calibration sweeper, read at boot. Empty is the normal state until // ROUTE_OPTIMIZER_URL is configured and deliveries accumulate — the // estimator falls back to the existing promise tables while it is. if res := db.Exec(`CREATE TABLE IF NOT EXISTS etacalibration ( calibrationid SERIAL PRIMARY KEY, scope VARCHAR(24) NOT NULL, zone VARCHAR(8), hourbucket INTEGER, weekday INTEGER, factor DOUBLE PRECISION NOT NULL, handling DOUBLE PRECISION NOT NULL DEFAULT 0, samples INTEGER NOT NULL, refreshedat TIMESTAMPTZ NOT NULL )`); res.Error != nil { utils.Error("❌ Failed to create etacalibration table", "error", res.Error) } else { utils.Info("✅ etacalibration table ready") } if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_etacalibration_lookup ON etacalibration (scope, zone, hourbucket, weekday)`); res.Error != nil { utils.Error("❌ Failed to create etacalibration lookup index", "error", res.Error) } else { utils.Info("✅ etacalibration lookup index ready") } // The calibration scan joins deliveries to their assignment. Without this // the refresh sequential-scans consignmenthistory on every run. // // CONCURRENTLY is not optional here. A plain CREATE INDEX takes a SHARE // lock, which blocks INSERTs for as long as the build takes — and // consignmenthistory gets a row on EVERY parcel status change. On a table // of any size that means riders cannot complete deliveries while the // migration runs: a revenue-path outage caused by an index for a // background job that is switched off by default. // // The tradeoffs CONCURRENTLY brings, and why they are acceptable: // - It cannot run inside a transaction. db.Exec is autocommit, so this // is fine, but it must never be moved inside a tx block. // - It can fail and leave an INVALID index behind, which is then not // used by the planner and must be dropped by hand. That degrades the // calibration scan to a sequential one; it does not affect any // booking. The log line below names the recovery. if res := db.Exec(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_consignmenthistory_status_consignment ON consignmenthistory (eventstatus, consignmentid)`); res.Error != nil { utils.Error("⚠️ consignmenthistory index not created — the ETA calibration scan will be slower. "+ "If this left an INVALID index, drop it before retrying: "+ "DROP INDEX IF EXISTS idx_consignmenthistory_status_consignment", "error", res.Error) } else { utils.Info("✅ consignmenthistory (eventstatus, consignmentid) index ready") } return nil }