293 lines
13 KiB
Go
293 lines
13 KiB
Go
package migrations
|
|
|
|
import (
|
|
"doormile/internal/ai/registry"
|
|
"doormile/models"
|
|
"doormile/utils"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
func Migrate(db *gorm.DB) error {
|
|
utils.Info("Starting database auto-migrations for 21 logistics tables...")
|
|
|
|
err := db.AutoMigrate(
|
|
&models.PartnerInfo{},
|
|
&models.Hub{},
|
|
&models.Vehicle{},
|
|
&models.AppUser{},
|
|
&models.MilerProfile{},
|
|
&models.AppCustomer{},
|
|
&models.AppCustomerLocation{},
|
|
&models.Consignment{},
|
|
&models.PickupBooking{},
|
|
&models.BookingParcel{},
|
|
&models.BookingServiceOption{},
|
|
&models.BookingPayment{},
|
|
&models.BookingAssignment{},
|
|
&models.BookingVehicleRequirement{},
|
|
&models.Tripsheet{},
|
|
&models.TripsheetItem{},
|
|
&models.DeliveryProof{},
|
|
&models.Pricing{},
|
|
&models.ConsignmentHistory{},
|
|
&models.ConsignmentException{},
|
|
&models.TenantLocation{},
|
|
&models.AppLocation{},
|
|
&models.Tenant{},
|
|
&models.Customer{},
|
|
&models.CustomerLocation{},
|
|
&models.DoormileClient{},
|
|
&models.DoormileAuth{},
|
|
&models.CompetitorBranch{},
|
|
&models.CarrierPricing{},
|
|
&models.DoormilePricing{},
|
|
&models.AgentDecision{},
|
|
&models.AISkillFinding{},
|
|
&models.DemandForecast{},
|
|
&models.HubStaffAccount{},
|
|
&models.HubConversation{},
|
|
&models.HubMessage{},
|
|
&models.MilerDutyLog{},
|
|
&models.MilerBreakLog{},
|
|
&models.MilerSupportTicket{},
|
|
|
|
// Customer app (doormile_cx). All additive: new tables plus new
|
|
// nullable/zero-default columns on pickupbookings, so an existing row
|
|
// and every console-created booking stay valid with no backfill.
|
|
&models.ServiceableState{},
|
|
&models.ServiceableDistrict{},
|
|
&models.PickupSlotTemplate{},
|
|
&models.CustomerBookingLimit{},
|
|
&models.BookingDestination{},
|
|
&models.BookingParcelPhoto{},
|
|
&models.BookingStageEvent{},
|
|
&models.CustomerRefreshToken{},
|
|
&models.CustomerDevice{},
|
|
|
|
// AI agent registry (agent-platform-plan Phase 1). Five new tables,
|
|
// nothing existing touched.
|
|
&models.AIAgent{},
|
|
&models.AITool{},
|
|
&models.AISkill{},
|
|
&models.AISkillTool{},
|
|
&models.AIRegistryAudit{},
|
|
|
|
// Agent runs recorded from AI_engine telemetry (plan Phase 4). One new
|
|
// append-only table, pruned after 30 days.
|
|
&models.AIAgentRun{},
|
|
)
|
|
|
|
if err != nil {
|
|
utils.Error("❌ Database migration failed", "error", err)
|
|
return err
|
|
}
|
|
|
|
utils.Info("✅ Database migration completed successfully!")
|
|
|
|
// Upsert the code-defined agent registry. A failure is logged, not fatal:
|
|
// the registry is read by Agent Studio and AI_engine, and neither is on a
|
|
// booking's path, so it must not stop the API from serving orders.
|
|
if err := registry.Seed(db); err != nil {
|
|
utils.Error("⚠️ AI registry seed failed", "error", err.Error())
|
|
}
|
|
|
|
// pgvector must exist before the vector column can be declared. This was
|
|
// missing: the ALTER TABLE below has always run against a database where
|
|
// the extension was installed by hand, and its failure is logged
|
|
// non-fatally — so on any database where it was not, the column and its
|
|
// index silently never existed and every similarity query 500s.
|
|
//
|
|
// Creating an extension needs rights a plain application role may not
|
|
// have. A failure here is logged and not fatal for the same reason the
|
|
// column add is not: the registry and decision log are not on a booking's
|
|
// path, and the API must keep serving orders.
|
|
if res := db.Exec(`CREATE EXTENSION IF NOT EXISTS vector`); res.Error != nil {
|
|
utils.Error("⚠️ pgvector extension unavailable — decision memory and similarity search are disabled",
|
|
"error", res.Error)
|
|
} else {
|
|
utils.Info("✅ pgvector extension ready")
|
|
}
|
|
|
|
if res := db.Exec(`ALTER TABLE agent_decisions ADD COLUMN IF NOT EXISTS context_embedding vector(1536)`); res.Error != nil {
|
|
utils.Error("❌ Failed to add context_embedding column", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ context_embedding vector column ready")
|
|
}
|
|
|
|
if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_agent_decisions_embedding ON agent_decisions USING ivfflat (context_embedding vector_cosine_ops) WITH (lists = 100)`); res.Error != nil {
|
|
utils.Error("❌ Failed to create ivfflat index on context_embedding", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ ivfflat index on context_embedding ready")
|
|
}
|
|
|
|
// The tripsheets.status CHECK constraint predates this codebase (not derived
|
|
// from any gorm tag) and only allowed Draft/Dispatched/Arrived/Cancelled.
|
|
// Hub batches need an intermediate "Ready" stage before dispatch.
|
|
db.Exec(`ALTER TABLE tripsheets DROP CONSTRAINT IF EXISTS tripsheets_status_check`)
|
|
if res := db.Exec(`ALTER TABLE tripsheets ADD CONSTRAINT tripsheets_status_check
|
|
CHECK (status::text = ANY (ARRAY['Draft','Ready','Dispatched','Arrived','Cancelled']::text[]))`); res.Error != nil {
|
|
utils.Error("❌ Failed to widen tripsheets_status_check constraint", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ tripsheets_status_check constraint includes Ready")
|
|
}
|
|
|
|
// The consignments.status CHECK constraint also predates this codebase (not
|
|
// derived from a gorm tag) and was missing two values the code actually
|
|
// writes: Collected_By_Miler — the hyperlocal two-step intermediate state,
|
|
// which a pickup-complete writes when MILER_COLLECTED_STATE_ENABLED is on —
|
|
// and Cancelled, which the admin console sets via PUT /consignments/:id/status.
|
|
// Without them the INSERT/UPDATE fails with a 23514 check violation. Widen it
|
|
// to the full set of constants.Consignment* statuses plus Cancelled.
|
|
db.Exec(`ALTER TABLE consignments DROP CONSTRAINT IF EXISTS consignments_status_check`)
|
|
if res := db.Exec(`ALTER TABLE consignments ADD CONSTRAINT consignments_status_check
|
|
CHECK (status::text = ANY (ARRAY['Created','Inwarded_at_Hub','Collected_By_Miler','Tripsheet_Loaded','In_Transit','Out_for_Delivery','Delivered','RTO_Initiated','Returned_to_Sender','Missing','Damaged','Cancelled']::text[]))`); res.Error != nil {
|
|
utils.Error("❌ Failed to widen consignments_status_check constraint", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ consignments_status_check constraint includes Collected_By_Miler")
|
|
}
|
|
|
|
// Human-facing identifiers for the customer app: DM-###### for a pickup
|
|
// booking, DMX######## for an order. Sequence-backed rather than random,
|
|
// because both columns are UNIQUE and a random short id collides long
|
|
// before a short id runs out — a collision here is a failed booking at the
|
|
// moment of payment, not a retry.
|
|
//
|
|
// No CYCLE and no MAXVALUE on purpose: past 999999 the format simply grows
|
|
// a digit (DM-1000000) instead of wrapping onto an id that already exists.
|
|
// Existing rows keep their old DM-BK-/DM-TRK- strings; nothing parses
|
|
// either format, so the two coexist safely.
|
|
if res := db.Exec(`CREATE SEQUENCE IF NOT EXISTS cx_booking_reference_seq START 100000 INCREMENT 1`); res.Error != nil {
|
|
utils.Error("❌ Failed to create cx_booking_reference_seq", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ cx_booking_reference_seq ready")
|
|
}
|
|
if res := db.Exec(`CREATE SEQUENCE IF NOT EXISTS cx_tracking_seq START 10000000 INCREMENT 1`); res.Error != nil {
|
|
utils.Error("❌ Failed to create cx_tracking_seq", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ cx_tracking_seq ready")
|
|
}
|
|
|
|
// One booking must not hold two destinations at the same position: the
|
|
// customer addresses a destination by index in
|
|
// PATCH /customer/bookings/{ref}/destinations/{index}, so a duplicate seq
|
|
// makes that route ambiguous and would let an edit land on the wrong
|
|
// address.
|
|
if res := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_bookingdestinations_booking_seq
|
|
ON bookingdestinations (bookingid, seq)`); res.Error != nil {
|
|
utils.Error("❌ Failed to create bookingdestinations (bookingid, seq) unique index", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ bookingdestinations (bookingid, seq) unique index ready")
|
|
}
|
|
|
|
// The timeline is read as "every event for this booking, oldest first" on
|
|
// every tracking poll, which is the hottest customer read there is.
|
|
if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_bookingstageevents_booking_time
|
|
ON bookingstageevents (bookingid, occurredat)`); res.Error != nil {
|
|
utils.Error("❌ Failed to create bookingstageevents (bookingid, occurredat) index", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ bookingstageevents (bookingid, occurredat) index ready")
|
|
}
|
|
|
|
// Reverse logistics for clients onboarded before the field existed.
|
|
//
|
|
// They have no delivery category and were all using returns, so they must
|
|
// keep them — new information must not withdraw a working capability. Done
|
|
// as a one-off backfill rather than a column default: a default makes GORM
|
|
// omit an explicit `false` on every future INSERT, which is how Food
|
|
// clients were silently created with returns enabled.
|
|
//
|
|
// Guarded on deliverycategory being empty, so it only ever touches rows
|
|
// that predate the field and can never re-enable returns for a client an
|
|
// operator has deliberately switched off.
|
|
if res := db.Exec(`UPDATE tenants SET reverselogisticsenabled = true
|
|
WHERE COALESCE(deliverycategory, '') = ''`); res.Error != nil {
|
|
utils.Error("❌ Failed to backfill tenants.reverselogisticsenabled", "error", res.Error)
|
|
} else if res.RowsAffected > 0 {
|
|
utils.Info("✅ reverse logistics kept on for pre-existing clients", "rows", res.RowsAffected)
|
|
}
|
|
|
|
// ── ETA calibration (docs/prediction-plan.md rung 1.1) ──────────────────
|
|
//
|
|
// consignment_booking resolves a consignment to its booking. There is no
|
|
// consignments.bookingid column, and the link runs two ways: through
|
|
// bookingdestinations for multi-destination customer pickups, and through
|
|
// the legacy pickupbookings.consignmentid for console/express bookings and
|
|
// anything written before the fan-out existed. That legacy column names
|
|
// only the FIRST order of a multi-destination pickup, which is why the
|
|
// bookingdestinations path is tried first and the fallback is guarded.
|
|
//
|
|
// Every query that needs a consignment's booking goes through this view.
|
|
// Joining consignments to bookings by hand attaches features to the wrong
|
|
// parcel for multi-destination bookings, silently — the defect CLAUDE.md
|
|
// §8.5 describes and docs/prediction-plan.md records as hazard H4.
|
|
if res := db.Exec(`CREATE OR REPLACE VIEW consignment_booking AS
|
|
SELECT c.consignmentid,
|
|
COALESCE(bd.bookingid, pb.bookingid) AS bookingid,
|
|
bd.bookingdestinationid
|
|
FROM consignments c
|
|
LEFT JOIN bookingdestinations bd ON bd.consignmentid = c.consignmentid
|
|
LEFT JOIN pickupbookings pb ON pb.consignmentid = c.consignmentid
|
|
AND bd.bookingid IS NULL`); res.Error != nil {
|
|
utils.Error("❌ Failed to create consignment_booking view", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ consignment_booking view ready")
|
|
}
|
|
|
|
// etacalibration holds the grouped p80 of actual-over-routed duration that
|
|
// internal/prediction multiplies a routed ETA by. Written only by the
|
|
// calibration sweeper, read at boot. Empty is the normal state until
|
|
// ROUTE_OPTIMIZER_URL is configured and deliveries accumulate — the
|
|
// estimator falls back to the existing promise tables while it is.
|
|
if res := db.Exec(`CREATE TABLE IF NOT EXISTS etacalibration (
|
|
calibrationid SERIAL PRIMARY KEY,
|
|
scope VARCHAR(24) NOT NULL,
|
|
zone VARCHAR(8),
|
|
hourbucket INTEGER,
|
|
weekday INTEGER,
|
|
factor DOUBLE PRECISION NOT NULL,
|
|
handling DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
samples INTEGER NOT NULL,
|
|
refreshedat TIMESTAMPTZ NOT NULL
|
|
)`); res.Error != nil {
|
|
utils.Error("❌ Failed to create etacalibration table", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ etacalibration table ready")
|
|
}
|
|
|
|
if res := db.Exec(`CREATE INDEX IF NOT EXISTS idx_etacalibration_lookup
|
|
ON etacalibration (scope, zone, hourbucket, weekday)`); res.Error != nil {
|
|
utils.Error("❌ Failed to create etacalibration lookup index", "error", res.Error)
|
|
} else {
|
|
utils.Info("✅ etacalibration lookup index ready")
|
|
}
|
|
|
|
// The calibration scan joins deliveries to their assignment. Without this
|
|
// the refresh sequential-scans consignmenthistory on every run.
|
|
//
|
|
// CONCURRENTLY is not optional here. A plain CREATE INDEX takes a SHARE
|
|
// lock, which blocks INSERTs for as long as the build takes — and
|
|
// consignmenthistory gets a row on EVERY parcel status change. On a table
|
|
// of any size that means riders cannot complete deliveries while the
|
|
// migration runs: a revenue-path outage caused by an index for a
|
|
// background job that is switched off by default.
|
|
//
|
|
// The tradeoffs CONCURRENTLY brings, and why they are acceptable:
|
|
// - It cannot run inside a transaction. db.Exec is autocommit, so this
|
|
// is fine, but it must never be moved inside a tx block.
|
|
// - It can fail and leave an INVALID index behind, which is then not
|
|
// used by the planner and must be dropped by hand. That degrades the
|
|
// calibration scan to a sequential one; it does not affect any
|
|
// booking. The log line below names the recovery.
|
|
if res := db.Exec(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_consignmenthistory_status_consignment
|
|
ON consignmenthistory (eventstatus, consignmentid)`); res.Error != nil {
|
|
utils.Error("⚠️ consignmenthistory index not created — the ETA calibration scan will be slower. "+
|
|
"If this left an INVALID index, drop it before retrying: "+
|
|
"DROP INDEX IF EXISTS idx_consignmenthistory_status_consignment",
|
|
"error", res.Error)
|
|
} else {
|
|
utils.Info("✅ consignmenthistory (eventstatus, consignmentid) index ready")
|
|
}
|
|
|
|
return nil
|
|
}
|