doormile_backend/internal/prediction/calibration.go

package prediction

import (
	"time"

	"doormile/constants"
	"doormile/db"
	"doormile/utils"

	"gorm.io/gorm"
)

// The calibration refresh.
//
// Every delivered consignment whose rider was sequenced carries two numbers:
// what the Route Optimization API predicted (bookingassignments.etaminutes,
// written by internal/routing) and what actually happened (the Delivered row in
// consignmenthistory). The ratio between them, grouped and taken at p80, is the
// factor ETAMinutes multiplies by.
//
// Why p80 and not the mean: docs/prediction-plan.md §8 decision 3. An ETA is a
// promise, and a mean is late half the time.
//
// Why a view and not a join here: consignments has no bookingid column (hazard
// H4 in the plan). The consignment_booking view resolves the two paths —
// bookingdestinations for multi-destination pickups, the legacy
// pickupbookings.consignmentid for console and pre-fan-out rows — once, in
// migrate.go. Joining through consignments directly attaches features to the
// wrong parcel for every multi-destination booking, silently.

// ETACalibration is one calibration cell as stored. Written only by Refresh,
// read at boot and after each refresh.
type ETACalibration struct {
	Calibrationid int       `gorm:"primaryKey;column:calibrationid;autoIncrement"`
	Scope         string    `gorm:"column:scope;size:24;not null;index:idx_etacalibration_lookup,priority:1"`
	Zone          string    `gorm:"column:zone;size:8;index:idx_etacalibration_lookup,priority:2"`
	Hourbucket    *int      `gorm:"column:hourbucket;index:idx_etacalibration_lookup,priority:3"`
	Weekday       *int      `gorm:"column:weekday;index:idx_etacalibration_lookup,priority:4"`
	Factor        float64   `gorm:"column:factor;not null"`
	Handling      float64   `gorm:"column:handling;not null;default:0"`
	Samples       int       `gorm:"column:samples;not null"`
	Refreshedat   time.Time `gorm:"column:refreshedat;not null"`
}

func (ETACalibration) TableName() string { return "etacalibration" }

// row is one group from the refresh query.
type row struct {
	Scope      string
	Zone       string
	Hourbucket *int
	Weekday    *int
	Factor     float64
	Samples    int
}

// Observed durations below this are almost certainly data errors — a Delivered
// event written in the same second as the consignment, or a backfill. Including
// them drags every factor toward zero.
const minObservedMinutes = 5

// And above this, the parcel sat for days for a reason that has nothing to do
// with road duration (held at hub, re-attempts, disputes). Calibrating a
// routed-duration multiplier on those teaches it the wrong thing.
const maxObservedMinutes = 48 * 60

// refreshSQL computes the p80 ratio of actual to routed duration, at three
// grains plus a global row, in one pass.
//
// The `actual` term uses consignmenthistory.createdat - consignments.createdat.
// Both come from CURRENT_TIMESTAMP defaults, so they share a tagging and their
// difference is correct regardless of hazard H1 — which is why this does not
// go near estimateddeliveryat, whose writes are not uniform
// (adminController.go:2738 uses time.Now(), elsewhere it is CURRENT_TIMESTAMP).
//
// EXTRACT(ISODOW) is 1..7 Monday-first, matching weekdayOf in eta.go. The hour
// bucket divides by 3, matching hourBucketSize. If either changes, both change.
// A booking can hold SEVERAL bookingassignments rows — Assigned, Rejected,
// Reassigned, Cancelled are all statuses it passes through. Joining them all
// would multiply one delivered parcel by its whole assignment history and pull
// the p80 toward whatever got reassigned most, so `assigned` picks exactly one
// row per booking: the most recently sequenced one, which is the routed ETA
// that was actually in force when the parcel was delivered.
//
// NULL::int / NULL::text are explicit because a UNION ALL resolves column types
// across its branches; an untyped NULL can be inferred as text and fail against
// the integer from the first branch — at runtime, against the real database,
// which is exactly where it is most expensive to discover.
const refreshSQL = `
WITH assigned AS (
    SELECT DISTINCT ON (ba.bookingid)
           ba.bookingid,
           ba.etaminutes
    FROM bookingassignments ba
    WHERE ba.etaminutes > 0
    ORDER BY ba.bookingid, ba.sequencedat DESC NULLS LAST, ba.bookingassignmentid DESC
),
observed AS (
    SELECT left(c.deliverypincode, 3)                              AS zone,
           (EXTRACT(HOUR FROM c.createdat)::int / ?)                AS hourbucket,
           EXTRACT(ISODOW FROM c.createdat)::int                    AS weekday,
           a.etaminutes::double precision                           AS routed,
           EXTRACT(EPOCH FROM (h.createdat - c.createdat)) / 60.0   AS actual
    FROM consignments c
    JOIN consignmenthistory h
           ON h.consignmentid = c.consignmentid
          AND h.eventstatus = ?
    JOIN consignment_booking cb
           ON cb.consignmentid = c.consignmentid
    JOIN assigned a
           ON a.bookingid = cb.bookingid
    WHERE c.deliverypincode IS NOT NULL
      AND length(c.deliverypincode) >= 3
),
clean AS (
    SELECT * FROM observed
    WHERE actual BETWEEN ? AND ?
      AND routed > 0
)
SELECT 'zone_hour_weekday'::text AS scope, zone, hourbucket, weekday,
       percentile_cont(0.8) WITHIN GROUP (ORDER BY actual / routed) AS factor,
       count(*)::int AS samples
FROM clean GROUP BY zone, hourbucket, weekday
UNION ALL
SELECT 'zone_weekday'::text, zone, NULL::int, weekday,
       percentile_cont(0.8) WITHIN GROUP (ORDER BY actual / routed), count(*)::int
FROM clean GROUP BY zone, weekday
UNION ALL
SELECT 'zone'::text, zone, NULL::int, NULL::int,
       percentile_cont(0.8) WITHIN GROUP (ORDER BY actual / routed), count(*)::int
FROM clean GROUP BY zone
UNION ALL
SELECT 'global'::text, ''::text, NULL::int, NULL::int,
       percentile_cont(0.8) WITHIN GROUP (ORDER BY actual / routed), count(*)::int
FROM clean
`

// Refresh recomputes the calibration from history, replaces the stored table,
// and swaps the in-memory snapshot.
//
// A failure leaves the previous calibration in place — a refresh that cannot
// run is not a reason to stop answering with the last good factors. If they go
// stale past staleAfter, ETAMinutes stops trusting them on its own.
func Refresh(gdb *gorm.DB) error {
	if gdb == nil {
		return nil
	}

	var rows []row
	if err := gdb.Raw(refreshSQL,
		hourBucketSize, constants.ConsignmentDelivered, minObservedMinutes, maxObservedMinutes,
	).Scan(&rows).Error; err != nil {
		return err
	}

	kept := make([]ETACalibration, 0, len(rows))
	now := utils.DBNow()
	for _, r := range rows {
		if r.Samples < minSamples {
			continue // a p80 over fewer than minSamples is noise
		}
		if r.Factor < minFactor || r.Factor > maxFactor {
			// Out of bounds is a signal about the data, not a usable factor.
			// Log it once here rather than discovering it per request.
			utils.Warn("prediction: discarding out-of-bounds calibration factor",
				"scope", r.Scope, "zone", r.Zone, "factor", r.Factor, "samples", r.Samples)
			continue
		}
		kept = append(kept, ETACalibration{
			Scope:       r.Scope,
			Zone:        r.Zone,
			Hourbucket:  r.Hourbucket,
			Weekday:     r.Weekday,
			Factor:      r.Factor,
			Samples:     r.Samples,
			Refreshedat: now,
		})
	}

	if len(kept) == 0 {
		// No cell cleared the floor. Expected until routing is on and history
		// accumulates; ETAMinutes keeps returning false and the promise tables
		// keep answering.
		utils.Info("prediction: no calibration cells met the sample floor",
			"groups_considered", len(rows), "min_samples", minSamples)
		return nil
	}

	// Replace wholesale in one transaction: a partial table would serve a mix
	// of old and new factors for the same zone.
	if err := gdb.Transaction(func(tx *gorm.DB) error {
		if err := tx.Exec(`DELETE FROM etacalibration`).Error; err != nil {
			return err
		}
		return tx.CreateInBatches(&kept, 200).Error
	}); err != nil {
		return err
	}

	Load(kept)
	utils.Info("prediction: calibration refreshed", "cells", len(kept))
	return nil
}

// Load swaps the in-memory snapshot. Exported so boot can populate from the
// stored table without recomputing, and so tests can install a known
// calibration without a database.
//
// builtAt is the NEWEST Refreshedat among the rows, not time.Now(): a restart
// that loads month-old rows from the store must still look month-old to the
// staleness guard in ETAMinutes. Taking the load time here would silently
// re-arm a stale calibration on every deploy. A row with no Refreshedat (a test
// fixture) is treated as fresh.
// Refreshedat is stored through utils.DBNow, which writes IST wall-clock digits
// labelled UTC. Reading it back as an instant is off by 5h30m, so it goes
// through utils.IST first — the same correction utils/epoch.go exists for.
// Without it a fresh calibration reads as 5.5 hours old, which is the very
// hazard docs/prediction-plan.md calls H1.
func Load(rows []ETACalibration) {
	t := &table{cells: make(map[string]cell, len(rows))}
	for _, r := range rows {
		if r.Refreshedat.IsZero() {
			continue
		}
		if at := utils.IST(r.Refreshedat); at.After(t.builtAt) {
			t.builtAt = at
		}
	}
	if t.builtAt.IsZero() {
		t.builtAt = time.Now() // a fixture with no Refreshedat counts as fresh
	}
	for _, r := range rows {
		c := cell{factor: r.Factor, handling: r.Handling, samples: r.Samples}
		switch r.Scope {
		case "global":
			t.global, t.hasGlobal = c, true
		case "zone":
			t.cells[keyZone(r.Zone)] = c
		case "zone_weekday":
			if r.Weekday != nil {
				t.cells[keyZoneWeekday(r.Zone, *r.Weekday)] = c
			}
		case "zone_hour_weekday":
			if r.Weekday != nil && r.Hourbucket != nil {
				t.cells[keyZoneHourWeekday(r.Zone, *r.Hourbucket, *r.Weekday)] = c
			}
		}
	}
	current.store(t)
}

// Reset clears the in-memory calibration. Tests only — it makes ETAMinutes
// return false, which is the state a fresh process is in before boot loads.
func Reset() { current.store(nil) }

// LoadFromDB populates the snapshot from the stored table at boot, so a restart
// does not wait for the next refresh to start answering.
func LoadFromDB() {
	if db.DB == nil {
		return
	}
	var rows []ETACalibration
	if err := db.DB.Find(&rows).Error; err != nil {
		utils.Warn("prediction: could not load stored calibration", "error", err)
		return
	}
	if len(rows) == 0 {
		return
	}
	Load(rows)
	utils.Info("prediction: calibration loaded from store", "cells", len(rows))
}