doormile_backend/internal/ai/outcomes/backfill.go

package outcomes

import (
	"encoding/json"
	"time"

	"doormile/constants"
	"doormile/models"
	"doormile/utils"

	"gorm.io/gorm"
)

// Seeding the decision memory from history, so recall is not useless for weeks.
//
// ─── The cold-start problem this solves ────────────────────────────────────
//
// `/internal/agent-decisions/similar` filters on `outcome IS NOT NULL`. A row
// becomes eligible only after the outcome sweeper has judged it, and a decision
// is only judged once its window has closed. So the sequence for a freshly
// switched-on memory is:
//
//	switch embeddings on -> wait for stalls to happen -> wait 48h per decision
//	-> the sweeper labels them -> only now does recall return anything
//
// With autonomy gates off and two decision types, that is weeks of recall
// returning [] — which reads as "retrieval does not help here" rather than
// "retrieval has nothing to retrieve yet". The first conclusion is wrong and
// expensive to un-learn.
//
// This backfills decisions from bookings whose outcome is ALREADY known.
//
// ─── What it does and does not claim ───────────────────────────────────────
//
// A backfilled row is not a decision the engine made. It is a record of a
// situation that occurred and how it ended, shaped so the retrieval path can
// use it as precedent. That distinction is recorded honestly:
//
//	decision.action   = "none"      — nothing was decided; nobody intervened
//	decision.source   = "backfill"  — so these are distinguishable forever
//	reasoning         = states plainly that this is historical, not a decision
//
// Why that matters: as_prompt_block renders `action` to the model. Writing a
// plausible-looking action here would teach the model that an action it never
// took produced the outcome that followed — which is worse than no memory.
// "none" is honest: this is what happened when nothing was done.
//
// ─── It writes no embeddings ───────────────────────────────────────────────
//
// Embedding is the engine's job (AI_engine/core/embeddings.py) and needs the
// provider key, which this process does not have. Backfilled rows land with
// NULL context_embedding and are invisible to similarity search until
// something embeds them. That is deliberate: a backfill that silently created
// un-embedded rows AND claimed to have seeded the memory would be the worse
// failure. BackfillStats reports the count so the caller knows what is owed.

// BackfillStats is what one run produced.
type BackfillStats struct {
	Scanned  int `json:"scanned"`
	Inserted int `json:"inserted"`
	Skipped  int `json:"skipped"`
	// NeedsEmbedding is Inserted — every backfilled row still needs a vector
	// before it can be retrieved. Surfaced separately so it cannot be missed.
	NeedsEmbedding int `json:"needsembedding"`
}

// historicalRow is one past booking whose ending is known.
//
// Joins through consignment_booking, never consignments.bookingid — that column
// does not exist, and the legacy pickupbookings.consignmentid link names only
// the first order of a multi-destination pickup (hazard H4).
type historicalRow struct {
	Bookingid       int
	Tenantid        *uint64
	Deliverypincode string
	Status          string
	Createdat       time.Time
	Deliveredat     *time.Time
	Sladueat        *time.Time
	Attemptcount    int
	Chargeableweight float64
}

const historicalSQL = `
SELECT pb.bookingid,
       pb.tenantid                        AS tenantid,
       COALESCE(c.deliverypincode, '')    AS deliverypincode,
       COALESCE(c.status, pb.status)      AS status,
       c.createdat                        AS createdat,
       del.createdat                      AS deliveredat,
       c.sladueat                         AS sladueat,
       COALESCE(c.attemptcount, 0)        AS attemptcount,
       COALESCE(c.chargeableweight, 0)    AS chargeableweight
FROM consignments c
JOIN consignment_booking cb ON cb.consignmentid = c.consignmentid
JOIN pickupbookings pb      ON pb.bookingid = cb.bookingid
LEFT JOIN (
    SELECT DISTINCT ON (consignmentid) consignmentid, createdat
    FROM consignmenthistory
    WHERE eventstatus = ?
    ORDER BY consignmentid, createdat ASC
) del ON del.consignmentid = c.consignmentid
WHERE c.deletedat IS NULL
  -- Only parcels whose story has ended. An in-flight parcel has no outcome to
  -- learn from, and guessing one is exactly what this must not do.
  AND (del.createdat IS NOT NULL OR c.status IN ?)
  -- Not already backfilled or decided for. The uniqueness is on
  -- (decision_type, booking_id), enforced here rather than by a constraint
  -- because real decisions legitimately repeat for one booking.
  AND NOT EXISTS (
      SELECT 1 FROM agent_decisions d
      WHERE d.booking_id = pb.bookingid AND d.decision_type = ?
  )
ORDER BY c.createdat DESC
LIMIT ?
`

// BackfillDecisionType is its own type, kept separate from the engine's
// `stall_response` and `assignment_failure`. Recall is per-type, so backfilled
// precedent is retrievable on purpose and never silently mixed into a type the
// engine thinks it authored.
const BackfillDecisionType = "historical_delivery"

// Backfill writes historical precedent rows. Idempotent: a booking that already
// has a decision of this type is skipped, so re-running adds only what is new.
//
// limit bounds one run — this scans delivery history, which is the largest
// table pair in the database.
func Backfill(gdb *gorm.DB, limit int) (BackfillStats, error) {
	if gdb == nil {
		return BackfillStats{}, nil
	}
	if limit <= 0 {
		limit = 2000
	}

	terminal := []string{
		constants.ConsignmentDelivered,
		"Returned_to_Sender",
		"Missing",
		"Damaged",
	}

	var rows []historicalRow
	if err := gdb.Raw(historicalSQL,
		constants.ConsignmentDelivered, terminal, BackfillDecisionType, limit,
	).Scan(&rows).Error; err != nil {
		return BackfillStats{}, err
	}

	stats := BackfillStats{Scanned: len(rows)}
	batch := make([]models.AgentDecision, 0, len(rows))

	for _, r := range rows {
		outcome := OutcomeFailure
		switch {
		case r.Deliveredat == nil:
			// Terminal but never delivered: returned, lost or damaged.
			outcome = OutcomeFailure
		case r.Sladueat != nil && r.Deliveredat.After(*r.Sladueat):
			// Delivered late. Counting this a success would teach that any
			// eventual delivery is a good outcome — the same trap the
			// stall_response rule avoids.
			outcome = OutcomeFailure
		default:
			outcome = OutcomeSuccess
		}

		// The facts shape mirrors what AI_engine puts in context.facts, so an
		// embedding of a backfilled row sits in the same space as a real one.
		// If these diverge, retrieval returns neighbours that are near in
		// vector space for the wrong reasons.
		facts := map[string]any{
			"booking_id":        r.Bookingid,
			"delivery_pincode":  r.Deliverypincode,
			"attempt_count":     r.Attemptcount,
			"chargeable_weight": r.Chargeableweight,
			"final_status":      r.Status,
		}
		if r.Deliveredat != nil {
			facts["hours_to_deliver"] = int(r.Deliveredat.Sub(r.Createdat).Hours())
		}

		contextJSON, err := json.Marshal(map[string]any{
			"facts":  facts,
			"model":  "none",
			"source": "backfill",
		})
		if err != nil {
			stats.Skipped++
			continue
		}
		decisionJSON, err := json.Marshal(map[string]any{
			// Honest: nobody decided anything. See the package comment — a
			// plausible-looking action here would be a fabricated lesson.
			"action":     "none",
			"confidence": 0.0,
			"source":     "backfill",
		})
		if err != nil {
			stats.Skipped++
			continue
		}

		recordedAt := utils.DBNow()
		batch = append(batch, models.AgentDecision{
			DecisionType:      BackfillDecisionType,
			BookingID:         u64(r.Bookingid),
			TenantID:          r.Tenantid,
			Context:           string(contextJSON),
			Decision:          string(decisionJSON),
			Reasoning:         "Historical outcome backfilled from delivery records. No agent decision was made for this booking; this row records what happened when nothing intervened.",
			Outcome:           &outcome,
			OutcomeRecordedAt: &recordedAt,
			CreatedAt:         r.Createdat,
		})
	}

	if len(batch) == 0 {
		return stats, nil
	}
	if err := gdb.CreateInBatches(&batch, 200).Error; err != nil {
		return stats, err
	}
	stats.Inserted = len(batch)
	stats.NeedsEmbedding = len(batch)

	utils.Info("outcomes: backfilled historical precedent",
		"scanned", stats.Scanned, "inserted", stats.Inserted, "skipped", stats.Skipped,
		"note", "rows have no embedding yet and are not retrievable until one is written")
	return stats, nil
}

func u64(n int) *uint64 {
	if n <= 0 {
		return nil
	}
	v := uint64(n)
	return &v
}