package outcomes import ( "testing" "time" ) func at(h int) time.Time { return time.Date(2026, 10, 5, h, 0, 0, 0, time.UTC) } func tp(t time.Time) *time.Time { return &t } const window = 48 * time.Hour // ─── Too soon to judge ───────────────────────────────────────────────────── func TestYoungDecisionIsLeftPending(t *testing.T) { decided := at(10) for _, dt := range []string{"stall_response", "assignment_failure", "something_new"} { _, ok := judge(dt, bookingFacts{DecidedAt: decided}, decided.Add(5*time.Minute), window) if ok { t.Errorf("%s: judged a 5-minute-old decision; the booking is still in flight", dt) } } } func TestPendingWhileInsideTheWindow(t *testing.T) { decided := at(10) // An hour in: no delivery yet, but the window has not closed. Recording // failure here would blame the decision for a parcel still on its way. if _, ok := judge("stall_response", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok { t.Error("stall_response judged before its window closed") } if _, ok := judge("assignment_failure", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok { t.Error("assignment_failure judged before its window closed") } } // ─── stall_response ──────────────────────────────────────────────────────── func TestStallDeliveredOnTimeIsSuccess(t *testing.T) { decided := at(10) f := bookingFacts{ DecidedAt: decided, DeliveredAt: tp(decided.Add(3 * time.Hour)), SLADueAt: tp(decided.Add(12 * time.Hour)), } got, ok := judge("stall_response", f, decided.Add(4*time.Hour), window) if !ok || got != OutcomeSuccess { t.Errorf("got %q ok=%v, want success", got, ok) } } // The case that matters most: counting any eventual delivery as success would // teach the model that the action is always vindicated, which is exactly the // wrong lesson. func TestStallDeliveredLateIsFailure(t *testing.T) { decided := at(10) f := bookingFacts{ DecidedAt: decided, DeliveredAt: tp(decided.Add(20 * time.Hour)), SLADueAt: tp(decided.Add(12 * time.Hour)), } got, ok := judge("stall_response", f, decided.Add(21*time.Hour), window) if !ok || got != OutcomeFailure { t.Errorf("got %q ok=%v, want failure — delivered 8h past SLA", got, ok) } } func TestStallDeliveredWithNoSLAIsSuccess(t *testing.T) { decided := at(10) // No sladueat recorded (an older row). Delivery is the best evidence // available and there is nothing to call it late against. f := bookingFacts{DecidedAt: decided, DeliveredAt: tp(decided.Add(5 * time.Hour))} got, ok := judge("stall_response", f, decided.Add(6*time.Hour), window) if !ok || got != OutcomeSuccess { t.Errorf("got %q ok=%v, want success", got, ok) } } func TestStallNeverDeliveredAfterWindowIsFailure(t *testing.T) { decided := at(10) got, ok := judge("stall_response", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window) if !ok || got != OutcomeFailure { t.Errorf("got %q ok=%v, want failure", got, ok) } } func TestStallCancelledIsFailureImmediately(t *testing.T) { decided := at(10) f := bookingFacts{DecidedAt: decided, Cancelled: true} // Cancellation is conclusive — no need to wait out the window. got, ok := judge("stall_response", f, decided.Add(time.Hour), window) if !ok || got != OutcomeFailure { t.Errorf("got %q ok=%v, want failure on cancellation", got, ok) } } // ─── assignment_failure ──────────────────────────────────────────────────── func TestAssignmentLaterAssignedIsSuccess(t *testing.T) { decided := at(10) f := bookingFacts{ DecidedAt: decided, Assigned: true, AssignedAt: tp(decided.Add(2 * time.Hour)), } got, ok := judge("assignment_failure", f, decided.Add(3*time.Hour), window) if !ok || got != OutcomeSuccess { t.Errorf("got %q ok=%v, want success", got, ok) } } // A rider assigned BEFORE the decision is not evidence the decision worked — // it is the assignment that was already there. Without the ordering check, // every reassignment decision would read as an instant success. func TestAssignmentPredatingTheDecisionIsNotSuccess(t *testing.T) { decided := at(10) f := bookingFacts{ DecidedAt: decided, Assigned: true, AssignedAt: tp(decided.Add(-2 * time.Hour)), } got, ok := judge("assignment_failure", f, decided.Add(time.Hour), window) if ok && got == OutcomeSuccess { t.Error("counted a pre-existing assignment as the decision's success") } } func TestAssignmentNeverAssignedAfterWindowIsFailure(t *testing.T) { decided := at(10) got, ok := judge("assignment_failure", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window) if !ok || got != OutcomeFailure { t.Errorf("got %q ok=%v, want failure", got, ok) } } func TestAssignmentCancelledIsFailure(t *testing.T) { decided := at(10) f := bookingFacts{DecidedAt: decided, Cancelled: true} got, ok := judge("assignment_failure", f, decided.Add(time.Hour), window) if !ok || got != OutcomeFailure { t.Errorf("got %q ok=%v, want failure", got, ok) } } // ─── Unknown types are never guessed ─────────────────────────────────────── func TestUnknownTypeIsNeverSuccessOrFailure(t *testing.T) { decided := at(10) // Inside the window: pending. if _, ok := judge("a_new_decision_type", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok { t.Error("judged an unknown decision type inside its window") } // Past it: unknown, so it stops being rescanned — but never a label that // would teach the model something nobody defined. got, ok := judge("a_new_decision_type", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window) if !ok || got != OutcomeUnknown { t.Errorf("got %q ok=%v, want unknown", got, ok) } } // A delivered unknown type must still not be called a success: the rule for // what success means for that type does not exist yet. func TestUnknownTypeWithDeliveryIsStillUnknown(t *testing.T) { decided := at(10) f := bookingFacts{DecidedAt: decided, DeliveredAt: tp(decided.Add(time.Hour))} got, ok := judge("a_new_decision_type", f, decided.Add(window+time.Hour), window) if !ok || got != OutcomeUnknown { t.Errorf("got %q ok=%v, want unknown", got, ok) } } // ─── Configuration ───────────────────────────────────────────────────────── func TestOutcomeWindowDefaultAndOverride(t *testing.T) { t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "") if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour { t.Errorf("default window = %v, want %v", got, defaultOutcomeWindowHours*time.Hour) } t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "12") if got := OutcomeWindow(); got != 12*time.Hour { t.Errorf("window = %v, want 12h", got) } // Garbage falls back rather than producing a zero window, which would // judge every decision the instant it passed minDecisionAge. t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "not-a-number") if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour { t.Errorf("window on garbage = %v, want the default", got) } t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "0") if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour { t.Errorf("window on 0 = %v, want the default", got) } } func TestRetentionDaysDefaultAndOverride(t *testing.T) { t.Setenv("AGENT_DECISION_RETENTION_DAYS", "") if got := RetentionDays(); got != 180 { t.Errorf("default retention = %d, want 180", got) } t.Setenv("AGENT_DECISION_RETENTION_DAYS", "90") if got := RetentionDays(); got != 90 { t.Errorf("retention = %d, want 90", got) } // Retention must be longer than aiagentruns' 30 days, because old // precedent is the point of this table. Not enforced in code — asserted // here so a future change to the default has to confront it. t.Setenv("AGENT_DECISION_RETENTION_DAYS", "") if RetentionDays() <= 30 { t.Error("decision retention is no longer than telemetry retention; precedent will be purged before it is useful") } }