updates on the ai agents and time series prediction and updates on the api to
This commit is contained in:
214
internal/ai/outcomes/rules_test.go
Normal file
214
internal/ai/outcomes/rules_test.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package outcomes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func at(h int) time.Time {
|
||||
return time.Date(2026, 10, 5, h, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func tp(t time.Time) *time.Time { return &t }
|
||||
|
||||
const window = 48 * time.Hour
|
||||
|
||||
// ─── Too soon to judge ─────────────────────────────────────────────────────
|
||||
|
||||
func TestYoungDecisionIsLeftPending(t *testing.T) {
|
||||
decided := at(10)
|
||||
for _, dt := range []string{"stall_response", "assignment_failure", "something_new"} {
|
||||
_, ok := judge(dt, bookingFacts{DecidedAt: decided}, decided.Add(5*time.Minute), window)
|
||||
if ok {
|
||||
t.Errorf("%s: judged a 5-minute-old decision; the booking is still in flight", dt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingWhileInsideTheWindow(t *testing.T) {
|
||||
decided := at(10)
|
||||
// An hour in: no delivery yet, but the window has not closed. Recording
|
||||
// failure here would blame the decision for a parcel still on its way.
|
||||
if _, ok := judge("stall_response", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok {
|
||||
t.Error("stall_response judged before its window closed")
|
||||
}
|
||||
if _, ok := judge("assignment_failure", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok {
|
||||
t.Error("assignment_failure judged before its window closed")
|
||||
}
|
||||
}
|
||||
|
||||
// ─── stall_response ────────────────────────────────────────────────────────
|
||||
|
||||
func TestStallDeliveredOnTimeIsSuccess(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{
|
||||
DecidedAt: decided,
|
||||
DeliveredAt: tp(decided.Add(3 * time.Hour)),
|
||||
SLADueAt: tp(decided.Add(12 * time.Hour)),
|
||||
}
|
||||
got, ok := judge("stall_response", f, decided.Add(4*time.Hour), window)
|
||||
if !ok || got != OutcomeSuccess {
|
||||
t.Errorf("got %q ok=%v, want success", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// The case that matters most: counting any eventual delivery as success would
|
||||
// teach the model that the action is always vindicated, which is exactly the
|
||||
// wrong lesson.
|
||||
func TestStallDeliveredLateIsFailure(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{
|
||||
DecidedAt: decided,
|
||||
DeliveredAt: tp(decided.Add(20 * time.Hour)),
|
||||
SLADueAt: tp(decided.Add(12 * time.Hour)),
|
||||
}
|
||||
got, ok := judge("stall_response", f, decided.Add(21*time.Hour), window)
|
||||
if !ok || got != OutcomeFailure {
|
||||
t.Errorf("got %q ok=%v, want failure — delivered 8h past SLA", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStallDeliveredWithNoSLAIsSuccess(t *testing.T) {
|
||||
decided := at(10)
|
||||
// No sladueat recorded (an older row). Delivery is the best evidence
|
||||
// available and there is nothing to call it late against.
|
||||
f := bookingFacts{DecidedAt: decided, DeliveredAt: tp(decided.Add(5 * time.Hour))}
|
||||
got, ok := judge("stall_response", f, decided.Add(6*time.Hour), window)
|
||||
if !ok || got != OutcomeSuccess {
|
||||
t.Errorf("got %q ok=%v, want success", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStallNeverDeliveredAfterWindowIsFailure(t *testing.T) {
|
||||
decided := at(10)
|
||||
got, ok := judge("stall_response", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window)
|
||||
if !ok || got != OutcomeFailure {
|
||||
t.Errorf("got %q ok=%v, want failure", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStallCancelledIsFailureImmediately(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{DecidedAt: decided, Cancelled: true}
|
||||
// Cancellation is conclusive — no need to wait out the window.
|
||||
got, ok := judge("stall_response", f, decided.Add(time.Hour), window)
|
||||
if !ok || got != OutcomeFailure {
|
||||
t.Errorf("got %q ok=%v, want failure on cancellation", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── assignment_failure ────────────────────────────────────────────────────
|
||||
|
||||
func TestAssignmentLaterAssignedIsSuccess(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{
|
||||
DecidedAt: decided,
|
||||
Assigned: true,
|
||||
AssignedAt: tp(decided.Add(2 * time.Hour)),
|
||||
}
|
||||
got, ok := judge("assignment_failure", f, decided.Add(3*time.Hour), window)
|
||||
if !ok || got != OutcomeSuccess {
|
||||
t.Errorf("got %q ok=%v, want success", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// A rider assigned BEFORE the decision is not evidence the decision worked —
|
||||
// it is the assignment that was already there. Without the ordering check,
|
||||
// every reassignment decision would read as an instant success.
|
||||
func TestAssignmentPredatingTheDecisionIsNotSuccess(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{
|
||||
DecidedAt: decided,
|
||||
Assigned: true,
|
||||
AssignedAt: tp(decided.Add(-2 * time.Hour)),
|
||||
}
|
||||
got, ok := judge("assignment_failure", f, decided.Add(time.Hour), window)
|
||||
if ok && got == OutcomeSuccess {
|
||||
t.Error("counted a pre-existing assignment as the decision's success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignmentNeverAssignedAfterWindowIsFailure(t *testing.T) {
|
||||
decided := at(10)
|
||||
got, ok := judge("assignment_failure", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window)
|
||||
if !ok || got != OutcomeFailure {
|
||||
t.Errorf("got %q ok=%v, want failure", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignmentCancelledIsFailure(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{DecidedAt: decided, Cancelled: true}
|
||||
got, ok := judge("assignment_failure", f, decided.Add(time.Hour), window)
|
||||
if !ok || got != OutcomeFailure {
|
||||
t.Errorf("got %q ok=%v, want failure", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Unknown types are never guessed ───────────────────────────────────────
|
||||
|
||||
func TestUnknownTypeIsNeverSuccessOrFailure(t *testing.T) {
|
||||
decided := at(10)
|
||||
// Inside the window: pending.
|
||||
if _, ok := judge("a_new_decision_type", bookingFacts{DecidedAt: decided}, decided.Add(time.Hour), window); ok {
|
||||
t.Error("judged an unknown decision type inside its window")
|
||||
}
|
||||
// Past it: unknown, so it stops being rescanned — but never a label that
|
||||
// would teach the model something nobody defined.
|
||||
got, ok := judge("a_new_decision_type", bookingFacts{DecidedAt: decided}, decided.Add(window+time.Hour), window)
|
||||
if !ok || got != OutcomeUnknown {
|
||||
t.Errorf("got %q ok=%v, want unknown", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivered unknown type must still not be called a success: the rule for
|
||||
// what success means for that type does not exist yet.
|
||||
func TestUnknownTypeWithDeliveryIsStillUnknown(t *testing.T) {
|
||||
decided := at(10)
|
||||
f := bookingFacts{DecidedAt: decided, DeliveredAt: tp(decided.Add(time.Hour))}
|
||||
got, ok := judge("a_new_decision_type", f, decided.Add(window+time.Hour), window)
|
||||
if !ok || got != OutcomeUnknown {
|
||||
t.Errorf("got %q ok=%v, want unknown", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestOutcomeWindowDefaultAndOverride(t *testing.T) {
|
||||
t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "")
|
||||
if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour {
|
||||
t.Errorf("default window = %v, want %v", got, defaultOutcomeWindowHours*time.Hour)
|
||||
}
|
||||
t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "12")
|
||||
if got := OutcomeWindow(); got != 12*time.Hour {
|
||||
t.Errorf("window = %v, want 12h", got)
|
||||
}
|
||||
// Garbage falls back rather than producing a zero window, which would
|
||||
// judge every decision the instant it passed minDecisionAge.
|
||||
t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "not-a-number")
|
||||
if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour {
|
||||
t.Errorf("window on garbage = %v, want the default", got)
|
||||
}
|
||||
t.Setenv("AGENT_OUTCOME_WINDOW_HOURS", "0")
|
||||
if got := OutcomeWindow(); got != defaultOutcomeWindowHours*time.Hour {
|
||||
t.Errorf("window on 0 = %v, want the default", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionDaysDefaultAndOverride(t *testing.T) {
|
||||
t.Setenv("AGENT_DECISION_RETENTION_DAYS", "")
|
||||
if got := RetentionDays(); got != 180 {
|
||||
t.Errorf("default retention = %d, want 180", got)
|
||||
}
|
||||
t.Setenv("AGENT_DECISION_RETENTION_DAYS", "90")
|
||||
if got := RetentionDays(); got != 90 {
|
||||
t.Errorf("retention = %d, want 90", got)
|
||||
}
|
||||
// Retention must be longer than aiagentruns' 30 days, because old
|
||||
// precedent is the point of this table. Not enforced in code — asserted
|
||||
// here so a future change to the default has to confront it.
|
||||
t.Setenv("AGENT_DECISION_RETENTION_DAYS", "")
|
||||
if RetentionDays() <= 30 {
|
||||
t.Error("decision retention is no longer than telemetry retention; precedent will be purged before it is useful")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user