package runtime import ( "context" "testing" "github.com/krow/krow-backend/go-api/internal/gateway" ) // standbyGateway is a gateway with somewhere else to go: `first` answers until // it is stood down, then `second` does. type standbyGateway struct { first gateway.Gateway second gateway.Gateway } func (s *standbyGateway) Complete(ctx context.Context, req gateway.Request) (*gateway.Response, error) { return s.first.Complete(ctx, req) } func (s *standbyGateway) Standby() (gateway.Gateway, bool) { if s.second == nil { return nil, false } return s.second, true } func rateLimited() error { return &gateway.Error{Code: gateway.CodeRateLimited, Status: 429, Message: "TPM limit 8000"} } // The production case: a rate limit on a run that had already called a tool, // which in-place failover will not move. func TestARateLimitedRunIsRetriedOnTheStandbyProvider(t *testing.T) { busy := &fakeGateway{err: rateLimited()} spare := &fakeGateway{text: "15 open roles"} exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil) res, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("how many open positions?")) if err != nil { t.Fatalf("the standby should have answered: %v", err) } if res.Termination != TerminationCompleted { t.Fatalf("Termination = %q, want Completed", res.Termination) } if res.Output != "15 open roles" { t.Errorf("Output = %q, want the standby's answer", res.Output) } if spare.calls == 0 { t.Error("the standby provider was never asked") } } // A run carrying a confirmation has performed an approved write. Re-running it // re-runs its tools, and a write twice is two shifts assigned. func TestAConfirmedRunIsNeverRestarted(t *testing.T) { busy := &fakeGateway{err: rateLimited()} spare := &fakeGateway{text: "should never be reached"} exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil) in := testInput("assign Maria to the Friday shift") in.Confirmation = "a-token-a-person-approved" res, _ := exec.ExecuteAgent(context.Background(), testAgent(), in) if res.Termination == TerminationCompleted { t.Error("a confirmed run was restarted; an approved write could run twice") } if spare.calls != 0 { t.Errorf("the standby was asked %d times; a confirmed run must not be replayed", spare.calls) } } // A credential or a model id fails the same way everywhere. Asking twice only // doubles the bill and hides the fault. func TestATerminalGatewayErrorIsNotRetriedElsewhere(t *testing.T) { busy := &fakeGateway{err: &gateway.Error{ Code: gateway.CodeUnauthorized, Status: 401, Message: "bad key", }} spare := &fakeGateway{text: "should never be reached"} exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil) res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything")) if res.Termination == TerminationCompleted { t.Error("a terminal error was retried on another provider") } if spare.calls != 0 { t.Errorf("the standby was asked %d times on a 401", spare.calls) } } // With one provider there is no standby, and nothing about the single-provider // path may change. func TestWithNoStandbyTheFailureStands(t *testing.T) { busy := &fakeGateway{err: rateLimited()} exec := NewModelExecutor(busy, &MemorySink{}, nil) res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything")) if res.Termination != TerminationGatewayFailure { t.Errorf("Termination = %q, want GatewayFailure", res.Termination) } if busy.calls == 0 { t.Error("the only provider was never asked") } } // Once, not until the providers run out. func TestTheStandbyIsAskedOnlyOnce(t *testing.T) { busy := &fakeGateway{err: rateLimited()} alsoBusy := &fakeGateway{err: rateLimited()} exec := NewModelExecutor(&standbyGateway{first: busy, second: alsoBusy}, &MemorySink{}, nil) res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything")) if res.Termination != TerminationGatewayFailure { t.Errorf("Termination = %q, want GatewayFailure", res.Termination) } if alsoBusy.calls == 0 { t.Error("the standby was never tried") } }