krow_backend/go-api/internal/runtime/standby_test.go

package runtime

import (
	"context"
	"testing"

	"github.com/krow/krow-backend/go-api/internal/gateway"
)

// standbyGateway is a gateway with somewhere else to go: `first` answers until
// it is stood down, then `second` does.
type standbyGateway struct {
	first  gateway.Gateway
	second gateway.Gateway
}

func (s *standbyGateway) Complete(ctx context.Context, req gateway.Request) (*gateway.Response, error) {
	return s.first.Complete(ctx, req)
}

func (s *standbyGateway) Standby() (gateway.Gateway, bool) {
	if s.second == nil {
		return nil, false
	}
	return s.second, true
}

func rateLimited() error {
	return &gateway.Error{Code: gateway.CodeRateLimited, Status: 429, Message: "TPM limit 8000"}
}

// The production case: a rate limit on a run that had already called a tool,
// which in-place failover will not move.
func TestARateLimitedRunIsRetriedOnTheStandbyProvider(t *testing.T) {
	busy := &fakeGateway{err: rateLimited()}
	spare := &fakeGateway{text: "15 open roles"}
	exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil)

	res, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("how many open positions?"))
	if err != nil {
		t.Fatalf("the standby should have answered: %v", err)
	}
	if res.Termination != TerminationCompleted {
		t.Fatalf("Termination = %q, want Completed", res.Termination)
	}
	if res.Output != "15 open roles" {
		t.Errorf("Output = %q, want the standby's answer", res.Output)
	}
	if spare.calls == 0 {
		t.Error("the standby provider was never asked")
	}
}

// A run carrying a confirmation has performed an approved write. Re-running it
// re-runs its tools, and a write twice is two shifts assigned.
func TestAConfirmedRunIsNeverRestarted(t *testing.T) {
	busy := &fakeGateway{err: rateLimited()}
	spare := &fakeGateway{text: "should never be reached"}
	exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil)

	in := testInput("assign Maria to the Friday shift")
	in.Confirmation = "a-token-a-person-approved"

	res, _ := exec.ExecuteAgent(context.Background(), testAgent(), in)
	if res.Termination == TerminationCompleted {
		t.Error("a confirmed run was restarted; an approved write could run twice")
	}
	if spare.calls != 0 {
		t.Errorf("the standby was asked %d times; a confirmed run must not be replayed", spare.calls)
	}
}

// A credential or a model id fails the same way everywhere. Asking twice only
// doubles the bill and hides the fault.
func TestATerminalGatewayErrorIsNotRetriedElsewhere(t *testing.T) {
	busy := &fakeGateway{err: &gateway.Error{
		Code: gateway.CodeUnauthorized, Status: 401, Message: "bad key",
	}}
	spare := &fakeGateway{text: "should never be reached"}
	exec := NewModelExecutor(&standbyGateway{first: busy, second: spare}, &MemorySink{}, nil)

	res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything"))
	if res.Termination == TerminationCompleted {
		t.Error("a terminal error was retried on another provider")
	}
	if spare.calls != 0 {
		t.Errorf("the standby was asked %d times on a 401", spare.calls)
	}
}

// With one provider there is no standby, and nothing about the single-provider
// path may change.
func TestWithNoStandbyTheFailureStands(t *testing.T) {
	busy := &fakeGateway{err: rateLimited()}
	exec := NewModelExecutor(busy, &MemorySink{}, nil)

	res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything"))
	if res.Termination != TerminationGatewayFailure {
		t.Errorf("Termination = %q, want GatewayFailure", res.Termination)
	}
	if busy.calls == 0 {
		t.Error("the only provider was never asked")
	}
}

// Once, not until the providers run out.
func TestTheStandbyIsAskedOnlyOnce(t *testing.T) {
	busy := &fakeGateway{err: rateLimited()}
	alsoBusy := &fakeGateway{err: rateLimited()}
	exec := NewModelExecutor(&standbyGateway{first: busy, second: alsoBusy}, &MemorySink{}, nil)

	res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("anything"))
	if res.Termination != TerminationGatewayFailure {
		t.Errorf("Termination = %q, want GatewayFailure", res.Termination)
	}
	if alsoBusy.calls == 0 {
		t.Error("the standby was never tried")
	}
}