diff --git a/go-api/internal/config/config.go b/go-api/internal/config/config.go index 8950fe2..0951d4c 100644 --- a/go-api/internal/config/config.go +++ b/go-api/internal/config/config.go @@ -177,6 +177,17 @@ type ModelConfig struct { // OpenAI-compatible wire. Off by default: reasoning models accept the // field and most others reject the entire request rather than ignoring it. ReasoningEffort bool + + // Fallbacks are further providers to ask when the one above cannot answer, + // in order. Empty is the ordinary case and carries no wrapper at all. + // + // A FREE TIER'S CEILING IS PER PROVIDER, so a second key is a second + // budget — which is the only thing that helps when a single run costs more + // tokens than a provider allows in a minute. Each entry is a whole + // ModelConfig because a fallback is a different service with its own + // credential, its own base URL and its own model ids; sharing any of those + // is what makes "the same request, somewhere else" impossible. + Fallbacks []ModelConfig } // SeedConfig locates the demo fixture. The file is generated from the frontend @@ -418,6 +429,7 @@ func Load() (*Config, error) { // unstreamed call, not the run's budget. MaxOutputTokens: intDefault("MODEL_MAX_OUTPUT_TOKENS", 16000), ReasoningEffort: boolDefault("MODEL_REASONING_EFFORT", false), + Fallbacks: loadFallbacks(), }, DB: DBConfig{ Host: required("DATABASE_HOST"), @@ -943,3 +955,42 @@ func (c *Config) validateOAuth() error { } return nil } + +// loadFallbacks reads MODEL_FALLBACK__* for n = 1, 2, 3… +// +// Numbered rather than comma-separated because each provider needs five fields, +// and a delimiter-packed string holding five fields times three providers is a +// parser nobody can read and an operator cannot edit under pressure: +// +// MODEL_FALLBACK_1_BASE_URL=https://api.cerebras.ai/v1 +// MODEL_FALLBACK_1_API_KEY=… +// MODEL_FALLBACK_1_BALANCED= +// +// Stops at the first gap, so a deployment cannot half-configure a third +// provider by deleting the second and have the third silently promoted. +// +// A fallback with no BASE_URL or no API_KEY is not a fallback, so both are +// required and the entry is skipped without one. The model ids fall back to the +// PRIMARY's — wrong for a different vendor, which is why each should be set, +// but an unset id produces a visible invalid_request rather than silence. +func loadFallbacks() []ModelConfig { + var out []ModelConfig + for n := 1; ; n++ { + prefix := fmt.Sprintf("MODEL_FALLBACK_%d_", n) + base := strings.TrimSpace(os.Getenv(prefix + "BASE_URL")) + key := strings.TrimSpace(os.Getenv(prefix + "API_KEY")) + if base == "" || key == "" { + return out + } + out = append(out, ModelConfig{ + Provider: strings.ToLower(strings.TrimSpace(os.Getenv(prefix + "PROVIDER"))), + APIKey: key, + BaseURL: base, + Fast: strings.TrimSpace(os.Getenv(prefix + "FAST")), + Balanced: strings.TrimSpace(os.Getenv(prefix + "BALANCED")), + Deep: strings.TrimSpace(os.Getenv(prefix + "DEEP")), + MaxOutputTokens: intDefault(prefix+"MAX_OUTPUT_TOKENS", 16000), + ReasoningEffort: boolDefault(prefix+"REASONING_EFFORT", false), + }) + } +} diff --git a/go-api/internal/gateway/failover.go b/go-api/internal/gateway/failover.go new file mode 100644 index 0000000..a70bd93 --- /dev/null +++ b/go-api/internal/gateway/failover.go @@ -0,0 +1,124 @@ +package gateway + +// Failover: a second and third provider, for when the first one says no. +// +// THE PROBLEM THIS SOLVES IS A CEILING, NOT A BUG. A free tier is a token +// budget per minute, and one agent run can exceed a whole minute's worth by +// itself — a three-call run measured 12,123 tokens against a ceiling of 8,000. +// withRetry already fires three times, and on a rate limit all three are +// refused, because waiting 1.6 seconds does not buy back a minute's budget. The +// run then ends GatewayFailure and a person reads "the model did not answer". +// +// Retrying harder cannot fix that. Asking somebody else can: the ceilings are +// per provider, so a second key is a second budget. Groq, Cerebras, Gemini, +// Mistral and OpenRouter all serve the same chat-completions shape, which is +// the whole reason this is a list of Configs and not a second implementation. +// +// WHAT IT DOES NOT DO, stated because the gap is where the next bug lives: +// it does not make a run cheaper, it does not raise any one provider's ceiling, +// and it does not help when every configured provider is exhausted at once. It +// converts "one busy provider" from an outage into a slower answer. + +import ( + "context" + "errors" +) + +// failover tries each provider in order until one answers. +type failover struct { + providers []Gateway +} + +// NewFailover builds a gateway that falls back through `rest` when `primary` +// cannot answer. With no fallbacks it returns the primary unchanged, so a +// single-provider deployment carries no wrapper and behaves exactly as before. +func NewFailover(primary Gateway, rest ...Gateway) Gateway { + if len(rest) == 0 { + return primary + } + return &failover{providers: append([]Gateway{primary}, rest...)} +} + +func (f *failover) Complete(ctx context.Context, req Request) (*Response, error) { + var last error + for i, p := range f.providers { + if i > 0 && !canFailOver(req, last) { + break + } + resp, err := p.Complete(ctx, req) + if err == nil { + return resp, nil + } + last = err + // The caller's deadline governs. A deployment with four providers must + // not spend four timeouts' worth of a person's patience discovering + // that none of them is available. + if ctx.Err() != nil { + break + } + } + return nil, last +} + +// Stream falls over only before the first fragment has been delivered. +// +// After a delta reaches the client, the answer has begun in the reader's own +// window. Starting a second provider would continue that sentence in a +// different voice from a different model, or repeat its opening — so once text +// is out, the error is the answer. +func (f *failover) Stream(ctx context.Context, req Request, onDelta func(string)) (*Response, error) { + var last error + for i, p := range f.providers { + if i > 0 && !canFailOver(req, last) { + break + } + var delivered bool + wrapped := func(s string) { + delivered = true + onDelta(s) + } + resp, err := StreamComplete(ctx, p, req, wrapped) + if err == nil { + return resp, nil + } + last = err + if delivered || ctx.Err() != nil { + break + } + } + return nil, last +} + +// canFailOver decides whether asking a DIFFERENT provider is sound. +// +// Two conditions, and both are necessary. +// +// 1. THE FAILURE MUST BE TRANSIENT. Error.Retryable() already draws that line +// for retries and it is the same line here: a rate limit or a 5xx is the +// provider being unable, and somebody else may be able. A 400 is a +// malformed request and will be malformed for everyone; a 401 is this +// deployment's own credential. Failing over on those turns one provider's +// configuration error into every provider's, and buries the fault. +// +// 2. THE CONVERSATION MUST NOT BE BOUND TO ITS PROVIDER. ToolCall.Extra +// carries provider metadata echoed back verbatim — Gemini 3's thought +// signature is the known case, and it REJECTS a follow-up that does not +// return it. That metadata is meaningless to a different provider and its +// absence is fatal to the one that issued it, so a conversation that +// already carries any is pinned to whoever produced it. In practice this +// means failover is available on the first model call of a run, which is +// where a rate limit usually lands anyway. +func canFailOver(req Request, err error) bool { + var gwErr *Error + if !errors.As(err, &gwErr) || !gwErr.Retryable() { + return false + } + for _, m := range req.Messages { + for _, tc := range m.ToolCalls { + if len(tc.Extra) > 0 { + return false + } + } + } + return true +} diff --git a/go-api/internal/gateway/failover_test.go b/go-api/internal/gateway/failover_test.go new file mode 100644 index 0000000..307cf21 --- /dev/null +++ b/go-api/internal/gateway/failover_test.go @@ -0,0 +1,121 @@ +package gateway + +import ( + "context" + "encoding/json" + "errors" + "testing" +) + +type scripted struct { + name string + err error + calls *[]string +} + +func (s *scripted) Complete(ctx context.Context, req Request) (*Response, error) { + *s.calls = append(*s.calls, s.name) + if s.err != nil { + return nil, s.err + } + return &Response{Text: "answered by " + s.name, Model: s.name}, nil +} + +func gwErr(code string, status int) error { + return &Error{Code: code, Status: status, Message: code} +} + +func TestFailoverAsksTheNextProviderOnARateLimit(t *testing.T) { + var calls []string + f := NewFailover( + &scripted{name: "groq", err: gwErr(CodeRateLimited, 429), calls: &calls}, + &scripted{name: "cerebras", calls: &calls}, + ) + resp, err := f.Complete(context.Background(), Request{ + Messages: []Message{{Role: RoleUser, Text: "hello"}}, + }) + if err != nil { + t.Fatalf("want an answer from the fallback, got %v", err) + } + if resp.Model != "cerebras" { + t.Errorf("answered by %q, want cerebras", resp.Model) + } + if len(calls) != 2 || calls[0] != "groq" { + t.Errorf("provider order was %v, want groq then cerebras", calls) + } +} + +func TestFailoverDoesNotMaskABadCredential(t *testing.T) { + // A 401 is THIS deployment's own configuration and fails identically + // everywhere. Trying three providers would turn one visible fault into + // three invisible ones and leave the operator nothing to fix. + var calls []string + f := NewFailover( + &scripted{name: "groq", err: gwErr(CodeUnauthorized, 401), calls: &calls}, + &scripted{name: "cerebras", calls: &calls}, + ) + _, err := f.Complete(context.Background(), Request{ + Messages: []Message{{Role: RoleUser, Text: "hello"}}, + }) + var e *Error + if !errors.As(err, &e) || e.Code != CodeUnauthorized { + t.Fatalf("want the unauthorized error raised, got %v", err) + } + if len(calls) != 1 { + t.Errorf("called %v; a terminal error must not reach the fallback", calls) + } +} + +func TestFailoverWillNotMoveAConversationBoundToItsProvider(t *testing.T) { + // ToolCall.Extra is provider metadata echoed back verbatim — Gemini's + // thought signature. Replaying it at a different vendor sends it a field it + // cannot read; dropping it kills the vendor that issued it. Either way the + // conversation belongs to whoever started it. + var calls []string + f := NewFailover( + &scripted{name: "gemini", err: gwErr(CodeRateLimited, 429), calls: &calls}, + &scripted{name: "groq", calls: &calls}, + ) + _, err := f.Complete(context.Background(), Request{ + Messages: []Message{ + {Role: RoleUser, Text: "how many open positions?"}, + {Role: RoleAssistant, ToolCalls: []ToolCall{{ + ID: "c1", Name: "open_positions", + Input: json.RawMessage(`{}`), + Extra: json.RawMessage(`{"thought_signature":"abc"}`), + }}}, + }, + }) + if err == nil { + t.Fatal("want the rate limit raised, not a second provider's answer") + } + if len(calls) != 1 { + t.Errorf("called %v; a pinned conversation must not fail over", calls) + } +} + +func TestFailoverWithNoFallbacksIsTheProviderItself(t *testing.T) { + var calls []string + p := &scripted{name: "groq", calls: &calls} + if got := NewFailover(p); got != Gateway(p) { + t.Error("with no fallbacks the primary must be returned unwrapped") + } +} + +func TestFailoverExhaustedReturnsTheLastError(t *testing.T) { + var calls []string + f := NewFailover( + &scripted{name: "a", err: gwErr(CodeRateLimited, 429), calls: &calls}, + &scripted{name: "b", err: gwErr(CodeUpstream, 503), calls: &calls}, + ) + _, err := f.Complete(context.Background(), Request{ + Messages: []Message{{Role: RoleUser, Text: "hi"}}, + }) + var e *Error + if !errors.As(err, &e) || e.Status != 503 { + t.Fatalf("want the LAST provider's error, got %v", err) + } + if len(calls) != 2 { + t.Errorf("called %v, want both tried", calls) + } +} diff --git a/go-api/internal/gateway/routing.go b/go-api/internal/gateway/routing.go index 8cd3a28..749a903 100644 --- a/go-api/internal/gateway/routing.go +++ b/go-api/internal/gateway/routing.go @@ -72,6 +72,10 @@ type Config struct { Balanced Routing Deep Routing + // Fallbacks are further providers to try, in order, when this one cannot + // answer. See failover.go for when that is sound and when it is not. + Fallbacks []Config + // MaxOutputTokens applies when a request does not set its own. MaxOutputTokens int64 @@ -104,7 +108,26 @@ type Config struct { // correctness matters more than cost, which is a judgement an operator makes // about a deployment, not one an agent author makes about a page. func FromConfig(c config.ModelConfig) Config { + var fallbacks []Config + for _, f := range c.Fallbacks { + // Model ids default to the primary's. Usually wrong for a different + // vendor and deliberately not silently corrected: an id the endpoint + // does not serve answers invalid_request, which is a visible fault an + // operator can fix, where a guessed substitution would be an invisible + // one nobody asked for. + if f.Fast == "" { + f.Fast = c.Fast + } + if f.Balanced == "" { + f.Balanced = c.Balanced + } + if f.Deep == "" { + f.Deep = c.Deep + } + fallbacks = append(fallbacks, FromConfig(f)) + } return Config{ + Fallbacks: fallbacks, Provider: c.Provider, APIKey: c.APIKey, BaseURL: c.BaseURL, @@ -123,7 +146,15 @@ func FromConfig(c config.ModelConfig) Config { // `gateway.New(gateway.FromConfig(...))` and should not learn a concrete type: // the next provider is a change here and nowhere else. func New(cfg Config) Gateway { - return NewOpenAI(cfg) + primary := NewOpenAI(cfg) + if len(cfg.Fallbacks) == 0 { + return primary + } + rest := make([]Gateway, 0, len(cfg.Fallbacks)) + for _, f := range cfg.Fallbacks { + rest = append(rest, NewOpenAI(f)) + } + return NewFailover(primary, rest...) } // routingFor resolves a tier against a table.