krow_backend/go-api/internal/gateway/retry.go

package gateway

import (
	"context"
	"errors"
	"time"
)

// MaxAttempts is how many times a transient failure is retried.
//
// Three total, not three retries. Past that the problem is not transient and a
// fourth call is just spending money on the same answer.
const MaxAttempts = 3

// retryBackoff is the pause before each retry.
//
// Short, and deliberately so: this sits inside a run that already has a
// wall-clock deadline, and a backoff long enough to be polite to the API is
// long enough to spend the caller's whole budget waiting. A run that cannot
// afford the wait dies on its deadline instead, which is the correct failure.
var retryBackoff = []time.Duration{400 * time.Millisecond, 1200 * time.Millisecond}

// withRetry runs one attempt until it succeeds, fails terminally, or runs out
// of attempts.
//
// THE RETRY IS NOT DEFENSIVE POLISH. Error.Retryable() has existed since this
// package was written and had ZERO callers — the classification was built and
// never used, so a 529 "overloaded" killed a run that would have succeeded four
// hundred milliseconds later. Found by a real overload during live testing,
// where it presented as "the agent could not finish" with nothing to act on.
//
// Only genuinely transient failures qualify: rate limits, timeouts, and 5xx.
// A 400 is a malformed request and will be malformed again; a 401 is a bad
// credential and retrying it three times just gets refused three times.
//
// The run's context governs. A retry that would outlive the caller's deadline
// does not happen — the deadline belongs to the run, not to this function, and
// waiting past it would turn a bounded run into an unbounded one.
//
// THIS FILE EXISTS BECAUSE THE POLICY OUTLIVED ITS FIRST PROVIDER. It was
// written inside the Anthropic implementation and used by both, so deleting
// that implementation would have deleted the retry policy of the one that
// remained — silently, because nothing about `openai.go` mentions it. The
// policy is a property of this platform's runs, not of any vendor's API, so it
// now lives somewhere no provider can take with it when it goes.
func withRetry(ctx context.Context, once func() (*Response, error)) (*Response, error) {
	var last error
	for attempt := 0; attempt < MaxAttempts; attempt++ {
		if attempt > 0 {
			pause := retryBackoff[min(attempt-1, len(retryBackoff)-1)]
			select {
			case <-time.After(pause):
			case <-ctx.Done():
				// Out of time. The ORIGINAL failure is returned rather than the
				// context error: "the model was overloaded" is what an operator
				// needs to see, and "context deadline exceeded" would hide it.
				return nil, last
			}
		}

		resp, err := once()
		if err == nil {
			return resp, nil
		}
		last = err

		var gwErr *Error
		if !errors.As(err, &gwErr) || !gwErr.Retryable() {
			return resp, err
		}
	}
	return nil, last
}