Add GatewayFailure: the provider not answering is not a tool failing

terminationFor sent every gateway error that was not Refused or Timeout
to ToolFailure, because the enum had nowhere else to put it. On
2026-09-22 that was 131 of 318 production runs, and not one of them was
a tool failing: 56 were retired model ids, 25 an exhausted Anthropic
balance, 45 Groq's free-tier rate limit -- the only one still happening.
An operator reading the termination column saw "a tool is broken" for
two weeks while the actual answer was "we are not paying for capacity".

GatewayFailure is the seventh termination. Rate limited, request
rejected, credential refused and unreachable land there; Refused and
Deadline keep their own reasons; a non-gateway error is still the tool
layer's. A delegation whose subagent died at the gateway now carries
that reason up to the parent instead of reading as a tool call that
failed.

Migration 000016 widens the CHECK that 000006 chose precisely so this
would be a migration rather than an ALTER TYPE. Its down folds any
GatewayFailure rows back to ToolFailure BEFORE narrowing the constraint,
which is the order that works; verified up, down and up again on a
scratch database. Existing rows are left as they are -- the trajectory
entries still carry the gateway.* code for anyone reclassifying history.

The surface wording is the one termination where "try again" is honest
advice, since the dominant cause clears within a minute.

Full suite run against a real database, including the tests that skip
without one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PJvibeSc1JYXjatankqM1g

go-api/internal/domain/definitions_schema_test.go

@@ -733,6 +733,9 @@ func TestMigrationPairsAreComplete(t *testing.T) {
 		// Phase 5: shared rate limit counters, so a limit means the same thing
 		// behind one instance and behind ten.
 		"000015_rate_limits.up.sql",
+		// A seventh termination reason. The CHECK in 000006 was chosen so
+		// this would be a migration rather than an ALTER TYPE; this is it.
+		"000016_gateway_failure_termination.up.sql",
 	}
 	if len(ups) != len(want) {
 		t.Fatalf("%d migrations, want %d — update this list deliberately", len(ups), len(want))

go-api/internal/httpserver/runs.go

@@ -205,7 +205,7 @@ func buildRunResponse(res *runtime.ExecutionResult) runResponse {
 // reached its budget before finishing" is accurate and means nothing to
 // somebody who has never heard of a token budget.
 //
-// Every one of the six is spelled out. A default that said "something went
+// Every one of the seven is spelled out. A default that said "something went
 // wrong" would be the place where a Refused run and a ToolFailure became
 // indistinguishable to the person best placed to tell us which it was.
 func terminationMessage(t runtime.Termination) string {
@@ -224,6 +224,12 @@ func terminationMessage(t runtime.Termination) string {
 			"Nothing was changed."
 	case runtime.TerminationRefused:
 		return "The agent declined to answer this one."
+	case runtime.TerminationGatewayFailure:
+		// The one termination where "try again" is honest advice: the
+		// dominant cause is a rate limit that clears within a minute, and
+		// nothing about the question itself was the problem.
+		return "The model behind this agent did not answer — usually it is busy. " +
+			"Wait a minute and ask again. Nothing was changed."
 	default:
 		return "The agent did not finish."
 	}

go-api/internal/runtime/budget.go

@@ -22,13 +22,26 @@ const (
 	TerminationConfirmationPending Termination = "ConfirmationPending"
 	TerminationToolFailure         Termination = "ToolFailure"
 	TerminationRefused             Termination = "Refused"
+
+	// GatewayFailure is the model provider failing to answer at all: rate
+	// limited, rejected the request, refused the credential, or unreachable.
+	// Added 2026-09-22 because until then every one of those was recorded as
+	// ToolFailure, and 131 of 318 production runs read as "a tool is broken"
+	// when no tool had failed — 45 of them were Groq's free-tier rate limit,
+	// which is a capacity decision, not a bug. The two are different questions
+	// to an operator ("what did we break" versus "what are we not paying
+	// for"), and an enum that could not tell them apart hid the answer for
+	// two weeks. Refused and Deadline keep their own reasons; this is the
+	// rest of the gateway's vocabulary.
+	TerminationGatewayFailure Termination = "GatewayFailure"
 )
 
-// Valid reports whether t is one of the six.
+// Valid reports whether t is one of the seven.
 func (t Termination) Valid() bool {
 	switch t {
 	case TerminationCompleted, TerminationBudgetExceeded, TerminationDeadline,
-		TerminationConfirmationPending, TerminationToolFailure, TerminationRefused:
+		TerminationConfirmationPending, TerminationToolFailure, TerminationRefused,
+		TerminationGatewayFailure:
 		return true
 	}
 	return false

go-api/internal/runtime/delegate.go

@@ -241,12 +241,16 @@ func (m *ModelExecutor) delegate(
 	rec.AddChildren(collected.Runs)
 
 	if res == nil {
-		msg := "the subagent returned nothing"
+		// The parent sees a delegation as a tool call, but the REASON it
+		// failed is still worth carrying: a subagent the provider rate limited
+		// should read as GatewayFailure in the parent's trajectory too, or the
+		// parent's operator goes looking for a tool that never broke.
+		msg, term := "the subagent returned nothing", TerminationToolFailure
 		if err != nil {
-			msg = err.Error()
+			msg, term = err.Error(), terminationFor(err)
 		}
 		return delegationAnswer{
-			Agent: sub.ID, Termination: string(TerminationToolFailure), Error: msg,
+			Agent: sub.ID, Termination: string(term), Error: msg,
 		}, nil
 	}
 

go-api/internal/runtime/loop.go

@@ -555,6 +555,12 @@ func (m *ModelExecutor) runTools(
 // a Refused run is one that must not be retried. Flattening them into a single
 // failure reason would make every one of those distinctions unanswerable from
 // the trajectory.
+//
+// Any other gateway error is GatewayFailure, not ToolFailure. The provider
+// being rate limited, rejecting the request or refusing the key is not a tool
+// failing, and calling it one sent two weeks of operators looking for a broken
+// tool that did not exist. A non-gateway error — a tool the model invented, a
+// result that could not be encoded — is still the tool layer's.
 func terminationFor(err error) Termination {
 	var gwErr *gateway.Error
 	if !errors.As(err, &gwErr) {
@@ -566,7 +572,7 @@ func terminationFor(err error) Termination {
 	case gateway.CodeTimeout:
 		return TerminationDeadline
 	default:
-		return TerminationToolFailure
+		return TerminationGatewayFailure
 	}
 }
 
@@ -657,6 +663,8 @@ func terminationMessage(t Termination) string {
 		return "the run is waiting on a confirmation"
 	case TerminationToolFailure:
 		return "the run failed"
+	case TerminationGatewayFailure:
+		return "the model provider did not answer"
 	default:
 		return string(t)
 	}

go-api/internal/runtime/loop_test.go

@@ -277,6 +277,7 @@ func TestTerminationValidRejectsInvented(t *testing.T) {
 	for _, ok := range []Termination{
 		TerminationCompleted, TerminationBudgetExceeded, TerminationDeadline,
 		TerminationConfirmationPending, TerminationToolFailure, TerminationRefused,
+		TerminationGatewayFailure,
 	} {
 		if !ok.Valid() {
 			t.Errorf("%q should be a valid termination", ok)
@@ -1075,3 +1076,31 @@ func TestTheTrajectoryRecordsWhichChunksGroundedTheAnswer(t *testing.T) {
 		t.Error("nothing in the trajectory says a retrieval happened")
 	}
 }
+
+// The whole reason GatewayFailure exists: a provider that will not answer is
+// not a tool that broke, and for two weeks the trajectory said it was.
+func TestTerminationForSeparatesGatewayFromTool(t *testing.T) {
+	gw := func(code string) error { return &gateway.Error{Code: code, Message: "x"} }
+	cases := []struct {
+		name string
+		err  error
+		want Termination
+	}{
+		{"rate limited is the gateway's", gw(gateway.CodeRateLimited), TerminationGatewayFailure},
+		{"invalid request is the gateway's", gw(gateway.CodeInvalidRequest), TerminationGatewayFailure},
+		{"bad credential is the gateway's", gw(gateway.CodeUnauthorized), TerminationGatewayFailure},
+		{"upstream is the gateway's", gw(gateway.CodeUpstream), TerminationGatewayFailure},
+		{"not configured is the gateway's", gw(gateway.CodeNotConfigured), TerminationGatewayFailure},
+		{"refused keeps its own reason", gw(gateway.CodeRefused), TerminationRefused},
+		{"timeout keeps its own reason", gw(gateway.CodeTimeout), TerminationDeadline},
+		{"a non-gateway error is still the tool layer's", errors.New("tool exploded"), TerminationToolFailure},
+		{"a wrapped gateway error is still found", fmt.Errorf("delegating: %w", gw(gateway.CodeRateLimited)), TerminationGatewayFailure},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := terminationFor(tc.err); got != tc.want {
+				t.Errorf("terminationFor(%v) = %s, want %s", tc.err, got, tc.want)
+			}
+		})
+	}
+}

go-api/internal/runtime/trajectory.go

@@ -285,7 +285,7 @@ func (r *Recorder) SetModel(model string) {
 
 // Finish closes the trajectory with its termination reason and returns it.
 //
-// A reason that is not one of the six is recorded as ToolFailure rather than
+// A reason that is not one of the seven is recorded as ToolFailure rather than
 // stored as-is: an unrecognised termination is a bug in the loop, and writing
 // it verbatim would let that bug propagate into every eval and dashboard that
 // groups by this column.