package tools_test import ( "context" "encoding/json" "sync" "sync/atomic" "testing" "time" "github.com/krow/krow-backend/go-api/internal/authctx" "github.com/krow/krow-backend/go-api/internal/tools" ) // These tests are about one question: what, exactly, does a person's approval // authorise? // // The answer I4 is usually given is "the write" — and if a confirmation were a // boolean, that answer would be wrong in a way nobody notices until it matters. // A yes given to "assign Maya to Friday" would equally authorise "assign Dan to // Saturday", because a boolean cannot tell them apart. Everything below exists // to prove the token can. /* ── A harness that records what actually ran ───────────────────────────── */ // spyWrite is a write tool that counts its own executions. // // The assertion that matters in most of these tests is not what Dispatch // returned but whether the handler ran at all. A refusal that still wrote is a // bug that a result-shaped assertion would sail straight past. type spyWrite struct { runs atomic.Int64 asked atomic.Int64 denied bool panics bool expiry time.Time } func (s *spyWrite) tool() tools.Tool { return tools.Tool{ Name: "assign_worker", Description: "Assign somebody to something.", InputSchema: map[string]any{"type": "object"}, Effect: tools.EffectWrite, Confirm: func(_ context.Context, tc tools.Context, in json.RawMessage) (*tools.Confirmation, *tools.Result) { s.asked.Add(1) if s.panics { panic("a renderer that blew up") } if s.denied { d := tools.Denied() return nil, &d } return &tools.Confirmation{ Title: "Assign somebody", Summary: "Somebody will be assigned to something.", Details: []tools.Detail{{Label: "Arguments", Value: string(in)}}, ExpiresAt: s.expiry, }, nil }, Handler: func(context.Context, tools.Context, json.RawMessage) tools.Result { s.runs.Add(1) return tools.OK(map[string]any{"written": true}) }, } } func caller(user, org string) tools.Context { return tools.Context{ Principal: authctx.Identity{UserID: user, OrgID: org, Role: "admin"}, RunID: "run_one", } } // ask dispatches a write with no token and returns the confirmation it raised. func ask(t *testing.T, reg *tools.Registry, tc tools.Context, args string) *tools.Confirmation { t.Helper() res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)) if res.Confirmation == nil { t.Fatalf("expected a confirmation, got %+v", res) } return res.Confirmation } // notApproved asserts that a call was not authorised: nothing was written, and // the caller was asked afresh rather than let through. // // "Asked afresh" is the shape of every refusal here, and it is deliberate. A // token that does not authorise THIS call — wrong arguments, wrong caller, // expired, already spent, invented — all mean the same thing, which is that // nobody has approved what is about to happen. The honest response to that is // to describe it and ask, not to hand the model an error it cannot act on. func notApproved(t *testing.T, res tools.Result, spy *spyWrite, staleToken string) { t.Helper() if spy.runs.Load() != 0 { t.Fatalf("the write ran %d times without an approval for it", spy.runs.Load()) } if res.Data != nil { t.Fatal("an unapproved write produced a result") } if res.Confirmation == nil { if res.Error == nil { t.Fatal("an unapproved write was neither refused nor re-described") } return } if staleToken != "" && res.Confirmation.Token == staleToken { t.Fatal("the stale token was handed straight back as if it were a fresh approval") } } /* ── The gate ───────────────────────────────────────────────────────────── */ func TestAWriteIsDescribedBeforeItIsDone(t *testing.T) { spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) c := ask(t, reg, caller("u1", "org1"), `{"worker":"maya"}`) if spy.runs.Load() != 0 { t.Fatal("the handler ran before anybody approved anything") } if c.Token == "" { t.Error("a confirmation with no token can never be answered") } if c.Tool != "assign_worker" { t.Errorf("confirmation names tool %q, want assign_worker", c.Tool) } if c.Title == "" || c.Summary == "" { t.Error("a person cannot approve a confirmation with nothing written on it") } if c.ExpiresAt.IsZero() { t.Error("a confirmation that never expires is a standing authorisation") } } func TestAnApprovalAuthorisesOnlyTheCallItDescribed(t *testing.T) { // The whole reason a confirmation is a binding rather than a flag. // // A person is shown "assign maya" and approves it. The model then calls the // same tool for a different worker, carrying the same token. If that // succeeded, the approval a person gave to one write would have silently // become approval of another — which is not a permissions bug the user // could ever detect, because the dialog they saw was accurate. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") approved := ask(t, reg, tc, `{"worker":"maya","shift":"friday"}`) tc.Confirmation = approved.Token res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(`{"worker":"dan","shift":"friday"}`)) // Not merely refused: the substituted call is DESCRIBED, so the person is // asked about the write that is actually being proposed. notApproved(t, res, spy, approved.Token) if res.Confirmation == nil { t.Fatal("the substituted call should have raised its own confirmation") } } func TestAnApprovalRunsTheCallItDescribed(t *testing.T) { spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") args := `{"worker":"maya","shift":"friday"}` approved := ask(t, reg, tc, args) tc.Confirmation = approved.Token res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)) if res.Error != nil { t.Fatalf("an approved write should run: %+v", res.Error) } if spy.runs.Load() != 1 { t.Fatalf("handler ran %d times, want exactly 1", spy.runs.Load()) } } func TestReorderedArgumentsAreStillTheSameCall(t *testing.T) { // The other direction, and the reason inputs are canonicalised rather than // hashed verbatim. A model that emits its arguments in a different order on // the resumed turn has not changed what it is asking for, and refusing it // would make approvals fail at random. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") approved := ask(t, reg, tc, `{"worker":"maya","shift":"friday"}`) tc.Confirmation = approved.Token res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(`{ "shift" : "friday", "worker" : "maya" }`)) if res.Error != nil { t.Fatalf("reordered and re-spaced arguments are the same call: %+v", res.Error) } if spy.runs.Load() != 1 { t.Fatal("the same call, written differently, should have run") } } func TestAnApprovalIsSpentOnce(t *testing.T) { spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") args := `{"worker":"maya"}` approved := ask(t, reg, tc, args) tc.Confirmation = approved.Token reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)) res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)) if spy.runs.Load() != 1 { t.Fatalf("one approval bought %d writes", spy.runs.Load()) } if res.Data != nil { t.Fatal("a spent token authorised a second write") } if res.Confirmation == nil { t.Fatal("the second call should have raised its own confirmation") } if res.Confirmation.Token == approved.Token { t.Fatal("a spent token was reissued") } } func TestConcurrentAttemptsSpendAnApprovalOnce(t *testing.T) { // Single-use has to survive two goroutines arriving at the same instant, or // it is only single-use in the happy path — and the unhappy path is a // duplicated assignment nobody ordered. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") args := `{"worker":"maya"}` tc.Confirmation = ask(t, reg, tc, args).Token var wg sync.WaitGroup for i := 0; i < 16; i++ { wg.Add(1) go func() { defer wg.Done() reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)) }() } wg.Wait() if got := spy.runs.Load(); got != 1 { t.Fatalf("16 concurrent attempts on one token produced %d writes, want 1", got) } } func TestAnApprovalDoesNotCrossCallers(t *testing.T) { // A token is not a bearer credential for the tool. It authorises one // person's decision, and a second caller holding it — in the same tenant, // same run, same arguments — is not that person. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) args := `{"worker":"maya"}` approved := ask(t, reg, caller("u1", "org1"), args) other := caller("u2", "org1") other.Confirmation = approved.Token notApproved(t, reg.Dispatch(context.Background(), other, "assign_worker", json.RawMessage(args)), spy, approved.Token) } func TestAnApprovalDoesNotCrossTenants(t *testing.T) { // I5, arriving by way of I4. The same user id in a different organization // is a different principal, and a confirmation issued in one tenant must // not act in another. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) args := `{"worker":"maya"}` approved := ask(t, reg, caller("u1", "org1"), args) elsewhere := caller("u1", "org2") elsewhere.Confirmation = approved.Token notApproved(t, reg.Dispatch(context.Background(), elsewhere, "assign_worker", json.RawMessage(args)), spy, approved.Token) } func TestAnApprovalSurvivesTheRunEnding(t *testing.T) { // The case the whole mechanism exists for, and the one an earlier version // of this code broke. // // A confirmation ends the run — that is the point: the model stops, a person // is asked, and the answer arrives later. The run that resumes is a NEW run // with a new id, so a token scoped to the run that raised it could never be // redeemed by the run that resumes. Binding on the run read as the tighter // choice and was in fact the choice that refused every legitimate approval. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) args := `{"worker":"maya"}` approved := ask(t, reg, caller("u1", "org1"), args) resumed := caller("u1", "org1") resumed.RunID = "run_two" // a different run, as a resumed one always is resumed.Confirmation = approved.Token if res := reg.Dispatch(context.Background(), resumed, "assign_worker", json.RawMessage(args)); res.Error != nil { t.Fatalf("an approval must survive the run that raised it: %+v", res.Error) } if spy.runs.Load() != 1 { t.Fatal("the approved write did not run on resumption") } } func TestAnExpiredApprovalIsRefused(t *testing.T) { // An approval is a judgement about a moment. Honouring a two-day-old yes // answers a question whose facts have moved on, and the person who clicked // had no way to know that. spy := &spyWrite{expiry: time.Now().Add(-time.Minute)} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") args := `{"worker":"maya"}` stale := ask(t, reg, tc, args).Token tc.Confirmation = stale notApproved(t, reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(args)), spy, stale) } func TestAnInventedTokenAuthorisesNothing(t *testing.T) { spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") tc.Confirmation = "cnf_this-looks-about-right" res := reg.Dispatch(context.Background(), tc, "assign_worker", json.RawMessage(`{}`)) notApproved(t, res, spy, tc.Confirmation) if res.Confirmation == nil { t.Fatal("an invented token should leave the call unapproved and described afresh") } } /* ── The renderer ───────────────────────────────────────────────────────── */ func TestARefusedRendererIssuesNothingAndSaysNothing(t *testing.T) { // A renderer authorizes on the same terms as the write. When it refuses, // the refusal must be the ordinary opaque one — a distinguishable "I cannot // describe that" would answer, at confirmation time, exactly the question // the denial exists to leave unanswered. spy := &spyWrite{denied: true} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) res := reg.Dispatch(context.Background(), caller("u1", "org1"), "assign_worker", json.RawMessage(`{}`)) if res.Confirmation != nil { t.Fatal("a refused caller was still handed a token") } if res.Error == nil || res.Error.Code != tools.CodeDenied { t.Fatalf("want the standard denial, got %+v", res.Error) } if res.Error.Message != tools.Denied().Error.Message { t.Error("a renderer's refusal must be worded identically to every other refusal") } if spy.runs.Load() != 0 { t.Fatal("a refused write ran anyway") } } func TestAPanickingRendererDoesNotWrite(t *testing.T) { // A renderer is author-written code running before any approval exists. It // gets the same containment a handler does, and the failure direction is // closed: no description, no token, no write. spy := &spyWrite{panics: true} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) res := reg.Dispatch(context.Background(), caller("u1", "org1"), "assign_worker", json.RawMessage(`{}`)) if res.Error == nil { t.Fatal("a renderer that panicked should have produced an error result") } if res.Confirmation != nil { t.Fatal("a panicking renderer still issued a token") } if spy.runs.Load() != 0 { t.Fatal("a write ran after its renderer panicked") } } func TestReadToolsAreNotGated(t *testing.T) { // The gate applies to effects, not to every tool. A read that had to be // approved would teach people to approve without reading, which is how a // confirmation dialog stops being a control. reg := tools.NewRegistry() var ran atomic.Int64 reg.MustRegister(tools.Tool{ Name: "activity_breakdown", Description: "Read.", Effect: tools.EffectRead, InputSchema: map[string]any{"type": "object"}, Handler: func(context.Context, tools.Context, json.RawMessage) tools.Result { ran.Add(1) return tools.OK(map[string]any{"ok": true}) }, }) res := reg.Dispatch(context.Background(), caller("u1", "org1"), "activity_breakdown", json.RawMessage(`{}`)) if res.Error != nil || ran.Load() != 1 { t.Fatalf("a read should run unasked: err=%+v ran=%d", res.Error, ran.Load()) } if res.Confirmation != nil { t.Error("a read raised a confirmation") } } /* ── Redeeming ──────────────────────────────────────────────────────────── */ func TestRedeemingAnApprovalPerformsExactlyWhatWasDescribed(t *testing.T) { // The path that makes a confirmation reliable rather than hopeful. // // Resolve asks "was THIS call approved?", which needs the caller to produce // the same call again. Redeem asks "what WAS approved?", so honouring an // approval does not depend on a model reproducing itself — which, against a // real model, it does not reliably do. var got json.RawMessage spy := &spyWrite{} tool := spy.tool() inner := tool.Handler tool.Handler = func(ctx context.Context, tc tools.Context, in json.RawMessage) tools.Result { got = in return inner(ctx, tc, in) } reg := tools.NewRegistry() reg.MustRegister(tool) tc := caller("u1", "org1") args := `{"shift":"friday","worker":"maya"}` approved := ask(t, reg, tc, args) // Nothing about the original call is supplied — only the token. out, ok := reg.DispatchApproved(context.Background(), caller("u1", "org1"), approved.Token) if !ok { t.Fatal("a valid token could not be redeemed") } if spy.runs.Load() != 1 { t.Fatalf("the write ran %d times, want 1", spy.runs.Load()) } if out.Tool != "assign_worker" { t.Errorf("redeemed tool = %q", out.Tool) } // The arguments are the ones that were described, recovered from the token. var recovered map[string]string if err := json.Unmarshal(got, &recovered); err != nil { t.Fatalf("the replayed arguments were not JSON: %s", got) } if recovered["worker"] != "maya" || recovered["shift"] != "friday" { t.Errorf("replayed %v, want the approved call", recovered) } } func TestARedeemedApprovalIsSpentOnce(t *testing.T) { spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") approved := ask(t, reg, tc, `{"worker":"maya"}`) if _, ok := reg.DispatchApproved(context.Background(), tc, approved.Token); !ok { t.Fatal("the first redemption failed") } if _, ok := reg.DispatchApproved(context.Background(), tc, approved.Token); ok { t.Fatal("a token was redeemed twice") } if spy.runs.Load() != 1 { t.Fatalf("one approval bought %d writes", spy.runs.Load()) } } func TestOnlyThePersonWhoWasAskedCanRedeem(t *testing.T) { // A token is not a bearer credential. Redeem does not check the arguments — // it is the source of them — so the caller check is the only thing standing // between a leaked token and somebody else's write. spy := &spyWrite{} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) approved := ask(t, reg, caller("u1", "org1"), `{"worker":"maya"}`) for name, other := range map[string]tools.Context{ "a different person": caller("u2", "org1"), "a different tenant": caller("u1", "org2"), } { if _, ok := reg.DispatchApproved(context.Background(), other, approved.Token); ok { t.Errorf("%s redeemed an approval that was not theirs", name) } } if spy.runs.Load() != 0 { t.Fatalf("%d writes happened for callers who never approved anything", spy.runs.Load()) } } func TestAnExpiredApprovalCannotBeRedeemed(t *testing.T) { spy := &spyWrite{expiry: time.Now().Add(-time.Minute)} reg := tools.NewRegistry() reg.MustRegister(spy.tool()) tc := caller("u1", "org1") approved := ask(t, reg, tc, `{"worker":"maya"}`) if _, ok := reg.DispatchApproved(context.Background(), tc, approved.Token); ok { t.Fatal("an expired approval was redeemed") } if spy.runs.Load() != 0 { t.Fatal("an expired approval produced a write") } }