package tools import ( "context" "errors" "testing" "time" ) // The registry is the only way to reach a tool, so these are the checks that // stand between a model and the data. Each one is a thing the model could ask // for and must not get. func okTool(name string) Tool { return Tool{ Name: name, Description: "a tool, for testing", Scope: ScopeRead, Schema: Schema{Fields: []Field{{ Name: "limit", Description: "how many", Kind: KindInt, Min: 1, Max: 50, Default: 10, }}}, Handler: func(_ context.Context, req Request) (Result, error) { return Result{Rows: []int{1, 2}, Count: 2, Scope: "all branches"}, nil }, } } func registryWith(t *testing.T, tools ...Tool) (*Registry, *CollectAudit) { t.Helper() audit := &CollectAudit{} r := New(audit) for _, tool := range tools { if err := r.Register(tool); err != nil { t.Fatalf("registering %s: %v", tool.Name, err) } } return r, audit } var anyone = Caller{Userid: 904, Tenantid: 1147} /* ── The five jobs ─────────────────────────────────────────────────────── */ func TestAToolRunsAndReturnsRows(t *testing.T) { r, _ := registryWith(t, okTool("thing")) agent := Agent{Name: "orders", Tools: []string{"thing"}} result, err := r.Call(context.Background(), agent, "thing", nil, anyone) if err != nil { t.Fatalf("calling: %v", err) } if result.Count != 2 { t.Fatalf("rows lost: %d", result.Count) } } func TestAnUnknownToolIsRefused(t *testing.T) { r, _ := registryWith(t, okTool("thing")) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, err := r.Call(context.Background(), agent, "invented", nil, anyone) if !errors.Is(err, ErrUnknownTool) { t.Fatalf("a tool the model made up was not refused: %v", err) } } func TestAToolOffTheAllowListIsRefusedByTheRegistryNotTheModel(t *testing.T) { // The whole point of the allow-list: a prompt is a request, this is a rule. r, _ := registryWith(t, okTool("thing"), okTool("other")) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, err := r.Call(context.Background(), agent, "other", nil, anyone) if !errors.Is(err, ErrNotAllowed) { t.Fatalf("an agent reached a tool it does not name: %v", err) } } func TestBadArgumentsNeverReachTheHandler(t *testing.T) { reached := false tool := okTool("thing") tool.Handler = func(context.Context, Request) (Result, error) { reached = true return Result{}, nil } r, _ := registryWith(t, tool) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, err := r.Call(context.Background(), agent, "thing", map[string]any{"limit": 999}, anyone) if !errors.Is(err, ErrBadArgument) { t.Fatalf("an out-of-range argument was accepted: %v", err) } if reached { t.Fatal("the handler ran on arguments the schema refused") } } func TestACallerScopedToNothingIsNotACallerScopedToEverything(t *testing.T) { // Go's zero value is 0, so an unset tenant and a platform account look // identical unless staff status is asked for separately. r, _ := registryWith(t, okTool("thing")) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, err := r.Call(context.Background(), agent, "thing", nil, Caller{Userid: 904}) if !errors.Is(err, ErrNoTenant) { t.Fatalf("a caller with no tenant was let through: %v", err) } } func TestStaffAreNotExemptFromAToolsScope(t *testing.T) { // A platform account carries no tenant, and "every merchant at once" is not // an answer to "what is stuck?". Staff are told to pick a shop, in the same // words a branch user would get — the exemption in WebAuth is about which // tenant they may NAME, not about reading all of them at once. r, _ := registryWith(t, okTool("thing")) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, err := r.Call(context.Background(), agent, "thing", nil, Caller{Userid: 12, Superadmin: true}) if !errors.Is(err, ErrNoTenant) { t.Fatalf("staff read a tenant-scoped tool with no tenant: %v", err) } // With a shop picked, the same call works. if _, err := r.Call(context.Background(), agent, "thing", nil, Caller{Userid: 12, Superadmin: true, Tenantid: 1147}); err != nil { t.Fatalf("staff were refused a shop they had picked: %v", err) } } /* ── The scope a tool declares ─────────────────────────────────────────── */ func TestAToolThatDeclaresNothingIsConfinedToOneMerchant(t *testing.T) { // Default-deny. The zero value of Requires is the strictest, so a tool // written next year without thinking about scope is safe rather than silent. tool := okTool("thing") if tool.Needs != RequiresTenant { t.Fatalf("the default scope is %v, not the strictest", tool.Needs) } } func TestABranchScopedToolRefusesAnAllBranchesCaller(t *testing.T) { tool := okTool("thing") tool.Needs = RequiresBranch r, _ := registryWith(t, tool) agent := Agent{Name: "orders", Tools: []string{"thing"}} if _, err := r.Call(context.Background(), agent, "thing", nil, anyone); !errors.Is(err, ErrNoTenant) { t.Fatalf("a branch-only tool answered for every branch: %v", err) } withBranch := Caller{Userid: 904, Tenantid: 1147, Locationid: 1172} if _, err := r.Call(context.Background(), agent, "thing", nil, withBranch); err != nil { t.Fatalf("a branch caller was refused: %v", err) } } func TestAToolNeedingNothingAnswersWithoutAShop(t *testing.T) { // There is one: the product help corpus, which carries nothing about // anybody. Requiring a tenant would refuse staff a help question for no // reason. tool := okTool("thing") tool.Needs = RequiresNothing r, _ := registryWith(t, tool) _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, Caller{Userid: 12, Superadmin: true}) if err != nil { t.Fatalf("a tool needing no shop was refused: %v", err) } } func TestTheScopeIsCheckedBeforeTheHandlerRuns(t *testing.T) { // So no query is ever built from a scope that was never established. ran := false tool := okTool("thing") tool.Needs = RequiresBranch tool.Handler = func(context.Context, Request) (Result, error) { ran = true return Result{}, nil } r, _ := registryWith(t, tool) _, _ = r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone) if ran { t.Fatal("the handler ran without the scope it declared") } } func TestAToolMayNotOfferAScopingArgument(t *testing.T) { // The structural guarantee. An argument is something the MODEL fills in, and // the model is the one part of this system that can be argued with — so a // tool offering `tenantid` is refused at registration rather than trusted to // ignore it. for _, name := range []string{"tenantid", "tenant_id", "locationid", "store_id", "partnerid", "customerid", "userid"} { tool := okTool("thing") tool.Schema = Schema{Fields: []Field{{Name: name, Description: "d", Kind: KindInt}}} if err := New(nil).Register(tool); err == nil { t.Fatalf("a tool offering %q as an argument was registered", name) } } } func TestNoRegisteredToolOffersAWayToChooseWhoseDataIsRead(t *testing.T) { // The sweep, over every tool that actually ships. This is the test that // catches the eighth tool somebody adds in a hurry. r := New(nil) for _, tool := range shippedTools(t) { if err := r.Register(tool); err != nil { t.Fatalf("%s: %v", tool.Name, err) } } } /* ── Arguments ─────────────────────────────────────────────────────────── */ func TestDefaultsAreFilledIn(t *testing.T) { var seen Request tool := okTool("thing") tool.Handler = func(_ context.Context, req Request) (Result, error) { seen = req return Result{}, nil } r, _ := registryWith(t, tool) if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone); err != nil { t.Fatalf("calling: %v", err) } if seen.Int("limit") != 10 { t.Fatalf("the default did not arrive: %d", seen.Int("limit")) } } func TestAJSONNumberIsAcceptedAsAnInteger(t *testing.T) { // Every argument arrives over HTTP, so an integer field that only accepted // Go ints would refuse every real call. var seen Request tool := okTool("thing") tool.Handler = func(_ context.Context, req Request) (Result, error) { seen = req return Result{}, nil } r, _ := registryWith(t, tool) if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", map[string]any{"limit": float64(20)}, anyone); err != nil { t.Fatalf("a JSON number was refused: %v", err) } if seen.Int("limit") != 20 { t.Fatalf("the value did not survive: %d", seen.Int("limit")) } } func TestAFractionIsNotAWholeNumber(t *testing.T) { r, _ := registryWith(t, okTool("thing")) _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", map[string]any{"limit": 2.5}, anyone) if !errors.Is(err, ErrBadArgument) { t.Fatalf("2.5 was accepted as a count: %v", err) } } func TestAnInventedArgumentIsDroppedNotPassedOn(t *testing.T) { // A handler must never receive a key it did not declare, or an argument the // model made up becomes one a handler might later start reading. var seen Request tool := okTool("thing") tool.Handler = func(_ context.Context, req Request) (Result, error) { seen = req return Result{}, nil } r, _ := registryWith(t, tool) args := map[string]any{"limit": 5, "tenantid": 916, "where": "1=1"} if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", args, anyone); err != nil { t.Fatalf("calling: %v", err) } if _, present := seen.Args["tenantid"]; present { t.Fatal("the model got to name a tenant") } if _, present := seen.Args["where"]; present { t.Fatal("the model got to pass a where clause") } } /* ── Registration ──────────────────────────────────────────────────────── */ func TestAToolCannotBeSilentlyReplaced(t *testing.T) { // Overwriting a registered tool is how a permission check disappears. r, _ := registryWith(t, okTool("thing")) if err := r.Register(okTool("thing")); err == nil { t.Fatal("a second tool took the same name") } } func TestAToolWithoutADescriptionIsRefused(t *testing.T) { // A model chooses between tools by their descriptions. One without is a // tool that gets called for the wrong question. r := New(nil) tool := okTool("thing") tool.Description = "" if err := r.Register(tool); err == nil { t.Fatal("a tool with no description was registered") } } /* ── What the model is told ────────────────────────────────────────────── */ func TestAnAgentIsOnlyToldAboutToolsItMayUse(t *testing.T) { // Describing a tool the agent would then be refused reads to a model as a // malfunction, and to a person as the assistant being broken. r, _ := registryWith(t, okTool("thing"), okTool("other")) defs := r.Definitions(Agent{Name: "orders", Tools: []string{"thing"}}) if len(defs) != 1 || defs[0]["name"] != "thing" { t.Fatalf("the agent was told about the wrong tools: %v", defs) } } func TestTheSchemaForbidsInventedArguments(t *testing.T) { r, _ := registryWith(t, okTool("thing")) defs := r.Definitions(Agent{Name: "orders", Tools: []string{"thing"}}) schema, _ := defs[0]["input_schema"].(map[string]any) if schema["additionalProperties"] != false { t.Fatal("the schema lets the model add its own arguments") } } /* ── The audit trail ───────────────────────────────────────────────────── */ func TestEveryCallLeavesExactlyOneRow(t *testing.T) { r, audit := registryWith(t, okTool("thing")) agent := Agent{Name: "orders", Tools: []string{"thing"}} _, _ = r.Call(context.Background(), agent, "thing", nil, anyone) if len(audit.Entries) != 1 { t.Fatalf("a successful call wrote %d rows", len(audit.Entries)) } entry, _ := audit.Last() if entry.Outcome != OutcomeOK || entry.Tool != "thing" || entry.Tenantid != 1147 { t.Fatalf("the row does not describe the call: %+v", entry) } } func TestARefusalIsAudited(t *testing.T) { // The refusals are the interesting ones. A trail of successes answers // "did anything try to read another tenant?" with silence, which reads the // same as "no". r, audit := registryWith(t, okTool("thing"), okTool("other")) _, _ = r.Call(context.Background(), Agent{Name: "orders", Tools: []string{"thing"}}, "other", nil, anyone) entry, ok := audit.Last() if !ok || entry.Outcome != OutcomeRefused { t.Fatalf("a refusal left no trace: %+v", entry) } if entry.Detail == "" { t.Fatal("the refusal does not say why") } } func TestABrokenToolIsFailedNotRefused(t *testing.T) { // Collapsing the two hides a broken tool inside a count of things working // as designed. tool := okTool("thing") tool.Handler = func(context.Context, Request) (Result, error) { return Result{}, errors.New("the database is down") } r, audit := registryWith(t, tool) _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone) if err == nil { t.Fatal("a broken handler reported success") } entry, _ := audit.Last() if entry.Outcome != OutcomeFailed { t.Fatalf("a handler error was recorded as %q", entry.Outcome) } } func TestTheAuditKeepsWhatRanNotWhatWasSent(t *testing.T) { // Defaults applied, invented keys dropped. What actually executed is the // thing worth being able to read back. r, audit := registryWith(t, okTool("thing")) _, _ = r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", map[string]any{"tenantid": 916}, anyone) entry, _ := audit.Last() if _, present := entry.Args["tenantid"]; present { t.Fatal("the audit kept an argument the handler never saw") } if entry.Args["limit"] != 10 { t.Fatalf("the applied default is missing from the trail: %+v", entry.Args) } } func TestAnAuditLineIsStableBetweenIdenticalCalls(t *testing.T) { // Go randomises map iteration, so without sorting the same call logs // differently every time and a grep for one of them finds one of them. entry := AuditEntry{ At: time.Now(), Agent: "orders", Tool: "thing", Userid: 904, Tenantid: 1147, Args: map[string]any{"b": 2, "a": 1, "c": 3}, Outcome: OutcomeOK, } first := entry.Line() for range 20 { if entry.Line() != first { t.Fatalf("two renderings of one entry differ:\n%s\n%s", first, entry.Line()) } } } func TestAnAuditLineSaysWhenThereWasNoTenant(t *testing.T) { // Explicitly, rather than by omission: no tenant on an assistant call is // either staff or a bug, and both are worth being able to search for. entry := AuditEntry{Agent: "orders", Tool: "thing", Userid: 12, Outcome: OutcomeOK} if got := entry.Line(); !contains(got, "tenant=none") { t.Fatalf("a tenantless call is invisible in the log: %s", got) } } func contains(haystack, needle string) bool { return len(haystack) >= len(needle) && (func() bool { for i := 0; i+len(needle) <= len(haystack); i++ { if haystack[i:i+len(needle)] == needle { return true } } return false })() } /* ── The silent zero ───────────────────────────────────────────────────── */ func TestAJSONNumberDoesNotBecomeASilentZero(t *testing.T) { // The root cause of the approval bug. `Validate` hands a handler a real int, // so this only matters on a path that skipped validation — and the cost of // getting it wrong is a zero id, which looks like a plausible argument // rather than a fault. "Request 0 is not waiting for approval" reads like a // stale card, not like a type error. req := Request{Args: map[string]any{"a": 41, "b": float64(41), "c": int64(41)}} for _, name := range []string{"a", "b", "c"} { if got := req.Int(name); got != 41 { t.Fatalf("%q read back as %d", name, got) } } } func TestAFractionIsNotQuietlyTruncated(t *testing.T) { // Silently rounding would be inventing an answer. Zero is wrong too, but it // is wrong in a way that shows up as "not found" rather than as the wrong // row being changed. req := Request{Args: map[string]any{"a": 41.5}} if got := req.Int("a"); got != 0 { t.Fatalf("41.5 became %d", got) } } func TestAnAbsentArgumentIsZero(t *testing.T) { req := Request{Args: map[string]any{}} if got := req.Int("missing"); got != 0 { t.Fatalf("an absent argument read as %d", got) } }