backend_fiesta/services/tools/registry_test.go

package tools

import (
	"context"
	"errors"
	"testing"
	"time"
)

// The registry is the only way to reach a tool, so these are the checks that
// stand between a model and the data. Each one is a thing the model could ask
// for and must not get.

func okTool(name string) Tool {
	return Tool{
		Name:        name,
		Description: "a tool, for testing",
		Scope:       ScopeRead,
		Schema: Schema{Fields: []Field{{
			Name: "limit", Description: "how many", Kind: KindInt, Min: 1, Max: 50, Default: 10,
		}}},
		Handler: func(_ context.Context, req Request) (Result, error) {
			return Result{Rows: []int{1, 2}, Count: 2, Scope: "all branches"}, nil
		},
	}
}

func registryWith(t *testing.T, tools ...Tool) (*Registry, *CollectAudit) {
	t.Helper()
	audit := &CollectAudit{}
	r := New(audit)
	for _, tool := range tools {
		if err := r.Register(tool); err != nil {
			t.Fatalf("registering %s: %v", tool.Name, err)
		}
	}
	return r, audit
}

var anyone = Caller{Userid: 904, Tenantid: 1147}

/* ── The five jobs ─────────────────────────────────────────────────────── */

func TestAToolRunsAndReturnsRows(t *testing.T) {
	r, _ := registryWith(t, okTool("thing"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	result, err := r.Call(context.Background(), agent, "thing", nil, anyone)
	if err != nil {
		t.Fatalf("calling: %v", err)
	}
	if result.Count != 2 {
		t.Fatalf("rows lost: %d", result.Count)
	}
}

func TestAnUnknownToolIsRefused(t *testing.T) {
	r, _ := registryWith(t, okTool("thing"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, err := r.Call(context.Background(), agent, "invented", nil, anyone)
	if !errors.Is(err, ErrUnknownTool) {
		t.Fatalf("a tool the model made up was not refused: %v", err)
	}
}

func TestAToolOffTheAllowListIsRefusedByTheRegistryNotTheModel(t *testing.T) {
	// The whole point of the allow-list: a prompt is a request, this is a rule.
	r, _ := registryWith(t, okTool("thing"), okTool("other"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, err := r.Call(context.Background(), agent, "other", nil, anyone)
	if !errors.Is(err, ErrNotAllowed) {
		t.Fatalf("an agent reached a tool it does not name: %v", err)
	}
}

func TestBadArgumentsNeverReachTheHandler(t *testing.T) {
	reached := false
	tool := okTool("thing")
	tool.Handler = func(context.Context, Request) (Result, error) {
		reached = true
		return Result{}, nil
	}
	r, _ := registryWith(t, tool)
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, err := r.Call(context.Background(), agent, "thing", map[string]any{"limit": 999}, anyone)
	if !errors.Is(err, ErrBadArgument) {
		t.Fatalf("an out-of-range argument was accepted: %v", err)
	}
	if reached {
		t.Fatal("the handler ran on arguments the schema refused")
	}
}

func TestACallerScopedToNothingIsNotACallerScopedToEverything(t *testing.T) {
	// Go's zero value is 0, so an unset tenant and a platform account look
	// identical unless staff status is asked for separately.
	r, _ := registryWith(t, okTool("thing"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, err := r.Call(context.Background(), agent, "thing", nil, Caller{Userid: 904})
	if !errors.Is(err, ErrNoTenant) {
		t.Fatalf("a caller with no tenant was let through: %v", err)
	}
}

func TestStaffAreNotExemptFromAToolsScope(t *testing.T) {
	// A platform account carries no tenant, and "every merchant at once" is not
	// an answer to "what is stuck?". Staff are told to pick a shop, in the same
	// words a branch user would get — the exemption in WebAuth is about which
	// tenant they may NAME, not about reading all of them at once.
	r, _ := registryWith(t, okTool("thing"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, err := r.Call(context.Background(), agent, "thing", nil, Caller{Userid: 12, Superadmin: true})
	if !errors.Is(err, ErrNoTenant) {
		t.Fatalf("staff read a tenant-scoped tool with no tenant: %v", err)
	}

	// With a shop picked, the same call works.
	if _, err := r.Call(context.Background(), agent, "thing", nil,
		Caller{Userid: 12, Superadmin: true, Tenantid: 1147}); err != nil {
		t.Fatalf("staff were refused a shop they had picked: %v", err)
	}
}

/* ── The scope a tool declares ─────────────────────────────────────────── */

func TestAToolThatDeclaresNothingIsConfinedToOneMerchant(t *testing.T) {
	// Default-deny. The zero value of Requires is the strictest, so a tool
	// written next year without thinking about scope is safe rather than silent.
	tool := okTool("thing")
	if tool.Needs != RequiresTenant {
		t.Fatalf("the default scope is %v, not the strictest", tool.Needs)
	}
}

func TestABranchScopedToolRefusesAnAllBranchesCaller(t *testing.T) {
	tool := okTool("thing")
	tool.Needs = RequiresBranch
	r, _ := registryWith(t, tool)
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	if _, err := r.Call(context.Background(), agent, "thing", nil, anyone); !errors.Is(err, ErrNoTenant) {
		t.Fatalf("a branch-only tool answered for every branch: %v", err)
	}

	withBranch := Caller{Userid: 904, Tenantid: 1147, Locationid: 1172}
	if _, err := r.Call(context.Background(), agent, "thing", nil, withBranch); err != nil {
		t.Fatalf("a branch caller was refused: %v", err)
	}
}

func TestAToolNeedingNothingAnswersWithoutAShop(t *testing.T) {
	// There is one: the product help corpus, which carries nothing about
	// anybody. Requiring a tenant would refuse staff a help question for no
	// reason.
	tool := okTool("thing")
	tool.Needs = RequiresNothing
	r, _ := registryWith(t, tool)

	_, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil,
		Caller{Userid: 12, Superadmin: true})
	if err != nil {
		t.Fatalf("a tool needing no shop was refused: %v", err)
	}
}

func TestTheScopeIsCheckedBeforeTheHandlerRuns(t *testing.T) {
	// So no query is ever built from a scope that was never established.
	ran := false
	tool := okTool("thing")
	tool.Needs = RequiresBranch
	tool.Handler = func(context.Context, Request) (Result, error) {
		ran = true
		return Result{}, nil
	}
	r, _ := registryWith(t, tool)

	_, _ = r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone)
	if ran {
		t.Fatal("the handler ran without the scope it declared")
	}
}

func TestAToolMayNotOfferAScopingArgument(t *testing.T) {
	// The structural guarantee. An argument is something the MODEL fills in, and
	// the model is the one part of this system that can be argued with — so a
	// tool offering `tenantid` is refused at registration rather than trusted to
	// ignore it.
	for _, name := range []string{"tenantid", "tenant_id", "locationid", "store_id", "partnerid", "customerid", "userid"} {
		tool := okTool("thing")
		tool.Schema = Schema{Fields: []Field{{Name: name, Description: "d", Kind: KindInt}}}
		if err := New(nil).Register(tool); err == nil {
			t.Fatalf("a tool offering %q as an argument was registered", name)
		}
	}
}

func TestNoRegisteredToolOffersAWayToChooseWhoseDataIsRead(t *testing.T) {
	// The sweep, over every tool that actually ships. This is the test that
	// catches the eighth tool somebody adds in a hurry.
	r := New(nil)
	for _, tool := range shippedTools(t) {
		if err := r.Register(tool); err != nil {
			t.Fatalf("%s: %v", tool.Name, err)
		}
	}
}

/* ── Arguments ─────────────────────────────────────────────────────────── */

func TestDefaultsAreFilledIn(t *testing.T) {
	var seen Request
	tool := okTool("thing")
	tool.Handler = func(_ context.Context, req Request) (Result, error) {
		seen = req
		return Result{}, nil
	}
	r, _ := registryWith(t, tool)

	if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone); err != nil {
		t.Fatalf("calling: %v", err)
	}
	if seen.Int("limit") != 10 {
		t.Fatalf("the default did not arrive: %d", seen.Int("limit"))
	}
}

func TestAJSONNumberIsAcceptedAsAnInteger(t *testing.T) {
	// Every argument arrives over HTTP, so an integer field that only accepted
	// Go ints would refuse every real call.
	var seen Request
	tool := okTool("thing")
	tool.Handler = func(_ context.Context, req Request) (Result, error) {
		seen = req
		return Result{}, nil
	}
	r, _ := registryWith(t, tool)

	if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", map[string]any{"limit": float64(20)}, anyone); err != nil {
		t.Fatalf("a JSON number was refused: %v", err)
	}
	if seen.Int("limit") != 20 {
		t.Fatalf("the value did not survive: %d", seen.Int("limit"))
	}
}

func TestAFractionIsNotAWholeNumber(t *testing.T) {
	r, _ := registryWith(t, okTool("thing"))
	_, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", map[string]any{"limit": 2.5}, anyone)
	if !errors.Is(err, ErrBadArgument) {
		t.Fatalf("2.5 was accepted as a count: %v", err)
	}
}

func TestAnInventedArgumentIsDroppedNotPassedOn(t *testing.T) {
	// A handler must never receive a key it did not declare, or an argument the
	// model made up becomes one a handler might later start reading.
	var seen Request
	tool := okTool("thing")
	tool.Handler = func(_ context.Context, req Request) (Result, error) {
		seen = req
		return Result{}, nil
	}
	r, _ := registryWith(t, tool)

	args := map[string]any{"limit": 5, "tenantid": 916, "where": "1=1"}
	if _, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", args, anyone); err != nil {
		t.Fatalf("calling: %v", err)
	}
	if _, present := seen.Args["tenantid"]; present {
		t.Fatal("the model got to name a tenant")
	}
	if _, present := seen.Args["where"]; present {
		t.Fatal("the model got to pass a where clause")
	}
}

/* ── Registration ──────────────────────────────────────────────────────── */

func TestAToolCannotBeSilentlyReplaced(t *testing.T) {
	// Overwriting a registered tool is how a permission check disappears.
	r, _ := registryWith(t, okTool("thing"))
	if err := r.Register(okTool("thing")); err == nil {
		t.Fatal("a second tool took the same name")
	}
}

func TestAToolWithoutADescriptionIsRefused(t *testing.T) {
	// A model chooses between tools by their descriptions. One without is a
	// tool that gets called for the wrong question.
	r := New(nil)
	tool := okTool("thing")
	tool.Description = ""
	if err := r.Register(tool); err == nil {
		t.Fatal("a tool with no description was registered")
	}
}

/* ── What the model is told ────────────────────────────────────────────── */

func TestAnAgentIsOnlyToldAboutToolsItMayUse(t *testing.T) {
	// Describing a tool the agent would then be refused reads to a model as a
	// malfunction, and to a person as the assistant being broken.
	r, _ := registryWith(t, okTool("thing"), okTool("other"))
	defs := r.Definitions(Agent{Name: "orders", Tools: []string{"thing"}})

	if len(defs) != 1 || defs[0]["name"] != "thing" {
		t.Fatalf("the agent was told about the wrong tools: %v", defs)
	}
}

func TestTheSchemaForbidsInventedArguments(t *testing.T) {
	r, _ := registryWith(t, okTool("thing"))
	defs := r.Definitions(Agent{Name: "orders", Tools: []string{"thing"}})
	schema, _ := defs[0]["input_schema"].(map[string]any)

	if schema["additionalProperties"] != false {
		t.Fatal("the schema lets the model add its own arguments")
	}
}

/* ── The audit trail ───────────────────────────────────────────────────── */

func TestEveryCallLeavesExactlyOneRow(t *testing.T) {
	r, audit := registryWith(t, okTool("thing"))
	agent := Agent{Name: "orders", Tools: []string{"thing"}}

	_, _ = r.Call(context.Background(), agent, "thing", nil, anyone)
	if len(audit.Entries) != 1 {
		t.Fatalf("a successful call wrote %d rows", len(audit.Entries))
	}
	entry, _ := audit.Last()
	if entry.Outcome != OutcomeOK || entry.Tool != "thing" || entry.Tenantid != 1147 {
		t.Fatalf("the row does not describe the call: %+v", entry)
	}
}

func TestARefusalIsAudited(t *testing.T) {
	// The refusals are the interesting ones. A trail of successes answers
	// "did anything try to read another tenant?" with silence, which reads the
	// same as "no".
	r, audit := registryWith(t, okTool("thing"), okTool("other"))

	_, _ = r.Call(context.Background(), Agent{Name: "orders", Tools: []string{"thing"}}, "other", nil, anyone)
	entry, ok := audit.Last()
	if !ok || entry.Outcome != OutcomeRefused {
		t.Fatalf("a refusal left no trace: %+v", entry)
	}
	if entry.Detail == "" {
		t.Fatal("the refusal does not say why")
	}
}

func TestABrokenToolIsFailedNotRefused(t *testing.T) {
	// Collapsing the two hides a broken tool inside a count of things working
	// as designed.
	tool := okTool("thing")
	tool.Handler = func(context.Context, Request) (Result, error) {
		return Result{}, errors.New("the database is down")
	}
	r, audit := registryWith(t, tool)

	_, err := r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing", nil, anyone)
	if err == nil {
		t.Fatal("a broken handler reported success")
	}
	entry, _ := audit.Last()
	if entry.Outcome != OutcomeFailed {
		t.Fatalf("a handler error was recorded as %q", entry.Outcome)
	}
}

func TestTheAuditKeepsWhatRanNotWhatWasSent(t *testing.T) {
	// Defaults applied, invented keys dropped. What actually executed is the
	// thing worth being able to read back.
	r, audit := registryWith(t, okTool("thing"))

	_, _ = r.Call(context.Background(), Agent{Name: "a", Tools: []string{"thing"}}, "thing",
		map[string]any{"tenantid": 916}, anyone)

	entry, _ := audit.Last()
	if _, present := entry.Args["tenantid"]; present {
		t.Fatal("the audit kept an argument the handler never saw")
	}
	if entry.Args["limit"] != 10 {
		t.Fatalf("the applied default is missing from the trail: %+v", entry.Args)
	}
}

func TestAnAuditLineIsStableBetweenIdenticalCalls(t *testing.T) {
	// Go randomises map iteration, so without sorting the same call logs
	// differently every time and a grep for one of them finds one of them.
	entry := AuditEntry{
		At: time.Now(), Agent: "orders", Tool: "thing", Userid: 904, Tenantid: 1147,
		Args: map[string]any{"b": 2, "a": 1, "c": 3}, Outcome: OutcomeOK,
	}
	first := entry.Line()
	for range 20 {
		if entry.Line() != first {
			t.Fatalf("two renderings of one entry differ:\n%s\n%s", first, entry.Line())
		}
	}
}

func TestAnAuditLineSaysWhenThereWasNoTenant(t *testing.T) {
	// Explicitly, rather than by omission: no tenant on an assistant call is
	// either staff or a bug, and both are worth being able to search for.
	entry := AuditEntry{Agent: "orders", Tool: "thing", Userid: 12, Outcome: OutcomeOK}
	if got := entry.Line(); !contains(got, "tenant=none") {
		t.Fatalf("a tenantless call is invisible in the log: %s", got)
	}
}

func contains(haystack, needle string) bool {
	return len(haystack) >= len(needle) && (func() bool {
		for i := 0; i+len(needle) <= len(haystack); i++ {
			if haystack[i:i+len(needle)] == needle {
				return true
			}
		}
		return false
	})()
}

/* ── The silent zero ───────────────────────────────────────────────────── */

func TestAJSONNumberDoesNotBecomeASilentZero(t *testing.T) {
	// The root cause of the approval bug. `Validate` hands a handler a real int,
	// so this only matters on a path that skipped validation — and the cost of
	// getting it wrong is a zero id, which looks like a plausible argument
	// rather than a fault. "Request 0 is not waiting for approval" reads like a
	// stale card, not like a type error.
	req := Request{Args: map[string]any{"a": 41, "b": float64(41), "c": int64(41)}}

	for _, name := range []string{"a", "b", "c"} {
		if got := req.Int(name); got != 41 {
			t.Fatalf("%q read back as %d", name, got)
		}
	}
}

func TestAFractionIsNotQuietlyTruncated(t *testing.T) {
	// Silently rounding would be inventing an answer. Zero is wrong too, but it
	// is wrong in a way that shows up as "not found" rather than as the wrong
	// row being changed.
	req := Request{Args: map[string]any{"a": 41.5}}
	if got := req.Int("a"); got != 0 {
		t.Fatalf("41.5 became %d", got)
	}
}

func TestAnAbsentArgumentIsZero(t *testing.T) {
	req := Request{Args: map[string]any{}}
	if got := req.Int("missing"); got != 0 {
		t.Fatalf("an absent argument read as %d", got)
	}
}