agent
This commit is contained in:
264
services/agents_test.go
Normal file
264
services/agents_test.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"nearle/services/tools"
|
||||
"nearle/utils"
|
||||
)
|
||||
|
||||
// realRegistry builds every tool the server registers, exactly as the facade
|
||||
// does — with nil services, because nothing here calls a handler.
|
||||
//
|
||||
// Replaces a hand-written list of tool names that went stale twice: once when
|
||||
// the help corpus arrived and once when the first write did. Both times the
|
||||
// shipped agents were correct and the TEST was wrong, which is the worst way
|
||||
// round — a check that has to be remembered is a check that will not be.
|
||||
func realRegistry(t *testing.T) *tools.Registry {
|
||||
t.Helper()
|
||||
|
||||
corpus, err := tools.LoadHelp()
|
||||
if err != nil {
|
||||
t.Fatalf("loading the help corpus: %v", err)
|
||||
}
|
||||
|
||||
r := tools.New(nil)
|
||||
for _, tool := range []tools.Tool{
|
||||
tools.StuckOrders(nil, nil),
|
||||
tools.DeliveryProgress(nil),
|
||||
tools.BranchPerformance(nil),
|
||||
tools.PendingApprovals(nil, nil),
|
||||
tools.LowStock(nil),
|
||||
tools.TillsNotSyncing(nil),
|
||||
tools.SalesByChannel(nil, nil, nil),
|
||||
tools.Help(corpus),
|
||||
tools.ApproveStockRequest(nil, nil),
|
||||
} {
|
||||
if err := r.Register(tool); err != nil {
|
||||
t.Fatalf("registering %s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// knownTool asks the real registry, so an agent naming a tool that does not
|
||||
// exist fails here and not on somebody's deployment.
|
||||
func knownTool(name string) bool { return liveRegistry.Has(name) }
|
||||
|
||||
// liveRegistry is built once, in TestMain, because knownTool has no *testing.T.
|
||||
var liveRegistry *tools.Registry
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
fake := &testing.T{}
|
||||
liveRegistry = realRegistry(fake)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// writeAgents drops YAML into a temporary directory and loads it.
|
||||
func writeAgents(t *testing.T, files map[string]string) (map[string]Agent, error) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for name, body := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
|
||||
t.Fatalf("writing %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
return LoadAgents(dir, knownTool)
|
||||
}
|
||||
|
||||
/* ── The agents that ship ──────────────────────────────────────────────── */
|
||||
|
||||
func TestTheEmbeddedAgentsLoad(t *testing.T) {
|
||||
// The defaults are compiled in so a binary always has working agents and a
|
||||
// deployment cannot be broken by a missing directory.
|
||||
agents, err := LoadAgents("", knownTool)
|
||||
if err != nil {
|
||||
t.Fatalf("the shipped agents do not load: %v", err)
|
||||
}
|
||||
for _, want := range []string{"orders", "inventory", "shopfloor"} {
|
||||
if _, ok := agents[want]; !ok {
|
||||
t.Fatalf("%q is missing: %v", want, AgentNames(agents))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryShippedAgentNamesOnlyRealTools(t *testing.T) {
|
||||
// The check that makes a typo a startup failure rather than an agent that
|
||||
// silently cannot do one of the things it claims.
|
||||
if _, err := LoadAgents("", knownTool); err != nil {
|
||||
t.Fatalf("a shipped agent names a tool that does not exist: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddingAnAgentTakesNoCodeChange(t *testing.T) {
|
||||
// Phase 3's whole point, stated as a test.
|
||||
agents, err := writeAgents(t, map[string]string{
|
||||
"sixth.yaml": "name: sixth\ntier: fast\ntools:\n - low_stock\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("loading: %v", err)
|
||||
}
|
||||
if _, ok := agents["sixth"]; !ok {
|
||||
t.Fatal("a new file did not become an agent")
|
||||
}
|
||||
}
|
||||
|
||||
/* ── The shared prompt ─────────────────────────────────────────────────── */
|
||||
|
||||
func TestEveryAgentInheritsTheRulesThatMustNotDrift(t *testing.T) {
|
||||
// Repeating "say when a result was truncated" in three files is three
|
||||
// chances for one of them to lose it.
|
||||
agents, err := LoadAgents("", knownTool)
|
||||
if err != nil {
|
||||
t.Fatalf("loading: %v", err)
|
||||
}
|
||||
for name, agent := range agents {
|
||||
if !strings.Contains(agent.System, "truncated") {
|
||||
t.Fatalf("agent %q was not told about truncated results", name)
|
||||
}
|
||||
if !strings.Contains(agent.System, "never estimate") {
|
||||
t.Fatalf("agent %q was not told to answer only from tools", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentsOwnWordsAreAddedNotSubstituted(t *testing.T) {
|
||||
agents, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "name: one\ntools:\n - low_stock\nsystem: |\n Mind the gap.\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("loading: %v", err)
|
||||
}
|
||||
system := agents["one"].System
|
||||
if !strings.Contains(system, "Mind the gap.") {
|
||||
t.Fatal("the agent's own words were dropped")
|
||||
}
|
||||
if !strings.Contains(system, "never estimate") {
|
||||
t.Fatal("the agent's own words replaced the shared rules")
|
||||
}
|
||||
}
|
||||
|
||||
/* ── What must not load quietly ────────────────────────────────────────── */
|
||||
|
||||
func TestATypoInAToolNameStopsTheServer(t *testing.T) {
|
||||
// Invisible at runtime otherwise: the agent never calls it, the model says
|
||||
// it cannot look something up, and everything reports healthy.
|
||||
_, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "name: one\ntools:\n - stcuk_orders\n",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("an agent naming a tool that does not exist was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "stcuk_orders") {
|
||||
t.Fatalf("the error does not name the typo: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentWithNoToolsIsRefused(t *testing.T) {
|
||||
// It could only answer from its prompt, which is the one thing this
|
||||
// assistant is built not to do.
|
||||
if _, err := writeAgents(t, map[string]string{"one.yaml": "name: one\ntier: fast\n"}); err == nil {
|
||||
t.Fatal("an agent with no tools was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownTierIsRefusedRatherThanFallingBack(t *testing.T) {
|
||||
// The gateway falls back to the balanced model for anything it does not
|
||||
// recognise, so a "deep" agent with a typo would run on the cheap model for
|
||||
// months with nothing to show for it.
|
||||
if _, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "name: one\ntier: thorough\ntools:\n - low_stock\n",
|
||||
}); err == nil {
|
||||
t.Fatal("an unknown tier was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFileThatDoesNotNameItsAgentIsRefused(t *testing.T) {
|
||||
// Named by contents, not filename: a file renamed on deploy would otherwise
|
||||
// silently become a different agent.
|
||||
if _, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "tier: fast\ntools:\n - low_stock\n",
|
||||
}); err == nil {
|
||||
t.Fatal("a file with no agent name was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoAgentsCannotShareAName(t *testing.T) {
|
||||
// One of them is being ignored, and which depends on directory order.
|
||||
_, err := writeAgents(t, map[string]string{
|
||||
"a.yaml": "name: same\ntools:\n - low_stock\n",
|
||||
"b.yaml": "name: same\ntools:\n - till_status\n",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("two agents took the same name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrokenYAMLIsRefusedWithTheFileNamed(t *testing.T) {
|
||||
_, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "name: one\n tools: [\n",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("unparseable YAML was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "one.yaml") {
|
||||
t.Fatalf("the error does not say which file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyDirectoryIsRefusedNotTreatedAsNoAgents(t *testing.T) {
|
||||
// Zero agents means every question answers "no assistant called that",
|
||||
// which reads as a bug in the console rather than a missing config.
|
||||
if _, err := LoadAgents(t.TempDir(), knownTool); err == nil {
|
||||
t.Fatal("an empty agents directory was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingDirectoryIsAnError(t *testing.T) {
|
||||
if _, err := LoadAgents(filepath.Join(t.TempDir(), "nope"), knownTool); err == nil {
|
||||
t.Fatal("a missing agents directory was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Defaults ──────────────────────────────────────────────────────────── */
|
||||
|
||||
func TestAnAgentThatNamesNoLimitsGetsSensibleOnes(t *testing.T) {
|
||||
// Zero would mean a loop that runs no steps at all and answers nothing.
|
||||
agents, err := writeAgents(t, map[string]string{
|
||||
"one.yaml": "name: one\ntools:\n - low_stock\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("loading: %v", err)
|
||||
}
|
||||
agent := agents["one"]
|
||||
if agent.MaxSteps <= 0 || agent.MaxToolCalls <= 0 {
|
||||
t.Fatalf("limits are unusable: %+v", agent)
|
||||
}
|
||||
if agent.Tier != utils.TierBalanced {
|
||||
t.Fatalf("an agent with no tier got %q", agent.Tier)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDiskDirectoryReplacesTheEmbeddedAgentsRatherThanMerging(t *testing.T) {
|
||||
// Merging would leave a deployment guessing which half of an agent it is
|
||||
// running.
|
||||
agents, err := writeAgents(t, map[string]string{
|
||||
"only.yaml": "name: only\ntools:\n - low_stock\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("loading: %v", err)
|
||||
}
|
||||
if len(agents) != 1 {
|
||||
t.Fatalf("the embedded agents leaked in: %v", AgentNames(agents))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentNamesAreSortedSoTwoDeploymentsLogTheSameLine(t *testing.T) {
|
||||
names := AgentNames(map[string]Agent{"zulu": {}, "alpha": {}, "mike": {}})
|
||||
if names[0] != "alpha" || names[2] != "zulu" {
|
||||
t.Fatalf("not sorted: %v", names)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user