265 lines
8.8 KiB
Go
265 lines
8.8 KiB
Go
package services
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"nearle/services/tools"
|
|
"nearle/utils"
|
|
)
|
|
|
|
// realRegistry builds every tool the server registers, exactly as the facade
|
|
// does — with nil services, because nothing here calls a handler.
|
|
//
|
|
// Replaces a hand-written list of tool names that went stale twice: once when
|
|
// the help corpus arrived and once when the first write did. Both times the
|
|
// shipped agents were correct and the TEST was wrong, which is the worst way
|
|
// round — a check that has to be remembered is a check that will not be.
|
|
func realRegistry(t *testing.T) *tools.Registry {
|
|
t.Helper()
|
|
|
|
corpus, err := tools.LoadHelp()
|
|
if err != nil {
|
|
t.Fatalf("loading the help corpus: %v", err)
|
|
}
|
|
|
|
r := tools.New(nil)
|
|
for _, tool := range []tools.Tool{
|
|
tools.StuckOrders(nil, nil),
|
|
tools.DeliveryProgress(nil),
|
|
tools.BranchPerformance(nil),
|
|
tools.PendingApprovals(nil, nil),
|
|
tools.LowStock(nil),
|
|
tools.TillsNotSyncing(nil),
|
|
tools.SalesByChannel(nil, nil, nil),
|
|
tools.Help(corpus),
|
|
tools.ApproveStockRequest(nil, nil),
|
|
} {
|
|
if err := r.Register(tool); err != nil {
|
|
t.Fatalf("registering %s: %v", tool.Name, err)
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// knownTool asks the real registry, so an agent naming a tool that does not
|
|
// exist fails here and not on somebody's deployment.
|
|
func knownTool(name string) bool { return liveRegistry.Has(name) }
|
|
|
|
// liveRegistry is built once, in TestMain, because knownTool has no *testing.T.
|
|
var liveRegistry *tools.Registry
|
|
|
|
func TestMain(m *testing.M) {
|
|
fake := &testing.T{}
|
|
liveRegistry = realRegistry(fake)
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
// writeAgents drops YAML into a temporary directory and loads it.
|
|
func writeAgents(t *testing.T, files map[string]string) (map[string]Agent, error) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
for name, body := range files {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
|
|
t.Fatalf("writing %s: %v", name, err)
|
|
}
|
|
}
|
|
return LoadAgents(dir, knownTool)
|
|
}
|
|
|
|
/* ── The agents that ship ──────────────────────────────────────────────── */
|
|
|
|
func TestTheEmbeddedAgentsLoad(t *testing.T) {
|
|
// The defaults are compiled in so a binary always has working agents and a
|
|
// deployment cannot be broken by a missing directory.
|
|
agents, err := LoadAgents("", knownTool)
|
|
if err != nil {
|
|
t.Fatalf("the shipped agents do not load: %v", err)
|
|
}
|
|
for _, want := range []string{"orders", "inventory", "shopfloor"} {
|
|
if _, ok := agents[want]; !ok {
|
|
t.Fatalf("%q is missing: %v", want, AgentNames(agents))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEveryShippedAgentNamesOnlyRealTools(t *testing.T) {
|
|
// The check that makes a typo a startup failure rather than an agent that
|
|
// silently cannot do one of the things it claims.
|
|
if _, err := LoadAgents("", knownTool); err != nil {
|
|
t.Fatalf("a shipped agent names a tool that does not exist: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAddingAnAgentTakesNoCodeChange(t *testing.T) {
|
|
// Phase 3's whole point, stated as a test.
|
|
agents, err := writeAgents(t, map[string]string{
|
|
"sixth.yaml": "name: sixth\ntier: fast\ntools:\n - low_stock\n",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("loading: %v", err)
|
|
}
|
|
if _, ok := agents["sixth"]; !ok {
|
|
t.Fatal("a new file did not become an agent")
|
|
}
|
|
}
|
|
|
|
/* ── The shared prompt ─────────────────────────────────────────────────── */
|
|
|
|
func TestEveryAgentInheritsTheRulesThatMustNotDrift(t *testing.T) {
|
|
// Repeating "say when a result was truncated" in three files is three
|
|
// chances for one of them to lose it.
|
|
agents, err := LoadAgents("", knownTool)
|
|
if err != nil {
|
|
t.Fatalf("loading: %v", err)
|
|
}
|
|
for name, agent := range agents {
|
|
if !strings.Contains(agent.System, "truncated") {
|
|
t.Fatalf("agent %q was not told about truncated results", name)
|
|
}
|
|
if !strings.Contains(agent.System, "never estimate") {
|
|
t.Fatalf("agent %q was not told to answer only from tools", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnAgentsOwnWordsAreAddedNotSubstituted(t *testing.T) {
|
|
agents, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "name: one\ntools:\n - low_stock\nsystem: |\n Mind the gap.\n",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("loading: %v", err)
|
|
}
|
|
system := agents["one"].System
|
|
if !strings.Contains(system, "Mind the gap.") {
|
|
t.Fatal("the agent's own words were dropped")
|
|
}
|
|
if !strings.Contains(system, "never estimate") {
|
|
t.Fatal("the agent's own words replaced the shared rules")
|
|
}
|
|
}
|
|
|
|
/* ── What must not load quietly ────────────────────────────────────────── */
|
|
|
|
func TestATypoInAToolNameStopsTheServer(t *testing.T) {
|
|
// Invisible at runtime otherwise: the agent never calls it, the model says
|
|
// it cannot look something up, and everything reports healthy.
|
|
_, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "name: one\ntools:\n - stcuk_orders\n",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("an agent naming a tool that does not exist was accepted")
|
|
}
|
|
if !strings.Contains(err.Error(), "stcuk_orders") {
|
|
t.Fatalf("the error does not name the typo: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnAgentWithNoToolsIsRefused(t *testing.T) {
|
|
// It could only answer from its prompt, which is the one thing this
|
|
// assistant is built not to do.
|
|
if _, err := writeAgents(t, map[string]string{"one.yaml": "name: one\ntier: fast\n"}); err == nil {
|
|
t.Fatal("an agent with no tools was accepted")
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownTierIsRefusedRatherThanFallingBack(t *testing.T) {
|
|
// The gateway falls back to the balanced model for anything it does not
|
|
// recognise, so a "deep" agent with a typo would run on the cheap model for
|
|
// months with nothing to show for it.
|
|
if _, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "name: one\ntier: thorough\ntools:\n - low_stock\n",
|
|
}); err == nil {
|
|
t.Fatal("an unknown tier was accepted")
|
|
}
|
|
}
|
|
|
|
func TestAFileThatDoesNotNameItsAgentIsRefused(t *testing.T) {
|
|
// Named by contents, not filename: a file renamed on deploy would otherwise
|
|
// silently become a different agent.
|
|
if _, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "tier: fast\ntools:\n - low_stock\n",
|
|
}); err == nil {
|
|
t.Fatal("a file with no agent name was accepted")
|
|
}
|
|
}
|
|
|
|
func TestTwoAgentsCannotShareAName(t *testing.T) {
|
|
// One of them is being ignored, and which depends on directory order.
|
|
_, err := writeAgents(t, map[string]string{
|
|
"a.yaml": "name: same\ntools:\n - low_stock\n",
|
|
"b.yaml": "name: same\ntools:\n - till_status\n",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("two agents took the same name")
|
|
}
|
|
}
|
|
|
|
func TestBrokenYAMLIsRefusedWithTheFileNamed(t *testing.T) {
|
|
_, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "name: one\n tools: [\n",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("unparseable YAML was accepted")
|
|
}
|
|
if !strings.Contains(err.Error(), "one.yaml") {
|
|
t.Fatalf("the error does not say which file: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyDirectoryIsRefusedNotTreatedAsNoAgents(t *testing.T) {
|
|
// Zero agents means every question answers "no assistant called that",
|
|
// which reads as a bug in the console rather than a missing config.
|
|
if _, err := LoadAgents(t.TempDir(), knownTool); err == nil {
|
|
t.Fatal("an empty agents directory was accepted")
|
|
}
|
|
}
|
|
|
|
func TestAMissingDirectoryIsAnError(t *testing.T) {
|
|
if _, err := LoadAgents(filepath.Join(t.TempDir(), "nope"), knownTool); err == nil {
|
|
t.Fatal("a missing agents directory was accepted")
|
|
}
|
|
}
|
|
|
|
/* ── Defaults ──────────────────────────────────────────────────────────── */
|
|
|
|
func TestAnAgentThatNamesNoLimitsGetsSensibleOnes(t *testing.T) {
|
|
// Zero would mean a loop that runs no steps at all and answers nothing.
|
|
agents, err := writeAgents(t, map[string]string{
|
|
"one.yaml": "name: one\ntools:\n - low_stock\n",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("loading: %v", err)
|
|
}
|
|
agent := agents["one"]
|
|
if agent.MaxSteps <= 0 || agent.MaxToolCalls <= 0 {
|
|
t.Fatalf("limits are unusable: %+v", agent)
|
|
}
|
|
if agent.Tier != utils.TierBalanced {
|
|
t.Fatalf("an agent with no tier got %q", agent.Tier)
|
|
}
|
|
}
|
|
|
|
func TestTheDiskDirectoryReplacesTheEmbeddedAgentsRatherThanMerging(t *testing.T) {
|
|
// Merging would leave a deployment guessing which half of an agent it is
|
|
// running.
|
|
agents, err := writeAgents(t, map[string]string{
|
|
"only.yaml": "name: only\ntools:\n - low_stock\n",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("loading: %v", err)
|
|
}
|
|
if len(agents) != 1 {
|
|
t.Fatalf("the embedded agents leaked in: %v", AgentNames(agents))
|
|
}
|
|
}
|
|
|
|
func TestAgentNamesAreSortedSoTwoDeploymentsLogTheSameLine(t *testing.T) {
|
|
names := AgentNames(map[string]Agent{"zulu": {}, "alpha": {}, "mike": {}})
|
|
if names[0] != "alpha" || names[2] != "zulu" {
|
|
t.Fatalf("not sorted: %v", names)
|
|
}
|
|
}
|