281 lines
11 KiB
Go
281 lines
11 KiB
Go
package playground
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"doormile/internal/ai/registry"
|
|
"doormile/models"
|
|
)
|
|
|
|
// fakeModel replays scripted replies and records every request.
|
|
type fakeModel struct {
|
|
replies []Reply
|
|
err error
|
|
reqs []Request
|
|
}
|
|
|
|
func (f *fakeModel) Next(_ context.Context, req Request) (Reply, error) {
|
|
f.reqs = append(f.reqs, req)
|
|
if f.err != nil {
|
|
return Reply{}, f.err
|
|
}
|
|
if len(f.replies) == 0 {
|
|
return Reply{StopReason: "end_turn", Blocks: []Block{{Type: "text", Text: "done"}}}, nil
|
|
}
|
|
r := f.replies[0]
|
|
f.replies = f.replies[1:]
|
|
return r, nil
|
|
}
|
|
|
|
func toolCall(id, name, input string) Block {
|
|
return Block{Type: "tool_use", ToolUse: &ToolUse{ID: id, Name: name, Input: json.RawMessage(input)}}
|
|
}
|
|
|
|
func testSnapshot() *registry.Snapshot {
|
|
agents := []models.AIAgent{
|
|
{Agentid: "EXCEPTION_AGENT", Name: "Exception", Purpose: "Handles stalled riders.", Model: "claude-sonnet-5-5"},
|
|
{Agentid: "CONSOLE_OPS_AGENT", Name: "Console Ops Agent", Purpose: "Watches the board."},
|
|
}
|
|
tools := []models.AITool{
|
|
{Toolname: "get_booking_cache", Kind: "read", Description: "Read a booking.", Inputschema: `{"type":"object"}`},
|
|
{Toolname: "nearby_milers", Kind: "read", Description: "Riders near a point.", Inputschema: `{"type":"object"}`},
|
|
{Toolname: "decide_stall_response", Kind: "read", Description: "Engine decision.", Inputschema: `{"type":"object"}`},
|
|
{Toolname: "reassign_booking", Kind: "write", Description: "Reassign.", Inputschema: `{"type":"object"}`},
|
|
{Toolname: "scan_bookings", Kind: "read", Description: "Scan.", Inputschema: `{"type":"object"}`},
|
|
}
|
|
skills := []models.AISkill{
|
|
{Skillid: "stall_response", Agentid: "EXCEPTION_AGENT", Title: "Stall", Description: "Respond to stalls.", Enabled: true},
|
|
{Skillid: "off_skill", Agentid: "EXCEPTION_AGENT", Title: "Off", Enabled: false},
|
|
{Skillid: "late_dispatch", Agentid: "CONSOLE_OPS_AGENT", Title: "Late", Enabled: true},
|
|
}
|
|
links := []models.AISkillTool{
|
|
{Skillid: "stall_response", Toolname: "get_booking_cache"},
|
|
{Skillid: "stall_response", Toolname: "nearby_milers"},
|
|
{Skillid: "stall_response", Toolname: "decide_stall_response"},
|
|
{Skillid: "stall_response", Toolname: "reassign_booking"},
|
|
{Skillid: "off_skill", Toolname: "scan_bookings"},
|
|
{Skillid: "late_dispatch", Toolname: "scan_bookings"},
|
|
}
|
|
return registry.Build(agents, tools, skills, links)
|
|
}
|
|
|
|
func mustPlan(t *testing.T, agent, skill string) *Plan {
|
|
t.Helper()
|
|
p, err := Prepare(testSnapshot(), agent, skill)
|
|
if err != nil {
|
|
t.Fatalf("Prepare: %v", err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func toolNames(p *Plan) []string {
|
|
var out []string
|
|
for _, t := range p.Tools {
|
|
out = append(out, t.Name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ── Prepare ─────────────────────────────────────────────────────────────────
|
|
|
|
func TestPrepareUsesSkillToolsAndPinnedModel(t *testing.T) {
|
|
p := mustPlan(t, "EXCEPTION_AGENT", "stall_response")
|
|
if p.Model != "claude-sonnet-5-5" {
|
|
t.Fatalf("model = %q, want the registry pin", p.Model)
|
|
}
|
|
got := strings.Join(toolNames(p), ",")
|
|
// Registry order (Load sorts by toolname; this fixture is in its own order).
|
|
if got != "get_booking_cache,nearby_milers,decide_stall_response,reassign_booking" {
|
|
t.Fatalf("tools = %s", got)
|
|
}
|
|
for _, td := range p.Tools {
|
|
marked := strings.Contains(td.Description, "NOT executed")
|
|
if (td.Name == "reassign_booking") != marked {
|
|
t.Fatalf("%s: write-tool marking = %v", td.Name, marked)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPrepareDefaultsModelAndSkipsDisabledSkillsWhenNoSkillGiven(t *testing.T) {
|
|
p := mustPlan(t, "CONSOLE_OPS_AGENT", "")
|
|
if p.Model != DefaultModel {
|
|
t.Fatalf("model = %q, want %q", p.Model, DefaultModel)
|
|
}
|
|
p = mustPlan(t, "EXCEPTION_AGENT", "")
|
|
for _, n := range toolNames(p) {
|
|
if n == "scan_bookings" {
|
|
t.Fatal("a disabled skill's tool was offered")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPrepareNotFound(t *testing.T) {
|
|
for _, c := range [][2]string{{"NOPE", ""}, {"EXCEPTION_AGENT", "late_dispatch"}, {"EXCEPTION_AGENT", "missing"}} {
|
|
if _, err := Prepare(testSnapshot(), c[0], c[1]); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("%v: err = %v, want ErrNotFound", c, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Run ─────────────────────────────────────────────────────────────────────
|
|
|
|
func TestRunTextOnly(t *testing.T) {
|
|
m := &fakeModel{replies: []Reply{{StopReason: "end_turn", InputTokens: 10, OutputTokens: 5,
|
|
Blocks: []Block{{Type: "thinking", Raw: json.RawMessage(`{"type":"thinking"}`)}, {Type: "text", Text: "All clear."}}}}}
|
|
tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "status?", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tr.Final != "All clear." || tr.Turns != 1 || tr.Inputtokens != 10 || tr.Outputtokens != 5 || tr.Model != "claude-sonnet-5-5" {
|
|
t.Fatalf("trace = %+v", tr)
|
|
}
|
|
if len(tr.Steps) != 1 || tr.Steps[0].Kind != "text" {
|
|
t.Fatalf("steps = %+v", tr.Steps)
|
|
}
|
|
req := m.reqs[0]
|
|
if req.Turns[0].Text != "status?" || req.MaxTokens != MaxTokens || len(req.Tools) != 4 || req.System == "" {
|
|
t.Fatalf("request = %+v", req)
|
|
}
|
|
}
|
|
|
|
func TestRunToolOutcomes(t *testing.T) {
|
|
executed := 0
|
|
execs := map[string]Executor{
|
|
"get_booking_cache": func(_ context.Context, in json.RawMessage) (any, error) {
|
|
executed++
|
|
return map[string]any{"bookingid": 5, "customername": "Ravi", "notes": "call 9876543210"}, nil
|
|
},
|
|
// Present but NOT in the skill — must never run.
|
|
"scan_bookings": func(context.Context, json.RawMessage) (any, error) {
|
|
t.Fatal("a tool outside the skill was executed")
|
|
return nil, nil
|
|
},
|
|
"nearby_milers": func(context.Context, json.RawMessage) (any, error) { return nil, errors.New("positions unavailable") },
|
|
}
|
|
m := &fakeModel{replies: []Reply{
|
|
{StopReason: "tool_use", Blocks: []Block{
|
|
toolCall("t1", "get_booking_cache", `{"booking_id":5}`),
|
|
toolCall("t2", "reassign_booking", `{"booking_id":5}`),
|
|
toolCall("t3", "scan_bookings", `{}`),
|
|
toolCall("t4", "decide_stall_response", `{}`),
|
|
toolCall("t5", "nearby_milers", `{"lat":11,"lon":77}`),
|
|
}},
|
|
{StopReason: "end_turn", Blocks: []Block{{Type: "text", Text: "Proposed a reassign."}}},
|
|
}}
|
|
tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "booking 5 is stuck", execs)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := map[string]string{
|
|
"get_booking_cache": OutcomeExecuted, "reassign_booking": OutcomeProposed, "scan_bookings": OutcomeRejected,
|
|
"decide_stall_response": OutcomeUnavailable, "nearby_milers": OutcomeError,
|
|
}
|
|
for _, s := range tr.Steps {
|
|
if s.Kind == "tool" && want[s.Tool] != s.Outcome {
|
|
t.Fatalf("%s outcome = %s, want %s", s.Tool, s.Outcome, want[s.Tool])
|
|
}
|
|
}
|
|
if executed != 1 {
|
|
t.Fatalf("read tool executed %d times", executed)
|
|
}
|
|
|
|
// The second request carries all five results, in order, and redacted.
|
|
results := m.reqs[1].Turns[2].Results
|
|
if len(results) != 5 || results[0].ToolUseID != "t1" {
|
|
t.Fatalf("results = %+v", results)
|
|
}
|
|
if strings.Contains(results[0].Content, "Ravi") || strings.Contains(results[0].Content, "9876543210") {
|
|
t.Fatalf("personal data reached the model: %s", results[0].Content)
|
|
}
|
|
if !strings.Contains(results[1].Content, `"executed":false`) || results[1].IsError {
|
|
t.Fatalf("write tool result = %+v", results[1])
|
|
}
|
|
for _, i := range []int{2, 3, 4} {
|
|
if !results[i].IsError {
|
|
t.Fatalf("result %d should be an error: %+v", i, results[i])
|
|
}
|
|
}
|
|
// The assistant turn (with its tool_use blocks) is echoed back before the results.
|
|
if m.reqs[1].Turns[1].Role != "assistant" || len(m.reqs[1].Turns[1].Assistant) != 5 {
|
|
t.Fatalf("assistant turn not echoed: %+v", m.reqs[1].Turns[1])
|
|
}
|
|
if tr.Final != "Proposed a reassign." || tr.Turns != 2 {
|
|
t.Fatalf("trace = %+v", tr)
|
|
}
|
|
}
|
|
|
|
func TestRunStopsAtMaxTurns(t *testing.T) {
|
|
var replies []Reply
|
|
for i := 0; i < MaxTurns+2; i++ {
|
|
replies = append(replies, Reply{StopReason: "tool_use", Blocks: []Block{toolCall("t", "reassign_booking", `{}`)}})
|
|
}
|
|
m := &fakeModel{replies: replies}
|
|
tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "loop", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tr.Turns != MaxTurns || tr.Stopreason != "max_turns" || len(m.reqs) != MaxTurns {
|
|
t.Fatalf("turns = %d, stop = %s, calls = %d", tr.Turns, tr.Stopreason, len(m.reqs))
|
|
}
|
|
}
|
|
|
|
func TestRunModelError(t *testing.T) {
|
|
tr, err := Run(context.Background(), &fakeModel{err: errors.New("overloaded")}, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "x", nil)
|
|
if err == nil || tr == nil || tr.Stopreason != "error" {
|
|
t.Fatalf("err = %v, trace = %+v", err, tr)
|
|
}
|
|
}
|
|
|
|
func TestInvalidToolInputBecomesEmptyObject(t *testing.T) {
|
|
m := &fakeModel{replies: []Reply{{StopReason: "tool_use", Blocks: []Block{toolCall("t", "reassign_booking", `{not json`)}}}}
|
|
tr, _ := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "x", nil)
|
|
if string(tr.Steps[0].Input) != "{}" {
|
|
t.Fatalf("input = %s", tr.Steps[0].Input)
|
|
}
|
|
}
|
|
|
|
func TestEncodeCapsLargeResults(t *testing.T) {
|
|
b := encode(map[string]string{"blob": strings.Repeat("x", maxResultBytes*2)})
|
|
if len(b) > maxResultBytes+1024 || !strings.Contains(string(b), `"truncated":true`) {
|
|
t.Fatalf("len = %d", len(b))
|
|
}
|
|
}
|
|
|
|
// ── Redact ──────────────────────────────────────────────────────────────────
|
|
|
|
func TestRedact(t *testing.T) {
|
|
in := map[string]any{
|
|
"bookingid": 7,
|
|
"customerName": "Ravi",
|
|
"pickupaddress": "12 MG Road",
|
|
"status": "Created",
|
|
"createdat_ist": "2026-09-29 12:30",
|
|
"deliverycity": "Coimbatore",
|
|
"cancelreason": "customer asked",
|
|
"nested": []any{map[string]any{"phone": "9876543210", "hub": "call +91 98765 43210 or a@b.co"}},
|
|
"missingnote": nil,
|
|
}
|
|
out := Redact(in).(map[string]any)
|
|
for _, k := range []string{"customerName", "pickupaddress", "cancelreason"} {
|
|
if out[k] != Redacted {
|
|
t.Fatalf("%s = %v, want redacted", k, out[k])
|
|
}
|
|
}
|
|
for k, want := range map[string]any{"status": "Created", "createdat_ist": "2026-09-29 12:30", "deliverycity": "Coimbatore", "bookingid": float64(7)} {
|
|
if out[k] != want {
|
|
t.Fatalf("%s = %v, want %v (must not be redacted)", k, out[k], want)
|
|
}
|
|
}
|
|
nested := out["nested"].([]any)[0].(map[string]any)
|
|
if nested["phone"] != Redacted || strings.ContainsAny(nested["hub"].(string), "@") || strings.Contains(nested["hub"].(string), "98765") {
|
|
t.Fatalf("nested = %v", nested)
|
|
}
|
|
if out["missingnote"] != nil {
|
|
t.Fatal("a null personal field should stay null")
|
|
}
|
|
}
|