updates on the ai and agent and all thse things awith onboarding

internal/ai/playground/playground_test.go

@@ -0,0 +1,280 @@
+package playground
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+
+	"doormile/internal/ai/registry"
+	"doormile/models"
+)
+
+// fakeModel replays scripted replies and records every request.
+type fakeModel struct {
+	replies []Reply
+	err     error
+	reqs    []Request
+}
+
+func (f *fakeModel) Next(_ context.Context, req Request) (Reply, error) {
+	f.reqs = append(f.reqs, req)
+	if f.err != nil {
+		return Reply{}, f.err
+	}
+	if len(f.replies) == 0 {
+		return Reply{StopReason: "end_turn", Blocks: []Block{{Type: "text", Text: "done"}}}, nil
+	}
+	r := f.replies[0]
+	f.replies = f.replies[1:]
+	return r, nil
+}
+
+func toolCall(id, name, input string) Block {
+	return Block{Type: "tool_use", ToolUse: &ToolUse{ID: id, Name: name, Input: json.RawMessage(input)}}
+}
+
+func testSnapshot() *registry.Snapshot {
+	agents := []models.AIAgent{
+		{Agentid: "EXCEPTION_AGENT", Name: "Exception", Purpose: "Handles stalled riders.", Model: "claude-sonnet-5-5"},
+		{Agentid: "CONSOLE_OPS_AGENT", Name: "Console Ops Agent", Purpose: "Watches the board."},
+	}
+	tools := []models.AITool{
+		{Toolname: "get_booking_cache", Kind: "read", Description: "Read a booking.", Inputschema: `{"type":"object"}`},
+		{Toolname: "nearby_milers", Kind: "read", Description: "Riders near a point.", Inputschema: `{"type":"object"}`},
+		{Toolname: "decide_stall_response", Kind: "read", Description: "Engine decision.", Inputschema: `{"type":"object"}`},
+		{Toolname: "reassign_booking", Kind: "write", Description: "Reassign.", Inputschema: `{"type":"object"}`},
+		{Toolname: "scan_bookings", Kind: "read", Description: "Scan.", Inputschema: `{"type":"object"}`},
+	}
+	skills := []models.AISkill{
+		{Skillid: "stall_response", Agentid: "EXCEPTION_AGENT", Title: "Stall", Description: "Respond to stalls.", Enabled: true},
+		{Skillid: "off_skill", Agentid: "EXCEPTION_AGENT", Title: "Off", Enabled: false},
+		{Skillid: "late_dispatch", Agentid: "CONSOLE_OPS_AGENT", Title: "Late", Enabled: true},
+	}
+	links := []models.AISkillTool{
+		{Skillid: "stall_response", Toolname: "get_booking_cache"},
+		{Skillid: "stall_response", Toolname: "nearby_milers"},
+		{Skillid: "stall_response", Toolname: "decide_stall_response"},
+		{Skillid: "stall_response", Toolname: "reassign_booking"},
+		{Skillid: "off_skill", Toolname: "scan_bookings"},
+		{Skillid: "late_dispatch", Toolname: "scan_bookings"},
+	}
+	return registry.Build(agents, tools, skills, links)
+}
+
+func mustPlan(t *testing.T, agent, skill string) *Plan {
+	t.Helper()
+	p, err := Prepare(testSnapshot(), agent, skill)
+	if err != nil {
+		t.Fatalf("Prepare: %v", err)
+	}
+	return p
+}
+
+func toolNames(p *Plan) []string {
+	var out []string
+	for _, t := range p.Tools {
+		out = append(out, t.Name)
+	}
+	return out
+}
+
+// ── Prepare ─────────────────────────────────────────────────────────────────
+
+func TestPrepareUsesSkillToolsAndPinnedModel(t *testing.T) {
+	p := mustPlan(t, "EXCEPTION_AGENT", "stall_response")
+	if p.Model != "claude-sonnet-5-5" {
+		t.Fatalf("model = %q, want the registry pin", p.Model)
+	}
+	got := strings.Join(toolNames(p), ",")
+	// Registry order (Load sorts by toolname; this fixture is in its own order).
+	if got != "get_booking_cache,nearby_milers,decide_stall_response,reassign_booking" {
+		t.Fatalf("tools = %s", got)
+	}
+	for _, td := range p.Tools {
+		marked := strings.Contains(td.Description, "NOT executed")
+		if (td.Name == "reassign_booking") != marked {
+			t.Fatalf("%s: write-tool marking = %v", td.Name, marked)
+		}
+	}
+}
+
+func TestPrepareDefaultsModelAndSkipsDisabledSkillsWhenNoSkillGiven(t *testing.T) {
+	p := mustPlan(t, "CONSOLE_OPS_AGENT", "")
+	if p.Model != DefaultModel {
+		t.Fatalf("model = %q, want %q", p.Model, DefaultModel)
+	}
+	p = mustPlan(t, "EXCEPTION_AGENT", "")
+	for _, n := range toolNames(p) {
+		if n == "scan_bookings" {
+			t.Fatal("a disabled skill's tool was offered")
+		}
+	}
+}
+
+func TestPrepareNotFound(t *testing.T) {
+	for _, c := range [][2]string{{"NOPE", ""}, {"EXCEPTION_AGENT", "late_dispatch"}, {"EXCEPTION_AGENT", "missing"}} {
+		if _, err := Prepare(testSnapshot(), c[0], c[1]); !errors.Is(err, ErrNotFound) {
+			t.Fatalf("%v: err = %v, want ErrNotFound", c, err)
+		}
+	}
+}
+
+// ── Run ─────────────────────────────────────────────────────────────────────
+
+func TestRunTextOnly(t *testing.T) {
+	m := &fakeModel{replies: []Reply{{StopReason: "end_turn", InputTokens: 10, OutputTokens: 5,
+		Blocks: []Block{{Type: "thinking", Raw: json.RawMessage(`{"type":"thinking"}`)}, {Type: "text", Text: "All clear."}}}}}
+	tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "status?", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tr.Final != "All clear." || tr.Turns != 1 || tr.Inputtokens != 10 || tr.Outputtokens != 5 || tr.Model != "claude-sonnet-5-5" {
+		t.Fatalf("trace = %+v", tr)
+	}
+	if len(tr.Steps) != 1 || tr.Steps[0].Kind != "text" {
+		t.Fatalf("steps = %+v", tr.Steps)
+	}
+	req := m.reqs[0]
+	if req.Turns[0].Text != "status?" || req.MaxTokens != MaxTokens || len(req.Tools) != 4 || req.System == "" {
+		t.Fatalf("request = %+v", req)
+	}
+}
+
+func TestRunToolOutcomes(t *testing.T) {
+	executed := 0
+	execs := map[string]Executor{
+		"get_booking_cache": func(_ context.Context, in json.RawMessage) (any, error) {
+			executed++
+			return map[string]any{"bookingid": 5, "customername": "Ravi", "notes": "call 9876543210"}, nil
+		},
+		// Present but NOT in the skill — must never run.
+		"scan_bookings": func(context.Context, json.RawMessage) (any, error) {
+			t.Fatal("a tool outside the skill was executed")
+			return nil, nil
+		},
+		"nearby_milers": func(context.Context, json.RawMessage) (any, error) { return nil, errors.New("positions unavailable") },
+	}
+	m := &fakeModel{replies: []Reply{
+		{StopReason: "tool_use", Blocks: []Block{
+			toolCall("t1", "get_booking_cache", `{"booking_id":5}`),
+			toolCall("t2", "reassign_booking", `{"booking_id":5}`),
+			toolCall("t3", "scan_bookings", `{}`),
+			toolCall("t4", "decide_stall_response", `{}`),
+			toolCall("t5", "nearby_milers", `{"lat":11,"lon":77}`),
+		}},
+		{StopReason: "end_turn", Blocks: []Block{{Type: "text", Text: "Proposed a reassign."}}},
+	}}
+	tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "booking 5 is stuck", execs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := map[string]string{
+		"get_booking_cache": OutcomeExecuted, "reassign_booking": OutcomeProposed, "scan_bookings": OutcomeRejected,
+		"decide_stall_response": OutcomeUnavailable, "nearby_milers": OutcomeError,
+	}
+	for _, s := range tr.Steps {
+		if s.Kind == "tool" && want[s.Tool] != s.Outcome {
+			t.Fatalf("%s outcome = %s, want %s", s.Tool, s.Outcome, want[s.Tool])
+		}
+	}
+	if executed != 1 {
+		t.Fatalf("read tool executed %d times", executed)
+	}
+
+	// The second request carries all five results, in order, and redacted.
+	results := m.reqs[1].Turns[2].Results
+	if len(results) != 5 || results[0].ToolUseID != "t1" {
+		t.Fatalf("results = %+v", results)
+	}
+	if strings.Contains(results[0].Content, "Ravi") || strings.Contains(results[0].Content, "9876543210") {
+		t.Fatalf("personal data reached the model: %s", results[0].Content)
+	}
+	if !strings.Contains(results[1].Content, `"executed":false`) || results[1].IsError {
+		t.Fatalf("write tool result = %+v", results[1])
+	}
+	for _, i := range []int{2, 3, 4} {
+		if !results[i].IsError {
+			t.Fatalf("result %d should be an error: %+v", i, results[i])
+		}
+	}
+	// The assistant turn (with its tool_use blocks) is echoed back before the results.
+	if m.reqs[1].Turns[1].Role != "assistant" || len(m.reqs[1].Turns[1].Assistant) != 5 {
+		t.Fatalf("assistant turn not echoed: %+v", m.reqs[1].Turns[1])
+	}
+	if tr.Final != "Proposed a reassign." || tr.Turns != 2 {
+		t.Fatalf("trace = %+v", tr)
+	}
+}
+
+func TestRunStopsAtMaxTurns(t *testing.T) {
+	var replies []Reply
+	for i := 0; i < MaxTurns+2; i++ {
+		replies = append(replies, Reply{StopReason: "tool_use", Blocks: []Block{toolCall("t", "reassign_booking", `{}`)}})
+	}
+	m := &fakeModel{replies: replies}
+	tr, err := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "loop", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tr.Turns != MaxTurns || tr.Stopreason != "max_turns" || len(m.reqs) != MaxTurns {
+		t.Fatalf("turns = %d, stop = %s, calls = %d", tr.Turns, tr.Stopreason, len(m.reqs))
+	}
+}
+
+func TestRunModelError(t *testing.T) {
+	tr, err := Run(context.Background(), &fakeModel{err: errors.New("overloaded")}, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "x", nil)
+	if err == nil || tr == nil || tr.Stopreason != "error" {
+		t.Fatalf("err = %v, trace = %+v", err, tr)
+	}
+}
+
+func TestInvalidToolInputBecomesEmptyObject(t *testing.T) {
+	m := &fakeModel{replies: []Reply{{StopReason: "tool_use", Blocks: []Block{toolCall("t", "reassign_booking", `{not json`)}}}}
+	tr, _ := Run(context.Background(), m, mustPlan(t, "EXCEPTION_AGENT", "stall_response"), "x", nil)
+	if string(tr.Steps[0].Input) != "{}" {
+		t.Fatalf("input = %s", tr.Steps[0].Input)
+	}
+}
+
+func TestEncodeCapsLargeResults(t *testing.T) {
+	b := encode(map[string]string{"blob": strings.Repeat("x", maxResultBytes*2)})
+	if len(b) > maxResultBytes+1024 || !strings.Contains(string(b), `"truncated":true`) {
+		t.Fatalf("len = %d", len(b))
+	}
+}
+
+// ── Redact ──────────────────────────────────────────────────────────────────
+
+func TestRedact(t *testing.T) {
+	in := map[string]any{
+		"bookingid":     7,
+		"customerName":  "Ravi",
+		"pickupaddress": "12 MG Road",
+		"status":        "Created",
+		"createdat_ist": "2026-09-29 12:30",
+		"deliverycity":  "Coimbatore",
+		"cancelreason":  "customer asked",
+		"nested":        []any{map[string]any{"phone": "9876543210", "hub": "call +91 98765 43210 or a@b.co"}},
+		"missingnote":   nil,
+	}
+	out := Redact(in).(map[string]any)
+	for _, k := range []string{"customerName", "pickupaddress", "cancelreason"} {
+		if out[k] != Redacted {
+			t.Fatalf("%s = %v, want redacted", k, out[k])
+		}
+	}
+	for k, want := range map[string]any{"status": "Created", "createdat_ist": "2026-09-29 12:30", "deliverycity": "Coimbatore", "bookingid": float64(7)} {
+		if out[k] != want {
+			t.Fatalf("%s = %v, want %v (must not be redacted)", k, out[k], want)
+		}
+	}
+	nested := out["nested"].([]any)[0].(map[string]any)
+	if nested["phone"] != Redacted || strings.ContainsAny(nested["hub"].(string), "@") || strings.Contains(nested["hub"].(string), "98765") {
+		t.Fatalf("nested = %v", nested)
+	}
+	if out["missingnote"] != nil {
+		t.Fatal("a null personal field should stay null")
+	}
+}