package runtime import ( "context" "errors" "strings" "testing" "github.com/krow/krow-backend/go-api/internal/authctx" "github.com/krow/krow-backend/go-api/internal/memory" ) type fakeMemory struct { records []memory.Record degraded string err error asked string } func (f *fakeMemory) Recall(_ context.Context, _ authctx.Identity, question string, _ int) ([]memory.Record, string, error) { f.asked = question return f.records, f.degraded, f.err } func TestARecalledMemoryReachesThePrompt(t *testing.T) { gw := &fakeGateway{text: "answered"} mem := &fakeMemory{records: []memory.Record{ {SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "Thursdays are short-staffed."}, }} exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem) if _, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")); err != nil { t.Fatalf("run failed: %v", err) } sent := gw.lastReq.Messages[0].Text if !strings.Contains(sent, "Thursdays are short-staffed.") { t.Errorf("the memory did not reach the model:\n%s", sent) } if !strings.Contains(sent, "") { t.Error("the memory was not fenced") } } // The question goes last. A model reads the last thing and answers it; put the // evidence after it and the evidence becomes the prompt. func TestTheQuestionStaysLastWhenMemoryIsCarried(t *testing.T) { gw := &fakeGateway{text: "answered"} mem := &fakeMemory{records: []memory.Record{ {SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."}, }} exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem) exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")) sent := gw.lastReq.Messages[0].Text if !strings.HasSuffix(strings.TrimSpace(sent), "who is free?") { t.Errorf("the question is not last:\n%s", sent) } } // A store that is down must not take the run with it: an answer without // memory is worse, not wrong. func TestAMemoryFailureDoesNotFailTheRun(t *testing.T) { gw := &fakeGateway{text: "answered anyway"} mem := &fakeMemory{err: errors.New("the memory store is unreachable")} sink := &MemorySink{} exec := NewModelExecutor(gw, sink, nil).WithMemory(mem) res, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")) if err != nil { t.Fatalf("a memory failure took the run with it: %v", err) } if res.Termination != TerminationCompleted { t.Errorf("Termination = %q, want Completed", res.Termination) } // ...and it is recorded, so a thin answer is explainable afterwards. var noted bool for _, e := range sink.Last().Entries { if strings.Contains(e.Name, "memory") || strings.Contains(e.Text, "memory store") { noted = true } } if !noted { t.Error("the memory failure left no trace in the trajectory") } } // Greetings skip memory for the same reason they skip retrieval: nobody needs // remembering to say good morning, and paying for it is how "hi" came to cost // six thousand tokens. func TestSmalltalkCarriesNoMemory(t *testing.T) { gw := &fakeGateway{text: "Good morning."} mem := &fakeMemory{records: []memory.Record{ {SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."}, }} exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem) exec.ExecuteAgent(context.Background(), testAgent(), testInput("good morning")) if strings.Contains(gw.lastReq.Messages[0].Text, "") { t.Errorf("a greeting carried memory:\n%s", gw.lastReq.Messages[0].Text) } } // With no store the loop is exactly what it was. func TestWithoutAMemoryStoreNothingChanges(t *testing.T) { gw := &fakeGateway{text: "answered"} exec := NewModelExecutor(gw, &MemorySink{}, nil) exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")) if gw.lastReq.Messages[0].Text != "who is free?" { t.Errorf("the question was altered with no memory configured:\n%q", gw.lastReq.Messages[0].Text) } }