krow_backend/go-api/internal/runtime/recall_test.go

package runtime

import (
	"context"
	"errors"
	"strings"
	"testing"

	"github.com/krow/krow-backend/go-api/internal/authctx"
	"github.com/krow/krow-backend/go-api/internal/memory"
)

type fakeMemory struct {
	records  []memory.Record
	degraded string
	err      error
	asked    string
}

func (f *fakeMemory) Recall(_ context.Context, _ authctx.Identity, question string, _ int) ([]memory.Record, string, error) {
	f.asked = question
	return f.records, f.degraded, f.err
}

func TestARecalledMemoryReachesThePrompt(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "Thursdays are short-staffed."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)

	if _, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")); err != nil {
		t.Fatalf("run failed: %v", err)
	}
	sent := gw.lastReq.Messages[0].Text
	if !strings.Contains(sent, "Thursdays are short-staffed.") {
		t.Errorf("the memory did not reach the model:\n%s", sent)
	}
	if !strings.Contains(sent, "<memory>") {
		t.Error("the memory was not fenced")
	}
}

// The question goes last. A model reads the last thing and answers it; put the
// evidence after it and the evidence becomes the prompt.
func TestTheQuestionStaysLastWhenMemoryIsCarried(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))

	sent := gw.lastReq.Messages[0].Text
	if !strings.HasSuffix(strings.TrimSpace(sent), "who is free?") {
		t.Errorf("the question is not last:\n%s", sent)
	}
}

// A store that is down must not take the run with it: an answer without
// memory is worse, not wrong.
func TestAMemoryFailureDoesNotFailTheRun(t *testing.T) {
	gw := &fakeGateway{text: "answered anyway"}
	mem := &fakeMemory{err: errors.New("the memory store is unreachable")}
	sink := &MemorySink{}
	exec := NewModelExecutor(gw, sink, nil).WithMemory(mem)

	res, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))
	if err != nil {
		t.Fatalf("a memory failure took the run with it: %v", err)
	}
	if res.Termination != TerminationCompleted {
		t.Errorf("Termination = %q, want Completed", res.Termination)
	}

	// ...and it is recorded, so a thin answer is explainable afterwards.
	var noted bool
	for _, e := range sink.Last().Entries {
		if strings.Contains(e.Name, "memory") || strings.Contains(e.Text, "memory store") {
			noted = true
		}
	}
	if !noted {
		t.Error("the memory failure left no trace in the trajectory")
	}
}

// Greetings skip memory for the same reason they skip retrieval: nobody needs
// remembering to say good morning, and paying for it is how "hi" came to cost
// six thousand tokens.
func TestSmalltalkCarriesNoMemory(t *testing.T) {
	gw := &fakeGateway{text: "Good morning."}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("good morning"))

	if strings.Contains(gw.lastReq.Messages[0].Text, "<memory>") {
		t.Errorf("a greeting carried memory:\n%s", gw.lastReq.Messages[0].Text)
	}
}

// With no store the loop is exactly what it was.
func TestWithoutAMemoryStoreNothingChanges(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	exec := NewModelExecutor(gw, &MemorySink{}, nil)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))

	if gw.lastReq.Messages[0].Text != "who is free?" {
		t.Errorf("the question was altered with no memory configured:\n%q", gw.lastReq.Messages[0].Text)
	}
}