krow_backend/go-api/internal/runtime/recall_test.go

package runtime

import (
	"context"
	"errors"
	"strings"
	"testing"

	"github.com/krow/krow-backend/go-api/internal/knowledge"

	"github.com/krow/krow-backend/go-api/internal/authctx"
	"github.com/krow/krow-backend/go-api/internal/memory"
)

type fakeMemory struct {
	records  []memory.Record
	degraded string
	err      error
	asked    string
}

func (f *fakeMemory) Recall(_ context.Context, _ authctx.Identity, question string, _ int) ([]memory.Record, string, error) {
	f.asked = question
	return f.records, f.degraded, f.err
}

func TestARecalledMemoryReachesThePrompt(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "Thursdays are short-staffed."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)

	if _, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?")); err != nil {
		t.Fatalf("run failed: %v", err)
	}
	sent := gw.lastReq.Messages[0].Text
	if !strings.Contains(sent, "Thursdays are short-staffed.") {
		t.Errorf("the memory did not reach the model:\n%s", sent)
	}
	if !strings.Contains(sent, "<memory>") {
		t.Error("the memory was not fenced")
	}
}

// The question goes last. A model reads the last thing and answers it; put the
// evidence after it and the evidence becomes the prompt.
func TestTheQuestionStaysLastWhenMemoryIsCarried(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))

	sent := gw.lastReq.Messages[0].Text
	if !strings.HasSuffix(strings.TrimSpace(sent), "who is free?") {
		t.Errorf("the question is not last:\n%s", sent)
	}
}

// A store that is down must not take the run with it: an answer without
// memory is worse, not wrong.
func TestAMemoryFailureDoesNotFailTheRun(t *testing.T) {
	gw := &fakeGateway{text: "answered anyway"}
	mem := &fakeMemory{err: errors.New("the memory store is unreachable")}
	sink := &MemorySink{}
	exec := NewModelExecutor(gw, sink, nil).WithMemory(mem)

	res, err := exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))
	if err != nil {
		t.Fatalf("a memory failure took the run with it: %v", err)
	}
	if res.Termination != TerminationCompleted {
		t.Errorf("Termination = %q, want Completed", res.Termination)
	}

	// ...and it is recorded, so a thin answer is explainable afterwards.
	var noted bool
	for _, e := range sink.Last().Entries {
		if strings.Contains(e.Name, "memory") || strings.Contains(e.Text, "memory store") {
			noted = true
		}
	}
	if !noted {
		t.Error("the memory failure left no trace in the trajectory")
	}
}

// Greetings skip memory for the same reason they skip retrieval: nobody needs
// remembering to say good morning, and paying for it is how "hi" came to cost
// six thousand tokens.
func TestSmalltalkCarriesNoMemory(t *testing.T) {
	gw := &fakeGateway{text: "Good morning."}
	mem := &fakeMemory{records: []memory.Record{
		{SubjectType: memory.SubjectWorkspace, Author: memory.AuthorModel, Text: "A remembered thing."},
	}}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithMemory(mem)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("good morning"))

	if strings.Contains(gw.lastReq.Messages[0].Text, "<memory>") {
		t.Errorf("a greeting carried memory:\n%s", gw.lastReq.Messages[0].Text)
	}
}

// With no store the loop is exactly what it was.
func TestWithoutAMemoryStoreNothingChanges(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	exec := NewModelExecutor(gw, &MemorySink{}, nil)
	exec.ExecuteAgent(context.Background(), testAgent(), testInput("who is free?"))

	if gw.lastReq.Messages[0].Text != "who is free?" {
		t.Errorf("the question was altered with no memory configured:\n%q", gw.lastReq.Messages[0].Text)
	}
}

/* ── Citable answers ─────────────────────────────────────────────────────── */

// Without this, a grounded answer and an invented one look identical to the
// reader: the ids reach the model and nothing reaches the panel, so the
// citations get stripped and the evidence disappears with them.
func TestSourcesComeBackWithTheAnswer(t *testing.T) {
	gw := &fakeGateway{text: "The policy says shifts are offered for four hours."}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithRetriever(stubRetriever{})

	agent := testAgent()
	agent.KnowledgeSources = []string{"policy_docs"}

	res, err := exec.ExecuteAgent(context.Background(), agent, testInput("how long is a shift offered?"))
	if err != nil {
		t.Fatalf("run failed: %v", err)
	}
	if len(res.Sources) == 0 {
		t.Fatal("the answer carries no sources, so no claim in it can be checked")
	}
	s := res.Sources[0]
	if s.ID == "" || s.Title == "" || s.Snippet == "" {
		t.Errorf("a source is missing what a reader needs: %+v", s)
	}
}

// The snippet is a recognisable opening, not the corpus delivered one answer
// at a time.
func TestASourceSnippetIsBounded(t *testing.T) {
	gw := &fakeGateway{text: "answered"}
	exec := NewModelExecutor(gw, &MemorySink{}, nil).WithRetriever(stubRetriever{long: true})

	agent := testAgent()
	agent.KnowledgeSources = []string{"policy_docs"}
	res, _ := exec.ExecuteAgent(context.Background(), agent, testInput("anything?"))

	if len(res.Sources) == 0 {
		t.Fatal("no sources")
	}
	if n := len([]rune(res.Sources[0].Snippet)); n > 260 {
		t.Errorf("snippet is %d runes; the response is becoming the corpus", n)
	}
}

// A run that retrieved nothing says so by carrying nothing, rather than an
// empty shell the panel would draw a heading for.
func TestARunWithoutRetrievalCarriesNoSources(t *testing.T) {
	gw := &fakeGateway{text: "answered from tools"}
	exec := NewModelExecutor(gw, &MemorySink{}, nil)
	res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("how many open roles?"))
	if len(res.Sources) != 0 {
		t.Errorf("got %d sources with no retriever", len(res.Sources))
	}
}

// stubRetriever returns one chunk, so the citation path can be exercised
// without a corpus or an embedder.
type stubRetriever struct{ long bool }

func (s stubRetriever) Retrieve(_ context.Context, _ knowledge.Query) (*knowledge.Results, error) {
	text := "Open shifts are offered to under-hour staff at the venue first, for four hours."
	if s.long {
		text = strings.Repeat("a long policy paragraph that goes on. ", 40)
	}
	return &knowledge.Results{Chunks: []knowledge.Result{{
		ChunkID: "chunk-1", DocumentID: "doc-1",
		Source: "policy_docs", Title: "Shift cover and cancellation",
		Heading: "Offering an open shift", Text: text,
	}}}, nil
}