Return the passages an answer was given, so a claim can be checked
The ids always travelled TO the model; nothing a reader could look at ever travelled back. So the panel stripped the citations the model wrote — there was nowhere to put them — and a grounded answer became indistinguishable from an invented one, which is the opposite of what citing is for. A run now carries its sources: the id the model was told to cite, the document title, the heading, and the opening of the passage. Both the JSON and the streamed paths return them, because both build the same response. Source is NOT knowledge.Result. That type carries ranks, scores and the whole chunk, which exist to debug a retrieval rather than to be shown: an RRF score is a rank and would be read as a percentage, and the full text would make the response larger than the answer. This is the subset a citation needs. Snippets are capped at 240 runes. A reader checking "the policy says X" needs to recognise the passage, not to receive the corpus one answer at a time. A run that retrieved nothing carries nothing rather than an empty list, so the panel has no heading to draw for absent evidence — which is most runs, since seven of nine agents answer from tools. Next, and only now possible: the panel can render these and stop stripping the citations that point at them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/krow/krow-backend/go-api/internal/knowledge"
|
||||
|
||||
"github.com/krow/krow-backend/go-api/internal/authctx"
|
||||
"github.com/krow/krow-backend/go-api/internal/memory"
|
||||
)
|
||||
@@ -111,3 +113,73 @@ func TestWithoutAMemoryStoreNothingChanges(t *testing.T) {
|
||||
t.Errorf("the question was altered with no memory configured:\n%q", gw.lastReq.Messages[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Citable answers ─────────────────────────────────────────────────────── */
|
||||
|
||||
// Without this, a grounded answer and an invented one look identical to the
|
||||
// reader: the ids reach the model and nothing reaches the panel, so the
|
||||
// citations get stripped and the evidence disappears with them.
|
||||
func TestSourcesComeBackWithTheAnswer(t *testing.T) {
|
||||
gw := &fakeGateway{text: "The policy says shifts are offered for four hours."}
|
||||
exec := NewModelExecutor(gw, &MemorySink{}, nil).WithRetriever(stubRetriever{})
|
||||
|
||||
agent := testAgent()
|
||||
agent.KnowledgeSources = []string{"policy_docs"}
|
||||
|
||||
res, err := exec.ExecuteAgent(context.Background(), agent, testInput("how long is a shift offered?"))
|
||||
if err != nil {
|
||||
t.Fatalf("run failed: %v", err)
|
||||
}
|
||||
if len(res.Sources) == 0 {
|
||||
t.Fatal("the answer carries no sources, so no claim in it can be checked")
|
||||
}
|
||||
s := res.Sources[0]
|
||||
if s.ID == "" || s.Title == "" || s.Snippet == "" {
|
||||
t.Errorf("a source is missing what a reader needs: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// The snippet is a recognisable opening, not the corpus delivered one answer
|
||||
// at a time.
|
||||
func TestASourceSnippetIsBounded(t *testing.T) {
|
||||
gw := &fakeGateway{text: "answered"}
|
||||
exec := NewModelExecutor(gw, &MemorySink{}, nil).WithRetriever(stubRetriever{long: true})
|
||||
|
||||
agent := testAgent()
|
||||
agent.KnowledgeSources = []string{"policy_docs"}
|
||||
res, _ := exec.ExecuteAgent(context.Background(), agent, testInput("anything?"))
|
||||
|
||||
if len(res.Sources) == 0 {
|
||||
t.Fatal("no sources")
|
||||
}
|
||||
if n := len([]rune(res.Sources[0].Snippet)); n > 260 {
|
||||
t.Errorf("snippet is %d runes; the response is becoming the corpus", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A run that retrieved nothing says so by carrying nothing, rather than an
|
||||
// empty shell the panel would draw a heading for.
|
||||
func TestARunWithoutRetrievalCarriesNoSources(t *testing.T) {
|
||||
gw := &fakeGateway{text: "answered from tools"}
|
||||
exec := NewModelExecutor(gw, &MemorySink{}, nil)
|
||||
res, _ := exec.ExecuteAgent(context.Background(), testAgent(), testInput("how many open roles?"))
|
||||
if len(res.Sources) != 0 {
|
||||
t.Errorf("got %d sources with no retriever", len(res.Sources))
|
||||
}
|
||||
}
|
||||
|
||||
// stubRetriever returns one chunk, so the citation path can be exercised
|
||||
// without a corpus or an embedder.
|
||||
type stubRetriever struct{ long bool }
|
||||
|
||||
func (s stubRetriever) Retrieve(_ context.Context, _ knowledge.Query) (*knowledge.Results, error) {
|
||||
text := "Open shifts are offered to under-hour staff at the venue first, for four hours."
|
||||
if s.long {
|
||||
text = strings.Repeat("a long policy paragraph that goes on. ", 40)
|
||||
}
|
||||
return &knowledge.Results{Chunks: []knowledge.Result{{
|
||||
ChunkID: "chunk-1", DocumentID: "doc-1",
|
||||
Source: "policy_docs", Title: "Shift cover and cancellation",
|
||||
Heading: "Offering an open shift", Text: text,
|
||||
}}}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user