package knowledge_test import ( "context" "fmt" "strings" "testing" "github.com/krow/krow-backend/go-api/internal/authctx" "github.com/krow/krow-backend/go-api/internal/domain" "github.com/krow/krow-backend/go-api/internal/knowledge" "github.com/krow/krow-backend/go-api/internal/testutil" ) // The knowledge layer's tests are almost entirely about who can see what. // // Retrieval quality is deliberately NOT asserted here, and it would be dishonest // to try: these run on the lexical stand-in embedder, which hashes words into a // vector and is not semantic. A test claiming "'time off' retrieves the annual // leave paragraph" would pass or fail on word overlap and would tell you nothing // about the system with a real embedder in it. // // What IS testable without a credential, and what actually carries the // invariants, is everything else: that the permission predicate runs before // scoring, that a caller cannot reach another tenant's corpus, that ingest // refuses a document nobody can read, that fusion is deterministic, and that a // document cannot break out of its context block. Those hold or fail // identically whichever embedder is underneath. /* ── Fixtures ───────────────────────────────────────────────────────────── */ type corpus struct { orgID string admin authctx.Identity talent authctx.Identity other authctx.Identity // an admin in a different tenant } func freshOrg(t *testing.T, h *testutil.Harness, slug string) string { t.Helper() var id string if err := h.Pool.QueryRow(context.Background(), `INSERT INTO organizations (name, slug) VALUES ($1, $2) RETURNING id::text`, slug, slug).Scan(&id); err != nil { t.Fatalf("create org %s: %v", slug, err) } return id } // seedCorpus ingests four documents whose audiences differ, in two tenants. // // The shapes matter. Each document is reachable by exactly one interesting set // of callers, so a leak in any direction is a specific, nameable failure rather // than "a test went red". func seedCorpus(t *testing.T, h *testutil.Harness, slug string) corpus { t.Helper() ctx := context.Background() mine := freshOrg(t, h, slug) theirs := freshOrg(t, h, slug+"-rival") c := corpus{ orgID: mine, admin: authctx.Identity{ UserID: "00000000-0000-0000-0000-000000000101", OrgID: mine, Role: "admin", Email: "boss@example.test", }, talent: authctx.Identity{ UserID: "00000000-0000-0000-0000-000000000102", OrgID: mine, Role: "talent", Email: "maya@example.test", }, other: authctx.Identity{ UserID: "00000000-0000-0000-0000-000000000103", OrgID: theirs, Role: "admin", Email: "rival@other.test", }, } ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128)) docs := []struct { org string doc knowledge.Document }{ {mine, knowledge.Document{ Source: "policy_docs", ExternalID: "handbook", Title: "Staff Handbook", Audience: knowledge.TenantWide(), Body: "# Attendance\n\n" + "Staff arriving more than ten minutes after the shift start are recorded as late. " + "Three late marks in a rolling month trigger a conversation with the venue manager.\n\n" + "# Breaks\n\n" + "A shift over six hours carries a thirty minute unpaid break. " + "Breaks are taken at a time agreed with the supervisor on duty.", }}, {mine, knowledge.Document{ Source: "policy_docs", ExternalID: "pay-review", Title: "Pay Review Guidance", Audience: knowledge.ForRoles(domain.RoleAdmin, domain.RoleEmployer), Body: "Managers set the annual uplift band before the review window opens. " + "The uplift budget for this year is capped at four percent of the wage bill.", }}, {mine, knowledge.Document{ Source: "worker_notes", ExternalID: "maya-review", Title: "Maya Chen — review note", Audience: knowledge.ForPerson("00000000-0000-0000-0000-000000000102", "maya@example.test"), Body: "Maya has covered eleven shifts this quarter and has asked about progressing " + "to a supervisor role. Attendance is spotless.", }}, {theirs, knowledge.Document{ Source: "policy_docs", ExternalID: "rival-handbook", Title: "Rival Co Handbook", Audience: knowledge.TenantWide(), Body: "Staff arriving more than ten minutes after the shift start are recorded as late. " + "Rival Co pays a retention bonus of nine hundred pounds after twelve months.", }}, } for _, d := range docs { if _, err := ing.Ingest(ctx, d.org, d.doc); err != nil { t.Fatalf("ingest %s: %v", d.doc.ExternalID, err) } } return c } func retriever(h *testutil.Harness) *knowledge.Retriever { return knowledge.NewRetriever(h.Pool, knowledge.NewLexical(128)) } func texts(res *knowledge.Results) string { var b strings.Builder for _, c := range res.Chunks { b.WriteString(c.Title) b.WriteString(" ") b.WriteString(c.Text) b.WriteString("\n") } return b.String() } /* ── I1: an agent reads what its caller could read ──────────────────────── */ func TestRetrievalRefusesACallerWithNoTenant(t *testing.T) { // §5: a retrieval function that accepts a query but not a caller principal // is wrong by construction. This package has one entry point and it takes a // principal — this asserts the run-time half, for a caller who assembled the // struct by hand with an empty identity. h := testutil.New(t) seedCorpus(t, h, "no-tenant") _, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "late", Principal: authctx.Identity{Role: "admin", Email: "x@example.test"}, Sources: []string{"policy_docs"}, }) if err == nil { t.Fatal("retrieval served a caller with no tenant") } var kErr *knowledge.Error if !asErr(err, &kErr) || kErr.Code != knowledge.ErrNoPrincipal { t.Errorf("want %s, got %v", knowledge.ErrNoPrincipal, err) } } func TestRetrievalRefusesAnUnlistedRole(t *testing.T) { h := testutil.New(t) c := seedCorpus(t, h, "unlisted-role") stranger := c.admin stranger.Role = "superuser" if _, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "late", Principal: stranger, Sources: []string{"policy_docs"}, }); err == nil { t.Fatal("retrieval served an unlisted role") } } func TestRetrievalRefusesAnEmptySourceList(t *testing.T) { // An agent whose spec named no knowledge has no knowledge. The dangerous // reading of an empty list is "all of them", and that reading is exactly // what a permissive default would ship. h := testutil.New(t) c := seedCorpus(t, h, "no-sources") _, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "late", Principal: c.admin, }) if err == nil { t.Fatal("an empty source list retrieved something") } var kErr *knowledge.Error if !asErr(err, &kErr) || kErr.Code != knowledge.ErrNoSources { t.Errorf("want %s, got %v", knowledge.ErrNoSources, err) } } func TestAnotherTenantsDocumentsAreInvisible(t *testing.T) { // The rival handbook contains the SAME sentence about ten minutes as ours, // so a query that matches ours matches theirs equally well. If tenancy were // a post-filter, the rival chunk would be fetched, ranked, and then dropped // — and its presence would still show in the result count. h := testutil.New(t) c := seedCorpus(t, h, "cross-tenant") res, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "arriving late after the shift start", Principal: c.admin, Sources: []string{"policy_docs"}, K: 20, }) if err != nil { t.Fatalf("retrieve: %v", err) } body := texts(res) for _, forbidden := range []string{"Rival Co", "retention bonus", "nine hundred"} { if strings.Contains(body, forbidden) { t.Errorf("another tenant's document leaked: %q appeared", forbidden) } } if len(res.Chunks) == 0 { t.Error("nothing came back at all; the query should match our own handbook") } } func TestTalentCannotReadAnOperatorDocument(t *testing.T) { h := testutil.New(t) c := seedCorpus(t, h, "role-scoped") res, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "annual uplift band review window budget", Principal: c.talent, Sources: []string{"policy_docs"}, K: 20, }) if err != nil { t.Fatalf("retrieve: %v", err) } if body := texts(res); strings.Contains(body, "uplift") { t.Errorf("a role-restricted document reached a talent caller: %s", body) } // And an operator DOES get it, so the test above is not passing because the // document failed to index. res, err = retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "annual uplift band review window budget", Principal: c.admin, Sources: []string{"policy_docs"}, K: 20, }) if err != nil { t.Fatalf("retrieve: %v", err) } if !strings.Contains(texts(res), "uplift") { t.Error("the operator document is not retrievable by an operator; the fixture is broken") } } func TestAPersonalDocumentReachesOnlyItsSubject(t *testing.T) { h := testutil.New(t) c := seedCorpus(t, h, "personal") r := retriever(h) ctx := context.Background() q := func(p authctx.Identity) string { res, err := r.Retrieve(ctx, knowledge.Query{ Text: "covered eleven shifts supervisor progression", Principal: p, Sources: []string{"worker_notes"}, K: 20, }) if err != nil { t.Fatalf("retrieve: %v", err) } return texts(res) } if !strings.Contains(q(c.talent), "eleven shifts") { t.Error("the subject of a personal note cannot read it") } // The admin is an operator and sees the whole tenant elsewhere — but this // document was addressed to a person, not to the organization, and an // operator's reach over OPERATIONAL rows is not a reach over every document // somebody filed about somebody. if strings.Contains(q(c.admin), "eleven shifts") { t.Error("a personal note reached someone it was not addressed to") } } func TestAnAgentCannotReadACorpusItsSpecDidNotName(t *testing.T) { // The source list is the agent's, not the caller's. A talent caller may // read their own note; an agent granted only policy_docs may not fetch it // on their behalf. Both halves have to hold, or `knowledge:` in a spec is // decoration. h := testutil.New(t) c := seedCorpus(t, h, "source-scoped") res, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "covered eleven shifts supervisor progression", Principal: c.talent, Sources: []string{"policy_docs"}, K: 20, }) if err != nil { t.Fatalf("retrieve: %v", err) } if strings.Contains(texts(res), "eleven shifts") { t.Error("a document from an undeclared source was retrieved") } } /* ── I2: the filter runs BEFORE scoring ─────────────────────────────────── */ func TestThePermissionFilterRunsBeforeScoring(t *testing.T) { // The distinction I2 turns on, made observable. // // A post-filter fetches k rows, drops the forbidden ones, and returns what // is left — so asking for k and getting back fewer than k, while permitted // matches still exist, is the fingerprint of post-filtering. A pre-filter // never sees the forbidden rows at all, so it fills its k from the caller's // own corpus. // // The fixture makes this sharp: 30 rival documents that match the query // perfectly, and 12 of our own that match it too. Under a post-filter the // rivals would crowd out the candidate window and the caller would get a // short, wrong result. Under a pre-filter they are invisible and the caller // gets a full k of their own. h := testutil.New(t) ctx := context.Background() mine := freshOrg(t, h, "prefilter-mine") theirs := freshOrg(t, h, "prefilter-theirs") ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128)) phrase := "lateness threshold ten minutes shift start recorded" for i := 0; i < 30; i++ { if _, err := ing.Ingest(ctx, theirs, knowledge.Document{ Source: "policy_docs", ExternalID: fmt.Sprintf("rival-%d", i), Title: fmt.Sprintf("Rival doc %d", i), Audience: knowledge.TenantWide(), Body: phrase + " — rival copy " + fmt.Sprint(i), }); err != nil { t.Fatalf("seed rival %d: %v", i, err) } } for i := 0; i < 12; i++ { if _, err := ing.Ingest(ctx, mine, knowledge.Document{ Source: "policy_docs", ExternalID: fmt.Sprintf("ours-%d", i), Title: fmt.Sprintf("Our doc %d", i), Audience: knowledge.TenantWide(), Body: phrase + " — our copy " + fmt.Sprint(i), }); err != nil { t.Fatalf("seed ours %d: %v", i, err) } } admin := authctx.Identity{ UserID: "00000000-0000-0000-0000-000000000201", OrgID: mine, Role: "admin", Email: "boss@prefilter.test", } res, err := retriever(h).Retrieve(ctx, knowledge.Query{ Text: phrase, Principal: admin, Sources: []string{"policy_docs"}, K: 10, }) if err != nil { t.Fatalf("retrieve: %v", err) } if len(res.Chunks) != 10 { t.Errorf("asked for 10 and got %d — a short result set with matches still available "+ "is the fingerprint of filtering AFTER scoring", len(res.Chunks)) } for _, c := range res.Chunks { if strings.Contains(c.Title, "Rival") { t.Fatalf("a rival document was returned: %s", c.Title) } } } /* ── §5: ingest rejects a document nobody can read ──────────────────────── */ func TestIngestRefusesADocumentWithNoAudience(t *testing.T) { // §5: chunks without ACL metadata are rejected at ingest. An empty ACL is // not "private" — it is a row the array-overlap operator can never match, // so the document reports as ingested and is silently unreachable forever. h := testutil.New(t) org := freshOrg(t, h, "no-audience") _, err := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128)). Ingest(context.Background(), org, knowledge.Document{ Source: "policy_docs", ExternalID: "orphan", Title: "Orphan", Body: "Nobody can read this.", // no Audience }) if err == nil { t.Fatal("a document with no audience was ingested") } var kErr *knowledge.Error if !asErr(err, &kErr) || kErr.Code != knowledge.ErrNoAudience { t.Errorf("want %s, got %v", knowledge.ErrNoAudience, err) } // And nothing was written. A refusal that left a half-document behind would // be worse than no refusal, because the row would then look ingested. var n int if err := h.Pool.QueryRow(context.Background(), `SELECT count(*) FROM knowledge_documents WHERE org_id = $1::uuid`, org).Scan(&n); err != nil { t.Fatalf("count: %v", err) } if n != 0 { t.Errorf("%d documents written by a refused ingest", n) } } func TestReIngestingAnUnchangedDocumentDoesNothing(t *testing.T) { h := testutil.New(t) ctx := context.Background() org := freshOrg(t, h, "unchanged") ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128)) doc := knowledge.Document{ Source: "policy_docs", ExternalID: "handbook", Title: "Handbook", Audience: knowledge.TenantWide(), Body: "Staff arriving more than ten minutes late are recorded as late.", } first, err := ing.Ingest(ctx, org, doc) if err != nil { t.Fatalf("first ingest: %v", err) } if first.Unchanged { t.Error("a first ingest reported itself unchanged") } second, err := ing.Ingest(ctx, org, doc) if err != nil { t.Fatalf("second ingest: %v", err) } if !second.Unchanged { t.Error("re-ingesting identical content re-chunked and re-embedded it") } if second.Chunks != first.Chunks { t.Errorf("chunk count changed on a no-op ingest: %d then %d", first.Chunks, second.Chunks) } } func TestChangingOnlyTheAudienceRewritesTheChunks(t *testing.T) { // The words did not change; who may read them did. The chunks carry a // denormalised copy of the tags, so treating this as "unchanged" would // leave every chunk permissioned by the OLD audience — a permission change // that silently did not take effect. h := testutil.New(t) ctx := context.Background() org := freshOrg(t, h, "audience-change") ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128)) doc := knowledge.Document{ Source: "policy_docs", ExternalID: "handbook", Title: "Handbook", Audience: knowledge.TenantWide(), Body: "The uplift budget this year is capped at four percent.", } if _, err := ing.Ingest(ctx, org, doc); err != nil { t.Fatalf("first ingest: %v", err) } doc.Audience = knowledge.ForRoles(domain.RoleAdmin) res, err := ing.Ingest(ctx, org, doc) if err != nil { t.Fatalf("second ingest: %v", err) } if res.Unchanged { t.Fatal("an audience change was treated as no change; the chunks would keep the old ACL") } // The talent caller must now be unable to reach it. talent := authctx.Identity{ UserID: "00000000-0000-0000-0000-000000000301", OrgID: org, Role: "talent", Email: "maya@audience.test", } out, err := retriever(h).Retrieve(ctx, knowledge.Query{ Text: "uplift budget capped four percent", Principal: talent, Sources: []string{"policy_docs"}, K: 10, }) if err != nil { t.Fatalf("retrieve: %v", err) } if strings.Contains(texts(out), "uplift") { t.Error("the chunks kept the old audience after a permission change") } } /* ── Determinism and shape ──────────────────────────────────────────────── */ func TestTheSameQueryReturnsTheSameOrder(t *testing.T) { // A retrieval whose ordering wobbles between identical calls makes every // downstream difference impossible to attribute — an eval that fails one // run in five is worse than no eval. h := testutil.New(t) c := seedCorpus(t, h, "determinism") r := retriever(h) ctx := context.Background() var previous []string for i := 0; i < 5; i++ { res, err := r.Retrieve(ctx, knowledge.Query{ Text: "late shift break supervisor", Principal: c.admin, Sources: []string{"policy_docs"}, K: 5, }) if err != nil { t.Fatalf("retrieve: %v", err) } var ids []string for _, ch := range res.Chunks { ids = append(ids, ch.ChunkID) } if previous != nil && strings.Join(ids, ",") != strings.Join(previous, ",") { t.Fatalf("ordering changed between identical queries:\n %v\n %v", previous, ids) } previous = ids } } func TestEveryResultCarriesACitation(t *testing.T) { // §5: retrieved chunks flow to the model with source ids, so a response can // cite — and so a claim without a citation can be told apart from a // grounded one. h := testutil.New(t) c := seedCorpus(t, h, "citations") res, err := retriever(h).Retrieve(context.Background(), knowledge.Query{ Text: "late break supervisor", Principal: c.admin, Sources: []string{"policy_docs"}, K: 5, }) if err != nil { t.Fatalf("retrieve: %v", err) } if len(res.Chunks) == 0 { t.Fatal("nothing retrieved") } for _, ch := range res.Chunks { if ch.ChunkID == "" || ch.DocumentID == "" { t.Errorf("a chunk came back with no citable id: %+v", ch) } if ch.Title == "" { t.Errorf("chunk %s has no document title to cite", ch.ChunkID) } if ch.Score <= 0 { t.Errorf("chunk %s has a non-positive fused score", ch.ChunkID) } } } func TestKeywordOnlyRetrievalSaysSo(t *testing.T) { // A retrieval that silently halved its own recall presents as the agent // getting worse for no reason anyone can find. With no embedder, results // still come back — and they say why they are only half the story. h := testutil.New(t) c := seedCorpus(t, h, "no-embedder") res, err := knowledge.NewRetriever(h.Pool, nil).Retrieve(context.Background(), knowledge.Query{ Text: "late", Principal: c.admin, Sources: []string{"policy_docs"}, K: 5, }) if err != nil { t.Fatalf("retrieve: %v", err) } if res.DenseSkipped == "" { t.Error("keyword-only results did not report that the dense half was skipped") } if len(res.Chunks) == 0 { t.Error("keyword-only retrieval returned nothing; it should still work") } } func TestVectorsFromAnotherModelAreNotSearched(t *testing.T) { // Vectors from two embedding models are not comparable — the numbers have // no shared meaning — so a corpus half-migrated returns confident nonsense // rather than failing. The model name on the row is what prevents it. h := testutil.New(t) ctx := context.Background() c := seedCorpus(t, h, "model-mismatch") // A retriever whose embedder produces a DIFFERENT model name over the same // corpus. Its dense half must match nothing. other := knowledge.NewRetriever(h.Pool, knowledge.NewLexical(64)) // different dims → different model name res, err := other.Retrieve(ctx, knowledge.Query{ Text: "late shift break", Principal: c.admin, Sources: []string{"policy_docs"}, K: 5, }) if err != nil { t.Fatalf("retrieve: %v", err) } // Keyword still works, so results come back — but none of them was ranked // by the dense half, because no row carries this model's vectors. for _, ch := range res.Chunks { if ch.DenseRank != 0 { t.Errorf("chunk %s was dense-ranked against a different model's vectors", ch.ChunkID) } } if len(res.Chunks) == 0 { t.Error("nothing came back; the keyword half should be unaffected") } } func asErr(err error, target **knowledge.Error) bool { if e, ok := err.(*knowledge.Error); ok { *target = e return true } return false } /* ── Re-embedding ───────────────────────────────────────────────────────── */ func TestReembeddingRestoresDenseSearchAfterAModelChange(t *testing.T) { // The silent failure this exists for. // // Vectors from two models are not comparable, so every chunk records which // model produced it and retrieval only searches matching ones. Change model // and the old vectors are not wrong — they are simply not looked at. // Retrieval keeps working, keeps citing, and quietly drops to keyword-only. // Nothing errors, and the only symptom is answers getting worse. h := testutil.New(t) ctx := context.Background() c := seedCorpus(t, h, "reembed") // A different embedder over the same corpus: same rows, incomparable // vectors. Its dense half matches nothing. other := knowledge.NewLexical(64) before, err := knowledge.NewRetriever(h.Pool, other).Retrieve(ctx, knowledge.Query{ Text: "late shift break supervisor", Principal: c.admin, Sources: []string{"policy_docs"}, K: 10, }) if err != nil { t.Fatalf("retrieve: %v", err) } for _, ch := range before.Chunks { if ch.DenseRank != 0 { t.Fatalf("chunk %s was dense-ranked before re-embedding; the fixture is wrong", ch.ChunkID) } } // Re-embed with the new model. done, err := knowledge.NewIngester(h.Pool, other).Reembed(ctx, c.orgID, 8, nil) if err != nil { t.Fatalf("reembed: %v", err) } if done == 0 { t.Fatal("re-embedding reported no work; the corpus should have needed it") } after, err := knowledge.NewRetriever(h.Pool, other).Retrieve(ctx, knowledge.Query{ Text: "late shift break supervisor", Principal: c.admin, Sources: []string{"policy_docs"}, K: 10, }) if err != nil { t.Fatalf("retrieve: %v", err) } var ranked int for _, ch := range after.Chunks { if ch.DenseRank != 0 { ranked++ } } if ranked == 0 { t.Error("dense search is still dead after re-embedding") } } func TestReembeddingTwiceDoesNothingTheSecondTime(t *testing.T) { // A corpus already carrying this model's vectors needs no work, and saying // so beats re-embedding it — which on a hosted provider is a bill for // nothing. h := testutil.New(t) ctx := context.Background() c := seedCorpus(t, h, "reembed-idempotent") e := knowledge.NewLexical(128) // the model the fixture already used done, err := knowledge.NewIngester(h.Pool, e).Reembed(ctx, c.orgID, 8, nil) if err != nil { t.Fatalf("reembed: %v", err) } if done != 0 { t.Errorf("%d chunks re-embedded with the model they already carried", done) } } func TestReembeddingKeepsTheHeadingInTheEmbeddedText(t *testing.T) { // Ingest embeds "heading\n\ntext". A re-embed that dropped the heading // would produce vectors subtly different from the ones ingest makes, and // the difference would surface as retrieval quality drifting after a // reindex — with nothing to point at. h := testutil.New(t) ctx := context.Background() c := seedCorpus(t, h, "reembed-heading") var heading, text string if err := h.Pool.QueryRow(ctx, ` SELECT heading, text FROM knowledge_chunks WHERE org_id = $1::uuid AND heading <> '' LIMIT 1`, c.orgID, ).Scan(&heading, &text); err != nil { t.Skipf("no headed chunk in the fixture: %v", err) } e := knowledge.NewLexical(64) if _, err := knowledge.NewIngester(h.Pool, e).Reembed(ctx, c.orgID, 8, nil); err != nil { t.Fatalf("reembed: %v", err) } // The stored vector must equal what the embedder produces for // heading+text, not for text alone. want, err := e.Embed(ctx, []string{heading + "\n\n" + text}, knowledge.KindDocument) if err != nil { t.Fatalf("embed: %v", err) } var stored []float32 if err := h.Pool.QueryRow(ctx, ` SELECT embedding FROM knowledge_chunks WHERE org_id = $1::uuid AND heading = $2 AND text = $3`, c.orgID, heading, text).Scan(&stored); err != nil { t.Fatalf("read back: %v", err) } if len(stored) != len(want[0]) { t.Fatalf("stored %d dims, embedder produces %d", len(stored), len(want[0])) } for i := range stored { if stored[i] != want[0][i] { t.Fatalf("the re-embedded vector does not match heading+text; "+ "the heading was dropped (first difference at %d)", i) } } }