krow_backend/go-api/internal/knowledge/corpus_test.go

package knowledge_test

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/krow/krow-backend/go-api/internal/authctx"
	"github.com/krow/krow-backend/go-api/internal/domain"
	"github.com/krow/krow-backend/go-api/internal/knowledge"
	"github.com/krow/krow-backend/go-api/internal/testutil"
)

// The corpus this product ships, tested as content rather than as machinery.
//
// The retrieval layer is covered elsewhere: pre-filtering, ingest refusing a
// document nobody can read, a poisoned document staying inside its block. What
// was not covered is the corpus itself — and once agents answer from it, the
// documents are product, not fixtures. A policy file with the wrong `audience:`
// line is a permission bug that no amount of correct retrieval code prevents,
// and it is one character away at all times.
//
// Read from knowledge/ rather than restated here, so the assertion is about the
// files that ship.

var frontMatter = regexp.MustCompile(`(?s)\A---\n(.*?)\n---\n`)

type corpusDoc struct {
	name, source, audience, title, body string
}

func loadCorpus(t *testing.T) []corpusDoc {
	t.Helper()
	dir := filepath.Join("..", "..", "..", "knowledge")
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatalf("read knowledge/: %v", err)
	}

	var docs []corpusDoc
	for _, e := range entries {
		if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || e.Name() == "README.md" {
			continue
		}
		raw, err := os.ReadFile(filepath.Join(dir, e.Name()))
		if err != nil {
			t.Fatalf("read %s: %v", e.Name(), err)
		}
		m := frontMatter.FindSubmatch(raw)
		if m == nil {
			t.Errorf("%s has no front matter; it cannot declare who may read it", e.Name())
			continue
		}
		d := corpusDoc{name: e.Name(), body: string(raw[len(m[0]):])}
		for _, line := range strings.Split(string(m[1]), "\n") {
			key, value, ok := strings.Cut(line, ":")
			if !ok {
				continue
			}
			switch strings.TrimSpace(key) {
			case "source":
				d.source = strings.TrimSpace(value)
			case "audience":
				d.audience = strings.TrimSpace(value)
			case "title":
				d.title = strings.TrimSpace(value)
			}
		}
		docs = append(docs, d)
	}
	return docs
}

// Every shipped document declares a source, a title and an audience.
func TestEveryShippedDocumentDeclaresItsReaders(t *testing.T) {
	docs := loadCorpus(t)
	if len(docs) < 2 {
		t.Fatalf("found %d documents in knowledge/; expected the shipped corpus", len(docs))
	}
	for _, d := range docs {
		if d.audience == "" {
			t.Errorf("%s declares no audience — ingest refuses it, and a document "+
				"nobody can read is not private, it is unreachable", d.name)
		}
		if d.source == "" {
			t.Errorf("%s declares no source; an agent grants corpora by name", d.name)
		}
		if d.title == "" {
			t.Errorf("%s has no title; a citation with no title cannot be followed", d.name)
		}
		if strings.TrimSpace(d.body) == "" {
			t.Errorf("%s has front matter and no body", d.name)
		}
	}
}

// mustNotBeTenantWide names the documents that are not for everyone.
//
// Written down rather than read from the files, because reading them is
// circular: a test that takes `audience:` from a document and then checks that
// document's audience is enforced passes whatever the line says, including
// after somebody widens it. Opening a restricted document is a one-character
// edit, it looks like every other edit in a diff, and it is the failure this
// corpus is most likely to have.
//
// So this is the judgement, held apart from the file: what these documents
// contain — an organisation's pay bands, how it screens people, whose
// right-to-work check is outstanding — is management guidance, and a worker
// reading it is a disclosure the organisation did not choose to make.
var mustNotBeTenantWide = map[string]string{
	"pay-and-progression.md":             "uplift bands and the wage-bill cap",
	"right-to-work-and-certification.md": "who has an outstanding or lapsed check",
	"screening-and-hiring-standards.md":  "how candidates are scored and rejected",
}

func TestDocumentsThatAreNotForEveryoneStayThatWay(t *testing.T) {
	byName := map[string]corpusDoc{}
	for _, d := range loadCorpus(t) {
		byName[d.name] = d
	}
	for name, why := range mustNotBeTenantWide {
		d, ok := byName[name]
		if !ok {
			t.Errorf("%s is named as restricted but is not in knowledge/; "+
				"if it was renamed, this list has to move with it", name)
			continue
		}
		if strings.Contains(d.audience, "tenant") {
			t.Errorf("%s is readable by the whole tenant, and holds %s. "+
				"If that is intended, remove it from mustNotBeTenantWide and say why "+
				"— but it is not the kind of thing to widen by accident", name, why)
		}
		if !strings.Contains(d.audience, "role:") {
			t.Errorf("%s restricts to nobody at all (audience: %q)", name, d.audience)
		}
	}
}

// A corpus with nothing restricted cannot demonstrate the boundary it relies on.
func TestTheCorpusKeepsSomethingBackFromTalent(t *testing.T) {
	docs := loadCorpus(t)
	var restricted, open int
	for _, d := range docs {
		if strings.Contains(d.audience, "role:") && !strings.Contains(d.audience, "tenant") {
			restricted++
		} else {
			open++
		}
	}
	if restricted == 0 {
		t.Error("no document is restricted to a role; the ACL is then decoration, " +
			"and nothing in the corpus would notice if the filter stopped working")
	}
	if open == 0 {
		t.Error("every document is restricted; a corpus talent cannot read at all " +
			"is one they will stop asking")
	}
	t.Logf("%d restricted to a role, %d open to the tenant", restricted, open)
}

// The boundary, over the documents that actually ship.
//
// Ingested into a throwaway tenant and queried as two roles. An admin-only
// document reaching a talent caller is a leak of this organisation's own
// guidance to its own workers, which is the quiet kind: same tenant, same
// corpus, wrong reader.
func TestShippedCorpusIsRetrievableAndScoped(t *testing.T) {
	h := testutil.New(t)
	ctx := context.Background()
	org := freshOrg(t, h, "shipped-corpus")

	docs := loadCorpus(t)
	ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128))

	var restrictedTitles []string
	for _, d := range docs {
		aud, err := audienceFrom(d.audience)
		if err != nil {
			t.Fatalf("%s: %v", d.name, err)
		}
		if _, err := ing.Ingest(ctx, org, knowledge.Document{
			Source: d.source, ExternalID: strings.TrimSuffix(d.name, ".md"),
			Title: d.title, Audience: aud, Body: d.body,
		}); err != nil {
			t.Fatalf("ingest %s: %v", d.name, err)
		}
		if len(aud.Roles) > 0 && !aud.Tenant {
			restrictedTitles = append(restrictedTitles, d.title)
		}
	}

	ask := func(role, email, text string) []knowledge.Result {
		res, err := retriever(h).Retrieve(ctx, knowledge.Query{
			Text: text,
			Principal: authctx.Identity{
				UserID: "00000000-0000-0000-0000-000000000301",
				OrgID:  org, Role: role, Email: email,
			},
			Sources: []string{"policy_docs"}, K: 12,
		})
		if err != nil {
			t.Fatalf("retrieve as %s: %v", role, err)
		}
		return res.Chunks
	}

	// The corpus answers at all.
	if got := ask("admin", "boss@corpus.test", "shift cover cancellation notice"); len(got) == 0 {
		t.Error("an admin asking about shift cover retrieved nothing from the shipped corpus")
	}

	// And the restricted documents stay behind the role that owns them.
	talent := ask("talent", "worker@corpus.test", strings.Join(restrictedTitles, " "))
	for _, c := range talent {
		for _, title := range restrictedTitles {
			if c.Title == title {
				t.Errorf("talent retrieved %q, which is restricted to a role", title)
			}
		}
	}
	if len(restrictedTitles) > 0 {
		admin := ask("admin", "boss@corpus.test", strings.Join(restrictedTitles, " "))
		var reached bool
		for _, c := range admin {
			for _, title := range restrictedTitles {
				if c.Title == title {
					reached = true
				}
			}
		}
		if !reached {
			t.Error("an admin could not reach a document restricted to admins — " +
				"the filter is not scoping, it is refusing")
		}
	}
}

// audienceFrom parses the front-matter `audience:` line the way cmd/ingest does.
//
// Deliberately a second implementation rather than an import: cmd/ingest is a
// main package and cannot be imported, and a test that reused the parser under
// test could not catch the parser being wrong about the files. This agreeing
// with ingest is the point — where they disagree, one of them is mis-reading a
// document's readers.
func audienceFrom(raw string) (knowledge.Audience, error) {
	var a knowledge.Audience
	if strings.TrimSpace(raw) == "" {
		return a, errors.New("no audience declared")
	}
	for _, part := range strings.Split(raw, ",") {
		part = strings.TrimSpace(part)
		switch {
		case part == "tenant":
			a.Tenant = true
		case strings.HasPrefix(part, "role:"):
			role, ok := domain.ParseRole(strings.TrimPrefix(part, "role:"))
			if !ok {
				return a, fmt.Errorf("%q is not a role", part)
			}
			a.Roles = append(a.Roles, role)
		case strings.HasPrefix(part, "email:"):
			a.Emails = append(a.Emails, strings.TrimPrefix(part, "email:"))
		default:
			return a, fmt.Errorf("unrecognised audience %q", part)
		}
	}
	return a, nil
}