Add evals for every shipped agent, a policy corpus, and CI

§9 says no agent ships without evals. Eight of the nine had none: the two
other suites in evals/ are harness fixtures rather than agents in the
registry, so the rule was being met by one agent in nine.

Evals — 40 new cases, five per agent, every one carrying mustNotLeak:

  - the agent is loaded from its real spec in agents/*.md rather than
    written out again in Go. A hand-copied agent tests the copy: it keeps
    passing after somebody edits the spec, which is the moment it most
    needed to fail.
  - callNamed calls the tool a case names. toolThenAnswer always called
    tools[0], so seven of positions-agent's eight tools were unreachable,
    and a boundary nothing calls is a boundary nothing tests.
  - seedWorkspace fills BOTH tenants. A leak test against an empty second
    tenant cannot fail.

Verified by breaking workersByScore's org predicate: six cases across four
agents fail with LEAKED "RIVAL".

Knowledge — six policy documents, taking the corpus from 2 to 8 (34
chunks). Three restricted to admin and employer, five tenant-wide. They
cover what the tools cannot: a tool reports how many shifts went unworked,
a policy says what cover costs inside 24 hours.

corpus_test.go treats those documents as product rather than fixtures. The
first version was tautological — it read audience: from a file and checked
that file's audience was enforced, so opening a restricted document passed.
mustNotBeTenantWide now holds that judgement apart from the files, with the
reason recorded for each.

CI — the checks this repository already had, made unskippable. testutil
calls t.Skipf on an unreachable database, so a dead service container would
produce a green build over a suite that ran almost nothing. Simulated: go
test exits 0 with 74 tests skipped, including every tenant-isolation test.
The guard exits 1 and names them, while still allowing TestLive* to skip
without a model key.

This CI tests; it does not deploy. The README's claim that migrations are
run by CI against the target database remains aspirational.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0186JgqQUCDS8ZwGmyw3ymWu

go-api/internal/evals/registry_agents_test.go

@@ -0,0 +1,277 @@
+package evals_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/krow/krow-backend/go-api/internal/definition"
+	"github.com/krow/krow-backend/go-api/internal/evals"
+	"github.com/krow/krow-backend/go-api/internal/gateway"
+	"github.com/krow/krow-backend/go-api/internal/knowledge"
+	"github.com/krow/krow-backend/go-api/internal/runtime"
+	"github.com/krow/krow-backend/go-api/internal/testutil"
+)
+
+// Suites for the agents this product actually ships.
+//
+// §9 says no agent ships without evals. Eight of the nine shipped without any:
+// `activity-agent` had a suite, and the other two suites in evals/ — coverage
+// and handbook — are fixtures built for the harness rather than agents in the
+// registry. So the rule was being met by one agent in nine.
+//
+// Two things are done differently here from the activity suite, both because
+// the point is to test what ships:
+//
+//   - the agent is loaded from its REAL spec in agents/*.md, not written out
+//     again in Go. A hand-copied agent tests the copy: it keeps passing after
+//     somebody edits the spec, which is the moment it most needed to fail.
+//   - the tool set is whatever that spec declares. If a spec names a tool the
+//     registry does not have, the suite says so rather than quietly running an
+//     agent with one capability fewer.
+//
+// What these prove is the boundary, not the prose. The model is scripted
+// (`toolThenAnswer`) and answers with the tool's output verbatim, so a case
+// asserts that a tool ran, that what it returned carries what it should, and —
+// the part that matters — that it carries nothing belonging to anyone else.
+
+// seedWorkspace fills both tenants with the records these agents read.
+//
+// Both, always. A leak test against an empty second tenant is a test that
+// cannot fail: `mustNotLeak` looks for the other tenant's rows in the answer,
+// and if that tenant has no rows there is nothing to find. Every table an
+// agent's tools touch is populated on both sides, with values distinctive
+// enough to spot in a blob of JSON.
+func seedWorkspace(t *testing.T, h *testutil.Harness) (otherOrg string) {
+	t.Helper()
+	ctx := context.Background()
+
+	if err := h.Pool.QueryRow(ctx,
+		`INSERT INTO organizations (name, slug) VALUES ('Rival Staffing', 'rival-staffing')
+		 RETURNING id::text`).Scan(&otherOrg); err != nil {
+		t.Fatalf("create rival org: %v", err)
+	}
+
+	type tenant struct {
+		org, tag string
+	}
+	for _, tn := range []tenant{{h.OrgID, "Ours"}, {otherOrg, "RIVAL"}} {
+		var postingID string
+		if err := h.Pool.QueryRow(ctx, `
+			INSERT INTO job_postings (org_id, title, status, headcount, location, priority)
+			VALUES ($1::uuid, $2, 'active', 3, $3, 'high') RETURNING id::text`,
+			tn.org, tn.tag+" Bar Supervisor", tn.tag+" Shoreditch").Scan(&postingID); err != nil {
+			t.Fatalf("seed posting (%s): %v", tn.tag, err)
+		}
+
+		for i, st := range []string{"applied", "ai_screened", "shortlisted", "interview", "hired"} {
+			if _, err := h.Pool.Exec(ctx, `
+				INSERT INTO job_applications
+				       (org_id, job_posting_id, applicant_name, email, status, ai_score, job_title)
+				VALUES ($1::uuid, $2::uuid, $3, $4, $5::application_status, $6, $7)`,
+				tn.org, postingID,
+				fmt.Sprintf("%s Applicant %d", tn.tag, i),
+				fmt.Sprintf("%s-applicant-%d@example.test", strings.ToLower(tn.tag), i),
+				st, 60+i*8, tn.tag+" Bar Supervisor"); err != nil {
+				t.Fatalf("seed application (%s): %v", tn.tag, err)
+			}
+		}
+
+		for i, name := range []string{"Worker One", "Worker Two"} {
+			if _, err := h.Pool.Exec(ctx, `
+				INSERT INTO worker_profiles
+				       (org_id, full_name, email, krow_score, reliability_score,
+				        attendance_score, performance_score, client_rating,
+				        experience_years, shifts_completed, current_position)
+				VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
+				tn.org, tn.tag+" "+name,
+				fmt.Sprintf("%s-worker-%d@example.test", strings.ToLower(tn.tag), i),
+				80+i*7, 85+i*5, 90+i*3, 82+i*4, 4.5, 3+i, 20+i*10,
+				tn.tag+" Bartender"); err != nil {
+				t.Fatalf("seed worker (%s): %v", tn.tag, err)
+			}
+		}
+
+		if _, err := h.Pool.Exec(ctx, `
+			INSERT INTO staff (org_id, name, email, role, status, ai_score, hire_date)
+			VALUES ($1::uuid, $2, $3, $4, 'active', 91, current_date - 30)`,
+			tn.org, tn.tag+" Hired Person",
+			fmt.Sprintf("%s-hire@example.test", strings.ToLower(tn.tag)),
+			tn.tag+" Bar Supervisor"); err != nil {
+			t.Fatalf("seed staff (%s): %v", tn.tag, err)
+		}
+
+		for i, st := range []string{"present", "present", "late", "absent", "no_show"} {
+			// A missed shift has no hours behind it — shift_records enforces
+			// that, and seeding around the constraint would be seeding data the
+			// product cannot hold.
+			missed := st == "absent" || st == "no_show"
+			worked, overtime, late := 8.0, float64(i), i*7
+			if missed {
+				worked, overtime, late = 0, 0, 0
+			}
+			if _, err := h.Pool.Exec(ctx, `
+				INSERT INTO shift_records
+				       (org_id, worker_name, worker_email, role, shift_date,
+				        scheduled_start, scheduled_end, created_date,
+				        status, scheduled_hours, actual_hours, overtime_hours, minutes_late)
+				VALUES ($1::uuid, $2, $3, $4, current_date - $5::int,
+				        (current_date - $5::int) + time '18:00',
+				        (current_date - $5::int) + time '02:00' + interval '1 day',
+				        (current_date - $5::int) + time '18:00',
+				        $6::shift_status, 8, $7, $8, $9)`,
+				tn.org, tn.tag+" Worker One",
+				fmt.Sprintf("%s-worker-0@example.test", strings.ToLower(tn.tag)),
+				tn.tag+" Bartender", i+1, st, worked, overtime, late); err != nil {
+				t.Fatalf("seed shift (%s): %v", tn.tag, err)
+			}
+		}
+
+		if _, err := h.Pool.Exec(ctx, `
+			INSERT INTO courses (org_id, title, category, status, xp)
+			VALUES ($1::uuid, $2, 'Bar', 'active', 50)`,
+			tn.org, tn.tag+" Cocktail Fundamentals"); err != nil {
+			t.Fatalf("seed course (%s): %v", tn.tag, err)
+		}
+
+		for _, ev := range []string{"apply_job", "hire_candidate", "delete_position"} {
+			if _, err := h.Pool.Exec(ctx, `
+				INSERT INTO user_activity (org_id, event_type, user_email, user_name)
+				VALUES ($1::uuid, $2, $3, $4)`,
+				tn.org, ev,
+				fmt.Sprintf("%s-actor@example.test", strings.ToLower(tn.tag)),
+				tn.tag+" Actor"); err != nil {
+				t.Fatalf("seed activity (%s): %v", tn.tag, err)
+			}
+		}
+	}
+	return otherOrg
+}
+
+// callNamed exercises the tool a case names, rather than always the first one.
+//
+// `toolThenAnswer` calls req.Tools[0], which is right for an agent carrying one
+// or two tools and useless for one carrying eight: seven of them would never be
+// reached, and a boundary nothing calls is a boundary nothing tests. A case
+// says which capability it is about through `expect.toolsCalled`, and this
+// calls that one. The assertions are still the case's own — this decides what
+// runs, not whether it passed.
+type callNamed struct {
+	want string
+	done bool
+}
+
+func (m *callNamed) Complete(_ context.Context, req gateway.Request) (*gateway.Response, error) {
+	last := req.Messages[len(req.Messages)-1]
+	if len(last.ToolResults) > 0 {
+		return &gateway.Response{
+			Text:       "Here is everything I was given: " + last.ToolResults[0].Content,
+			StopReason: "end_turn", Model: "scripted",
+		}, nil
+	}
+	if len(req.Tools) == 0 {
+		return &gateway.Response{
+			Text: "I have no way to look that up.", StopReason: "end_turn", Model: "scripted",
+		}, nil
+	}
+	pick := req.Tools[0].Name
+	for _, tool := range req.Tools {
+		if tool.Name == m.want {
+			pick = tool.Name
+			break
+		}
+	}
+	return &gateway.Response{
+		ToolCalls:  []gateway.ToolCall{{ID: "call_1", Name: pick, Input: json.RawMessage(`{}`)}},
+		StopReason: "tool_use", Model: "scripted",
+	}, nil
+}
+
+// loadShippedAgent reads an agent from the spec this product ships.
+func loadShippedAgent(t *testing.T, key string) *runtime.Agent {
+	t.Helper()
+	path := filepath.Join("..", "..", "..", "agents", key+".md")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read spec %s: %v", path, err)
+	}
+	parsed, err := definition.ParseAgent(string(raw), definition.Options{})
+	if err != nil {
+		t.Fatalf("parse spec %s: %v", key, err)
+	}
+	return &runtime.Agent{
+		ID: parsed.ID, Name: parsed.Name, Version: parsed.Version,
+		Description: parsed.Description, Reasoning: parsed.Reasoning,
+		Pages: parsed.Pages, Instructions: parsed.Instructions,
+		Skills: parsed.Skills, Tools: parsed.Tools,
+		KnowledgeSources: parsed.Sources,
+	}
+}
+
+// TestShippedAgentSuites runs every shipped agent against its own suite.
+//
+// One test over a table rather than eight near-identical functions: the agents
+// differ in their spec and their cases, not in how they are exercised, and
+// eight copies of this loop would drift apart one edit at a time.
+func TestShippedAgentSuites(t *testing.T) {
+	for _, key := range []string{
+		"analytics-agent", "candidates-agent", "control-center-agent",
+		"hired-history-agent", "krow-forge-agent", "krow-workforce-agent",
+		"positions-agent", "talent-pool-agent",
+	} {
+		t.Run(key, func(t *testing.T) {
+			h := testutil.New(t)
+			ctx := context.Background()
+			seedWorkspace(t, h)
+
+			suite, err := evals.LoadSuite(resolveSuite(t, key+".json"))
+			if err != nil {
+				t.Fatalf("load suite: %v", err)
+			}
+			agent := loadShippedAgent(t, key)
+
+			// The real registry, so a case exercises the tool that ships rather
+			// than a stand-in written to pass.
+			reg := runtime.DefaultTools(h.Pool, knowledge.NewRetriever(h.Pool, nil))
+
+			// A spec naming a tool the registry does not have is an agent with a
+			// capability its author believes it has. Said here rather than left
+			// for the runtime to drop in silence.
+			if unknown := reg.Known(agent.Tools); len(unknown) > 0 {
+				t.Fatalf("%s declares tools that are not registered: %s",
+					key, strings.Join(unknown, ", "))
+			}
+
+			users := seedPrincipals(t, h, map[string]string{
+				"$ADMIN_ID":  "boss@example.test",
+				"$TALENT_ID": "worker@example.test",
+			})
+
+			var results []evals.Result
+			for _, c := range suite.Cases {
+				want := ""
+				if len(c.Expect.ToolsCalled) > 0 {
+					want = c.Expect.ToolsCalled[0]
+				}
+				runner := evals.NewRunner(func(sink runtime.Sink) runtime.AgentExecutor {
+					return runtime.NewModelExecutor(&callNamed{want: want}, sink, reg)
+				}, agent)
+				results = append(results, runner.Run(ctx, substitute(c, h.OrgID, users)))
+			}
+
+			t.Log("\n" + evals.Report(suite.Agent, results))
+			for _, r := range results {
+				if !r.Passed {
+					t.Errorf("%s failed: %v", r.CaseID, r.Failures)
+				}
+			}
+			if len(results) < 5 {
+				t.Errorf("%s has %d cases; §9 requires at least 5", key, len(results))
+			}
+		})
+	}
+}

go-api/internal/knowledge/corpus_test.go

@@ -0,0 +1,273 @@
+package knowledge_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/krow/krow-backend/go-api/internal/authctx"
+	"github.com/krow/krow-backend/go-api/internal/domain"
+	"github.com/krow/krow-backend/go-api/internal/knowledge"
+	"github.com/krow/krow-backend/go-api/internal/testutil"
+)
+
+// The corpus this product ships, tested as content rather than as machinery.
+//
+// The retrieval layer is covered elsewhere: pre-filtering, ingest refusing a
+// document nobody can read, a poisoned document staying inside its block. What
+// was not covered is the corpus itself — and once agents answer from it, the
+// documents are product, not fixtures. A policy file with the wrong `audience:`
+// line is a permission bug that no amount of correct retrieval code prevents,
+// and it is one character away at all times.
+//
+// Read from knowledge/ rather than restated here, so the assertion is about the
+// files that ship.
+
+var frontMatter = regexp.MustCompile(`(?s)\A---\n(.*?)\n---\n`)
+
+type corpusDoc struct {
+	name, source, audience, title, body string
+}
+
+func loadCorpus(t *testing.T) []corpusDoc {
+	t.Helper()
+	dir := filepath.Join("..", "..", "..", "knowledge")
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("read knowledge/: %v", err)
+	}
+
+	var docs []corpusDoc
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || e.Name() == "README.md" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(dir, e.Name()))
+		if err != nil {
+			t.Fatalf("read %s: %v", e.Name(), err)
+		}
+		m := frontMatter.FindSubmatch(raw)
+		if m == nil {
+			t.Errorf("%s has no front matter; it cannot declare who may read it", e.Name())
+			continue
+		}
+		d := corpusDoc{name: e.Name(), body: string(raw[len(m[0]):])}
+		for _, line := range strings.Split(string(m[1]), "\n") {
+			key, value, ok := strings.Cut(line, ":")
+			if !ok {
+				continue
+			}
+			switch strings.TrimSpace(key) {
+			case "source":
+				d.source = strings.TrimSpace(value)
+			case "audience":
+				d.audience = strings.TrimSpace(value)
+			case "title":
+				d.title = strings.TrimSpace(value)
+			}
+		}
+		docs = append(docs, d)
+	}
+	return docs
+}
+
+// Every shipped document declares a source, a title and an audience.
+func TestEveryShippedDocumentDeclaresItsReaders(t *testing.T) {
+	docs := loadCorpus(t)
+	if len(docs) < 2 {
+		t.Fatalf("found %d documents in knowledge/; expected the shipped corpus", len(docs))
+	}
+	for _, d := range docs {
+		if d.audience == "" {
+			t.Errorf("%s declares no audience — ingest refuses it, and a document "+
+				"nobody can read is not private, it is unreachable", d.name)
+		}
+		if d.source == "" {
+			t.Errorf("%s declares no source; an agent grants corpora by name", d.name)
+		}
+		if d.title == "" {
+			t.Errorf("%s has no title; a citation with no title cannot be followed", d.name)
+		}
+		if strings.TrimSpace(d.body) == "" {
+			t.Errorf("%s has front matter and no body", d.name)
+		}
+	}
+}
+
+// mustNotBeTenantWide names the documents that are not for everyone.
+//
+// Written down rather than read from the files, because reading them is
+// circular: a test that takes `audience:` from a document and then checks that
+// document's audience is enforced passes whatever the line says, including
+// after somebody widens it. Opening a restricted document is a one-character
+// edit, it looks like every other edit in a diff, and it is the failure this
+// corpus is most likely to have.
+//
+// So this is the judgement, held apart from the file: what these documents
+// contain — an organisation's pay bands, how it screens people, whose
+// right-to-work check is outstanding — is management guidance, and a worker
+// reading it is a disclosure the organisation did not choose to make.
+var mustNotBeTenantWide = map[string]string{
+	"pay-and-progression.md":             "uplift bands and the wage-bill cap",
+	"right-to-work-and-certification.md": "who has an outstanding or lapsed check",
+	"screening-and-hiring-standards.md":  "how candidates are scored and rejected",
+}
+
+func TestDocumentsThatAreNotForEveryoneStayThatWay(t *testing.T) {
+	byName := map[string]corpusDoc{}
+	for _, d := range loadCorpus(t) {
+		byName[d.name] = d
+	}
+	for name, why := range mustNotBeTenantWide {
+		d, ok := byName[name]
+		if !ok {
+			t.Errorf("%s is named as restricted but is not in knowledge/; "+
+				"if it was renamed, this list has to move with it", name)
+			continue
+		}
+		if strings.Contains(d.audience, "tenant") {
+			t.Errorf("%s is readable by the whole tenant, and holds %s. "+
+				"If that is intended, remove it from mustNotBeTenantWide and say why "+
+				"— but it is not the kind of thing to widen by accident", name, why)
+		}
+		if !strings.Contains(d.audience, "role:") {
+			t.Errorf("%s restricts to nobody at all (audience: %q)", name, d.audience)
+		}
+	}
+}
+
+// A corpus with nothing restricted cannot demonstrate the boundary it relies on.
+func TestTheCorpusKeepsSomethingBackFromTalent(t *testing.T) {
+	docs := loadCorpus(t)
+	var restricted, open int
+	for _, d := range docs {
+		if strings.Contains(d.audience, "role:") && !strings.Contains(d.audience, "tenant") {
+			restricted++
+		} else {
+			open++
+		}
+	}
+	if restricted == 0 {
+		t.Error("no document is restricted to a role; the ACL is then decoration, " +
+			"and nothing in the corpus would notice if the filter stopped working")
+	}
+	if open == 0 {
+		t.Error("every document is restricted; a corpus talent cannot read at all " +
+			"is one they will stop asking")
+	}
+	t.Logf("%d restricted to a role, %d open to the tenant", restricted, open)
+}
+
+// The boundary, over the documents that actually ship.
+//
+// Ingested into a throwaway tenant and queried as two roles. An admin-only
+// document reaching a talent caller is a leak of this organisation's own
+// guidance to its own workers, which is the quiet kind: same tenant, same
+// corpus, wrong reader.
+func TestShippedCorpusIsRetrievableAndScoped(t *testing.T) {
+	h := testutil.New(t)
+	ctx := context.Background()
+	org := freshOrg(t, h, "shipped-corpus")
+
+	docs := loadCorpus(t)
+	ing := knowledge.NewIngester(h.Pool, knowledge.NewLexical(128))
+
+	var restrictedTitles []string
+	for _, d := range docs {
+		aud, err := audienceFrom(d.audience)
+		if err != nil {
+			t.Fatalf("%s: %v", d.name, err)
+		}
+		if _, err := ing.Ingest(ctx, org, knowledge.Document{
+			Source: d.source, ExternalID: strings.TrimSuffix(d.name, ".md"),
+			Title: d.title, Audience: aud, Body: d.body,
+		}); err != nil {
+			t.Fatalf("ingest %s: %v", d.name, err)
+		}
+		if len(aud.Roles) > 0 && !aud.Tenant {
+			restrictedTitles = append(restrictedTitles, d.title)
+		}
+	}
+
+	ask := func(role, email, text string) []knowledge.Result {
+		res, err := retriever(h).Retrieve(ctx, knowledge.Query{
+			Text: text,
+			Principal: authctx.Identity{
+				UserID: "00000000-0000-0000-0000-000000000301",
+				OrgID:  org, Role: role, Email: email,
+			},
+			Sources: []string{"policy_docs"}, K: 12,
+		})
+		if err != nil {
+			t.Fatalf("retrieve as %s: %v", role, err)
+		}
+		return res.Chunks
+	}
+
+	// The corpus answers at all.
+	if got := ask("admin", "boss@corpus.test", "shift cover cancellation notice"); len(got) == 0 {
+		t.Error("an admin asking about shift cover retrieved nothing from the shipped corpus")
+	}
+
+	// And the restricted documents stay behind the role that owns them.
+	talent := ask("talent", "worker@corpus.test", strings.Join(restrictedTitles, " "))
+	for _, c := range talent {
+		for _, title := range restrictedTitles {
+			if c.Title == title {
+				t.Errorf("talent retrieved %q, which is restricted to a role", title)
+			}
+		}
+	}
+	if len(restrictedTitles) > 0 {
+		admin := ask("admin", "boss@corpus.test", strings.Join(restrictedTitles, " "))
+		var reached bool
+		for _, c := range admin {
+			for _, title := range restrictedTitles {
+				if c.Title == title {
+					reached = true
+				}
+			}
+		}
+		if !reached {
+			t.Error("an admin could not reach a document restricted to admins — " +
+				"the filter is not scoping, it is refusing")
+		}
+	}
+}
+
+// audienceFrom parses the front-matter `audience:` line the way cmd/ingest does.
+//
+// Deliberately a second implementation rather than an import: cmd/ingest is a
+// main package and cannot be imported, and a test that reused the parser under
+// test could not catch the parser being wrong about the files. This agreeing
+// with ingest is the point — where they disagree, one of them is mis-reading a
+// document's readers.
+func audienceFrom(raw string) (knowledge.Audience, error) {
+	var a knowledge.Audience
+	if strings.TrimSpace(raw) == "" {
+		return a, errors.New("no audience declared")
+	}
+	for _, part := range strings.Split(raw, ",") {
+		part = strings.TrimSpace(part)
+		switch {
+		case part == "tenant":
+			a.Tenant = true
+		case strings.HasPrefix(part, "role:"):
+			role, ok := domain.ParseRole(strings.TrimPrefix(part, "role:"))
+			if !ok {
+				return a, fmt.Errorf("%q is not a role", part)
+			}
+			a.Roles = append(a.Roles, role)
+		case strings.HasPrefix(part, "email:"):
+			a.Emails = append(a.Emails, strings.TrimPrefix(part, "email:"))
+		default:
+			return a, fmt.Errorf("unrecognised audience %q", part)
+		}
+	}
+	return a, nil
+}