agent

services/tools/help_test.go

@@ -0,0 +1,238 @@
+package tools
+
+import (
+	"strings"
+	"testing"
+)
+
+func loadedHelp(t *testing.T) []HelpEntry {
+	t.Helper()
+	corpus, err := LoadHelp()
+	if err != nil {
+		t.Fatalf("the shipped corpus does not load: %v", err)
+	}
+	return corpus
+}
+
+func ask(t *testing.T, question string) []HelpAnswer {
+	t.Helper()
+	result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": question}, merchantCaller)
+	if err != nil {
+		t.Fatalf("asking %q: %v", question, err)
+	}
+	answers, ok := result.Rows.([]HelpAnswer)
+	if !ok {
+		t.Fatalf("rows are not help answers: %T", result.Rows)
+	}
+	return answers
+}
+
+/* ── The question phase 4 is measured on ───────────────────────────────── */
+
+func TestHowDoIAddACashierIsAnsweredFromTheCorpus(t *testing.T) {
+	answers := ask(t, "How do I add a cashier?")
+	if len(answers) == 0 {
+		t.Fatal("the corpus answered nothing")
+	}
+	if !strings.Contains(strings.ToLower(answers[0].Answer), "supervisor") {
+		t.Fatalf("the top answer is not about till accounts: %q", answers[0].Question)
+	}
+}
+
+func TestTheSameQuestionInSomebodyElsesWords(t *testing.T) {
+	// A shopkeeper does not type the heading. `also` carries the phrasings they
+	// actually use, which is most of why keyword matching is enough here.
+	for _, phrasing := range []string{
+		"how do I create a till account",
+		"give someone access to the terminal",
+	} {
+		answers := ask(t, phrasing)
+		if len(answers) == 0 {
+			t.Fatalf("no answer for %q", phrasing)
+		}
+	}
+}
+
+func TestQuestionsAcrossTheCorpusFindTheirOwnEntry(t *testing.T) {
+	// Asserts which ENTRY came back, not which words are in it. Checking for a
+	// phrase in the body ties the test to wording that is meant to be rewritten
+	// as the source comments change — the first version of this failed because
+	// the re-import answer says "second copies" rather than "duplicates", which
+	// was the test being wrong rather than the retrieval.
+	for question, wantEntry := range map[string]string{
+		"why can I not delete someone":                          "Why can I not delete someone?",
+		"what happens when I assign a rider":                    "What happens when I assign a rider?",
+		"will importing again create duplicates":                "What does re-importing a product do?",
+		"why is a product in my catalogue but not on the shelf": "Why is a product in my catalogue but not on the shelf?",
+		"why do my counter takings not show in revenue":         "Why do online and counter sales not add up to one total?",
+	} {
+		answers := ask(t, question)
+		if len(answers) == 0 {
+			t.Fatalf("no answer for %q", question)
+		}
+		if answers[0].Question != wantEntry {
+			t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry)
+		}
+	}
+}
+
+/* ── Saying nothing, rather than inventing ─────────────────────────────── */
+
+func TestAQuestionTheCorpusDoesNotCoverSaysSo(t *testing.T) {
+	// A model handed an empty list will otherwise answer from what it knows
+	// about retail software in general, which is the behaviour this whole
+	// design exists to prevent.
+	result, err := call1(t, Help(loadedHelp(t)),
+		map[string]any{"question": "what is the capital of France"}, merchantCaller)
+	if err != nil {
+		t.Fatalf("asking: %v", err)
+	}
+	if result.Count != 0 {
+		t.Fatalf("an unrelated question matched %d entries", result.Count)
+	}
+	if !strings.Contains(strings.ToLower(result.Note), "do not answer from general knowledge") {
+		t.Fatalf("nothing told the model to stop: %q", result.Note)
+	}
+}
+
+func TestAnswersAreCappedSoOneQuestionDoesNotReturnTheBook(t *testing.T) {
+	result, _ := call1(t, Help(loadedHelp(t)),
+		map[string]any{"question": "account cashier till console stock order delivery catalogue product"}, merchantCaller)
+	if result.Count > helpTopK {
+		t.Fatalf("%d answers came back", result.Count)
+	}
+}
+
+/* ── Retrieved text is data, not instructions ──────────────────────────── */
+
+func TestAPassageIsLabelledAsReferenceMaterial(t *testing.T) {
+	// The corpus is written by us today, so this is belt and braces — but it is
+	// meant to grow from generated text, and the label has to exist before it
+	// does.
+	result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, merchantCaller)
+	if !strings.Contains(strings.ToLower(result.Note), "not instructions") {
+		t.Fatalf("passages are not labelled as data: %q", result.Note)
+	}
+}
+
+/* ── Nothing about a real shop ships ───────────────────────────────────── */
+
+func TestTheShippedCorpusCarriesNoShopData(t *testing.T) {
+	// The comments this is drawn from are full of it: row counts for named
+	// tenants, a terminal id from a specific shop. Useful to a developer, and
+	// one merchant's data if it reaches another merchant's screen.
+	if _, err := LoadHelp(); err != nil {
+		t.Fatalf("the corpus carries production detail: %v", err)
+	}
+}
+
+func TestAPassageCarryingATenantIsRefused(t *testing.T) {
+	for _, body := range []string{
+		"Measured on tenant 1147, all 181 rows were empty.",
+		"A till named store_id=1185 in a URL and was believed.",
+		"Terminal T5EDD stopped reporting in.",
+		"Rider 897 carries both a name and a status.",
+	} {
+		err := checkCorpusSafety("test.md", HelpEntry{Question: "q", Body: body})
+		if err == nil {
+			t.Fatalf("this would have shipped: %q", body)
+		}
+	}
+}
+
+func TestOrdinaryHelpTextIsNotFlagged(t *testing.T) {
+	// A check that refuses everything protects nothing, because the next person
+	// turns it off.
+	err := checkCorpusSafety("test.md", HelpEntry{
+		Question: "How do I add a cashier?",
+		Body:     "There are two till roles: a supervisor runs the terminal, a cashier bills and nothing else.",
+	})
+	if err != nil {
+		t.Fatalf("ordinary help was refused: %v", err)
+	}
+}
+
+/* ── Provenance ────────────────────────────────────────────────────────── */
+
+func TestEveryEntryNamesWhereItsWordingCameFrom(t *testing.T) {
+	// A stale entry has to be traceable to the comment that moved on without it.
+	for _, entry := range loadedHelp(t) {
+		if strings.TrimSpace(entry.Source) == "" {
+			t.Fatalf("entry %q cites no source", entry.Question)
+		}
+	}
+}
+
+func TestTheSourceReachesTheAnswer(t *testing.T) {
+	answers := ask(t, "how do I add a cashier")
+	if answers[0].Source == "" {
+		t.Fatal("the answer dropped its provenance")
+	}
+}
+
+/* ── Parsing ───────────────────────────────────────────────────────────── */
+
+func TestAnEntryWithNoAnswerIsRefused(t *testing.T) {
+	if _, err := parseHelpEntry("---\nquestion: q\n---\n"); err == nil {
+		t.Fatal("an entry with no body was accepted")
+	}
+}
+
+func TestAnEntryWithNoQuestionIsRefused(t *testing.T) {
+	if _, err := parseHelpEntry("---\narea: people\n---\nsome words"); err == nil {
+		t.Fatal("an entry with no question was accepted")
+	}
+}
+
+func TestAFileWithNoFrontMatterIsRefused(t *testing.T) {
+	if _, err := parseHelpEntry("just some prose"); err == nil {
+		t.Fatal("a file with no front matter was accepted")
+	}
+}
+
+/* ── Scoring ───────────────────────────────────────────────────────────── */
+
+func TestATitleMatchOutranksTheSameWordBuriedInABody(t *testing.T) {
+	// Otherwise "delivery" appearing once in a long answer outranks the entry
+	// whose title is the question being asked.
+	titled := HelpEntry{Question: "How do I add a cashier?", Body: "unrelated words entirely"}
+	buried := HelpEntry{Question: "Something else", Body: strings.Repeat("cashier ", 20)}
+
+	want := terms("how do I add a cashier")
+	if score(titled, want, "") <= score(buried, want, "") {
+		t.Fatal("a buried mention outranked a title match")
+	}
+}
+
+func TestTheAreaHintNudgesButDoesNotFilter(t *testing.T) {
+	// Somebody on the Sales page asking about cashiers should still be answered.
+	answers := ask(t, "how do I add a cashier")
+	if len(answers) == 0 {
+		t.Fatal("no answer without an area")
+	}
+
+	result, err := call1(t, Help(loadedHelp(t)),
+		map[string]any{"question": "how do I add a cashier", "area": "orders"}, merchantCaller)
+	if err != nil {
+		t.Fatalf("asking: %v", err)
+	}
+	if result.Count == 0 {
+		t.Fatal("a mismatched area filtered the answer away")
+	}
+}
+
+/* ── Scope ─────────────────────────────────────────────────────────────── */
+
+func TestHelpNeedsNoTenant(t *testing.T) {
+	// The only tool without a tenant check: this corpus describes the product
+	// and contains nothing about any shop. Refusing staff a help question would
+	// be for no reason.
+	result, err := call1(t, Help(loadedHelp(t)),
+		map[string]any{"question": "how do I add a cashier"}, Caller{Userid: 12, Superadmin: true})
+	if err != nil {
+		t.Fatalf("staff were refused a help question: %v", err)
+	}
+	if result.Count == 0 {
+		t.Fatal("staff got no answer")
+	}
+}