package tools import ( "strings" "testing" ) func loadedHelp(t *testing.T) []HelpEntry { t.Helper() corpus, err := LoadHelp() if err != nil { t.Fatalf("the shipped corpus does not load: %v", err) } return corpus } func ask(t *testing.T, question string) []HelpAnswer { t.Helper() result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": question}, merchantCaller) if err != nil { t.Fatalf("asking %q: %v", question, err) } answers, ok := result.Rows.([]HelpAnswer) if !ok { t.Fatalf("rows are not help answers: %T", result.Rows) } return answers } /* ── The question phase 4 is measured on ───────────────────────────────── */ func TestHowDoIAddACashierIsAnsweredFromTheCorpus(t *testing.T) { answers := ask(t, "How do I add a cashier?") if len(answers) == 0 { t.Fatal("the corpus answered nothing") } if !strings.Contains(strings.ToLower(answers[0].Answer), "supervisor") { t.Fatalf("the top answer is not about till accounts: %q", answers[0].Question) } } func TestTheSameQuestionInSomebodyElsesWords(t *testing.T) { // A shopkeeper does not type the heading. `also` carries the phrasings they // actually use, which is most of why keyword matching is enough here. for _, phrasing := range []string{ "how do I create a till account", "give someone access to the terminal", } { answers := ask(t, phrasing) if len(answers) == 0 { t.Fatalf("no answer for %q", phrasing) } } } func TestQuestionsAcrossTheCorpusFindTheirOwnEntry(t *testing.T) { // Asserts which ENTRY came back, not which words are in it. Checking for a // phrase in the body ties the test to wording that is meant to be rewritten // as the source comments change — the first version of this failed because // the re-import answer says "second copies" rather than "duplicates", which // was the test being wrong rather than the retrieval. for question, wantEntry := range map[string]string{ "why can I not delete someone": "Why can I not delete someone?", "what happens when I assign a rider": "What happens when I assign a rider?", "will importing again create duplicates": "What does re-importing a product do?", "why is a product in my catalogue but not on the shelf": "Why is a product in my catalogue but not on the shelf?", "why do my counter takings not show in revenue": "Why do online and counter sales not add up to one total?", } { answers := ask(t, question) if len(answers) == 0 { t.Fatalf("no answer for %q", question) } if answers[0].Question != wantEntry { t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry) } } } /* ── Saying nothing, rather than inventing ─────────────────────────────── */ func TestAQuestionTheCorpusDoesNotCoverSaysSo(t *testing.T) { // A model handed an empty list will otherwise answer from what it knows // about retail software in general, which is the behaviour this whole // design exists to prevent. result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "what is the capital of France"}, merchantCaller) if err != nil { t.Fatalf("asking: %v", err) } if result.Count != 0 { t.Fatalf("an unrelated question matched %d entries", result.Count) } if !strings.Contains(strings.ToLower(result.Note), "do not answer from general knowledge") { t.Fatalf("nothing told the model to stop: %q", result.Note) } } func TestAnswersAreCappedSoOneQuestionDoesNotReturnTheBook(t *testing.T) { result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "account cashier till console stock order delivery catalogue product"}, merchantCaller) if result.Count > helpTopK { t.Fatalf("%d answers came back", result.Count) } } /* ── Retrieved text is data, not instructions ──────────────────────────── */ func TestAPassageIsLabelledAsReferenceMaterial(t *testing.T) { // The corpus is written by us today, so this is belt and braces — but it is // meant to grow from generated text, and the label has to exist before it // does. result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, merchantCaller) if !strings.Contains(strings.ToLower(result.Note), "not instructions") { t.Fatalf("passages are not labelled as data: %q", result.Note) } } /* ── Nothing about a real shop ships ───────────────────────────────────── */ func TestTheShippedCorpusCarriesNoShopData(t *testing.T) { // The comments this is drawn from are full of it: row counts for named // tenants, a terminal id from a specific shop. Useful to a developer, and // one merchant's data if it reaches another merchant's screen. if _, err := LoadHelp(); err != nil { t.Fatalf("the corpus carries production detail: %v", err) } } func TestAPassageCarryingATenantIsRefused(t *testing.T) { for _, body := range []string{ "Measured on tenant 1147, all 181 rows were empty.", "A till named store_id=1185 in a URL and was believed.", "Terminal T5EDD stopped reporting in.", "Rider 897 carries both a name and a status.", } { err := checkCorpusSafety("test.md", HelpEntry{Question: "q", Body: body}) if err == nil { t.Fatalf("this would have shipped: %q", body) } } } func TestOrdinaryHelpTextIsNotFlagged(t *testing.T) { // A check that refuses everything protects nothing, because the next person // turns it off. err := checkCorpusSafety("test.md", HelpEntry{ Question: "How do I add a cashier?", Body: "There are two till roles: a supervisor runs the terminal, a cashier bills and nothing else.", }) if err != nil { t.Fatalf("ordinary help was refused: %v", err) } } /* ── Provenance ────────────────────────────────────────────────────────── */ func TestEveryEntryNamesWhereItsWordingCameFrom(t *testing.T) { // A stale entry has to be traceable to the comment that moved on without it. for _, entry := range loadedHelp(t) { if strings.TrimSpace(entry.Source) == "" { t.Fatalf("entry %q cites no source", entry.Question) } } } func TestTheSourceReachesTheAnswer(t *testing.T) { answers := ask(t, "how do I add a cashier") if answers[0].Source == "" { t.Fatal("the answer dropped its provenance") } } /* ── Parsing ───────────────────────────────────────────────────────────── */ func TestAnEntryWithNoAnswerIsRefused(t *testing.T) { if _, err := parseHelpEntry("---\nquestion: q\n---\n"); err == nil { t.Fatal("an entry with no body was accepted") } } func TestAnEntryWithNoQuestionIsRefused(t *testing.T) { if _, err := parseHelpEntry("---\narea: people\n---\nsome words"); err == nil { t.Fatal("an entry with no question was accepted") } } func TestAFileWithNoFrontMatterIsRefused(t *testing.T) { if _, err := parseHelpEntry("just some prose"); err == nil { t.Fatal("a file with no front matter was accepted") } } /* ── Scoring ───────────────────────────────────────────────────────────── */ func TestATitleMatchOutranksTheSameWordBuriedInABody(t *testing.T) { // Otherwise "delivery" appearing once in a long answer outranks the entry // whose title is the question being asked. titled := HelpEntry{Question: "How do I add a cashier?", Body: "unrelated words entirely"} buried := HelpEntry{Question: "Something else", Body: strings.Repeat("cashier ", 20)} want := terms("how do I add a cashier") if score(titled, want, "") <= score(buried, want, "") { t.Fatal("a buried mention outranked a title match") } } func TestTheAreaHintNudgesButDoesNotFilter(t *testing.T) { // Somebody on the Sales page asking about cashiers should still be answered. answers := ask(t, "how do I add a cashier") if len(answers) == 0 { t.Fatal("no answer without an area") } result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier", "area": "orders"}, merchantCaller) if err != nil { t.Fatalf("asking: %v", err) } if result.Count == 0 { t.Fatal("a mismatched area filtered the answer away") } } /* ── Scope ─────────────────────────────────────────────────────────────── */ func TestHelpNeedsNoTenant(t *testing.T) { // The only tool without a tenant check: this corpus describes the product // and contains nothing about any shop. Refusing staff a help question would // be for no reason. result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, Caller{Userid: 12, Superadmin: true}) if err != nil { t.Fatalf("staff were refused a help question: %v", err) } if result.Count == 0 { t.Fatal("staff got no answer") } }