package tools import ( "strings" "testing" ) func loadedHelp(t *testing.T) []HelpEntry { t.Helper() corpus, err := LoadHelp() if err != nil { t.Fatalf("the shipped corpus does not load: %v", err) } return corpus } func ask(t *testing.T, question string) []HelpAnswer { t.Helper() result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": question}, merchantCaller) if err != nil { t.Fatalf("asking %q: %v", question, err) } answers, ok := result.Rows.([]HelpAnswer) if !ok { t.Fatalf("rows are not help answers: %T", result.Rows) } return answers } /* ── The question phase 4 is measured on ───────────────────────────────── */ func TestHowDoIAddACashierIsAnsweredFromTheCorpus(t *testing.T) { answers := ask(t, "How do I add a cashier?") if len(answers) == 0 { t.Fatal("the corpus answered nothing") } if !strings.Contains(strings.ToLower(answers[0].Answer), "supervisor") { t.Fatalf("the top answer is not about till accounts: %q", answers[0].Question) } } func TestTheSameQuestionInSomebodyElsesWords(t *testing.T) { // A shopkeeper does not type the heading. `also` carries the phrasings they // actually use, which is most of why keyword matching is enough here. for _, phrasing := range []string{ "how do I create a till account", "give someone access to the terminal", } { answers := ask(t, phrasing) if len(answers) == 0 { t.Fatalf("no answer for %q", phrasing) } } } func TestQuestionsAcrossTheCorpusFindTheirOwnEntry(t *testing.T) { // Asserts which ENTRY came back, not which words are in it. Checking for a // phrase in the body ties the test to wording that is meant to be rewritten // as the source comments change — the first version of this failed because // the re-import answer says "second copies" rather than "duplicates", which // was the test being wrong rather than the retrieval. for question, wantEntry := range map[string]string{ "why can I not delete someone": "Why can I not delete someone?", "what happens when I assign a rider": "What happens when I assign a rider?", "will importing again create duplicates": "What does re-importing a product do?", "why is a product in my catalogue but not on the shelf": "Why is a product in my catalogue but not on the shelf?", "why do my counter takings not show in revenue": "Why do online and counter sales not add up to one total?", } { answers := ask(t, question) if len(answers) == 0 { t.Fatalf("no answer for %q", question) } if answers[0].Question != wantEntry { t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry) } } } /* ── Saying nothing, rather than inventing ─────────────────────────────── */ func TestAQuestionTheCorpusDoesNotCoverSaysSo(t *testing.T) { // A model handed an empty list will otherwise answer from what it knows // about retail software in general, which is the behaviour this whole // design exists to prevent. result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "what is the capital of France"}, merchantCaller) if err != nil { t.Fatalf("asking: %v", err) } if result.Count != 0 { t.Fatalf("an unrelated question matched %d entries", result.Count) } if !strings.Contains(strings.ToLower(result.Note), "do not answer from general knowledge") { t.Fatalf("nothing told the model to stop: %q", result.Note) } } func TestAnswersAreCappedSoOneQuestionDoesNotReturnTheBook(t *testing.T) { result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "account cashier till console stock order delivery catalogue product"}, merchantCaller) if result.Count > helpTopK { t.Fatalf("%d answers came back", result.Count) } } /* ── Retrieved text is data, not instructions ──────────────────────────── */ func TestAPassageIsLabelledAsReferenceMaterial(t *testing.T) { // The corpus is written by us today, so this is belt and braces — but it is // meant to grow from generated text, and the label has to exist before it // does. result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, merchantCaller) if !strings.Contains(strings.ToLower(result.Note), "not instructions") { t.Fatalf("passages are not labelled as data: %q", result.Note) } } /* ── Nothing about a real shop ships ───────────────────────────────────── */ func TestTheShippedCorpusCarriesNoShopData(t *testing.T) { // The comments this is drawn from are full of it: row counts for named // tenants, a terminal id from a specific shop. Useful to a developer, and // one merchant's data if it reaches another merchant's screen. if _, err := LoadHelp(); err != nil { t.Fatalf("the corpus carries production detail: %v", err) } } func TestAPassageCarryingATenantIsRefused(t *testing.T) { for _, body := range []string{ "Measured on tenant 1147, all 181 rows were empty.", "A till named store_id=1185 in a URL and was believed.", "Terminal T5EDD stopped reporting in.", "Rider 897 carries both a name and a status.", } { err := checkCorpusSafety("test.md", HelpEntry{Question: "q", Body: body}) if err == nil { t.Fatalf("this would have shipped: %q", body) } } } func TestOrdinaryHelpTextIsNotFlagged(t *testing.T) { // A check that refuses everything protects nothing, because the next person // turns it off. err := checkCorpusSafety("test.md", HelpEntry{ Question: "How do I add a cashier?", Body: "There are two till roles: a supervisor runs the terminal, a cashier bills and nothing else.", }) if err != nil { t.Fatalf("ordinary help was refused: %v", err) } } /* ── Provenance ────────────────────────────────────────────────────────── */ func TestEveryEntryNamesWhereItsWordingCameFrom(t *testing.T) { // A stale entry has to be traceable to the comment that moved on without it. for _, entry := range loadedHelp(t) { if strings.TrimSpace(entry.Source) == "" { t.Fatalf("entry %q cites no source", entry.Question) } } } func TestTheSourceReachesTheAnswer(t *testing.T) { answers := ask(t, "how do I add a cashier") if answers[0].Source == "" { t.Fatal("the answer dropped its provenance") } } /* ── Parsing ───────────────────────────────────────────────────────────── */ func TestAnEntryWithNoAnswerIsRefused(t *testing.T) { if _, err := parseHelpEntry("---\nquestion: q\n---\n"); err == nil { t.Fatal("an entry with no body was accepted") } } func TestAnEntryWithNoQuestionIsRefused(t *testing.T) { if _, err := parseHelpEntry("---\narea: people\n---\nsome words"); err == nil { t.Fatal("an entry with no question was accepted") } } func TestAFileWithNoFrontMatterIsRefused(t *testing.T) { if _, err := parseHelpEntry("just some prose"); err == nil { t.Fatal("a file with no front matter was accepted") } } /* ── Scoring ───────────────────────────────────────────────────────────── */ func TestATitleMatchOutranksTheSameWordBuriedInABody(t *testing.T) { // Otherwise "delivery" appearing once in a long answer outranks the entry // whose title is the question being asked. titled := HelpEntry{Question: "How do I add a cashier?", Body: "unrelated words entirely"} buried := HelpEntry{Question: "Something else", Body: strings.Repeat("cashier ", 20)} want := terms("how do I add a cashier") if score(titled, want, "") <= score(buried, want, "") { t.Fatal("a buried mention outranked a title match") } } func TestTheAreaHintNudgesButDoesNotFilter(t *testing.T) { // Somebody on the Sales page asking about cashiers should still be answered. answers := ask(t, "how do I add a cashier") if len(answers) == 0 { t.Fatal("no answer without an area") } result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier", "area": "orders"}, merchantCaller) if err != nil { t.Fatalf("asking: %v", err) } if result.Count == 0 { t.Fatal("a mismatched area filtered the answer away") } } /* ── Scope ─────────────────────────────────────────────────────────────── */ func TestHelpNeedsNoTenant(t *testing.T) { // The only tool without a tenant check: this corpus describes the product // and contains nothing about any shop. Refusing staff a help question would // be for no reason. result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, Caller{Userid: 12, Superadmin: true}) if err != nil { t.Fatalf("staff were refused a help question: %v", err) } if result.Count == 0 { t.Fatal("staff got no answer") } } /* ── The platform passages ──────────────────────────────────────────────── */ // Nearle's own staff, whose questions are about the platform rather than about // a shop. // // These exist because the `platform` agent holds ONE tool — `help` — and can // hold no other: a staff account carries no tenant, so every tool that reads a // shop's data refuses it. The corpus is therefore the entire capability of the // assistant in the Nearle Admin workspace, and a passage that cannot be // retrieved is a question that has no answer at all there. func TestThePlatformQuestionsFindTheirOwnEntry(t *testing.T) { for question, wantEntry := range map[string]string{ "new outlet cannot sign in": "What does creating a branch actually create?", "do I need a user before I add a branch": "What does creating a branch actually create?", "how does a shop get riders": "What does assigning a delivery partner to a merchant do?", "what does partnerid zero mean": "What does assigning a delivery partner to a merchant do?", "riders not showing in a new city": "What is a district, and what happens when a partner is onboarded in a new one?", "nearle admin cannot see sales": "Why can I not see a shop's orders or stock from the platform workspace?", "how do I check what is stuck for a tenant": "Why can I not see a shop's orders or stock from the platform workspace?", } { answers := ask(t, question) if len(answers) == 0 { t.Fatalf("no answer for %q", question) } if answers[0].Question != wantEntry { t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry) } } } func TestThePlatformPassagesDoNotSwallowTheMerchantOnes(t *testing.T) { // Four passages were added at once, and they share vocabulary with the // merchant ones — branches, riders, partners, shops. A new passage that // outranks an existing answer breaks a question that used to work, silently, // and the person sees a confident answer to something they did not ask. for question, wantEntry := range map[string]string{ "how do I add a cashier": "How do I add a cashier?", "what happens when I assign a rider": "What happens when I assign a rider?", "till login not working on the website": "What is the difference between a till account and a console login?", "re-import from the global catalogue": "What does re-importing a product do?", } { answers := ask(t, question) if len(answers) == 0 || answers[0].Question != wantEntry { got := "nothing" if len(answers) > 0 { got = answers[0].Question } t.Fatalf("%q now answers with %q, wanted %q", question, got, wantEntry) } } } func TestEveryChipOnThePlatformPagesHasAnAnswer(t *testing.T) { // The console's own chips, verbatim from `assistantContext.ts`. // // The rule that file states: a page is only given an agent when that agent // can answer every chip on it, because a chip coming back "I cannot look // that up" reads as a broken assistant rather than an unbuilt feature. In // the platform workspace the corpus IS the assistant, so a chip with no // passage behind it is a dead button. // // Duplicated here rather than imported because the two repositories deploy // separately: this is the half that can fail on its own. for _, chip := range []string{ "What does creating a branch actually create?", "Why can I not see a shop's orders from here?", "How does a shop get riders?", "What is the difference between a till account and a console login?", "New outlet cannot sign in", "What is a district?", "What does re-importing a product do?", "Why is a product in my catalogue but not on the shelf?", "Why can I not see a shop's stock from here?", } { if answers := ask(t, chip); len(answers) == 0 { t.Fatalf("the chip %q has no passage behind it — it is a dead button", chip) } } }