239 lines
9.1 KiB
Go
239 lines
9.1 KiB
Go
package tools
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func loadedHelp(t *testing.T) []HelpEntry {
|
|
t.Helper()
|
|
corpus, err := LoadHelp()
|
|
if err != nil {
|
|
t.Fatalf("the shipped corpus does not load: %v", err)
|
|
}
|
|
return corpus
|
|
}
|
|
|
|
func ask(t *testing.T, question string) []HelpAnswer {
|
|
t.Helper()
|
|
result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": question}, merchantCaller)
|
|
if err != nil {
|
|
t.Fatalf("asking %q: %v", question, err)
|
|
}
|
|
answers, ok := result.Rows.([]HelpAnswer)
|
|
if !ok {
|
|
t.Fatalf("rows are not help answers: %T", result.Rows)
|
|
}
|
|
return answers
|
|
}
|
|
|
|
/* ── The question phase 4 is measured on ───────────────────────────────── */
|
|
|
|
func TestHowDoIAddACashierIsAnsweredFromTheCorpus(t *testing.T) {
|
|
answers := ask(t, "How do I add a cashier?")
|
|
if len(answers) == 0 {
|
|
t.Fatal("the corpus answered nothing")
|
|
}
|
|
if !strings.Contains(strings.ToLower(answers[0].Answer), "supervisor") {
|
|
t.Fatalf("the top answer is not about till accounts: %q", answers[0].Question)
|
|
}
|
|
}
|
|
|
|
func TestTheSameQuestionInSomebodyElsesWords(t *testing.T) {
|
|
// A shopkeeper does not type the heading. `also` carries the phrasings they
|
|
// actually use, which is most of why keyword matching is enough here.
|
|
for _, phrasing := range []string{
|
|
"how do I create a till account",
|
|
"give someone access to the terminal",
|
|
} {
|
|
answers := ask(t, phrasing)
|
|
if len(answers) == 0 {
|
|
t.Fatalf("no answer for %q", phrasing)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQuestionsAcrossTheCorpusFindTheirOwnEntry(t *testing.T) {
|
|
// Asserts which ENTRY came back, not which words are in it. Checking for a
|
|
// phrase in the body ties the test to wording that is meant to be rewritten
|
|
// as the source comments change — the first version of this failed because
|
|
// the re-import answer says "second copies" rather than "duplicates", which
|
|
// was the test being wrong rather than the retrieval.
|
|
for question, wantEntry := range map[string]string{
|
|
"why can I not delete someone": "Why can I not delete someone?",
|
|
"what happens when I assign a rider": "What happens when I assign a rider?",
|
|
"will importing again create duplicates": "What does re-importing a product do?",
|
|
"why is a product in my catalogue but not on the shelf": "Why is a product in my catalogue but not on the shelf?",
|
|
"why do my counter takings not show in revenue": "Why do online and counter sales not add up to one total?",
|
|
} {
|
|
answers := ask(t, question)
|
|
if len(answers) == 0 {
|
|
t.Fatalf("no answer for %q", question)
|
|
}
|
|
if answers[0].Question != wantEntry {
|
|
t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry)
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ── Saying nothing, rather than inventing ─────────────────────────────── */
|
|
|
|
func TestAQuestionTheCorpusDoesNotCoverSaysSo(t *testing.T) {
|
|
// A model handed an empty list will otherwise answer from what it knows
|
|
// about retail software in general, which is the behaviour this whole
|
|
// design exists to prevent.
|
|
result, err := call1(t, Help(loadedHelp(t)),
|
|
map[string]any{"question": "what is the capital of France"}, merchantCaller)
|
|
if err != nil {
|
|
t.Fatalf("asking: %v", err)
|
|
}
|
|
if result.Count != 0 {
|
|
t.Fatalf("an unrelated question matched %d entries", result.Count)
|
|
}
|
|
if !strings.Contains(strings.ToLower(result.Note), "do not answer from general knowledge") {
|
|
t.Fatalf("nothing told the model to stop: %q", result.Note)
|
|
}
|
|
}
|
|
|
|
func TestAnswersAreCappedSoOneQuestionDoesNotReturnTheBook(t *testing.T) {
|
|
result, _ := call1(t, Help(loadedHelp(t)),
|
|
map[string]any{"question": "account cashier till console stock order delivery catalogue product"}, merchantCaller)
|
|
if result.Count > helpTopK {
|
|
t.Fatalf("%d answers came back", result.Count)
|
|
}
|
|
}
|
|
|
|
/* ── Retrieved text is data, not instructions ──────────────────────────── */
|
|
|
|
func TestAPassageIsLabelledAsReferenceMaterial(t *testing.T) {
|
|
// The corpus is written by us today, so this is belt and braces — but it is
|
|
// meant to grow from generated text, and the label has to exist before it
|
|
// does.
|
|
result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, merchantCaller)
|
|
if !strings.Contains(strings.ToLower(result.Note), "not instructions") {
|
|
t.Fatalf("passages are not labelled as data: %q", result.Note)
|
|
}
|
|
}
|
|
|
|
/* ── Nothing about a real shop ships ───────────────────────────────────── */
|
|
|
|
func TestTheShippedCorpusCarriesNoShopData(t *testing.T) {
|
|
// The comments this is drawn from are full of it: row counts for named
|
|
// tenants, a terminal id from a specific shop. Useful to a developer, and
|
|
// one merchant's data if it reaches another merchant's screen.
|
|
if _, err := LoadHelp(); err != nil {
|
|
t.Fatalf("the corpus carries production detail: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAPassageCarryingATenantIsRefused(t *testing.T) {
|
|
for _, body := range []string{
|
|
"Measured on tenant 1147, all 181 rows were empty.",
|
|
"A till named store_id=1185 in a URL and was believed.",
|
|
"Terminal T5EDD stopped reporting in.",
|
|
"Rider 897 carries both a name and a status.",
|
|
} {
|
|
err := checkCorpusSafety("test.md", HelpEntry{Question: "q", Body: body})
|
|
if err == nil {
|
|
t.Fatalf("this would have shipped: %q", body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOrdinaryHelpTextIsNotFlagged(t *testing.T) {
|
|
// A check that refuses everything protects nothing, because the next person
|
|
// turns it off.
|
|
err := checkCorpusSafety("test.md", HelpEntry{
|
|
Question: "How do I add a cashier?",
|
|
Body: "There are two till roles: a supervisor runs the terminal, a cashier bills and nothing else.",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("ordinary help was refused: %v", err)
|
|
}
|
|
}
|
|
|
|
/* ── Provenance ────────────────────────────────────────────────────────── */
|
|
|
|
func TestEveryEntryNamesWhereItsWordingCameFrom(t *testing.T) {
|
|
// A stale entry has to be traceable to the comment that moved on without it.
|
|
for _, entry := range loadedHelp(t) {
|
|
if strings.TrimSpace(entry.Source) == "" {
|
|
t.Fatalf("entry %q cites no source", entry.Question)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheSourceReachesTheAnswer(t *testing.T) {
|
|
answers := ask(t, "how do I add a cashier")
|
|
if answers[0].Source == "" {
|
|
t.Fatal("the answer dropped its provenance")
|
|
}
|
|
}
|
|
|
|
/* ── Parsing ───────────────────────────────────────────────────────────── */
|
|
|
|
func TestAnEntryWithNoAnswerIsRefused(t *testing.T) {
|
|
if _, err := parseHelpEntry("---\nquestion: q\n---\n"); err == nil {
|
|
t.Fatal("an entry with no body was accepted")
|
|
}
|
|
}
|
|
|
|
func TestAnEntryWithNoQuestionIsRefused(t *testing.T) {
|
|
if _, err := parseHelpEntry("---\narea: people\n---\nsome words"); err == nil {
|
|
t.Fatal("an entry with no question was accepted")
|
|
}
|
|
}
|
|
|
|
func TestAFileWithNoFrontMatterIsRefused(t *testing.T) {
|
|
if _, err := parseHelpEntry("just some prose"); err == nil {
|
|
t.Fatal("a file with no front matter was accepted")
|
|
}
|
|
}
|
|
|
|
/* ── Scoring ───────────────────────────────────────────────────────────── */
|
|
|
|
func TestATitleMatchOutranksTheSameWordBuriedInABody(t *testing.T) {
|
|
// Otherwise "delivery" appearing once in a long answer outranks the entry
|
|
// whose title is the question being asked.
|
|
titled := HelpEntry{Question: "How do I add a cashier?", Body: "unrelated words entirely"}
|
|
buried := HelpEntry{Question: "Something else", Body: strings.Repeat("cashier ", 20)}
|
|
|
|
want := terms("how do I add a cashier")
|
|
if score(titled, want, "") <= score(buried, want, "") {
|
|
t.Fatal("a buried mention outranked a title match")
|
|
}
|
|
}
|
|
|
|
func TestTheAreaHintNudgesButDoesNotFilter(t *testing.T) {
|
|
// Somebody on the Sales page asking about cashiers should still be answered.
|
|
answers := ask(t, "how do I add a cashier")
|
|
if len(answers) == 0 {
|
|
t.Fatal("no answer without an area")
|
|
}
|
|
|
|
result, err := call1(t, Help(loadedHelp(t)),
|
|
map[string]any{"question": "how do I add a cashier", "area": "orders"}, merchantCaller)
|
|
if err != nil {
|
|
t.Fatalf("asking: %v", err)
|
|
}
|
|
if result.Count == 0 {
|
|
t.Fatal("a mismatched area filtered the answer away")
|
|
}
|
|
}
|
|
|
|
/* ── Scope ─────────────────────────────────────────────────────────────── */
|
|
|
|
func TestHelpNeedsNoTenant(t *testing.T) {
|
|
// The only tool without a tenant check: this corpus describes the product
|
|
// and contains nothing about any shop. Refusing staff a help question would
|
|
// be for no reason.
|
|
result, err := call1(t, Help(loadedHelp(t)),
|
|
map[string]any{"question": "how do I add a cashier"}, Caller{Userid: 12, Superadmin: true})
|
|
if err != nil {
|
|
t.Fatalf("staff were refused a help question: %v", err)
|
|
}
|
|
if result.Count == 0 {
|
|
t.Fatal("staff got no answer")
|
|
}
|
|
}
|