backend_fiesta/services/tools/help_test.go

package tools

import (
	"strings"
	"testing"
)

func loadedHelp(t *testing.T) []HelpEntry {
	t.Helper()
	corpus, err := LoadHelp()
	if err != nil {
		t.Fatalf("the shipped corpus does not load: %v", err)
	}
	return corpus
}

func ask(t *testing.T, question string) []HelpAnswer {
	t.Helper()
	result, err := call1(t, Help(loadedHelp(t)), map[string]any{"question": question}, merchantCaller)
	if err != nil {
		t.Fatalf("asking %q: %v", question, err)
	}
	answers, ok := result.Rows.([]HelpAnswer)
	if !ok {
		t.Fatalf("rows are not help answers: %T", result.Rows)
	}
	return answers
}

/* ── The question phase 4 is measured on ───────────────────────────────── */

func TestHowDoIAddACashierIsAnsweredFromTheCorpus(t *testing.T) {
	answers := ask(t, "How do I add a cashier?")
	if len(answers) == 0 {
		t.Fatal("the corpus answered nothing")
	}
	if !strings.Contains(strings.ToLower(answers[0].Answer), "supervisor") {
		t.Fatalf("the top answer is not about till accounts: %q", answers[0].Question)
	}
}

func TestTheSameQuestionInSomebodyElsesWords(t *testing.T) {
	// A shopkeeper does not type the heading. `also` carries the phrasings they
	// actually use, which is most of why keyword matching is enough here.
	for _, phrasing := range []string{
		"how do I create a till account",
		"give someone access to the terminal",
	} {
		answers := ask(t, phrasing)
		if len(answers) == 0 {
			t.Fatalf("no answer for %q", phrasing)
		}
	}
}

func TestQuestionsAcrossTheCorpusFindTheirOwnEntry(t *testing.T) {
	// Asserts which ENTRY came back, not which words are in it. Checking for a
	// phrase in the body ties the test to wording that is meant to be rewritten
	// as the source comments change — the first version of this failed because
	// the re-import answer says "second copies" rather than "duplicates", which
	// was the test being wrong rather than the retrieval.
	for question, wantEntry := range map[string]string{
		"why can I not delete someone":                          "Why can I not delete someone?",
		"what happens when I assign a rider":                    "What happens when I assign a rider?",
		"will importing again create duplicates":                "What does re-importing a product do?",
		"why is a product in my catalogue but not on the shelf": "Why is a product in my catalogue but not on the shelf?",
		"why do my counter takings not show in revenue":         "Why do online and counter sales not add up to one total?",
	} {
		answers := ask(t, question)
		if len(answers) == 0 {
			t.Fatalf("no answer for %q", question)
		}
		if answers[0].Question != wantEntry {
			t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry)
		}
	}
}

/* ── Saying nothing, rather than inventing ─────────────────────────────── */

func TestAQuestionTheCorpusDoesNotCoverSaysSo(t *testing.T) {
	// A model handed an empty list will otherwise answer from what it knows
	// about retail software in general, which is the behaviour this whole
	// design exists to prevent.
	result, err := call1(t, Help(loadedHelp(t)),
		map[string]any{"question": "what is the capital of France"}, merchantCaller)
	if err != nil {
		t.Fatalf("asking: %v", err)
	}
	if result.Count != 0 {
		t.Fatalf("an unrelated question matched %d entries", result.Count)
	}
	if !strings.Contains(strings.ToLower(result.Note), "do not answer from general knowledge") {
		t.Fatalf("nothing told the model to stop: %q", result.Note)
	}
}

func TestAnswersAreCappedSoOneQuestionDoesNotReturnTheBook(t *testing.T) {
	result, _ := call1(t, Help(loadedHelp(t)),
		map[string]any{"question": "account cashier till console stock order delivery catalogue product"}, merchantCaller)
	if result.Count > helpTopK {
		t.Fatalf("%d answers came back", result.Count)
	}
}

/* ── Retrieved text is data, not instructions ──────────────────────────── */

func TestAPassageIsLabelledAsReferenceMaterial(t *testing.T) {
	// The corpus is written by us today, so this is belt and braces — but it is
	// meant to grow from generated text, and the label has to exist before it
	// does.
	result, _ := call1(t, Help(loadedHelp(t)), map[string]any{"question": "how do I add a cashier"}, merchantCaller)
	if !strings.Contains(strings.ToLower(result.Note), "not instructions") {
		t.Fatalf("passages are not labelled as data: %q", result.Note)
	}
}

/* ── Nothing about a real shop ships ───────────────────────────────────── */

func TestTheShippedCorpusCarriesNoShopData(t *testing.T) {
	// The comments this is drawn from are full of it: row counts for named
	// tenants, a terminal id from a specific shop. Useful to a developer, and
	// one merchant's data if it reaches another merchant's screen.
	if _, err := LoadHelp(); err != nil {
		t.Fatalf("the corpus carries production detail: %v", err)
	}
}

func TestAPassageCarryingATenantIsRefused(t *testing.T) {
	for _, body := range []string{
		"Measured on tenant 1147, all 181 rows were empty.",
		"A till named store_id=1185 in a URL and was believed.",
		"Terminal T5EDD stopped reporting in.",
		"Rider 897 carries both a name and a status.",
	} {
		err := checkCorpusSafety("test.md", HelpEntry{Question: "q", Body: body})
		if err == nil {
			t.Fatalf("this would have shipped: %q", body)
		}
	}
}

func TestOrdinaryHelpTextIsNotFlagged(t *testing.T) {
	// A check that refuses everything protects nothing, because the next person
	// turns it off.
	err := checkCorpusSafety("test.md", HelpEntry{
		Question: "How do I add a cashier?",
		Body:     "There are two till roles: a supervisor runs the terminal, a cashier bills and nothing else.",
	})
	if err != nil {
		t.Fatalf("ordinary help was refused: %v", err)
	}
}

/* ── Provenance ────────────────────────────────────────────────────────── */

func TestEveryEntryNamesWhereItsWordingCameFrom(t *testing.T) {
	// A stale entry has to be traceable to the comment that moved on without it.
	for _, entry := range loadedHelp(t) {
		if strings.TrimSpace(entry.Source) == "" {
			t.Fatalf("entry %q cites no source", entry.Question)
		}
	}
}

func TestTheSourceReachesTheAnswer(t *testing.T) {
	answers := ask(t, "how do I add a cashier")
	if answers[0].Source == "" {
		t.Fatal("the answer dropped its provenance")
	}
}

/* ── Parsing ───────────────────────────────────────────────────────────── */

func TestAnEntryWithNoAnswerIsRefused(t *testing.T) {
	if _, err := parseHelpEntry("---\nquestion: q\n---\n"); err == nil {
		t.Fatal("an entry with no body was accepted")
	}
}

func TestAnEntryWithNoQuestionIsRefused(t *testing.T) {
	if _, err := parseHelpEntry("---\narea: people\n---\nsome words"); err == nil {
		t.Fatal("an entry with no question was accepted")
	}
}

func TestAFileWithNoFrontMatterIsRefused(t *testing.T) {
	if _, err := parseHelpEntry("just some prose"); err == nil {
		t.Fatal("a file with no front matter was accepted")
	}
}

/* ── Scoring ───────────────────────────────────────────────────────────── */

func TestATitleMatchOutranksTheSameWordBuriedInABody(t *testing.T) {
	// Otherwise "delivery" appearing once in a long answer outranks the entry
	// whose title is the question being asked.
	titled := HelpEntry{Question: "How do I add a cashier?", Body: "unrelated words entirely"}
	buried := HelpEntry{Question: "Something else", Body: strings.Repeat("cashier ", 20)}

	want := terms("how do I add a cashier")
	if score(titled, want, "") <= score(buried, want, "") {
		t.Fatal("a buried mention outranked a title match")
	}
}

func TestTheAreaHintNudgesButDoesNotFilter(t *testing.T) {
	// Somebody on the Sales page asking about cashiers should still be answered.
	answers := ask(t, "how do I add a cashier")
	if len(answers) == 0 {
		t.Fatal("no answer without an area")
	}

	result, err := call1(t, Help(loadedHelp(t)),
		map[string]any{"question": "how do I add a cashier", "area": "orders"}, merchantCaller)
	if err != nil {
		t.Fatalf("asking: %v", err)
	}
	if result.Count == 0 {
		t.Fatal("a mismatched area filtered the answer away")
	}
}

/* ── Scope ─────────────────────────────────────────────────────────────── */

func TestHelpNeedsNoTenant(t *testing.T) {
	// The only tool without a tenant check: this corpus describes the product
	// and contains nothing about any shop. Refusing staff a help question would
	// be for no reason.
	result, err := call1(t, Help(loadedHelp(t)),
		map[string]any{"question": "how do I add a cashier"}, Caller{Userid: 12, Superadmin: true})
	if err != nil {
		t.Fatalf("staff were refused a help question: %v", err)
	}
	if result.Count == 0 {
		t.Fatal("staff got no answer")
	}
}

/* ── The platform passages ──────────────────────────────────────────────── */

// Nearle's own staff, whose questions are about the platform rather than about
// a shop.
//
// These exist because the `platform` agent holds ONE tool — `help` — and can
// hold no other: a staff account carries no tenant, so every tool that reads a
// shop's data refuses it. The corpus is therefore the entire capability of the
// assistant in the Nearle Admin workspace, and a passage that cannot be
// retrieved is a question that has no answer at all there.
func TestThePlatformQuestionsFindTheirOwnEntry(t *testing.T) {
	for question, wantEntry := range map[string]string{
		"new outlet cannot sign in":                 "What does creating a branch actually create?",
		"do I need a user before I add a branch":    "What does creating a branch actually create?",
		"how does a shop get riders":                "What does assigning a delivery partner to a merchant do?",
		"what does partnerid zero mean":             "What does assigning a delivery partner to a merchant do?",
		"riders not showing in a new city":          "What is a district, and what happens when a partner is onboarded in a new one?",
		"nearle admin cannot see sales":             "Why can I not see a shop's orders or stock from the platform workspace?",
		"how do I check what is stuck for a tenant": "Why can I not see a shop's orders or stock from the platform workspace?",
	} {
		answers := ask(t, question)
		if len(answers) == 0 {
			t.Fatalf("no answer for %q", question)
		}
		if answers[0].Question != wantEntry {
			t.Fatalf("%q was answered with %q, wanted %q", question, answers[0].Question, wantEntry)
		}
	}
}

func TestThePlatformPassagesDoNotSwallowTheMerchantOnes(t *testing.T) {
	// Four passages were added at once, and they share vocabulary with the
	// merchant ones — branches, riders, partners, shops. A new passage that
	// outranks an existing answer breaks a question that used to work, silently,
	// and the person sees a confident answer to something they did not ask.
	for question, wantEntry := range map[string]string{
		"how do I add a cashier":                "How do I add a cashier?",
		"what happens when I assign a rider":    "What happens when I assign a rider?",
		"till login not working on the website": "What is the difference between a till account and a console login?",
		"re-import from the global catalogue":   "What does re-importing a product do?",
	} {
		answers := ask(t, question)
		if len(answers) == 0 || answers[0].Question != wantEntry {
			got := "nothing"
			if len(answers) > 0 {
				got = answers[0].Question
			}
			t.Fatalf("%q now answers with %q, wanted %q", question, got, wantEntry)
		}
	}
}

func TestEveryChipOnThePlatformPagesHasAnAnswer(t *testing.T) {
	// The console's own chips, verbatim from `assistantContext.ts`.
	//
	// The rule that file states: a page is only given an agent when that agent
	// can answer every chip on it, because a chip coming back "I cannot look
	// that up" reads as a broken assistant rather than an unbuilt feature. In
	// the platform workspace the corpus IS the assistant, so a chip with no
	// passage behind it is a dead button.
	//
	// Duplicated here rather than imported because the two repositories deploy
	// separately: this is the half that can fail on its own.
	for _, chip := range []string{
		"What does creating a branch actually create?",
		"Why can I not see a shop's orders from here?",
		"How does a shop get riders?",
		"What is the difference between a till account and a console login?",
		"New outlet cannot sign in",
		"What is a district?",
		"What does re-importing a product do?",
		"Why is a product in my catalogue but not on the shelf?",
		"Why can I not see a shop's stock from here?",
	} {
		if answers := ask(t, chip); len(answers) == 0 {
			t.Fatalf("the chip %q has no passage behind it — it is a dead button", chip)
		}
	}
}