image search test

utils/geo.go

@@ -69,23 +69,61 @@ func parseClock(s string) (int, bool) {
 }
 
 // SearchTokens splits a label into the words worth matching on: lowercased,
-// punctuation stripped, single characters and pack-size noise dropped. "Milk
-// Bikis 100g" → ["milk", "bikis"]; the size is matched separately, if at all.
+// punctuation stripped, pack-size noise dropped. "Milk Bikis 100g" →
+// ["milk", "bikis"]; the size is matched separately, if at all.
+//
+// A single character is kept when it follows a word, because in this
+// catalogue that character is often the whole product: the "G" in "Parle G",
+// the "K" in "Special K". Dropping it made "Parle G" score the same against
+// "Parle-G Original Glucose Biscuits" as against "Parle Monaco Classic", and
+// the tie went to Monaco. It is still dropped when it stands alone — a
+// one-letter label is not a search — and bare multipliers ("2 x 50gm") are
+// never words.
 func SearchTokens(label string) []string {
 	var tokens []string
 	seen := make(map[string]bool)
+	kept := 0 // multi-character tokens so far: a lone letter needs one
 	for _, raw := range strings.FieldsFunc(strings.ToLower(label), func(r rune) bool {
 		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
 	}) {
-		if len(raw) < 2 || isPackSize(raw) || seen[raw] {
+		if isPackSize(raw) || seen[raw] {
 			continue
 		}
+		if len(raw) < 2 && (kept == 0 || isMultiplier(raw)) {
+			continue
+		}
+		if len(raw) > 1 {
+			kept++
+		}
 		seen[raw] = true
 		tokens = append(tokens, raw)
 	}
 	return tokens
 }
 
+// isMultiplier is the "x" of "2 x 50gm" and the "n" of a multipack — a single
+// character that joins sizes rather than naming a product.
+func isMultiplier(tok string) bool {
+	return tok == "x" || tok == "n"
+}
+
+// FoldSeparators turns every run of punctuation into one space, so a label
+// typed without the brand's own punctuation still matches it: "Parle G" and
+// "Parle-G" both fold to "parle g". Lens reads letterforms off a packet, and
+// people type what they see, so the hyphen is not reliably either present or
+// absent on the way in.
+func FoldSeparators(s string) string {
+	return strings.Join(strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
+		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
+	}), " ")
+}
+
+// TightenLabel removes separators outright rather than folding them, catching
+// the other way people write a hyphenated name: "ParleG" against "Parle-G".
+func TightenLabel(s string) string {
+	return strings.ReplaceAll(FoldSeparators(s), " ", "")
+}
+
 // isPackSize is "100g", "1kg", "500ml", "2l", "250gm" — a number with a unit
 // glued on, or a bare number.
 func isPackSize(tok string) bool {

utils/geo_test.go

@@ -63,3 +63,42 @@ func TestSearchTokens(t *testing.T) {
 		t.Error("pack sizes alone are not searchable")
 	}
 }
+
+// A single letter is often the whole product name in this catalogue, so it
+// survives when it follows a word — but not when it stands alone, and not
+// when it is the multiplier in a pack size.
+func TestSearchTokensKeepsALetterThatFollowsAWord(t *testing.T) {
+	for _, tc := range []struct {
+		label string
+		want  []string
+	}{
+		{"Parle G", []string{"parle", "g"}},
+		{"Parle-G", []string{"parle", "g"}},
+		{"Special K Original", []string{"special", "k", "original"}},
+		{"G", nil},        // a letter alone is not a search
+		{"2 x 50gm", nil}, // multiplier and pack size, no words
+		{"Milk Bikis 100g, Britannia (2 x 50gm)", []string{"milk", "bikis", "britannia"}},
+	} {
+		got := SearchTokens(tc.label)
+		if len(got) != len(tc.want) {
+			t.Fatalf("%q: got %v want %v", tc.label, got, tc.want)
+		}
+		for i := range tc.want {
+			if got[i] != tc.want[i] {
+				t.Fatalf("%q: got %v want %v", tc.label, got, tc.want)
+			}
+		}
+	}
+}
+
+func TestFoldAndTightenSeparators(t *testing.T) {
+	if got := FoldSeparators("Parle-G Original"); got != "parle g original" {
+		t.Fatalf("fold: got %q", got)
+	}
+	if FoldSeparators("Parle-G") != FoldSeparators("Parle G") {
+		t.Error("a hyphen and a space are the same separator to us")
+	}
+	if got := TightenLabel("Parle-G"); got != "parleg" {
+		t.Fatalf("tighten: got %q", got)
+	}
+}