image search test

utils/geo.go

@@ -69,23 +69,61 @@ func parseClock(s string) (int, bool) {
 }
 
 // SearchTokens splits a label into the words worth matching on: lowercased,
-// punctuation stripped, single characters and pack-size noise dropped. "Milk
-// Bikis 100g" → ["milk", "bikis"]; the size is matched separately, if at all.
+// punctuation stripped, pack-size noise dropped. "Milk Bikis 100g" →
+// ["milk", "bikis"]; the size is matched separately, if at all.
+//
+// A single character is kept when it follows a word, because in this
+// catalogue that character is often the whole product: the "G" in "Parle G",
+// the "K" in "Special K". Dropping it made "Parle G" score the same against
+// "Parle-G Original Glucose Biscuits" as against "Parle Monaco Classic", and
+// the tie went to Monaco. It is still dropped when it stands alone — a
+// one-letter label is not a search — and bare multipliers ("2 x 50gm") are
+// never words.
 func SearchTokens(label string) []string {
 	var tokens []string
 	seen := make(map[string]bool)
+	kept := 0 // multi-character tokens so far: a lone letter needs one
 	for _, raw := range strings.FieldsFunc(strings.ToLower(label), func(r rune) bool {
 		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
 	}) {
-		if len(raw) < 2 || isPackSize(raw) || seen[raw] {
+		if isPackSize(raw) || seen[raw] {
 			continue
 		}
+		if len(raw) < 2 && (kept == 0 || isMultiplier(raw)) {
+			continue
+		}
+		if len(raw) > 1 {
+			kept++
+		}
 		seen[raw] = true
 		tokens = append(tokens, raw)
 	}
 	return tokens
 }
 
+// isMultiplier is the "x" of "2 x 50gm" and the "n" of a multipack — a single
+// character that joins sizes rather than naming a product.
+func isMultiplier(tok string) bool {
+	return tok == "x" || tok == "n"
+}
+
+// FoldSeparators turns every run of punctuation into one space, so a label
+// typed without the brand's own punctuation still matches it: "Parle G" and
+// "Parle-G" both fold to "parle g". Lens reads letterforms off a packet, and
+// people type what they see, so the hyphen is not reliably either present or
+// absent on the way in.
+func FoldSeparators(s string) string {
+	return strings.Join(strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
+		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
+	}), " ")
+}
+
+// TightenLabel removes separators outright rather than folding them, catching
+// the other way people write a hyphenated name: "ParleG" against "Parle-G".
+func TightenLabel(s string) string {
+	return strings.ReplaceAll(FoldSeparators(s), " ", "")
+}
+
 // isPackSize is "100g", "1kg", "500ml", "2l", "250gm" — a number with a unit
 // glued on, or a bare number.
 func isPackSize(tok string) bool {