text search
This commit is contained in:
@@ -467,9 +467,22 @@ func (r *scanRepository) VectorSearch(ctx context.Context, vector []float32, lim
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// minTokenHits is how many of the label's words a row must carry to be worth
|
||||
// looking at. Every word was once required, which meant a single word the
|
||||
// catalogue does not use — "Parle G biscuit pack", "Milk Bikis pack" — kept
|
||||
// the right product out of the result entirely, leaving the vector search to
|
||||
// answer alone and confidently wrong. Most of them is enough; scoring sorts
|
||||
// out the rest.
|
||||
func minTokenHits(n int) int {
|
||||
if n <= 2 {
|
||||
return n
|
||||
}
|
||||
return (n*2 + 2) / 3 // two thirds, rounded up; never below 2 for n >= 3
|
||||
}
|
||||
|
||||
// TextSearch is the fallback when there is no embedder, and the tie-breaker
|
||||
// beside it when there is: rows whose name or title contains the label, or
|
||||
// contains every word of it.
|
||||
// carry most of its words.
|
||||
func (r *scanRepository) TextSearch(ctx context.Context, label string, limit int) ([]CatalogueHit, error) {
|
||||
tables, err := r.brandTables(ctx)
|
||||
if err != nil {
|
||||
@@ -495,18 +508,32 @@ func (r *scanRepository) TextSearch(ctx context.Context, label string, limit int
|
||||
hay = "LOWER(COALESCE(product_name, '') || ' ' || COALESCE(title, '') || ' ' || COALESCE(search_query, ''))"
|
||||
}
|
||||
|
||||
conds := []string{hay + " LIKE ?"}
|
||||
args = append(args, "%"+label+"%")
|
||||
all := make([]string, 0, len(tokens))
|
||||
for _, tok := range tokens {
|
||||
all = append(all, hay+" LIKE ?")
|
||||
args = append(args, "%"+tok+"%")
|
||||
// How well a row matches, as a number: the whole label as a substring
|
||||
// outweighs any number of loose words, then one point per word found.
|
||||
hits := make([]string, 0, len(tokens)+1)
|
||||
hits = append(hits, "(CASE WHEN "+hay+" LIKE ? THEN 100 ELSE 0 END)")
|
||||
for range tokens {
|
||||
hits = append(hits, "(CASE WHEN "+hay+" LIKE ? THEN 1 ELSE 0 END)")
|
||||
}
|
||||
conds = append(conds, "("+strings.Join(all, " AND ")+")")
|
||||
rank := strings.Join(hits, " + ")
|
||||
|
||||
// The expression appears twice in the SQL — once to filter, once to
|
||||
// order — so its arguments are bound twice, in that order.
|
||||
bind := func() {
|
||||
args = append(args, "%"+label+"%")
|
||||
for _, tok := range tokens {
|
||||
args = append(args, "%"+tok+"%")
|
||||
}
|
||||
}
|
||||
bind()
|
||||
bind()
|
||||
|
||||
// Ordering matters as much as the threshold: a looser WHERE lets more
|
||||
// rows qualify, and an unordered LIMIT would then be free to return
|
||||
// the wrong ones. Best match per brand first, id to keep it stable.
|
||||
branches = append(branches, fmt.Sprintf(
|
||||
`(SELECT %s, -1::float8 AS distance FROM %s WHERE %s LIMIT %d)`,
|
||||
hitColumns(brand, cols), table, strings.Join(conds, " OR "), limit))
|
||||
`(SELECT %s, -1::float8 AS distance FROM %s WHERE (%s) >= %d ORDER BY (%s) DESC, id LIMIT %d)`,
|
||||
hitColumns(brand, cols), table, rank, minTokenHits(len(tokens)), rank, limit))
|
||||
}
|
||||
if len(branches) == 0 {
|
||||
return nil, nil
|
||||
|
||||
Reference in New Issue
Block a user