backend_fiesta/scratch/cataloguedims/main.go

// Reports how the catalogue's `embedding` columns are shaped — width, how
// many rows are filled, and a sample norm — so the embedding model Fiesta
// calls can be matched to the one that indexed the catalogue. Metadata and
// counts only, on a read-only transaction; it never writes.
//
//	go run ./scratch/cataloguedims   # reads CATALOGUE_DB_* from .env.production
package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"

	"github.com/joho/godotenv"
	"gorm.io/driver/postgres"
	"gorm.io/gorm"
)

func main() {
	sample := flag.String("sample", "", "print one row's texts and stored vector from this table, to check which model produced it")
	flag.Parse()
	_ = godotenv.Load(".env.production")
	dsn := url.URL{
		Scheme: "postgres",
		User:   url.UserPassword(os.Getenv("CATALOGUE_DB_USER"), os.Getenv("CATALOGUE_DB_PASSWORD")),
		Host:   os.Getenv("CATALOGUE_DB_HOST") + ":" + os.Getenv("CATALOGUE_DB_PORT"),
		Path:   "/" + os.Getenv("CATALOGUE_DB_NAME"),
	}
	q := dsn.Query()
	q.Set("sslmode", "disable")
	q.Set("default_transaction_read_only", "on")
	dsn.RawQuery = q.Encode()

	db, err := gorm.Open(postgres.Open(dsn.String()), &gorm.Config{})
	if err != nil {
		log.Fatal(err)
	}

	if *sample != "" {
		var row struct {
			ProductName string
			Title       string
			SearchQuery string
			Embedding   string
		}
		db.Raw(fmt.Sprintf(`SELECT product_name, COALESCE(title, '') AS title, COALESCE(search_query, '') AS search_query,
			embedding::text AS embedding FROM %s WHERE embedding IS NOT NULL ORDER BY id LIMIT 1`, *sample)).Scan(&row)
		fmt.Printf("product_name: %s\ntitle: %s\nsearch_query: %s\nembedding: %s\n", row.ProductName, row.Title, row.SearchQuery, row.Embedding)
		return
	}

	var cols []struct {
		Relname   string
		Attname   string
		Typname   string
		Atttypmod int
	}
	if err := db.Raw(`
		SELECT c.relname, a.attname, t.typname, a.atttypmod
		FROM pg_attribute a
		JOIN pg_class c ON c.oid = a.attrelid
		JOIN pg_type  t ON t.oid = a.atttypid
		WHERE t.typname = 'vector' AND a.attnum > 0 AND c.relname LIKE 'brand\_%'
		ORDER BY c.relname, a.attname`).Scan(&cols).Error; err != nil {
		log.Fatal(err)
	}
	if len(cols) == 0 {
		fmt.Println("no brand_* table has a vector column")
		return
	}
	fmt.Printf("%-24s %-16s %-8s %5s %5s %5s\n", "table", "column", "type", "dims", "rows", "filled")
	for _, c := range cols {
		var total, filled int64
		db.Raw(fmt.Sprintf(`SELECT COUNT(1) FROM %s`, c.Relname)).Scan(&total)
		db.Raw(fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE %s IS NOT NULL`, c.Relname, c.Attname)).Scan(&filled)
		fmt.Printf("%-24s %-16s %-8s %5d %5d %5d\n", c.Relname, c.Attname, c.Typname, c.Atttypmod, total, filled)
	}

	// nomic/bge emit unit vectors; a norm far from 1 means another pipeline.
	for _, c := range cols {
		var norm float64
		db.Raw(fmt.Sprintf(`SELECT vector_norm(embedding) FROM %s WHERE embedding IS NOT NULL LIMIT 1`, c.Relname)).Scan(&norm)
		if norm > 0 {
			fmt.Printf("sample vector norm (%s): %.4f\n", c.Relname, norm)
			break
		}
	}
}