// Reports how the catalogue's `embedding` columns are shaped — width, how // many rows are filled, and a sample norm — so the embedding model Fiesta // calls can be matched to the one that indexed the catalogue. Metadata and // counts only, on a read-only transaction; it never writes. // // go run ./scratch/cataloguedims # reads CATALOGUE_DB_* from .env.production package main import ( "flag" "fmt" "log" "net/url" "os" "github.com/joho/godotenv" "gorm.io/driver/postgres" "gorm.io/gorm" ) func main() { sample := flag.String("sample", "", "print one row's texts and stored vector from this table, to check which model produced it") flag.Parse() _ = godotenv.Load(".env.production") dsn := url.URL{ Scheme: "postgres", User: url.UserPassword(os.Getenv("CATALOGUE_DB_USER"), os.Getenv("CATALOGUE_DB_PASSWORD")), Host: os.Getenv("CATALOGUE_DB_HOST") + ":" + os.Getenv("CATALOGUE_DB_PORT"), Path: "/" + os.Getenv("CATALOGUE_DB_NAME"), } q := dsn.Query() q.Set("sslmode", "disable") q.Set("default_transaction_read_only", "on") dsn.RawQuery = q.Encode() db, err := gorm.Open(postgres.Open(dsn.String()), &gorm.Config{}) if err != nil { log.Fatal(err) } if *sample != "" { var row struct { ProductName string Title string SearchQuery string Embedding string } db.Raw(fmt.Sprintf(`SELECT product_name, COALESCE(title, '') AS title, COALESCE(search_query, '') AS search_query, embedding::text AS embedding FROM %s WHERE embedding IS NOT NULL ORDER BY id LIMIT 1`, *sample)).Scan(&row) fmt.Printf("product_name: %s\ntitle: %s\nsearch_query: %s\nembedding: %s\n", row.ProductName, row.Title, row.SearchQuery, row.Embedding) return } var cols []struct { Relname string Attname string Typname string Atttypmod int } if err := db.Raw(` SELECT c.relname, a.attname, t.typname, a.atttypmod FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_type t ON t.oid = a.atttypid WHERE t.typname = 'vector' AND a.attnum > 0 AND c.relname LIKE 'brand\_%' ORDER BY c.relname, a.attname`).Scan(&cols).Error; err != nil { log.Fatal(err) } if len(cols) == 0 { fmt.Println("no brand_* table has a vector column") return } fmt.Printf("%-24s %-16s %-8s %5s %5s %5s\n", "table", "column", "type", "dims", "rows", "filled") for _, c := range cols { var total, filled int64 db.Raw(fmt.Sprintf(`SELECT COUNT(1) FROM %s`, c.Relname)).Scan(&total) db.Raw(fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE %s IS NOT NULL`, c.Relname, c.Attname)).Scan(&filled) fmt.Printf("%-24s %-16s %-8s %5d %5d %5d\n", c.Relname, c.Attname, c.Typname, c.Atttypmod, total, filled) } // nomic/bge emit unit vectors; a norm far from 1 means another pipeline. for _, c := range cols { var norm float64 db.Raw(fmt.Sprintf(`SELECT vector_norm(embedding) FROM %s WHERE embedding IS NOT NULL LIMIT 1`, c.Relname)).Scan(&norm) if norm > 0 { fmt.Printf("sample vector norm (%s): %.4f\n", c.Relname, norm) break } } }