Loyaly/Behavision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 22 Wiki Activity

Behavision: face recognition for retail, edge to head office

Browse Source 
Five components that ship as one product:

- behavision/  the recognition engine. RTSP ingest, YuNet detection, IoU
               tracking, ArcFace embeddings, a FAISS/SQLite gallery, and a
               FastAPI dashboard. Identity is decided once per TRACK from an
               average of at least three embeddings, never per frame.
- agent/       the Go edge agent: supervises the engine, holds a durable
               spool, and drains it to MQTT. Nothing is acked before the
               broker confirms.
- desktop/     the shop PC application (Wails + React + tray).
- server/      the cloud API, MQTT consumer, reports and assistant.
- web/         platform.loyaly.ai, the head-office app, embedded in the
               server binary.

The gallery stores 512-float embeddings and timestamps - no images unless
`app.store_faces` is switched on. Those embeddings are biometric personal
data under GDPR and India's DPDP: template inversion reconstructs a
recognisable face from an ArcFace vector, so data/behavision.db is treated
as a biometric database and DELETE /api/visitors/{id} is a real erasure.

CLAUDE.md carries the reasoning behind every non-obvious decision here,
including the ones that were measured and the ones that were wrong first.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HViLj9gYNRtSr7YVZmW5sn
This commit is contained in:
Suriyakumarvijayanayagam
2026-09-04 11:14:18 +05:30
commit dad04e8cda
216 changed files with 40473 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
server/internal/store/api_admin.go Normal file
View File

@@ -0,0 +1,117 @@
package store
import (
"context"
"crypto/rand"
"encoding/base32"
"fmt"
"strings"
"time"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/auth"
)
// CreateClientWithOwner creates a tenant and the account that owns it.
//
// ONE transaction, deliberately. A client row with no owner is a tenant nobody
// can sign into, and it is invisible: it looks exactly like a normal client in
// every list, so the operator finds out weeks later when the customer says
// their login does not work. Rolling the whole thing back on a duplicate email
// is the only outcome that leaves the database describing something real.
func (s *Store) CreateClientWithOwner(ctx context.Context, in api.NewClientInput) (
api.NewClientResult, error) {
var out api.NewClientResult
password := in.Password
if password == "" {
// Generated rather than defaulted. An operator inventing a password for
// somebody else invents a weak one and then sends it over chat.
p, err := randomPassword()
if err != nil {
return out, err
}
password = p
}
hash, err := auth.HashPassword(password)
if err != nil {
// The policy message is user-facing text an operator can act on
// ("password must be at least 8 characters"), so it travels out as-is.
return out, err
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return out, err
}
defer tx.Rollback(ctx) //nolint:errcheck // no-op once committed
// No ON CONFLICT DO UPDATE here, unlike the provisioning CLI. On this path
// a clashing slug means the operator is about to hand someone else's tenant
// to a new owner; it has to fail and say so.
if err := tx.QueryRow(ctx, `
INSERT INTO clients (slug, name) VALUES ($1, $2)
RETURNING id::text`, in.Slug, in.CompanyName).Scan(&out.ClientID); err != nil {
return out, fmt.Errorf("create client: %w", err)
}
// The owner, not a manager: this is the account the customer runs their
// business from, and it must be able to add the staff who come after it.
if _, err := tx.Exec(ctx, `
INSERT INTO app_users (client_id, email, password_hash, full_name, role)
VALUES ($1::uuid, $2, $3, $4, 'owner')`,
out.ClientID, in.OwnerEmail, hash, in.OwnerName); err != nil {
return out, fmt.Errorf("create owner: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return out, err
}
out.Slug, out.OwnerEmail, out.Password = in.Slug, in.OwnerEmail, password
return out, nil
}
// ListClients is the platform-admin overview.
//
// Counts come from correlated subqueries rather than joins: a client with two
// sites and three users would otherwise appear six times and be counted wrong
// in whichever direction the operator's eye went first.
func (s *Store) ListClients(ctx context.Context) ([]api.ClientRow, error) {
rows, err := s.pool.Query(ctx, `
SELECT c.id::text, c.slug, c.name, c.created_at,
(SELECT count(*) FROM sites si WHERE si.client_id = c.id),
(SELECT count(*) FROM app_users au WHERE au.client_id = c.id)
FROM clients c
ORDER BY c.created_at DESC`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.ClientRow
for rows.Next() {
var c api.ClientRow
var at time.Time
if err := rows.Scan(&c.ID, &c.Slug, &c.Name, &at, &c.Sites, &c.Users); err != nil {
return nil, err
}
c.CreatedAt = at.UTC().Format(time.RFC3339)
out = append(out, c)
}
return out, rows.Err()
}
// randomPassword mints an owner's first password.
//
// base32 without padding, matching the rest of this system's generated
// secrets: it gets read down a phone line and pasted into a form, and base64's
// + / = survive neither.
func randomPassword() (string, error) {
b := make([]byte, 10) // 80 bits -> 16 characters
if _, err := rand.Read(b); err != nil {
return "", err
}
return strings.ToLower(base32.StdEncoding.
WithPadding(base32.NoPadding).EncodeToString(b)), nil
}

134
server/internal/store/api_arrivals.go Normal file
View File

@@ -0,0 +1,134 @@
package store
import (
"context"
"time"
"github.com/loyaly/behavision-server/internal/api"
)
// arrivalColumns is shared by both directions of the query below so the two
// cannot drift apart - a column present in one and missing from the other would
// mean the first poll of a feed and every poll after it returned different
// shapes, which is the kind of bug that only shows up under load.
const arrivalColumns = `
vi.id::text, vi.seq, vi.occurred_at, vi.site_id::text, si.name, vi.camera_id,
vi.is_new_visitor, vi.similarity, vi.quality, vi.attributes, vi.image_key,
COALESCE(vi.visitor_id::text, ''),
COALESCE(vs.label, ''),
COALESCE(p.full_name, '')`
const arrivalFrom = `
FROM visits vi
JOIN sites si ON si.id = vi.site_id
-- LEFT, not INNER, three times over. A visit with no visitor_id is a site
-- reporting footfall without templates; an erased customer has their
-- visitor row flagged deleted. Both are real arrivals and an inner join
-- would silently drop them, making the feed disagree with the footfall
-- report about how many people came in.
LEFT JOIN visitors vs
ON vs.id = vi.visitor_id AND vs.deleted_at IS NULL
LEFT JOIN visitor_profiles p
ON p.visitor_id = vi.visitor_id AND p.client_id = vi.client_id
WHERE vi.client_id = $1
AND ($2 = '' OR vi.site_id = $2::uuid)`
// Arrivals reads a window of the live feed, oldest first.
//
// Ordered by `seq` - the server-assigned position - and never by occurred_at.
// That is the whole correctness argument for this endpoint and it is not
// obvious, so:
//
// - occurred_at is the CAMERA's clock. Four people through one door share it
// to the microsecond, so it cannot order them; and a site that was offline
// for a day floods in carrying yesterday's timestamps, which a reader whose
// cursor has passed them would skip entirely.
// - A (occurred_at, id) tie-break does not save it either, because id is a
// random uuid: a row that COMMITS after the reader moved its cursor but
// carries a lower uuid sorts behind that cursor and is never delivered.
// Measured live before this was fixed - four simultaneous visits, two
// delivered, and nothing downstream able to tell.
//
// So the feed is ordered by when the server LEARNED of a visit. Each row still
// carries occurred_at for display; seq is only ever a position.
//
// This depends on visits being inserted one at a time, which the MQTT consumer
// guarantees with SetOrderMatters(true) - a single ordered handler goroutine,
// so seq order is commit order. Running two server instances against one
// database would break that assumption, and the fix then is a commit-ordered
// cursor, not a bigger sequence.
//
// Keyset, never OFFSET: rows arrive into this table continuously, so an offset
// shifts under the caller between polls and a feed built on it both repeats and
// skips people.
func (s *Store) Arrivals(ctx context.Context, q api.ArrivalQuery) ([]api.Arrival, error) {
var sql string
var args []any
switch {
case q.AfterSeq != nil:
sql = `SELECT ` + arrivalColumns + arrivalFrom + `
AND vi.seq > $3
ORDER BY vi.seq ASC
LIMIT $4`
args = []any{q.ClientID, q.SiteID, *q.AfterSeq, q.Limit}
case q.Since != nil:
// "Everything I have not been told about since this instant." Resolved
// against received_at, not occurred_at, so it means the same thing as
// the cursor it turns into on the next poll - a caller must not get a
// different feed depending on which of the two it started with.
sql = `SELECT ` + arrivalColumns + arrivalFrom + `
AND vi.seq > COALESCE(
(SELECT max(v2.seq) FROM visits v2
WHERE v2.client_id = $1 AND v2.received_at < $3), 0)
ORDER BY vi.seq ASC
LIMIT $4`
args = []any{q.ClientID, q.SiteID, *q.Since, q.Limit}
default:
// No cursor: an app that has just opened. It wants the last few
// arrivals, not the first few ever recorded, so take the newest rows
// and reverse them - the response is still ascending, so the caller's
// cursor handling is identical on the first poll and every one after.
sql = `SELECT * FROM (
SELECT ` + arrivalColumns + arrivalFrom + `
ORDER BY vi.seq DESC
LIMIT $3
) t ORDER BY t.seq ASC`
args = []any{q.ClientID, q.SiteID, q.Limit}
}
rows, err := s.pool.Query(ctx, sql, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.Arrival
for rows.Next() {
var a api.Arrival
var at time.Time
var sim, qual *float64
var imageKey string
if err := rows.Scan(&a.VisitID, &a.Seq, &at, &a.SiteID, &a.Site, &a.CameraID,
&a.IsNew, &sim, &qual, &a.Attributes, &imageKey,
&a.VisitorID, &a.Label, &a.Name); err != nil {
return nil, err
}
a.OccurredAt = at.UTC().Format(time.RFC3339Nano)
if sim != nil {
a.Similarity = *sim
}
if qual != nil {
a.Quality = *qual
}
// The store never presigns. It has no bucket and no idea whether this
// caller is allowed to look, and a query that mints credentials is one
// refactor away from doing it on a path that never checked. ImageKey
// is json:"-", so a handler that forgets to swap it leaks nothing.
a.ImageKey = imageKey
out = append(out, a)
}
return out, rows.Err()
}

414
server/internal/store/api_arrivals_live_test.go Normal file
View File

@@ -0,0 +1,414 @@
package store
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/loyaly/behavision-server/internal/api"
)
// Live database tests for the arrivals feed.
//
// Skipped unless TEST_DATABASE_URL is set, following the same rule as the
// bucket tests: the suite must stay runnable with no network and no services.
// They exist because the rest of the arrivals suite runs against an in-memory
// fake, and a fake cannot catch what actually goes wrong in this file - a
// keyset comparison Postgres plans differently than expected, a LEFT JOIN
// silently promoted to an inner one by a WHERE clause, a column list that
// drifts between the two directions of the query. Those only fail against a
// real planner.
//
// docker run -d -p 55432:5432 -e POSTGRES_PASSWORD=test \
// -e POSTGRES_DB=behavision pgvector/pgvector:pg16
// psql < server/migrations/*.sql
// TEST_DATABASE_URL='postgres://postgres:test@127.0.0.1:55432/behavision' go test ./internal/store/
func liveStore(t *testing.T) *Store {
t.Helper()
dsn := os.Getenv("TEST_DATABASE_URL")
if dsn == "" {
t.Skip("set TEST_DATABASE_URL to run the live store tests")
}
st, err := Open(context.Background(), dsn)
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(st.Close)
return st
}
// seedTenant builds a client, a site and n visits, and returns the client id.
// Every test gets its own tenant so they can run in any order without a
// truncate between them - and so the isolation assertions below have a real
// neighbour to be isolated from.
func seedTenant(t *testing.T, st *Store, name string, n int, withImages bool) (clientID, siteID string) {
t.Helper()
ctx := context.Background()
err := st.pool.QueryRow(ctx, `
INSERT INTO clients (name, slug) VALUES ($1, $1) RETURNING id::text`, name).Scan(&clientID)
if err != nil {
t.Fatalf("seed client: %v", err)
}
err = st.pool.QueryRow(ctx, `
INSERT INTO sites (client_id, name, slug) VALUES ($1::uuid, $2, $3)
RETURNING id::text`, clientID, name+" Main", name+"-main").Scan(&siteID)
if err != nil {
t.Fatalf("seed site: %v", err)
}
start := time.Date(2026, 9, 2, 10, 0, 0, 0, time.UTC)
for i := 0; i < n; i++ {
var visitorID string
if err := st.pool.QueryRow(ctx, `
INSERT INTO visitors (client_id, label, first_seen_at)
VALUES ($1::uuid, $2, $3) RETURNING id::text`,
clientID, fmt.Sprintf("Visitor %d", i), start).Scan(&visitorID); err != nil {
t.Fatalf("seed visitor: %v", err)
}
key := ""
if withImages {
key = fmt.Sprintf("behavision/v2/%s/main/2026/09/02/%d.jpg", name, i)
}
if _, err := st.pool.Exec(ctx, `
INSERT INTO visits (client_id, site_id, visitor_id, source_event_id,
occurred_at, camera_id, is_new_visitor,
similarity, quality, image_key)
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, 'door', $6, 0.71, 0.66, $7)`,
clientID, siteID, visitorID, fmt.Sprintf("%s-e%d", name, i),
start.Add(time.Duration(i)*time.Second), i == 0, key); err != nil {
t.Fatalf("seed visit: %v", err)
}
}
return clientID, siteID
}
func TestLiveArrivalsWalkTheFeedWithoutLosingAnyone(t *testing.T) {
st := liveStore(t)
clientID, _ := seedTenant(t, st, "walk"+stamp(), 25, true)
ctx := context.Background()
seen := map[string]int{}
// Position zero: replay from the very beginning. Positions start at 1, so
// nothing is excluded.
from := int64(0)
after := &from
for poll := 0; poll < 6; poll++ {
rows, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: clientID, AfterSeq: after, Limit: 10})
if err != nil {
t.Fatalf("poll %d: %v", poll, err)
}
for _, a := range rows {
seen[a.VisitID]++
}
if len(rows) == 0 {
break
}
last := rows[len(rows)-1].Seq
after = &last
}
if len(seen) != 25 {
t.Fatalf("saw %d of 25 visits", len(seen))
}
for id, n := range seen {
if n != 1 {
t.Errorf("visit %s delivered %d times", id, n)
}
}
}
// The case the tuple comparison exists for. Four people through a door at once
// share a timestamp to the microsecond; ordering on time alone either repeats
// them forever or skips three of them.
func TestLiveArrivalsPageThroughASimultaneousBurst(t *testing.T) {
st := liveStore(t)
name := "burst" + stamp()
clientID, siteID := seedTenant(t, st, name, 0, false)
ctx := context.Background()
at := time.Date(2026, 9, 2, 11, 0, 0, 0, time.UTC)
for i := 0; i < 4; i++ {
if _, err := st.pool.Exec(ctx, `
INSERT INTO visits (client_id, site_id, source_event_id, occurred_at, camera_id)
VALUES ($1::uuid, $2::uuid, $3, $4, 'door')`,
clientID, siteID, fmt.Sprintf("%s-b%d", name, i), at); err != nil {
t.Fatal(err)
}
}
seen := map[string]bool{}
from := int64(0)
after := &from
for poll := 0; poll < 5; poll++ {
rows, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: clientID, AfterSeq: after, Limit: 2})
if err != nil {
t.Fatal(err)
}
if len(rows) == 0 {
break
}
for _, a := range rows {
if seen[a.VisitID] {
t.Fatalf("visit %s came back twice - the cursor is stuck", a.VisitID)
}
seen[a.VisitID] = true
}
last := rows[len(rows)-1].Seq
after = &last
}
if len(seen) != 4 {
t.Fatalf("paged a 4-person burst two at a time and saw %d", len(seen))
}
}
// One tenant's feed must never contain another's customers. The site filter is
// caller-supplied, so this asks for a site id that exists - and belongs to
// somebody else.
func TestLiveArrivalsCannotReadAnotherTenant(t *testing.T) {
st := liveStore(t)
mine, _ := seedTenant(t, st, "mine"+stamp(), 3, false)
_, theirSite := seedTenant(t, st, "theirs"+stamp(), 3, false)
ctx := context.Background()
rows, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: mine, SiteID: theirSite, Limit: 50})
if err != nil {
t.Fatal(err)
}
if len(rows) != 0 {
t.Fatalf("read %d visits from another tenant's site", len(rows))
}
}
// A visit with no visitor_id is a site sending counts without templates. It is
// real footfall by an unknown person and an inner join would delete it from the
// feed while the footfall report still counted it.
func TestLiveArrivalsKeepVisitsWithNoVisitor(t *testing.T) {
st := liveStore(t)
name := "anon" + stamp()
clientID, siteID := seedTenant(t, st, name, 0, false)
ctx := context.Background()
if _, err := st.pool.Exec(ctx, `
INSERT INTO visits (client_id, site_id, source_event_id, occurred_at, camera_id)
VALUES ($1::uuid, $2::uuid, $3, now(), 'door')`,
clientID, siteID, name+"-anon"); err != nil {
t.Fatal(err)
}
rows, err := st.Arrivals(ctx, api.ArrivalQuery{ClientID: clientID, Limit: 10})
if err != nil {
t.Fatal(err)
}
if len(rows) != 1 {
t.Fatalf("an anonymous visit vanished from the feed (%d rows)", len(rows))
}
if rows[0].VisitorID != "" {
t.Errorf("visitor id should be empty, got %q", rows[0].VisitorID)
}
}
// An erased customer's visits stay, unlinked - that is the documented erasure
// contract. The feed must still show them, or a shop's live count silently
// drops every time someone exercises their rights.
func TestLiveArrivalsKeepVisitsOfAnErasedCustomer(t *testing.T) {
st := liveStore(t)
clientID, _ := seedTenant(t, st, "erased"+stamp(), 2, false)
ctx := context.Background()
if _, err := st.pool.Exec(ctx,
`UPDATE visitors SET deleted_at = now() WHERE client_id = $1::uuid`,
clientID); err != nil {
t.Fatal(err)
}
rows, err := st.Arrivals(ctx, api.ArrivalQuery{ClientID: clientID, Limit: 10})
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 {
t.Fatalf("erasing a customer removed %d visits from the feed", 2-len(rows))
}
if rows[0].Label != "" {
t.Errorf("an erased customer's label leaked into the feed: %q", rows[0].Label)
}
}
// A profile name must reach the feed, or a shop screen shows "Visitor 12" for
// a regular whose name staff typed in last week.
func TestLiveArrivalsCarryTheProfileName(t *testing.T) {
st := liveStore(t)
clientID, _ := seedTenant(t, st, "named"+stamp(), 1, false)
ctx := context.Background()
var visitorID string
if err := st.pool.QueryRow(ctx,
`SELECT id::text FROM visitors WHERE client_id = $1::uuid`, clientID).
Scan(&visitorID); err != nil {
t.Fatal(err)
}
if _, err := st.pool.Exec(ctx, `
INSERT INTO visitor_profiles (client_id, visitor_id, full_name)
VALUES ($1::uuid, $2::uuid, 'Asha Menon')`, clientID, visitorID); err != nil {
t.Fatal(err)
}
rows, err := st.Arrivals(ctx, api.ArrivalQuery{ClientID: clientID, Limit: 10})
if err != nil {
t.Fatal(err)
}
if rows[0].Name != "Asha Menon" {
t.Fatalf("profile name did not reach the feed: %q", rows[0].Name)
}
if rows[0].Label == "" {
t.Error("the system label should travel alongside the typed name")
}
}
// No cursor means "an app that has just opened": it wants the LAST few
// arrivals, not the first few ever recorded - but still ascending, so the
// caller's cursor handling is identical on every poll.
func TestLiveArrivalsFirstPollIsTheNewestWindowAscending(t *testing.T) {
st := liveStore(t)
clientID, _ := seedTenant(t, st, "newest"+stamp(), 12, false)
ctx := context.Background()
rows, err := st.Arrivals(ctx, api.ArrivalQuery{ClientID: clientID, Limit: 4})
if err != nil {
t.Fatal(err)
}
if len(rows) != 4 {
t.Fatalf("got %d rows", len(rows))
}
for i := 1; i < len(rows); i++ {
if rows[i-1].OccurredAt >= rows[i].OccurredAt {
t.Fatalf("not ascending at %d", i)
}
}
// Seeded one second apart from 10:00:00, so the newest four start at :08.
if want := "2026-09-02T10:00:08Z"; rows[0].OccurredAt != want {
t.Errorf("first poll started at %s, want the newest window at %s",
rows[0].OccurredAt, want)
}
}
func TestLiveArrivalsCarryTheImageKeyForPresigning(t *testing.T) {
st := liveStore(t)
clientID, _ := seedTenant(t, st, "img"+stamp(), 1, true)
rows, err := st.Arrivals(context.Background(),
api.ArrivalQuery{ClientID: clientID, Limit: 10})
if err != nil {
t.Fatal(err)
}
if rows[0].ImageKey == "" {
t.Fatal("no image key reached the handler, so no photo can be signed")
}
if rows[0].Image.URL != "" {
t.Error("the store must not presign - it has no bucket and checks nobody")
}
}
func stamp() string { return fmt.Sprintf("%d", time.Now().UnixNano()) }
// The regression test for the bug that shipped, and was caught only by running
// the real thing against a real broker.
//
// The feed used to be ordered by (occurred_at, id). Four people through one
// door share occurred_at to the microsecond, so the tie-break fell to `id` - a
// RANDOM uuid. A visit that committed AFTER the reader had moved its cursor but
// carried a lower uuid sorted behind that cursor and was never delivered.
// Measured live: four simultaneous visits published, two delivered, and no
// counter anywhere that would show the other two had been dropped.
//
// This reproduces the exact shape: read, move the cursor, THEN insert more rows
// carrying the same occurred_at. Every one of them must still arrive.
func TestLiveArrivalsDeliverLateInsertsThatShareATimestamp(t *testing.T) {
st := liveStore(t)
name := "late" + stamp()
clientID, siteID := seedTenant(t, st, name, 0, false)
ctx := context.Background()
// One instant for everybody - this is a single frame of one camera.
at := time.Date(2026, 9, 2, 12, 0, 0, 0, time.UTC)
insert := func(tag string) {
t.Helper()
if _, err := st.pool.Exec(ctx, `
INSERT INTO visits (client_id, site_id, source_event_id, occurred_at, camera_id)
VALUES ($1::uuid, $2::uuid, $3, $4, 'door')`,
clientID, siteID, name+"-"+tag, at); err != nil {
t.Fatal(err)
}
}
insert("a")
insert("b")
from := int64(0)
rows, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: clientID, AfterSeq: &from, Limit: 50})
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 {
t.Fatalf("first read got %d rows, want 2", len(rows))
}
cursor := rows[len(rows)-1].Seq
// Now two more arrive at the SAME instant, after the cursor has moved.
// Under the old ordering roughly half of these vanished, depending on how
// their random uuids happened to sort.
insert("c")
insert("d")
rest, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: clientID, AfterSeq: &cursor, Limit: 50})
if err != nil {
t.Fatal(err)
}
if len(rest) != 2 {
t.Fatalf("late inserts sharing a timestamp: got %d of 2 - people are being dropped from the feed",
len(rest))
}
}
// Run the same shape many times over. The old bug was probabilistic - it
// depended on how random uuids happened to sort - so a single pass could pass
// by luck. This one cannot.
func TestLiveArrivalsNeverDropAnyoneAcrossManySimultaneousBursts(t *testing.T) {
st := liveStore(t)
name := "many" + stamp()
clientID, siteID := seedTenant(t, st, name, 0, false)
ctx := context.Background()
at := time.Date(2026, 9, 2, 13, 0, 0, 0, time.UTC)
cursor := int64(0)
delivered := 0
for round := 0; round < 30; round++ {
for i := 0; i < 4; i++ {
if _, err := st.pool.Exec(ctx, `
INSERT INTO visits (client_id, site_id, source_event_id, occurred_at, camera_id)
VALUES ($1::uuid, $2::uuid, $3, $4, 'door')`,
clientID, siteID, fmt.Sprintf("%s-r%d-%d", name, round, i), at); err != nil {
t.Fatal(err)
}
}
rows, err := st.Arrivals(ctx, api.ArrivalQuery{
ClientID: clientID, AfterSeq: &cursor, Limit: 50})
if err != nil {
t.Fatal(err)
}
delivered += len(rows)
if len(rows) > 0 {
cursor = rows[len(rows)-1].Seq
}
}
if delivered != 120 {
t.Fatalf("30 bursts of 4 people delivered %d of 120", delivered)
}
}

291
server/internal/store/api_cameras.go Normal file
View File

@@ -0,0 +1,291 @@
package store
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/loyaly/behavision-server/internal/api"
)
// ErrNoSecrets is the API package's sentinel, aliased rather than redeclared.
//
// Two variables with the same text would compare unequal under errors.Is, so
// the handler's check would silently fall through to a 500 - the failure this
// error exists to replace with a sentence an operator can act on.
var ErrNoSecrets = api.ErrNoSecrets
const cameraCols = `
c.id::text, c.site_id::text, si.name, c.camera_id, c.label,
c.host, c.port, c.path, c.username, (c.password_enc IS NOT NULL),
c.max_width, c.tuning, c.enabled, c.revision,
c.connected, c.last_seen_at, c.snapshot_key, c.snapshot_at,
c.check_kind, c.check_requested_at, c.check_started_at, c.check_finished_at,
c.check_seconds, c.check_result, c.check_image_key`
func scanCamera(row pgx.Row) (api.Camera, error) {
var c api.Camera
var lastSeen, snapAt *time.Time
var snapKey string
var checkKind *string
var reqAt, startAt, finAt *time.Time
var checkSeconds int
var checkResult []byte
var checkImage string
if err := row.Scan(&c.ID, &c.SiteID, &c.Site, &c.CameraID, &c.Label,
&c.Host, &c.Port, &c.Path, &c.Username, &c.HasPassword,
&c.MaxWidth, &c.Tuning, &c.Enabled, &c.Revision,
&c.Connected, &lastSeen, &snapKey, &snapAt,
&checkKind, &reqAt, &startAt, &finAt,
&checkSeconds, &checkResult, &checkImage); err != nil {
return c, err
}
kind := ""
if checkKind != nil {
kind = *checkKind
}
// Carried on the camera rather than fetched separately: "is this camera set
// up" and "has anyone proved it works" are the same question to the person
// asking, and two requests to answer it is two chances for the screen to
// show a camera and its verdict from different moments.
c.Check = checkOf(kind, reqAt, startAt, finAt, checkSeconds, checkResult, checkImage)
if lastSeen != nil {
c.LastSeenAt = lastSeen.UTC().Format(time.RFC3339)
}
if snapAt != nil {
c.SnapshotAt = snapAt.UTC().Format(time.RFC3339)
}
// The KEY travels in ImageKey, which is json:"-", and the handler swaps it
// for a signed link. Same rule as an arrival's face.
c.Snapshot.Key = snapKey
return c, nil
}
// Cameras lists a tenant's cameras, optionally for one site.
//
// Never returns a password, and structurally cannot: the column is not in the
// select list at all, only whether it is set.
func (s *Store) Cameras(ctx context.Context, clientID, siteID string) ([]api.Camera, error) {
rows, err := s.pool.Query(ctx, `
SELECT `+cameraCols+`
FROM site_cameras c
JOIN sites si ON si.id = c.site_id
WHERE c.client_id = $1 AND c.deleted_at IS NULL
AND ($2 = '' OR c.site_id = $2::uuid)
ORDER BY si.name, c.camera_id`, clientID, siteID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.Camera
for rows.Next() {
c, err := scanCamera(rows)
if err != nil {
return nil, err
}
out = append(out, c)
}
return out, rows.Err()
}
// SaveCamera creates or updates one camera and bumps its revision.
//
// The revision bump is what makes the agent's reconcile cheap: it compares one
// integer instead of diffing every field, so a sync on an unchanged site costs
// a single query and no engine calls.
func (s *Store) SaveCamera(ctx context.Context, clientID, siteID, cameraID string,
in api.CameraInput) (api.Camera, error) {
var out api.Camera
if in.Password != nil && *in.Password != "" && s.secrets == nil {
return out, ErrNoSecrets
}
// Site must belong to this tenant. Checked in SQL rather than trusted from
// the path: a site id is caller-supplied and this would otherwise write a
// camera into somebody else's shop.
var owns bool
if err := s.pool.QueryRow(ctx,
`SELECT EXISTS (SELECT 1 FROM sites WHERE id = $1::uuid AND client_id = $2::uuid)`,
siteID, clientID).Scan(&owns); err != nil {
return out, err
}
if !owns {
return out, pgx.ErrNoRows
}
var sealed []byte
if in.Password != nil && *in.Password != "" {
// Sealed with the SITE id as additional data, so a row copied between
// sites in the database does not decrypt into a working credential.
b, err := s.secrets.SealString(*in.Password, siteID)
if err != nil {
return out, err
}
sealed = b
}
// COALESCE on every field: a nil pointer means "leave this alone". An
// operator editing a label must not blank the password, and the form does
// not send one because the API never gave it back.
row := s.pool.QueryRow(ctx, `
INSERT INTO site_cameras (client_id, site_id, camera_id, label, host, port,
path, username, password_enc, max_width, tuning, enabled)
VALUES ($1::uuid, $2::uuid, $3,
COALESCE($4, ''), COALESCE($5, ''), COALESCE($6, 554),
COALESCE($7, '/'), COALESCE($8, ''), $9,
COALESCE($10, 1280), COALESCE($11, '{}'::jsonb), COALESCE($12, true))
ON CONFLICT (site_id, camera_id) DO UPDATE SET
label = COALESCE($4, site_cameras.label),
host = COALESCE($5, site_cameras.host),
port = COALESCE($6, site_cameras.port),
path = COALESCE($7, site_cameras.path),
username = COALESCE($8, site_cameras.username),
password_enc = COALESCE($9, site_cameras.password_enc),
max_width = COALESCE($10, site_cameras.max_width),
tuning = COALESCE($11, site_cameras.tuning),
enabled = COALESCE($12, site_cameras.enabled),
revision = site_cameras.revision + 1,
updated_at = now(),
-- Re-saving a deleted camera revives it. An operator adding back a
-- camera they removed should get their camera, not a unique-key
-- error about a row they cannot see.
deleted_at = NULL
RETURNING id`, clientID, siteID, cameraID,
in.Label, in.Host, in.Port, in.Path, in.Username, sealed,
in.MaxWidth, in.Tuning, in.Enabled)
var id string
if err := row.Scan(&id); err != nil {
return out, err
}
return s.CameraByID(ctx, clientID, id)
}
func (s *Store) CameraByID(ctx context.Context, clientID, id string) (api.Camera, error) {
return scanCamera(s.pool.QueryRow(ctx, `
SELECT `+cameraCols+`
FROM site_cameras c
JOIN sites si ON si.id = c.site_id
WHERE c.client_id = $1 AND c.id = $2::uuid AND c.deleted_at IS NULL`,
clientID, id))
}
// DeleteCamera tombstones a camera.
//
// A tombstone rather than a DELETE, because the agent adopts cameras it finds
// configured on the shop PC. A hard delete here would be undone on the next
// sync by the very camera the operator just removed - and they would have no
// idea why it kept coming back.
func (s *Store) DeleteCamera(ctx context.Context, clientID, id string) (api.Camera, error) {
cam, err := s.CameraByID(ctx, clientID, id)
if err != nil {
return cam, err
}
_, err = s.pool.Exec(ctx, `
UPDATE site_cameras
SET deleted_at = now(), revision = revision + 1, updated_at = now()
WHERE client_id = $1 AND id = $2::uuid`, clientID, id)
return cam, err
}
// AgentCameras is the desired configuration for one site, WITH passwords.
//
// The only route that decrypts them, and it is reachable only with that site's
// own agent token. Deleted cameras are included, flagged: the agent cannot
// distinguish "head office removed this" from "head office has not seen this
// yet" by absence, and would re-adopt what was just deleted.
func (s *Store) AgentCameras(ctx context.Context, siteID string) ([]api.AgentCamera, error) {
rows, err := s.pool.Query(ctx, `
SELECT camera_id, label, host, port, path, username, password_enc,
max_width, tuning, enabled, revision, (deleted_at IS NOT NULL)
FROM site_cameras
WHERE site_id = $1::uuid
ORDER BY camera_id`, siteID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.AgentCamera
for rows.Next() {
var c api.AgentCamera
var sealed []byte
if err := rows.Scan(&c.CameraID, &c.Label, &c.Host, &c.Port, &c.Path,
&c.Username, &sealed, &c.MaxWidth, &c.Tuning, &c.Enabled,
&c.Revision, &c.Deleted); err != nil {
return nil, err
}
if len(sealed) > 0 && s.secrets != nil {
// A password that will not decrypt is sent as empty rather than
// failing the whole sync: one unreadable camera must not stop the
// other three being configured. The agent reports the connection
// failure, which is the symptom an operator can actually act on.
if pw, err := s.secrets.OpenString(sealed, siteID); err == nil {
c.Password = pw
} else {
s.auditFailed("camera password decrypt", err)
}
}
out = append(out, c)
}
return out, rows.Err()
}
// ApplyAgentReport records what a shop PC observes, and adopts any camera it
// is running that head office does not know about.
//
// Adoption is what makes turning this on safe. Every existing site already has
// cameras configured locally - including the office camera this was tested with
// - and a reconcile that only pushed downwards would delete all of them on
// first sync.
func (s *Store) ApplyAgentReport(ctx context.Context, clientID, siteID string,
rep api.AgentCameraReport) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx) //nolint:errcheck
for _, cam := range rep.Adopt {
var sealed []byte
if cam.Password != "" && s.secrets != nil {
if b, err := s.secrets.SealString(cam.Password, siteID); err == nil {
sealed = b
}
}
// DO NOTHING on conflict, deliberately. Adoption must never overwrite
// head office's configuration with what the shop PC happens to hold -
// that would make an edit here silently revert on the next sync. It
// only fills in cameras nobody has configured centrally, tombstones
// included, so a deleted camera stays deleted.
if _, err := tx.Exec(ctx, `
INSERT INTO site_cameras (client_id, site_id, camera_id, label, host,
port, path, username, password_enc,
max_width, tuning, enabled)
VALUES ($1::uuid, $2::uuid, $3, $4, $5, $6, $7, $8, $9, $10,
COALESCE($11, '{}'::jsonb), $12)
ON CONFLICT (site_id, camera_id) DO NOTHING`,
clientID, siteID, cam.CameraID, cam.Label, cam.Host, cam.Port,
cam.Path, cam.Username, sealed, cam.MaxWidth, cam.Tuning,
cam.Enabled); err != nil {
return fmt.Errorf("adopt %q: %w", cam.CameraID, err)
}
}
for _, st := range rep.State {
if _, err := tx.Exec(ctx, `
UPDATE site_cameras
SET connected = $3, last_seen_at = now(),
snapshot_key = CASE WHEN $4 = '' THEN snapshot_key ELSE $4 END,
snapshot_at = CASE WHEN $4 = '' THEN snapshot_at ELSE now() END
WHERE site_id = $1::uuid AND camera_id = $2`,
siteID, st.CameraID, st.Connected, st.SnapshotKey); err != nil {
return fmt.Errorf("state %q: %w", st.CameraID, err)
}
}
return tx.Commit(ctx)
}

222
server/internal/store/api_cameras_live_test.go Normal file
View File

@@ -0,0 +1,222 @@
package store
import (
"context"
"crypto/rand"
"strings"
"testing"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/secret"
)
// Live database tests for camera onboarding.
//
// These exist because the fake in the API package cannot catch what actually
// goes wrong here: a uuid column handed a slug, an ON CONFLICT that overwrites
// what it should preserve, a tombstone that a later insert quietly revives.
// The first of those shipped and was caught only by running it.
func sealedStore(t *testing.T) *Store {
t.Helper()
st := liveStore(t)
var key [32]byte
if _, err := rand.Read(key[:]); err != nil {
t.Fatal(err)
}
box, err := secret.New(key[:])
if err != nil {
t.Fatal(err)
}
st.UseSecrets(box)
return st
}
func ptr[T any](v T) *T { return &v }
func TestLiveACameraPasswordSurvivesTheRoundTripEncrypted(t *testing.T) {
st := sealedStore(t)
client, site := seedTenant(t, st, "cam"+stamp(), 0, false)
ctx := context.Background()
if _, err := st.SaveCamera(ctx, client, site, "entrance", api.CameraInput{
Label: ptr("Entrance"), Host: ptr("192.168.0.138"),
Username: ptr("admin"), Password: ptr("office-cam-secret"),
}); err != nil {
t.Fatal(err)
}
// Nothing a person can reach carries the password.
cams, err := st.Cameras(ctx, client, "")
if err != nil {
t.Fatal(err)
}
if len(cams) != 1 || !cams[0].HasPassword {
t.Fatalf("camera not stored with a password: %+v", cams)
}
// The agent, and only the agent, gets it back.
agent, err := st.AgentCameras(ctx, site)
if err != nil {
t.Fatal(err)
}
if agent[0].Password != "office-cam-secret" {
t.Fatalf("password did not survive: %q", agent[0].Password)
}
// And it is genuinely encrypted at rest, not merely hidden by the query.
var raw []byte
if err := st.pool.QueryRow(ctx,
`SELECT password_enc FROM site_cameras WHERE site_id = $1::uuid`, site).
Scan(&raw); err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "office-cam-secret") {
t.Fatal("the password is stored in the clear")
}
}
// The aad is the site id, so a row copied between sites in the database does
// not decrypt into a working credential.
func TestLiveACameraRowCopiedToAnotherSiteDoesNotDecrypt(t *testing.T) {
st := sealedStore(t)
client, siteA := seedTenant(t, st, "aad"+stamp(), 0, false)
ctx := context.Background()
var siteB string
if err := st.pool.QueryRow(ctx, `
INSERT INTO sites (client_id, name, slug) VALUES ($1::uuid, 'Other', $2)
RETURNING id::text`, client, "other"+stamp()).Scan(&siteB); err != nil {
t.Fatal(err)
}
if _, err := st.SaveCamera(ctx, client, siteA, "entrance", api.CameraInput{
Host: ptr("10.0.0.5"), Password: ptr("office-cam-secret")}); err != nil {
t.Fatal(err)
}
// Move the row, as a database-level attacker would.
if _, err := st.pool.Exec(ctx,
`UPDATE site_cameras SET site_id = $1::uuid WHERE site_id = $2::uuid`,
siteB, siteA); err != nil {
t.Fatal(err)
}
got, err := st.AgentCameras(ctx, siteB)
if err != nil {
t.Fatal(err)
}
if got[0].Password != "" {
t.Fatalf("a relocated row decrypted into a usable credential: %q", got[0].Password)
}
}
// Adoption must never overwrite head office's configuration with whatever the
// shop PC happens to hold - an edit made here would silently revert on the
// agent's next sync.
func TestLiveAdoptionNeverOverwritesHeadOffice(t *testing.T) {
st := sealedStore(t)
client, site := seedTenant(t, st, "adopt"+stamp(), 0, false)
ctx := context.Background()
if _, err := st.SaveCamera(ctx, client, site, "entrance", api.CameraInput{
Label: ptr("Front entrance"), Host: ptr("192.168.0.138")}); err != nil {
t.Fatal(err)
}
// The shop PC reports an older, different configuration.
if err := st.ApplyAgentReport(ctx, client, site, api.AgentCameraReport{
Adopt: []api.AgentCamera{{CameraID: "entrance", Label: "stale",
Host: "10.9.9.9", Enabled: true}},
}); err != nil {
t.Fatal(err)
}
cams, err := st.Cameras(ctx, client, "")
if err != nil {
t.Fatal(err)
}
if cams[0].Host != "192.168.0.138" || cams[0].Label != "Front entrance" {
t.Fatalf("adoption clobbered head office: %+v", cams[0])
}
}
// A hard delete would be undone on the next sync by the very camera the
// operator just removed, and they would have no idea why it kept coming back.
func TestLiveADeletedCameraIsNotResurrectedByAdoption(t *testing.T) {
st := sealedStore(t)
client, site := seedTenant(t, st, "tomb"+stamp(), 0, false)
ctx := context.Background()
cam, err := st.SaveCamera(ctx, client, site, "entrance",
api.CameraInput{Host: ptr("10.0.0.5")})
if err != nil {
t.Fatal(err)
}
if _, err := st.DeleteCamera(ctx, client, cam.ID); err != nil {
t.Fatal(err)
}
if err := st.ApplyAgentReport(ctx, client, site, api.AgentCameraReport{
Adopt: []api.AgentCamera{{CameraID: "entrance", Host: "10.0.0.5", Enabled: true}},
}); err != nil {
t.Fatal(err)
}
cams, err := st.Cameras(ctx, client, "")
if err != nil {
t.Fatal(err)
}
if len(cams) != 0 {
t.Fatalf("a deleted camera came back: %+v", cams)
}
// The agent must still be TOLD it is deleted, or it keeps running it.
agent, err := st.AgentCameras(ctx, site)
if err != nil {
t.Fatal(err)
}
if len(agent) != 1 || !agent[0].Deleted {
t.Fatalf("the agent was not told to stop: %+v", agent)
}
}
// Editing one field must not blank the others - especially not the password,
// which the form cannot resend because the API never returned it.
func TestLiveEditingALabelKeepsTheStoredPassword(t *testing.T) {
st := sealedStore(t)
client, site := seedTenant(t, st, "edit"+stamp(), 0, false)
ctx := context.Background()
if _, err := st.SaveCamera(ctx, client, site, "entrance", api.CameraInput{
Label: ptr("Entrance"), Host: ptr("192.168.0.138"),
Username: ptr("admin"), Password: ptr("office-cam-secret")}); err != nil {
t.Fatal(err)
}
if _, err := st.SaveCamera(ctx, client, site, "entrance",
api.CameraInput{Label: ptr("Front door")}); err != nil {
t.Fatal(err)
}
agent, err := st.AgentCameras(ctx, site)
if err != nil {
t.Fatal(err)
}
if agent[0].Password != "office-cam-secret" {
t.Fatalf("the password was lost by a label edit: %q", agent[0].Password)
}
if agent[0].Host != "192.168.0.138" {
t.Fatalf("the address was lost: %q", agent[0].Host)
}
if agent[0].Label != "Front door" {
t.Fatalf("the edit did not apply: %q", agent[0].Label)
}
// The revision has to move, or the agent will not re-apply it.
if agent[0].Revision < 2 {
t.Fatalf("revision %d - the shop PC would never pick this up", agent[0].Revision)
}
}
// One tenant must not be able to write a camera into another's shop, even
// naming a site id that really exists.
func TestLiveACameraCannotBeWrittenIntoAnotherTenantsShop(t *testing.T) {
st := sealedStore(t)
mine, _ := seedTenant(t, st, "mine"+stamp(), 0, false)
_, theirSite := seedTenant(t, st, "theirs"+stamp(), 0, false)
if _, err := st.SaveCamera(context.Background(), mine, theirSite, "entrance",
api.CameraInput{Host: ptr("10.0.0.5")}); err == nil {
t.Fatal("wrote a camera into another tenant's site")
}
}

142
server/internal/store/api_checks.go Normal file
View File

@@ -0,0 +1,142 @@
package store
import (
"context"
"encoding/json"
"time"
"github.com/jackc/pgx/v5"
"github.com/loyaly/behavision-server/internal/api"
)
// RequestCheck queues a check for the shop PC to run on its next sync.
//
// Overwrites any previous request for the same camera rather than queuing a
// second: an operator who presses Check twice wants one answer, now, not two
// answers several minutes apart in an order they cannot predict.
func (s *Store) RequestCheck(ctx context.Context, clientID, id, kind string,
seconds int) error {
tag, err := s.pool.Exec(ctx, `
UPDATE site_cameras
SET check_kind = $3, check_seconds = $4, check_requested_at = now(),
check_started_at = NULL, check_finished_at = NULL,
check_result = NULL, check_image_key = ''
WHERE client_id = $1 AND id = $2::uuid AND deleted_at IS NULL`,
clientID, id, kind, seconds)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return pgx.ErrNoRows
}
return nil
}
// ClaimChecks hands a site its pending checks and marks them started.
//
// One statement, so two syncs racing cannot both claim the same job. A
// placement check asks a human to walk about for 25 seconds; running it twice
// because the agent polled while the first was still going would give the
// operator two contradictory verdicts for one walk.
func (s *Store) ClaimChecks(ctx context.Context, siteID string) ([]api.AgentCheckJob, error) {
rows, err := s.pool.Query(ctx, `
UPDATE site_cameras
SET check_started_at = now()
WHERE site_id = $1::uuid
AND check_requested_at IS NOT NULL
AND check_finished_at IS NULL
AND check_started_at IS NULL
RETURNING camera_id, check_kind, check_seconds`, siteID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.AgentCheckJob
for rows.Next() {
var j api.AgentCheckJob
if err := rows.Scan(&j.CameraID, &j.Kind, &j.Seconds); err != nil {
return nil, err
}
out = append(out, j)
}
return out, rows.Err()
}
// RecordCheckResult stores what the shop PC found.
func (s *Store) RecordCheckResult(ctx context.Context, siteID string,
res api.AgentCheckResult) error {
detail := res.Detail
if detail == nil {
detail = map[string]any{}
}
body, err := json.Marshal(map[string]any{
"ok": res.OK, "verdict": res.Verdict, "headline": res.Headline,
"advice": res.Advice, "detail": detail,
})
if err != nil {
return err
}
_, err = s.pool.Exec(ctx, `
UPDATE site_cameras
SET check_finished_at = now(), check_result = $3::jsonb,
check_image_key = CASE WHEN $4 = '' THEN check_image_key ELSE $4 END
WHERE site_id = $1::uuid AND camera_id = $2`,
siteID, res.CameraID, body, res.ImageKey)
return err
}
// ReleaseStaleChecks un-claims checks a shop PC took and never finished.
//
// Without this a PC that is restarted mid-check leaves the camera showing
// "checking..." for ever, and the operator's only recourse is to guess that
// pressing Check again will help - which it would not, because the request is
// still marked started.
func (s *Store) ReleaseStaleChecks(ctx context.Context, olderThan time.Duration) error {
_, err := s.pool.Exec(ctx, `
UPDATE site_cameras
SET check_started_at = NULL
WHERE check_requested_at IS NOT NULL
AND check_finished_at IS NULL
AND check_started_at < now() - $1::interval`,
olderThan.String())
return err
}
// checkOf reads the stored check for one camera row.
func checkOf(kind string, requested, started, finished *time.Time,
seconds int, result []byte, imageKey string) api.CameraCheck {
if requested == nil {
return api.CameraCheck{}
}
c := api.CameraCheck{
Kind: kind, Seconds: seconds,
RequestedAt: requested.UTC().Format(time.RFC3339),
State: "requested",
}
if started != nil {
c.State = "running"
}
if finished != nil {
c.State = "done"
c.FinishedAt = finished.UTC().Format(time.RFC3339)
}
if len(result) > 0 {
var body struct {
OK bool `json:"ok"`
Verdict string `json:"verdict"`
Headline string `json:"headline"`
Advice []string `json:"advice"`
Detail map[string]any `json:"detail"`
}
if err := json.Unmarshal(result, &body); err == nil {
c.OK, c.Verdict, c.Headline = body.OK, body.Verdict, body.Headline
c.Advice, c.Detail = body.Advice, body.Detail
}
}
c.Image.Key = imageKey
return c
}

61
server/internal/store/api_enrolment.go Normal file
View File

@@ -0,0 +1,61 @@
package store
import (
"context"
"time"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/auth"
)
// IssueEnrolmentCode mints the code a shop PC redeems, for one site of one
// client.
//
// The same row the provisioning command writes, hashed by the same function.
// It exists as an API as well because a code is not a one-off: a PC is
// replaced, reinstalled, or moved between shops, and until now every one of
// those was a support ticket and an SSH session. The provisioning command
// remains the bootstrap - a brand new customer has nobody to sign in as yet.
//
// The site id is checked against the CALLER'S client in the same statement
// that inserts, so a code for another tenant's shop cannot be minted by
// guessing a uuid.
func (s *Store) IssueEnrolmentCode(ctx context.Context, clientID, siteID,
actorID, label string, ttl time.Duration) (api.EnrolmentCode, error) {
if ttl <= 0 {
ttl = 7 * 24 * time.Hour
}
code, err := auth.NewEnrolmentCode()
if err != nil {
return api.EnrolmentCode{}, err
}
out := api.EnrolmentCode{Code: code, Label: label, SiteID: siteID}
expires := time.Now().Add(ttl).UTC()
err = s.pool.QueryRow(ctx, `
INSERT INTO site_enrolment_tokens (client_id, site_id, token_hash,
label, expires_at, created_by)
SELECT $1::uuid, si.id, $3, $4, $5, $6::uuid
FROM sites si
WHERE si.id = $2::uuid AND si.client_id = $1::uuid
RETURNING expires_at, (SELECT name FROM sites WHERE id = $2::uuid)`,
clientID, siteID, auth.HashToken(auth.NormalizeCode(code)),
label, expires, nullableUUID(actorID)).
Scan(&out.ExpiresAt, &out.SiteName)
// pgx.ErrNoRows travels up as-is, the way RequestCheck already signals a
// missing row. It means no such site FOR THIS CLIENT, and the handler turns
// it into a 404: a tenant has no business learning that another tenant's
// shop exists.
if err != nil {
return api.EnrolmentCode{}, err
}
return out, nil
}
func nullableUUID(s string) any {
if s == "" {
return nil
}
return s
}

152
server/internal/store/api_images.go Normal file
View File

@@ -0,0 +1,152 @@
package store
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/loyaly/behavision-server/internal/api"
)
// SetAgentAPIToken stores the hash of a store PC's HTTPS credential.
//
// Hashed, not encrypted, unlike the broker password: this one is never handed
// back out. It is shown once at enrolment and the agent keeps it, so a database
// dump contains nothing usable.
func (s *Store) SetAgentAPIToken(ctx context.Context, agentID string, hash []byte) error {
_, err := s.pool.Exec(ctx,
`UPDATE agents SET api_token_hash = $2 WHERE id = $1::uuid`, agentID, hash)
return err
}
func (s *Store) AgentByToken(ctx context.Context, hash []byte) (api.AgentPrincipal, error) {
var ap api.AgentPrincipal
err := s.pool.QueryRow(ctx, `
SELECT a.id::text, a.client_id::text, a.site_id::text, a.mqtt_username,
c.slug, si.slug
FROM agents a
JOIN sites si ON si.id = a.site_id AND si.active
JOIN clients c ON c.id = a.client_id AND c.active
WHERE a.api_token_hash = $1`, hash).
Scan(&ap.AgentID, &ap.ClientID, &ap.SiteID, &ap.Slug, &ap.Client, &ap.Site)
if errors.Is(err, pgx.ErrNoRows) {
return api.AgentPrincipal{}, errors.New("no such agent")
}
return ap, err
}
// VisitorImageKey is the most recent surviving photo of one person.
//
// image_deleted_at is checked, not just image_key: a key that has been erased
// is still in the row as the record that it WAS erased, and handing it to the
// presigner would produce a link to an object that is gone - or, worse, to one
// that was re-created under the same name.
func (s *Store) VisitorImageKey(ctx context.Context, clientID, visitorID string) (string, error) {
var key string
err := s.pool.QueryRow(ctx, `
SELECT image_key FROM visits
WHERE client_id = $1 AND visitor_id = $2::uuid
AND image_key <> '' AND image_deleted_at IS NULL
ORDER BY occurred_at DESC
LIMIT 1`, clientID, visitorID).Scan(&key)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return key, err
}
// VisitorImageKeys is every object belonging to one person - the first step of
// an erasure request.
func (s *Store) VisitorImageKeys(ctx context.Context, clientID, visitorID string) ([]string, error) {
rows, err := s.pool.Query(ctx, `
SELECT image_key FROM visits
WHERE client_id = $1 AND visitor_id = $2::uuid
AND image_key <> '' AND image_deleted_at IS NULL`, clientID, visitorID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []string
for rows.Next() {
var k string
if err := rows.Scan(&k); err != nil {
return nil, err
}
out = append(out, k)
}
return out, rows.Err()
}
// ForgetVisitor is the database half of erasure.
//
// What goes and what stays is a deliberate line:
//
// - the biometric template is DELETED outright, not flagged. Template
// inversion reconstructs a recognisable face from an ArcFace embedding, so
// a soft-deleted vector is a retained photograph by another name.
// - the profile goes: a name, a phone number and a date of birth are exactly
// what the request is about.
// - visits STAY, with the person unlinked. They are the shop's own footfall
// history, and silently changing last quarter's numbers because one
// customer exercised a right is both wrong and detectable.
// - the visitors row stays with deleted_at set, so the same face cannot be
// re-enrolled as a brand new person the next time they walk in.
func (s *Store) ForgetVisitor(ctx context.Context, clientID, visitorID string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx) //nolint:errcheck
var exists bool
err = tx.QueryRow(ctx,
`SELECT true FROM visitors WHERE id = $1::uuid AND client_id = $2`,
visitorID, clientID).Scan(&exists)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("no such visitor")
}
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `
DELETE FROM visitor_embeddings
WHERE visitor_id = $1::uuid AND client_id = $2`,
visitorID, clientID); err != nil {
return fmt.Errorf("delete templates: %w", err)
}
if _, err := tx.Exec(ctx, `
DELETE FROM visitor_profiles
WHERE visitor_id = $1::uuid AND client_id = $2`,
visitorID, clientID); err != nil {
return fmt.Errorf("delete profile: %w", err)
}
// The consent record itself survives as a revocation. Deleting it would
// destroy the proof of what we were permitted to do and when, which is the
// thing an auditor actually asks for.
if _, err := tx.Exec(ctx, `
UPDATE consents SET revoked_at = COALESCE(revoked_at, now())
WHERE visitor_id = $1::uuid AND client_id = $2`,
visitorID, clientID); err != nil {
return fmt.Errorf("revoke consents: %w", err)
}
// The objects are already gone from storage by the time this runs; this
// records that, and stops anything presigning a dead key.
if _, err := tx.Exec(ctx, `
UPDATE visits SET image_deleted_at = now()
WHERE client_id = $1 AND visitor_id = $2::uuid
AND image_key <> '' AND image_deleted_at IS NULL`,
clientID, visitorID); err != nil {
return fmt.Errorf("mark images deleted: %w", err)
}
if _, err := tx.Exec(ctx, `
UPDATE visitors
SET deleted_at = now(), label = 'Erased'
WHERE id = $1::uuid AND client_id = $2`,
visitorID, clientID); err != nil {
return fmt.Errorf("mark visitor erased: %w", err)
}
return tx.Commit(ctx)
}

352
server/internal/store/api_people.go Normal file
View File

@@ -0,0 +1,352 @@
package store
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/secret"
)
// Secrets decrypts values the server must hand back out - today, each site's
// broker password. Nil until configured, and every path that needs it says so
// rather than silently returning an empty credential.
func (s *Store) UseSecrets(b *secret.Box) { s.secrets = b }
// likePattern escapes the wildcards so a customer searching for "50%" finds
// the person called "50%" instead of matching everybody.
func likePattern(q string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return "%" + r.Replace(q) + "%"
}
func (s *Store) SearchVisitors(ctx context.Context, clientID, query string, limit int) (
[]api.Customer, error) {
rows, err := s.pool.Query(ctx, `
SELECT v.id::text, v.label,
COALESCE(p.full_name, ''), COALESCE(p.phone, ''), COALESCE(p.email, ''),
v.visit_count, v.first_seen_at, v.last_seen_at,
(p.id IS NOT NULL),
EXISTS (SELECT 1 FROM consents c
WHERE c.visitor_id = v.id AND c.revoked_at IS NULL)
FROM visitors v
LEFT JOIN visitor_profiles p
ON p.visitor_id = v.id AND p.client_id = v.client_id
WHERE v.client_id = $1 AND v.deleted_at IS NULL
AND ($2 = '' OR v.label ILIKE $3 ESCAPE '\'
OR p.full_name ILIKE $3 ESCAPE '\'
OR p.phone ILIKE $3 ESCAPE '\'
OR p.email ILIKE $3 ESCAPE '\')
ORDER BY v.last_seen_at DESC NULLS LAST, v.first_seen_at DESC
LIMIT $4`,
clientID, strings.TrimSpace(query), likePattern(strings.TrimSpace(query)), limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.Customer
for rows.Next() {
var c api.Customer
var first time.Time
var last *time.Time
if err := rows.Scan(&c.ID, &c.Label, &c.FullName, &c.Phone, &c.Email,
&c.VisitCount, &first, &last, &c.HasProfile, &c.HasConsent); err != nil {
return nil, err
}
c.FirstSeenAt = first.UTC().Format(time.RFC3339)
if last != nil {
c.LastSeenAt = last.UTC().Format(time.RFC3339)
}
out = append(out, c)
}
return out, rows.Err()
}
func (s *Store) VisitorHistory(ctx context.Context, clientID, visitorID string, limit int) (
[]api.VisitRow, error) {
rows, err := s.pool.Query(ctx, `
SELECT vi.id::text, vi.occurred_at, si.name, vi.camera_id,
vi.is_new_visitor, vi.similarity, vi.quality, vi.attributes
FROM visits vi
JOIN sites si ON si.id = vi.site_id
WHERE vi.client_id = $1 AND vi.visitor_id = $2::uuid
ORDER BY vi.occurred_at DESC
LIMIT $3`, clientID, visitorID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.VisitRow
for rows.Next() {
var v api.VisitRow
var at time.Time
var sim, qual *float64
if err := rows.Scan(&v.ID, &at, &v.Site, &v.CameraID, &v.IsNew,
&sim, &qual, &v.Attributes); err != nil {
return nil, err
}
v.OccurredAt = at.UTC().Format(time.RFC3339)
if sim != nil {
v.Similarity = *sim
}
if qual != nil {
v.Quality = *qual
}
out = append(out, v)
}
return out, rows.Err()
}
// SaveProfile writes the in-store form, and the consent record with it.
//
// One transaction: a name saved without its consent row is a customer whose
// personal data we hold with no record of being allowed to, which is the exact
// state the consents table exists to make impossible.
func (s *Store) SaveProfile(ctx context.Context, clientID string, p api.Profile,
actor string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx) //nolint:errcheck // no-op once committed
// Scoped to the client, so an id from another tenant is simply not found -
// the same answer as a typo, which is what it should look like.
var exists bool
err = tx.QueryRow(ctx, `
SELECT true FROM visitors
WHERE id = $1::uuid AND client_id = $2 AND deleted_at IS NULL`,
p.VisitorID, clientID).Scan(&exists)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("no such visitor")
}
if err != nil {
return err
}
var dob any
if p.DateOfBirth != "" {
dob = p.DateOfBirth
}
if _, err := tx.Exec(ctx, `
INSERT INTO visitor_profiles (visitor_id, client_id, full_name, phone,
email, gender, date_of_birth, notes,
collected_by)
VALUES ($1::uuid, $2, $3, $4, $5, $6, $7::date, $8, NULLIF($9, '')::uuid)
ON CONFLICT (visitor_id) DO UPDATE SET
full_name = EXCLUDED.full_name,
phone = EXCLUDED.phone,
email = EXCLUDED.email,
gender = EXCLUDED.gender,
date_of_birth = EXCLUDED.date_of_birth,
notes = EXCLUDED.notes,
collected_by = EXCLUDED.collected_by,
updated_at = now()`,
p.VisitorID, clientID, p.FullName, p.Phone, p.Email, p.Gender,
dob, p.Notes, actor); err != nil {
return fmt.Errorf("save profile: %w", err)
}
if p.Consent {
// Only if there is not already a live one. Re-saving the form must not
// stack up consent records, or the audit trail stops being readable.
if _, err := tx.Exec(ctx, `
INSERT INTO consents (visitor_id, client_id, scope, method,
collected_by, evidence)
SELECT $1::uuid, $2, 'biometric', 'in_store_form',
NULLIF($3, '')::uuid, '{}'::jsonb
WHERE NOT EXISTS (
SELECT 1 FROM consents
WHERE visitor_id = $1::uuid AND scope = 'biometric'
AND revoked_at IS NULL)`,
p.VisitorID, clientID, actor); err != nil {
return fmt.Errorf("record consent: %w", err)
}
} else {
// Unticking the box is a withdrawal, and a withdrawal is a timestamp,
// never a delete: the fact that they withdrew is itself the thing an
// auditor asks to see.
if _, err := tx.Exec(ctx, `
UPDATE consents SET revoked_at = now()
WHERE visitor_id = $1::uuid AND client_id = $2
AND scope = 'biometric' AND revoked_at IS NULL`,
p.VisitorID, clientID); err != nil {
return fmt.Errorf("revoke consent: %w", err)
}
}
return tx.Commit(ctx)
}
// RecordPurchase books a sale against a customer.
//
// When no site is given it uses the one where this customer was most recently
// seen, which is what "the assistant on the floor just sold them something"
// means. If they have never been seen anywhere the caller is told to pass a
// site rather than being handed a foreign key error.
func (s *Store) RecordPurchase(ctx context.Context, clientID string,
p api.PurchaseInput, actor string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx) //nolint:errcheck
var exists bool
err = tx.QueryRow(ctx, `
SELECT true FROM visitors
WHERE id = $1::uuid AND client_id = $2 AND deleted_at IS NULL`,
p.VisitorID, clientID).Scan(&exists)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("no such visitor")
}
if err != nil {
return err
}
siteID := p.SiteID
var visitID any
if siteID == "" {
var sid, vid *string
err = tx.QueryRow(ctx, `
SELECT site_id::text, id::text FROM visits
WHERE client_id = $1 AND visitor_id = $2::uuid
ORDER BY occurred_at DESC LIMIT 1`,
clientID, p.VisitorID).Scan(&sid, &vid)
if errors.Is(err, pgx.ErrNoRows) || sid == nil {
return errors.New("no site for this visitor")
}
if err != nil {
return err
}
siteID = *sid
// Attaching the sale to the visit it belongs to is what makes
// "did this visit convert" answerable at all, rather than only
// "did this person ever buy".
visitID = vid
} else {
// A site passed in must still belong to the caller's client.
var ok bool
err = tx.QueryRow(ctx,
`SELECT true FROM sites WHERE id = $1::uuid AND client_id = $2`,
siteID, clientID).Scan(&ok)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("no site for this visitor")
}
if err != nil {
return err
}
}
items := p.Items
if items == nil {
items = []string{}
}
if _, err := tx.Exec(ctx, `
INSERT INTO purchases (client_id, site_id, visitor_id, visit_id, amount,
currency, items, source, external_ref, recorded_by)
VALUES ($1, $2::uuid, $3::uuid, $4::uuid, $5, $6, $7, $8, $9,
NULLIF($10, '')::uuid)`,
clientID, siteID, p.VisitorID, visitID, p.Amount, p.Currency,
items, p.Source, p.Notes, actor); err != nil {
return fmt.Errorf("insert purchase: %w", err)
}
return tx.Commit(ctx)
}
// ---------------------------------------------------------------- enrolment
// RedeemEnrolment spends an installation code and returns the broker login.
//
// Single-use is enforced by the UPDATE itself: the `used_at IS NULL` predicate
// and the write are one statement, so two PCs racing on the same code cannot
// both win. Checking first and updating after would be exactly that race.
func (s *Store) RedeemEnrolment(ctx context.Context, hash []byte) (api.Enrolment, error) {
var en api.Enrolment
err := s.pool.QueryRow(ctx, `
UPDATE site_enrolment_tokens
SET used_at = now()
WHERE token_hash = $1 AND used_at IS NULL AND expires_at > now()
RETURNING client_id::text, site_id::text`, hash).
Scan(&en.ClientID, &en.SiteID)
if errors.Is(err, pgx.ErrNoRows) {
return en, errors.New("enrolment token is unknown, expired or already used")
}
if err != nil {
return en, err
}
var sealed []byte
if err := s.pool.QueryRow(ctx, `
SELECT si.name, si.slug, a.id::text, a.mqtt_username, a.mqtt_password_enc
FROM sites si
JOIN agents a ON a.site_id = si.id
WHERE si.id = $1::uuid AND si.client_id = $2`,
en.SiteID, en.ClientID).
Scan(&en.SiteName, &en.SiteSlug, &en.AgentID, &en.MQTTUser, &sealed); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return en, errors.New("site has no agent provisioned - " +
"create the broker user before issuing an enrolment token")
}
return en, err
}
if len(sealed) == 0 {
return en, errors.New("site has no broker password stored")
}
if s.secrets == nil {
return en, errors.New("BEHAVISION_SECRET_KEY is not configured, " +
"so stored broker passwords cannot be read")
}
pass, err := s.secrets.OpenString(sealed, en.AgentID)
if err != nil {
return en, fmt.Errorf("broker password for %s: %w", en.SiteSlug, err)
}
en.MQTTPass = pass
return en, nil
}
// SetAgentSecret stores a site's broker password, sealed to that agent's id.
// Used by provisioning, never by a request handler.
func (s *Store) SetAgentSecret(ctx context.Context, agentID, password string) error {
if s.secrets == nil {
return errors.New("BEHAVISION_SECRET_KEY is not configured")
}
sealed, err := s.secrets.SealString(password, agentID)
if err != nil {
return err
}
_, err = s.pool.Exec(ctx,
`UPDATE agents SET mqtt_password_enc = $2 WHERE id = $1::uuid`,
agentID, sealed)
return err
}
// Audit never fails a request.
//
// A refused audit write is worth knowing about, but refusing the action it was
// recording is worse: it would mean an outage in the logging table stops staff
// serving customers.
func (s *Store) Audit(ctx context.Context, e api.AuditEntry) {
detail := e.Detail
if detail == nil {
detail = map[string]any{}
}
if _, err := s.pool.Exec(ctx, `
INSERT INTO audit_log (client_id, actor_id, actor_kind, action,
entity, entity_id, detail)
VALUES (NULLIF($1, '')::uuid, NULLIF($2, '')::uuid, $3, $4, $5, $6, $7)`,
e.ClientID, e.ActorID, e.ActorKind, e.Action,
e.Entity, e.EntityID, detail); err != nil {
s.auditFailed(e.Action, err)
}
}

365
server/internal/store/api_store.go Normal file
View File

@@ -0,0 +1,365 @@
package store
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/auth"
)
// ---------------------------------------------------------------- identity
func (s *Store) UserByEmail(ctx context.Context, email string) (api.UserRecord, error) {
var u api.UserRecord
err := s.pool.QueryRow(ctx, `
SELECT u.id::text, COALESCE(u.client_id::text, ''), COALESCE(c.name, ''),
u.email, u.full_name, u.role, u.active, u.password_hash
FROM app_users u
LEFT JOIN clients c ON c.id = u.client_id
WHERE lower(u.email) = $1`, email).
Scan(&u.ID, &u.ClientID, &u.ClientName, &u.Email, &u.FullName,
&u.Role, &u.Active, &u.PasswordHash)
if errors.Is(err, pgx.ErrNoRows) {
// Not an error. The handler must still spend the same time verifying a
// password, so "no such user" has to come back as data rather than as a
// short-circuit.
return api.UserRecord{Found: false}, nil
}
if err != nil {
return api.UserRecord{}, err
}
// A user whose client has been deactivated must not be able to sign in and
// read that client's customers.
if u.ClientID != "" {
var active bool
if err := s.pool.QueryRow(ctx,
`SELECT active FROM clients WHERE id = $1`, u.ClientID).
Scan(&active); err != nil {
return api.UserRecord{}, err
}
u.Active = u.Active && active
}
u.Found = true
return u, nil
}
func (s *Store) TouchUserLogin(ctx context.Context, userID string) error {
_, err := s.pool.Exec(ctx,
`UPDATE app_users SET last_login_at = now() WHERE id = $1`, userID)
return err
}
func (s *Store) CreateSession(ctx context.Context, n api.NewSession) error {
_, err := s.pool.Exec(ctx, `
INSERT INTO sessions (user_id, client_id, access_hash, refresh_hash,
access_expires_at, refresh_expires_at, device)
VALUES ($1, NULLIF($2, '')::uuid, $3, $4, $5, $6, $7)`,
n.UserID, n.ClientID, n.AccessHash, n.RefreshHash,
n.AccessExpiry, n.RefreshExp, n.Device)
return err
}
// sessionQuery is shared by the access and refresh lookups so the two can
// never disagree about what makes a session valid.
const sessionQuery = `
SELECT s.id::text, s.user_id::text, COALESCE(s.client_id::text, ''),
COALESCE(c.name, ''), u.email, u.full_name, u.role, %s
FROM sessions s
JOIN app_users u ON u.id = s.user_id AND u.active
LEFT JOIN clients c ON c.id = s.client_id
WHERE s.%s = $1 AND s.revoked_at IS NULL`
func (s *Store) SessionByAccess(ctx context.Context, hash []byte) (auth.Principal, time.Time, error) {
// last_used_at is refreshed at most every five minutes. Writing it on every
// authenticated request would turn a read-only API call into a row update
// and a WAL record, for a column nothing needs to the second.
if _, err := s.pool.Exec(ctx, `
UPDATE sessions SET last_used_at = now()
WHERE access_hash = $1 AND revoked_at IS NULL
AND (last_used_at IS NULL OR last_used_at < now() - interval '5 minutes')`,
hash); err != nil {
// Bookkeeping. Refusing the request because a timestamp would not
// update would log everybody out over nothing.
_ = err
}
return s.session(ctx, hash, "access_expires_at", "access_hash")
}
func (s *Store) SessionByRefresh(ctx context.Context, hash []byte) (auth.Principal, time.Time, error) {
return s.session(ctx, hash, "refresh_expires_at", "refresh_hash")
}
func (s *Store) session(ctx context.Context, hash []byte, expiryCol, hashCol string) (
auth.Principal, time.Time, error) {
var p auth.Principal
var expires time.Time
err := s.pool.QueryRow(ctx,
fmt.Sprintf(sessionQuery, expiryCol, hashCol), hash).
Scan(&p.SessionID, &p.UserID, &p.ClientID, &p.ClientName,
&p.Email, &p.FullName, &p.Role, &expires)
if errors.Is(err, pgx.ErrNoRows) {
return auth.Principal{}, time.Time{}, auth.ErrNoSession
}
return p, expires, err
}
// RotateSession replaces the tokens on an existing row rather than inserting a
// new one. The old refresh token stops working the moment this commits, which
// is the point: a token copied off a resold shop PC must not keep working
// alongside the real one.
func (s *Store) RotateSession(ctx context.Context, sessionID string, n api.NewSession) error {
tag, err := s.pool.Exec(ctx, `
UPDATE sessions
SET access_hash = $2, refresh_hash = $3,
access_expires_at = $4, refresh_expires_at = $5,
last_used_at = now(),
device = COALESCE(NULLIF($6, ''), device)
WHERE id = $1 AND revoked_at IS NULL`,
sessionID, n.AccessHash, n.RefreshHash, n.AccessExpiry, n.RefreshExp, n.Device)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return auth.ErrNoSession
}
return nil
}
func (s *Store) RevokeSession(ctx context.Context, sessionID string) error {
_, err := s.pool.Exec(ctx,
`UPDATE sessions SET revoked_at = now()
WHERE id = $1 AND revoked_at IS NULL`, sessionID)
return err
}
// ---------------------------------------------------------------- reports
func nullUUID(s string) any {
if strings.TrimSpace(s) == "" {
return nil
}
return s
}
// Footfall buckets visits in the requested timezone.
//
// Two things here are easy to get wrong and expensive to notice:
//
// - "New" means first-ever, computed over all of time, not first-in-window.
// Otherwise every report re-labels your regulars as new customers the
// moment the window starts after their last visit.
// - A visit with no visitor_id (a site sending counts without templates) is
// real footfall but an unknown person. It is counted in `visitors` and in
// neither `new` nor `returning`, so those two may sum to less than the
// total. Guessing either way would put a number in a marketing report that
// nothing supports.
func (s *Store) Footfall(ctx context.Context, q api.ReportQuery) (
[]api.FootfallPoint, api.Totals, error) {
site := nullUUID(q.SiteID)
rows, err := s.pool.Query(ctx, `
WITH scoped AS (
SELECT v.visitor_id, v.occurred_at
FROM visits v
WHERE v.client_id = $1
AND v.occurred_at >= $2 AND v.occurred_at < $3
AND ($4::uuid IS NULL OR v.site_id = $4::uuid)
),
firsts AS (
SELECT v.visitor_id, min(v.occurred_at) AS first_at
FROM visits v
WHERE v.client_id = $1
AND v.visitor_id IS NOT NULL
AND ($4::uuid IS NULL OR v.site_id = $4::uuid)
GROUP BY v.visitor_id
)
SELECT date_trunc($5, s.occurred_at AT TIME ZONE $6) AS bucket,
count(DISTINCT s.visitor_id) AS identified,
count(*) FILTER (WHERE s.visitor_id IS NULL) AS anonymous,
count(DISTINCT s.visitor_id) FILTER (
WHERE date_trunc($5, f.first_at AT TIME ZONE $6)
= date_trunc($5, s.occurred_at AT TIME ZONE $6)) AS newcomers
FROM scoped s
LEFT JOIN firsts f ON f.visitor_id = s.visitor_id
GROUP BY 1
ORDER BY 1`,
q.ClientID, q.From, q.To, site, q.Bucket, q.Timezone)
if err != nil {
return nil, api.Totals{}, fmt.Errorf("footfall buckets: %w", err)
}
defer rows.Close()
var points []api.FootfallPoint
for rows.Next() {
var t time.Time
var identified, anonymous, newcomers int
if err := rows.Scan(&t, &identified, &anonymous, &newcomers); err != nil {
return nil, api.Totals{}, err
}
points = append(points, api.FootfallPoint{
// Local wall time, with no offset, because the label belongs to the
// timezone named alongside it in the report. Stamping it with Z
// would say 09:00 UTC when the shop means 09:00 in Chennai.
Bucket: t.Format("2006-01-02T15:04:05"),
Visitors: identified + anonymous,
New: newcomers,
Returning: identified - newcomers,
})
}
if err := rows.Err(); err != nil {
return nil, api.Totals{}, err
}
if points == nil {
points = []api.FootfallPoint{}
}
var totals api.Totals
var identified, anonymous int
if err := s.pool.QueryRow(ctx, `
SELECT count(DISTINCT v.visitor_id),
count(*) FILTER (WHERE v.visitor_id IS NULL),
count(*)
FROM visits v
WHERE v.client_id = $1
AND v.occurred_at >= $2 AND v.occurred_at < $3
AND ($4::uuid IS NULL OR v.site_id = $4::uuid)`,
q.ClientID, q.From, q.To, site).
Scan(&identified, &anonymous, &totals.Visits); err != nil {
return nil, api.Totals{}, fmt.Errorf("footfall totals: %w", err)
}
totals.UniqueVisitors = identified + anonymous
// Worst site, not the average. One badly placed camera is a hole in this
// report, and averaging it against three good ones hides the only site
// anyone needs to do something about.
err = s.pool.QueryRow(ctx, `
SELECT a.fraction_below_gate, si.name
FROM agents a
JOIN sites si ON si.id = a.site_id
WHERE a.client_id = $1
AND a.fraction_below_gate IS NOT NULL
AND ($2::uuid IS NULL OR a.site_id = $2::uuid)
ORDER BY a.fraction_below_gate DESC
LIMIT 1`, q.ClientID, site).
Scan(&totals.FractionBelowGate, &totals.WorstSite)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return nil, api.Totals{}, fmt.Errorf("gate fraction: %w", err)
}
return points, totals, nil
}
// Conversion answers "how many of the people who walked in bought something".
//
// Revenue is summed for ONE currency - whichever accounts for the most of it.
// Adding rupees to dollars produces a number that looks like money and is not,
// and this figure is the one a customer judges the product by.
func (s *Store) Conversion(ctx context.Context, q api.ReportQuery) (api.SalesReport, error) {
site := nullUUID(q.SiteID)
var rep api.SalesReport
var identified, anonymous int
if err := s.pool.QueryRow(ctx, `
SELECT count(DISTINCT v.visitor_id),
count(*) FILTER (WHERE v.visitor_id IS NULL)
FROM visits v
WHERE v.client_id = $1
AND v.occurred_at >= $2 AND v.occurred_at < $3
AND ($4::uuid IS NULL OR v.site_id = $4::uuid)`,
q.ClientID, q.From, q.To, site).Scan(&identified, &anonymous); err != nil {
return rep, fmt.Errorf("conversion visitors: %w", err)
}
rep.Visitors = identified + anonymous
var baskets int
if err := s.pool.QueryRow(ctx, `
WITH scoped AS (
SELECT p.visitor_id, p.amount, p.currency
FROM purchases p
WHERE p.client_id = $1
AND p.occurred_at >= $2 AND p.occurred_at < $3
AND ($4::uuid IS NULL OR p.site_id = $4::uuid)
),
dominant AS (
SELECT currency FROM scoped
GROUP BY currency ORDER BY sum(amount) DESC LIMIT 1
)
SELECT COALESCE((SELECT currency FROM dominant), 'INR'),
count(DISTINCT s.visitor_id),
count(*),
COALESCE(sum(s.amount), 0)::float8
FROM scoped s
WHERE s.currency = COALESCE((SELECT currency FROM dominant), 'INR')`,
q.ClientID, q.From, q.To, site).
Scan(&rep.Currency, &rep.Purchasers, &baskets, &rep.Revenue); err != nil {
return rep, fmt.Errorf("conversion purchases: %w", err)
}
if rep.Visitors > 0 {
rep.Conversion = float64(rep.Purchasers) / float64(rep.Visitors)
}
if baskets > 0 {
// Per basket, not per purchaser: a customer who bought twice in the
// window had two baskets, and averaging over people would overstate
// what a single transaction is worth.
rep.AvgBasket = rep.Revenue / float64(baskets)
}
return rep, nil
}
func (s *Store) SiteHealth(ctx context.Context, clientID string) ([]api.SiteHealth, error) {
rows, err := s.pool.Query(ctx, `
SELECT si.id::text, si.slug, si.name, si.timezone,
a.last_heartbeat_at, a.last_event_at,
COALESCE(a.recognition_model, ''), COALESCE(a.agent_version, ''),
COALESCE(a.cameras_up, 0), COALESCE(a.cameras_total, 0),
a.fraction_below_gate,
COALESCE(a.spool_queued, 0), COALESCE(a.spool_dropped, 0)
FROM sites si
LEFT JOIN agents a ON a.site_id = si.id
WHERE si.client_id = $1 AND si.active
ORDER BY si.name`, clientID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []api.SiteHealth
for rows.Next() {
var h api.SiteHealth
var beat, event *time.Time
var gate *float64
if err := rows.Scan(&h.SiteID, &h.Slug, &h.Name, &h.Timezone,
&beat, &event, &h.RecognitionModel, &h.AgentVersion,
&h.CamerasUp, &h.CamerasTotal, &gate,
&h.Queued, &h.Dropped); err != nil {
return nil, err
}
if beat != nil {
h.LastHeartbeatAt = beat.UTC().Format(time.RFC3339)
// Three missed beats. One missed beat is a dropped packet; three is
// a site that has actually gone away, and calling that out too
// eagerly trains people to ignore the indicator.
h.Online = time.Since(*beat) < 3*time.Minute
}
if event != nil {
h.LastEventAt = event.UTC().Format(time.RFC3339)
}
if gate != nil {
h.FractionBelowGate = *gate
}
out = append(out, h)
}
return out, rows.Err()
}
// Compile-time proof that the store satisfies what the API asks for. Without
// it a missing method is only discovered when main.go is wired up, which is the
// one file least covered by tests.
var _ api.Store = (*Store)(nil)

334
server/internal/store/store.go Normal file
View File

@@ -0,0 +1,334 @@
// Package store is the Postgres implementation of the ingest Store.
//
// Every statement filters or writes client_id explicitly, even where a join
// could derive it. That redundancy is the point: a cross-tenant leak then
// requires a deliberately wrong WHERE clause rather than one forgotten join
// condition.
package store
import (
"context"
"errors"
"fmt"
"log"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/loyaly/behavision-server/internal/contract"
"github.com/loyaly/behavision-server/internal/ingest"
"github.com/loyaly/behavision-server/internal/secret"
)
type Store struct {
pool *pgxpool.Pool
// secrets decrypts the few values the server must hand back out - today,
// each site's broker password. Nil until UseSecrets is called.
secrets *secret.Box
log *log.Logger
}
// UseLogger gives the store somewhere to report failures it deliberately does
// not surface to the caller, such as a refused audit write.
func (s *Store) UseLogger(l *log.Logger) { s.log = l }
func (s *Store) auditFailed(action string, err error) {
if s.log != nil {
s.log.Printf("WARN audit write failed for %s: %v", action, err)
}
}
func Open(ctx context.Context, dsn string) (*Store, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, fmt.Errorf("bad database url: %w", err)
}
// Small pool on purpose. This box has 2 vCPU and runs someone else's
// services; a large idle pool costs memory to no benefit at this volume.
cfg.MaxConns = 8
cfg.MinConns = 1
cfg.MaxConnLifetime = time.Hour
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, err
}
if err := pool.Ping(ctx); err != nil {
pool.Close()
return nil, fmt.Errorf("database unreachable: %w", err)
}
return &Store{pool: pool}, nil
}
func (s *Store) Close() { s.pool.Close() }
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
// ResolveSite maps an authenticated MQTT username to a provisioned tenant.
// It only ever reads: see the comment in ingest.Consumer.Handle.
func (s *Store) ResolveSite(ctx context.Context, mqttUsername string) (ingest.Site, error) {
var site ingest.Site
err := s.pool.QueryRow(ctx, `
SELECT a.client_id::text, a.site_id::text, a.id::text, a.mqtt_username
FROM agents a
JOIN sites si ON si.id = a.site_id AND si.active
JOIN clients c ON c.id = a.client_id AND c.active
WHERE a.mqtt_username = $1`, mqttUsername).
Scan(&site.ClientID, &site.SiteID, &site.AgentID, &site.Slug)
if errors.Is(err, pgx.ErrNoRows) {
return ingest.Site{}, ingest.ErrUnknownSite
}
if err != nil {
return ingest.Site{}, err
}
return site, nil
}
// RecordVisit writes one visit, resolving or creating the visitor.
//
// Returns inserted=false when the event was already stored. That is not an
// error: MQTT delivery is at-least-once, so a redelivery after a reconnect is
// expected, and treating it as a failure would make every reconnect look like
// an outage.
func (s *Store) RecordVisit(ctx context.Context, site ingest.Site,
v *contract.Visit) (bool, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(ctx) //nolint:errcheck // no-op once committed
// Claim the event id first. If it is already there, nothing else in this
// transaction should run - in particular we must not create a second
// visitor for a visit we already recorded.
var visitID string
err = tx.QueryRow(ctx, `
INSERT INTO visits (client_id, site_id, source_event_id, occurred_at,
camera_id, is_new_visitor, similarity, quality,
attributes, image_key)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, COALESCE($9, '{}'::jsonb), $10)
ON CONFLICT (client_id, source_event_id) DO NOTHING
RETURNING id::text`,
site.ClientID, site.SiteID, v.EventID, v.OccurredAt, v.CameraID,
v.IsNew, nullFloat(v.Similarity), nullFloat(v.Quality),
v.Attributes, v.ImageKey).Scan(&visitID)
if errors.Is(err, pgx.ErrNoRows) {
return false, tx.Commit(ctx) // already recorded
}
if err != nil {
return false, fmt.Errorf("insert visit: %w", err)
}
// Only now decide who this was. Matching is scoped to the client, never
// global: linking a face across unrelated clients would build a
// cross-company biometric tracking network.
if len(v.Embedding) == contract.EmbeddingDim {
visitorID, err := s.matchOrCreateVisitor(ctx, tx, site, v)
if err != nil {
return false, err
}
if _, err := tx.Exec(ctx,
`UPDATE visits SET visitor_id = $1 WHERE id = $2`,
visitorID, visitID); err != nil {
return false, err
}
if _, err := tx.Exec(ctx, `
UPDATE visitors
SET last_seen_at = GREATEST(COALESCE(last_seen_at, $2), $2),
visit_count = visit_count + 1
WHERE id = $1 AND client_id = $3`,
visitorID, v.OccurredAt, site.ClientID); err != nil {
return false, err
}
}
if _, err := tx.Exec(ctx,
`UPDATE agents SET last_event_at = now() WHERE id = $1`,
site.AgentID); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
// Thresholds mirror the edge defaults. Server-side matching answers a
// different question than the agent's - "has this person been to ANY of this
// client's sites" - but the vectors and the geometry are identical, so the
// numbers must be too. Diverging would mean two components disagreeing about
// who someone is.
const (
matchThreshold = 0.42
enrollThreshold = 0.32
reinforceThreshold = 0.55
maxEmbeddings = 5
// The server cannot know each camera's own quality gate, and every camera
// writes into ONE client-wide gallery, so it applies its own floor. Without
// it a loosely-gated camera could weld a poor view onto an identity that a
// strict camera then trusts.
minReinforceQuality = 0.45
)
func (s *Store) matchOrCreateVisitor(ctx context.Context, tx pgx.Tx,
site ingest.Site, v *contract.Visit) (string, error) {
vec := pgVector(v.Embedding)
// Exact nearest neighbour, scoped to this client and this encoder.
// `<=>` is cosine distance, so similarity is 1 - distance.
var visitorID string
var similarity float64
err := tx.QueryRow(ctx, `
SELECT e.visitor_id::text, 1 - (e.embedding <=> $1::vector) AS sim
FROM visitor_embeddings e
JOIN visitors vi ON vi.id = e.visitor_id AND vi.deleted_at IS NULL
WHERE e.client_id = $2 AND e.model = $3
ORDER BY e.embedding <=> $1::vector
LIMIT 1`, vec, site.ClientID, v.Model).Scan(&visitorID, &similarity)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return "", fmt.Errorf("match visitor: %w", err)
}
if err == nil && similarity >= matchThreshold {
// Known person. Consider keeping this view too.
//
// Without this an identity is born holding the single embedding from
// the first second it was ever seen, and the next encounter at a
// different angle has one reference vector to beat. That is not
// hypothetical: measured live on the Office1 camera, exactly this
// produced one person as two identities at similarity 0.304. The edge
// fixes it with reinforce_identity; the server has the same problem
// with the same cause and needs the same fix.
if err := reinforce(ctx, tx, site, v, visitorID, similarity, vec); err != nil {
return "", err
}
return visitorID, nil
}
// New person for this client.
var newID string
if err := tx.QueryRow(ctx, `
INSERT INTO visitors (client_id, label, first_seen_at)
VALUES ($1, '', $2) RETURNING id::text`,
site.ClientID, v.OccurredAt).Scan(&newID); err != nil {
return "", fmt.Errorf("create visitor: %w", err)
}
if _, err := tx.Exec(ctx, `
UPDATE visitors SET label = 'Visitor ' || left(id::text, 8)
WHERE id = $1 AND label = ''`, newID); err != nil {
return "", err
}
if _, err := tx.Exec(ctx, `
INSERT INTO visitor_embeddings
(visitor_id, client_id, model, embedding, quality, source_site_id)
VALUES ($1, $2, $3, $4::vector, $5, $6)`,
newID, site.ClientID, v.Model, vec, v.Quality, site.SiteID); err != nil {
return "", fmt.Errorf("store embedding: %w", err)
}
return newID, nil
}
func (s *Store) RecordHeartbeat(ctx context.Context, site ingest.Site,
h *contract.Heartbeat) error {
up, total := 0, len(h.Cameras)
for _, ok := range h.Cameras {
if ok {
up++
}
}
// fraction_below_gate is NULL until a site has actually measured one.
// Storing 0.0 for "not reported" would read as a perfectly placed camera,
// which is the opposite of what an unmeasured site means.
var gate any
if h.FractionBelowGate > 0 {
gate = h.FractionBelowGate
}
_, err := s.pool.Exec(ctx, `
UPDATE agents
SET last_heartbeat_at = now(),
agent_version = COALESCE(NULLIF($2, ''), agent_version),
engine_version = COALESCE(NULLIF($3, ''), engine_version),
recognition_model = COALESCE(NULLIF($4, ''), recognition_model),
cameras_up = $5,
cameras_total = $6,
spool_queued = $7,
-- Never decreases. Dropped events are footfall a site permanently
-- lost; a restart that reset the agent's own counter must not make
-- that loss disappear from the report.
spool_dropped = GREATEST(spool_dropped, $8),
fraction_below_gate = COALESCE($9::real, fraction_below_gate)
WHERE id = $1`,
site.AgentID, h.AgentVersion, h.EngineVersion, h.RecognitionModel,
up, total, h.Queued, int64(h.Dropped), gate)
return err
}
// reinforce adds another view of an already-identified person.
//
// Guarded the same three ways as the edge, and for the same reasons:
//
// - Similar enough to believe it is them. Below enrollThreshold the matcher
// would call this vector a DIFFERENT person, so attaching it here would
// contradict the number driving every other decision.
// - Different enough to be worth storing. Above reinforceThreshold it is a
// near-duplicate of what we already hold and teaches the gallery nothing.
// - Good enough to keep. A blurred view welded onto an identity is
// unrecoverable; a missed hard angle is not. The risk is asymmetric, so
// the gate leans towards refusing.
//
// Capped, because an identity holding fifty vectors starts matching everyone.
func reinforce(ctx context.Context, tx pgx.Tx, site ingest.Site,
v *contract.Visit, visitorID string, similarity float64, vec string) error {
if similarity < enrollThreshold || similarity >= reinforceThreshold {
return nil
}
if v.Quality < minReinforceQuality {
return nil
}
var n int
if err := tx.QueryRow(ctx, `
SELECT count(*) FROM visitor_embeddings
WHERE visitor_id = $1 AND client_id = $2 AND model = $3`,
visitorID, site.ClientID, v.Model).Scan(&n); err != nil {
return fmt.Errorf("count embeddings: %w", err)
}
if n >= maxEmbeddings {
return nil
}
if _, err := tx.Exec(ctx, `
INSERT INTO visitor_embeddings
(visitor_id, client_id, model, embedding, quality, source_site_id)
VALUES ($1, $2, $3, $4::vector, $5, $6)`,
visitorID, site.ClientID, v.Model, vec, v.Quality, site.SiteID); err != nil {
return fmt.Errorf("reinforce: %w", err)
}
return nil
}
// pgVector renders a float slice in pgvector's literal form. Built by hand
// rather than with a driver type so the store has no dependency on a pgvector
// Go package - the format is a bracketed comma list and nothing more.
func pgVector(v []float32) string {
var b strings.Builder
b.Grow(len(v) * 12)
b.WriteByte('[')
for i, f := range v {
if i > 0 {
b.WriteByte(',')
}
fmt.Fprintf(&b, "%g", f)
}
b.WriteByte(']')
return b.String()
}
// nullFloat keeps "not measured" distinct from "measured as zero". A track
// that never reached a decision has no similarity, and storing 0 would drag
// every percentile down.
func nullFloat(f float32) any {
if f == 0 {
return nil
}
return f
}