Files
Suriyakumarvijayanayagam dad04e8cda Behavision: face recognition for retail, edge to head office
Five components that ship as one product:

- behavision/  the recognition engine. RTSP ingest, YuNet detection, IoU
               tracking, ArcFace embeddings, a FAISS/SQLite gallery, and a
               FastAPI dashboard. Identity is decided once per TRACK from an
               average of at least three embeddings, never per frame.
- agent/       the Go edge agent: supervises the engine, holds a durable
               spool, and drains it to MQTT. Nothing is acked before the
               broker confirms.
- desktop/     the shop PC application (Wails + React + tray).
- server/      the cloud API, MQTT consumer, reports and assistant.
- web/         platform.loyaly.ai, the head-office app, embedded in the
               server binary.

The gallery stores 512-float embeddings and timestamps - no images unless
`app.store_faces` is switched on. Those embeddings are biometric personal
data under GDPR and India's DPDP: template inversion reconstructs a
recognisable face from an ArcFace vector, so data/behavision.db is treated
as a biometric database and DELETE /api/visitors/{id} is a real erasure.

CLAUDE.md carries the reasoning behind every non-obvious decision here,
including the ones that were measured and the ones that were wrong first.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HViLj9gYNRtSr7YVZmW5sn
2026-09-04 11:14:18 +05:30

397 lines
14 KiB
Go

// Package assistant answers questions about a shop in plain language, by
// calling the same business questions the screens ask.
//
// The tools here are deliberately BUSINESS-level - `store_footfall`, not
// `execute_sql`. That is not a stylistic preference. An assistant handed raw
// SQL has to invent the arithmetic, and this product's arithmetic is full of
// traps that produce a plausible wrong number rather than an error:
//
// - unique visitors is not the sum of the daily bars
// - "new" means first-ever, not first-in-this-window
// - new + returning can be less than the total, because a site sending
// counts without templates records real people nobody identified
// - revenue is one currency; adding rupees to dollars produces something
// that looks like money and is not
//
// Every one of those is already settled, tested, and used by the reports. A
// tool that returns the settled answer cannot get them wrong; a tool that
// returns rows invites the model to re-derive them badly.
//
// The same registry is what an MCP server would expose. Nothing here depends
// on the Anthropic SDK - that lives in one file next door - so a second
// consumer needs no changes.
package assistant
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/loyaly/behavision-server/internal/api"
"github.com/loyaly/behavision-server/internal/auth"
)
// Store is what the assistant needs. A narrow interface, and read-mostly on
// purpose: the one thing it can change is asking a shop PC to check a camera,
// which is reversible and is the single action a support conversation actually
// needs to take.
type Store interface {
SiteHealth(ctx context.Context, clientID string) ([]api.SiteHealth, error)
Cameras(ctx context.Context, clientID, siteID string) ([]api.Camera, error)
CameraByID(ctx context.Context, clientID, id string) (api.Camera, error)
RequestCheck(ctx context.Context, clientID, id, kind string, seconds int) error
Footfall(ctx context.Context, q api.ReportQuery) ([]api.FootfallPoint, api.Totals, error)
Conversion(ctx context.Context, q api.ReportQuery) (api.SalesReport, error)
SearchVisitors(ctx context.Context, clientID, query string, limit int) ([]api.Customer, error)
}
// Tool is one business question, independent of any LLM SDK.
type Tool struct {
Name string
Description string
// Schema is a JSON Schema object for the arguments.
Schema map[string]any
// Run answers it. The principal is the SIGNED-IN USER, passed in by the
// caller and never taken from the arguments - see Registry.Call.
Run func(ctx context.Context, p auth.Principal, args json.RawMessage) (any, error)
}
type Registry struct {
Store Store
// SiteChecker is the smoke test. A func rather than a method so the
// judgement stays in the api package beside the screens that show it.
SiteChecker func(site *api.SiteHealth, cams []api.Camera, now time.Time) []api.CheckStep
Now func() time.Time
tools []Tool
}
func (r *Registry) now() time.Time {
if r.Now != nil {
return r.Now()
}
return time.Now().UTC()
}
// Call runs a tool as a specific signed-in user.
//
// The principal comes from the SESSION and is passed in here; no tool takes a
// client_id argument, so there is nothing for the model to set. That is the
// whole tenancy story for the assistant, and it is a property of the
// signatures rather than of anybody remembering to check.
func (r *Registry) Call(ctx context.Context, p auth.Principal, name string,
args json.RawMessage) (string, error) {
for _, t := range r.Tools() {
if t.Name != name {
continue
}
out, err := t.Run(ctx, p, args)
if err != nil {
// Returned as a RESULT, not an error: the model can usually
// recover ("that shop does not exist - here are the ones that
// do"), and killing the turn would leave the user with nothing.
return fmt.Sprintf("That did not work: %v", err), nil
}
body, err := json.Marshal(out)
if err != nil {
return "", err
}
return string(body), nil
}
return "", fmt.Errorf("no such tool %q", name)
}
func (r *Registry) Tools() []Tool {
if r.tools == nil {
r.tools = r.build()
}
return r.tools
}
func obj(props map[string]any, required ...string) map[string]any {
if required == nil {
required = []string{}
}
return map[string]any{
"type": "object", "properties": props,
"required": required, "additionalProperties": false,
}
}
func str(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} }
func (r *Registry) build() []Tool {
return []Tool{
{
Name: "list_shops",
Description: "Every shop this account can see, with whether its PC is online, " +
"how many cameras are connected, and what share of the faces its cameras " +
"saw were too poor to recognise. Start here when a question names a shop.",
Schema: obj(map[string]any{}),
Run: func(ctx context.Context, p auth.Principal, _ json.RawMessage) (any, error) {
return r.Store.SiteHealth(ctx, p.ClientID)
},
},
{
Name: "check_shop",
Description: "Run the end-to-end check on one shop: is the PC online, is " +
"recognition running, are cameras connected, can they recognise faces, " +
"are visits reaching head office. Use this for 'is X working' and for " +
"any complaint that footfall looks wrong or too low.",
Schema: obj(map[string]any{"shop": str("The shop's name or id")}, "shop"),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
var in struct {
Shop string `json:"shop"`
}
if err := json.Unmarshal(raw, &in); err != nil {
return nil, err
}
site, err := r.findSite(ctx, p, in.Shop)
if err != nil {
return nil, err
}
cams, err := r.Store.Cameras(ctx, p.ClientID, site.SiteID)
if err != nil {
return nil, err
}
return map[string]any{
"shop": site.Name,
"steps": r.SiteChecker(site, cams, r.now()),
}, nil
},
},
{
Name: "list_cameras",
Description: "Cameras, with whether each is connected and whether anyone has " +
"proved it can actually recognise a face. Those are different things: a " +
"camera can stream perfectly and still produce views nothing can recognise.",
Schema: obj(map[string]any{"shop": str("Optional: limit to one shop")}),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
var in struct {
Shop string `json:"shop"`
}
_ = json.Unmarshal(raw, &in)
siteID := ""
if strings.TrimSpace(in.Shop) != "" {
site, err := r.findSite(ctx, p, in.Shop)
if err != nil {
return nil, err
}
siteID = site.SiteID
}
cams, err := r.Store.Cameras(ctx, p.ClientID, siteID)
if err != nil {
return nil, err
}
// Trimmed deliberately. The full row carries object keys and
// connection details that are of no use in an answer and would
// spend context on every turn.
out := make([]map[string]any, 0, len(cams))
for _, c := range cams {
out = append(out, map[string]any{
"id": c.ID, "name": c.Label, "shop": c.Site,
"connected": c.Connected,
"verified": verificationOf(c),
})
}
return out, nil
},
},
{
Name: "check_camera",
Description: "Ask a shop's PC to test one camera. kind=connection asks whether " +
"it can open the stream; kind=placement watches for 25 seconds and judges " +
"whether somebody walking past produces a view good enough to recognise - " +
"that one needs a person to actually walk past. The answer arrives in a " +
"couple of minutes, not immediately.",
Schema: obj(map[string]any{
"camera_id": str("The camera's id, from list_cameras"),
"kind": map[string]any{"type": "string", "enum": []string{"connection", "placement"}},
}, "camera_id", "kind"),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
var in struct {
CameraID string `json:"camera_id"`
Kind string `json:"kind"`
}
if err := json.Unmarshal(raw, &in); err != nil {
return nil, err
}
if !p.CanManageSites() {
// Refused here rather than left to the prompt. An
// instruction not to do something is not a permission
// check, and this one writes to a shop's PC.
return nil, fmt.Errorf(
"this account cannot run camera checks - a manager or owner can")
}
if in.Kind != "connection" && in.Kind != "placement" {
return nil, fmt.Errorf(`kind must be "connection" or "placement"`)
}
if err := r.Store.RequestCheck(ctx, p.ClientID, in.CameraID, in.Kind, 25); err != nil {
return nil, fmt.Errorf("no camera with that id")
}
cam, err := r.Store.CameraByID(ctx, p.ClientID, in.CameraID)
if err != nil {
return nil, err
}
return map[string]any{
"requested": in.Kind, "camera": cam.Label, "shop": cam.Site,
"note": "The shop's PC picks this up within a couple of minutes. " +
"Tell the person to watch the camera's card for the result.",
}, nil
},
},
{
Name: "footfall",
Description: "How many people visited. Returns unique people over the window " +
"AND the visit count, which are different numbers - a regular is one " +
"person and several visits. Also returns what share of faces were too " +
"poor to recognise, which says whether the figure can be believed at all.",
Schema: obj(map[string]any{
"from": str("YYYY-MM-DD"), "to": str("YYYY-MM-DD (inclusive)"),
"shop": str("Optional: one shop"),
"bucket": map[string]any{"type": "string", "enum": []string{"hour", "day", "week", "month"}},
}, "from", "to"),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
q, err := r.reportQuery(ctx, p, raw)
if err != nil {
return nil, err
}
points, totals, err := r.Store.Footfall(ctx, q)
if err != nil {
return nil, err
}
return map[string]any{
"unique_people": totals.UniqueVisitors,
"visits": totals.Visits,
"per_bucket": points,
"timezone": q.Timezone,
"fraction_of_faces_too_poor_to_recognise": totals.FractionBelowGate,
"worst_shop": totals.WorstSite,
"note": "unique_people is people; visits counts every appearance. " +
"They differ because regulars come back - do not add the buckets up " +
"to get unique_people.",
}, nil
},
},
{
Name: "sales",
Description: "How many visitors bought something, the conversion rate, revenue " +
"and average basket. Revenue is a single currency - whichever accounts for " +
"most of it - and average basket is per basket, not per person.",
Schema: obj(map[string]any{
"from": str("YYYY-MM-DD"), "to": str("YYYY-MM-DD (inclusive)"),
"shop": str("Optional: one shop"),
}, "from", "to"),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
q, err := r.reportQuery(ctx, p, raw)
if err != nil {
return nil, err
}
return r.Store.Conversion(ctx, q)
},
},
{
Name: "find_customer",
Description: "Search customers by name, phone or email. Returns how many times " +
"each has visited and whether they have given consent for their details to " +
"be kept.",
Schema: obj(map[string]any{"query": str("Part of a name, phone or email")}, "query"),
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
var in struct {
Query string `json:"query"`
}
if err := json.Unmarshal(raw, &in); err != nil {
return nil, err
}
return r.Store.SearchVisitors(ctx, p.ClientID, in.Query, 10)
},
},
}
}
// findSite resolves what a person typed to a shop they can see.
//
// Matched against the tenant's OWN shops, so a name the model invented or a
// caller supplied cannot reach another tenant: the candidate list never
// contains anybody else's shops in the first place.
func (r *Registry) findSite(ctx context.Context, p auth.Principal, want string) (*api.SiteHealth, error) {
sites, err := r.Store.SiteHealth(ctx, p.ClientID)
if err != nil {
return nil, err
}
want = strings.ToLower(strings.TrimSpace(want))
if want == "" && len(sites) == 1 {
// One shop and no name given: the question can only be about that one.
return &sites[0], nil
}
for i := range sites {
if strings.EqualFold(sites[i].SiteID, want) ||
strings.EqualFold(sites[i].Slug, want) ||
strings.Contains(strings.ToLower(sites[i].Name), want) {
return &sites[i], nil
}
}
names := make([]string, 0, len(sites))
for _, s := range sites {
names = append(names, s.Name)
}
return nil, fmt.Errorf("no shop matching %q. This account has: %s",
want, strings.Join(names, ", "))
}
func (r *Registry) reportQuery(ctx context.Context, p auth.Principal,
raw json.RawMessage) (api.ReportQuery, error) {
var in struct {
From, To, Shop, Bucket string
}
if err := json.Unmarshal(raw, &in); err != nil {
return api.ReportQuery{}, err
}
q := api.ReportQuery{ClientID: p.ClientID, Bucket: in.Bucket, Timezone: "Asia/Kolkata"}
if q.Bucket == "" {
q.Bucket = "day"
}
from, err := time.Parse("2006-01-02", in.From)
if err != nil {
return q, fmt.Errorf("from must be a date like 2026-09-01")
}
to, err := time.Parse("2006-01-02", in.To)
if err != nil {
return q, fmt.Errorf("to must be a date like 2026-09-02")
}
// `to` is inclusive to a person and exclusive in SQL. Converted here, in
// one place, exactly as the report handler does - otherwise a question
// about "the 1st to the 7th" quietly loses the 7th's trade.
q.From, q.To = from, to.AddDate(0, 0, 1)
if !q.To.After(q.From) {
return q, fmt.Errorf("to must be on or after from")
}
if strings.TrimSpace(in.Shop) != "" {
site, err := r.findSite(ctx, p, in.Shop)
if err != nil {
return q, err
}
q.SiteID = site.SiteID
}
return q, nil
}
// verificationOf is the same two-claim distinction the camera card makes.
func verificationOf(c api.Camera) string {
switch {
case c.Check.State != "done":
return "never checked"
case c.Check.Kind == "placement" && c.Check.OK:
return "proved it can recognise faces"
case c.Check.Kind == "placement":
return "checked and CANNOT recognise faces here: " + c.Check.Headline
case c.Check.OK:
return "stream works, but nobody has proved it can recognise a face"
default:
return "could not be reached: " + c.Check.Headline
}
}