Five components that ship as one product:
- behavision/ the recognition engine. RTSP ingest, YuNet detection, IoU
tracking, ArcFace embeddings, a FAISS/SQLite gallery, and a
FastAPI dashboard. Identity is decided once per TRACK from an
average of at least three embeddings, never per frame.
- agent/ the Go edge agent: supervises the engine, holds a durable
spool, and drains it to MQTT. Nothing is acked before the
broker confirms.
- desktop/ the shop PC application (Wails + React + tray).
- server/ the cloud API, MQTT consumer, reports and assistant.
- web/ platform.loyaly.ai, the head-office app, embedded in the
server binary.
The gallery stores 512-float embeddings and timestamps - no images unless
`app.store_faces` is switched on. Those embeddings are biometric personal
data under GDPR and India's DPDP: template inversion reconstructs a
recognisable face from an ArcFace vector, so data/behavision.db is treated
as a biometric database and DELETE /api/visitors/{id} is a real erasure.
CLAUDE.md carries the reasoning behind every non-obvious decision here,
including the ones that were measured and the ones that were wrong first.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HViLj9gYNRtSr7YVZmW5sn
397 lines
14 KiB
Go
397 lines
14 KiB
Go
// Package assistant answers questions about a shop in plain language, by
|
|
// calling the same business questions the screens ask.
|
|
//
|
|
// The tools here are deliberately BUSINESS-level - `store_footfall`, not
|
|
// `execute_sql`. That is not a stylistic preference. An assistant handed raw
|
|
// SQL has to invent the arithmetic, and this product's arithmetic is full of
|
|
// traps that produce a plausible wrong number rather than an error:
|
|
//
|
|
// - unique visitors is not the sum of the daily bars
|
|
// - "new" means first-ever, not first-in-this-window
|
|
// - new + returning can be less than the total, because a site sending
|
|
// counts without templates records real people nobody identified
|
|
// - revenue is one currency; adding rupees to dollars produces something
|
|
// that looks like money and is not
|
|
//
|
|
// Every one of those is already settled, tested, and used by the reports. A
|
|
// tool that returns the settled answer cannot get them wrong; a tool that
|
|
// returns rows invites the model to re-derive them badly.
|
|
//
|
|
// The same registry is what an MCP server would expose. Nothing here depends
|
|
// on the Anthropic SDK - that lives in one file next door - so a second
|
|
// consumer needs no changes.
|
|
package assistant
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/loyaly/behavision-server/internal/api"
|
|
"github.com/loyaly/behavision-server/internal/auth"
|
|
)
|
|
|
|
// Store is what the assistant needs. A narrow interface, and read-mostly on
|
|
// purpose: the one thing it can change is asking a shop PC to check a camera,
|
|
// which is reversible and is the single action a support conversation actually
|
|
// needs to take.
|
|
type Store interface {
|
|
SiteHealth(ctx context.Context, clientID string) ([]api.SiteHealth, error)
|
|
Cameras(ctx context.Context, clientID, siteID string) ([]api.Camera, error)
|
|
CameraByID(ctx context.Context, clientID, id string) (api.Camera, error)
|
|
RequestCheck(ctx context.Context, clientID, id, kind string, seconds int) error
|
|
Footfall(ctx context.Context, q api.ReportQuery) ([]api.FootfallPoint, api.Totals, error)
|
|
Conversion(ctx context.Context, q api.ReportQuery) (api.SalesReport, error)
|
|
SearchVisitors(ctx context.Context, clientID, query string, limit int) ([]api.Customer, error)
|
|
}
|
|
|
|
// Tool is one business question, independent of any LLM SDK.
|
|
type Tool struct {
|
|
Name string
|
|
Description string
|
|
// Schema is a JSON Schema object for the arguments.
|
|
Schema map[string]any
|
|
// Run answers it. The principal is the SIGNED-IN USER, passed in by the
|
|
// caller and never taken from the arguments - see Registry.Call.
|
|
Run func(ctx context.Context, p auth.Principal, args json.RawMessage) (any, error)
|
|
}
|
|
|
|
type Registry struct {
|
|
Store Store
|
|
// SiteChecker is the smoke test. A func rather than a method so the
|
|
// judgement stays in the api package beside the screens that show it.
|
|
SiteChecker func(site *api.SiteHealth, cams []api.Camera, now time.Time) []api.CheckStep
|
|
Now func() time.Time
|
|
tools []Tool
|
|
}
|
|
|
|
func (r *Registry) now() time.Time {
|
|
if r.Now != nil {
|
|
return r.Now()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
// Call runs a tool as a specific signed-in user.
|
|
//
|
|
// The principal comes from the SESSION and is passed in here; no tool takes a
|
|
// client_id argument, so there is nothing for the model to set. That is the
|
|
// whole tenancy story for the assistant, and it is a property of the
|
|
// signatures rather than of anybody remembering to check.
|
|
func (r *Registry) Call(ctx context.Context, p auth.Principal, name string,
|
|
args json.RawMessage) (string, error) {
|
|
|
|
for _, t := range r.Tools() {
|
|
if t.Name != name {
|
|
continue
|
|
}
|
|
out, err := t.Run(ctx, p, args)
|
|
if err != nil {
|
|
// Returned as a RESULT, not an error: the model can usually
|
|
// recover ("that shop does not exist - here are the ones that
|
|
// do"), and killing the turn would leave the user with nothing.
|
|
return fmt.Sprintf("That did not work: %v", err), nil
|
|
}
|
|
body, err := json.Marshal(out)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(body), nil
|
|
}
|
|
return "", fmt.Errorf("no such tool %q", name)
|
|
}
|
|
|
|
func (r *Registry) Tools() []Tool {
|
|
if r.tools == nil {
|
|
r.tools = r.build()
|
|
}
|
|
return r.tools
|
|
}
|
|
|
|
func obj(props map[string]any, required ...string) map[string]any {
|
|
if required == nil {
|
|
required = []string{}
|
|
}
|
|
return map[string]any{
|
|
"type": "object", "properties": props,
|
|
"required": required, "additionalProperties": false,
|
|
}
|
|
}
|
|
|
|
func str(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} }
|
|
|
|
func (r *Registry) build() []Tool {
|
|
return []Tool{
|
|
{
|
|
Name: "list_shops",
|
|
Description: "Every shop this account can see, with whether its PC is online, " +
|
|
"how many cameras are connected, and what share of the faces its cameras " +
|
|
"saw were too poor to recognise. Start here when a question names a shop.",
|
|
Schema: obj(map[string]any{}),
|
|
Run: func(ctx context.Context, p auth.Principal, _ json.RawMessage) (any, error) {
|
|
return r.Store.SiteHealth(ctx, p.ClientID)
|
|
},
|
|
},
|
|
{
|
|
Name: "check_shop",
|
|
Description: "Run the end-to-end check on one shop: is the PC online, is " +
|
|
"recognition running, are cameras connected, can they recognise faces, " +
|
|
"are visits reaching head office. Use this for 'is X working' and for " +
|
|
"any complaint that footfall looks wrong or too low.",
|
|
Schema: obj(map[string]any{"shop": str("The shop's name or id")}, "shop"),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
var in struct {
|
|
Shop string `json:"shop"`
|
|
}
|
|
if err := json.Unmarshal(raw, &in); err != nil {
|
|
return nil, err
|
|
}
|
|
site, err := r.findSite(ctx, p, in.Shop)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cams, err := r.Store.Cameras(ctx, p.ClientID, site.SiteID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{
|
|
"shop": site.Name,
|
|
"steps": r.SiteChecker(site, cams, r.now()),
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
Name: "list_cameras",
|
|
Description: "Cameras, with whether each is connected and whether anyone has " +
|
|
"proved it can actually recognise a face. Those are different things: a " +
|
|
"camera can stream perfectly and still produce views nothing can recognise.",
|
|
Schema: obj(map[string]any{"shop": str("Optional: limit to one shop")}),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
var in struct {
|
|
Shop string `json:"shop"`
|
|
}
|
|
_ = json.Unmarshal(raw, &in)
|
|
siteID := ""
|
|
if strings.TrimSpace(in.Shop) != "" {
|
|
site, err := r.findSite(ctx, p, in.Shop)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
siteID = site.SiteID
|
|
}
|
|
cams, err := r.Store.Cameras(ctx, p.ClientID, siteID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Trimmed deliberately. The full row carries object keys and
|
|
// connection details that are of no use in an answer and would
|
|
// spend context on every turn.
|
|
out := make([]map[string]any, 0, len(cams))
|
|
for _, c := range cams {
|
|
out = append(out, map[string]any{
|
|
"id": c.ID, "name": c.Label, "shop": c.Site,
|
|
"connected": c.Connected,
|
|
"verified": verificationOf(c),
|
|
})
|
|
}
|
|
return out, nil
|
|
},
|
|
},
|
|
{
|
|
Name: "check_camera",
|
|
Description: "Ask a shop's PC to test one camera. kind=connection asks whether " +
|
|
"it can open the stream; kind=placement watches for 25 seconds and judges " +
|
|
"whether somebody walking past produces a view good enough to recognise - " +
|
|
"that one needs a person to actually walk past. The answer arrives in a " +
|
|
"couple of minutes, not immediately.",
|
|
Schema: obj(map[string]any{
|
|
"camera_id": str("The camera's id, from list_cameras"),
|
|
"kind": map[string]any{"type": "string", "enum": []string{"connection", "placement"}},
|
|
}, "camera_id", "kind"),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
var in struct {
|
|
CameraID string `json:"camera_id"`
|
|
Kind string `json:"kind"`
|
|
}
|
|
if err := json.Unmarshal(raw, &in); err != nil {
|
|
return nil, err
|
|
}
|
|
if !p.CanManageSites() {
|
|
// Refused here rather than left to the prompt. An
|
|
// instruction not to do something is not a permission
|
|
// check, and this one writes to a shop's PC.
|
|
return nil, fmt.Errorf(
|
|
"this account cannot run camera checks - a manager or owner can")
|
|
}
|
|
if in.Kind != "connection" && in.Kind != "placement" {
|
|
return nil, fmt.Errorf(`kind must be "connection" or "placement"`)
|
|
}
|
|
if err := r.Store.RequestCheck(ctx, p.ClientID, in.CameraID, in.Kind, 25); err != nil {
|
|
return nil, fmt.Errorf("no camera with that id")
|
|
}
|
|
cam, err := r.Store.CameraByID(ctx, p.ClientID, in.CameraID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{
|
|
"requested": in.Kind, "camera": cam.Label, "shop": cam.Site,
|
|
"note": "The shop's PC picks this up within a couple of minutes. " +
|
|
"Tell the person to watch the camera's card for the result.",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
Name: "footfall",
|
|
Description: "How many people visited. Returns unique people over the window " +
|
|
"AND the visit count, which are different numbers - a regular is one " +
|
|
"person and several visits. Also returns what share of faces were too " +
|
|
"poor to recognise, which says whether the figure can be believed at all.",
|
|
Schema: obj(map[string]any{
|
|
"from": str("YYYY-MM-DD"), "to": str("YYYY-MM-DD (inclusive)"),
|
|
"shop": str("Optional: one shop"),
|
|
"bucket": map[string]any{"type": "string", "enum": []string{"hour", "day", "week", "month"}},
|
|
}, "from", "to"),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
q, err := r.reportQuery(ctx, p, raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
points, totals, err := r.Store.Footfall(ctx, q)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{
|
|
"unique_people": totals.UniqueVisitors,
|
|
"visits": totals.Visits,
|
|
"per_bucket": points,
|
|
"timezone": q.Timezone,
|
|
"fraction_of_faces_too_poor_to_recognise": totals.FractionBelowGate,
|
|
"worst_shop": totals.WorstSite,
|
|
"note": "unique_people is people; visits counts every appearance. " +
|
|
"They differ because regulars come back - do not add the buckets up " +
|
|
"to get unique_people.",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
Name: "sales",
|
|
Description: "How many visitors bought something, the conversion rate, revenue " +
|
|
"and average basket. Revenue is a single currency - whichever accounts for " +
|
|
"most of it - and average basket is per basket, not per person.",
|
|
Schema: obj(map[string]any{
|
|
"from": str("YYYY-MM-DD"), "to": str("YYYY-MM-DD (inclusive)"),
|
|
"shop": str("Optional: one shop"),
|
|
}, "from", "to"),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
q, err := r.reportQuery(ctx, p, raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return r.Store.Conversion(ctx, q)
|
|
},
|
|
},
|
|
{
|
|
Name: "find_customer",
|
|
Description: "Search customers by name, phone or email. Returns how many times " +
|
|
"each has visited and whether they have given consent for their details to " +
|
|
"be kept.",
|
|
Schema: obj(map[string]any{"query": str("Part of a name, phone or email")}, "query"),
|
|
Run: func(ctx context.Context, p auth.Principal, raw json.RawMessage) (any, error) {
|
|
var in struct {
|
|
Query string `json:"query"`
|
|
}
|
|
if err := json.Unmarshal(raw, &in); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.Store.SearchVisitors(ctx, p.ClientID, in.Query, 10)
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// findSite resolves what a person typed to a shop they can see.
|
|
//
|
|
// Matched against the tenant's OWN shops, so a name the model invented or a
|
|
// caller supplied cannot reach another tenant: the candidate list never
|
|
// contains anybody else's shops in the first place.
|
|
func (r *Registry) findSite(ctx context.Context, p auth.Principal, want string) (*api.SiteHealth, error) {
|
|
sites, err := r.Store.SiteHealth(ctx, p.ClientID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
want = strings.ToLower(strings.TrimSpace(want))
|
|
if want == "" && len(sites) == 1 {
|
|
// One shop and no name given: the question can only be about that one.
|
|
return &sites[0], nil
|
|
}
|
|
for i := range sites {
|
|
if strings.EqualFold(sites[i].SiteID, want) ||
|
|
strings.EqualFold(sites[i].Slug, want) ||
|
|
strings.Contains(strings.ToLower(sites[i].Name), want) {
|
|
return &sites[i], nil
|
|
}
|
|
}
|
|
names := make([]string, 0, len(sites))
|
|
for _, s := range sites {
|
|
names = append(names, s.Name)
|
|
}
|
|
return nil, fmt.Errorf("no shop matching %q. This account has: %s",
|
|
want, strings.Join(names, ", "))
|
|
}
|
|
|
|
func (r *Registry) reportQuery(ctx context.Context, p auth.Principal,
|
|
raw json.RawMessage) (api.ReportQuery, error) {
|
|
|
|
var in struct {
|
|
From, To, Shop, Bucket string
|
|
}
|
|
if err := json.Unmarshal(raw, &in); err != nil {
|
|
return api.ReportQuery{}, err
|
|
}
|
|
q := api.ReportQuery{ClientID: p.ClientID, Bucket: in.Bucket, Timezone: "Asia/Kolkata"}
|
|
if q.Bucket == "" {
|
|
q.Bucket = "day"
|
|
}
|
|
from, err := time.Parse("2006-01-02", in.From)
|
|
if err != nil {
|
|
return q, fmt.Errorf("from must be a date like 2026-09-01")
|
|
}
|
|
to, err := time.Parse("2006-01-02", in.To)
|
|
if err != nil {
|
|
return q, fmt.Errorf("to must be a date like 2026-09-02")
|
|
}
|
|
// `to` is inclusive to a person and exclusive in SQL. Converted here, in
|
|
// one place, exactly as the report handler does - otherwise a question
|
|
// about "the 1st to the 7th" quietly loses the 7th's trade.
|
|
q.From, q.To = from, to.AddDate(0, 0, 1)
|
|
if !q.To.After(q.From) {
|
|
return q, fmt.Errorf("to must be on or after from")
|
|
}
|
|
if strings.TrimSpace(in.Shop) != "" {
|
|
site, err := r.findSite(ctx, p, in.Shop)
|
|
if err != nil {
|
|
return q, err
|
|
}
|
|
q.SiteID = site.SiteID
|
|
}
|
|
return q, nil
|
|
}
|
|
|
|
// verificationOf is the same two-claim distinction the camera card makes.
|
|
func verificationOf(c api.Camera) string {
|
|
switch {
|
|
case c.Check.State != "done":
|
|
return "never checked"
|
|
case c.Check.Kind == "placement" && c.Check.OK:
|
|
return "proved it can recognise faces"
|
|
case c.Check.Kind == "placement":
|
|
return "checked and CANNOT recognise faces here: " + c.Check.Headline
|
|
case c.Check.OK:
|
|
return "stream works, but nobody has proved it can recognise a face"
|
|
default:
|
|
return "could not be reached: " + c.Check.Headline
|
|
}
|
|
}
|