Loyaly/Behavision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 22 Wiki Activity

Behavision: face recognition for retail, edge to head office

Browse Source 
Five components that ship as one product:

- behavision/  the recognition engine. RTSP ingest, YuNet detection, IoU
               tracking, ArcFace embeddings, a FAISS/SQLite gallery, and a
               FastAPI dashboard. Identity is decided once per TRACK from an
               average of at least three embeddings, never per frame.
- agent/       the Go edge agent: supervises the engine, holds a durable
               spool, and drains it to MQTT. Nothing is acked before the
               broker confirms.
- desktop/     the shop PC application (Wails + React + tray).
- server/      the cloud API, MQTT consumer, reports and assistant.
- web/         platform.loyaly.ai, the head-office app, embedded in the
               server binary.

The gallery stores 512-float embeddings and timestamps - no images unless
`app.store_faces` is switched on. Those embeddings are biometric personal
data under GDPR and India's DPDP: template inversion reconstructs a
recognisable face from an ArcFace vector, so data/behavision.db is treated
as a biometric database and DELETE /api/visitors/{id} is a real erasure.

CLAUDE.md carries the reasoning behind every non-obvious decision here,
including the ones that were measured and the ones that were wrong first.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HViLj9gYNRtSr7YVZmW5sn
This commit is contained in:
Suriyakumarvijayanayagam
2026-09-04 11:14:18 +05:30
commit dad04e8cda
216 changed files with 40473 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
agent/pkg/engine/health_test.go Normal file
View File

@@ -0,0 +1,44 @@
package engine
import (
"context"
"net/http"
"net/http/httptest"
"testing"
)
// The engine's REAL reply, copied from a running instance. The point of this
// test is the `"frozen": false` inside `paths`: Health.Paths was
// map[string]string, so decoding failed on that one bool, and because a failed
// decode fails the whole document a working engine was reported unreachable -
// red tray, and a heartbeat carrying neither the model nor the cameras, so head
// office showed "0 of 0 cameras" for a site that was watching one.
const realHealthBody = `{"status":"ok","recognition_model":"w600k_r50",
"paths":{"frozen":false,"install_root":"/opt/behavision","state_root":"/var/behavision",
"config":"/var/behavision/config/default.yaml","data_dir":"/var/behavision/data",
"models_dir":"/var/behavision/models"},"cameras":{"cam1":true},"uptime_seconds":42.5}`
func TestHealthDecodesWhatTheEngineActuallySends(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(realHealthBody))
}))
defer srv.Close()
s := New(Options{HealthURL: srv.URL})
h, err := s.Health(context.Background())
if err != nil {
t.Fatalf("the engine's own reply did not decode: %v", err)
}
if h.RecognitionModel != "w600k_r50" {
t.Errorf("model = %q", h.RecognitionModel)
}
// The two fields the heartbeat carries. Losing these is what made a
// working site look empty at head office.
if up, ok := h.Cameras["cam1"]; !ok || !up {
t.Errorf("cameras = %v, want cam1 connected", h.Cameras)
}
if h.Paths.StateRoot != "/var/behavision" || h.Paths.Frozen {
t.Errorf("paths = %+v", h.Paths)
}
}

355
agent/pkg/engine/supervisor.go Normal file
View File

@@ -0,0 +1,355 @@
// Package engine starts, watches and stops the Python recognition engine.
//
// Go cannot run ONNX, OpenCV or FAISS, so the engine stays Python and ships
// frozen. What Go owns is its lifecycle: start it, keep it up, capture its
// output, and stop it when the user asks — which is what the tray's start/stop
// buttons actually drive.
//
// Deliberately not a Windows service. A service runs in session 0 and cannot
// draw a tray icon, and spawning a child process needs no elevation while
// controlling a service does. A service wrapper can be layered on later
// without touching anything here.
package engine
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"sync"
"time"
)
// State is what the tray icon colours itself from.
type State string
const (
Stopped State = "stopped" // not running, and not meant to be
Starting State = "starting" // process spawned, not yet answering
Running State = "running" // answering /api/health
Backoff State = "backoff" // crashed, waiting to retry
Failed State = "failed" // gave up
)
const (
minBackoff = 1 * time.Second
maxBackoff = 30 * time.Second
// A run that lasted this long counts as healthy, so the next crash starts
// its backoff from the bottom again. Without this a process that runs fine
// for hours and then crashes once waits the full 30s to come back.
stableRun = 60 * time.Second
// How long a stopping process gets to exit on its own before it is killed.
stopGrace = 10 * time.Second
)
// Options configures a Supervisor.
type Options struct {
// Command builds the process to run. Injected rather than hardcoded so
// tests can supervise /bin/sh instead of a 200 MB frozen engine.
Command func(ctx context.Context) *exec.Cmd
// LogWriter receives the engine's stdout and stderr. A crashed engine with
// no captured output is undiagnosable, which on a customer site means a
// site visit.
LogWriter io.Writer
// HealthURL, StatsURL, User, Password address the engine's own API.
HealthURL string
StatsURL string
User string
Password string
// MaxRestarts of 0 means never give up. Non-zero is for tests.
MaxRestarts int
now func() time.Time
}
// Supervisor keeps one engine process running. Safe for concurrent use.
type Supervisor struct {
opts Options
mu sync.Mutex
state State
lastErr error
restarts int
cancel context.CancelFunc
done chan struct{}
}
func New(opts Options) *Supervisor {
if opts.LogWriter == nil {
opts.LogWriter = io.Discard
}
if opts.now == nil {
opts.now = time.Now
}
return &Supervisor{opts: opts, state: Stopped}
}
// Start launches the engine and keeps it running until Stop. Calling it while
// already running is a no-op rather than a second process — two engines on one
// SQLite WAL and one camera is exactly the failure this package exists to
// avoid.
func (s *Supervisor) Start() {
s.mu.Lock()
if s.cancel != nil {
s.mu.Unlock()
return
}
ctx, cancel := context.WithCancel(context.Background())
s.cancel = cancel
s.done = make(chan struct{})
s.state = Starting
s.restarts = 0
done := s.done
s.mu.Unlock()
go s.supervise(ctx, done)
}
// Stop asks the engine to exit and waits for it.
func (s *Supervisor) Stop() {
s.mu.Lock()
cancel, done := s.cancel, s.done
s.cancel = nil
s.mu.Unlock()
if cancel == nil {
return
}
cancel()
if done != nil {
<-done
}
s.setState(Stopped, nil)
}
// State reports what the supervisor is doing, plus the last error if any.
func (s *Supervisor) State() (State, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.state, s.lastErr
}
// Restarts counts crash-restarts since Start.
func (s *Supervisor) Restarts() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.restarts
}
// -- the loop --------------------------------------------------------------
func (s *Supervisor) supervise(ctx context.Context, done chan struct{}) {
defer close(done)
backoff := minBackoff
for {
if ctx.Err() != nil {
return
}
s.setState(Starting, nil)
started := s.opts.now()
err := s.runOnce(ctx)
ran := s.opts.now().Sub(started)
// A cancelled context means the user pressed Stop. Exiting then is
// success, not a crash, and restarting would be the single most
// annoying bug a tray app can have.
if ctx.Err() != nil {
return
}
s.mu.Lock()
s.restarts++
restarts := s.restarts
s.mu.Unlock()
if s.opts.MaxRestarts > 0 && restarts >= s.opts.MaxRestarts {
s.setState(Failed, err)
return
}
if ran >= stableRun {
backoff = minBackoff
}
s.setState(Backoff, err)
select {
case <-ctx.Done():
return
case <-time.After(backoff):
}
if backoff < maxBackoff {
backoff *= 2
if backoff > maxBackoff {
backoff = maxBackoff
}
}
}
}
func (s *Supervisor) runOnce(ctx context.Context) error {
cmd := s.opts.Command(ctx)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
cmd.Stderr = cmd.Stdout
if err := cmd.Start(); err != nil {
return fmt.Errorf("engine failed to start: %w", err)
}
pumped := make(chan struct{})
go func() {
defer close(pumped)
sc := bufio.NewScanner(stdout)
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for sc.Scan() {
fmt.Fprintln(s.opts.LogWriter, sc.Text())
}
}()
s.setState(Running, nil)
waitErr := cmd.Wait()
<-pumped
// A context cancel terminates the child through exec's own handling; the
// resulting error is expected, not a fault.
if ctx.Err() != nil {
return nil
}
if waitErr != nil {
return fmt.Errorf("engine exited: %w", waitErr)
}
return errors.New("engine exited unexpectedly with status 0")
}
func (s *Supervisor) setState(st State, err error) {
s.mu.Lock()
s.state = st
if err != nil {
s.lastErr = err
}
s.mu.Unlock()
}
// -- health ----------------------------------------------------------------
// Health is the subset of /api/health the tray and the server care about.
type Health struct {
Status string `json:"status"`
RecognitionModel string `json:"recognition_model"`
Cameras map[string]bool `json:"cameras"`
// Paths is a STRUCT, not map[string]string, because `frozen` is a bool.
// It was a map of strings, so decoding the engine's real reply failed with
// "cannot unmarshal bool into Go struct field Health.paths" - and because
// one bad field fails the whole document, a perfectly healthy engine was
// reported unreachable: red tray, and a heartbeat carrying neither the
// model nor the camera list, so head office showed 0 of 0 cameras for a
// site that was watching one.
Paths EnginePaths `json:"paths"`
}
// EnginePaths mirrors what `behavision paths` and /api/health report. Unknown
// fields are ignored by encoding/json, so the engine can add to it freely.
type EnginePaths struct {
Frozen bool `json:"frozen"`
InstallRoot string `json:"install_root"`
StateRoot string `json:"state_root"`
Config string `json:"config"`
DataDir string `json:"data_dir"`
ModelsDir string `json:"models_dir"`
}
// Health polls the engine's own API. A running process is not the same as a
// working engine: the model can fail to load and the process stays up.
func (s *Supervisor) Health(ctx context.Context) (*Health, error) {
if s.opts.HealthURL == "" {
return nil, errors.New("no health url configured")
}
var h Health
if err := s.getJSON(ctx, s.opts.HealthURL, &h); err != nil {
return nil, err
}
return &h, nil
}
// Stats is the slice of /api/stats the heartbeat carries.
//
// Only fraction_below_gate, because that is the number that decides whether a
// site's footfall can be believed at all - the share of faces its cameras saw
// and discarded before they ever became a visit. Everything else in /api/stats
// is a local diagnostic and belongs on the local dashboard, not on the wire
// every thirty seconds.
type Stats struct {
Cameras []struct {
CameraID string `json:"camera_id"`
Pipeline struct {
BestQuality struct {
N int `json:"n"`
FractionBelowGate float64 `json:"fraction_below_gate"`
} `json:"best_quality"`
} `json:"pipeline"`
} `json:"cameras"`
}
// WorstBelowGate returns the worst camera's figure, and whether any camera has
// measured enough faces to have an opinion.
//
// Worst rather than average: one badly placed camera is a hole in the report,
// and averaging it against three good ones hides the only camera anyone needs
// to move. The sample floor is there because three faces is an anecdote -
// reporting 1.00 from a single below-gate track would raise an alarm about a
// camera nobody has walked past yet.
func (s *Stats) WorstBelowGate() (float64, bool) {
const minSamples = 10
worst, found := 0.0, false
for _, c := range s.Cameras {
if c.Pipeline.BestQuality.N < minSamples {
continue
}
if !found || c.Pipeline.BestQuality.FractionBelowGate > worst {
worst, found = c.Pipeline.BestQuality.FractionBelowGate, true
}
}
return worst, found
}
// Stats polls the engine's pipeline counters.
func (s *Supervisor) Stats(ctx context.Context) (*Stats, error) {
if s.opts.StatsURL == "" {
return nil, errors.New("no stats url configured")
}
var out Stats
if err := s.getJSON(ctx, s.opts.StatsURL, &out); err != nil {
return nil, err
}
return &out, nil
}
func (s *Supervisor) getJSON(ctx context.Context, url string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
if s.opts.User != "" {
req.SetBasicAuth(s.opts.User, s.opts.Password)
}
resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s returned %s", url, resp.Status)
}
return json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(out)
}
// LogFile opens the engine log, rotating aside anything already there so one
// run's output cannot be mistaken for another's.
func LogFile(path string) (*os.File, error) {
if _, err := os.Stat(path); err == nil {
os.Rename(path, path+".1")
}
return os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
}

240
agent/pkg/engine/supervisor_test.go Normal file
View File

@@ -0,0 +1,240 @@
package engine
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os/exec"
"sync"
"testing"
"time"
)
// sh supervises /bin/sh instead of a 200 MB frozen engine. The Command hook
// exists for exactly this.
func sh(script string) func(context.Context) *exec.Cmd {
return func(ctx context.Context) *exec.Cmd {
return exec.CommandContext(ctx, "/bin/sh", "-c", script)
}
}
func waitFor(t *testing.T, s *Supervisor, want State, within time.Duration) {
t.Helper()
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
if got, _ := s.State(); got == want {
return
}
time.Sleep(5 * time.Millisecond)
}
got, err := s.State()
t.Fatalf("state %q (err %v), want %q within %s", got, err, want, within)
}
func TestItRunsAndReportsRunning(t *testing.T) {
s := New(Options{Command: sh("sleep 5")})
s.Start()
defer s.Stop()
waitFor(t, s, Running, 2*time.Second)
}
func TestStopDoesNotTriggerARestart(t *testing.T) {
// The classic supervisor bug: the user presses Stop, the child exits, the
// loop reads that as a crash and starts it again.
s := New(Options{Command: sh("sleep 30")})
s.Start()
waitFor(t, s, Running, 2*time.Second)
s.Stop()
if got, _ := s.State(); got != Stopped {
t.Fatalf("state after Stop is %q", got)
}
if n := s.Restarts(); n != 0 {
t.Fatalf("Stop counted as %d crash-restarts", n)
}
time.Sleep(200 * time.Millisecond)
if got, _ := s.State(); got != Stopped {
t.Fatalf("it restarted itself after Stop: %q", got)
}
}
func TestStopIsSynchronous(t *testing.T) {
// Stop must not return while the child still holds the SQLite WAL, or the
// next Start races the previous process.
s := New(Options{Command: sh("sleep 30")})
s.Start()
waitFor(t, s, Running, 2*time.Second)
done := make(chan struct{})
go func() { s.Stop(); close(done) }()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("Stop did not return")
}
}
func TestACrashIsRestarted(t *testing.T) {
s := New(Options{Command: sh("exit 1")})
s.Start()
defer s.Stop()
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if s.Restarts() >= 2 {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("only %d restarts - is it backing off correctly?", s.Restarts())
}
func TestItDoesNotSpinOnAProcessThatCannotStart(t *testing.T) {
// A tight restart loop on a broken install pins a core and fills the disk
// with log lines. Backoff must space the attempts out.
s := New(Options{Command: sh("exit 1")})
s.Start()
defer s.Stop()
time.Sleep(1500 * time.Millisecond)
// 1s + 2s backoff means at most ~2 attempts in 1.5s; a spin would be
// thousands.
if n := s.Restarts(); n > 4 {
t.Fatalf("%d restarts in 1.5s - not backing off", n)
}
}
func TestItGivesUpAfterMaxRestarts(t *testing.T) {
s := New(Options{Command: sh("exit 1"), MaxRestarts: 2})
s.Start()
defer s.Stop()
waitFor(t, s, Failed, 5*time.Second)
if _, err := s.State(); err == nil {
t.Fatal("Failed state carries no reason")
}
}
func TestEngineOutputIsCaptured(t *testing.T) {
// A crashed engine with no captured output means a site visit to diagnose.
var mu sync.Mutex
buf := &lockedBuf{mu: &mu}
s := New(Options{Command: sh("echo model-load-failed; exit 1"),
LogWriter: buf, MaxRestarts: 1})
s.Start()
defer s.Stop()
waitFor(t, s, Failed, 5*time.Second)
if got := buf.String(); !bytes.Contains([]byte(got), []byte("model-load-failed")) {
t.Fatalf("engine output not captured, got %q", got)
}
}
func TestStartTwiceDoesNotRunTwoEngines(t *testing.T) {
// Two engines on one SQLite WAL and one camera is the failure this whole
// package exists to prevent.
s := New(Options{Command: sh("sleep 5")})
s.Start()
s.Start()
defer s.Stop()
waitFor(t, s, Running, 2*time.Second)
if n := s.Restarts(); n != 0 {
t.Fatalf("second Start disturbed the first: %d restarts", n)
}
}
func TestHealthReportsTheModelThatActuallyLoaded(t *testing.T) {
// A running process is not a working engine: on a memory-starved box the
// big model loses the fallback chain and the process stays up regardless.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, ok := r.BasicAuth()
if !ok || user != "u" || pass != "p" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Write([]byte(`{"status":"ok","recognition_model":"w600k_mbf.onnx",
"cameras":{"entrance":true}}`))
}))
defer srv.Close()
s := New(Options{Command: sh("sleep 1"), HealthURL: srv.URL,
User: "u", Password: "p"})
h, err := s.Health(context.Background())
if err != nil {
t.Fatal(err)
}
if h.RecognitionModel != "w600k_mbf.onnx" || !h.Cameras["entrance"] {
t.Fatalf("bad health: %+v", h)
}
}
func TestHealthFailsClosedOnBadCredentials(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
}))
defer srv.Close()
s := New(Options{Command: sh("true"), HealthURL: srv.URL, User: "u", Password: "wrong"})
if _, err := s.Health(context.Background()); err == nil {
t.Fatal("401 reported as healthy")
}
}
type lockedBuf struct {
mu *sync.Mutex
buf bytes.Buffer
}
func (l *lockedBuf) Write(p []byte) (int, error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.Write(p)
}
func (l *lockedBuf) String() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.String()
}
// The engine has to be TOLD where to post detections, and the only place that
// can happen is when the child is launched: the bridge picks a random loopback
// port after the supervisor is built, and a restarted engine has to be told
// again. This pins that the Command hook is consulted per launch rather than
// captured once - the wiring that was missing while the bridge's own doc
// comment claimed it existed.
func TestTheChildIsBuiltFreshOnEveryLaunch(t *testing.T) {
var mu sync.Mutex
url := "http://127.0.0.1:1111/e"
var seen []string
s := New(Options{Command: func(ctx context.Context) *exec.Cmd {
mu.Lock()
seen = append(seen, url)
mu.Unlock()
return exec.CommandContext(ctx, "/bin/sh", "-c", "exit 1")
}})
s.Start()
waitFor(t, s, Backoff, 2*time.Second)
// The port changes, exactly as it does when the bridge restarts.
mu.Lock()
url = "http://127.0.0.1:2222/e"
mu.Unlock()
// One backoff (1s) plus room for the relaunch.
deadline := time.Now().Add(4 * time.Second)
for time.Now().Before(deadline) {
mu.Lock()
n := len(seen)
mu.Unlock()
if n >= 2 {
break
}
time.Sleep(20 * time.Millisecond)
}
s.Stop()
mu.Lock()
defer mu.Unlock()
if len(seen) < 2 {
t.Fatalf("the command hook ran %d times, so a restart could not be told a new URL", len(seen))
}
if seen[len(seen)-1] != "http://127.0.0.1:2222/e" {
t.Fatalf("the last launch used %q - the hook captured a stale value", seen[len(seen)-1])
}
}