Opening a shop is an API call; the broker learns of it in the same request

The last step of onboarding that needed a shell: provision site printed
a broker password and a person typed it into Mosquitto's passwd file on
the host - mounted read-only in the container, so the first attempt
failed silently and the password was re-rolled. No tenant could open a
second branch without us.

The server now drives Mosquitto's dynamic-security plugin over its own
broker login: POST /api/sites (owner) writes the row and the sealed
password, registers the login and a per-site role with literal topics
(the 2.0 plugin does not substitute %u - measured), and removes the row
again if the broker refuses, so a shop cannot exist in the database and
not on the broker. provision site goes through the same path. The
head-office Shops screen gets 'Open a new shop'.

broker-init converts the existing passwd file into the plugin's store
with every hash intact - PBKDF2-SHA512 both sides - so the cutover
re-claims no shop PC. Rehearsed locally: old logins keep working,
isolation holds, the health probe works, and a PC claiming a shop opened
through the API connects as that shop. run-local.sh now brings the
broker up the same way.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KGcjxF1cNLcuwc3DAPcnfj
This commit is contained in:
2026-09-19 11:55:26 +05:30
parent 5f83a1077d
commit 4c750cb2ac
21 changed files with 1310 additions and 54 deletions

View File

@@ -0,0 +1,325 @@
// Package broker creates and removes a site's broker login at runtime.
//
// Until this existed a shop was created in two places by two mechanisms:
// `provision site` wrote the row and printed a password, and a person then
// typed that password into Mosquitto's passwd file on the host and reloaded
// the broker. Nothing else in the product needed a shell, so this one step
// was what stopped a tenant opening a second branch on their own - and it was
// fragile even for us: the file was mounted read-only in the container, the
// first attempt failed, and the password had to be re-rolled.
//
// Mosquitto 2.0's dynamic-security plugin takes the same operations as
// commands on a control topic, from a client that holds the `admin` role.
// The server already holds a broker login; this gives it that role and uses
// it. No file, no reload, no docker socket, and the per-site credential
// model is unchanged - one username per shop, topics only under its own
// prefix.
//
// Isolation is a ROLE PER SITE with literal topics, not one role with `%u`:
// the 2.0 plugin does not substitute `%u` in ACL topics (measured - the
// publish was denied). The role is created and deleted with the client, so
// there is still exactly one thing to get right and it is done in one place.
package broker
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
paho "github.com/eclipse/paho.mqtt.golang"
)
const (
controlTopic = "$CONTROL/dynamic-security/v1"
responseTopic = "$CONTROL/dynamic-security/v1/response"
// A control round trip on a healthy broker is milliseconds; this is a
// bound on a broker that is up but not answering control commands, which
// is what a broker WITHOUT the plugin looks like.
commandTimeout = 8 * time.Second
connectTimeout = 10 * time.Second
)
// ErrUnavailable is returned when the broker cannot be reached or does not
// answer control commands. Callers turn it into a 502/503, never a 500: the
// operator's next step is to look at the broker, not at the server.
var ErrUnavailable = errors.New("broker: dynamic security not available")
// SiteBroker is what the API and the provisioner depend on. A fake satisfies
// it in tests; Dynsec satisfies it in production.
type SiteBroker interface {
// EnsureSite creates the broker login for one site, or resets its password
// if it already exists. Idempotent: safe to run again after any failure.
EnsureSite(ctx context.Context, username, password string) error
// DeleteSite removes the login and its role. Missing is not an error.
DeleteSite(ctx context.Context, username string) error
}
// Dynsec drives the plugin over its own connection - not the ingest client's.
// That one has SetOrderMatters and blocking handlers, and a provisioning call
// must neither wait behind a slow visit nor delay one.
type Dynsec struct {
url, user, pass string
log *log.Logger
mu sync.Mutex
client paho.Client
pending map[string]chan []response // keyed by batch id
}
func New(url, user, pass string, logger *log.Logger) *Dynsec {
if logger == nil {
logger = log.Default()
}
return &Dynsec{url: url, user: user, pass: pass, log: logger, pending: map[string]chan []response{}}
}
type response struct {
Command string `json:"command"`
Error string `json:"error,omitempty"`
CorrelationData string `json:"correlationData,omitempty"`
Data json.RawMessage `json:"data,omitempty"`
}
// SiteTopics are exactly what a shop PC may do: publish its own visits,
// heartbeats and status, and read its own commands. This mirrors the acl
// file the broker used to be configured with, rule for rule.
func siteACLs(username string) []map[string]any {
prefix := "bv/" + username
acl := func(kind, topic string) map[string]any {
return map[string]any{"acltype": kind, "topic": topic, "allow": true, "priority": 0}
}
return []map[string]any{
acl("publishClientSend", prefix+"/visit"),
acl("publishClientSend", prefix+"/heartbeat"),
acl("publishClientSend", prefix+"/status"),
acl("publishClientReceive", prefix+"/cmd/#"),
acl("subscribePattern", prefix+"/cmd/#"),
}
}
// RoleName is the per-site role. Exported so the migration can name it the
// same way.
func RoleName(username string) string { return "site." + username }
func (d *Dynsec) EnsureSite(ctx context.Context, username, password string) error {
if strings.TrimSpace(username) == "" || password == "" {
return errors.New("broker: username and password are required")
}
role := RoleName(username)
cmds := []map[string]any{{"command": "createRole", "rolename": role}}
for _, a := range siteACLs(username) {
c := map[string]any{"command": "addRoleACL", "rolename": role}
for k, v := range a {
c[k] = v
}
cmds = append(cmds, c)
}
cmds = append(cmds, map[string]any{
"command": "createClient", "username": username, "password": password,
"roles": []map[string]any{{"rolename": role}},
})
res, err := d.run(ctx, cmds)
if err != nil {
return err
}
clientExisted := false
for _, r := range res {
switch {
case r.Error == "":
case strings.Contains(r.Error, "already exists") && r.Command != "createClient":
// A re-run after a partial failure. Fine.
case r.Command == "createClient" && strings.Contains(r.Error, "already exists"):
clientExisted = true
default:
return fmt.Errorf("broker: %s: %s", r.Command, r.Error)
}
}
if !clientExisted {
return nil
}
// The login exists from an earlier run: make its password THIS one - the
// database holds this one and the enrolment hands it out - and make sure
// it carries the role. addClientRole on a client that already has it
// answers "Internal error", so check first rather than guess from prose.
res, err = d.run(ctx, []map[string]any{
{"command": "setClientPassword", "username": username, "password": password},
{"command": "getClient", "username": username},
})
if err != nil {
return err
}
hasRole := false
for _, r := range res {
if r.Error != "" {
return fmt.Errorf("broker: %s: %s", r.Command, r.Error)
}
if r.Command == "getClient" {
var got struct {
Client struct {
Roles []struct{ Rolename string } `json:"roles"`
} `json:"client"`
}
_ = json.Unmarshal(r.Data, &got)
for _, rr := range got.Client.Roles {
if rr.Rolename == role {
hasRole = true
}
}
}
}
if hasRole {
return nil
}
res, err = d.run(ctx, []map[string]any{{"command": "addClientRole", "username": username, "rolename": role}})
if err != nil {
return err
}
if res[0].Error != "" {
return fmt.Errorf("broker: addClientRole: %s", res[0].Error)
}
return nil
}
func (d *Dynsec) DeleteSite(ctx context.Context, username string) error {
res, err := d.run(ctx, []map[string]any{
{"command": "deleteClient", "username": username},
{"command": "deleteRole", "rolename": RoleName(username)},
})
if err != nil {
return err
}
for _, r := range res {
if r.Error != "" && !strings.Contains(r.Error, "not found") {
return fmt.Errorf("broker: %s: %s", r.Command, r.Error)
}
}
return nil
}
// Close drops the control connection. Safe when never connected.
func (d *Dynsec) Close() {
d.mu.Lock()
c := d.client
d.client = nil
d.mu.Unlock()
if c != nil && c.IsConnected() {
c.Disconnect(250)
}
}
// run sends one batch and waits for its one response message. Every command
// carries the batch id as correlationData, and the plugin echoes it, so a
// response for somebody else's batch - two servers, or a retry - is never
// mistaken for ours.
func (d *Dynsec) run(ctx context.Context, cmds []map[string]any) ([]response, error) {
if err := d.connect(ctx); err != nil {
return nil, err
}
id := newID()
for _, c := range cmds {
c["correlationData"] = id
}
body, err := json.Marshal(map[string]any{"commands": cmds})
if err != nil {
return nil, err
}
ch := make(chan []response, 1)
d.mu.Lock()
d.pending[id] = ch
client := d.client
d.mu.Unlock()
defer func() {
d.mu.Lock()
delete(d.pending, id)
d.mu.Unlock()
}()
tok := client.Publish(controlTopic, 1, false, body)
if !tok.WaitTimeout(commandTimeout) {
return nil, fmt.Errorf("%w: publish timed out", ErrUnavailable)
}
if tok.Error() != nil {
return nil, fmt.Errorf("%w: %v", ErrUnavailable, tok.Error())
}
select {
case res := <-ch:
return res, nil
case <-time.After(commandTimeout):
return nil, fmt.Errorf("%w: no answer on %s - is the dynamic-security plugin enabled and does %q hold the admin role?", ErrUnavailable, responseTopic, d.user)
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (d *Dynsec) onResponse(_ paho.Client, m paho.Message) {
var env struct {
Responses []response `json:"responses"`
}
if err := json.Unmarshal(m.Payload(), &env); err != nil || len(env.Responses) == 0 {
return
}
id := env.Responses[0].CorrelationData
d.mu.Lock()
ch, ok := d.pending[id]
d.mu.Unlock()
if ok {
select {
case ch <- env.Responses:
default:
}
}
}
func (d *Dynsec) connect(ctx context.Context) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.client != nil && d.client.IsConnectionOpen() {
return nil
}
opts := paho.NewClientOptions().
AddBroker(d.url).
SetClientID(fmt.Sprintf("behavision-dynsec-%s", newID()[:8])).
SetUsername(d.user).
SetPassword(d.pass).
SetAutoReconnect(true).
SetCleanSession(true).
SetKeepAlive(30 * time.Second).
SetConnectTimeout(connectTimeout)
opts.OnConnect = func(c paho.Client) {
// Re-subscribed on every (re)connect: clean session keeps nothing.
c.Subscribe(responseTopic, 1, d.onResponse)
}
c := paho.NewClient(opts)
tok := c.Connect()
if !tok.WaitTimeout(connectTimeout) {
return fmt.Errorf("%w: connect to %s timed out", ErrUnavailable, d.url)
}
if tok.Error() != nil {
return fmt.Errorf("%w: %v", ErrUnavailable, tok.Error())
}
// The subscription must be in place before the first command is sent, or
// its answer is published to nobody.
st := c.Subscribe(responseTopic, 1, d.onResponse)
if !st.WaitTimeout(connectTimeout) || st.Error() != nil {
c.Disconnect(100)
return fmt.Errorf("%w: cannot subscribe to %s (does %q hold the admin role?)", ErrUnavailable, responseTopic, d.user)
}
d.client = c
return nil
}
func newID() string {
var b [12]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}

View File

@@ -0,0 +1,96 @@
package broker
import (
"context"
"os"
"testing"
"time"
paho "github.com/eclipse/paho.mqtt.golang"
)
// Runs against a real Mosquitto with the dynamic-security plugin, because a
// fake broker would only prove the JSON matches what I believe the plugin
// wants. Gated on the environment like the store's live tests:
//
// DYNSEC_TEST_URL=tcp://127.0.0.1:51884 DYNSEC_TEST_USER=behavision-backend \
// DYNSEC_TEST_PASS=pw-backend go test ./internal/broker -run Live -v
func TestLiveEnsureAndDeleteSite(t *testing.T) {
url, user, pass := os.Getenv("DYNSEC_TEST_URL"), os.Getenv("DYNSEC_TEST_USER"), os.Getenv("DYNSEC_TEST_PASS")
if url == "" {
t.Skip("DYNSEC_TEST_URL not set")
}
d := New(url, user, pass, nil)
defer d.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
site := "livetest.shop1"
if err := d.EnsureSite(ctx, site, "first-pw"); err != nil {
t.Fatalf("EnsureSite: %v", err)
}
// Idempotent, and the password becomes the one given THIS time.
if err := d.EnsureSite(ctx, site, "second-pw"); err != nil {
t.Fatalf("EnsureSite again: %v", err)
}
if err := canConnect(url, site, "first-pw"); err == nil {
t.Fatalf("old password still accepted after re-ensure")
}
if err := canConnect(url, site, "second-pw"); err != nil {
t.Fatalf("new password refused: %v", err)
}
// Own topic allowed, another site's refused. A denied publish at QoS 1
// still gets a PUBACK, so this is observed through the backend's inbox.
got := make(chan string, 4)
backend := connect(t, url, user, pass)
defer backend.Disconnect(100)
backend.Subscribe("bv/#", 1, func(_ paho.Client, m paho.Message) { got <- m.Topic() }).Wait()
shop := connect(t, url, site, "second-pw")
defer shop.Disconnect(100)
shop.Publish("bv/other.shop/visit", 1, false, "leak").Wait()
shop.Publish("bv/"+site+"/visit", 1, false, "ok").Wait()
select {
case topic := <-got:
if topic != "bv/"+site+"/visit" {
t.Fatalf("first delivered topic was %s", topic)
}
case <-time.After(5 * time.Second):
t.Fatal("own-topic publish never arrived")
}
select {
case topic := <-got:
t.Fatalf("unexpected second delivery: %s", topic)
case <-time.After(1500 * time.Millisecond):
}
if err := d.DeleteSite(ctx, site); err != nil {
t.Fatalf("DeleteSite: %v", err)
}
if err := d.DeleteSite(ctx, site); err != nil {
t.Fatalf("DeleteSite twice: %v", err)
}
if err := canConnect(url, site, "second-pw"); err == nil {
t.Fatal("deleted site can still connect")
}
}
func connect(t *testing.T, url, user, pass string) paho.Client {
t.Helper()
c := paho.NewClient(paho.NewClientOptions().AddBroker(url).SetUsername(user).SetPassword(pass).SetConnectTimeout(5 * time.Second))
tok := c.Connect()
tok.Wait()
if tok.Error() != nil {
t.Fatalf("connect as %s: %v", user, tok.Error())
}
return c
}
func canConnect(url, user, pass string) error {
c := paho.NewClient(paho.NewClientOptions().AddBroker(url).SetUsername(user).SetPassword(pass).SetConnectTimeout(5 * time.Second))
tok := c.Connect()
tok.Wait()
if tok.Error() == nil {
c.Disconnect(50)
}
return tok.Error()
}

View File

@@ -0,0 +1,132 @@
package broker
import (
"bufio"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
)
// Store is the plugin's on-disk shape - the part of it this code writes.
type Store struct {
Clients []Client `json:"clients"`
Roles []Role `json:"roles"`
DefaultACLAccess map[string]bool `json:"defaultACLAccess"`
}
type Client struct {
Username string `json:"username"`
TextName string `json:"textName,omitempty"`
Password string `json:"password"`
Salt string `json:"salt"`
Iterations int `json:"iterations"`
Roles []RoleRef `json:"roles"`
}
type RoleRef struct {
Rolename string `json:"rolename"`
}
type Role struct {
Rolename string `json:"rolename"`
ACLs []ACL `json:"acls"`
}
type ACL struct {
ACLType string `json:"acltype"`
Topic string `json:"topic"`
Allow bool `json:"allow"`
Priority int `json:"priority"`
}
// FromPasswd converts a mosquitto_passwd file into the plugin's store,
// keeping every password exactly as it is.
//
// This is the cutover for a broker that already has sites: the hashes in the
// passwd file are PBKDF2-SHA512 ($7$<iterations>$<salt>$<digest>, base64),
// which is the same thing the plugin stores as password/salt/iterations - so
// no shop PC has to be re-claimed and no credential changes hands. The roles
// reproduce the acl file rule for rule: the backend reads everything and
// drives the plugin, the health probe reads uptime, and every other user is a
// site that may write under its own prefix and read its own commands.
func FromPasswd(r io.Reader, backendUser, healthUser string) (*Store, error) {
st := &Store{
DefaultACLAccess: map[string]bool{
"publishClientSend": false, "publishClientReceive": false,
"subscribe": false, "unsubscribe": true,
},
}
st.Roles = append(st.Roles,
Role{Rolename: "admin", ACLs: []ACL{
{"publishClientSend", "$CONTROL/dynamic-security/#", true, 0},
{"publishClientReceive", "$CONTROL/dynamic-security/#", true, 0},
{"subscribePattern", "$CONTROL/dynamic-security/#", true, 0},
}},
Role{Rolename: "backend", ACLs: []ACL{
{"publishClientSend", "bv/#", true, 0},
{"publishClientReceive", "bv/#", true, 0},
{"subscribePattern", "bv/#", true, 0},
{"publishClientReceive", "$SYS/#", true, 0},
{"subscribePattern", "$SYS/#", true, 0},
}},
Role{Rolename: "health", ACLs: []ACL{
{"publishClientReceive", "$SYS/broker/uptime", true, 0},
{"subscribePattern", "$SYS/broker/uptime", true, 0},
}},
)
sc := bufio.NewScanner(r)
line := 0
for sc.Scan() {
line++
text := strings.TrimSpace(sc.Text())
if text == "" || strings.HasPrefix(text, "#") {
continue
}
user, hash, ok := strings.Cut(text, ":")
if !ok {
return nil, fmt.Errorf("passwd line %d: no ':'", line)
}
parts := strings.Split(hash, "$")
// "", "7", iterations, salt, digest
if len(parts) != 5 || parts[1] != "7" {
return nil, fmt.Errorf("passwd line %d (%s): not a $7$ PBKDF2 hash; re-set that password with mosquitto_passwd first", line, user)
}
iters, err := strconv.Atoi(parts[2])
if err != nil {
return nil, fmt.Errorf("passwd line %d (%s): iterations %q", line, user, parts[2])
}
c := Client{Username: user, Password: parts[4], Salt: parts[3], Iterations: iters}
switch user {
case backendUser:
c.TextName = "Behavision server"
c.Roles = []RoleRef{{"admin"}, {"backend"}}
case healthUser:
c.TextName = "health probe"
c.Roles = []RoleRef{{"health"}}
default:
role := RoleName(user)
var acls []ACL
for _, a := range siteACLs(user) {
acls = append(acls, ACL{a["acltype"].(string), a["topic"].(string), true, 0})
}
st.Roles = append(st.Roles, Role{Rolename: role, ACLs: acls})
c.TextName = "site " + user
c.Roles = []RoleRef{{role}}
}
st.Clients = append(st.Clients, c)
}
if err := sc.Err(); err != nil {
return nil, err
}
return st, nil
}
// Encode writes the store as the plugin reads it.
func (s *Store) Encode(w io.Writer) error {
enc := json.NewEncoder(w)
enc.SetIndent("", "\t")
return enc.Encode(s)
}

View File

@@ -0,0 +1,73 @@
package broker
import (
"bytes"
"encoding/json"
"strings"
"testing"
)
const passwd = `behavision-backend:$7$101$c2FsdA==$ZGlnZXN0
health:$7$101$aGVhbHRo$aGFzaA==
acme.store1:$7$101$c2l0ZQ==$c2l0ZWhhc2g=
`
func TestPasswdBecomesTheStoreWithHashesIntact(t *testing.T) {
st, err := FromPasswd(strings.NewReader(passwd), "behavision-backend", "health")
if err != nil {
t.Fatal(err)
}
if len(st.Clients) != 3 {
t.Fatalf("clients: %d", len(st.Clients))
}
site := st.Clients[2]
if site.Password != "c2l0ZWhhc2g=" || site.Salt != "c2l0ZQ==" || site.Iterations != 101 {
t.Fatalf("hash not carried over intact: %+v", site)
}
if site.Roles[0].Rolename != "site.acme.store1" {
t.Fatalf("site role: %+v", site.Roles)
}
var siteRole *Role
for i := range st.Roles {
if st.Roles[i].Rolename == "site.acme.store1" {
siteRole = &st.Roles[i]
}
}
if siteRole == nil {
t.Fatal("no role for the site")
}
topics := map[string]bool{}
for _, a := range siteRole.ACLs {
topics[a.ACLType+" "+a.Topic] = true
}
for _, want := range []string{
"publishClientSend bv/acme.store1/visit",
"publishClientSend bv/acme.store1/heartbeat",
"publishClientSend bv/acme.store1/status",
"subscribePattern bv/acme.store1/cmd/#",
} {
if !topics[want] {
t.Errorf("missing %q in %v", want, topics)
}
}
if st.Clients[0].Roles[0].Rolename != "admin" {
t.Fatalf("backend is not an admin: %+v", st.Clients[0].Roles)
}
if st.DefaultACLAccess["publishClientSend"] || st.DefaultACLAccess["subscribe"] {
t.Fatal("default access must be deny")
}
var buf bytes.Buffer
if err := st.Encode(&buf); err != nil {
t.Fatal(err)
}
if !json.Valid(buf.Bytes()) {
t.Fatal("encoded store is not valid JSON")
}
}
func TestANonPBKDF2LineIsRefusedByName(t *testing.T) {
_, err := FromPasswd(strings.NewReader("old:$6$abc$def\n"), "b", "h")
if err == nil || !strings.Contains(err.Error(), "old") {
t.Fatalf("expected a named refusal, got %v", err)
}
}