Process the MQTT ingest on a bounded worker pool

paho delivers on one goroutine, so bills were committed strictly one
after another. Each is a full Postgres transaction — advisory lock,
dedup, stock row locks, availability check, four inserts, commit — which
is 10-30ms, so the ceiling was roughly 30-100 bills a second and a
shop's backlog draining after an outage took minutes to land.

A fixed pool behind a bounded queue, rather than a goroutine per
message. Unbounded concurrency would open a transaction per message and
exhaust the connection pool under a storm, stalling every one of them at
once — a slow minute turned into a dead one. When the queue fills,
submit blocks: paho stops acknowledging, the broker's in-flight window
fills, it stops sending, and the backpressure reaches the till, which
holds its bills and retries. Slow, but nothing is dropped.

Heartbeats get their own pool. Sharing one would let a backlog of bills
delay presence, so every till would appear to go dark at exactly the
moment the system was busiest — the worst time to be blind to which
counters are alive.

Payloads are copied on the way in. paho reuses its buffer once a handler
returns and the work now happens after that, so a queued bill would
otherwise be read as whatever message arrived next — silently, and as
valid JSON often enough to commit the wrong sale.

One bug found by its own test: submit-after-stop selected between a
done-channel and the job channel, and once both were ready Go picks at
random. Picking the send panics on a closed channel. It would have shown
up in production as an occasional crash during shutdown and nowhere
else. Now guarded by an RWMutex held across the send, so the queue
cannot be closed under one in progress.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

messaging/posworkers.go

@@ -0,0 +1,174 @@
+package messaging
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"sync"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+)
+
+// Concurrency for the MQTT ingest.
+//
+// paho delivers messages on a single goroutine, so without this every bill is
+// committed one after another. A bill is a full Postgres transaction — advisory
+// lock, dedup check, stock row locks, availability check, four inserts, commit —
+// which realistically costs 10–30ms. Serially that is 30–100 bills a second,
+// and a shop-wide backlog draining after an outage would take minutes to land.
+//
+// ### Why a bounded pool rather than a goroutine per message
+//
+// paho can be told to call handlers concurrently, but it spawns without limit.
+// A storm would then open a database transaction per message, exhaust the
+// connection pool, and stall every one of them at once — turning a slow minute
+// into a dead one.
+//
+// A fixed pool behind a bounded queue does the opposite. When the queue fills,
+// submitting **blocks**, which is the point: paho stops acknowledging, the
+// broker's in-flight window fills, and it stops sending. Backpressure travels
+// all the way back to the till, which holds its bills and retries. Slow, but
+// nothing is dropped and nothing is lost.
+//
+// ### Why bills and heartbeats have separate pools
+//
+// A heartbeat is one Redis write and a bill is a transaction. Sharing a queue
+// would let a backlog of bills delay presence, and every till would appear to
+// go dark at exactly the moment the system was busiest — the worst possible
+// time to be blind to which counters are alive.
+
+// posPool is a fixed set of workers reading a bounded queue.
+type posPool struct {
+	name string
+	jobs chan func()
+	wg   sync.WaitGroup
+	once sync.Once
+
+	// Guards the transition to closed. A plain `select` over a done-channel and
+	// the job channel is not enough: once both are ready Go picks between them
+	// at random, and picking the send panics on a closed channel. Held for
+	// reading across the whole of submit, so stop cannot close the queue out
+	// from under a send already in progress.
+	mu     sync.RWMutex
+	closed bool
+}
+
+func newPosPool(name string, workers, queue int) *posPool {
+	p := &posPool{
+		name: name,
+		jobs: make(chan func(), queue),
+	}
+
+	p.wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer p.wg.Done()
+			for job := range p.jobs {
+				job()
+			}
+		}()
+	}
+
+	log.Printf("pos: %s pool started with %d workers, queue %d", name, workers, queue)
+	return p
+}
+
+// submit queues work, blocking when the queue is full.
+//
+// Blocking is deliberate. Dropping would lose a bill outright; the terminal
+// would eventually re-send it, but only after its ack timeout, and meanwhile we
+// would have thrown away work we had already accepted. Blocking instead pushes
+// back through paho to the broker to the till, which is exactly where the
+// decision to slow down belongs.
+func (p *posPool) submit(job func()) {
+	p.mu.RLock()
+
+	if p.closed {
+		p.mu.RUnlock()
+		// Shutting down. Running it inline still gets the work done and its ack
+		// published, rather than discarding a bill that already reached us.
+		job()
+		return
+	}
+
+	// The read lock is held across the send. Blocking here while the queue is
+	// full cannot deadlock against stop: the workers only exit once the channel
+	// is closed, and that happens under the write lock this send is holding
+	// off — so they stay alive and keep draining until this send completes.
+	p.jobs <- job
+	p.mu.RUnlock()
+}
+
+// stop drains the queue and waits for in-flight work.
+//
+// Every job already accepted runs to completion, so a bill mid-commit still
+// gets its ack. Without one the terminal would hold it and send it again on
+// restart — harmless, but avoidable.
+func (p *posPool) stop() {
+	p.once.Do(func() {
+		// The write lock waits for every submit already in progress, so the
+		// channel is never closed while something is mid-send.
+		p.mu.Lock()
+		p.closed = true
+		close(p.jobs)
+		p.mu.Unlock()
+
+		p.wg.Wait()
+		log.Printf("pos: %s pool drained", p.name)
+	})
+}
+
+// posPoolSize reads a worker count from the environment.
+//
+// The default is deliberately modest. Each worker holds a database transaction
+// while it runs, so the useful ceiling is the Postgres connection pool, not the
+// CPU — set this above what the database can serve and the workers simply queue
+// inside the driver instead, where there is no backpressure to feel.
+func posPoolSize(key string, fallback int) int {
+	v, err := strconv.Atoi(os.Getenv(key))
+	if err != nil || v <= 0 {
+		return fallback
+	}
+	if v > 128 {
+		return 128
+	}
+	return v
+}
+
+// posOrderedDelivery reports whether paho should preserve message order.
+//
+// Left on: paho then delivers on one goroutine, which hands work to the pool
+// and blocks when it is full. That single delivery goroutine is what makes
+// backpressure reach the broker at all — with concurrent delivery paho would
+// keep reading regardless of how far behind the workers were.
+const posOrderedDelivery = true
+
+// wrapHandler puts a paho message handler behind a pool.
+//
+// The payload is copied because paho reuses its buffer once the handler
+// returns, and the work now happens after that.
+func wrapHandler(pool *posPool, h mqtt.MessageHandler) mqtt.MessageHandler {
+	return func(client mqtt.Client, msg mqtt.Message) {
+		topic := msg.Topic()
+		payload := make([]byte, len(msg.Payload()))
+		copy(payload, msg.Payload())
+
+		pool.submit(func() {
+			h(client, copiedMessage{topic: topic, payload: payload})
+		})
+	}
+}
+
+// copiedMessage carries a payload that outlives paho's buffer.
+type copiedMessage struct {
+	topic   string
+	payload []byte
+}
+
+func (m copiedMessage) Duplicate() bool   { return false }
+func (m copiedMessage) Qos() byte         { return 1 }
+func (m copiedMessage) Retained() bool    { return false }
+func (m copiedMessage) Topic() string     { return m.topic }
+func (m copiedMessage) MessageID() uint16 { return 0 }
+func (m copiedMessage) Payload() []byte   { return m.payload }
+func (m copiedMessage) Ack()              {}