Process the MQTT ingest on a bounded worker pool

paho delivers on one goroutine, so bills were committed strictly one
after another. Each is a full Postgres transaction — advisory lock,
dedup, stock row locks, availability check, four inserts, commit — which
is 10-30ms, so the ceiling was roughly 30-100 bills a second and a
shop's backlog draining after an outage took minutes to land.

A fixed pool behind a bounded queue, rather than a goroutine per
message. Unbounded concurrency would open a transaction per message and
exhaust the connection pool under a storm, stalling every one of them at
once — a slow minute turned into a dead one. When the queue fills,
submit blocks: paho stops acknowledging, the broker's in-flight window
fills, it stops sending, and the backpressure reaches the till, which
holds its bills and retries. Slow, but nothing is dropped.

Heartbeats get their own pool. Sharing one would let a backlog of bills
delay presence, so every till would appear to go dark at exactly the
moment the system was busiest — the worst time to be blind to which
counters are alive.

Payloads are copied on the way in. paho reuses its buffer once a handler
returns and the work now happens after that, so a queued bill would
otherwise be read as whatever message arrived next — silently, and as
valid JSON often enough to commit the wrong sale.

One bug found by its own test: submit-after-stop selected between a
done-channel and the job channel, and once both were ready Go picks at
random. Picking the send panics on a closed channel. It would have shown
up in production as an occasional crash during shutdown and nowhere
else. Now guarded by an RWMutex held across the send, so the queue
cannot be closed under one in progress.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

messaging/posmqtt.go

@@ -42,6 +42,12 @@ const (
 type PosMqttConsumer struct {
 	client mqtt.Client
 	svc    services.PosService
+
+	// Bills and registrations share a pool; heartbeats get their own, so a
+	// backlog of sales cannot make every till look dark at the moment the
+	// system is busiest.
+	ingest *posPool
+	health *posPool
 }
 
 // StartPosMqttConsumer connects and subscribes.
@@ -72,7 +78,15 @@ func StartPosMqttConsumer(svc services.PosService) (*PosMqttConsumer, error) {
 		return nil, nil
 	}
 
-	c := &PosMqttConsumer{svc: svc}
+	// Each ingest worker holds a database transaction while it runs, so the
+	// real ceiling is the Postgres connection pool rather than the CPU. The
+	// queue is deep enough to absorb a burst and shallow enough that a genuine
+	// overload is felt as backpressure rather than hidden as latency.
+	c := &PosMqttConsumer{
+		svc:    svc,
+		ingest: newPosPool("ingest", posPoolSize("POS_INGEST_WORKERS", 8), 256),
+		health: newPosPool("health", posPoolSize("POS_HEALTH_WORKERS", 2), 512),
+	}
 
 	opts := mqtt.NewClientOptions().
 		AddBroker(url).
@@ -84,6 +98,10 @@ func StartPosMqttConsumer(svc services.PosService) (*PosMqttConsumer, error) {
 		SetClientID(getEnvDefault("MQTT_CLIENT_ID",
 			getEnvDefault("HOSTNAME", "nearle-pos-ingest"))).
 		SetCleanSession(false).
+		// Ordered delivery keeps paho on one goroutine, which is what lets a
+		// full queue push back on the broker. With concurrent delivery paho
+		// would keep reading no matter how far behind the workers were.
+		SetOrderMatters(posOrderedDelivery).
 		SetAutoReconnect(true).
 		SetMaxReconnectInterval(30 * time.Second).
 		SetKeepAlive(30 * time.Second).
@@ -101,9 +119,9 @@ func StartPosMqttConsumer(svc services.PosService) (*PosMqttConsumer, error) {
 	opts.SetOnConnectHandler(func(client mqtt.Client) {
 		log.Printf("pos: connected to MQTT broker %s", url)
 		for topic, handler := range map[string]mqtt.MessageHandler{
-			topicOrders:    c.handleOrders,
-			topicCustomers: c.handleCustomers,
-			topicHealth:    c.handleHealth,
+			topicOrders:    wrapHandler(c.ingest, c.handleOrders),
+			topicCustomers: wrapHandler(c.ingest, c.handleCustomers),
+			topicHealth:    wrapHandler(c.health, c.handleHealth),
 		} {
 			if token := client.Subscribe(topic, 1, handler); token.Wait() && token.Error() != nil {
 				log.Printf("pos: could not subscribe to %s: %v", topic, token.Error())
@@ -270,6 +288,12 @@ func (c *PosMqttConsumer) Close() {
 	if c == nil || c.client == nil {
 		return
 	}
+	// Workers drain before the connection closes, so a bill mid-commit still
+	// gets its ack out. Disconnecting first would strand it: committed here,
+	// unacknowledged there, and sent again on the till's next attempt.
+	c.ingest.stop()
+	c.health.stop()
+
 	quiesce, err := strconv.Atoi(getEnvDefault("MQTT_QUIESCE_MS", "2000"))
 	if err != nil || quiesce < 0 {
 		quiesce = 2000