package messaging import ( "log" "os" "strconv" "sync" mqtt "github.com/eclipse/paho.mqtt.golang" ) // Concurrency for the MQTT ingest. // // paho delivers messages on a single goroutine, so without this every bill is // committed one after another. A bill is a full Postgres transaction — advisory // lock, dedup check, stock row locks, availability check, four inserts, commit — // which realistically costs 10–30ms. Serially that is 30–100 bills a second, // and a shop-wide backlog draining after an outage would take minutes to land. // // ### Why a bounded pool rather than a goroutine per message // // paho can be told to call handlers concurrently, but it spawns without limit. // A storm would then open a database transaction per message, exhaust the // connection pool, and stall every one of them at once — turning a slow minute // into a dead one. // // A fixed pool behind a bounded queue does the opposite. When the queue fills, // submitting **blocks**, which is the point: paho stops acknowledging, the // broker's in-flight window fills, and it stops sending. Backpressure travels // all the way back to the till, which holds its bills and retries. Slow, but // nothing is dropped and nothing is lost. // // ### Why bills and heartbeats have separate pools // // A heartbeat is one Redis write and a bill is a transaction. Sharing a queue // would let a backlog of bills delay presence, and every till would appear to // go dark at exactly the moment the system was busiest — the worst possible // time to be blind to which counters are alive. // posPool is a fixed set of workers reading a bounded queue. type posPool struct { name string jobs chan func() wg sync.WaitGroup once sync.Once // Guards the transition to closed. A plain `select` over a done-channel and // the job channel is not enough: once both are ready Go picks between them // at random, and picking the send panics on a closed channel. Held for // reading across the whole of submit, so stop cannot close the queue out // from under a send already in progress. mu sync.RWMutex closed bool } func newPosPool(name string, workers, queue int) *posPool { p := &posPool{ name: name, jobs: make(chan func(), queue), } p.wg.Add(workers) for i := 0; i < workers; i++ { go func() { defer p.wg.Done() for job := range p.jobs { job() } }() } log.Printf("pos: %s pool started with %d workers, queue %d", name, workers, queue) return p } // submit queues work, blocking when the queue is full. // // Blocking is deliberate. Dropping would lose a bill outright; the terminal // would eventually re-send it, but only after its ack timeout, and meanwhile we // would have thrown away work we had already accepted. Blocking instead pushes // back through paho to the broker to the till, which is exactly where the // decision to slow down belongs. func (p *posPool) submit(job func()) { p.mu.RLock() if p.closed { p.mu.RUnlock() // Shutting down. Running it inline still gets the work done and its ack // published, rather than discarding a bill that already reached us. job() return } // The read lock is held across the send. Blocking here while the queue is // full cannot deadlock against stop: the workers only exit once the channel // is closed, and that happens under the write lock this send is holding // off — so they stay alive and keep draining until this send completes. p.jobs <- job p.mu.RUnlock() } // stop drains the queue and waits for in-flight work. // // Every job already accepted runs to completion, so a bill mid-commit still // gets its ack. Without one the terminal would hold it and send it again on // restart — harmless, but avoidable. func (p *posPool) stop() { p.once.Do(func() { // The write lock waits for every submit already in progress, so the // channel is never closed while something is mid-send. p.mu.Lock() p.closed = true close(p.jobs) p.mu.Unlock() p.wg.Wait() log.Printf("pos: %s pool drained", p.name) }) } // posPoolSize reads a worker count from the environment. // // The default is deliberately modest. Each worker holds a database transaction // while it runs, so the useful ceiling is the Postgres connection pool, not the // CPU — set this above what the database can serve and the workers simply queue // inside the driver instead, where there is no backpressure to feel. func posPoolSize(key string, fallback int) int { v, err := strconv.Atoi(os.Getenv(key)) if err != nil || v <= 0 { return fallback } if v > 128 { return 128 } return v } // posOrderedDelivery reports whether paho should preserve message order. // // Left on: paho then delivers on one goroutine, which hands work to the pool // and blocks when it is full. That single delivery goroutine is what makes // backpressure reach the broker at all — with concurrent delivery paho would // keep reading regardless of how far behind the workers were. const posOrderedDelivery = true // wrapHandler puts a paho message handler behind a pool. // // The payload is copied because paho reuses its buffer once the handler // returns, and the work now happens after that. func wrapHandler(pool *posPool, h mqtt.MessageHandler) mqtt.MessageHandler { return func(client mqtt.Client, msg mqtt.Message) { topic := msg.Topic() payload := make([]byte, len(msg.Payload())) copy(payload, msg.Payload()) pool.submit(func() { h(client, copiedMessage{topic: topic, payload: payload}) }) } } // copiedMessage carries a payload that outlives paho's buffer. type copiedMessage struct { topic string payload []byte } func (m copiedMessage) Duplicate() bool { return false } func (m copiedMessage) Qos() byte { return 1 } func (m copiedMessage) Retained() bool { return false } func (m copiedMessage) Topic() string { return m.topic } func (m copiedMessage) MessageID() uint16 { return 0 } func (m copiedMessage) Payload() []byte { return m.payload } func (m copiedMessage) Ack() {}