Process the MQTT ingest on a bounded worker pool
paho delivers on one goroutine, so bills were committed strictly one after another. Each is a full Postgres transaction — advisory lock, dedup, stock row locks, availability check, four inserts, commit — which is 10-30ms, so the ceiling was roughly 30-100 bills a second and a shop's backlog draining after an outage took minutes to land. A fixed pool behind a bounded queue, rather than a goroutine per message. Unbounded concurrency would open a transaction per message and exhaust the connection pool under a storm, stalling every one of them at once — a slow minute turned into a dead one. When the queue fills, submit blocks: paho stops acknowledging, the broker's in-flight window fills, it stops sending, and the backpressure reaches the till, which holds its bills and retries. Slow, but nothing is dropped. Heartbeats get their own pool. Sharing one would let a backlog of bills delay presence, so every till would appear to go dark at exactly the moment the system was busiest — the worst time to be blind to which counters are alive. Payloads are copied on the way in. paho reuses its buffer once a handler returns and the work now happens after that, so a queued bill would otherwise be read as whatever message arrived next — silently, and as valid JSON often enough to commit the wrong sale. One bug found by its own test: submit-after-stop selected between a done-channel and the job channel, and once both were ready Go picks at random. Picking the send panics on a closed channel. It would have shown up in production as an occasional crash during shutdown and nowhere else. Now guarded by an RWMutex held across the send, so the queue cannot be closed under one in progress. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Suriya
247
messaging/posworkers_test.go
Normal file
247
messaging/posworkers_test.go
Normal file
@@ -0,0 +1,247 @@
|
||||
package messaging
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
)
|
||||
|
||||
func TestEveryJobRuns(t *testing.T) {
|
||||
pool := newPosPool("test", 4, 16)
|
||||
|
||||
var done int64
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
pool.submit(func() {
|
||||
defer wg.Done()
|
||||
atomic.AddInt64(&done, 1)
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
pool.stop()
|
||||
|
||||
if got := atomic.LoadInt64(&done); got != 100 {
|
||||
t.Errorf("ran %d jobs, want 100", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyIsBounded(t *testing.T) {
|
||||
// The reason the pool exists. Unbounded concurrency would open a database
|
||||
// transaction per message and exhaust the connection pool under a storm,
|
||||
// stalling every one of them at once.
|
||||
const workers = 4
|
||||
pool := newPosPool("test", workers, 64)
|
||||
|
||||
var inFlight, peak int64
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
wg.Add(1)
|
||||
pool.submit(func() {
|
||||
defer wg.Done()
|
||||
|
||||
now := atomic.AddInt64(&inFlight, 1)
|
||||
for {
|
||||
was := atomic.LoadInt64(&peak)
|
||||
if now <= was || atomic.CompareAndSwapInt64(&peak, was, now) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
atomic.AddInt64(&inFlight, -1)
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
pool.stop()
|
||||
|
||||
if got := atomic.LoadInt64(&peak); got > workers {
|
||||
t.Errorf("peak concurrency %d exceeded the %d workers", got, workers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitBlocksRatherThanDroppingWork(t *testing.T) {
|
||||
// A full queue must slow the caller down, not discard a bill. Dropping
|
||||
// would throw away work already accepted from the broker, and the terminal
|
||||
// would only find out at its ack timeout.
|
||||
pool := newPosPool("test", 1, 1)
|
||||
|
||||
release := make(chan struct{})
|
||||
var ran int64
|
||||
|
||||
// Occupy the single worker.
|
||||
pool.submit(func() {
|
||||
<-release
|
||||
atomic.AddInt64(&ran, 1)
|
||||
})
|
||||
|
||||
// Fill the queue, then a third submit must block until the worker frees up.
|
||||
pool.submit(func() { atomic.AddInt64(&ran, 1) })
|
||||
|
||||
blocked := make(chan struct{})
|
||||
go func() {
|
||||
pool.submit(func() { atomic.AddInt64(&ran, 1) })
|
||||
close(blocked)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-blocked:
|
||||
t.Fatal("submit returned while the queue was full; work would be dropped under load")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// Correctly blocked.
|
||||
}
|
||||
|
||||
close(release)
|
||||
|
||||
select {
|
||||
case <-blocked:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("submit never unblocked after the worker freed up")
|
||||
}
|
||||
|
||||
pool.stop()
|
||||
|
||||
if got := atomic.LoadInt64(&ran); got != 3 {
|
||||
t.Errorf("ran %d jobs, want 3 — none may be lost", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopDrainsAcceptedWork(t *testing.T) {
|
||||
// A bill mid-commit must still get its ack. Without one the terminal holds
|
||||
// it and sends it again on restart — harmless, but avoidable.
|
||||
pool := newPosPool("test", 2, 64)
|
||||
|
||||
var done int64
|
||||
for i := 0; i < 50; i++ {
|
||||
pool.submit(func() {
|
||||
time.Sleep(time.Millisecond)
|
||||
atomic.AddInt64(&done, 1)
|
||||
})
|
||||
}
|
||||
|
||||
pool.stop()
|
||||
|
||||
if got := atomic.LoadInt64(&done); got != 50 {
|
||||
t.Errorf("only %d of 50 jobs completed before shutdown finished", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitAfterStopStillRunsTheWork(t *testing.T) {
|
||||
// A message that arrived during shutdown has already been taken from the
|
||||
// broker. Discarding it would lose a bill we accepted responsibility for.
|
||||
pool := newPosPool("test", 2, 8)
|
||||
pool.stop()
|
||||
|
||||
var ran int64
|
||||
pool.submit(func() { atomic.AddInt64(&ran, 1) })
|
||||
|
||||
if got := atomic.LoadInt64(&ran); got != 1 {
|
||||
t.Error("work submitted during shutdown was discarded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopIsIdempotent(t *testing.T) {
|
||||
// Close() may be reached twice on a shutdown path; a second close of the
|
||||
// jobs channel would panic and take the process down mid-drain.
|
||||
pool := newPosPool("test", 2, 8)
|
||||
pool.stop()
|
||||
pool.stop()
|
||||
pool.stop()
|
||||
}
|
||||
|
||||
func TestPoolSizeFallsBackAndClamps(t *testing.T) {
|
||||
t.Setenv("POS_TEST_WORKERS", "")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 8 {
|
||||
t.Errorf("unset = %d, want the fallback 8", got)
|
||||
}
|
||||
|
||||
t.Setenv("POS_TEST_WORKERS", "not a number")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 8 {
|
||||
t.Errorf("garbage = %d, want the fallback 8", got)
|
||||
}
|
||||
|
||||
t.Setenv("POS_TEST_WORKERS", "0")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 8 {
|
||||
t.Errorf("zero = %d, want the fallback 8", got)
|
||||
}
|
||||
|
||||
t.Setenv("POS_TEST_WORKERS", "-4")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 8 {
|
||||
t.Errorf("negative = %d, want the fallback 8", got)
|
||||
}
|
||||
|
||||
t.Setenv("POS_TEST_WORKERS", "24")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 24 {
|
||||
t.Errorf("explicit = %d, want 24", got)
|
||||
}
|
||||
|
||||
// Clamped: more workers than the database can serve just moves the queue
|
||||
// inside the driver, where there is no backpressure to feel.
|
||||
t.Setenv("POS_TEST_WORKERS", "100000")
|
||||
if got := posPoolSize("POS_TEST_WORKERS", 8); got != 128 {
|
||||
t.Errorf("absurd = %d, want the 128 clamp", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWrappedHandlerCopiesThePayload(t *testing.T) {
|
||||
// paho reuses its buffer once a handler returns, and with a pool the work
|
||||
// now happens *after* that. Without a copy a queued bill would be read as
|
||||
// whatever message happened to arrive next — silently, and as valid JSON
|
||||
// often enough to commit the wrong sale.
|
||||
pool := newPosPool("test", 1, 4)
|
||||
|
||||
seen := make(chan string, 1)
|
||||
wrapped := wrapHandler(pool, func(_ mqtt.Client, msg mqtt.Message) {
|
||||
seen <- string(msg.Payload())
|
||||
})
|
||||
|
||||
// A buffer paho would reuse.
|
||||
buffer := []byte(`{"batch_id":"original"}`)
|
||||
wrapped(nil, fakeMessage{topic: "nearle/pos/12/T4A9/order", payload: buffer})
|
||||
|
||||
// Overwrite it the instant the handler returns, exactly as paho would.
|
||||
for i := range buffer {
|
||||
buffer[i] = 'X'
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-seen:
|
||||
if got != `{"batch_id":"original"}` {
|
||||
t.Errorf("handler saw %q — the payload was not copied before queueing", got)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the wrapped handler never ran")
|
||||
}
|
||||
|
||||
pool.stop()
|
||||
}
|
||||
|
||||
func TestAWrappedHandlerKeepsTheTopic(t *testing.T) {
|
||||
// Store and terminal are read from the topic, never the body. Losing it in
|
||||
// the hand-off would leave the ack with nowhere to go.
|
||||
pool := newPosPool("test", 1, 4)
|
||||
|
||||
seen := make(chan string, 1)
|
||||
wrapped := wrapHandler(pool, func(_ mqtt.Client, msg mqtt.Message) {
|
||||
seen <- msg.Topic()
|
||||
})
|
||||
|
||||
wrapped(nil, fakeMessage{topic: "nearle/pos/1135/T4A9/order", payload: []byte("{}")})
|
||||
|
||||
select {
|
||||
case got := <-seen:
|
||||
if got != "nearle/pos/1135/T4A9/order" {
|
||||
t.Errorf("topic = %q, want nearle/pos/1135/T4A9/order", got)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the wrapped handler never ran")
|
||||
}
|
||||
|
||||
pool.stop()
|
||||
}
|
||||
Reference in New Issue
Block a user