Files
AI_engine/tests/test_dispatch_agent.py
Suriyakumarvijayanayagam 4283c602f6 fix(dispatch): liveness-aware coverage check, escalation rate limit, real alert sink
Prod findings (2026-09-22): a backend retry sweep failed 1,001 bookings in
60s; the agent called each a "coverage gap" because GEORADIUS found a miler
in the geo index (last seen in June), then sent 1,001 ops_alert tasks to
CUSTOMER_AGENT, which has no such handler.

- _find_zone filters GEORADIUS candidates by the backend's miler_status:<id>
  key; only status=Available counts. Facts now carry
  nearest_available_miler_within_km plus milers_in_geo_index_within_30km so
  the decision can separate "no riders here" from "riders exist, none on duty".
- Rate limit per zone per day: after the first alert, further failures only
  bump the counter (no LLM call); a summary re-alert goes out every
  DISPATCH_REALERT_EVERY (default 100).
- _ops_alert / _escalate_dispatch send EXCEPTION_DETECTED to JARVIS (the path
  that is actually handled); customer delay notice uses CUSTOMER_AGENT's real
  send_notification contract.
- JARVIS: escalation inbox (_escalations, pending_escalations()) and
  human_review/ops_alert task types are recorded instead of dropped.
- ExceptionAgent pull loops: also catch asyncio.TimeoutError (distinct from
  nats.errors.TimeoutError on 3.11) and log the exception type — the blank
  "pull loop error:" lines.
- Prompt + eval cases updated for the renamed facts; new case for the
  observed index-full/nobody-on-duty pattern. Tests for liveness filtering,
  burst suppression, fallback heuristic, sinks, and the JARVIS inbox.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012AJLYcbTHCe45fyFnMfEin
2026-09-22 16:26:27 +05:30

290 lines
12 KiB
Python

"""
Unit tests for the DispatchAgent assignment-failure context gatherer.
Focus is the same as the ExceptionAgent tests: the gatherer must degrade
gracefully (no coords, a failing GEORADIUS, a failing counter) and never raise.
The real `_find_zone` is exercised through an in-memory fake Redis; the agent is
built with __new__ so NATS / the message bus are never touched.
Run:
python -m unittest discover -s tests
"""
import unittest
import nats.js.errors
from agents.dispatch_agent import DispatchAgent
class FakeRedis:
"""decode_responses=True style. `geo` is the GEORADIUS result for every call.
`available` is the set of miler ids whose miler_status:<id> says Available;
every other geo member is treated as present-but-off-duty (the stale-index case)."""
def __init__(self, geo=None, counter_start=0, raise_on=(), available=None, alerted=False):
self._geo = list(geo) if geo is not None else []
self._counter = counter_start
self._raise_on = set(raise_on)
self._available = set(available) if available is not None else set(self._geo)
self._kv = {}
if alerted:
self._kv["__alerted__"] = "1"
self.incr_calls = []
self.expire_calls = []
self.set_calls = []
async def get(self, key):
if key.startswith("miler_status:"):
mid = key.split(":", 1)[1]
if mid in self._available:
return '{"userid": %s, "status": "Available"}' % (mid if mid.isdigit() else 0)
if mid in self._geo:
return '{"userid": 0, "status": "Break"}'
return None
return self._kv.get(key)
async def exists(self, key):
if key.startswith("assignment_alert_sent:"):
return 1 if "__alerted__" in self._kv or key in self._kv else 0
return 1 if key in self._kv else 0
async def set(self, key, value, ex=None, nx=False):
self.set_calls.append((key, ex, nx))
if nx and key in self._kv:
return None
self._kv[key] = value
return True
async def georadius(self, key, lon, lat, radius, unit, sort=None, count=None):
if "georadius" in self._raise_on:
raise RuntimeError("geo down")
return list(self._geo)
async def hgetall(self, key):
return {"hub_id": "H1", "zone_id": "z1", "avg_delivery_time": "60"}
async def incr(self, key):
if "incr" in self._raise_on:
raise RuntimeError("redis down")
self._counter += 1
self.incr_calls.append(key)
return self._counter
async def expire(self, key, ttl):
self.expire_calls.append((key, ttl))
return True
def make_agent(redis):
agent = DispatchAgent.__new__(DispatchAgent) # skip heavy __init__ / message-bus registration
agent._redis = redis
return agent
class FakeJS:
"""Stand-in JetStream context for binding tests."""
def __init__(self, stream_name=None, not_found=False):
self._stream = stream_name
self._not_found = not_found
self.subscribe_calls = []
self.add_stream_calls = []
async def find_stream_name_by_subject(self, subject):
if self._not_found:
raise nats.js.errors.NotFoundError()
return self._stream
async def subscribe(self, subject, durable=None, stream=None, cb=None):
self.subscribe_calls.append((subject, durable, stream))
return object() # stand-in subscription
async def add_stream(self, **kwargs):
self.add_stream_calls.append(kwargs)
async def _cb(msg):
pass
def make_agent_js(js):
agent = DispatchAgent.__new__(DispatchAgent)
agent._nats_js = js
return agent
class TestGatherAssignmentFacts(unittest.IsolatedAsyncioTestCase):
async def test_rider_nearby(self):
redis = FakeRedis(geo=["m5"], counter_start=0) # found on the first (10km) sweep
facts, count = await make_agent(redis)._gather_assignment_facts("hyderabad", 17.4, 78.4)
self.assertEqual(facts["zone_id"], "hyderabad")
self.assertTrue(facts["has_coordinates"])
self.assertEqual(facts["nearest_available_miler_within_km"], 10)
self.assertEqual(facts["milers_in_geo_index_within_30km"], 1)
self.assertFalse(facts["alert_already_sent_today"])
self.assertEqual(facts["failures_today"], 1)
self.assertEqual(count, 1)
self.assertEqual(redis.expire_calls[0][1], 172800)
async def test_no_rider_within_30km(self):
redis = FakeRedis(geo=[], counter_start=2) # every sweep empty
facts, count = await make_agent(redis)._gather_assignment_facts("pune", 18.5, 73.8)
self.assertEqual(facts["nearest_available_miler_within_km"], "none within 30km")
self.assertEqual(facts["milers_in_geo_index_within_30km"], 0)
self.assertEqual(count, 3)
async def test_no_coordinates(self):
redis = FakeRedis(geo=["m1"])
facts, count = await make_agent(redis)._gather_assignment_facts("unknown", None, None)
self.assertFalse(facts["has_coordinates"])
self.assertEqual(facts["nearest_available_miler_within_km"], "unknown (no coordinates)")
self.assertNotIn("milers_in_geo_index_within_30km", facts)
self.assertEqual(count, 1) # counter still incremented
self.assertEqual(redis.incr_calls and 1, 1)
async def test_georadius_error_is_swallowed(self):
redis = FakeRedis(raise_on={"georadius"})
facts, count = await make_agent(redis)._gather_assignment_facts("z", 1.0, 2.0) # must not raise
self.assertEqual(facts["nearest_available_miler_within_km"], "none within 30km")
self.assertEqual(facts["failures_today"], 1)
async def test_counter_error_is_swallowed(self):
redis = FakeRedis(geo=["m9"], raise_on={"incr"})
facts, count = await make_agent(redis)._gather_assignment_facts("z", 1.0, 2.0) # must not raise
self.assertEqual(facts["nearest_available_miler_within_km"], 10) # coverage still gathered
self.assertEqual(facts["failures_today"], 0) # counter degraded to 0
self.assertEqual(count, 0)
class TestLiveness(unittest.IsolatedAsyncioTestCase):
"""The prod incident: 26 riders in the geo index, none on duty. The old
check said 'rider within 20 km' and called it a coverage gap."""
async def test_stale_index_entries_are_not_available_riders(self):
redis = FakeRedis(geo=["4", "7", "12"], available=[]) # all present, all off-duty
facts, _ = await make_agent(redis)._gather_assignment_facts("coimbatore", 11.0, 76.9)
self.assertEqual(facts["nearest_available_miler_within_km"], "none within 30km")
self.assertEqual(facts["milers_in_geo_index_within_30km"], 3) # but the index knows them
async def test_first_available_candidate_wins(self):
redis = FakeRedis(geo=["4", "7", "12"], available=["12"])
found = await make_agent(redis)._find_zone(11.0, 76.9, radius_km=10)
self.assertEqual(found["miler_id"], "12")
self.assertEqual(found["stale_candidates_skipped"], 2)
async def test_missing_status_key_means_unavailable(self):
redis = FakeRedis(geo=["99"], available=[])
redis._geo = [] # nothing in geo → get() returns None for status
self.assertFalse(await make_agent(redis)._miler_is_available("99"))
class FakeMessages:
def __init__(self):
self.sent = []
async def send_message(self, recipient, message_type, payload, correlation_id=None):
self.sent.append((recipient, message_type, payload))
def make_handler_agent(redis, decision):
"""Agent with send_message captured and the LLM stubbed to a fixed decision (or None)."""
from agents import dispatch_agent as mod
agent = DispatchAgent.__new__(DispatchAgent)
agent._redis = redis
agent.agent_id = "DISPATCH_AGENT"
box = FakeMessages()
agent.send_message = box.send_message
calls = []
async def fake_decide(context):
calls.append(context)
return decision
mod.decide_assignment_failure = fake_decide
return agent, box, calls
class FakeMsg:
def __init__(self, payload):
import json
self.data = json.dumps(payload).encode()
self.acked = False
async def ack(self):
self.acked = True
class TestRateLimitAndSinks(unittest.IsolatedAsyncioTestCase):
def setUp(self):
from agents import dispatch_agent as mod
self._orig = mod.decide_assignment_failure
def tearDown(self):
from agents import dispatch_agent as mod
mod.decide_assignment_failure = self._orig
async def test_ops_alert_goes_to_jarvis_as_exception_and_marks_zone(self):
from core.types import MessageType
from core.llm import AssignmentDecision
redis = FakeRedis(geo=[], counter_start=2)
agent, box, calls = make_handler_agent(redis, AssignmentDecision("ops_alert", "gap", 0.9))
msg = FakeMsg({"booking_id": 1, "lat": 11.0, "lon": 76.9})
await agent._on_nats_booking_assignment_failed(msg)
self.assertTrue(msg.acked)
self.assertEqual(len(calls), 1) # LLM consulted once
recipient, mtype, payload = box.sent[0]
self.assertEqual(recipient, "JARVIS") # not CUSTOMER_AGENT
self.assertEqual(mtype, MessageType.EXCEPTION_DETECTED) # the path JARVIS handles
self.assertEqual(payload["exception_type"], "coverage_gap")
self.assertTrue(any(k.startswith("assignment_alert_sent:") for k, _, _ in redis.set_calls))
async def test_burst_after_alert_skips_llm_and_realerts_every_n(self):
from agents import dispatch_agent as mod
from core.llm import AssignmentDecision
redis = FakeRedis(geo=[], counter_start=0, alerted=True) # zone already alerted today
agent, box, calls = make_handler_agent(redis, AssignmentDecision("ops_alert", "gap", 0.9))
old = mod.DISPATCH_REALERT_EVERY
mod.DISPATCH_REALERT_EVERY = 100
try:
for i in range(1, 251):
await agent._on_nats_booking_assignment_failed(FakeMsg({"booking_id": i, "lat": 11.0, "lon": 76.9}))
finally:
mod.DISPATCH_REALERT_EVERY = old
self.assertEqual(calls, []) # zero LLM calls during the burst
self.assertEqual(len(box.sent), 2) # re-alerts at 100 and 200 only
self.assertIn("200 failed assignments", box.sent[1][2]["reasoning"])
async def test_llm_unavailable_falls_back_to_heuristic(self):
redis = FakeRedis(geo=[], counter_start=2)
agent, box, calls = make_handler_agent(redis, None) # LLM down
await agent._on_nats_booking_assignment_failed(FakeMsg({"booking_id": 5, "lat": 1.0, "lon": 2.0}))
self.assertEqual(len(box.sent), 1) # count reached 3 → heuristic alert
self.assertEqual(box.sent[0][0], "JARVIS")
async def test_escalate_goes_to_jarvis_as_exception(self):
from core.types import MessageType
from core.llm import AssignmentDecision
redis = FakeRedis(geo=["4"], available=["4"], counter_start=4)
agent, box, calls = make_handler_agent(redis, AssignmentDecision("escalate", "riders near, still failing", 0.8))
await agent._on_nats_booking_assignment_failed(FakeMsg({"booking_id": 9, "lat": 1.0, "lon": 2.0}))
recipient, mtype, payload = box.sent[0]
self.assertEqual((recipient, mtype), ("JARVIS", MessageType.EXCEPTION_DETECTED))
self.assertEqual(payload["proposed_action"], "escalate")
class TestBindConsumer(unittest.IsolatedAsyncioTestCase):
async def test_binds_to_discovered_stream_without_creating(self):
js = FakeJS(stream_name="TRACKING")
sub = await make_agent_js(js)._bind_consumer("booking.assigned", "d1", _cb)
self.assertIsNotNone(sub)
# Bound explicitly on the discovered stream...
self.assertEqual(js.subscribe_calls, [("booking.assigned", "d1", "TRACKING")])
# ...and never tried to create a stream (the old overlap bug).
self.assertEqual(js.add_stream_calls, [])
async def test_returns_none_when_no_stream_carries_subject(self):
js = FakeJS(not_found=True)
sub = await make_agent_js(js)._bind_consumer("booking.assigned", "d1", _cb)
self.assertIsNone(sub) # clear failure, not a raise
self.assertEqual(js.subscribe_calls, []) # did not attempt to subscribe
if __name__ == "__main__":
unittest.main()