Initial commit

This commit is contained in:
2026-06-26 16:08:31 +05:30
commit f49193ee73
23 changed files with 6056 additions and 0 deletions

607
agents/exception_agent.py Normal file
View File

@@ -0,0 +1,607 @@
"""
Exception Agent — Miler stall detection + general exception handling.
Stall detection flow:
JetStream TRACKING/miler.location.updated (pull) -> Redis HSET -> stall check -> miler.stalled
JetStream TRACKING/miler.stalled (pull) -> POST reassign + POST notify
StallDetector sweep (every 60 s):
Postgres pickupbookings (active status) -> Redis movement.updated_at age -> miler.stalled
"""
import asyncio
import json
import uuid
from datetime import datetime, timedelta
from typing import Dict, List, Any, Optional, Set
from dataclasses import dataclass, field
from enum import Enum
import asyncpg
import nats
import nats.errors
import redis.asyncio as aioredis
from core.agent import SpecializedAgent
from core.types import AgentTask, MessageType
from core.logger import logger
from core.http_client import api_post
from config.system_config import (
GO_API_BASE_URL, INTERNAL_API_KEY,
DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD,
NATS_HOST, NATS_PORT, NATS_USER, NATS_PASSWORD,
REDIS_HOST, REDIS_PORT, REDIS_PASSWORD,
)
STALL_MINUTES = 10
ACTIVE_STATUSES = ["Miler_Assigned", "Pickup_Scheduled"]
TRACKING_STREAM = "TRACKING"
class ExceptionType(str, Enum):
DELAY = "delay"
CANCELLATION = "cancellation"
RESCHEDULE = "reschedule"
REROUTE = "reroute"
FAILED_DELIVERY = "failed_delivery"
VEHICLE_BREAKDOWN = "vehicle_breakdown"
HUB_OVERFLOW = "hub_overflow"
WEATHER_IMPACT = "weather_impact"
CUSTOMER_UNAVAILABLE = "customer_unavailable"
MILER_STALLED = "miler_stalled"
class ExceptionSeverity(str, Enum):
LOW = "low"
MEDIUM = "medium"
HIGH = "high"
CRITICAL = "critical"
@dataclass
class ExceptionRecord:
exception_id: str
order_id: str
exception_type: ExceptionType
severity: ExceptionSeverity
description: str
detected_at: datetime
resolved_at: Optional[datetime]
resolution: Optional[str]
assigned_to: Optional[str]
status: str
actions_taken: List[str] = field(default_factory=list)
class ExceptionAgent(SpecializedAgent):
"""
Handles miler stall detection via JetStream pull-subscribe + Postgres + Redis,
and all existing order exception types.
"""
def __init__(self):
super().__init__(
agent_id="EXCEPTION_AGENT",
domain="exception_handling",
description="Stall detection, exception resolution, reassignment",
)
self._nc: Optional[nats.aio.client.Client] = None
self._js = None
self._pg: Optional[asyncpg.Pool] = None
self._redis = aioredis.Redis(
host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD,
decode_responses=True,
)
self._location_sub = None
self._stalled_sub = None
self._notified_stalls: Set[str] = set()
self._exceptions: Dict[str, ExceptionRecord] = {}
self._strategies = self._init_strategies()
self._escalation_rules = self._init_escalation_rules()
# ── Startup ───────────────────────────────────────────────────────────────
async def start(self):
await self._connect_infra()
await self._setup_nats_pull_subs()
asyncio.create_task(self._pull_location_loop())
asyncio.create_task(self._pull_stalled_loop())
asyncio.create_task(self._stall_detector_loop())
await super().start()
async def _connect_infra(self):
try:
self._nc = await nats.connect(
servers=[f"nats://{NATS_HOST}:{NATS_PORT}"],
user=NATS_USER,
password=NATS_PASSWORD,
max_reconnect_attempts=10,
)
self._js = self._nc.jetstream()
logger.info("EXCEPTION_AGENT connected to NATS JetStream")
except Exception as e:
logger.warning(f"EXCEPTION_AGENT NATS connect failed: {e}")
try:
self._pg = await asyncpg.create_pool(
host=DB_HOST, port=DB_PORT, database=DB_NAME,
user=DB_USER, password=DB_PASSWORD,
min_size=1, max_size=5,
)
logger.info("EXCEPTION_AGENT connected to Postgres")
except Exception as e:
logger.warning(f"EXCEPTION_AGENT Postgres connect failed: {e}")
async def _setup_nats_pull_subs(self):
if not self._js:
logger.warning("EXCEPTION_AGENT: skipping NATS pull subs (no JetStream connection)")
return
try:
self._location_sub = await self._js.pull_subscribe(
"miler.location.updated",
"tracking_location_consumer",
stream=TRACKING_STREAM,
)
logger.info("EXCEPTION_AGENT pull-subscribed: miler.location.updated")
except Exception as e:
logger.warning(f"pull_subscribe miler.location.updated failed: {e}")
try:
self._stalled_sub = await self._js.pull_subscribe(
"miler.stalled",
"tracking_stall_consumer",
stream=TRACKING_STREAM,
)
logger.info("EXCEPTION_AGENT pull-subscribed: miler.stalled")
except Exception as e:
logger.warning(f"pull_subscribe miler.stalled failed: {e}")
# ── Pull loops ────────────────────────────────────────────────────────────
async def _pull_location_loop(self):
while self._running:
if not self._location_sub:
await asyncio.sleep(5)
continue
try:
msgs = await self._location_sub.fetch(batch=20, timeout=1.0)
for msg in msgs:
try:
await self._on_location_update(msg)
except Exception as e:
logger.error(f"Location handler error: {e}")
finally:
await msg.ack()
except nats.errors.TimeoutError:
pass
except Exception as e:
logger.error(f"Location pull loop error: {e}")
await asyncio.sleep(2)
async def _pull_stalled_loop(self):
while self._running:
if not self._stalled_sub:
await asyncio.sleep(5)
continue
try:
msgs = await self._stalled_sub.fetch(batch=10, timeout=1.0)
for msg in msgs:
try:
await self._on_miler_stalled(msg)
except Exception as e:
logger.error(f"Stalled handler error: {e}")
finally:
await msg.ack()
except nats.errors.TimeoutError:
pass
except Exception as e:
logger.error(f"Stalled pull loop error: {e}")
await asyncio.sleep(2)
# ── Stall detection: per location ping ───────────────────────────────────
async def _on_location_update(self, msg):
try:
data = json.loads(msg.data.decode())
except Exception:
return
miler_id = str(data.get("miler_id", ""))
lat = str(round(float(data.get("lat", 0)), 6))
lon = str(round(float(data.get("lon", 0)), 6))
now_ts = datetime.now().isoformat()
redis_key = f"miler:{miler_id}:movement"
prev = await self._redis.hgetall(redis_key)
prev_lat = prev.get("lat", "")
prev_lon = prev.get("lon", "")
if lat != prev_lat or lon != prev_lon:
await self._redis.hset(redis_key, mapping={
"lat": lat, "lon": lon,
"updated_at": now_ts,
"position_unchanged_since": now_ts,
})
else:
await self._redis.hset(redis_key, mapping={"lat": lat, "lon": lon, "updated_at": now_ts})
unchanged_since_str = prev.get("position_unchanged_since", now_ts)
try:
unchanged_since = datetime.fromisoformat(unchanged_since_str)
except ValueError:
unchanged_since = datetime.now()
minutes_stalled = (datetime.now() - unchanged_since).total_seconds() / 60
if minutes_stalled >= STALL_MINUTES:
booking = await self._get_active_booking(miler_id)
if booking:
booking_id = booking["booking_id"]
if booking_id not in self._notified_stalls:
await self._publish_stall(miler_id, booking_id, minutes_stalled)
# ── Stall detection: background sweep ────────────────────────────────────
async def _stall_detector_loop(self):
while self._running:
await asyncio.sleep(60)
try:
await self._sweep_active_bookings()
except Exception as e:
logger.error(f"StallDetector error: {e}")
async def _sweep_active_bookings(self):
if not self._pg:
return
logger.debug("StallDetector: sweeping active bookings")
try:
async with self._pg.acquire() as conn:
rows = await conn.fetch(
"""
SELECT DISTINCT
assignedmileruserid::text AS miler_id,
bookingid::text AS booking_id
FROM pickupbookings
WHERE status = ANY($1)
""",
ACTIVE_STATUSES,
)
except Exception as e:
logger.error(f"StallDetector Postgres error: {e}")
return
now = datetime.now()
for row in rows:
miler_id = row["miler_id"]
booking_id = row["booking_id"]
if booking_id in self._notified_stalls:
continue
movement = await self._redis.hgetall(f"miler:{miler_id}:movement")
if not movement or "updated_at" not in movement:
continue
try:
updated_at = datetime.fromisoformat(movement["updated_at"])
minutes_stale = (now - updated_at).total_seconds() / 60
except ValueError:
continue
if minutes_stale >= STALL_MINUTES:
logger.warning(f"StallDetector: miler {miler_id} stale {minutes_stale:.1f} min (booking {booking_id})")
await self._publish_stall(miler_id, booking_id, minutes_stale)
# ── Stall: publish + handle ───────────────────────────────────────────────
async def _publish_stall(self, miler_id: str, booking_id: str, minutes_stalled: float):
self._notified_stalls.add(booking_id)
payload = json.dumps({
"miler_id": miler_id,
"booking_id": booking_id,
"minutes_stalled": round(minutes_stalled, 1),
}).encode()
try:
await self._js.publish("miler.stalled", payload)
logger.info(f"miler.stalled published: miler={miler_id} booking={booking_id} ({minutes_stalled:.1f} min)")
except Exception as e:
logger.error(f"Failed to publish miler.stalled: {e}")
async def _on_miler_stalled(self, msg):
"""Reassign booking and notify customer via Go API."""
try:
data = json.loads(msg.data.decode())
except Exception:
return
miler_id = data.get("miler_id", "")
booking_id = data.get("booking_id", "")
minutes_stalled = data.get("minutes_stalled", 0)
logger.warning(f"EXCEPTION_AGENT stall: miler={miler_id} booking={booking_id} ({minutes_stalled} min)")
headers = {"X-Internal-Key": INTERNAL_API_KEY}
reassign_result = await api_post(
f"{GO_API_BASE_URL}/api/v1/internal/bookings/{booking_id}/reassign",
json={"reason": "miler_stalled"},
headers=headers,
)
logger.info(f"Reassign {'OK' if reassign_result is not None else 'FAILED'} for booking {booking_id}")
notify_result = await api_post(
f"{GO_API_BASE_URL}/api/v1/internal/notify",
json={
"booking_id": booking_id,
"message": "We detected a delay, finding you a new miler",
"target": "customer",
},
headers=headers,
)
logger.info(f"Notify {'OK' if notify_result is not None else 'FAILED'} for booking {booking_id}")
exc_id = f"EXC-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
self._exceptions[exc_id] = ExceptionRecord(
exception_id=exc_id, order_id=booking_id,
exception_type=ExceptionType.MILER_STALLED,
severity=ExceptionSeverity.HIGH,
description=f"Miler {miler_id} stalled {minutes_stalled} min",
detected_at=datetime.now(), resolved_at=datetime.now(),
resolution="Reassignment triggered + customer notified",
assigned_to=None, status="resolved",
actions_taken=["reassign", "notify_customer"],
)
# ── Postgres helper ───────────────────────────────────────────────────────
async def _get_active_booking(self, miler_id: str) -> Optional[Dict]:
if not self._pg:
return None
try:
async with self._pg.acquire() as conn:
row = await conn.fetchrow(
"""
SELECT bookingid::text AS booking_id, status
FROM pickupbookings
WHERE assignedmileruserid::text = $1
AND status = ANY($2)
ORDER BY createdat DESC
LIMIT 1
""",
miler_id,
ACTIVE_STATUSES,
)
if row:
return {"booking_id": row["booking_id"], "status": row["status"]}
except Exception as e:
logger.error(f"Postgres get_active_booking error: {e}")
return None
# ── Task handler ──────────────────────────────────────────────────────────
async def handle_task(self, task: AgentTask) -> Dict[str, Any]:
handlers = {
"detect_exception": self._detect_exception,
"analyze_exception": self._analyze_exception,
"resolve_exception": self._resolve_exception,
"cancel_order": self._cancel_order,
"reschedule_delivery": self._reschedule_delivery,
"handle_delay": self._handle_delay,
"get_exception_status": self._get_exception_status,
"get_exception_history": self._get_exception_history,
"escalate_exception": self._escalate_exception,
}
handler = handlers.get(task.task_type, self._unknown_task)
return await handler(task)
async def _detect_exception(self, task: AgentTask) -> Dict[str, Any]:
order_id = task.data.get("order_id")
exception_type = task.data.get("exception_type")
severity = task.data.get("severity", "medium")
description = task.data.get("description", "")
logger.warning(f"Exception Agent: {exception_type} for {order_id} (severity={severity})")
exc_id = f"EXC-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
record = ExceptionRecord(
exception_id=exc_id, order_id=order_id,
exception_type=ExceptionType(exception_type),
severity=ExceptionSeverity(severity), description=description,
detected_at=datetime.now(), resolved_at=None,
resolution=None, assigned_to=None, status="detected",
)
self._exceptions[exc_id] = record
if severity in ("high", "critical"):
await self._analyze_exception(AgentTask(
task_id=f"{exc_id}_analyze", agent_type="exception",
task_type="analyze_exception", data={"exception_id": exc_id},
))
await self.send_message(
recipient="JARVIS", message_type=MessageType.EXCEPTION_DETECTED,
payload={"exception_id": exc_id, "order_id": order_id,
"exception_type": exception_type, "severity": severity},
)
return {"status": "detected", "exception_id": exc_id,
"exception_type": exception_type, "severity": severity}
async def _analyze_exception(self, task: AgentTask) -> Dict[str, Any]:
exc_id = task.data.get("exception_id")
if exc_id not in self._exceptions:
return {"status": "error", "message": f"Exception {exc_id} not found"}
record = self._exceptions[exc_id]
record.status = "analyzing"
strategy = self._strategies.get(record.exception_type, {})
auto_actions = strategy.get("auto_actions", [])
record.actions_taken.append("analyzed")
return {"status": "analyzed", "exception_id": exc_id, "strategy": auto_actions}
async def _resolve_exception(self, task: AgentTask) -> Dict[str, Any]:
exc_id = task.data.get("exception_id")
if exc_id not in self._exceptions:
return {"status": "error", "message": f"Exception {exc_id} not found"}
record = self._exceptions[exc_id]
record.status = "resolving"
for action in self._strategies.get(record.exception_type, {}).get("auto_actions", []):
record.actions_taken.append(action)
record.status = "resolved"
record.resolved_at = datetime.now()
record.resolution = f"Completed {len(record.actions_taken)} actions"
await self.send_message(
recipient="CUSTOMER_AGENT", message_type=MessageType.NOTIFICATION_SENT,
payload={"order_id": record.order_id, "message_type": "exception_resolved",
"data": {"exception_type": record.exception_type.value}},
)
return {"status": "resolved", "exception_id": exc_id, "resolution": record.resolution}
async def _cancel_order(self, task: AgentTask) -> Dict[str, Any]:
order_id = task.data.get("order_id")
reason = task.data.get("reason", "Customer requested cancellation")
exc_id = f"EXC-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
self._exceptions[exc_id] = ExceptionRecord(
exception_id=exc_id, order_id=order_id,
exception_type=ExceptionType.CANCELLATION, severity=ExceptionSeverity.MEDIUM,
description=reason, detected_at=datetime.now(), resolved_at=datetime.now(),
resolution="Order cancelled", assigned_to=None, status="resolved",
actions_taken=["release_vehicle", "notify_customer"],
)
await self.send_message(
recipient="FLEET_AGENT", message_type=MessageType.AGENT_TASK,
payload={"task_type": "release_vehicle_for_cancel", "order_id": order_id},
)
await self.send_message(
recipient="CUSTOMER_AGENT", message_type=MessageType.ORDER_CANCELLED,
payload={"order_id": order_id, "reason": reason,
"refund_info": "Refund within 5-7 business days"},
)
return {"status": "cancelled", "exception_id": exc_id, "order_id": order_id}
async def _reschedule_delivery(self, task: AgentTask) -> Dict[str, Any]:
order_id = task.data.get("order_id")
new_date = task.data.get("new_date")
reason = task.data.get("reason", "Operational constraints")
exc_id = f"EXC-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
self._exceptions[exc_id] = ExceptionRecord(
exception_id=exc_id, order_id=order_id,
exception_type=ExceptionType.RESCHEDULE, severity=ExceptionSeverity.MEDIUM,
description=f"Rescheduled to {new_date}: {reason}",
detected_at=datetime.now(), resolved_at=None, resolution=None,
assigned_to=None, status="resolving",
)
await self.send_message(
recipient="DISPATCH_AGENT", message_type=MessageType.AGENT_TASK,
payload={"task_type": "reschedule_route", "order_id": order_id, "new_date": new_date},
)
await self.send_message(
recipient="CUSTOMER_AGENT", message_type=MessageType.NOTIFICATION_SENT,
payload={"order_id": order_id, "message_type": "rescheduled",
"data": {"new_date": new_date, "reason": reason}},
)
return {"status": "rescheduled", "exception_id": exc_id, "order_id": order_id, "new_date": new_date}
async def _handle_delay(self, task: AgentTask) -> Dict[str, Any]:
order_id = task.data.get("order_id")
delay_reason = task.data.get("reason", "Unknown")
new_eta = task.data.get("new_eta")
severity = task.data.get("severity", "medium")
exc_id = f"EXC-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
self._exceptions[exc_id] = ExceptionRecord(
exception_id=exc_id, order_id=order_id,
exception_type=ExceptionType.DELAY, severity=ExceptionSeverity(severity),
description=f"Delay: {delay_reason}", detected_at=datetime.now(),
resolved_at=None, resolution=None, assigned_to=None, status="resolving",
)
if new_eta:
await self.send_message(
recipient="DISPATCH_AGENT", message_type=MessageType.ORDER_DELAYED,
payload={"order_id": order_id, "new_eta": new_eta, "reason": delay_reason},
)
await self.send_message(
recipient="CUSTOMER_AGENT", message_type=MessageType.NOTIFICATION_SENT,
payload={"order_id": order_id, "message_type": "delayed",
"data": {"delay_reason": delay_reason, "new_eta": new_eta}},
)
return {"status": "delay_handled", "exception_id": exc_id,
"order_id": order_id, "new_eta": new_eta, "customer_notified": True}
async def _get_exception_status(self, task: AgentTask) -> Dict[str, Any]:
exc_id = task.data.get("exception_id")
if exc_id not in self._exceptions:
return {"status": "error", "message": f"Exception {exc_id} not found"}
r = self._exceptions[exc_id]
return {
"exception_id": exc_id, "order_id": r.order_id,
"type": r.exception_type.value, "severity": r.severity.value,
"status": r.status, "description": r.description,
"detected_at": r.detected_at.isoformat(),
"resolved_at": r.resolved_at.isoformat() if r.resolved_at else None,
"actions_taken": r.actions_taken,
}
async def _get_exception_history(self, task: AgentTask) -> Dict[str, Any]:
exc_type = task.data.get("exception_type")
status_f = task.data.get("status")
items = list(self._exceptions.values())
if exc_type:
items = [e for e in items if e.exception_type.value == exc_type]
if status_f:
items = [e for e in items if e.status == status_f]
return {
"total_exceptions": len(items),
"exceptions": [
{"exception_id": e.exception_id, "order_id": e.order_id,
"type": e.exception_type.value, "severity": e.severity.value,
"status": e.status, "detected_at": e.detected_at.isoformat()}
for e in items[-50:]
],
}
async def _escalate_exception(self, task: AgentTask) -> Dict[str, Any]:
exc_id = task.data.get("exception_id")
if exc_id not in self._exceptions:
return {"status": "error", "message": f"Exception {exc_id} not found"}
record = self._exceptions[exc_id]
escalate_to = self._escalation_rules.get(record.exception_type, {}).get("escalate_to", "JARVIS")
record.actions_taken.append(f"escalated_to_{escalate_to}")
await self.send_message(
recipient=escalate_to, message_type=MessageType.EXCEPTION_DETECTED,
payload={"exception_id": exc_id, "order_id": record.order_id,
"exception_type": record.exception_type.value,
"severity": record.severity.value, "urgency": "high"},
)
return {"status": "escalated", "exception_id": exc_id, "escalated_to": escalate_to}
async def _unknown_task(self, task: AgentTask) -> Dict[str, Any]:
return {"status": "error", "message": f"Unknown task: {task.task_type}"}
def _init_strategies(self) -> Dict[str, Any]:
return {
ExceptionType.DELAY: {"auto_actions": ["notify_customer", "update_eta", "reroute_if_possible"]},
ExceptionType.CANCELLATION: {"auto_actions": ["release_vehicle", "refund_check", "notify_customer"]},
ExceptionType.RESCHEDULE: {"auto_actions": ["find_new_slot", "update_route", "notify_customer"]},
ExceptionType.REROUTE: {"auto_actions": ["recalculate_route", "update_vehicle", "notify_customer"]},
ExceptionType.FAILED_DELIVERY: {"auto_actions": ["retry_attempt", "return_to_hub"]},
ExceptionType.VEHICLE_BREAKDOWN: {"auto_actions": ["find_replacement_vehicle", "reassign_route", "notify_customer"]},
ExceptionType.HUB_OVERFLOW: {"auto_actions": ["reroute_to_alternate", "increase_processing"]},
ExceptionType.WEATHER_IMPACT: {"auto_actions": ["delay_notification", "route_avoidance"]},
ExceptionType.MILER_STALLED: {"auto_actions": ["reassign", "notify_customer"]},
}
def _init_escalation_rules(self) -> Dict[str, Any]:
return {
ExceptionType.VEHICLE_BREAKDOWN: {"escalate_to": "FLEET_AGENT"},
ExceptionType.HUB_OVERFLOW: {"escalate_to": "HUB_AGENT"},
ExceptionType.FAILED_DELIVERY: {"escalate_to": "DISPATCH_AGENT"},
ExceptionType.DELAY: {"escalate_to": "JARVIS"},
ExceptionType.MILER_STALLED: {"escalate_to": "JARVIS"},
}
async def think(self, context: str, options: List[str] = None) -> str:
return f"[EXCEPTION_AGENT reasoning]: {context}"