implemenation on the ai agents and registry
This commit is contained in:
@@ -213,7 +213,7 @@ def make_handler_agent(redis, decision):
|
||||
box = FakeMessages()
|
||||
agent.send_message = box.send_message
|
||||
calls = []
|
||||
async def fake_decide(context):
|
||||
async def fake_decide(context, model=None):
|
||||
calls.append(context)
|
||||
return decision
|
||||
mod.decide_assignment_failure = fake_decide
|
||||
|
||||
413
tests/test_registry_phase5.py
Normal file
413
tests/test_registry_phase5.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""
|
||||
Tests for Phase 5 of krow_talent_app/docs/agent-platform-plan.md: the engine
|
||||
reads operator settings from the backend's agent registry, records its model
|
||||
decisions, and the six agent-to-agent message bugs are fixed.
|
||||
|
||||
No network, no NATS, no Redis — every external edge is faked.
|
||||
|
||||
Run:
|
||||
python -m unittest discover -s tests
|
||||
"""
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import core.decisions as decisions
|
||||
import core.llm as llm
|
||||
from core.message_bus import message_bus
|
||||
from core.registry import RegistryClient, registry
|
||||
from core.types import AgentMessage, AgentTask, MessageType
|
||||
|
||||
|
||||
SNAPSHOT = {
|
||||
"agents": [
|
||||
{"agentid": "EXCEPTION_AGENT", "autonomous": True, "model": "claude-sonnet-5-5"},
|
||||
{"agentid": "DISPATCH_AGENT", "autonomous": False, "model": ""},
|
||||
],
|
||||
"skills": [
|
||||
{"skillid": "stall_response", "enabled": True,
|
||||
"thresholds": {"stallMinutes": 15, "reassignConfidence": 0.9, "bogus": "x", "flag": True}},
|
||||
{"skillid": "customer_notifications", "enabled": False},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _task(task_type, **data):
|
||||
return AgentTask(task_id="t", agent_type="x", task_type=task_type, data=data)
|
||||
|
||||
|
||||
class _RegistryIsolation:
|
||||
"""Save and restore the shared registry singleton the agents read."""
|
||||
|
||||
def _save_registry(self):
|
||||
self._saved = (registry._agents, registry._skills, registry._etag, registry.loaded)
|
||||
|
||||
def _restore_registry(self):
|
||||
registry._agents, registry._skills, registry._etag, registry.loaded = self._saved
|
||||
|
||||
|
||||
# ── RegistryClient: precedence ───────────────────────────────────────────────
|
||||
|
||||
class TestRegistryPrecedence(unittest.TestCase):
|
||||
def test_env_defaults_before_load(self):
|
||||
r = RegistryClient()
|
||||
self.assertTrue(r.autonomous("EXCEPTION_AGENT", True))
|
||||
self.assertFalse(r.autonomous("EXCEPTION_AGENT", False))
|
||||
self.assertIsNone(r.model("EXCEPTION_AGENT"))
|
||||
self.assertTrue(r.skill_enabled("customer_notifications"))
|
||||
self.assertFalse(r.skill_enabled("anything", default=False))
|
||||
self.assertEqual(r.threshold("stall_response", "stallMinutes", 10), 10)
|
||||
|
||||
def test_registry_wins_once_loaded(self):
|
||||
r = RegistryClient()
|
||||
r.apply(SNAPSHOT)
|
||||
self.assertTrue(r.loaded)
|
||||
self.assertTrue(r.autonomous("EXCEPTION_AGENT", False))
|
||||
self.assertFalse(r.autonomous("DISPATCH_AGENT", True))
|
||||
self.assertEqual(r.model("EXCEPTION_AGENT"), "claude-sonnet-5-5")
|
||||
self.assertIsNone(r.model("DISPATCH_AGENT")) # "" means engine default
|
||||
self.assertFalse(r.skill_enabled("customer_notifications"))
|
||||
self.assertEqual(r.threshold("stall_response", "stallMinutes", 10), 15)
|
||||
self.assertEqual(r.threshold("stall_response", "reassignConfidence", 0.75), 0.9)
|
||||
|
||||
def test_unknown_ids_fall_back(self):
|
||||
r = RegistryClient()
|
||||
r.apply(SNAPSHOT)
|
||||
self.assertTrue(r.autonomous("HUB_AGENT", True))
|
||||
self.assertTrue(r.skill_enabled("not_in_registry"))
|
||||
self.assertEqual(r.threshold("not_in_registry", "k", 3), 3)
|
||||
|
||||
def test_non_numeric_thresholds_ignored(self):
|
||||
r = RegistryClient()
|
||||
r.apply(SNAPSHOT)
|
||||
self.assertEqual(r.threshold("stall_response", "bogus", 7), 7)
|
||||
self.assertEqual(r.threshold("stall_response", "flag", 7), 7) # bool is not a number here
|
||||
|
||||
def test_rows_without_ids_skipped(self):
|
||||
r = RegistryClient()
|
||||
r.apply({"agents": [{"autonomous": True}], "skills": [{"enabled": False}]})
|
||||
self.assertTrue(r.loaded)
|
||||
self.assertEqual(r._agents, {})
|
||||
self.assertEqual(r._skills, {})
|
||||
|
||||
|
||||
# ── RegistryClient: polling ──────────────────────────────────────────────────
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, status, body=None, etag=None, raise_on_json=False):
|
||||
self.status = status
|
||||
self._body = body
|
||||
self.headers = {"ETag": etag} if etag else {}
|
||||
self._raise = raise_on_json
|
||||
|
||||
async def json(self):
|
||||
if self._raise:
|
||||
raise ValueError("bad json")
|
||||
return self._body
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, *responses, exc=None):
|
||||
self._responses = list(responses)
|
||||
self._exc = exc
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, headers=None, timeout=None):
|
||||
self.calls.append((url, dict(headers or {})))
|
||||
if self._exc:
|
||||
raise self._exc
|
||||
return self._responses.pop(0)
|
||||
|
||||
|
||||
class TestRegistryFetch(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_200_then_304_uses_etag(self):
|
||||
r = RegistryClient()
|
||||
s = FakeSession(FakeResp(200, {"data": SNAPSHOT}, etag='"v1"'), FakeResp(304))
|
||||
self.assertEqual(await r.fetch_once(s, "http://api", "k"), "updated")
|
||||
self.assertTrue(r.loaded)
|
||||
self.assertEqual(await r.fetch_once(s, "http://api", "k"), "unchanged")
|
||||
url, first = s.calls[0]
|
||||
self.assertEqual(url, "http://api/api/v1/internal/ai/registry")
|
||||
self.assertEqual(first["X-Internal-Key"], "k")
|
||||
self.assertNotIn("If-None-Match", first)
|
||||
self.assertEqual(s.calls[1][1]["If-None-Match"], '"v1"')
|
||||
|
||||
async def test_refused_and_unavailable_keep_last_copy(self):
|
||||
r = RegistryClient()
|
||||
r.apply(SNAPSHOT)
|
||||
for resp, want in ((FakeResp(401), "refused"), (FakeResp(403), "refused"),
|
||||
(FakeResp(500), "unavailable"), (FakeResp(200, raise_on_json=True), "unavailable")):
|
||||
self.assertEqual(await r.fetch_once(FakeSession(resp), "http://api", "k"), want)
|
||||
self.assertEqual(await r.fetch_once(FakeSession(exc=OSError("down")), "http://api", "k"), "unavailable")
|
||||
# Still the operator's settings, not the env defaults.
|
||||
self.assertTrue(r.autonomous("EXCEPTION_AGENT", False))
|
||||
self.assertFalse(r.skill_enabled("customer_notifications"))
|
||||
|
||||
async def test_run_without_key_returns_at_once(self):
|
||||
r = RegistryClient()
|
||||
await r.run("http://api", "", poll_seconds=0) # must not loop forever
|
||||
self.assertFalse(r.loaded)
|
||||
|
||||
|
||||
# ── Decision log payload ─────────────────────────────────────────────────────
|
||||
|
||||
class TestDecisions(unittest.IsolatedAsyncioTestCase):
|
||||
def test_build_payload_shape(self):
|
||||
p = decisions.build_payload("stall_response", "42", {"eta": 5}, "reassign", 0.87654, "slow", "m")
|
||||
self.assertEqual(p, {
|
||||
"decision_type": "stall_response",
|
||||
"booking_id": 42,
|
||||
"context": {"facts": {"eta": 5}, "model": "m"},
|
||||
"decision": {"action": "reassign", "confidence": 0.877},
|
||||
"reasoning": "slow",
|
||||
})
|
||||
|
||||
def test_bad_booking_ids_omitted(self):
|
||||
for bad in (None, "", "abc", 0, -3):
|
||||
self.assertIsNone(decisions.build_payload("t", bad, None, "a", 0, None, "m")["booking_id"])
|
||||
self.assertEqual(decisions.build_payload("t", 7, None, "a", 0, None, "m")["context"]["facts"], {})
|
||||
|
||||
def test_no_running_loop_returns_none(self):
|
||||
with patch.object(decisions, "INTERNAL_API_KEY", "k"):
|
||||
self.assertIsNone(decisions.record_decision("t", 1, {}, "a", 0.5, "r", "m"))
|
||||
|
||||
async def test_no_key_returns_none(self):
|
||||
with patch.object(decisions, "INTERNAL_API_KEY", ""):
|
||||
self.assertIsNone(decisions.record_decision("t", 1, {}, "a", 0.5, "r", "m"))
|
||||
|
||||
async def test_posts_to_decision_log(self):
|
||||
sent = []
|
||||
|
||||
async def fake_post(url, json=None, headers=None, **kw):
|
||||
sent.append((url, json, headers))
|
||||
|
||||
with patch.object(decisions, "INTERNAL_API_KEY", "k"), \
|
||||
patch.object(decisions, "GO_API_BASE_URL", "http://api"), \
|
||||
patch.object(decisions, "api_post", fake_post):
|
||||
task = decisions.record_decision("assignment_failure", 9, {}, "notify", 0.5, "r", "m")
|
||||
self.assertIsNotNone(task)
|
||||
await task
|
||||
url, body, headers = sent[0]
|
||||
self.assertEqual(url, "http://api/api/v1/internal/agent-decisions")
|
||||
self.assertEqual(body["booking_id"], 9)
|
||||
self.assertEqual(headers, {"X-Internal-Key": "k"})
|
||||
|
||||
|
||||
# ── Model-dependent request params ──────────────────────────────────────────
|
||||
|
||||
class TestRequestParams(unittest.TestCase):
|
||||
def test_default_model_has_thinking_and_effort(self):
|
||||
p = llm.request_params()
|
||||
self.assertEqual(p["model"], llm.LLM_MODEL)
|
||||
self.assertEqual(p["thinking"], {"type": "adaptive"})
|
||||
self.assertIn("effort", p["output_config"])
|
||||
|
||||
def test_override_model_used(self):
|
||||
self.assertEqual(llm.request_params("claude-sonnet-5-5")["model"], "claude-sonnet-5-5")
|
||||
|
||||
def test_haiku_omits_thinking_and_effort(self):
|
||||
p = llm.request_params("claude-haiku-4-5")
|
||||
self.assertNotIn("thinking", p)
|
||||
self.assertNotIn("effort", p["output_config"])
|
||||
|
||||
def test_calls_do_not_share_output_config(self):
|
||||
a, b = llm.request_params(), llm.request_params()
|
||||
a["output_config"]["format"] = {}
|
||||
self.assertNotIn("format", b["output_config"])
|
||||
|
||||
|
||||
# ── ExceptionAgent: registry gates and the LLM-down fallback ────────────────
|
||||
|
||||
class FakeStallMsg:
|
||||
def __init__(self, payload):
|
||||
self.data = json.dumps(payload).encode()
|
||||
|
||||
|
||||
class TestExceptionAgentRegistry(_RegistryIsolation, unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
from agents import exception_agent as mod
|
||||
self.mod = mod
|
||||
self._save_registry()
|
||||
self._orig_decide = mod.decide_stall_response
|
||||
self._orig_record = mod.record_decision
|
||||
|
||||
def tearDown(self):
|
||||
self._restore_registry()
|
||||
self.mod.decide_stall_response = self._orig_decide
|
||||
self.mod.record_decision = self._orig_record
|
||||
|
||||
def _agent(self, decision=None):
|
||||
from agents.exception_agent import ExceptionAgent
|
||||
agent = ExceptionAgent.__new__(ExceptionAgent)
|
||||
agent.acts = []
|
||||
|
||||
async def claim(_b):
|
||||
return True
|
||||
|
||||
async def facts(_m, _b):
|
||||
return {}
|
||||
|
||||
async def reassign(b, reason):
|
||||
agent.acts.append("reassign")
|
||||
return True
|
||||
|
||||
async def notify(b, m):
|
||||
agent.acts.append("notify")
|
||||
return True
|
||||
|
||||
async def escalate(m, b, mins, d):
|
||||
agent.acts.append(("escalate", d.action))
|
||||
|
||||
agent._claim_stall_handled = claim
|
||||
agent._gather_stall_facts = facts
|
||||
agent._reassign = reassign
|
||||
agent._notify_customer = notify
|
||||
agent._escalate_to_human = escalate
|
||||
agent._record_stall_exception = lambda *a, **k: None
|
||||
|
||||
self.models = []
|
||||
self.recorded = []
|
||||
|
||||
async def fake_decide(context, model=None):
|
||||
self.models.append(model)
|
||||
return decision
|
||||
|
||||
self.mod.decide_stall_response = fake_decide
|
||||
self.mod.record_decision = lambda *a: self.recorded.append(a)
|
||||
return agent
|
||||
|
||||
def _load(self, autonomous, enabled=True, confidence=0.75):
|
||||
registry.apply({
|
||||
"agents": [{"agentid": "EXCEPTION_AGENT", "autonomous": autonomous, "model": "claude-sonnet-5-5"}],
|
||||
"skills": [{"skillid": "stall_response", "enabled": enabled,
|
||||
"thresholds": {"reassignConfidence": confidence}}],
|
||||
})
|
||||
|
||||
async def _stall(self, agent):
|
||||
await agent._on_miler_stalled(FakeStallMsg({"miler_id": "m1", "booking_id": "5", "minutes_stalled": 12}))
|
||||
|
||||
async def test_llm_down_not_autonomous_escalates(self):
|
||||
self._load(autonomous=False)
|
||||
agent = self._agent(decision=None)
|
||||
await self._stall(agent)
|
||||
self.assertEqual(agent.acts, [("escalate", "escalate")])
|
||||
|
||||
async def test_llm_down_autonomous_reassigns(self):
|
||||
self._load(autonomous=True)
|
||||
agent = self._agent(decision=None)
|
||||
await self._stall(agent)
|
||||
self.assertEqual(agent.acts, ["reassign", "notify"])
|
||||
|
||||
async def test_skill_disabled_does_nothing(self):
|
||||
self._load(autonomous=True, enabled=False)
|
||||
agent = self._agent(decision=None)
|
||||
await self._stall(agent)
|
||||
self.assertEqual(agent.acts, [])
|
||||
self.assertEqual(self.models, []) # the model was never asked
|
||||
|
||||
async def test_model_pin_and_decision_recorded(self):
|
||||
self._load(autonomous=True, confidence=0.95)
|
||||
agent = self._agent(decision=llm.StallDecision("reassign", "stuck", 0.9))
|
||||
await self._stall(agent)
|
||||
self.assertEqual(self.models, ["claude-sonnet-5-5"])
|
||||
# 0.9 is below the registry's 0.95, so the reassign goes to a human.
|
||||
self.assertEqual(agent.acts, [("escalate", "reassign")])
|
||||
self.assertEqual(self.recorded[0][0], "stall_response")
|
||||
self.assertEqual(self.recorded[0][-1], "claude-sonnet-5-5")
|
||||
|
||||
async def test_stall_minutes_from_registry(self):
|
||||
registry.loaded = False
|
||||
self.assertEqual(self.mod.stall_minutes(), self.mod.STALL_MINUTES)
|
||||
registry.apply({"skills": [{"skillid": "stall_response", "enabled": True, "thresholds": {"stallMinutes": 20}}]})
|
||||
self.assertEqual(self.mod.stall_minutes(), 20)
|
||||
|
||||
|
||||
# ── CustomerAgent: skill gate and recorded events ───────────────────────────
|
||||
|
||||
class TestCustomerAgent(_RegistryIsolation, unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
self._save_registry()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self._restore_registry()
|
||||
message_bus.unregister_agent("CUSTOMER_AGENT")
|
||||
|
||||
async def test_notifications_disabled_skips_send(self):
|
||||
from agents.customer_agent import CustomerAgent
|
||||
registry.apply({"skills": [{"skillid": "customer_notifications", "enabled": False}]})
|
||||
result = await CustomerAgent()._send_notification(_task("send_notification", order_id="O1"))
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
|
||||
async def test_cancel_and_notification_events_recorded(self):
|
||||
from agents.customer_agent import CustomerAgent
|
||||
agent = CustomerAgent()
|
||||
for mt in (MessageType.ORDER_CANCELLED, MessageType.NOTIFICATION_SENT, MessageType.HEARTBEAT):
|
||||
await agent.handle_message(AgentMessage("id", "EXCEPTION_AGENT", "CUSTOMER_AGENT", mt,
|
||||
{"order_id": "O9"}, datetime.now()))
|
||||
self.assertEqual([e["type"] for e in agent._received_events], ["ORDER_CANCELLED", "NOTIFICATION_SENT"])
|
||||
self.assertEqual(agent._received_events[0]["order_id"], "O9")
|
||||
|
||||
|
||||
# ── Message-bug fixes ────────────────────────────────────────────────────────
|
||||
|
||||
class TestMessageFixes(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncTearDown(self):
|
||||
for agent_id in ("FLEET_AGENT", "ORDER_AGENT", "JARVIS", "FAKE_ORDER", "FAKE_CUSTOMER"):
|
||||
message_bus.unregister_agent(agent_id)
|
||||
|
||||
async def test_fleet_release_for_cancel_finds_vehicle_by_order(self):
|
||||
from agents.fleet_agent import FleetAgent, VehicleAssignment
|
||||
fleet = FleetAgent()
|
||||
vehicle_id = next(iter(fleet._vehicles))
|
||||
fleet._vehicles[vehicle_id]["status"] = "in_transit"
|
||||
fleet._assignments["A1"] = VehicleAssignment(
|
||||
"A1", vehicle_id, "R1", ["O1", "O2"], datetime.now(), datetime.now(), {})
|
||||
|
||||
missing = await fleet.handle_task(_task("release_vehicle_for_cancel", order_id="NOPE"))
|
||||
self.assertEqual(missing["status"], "no_assignment")
|
||||
|
||||
await fleet.handle_task(_task("release_vehicle_for_cancel", order_id="O2"))
|
||||
self.assertEqual(fleet._vehicles[vehicle_id]["status"], "available")
|
||||
self.assertEqual(fleet._assignments["A1"].status, "returned")
|
||||
|
||||
async def test_order_agent_refuses_backend_calls(self):
|
||||
from agents.order_agent import OrderAgent
|
||||
agent = OrderAgent()
|
||||
with patch("core.http_client.api_post") as post, patch("core.http_client.api_get") as get:
|
||||
self.assertIsNone(await agent._api_post("/api/v1/admin/crmbooking", {}))
|
||||
self.assertIsNone(await agent._api_get("/api/v1/admin/x"))
|
||||
self.assertIsNone(await agent._api_patch("/api/v1/admin/x", {}))
|
||||
post.assert_not_called()
|
||||
get.assert_not_called()
|
||||
|
||||
async def test_jarvis_validate_payload_carries_order_id(self):
|
||||
from core.agent import MasterAgent
|
||||
|
||||
class Sink:
|
||||
def __init__(self, agent_id):
|
||||
self.agent_id = agent_id
|
||||
self.tasks = []
|
||||
|
||||
async def submit_task(self, task):
|
||||
self.tasks.append(task)
|
||||
|
||||
jarvis = MasterAgent()
|
||||
order, customer = Sink("ORDER_AGENT"), Sink("CUSTOMER_AGENT")
|
||||
jarvis._sub_agents = {"ORDER_AGENT": order, "CUSTOMER_AGENT": customer}
|
||||
await jarvis.handle_task(_task("orchestrate_order", order={"order_id": "O7"}))
|
||||
self.assertEqual(order.tasks[0].task_type, "validate_order")
|
||||
self.assertEqual(order.tasks[0].data["order_id"], "O7")
|
||||
|
||||
def test_order_status_update_message_type_exists(self):
|
||||
self.assertEqual(MessageType.ORDER_STATUS_UPDATE.value, "ORDER_STATUS_UPDATE")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user