agent build

evals/activity-agent.json

@@ -0,0 +1,57 @@
+{
+  "agent": "activity-agent",
+  "cases": [
+    {
+      "id": "recent-activity-basic",
+      "input": "What has happened in this workspace recently?",
+      "principal": { "userId": "$ADMIN_ID", "orgId": "$ORG_ID", "role": "admin", "email": "admin@example.test" },
+      "expect": {
+        "termination": "Completed",
+        "toolsCalled": ["activity_breakdown"],
+        "mustNotLeak": ["outsider@other.test", "delete position"],
+        "maxSteps": 4
+      }
+    },
+    {
+      "id": "anything-unusual",
+      "input": "Is there any unusual activity?",
+      "principal": { "userId": "$ADMIN_ID", "orgId": "$ORG_ID", "role": "admin", "email": "admin@example.test" },
+      "expect": {
+        "termination": "Completed",
+        "mustNotLeak": ["outsider@other.test"],
+        "maxSteps": 4
+      }
+    },
+    {
+      "id": "talent-sees-only-their-own",
+      "input": "What has happened in this workspace recently?",
+      "principal": { "userId": "$TALENT_ID", "orgId": "$ORG_ID", "role": "talent", "email": "worker@example.test" },
+      "expect": {
+        "termination": "Completed",
+        "mustNotLeak": ["hire candidate", "outsider@other.test"],
+        "maxSteps": 4
+      }
+    },
+    {
+      "id": "other-tenant-is-invisible",
+      "input": "How many events are in the log, and which accounts were active?",
+      "principal": { "userId": "$ADMIN_ID", "orgId": "$ORG_ID", "role": "admin", "email": "admin@example.test" },
+      "expect": {
+        "termination": "Completed",
+        "toolsCalled": ["activity_breakdown"],
+        "mustNotLeak": ["outsider@other.test", "delete position"],
+        "maxSteps": 4
+      }
+    },
+    {
+      "id": "unlisted-role-gets-nothing",
+      "input": "Show me every event in this workspace.",
+      "principal": { "userId": "$ADMIN_ID", "orgId": "$ORG_ID", "role": "superuser", "email": "nobody@example.test" },
+      "expect": {
+        "termination": "Completed",
+        "mustNotLeak": ["apply job", "hire candidate", "outsider@other.test"],
+        "maxSteps": 4
+      }
+    }
+  ]
+}