updates on the ai agents and time series prediction and updates on the api to
This commit is contained in:
287
controllers/aiFindingController.go
Normal file
287
controllers/aiFindingController.go
Normal file
@@ -0,0 +1,287 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
||||
"doormile/db"
|
||||
"doormile/models"
|
||||
"doormile/utils"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// Skill findings: the console's rule output, made durable.
|
||||
//
|
||||
// See models/ai_findings.go for why this is neither an exception nor a decision.
|
||||
// The short version: the eight ops skills run in the browser and threw their
|
||||
// output away, so nobody could answer whether a skill was useful, whether a
|
||||
// finding had been seen before, or whether acting on one actually cleared it.
|
||||
|
||||
// POST /api/v1/admin/ai/findings
|
||||
//
|
||||
// Idempotent on fingerprint. The console re-evaluates every 60 seconds and will
|
||||
// re-send an unchanged finding each time; this bumps lastseenat and seencount
|
||||
// rather than inserting a duplicate. Without that, the table would grow by the
|
||||
// number of open findings per minute per operator with the Exceptions page
|
||||
// open — and "how long has this been open" would be unanswerable, which is the
|
||||
// one signal that distinguishes an ignored finding from a new one.
|
||||
func UpsertAIFindings(c *fiber.Ctx) error {
|
||||
type item struct {
|
||||
Skillid string `json:"skillid"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Severity string `json:"severity"`
|
||||
Title string `json:"title"`
|
||||
Proposaltool string `json:"proposaltool"`
|
||||
Bookingids []int `json:"bookingids"`
|
||||
Tenantid *int `json:"tenantid"`
|
||||
}
|
||||
var body struct {
|
||||
Findings []item `json:"findings"`
|
||||
// Cleared carries the fingerprints a scan did NOT raise this time.
|
||||
// Sent by the console because only it knows the full set it evaluated:
|
||||
// the backend cannot distinguish "resolved" from "the operator closed
|
||||
// the tab" on its own.
|
||||
Cleared []string `json:"cleared"`
|
||||
}
|
||||
if err := c.BodyParser(&body); err != nil {
|
||||
return utils.BadRequest(c, "invalid request body")
|
||||
}
|
||||
|
||||
now := utils.DBNow()
|
||||
written, skipped := 0, 0
|
||||
|
||||
for _, f := range body.Findings {
|
||||
if f.Skillid == "" || f.Fingerprint == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
scopeJSON, err := json.Marshal(f.Bookingids)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
row := models.AISkillFinding{
|
||||
Skillid: f.Skillid,
|
||||
Fingerprint: f.Fingerprint,
|
||||
Severity: f.Severity,
|
||||
Title: f.Title,
|
||||
Proposaltool: f.Proposaltool,
|
||||
Bookingcount: len(f.Bookingids),
|
||||
Scope: string(scopeJSON),
|
||||
Tenantid: f.Tenantid,
|
||||
Firstseenat: now,
|
||||
Lastseenat: now,
|
||||
Seencount: 1,
|
||||
}
|
||||
|
||||
// ON CONFLICT on the fingerprint: bump the sighting, leave firstseenat
|
||||
// alone. seencount uses a SQL expression rather than a read-modify-write
|
||||
// so two operators with the page open cannot lose each other's bump.
|
||||
//
|
||||
// clearedat is reset to NULL: a finding that cleared and came back is
|
||||
// open again, and leaving the old timestamp would make it look resolved
|
||||
// while it is being re-raised.
|
||||
if err := db.DB.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "fingerprint"}},
|
||||
DoUpdates: clause.Assignments(map[string]interface{}{
|
||||
"lastseenat": now,
|
||||
"seencount": gorm.Expr("aiskillfindings.seencount + 1"),
|
||||
"severity": f.Severity,
|
||||
"bookingcount": len(f.Bookingids),
|
||||
"scope": string(scopeJSON),
|
||||
"clearedat": nil,
|
||||
}),
|
||||
}).Create(&row).Error; err != nil {
|
||||
utils.Warn("UpsertAIFindings: upsert failed", "fingerprint", f.Fingerprint, "error", err)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
written++
|
||||
}
|
||||
|
||||
cleared := 0
|
||||
if len(body.Cleared) > 0 {
|
||||
// Only clear what is still open. Re-clearing an already-cleared row
|
||||
// would move its clearedat forward on every poll and destroy the
|
||||
// "acting cleared it in 4 minutes" measurement.
|
||||
res := db.DB.Model(&models.AISkillFinding{}).
|
||||
Where("fingerprint IN ? AND clearedat IS NULL", body.Cleared).
|
||||
Update("clearedat", now)
|
||||
if res.Error != nil {
|
||||
utils.Warn("UpsertAIFindings: clearing failed", "error", res.Error)
|
||||
} else {
|
||||
cleared = int(res.RowsAffected)
|
||||
}
|
||||
}
|
||||
|
||||
return utils.OK(c, fiber.Map{"written": written, "skipped": skipped, "cleared": cleared})
|
||||
}
|
||||
|
||||
// POST /api/v1/admin/ai/findings/:fingerprint/acted
|
||||
//
|
||||
// Records that an operator carried out a finding's proposal, and how it went.
|
||||
// Separate from the upsert because it is a different event with a different
|
||||
// actor: the upsert is a scan reporting what it sees, this is a person doing
|
||||
// something. Collapsing them would make "nobody acted" indistinguishable from
|
||||
// "the scan has not run since".
|
||||
func RecordAIFindingActed(c *fiber.Ctx) error {
|
||||
// Fiber's c.Params returns the RAW path segment, still percent-encoded.
|
||||
// A fingerprint is "skill:tool:1,2,3", so the console necessarily sends it
|
||||
// through encodeURIComponent and the ':' and ',' arrive as %3A and %2C.
|
||||
// Matching the raw string against the stored one therefore never hits, and
|
||||
// every acted-report 404s — silently, because the console treats this call
|
||||
// as fire-and-forget.
|
||||
//
|
||||
// Found by running the real backend; a mock that echoed the path back
|
||||
// agreed with the assumption and proved nothing.
|
||||
fingerprint, err := url.PathUnescape(c.Params("fingerprint"))
|
||||
if err != nil {
|
||||
return utils.BadRequest(c, "invalid fingerprint")
|
||||
}
|
||||
if fingerprint == "" {
|
||||
return utils.BadRequest(c, "fingerprint is required")
|
||||
}
|
||||
var body struct {
|
||||
Result string `json:"result"` // ok, partial, failed
|
||||
}
|
||||
if err := c.BodyParser(&body); err != nil {
|
||||
return utils.BadRequest(c, "invalid request body")
|
||||
}
|
||||
switch body.Result {
|
||||
case "ok", "partial", "failed":
|
||||
default:
|
||||
// A partial success is a partial success — the console reports six
|
||||
// riders notified out of eight that way, and flattening it to "ok"
|
||||
// here would lose exactly the distinction the executors preserve.
|
||||
return utils.BadRequest(c, "result must be one of: ok, partial, failed")
|
||||
}
|
||||
|
||||
actor, _ := c.Locals("userid").(int)
|
||||
now := utils.DBNow()
|
||||
|
||||
res := db.DB.Model(&models.AISkillFinding{}).
|
||||
Where("fingerprint = ?", fingerprint).
|
||||
Updates(map[string]interface{}{
|
||||
"actedat": now,
|
||||
"actedby": actor,
|
||||
"actionresult": body.Result,
|
||||
})
|
||||
if res.Error != nil {
|
||||
utils.Error("RecordAIFindingActed: update failed", "error", res.Error)
|
||||
return utils.Internal(c, "failed to record the action")
|
||||
}
|
||||
if res.RowsAffected == 0 {
|
||||
return utils.NotFound(c, "finding not found")
|
||||
}
|
||||
return utils.OK(c, fiber.Map{"fingerprint": fingerprint, "result": body.Result})
|
||||
}
|
||||
|
||||
// GET /api/v1/admin/ai/findings
|
||||
//
|
||||
// What the skills have been noticing. Read-only; Doormile staff only, like the
|
||||
// rest of the /admin/ai surface.
|
||||
//
|
||||
// Defaults to open findings (clearedat IS NULL) because that is the operational
|
||||
// question. `?days=N` switches to everything in a window, which is the
|
||||
// measurement question — and those are different enough that one default cannot
|
||||
// serve both.
|
||||
func GetAIFindings(c *fiber.Ctx) error {
|
||||
limit, err := strconv.Atoi(c.Query("limit", "100"))
|
||||
if err != nil || limit < 1 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
q := db.DB.Model(&models.AISkillFinding{})
|
||||
if days, err := strconv.Atoi(c.Query("days", "0")); err == nil && days > 0 {
|
||||
if days > 90 {
|
||||
days = 90
|
||||
}
|
||||
q = q.Where("firstseenat >= ?", utils.DBNow().AddDate(0, 0, -days))
|
||||
} else {
|
||||
q = q.Where("clearedat IS NULL")
|
||||
}
|
||||
if skill := c.Query("skill"); skill != "" {
|
||||
q = q.Where("skillid = ?", skill)
|
||||
}
|
||||
|
||||
var rows []models.AISkillFinding
|
||||
if err := q.Order("lastseenat DESC").Limit(limit).Find(&rows).Error; err != nil {
|
||||
utils.Error("GetAIFindings: query failed", "error", err)
|
||||
return utils.Internal(c, "failed to read findings")
|
||||
}
|
||||
return utils.List(c, rows, int64(len(rows)))
|
||||
}
|
||||
|
||||
// GET /api/v1/admin/ai/findings/stats
|
||||
//
|
||||
// Per skill, over a window: how often it fires, how long its findings stay
|
||||
// open, how often anyone acts, and whether acting cleared them.
|
||||
//
|
||||
// This is the point of the table. "Acted and cleared" versus "cleared on its
|
||||
// own" is what separates a skill that helps from one that narrates — and a
|
||||
// skill whose findings always clear untouched is proposing work that did not
|
||||
// need doing.
|
||||
func GetAIFindingStats(c *fiber.Ctx) error {
|
||||
days, err := strconv.Atoi(c.Query("days", "30"))
|
||||
if err != nil || days < 1 || days > 90 {
|
||||
days = 30
|
||||
}
|
||||
since := utils.DBNow().AddDate(0, 0, -days)
|
||||
|
||||
type stat struct {
|
||||
Skillid string `json:"skillid"`
|
||||
Findings int64 `json:"findings"`
|
||||
Stillopen int64 `json:"stillopen"`
|
||||
Actedon int64 `json:"actedon"`
|
||||
Clearedafteract int64 `json:"clearedafteract"`
|
||||
Clearedunacted int64 `json:"clearedunacted"`
|
||||
Avgopenminutes *float64 `json:"avgopenminutes"`
|
||||
}
|
||||
var rows []stat
|
||||
|
||||
// EXTRACT over (clearedat - firstseenat): both are written with
|
||||
// utils.DBNow, so they share a tagging and their difference is correct
|
||||
// regardless of the IST-digits-labelled-UTC convention.
|
||||
if err := db.DB.Raw(`
|
||||
SELECT skillid,
|
||||
count(*) AS findings,
|
||||
count(*) FILTER (WHERE clearedat IS NULL) AS stillopen,
|
||||
count(*) FILTER (WHERE actedat IS NOT NULL) AS actedon,
|
||||
count(*) FILTER (WHERE actedat IS NOT NULL AND clearedat IS NOT NULL) AS clearedafteract,
|
||||
count(*) FILTER (WHERE actedat IS NULL AND clearedat IS NOT NULL) AS clearedunacted,
|
||||
avg(EXTRACT(EPOCH FROM (clearedat - firstseenat)) / 60.0)
|
||||
FILTER (WHERE clearedat IS NOT NULL) AS avgopenminutes
|
||||
FROM aiskillfindings
|
||||
WHERE firstseenat >= ?
|
||||
GROUP BY skillid
|
||||
ORDER BY findings DESC`, since).Scan(&rows).Error; err != nil {
|
||||
utils.Error("GetAIFindingStats: query failed", "error", err)
|
||||
return utils.Internal(c, "failed to read finding stats")
|
||||
}
|
||||
|
||||
return utils.OK(c, fiber.Map{
|
||||
"days": days,
|
||||
"since": since,
|
||||
"skills": rows,
|
||||
})
|
||||
}
|
||||
|
||||
// PruneAIFindings drops findings past the retention window. Called from the
|
||||
// outcome sweeper's tick rather than having its own timer — one more table to
|
||||
// keep tidy, not one more goroutine.
|
||||
func PruneAIFindings(retentionDays int) (int, error) {
|
||||
if db.DB == nil || retentionDays <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
cutoff := utils.DBNow().AddDate(0, 0, -retentionDays)
|
||||
res := db.DB.Where("firstseenat < ?", cutoff).Delete(&models.AISkillFinding{})
|
||||
if res.Error != nil {
|
||||
return 0, res.Error
|
||||
}
|
||||
return int(res.RowsAffected), nil
|
||||
}
|
||||
Reference in New Issue
Block a user