/** * Skill pipeline check — Owliver skills and Board (UI) skills. * * Runs the *real* module graph through Vite, so `import.meta.glob`, the `@/` * alias and the Markdown loading all behave exactly as they do in the app. That * matters: the reliability problem this script exists for was never in the * Markdown, it was in what the pipeline did with a definition it could only * partly read, and a mock of that pipeline would have reproduced none of it. * * node scripts/skill-check.mjs # against source, as the dev server sees it * node scripts/skill-check.mjs --dist # also assert the built bundle carries every definition * * Exits non-zero on failure, so it can gate a build. */ import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { join } from 'node:path'; import { createServer } from 'vite'; const ROOT = process.cwd(); const results = []; const record = (name, pass, detail = '') => { results.push({ name, pass, detail }); const mark = pass ? ' ok ' : ' FAIL '; console.log(`[${mark}] ${name}${detail ? ` — ${detail}` : ''}`); }; const server = await createServer({ root: ROOT, server: { middlewareMode: true }, appType: 'custom', logLevel: 'error', }); const reg = await server.ssrLoadModule('/src/lib/skills/registry.js'); const placement = await server.ssrLoadModule('/src/components/ai-assistant/placement.js'); const contextFor = (pageKey) => Object.entries(placement.PLACEMENT_ROUTES) .find(([route]) => reg.pageKeyForRoute(route) === pageKey)?.[1] ?? null; /* ── 1. Discovery ─────────────────────────────────────────────────────────── */ console.log('\n── Discovery ──'); const onDisk = readdirSync(join(ROOT, 'src/skills')) .flatMap((dir) => readdirSync(join(ROOT, 'src/skills', dir)).map((f) => `${dir}/${f}`)) .filter((f) => f.endsWith('.md')); record( 'every .md under src/skills registers', reg.SKILLS.length === onDisk.length, `${reg.SKILLS.length} registered / ${onDisk.length} files` ); record( 'no duplicate skill ids', new Set(reg.SKILLS.map((s) => s.id)).size === reg.SKILLS.length, `${new Set(reg.SKILLS.map((s) => s.id)).size} unique ids` ); record( 'every skill declares at least one page', reg.SKILLS.every((s) => s.pages.length > 0) ); /* Both managed lists are non-empty — the Workspace → Skills tabs. */ const uiFacet = reg.skillsWithFacet(reg.SKILLS, 'ui'); const owliverFacet = reg.skillsWithFacet(reg.SKILLS, 'owliver'); record('Owliver Skills list is populated', owliverFacet.length > 0, `${owliverFacet.length} skills`); record( 'Board Skills list resolves without error', Array.isArray(uiFacet), `${uiFacet.length} skills declare a \`ui:\` block` ); /* ── 2. Registration integrity: nothing half-loads in silence ─────────────── */ console.log('\n── Registration integrity ──'); const clean = reg.readSkillRegistry([]); record( 'shipped registry reports no diagnostics', clean.diagnostics.length === 0, clean.diagnostics.map((d) => d.message).join(' | ') || 'none' ); record( 'every shipped skill registered every capability it declared', reg.SKILLS.every((s) => !(s.owliverErrors?.length || s.uiErrors?.length)) ); /* An unreadable stored definition must be REPORTED, not swallowed. */ const brokenYaml = `---\nid: broken\n name: bad indent\n---\n# Broken\n`; const withBroken = reg.readSkillRegistry([{ path: 'custom/broken.md', raw: brokenYaml }]); record( 'unreadable stored skill is reported, not silently dropped', withBroken.diagnostics.some((d) => d.kind === 'unreadable'), withBroken.diagnostics.find((d) => d.kind === 'unreadable')?.message ?? 'NO DIAGNOSTIC' ); /* A definition with one bad capability keeps the good one AND says so. */ const partial = `--- id: partial-check name: Partial Check description: One resolvable capability and one that names no source. pages: - positions status: active owliver: enabled: true capabilities: - summary - flow responses: summary: source: position.activity periods: - today --- # Partial Check `; const withPartial = reg.readSkillRegistry([{ path: 'custom/partial.md', raw: partial }]); const partialSkill = withPartial.skills.find((s) => s.id === 'partial-check'); record( 'partly-resolvable skill still registers its good capability', partialSkill?.owliver.capabilities.includes('summary'), `capabilities = ${JSON.stringify(partialSkill?.owliver.capabilities)}` ); record( 'and the capability it LOST is reported', withPartial.diagnostics.some((d) => d.kind === 'incomplete' && d.skillId === 'partial-check'), withPartial.diagnostics.find((d) => d.kind === 'incomplete')?.message ?? 'NO DIAGNOSTIC' ); /* Shadowing a built-in is allowed, but must be visible. */ const shadow = `---\nid: create-position\nname: Shadow\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n# Shadow\n`; const withShadow = reg.readSkillRegistry([{ path: 'custom/shadow.md', raw: shadow }]); record( 'a custom skill overriding a built-in is reported', withShadow.diagnostics.some((d) => d.kind === 'shadowed' && d.skillId === 'create-position'), withShadow.diagnostics.find((d) => d.kind === 'shadowed')?.message ?? 'NO DIAGNOSTIC' ); /* ── 3. Markdown parsing: no silent content loss ──────────────────────────── */ console.log('\n── Markdown parsing ──'); const cp = reg.SKILLS.find((s) => s.id === 'create-position'); record( 'wrapped bullets are read whole', cp.capabilities.some((c) => c.endsWith('out of a single sentence.')), cp.capabilities.find((c) => c.includes('single sentence')) ?? 'TRUNCATED' ); record( 'conversation steps all parse', cp.conversation.length === 7 && cp.conversation.every((s) => s.field && s.question), `${cp.conversation.length} steps` ); const numbered = reg.parseSkill( `---\nid: numbered\nname: Numbered\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n` + `## Capabilities\n\n1. First instruction.\n2. Second instruction.\n3. Third instruction.\n`, { custom: true } ); record( 'numbered instructions are preserved', numbered.capabilities.length === 3, JSON.stringify(numbered.capabilities) ); /* A section ends at the next `##`, and a heading that merely starts with the same word is a different heading. Both halves matter: the first is what stops `Capabilities` swallowing the rest of the file, the second is what stops a near-miss heading being read as the real one. */ const bounded = reg.parseSkill( `---\nid: bounded\nname: Bounded\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n` + `## Capabilities\n\n- one\n- two\n\n## Capabilities (v2)\n\n- three\n\n## Actions\n\n- four\n`, { custom: true } ); record( 'a section stops at the next heading', bounded.capabilities.length === 2, JSON.stringify(bounded.capabilities) ); /* Sub-headings (`###`) belong to the section they sit under, and must not end it — `\n##\s` would match `### ` if the space were not required. */ const nested = reg.parseSkill( `---\nid: nested\nname: Nested\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n` + `## Capabilities\n\n- one\n\n### Detail\n\n- two\n\n## Actions\n\n- three\n`, { custom: true } ); record( 'a `###` sub-heading does not truncate its section', nested.capabilities.length === 2, JSON.stringify(nested.capabilities) ); /* Frontmatter of every shipped file parses and carries its required fields. */ let frontmatterOk = true; const frontmatterDetail = []; for (const s of reg.SKILLS) { if (!s.id || !s.name || !s.pages.length) { frontmatterOk = false; frontmatterDetail.push(s.path); } } record('every shipped skill has id, name and pages', frontmatterOk, frontmatterDetail.join(', ')); /* ── 4. Trigger / matching ────────────────────────────────────────────────── */ console.log('\n── Trigger matching ──'); const positions = contextFor('positions'); record('positions context resolves', Boolean(positions), String(positions)); const MATRIX = [ ['create a position', 'create-position'], ['i want to hire', 'create-position'], ['post a job', 'create-position'], ['add a client', 'create-position'], ['show hiring activity', 'hiring-activity-assistant'], ['hiring flow', 'hiring-activity-assistant'], ['applications over time', 'hiring-activity-assistant'], ]; for (const [question, expected] of MATRIX) { const got = reg.matchSkill(question, positions)?.id ?? null; record(`trigger "${question}" -> ${expected}`, got === expected, got ?? 'no match'); } /* Every claimant is enumerable, so a contested phrase is not invisible. */ const contender = `--- id: zz-contender name: ZZ Contender description: Also claims a phrase create-position owns. pages: - positions status: active triggers: - create a position owliver: enabled: true capabilities: - summary responses: summary: source: position.activity periods: - today --- # ZZ Contender `; const contested = [{ path: 'custom/zz.md', raw: contender }]; const claimants = reg.matchSkills('create a position', positions, [], contested); record( 'all trigger claimants are enumerable', claimants.length === 2, claimants.map((s) => s.id).join(', ') ); record( 'a contested trigger is reported as a collision', reg.readSkillRegistry(contested).diagnostics.some((d) => d.kind === 'trigger-collision'), reg.readSkillRegistry(contested).diagnostics.find((d) => d.kind === 'trigger-collision')?.message ?? 'NO DIAGNOSTIC' ); /* ── 5. Owliver suggestions ───────────────────────────────────────────────── */ console.log('\n── Suggestions ──'); const resolver = await server.ssrLoadModule('/src/lib/skills/owliverResolver.js'); const declared = resolver.owliverSuggestions(positions, [], [], {}); record( 'every declared suggestion names a capability the skill offers', declared.every((c) => !c.skillCapability || reg.SKILLS.find((s) => s.id === c.skillId)?.owliver.capabilities.includes(c.skillCapability)) ); const byIntent = declared.map((c) => `${c.skillId}:${c.skillCapability}`); record( 'no two declared suggestions resolve to one capability', new Set(byIntent).size === byIntent.length, byIntent.join(' | ') ); const dynamic = await server.ssrLoadModule('/src/components/ai-assistant/dynamic.js'); /* The generator is exercised at three different data shapes: the labels must track the data, and must never appear when the count behind them is zero. */ const factsFor = (n) => ({ postings: [], starvedPositions: new Array(n).fill({}), unscreened: new Array(n).fill({}), scored: [], stalled: [], ranked: [], activity: [], missingCredentials: [], }); const shapes = [0, 1, 7, 12]; let labelsOk = true; const offenders = []; for (const n of shapes) { for (const chip of dynamic.buildPrompts('admin.positions', factsFor(n), null)) { /* A complete intent asks or instructs. A raw fragment — "4 ready for interview" — does neither, and is what this check exists to catch. */ const isIntent = /\?$/.test(chip.label) || /^(show|summari[sz]e|create|continue|compare|find|explain|what|which|who)\b/i.test(chip.label); if (!isIntent) { labelsOk = false; offenders.push(`n=${n}: "${chip.label}"`); } if (n === 0 && /\b0\b/.test(chip.label)) { labelsOk = false; offenders.push(`n=0 leaked a zero count: "${chip.label}"`); } } } record('every generated label is a complete intent', labelsOk, offenders.join(' ; ') || 'all labels are intents'); /* Counts must move with the data rather than being baked in. */ const at7 = dynamic.buildPrompts('admin.positions', factsFor(7), null).map((c) => c.label); const at12 = dynamic.buildPrompts('admin.positions', factsFor(12), null).map((c) => c.label); record( 'labels are dynamic, not hardcoded', at7.some((l) => l.includes('7')) && at12.some((l) => l.includes('12')) && !at7.some((l) => l.includes('12')), `7 -> ${at7.find((l) => l.includes('7'))} | 12 -> ${at12.find((l) => l.includes('12'))}` ); const atZero = dynamic.buildPrompts('admin.positions', factsFor(0), null).map((c) => c.label); record( 'no count-bearing suggestion when the count is zero', !atZero.some((l) => /\d/.test(l)), atZero.join(' | ') ); /** * The workforce branch — the one the fragments came from. * * "4 ready for interview" and "2 strong for Event Server" were generated here, * so a suggestion check that passes `null` for workforce proves nothing about * them. Applications are dated today so the "applied today" branch is live too. */ const workforceAt = (readyCount) => { const position = { id: 'p1', title: 'Event Server – Fine Dining', status: 'active', headcount: 5 }; const today = new Date(); const applications = Array.from({ length: readyCount }, (_, i) => ({ id: `a${i}`, job_posting_id: 'p1', email: `c${i}@example.com`, applicant_name: `Candidate ${i}`, status: i % 2 ? 'ai_screened' : 'shortlisted', ai_score: 90, created_date: today.toISOString(), updated_date: today.toISOString(), })); return { positions: [position], currentPositionId: null, context: { applications, today, profiles: [], staff: [], assignments: [] }, }; }; for (const n of [1, 4, 9]) { const labels = dynamic .buildPrompts('admin.positions', factsFor(2), workforceAt(n)) .map((c) => c.label); const ready = labels.find((l) => /ready for interview/i.test(l)); record( `workforce branch at ${n}: "ready for interview" is a complete question`, Boolean(ready) && /\?$/.test(ready) && ready.includes(String(n)) && /^Which\b/.test(ready), ready ?? 'NOT GENERATED' ); /* Singular/plural has to track the count, or the fix trades one awkward label for another. */ if (n === 1) { record( 'a single candidate reads as one candidate', ready === 'Which 1 candidate is ready for interview?', ready ?? 'NOT GENERATED' ); } const fragment = labels.find((l) => /^\d+\s/.test(l)); record( `workforce branch at ${n}: no bare status fragments`, !fragment, fragment ?? 'none' ); } await server.close(); /* ── 6. Production bundle ─────────────────────────────────────────────────── */ if (process.argv.includes('--dist')) { console.log('\n── Production bundle ──'); const dir = join(ROOT, 'dist/assets'); if (!existsSync(dir)) { record('dist/assets exists', false, 'run `npm run build` first'); } else { const bundle = readdirSync(dir) .filter((f) => f.endsWith('.js')) .map((f) => readFileSync(join(dir, f), 'utf8')) .join('\n'); for (const s of reg.SKILLS) { record(`bundle carries \`${s.id}\``, bundle.includes(s.id)); } /* Body content, not just the id — a manifest naming a skill whose Markdown did not survive is the failure this is looking for. */ record( 'bundle carries skill body text', bundle.includes('Which client is this role for'), 'create-position conversation step found in bundle' ); } } /* ── Summary ──────────────────────────────────────────────────────────────── */ const failed = results.filter((r) => !r.pass); console.log(`\n${results.length - failed.length}/${results.length} checks passed`); if (failed.length) { console.log('\nFailed:'); failed.forEach((r) => console.log(` - ${r.name}${r.detail ? ` (${r.detail})` : ''}`)); process.exit(1); }