413 lines
16 KiB
JavaScript
413 lines
16 KiB
JavaScript
/**
|
||
* Skill pipeline check — Owliver skills and Board (UI) skills.
|
||
*
|
||
* Runs the *real* module graph through Vite, so `import.meta.glob`, the `@/`
|
||
* alias and the Markdown loading all behave exactly as they do in the app. That
|
||
* matters: the reliability problem this script exists for was never in the
|
||
* Markdown, it was in what the pipeline did with a definition it could only
|
||
* partly read, and a mock of that pipeline would have reproduced none of it.
|
||
*
|
||
* node scripts/skill-check.mjs # against source, as the dev server sees it
|
||
* node scripts/skill-check.mjs --dist # also assert the built bundle carries every definition
|
||
*
|
||
* Exits non-zero on failure, so it can gate a build.
|
||
*/
|
||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||
import { join } from 'node:path';
|
||
import { createServer } from 'vite';
|
||
|
||
const ROOT = process.cwd();
|
||
const results = [];
|
||
const record = (name, pass, detail = '') => {
|
||
results.push({ name, pass, detail });
|
||
const mark = pass ? ' ok ' : ' FAIL ';
|
||
console.log(`[${mark}] ${name}${detail ? ` — ${detail}` : ''}`);
|
||
};
|
||
|
||
const server = await createServer({
|
||
root: ROOT,
|
||
server: { middlewareMode: true },
|
||
appType: 'custom',
|
||
logLevel: 'error',
|
||
});
|
||
|
||
const reg = await server.ssrLoadModule('/src/lib/skills/registry.js');
|
||
const placement = await server.ssrLoadModule('/src/components/ai-assistant/placement.js');
|
||
|
||
const contextFor = (pageKey) => Object.entries(placement.PLACEMENT_ROUTES)
|
||
.find(([route]) => reg.pageKeyForRoute(route) === pageKey)?.[1] ?? null;
|
||
|
||
/* ── 1. Discovery ─────────────────────────────────────────────────────────── */
|
||
console.log('\n── Discovery ──');
|
||
|
||
const onDisk = readdirSync(join(ROOT, 'src/skills'))
|
||
.flatMap((dir) => readdirSync(join(ROOT, 'src/skills', dir)).map((f) => `${dir}/${f}`))
|
||
.filter((f) => f.endsWith('.md'));
|
||
|
||
record(
|
||
'every .md under src/skills registers',
|
||
reg.SKILLS.length === onDisk.length,
|
||
`${reg.SKILLS.length} registered / ${onDisk.length} files`
|
||
);
|
||
|
||
record(
|
||
'no duplicate skill ids',
|
||
new Set(reg.SKILLS.map((s) => s.id)).size === reg.SKILLS.length,
|
||
`${new Set(reg.SKILLS.map((s) => s.id)).size} unique ids`
|
||
);
|
||
|
||
record(
|
||
'every skill declares at least one page',
|
||
reg.SKILLS.every((s) => s.pages.length > 0)
|
||
);
|
||
|
||
/* Both managed lists are non-empty — the Workspace → Skills tabs. */
|
||
const uiFacet = reg.skillsWithFacet(reg.SKILLS, 'ui');
|
||
const owliverFacet = reg.skillsWithFacet(reg.SKILLS, 'owliver');
|
||
record('Owliver Skills list is populated', owliverFacet.length > 0, `${owliverFacet.length} skills`);
|
||
record(
|
||
'Board Skills list resolves without error',
|
||
Array.isArray(uiFacet),
|
||
`${uiFacet.length} skills declare a \`ui:\` block`
|
||
);
|
||
|
||
/* ── 2. Registration integrity: nothing half-loads in silence ─────────────── */
|
||
console.log('\n── Registration integrity ──');
|
||
|
||
const clean = reg.readSkillRegistry([]);
|
||
record(
|
||
'shipped registry reports no diagnostics',
|
||
clean.diagnostics.length === 0,
|
||
clean.diagnostics.map((d) => d.message).join(' | ') || 'none'
|
||
);
|
||
|
||
record(
|
||
'every shipped skill registered every capability it declared',
|
||
reg.SKILLS.every((s) => !(s.owliverErrors?.length || s.uiErrors?.length))
|
||
);
|
||
|
||
/* An unreadable stored definition must be REPORTED, not swallowed. */
|
||
const brokenYaml = `---\nid: broken\n name: bad indent\n---\n# Broken\n`;
|
||
const withBroken = reg.readSkillRegistry([{ path: 'custom/broken.md', raw: brokenYaml }]);
|
||
record(
|
||
'unreadable stored skill is reported, not silently dropped',
|
||
withBroken.diagnostics.some((d) => d.kind === 'unreadable'),
|
||
withBroken.diagnostics.find((d) => d.kind === 'unreadable')?.message ?? 'NO DIAGNOSTIC'
|
||
);
|
||
|
||
/* A definition with one bad capability keeps the good one AND says so. */
|
||
const partial = `---
|
||
id: partial-check
|
||
name: Partial Check
|
||
description: One resolvable capability and one that names no source.
|
||
pages:
|
||
- positions
|
||
status: active
|
||
owliver:
|
||
enabled: true
|
||
capabilities:
|
||
- summary
|
||
- flow
|
||
responses:
|
||
summary:
|
||
source: position.activity
|
||
periods:
|
||
- today
|
||
---
|
||
|
||
# Partial Check
|
||
`;
|
||
const withPartial = reg.readSkillRegistry([{ path: 'custom/partial.md', raw: partial }]);
|
||
const partialSkill = withPartial.skills.find((s) => s.id === 'partial-check');
|
||
record(
|
||
'partly-resolvable skill still registers its good capability',
|
||
partialSkill?.owliver.capabilities.includes('summary'),
|
||
`capabilities = ${JSON.stringify(partialSkill?.owliver.capabilities)}`
|
||
);
|
||
record(
|
||
'and the capability it LOST is reported',
|
||
withPartial.diagnostics.some((d) => d.kind === 'incomplete' && d.skillId === 'partial-check'),
|
||
withPartial.diagnostics.find((d) => d.kind === 'incomplete')?.message ?? 'NO DIAGNOSTIC'
|
||
);
|
||
|
||
/* Shadowing a built-in is allowed, but must be visible. */
|
||
const shadow = `---\nid: create-position\nname: Shadow\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n# Shadow\n`;
|
||
const withShadow = reg.readSkillRegistry([{ path: 'custom/shadow.md', raw: shadow }]);
|
||
record(
|
||
'a custom skill overriding a built-in is reported',
|
||
withShadow.diagnostics.some((d) => d.kind === 'shadowed' && d.skillId === 'create-position'),
|
||
withShadow.diagnostics.find((d) => d.kind === 'shadowed')?.message ?? 'NO DIAGNOSTIC'
|
||
);
|
||
|
||
/* ── 3. Markdown parsing: no silent content loss ──────────────────────────── */
|
||
console.log('\n── Markdown parsing ──');
|
||
|
||
const cp = reg.SKILLS.find((s) => s.id === 'create-position');
|
||
record(
|
||
'wrapped bullets are read whole',
|
||
cp.capabilities.some((c) => c.endsWith('out of a single sentence.')),
|
||
cp.capabilities.find((c) => c.includes('single sentence')) ?? 'TRUNCATED'
|
||
);
|
||
|
||
record(
|
||
'conversation steps all parse',
|
||
cp.conversation.length === 7 && cp.conversation.every((s) => s.field && s.question),
|
||
`${cp.conversation.length} steps`
|
||
);
|
||
|
||
const numbered = reg.parseSkill(
|
||
`---\nid: numbered\nname: Numbered\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n`
|
||
+ `## Capabilities\n\n1. First instruction.\n2. Second instruction.\n3. Third instruction.\n`,
|
||
{ custom: true }
|
||
);
|
||
record(
|
||
'numbered instructions are preserved',
|
||
numbered.capabilities.length === 3,
|
||
JSON.stringify(numbered.capabilities)
|
||
);
|
||
|
||
/* A section ends at the next `##`, and a heading that merely starts with the
|
||
same word is a different heading. Both halves matter: the first is what stops
|
||
`Capabilities` swallowing the rest of the file, the second is what stops a
|
||
near-miss heading being read as the real one. */
|
||
const bounded = reg.parseSkill(
|
||
`---\nid: bounded\nname: Bounded\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n`
|
||
+ `## Capabilities\n\n- one\n- two\n\n## Capabilities (v2)\n\n- three\n\n## Actions\n\n- four\n`,
|
||
{ custom: true }
|
||
);
|
||
record(
|
||
'a section stops at the next heading',
|
||
bounded.capabilities.length === 2,
|
||
JSON.stringify(bounded.capabilities)
|
||
);
|
||
|
||
/* Sub-headings (`###`) belong to the section they sit under, and must not end
|
||
it — `\n##\s` would match `### ` if the space were not required. */
|
||
const nested = reg.parseSkill(
|
||
`---\nid: nested\nname: Nested\ndescription: d\npages:\n - positions\nstatus: active\n---\n\n`
|
||
+ `## Capabilities\n\n- one\n\n### Detail\n\n- two\n\n## Actions\n\n- three\n`,
|
||
{ custom: true }
|
||
);
|
||
record(
|
||
'a `###` sub-heading does not truncate its section',
|
||
nested.capabilities.length === 2,
|
||
JSON.stringify(nested.capabilities)
|
||
);
|
||
|
||
/* Frontmatter of every shipped file parses and carries its required fields. */
|
||
let frontmatterOk = true;
|
||
const frontmatterDetail = [];
|
||
for (const s of reg.SKILLS) {
|
||
if (!s.id || !s.name || !s.pages.length) {
|
||
frontmatterOk = false;
|
||
frontmatterDetail.push(s.path);
|
||
}
|
||
}
|
||
record('every shipped skill has id, name and pages', frontmatterOk, frontmatterDetail.join(', '));
|
||
|
||
/* ── 4. Trigger / matching ────────────────────────────────────────────────── */
|
||
console.log('\n── Trigger matching ──');
|
||
|
||
const positions = contextFor('positions');
|
||
record('positions context resolves', Boolean(positions), String(positions));
|
||
|
||
const MATRIX = [
|
||
['create a position', 'create-position'],
|
||
['i want to hire', 'create-position'],
|
||
['post a job', 'create-position'],
|
||
['add a client', 'create-position'],
|
||
['show hiring activity', 'hiring-activity-assistant'],
|
||
['hiring flow', 'hiring-activity-assistant'],
|
||
['applications over time', 'hiring-activity-assistant'],
|
||
];
|
||
for (const [question, expected] of MATRIX) {
|
||
const got = reg.matchSkill(question, positions)?.id ?? null;
|
||
record(`trigger "${question}" -> ${expected}`, got === expected, got ?? 'no match');
|
||
}
|
||
|
||
/* Every claimant is enumerable, so a contested phrase is not invisible. */
|
||
const contender = `---
|
||
id: zz-contender
|
||
name: ZZ Contender
|
||
description: Also claims a phrase create-position owns.
|
||
pages:
|
||
- positions
|
||
status: active
|
||
triggers:
|
||
- create a position
|
||
owliver:
|
||
enabled: true
|
||
capabilities:
|
||
- summary
|
||
responses:
|
||
summary:
|
||
source: position.activity
|
||
periods:
|
||
- today
|
||
---
|
||
|
||
# ZZ Contender
|
||
`;
|
||
const contested = [{ path: 'custom/zz.md', raw: contender }];
|
||
const claimants = reg.matchSkills('create a position', positions, [], contested);
|
||
record(
|
||
'all trigger claimants are enumerable',
|
||
claimants.length === 2,
|
||
claimants.map((s) => s.id).join(', ')
|
||
);
|
||
record(
|
||
'a contested trigger is reported as a collision',
|
||
reg.readSkillRegistry(contested).diagnostics.some((d) => d.kind === 'trigger-collision'),
|
||
reg.readSkillRegistry(contested).diagnostics.find((d) => d.kind === 'trigger-collision')?.message ?? 'NO DIAGNOSTIC'
|
||
);
|
||
|
||
/* ── 5. Owliver suggestions ───────────────────────────────────────────────── */
|
||
console.log('\n── Suggestions ──');
|
||
|
||
const resolver = await server.ssrLoadModule('/src/lib/skills/owliverResolver.js');
|
||
const declared = resolver.owliverSuggestions(positions, [], [], {});
|
||
|
||
record(
|
||
'every declared suggestion names a capability the skill offers',
|
||
declared.every((c) => !c.skillCapability
|
||
|| reg.SKILLS.find((s) => s.id === c.skillId)?.owliver.capabilities.includes(c.skillCapability))
|
||
);
|
||
|
||
const byIntent = declared.map((c) => `${c.skillId}:${c.skillCapability}`);
|
||
record(
|
||
'no two declared suggestions resolve to one capability',
|
||
new Set(byIntent).size === byIntent.length,
|
||
byIntent.join(' | ')
|
||
);
|
||
|
||
const dynamic = await server.ssrLoadModule('/src/components/ai-assistant/dynamic.js');
|
||
|
||
/* The generator is exercised at three different data shapes: the labels must
|
||
track the data, and must never appear when the count behind them is zero. */
|
||
const factsFor = (n) => ({
|
||
postings: [], starvedPositions: new Array(n).fill({}), unscreened: new Array(n).fill({}),
|
||
scored: [], stalled: [], ranked: [], activity: [], missingCredentials: [],
|
||
});
|
||
const shapes = [0, 1, 7, 12];
|
||
let labelsOk = true;
|
||
const offenders = [];
|
||
for (const n of shapes) {
|
||
for (const chip of dynamic.buildPrompts('admin.positions', factsFor(n), null)) {
|
||
/* A complete intent asks or instructs. A raw fragment — "4 ready for
|
||
interview" — does neither, and is what this check exists to catch. */
|
||
const isIntent = /\?$/.test(chip.label) || /^(show|summari[sz]e|create|continue|compare|find|explain|what|which|who)\b/i.test(chip.label);
|
||
if (!isIntent) { labelsOk = false; offenders.push(`n=${n}: "${chip.label}"`); }
|
||
if (n === 0 && /\b0\b/.test(chip.label)) { labelsOk = false; offenders.push(`n=0 leaked a zero count: "${chip.label}"`); }
|
||
}
|
||
}
|
||
record('every generated label is a complete intent', labelsOk, offenders.join(' ; ') || 'all labels are intents');
|
||
|
||
/* Counts must move with the data rather than being baked in. */
|
||
const at7 = dynamic.buildPrompts('admin.positions', factsFor(7), null).map((c) => c.label);
|
||
const at12 = dynamic.buildPrompts('admin.positions', factsFor(12), null).map((c) => c.label);
|
||
record(
|
||
'labels are dynamic, not hardcoded',
|
||
at7.some((l) => l.includes('7')) && at12.some((l) => l.includes('12')) && !at7.some((l) => l.includes('12')),
|
||
`7 -> ${at7.find((l) => l.includes('7'))} | 12 -> ${at12.find((l) => l.includes('12'))}`
|
||
);
|
||
|
||
const atZero = dynamic.buildPrompts('admin.positions', factsFor(0), null).map((c) => c.label);
|
||
record(
|
||
'no count-bearing suggestion when the count is zero',
|
||
!atZero.some((l) => /\d/.test(l)),
|
||
atZero.join(' | ')
|
||
);
|
||
|
||
/**
|
||
* The workforce branch — the one the fragments came from.
|
||
*
|
||
* "4 ready for interview" and "2 strong for Event Server" were generated here,
|
||
* so a suggestion check that passes `null` for workforce proves nothing about
|
||
* them. Applications are dated today so the "applied today" branch is live too.
|
||
*/
|
||
const workforceAt = (readyCount) => {
|
||
const position = { id: 'p1', title: 'Event Server – Fine Dining', status: 'active', headcount: 5 };
|
||
const today = new Date();
|
||
const applications = Array.from({ length: readyCount }, (_, i) => ({
|
||
id: `a${i}`,
|
||
job_posting_id: 'p1',
|
||
email: `c${i}@example.com`,
|
||
applicant_name: `Candidate ${i}`,
|
||
status: i % 2 ? 'ai_screened' : 'shortlisted',
|
||
ai_score: 90,
|
||
created_date: today.toISOString(),
|
||
updated_date: today.toISOString(),
|
||
}));
|
||
return {
|
||
positions: [position],
|
||
currentPositionId: null,
|
||
context: { applications, today, profiles: [], staff: [], assignments: [] },
|
||
};
|
||
};
|
||
|
||
for (const n of [1, 4, 9]) {
|
||
const labels = dynamic
|
||
.buildPrompts('admin.positions', factsFor(2), workforceAt(n))
|
||
.map((c) => c.label);
|
||
|
||
const ready = labels.find((l) => /ready for interview/i.test(l));
|
||
record(
|
||
`workforce branch at ${n}: "ready for interview" is a complete question`,
|
||
Boolean(ready) && /\?$/.test(ready) && ready.includes(String(n)) && /^Which\b/.test(ready),
|
||
ready ?? 'NOT GENERATED'
|
||
);
|
||
|
||
/* Singular/plural has to track the count, or the fix trades one awkward
|
||
label for another. */
|
||
if (n === 1) {
|
||
record(
|
||
'a single candidate reads as one candidate',
|
||
ready === 'Which 1 candidate is ready for interview?',
|
||
ready ?? 'NOT GENERATED'
|
||
);
|
||
}
|
||
|
||
const fragment = labels.find((l) => /^\d+\s/.test(l));
|
||
record(
|
||
`workforce branch at ${n}: no bare status fragments`,
|
||
!fragment,
|
||
fragment ?? 'none'
|
||
);
|
||
}
|
||
|
||
await server.close();
|
||
|
||
/* ── 6. Production bundle ─────────────────────────────────────────────────── */
|
||
if (process.argv.includes('--dist')) {
|
||
console.log('\n── Production bundle ──');
|
||
const dir = join(ROOT, 'dist/assets');
|
||
if (!existsSync(dir)) {
|
||
record('dist/assets exists', false, 'run `npm run build` first');
|
||
} else {
|
||
const bundle = readdirSync(dir)
|
||
.filter((f) => f.endsWith('.js'))
|
||
.map((f) => readFileSync(join(dir, f), 'utf8'))
|
||
.join('\n');
|
||
|
||
for (const s of reg.SKILLS) {
|
||
record(`bundle carries \`${s.id}\``, bundle.includes(s.id));
|
||
}
|
||
/* Body content, not just the id — a manifest naming a skill whose Markdown
|
||
did not survive is the failure this is looking for. */
|
||
record(
|
||
'bundle carries skill body text',
|
||
bundle.includes('Which client is this role for'),
|
||
'create-position conversation step found in bundle'
|
||
);
|
||
}
|
||
}
|
||
|
||
/* ── Summary ──────────────────────────────────────────────────────────────── */
|
||
const failed = results.filter((r) => !r.pass);
|
||
console.log(`\n${results.length - failed.length}/${results.length} checks passed`);
|
||
if (failed.length) {
|
||
console.log('\nFailed:');
|
||
failed.forEach((r) => console.log(` - ${r.name}${r.detail ? ` (${r.detail})` : ''}`));
|
||
process.exit(1);
|
||
}
|