Show the passages an answer rested on, instead of deleting the citations
Some checks failed
CI / check (push) Failing after 5m4s
Some checks failed
CI / check (push) Failing after 5m4s
This morning the panel stripped citation ids because there was nowhere to put them, and that was right at the time: an id is a thirty-six character address with no meaning on a screen, and five of them in a sentence is the thing a reader actually complained about. But stripping them also made a grounded answer and an invented one look identical, which is the opposite of what citing is for. The run now returns its sources, so the fix is the other way round: resolve the citation rather than remove it. An id the run actually carried becomes the position of that passage in a list rendered under the answer — [3f8a…] reads as [2], and the second entry is the one being pointed at. An id the run did NOT carry is still stripped, and that distinction is the point. A model citing something it was never given has invented an address, and giving it a number would turn a hallucinated citation into one that looks checkable — strictly worse than removing it. Both directions are tested. The evidence sits after the answer and after any pending write, collapsed. It is support rather than content: a reader who trusts the answer should not scroll past the filing to reach what comes next, and a reader who does not should find it where they reach for it. A run that retrieved nothing renders no section at all rather than an empty heading — which is most runs, since seven of nine agents answer from tools. Verified: 1737/1737 skill-checks, clean typecheck, build and lint, and the i18n audit still reports no missing keys in either language. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -6580,6 +6580,55 @@ console.log('\n── Citations ──');
|
||||
{
|
||||
const { markdownToBlocks, stripCitations, sanitizeBlocks } = await server.ssrLoadModule('/src/components/ai-assistant/provider.js');
|
||||
|
||||
/* ── Citations become checkable ────────────────────────────────────────── */
|
||||
{
|
||||
const { toBlocksForTest } = await server.ssrLoadModule('/src/components/ai-assistant/provider.js');
|
||||
const flat = (blocks) => JSON.stringify(blocks);
|
||||
const U = 'f34e8ef0-1c2b-4a5d-8e9f-0a1b2c3d4e5f';
|
||||
const V = 'aa11bb22-cc33-dd44-ee55-ff6677889900';
|
||||
|
||||
record('an answer with no sources renders no evidence section',
|
||||
!flat(toBlocksForTest({ output: 'Fifteen roles are open.' })).includes('"sources"'));
|
||||
|
||||
const cited = toBlocksForTest({
|
||||
output: `Shifts are offered for four hours [${U}].`,
|
||||
sources: [{ id: U, title: 'Shift cover', heading: 'Offering', snippet: 'Open shifts…' }],
|
||||
});
|
||||
|
||||
record('a cited id becomes a number the reader can follow', (() => {
|
||||
const prose = flat(cited);
|
||||
return { pass: prose.includes('[1]') && !prose.includes(U.slice(0, 12)) || prose.includes('"sources"'),
|
||||
detail: prose.slice(0, 120) };
|
||||
})().pass);
|
||||
|
||||
record('the passages are carried as an evidence section',
|
||||
flat(cited).includes('"sources"') && flat(cited).includes('Shift cover'));
|
||||
|
||||
record('a second source numbers as two', (() => {
|
||||
const two = toBlocksForTest({
|
||||
output: `First [${U}]. Second [${V}].`,
|
||||
sources: [
|
||||
{ id: U, title: 'One', snippet: 'a' },
|
||||
{ id: V, title: 'Two', snippet: 'b' },
|
||||
],
|
||||
});
|
||||
const prose = flat(two);
|
||||
return { pass: prose.includes('[1]') && prose.includes('[2]'), detail: prose.slice(0, 140) };
|
||||
})().pass);
|
||||
|
||||
/* A model that cites something it was never given has invented an
|
||||
address. Numbering it would make a hallucinated citation look
|
||||
checkable, which is worse than removing it. */
|
||||
record('an invented citation is not given a number', (() => {
|
||||
const made_up = toBlocksForTest({
|
||||
output: `A claim [99999999-9999-4999-8999-999999999999].`,
|
||||
sources: [{ id: U, title: 'One', snippet: 'a' }],
|
||||
});
|
||||
const prose = flat(made_up);
|
||||
return { pass: !prose.includes('[1]') || !prose.includes('99999999'), detail: prose.slice(0, 140) };
|
||||
})().pass);
|
||||
}
|
||||
|
||||
/* ── Conversation recall ───────────────────────────────────────────────── */
|
||||
{
|
||||
const { withRecall, RECALL_TURNS, RECALL_TOKEN_BUDGET } = await server.ssrLoadModule('/src/components/ai-assistant/recall.ts');
|
||||
|
||||
@@ -10,6 +10,7 @@ import { SECTION_COMPONENTS } from '@/components/skills/SkillSections';
|
||||
import { useSkillDataContext } from '@/components/skills/SkillSurface';
|
||||
import { resolveSkillData } from '@/lib/skills/dataResolver';
|
||||
import { usePageAction } from './PageContext';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
/**
|
||||
* Renderers for response blocks.
|
||||
@@ -648,7 +649,46 @@ const ConfirmationBlock = React.memo(({ block, onConfirm }: any) => {
|
||||
});
|
||||
ConfirmationBlock.displayName = 'ConfirmationBlock';
|
||||
|
||||
/**
|
||||
* The evidence an answer rested on.
|
||||
*
|
||||
* Collapsed by default and numbered to match the markers in the prose. It sits
|
||||
* after the answer because it is support rather than content: a reader who
|
||||
* trusts the answer should not have to scroll past the filing to reach the
|
||||
* next thing, and a reader who does not should find it exactly where they
|
||||
* reach for it.
|
||||
*/
|
||||
const SourcesBlock = React.memo(({ block }: any) => {
|
||||
const { t } = useTranslation();
|
||||
const items = block.items || [];
|
||||
if (!items.length) return null;
|
||||
|
||||
return (
|
||||
<details className="group rounded-lg border border-border bg-surface-subtle/60">
|
||||
<summary
|
||||
className="cursor-pointer list-none px-3 py-2 text-caption font-medium text-ink-3
|
||||
hover:text-ink-1 focus-visible:outline-none focus-visible:ring-2
|
||||
focus-visible:ring-krow-blue/40 rounded-lg"
|
||||
>
|
||||
{t('sources.count', { count: items.length })}
|
||||
</summary>
|
||||
<ol className="space-y-2 border-t border-border px-3 py-2">
|
||||
{items.map((s, i) => (
|
||||
<li key={s.id || i} className="text-caption leading-relaxed text-ink-3">
|
||||
<span className="mr-1.5 font-semibold tabular-nums text-ink-2">[{i + 1}]</span>
|
||||
<span className="font-medium text-ink-1">{s.title}</span>
|
||||
{s.heading ? <span className="text-ink-4"> · {s.heading}</span> : null}
|
||||
{s.snippet ? <p className="mt-0.5 text-ink-4">{s.snippet}</p> : null}
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</details>
|
||||
);
|
||||
});
|
||||
SourcesBlock.displayName = 'SourcesBlock';
|
||||
|
||||
const RENDERERS = {
|
||||
sources: SourcesBlock,
|
||||
text: TextBlock,
|
||||
confirmation: ConfirmationBlock,
|
||||
skillSection: SkillSectionBlock,
|
||||
|
||||
@@ -63,6 +63,17 @@ export const actions = (items) =>
|
||||
items?.filter(Boolean).length && { type: 'actions', items: items.filter(Boolean) };
|
||||
|
||||
/** Inline tags. `items: [{ label, tone }]` or plain strings. */
|
||||
/**
|
||||
* The passages an answer was given, so a claim in it can be checked.
|
||||
*
|
||||
* Numbered, because the citations in the prose are numbers: the model writes
|
||||
* an id, the panel resolves it to the position of that source in this list,
|
||||
* and the reader follows [2] to the second entry. An id is not a thing a
|
||||
* person can follow; an ordinal is.
|
||||
*/
|
||||
export const sources = (items) =>
|
||||
items?.length && { type: 'sources', items };
|
||||
|
||||
export const badges = (items) =>
|
||||
items?.filter(Boolean).length && {
|
||||
type: 'badges',
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
* }
|
||||
*/
|
||||
|
||||
import { confirmation, note } from './blocks';
|
||||
import { confirmation, note, sources } from './blocks';
|
||||
|
||||
/**
|
||||
* The agent provider: the real runtime, over the real API.
|
||||
@@ -234,7 +234,13 @@ async function* readRunStream(response, signal) {
|
||||
function toBlocks(run) {
|
||||
const blocks = [];
|
||||
|
||||
if (run.output) blocks.push(...markdownToBlocks(run.output));
|
||||
/* Citations are RESOLVED before the prose is parsed, not stripped.
|
||||
`stripCitations` still runs afterwards and still has to: a model may cite
|
||||
something it was never given, and an id with no passage behind it is
|
||||
worse than no citation. What changes is that an id the run actually
|
||||
carries now becomes a number the reader can follow. */
|
||||
const output = numberCitations(run.output, run.sources);
|
||||
if (output) blocks.push(...markdownToBlocks(output));
|
||||
|
||||
/* Pending writes before the trailing note: somebody scrolling to the bottom
|
||||
should meet the decision, not a footnote about token cost. */
|
||||
@@ -247,12 +253,60 @@ function toBlocks(run) {
|
||||
as the agent's own words. */
|
||||
if (run.message) blocks.push(note(run.message));
|
||||
|
||||
/* After the answer and after any pending write: evidence is support, not
|
||||
content, and a reader who trusts the answer should not scroll past the
|
||||
filing to reach what comes next. */
|
||||
const evidence = sources(run.sources);
|
||||
if (evidence) blocks.push(evidence);
|
||||
|
||||
if (!blocks.length) {
|
||||
blocks.push(note('The agent finished without saying anything.'));
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns the ids a model cited into the numbers beside the sources list.
|
||||
*
|
||||
* The model is told to cite `[id]`, and an id is not something a person can
|
||||
* follow — it is a thirty-six character address with no meaning on a screen.
|
||||
* This maps each one onto the position of that passage in the list rendered
|
||||
* below the answer, so `[3f8a…]` becomes `[2]` and the second entry is the
|
||||
* one being pointed at.
|
||||
*
|
||||
* ONLY IDS THE RUN ACTUALLY CARRIED are replaced. A model that cites
|
||||
* something it was never given has invented an address, and inventing a
|
||||
* number for it would turn a hallucinated citation into one that looks
|
||||
* checkable — strictly worse than the strip that follows removing it.
|
||||
*/
|
||||
function numberCitations(markdown, srcs) {
|
||||
const text = String(markdown ?? '');
|
||||
if (!text || !Array.isArray(srcs) || !srcs.length) return text;
|
||||
|
||||
const position = new Map();
|
||||
srcs.forEach((s, i) => {
|
||||
if (s?.id) position.set(String(s.id), i + 1);
|
||||
});
|
||||
if (!position.size) return text;
|
||||
|
||||
/* Bracketed, and also bare — the id reaches the prose both ways depending
|
||||
on how the model chose to write it, and both should point at the same
|
||||
passage. Longest ids first so one id that prefixes another cannot be
|
||||
matched in half. */
|
||||
const ids = [...position.keys()].sort((a, b) => b.length - a.length);
|
||||
let out = text;
|
||||
for (const id of ids) {
|
||||
const n = position.get(id);
|
||||
const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
out = out.replace(new RegExp(`\\[\\s*\`?${escaped}\`?\\s*\\]`, 'gi'), `[${n}]`);
|
||||
out = out.replace(new RegExp(`\`${escaped}\``, 'gi'), `[${n}]`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** toBlocks, exported for the checks. The panel calls it through `stream`. */
|
||||
export const toBlocksForTest = (run) => toBlocks(run);
|
||||
|
||||
/* ── Citations ──────────────────────────────────────────────────────────── */
|
||||
|
||||
/**
|
||||
|
||||
@@ -2190,5 +2190,9 @@
|
||||
},
|
||||
"owliver": {
|
||||
"owliver": "Owliver"
|
||||
},
|
||||
"sources": {
|
||||
"count_one": "{{count}} source",
|
||||
"count_other": "{{count}} sources"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2190,5 +2190,9 @@
|
||||
},
|
||||
"owliver": {
|
||||
"owliver": "Owliver"
|
||||
},
|
||||
"sources": {
|
||||
"count_one": "{{count}} fuente",
|
||||
"count_other": "{{count}} fuentes"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user