/** * Verification-before-completion gate (premium harness D3 — structural). * * R3 (`f9be0b4`) put the "a task is not done until verified" contract in * BEHAVIORAL_SPEC as prose. R6 is direct evidence that a model does not * reliably obey in-context discipline. Premium D3 ("not done until * verified; reproduce the check") therefore needs a STRUCTURAL gate, the * same way R2/R5a made memory integrity structural rather than prose. * * This classifier flags a final turn that *asserts* the work is * verified / passing / working while NO verification-class tool was run * that turn — i.e. an unverified completion claim, the exact * confabulation R3 prohibits. The agent loop uses it to force ONE * corrective turn before accepting completion (one-shot; loop-guard / * maxTurns still bound the loop). * * Pure + conservatively tuned: it must fire on explicit success/verified * assertions and NOT on neutral "done", honest "I could not verify" * non-claims, or turns where a check actually ran. False negatives are * safe (no behavior change); false positives would disrupt normal flows, * so the bar to fire is deliberately high. */ /** Exact general-purpose tools that can run or inspect a real check. */ const VERIFICATION_TOOL_EXACT = new Set([ 'bash', 'shell', 'terminal', 'powershell', 'cmd', 'run', 'run_code', 'run_harness', 'exec', 'exec_command', 'execute_command', 'cli_execute', 'lsp_diagnostics', ]); /** Verification-specific whole name segments; avoids `inspect_*` matching `spec`. */ const VERIFICATION_TOOL_SEGMENT = /(?:^|[_:-])(?:tests?|build|verification?|verify|lint|typecheck|tsc|pytest|jest|vitest|compile|diagnostics?|spec)(?:$|[_:-])/i; export function isVerificationToolName(name: string): boolean { const normalized = name.trim().toLowerCase(); return VERIFICATION_TOOL_EXACT.has(normalized) || VERIFICATION_TOOL_SEGMENT.test(normalized); } /** Explicit "the work is verified / passing / working" success assertions. */ const SUCCESS_ASSERTION: RegExp[] = [ /\b(?:all\s+)?(?:tests?|suite|specs?)\s+(?:(?:are|is)\s+)?(?:pass(?:ed|ing)?|green)\b/i, /\b\d+\s+tests?\s+(?:pass(?:ed|ing)?|green)\b/i, /\bbuild\s+(?:succeed(?:s|ed)?|passes|is\s+green)\b/i, /\bit\s+(?:now\s+)?compiles?\b|\beverything\s+compiles\b/i, /\bI(?:'ve| have)?\s+verified\b|\bverified\s+(?:that\s+)?(?:everything|it|the)\b/i, /\bit\s+works\s+(?:now|correctly|as\s+expected)\b|\beverything\s+works\b/i, /\bI\s+ran\s+it\b[^.]*\b(?:correct|works?|pass(?:es|ed)?)\b/i, /\b(?:confirmed|validated)\s+(?:it|the|that)\b[^.]*\b(?:works?|passes?|correct)\b/i, ]; /** Requests whose output is expected to preserve claims supplied by the user. */ const SOURCE_TRANSFORM_REQUEST = /\b(?:rewrite|rephrase|summari[sz]e|translate|preserve|quote|extract|polish|edit this|turn this into)\b/i; /** Context that makes a success phrase a future condition rather than a completion claim. */ const PLANNING_CONTEXT = /(?:\b(?:exit|acceptance|release|completion|success|quality)\s+(?:criteria|criterion|gate)\b|\bdefinition of done\b|\b(?:if|when|once|until|unless)\b|\b(?:must|should|needs? to|required|requires?|target|goal|planned|plan to|will)\b)/i; const PLANNING_HEADER = /(?:criteria|criterion|gate|definition of done|requirements?|target|goal|next checks?)/i; const TABLE_PLANNING_HEADER = /(?:pass conditions?|next checks?|exit criteria|acceptance criteria|requirements?)/i; const ATTRIBUTED_CONTEXT = /\b(?:you (?:said|reported|stated|provided)|according to (?:you|your message)|the supplied (?:text|claim)|reported|claimed)\b/i; function normalizeAssertion(value: string): string { return value.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); } function assertionContext(content: string, index: number): { line: string; previousLine: string } { const lineStart = content.lastIndexOf('\n', Math.max(0, index - 1)) + 1; const lineEndMatch = content.indexOf('\n', index); const lineEnd = lineEndMatch === -1 ? content.length : lineEndMatch; const previousEnd = Math.max(0, lineStart - 1); const previousStart = content.lastIndexOf('\n', Math.max(0, previousEnd - 1)) + 1; return { line: content.slice(lineStart, lineEnd), previousLine: content.slice(previousStart, previousEnd), }; } interface MarkdownTableCell { text: string; start: number; end: number; } function isEscapedDelimiter(line: string, index: number): boolean { let backslashes = 0; for (let cursor = index - 1; cursor >= 0 && line[cursor] === '\\'; cursor--) backslashes++; return backslashes % 2 === 1; } function markdownTableCells(line: string): MarkdownTableCell[] | null { const delimiters: number[] = []; for (let index = 0; index < line.length; index++) { if (line[index] === '|' && !isEscapedDelimiter(line, index)) delimiters.push(index); } if (delimiters.length === 0) return null; const boundaries = [-1, ...delimiters, line.length]; const cells: MarkdownTableCell[] = []; for (let index = 0; index < boundaries.length - 1; index++) { const start = boundaries[index] + 1; const end = boundaries[index + 1]; cells.push({ text: line.slice(start, end).trim().replace(/\\\|/g, '|'), start, end, }); } if (cells[0]?.text === '') cells.shift(); if (cells.at(-1)?.text === '') cells.pop(); return cells.length >= 2 ? cells : null; } function tableAssertionContext( content: string, line: string, lineStart: number, assertionIndex: number, ): { cell: string; header: string } | null { const cells = markdownTableCells(line); if (!cells) return null; const relativeIndex = Math.max(0, assertionIndex - lineStart); const columnIndex = cells.findIndex(cell => relativeIndex >= cell.start && relativeIndex < cell.end); if (columnIndex < 0) return null; const priorLines = content.slice(0, lineStart).split('\n'); let cursor = priorLines.length - 1; if (priorLines[cursor] === '') cursor--; let header = ''; for (; cursor >= 1; cursor--) { const rowCells = markdownTableCells(priorLines[cursor]); if (!rowCells) break; if (rowCells.every(cell => /^:?-{3,}:?$/.test(cell.text))) { const headerCells = markdownTableCells(priorLines[cursor - 1]); if (!headerCells || headerCells.length !== cells.length || rowCells.length !== cells.length) return null; header = headerCells[columnIndex]?.text ?? ''; break; } } return { cell: cells[columnIndex]?.text ?? '', header }; } function isPlanningListSection(content: string, lineStart: number): boolean { const headingStack: Array<{ level: number; text: string }> = []; let colonLabel = ''; for (const line of content.slice(0, lineStart).split('\n')) { const heading = line.match(/^\s*(#{1,6})\s+(.+?)\s*$/); if (heading) { const level = heading[1].length; while (headingStack.at(-1)?.level !== undefined && headingStack.at(-1)!.level >= level) { headingStack.pop(); } headingStack.push({ level, text: heading[2] }); colonLabel = ''; continue; } if (!/^\s*(?:[-*]|\d+[.)])\s+/.test(line) && /^\s*[^|#\r\n]{1,120}:\s*$/.test(line)) { colonLabel = line; } } return headingStack.some(heading => PLANNING_HEADER.test(heading.text)) || PLANNING_HEADER.test(colonLabel); } function isPlanningCondition(content: string, index: number): boolean { const { line } = assertionContext(content, index); const isListItem = /^\s*(?:[-*]|\d+[.)])\s+/.test(line); const lineStart = content.lastIndexOf('\n', Math.max(0, index - 1)) + 1; const tableContext = tableAssertionContext(content, line, lineStart, index); if (tableContext) { return PLANNING_CONTEXT.test(tableContext.cell) || TABLE_PLANNING_HEADER.test(tableContext.header); } if (PLANNING_CONTEXT.test(line)) return true; if (!isListItem) return false; return isPlanningListSection(content, lineStart); } function isSuppliedClaim( assertion: RegExp, matchText: string, content: string, index: number, userRequest: string, ): boolean { if (SOURCE_TRANSFORM_REQUEST.test(userRequest)) { const flags = assertion.flags.replaceAll('g', ''); if (new RegExp(assertion.source, flags).test(userRequest)) return true; } const normalizedMatch = normalizeAssertion(matchText); if (!normalizedMatch || !normalizeAssertion(userRequest).includes(normalizedMatch)) return false; return ATTRIBUTED_CONTEXT.test(assertionContext(content, index).line); } /** * True when `content` asserts verified/passing/working completion but * none of `toolsUsed` is a verification-class tool — an unverified * completion claim that must not be accepted as "done". */ export function assertsUnverifiedCompletion( content: string, toolsUsed: readonly string[], userRequest = '', ): boolean { if (!content || content.length < 12) return false; // A check actually ran this turn → the claim is grounded; do not fire. if (toolsUsed.some(isVerificationToolName)) return false; for (const assertion of SUCCESS_ASSERTION) { const flags = assertion.flags.includes('g') ? assertion.flags : `${assertion.flags}g`; for (const match of content.matchAll(new RegExp(assertion.source, flags))) { const index = match.index ?? 0; if (isPlanningCondition(content, index)) continue; if (isSuppliedClaim(assertion, match[0], content, index, userRequest)) continue; return true; } } return false; } /** * One-shot corrective directive injected when the gate fires. Mirrors * BEHAVIORAL_SPEC's verification contract into an actionable instruction. */ export const VERIFICATION_GATE_DIRECTIVE = 'You asserted the work is verified/passing/working but ran no verification ' + 'tool this turn. Per the verification-before-completion contract: actually ' + 'run the check now (the tests / build / the original reproduction) and quote ' + 'the real output, OR explicitly label the result UNVERIFIED and say what ' + 'remains unchecked. Do not reassert success without evidence.'; export const VERIFICATION_NO_TOOL_DISCLOSURE = '\n\n**Verification: UNVERIFIED** — No verification-capable tool was available in this turn. ' + 'Any success or readiness condition above is therefore a proposed criterion, not a measured result.';