200 lines
7.5 KiB
TypeScript
200 lines
7.5 KiB
TypeScript
/**
|
|
* D3 — runtime verification-before-completion gate (premium pillar:
|
|
* "not done until verified; reproduce the check"). The behavioral-spec
|
|
* carries the contract as prose (R3/f9be0b4); R6 is direct evidence a
|
|
* model does not reliably obey in-context discipline. Premium D3 needs
|
|
* a STRUCTURAL gate, not more prose: detect a turn that asserts the
|
|
* work is verified/passing while NO verification-class tool was run,
|
|
* and force one corrective turn before completion is accepted.
|
|
*
|
|
* Pure classifier — tuned conservatively (like memory-sign-gate): fire
|
|
* on explicit "it's verified / tests pass / it works" *claims*, never
|
|
* on neutral "done" or honest "I could not verify" non-claims.
|
|
*/
|
|
import { describe, it, expect } from 'vitest';
|
|
import {
|
|
assertsUnverifiedCompletion,
|
|
VERIFICATION_GATE_DIRECTIVE,
|
|
} from '../src/verification-gate.js';
|
|
|
|
describe('assertsUnverifiedCompletion', () => {
|
|
// Asserts verified/passing success BUT no verification-class tool ran.
|
|
const UNVERIFIED: Array<[string, string[]]> = [
|
|
['All tests pass and the build succeeds.', []],
|
|
['Done — I verified everything works correctly.', ['read_file', 'edit_file']],
|
|
['The fix is complete and the suite is green.', ['grep']],
|
|
['It works now; I ran it and the output is correct.', []],
|
|
['Everything compiles and all 141 tests are passing.', ['write_file']],
|
|
];
|
|
|
|
it('fires on an unverified completion/success assertion', () => {
|
|
for (const [c, tools] of UNVERIFIED) {
|
|
expect(assertsUnverifiedCompletion(c, tools), c).toBe(true);
|
|
}
|
|
});
|
|
|
|
// Must NOT fire — either a verify tool actually ran, or no success claim.
|
|
const OK: Array<[string, string[]]> = [
|
|
['All tests pass and the build succeeds.', ['run_tests']], // actually verified
|
|
['The suite is green.', ['bash']], // ran a check
|
|
['Build succeeds.', ['typecheck']], // ran a check
|
|
['I updated the config as you asked.', []], // neutral, no claim
|
|
['Here is the summary of the architecture you requested.', ['read_file']],
|
|
["I could not run the tests here — you'll need to run them.", []], // honest non-claim
|
|
['Done.', []], // bare ack, no success claim
|
|
];
|
|
|
|
it('does NOT fire when verification ran, or when there is no success claim', () => {
|
|
for (const [c, tools] of OK) {
|
|
expect(assertsUnverifiedCompletion(c, tools), c).toBe(false);
|
|
}
|
|
});
|
|
|
|
it('preserves success claims supplied by a closed-world rewrite request', () => {
|
|
const request = 'Rewrite this and preserve the facts: API tests pass. Browser tests have two failures.';
|
|
const response = 'API tests are passing. Browser tests still have two failures.';
|
|
expect(assertsUnverifiedCompletion(response, [], request)).toBe(false);
|
|
});
|
|
|
|
it('does not mistake future exit criteria for completed verification', () => {
|
|
const response = [
|
|
'Exit criteria:',
|
|
'- All tests pass.',
|
|
'- The build succeeds.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(false);
|
|
});
|
|
|
|
it('does not mistake a pass-condition table for completed verification', () => {
|
|
const response = [
|
|
'## Minimum next checks (evidence-only)',
|
|
'| Check | Pass condition |',
|
|
'|---|---|',
|
|
'| Verify the cited CI build | Build passes for the release commit |',
|
|
'**VERDICT: FAIL** - Production readiness is not established.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [], 'Give an evidence-only production verdict.')).toBe(false);
|
|
});
|
|
|
|
it('still fires on an unsupported status claim inside a table', () => {
|
|
const response = [
|
|
'## Next checks',
|
|
'| Component | Current status | Pass condition |',
|
|
'|---|---|---|',
|
|
'| Web build | All tests pass | Build passes for the release commit |',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('does not let planning text in an adjacent cell hide a status claim', () => {
|
|
const response = [
|
|
'| Current status | Required follow-up |',
|
|
'|---|---|',
|
|
'| All tests pass | Must rerun on Windows |',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('maps escaped Markdown pipes to the correct status column', () => {
|
|
const response = [
|
|
'| Detail \\| notes | Pass condition | Current status |',
|
|
'|---|---|---|',
|
|
'| Matrix | Must be green | All tests pass |',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('recognizes pass-condition tables without outer pipes', () => {
|
|
const response = [
|
|
'Check | Pass condition',
|
|
'--- | ---',
|
|
'Verify the cited CI build | Build passes for the release commit',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(false);
|
|
});
|
|
|
|
it('stops planning context at adjacent table boundaries', () => {
|
|
const response = [
|
|
'Check | Pass condition',
|
|
'--- | ---',
|
|
'Verify the cited CI build | Build passes for the release commit',
|
|
'',
|
|
'Component | Current status',
|
|
'--- | ---',
|
|
'Web build | All tests pass',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('stops list planning context at the nearest section heading', () => {
|
|
const response = [
|
|
'## Next checks',
|
|
'- Run CI on Windows.',
|
|
'',
|
|
'## Current status',
|
|
'- All tests pass.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('still recognizes a success phrase as prospective inside next checks', () => {
|
|
const response = [
|
|
'## Next checks',
|
|
'- All tests pass.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(false);
|
|
});
|
|
|
|
it('does not treat a previous bullet as the current planning header', () => {
|
|
const response = [
|
|
'## Current status',
|
|
'- Next checks are documented separately.',
|
|
'- All tests pass.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(true);
|
|
});
|
|
|
|
it('inherits planning context through nested Markdown sections', () => {
|
|
const response = [
|
|
'## Next checks',
|
|
'### Windows',
|
|
'- All tests pass.',
|
|
].join('\n');
|
|
expect(assertsUnverifiedCompletion(response, [])).toBe(false);
|
|
});
|
|
|
|
it('still fires on an unsupported claim when the user did not supply it', () => {
|
|
expect(assertsUnverifiedCompletion(
|
|
'All tests pass and the build succeeds.',
|
|
[],
|
|
'Fix the failing tests.',
|
|
)).toBe(true);
|
|
});
|
|
|
|
it('does not let narrative use of "after" hide an unsupported completion claim', () => {
|
|
expect(assertsUnverifiedCompletion(
|
|
'After the fix, all tests pass.',
|
|
[],
|
|
'Fix the failing tests.',
|
|
)).toBe(true);
|
|
});
|
|
|
|
it('accepts an honestly attributed user claim without upgrading it', () => {
|
|
expect(assertsUnverifiedCompletion(
|
|
'You reported that all tests pass; I have not independently verified that claim.',
|
|
[],
|
|
'All tests pass.',
|
|
)).toBe(false);
|
|
});
|
|
|
|
it('ignores trivially short content', () => {
|
|
expect(assertsUnverifiedCompletion('', [])).toBe(false);
|
|
expect(assertsUnverifiedCompletion('ok', [])).toBe(false);
|
|
});
|
|
|
|
it('exposes a corrective directive that demands a real check or an UNVERIFIED label', () => {
|
|
expect(VERIFICATION_GATE_DIRECTIVE.length).toBeGreaterThan(40);
|
|
expect(VERIFICATION_GATE_DIRECTIVE).toMatch(/verif|check|UNVERIFIED/i);
|
|
});
|
|
});
|