moving
This commit is contained in:
Oleg Maslov
329
tests/vision/persona-acceptance-seal.test.ts
Normal file
329
tests/vision/persona-acceptance-seal.test.ts
Normal file
@@ -0,0 +1,329 @@
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildPersonaAcceptanceSeal,
|
||||
type PersonaAcceptanceSealManifest,
|
||||
type ReceiptScore,
|
||||
} from './persona-acceptance-seal';
|
||||
import { PERSONA_CASES } from './persona-cases';
|
||||
|
||||
const passingScore: ReceiptScore = {
|
||||
score: 100,
|
||||
capturedScore: 100,
|
||||
scoreMode: 'captured',
|
||||
passed: true,
|
||||
criticalFailures: [],
|
||||
};
|
||||
|
||||
function artifact(personaId: string, repeat: number, overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
schemaVersion: 7,
|
||||
runId: 'paid-run-1',
|
||||
source: {
|
||||
gitRevision: 'da25b5097e8f735608d2ad1204ce89048008cec3',
|
||||
relevantWorkingTreeClean: true,
|
||||
},
|
||||
persona: { id: personaId, repeat, repeatCount: 1, gating: true },
|
||||
request: { exactPrompt: 'Prompt', personaId, sessionId: `session-${repeat}`, payload: { workspaceId: `workspace-${repeat}` } },
|
||||
response: {
|
||||
exact: 'Answer',
|
||||
tokenStreamExact: 'Answer',
|
||||
renderedAssistantExact: 'Answer',
|
||||
visibleAssistantTextExact: 'Answer',
|
||||
expectedCodeSegmentsExact: [],
|
||||
visibleCodeSegmentsExact: [],
|
||||
persistedExact: 'Answer',
|
||||
persistedPromptExact: 'Prompt',
|
||||
persistedSessionId: `session-${repeat}`,
|
||||
persistedMessageCount: 2,
|
||||
doneEventCount: 1,
|
||||
httpStatus: 200,
|
||||
durationMs: 10,
|
||||
model: 'openrouter/anthropic/claude-sonnet-5',
|
||||
estimatedCostUsd: 0.010001,
|
||||
tokens: { input: 10, output: 10 },
|
||||
toolsUsed: [],
|
||||
sseEvents: [{
|
||||
event: 'done',
|
||||
data: {
|
||||
content: 'Answer',
|
||||
model: 'openrouter/anthropic/claude-sonnet-5',
|
||||
cost: 0.010001,
|
||||
},
|
||||
}],
|
||||
parseErrors: [],
|
||||
transportError: null,
|
||||
},
|
||||
runtime: {
|
||||
healthStatus: 200,
|
||||
llmHealthy: true,
|
||||
expectedProvider: 'anthropic-proxy',
|
||||
expectedDetail: 'credential verified',
|
||||
health: { llm: { provider: 'anthropic-proxy', health: 'healthy', detail: 'OpenRouter credential verified' } },
|
||||
},
|
||||
workspace: { workspaceId: `workspace-${repeat}`, personaPersisted: true },
|
||||
journey: { memoryJourneyOk: true, memoryText: 'Memory', leakedSnippets: [] },
|
||||
codeValidation: {},
|
||||
score: { score: 100, passed: true, criticalFailures: [] },
|
||||
browser: {
|
||||
criticalConsoleErrors: [],
|
||||
pageErrors: [],
|
||||
criticalNetworkFailures: [],
|
||||
screenshotErrors: [],
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function writeArtifact(value: unknown): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'waggle-persona-seal-'));
|
||||
const path = join(dir, 'receipt.json');
|
||||
writeFileSync(path, JSON.stringify(value));
|
||||
return path;
|
||||
}
|
||||
|
||||
function manifest(artifactPath: string): PersonaAcceptanceSealManifest {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
benchmarkId: 'paid-30-final',
|
||||
runId: 'paid-run-1',
|
||||
sourceRevision: 'da25b5097e8f735608d2ad1204ce89048008cec3',
|
||||
threshold: 95,
|
||||
repeats: 1,
|
||||
expectedProvider: 'anthropic-proxy',
|
||||
expectedDetail: 'credential verified',
|
||||
allowedModels: ['openrouter/anthropic/claude-sonnet-5'],
|
||||
receipts: [{ artifactPath }],
|
||||
diagnosticCostLedger: [
|
||||
{ id: 'diagnostic', amountUsd: '0.000009', evidence: 'provider usage export diagnostic-1' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const options = {
|
||||
expectedPersonaIds: ['general-purpose'] as const,
|
||||
repeats: 1,
|
||||
scoreArtifact: () => passingScore,
|
||||
runtimeProvenance: {
|
||||
gitRevision: 'da25b5097e8f735608d2ad1204ce89048008cec3',
|
||||
relevantWorkingTreeClean: true,
|
||||
},
|
||||
};
|
||||
|
||||
describe('persona acceptance seal', () => {
|
||||
it('maps persisted schema-7 Python validation into a Data Engineer rescore', () => {
|
||||
const persona = PERSONA_CASES.find(item => item.id === 'data-engineer')!;
|
||||
const python = [
|
||||
'import json',
|
||||
'import sqlite3',
|
||||
'',
|
||||
'with sqlite3.connect("events.db") as connection:',
|
||||
' connection.execute("BEGIN")',
|
||||
' connection.execute("INSERT OR IGNORE INTO events VALUES (?, ?)", ("event-1", json.dumps({})))',
|
||||
].join('\n');
|
||||
const response = [
|
||||
'CREATE TABLE events (dedup_key TEXT PRIMARY KEY);',
|
||||
'The deduplication key is stable for every source event.',
|
||||
'Use a BEGIN transaction for each batch and commit its checkpoint atomically.',
|
||||
'Retry lock failures with exponential backoff and busy_timeout.',
|
||||
'```python',
|
||||
python,
|
||||
'```',
|
||||
].join('\n');
|
||||
const value = artifact('data-engineer', 1);
|
||||
value.request.exactPrompt = persona.prompt;
|
||||
value.response.exact = response;
|
||||
value.response.tokenStreamExact = response;
|
||||
value.response.renderedAssistantExact = response;
|
||||
value.response.visibleAssistantTextExact = response;
|
||||
value.response.expectedCodeSegmentsExact = [python];
|
||||
value.response.visibleCodeSegmentsExact = [python];
|
||||
value.response.persistedExact = response;
|
||||
value.response.persistedPromptExact = persona.prompt;
|
||||
value.response.sseEvents[0].data.content = response;
|
||||
value.codeValidation = {
|
||||
available: true,
|
||||
syntaxValid: true,
|
||||
importsPresent: true,
|
||||
};
|
||||
const artifactPath = writeArtifact(value);
|
||||
|
||||
const seal = buildPersonaAcceptanceSeal(manifest(artifactPath), {
|
||||
expectedPersonaIds: ['data-engineer'],
|
||||
repeats: 1,
|
||||
runtimeProvenance: options.runtimeProvenance,
|
||||
});
|
||||
|
||||
expect(seal.status).toBe('ready');
|
||||
expect(seal.invalidReceipts).toEqual([]);
|
||||
expect(seal.receipts[0]).toMatchObject({ score: 100, capturedScore: 100 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['unavailable validation', { available: false, syntaxValid: true, importsPresent: true }, '```python\nimport sqlite3\n```'],
|
||||
['failed syntax validation', { available: true, syntaxValid: false, importsPresent: true }, '```python\nimport sqlite3\n```'],
|
||||
['missing import validation', { available: true, syntaxValid: true, importsPresent: false }, '```python\nimport sqlite3\n```'],
|
||||
['missing validation fields', { available: true }, '```python\nimport sqlite3\n```'],
|
||||
['no Python evidence', { available: true, syntaxValid: true, importsPresent: true }, 'No Python block is present.'],
|
||||
['invalid Python evidence', { available: true, syntaxValid: true, importsPresent: true }, '```python\nimport sqlite3\nif True print("broken")\n```'],
|
||||
['Python evidence without imports', { available: true, syntaxValid: true, importsPresent: true }, '```python\nprint("valid but unimported")\n```'],
|
||||
])('fails closed on %s', (_label, codeValidation, codeEvidence) => {
|
||||
const persona = PERSONA_CASES.find(item => item.id === 'data-engineer')!;
|
||||
const response = [
|
||||
'CREATE TABLE events (dedup_key TEXT PRIMARY KEY);',
|
||||
'The deduplication key is stable for every source event.',
|
||||
'Use a BEGIN transaction for each batch and commit its checkpoint atomically.',
|
||||
'Retry lock failures with exponential backoff and busy_timeout.',
|
||||
codeEvidence,
|
||||
].join('\n');
|
||||
const codeSegment = codeEvidence.match(/```python\n([\s\S]*?)\n```/)?.[1];
|
||||
const value = artifact('data-engineer', 1);
|
||||
value.request.exactPrompt = persona.prompt;
|
||||
value.response.exact = response;
|
||||
value.response.tokenStreamExact = response;
|
||||
value.response.renderedAssistantExact = response;
|
||||
value.response.visibleAssistantTextExact = response;
|
||||
value.response.expectedCodeSegmentsExact = codeSegment ? [codeSegment] : [];
|
||||
value.response.visibleCodeSegmentsExact = codeSegment ? [codeSegment] : [];
|
||||
value.response.persistedExact = response;
|
||||
value.response.persistedPromptExact = persona.prompt;
|
||||
value.response.sseEvents[0].data.content = response;
|
||||
value.codeValidation = codeValidation;
|
||||
|
||||
const seal = buildPersonaAcceptanceSeal(manifest(writeArtifact(value)), {
|
||||
expectedPersonaIds: ['data-engineer'],
|
||||
repeats: 1,
|
||||
runtimeProvenance: options.runtimeProvenance,
|
||||
});
|
||||
|
||||
expect(seal.status).toBe('failed');
|
||||
expect(seal.receipts).toEqual([]);
|
||||
});
|
||||
|
||||
it('seals exact slot coverage with immutable hashes and decimal cost accounting', () => {
|
||||
const artifactPath = writeArtifact(artifact('general-purpose', 1));
|
||||
const seal = buildPersonaAcceptanceSeal(manifest(artifactPath), options);
|
||||
|
||||
expect(seal.status).toBe('ready');
|
||||
expect(seal.expectedReceiptCount).toBe(1);
|
||||
expect(seal.completedReceiptCount).toBe(1);
|
||||
expect(seal.acceptedEstimatedCostUsd).toBe('0.010001');
|
||||
expect(seal.diagnosticRecordedCostUsd).toBe('0.000009');
|
||||
expect(seal.totalRecordedSpendUsd).toBe('0.010010');
|
||||
expect(seal.manifestSha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(seal.diagnosticCostLedger).toEqual([
|
||||
{ id: 'diagnostic', amountUsd: '0.000009', evidence: 'provider usage export diagnostic-1' },
|
||||
]);
|
||||
expect(seal.receipts[0]).toMatchObject({
|
||||
slot: 'general-purpose#1',
|
||||
score: 100,
|
||||
scoreMode: 'captured',
|
||||
sourceRevision: 'da25b5097e8f735608d2ad1204ce89048008cec3',
|
||||
estimatedCostUsd: '0.010001',
|
||||
});
|
||||
expect(seal.receipts[0]?.artifactSha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(seal.receipts[0]?.responseSha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
it('fails closed on a duplicate persona and repeat slot', () => {
|
||||
const first = writeArtifact(artifact('general-purpose', 1));
|
||||
const second = writeArtifact(artifact('general-purpose', 1));
|
||||
const value = manifest(first);
|
||||
value.receipts.push({ artifactPath: second });
|
||||
|
||||
const seal = buildPersonaAcceptanceSeal(value, options);
|
||||
|
||||
expect(seal.status).toBe('failed');
|
||||
expect(seal.duplicateSlots).toEqual(['general-purpose#1']);
|
||||
});
|
||||
|
||||
it('reports incomplete without treating missing receipts as a passing report', () => {
|
||||
const value = manifest(writeArtifact(artifact('general-purpose', 1)));
|
||||
value.receipts = [];
|
||||
|
||||
const seal = buildPersonaAcceptanceSeal(value, options);
|
||||
|
||||
expect(seal.status).toBe('incomplete');
|
||||
expect(seal.missingSlots).toEqual(['general-purpose#1']);
|
||||
expect(seal.completedReceiptCount).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects a receipt whose live-provider health evidence is not healthy', () => {
|
||||
const unhealthy = artifact('general-purpose', 1, {
|
||||
runtime: { healthStatus: 200, llmHealthy: false, health: { llm: { health: 'unhealthy' } } },
|
||||
});
|
||||
const seal = buildPersonaAcceptanceSeal(manifest(writeArtifact(unhealthy)), options);
|
||||
|
||||
expect(seal.status).toBe('failed');
|
||||
expect(seal.invalidReceipts[0]?.reasons).toContain('runtime LLM was not healthy');
|
||||
});
|
||||
|
||||
it('records a current-scorer rescore instead of silently mutating the captured score', () => {
|
||||
const artifactPath = writeArtifact(artifact('general-purpose', 1, {
|
||||
score: { score: 80, passed: false, criticalFailures: [] },
|
||||
}));
|
||||
const seal = buildPersonaAcceptanceSeal(manifest(artifactPath), {
|
||||
...options,
|
||||
scoreArtifact: () => ({
|
||||
...passingScore,
|
||||
capturedScore: 80,
|
||||
scoreMode: 'derived-rescore',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(seal.status).toBe('ready');
|
||||
expect(seal.receipts[0]).toMatchObject({
|
||||
capturedScore: 80,
|
||||
score: 100,
|
||||
scoreMode: 'derived-rescore',
|
||||
scorerRevision: 'da25b5097e8f735608d2ad1204ce89048008cec3',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects artifact-backed provenance captured from a dirty relevant tree', () => {
|
||||
const sourceRevision = 'da25b5097e8f735608d2ad1204ce89048008cec3';
|
||||
const value = artifact('general-purpose', 1, {
|
||||
schemaVersion: 7,
|
||||
source: { gitRevision: sourceRevision, relevantWorkingTreeClean: false },
|
||||
});
|
||||
const artifactPath = writeArtifact(value);
|
||||
const receiptManifest = manifest(artifactPath);
|
||||
|
||||
const seal = buildPersonaAcceptanceSeal(receiptManifest, options);
|
||||
|
||||
expect(seal.status).toBe('failed');
|
||||
expect(seal.invalidReceipts[0]?.reasons).toContain(
|
||||
'artifact was not captured from a clean relevant working tree',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects receipts without matching visible assistant DOM evidence', () => {
|
||||
const missing = artifact('general-purpose', 1);
|
||||
delete (missing.response as Record<string, unknown>).visibleAssistantTextExact;
|
||||
delete (missing.response as Record<string, unknown>).visibleCodeSegmentsExact;
|
||||
const missingSeal = buildPersonaAcceptanceSeal(manifest(writeArtifact(missing)), options);
|
||||
|
||||
expect(missingSeal.status).toBe('failed');
|
||||
expect(missingSeal.invalidReceipts[0]?.reasons).toEqual(expect.arrayContaining([
|
||||
'visible assistant DOM text evidence is missing',
|
||||
'visible assistant DOM code evidence is missing or malformed',
|
||||
]));
|
||||
|
||||
const corrupted = artifact('general-purpose', 1, {
|
||||
response: {
|
||||
...(artifact('general-purpose', 1).response as Record<string, unknown>),
|
||||
exact: 'Use `search_files("**/*")`.',
|
||||
visibleAssistantTextExact: 'Use search_files("*/").',
|
||||
visibleCodeSegmentsExact: ['search_files("*/")'],
|
||||
},
|
||||
});
|
||||
const corruptedSeal = buildPersonaAcceptanceSeal(manifest(writeArtifact(corrupted)), options);
|
||||
|
||||
expect(corruptedSeal.status).toBe('failed');
|
||||
expect(corruptedSeal.invalidReceipts[0]?.reasons).toContain(
|
||||
'visible assistant DOM code did not match the response Markdown',
|
||||
);
|
||||
});
|
||||
});
|
||||
495
tests/vision/persona-acceptance-seal.ts
Normal file
495
tests/vision/persona-acceptance-seal.ts
Normal file
@@ -0,0 +1,495 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import {
|
||||
ACCEPTANCE_PERSONA_IDS,
|
||||
PERSONA_CASES,
|
||||
type PersonaAcceptanceCase,
|
||||
} from './persona-cases';
|
||||
import {
|
||||
containsFailureCopy,
|
||||
markdownCodeSegmentsMatch,
|
||||
scorePersonaTrial,
|
||||
validatePythonSyntax,
|
||||
visibleMarkdownPreservesText,
|
||||
type CapturedSseEvent,
|
||||
type PersonaTrialEvidence,
|
||||
} from './persona-scorer';
|
||||
|
||||
export interface PersonaAcceptanceReceiptManifest {
|
||||
artifactPath: string;
|
||||
}
|
||||
|
||||
export interface PersonaAcceptanceDiagnosticCostEntry {
|
||||
id: string;
|
||||
/** Exact decimal string with six fractional digits. */
|
||||
amountUsd: string;
|
||||
/** Human-auditable provider usage export, invoice, or receipt reference. */
|
||||
evidence: string;
|
||||
}
|
||||
|
||||
export interface PersonaAcceptanceSealManifest {
|
||||
schemaVersion: 1;
|
||||
benchmarkId: string;
|
||||
runId: string;
|
||||
sourceRevision: string;
|
||||
threshold: 95;
|
||||
repeats: number;
|
||||
expectedProvider: string;
|
||||
expectedDetail: string;
|
||||
allowedModels: string[];
|
||||
receipts: PersonaAcceptanceReceiptManifest[];
|
||||
diagnosticCostLedger: PersonaAcceptanceDiagnosticCostEntry[];
|
||||
}
|
||||
|
||||
export interface ReceiptScore {
|
||||
score: number;
|
||||
capturedScore: number | null;
|
||||
scoreMode: 'captured' | 'derived-rescore';
|
||||
passed: boolean;
|
||||
criticalFailures: readonly unknown[];
|
||||
}
|
||||
|
||||
interface SealOptions {
|
||||
expectedPersonaIds?: readonly string[];
|
||||
repeats?: number;
|
||||
scoreArtifact?: (artifact: Record<string, unknown>) => ReceiptScore;
|
||||
runtimeProvenance?: RuntimeProvenance;
|
||||
}
|
||||
|
||||
interface RuntimeProvenance {
|
||||
gitRevision: string | null;
|
||||
relevantWorkingTreeClean: boolean;
|
||||
}
|
||||
|
||||
interface InvalidReceipt {
|
||||
artifactPath: string;
|
||||
slot: string | null;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
export interface SealedPersonaReceipt {
|
||||
slot: string;
|
||||
personaId: string;
|
||||
repeat: number;
|
||||
runId: string;
|
||||
artifactPath: string;
|
||||
artifactSha256: string;
|
||||
responseSha256: string;
|
||||
sourceRevision: string;
|
||||
scorerRevision: string;
|
||||
capturedScore: number | null;
|
||||
score: number;
|
||||
scoreMode: ReceiptScore['scoreMode'];
|
||||
model: string | null;
|
||||
provider: string;
|
||||
estimatedCostUsd: string;
|
||||
workspaceId: string;
|
||||
sessionId: string;
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
durationMs: number | null;
|
||||
}
|
||||
|
||||
export interface PersonaAcceptanceSeal {
|
||||
schemaVersion: 1;
|
||||
benchmarkId: string;
|
||||
manifestSha256: string;
|
||||
status: 'ready' | 'incomplete' | 'failed';
|
||||
threshold: 95;
|
||||
repeats: number;
|
||||
expectedReceiptCount: number;
|
||||
completedReceiptCount: number;
|
||||
missingSlots: string[];
|
||||
duplicateSlots: string[];
|
||||
invalidReceipts: InvalidReceipt[];
|
||||
manifestErrors: string[];
|
||||
acceptedEstimatedCostUsd: string;
|
||||
diagnosticRecordedCostUsd: string;
|
||||
totalRecordedSpendUsd: string;
|
||||
diagnosticCostLedger: PersonaAcceptanceDiagnosticCostEntry[];
|
||||
receipts: SealedPersonaReceipt[];
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
function strings(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : [];
|
||||
}
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function sha256(value: string | Buffer): string {
|
||||
return createHash('sha256').update(value).digest('hex');
|
||||
}
|
||||
|
||||
function slot(personaId: string, repeat: number): string {
|
||||
return `${personaId}#${repeat}`;
|
||||
}
|
||||
|
||||
function parseUsdMicros(value: string): bigint | null {
|
||||
const match = value.match(/^(0|[1-9]\d*)\.(\d{6})$/);
|
||||
return match ? (BigInt(match[1]) * 1_000_000n) + BigInt(match[2]) : null;
|
||||
}
|
||||
|
||||
function formatUsdMicros(value: bigint): string {
|
||||
const whole = value / 1_000_000n;
|
||||
return `${whole}.${(value % 1_000_000n).toString().padStart(6, '0')}`;
|
||||
}
|
||||
|
||||
function estimatedCostMicros(value: unknown): bigint | null {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return null;
|
||||
const scaled = Math.round(value * 1_000_000);
|
||||
return Math.abs(value - (scaled / 1_000_000)) <= 1e-9 ? BigInt(scaled) : null;
|
||||
}
|
||||
|
||||
function gitOutput(args: string[]): string | null {
|
||||
try {
|
||||
return execFileSync('git', args, {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
windowsHide: true,
|
||||
}).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function currentRuntimeProvenance(): RuntimeProvenance {
|
||||
const status = gitOutput([
|
||||
'status',
|
||||
'--porcelain',
|
||||
'--untracked-files=all',
|
||||
'--',
|
||||
'.',
|
||||
':(exclude)output/**',
|
||||
':(exclude)test-results/**',
|
||||
':(exclude)playwright-report/**',
|
||||
':(exclude).playwright-cli/**',
|
||||
]);
|
||||
return {
|
||||
gitRevision: gitOutput(['rev-parse', 'HEAD']),
|
||||
relevantWorkingTreeClean: status === '',
|
||||
};
|
||||
}
|
||||
|
||||
function scoreWithCurrentScorer(artifact: Record<string, unknown>): ReceiptScore {
|
||||
const personaRecord = record(artifact.persona);
|
||||
const persona = PERSONA_CASES.find(item => item.id === personaRecord.id) as PersonaAcceptanceCase | undefined;
|
||||
const capturedScore = numberOrNull(record(artifact.score).score);
|
||||
if (!persona) {
|
||||
return {
|
||||
score: 0,
|
||||
capturedScore,
|
||||
scoreMode: 'derived-rescore',
|
||||
passed: false,
|
||||
criticalFailures: [{ code: 'persona_mismatch', detail: 'No canonical scorer case exists.' }],
|
||||
};
|
||||
}
|
||||
|
||||
const request = record(artifact.request);
|
||||
const requestPayload = record(request.payload);
|
||||
const response = record(artifact.response);
|
||||
const responseTokens = record(response.tokens);
|
||||
const runtime = record(artifact.runtime);
|
||||
const workspace = record(artifact.workspace);
|
||||
const journey = record(artifact.journey);
|
||||
const browser = record(artifact.browser);
|
||||
const codeValidation = record(artifact.codeValidation);
|
||||
const exactResponse = typeof response.exact === 'string' ? response.exact : '';
|
||||
const verifiedPython = validatePythonSyntax(exactResponse);
|
||||
const inputTokens = numberOrNull(responseTokens.input) ?? 0;
|
||||
const outputTokens = numberOrNull(responseTokens.output) ?? 0;
|
||||
const doneEventCount = numberOrNull(response.doneEventCount) ?? 0;
|
||||
const parseErrors = Array.isArray(response.parseErrors) ? response.parseErrors : [];
|
||||
const criticalBrowserErrors = [
|
||||
...(Array.isArray(browser.criticalConsoleErrors) ? browser.criticalConsoleErrors : []),
|
||||
...(Array.isArray(browser.pageErrors) ? browser.pageErrors : []),
|
||||
...(Array.isArray(browser.criticalNetworkFailures) ? browser.criticalNetworkFailures : []),
|
||||
...(Array.isArray(browser.screenshotErrors) ? browser.screenshotErrors : []),
|
||||
];
|
||||
const llmHealthy = runtime.llmHealthy === true;
|
||||
const transportError = typeof response.transportError === 'string' ? response.transportError : '';
|
||||
const evidence: PersonaTrialEvidence = {
|
||||
prompt: typeof request.exactPrompt === 'string' ? request.exactPrompt : '',
|
||||
response: exactResponse,
|
||||
persistedResponse: typeof response.persistedExact === 'string' ? response.persistedExact : '',
|
||||
sseEvents: (Array.isArray(response.sseEvents) ? response.sseEvents : []) as CapturedSseEvent[],
|
||||
toolsUsed: strings(response.toolsUsed),
|
||||
durationMs: numberOrNull(response.durationMs) ?? Number.POSITIVE_INFINITY,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
personaPersisted: workspace.personaPersisted === true,
|
||||
requestPersonaId: typeof request.personaId === 'string' ? request.personaId : null,
|
||||
expectedWorkspaceId: typeof workspace.workspaceId === 'string' ? workspace.workspaceId : '',
|
||||
requestWorkspaceId: typeof requestPayload.workspaceId === 'string' ? requestPayload.workspaceId : null,
|
||||
requestSessionId: typeof request.sessionId === 'string' ? request.sessionId : null,
|
||||
persistedSessionId: typeof response.persistedSessionId === 'string' ? response.persistedSessionId : null,
|
||||
persistedPrompt: typeof response.persistedPromptExact === 'string' ? response.persistedPromptExact : '',
|
||||
persistedMessageCount: numberOrNull(response.persistedMessageCount) ?? 0,
|
||||
tokenStreamResponse: typeof response.tokenStreamExact === 'string' ? response.tokenStreamExact : '',
|
||||
doneEventCount,
|
||||
renderedAssistantResponse: typeof response.renderedAssistantExact === 'string' ? response.renderedAssistantExact : '',
|
||||
visibleAssistantText: typeof response.visibleAssistantTextExact === 'string'
|
||||
? response.visibleAssistantTextExact
|
||||
: '',
|
||||
visibleCodeSegments: strings(response.visibleCodeSegmentsExact),
|
||||
memoryEvidencePresent: journey.memoryJourneyOk === true
|
||||
&& typeof journey.memoryText === 'string'
|
||||
&& journey.memoryText.trim().length > 0,
|
||||
workspaceLeak: Array.isArray(journey.leakedSnippets) && journey.leakedSnippets.length > 0,
|
||||
completed: response.completed === true || doneEventCount === 1,
|
||||
timedOut: response.timedOut === true || /timed?\s*out/i.test(transportError),
|
||||
corrupted: response.httpStatus !== 200
|
||||
|| parseErrors.length > 0
|
||||
|| criticalBrowserErrors.length > 0
|
||||
|| !llmHealthy
|
||||
|| inputTokens <= 0
|
||||
|| outputTokens <= 0
|
||||
|| containsFailureCopy(exactResponse),
|
||||
codeValidation: {
|
||||
pythonSyntaxValid: codeValidation.available === true
|
||||
&& codeValidation.syntaxValid === true
|
||||
&& verifiedPython.available
|
||||
&& verifiedPython.syntaxValid,
|
||||
pythonImportsPresent: codeValidation.available === true
|
||||
&& codeValidation.importsPresent === true
|
||||
&& verifiedPython.available
|
||||
&& verifiedPython.importsPresent,
|
||||
},
|
||||
};
|
||||
const current = scorePersonaTrial(persona, evidence);
|
||||
const capturedPassed = record(artifact.score).passed === true;
|
||||
return {
|
||||
score: current.score,
|
||||
capturedScore,
|
||||
scoreMode: capturedScore === current.score && capturedPassed === current.passed
|
||||
? 'captured'
|
||||
: 'derived-rescore',
|
||||
passed: current.passed,
|
||||
criticalFailures: current.criticalFailures,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildPersonaAcceptanceSeal(
|
||||
manifest: PersonaAcceptanceSealManifest,
|
||||
options: SealOptions = {},
|
||||
): PersonaAcceptanceSeal {
|
||||
const expectedPersonaIds = options.expectedPersonaIds ?? ACCEPTANCE_PERSONA_IDS;
|
||||
const repeats = options.repeats ?? 3;
|
||||
const scoreArtifact = options.scoreArtifact ?? scoreWithCurrentScorer;
|
||||
const runtimeProvenance = options.runtimeProvenance ?? currentRuntimeProvenance();
|
||||
const allowedModels = Array.isArray(manifest.allowedModels) ? manifest.allowedModels : [];
|
||||
const manifestErrors: string[] = [];
|
||||
const invalidReceipts: InvalidReceipt[] = [];
|
||||
const duplicateSlots = new Set<string>();
|
||||
const acceptedBySlot = new Map<string, SealedPersonaReceipt>();
|
||||
const seenWorkspaceIds = new Set<string>();
|
||||
const seenSessionIds = new Set<string>();
|
||||
const expectedSlots = expectedPersonaIds.flatMap(personaId =>
|
||||
Array.from({ length: repeats }, (_, index) => slot(personaId, index + 1)),
|
||||
);
|
||||
const expectedSlotSet = new Set(expectedSlots);
|
||||
|
||||
if (manifest.schemaVersion !== 1) manifestErrors.push('manifest schemaVersion must be 1');
|
||||
if (!manifest.benchmarkId?.trim()) manifestErrors.push('benchmarkId is required');
|
||||
if (!manifest.runId?.trim()) manifestErrors.push('runId is required');
|
||||
if (manifest.threshold !== 95) manifestErrors.push('threshold must be 95');
|
||||
if (manifest.repeats !== repeats) manifestErrors.push(`manifest repeats must be ${repeats}`);
|
||||
if (!/^[a-f0-9]{40}$/i.test(manifest.sourceRevision)) {
|
||||
manifestErrors.push('sourceRevision must be a full 40-character Git revision');
|
||||
}
|
||||
if (runtimeProvenance.gitRevision !== manifest.sourceRevision) {
|
||||
manifestErrors.push('executed scorer revision does not match sourceRevision');
|
||||
}
|
||||
if (!runtimeProvenance.relevantWorkingTreeClean) {
|
||||
manifestErrors.push('executed scorer relevant working tree is not clean');
|
||||
}
|
||||
if (!manifest.expectedProvider?.trim()) manifestErrors.push('expectedProvider is required');
|
||||
if (!manifest.expectedDetail?.trim()) manifestErrors.push('expectedDetail is required');
|
||||
if (allowedModels.length === 0) {
|
||||
manifestErrors.push('allowedModels must contain at least one paid model');
|
||||
} else if (new Set(allowedModels).size !== allowedModels.length) {
|
||||
manifestErrors.push('allowedModels must not contain duplicates');
|
||||
}
|
||||
|
||||
const costIds = new Set<string>();
|
||||
let diagnosticCostMicros = 0n;
|
||||
for (const entry of manifest.diagnosticCostLedger ?? []) {
|
||||
if (!entry.id?.trim()) manifestErrors.push('every cost ledger entry requires an id');
|
||||
if (costIds.has(entry.id)) manifestErrors.push(`duplicate cost ledger id: ${entry.id}`);
|
||||
costIds.add(entry.id);
|
||||
if (!entry.evidence?.trim()) manifestErrors.push(`diagnostic cost ${entry.id || '(missing id)'} requires evidence`);
|
||||
const amount = parseUsdMicros(entry.amountUsd);
|
||||
if (amount === null) manifestErrors.push(`cost ${entry.id || '(missing id)'} must use exactly six decimal places`);
|
||||
else diagnosticCostMicros += amount;
|
||||
}
|
||||
|
||||
const seenArtifactPaths = new Set<string>();
|
||||
let acceptedCostMicros = 0n;
|
||||
for (const entry of manifest.receipts ?? []) {
|
||||
const artifactPath = resolve(entry.artifactPath);
|
||||
const reasons: string[] = [];
|
||||
if (seenArtifactPaths.has(artifactPath)) reasons.push('artifact path is selected more than once');
|
||||
seenArtifactPaths.add(artifactPath);
|
||||
|
||||
let artifactBytes: Buffer | null = null;
|
||||
let artifact: Record<string, unknown> = {};
|
||||
try {
|
||||
artifactBytes = readFileSync(artifactPath);
|
||||
artifact = record(JSON.parse(artifactBytes.toString('utf8')));
|
||||
} catch (error) {
|
||||
reasons.push(`artifact could not be read as JSON: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
|
||||
const persona = record(artifact.persona);
|
||||
const personaId = typeof persona.id === 'string' ? persona.id : '';
|
||||
const repeat = numberOrNull(persona.repeat);
|
||||
const receiptSlot = personaId && repeat !== null ? slot(personaId, repeat) : null;
|
||||
if (artifact.schemaVersion !== 7) reasons.push('artifact schemaVersion must be 7');
|
||||
if (!receiptSlot || !expectedSlotSet.has(receiptSlot)) reasons.push('artifact persona/repeat is outside the expected matrix');
|
||||
if (persona.repeatCount !== repeats) reasons.push(`artifact repeatCount must be ${repeats}`);
|
||||
if (persona.gating !== true) reasons.push('artifact was not captured in gating mode');
|
||||
if (artifact.runId !== manifest.runId) reasons.push('artifact runId does not match the manifest runId');
|
||||
const artifactSource = record(artifact.source);
|
||||
if (artifactSource.gitRevision !== manifest.sourceRevision) {
|
||||
reasons.push('artifact source revision does not match the manifest');
|
||||
}
|
||||
if (artifactSource.relevantWorkingTreeClean !== true) {
|
||||
reasons.push('artifact was not captured from a clean relevant working tree');
|
||||
}
|
||||
|
||||
const runtime = record(artifact.runtime);
|
||||
const runtimeLlm = record(record(runtime.health).llm);
|
||||
if (runtime.healthStatus !== 200) reasons.push('runtime health endpoint did not return 200');
|
||||
if (runtime.llmHealthy !== true || runtimeLlm.health !== 'healthy') reasons.push('runtime LLM was not healthy');
|
||||
if (runtime.expectedProvider !== manifest.expectedProvider || runtimeLlm.provider !== manifest.expectedProvider) {
|
||||
reasons.push('runtime provider does not match the paid provider contract');
|
||||
}
|
||||
if (
|
||||
runtime.expectedDetail !== manifest.expectedDetail
|
||||
|| typeof runtimeLlm.detail !== 'string'
|
||||
|| !runtimeLlm.detail.includes(manifest.expectedDetail)
|
||||
) {
|
||||
reasons.push('runtime provider detail does not match the paid provider contract');
|
||||
}
|
||||
const response = record(artifact.response);
|
||||
if (response.httpStatus !== 200) reasons.push('chat response did not return 200');
|
||||
if (response.doneEventCount !== 1) reasons.push('chat stream did not contain exactly one done event');
|
||||
if (!Array.isArray(response.parseErrors) || response.parseErrors.length > 0) reasons.push('chat stream contained parse errors or omitted parse-error evidence');
|
||||
if (typeof response.exact !== 'string' || !response.exact.trim()) reasons.push('exact response is missing');
|
||||
const exactResponse = typeof response.exact === 'string' ? response.exact : '';
|
||||
const visibleAssistantText = typeof response.visibleAssistantTextExact === 'string'
|
||||
? response.visibleAssistantTextExact
|
||||
: '';
|
||||
if (!visibleAssistantText.trim()) reasons.push('visible assistant DOM text evidence is missing');
|
||||
else if (!visibleMarkdownPreservesText(exactResponse, visibleAssistantText)) {
|
||||
reasons.push('visible assistant DOM text did not preserve the response content');
|
||||
}
|
||||
if (
|
||||
!Array.isArray(response.visibleCodeSegmentsExact)
|
||||
|| response.visibleCodeSegmentsExact.some(segment => typeof segment !== 'string')
|
||||
) {
|
||||
reasons.push('visible assistant DOM code evidence is missing or malformed');
|
||||
} else if (!markdownCodeSegmentsMatch(exactResponse, response.visibleCodeSegmentsExact as string[])) {
|
||||
reasons.push('visible assistant DOM code did not match the response Markdown');
|
||||
}
|
||||
const model = typeof response.model === 'string' ? response.model : '';
|
||||
if (!model || !allowedModels.includes(model)) reasons.push('response model is missing or not allowed');
|
||||
const costMicros = estimatedCostMicros(response.estimatedCostUsd);
|
||||
if (costMicros === null) reasons.push('positive Waggle-estimated cost with at most six decimal places is required');
|
||||
const doneEvents = Array.isArray(response.sseEvents)
|
||||
? response.sseEvents.map(record).filter(event => event.event === 'done')
|
||||
: [];
|
||||
const doneData = record(doneEvents[0]?.data);
|
||||
const doneCostMicros = estimatedCostMicros(doneData.cost);
|
||||
if (doneEvents.length !== 1 || doneCostMicros === null || doneCostMicros !== costMicros) {
|
||||
reasons.push('estimated cost does not match the single done event');
|
||||
}
|
||||
if (doneData.model !== model) reasons.push('response model does not match the provider done event');
|
||||
const workspace = record(artifact.workspace);
|
||||
const request = record(artifact.request);
|
||||
const workspaceId = typeof workspace.workspaceId === 'string' ? workspace.workspaceId : '';
|
||||
const sessionId = typeof request.sessionId === 'string' ? request.sessionId : '';
|
||||
if (!workspaceId) reasons.push('workspace id is missing');
|
||||
else if (seenWorkspaceIds.has(workspaceId)) reasons.push('workspace id is reused across receipts');
|
||||
if (!sessionId) reasons.push('session id is missing');
|
||||
else if (seenSessionIds.has(sessionId)) reasons.push('session id is reused across receipts');
|
||||
if (workspaceId) seenWorkspaceIds.add(workspaceId);
|
||||
if (sessionId) seenSessionIds.add(sessionId);
|
||||
const browser = record(artifact.browser);
|
||||
for (const key of ['criticalConsoleErrors', 'pageErrors', 'criticalNetworkFailures', 'screenshotErrors']) {
|
||||
if (!Array.isArray(browser[key]) || (browser[key] as unknown[]).length > 0) {
|
||||
reasons.push(`browser ${key} evidence is missing or non-empty`);
|
||||
}
|
||||
}
|
||||
|
||||
const rescored = scoreArtifact(artifact);
|
||||
if (!rescored.passed || rescored.score < manifest.threshold) reasons.push(`current scorer returned ${rescored.score}/100`);
|
||||
if (rescored.criticalFailures.length > 0) reasons.push('current scorer reported critical failures');
|
||||
if (receiptSlot && acceptedBySlot.has(receiptSlot)) {
|
||||
duplicateSlots.add(receiptSlot);
|
||||
reasons.push('persona/repeat slot is selected more than once');
|
||||
}
|
||||
|
||||
if (reasons.length > 0 || !receiptSlot || !artifactBytes || repeat === null || costMicros === null) {
|
||||
invalidReceipts.push({ artifactPath, slot: receiptSlot, reasons });
|
||||
continue;
|
||||
}
|
||||
|
||||
acceptedCostMicros += costMicros;
|
||||
const responseTokens = record(response.tokens);
|
||||
acceptedBySlot.set(receiptSlot, {
|
||||
slot: receiptSlot,
|
||||
personaId,
|
||||
repeat,
|
||||
runId: manifest.runId,
|
||||
artifactPath,
|
||||
artifactSha256: sha256(artifactBytes),
|
||||
responseSha256: sha256(response.exact as string),
|
||||
sourceRevision: manifest.sourceRevision,
|
||||
scorerRevision: manifest.sourceRevision,
|
||||
capturedScore: rescored.capturedScore,
|
||||
score: rescored.score,
|
||||
scoreMode: rescored.scoreMode,
|
||||
model,
|
||||
provider: manifest.expectedProvider,
|
||||
estimatedCostUsd: formatUsdMicros(costMicros),
|
||||
workspaceId,
|
||||
sessionId,
|
||||
inputTokens: numberOrNull(responseTokens.input),
|
||||
outputTokens: numberOrNull(responseTokens.output),
|
||||
durationMs: numberOrNull(response.durationMs),
|
||||
});
|
||||
}
|
||||
|
||||
const missingSlots = expectedSlots.filter(item => !acceptedBySlot.has(item));
|
||||
const receipts = [...acceptedBySlot.values()].sort((a, b) => a.slot.localeCompare(b.slot));
|
||||
const failed = manifestErrors.length > 0 || invalidReceipts.length > 0 || duplicateSlots.size > 0;
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
benchmarkId: manifest.benchmarkId,
|
||||
manifestSha256: sha256(JSON.stringify(manifest)),
|
||||
status: failed ? 'failed' : missingSlots.length > 0 ? 'incomplete' : 'ready',
|
||||
threshold: 95,
|
||||
repeats,
|
||||
expectedReceiptCount: expectedSlots.length,
|
||||
completedReceiptCount: receipts.length,
|
||||
missingSlots,
|
||||
duplicateSlots: [...duplicateSlots].sort(),
|
||||
invalidReceipts,
|
||||
manifestErrors,
|
||||
acceptedEstimatedCostUsd: formatUsdMicros(acceptedCostMicros),
|
||||
diagnosticRecordedCostUsd: formatUsdMicros(diagnosticCostMicros),
|
||||
totalRecordedSpendUsd: formatUsdMicros(acceptedCostMicros + diagnosticCostMicros),
|
||||
diagnosticCostLedger: (manifest.diagnosticCostLedger ?? []).map(entry => ({ ...entry })),
|
||||
receipts,
|
||||
};
|
||||
}
|
||||
378
tests/vision/persona-cases.ts
Normal file
378
tests/vision/persona-cases.ts
Normal file
@@ -0,0 +1,378 @@
|
||||
import {
|
||||
CANONICAL_VERIFIER_REPORT,
|
||||
VERIFIER_BLOCKER_CHECK_PAIRS,
|
||||
VERIFIER_NEXT_CHECK_KEYS,
|
||||
VERIFIER_REPORT_CLOSE,
|
||||
VERIFIER_REPORT_OPEN,
|
||||
VERIFIER_TOP_LEVEL_KEYS,
|
||||
} from './verifier-contract';
|
||||
|
||||
export const ACCEPTANCE_PERSONA_IDS = [
|
||||
'general-purpose',
|
||||
'researcher',
|
||||
'writer',
|
||||
'project-manager',
|
||||
'executive-assistant',
|
||||
'finance-owner',
|
||||
'coder',
|
||||
'data-engineer',
|
||||
'verifier',
|
||||
'coordinator',
|
||||
] as const;
|
||||
|
||||
export type AcceptancePersonaId = typeof ACCEPTANCE_PERSONA_IDS[number];
|
||||
|
||||
interface BaseResponseRule {
|
||||
id: string;
|
||||
description: string;
|
||||
points: number;
|
||||
}
|
||||
|
||||
export type PersonaResponseRule =
|
||||
| (BaseResponseRule & { kind: 'pattern'; pattern: RegExp })
|
||||
| (BaseResponseRule & { kind: 'dependencyMap' })
|
||||
| (BaseResponseRule & { kind: 'timedAgenda'; durationMinutes: number; minimumBlocks: number })
|
||||
| (BaseResponseRule & { kind: 'runwayResult' })
|
||||
| (BaseResponseRule & { kind: 'runwayFormula' })
|
||||
| (BaseResponseRule & { kind: 'runwayAssumption' })
|
||||
| (BaseResponseRule & { kind: 'runwayActions'; patterns: readonly RegExp[] })
|
||||
| (BaseResponseRule & { kind: 'writerReleaseFacts'; patterns: readonly RegExp[] })
|
||||
| (BaseResponseRule & { kind: 'emptyWorkspaceResult' })
|
||||
| (BaseResponseRule & { kind: 'boundedWorkspaceClaims' })
|
||||
| (BaseResponseRule & {
|
||||
kind: 'prioritizationJustification';
|
||||
criteria: readonly {
|
||||
topic: RegExp;
|
||||
basis: RegExp;
|
||||
basisFamilies?: readonly RegExp[];
|
||||
}[];
|
||||
})
|
||||
| (BaseResponseRule & { kind: 'allPatterns'; patterns: readonly RegExp[] })
|
||||
| (BaseResponseRule & { kind: 'notPattern'; pattern: RegExp })
|
||||
| (BaseResponseRule & { kind: 'verifierContract' })
|
||||
| (BaseResponseRule & { kind: 'maxWords'; maxWords: number })
|
||||
| (BaseResponseRule & {
|
||||
kind: 'primaryEvidence';
|
||||
minimum: number;
|
||||
allowedDomains: readonly string[];
|
||||
requiredSourceGroups?: readonly (readonly string[])[];
|
||||
})
|
||||
| (BaseResponseRule & { kind: 'codeValidation'; language: 'python' });
|
||||
|
||||
export interface PersonaAcceptanceCase {
|
||||
id: AcceptancePersonaId;
|
||||
label: string;
|
||||
prompt: string;
|
||||
/** Every acceptance prompt is intentionally advisory/read-only. */
|
||||
readOnly: true;
|
||||
maxDurationMs: number;
|
||||
maxInputTokens: number;
|
||||
maxOutputTokens: number;
|
||||
/** At least one successful tool must match every listed pattern. */
|
||||
requiredToolPatterns: readonly RegExp[];
|
||||
responseRules: readonly PersonaResponseRule[];
|
||||
}
|
||||
|
||||
const sqlitePrimaryResearchDomains = [
|
||||
'sqlite.org',
|
||||
'sqlite.ai',
|
||||
'github.com/sqliteai/sqlite-vector',
|
||||
'github.com/asg017/sqlite-vec',
|
||||
'raw.githubusercontent.com/asg017/sqlite-vec',
|
||||
] as const;
|
||||
|
||||
const postgresPrimaryResearchDomains = [
|
||||
'postgresql.org',
|
||||
'github.com/pgvector/pgvector',
|
||||
'raw.githubusercontent.com/pgvector/pgvector',
|
||||
] as const;
|
||||
|
||||
const primaryResearchDomains = [
|
||||
...sqlitePrimaryResearchDomains,
|
||||
...postgresPrimaryResearchDomains,
|
||||
] as const;
|
||||
|
||||
const affirmedFactClause = String.raw`(?<!not true that )(?<!not true that the )(?<!not true that \*\*)(?<!not true that __)(?<!not true that \*)(?<!not true that _)(?<!false that )(?<!false that the )(?<!false that \*\*)(?<!false that __)(?<!false that \*)(?<!false that _)`;
|
||||
const affirmedBrowserTests = `${affirmedFactClause}${String.raw`\bbrowser test(?:s|ing)\b`}`;
|
||||
const positiveFailureVerb = String.raw`(?<!not )(?<!cannot )(?<!can't )(?<!don't )(?<!doesn't )(?<!didn't )(?<!aren't )(?<!isn't )(?<!never )(?<!no longer )\b(?:currently\s+show(?:s|ing)?\s+(?:two|2)\s+remaining\s+failures?|(?:(?:still\s+)?(?:show(?:s|ing)?|have|report(?:s|ing)?|return(?:s|ing)?|produce(?:s|ing)?)|remain(?:s|ing)?)\s+(?:two|2)\s+failures?)\b`;
|
||||
const windowsBrowserFailuresPattern = new RegExp([
|
||||
`${String.raw`(?<!could )(?<!can )(?<!may )(?<!might )`}${affirmedBrowserTests}${String.raw`(?![^.\r\n]*\?)[ \t]+(?:currently[ \t]+|still[ \t]+)?(?:show(?:s|ing)?|is[ \t]+showing|report(?:s|ing)?|has|found)\s+(?:two|2)\s+(?:unresolved|open)\s+failures?\b(?![^.\r\n]{0,80}\b(?:incorrect|wrong|false|resolved|untrue|not[ \t]+true|disputed)\b)[^.\r\n]{0,60}\bWindows\b`}`,
|
||||
`${affirmedBrowserTests}${String.raw`[^.\r\n]{0,80}\bWindows\b[^.\r\n]{0,50}`}${positiveFailureVerb}`,
|
||||
`${affirmedBrowserTests}${String.raw`[^.\r\n]{0,60}`}${positiveFailureVerb}${String.raw`[^.\r\n]{0,60}\bWindows\b`}`,
|
||||
`${affirmedBrowserTests}${String.raw`[^.\r\n]{0,30}\b(?:two|2)\s+failures?\b[^.\r\n]{0,20}\b(?:remain|persist|exist)\b[^.\r\n]{0,60}\bWindows\b`}`,
|
||||
`${affirmedFactClause}${String.raw`(?<!not )(?<!no longer )\b(?:two|2)\s+browser test failures?\s+(?:still\s+)?(?:persist|remain|exist)\b[^.\r\n]{0,60}\bWindows\b`}`,
|
||||
].join('|'), 'i');
|
||||
const positiveRecommendationLead = String.raw`(?:(?<!cannot )(?<!can't )(?<!not )\b(?:recommend(?:ation|ed)?)\b(?:(?!\b(?:not|never|cannot|can't|avoid|against)\b)[\s\S]){0,80}|(?:^|[\r\n])[ \t]*(?:[-*#>]+[ \t]*)?(?:\*\*)?|(?:^|[.!?]\s+|[\r\n])[ \t]*(?:[-*#>]+[ \t]*)?(?:we|you|the team)[ \t]+should[ \t]+)`;
|
||||
const delayRecommendationPattern = new RegExp(`${positiveRecommendationLead}${String.raw`\bdelay(?:ing)?\s+(?:the\s+)?release\b[\s\S]{0,240}\b(?:until|once)\b[\s\S]{0,180}(?:gaps?|failures?|smart router|cloud credentials)`}`, 'im');
|
||||
const positiveActionLead = String.raw`(?:(?:^|[.!?]\s+|[\r\n])[ \t]*(?:(?:\d+[.)]|[-*])[ \t]*|\|[ \t]*\d+[ \t]*\|[ \t]*)?(?:\*\*)?(?:(?:we|you|the team)[ \t]+should[ \t]+)?|\b(?:actions?|recommend(?:ation|ed)?)\b(?:(?!\b(?:not|never|cannot|can't|avoid|against)\b)[^.\r\n]){0,80})`;
|
||||
const positiveActionSuffix = String.raw`(?![^.\r\n]{0,80}(?:\?|\b(?:cannot|can't|do not|don't|must not|should not|never|impossible|merely reported|no longer recommend(?:ed|ing)?|(?:not|(?:is|are|was|were)n['’]t)[ \t]+(?:(?:an?|the|this|that|my|your|our|their|his|her|its)[ \t]+)?recommendations?|decid(?:e[sd]?|ing) against|not (?:advisable|feasible|possible|recommended))\b))`;
|
||||
const costActionPattern = new RegExp(`${positiveActionLead}${String.raw`\b(?:reduce|cut|lower)\b[^.\r\n]{0,60}(?:costs?|burn)`}${positiveActionSuffix}`, 'im');
|
||||
const cashActionPattern = new RegExp(`${positiveActionLead}${String.raw`\b(?:(?:increase|generate|grow|close|raise|start[ \t]+generating)\b[^.\r\n]{0,80}(?:revenue|customers?|funding|cash inflows?)|(?:create|add)\b[ \t]+near[- ]term[ \t]+(?:revenue|cash inflows?)|(?:pull forward|accelerate|improve|speed up)\b[^.\r\n]{0,80}(?:cash inflows?|payments?|collections?|receivables?))`}${positiveActionSuffix}`, 'im');
|
||||
const verifierPairInstructions = VERIFIER_BLOCKER_CHECK_PAIRS
|
||||
.map(([blocker, [operation, target, passCondition]]) => (
|
||||
`${blocker} => ${JSON.stringify({ operation, target, passCondition })}`
|
||||
))
|
||||
.join('; ');
|
||||
|
||||
const verifierAcceptancePrompt = [
|
||||
'A teammate claims the product is production-ready because the web build passed.',
|
||||
`Return exactly one ${VERIFIER_REPORT_OPEN}...${VERIFIER_REPORT_CLOSE} JSON envelope and no text before or after it.`,
|
||||
`Use schemaVersion ${CANONICAL_VERIFIER_REPORT.schemaVersion}, scenarioId ${JSON.stringify(CANONICAL_VERIFIER_REPORT.scenarioId)}, evidenceScope ${JSON.stringify(CANONICAL_VERIFIER_REPORT.evidenceScope)}, facts exactly ${JSON.stringify(CANONICAL_VERIFIER_REPORT.facts)}, unsupportedClaims exactly ${JSON.stringify(CANONICAL_VERIFIER_REPORT.unsupportedClaims)}, verdict ${JSON.stringify(CANONICAL_VERIFIER_REPORT.verdict)}, and releaseDecision ${JSON.stringify(CANONICAL_VERIFIER_REPORT.releaseDecision)}.`,
|
||||
`Use exactly these top-level keys and no others: ${VERIFIER_TOP_LEVEL_KEYS.join(', ')}. Each nextChecks object has exactly these keys and no others: ${VERIFIER_NEXT_CHECK_KEYS.join(', ')}. Spell all keys literally; do not escape or duplicate keys.`,
|
||||
`Include one or more unique blocker/check pairs and no unmatched blockers or checks: ${verifierPairInstructions}.`,
|
||||
'Put selected blocker ids in blockerCodes and their paired check objects in nextChecks. Do not create or edit files.',
|
||||
].join(' ');
|
||||
|
||||
export const PERSONA_CASES: readonly PersonaAcceptanceCase[] = [
|
||||
{
|
||||
id: 'general-purpose',
|
||||
label: 'Prioritization under ambiguity',
|
||||
prompt: 'I have three priorities this week: close one customer, repair onboarding friction, and investigate a production memory bug. Choose the order, justify it in one concise plan, and identify the first action for today. Do not ask clarifying questions; make reasonable assumptions.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 45_000,
|
||||
maxInputTokens: 15_000,
|
||||
maxOutputTokens: 2_500,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'all-priorities', description: 'Addresses all three supplied priorities', kind: 'allPatterns', patterns: [/customer/i, /onboarding/i, /memory (?:bug|issue)/i], points: 10 },
|
||||
{ id: 'ordered-plan', description: 'Provides an explicit order', kind: 'pattern', pattern: /(?:priority order|\b1[.)]|\bfirst\b[\s\S]*\bsecond\b|(?:^|[\r\n])\s*(?:\*\*)?order(?:\*\*)?\s*:\s*[^\r\n]*(?:\u2192|->|=>)[^\r\n]*(?:\u2192|->|=>))/im, points: 10 },
|
||||
{
|
||||
id: 'justification',
|
||||
description: 'Links each priority to a relevant decision basis',
|
||||
kind: 'prioritizationJustification',
|
||||
criteria: [
|
||||
{
|
||||
topic: /\b(?:(?:production|memory) (?:bugs?|issues?)|memory leak)\b/i,
|
||||
basis: /(?:\b(?:live|active) in production\b[^.!?\r\n]{0,220}\b(?:may|might|could|would)\b(?:(?![.!?\r\n]|\b(?:not|never|no|without|lacks?|cannot|fails?|unlikely)\b).){0,140}\b(?:degrad(?:e[ds]?|ation)|outage)\b(?:(?![.!?\r\n]|\b(?:not|never|no|without|lacks?|cannot|fails?|unlikely)\b).){0,180}\bcompounding (?:downside )?risk if delayed\b|\b(?:risk|reliab(?:ility|le)|stabil(?:ity|ize)|outage|trust|blast radius|unbounded downside|degrad(?:e[ds]?|ation)|crash(?:es|ed|ing)?)\b)/i,
|
||||
basisFamilies: [
|
||||
/\b(?:reliab(?:ility|le)|stabil(?:ity|ize)|trust)\b/i,
|
||||
/\b(?:outage|degrad(?:e[ds]?|ation)|crash(?:es|ed|ing)?)\b/i,
|
||||
/\b(?:risk|blast radius|unbounded downside)\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
topic: /\b(?:customer|deal)s?\b/i,
|
||||
basis: /\b(?:revenue|pipeline|cash|commercial|near[- ]term|closable|proof points?|de-risk|signature|close date|deadline|immediate (?:payoff|value)|high(?:est)?[- ](?:value|leverage)|time[- ](?:sensitive|boxed)|decision (?:clock|point)|external momentum|deal urgency|urgency|momentum)\b/i,
|
||||
basisFamilies: [
|
||||
/\b(?:revenue|cash|commercial|near[- ]term|immediate (?:payoff|value)|high(?:est)?[- ](?:value|leverage))\b/i,
|
||||
/\b(?:pipeline|closable|proof points?|de-risk|signature|close date|deadline|time[- ](?:sensitive|boxed)|decision (?:clock|point)|external momentum|deal urgency|urgency|momentum)\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
topic: /\bonboarding\b/i,
|
||||
basis: /\b(?:conversion|retention|activation|drop[- ]?off|sales drag|high leverage|less urgent|not urgent|structural|future throughput|support load|reliab(?:ility|le)|friction|crash|retry|user experience|growth)\b/i,
|
||||
basisFamilies: [
|
||||
/\b(?:conversion|activation|drop[- ]?off|sales drag|growth)\b/i,
|
||||
/\b(?:retention|support load|friction|retry|user experience)\b/i,
|
||||
/\b(?:reliab(?:ility|le)|crash)\b/i,
|
||||
/\b(?:high leverage|less urgent|not urgent|structural|future throughput)\b/i,
|
||||
],
|
||||
},
|
||||
],
|
||||
points: 10,
|
||||
},
|
||||
{ id: 'first-action', description: 'Names the first action for today', kind: 'pattern', pattern: /(?:first action|today(?:'s)? action|start today|begin today|\btoday\s*:)/i, points: 10 },
|
||||
{ id: 'no-followup', description: 'Does not end by reopening clarification', kind: 'notPattern', pattern: /\?\s*$/, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'researcher',
|
||||
label: 'Primary-source technical comparison',
|
||||
prompt: 'Use current primary sources to compare SQLite vector search with PostgreSQL plus pgvector for a single-user desktop AI memory store. Give a decision table and a recommendation. Cite source URLs, distinguish facts from inference, and do not claim a benchmark you did not find.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 120_000,
|
||||
maxInputTokens: 60_000,
|
||||
maxOutputTokens: 6_000,
|
||||
requiredToolPatterns: [/(?:search|fetch|browse)/i],
|
||||
responseRules: [
|
||||
{
|
||||
id: 'primary-sources',
|
||||
description: 'Cites and fetches primary evidence for both sides of the comparison',
|
||||
kind: 'primaryEvidence',
|
||||
minimum: 2,
|
||||
allowedDomains: primaryResearchDomains,
|
||||
requiredSourceGroups: [sqlitePrimaryResearchDomains, postgresPrimaryResearchDomains],
|
||||
points: 10,
|
||||
},
|
||||
{ id: 'decision-table', description: 'Includes a comparison or decision table', kind: 'pattern', pattern: /(?:decision table|\|\s*(?:criterion|dimension|factor|consideration)\s*\|)/i, points: 10 },
|
||||
{ id: 'recommendation', description: 'Makes a recommendation for the stated desktop use case', kind: 'pattern', pattern: /recommend(?:ation|ed)?/i, points: 10 },
|
||||
{
|
||||
id: 'fact-inference',
|
||||
description: 'Separates sourced facts from inference',
|
||||
kind: 'allPatterns',
|
||||
patterns: [
|
||||
/(?:(?:^|\n)#{1,6}\s*(?:key\s+)?facts?\b(?=[ \t]*(?::|$)|[ \t]+(?:from|based[ \t]+on|verified|confirmed|sourced)\b)|(?:^|\n)#{1,6}[ \t]*(?:what(?:'s| is)[ \t]+)?(?:verified|confirmed|sourced)\b|\*\*(?:what(?:'s| is)\s+)?(?:verified|confirmed|sourced)\b[^*\r\n]{0,80}\*\*|\*\*(?:key\s+)?facts?\b(?=[ \t]*(?::|\*\*)|[ \t]+(?:from|based[ \t]+on|verified|confirmed|sourced|supporting[ \t]+(?:this|the)[ \t]+(?:recommendation|comparison|decision))\b)[^*\r\n]{0,80}\*\*|\(\s*facts?\b(?=[ \t]*(?::|\))|[ \t]+(?:from|based[ \t]+on|verified|confirmed|sourced)\b)[^)]{0,200}\)|\(\s*facts?\s*,\s*\[(?![^\]\r\n]{0,80}\b(?:not(?:\s+(?:yet|independently))?\s+(?:verified|confirmed|sourced)|never\s+sourced|no\s+(?:facts?|source|evidence)|unverified|unavailable|missing|unknown|unsourced|unconfirmed|none|opinion)\b)[^\]\r\n]+\]\(https?:\/\/[^)\s]+\)\s*\)|(?:^|[|(\r\n.])[ \t]*(?:[-*+][ \t]+)?facts?(?:[ \t]*\/\s*inference\s*)?(?:[ \t]*[:)]|[ \t]+(?:[\u2013\u2014-]|(?:for|from)\b))(?![^\r\n|]{0,80}\b(?:not(?:\s+(?:yet|independently))?\s+(?:verified|confirmed|sourced)|never\s+sourced|no\s+(?:facts?|source|evidence)|unverified|unavailable|missing|unknown|unsourced|unconfirmed|none)\b)|(?:^|[|\r\n.])[ \t]*(?:[-*+][ \t]+)?(?:\*\*)?facts?[ \t]*\((?![^\r\n)]{0,80}\b(?:inferences?|not|never|no|unverified|unavailable|missing|unknown|unsourced|unconfirmed)\b)[^\r\n)]{1,80}\))/im,
|
||||
/(?:(?:^|\n)#{1,6}\s*(?:key\s+)?inferences?\b|\*\*[^*\r\n]{0,80}\binferences?(?:\s*\/\s*fact)?\b[^*\r\n]{0,80}\*\*|\(\s*inferences?(?:\s*\/\s*fact)?\b[^)]{0,200}\)|\binferences?\s*(?:\/\s*fact\s*)?[:)]|(?:^|[|\r\n.])[ \t]*(?:[-*+][ \t]+)?(?:\*\*)?inferences?[ \t]*\((?![^\r\n)]{0,80}\bfacts?\b)[^\r\n)]{1,80}\))/im,
|
||||
],
|
||||
points: 10,
|
||||
},
|
||||
{ id: 'source-quality', description: 'Avoids known secondary AI-synthesized sources', kind: 'notPattern', pattern: /(?:grokipedia|deepwiki)/i, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'writer',
|
||||
label: 'Fact-preserving executive rewrite',
|
||||
prompt: 'Rewrite this into a crisp executive memo of at most 120 words. Preserve the facts and add no new claims: We planned to ship Friday. API tests pass. Browser tests still have two failures on Windows. The smart router has not been exercised without cloud credentials. Recommendation: delay release until those gaps are closed.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 30_000,
|
||||
maxInputTokens: 12_000,
|
||||
maxOutputTokens: 1_500,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'word-limit', description: 'Stays within 120 words', kind: 'maxWords', maxWords: 120, points: 10 },
|
||||
{ id: 'release-facts', description: 'Preserves Friday, passing API tests, and two Windows browser-test failures', kind: 'writerReleaseFacts', patterns: [/Friday/i, /API tests?\b\s*(?:(?:\*\*|__)\s*)?:?\s*(?:(?:\*\*|__)\s*)?(?:(?:are\s+)?pass(?:ed|ing)?|have\s+passed)\b/i, windowsBrowserFailuresPattern], points: 10 },
|
||||
{ id: 'router-fact', description: 'Preserves the unexercised smart-router/cloud-credentials fact', kind: 'allPatterns', patterns: [/smart router/i, /not (?:(?:yet|been|fully|thoroughly)\s+)*(?:exercised|tested|validated)/i, /cloud credentials/i], points: 10 },
|
||||
{ id: 'recommendation', description: 'Preserves a positive delay recommendation and its condition', kind: 'pattern', pattern: delayRecommendationPattern, points: 10 },
|
||||
{ id: 'no-new-claims', description: 'Avoids known invented risk and schedule claims', kind: 'notPattern', pattern: /(?:production-equivalent|unacceptable (?:post-release )?incident risk|short hold|not a scope change|revised ship date|\bunverified\s+risk\b|\brisk\s+to\s+(?:release\s+)?stability\b)/i, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'project-manager',
|
||||
label: 'Evidence-bounded release plan',
|
||||
prompt: 'Turn this release goal into milestones, dependencies, owners by role, risks, and exit criteria: production-ready solo installation with no Docker dependency, local models and proxy included, a functioning smart router, and verified Windows behavior. Do not create or edit anything.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 45_000,
|
||||
maxInputTokens: 18_000,
|
||||
maxOutputTokens: 3_500,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'milestones', description: 'Defines milestones', kind: 'pattern', pattern: /milestones?/i, points: 10 },
|
||||
{ id: 'dependencies', description: 'Maps dependencies', kind: 'dependencyMap', points: 10 },
|
||||
{ id: 'owners', description: 'Assigns owners by role', kind: 'allPatterns', patterns: [/owners?/i, /role/i], points: 10 },
|
||||
{ id: 'risks-exit', description: 'Includes risks and exit criteria', kind: 'allPatterns', patterns: [/risks?/i, /exit criteria/i], points: 10 },
|
||||
{ id: 'no-invented-schedule', description: 'Does not invent a calendar schedule', kind: 'notPattern', pattern: /(?:week\s*\d+|\d+[ -]?week effort|target date:)/i, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'executive-assistant',
|
||||
label: 'Launch-readiness agenda',
|
||||
prompt: 'Draft a 30-minute launch-readiness meeting agenda with time blocks, desired decisions, and a short pre-read checklist. Participants are product, engineering, QA, and support. Do not create a calendar event and do not ask follow-up questions.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 30_000,
|
||||
maxInputTokens: 12_000,
|
||||
maxOutputTokens: 2_000,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'duration-blocks', description: 'Uses time blocks for a 30-minute meeting', kind: 'timedAgenda', durationMinutes: 30, minimumBlocks: 2, points: 10 },
|
||||
{ id: 'decisions', description: 'Names desired decisions', kind: 'pattern', pattern: /desired decisions?|decision(?:s| owner)/i, points: 10 },
|
||||
{ id: 'preread', description: 'Provides a pre-read checklist', kind: 'allPatterns', patterns: [/pre-read/i, /(?:checklist|\[[ x]\])/i], points: 10 },
|
||||
{ id: 'participants', description: 'Covers all four participant groups', kind: 'allPatterns', patterns: [/product/i, /engineering/i, /\bQA\b/i, /support/i], points: 10 },
|
||||
{ id: 'no-followup', description: 'Does not ask a follow-up or offer an action', kind: 'notPattern', pattern: /\?\s*$/, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'finance-owner',
|
||||
label: 'Runway calculation and action',
|
||||
prompt: 'Cash is 40000 dollars, monthly burn is 10000 dollars, and revenue is zero. Calculate runway in months, state the formula, name the biggest assumption, and give two actions that improve runway. Do not create files or schedules.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 30_000,
|
||||
maxInputTokens: 12_000,
|
||||
maxOutputTokens: 2_000,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'runway', description: 'Positively calculates four months of runway', kind: 'runwayResult', points: 10 },
|
||||
{ id: 'formula', description: 'States cash divided by monthly net burn', kind: 'runwayFormula', points: 10 },
|
||||
{ id: 'assumption', description: 'Names the constant-burn/no-revenue assumption', kind: 'runwayAssumption', points: 10 },
|
||||
{ id: 'two-actions', description: 'Gives positive cost and revenue or cash-inflow actions', kind: 'runwayActions', patterns: [costActionPattern, cashActionPattern], points: 10 },
|
||||
{ id: 'no-false-impact', description: 'Avoids false dollar-to-month claims and schedule CTAs', kind: 'notPattern', pattern: /(?:each dollar saved.*(?:one|1).*month|\/schedule|calendar event)/i, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'coder',
|
||||
label: 'Workspace-bounded inspection',
|
||||
prompt: 'Inspect only this current virtual workspace and report exactly what files exist before recommending one next engineering step. Do not create or edit files. Do not inspect parent directories or any repository outside this workspace. Do not claim inspection unless a tool succeeds.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 45_000,
|
||||
maxInputTokens: 20_000,
|
||||
maxOutputTokens: 2_500,
|
||||
requiredToolPatterns: [/(?:search_files|list_workspace_files|read_file)/i],
|
||||
responseRules: [
|
||||
{ id: 'workspace-scope', description: 'Reports on the current workspace', kind: 'pattern', pattern: /workspace/i, points: 10 },
|
||||
{ id: 'empty-result', description: 'Accurately reports the fresh virtual workspace as empty', kind: 'emptyWorkspaceResult', points: 10 },
|
||||
{ id: 'next-step', description: 'Recommends one next engineering step', kind: 'pattern', pattern: /(?:next (?:engineering )?step|recommended next step)/i, points: 10 },
|
||||
{ id: 'bounded-claim', description: 'Does not claim parent or external repository contents', kind: 'boundedWorkspaceClaims', points: 10 },
|
||||
{ id: 'concise', description: 'Keeps an empty-workspace report concise', kind: 'maxWords', maxWords: 300, points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'data-engineer',
|
||||
label: 'Idempotent ETL design',
|
||||
prompt: 'Design an idempotent ETL from newline-delimited JSON events into SQLite. Include schema, deduplication key, transaction strategy, retry behavior, and a compact Python example. The example must be syntactically valid and include all imports. Do not write files or execute code; provide the example as text only.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 60_000,
|
||||
maxInputTokens: 25_000,
|
||||
maxOutputTokens: 4_500,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'schema', description: 'Includes a concrete SQLite schema', kind: 'pattern', pattern: /CREATE\s+TABLE/i, points: 10 },
|
||||
{ id: 'deduplication', description: 'Defines a deduplication key or constraint', kind: 'pattern', pattern: /(?:dedup(?:lication)? key|PRIMARY KEY|UNIQUE\s*\()/i, points: 10 },
|
||||
{ id: 'transaction', description: 'Defines transaction boundaries', kind: 'pattern', pattern: /(?:BEGIN\b|transaction)/i, points: 10 },
|
||||
{ id: 'retry', description: 'Defines retry/backoff behavior', kind: 'pattern', pattern: /(?:retry|backoff|busy_timeout)/i, points: 10 },
|
||||
{ id: 'python-valid', description: 'Provides syntactically valid Python with imports', kind: 'codeValidation', language: 'python', points: 10 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'verifier',
|
||||
label: 'Typed evidence-only production verdict',
|
||||
prompt: verifierAcceptancePrompt,
|
||||
readOnly: true,
|
||||
maxDurationMs: 30_000,
|
||||
maxInputTokens: 12_000,
|
||||
maxOutputTokens: 2_500,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{
|
||||
id: 'verifier-contract',
|
||||
description: 'Emits one strict, internally consistent, evidence-bounded VerifierReportV1 contract',
|
||||
kind: 'verifierContract',
|
||||
points: 50,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'coordinator',
|
||||
label: 'Two-lane review decomposition',
|
||||
prompt: 'Decompose a production-readiness review into one researcher lane and one coder lane. Specify each lane objective, inputs, deliverables, dependencies, merge criteria, and what the coordinator must verify before accepting either result. Do not create or edit files and do not launch agents.',
|
||||
readOnly: true,
|
||||
maxDurationMs: 45_000,
|
||||
maxInputTokens: 18_000,
|
||||
maxOutputTokens: 3_000,
|
||||
requiredToolPatterns: [],
|
||||
responseRules: [
|
||||
{ id: 'two-lanes', description: 'Defines researcher and coder lanes', kind: 'allPatterns', patterns: [/(?:\bresearcher\s+lane\b|^[ \t]*(?:#{1,6}[ \t]+)?(?:\*\*)?lane\s+\d+\s*[-\u2013\u2014:]\s*researcher(?:[ \t]+\([^\r\n)]+\))?[ \t]*(?:\*\*)?[ \t]*:?[ \t]*$|^[ \t]*#{1,6}[ \t]+(?:\*\*)?lane\s+\d+\s*[-\u2013\u2014:]\s*researcher(?:[ \t]+\([^\r\n)]+\))?[ \t]*(?:\*\*)?[ \t]*[-\u2013\u2014:][ \t]+(?!not\b)\S[^\r\n]*$)/im, /(?:\bcoder\s+lane\b|^[ \t]*(?:#{1,6}[ \t]+)?(?:\*\*)?lane\s+\d+\s*[-\u2013\u2014:]\s*coder(?:[ \t]+\([^\r\n)]+\))?[ \t]*(?:\*\*)?[ \t]*:?[ \t]*$|^[ \t]*#{1,6}[ \t]+(?:\*\*)?lane\s+\d+\s*[-\u2013\u2014:]\s*coder(?:[ \t]+\([^\r\n)]+\))?[ \t]*(?:\*\*)?[ \t]*[-\u2013\u2014:][ \t]+(?!not\b)\S[^\r\n]*$)/im], points: 10 },
|
||||
{ id: 'lane-contracts', description: 'Provides objectives, inputs, and deliverables', kind: 'allPatterns', patterns: [/objectives?/i, /inputs?/i, /deliverables?/i], points: 10 },
|
||||
{ id: 'dependencies', description: 'Defines dependencies', kind: 'pattern', pattern: /dependenc(?:y|ies)/i, points: 10 },
|
||||
{ id: 'merge', description: 'Defines merge criteria', kind: 'pattern', pattern: /merge criteria/i, points: 10 },
|
||||
{ id: 'acceptance', description: 'Defines coordinator verification before acceptance', kind: 'allPatterns', patterns: [/coordinator/i, /verif(?:y|ication)|accept/i], points: 5 },
|
||||
{ id: 'no-generic-inventions', description: 'Avoids unrelated compliance and deployment inventions', kind: 'notPattern', pattern: /(?:SOC\s*2|HIPAA|Helm chart)/i, points: 5 },
|
||||
],
|
||||
},
|
||||
] as const;
|
||||
|
||||
export function parsePersonaRepeats(raw: string | undefined): number {
|
||||
const parsed = Number.parseInt(raw ?? '', 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) return 3;
|
||||
return Math.min(parsed, 10);
|
||||
}
|
||||
|
||||
export interface PersonaRunMode {
|
||||
gating: boolean;
|
||||
repeats: number;
|
||||
}
|
||||
|
||||
/** Acceptance is always 10 x 3. Smaller runs require an explicit debug label. */
|
||||
export function resolvePersonaRunMode(
|
||||
nonGatingDebugRaw: string | undefined,
|
||||
repeatsRaw: string | undefined,
|
||||
): PersonaRunMode {
|
||||
const nonGatingDebug = nonGatingDebugRaw === '1';
|
||||
if (repeatsRaw !== undefined && !nonGatingDebug) {
|
||||
throw new Error(
|
||||
'WAGGLE_PERSONA_REPEATS is allowed only in non-gating debug mode with WAGGLE_PERSONA_NON_GATING_DEBUG=1; acceptance is locked to 3 repeats.',
|
||||
);
|
||||
}
|
||||
return {
|
||||
gating: !nonGatingDebug,
|
||||
repeats: nonGatingDebug ? parsePersonaRepeats(repeatsRaw) : 3,
|
||||
};
|
||||
}
|
||||
6713
tests/vision/persona-scorer.test.ts
Normal file
6713
tests/vision/persona-scorer.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
3003
tests/vision/persona-scorer.ts
Normal file
3003
tests/vision/persona-scorer.ts
Normal file
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
478
tests/vision/verifier-contract.test.ts
Normal file
478
tests/vision/verifier-contract.test.ts
Normal file
@@ -0,0 +1,478 @@
|
||||
import { describe, expect, expectTypeOf, it } from 'vitest';
|
||||
import { detectTaskShape } from '../../packages/agent/src/task-shape';
|
||||
import { PERSONA_CASES } from './persona-cases';
|
||||
import {
|
||||
CANONICAL_VERIFIER_REPORT,
|
||||
VERIFIER_BLOCKER_CHECK_PAIRS,
|
||||
VERIFIER_BLOCKER_CODES,
|
||||
VERIFIER_FACT_IDS,
|
||||
VERIFIER_NEXT_CHECK_KEYS,
|
||||
VERIFIER_NEXT_CHECKS,
|
||||
VERIFIER_REPORT_CLOSE,
|
||||
VERIFIER_REPORT_OPEN,
|
||||
VERIFIER_TOP_LEVEL_KEYS,
|
||||
evaluateVerifierContract,
|
||||
renderVerifierReportEnvelope,
|
||||
type VerifierNextCheckV1,
|
||||
} from './verifier-contract';
|
||||
|
||||
function cloneReport(): Record<string, unknown> {
|
||||
return JSON.parse(JSON.stringify(CANONICAL_VERIFIER_REPORT)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function envelope(value: unknown): string {
|
||||
return `${VERIFIER_REPORT_OPEN}\n${JSON.stringify(value, null, 2)}\n${VERIFIER_REPORT_CLOSE}`;
|
||||
}
|
||||
|
||||
function mutateReport(mutator: (report: Record<string, unknown>) => void): string {
|
||||
const report = cloneReport();
|
||||
mutator(report);
|
||||
return envelope(report);
|
||||
}
|
||||
|
||||
function contractPassed(response: string): boolean {
|
||||
return evaluateVerifierContract(response).passed;
|
||||
}
|
||||
|
||||
const blockerForTarget = {
|
||||
release_artifact: 'release_artifact_missing',
|
||||
windows_installer: 'windows_installer_validation_missing',
|
||||
runtime_smoke_suite: 'runtime_validation_missing',
|
||||
security_scan: 'security_validation_missing',
|
||||
rollback_recovery: 'rollback_validation_missing',
|
||||
smart_router: 'smart_router_validation_missing',
|
||||
local_model_proxy: 'local_model_proxy_validation_missing',
|
||||
} as const;
|
||||
|
||||
describe('VerifierReportV1 deterministic contract', () => {
|
||||
it('models next checks as an exact discriminated tuple union', () => {
|
||||
expectTypeOf<{
|
||||
operation: 'inspect';
|
||||
target: 'release_artifact';
|
||||
passCondition: 'artifact_matches_release_commit';
|
||||
}>().toMatchTypeOf<VerifierNextCheckV1>();
|
||||
expectTypeOf<{
|
||||
operation: 'inspect';
|
||||
target: 'local_model_proxy';
|
||||
passCondition: 'clean_windows_install_passes';
|
||||
}>().not.toMatchTypeOf<VerifierNextCheckV1>();
|
||||
});
|
||||
|
||||
it('locks the public contract vocabulary independently of the validator implementation', () => {
|
||||
expect(VERIFIER_REPORT_OPEN).toBe('<waggle-verifier-report-v1>');
|
||||
expect(VERIFIER_REPORT_CLOSE).toBe('</waggle-verifier-report-v1>');
|
||||
expect(VERIFIER_FACT_IDS).toEqual([
|
||||
'teammate_claims_production_ready',
|
||||
'web_build_pass_reported',
|
||||
]);
|
||||
expect(VERIFIER_BLOCKER_CODES).toEqual([
|
||||
'release_artifact_missing',
|
||||
'runtime_validation_missing',
|
||||
'windows_installer_validation_missing',
|
||||
'security_validation_missing',
|
||||
'rollback_validation_missing',
|
||||
'smart_router_validation_missing',
|
||||
'local_model_proxy_validation_missing',
|
||||
]);
|
||||
expect(VERIFIER_NEXT_CHECKS).toEqual([
|
||||
['inspect', 'release_artifact', 'artifact_matches_release_commit'],
|
||||
['run', 'runtime_smoke_suite', 'critical_journeys_pass'],
|
||||
['run', 'windows_installer', 'clean_windows_install_passes'],
|
||||
['inspect', 'security_scan', 'no_reportable_high_severity_findings'],
|
||||
['run', 'rollback_recovery', 'rollback_restores_service'],
|
||||
['run', 'smart_router', 'routes_without_cloud_credentials'],
|
||||
['run', 'local_model_proxy', 'local_inference_succeeds'],
|
||||
]);
|
||||
expect(VERIFIER_TOP_LEVEL_KEYS).toEqual([
|
||||
'schemaVersion',
|
||||
'scenarioId',
|
||||
'evidenceScope',
|
||||
'facts',
|
||||
'unsupportedClaims',
|
||||
'blockerCodes',
|
||||
'nextChecks',
|
||||
'verdict',
|
||||
'releaseDecision',
|
||||
]);
|
||||
expect(VERIFIER_NEXT_CHECK_KEYS).toEqual(['operation', 'target', 'passCondition']);
|
||||
});
|
||||
|
||||
it('accepts the canonical report and returns every atomic diagnostic', () => {
|
||||
const result = evaluateVerifierContract(renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT));
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.report).toEqual(CANONICAL_VERIFIER_REPORT);
|
||||
expect(result.checks).toHaveLength(10);
|
||||
expect(result.checks.every(check => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns a null report and the complete ordered diagnostic set on failure', () => {
|
||||
const result = evaluateVerifierContract('VERDICT: FAIL');
|
||||
|
||||
expect(result.report).toBeNull();
|
||||
expect(result.checks.map(check => check.id)).toEqual([
|
||||
'envelope',
|
||||
'json',
|
||||
'schema',
|
||||
'scenario',
|
||||
'evidence-scope',
|
||||
'facts',
|
||||
'unsupported-claims',
|
||||
'blockers',
|
||||
'next-checks',
|
||||
'decision',
|
||||
]);
|
||||
expect(result.checks.every(check => check.passed === false)).toBe(true);
|
||||
});
|
||||
|
||||
it('generates the acceptance prompt from every closed contract value and pair', () => {
|
||||
const prompt = PERSONA_CASES.find(persona => persona.id === 'verifier')!.prompt;
|
||||
|
||||
expect(prompt).toContain(VERIFIER_REPORT_OPEN);
|
||||
expect(prompt).toContain(VERIFIER_REPORT_CLOSE);
|
||||
expect(prompt).toContain('schemaVersion 1');
|
||||
expect(prompt).toContain(`scenarioId ${JSON.stringify(CANONICAL_VERIFIER_REPORT.scenarioId)}`);
|
||||
expect(prompt).toContain(`evidenceScope ${JSON.stringify(CANONICAL_VERIFIER_REPORT.evidenceScope)}`);
|
||||
expect(prompt).toContain(`unsupportedClaims exactly ${JSON.stringify(CANONICAL_VERIFIER_REPORT.unsupportedClaims)}`);
|
||||
expect(prompt).toContain(`exactly these top-level keys and no others: ${VERIFIER_TOP_LEVEL_KEYS.join(', ')}`);
|
||||
expect(prompt).toContain(`exactly these keys and no others: ${VERIFIER_NEXT_CHECK_KEYS.join(', ')}`);
|
||||
expect(prompt).toContain('Spell all keys literally; do not escape or duplicate keys.');
|
||||
for (const fact of VERIFIER_FACT_IDS) expect(prompt).toContain(fact);
|
||||
for (const [blocker, [operation, target, passCondition]] of VERIFIER_BLOCKER_CHECK_PAIRS) {
|
||||
expect(prompt).toContain(`${blocker} => ${JSON.stringify({ operation, target, passCondition })}`);
|
||||
}
|
||||
expect(prompt).toContain(`verdict ${JSON.stringify(CANONICAL_VERIFIER_REPORT.verdict)}`);
|
||||
expect(prompt).toContain(`releaseDecision ${JSON.stringify(CANONICAL_VERIFIER_REPORT.releaseDecision)}`);
|
||||
});
|
||||
|
||||
it('does not trigger a response scaffold that conflicts with the exact JSON envelope', () => {
|
||||
const prompt = PERSONA_CASES.find(persona => persona.id === 'verifier')!.prompt;
|
||||
const shape = detectTaskShape(prompt);
|
||||
|
||||
expect(shape.signals).toEqual([]);
|
||||
expect(shape.confidence).toBe(0.1);
|
||||
});
|
||||
|
||||
it('accepts insignificant JSON whitespace, key order, and closed-array order', () => {
|
||||
const reordered = {
|
||||
releaseDecision: 'block',
|
||||
verdict: 'fail',
|
||||
nextChecks: [...CANONICAL_VERIFIER_REPORT.nextChecks].reverse(),
|
||||
blockerCodes: [...CANONICAL_VERIFIER_REPORT.blockerCodes].reverse(),
|
||||
unsupportedClaims: ['production_readiness'],
|
||||
facts: [...VERIFIER_FACT_IDS].reverse(),
|
||||
evidenceScope: 'supplied_only',
|
||||
scenarioId: 'web-build-only-readiness-v1',
|
||||
schemaVersion: 1,
|
||||
};
|
||||
const response = `${VERIFIER_REPORT_OPEN}\n ${JSON.stringify(reordered)} \n${VERIFIER_REPORT_CLOSE}`;
|
||||
|
||||
expect(contractPassed(response)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['free-form response', 'VERDICT: FAIL'],
|
||||
['prefix prose', `note\n${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}`],
|
||||
['suffix prose', `${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}\nnote`],
|
||||
['opening envelope only', `${VERIFIER_REPORT_OPEN}{}`],
|
||||
['closing envelope only', `{}` + VERIFIER_REPORT_CLOSE],
|
||||
['two envelopes', `${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}\n${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}`],
|
||||
[
|
||||
'two openings and one closing',
|
||||
`${VERIFIER_REPORT_OPEN}${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}`,
|
||||
],
|
||||
[
|
||||
'one opening and two closings',
|
||||
`${renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)}${VERIFIER_REPORT_CLOSE}`,
|
||||
],
|
||||
['oversized response', `${VERIFIER_REPORT_OPEN}${' '.repeat(20_001)}${VERIFIER_REPORT_CLOSE}`],
|
||||
])('rejects an invalid envelope boundary: %s', (_name, response) => {
|
||||
const result = evaluateVerifierContract(response);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'envelope')?.passed).toBe(false);
|
||||
expect(result.checks).toHaveLength(10);
|
||||
});
|
||||
|
||||
it('accepts the exact response-size boundary and rejects one character beyond it', () => {
|
||||
const canonical = renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT);
|
||||
const atLimit = `${' '.repeat(20_000 - canonical.length)}${canonical}`;
|
||||
const overLimit = ` ${atLimit}`;
|
||||
|
||||
expect(atLimit).toHaveLength(20_000);
|
||||
expect(contractPassed(atLimit)).toBe(true);
|
||||
expect(evaluateVerifierContract(overLimit).checks.find(check => check.id === 'envelope')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts the exact JSON-payload boundary and rejects one character beyond it', () => {
|
||||
const compact = JSON.stringify(CANONICAL_VERIFIER_REPORT);
|
||||
const atLimitPayload = `{${' '.repeat(18_000 - compact.length)}${compact.slice(1)}`;
|
||||
const overLimitPayload = `{ ${atLimitPayload.slice(1)}`;
|
||||
|
||||
expect(atLimitPayload).toHaveLength(18_000);
|
||||
expect(contractPassed(`${VERIFIER_REPORT_OPEN}${atLimitPayload}${VERIFIER_REPORT_CLOSE}`)).toBe(true);
|
||||
const over = evaluateVerifierContract(`${VERIFIER_REPORT_OPEN}${overLimitPayload}${VERIFIER_REPORT_CLOSE}`);
|
||||
expect(over.checks.find(check => check.id === 'json')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['malformed JSON', `${VERIFIER_REPORT_OPEN}{${VERIFIER_REPORT_CLOSE}`],
|
||||
['JSON array', envelope([])],
|
||||
['JSON scalar', envelope('fail')],
|
||||
['JSON null', envelope(null)],
|
||||
])('rejects a non-object JSON payload: %s', (_name, response) => {
|
||||
const result = evaluateVerifierContract(response);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'json')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['missing key', mutateReport(report => { delete report.releaseDecision; })],
|
||||
['extra key', mutateReport(report => { report.notes = 'not allowed'; })],
|
||||
['wrong version', mutateReport(report => { report.schemaVersion = 2; })],
|
||||
[
|
||||
'duplicate top-level key',
|
||||
renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)
|
||||
.replace('"schemaVersion": 1,', '"schemaVersion": 1,\n "schemaVersion": 1,'),
|
||||
],
|
||||
[
|
||||
'Unicode-escaped top-level alias',
|
||||
renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)
|
||||
.replace('"schemaVersion": 1,', '"schema\\u0056ersion": 999,\n "schemaVersion": 1,'),
|
||||
],
|
||||
[
|
||||
'escaped canonical key without a duplicate',
|
||||
renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)
|
||||
.replace('"schemaVersion": 1,', '"schema\\u0056ersion": 1,'),
|
||||
],
|
||||
])('rejects a schema mutation: %s', (_name, response) => {
|
||||
const result = evaluateVerifierContract(response);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'schema')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
const topLevelKeys = [
|
||||
'schemaVersion',
|
||||
'scenarioId',
|
||||
'evidenceScope',
|
||||
'facts',
|
||||
'unsupportedClaims',
|
||||
'blockerCodes',
|
||||
'nextChecks',
|
||||
'verdict',
|
||||
'releaseDecision',
|
||||
];
|
||||
|
||||
it.each(topLevelKeys)('rejects a duplicate top-level %s key', (key) => {
|
||||
const report = cloneReport();
|
||||
const duplicate = `"${key}":${JSON.stringify(report[key])},`;
|
||||
const response = `${VERIFIER_REPORT_OPEN}{${duplicate}${JSON.stringify(report).slice(1)}${VERIFIER_REPORT_CLOSE}`;
|
||||
const result = evaluateVerifierContract(response);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'schema')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(topLevelKeys)('rejects a missing top-level %s key', (key) => {
|
||||
const response = mutateReport(report => { delete report[key]; });
|
||||
expect(contractPassed(response)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['schemaVersion', '1', 'schema'],
|
||||
['scenarioId', null, 'scenario'],
|
||||
['evidenceScope', [], 'evidence-scope'],
|
||||
['facts', 'reported', 'facts'],
|
||||
['unsupportedClaims', 'production_readiness', 'unsupported-claims'],
|
||||
['blockerCodes', 'runtime_validation_missing', 'blockers'],
|
||||
['nextChecks', {}, 'next-checks'],
|
||||
['verdict', false, 'decision'],
|
||||
['releaseDecision', 0, 'decision'],
|
||||
])('rejects wrong type for %s', (key, value, checkId) => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report[key as string] = value; }));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === checkId)?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['wrong scenario', mutateReport(report => { report.scenarioId = 'other'; }), 'scenario'],
|
||||
['outside evidence', mutateReport(report => { report.evidenceScope = 'external_allowed'; }), 'evidence-scope'],
|
||||
['approve decision', mutateReport(report => { report.releaseDecision = 'approve'; }), 'decision'],
|
||||
['pass verdict', mutateReport(report => { report.verdict = 'pass'; }), 'decision'],
|
||||
])('rejects a closed invariant mutation: %s', (_name, response, checkId) => {
|
||||
const result = evaluateVerifierContract(response);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === checkId)?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('returns a null report after a late decision-invariant failure', () => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report.releaseDecision = 'approve'; }));
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.report).toBeNull();
|
||||
expect(result.checks.find(check => check.id === 'decision')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['missing fact', [VERIFIER_FACT_IDS[0]]],
|
||||
['invented fact', [...VERIFIER_FACT_IDS, 'runtime_passed']],
|
||||
['promoted verified fact', [VERIFIER_FACT_IDS[0], 'web_build_pass_verified']],
|
||||
['duplicate fact', [VERIFIER_FACT_IDS[0], VERIFIER_FACT_IDS[0]]],
|
||||
['empty facts', []],
|
||||
])('rejects invalid fact provenance: %s', (_name, facts) => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report.facts = facts; }));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'facts')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['missing unsupported claim', []],
|
||||
['wrong unsupported claim', ['runtime_validation']],
|
||||
['duplicate unsupported claim', ['production_readiness', 'production_readiness']],
|
||||
])('rejects invalid unsupported-claim state: %s', (_name, claims) => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report.unsupportedClaims = claims; }));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'unsupported-claims')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['empty blockers', []],
|
||||
['none blocker', ['none']],
|
||||
['unknown blocker', ['unicorn_missing']],
|
||||
['duplicate blocker', [VERIFIER_BLOCKER_CODES[0], VERIFIER_BLOCKER_CODES[0]]],
|
||||
])('rejects invalid blocker state: %s', (_name, blockers) => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report.blockerCodes = blockers; }));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'blockers')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(VERIFIER_NEXT_CHECKS)('accepts closed next-check tuple %s/%s/%s', (operation, target, passCondition) => {
|
||||
const response = mutateReport((report) => {
|
||||
report.blockerCodes = [blockerForTarget[target]];
|
||||
report.nextChecks = [{ operation, target, passCondition }];
|
||||
});
|
||||
expect(contractPassed(response)).toBe(true);
|
||||
});
|
||||
|
||||
it('accepts the maximal one-to-one blocker/check set', () => {
|
||||
const response = mutateReport((report) => {
|
||||
report.blockerCodes = [
|
||||
'release_artifact_missing',
|
||||
'runtime_validation_missing',
|
||||
'windows_installer_validation_missing',
|
||||
'security_validation_missing',
|
||||
'rollback_validation_missing',
|
||||
'smart_router_validation_missing',
|
||||
'local_model_proxy_validation_missing',
|
||||
];
|
||||
report.nextChecks = [
|
||||
{ operation: 'inspect', target: 'release_artifact', passCondition: 'artifact_matches_release_commit' },
|
||||
{ operation: 'run', target: 'runtime_smoke_suite', passCondition: 'critical_journeys_pass' },
|
||||
{ operation: 'run', target: 'windows_installer', passCondition: 'clean_windows_install_passes' },
|
||||
{ operation: 'inspect', target: 'security_scan', passCondition: 'no_reportable_high_severity_findings' },
|
||||
{ operation: 'run', target: 'rollback_recovery', passCondition: 'rollback_restores_service' },
|
||||
{ operation: 'run', target: 'smart_router', passCondition: 'routes_without_cloud_credentials' },
|
||||
{ operation: 'run', target: 'local_model_proxy', passCondition: 'local_inference_succeeds' },
|
||||
];
|
||||
});
|
||||
|
||||
expect(contractPassed(response)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
'missing check for selected blocker',
|
||||
mutateReport((report) => {
|
||||
report.blockerCodes = ['security_validation_missing'];
|
||||
report.nextChecks = [CANONICAL_VERIFIER_REPORT.nextChecks[0]];
|
||||
}),
|
||||
],
|
||||
[
|
||||
'unrelated extra check',
|
||||
mutateReport((report) => {
|
||||
report.blockerCodes = ['release_artifact_missing'];
|
||||
report.nextChecks = [...CANONICAL_VERIFIER_REPORT.nextChecks];
|
||||
}),
|
||||
],
|
||||
])('rejects blocker/check coverage mismatch: %s', (_name, response) => {
|
||||
const result = evaluateVerifierContract(response);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects partial coverage of a two-blocker report', () => {
|
||||
const response = mutateReport((report) => {
|
||||
report.nextChecks = [CANONICAL_VERIFIER_REPORT.nextChecks[0]];
|
||||
});
|
||||
const result = evaluateVerifierContract(response);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
const operations = [...new Set(VERIFIER_NEXT_CHECKS.map(tuple => tuple[0]))];
|
||||
const targets = [...new Set(VERIFIER_NEXT_CHECKS.map(tuple => tuple[1]))];
|
||||
const passConditions = [...new Set(VERIFIER_NEXT_CHECKS.map(tuple => tuple[2]))];
|
||||
const validTuples = new Set(VERIFIER_NEXT_CHECKS.map(tuple => tuple.join('|')));
|
||||
const invalidCrossProduct = operations.flatMap(operation =>
|
||||
targets.flatMap(target =>
|
||||
passConditions
|
||||
.filter(passCondition => !validTuples.has(`${operation}|${target}|${passCondition}`))
|
||||
.map(passCondition => [operation, target, passCondition] as const),
|
||||
),
|
||||
);
|
||||
|
||||
it.each(invalidCrossProduct)('rejects incompatible next-check tuple %s/%s/%s', (operation, target, passCondition) => {
|
||||
const result = evaluateVerifierContract(mutateReport((report) => {
|
||||
report.nextChecks = [{ operation, target, passCondition }];
|
||||
}));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['empty next checks', []],
|
||||
['unknown operation', [{ operation: 'guess', target: 'release_artifact', passCondition: 'artifact_matches_release_commit' }]],
|
||||
['unknown target', [{ operation: 'inspect', target: 'imaginary_artifact', passCondition: 'artifact_matches_release_commit' }]],
|
||||
['unknown condition', [{ operation: 'inspect', target: 'release_artifact', passCondition: 'assume_success' }]],
|
||||
['missing field', [{ operation: 'inspect', target: 'release_artifact' }]],
|
||||
['extra field', [{ operation: 'inspect', target: 'release_artifact', passCondition: 'artifact_matches_release_commit', notes: 'trust me' }]],
|
||||
['duplicate tuple', [CANONICAL_VERIFIER_REPORT.nextChecks[0], CANONICAL_VERIFIER_REPORT.nextChecks[0]]],
|
||||
['null element', [null]],
|
||||
['scalar element', ['inspect release artifact']],
|
||||
])('rejects malformed next-check state: %s', (_name, nextChecks) => {
|
||||
const result = evaluateVerifierContract(mutateReport(report => { report.nextChecks = nextChecks; }));
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects a duplicate nested key before JSON last-write-wins can hide it', () => {
|
||||
const response = renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)
|
||||
.replace('"operation": "inspect",', '"operation": "inspect",\n "operation": "inspect",');
|
||||
const result = evaluateVerifierContract(response);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects a Unicode-escaped duplicate nested key', () => {
|
||||
const response = renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT)
|
||||
.replace('"operation": "inspect",', '"operatio\\u006e": "guess",\n "operation": "inspect",');
|
||||
const result = evaluateVerifierContract(response);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'schema')?.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['operation', 'target', 'passCondition'])('rejects a duplicate nested %s key', (key) => {
|
||||
const report = cloneReport();
|
||||
const nextChecks = report.nextChecks as Array<Record<string, unknown>>;
|
||||
const duplicate = `"${key}":${JSON.stringify(nextChecks[0][key])},`;
|
||||
const compact = JSON.stringify(report);
|
||||
const response = `${VERIFIER_REPORT_OPEN}${compact.replace('{"operation"', `{${duplicate}"operation"`).replace('{{', '{')}${VERIFIER_REPORT_CLOSE}`;
|
||||
const result = evaluateVerifierContract(response);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find(check => check.id === 'next-checks')?.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
351
tests/vision/verifier-contract.ts
Normal file
351
tests/vision/verifier-contract.ts
Normal file
@@ -0,0 +1,351 @@
|
||||
export const VERIFIER_REPORT_OPEN = '<waggle-verifier-report-v1>';
|
||||
export const VERIFIER_REPORT_CLOSE = '</waggle-verifier-report-v1>';
|
||||
export const VERIFIER_SCENARIO_ID = 'web-build-only-readiness-v1';
|
||||
|
||||
export const VERIFIER_FACT_IDS = [
|
||||
'teammate_claims_production_ready',
|
||||
'web_build_pass_reported',
|
||||
] as const;
|
||||
|
||||
type VerifierFactId = typeof VERIFIER_FACT_IDS[number];
|
||||
|
||||
export const VERIFIER_BLOCKER_CHECK_PAIRS = [
|
||||
['release_artifact_missing', ['inspect', 'release_artifact', 'artifact_matches_release_commit']],
|
||||
['runtime_validation_missing', ['run', 'runtime_smoke_suite', 'critical_journeys_pass']],
|
||||
['windows_installer_validation_missing', ['run', 'windows_installer', 'clean_windows_install_passes']],
|
||||
['security_validation_missing', ['inspect', 'security_scan', 'no_reportable_high_severity_findings']],
|
||||
['rollback_validation_missing', ['run', 'rollback_recovery', 'rollback_restores_service']],
|
||||
['smart_router_validation_missing', ['run', 'smart_router', 'routes_without_cloud_credentials']],
|
||||
['local_model_proxy_validation_missing', ['run', 'local_model_proxy', 'local_inference_succeeds']],
|
||||
] as const;
|
||||
|
||||
type VerifierBlockerCheckPair = typeof VERIFIER_BLOCKER_CHECK_PAIRS[number];
|
||||
type VerifierBlockerCode = VerifierBlockerCheckPair[0];
|
||||
type VerifierNextCheckTuple = VerifierBlockerCheckPair[1];
|
||||
type VerifierOperation = VerifierNextCheckTuple[0];
|
||||
type VerifierTarget = VerifierNextCheckTuple[1];
|
||||
type VerifierPassCondition = VerifierNextCheckTuple[2];
|
||||
|
||||
export const VERIFIER_BLOCKER_CODES: readonly VerifierBlockerCode[] =
|
||||
VERIFIER_BLOCKER_CHECK_PAIRS.map(([blocker]) => blocker);
|
||||
|
||||
export const VERIFIER_NEXT_CHECKS: readonly VerifierNextCheckTuple[] =
|
||||
VERIFIER_BLOCKER_CHECK_PAIRS.map(([, nextCheck]) => nextCheck);
|
||||
|
||||
type NextCheckFromTuple<T> = T extends readonly [
|
||||
infer Operation extends VerifierOperation,
|
||||
infer Target extends VerifierTarget,
|
||||
infer PassCondition extends VerifierPassCondition,
|
||||
]
|
||||
? { operation: Operation; target: Target; passCondition: PassCondition }
|
||||
: never;
|
||||
|
||||
const BLOCKER_TARGET = Object.fromEntries(
|
||||
VERIFIER_BLOCKER_CHECK_PAIRS.map(([blocker, [, target]]) => [blocker, target]),
|
||||
) as Record<VerifierBlockerCode, VerifierTarget>;
|
||||
|
||||
export type VerifierNextCheckV1 = NextCheckFromTuple<VerifierNextCheckTuple>;
|
||||
|
||||
export interface VerifierReportV1 {
|
||||
schemaVersion: 1;
|
||||
scenarioId: typeof VERIFIER_SCENARIO_ID;
|
||||
evidenceScope: 'supplied_only';
|
||||
facts: VerifierFactId[];
|
||||
unsupportedClaims: ['production_readiness'];
|
||||
blockerCodes: VerifierBlockerCode[];
|
||||
nextChecks: VerifierNextCheckV1[];
|
||||
verdict: 'fail';
|
||||
releaseDecision: 'block';
|
||||
}
|
||||
|
||||
export const CANONICAL_VERIFIER_REPORT: VerifierReportV1 = {
|
||||
schemaVersion: 1,
|
||||
scenarioId: VERIFIER_SCENARIO_ID,
|
||||
evidenceScope: 'supplied_only',
|
||||
facts: [...VERIFIER_FACT_IDS],
|
||||
unsupportedClaims: ['production_readiness'],
|
||||
blockerCodes: ['release_artifact_missing', 'runtime_validation_missing'],
|
||||
nextChecks: [
|
||||
{
|
||||
operation: 'inspect',
|
||||
target: 'release_artifact',
|
||||
passCondition: 'artifact_matches_release_commit',
|
||||
},
|
||||
{
|
||||
operation: 'run',
|
||||
target: 'runtime_smoke_suite',
|
||||
passCondition: 'critical_journeys_pass',
|
||||
},
|
||||
],
|
||||
verdict: 'fail',
|
||||
releaseDecision: 'block',
|
||||
};
|
||||
|
||||
export type VerifierContractCheckId =
|
||||
| 'envelope'
|
||||
| 'json'
|
||||
| 'schema'
|
||||
| 'scenario'
|
||||
| 'evidence-scope'
|
||||
| 'facts'
|
||||
| 'unsupported-claims'
|
||||
| 'blockers'
|
||||
| 'next-checks'
|
||||
| 'decision';
|
||||
|
||||
export interface VerifierContractCheck {
|
||||
id: VerifierContractCheckId;
|
||||
passed: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface VerifierContractResult {
|
||||
passed: boolean;
|
||||
report: VerifierReportV1 | null;
|
||||
checks: VerifierContractCheck[];
|
||||
}
|
||||
|
||||
const CHECK_IDS: readonly VerifierContractCheckId[] = [
|
||||
'envelope',
|
||||
'json',
|
||||
'schema',
|
||||
'scenario',
|
||||
'evidence-scope',
|
||||
'facts',
|
||||
'unsupported-claims',
|
||||
'blockers',
|
||||
'next-checks',
|
||||
'decision',
|
||||
];
|
||||
|
||||
export const VERIFIER_TOP_LEVEL_KEYS = [
|
||||
'schemaVersion',
|
||||
'scenarioId',
|
||||
'evidenceScope',
|
||||
'facts',
|
||||
'unsupportedClaims',
|
||||
'blockerCodes',
|
||||
'nextChecks',
|
||||
'verdict',
|
||||
'releaseDecision',
|
||||
] as const;
|
||||
|
||||
export const VERIFIER_NEXT_CHECK_KEYS = ['operation', 'target', 'passCondition'] as const;
|
||||
const MAX_RESPONSE_CHARS = 20_000;
|
||||
const MAX_PAYLOAD_CHARS = 18_000;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function exactKeys(value: Record<string, unknown>, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const wanted = [...expected].sort();
|
||||
return actual.length === wanted.length
|
||||
&& actual.every((key, index) => key === wanted[index]);
|
||||
}
|
||||
|
||||
interface JsonKeyScan {
|
||||
counts: Map<string, number>;
|
||||
escapedKey: boolean;
|
||||
}
|
||||
|
||||
function scanJsonKeys(payload: string): JsonKeyScan | null {
|
||||
const counts = new Map<string, number>();
|
||||
let escapedKey = false;
|
||||
for (const match of payload.matchAll(/"((?:\\.|[^"\\])*)"\s*:/g)) {
|
||||
const rawKey = match[1];
|
||||
let decoded: unknown;
|
||||
try {
|
||||
decoded = JSON.parse(`"${rawKey}"`);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof decoded !== 'string') return null;
|
||||
if (rawKey.includes('\\')) escapedKey = true;
|
||||
counts.set(decoded, (counts.get(decoded) ?? 0) + 1);
|
||||
}
|
||||
return { counts, escapedKey };
|
||||
}
|
||||
|
||||
function isUniqueStringArray(
|
||||
value: unknown,
|
||||
allowed: ReadonlySet<string>,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): value is string[] {
|
||||
if (!Array.isArray(value) || value.length < minimum || value.length > maximum) return false;
|
||||
if (!value.every(item => typeof item === 'string' && allowed.has(item))) return false;
|
||||
return new Set(value).size === value.length;
|
||||
}
|
||||
|
||||
function sameSet(actual: readonly string[], expected: readonly string[]): boolean {
|
||||
return actual.length === expected.length
|
||||
&& expected.every(value => actual.includes(value));
|
||||
}
|
||||
|
||||
function appendNotEvaluated(checks: VerifierContractCheck[]): VerifierContractResult {
|
||||
const completed = new Set(checks.map(check => check.id));
|
||||
for (const id of CHECK_IDS) {
|
||||
if (!completed.has(id)) checks.push({ id, passed: false, detail: 'Not evaluated because an earlier contract boundary failed.' });
|
||||
}
|
||||
return { passed: false, report: null, checks };
|
||||
}
|
||||
|
||||
export function renderVerifierReportEnvelope(report: VerifierReportV1): string {
|
||||
return `${VERIFIER_REPORT_OPEN}\n${JSON.stringify(report, null, 2)}\n${VERIFIER_REPORT_CLOSE}`;
|
||||
}
|
||||
|
||||
export function evaluateVerifierContract(response: string): VerifierContractResult {
|
||||
const checks: VerifierContractCheck[] = [];
|
||||
const trimmed = response.trim();
|
||||
const openCount = trimmed.split(VERIFIER_REPORT_OPEN).length - 1;
|
||||
const closeCount = trimmed.split(VERIFIER_REPORT_CLOSE).length - 1;
|
||||
const envelopeValid = response.length <= MAX_RESPONSE_CHARS
|
||||
&& openCount === 1
|
||||
&& closeCount === 1
|
||||
&& trimmed.startsWith(VERIFIER_REPORT_OPEN)
|
||||
&& trimmed.endsWith(VERIFIER_REPORT_CLOSE);
|
||||
checks.push({
|
||||
id: 'envelope',
|
||||
passed: envelopeValid,
|
||||
detail: envelopeValid
|
||||
? 'Exactly one report envelope contains the entire response.'
|
||||
: 'Response must contain only one bounded verifier-report-v1 envelope.',
|
||||
});
|
||||
if (!envelopeValid) return appendNotEvaluated(checks);
|
||||
|
||||
const payload = trimmed.slice(VERIFIER_REPORT_OPEN.length, -VERIFIER_REPORT_CLOSE.length).trim();
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = payload.length > 0 && payload.length <= MAX_PAYLOAD_CHARS
|
||||
? JSON.parse(payload)
|
||||
: null;
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
const jsonValid = isRecord(parsed);
|
||||
checks.push({
|
||||
id: 'json',
|
||||
passed: jsonValid,
|
||||
detail: jsonValid ? 'Envelope payload is one JSON object.' : 'Envelope payload is not a bounded JSON object.',
|
||||
});
|
||||
if (!jsonValid) return appendNotEvaluated(checks);
|
||||
|
||||
const report = parsed as Record<string, unknown>;
|
||||
const keyScan = scanJsonKeys(payload);
|
||||
const topLevelCountsValid = keyScan !== null
|
||||
&& !keyScan.escapedKey
|
||||
&& VERIFIER_TOP_LEVEL_KEYS.every(key => keyScan.counts.get(key) === 1);
|
||||
const schemaValid = exactKeys(report, VERIFIER_TOP_LEVEL_KEYS)
|
||||
&& topLevelCountsValid
|
||||
&& report.schemaVersion === 1;
|
||||
checks.push({
|
||||
id: 'schema',
|
||||
passed: schemaValid,
|
||||
detail: schemaValid
|
||||
? 'Schema version and exact top-level keys are valid.'
|
||||
: 'Schema version, key set, or duplicate-key invariant failed.',
|
||||
});
|
||||
|
||||
const scenarioValid = report.scenarioId === VERIFIER_SCENARIO_ID;
|
||||
checks.push({
|
||||
id: 'scenario',
|
||||
passed: scenarioValid,
|
||||
detail: scenarioValid ? 'Scenario id matches the frozen acceptance case.' : 'Scenario id is missing or does not match.',
|
||||
});
|
||||
|
||||
const scopeValid = report.evidenceScope === 'supplied_only';
|
||||
checks.push({
|
||||
id: 'evidence-scope',
|
||||
passed: scopeValid,
|
||||
detail: scopeValid ? 'Evidence scope is supplied_only.' : 'Evidence scope is not supplied_only.',
|
||||
});
|
||||
|
||||
const factsValid = isUniqueStringArray(report.facts, new Set(VERIFIER_FACT_IDS), 2, 2)
|
||||
&& sameSet(report.facts, VERIFIER_FACT_IDS);
|
||||
checks.push({
|
||||
id: 'facts',
|
||||
passed: factsValid,
|
||||
detail: factsValid
|
||||
? 'Both prompt facts are represented as reported facts only.'
|
||||
: 'Facts must be the two closed reported-fact ids; verified or invented facts are forbidden.',
|
||||
});
|
||||
|
||||
const unsupportedValid = isUniqueStringArray(
|
||||
report.unsupportedClaims,
|
||||
new Set(['production_readiness']),
|
||||
1,
|
||||
1,
|
||||
) && report.unsupportedClaims[0] === 'production_readiness';
|
||||
checks.push({
|
||||
id: 'unsupported-claims',
|
||||
passed: unsupportedValid,
|
||||
detail: unsupportedValid
|
||||
? 'Production readiness remains explicitly unsupported.'
|
||||
: 'The production_readiness unsupported claim is required.',
|
||||
});
|
||||
|
||||
const blockersValid = isUniqueStringArray(
|
||||
report.blockerCodes,
|
||||
new Set(VERIFIER_BLOCKER_CODES),
|
||||
1,
|
||||
VERIFIER_BLOCKER_CODES.length,
|
||||
);
|
||||
checks.push({
|
||||
id: 'blockers',
|
||||
passed: blockersValid,
|
||||
detail: blockersValid ? 'At least one closed blocker code is present.' : 'Blocker codes are empty, duplicated, or outside the closed set.',
|
||||
});
|
||||
|
||||
const nextChecksArray = Array.isArray(report.nextChecks) ? report.nextChecks : [];
|
||||
const rawNextCheckKeysValid = keyScan !== null
|
||||
&& VERIFIER_NEXT_CHECK_KEYS.every(key => keyScan.counts.get(key) === nextChecksArray.length);
|
||||
const validTuples = new Set(VERIFIER_NEXT_CHECKS.map(tuple => tuple.join('|')));
|
||||
const normalizedChecks: string[] = [];
|
||||
const structuralNextChecksValid = nextChecksArray.length >= 1
|
||||
&& nextChecksArray.length <= VERIFIER_NEXT_CHECKS.length
|
||||
&& rawNextCheckKeysValid
|
||||
&& nextChecksArray.every((value) => {
|
||||
if (!isRecord(value) || !exactKeys(value, VERIFIER_NEXT_CHECK_KEYS)) return false;
|
||||
if (
|
||||
typeof value.operation !== 'string'
|
||||
|| typeof value.target !== 'string'
|
||||
|| typeof value.passCondition !== 'string'
|
||||
) return false;
|
||||
const tuple = `${value.operation}|${value.target}|${value.passCondition}`;
|
||||
normalizedChecks.push(tuple);
|
||||
return validTuples.has(tuple);
|
||||
})
|
||||
&& new Set(normalizedChecks).size === normalizedChecks.length;
|
||||
const selectedBlockers = blockersValid ? report.blockerCodes as VerifierBlockerCode[] : [];
|
||||
const expectedTargets = new Set(selectedBlockers.map(blocker => BLOCKER_TARGET[blocker]));
|
||||
const actualTargets = new Set(normalizedChecks.map(tuple => tuple.split('|')[1]));
|
||||
const blockerCoverageValid = blockersValid
|
||||
&& expectedTargets.size === actualTargets.size
|
||||
&& [...expectedTargets].every(target => actualTargets.has(target));
|
||||
const nextChecksValid = structuralNextChecksValid && blockerCoverageValid;
|
||||
checks.push({
|
||||
id: 'next-checks',
|
||||
passed: nextChecksValid,
|
||||
detail: nextChecksValid
|
||||
? 'Every blocker maps one-to-one to a unique executable closed next-check tuple.'
|
||||
: 'Next checks are malformed, outside the closed tuples, or do not map one-to-one to blocker codes.',
|
||||
});
|
||||
|
||||
const decisionValid = report.verdict === 'fail' && report.releaseDecision === 'block';
|
||||
checks.push({
|
||||
id: 'decision',
|
||||
passed: decisionValid,
|
||||
detail: decisionValid
|
||||
? 'Fail verdict and block decision are consistent.'
|
||||
: 'Verdict must be fail and releaseDecision must be block.',
|
||||
});
|
||||
|
||||
const passed = checks.every(check => check.passed);
|
||||
return {
|
||||
passed,
|
||||
report: passed ? report as unknown as VerifierReportV1 : null,
|
||||
checks,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user