import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { ACCEPTANCE_PERSONA_IDS, PERSONA_CASES, type PersonaAcceptanceCase, } from './persona-cases'; import { containsFailureCopy, markdownCodeSegmentsMatch, scorePersonaTrial, validatePythonSyntax, visibleMarkdownPreservesText, type CapturedSseEvent, type PersonaTrialEvidence, } from './persona-scorer'; export interface PersonaAcceptanceReceiptManifest { artifactPath: string; } export interface PersonaAcceptanceDiagnosticCostEntry { id: string; /** Exact decimal string with six fractional digits. */ amountUsd: string; /** Human-auditable provider usage export, invoice, or receipt reference. */ evidence: string; } export interface PersonaAcceptanceSealManifest { schemaVersion: 1; benchmarkId: string; runId: string; sourceRevision: string; threshold: 95; repeats: number; expectedProvider: string; expectedDetail: string; allowedModels: string[]; receipts: PersonaAcceptanceReceiptManifest[]; diagnosticCostLedger: PersonaAcceptanceDiagnosticCostEntry[]; } export interface ReceiptScore { score: number; capturedScore: number | null; scoreMode: 'captured' | 'derived-rescore'; passed: boolean; criticalFailures: readonly unknown[]; } interface SealOptions { expectedPersonaIds?: readonly string[]; repeats?: number; scoreArtifact?: (artifact: Record) => ReceiptScore; runtimeProvenance?: RuntimeProvenance; } interface RuntimeProvenance { gitRevision: string | null; relevantWorkingTreeClean: boolean; } interface InvalidReceipt { artifactPath: string; slot: string | null; reasons: string[]; } export interface SealedPersonaReceipt { slot: string; personaId: string; repeat: number; runId: string; artifactPath: string; artifactSha256: string; responseSha256: string; sourceRevision: string; scorerRevision: string; capturedScore: number | null; score: number; scoreMode: ReceiptScore['scoreMode']; model: string | null; provider: string; estimatedCostUsd: string; workspaceId: string; sessionId: string; inputTokens: number | null; outputTokens: number | null; durationMs: number | null; } export interface PersonaAcceptanceSeal { schemaVersion: 1; benchmarkId: string; manifestSha256: string; status: 'ready' | 'incomplete' | 'failed'; threshold: 95; repeats: number; expectedReceiptCount: number; completedReceiptCount: number; missingSlots: string[]; duplicateSlots: string[]; invalidReceipts: InvalidReceipt[]; manifestErrors: string[]; acceptedEstimatedCostUsd: string; diagnosticRecordedCostUsd: string; totalRecordedSpendUsd: string; diagnosticCostLedger: PersonaAcceptanceDiagnosticCostEntry[]; receipts: SealedPersonaReceipt[]; } function record(value: unknown): Record { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; } function strings(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; } function numberOrNull(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null; } function sha256(value: string | Buffer): string { return createHash('sha256').update(value).digest('hex'); } function slot(personaId: string, repeat: number): string { return `${personaId}#${repeat}`; } function parseUsdMicros(value: string): bigint | null { const match = value.match(/^(0|[1-9]\d*)\.(\d{6})$/); return match ? (BigInt(match[1]) * 1_000_000n) + BigInt(match[2]) : null; } function formatUsdMicros(value: bigint): string { const whole = value / 1_000_000n; return `${whole}.${(value % 1_000_000n).toString().padStart(6, '0')}`; } function estimatedCostMicros(value: unknown): bigint | null { if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return null; const scaled = Math.round(value * 1_000_000); return Math.abs(value - (scaled / 1_000_000)) <= 1e-9 ? BigInt(scaled) : null; } function gitOutput(args: string[]): string | null { try { return execFileSync('git', args, { cwd: process.cwd(), encoding: 'utf8', windowsHide: true, }).trim(); } catch { return null; } } function currentRuntimeProvenance(): RuntimeProvenance { const status = gitOutput([ 'status', '--porcelain', '--untracked-files=all', '--', '.', ':(exclude)output/**', ':(exclude)test-results/**', ':(exclude)playwright-report/**', ':(exclude).playwright-cli/**', ]); return { gitRevision: gitOutput(['rev-parse', 'HEAD']), relevantWorkingTreeClean: status === '', }; } function scoreWithCurrentScorer(artifact: Record): ReceiptScore { const personaRecord = record(artifact.persona); const persona = PERSONA_CASES.find(item => item.id === personaRecord.id) as PersonaAcceptanceCase | undefined; const capturedScore = numberOrNull(record(artifact.score).score); if (!persona) { return { score: 0, capturedScore, scoreMode: 'derived-rescore', passed: false, criticalFailures: [{ code: 'persona_mismatch', detail: 'No canonical scorer case exists.' }], }; } const request = record(artifact.request); const requestPayload = record(request.payload); const response = record(artifact.response); const responseTokens = record(response.tokens); const runtime = record(artifact.runtime); const workspace = record(artifact.workspace); const journey = record(artifact.journey); const browser = record(artifact.browser); const codeValidation = record(artifact.codeValidation); const exactResponse = typeof response.exact === 'string' ? response.exact : ''; const verifiedPython = validatePythonSyntax(exactResponse); const inputTokens = numberOrNull(responseTokens.input) ?? 0; const outputTokens = numberOrNull(responseTokens.output) ?? 0; const doneEventCount = numberOrNull(response.doneEventCount) ?? 0; const parseErrors = Array.isArray(response.parseErrors) ? response.parseErrors : []; const criticalBrowserErrors = [ ...(Array.isArray(browser.criticalConsoleErrors) ? browser.criticalConsoleErrors : []), ...(Array.isArray(browser.pageErrors) ? browser.pageErrors : []), ...(Array.isArray(browser.criticalNetworkFailures) ? browser.criticalNetworkFailures : []), ...(Array.isArray(browser.screenshotErrors) ? browser.screenshotErrors : []), ]; const llmHealthy = runtime.llmHealthy === true; const transportError = typeof response.transportError === 'string' ? response.transportError : ''; const evidence: PersonaTrialEvidence = { prompt: typeof request.exactPrompt === 'string' ? request.exactPrompt : '', response: exactResponse, persistedResponse: typeof response.persistedExact === 'string' ? response.persistedExact : '', sseEvents: (Array.isArray(response.sseEvents) ? response.sseEvents : []) as CapturedSseEvent[], toolsUsed: strings(response.toolsUsed), durationMs: numberOrNull(response.durationMs) ?? Number.POSITIVE_INFINITY, inputTokens, outputTokens, personaPersisted: workspace.personaPersisted === true, requestPersonaId: typeof request.personaId === 'string' ? request.personaId : null, expectedWorkspaceId: typeof workspace.workspaceId === 'string' ? workspace.workspaceId : '', requestWorkspaceId: typeof requestPayload.workspaceId === 'string' ? requestPayload.workspaceId : null, requestSessionId: typeof request.sessionId === 'string' ? request.sessionId : null, persistedSessionId: typeof response.persistedSessionId === 'string' ? response.persistedSessionId : null, persistedPrompt: typeof response.persistedPromptExact === 'string' ? response.persistedPromptExact : '', persistedMessageCount: numberOrNull(response.persistedMessageCount) ?? 0, tokenStreamResponse: typeof response.tokenStreamExact === 'string' ? response.tokenStreamExact : '', doneEventCount, renderedAssistantResponse: typeof response.renderedAssistantExact === 'string' ? response.renderedAssistantExact : '', visibleAssistantText: typeof response.visibleAssistantTextExact === 'string' ? response.visibleAssistantTextExact : '', visibleCodeSegments: strings(response.visibleCodeSegmentsExact), memoryEvidencePresent: journey.memoryJourneyOk === true && typeof journey.memoryText === 'string' && journey.memoryText.trim().length > 0, workspaceLeak: Array.isArray(journey.leakedSnippets) && journey.leakedSnippets.length > 0, completed: response.completed === true || doneEventCount === 1, timedOut: response.timedOut === true || /timed?\s*out/i.test(transportError), corrupted: response.httpStatus !== 200 || parseErrors.length > 0 || criticalBrowserErrors.length > 0 || !llmHealthy || inputTokens <= 0 || outputTokens <= 0 || containsFailureCopy(exactResponse), codeValidation: { pythonSyntaxValid: codeValidation.available === true && codeValidation.syntaxValid === true && verifiedPython.available && verifiedPython.syntaxValid, pythonImportsPresent: codeValidation.available === true && codeValidation.importsPresent === true && verifiedPython.available && verifiedPython.importsPresent, }, }; const current = scorePersonaTrial(persona, evidence); const capturedPassed = record(artifact.score).passed === true; return { score: current.score, capturedScore, scoreMode: capturedScore === current.score && capturedPassed === current.passed ? 'captured' : 'derived-rescore', passed: current.passed, criticalFailures: current.criticalFailures, }; } export function buildPersonaAcceptanceSeal( manifest: PersonaAcceptanceSealManifest, options: SealOptions = {}, ): PersonaAcceptanceSeal { const expectedPersonaIds = options.expectedPersonaIds ?? ACCEPTANCE_PERSONA_IDS; const repeats = options.repeats ?? 3; const scoreArtifact = options.scoreArtifact ?? scoreWithCurrentScorer; const runtimeProvenance = options.runtimeProvenance ?? currentRuntimeProvenance(); const allowedModels = Array.isArray(manifest.allowedModels) ? manifest.allowedModels : []; const manifestErrors: string[] = []; const invalidReceipts: InvalidReceipt[] = []; const duplicateSlots = new Set(); const acceptedBySlot = new Map(); const seenWorkspaceIds = new Set(); const seenSessionIds = new Set(); const expectedSlots = expectedPersonaIds.flatMap(personaId => Array.from({ length: repeats }, (_, index) => slot(personaId, index + 1)), ); const expectedSlotSet = new Set(expectedSlots); if (manifest.schemaVersion !== 1) manifestErrors.push('manifest schemaVersion must be 1'); if (!manifest.benchmarkId?.trim()) manifestErrors.push('benchmarkId is required'); if (!manifest.runId?.trim()) manifestErrors.push('runId is required'); if (manifest.threshold !== 95) manifestErrors.push('threshold must be 95'); if (manifest.repeats !== repeats) manifestErrors.push(`manifest repeats must be ${repeats}`); if (!/^[a-f0-9]{40}$/i.test(manifest.sourceRevision)) { manifestErrors.push('sourceRevision must be a full 40-character Git revision'); } if (runtimeProvenance.gitRevision !== manifest.sourceRevision) { manifestErrors.push('executed scorer revision does not match sourceRevision'); } if (!runtimeProvenance.relevantWorkingTreeClean) { manifestErrors.push('executed scorer relevant working tree is not clean'); } if (!manifest.expectedProvider?.trim()) manifestErrors.push('expectedProvider is required'); if (!manifest.expectedDetail?.trim()) manifestErrors.push('expectedDetail is required'); if (allowedModels.length === 0) { manifestErrors.push('allowedModels must contain at least one paid model'); } else if (new Set(allowedModels).size !== allowedModels.length) { manifestErrors.push('allowedModels must not contain duplicates'); } const costIds = new Set(); let diagnosticCostMicros = 0n; for (const entry of manifest.diagnosticCostLedger ?? []) { if (!entry.id?.trim()) manifestErrors.push('every cost ledger entry requires an id'); if (costIds.has(entry.id)) manifestErrors.push(`duplicate cost ledger id: ${entry.id}`); costIds.add(entry.id); if (!entry.evidence?.trim()) manifestErrors.push(`diagnostic cost ${entry.id || '(missing id)'} requires evidence`); const amount = parseUsdMicros(entry.amountUsd); if (amount === null) manifestErrors.push(`cost ${entry.id || '(missing id)'} must use exactly six decimal places`); else diagnosticCostMicros += amount; } const seenArtifactPaths = new Set(); let acceptedCostMicros = 0n; for (const entry of manifest.receipts ?? []) { const artifactPath = resolve(entry.artifactPath); const reasons: string[] = []; if (seenArtifactPaths.has(artifactPath)) reasons.push('artifact path is selected more than once'); seenArtifactPaths.add(artifactPath); let artifactBytes: Buffer | null = null; let artifact: Record = {}; try { artifactBytes = readFileSync(artifactPath); artifact = record(JSON.parse(artifactBytes.toString('utf8'))); } catch (error) { reasons.push(`artifact could not be read as JSON: ${error instanceof Error ? error.message : String(error)}`); } const persona = record(artifact.persona); const personaId = typeof persona.id === 'string' ? persona.id : ''; const repeat = numberOrNull(persona.repeat); const receiptSlot = personaId && repeat !== null ? slot(personaId, repeat) : null; if (artifact.schemaVersion !== 7) reasons.push('artifact schemaVersion must be 7'); if (!receiptSlot || !expectedSlotSet.has(receiptSlot)) reasons.push('artifact persona/repeat is outside the expected matrix'); if (persona.repeatCount !== repeats) reasons.push(`artifact repeatCount must be ${repeats}`); if (persona.gating !== true) reasons.push('artifact was not captured in gating mode'); if (artifact.runId !== manifest.runId) reasons.push('artifact runId does not match the manifest runId'); const artifactSource = record(artifact.source); if (artifactSource.gitRevision !== manifest.sourceRevision) { reasons.push('artifact source revision does not match the manifest'); } if (artifactSource.relevantWorkingTreeClean !== true) { reasons.push('artifact was not captured from a clean relevant working tree'); } const runtime = record(artifact.runtime); const runtimeLlm = record(record(runtime.health).llm); if (runtime.healthStatus !== 200) reasons.push('runtime health endpoint did not return 200'); if (runtime.llmHealthy !== true || runtimeLlm.health !== 'healthy') reasons.push('runtime LLM was not healthy'); if (runtime.expectedProvider !== manifest.expectedProvider || runtimeLlm.provider !== manifest.expectedProvider) { reasons.push('runtime provider does not match the paid provider contract'); } if ( runtime.expectedDetail !== manifest.expectedDetail || typeof runtimeLlm.detail !== 'string' || !runtimeLlm.detail.includes(manifest.expectedDetail) ) { reasons.push('runtime provider detail does not match the paid provider contract'); } const response = record(artifact.response); if (response.httpStatus !== 200) reasons.push('chat response did not return 200'); if (response.doneEventCount !== 1) reasons.push('chat stream did not contain exactly one done event'); if (!Array.isArray(response.parseErrors) || response.parseErrors.length > 0) reasons.push('chat stream contained parse errors or omitted parse-error evidence'); if (typeof response.exact !== 'string' || !response.exact.trim()) reasons.push('exact response is missing'); const exactResponse = typeof response.exact === 'string' ? response.exact : ''; const visibleAssistantText = typeof response.visibleAssistantTextExact === 'string' ? response.visibleAssistantTextExact : ''; if (!visibleAssistantText.trim()) reasons.push('visible assistant DOM text evidence is missing'); else if (!visibleMarkdownPreservesText(exactResponse, visibleAssistantText)) { reasons.push('visible assistant DOM text did not preserve the response content'); } if ( !Array.isArray(response.visibleCodeSegmentsExact) || response.visibleCodeSegmentsExact.some(segment => typeof segment !== 'string') ) { reasons.push('visible assistant DOM code evidence is missing or malformed'); } else if (!markdownCodeSegmentsMatch(exactResponse, response.visibleCodeSegmentsExact as string[])) { reasons.push('visible assistant DOM code did not match the response Markdown'); } const model = typeof response.model === 'string' ? response.model : ''; if (!model || !allowedModels.includes(model)) reasons.push('response model is missing or not allowed'); const costMicros = estimatedCostMicros(response.estimatedCostUsd); if (costMicros === null) reasons.push('positive Waggle-estimated cost with at most six decimal places is required'); const doneEvents = Array.isArray(response.sseEvents) ? response.sseEvents.map(record).filter(event => event.event === 'done') : []; const doneData = record(doneEvents[0]?.data); const doneCostMicros = estimatedCostMicros(doneData.cost); if (doneEvents.length !== 1 || doneCostMicros === null || doneCostMicros !== costMicros) { reasons.push('estimated cost does not match the single done event'); } if (doneData.model !== model) reasons.push('response model does not match the provider done event'); const workspace = record(artifact.workspace); const request = record(artifact.request); const workspaceId = typeof workspace.workspaceId === 'string' ? workspace.workspaceId : ''; const sessionId = typeof request.sessionId === 'string' ? request.sessionId : ''; if (!workspaceId) reasons.push('workspace id is missing'); else if (seenWorkspaceIds.has(workspaceId)) reasons.push('workspace id is reused across receipts'); if (!sessionId) reasons.push('session id is missing'); else if (seenSessionIds.has(sessionId)) reasons.push('session id is reused across receipts'); if (workspaceId) seenWorkspaceIds.add(workspaceId); if (sessionId) seenSessionIds.add(sessionId); const browser = record(artifact.browser); for (const key of ['criticalConsoleErrors', 'pageErrors', 'criticalNetworkFailures', 'screenshotErrors']) { if (!Array.isArray(browser[key]) || (browser[key] as unknown[]).length > 0) { reasons.push(`browser ${key} evidence is missing or non-empty`); } } const rescored = scoreArtifact(artifact); if (!rescored.passed || rescored.score < manifest.threshold) reasons.push(`current scorer returned ${rescored.score}/100`); if (rescored.criticalFailures.length > 0) reasons.push('current scorer reported critical failures'); if (receiptSlot && acceptedBySlot.has(receiptSlot)) { duplicateSlots.add(receiptSlot); reasons.push('persona/repeat slot is selected more than once'); } if (reasons.length > 0 || !receiptSlot || !artifactBytes || repeat === null || costMicros === null) { invalidReceipts.push({ artifactPath, slot: receiptSlot, reasons }); continue; } acceptedCostMicros += costMicros; const responseTokens = record(response.tokens); acceptedBySlot.set(receiptSlot, { slot: receiptSlot, personaId, repeat, runId: manifest.runId, artifactPath, artifactSha256: sha256(artifactBytes), responseSha256: sha256(response.exact as string), sourceRevision: manifest.sourceRevision, scorerRevision: manifest.sourceRevision, capturedScore: rescored.capturedScore, score: rescored.score, scoreMode: rescored.scoreMode, model, provider: manifest.expectedProvider, estimatedCostUsd: formatUsdMicros(costMicros), workspaceId, sessionId, inputTokens: numberOrNull(responseTokens.input), outputTokens: numberOrNull(responseTokens.output), durationMs: numberOrNull(response.durationMs), }); } const missingSlots = expectedSlots.filter(item => !acceptedBySlot.has(item)); const receipts = [...acceptedBySlot.values()].sort((a, b) => a.slot.localeCompare(b.slot)); const failed = manifestErrors.length > 0 || invalidReceipts.length > 0 || duplicateSlots.size > 0; return { schemaVersion: 1, benchmarkId: manifest.benchmarkId, manifestSha256: sha256(JSON.stringify(manifest)), status: failed ? 'failed' : missingSlots.length > 0 ? 'incomplete' : 'ready', threshold: 95, repeats, expectedReceiptCount: expectedSlots.length, completedReceiptCount: receipts.length, missingSlots, duplicateSlots: [...duplicateSlots].sort(), invalidReceipts, manifestErrors, acceptedEstimatedCostUsd: formatUsdMicros(acceptedCostMicros), diagnosticRecordedCostUsd: formatUsdMicros(diagnosticCostMicros), totalRecordedSpendUsd: formatUsdMicros(acceptedCostMicros + diagnosticCostMicros), diagnosticCostLedger: (manifest.diagnosticCostLedger ?? []).map(entry => ({ ...entry })), receipts, }; }