moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,367 @@
/**
* Long-task checkpointing — Phase 3.1 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.1)
*
* Per-step state serialization for long-task agent runs. Lets the recovery
* layer (Phase 3.2) resume from the last successful step after a crash, a
* mid-run kill, or a transient tool failure.
*
* Storage layout (filesystem JSON, atomic writes):
* <rootDir>/<task-id>/step-000000.json
* <rootDir>/<task-id>/step-000001.json
* ...
*
* Each step file is written via tmp+rename so concurrent reads NEVER observe
* a partial file. Per-task isolation via the directory prefix matches the
* pilot 2026-04-26 amendment v1 §4 "per-task SessionStore" pattern (one
* corpus per task, deleted after results are persisted).
*
* HARD RULES (from PM brief §3.1):
* - schema_version is REQUIRED and validated on save (forward-compat)
* - step_index is monotonic 0,1,2,... (verifyIntegrity flags gaps)
* - dispose() recursively deletes the per-task directory
* - Reads ignore *.tmp files (crash residue from a partial write)
* - Per-task isolation: state.task_id MUST match the store's taskId
*/
import * as fsp from 'node:fs/promises';
import * as path from 'node:path';
import * as crypto from 'node:crypto';
export const CHECKPOINT_SCHEMA_VERSION = 1;
// ─────────────────────────────────────────────────────────────────────────
// State shape
// ─────────────────────────────────────────────────────────────────────────
/**
* A single decision recorded in the agent's history. Used by the recovery
* layer to inform retry strategy and by audit / replay tooling.
*/
export interface Decision {
/** step_index where the decision was made. */
step_index: number;
/** What was decided (one short sentence). */
decision: string;
/** Optional rationale captured at decision time. */
rationale?: string;
/** Optional list of alternatives the agent considered. */
alternatives_considered?: readonly string[];
}
/**
* Per-step agent state. JSON-serializable. One file per step on disk.
*
* Persistent fields (carried forward across steps by `nextStateFrom`):
* accumulated_context · retrieval_cache · decision_history
*
* Per-step fields (overwritten each step):
* step_action · step_input · step_output · cost_usd · latency_ms · error
*/
export interface CheckpointStepState {
/** Forward-compat schema version. */
schema_version: number;
/** Logical task identifier (matches per-task directory name). */
task_id: string;
/** Run-level UUID grouping multiple tasks in the same run. */
run_id: string;
/** Monotonic 0-indexed step number. */
step_index: number;
/** ISO 8601 UTC timestamp at the moment this checkpoint was captured. */
timestamp_iso: string;
/** Action / tool / cell label that produced this step. */
step_action: string;
/** Inputs the step received. */
step_input: Readonly<Record<string, unknown>>;
/** Outputs the step produced. */
step_output: Readonly<Record<string, unknown>>;
/** Running scratchpad / synthesis built up across all steps. */
accumulated_context: string;
/** Cache of retrieval / search query → results, keyed by canonical query string. */
retrieval_cache: Readonly<Record<string, unknown>>;
/** History of decisions made up to and including this step. */
decision_history: readonly Decision[];
/** Optional per-step cost (USD). */
cost_usd?: number;
/** Optional per-step latency (ms). */
latency_ms?: number;
/** Optional error string if this step failed. Recovery layer reads this. */
error?: string;
}
// ─────────────────────────────────────────────────────────────────────────
// Store
// ─────────────────────────────────────────────────────────────────────────
const STEP_FILE_RE = /^step-(\d{6})\.json$/;
const PADDING_WIDTH = 6;
function stepFilename(stepIndex: number): string {
return `step-${String(stepIndex).padStart(PADDING_WIDTH, '0')}.json`;
}
async function fileExists(filePath: string): Promise<boolean> {
try {
await fsp.access(filePath);
return true;
} catch {
return false;
}
}
export interface CheckpointStoreOptions {
/** Parent directory under which the per-task subfolder lives. */
rootDir: string;
/** Logical task ID — used as the per-task subfolder name. */
taskId: string;
}
export interface IntegrityReport {
ok: boolean;
issues: readonly string[];
}
/**
* Per-task checkpoint store. One instance corresponds to exactly one task —
* mirrors the per-task SessionStore isolation pattern from the pilot
* 2026-04-26 amendment v1 §4.
*/
export class CheckpointStore {
private readonly taskIdInternal: string;
private readonly taskDir: string;
constructor(opts: CheckpointStoreOptions) {
if (!opts.rootDir) throw new Error('CheckpointStore: rootDir is required');
if (!opts.taskId) throw new Error('CheckpointStore: taskId is required');
if (opts.taskId.includes(path.sep) || opts.taskId.includes('/') || opts.taskId.includes('\\')) {
throw new Error(`CheckpointStore: taskId must not contain path separators (got "${opts.taskId}")`);
}
if (opts.taskId === '.' || opts.taskId === '..') {
throw new Error(`CheckpointStore: taskId must not be "." or ".."`);
}
this.taskIdInternal = opts.taskId;
this.taskDir = path.join(opts.rootDir, opts.taskId);
}
/** Per-task directory path (read-only accessor). */
get directory(): string {
return this.taskDir;
}
/** Logical task ID (read-only accessor). */
get taskId(): string {
return this.taskIdInternal;
}
/** Idempotent: ensures per-task directory exists. Safe to call repeatedly. */
async init(): Promise<void> {
await fsp.mkdir(this.taskDir, { recursive: true });
}
/**
* Atomically writes a step's state. Implementation: write to <file>.<rand>.tmp,
* then rename to the final filename. Reads never observe partial files.
*
* Per-task directory is mkdir'd on every save, so callers don't strictly
* need to call init() first.
*
* Returns the absolute path of the finalized step file.
*/
async save(state: CheckpointStepState): Promise<string> {
if (state.task_id !== this.taskIdInternal) {
throw new Error(
`CheckpointStore: state.task_id="${state.task_id}" does not match store.taskId="${this.taskIdInternal}"`,
);
}
if (!Number.isInteger(state.step_index) || state.step_index < 0) {
throw new Error(
`CheckpointStore: step_index must be a non-negative integer (got ${String(state.step_index)})`,
);
}
if (state.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
throw new Error(
`CheckpointStore: schema_version mismatch — store expects ${CHECKPOINT_SCHEMA_VERSION}, state has ${String(state.schema_version)}`,
);
}
await fsp.mkdir(this.taskDir, { recursive: true });
const finalPath = path.join(this.taskDir, stepFilename(state.step_index));
// Randomized .tmp suffix so two concurrent writers at the same step_index
// (e.g. retry races) don't collide on a shared tmp filename.
const tmpPath = `${finalPath}.${crypto.randomBytes(4).toString('hex')}.tmp`;
const json = JSON.stringify(state, null, 2);
await fsp.writeFile(tmpPath, json, { encoding: 'utf-8' });
try {
await fsp.rename(tmpPath, finalPath);
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
// Windows quirk: fs.rename refuses to overwrite an existing destination
// (POSIX silently overwrites). When two writers race the same step_index
// (e.g. a retry that overlapped its predecessor), the loser yields:
// there is already a valid state at finalPath from the winner. Drop our
// tmp file and report the destination path. Any other error is fatal.
if ((code === 'EPERM' || code === 'EEXIST') && (await fileExists(finalPath))) {
await fsp.rm(tmpPath, { force: true });
return finalPath;
}
await fsp.rm(tmpPath, { force: true });
throw err;
}
return finalPath;
}
/** Loads a specific step. Returns undefined if no file exists at that index. */
async load(stepIndex: number): Promise<CheckpointStepState | undefined> {
const filePath = path.join(this.taskDir, stepFilename(stepIndex));
let json: string;
try {
json = await fsp.readFile(filePath, 'utf-8');
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
throw err;
}
return JSON.parse(json) as CheckpointStepState;
}
/** Loads the highest-indexed step. Returns undefined if no steps. */
async loadLatest(): Promise<CheckpointStepState | undefined> {
const indices = await this.listSteps();
if (indices.length === 0) return undefined;
return this.load(indices[indices.length - 1]!);
}
/**
* Returns sorted-ascending step indices that have finalized files in the
* per-task directory. Ignores *.tmp residue (crash artifact from a
* partial save) and any non-matching filenames.
*
* Returns [] if the per-task directory does not exist (treated as
* "no steps yet" rather than an error).
*/
async listSteps(): Promise<number[]> {
let entries: string[];
try {
entries = await fsp.readdir(this.taskDir);
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return [];
throw err;
}
const indices: number[] = [];
for (const name of entries) {
const m = name.match(STEP_FILE_RE);
if (!m) continue;
indices.push(Number.parseInt(m[1]!, 10));
}
indices.sort((a, b) => a - b);
return indices;
}
/**
* Walks every step file and verifies:
* 1. step_index sequence is monotonic 0,1,2,... with no gaps
* 2. file's step_index matches its filename's step_index
* 3. schema_version matches CHECKPOINT_SCHEMA_VERSION
*
* Returns ok=true iff all checks pass; otherwise returns the issues list.
*/
async verifyIntegrity(): Promise<IntegrityReport> {
const indices = await this.listSteps();
const issues: string[] = [];
for (let expected = 0; expected < indices.length; expected += 1) {
const observed = indices[expected]!;
if (observed !== expected) {
issues.push(`gap or out-of-order: expected step_index=${expected}, found ${observed}`);
// Stop on first gap — the rest of the report would be noise.
break;
}
const state = await this.load(observed);
if (!state) {
issues.push(`step ${observed}: file disappeared mid-verify`);
continue;
}
if (state.step_index !== observed) {
issues.push(`step ${observed}: file.step_index=${state.step_index} does not match filename`);
}
if (state.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
issues.push(`step ${observed}: schema_version=${state.schema_version} (expected ${CHECKPOINT_SCHEMA_VERSION})`);
}
}
return { ok: issues.length === 0, issues };
}
/**
* Recursively deletes the per-task directory. Per amendment v2 §4 pattern:
* "deleted after each task once results are persisted". Idempotent —
* no-ops if the directory does not exist.
*/
async dispose(): Promise<void> {
await fsp.rm(this.taskDir, { recursive: true, force: true });
}
}
// ─────────────────────────────────────────────────────────────────────────
// Helpers (used by Phase 3.2 recovery + Phase 3.4 agent-loop integration)
// ─────────────────────────────────────────────────────────────────────────
/**
* Builds an empty CheckpointStepState shell. Useful when no prior checkpoint
* exists — Phase 3.2 recovery uses this to construct a fresh state.
*/
export function makeInitialState(opts: {
task_id: string;
run_id: string;
step_action?: string;
}): CheckpointStepState {
return {
schema_version: CHECKPOINT_SCHEMA_VERSION,
task_id: opts.task_id,
run_id: opts.run_id,
step_index: 0,
timestamp_iso: new Date().toISOString(),
step_action: opts.step_action ?? 'init',
step_input: {},
step_output: {},
accumulated_context: '',
retrieval_cache: {},
decision_history: [],
};
}
/**
* Builds the next step state from the prior step's state. Increments
* step_index, copies forward the persistent fields (accumulated_context,
* retrieval_cache, decision_history) and merges in caller-supplied additions.
*
* Per-step fields (step_action / step_input / step_output / cost_usd /
* latency_ms / error) are taken from `opts`, not carried forward.
*/
export function nextStateFrom(prior: CheckpointStepState, opts: {
step_action: string;
step_input?: Record<string, unknown>;
step_output?: Record<string, unknown>;
decisions?: readonly Decision[];
retrieval_additions?: Record<string, unknown>;
appended_context?: string;
cost_usd?: number;
latency_ms?: number;
error?: string;
}): CheckpointStepState {
return {
schema_version: CHECKPOINT_SCHEMA_VERSION,
task_id: prior.task_id,
run_id: prior.run_id,
step_index: prior.step_index + 1,
timestamp_iso: new Date().toISOString(),
step_action: opts.step_action,
step_input: { ...(opts.step_input ?? {}) },
step_output: { ...(opts.step_output ?? {}) },
accumulated_context: opts.appended_context !== undefined
? prior.accumulated_context + opts.appended_context
: prior.accumulated_context,
retrieval_cache: { ...prior.retrieval_cache, ...(opts.retrieval_additions ?? {}) },
decision_history: [...prior.decision_history, ...(opts.decisions ?? [])],
cost_usd: opts.cost_usd,
latency_ms: opts.latency_ms,
error: opts.error,
};
}

View File

@@ -0,0 +1,495 @@
/**
* Long-task context management — Phase 3.3 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.3)
*
* Handles context-window exhaustion in multi-hour agent runs. Phase 3.1's
* accumulated_context grows monotonically across steps; eventually exceeds
* the model's context-window budget. ContextManager intelligently shrinks
* accumulated_context, decision_history, and retrieval_cache while
* preserving task-progress signal.
*
* THREE SHRINKING DIMENSIONS:
* 1. accumulated_context (string) — compress via LLM summarization,
* retrieve-only truncation, or hybrid (summarize + emit archive event
* so caller can index for later retrieval).
* 2. decision_history (Decision[]) — bucket-summarize older decisions
* into one rolled-up Decision; keep last K verbatim. Optional archive
* callback for audit-trail persistence.
* 3. retrieval_cache (Record<string, unknown>) — LRU-evict to maxSize.
* Defaults to insertion order; caller can supply accessOrder for true
* LRU.
*
* REUSE / NON-DUPLICATION:
* - estimateStringTokens from existing context-compressor.ts (content-aware:
* prose 4.0 / code 3.2 / json 3.5 / mixed 2.5 chars/token).
* - LlmCallFn / RetrievalSearchFn from Phase 2.1 (retrieval-agent-loop.ts).
* - Decision type from Phase 3.1 (checkpoint.ts).
*
* HALT-AND-PING TRIGGER RESOLUTIONS (locked before code, per PM brief §3.3):
* 1. Tokenizer model-class dependencies → injectable estimateTokensFn,
* defaults to estimateStringTokens (no new package deps).
* 2. Compression LLM-call cost → threshold-gated (default 0.7), caller
* chooses summarizationModel, cost telemetry via onCompressionEvent.
* 3. Hive-mind retrieval ↔ Memory Sync Repair conflict → ContextManager
* does NOT touch hive-mind; consumes RetrievalSearchFn abstraction;
* caller wires it to per-task SessionStore (scratch, NOT synced).
* 4. Decision history compression dropping critical decisions → bucket-
* summarize preserves Decision shape; optional archiveDecisionsTo
* callback for lossless on-disk audit trail.
*/
import { estimateStringTokens } from '../context-compressor.js';
import type { LlmCallFn, RetrievalSearchFn } from '../retrieval-agent-loop.js';
import {
CHECKPOINT_SCHEMA_VERSION,
type CheckpointStepState,
type Decision,
} from './checkpoint.js';
// ─────────────────────────────────────────────────────────────────────────
// Public types
// ─────────────────────────────────────────────────────────────────────────
export type CompressionStrategy = 'summarize-only' | 'retrieve-only' | 'hybrid';
export interface ArchivedContextRange {
/** The text that was archived/replaced. */
archived_text: string;
/** Char offset where the archived range began (0-indexed) in the original context. */
range_start: number;
/** Char offset where the archived range ended (exclusive) in the original context. */
range_end: number;
}
export interface ContextCompressionEvent {
type: 'context-compressed' | 'context-compression-skipped';
/** Tokens before the operation. */
before_tokens: number;
/** Tokens after the operation. */
after_tokens: number;
/** Strategy used. */
strategy: CompressionStrategy;
/** USD cost incurred (LLM-summarize calls). 0 for retrieve-only or skipped. */
cost_usd: number;
/** Archived range (set on retrieve-only and hybrid). */
archived?: ArchivedContextRange;
}
export interface DecisionsCompressionEvent {
type: 'decisions-compressed';
/** Number of decisions before. */
before_count: number;
/** Number of decisions after. */
after_count: number;
/** Number archived (= before - after - 1 summary entry, when summary added). */
evicted_count: number;
/** USD cost (0 if heuristic summary used). */
cost_usd: number;
}
export interface CacheEvictionEvent {
type: 'cache-evicted';
/** Cache size before eviction. */
before_size: number;
/** Cache size after eviction. */
after_size: number;
/** Number of entries dropped. */
evicted_count: number;
}
export type CompressionEvent = ContextCompressionEvent | DecisionsCompressionEvent | CacheEvictionEvent;
export interface ContextManagerOptions {
/** Token budget for accumulated_context. Caller picks based on model. */
contextTokenBudget: number;
/** Threshold (fraction of budget) that triggers proactive compression. Default 0.7. */
compressionThreshold?: number;
/** Compression strategy. Default 'hybrid'. */
strategy?: CompressionStrategy;
/** LLM call for summarization. Required for 'summarize-only' / 'hybrid'. */
llmCall?: LlmCallFn;
/** Optional retrieval function (consumed by Phase 3.4, not by ContextManager directly). */
retrievalSearch?: RetrievalSearchFn;
/** Optional override for token estimation. Default = estimateStringTokens. */
estimateTokensFn?: (text: string) => number;
/** Model alias for summarization. Required if llmCall provided. */
summarizationModel?: string;
/** max_tokens for the summarization LLM call. Default 1500. */
summarizationMaxTokens?: number;
/** Recent verbatim chars to retain at the END of accumulated_context. Default 4000. */
retainRecentChars?: number;
/** Decision history compression: keep last K decisions verbatim. Default 10. */
retainRecentDecisions?: number;
/** Retrieval cache LRU max size. Default 500 entries. */
retrievalCacheMaxSize?: number;
/** Optional callback for compression telemetry (any compression event type). */
onCompressionEvent?: (event: CompressionEvent) => void;
/** Optional async callback to persist evicted decisions (audit trail). */
archiveDecisionsTo?: (decisions: readonly Decision[]) => Promise<void>;
/** Optional async callback to persist evicted retrieval cache entries. */
archiveCacheTo?: (entries: ReadonlyArray<readonly [string, unknown]>) => Promise<void>;
}
export interface CompressOptions {
/**
* Optional query string for retrieval-aware compression. Reserved for
* future use; ContextManager does not currently call retrievalSearch
* directly (Phase 3.4 will wire that into the agent loop). Stored on
* the event for caller introspection.
*/
stepQuery?: string;
}
// ─────────────────────────────────────────────────────────────────────────
// Constants + helpers
// ─────────────────────────────────────────────────────────────────────────
const DEFAULT_COMPRESSION_THRESHOLD = 0.7;
const DEFAULT_STRATEGY: CompressionStrategy = 'hybrid';
const DEFAULT_RETAIN_RECENT_CHARS = 4000;
const DEFAULT_RETAIN_RECENT_DECISIONS = 10;
const DEFAULT_CACHE_MAX_SIZE = 500;
const DEFAULT_SUMMARIZATION_MAX_TOKENS = 1500;
const CONTEXT_COMPACTION_SYSTEM_PROMPT = [
'You compress an agent\'s scratchpad / running context.',
'Preserve every material fact, decision, unresolved question, and open thread.',
'Drop verbose narration, repetition, and intermediate reasoning that was already concluded.',
'Output a tight, structured summary — no preamble, no apologies.',
].join(' ');
const DECISION_COMPACTION_SYSTEM_PROMPT = [
'You summarize an agent\'s history of decisions.',
'Output a single concise paragraph capturing the gist of all decisions provided —',
'what was chosen, what was rejected, and why.',
'No preamble, no apologies.',
].join(' ');
// ─────────────────────────────────────────────────────────────────────────
// ContextManager
// ─────────────────────────────────────────────────────────────────────────
export class ContextManager {
private readonly contextTokenBudget: number;
private readonly compressionThreshold: number;
private readonly strategy: CompressionStrategy;
private readonly llmCall: LlmCallFn | undefined;
private readonly retrievalSearch: RetrievalSearchFn | undefined;
private readonly estimateFn: (text: string) => number;
private readonly summarizationModel: string | undefined;
private readonly summarizationMaxTokens: number;
private readonly retainRecentChars: number;
private readonly retainRecentDecisions: number;
private readonly retrievalCacheMaxSize: number;
private readonly emit: (event: CompressionEvent) => void;
private readonly archiveDecisionsTo: ((decisions: readonly Decision[]) => Promise<void>) | undefined;
private readonly archiveCacheTo: ((entries: ReadonlyArray<readonly [string, unknown]>) => Promise<void>) | undefined;
constructor(opts: ContextManagerOptions) {
if (opts.contextTokenBudget <= 0) {
throw new Error(`ContextManager: contextTokenBudget must be > 0 (got ${opts.contextTokenBudget})`);
}
const threshold = opts.compressionThreshold ?? DEFAULT_COMPRESSION_THRESHOLD;
if (threshold <= 0 || threshold > 1) {
throw new Error(`ContextManager: compressionThreshold must be in (0, 1] (got ${threshold})`);
}
const strategy = opts.strategy ?? DEFAULT_STRATEGY;
const needsLlm = strategy === 'summarize-only' || strategy === 'hybrid';
if (needsLlm && (!opts.llmCall || !opts.summarizationModel)) {
throw new Error(
`ContextManager: strategy='${strategy}' requires both llmCall and summarizationModel`,
);
}
this.contextTokenBudget = opts.contextTokenBudget;
this.compressionThreshold = threshold;
this.strategy = strategy;
this.llmCall = opts.llmCall;
this.retrievalSearch = opts.retrievalSearch;
this.estimateFn = opts.estimateTokensFn ?? estimateStringTokens;
this.summarizationModel = opts.summarizationModel;
this.summarizationMaxTokens = opts.summarizationMaxTokens ?? DEFAULT_SUMMARIZATION_MAX_TOKENS;
this.retainRecentChars = opts.retainRecentChars ?? DEFAULT_RETAIN_RECENT_CHARS;
this.retainRecentDecisions = opts.retainRecentDecisions ?? DEFAULT_RETAIN_RECENT_DECISIONS;
this.retrievalCacheMaxSize = opts.retrievalCacheMaxSize ?? DEFAULT_CACHE_MAX_SIZE;
this.emit = opts.onCompressionEvent ?? (() => {});
this.archiveDecisionsTo = opts.archiveDecisionsTo;
this.archiveCacheTo = opts.archiveCacheTo;
if (this.retainRecentChars < 0) {
throw new Error(`ContextManager: retainRecentChars must be >= 0 (got ${this.retainRecentChars})`);
}
if (this.retainRecentDecisions < 0 || !Number.isInteger(this.retainRecentDecisions)) {
throw new Error(`ContextManager: retainRecentDecisions must be a non-negative integer (got ${this.retainRecentDecisions})`);
}
if (this.retrievalCacheMaxSize < 0 || !Number.isInteger(this.retrievalCacheMaxSize)) {
throw new Error(`ContextManager: retrievalCacheMaxSize must be a non-negative integer (got ${this.retrievalCacheMaxSize})`);
}
}
/** Estimate token count of a single text string. */
estimateTokens(text: string): number {
return this.estimateFn(text);
}
/** Returns true if the state's accumulated_context exceeds the configured threshold. */
needsCompression(state: CheckpointStepState): boolean {
const tokens = this.estimateTokens(state.accumulated_context);
return tokens > this.contextTokenBudget * this.compressionThreshold;
}
/**
* Compress accumulated_context per the configured strategy. Returns a NEW
* state object (does not mutate input). If state's context is below the
* threshold OR shorter than retainRecentChars, returns input unchanged.
*
* Strategies:
* - 'summarize-only': LLM-summarize older portion + retain recent
* - 'retrieve-only': drop older portion + retain recent + emit archive event
* - 'hybrid': summarize older + emit archive event for retrieval indexing
*/
async compress(state: CheckpointStepState, opts: CompressOptions = {}): Promise<CheckpointStepState> {
void opts; // stepQuery reserved for Phase 3.4 retrieval-aware compression
const beforeTokens = this.estimateTokens(state.accumulated_context);
if (!this.needsCompression(state)) {
this.emit({
type: 'context-compression-skipped',
before_tokens: beforeTokens,
after_tokens: beforeTokens,
strategy: this.strategy,
cost_usd: 0,
});
return state;
}
if (state.accumulated_context.length <= this.retainRecentChars) {
// Context exceeds token budget but the entire string fits in retainRecentChars.
// Nothing to safely compress without dropping the recent verbatim region.
this.emit({
type: 'context-compression-skipped',
before_tokens: beforeTokens,
after_tokens: beforeTokens,
strategy: this.strategy,
cost_usd: 0,
});
return state;
}
const splitAt = state.accumulated_context.length - this.retainRecentChars;
const olderText = state.accumulated_context.slice(0, splitAt);
const recentText = state.accumulated_context.slice(splitAt);
const archivedRange: ArchivedContextRange = {
archived_text: olderText,
range_start: 0,
range_end: splitAt,
};
let newContext: string;
let costUsd = 0;
if (this.strategy === 'retrieve-only') {
newContext = `[archived: ${olderText.length} chars; see retrieval index]\n\n${recentText}`;
} else {
// summarize-only or hybrid — LLM-summarize the older portion.
const summaryResult = await this._summarize(olderText);
costUsd = summaryResult.cost_usd;
const summaryText = summaryResult.summary;
newContext = `[summary of ${olderText.length} archived chars]\n${summaryText}\n\n${recentText}`;
}
const afterTokens = this.estimateTokens(newContext);
const newState: CheckpointStepState = {
...state,
accumulated_context: newContext,
};
const event: ContextCompressionEvent = {
type: 'context-compressed',
before_tokens: beforeTokens,
after_tokens: afterTokens,
strategy: this.strategy,
cost_usd: costUsd,
};
if (this.strategy === 'retrieve-only' || this.strategy === 'hybrid') {
event.archived = archivedRange;
}
this.emit(event);
return newState;
}
/**
* LRU-evict retrieval_cache entries to fit retrievalCacheMaxSize. Returns
* a NEW state object (does not mutate input). If cache size already fits,
* returns input unchanged.
*
* If accessOrder is provided (keys ordered most-recent-LAST), entries
* earlier in accessOrder are evicted first; keys not in accessOrder are
* treated as oldest. Without accessOrder, evicts in insertion order
* (JS spec preserves insertion order for string keys).
*/
evictRetrievalCache(state: CheckpointStepState, opts: { accessOrder?: readonly string[] } = {}): CheckpointStepState {
const entries = Object.entries(state.retrieval_cache);
const beforeSize = entries.length;
if (beforeSize <= this.retrievalCacheMaxSize) {
return state;
}
let orderedKeys: string[];
if (opts.accessOrder) {
// accessOrder = recent-last → oldest at index 0.
// Keys not present in accessOrder are oldest of all (prepend).
const accessed = new Set(opts.accessOrder);
const unaccessed = Object.keys(state.retrieval_cache).filter(k => !accessed.has(k));
// Re-order accessOrder to only include keys actually in cache.
const accessedInCache = opts.accessOrder.filter(k => k in state.retrieval_cache);
orderedKeys = [...unaccessed, ...accessedInCache];
} else {
// Insertion order — already provided by Object.keys.
orderedKeys = Object.keys(state.retrieval_cache);
}
const evictCount = beforeSize - this.retrievalCacheMaxSize;
const evictedKeys = orderedKeys.slice(0, evictCount);
const evictedSet = new Set(evictedKeys);
const newCache: Record<string, unknown> = {};
const evictedEntries: Array<[string, unknown]> = [];
for (const [k, v] of entries) {
if (evictedSet.has(k)) {
evictedEntries.push([k, v]);
} else {
newCache[k] = v;
}
}
if (this.archiveCacheTo) {
void this.archiveCacheTo(evictedEntries);
}
this.emit({
type: 'cache-evicted',
before_size: beforeSize,
after_size: Object.keys(newCache).length,
evicted_count: evictedEntries.length,
});
return { ...state, retrieval_cache: newCache };
}
/**
* Compress decision_history to keep only the last K verbatim plus one
* rolled-up summary Decision representing the older bucket. Returns a NEW
* state object (does not mutate input). If history length ≤ retainRecentDecisions,
* returns input unchanged.
*
* If llmCall is configured, the older bucket is summarized via LLM call.
* Otherwise a heuristic summary (concatenation of decision strings) is used.
* If archiveDecisionsTo is set, the original older bucket is persisted
* before compression for lossless audit trail.
*/
async compressDecisionHistory(state: CheckpointStepState): Promise<CheckpointStepState> {
const history = state.decision_history;
const beforeCount = history.length;
if (beforeCount <= this.retainRecentDecisions) {
return state;
}
const splitAt = beforeCount - this.retainRecentDecisions;
const olderBucket = history.slice(0, splitAt);
const recentBucket = history.slice(splitAt);
if (this.archiveDecisionsTo) {
void this.archiveDecisionsTo(olderBucket);
}
let summaryText: string;
let costUsd = 0;
if (this.llmCall && this.summarizationModel) {
const result = await this._summarizeDecisions(olderBucket);
summaryText = result.summary;
costUsd = result.cost_usd;
} else {
summaryText = buildHeuristicDecisionSummary(olderBucket);
}
const summaryDecision: Decision = {
step_index: olderBucket[0]?.step_index ?? 0,
decision: `[rolled-up summary of ${olderBucket.length} earlier decisions]`,
rationale: summaryText,
};
const newHistory = [summaryDecision, ...recentBucket];
const afterCount = newHistory.length;
this.emit({
type: 'decisions-compressed',
before_count: beforeCount,
after_count: afterCount,
evicted_count: olderBucket.length,
cost_usd: costUsd,
});
void CHECKPOINT_SCHEMA_VERSION; // pin import for forward-compat use
return { ...state, decision_history: newHistory };
}
// ──────────────────────────────────────────────────────────────────────
// Private — LLM summarization helpers
// ──────────────────────────────────────────────────────────────────────
private async _summarize(text: string): Promise<{ summary: string; cost_usd: number }> {
if (!this.llmCall || !this.summarizationModel) {
throw new Error('ContextManager: _summarize called without llmCall — constructor invariant violated');
}
const result = await this.llmCall({
model: this.summarizationModel,
messages: [
{ role: 'system', content: CONTEXT_COMPACTION_SYSTEM_PROMPT },
{ role: 'user', content: text },
],
maxTokens: this.summarizationMaxTokens,
temperature: 0.1,
});
return { summary: result.content, cost_usd: result.costUsd };
}
private async _summarizeDecisions(decisions: readonly Decision[]): Promise<{ summary: string; cost_usd: number }> {
if (!this.llmCall || !this.summarizationModel) {
throw new Error('ContextManager: _summarizeDecisions called without llmCall — constructor invariant violated');
}
const formatted = decisions.map(d => {
const parts = [`Step ${d.step_index}: ${d.decision}`];
if (d.rationale) parts.push(` rationale: ${d.rationale}`);
if (d.alternatives_considered && d.alternatives_considered.length > 0) {
parts.push(` alternatives: ${d.alternatives_considered.join(' | ')}`);
}
return parts.join('\n');
}).join('\n\n');
const result = await this.llmCall({
model: this.summarizationModel,
messages: [
{ role: 'system', content: DECISION_COMPACTION_SYSTEM_PROMPT },
{ role: 'user', content: formatted },
],
maxTokens: this.summarizationMaxTokens,
temperature: 0.1,
});
return { summary: result.content, cost_usd: result.costUsd };
}
}
// ─────────────────────────────────────────────────────────────────────────
// Heuristic summary fallback (no LLM)
// ─────────────────────────────────────────────────────────────────────────
function buildHeuristicDecisionSummary(decisions: readonly Decision[]): string {
const lines = decisions.map(d => {
const stepBit = `step ${d.step_index}: `;
const decisionBit = d.decision.length > 80 ? d.decision.slice(0, 77) + '…' : d.decision;
return stepBit + decisionBit;
});
return lines.join('; ');
}

View File

@@ -0,0 +1,594 @@
/**
* Failure classifier — Phase 4.1 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.1)
*
* Classifies a model's failed output against a gold answer into one of ten
* pre-registered failure categories. Rule-based first (fast, free, deterministic);
* LLM-judge fallback only for ambiguous cases when the caller provides one.
*
* The ten categories are pre-registered in the sprint plan and reflect the
* 2026-04-26 pilot's empirical failure-mode taxonomy. Phase 4.3 re-score uses
* this classifier to bucket the 12-cell pilot's H2/H3/H4 failures into
* Tier 1 (presentation / fixable by output-normalize layer) vs Tier 2
* (real harness-design issues).
*
* The classifier walks rules in PRIORITY ORDER (most-specific first):
* 1. retrieval_or_harness_error (upstream signal overrides everything)
* 2. thinking_leakage (explicit <think> tags or chain-of-thought prefixes)
* 3. metadata_copy (literal substrate metadata in output)
* 4. format_violation (bullet list when prose expected, JSON when text, etc.)
* 5. unknown_false_negative (model abstained but gold is answerable)
* 6. punctuation_or_case_only (normalize-equal but raw-different)
* 7. correct_answer_with_extra_text (gold-substring-match but output is verbose)
* 8. wrong_span (high token overlap, but no substring match)
* 9. wrong_entity (low token overlap, definitive statement)
* 10. hallucination (default fallback for "wrong but uncategorized")
*
* Each category emits a `confidence` level:
* - 'high': explicit signal (e.g., literal <think> tag, exact token-set equality)
* - 'medium': heuristic signal (e.g., format-prefix mismatch, token-overlap threshold)
* - 'low': no clear signal — defaults to hallucination
*
* Callers MUST decide whether to act on low-confidence classifications
* (e.g., surface for human review, or invoke LLM judge fallback).
*/
import type { LlmCallFn } from '../retrieval-agent-loop.js';
// ─────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────
export type FailureCategory =
| 'thinking_leakage'
| 'correct_answer_with_extra_text'
| 'unknown_false_negative'
| 'metadata_copy'
| 'format_violation'
| 'punctuation_or_case_only'
| 'wrong_span'
| 'wrong_entity'
| 'hallucination'
| 'retrieval_or_harness_error';
export const FAILURE_CATEGORIES: readonly FailureCategory[] = [
'thinking_leakage',
'correct_answer_with_extra_text',
'unknown_false_negative',
'metadata_copy',
'format_violation',
'punctuation_or_case_only',
'wrong_span',
'wrong_entity',
'hallucination',
'retrieval_or_harness_error',
] as const;
export type ConfidenceLevel = 'high' | 'medium' | 'low';
export interface FailureClassification {
primary: FailureCategory;
confidence: ConfidenceLevel;
secondary?: FailureCategory;
rationale: string;
/** Names of rules / heuristics that fired. Empty for LLM-judge-only verdicts. */
rules_fired: readonly string[];
/** Whether the LLM-judge fallback was invoked. */
llm_judge_invoked: boolean;
}
export interface ClassifierInput {
model_output: string;
/** Single string or list of acceptable gold answers (any-match = passing). */
gold_answer: string | readonly string[];
/** Optional question text — passed to LLM judge if invoked. */
question?: string;
/** Optional substrate / retrieved context — used to detect metadata leakage. */
substrate?: string;
/** Optional upstream error string (retrieval/harness). If set, overrides to category #1. */
upstream_error?: string;
}
export interface ClassifierOptions {
/**
* Optional LLM judge function for low-confidence rule cases. If absent, the
* classifier returns the rule's verdict at the rule's confidence level.
*/
llmJudge?: LlmCallFn;
/** Model alias for the LLM judge. Required if llmJudge is provided. */
judgeModel?: string;
/**
* If true (default), the LLM judge is invoked for 'low' confidence verdicts
* to potentially upgrade them. If false, never invoke the judge regardless.
*/
invokeJudgeOnLowConfidence?: boolean;
/** Per-call max_tokens for the LLM judge. Default 256. */
judgeMaxTokens?: number;
}
// ─────────────────────────────────────────────────────────────────────────
// Detector regexes / constants
// ─────────────────────────────────────────────────────────────────────────
const THINKING_TAG_RE = /<\s*\/?\s*think(?:ing)?\s*[>\s]/i;
const THINKING_PREFIX_RE = /^(let me think|first[,\s]|step \d|i need to|i'?ll start|let's break|to answer this|let me analyze|let me work through)/i;
const METADATA_LITERAL_PATTERNS = [
/\[memory:[^\]]*\]/i,
/\[ref:[^\]]*\]/i,
/\[source:[^\]]*\]/i,
/from session\s*(?:#?\d+|"[^"]+")/i,
/\[\s*context:[^\]]*\]/i,
/^\s*<<\s*[a-z_]+\s*>>/i,
];
const UNKNOWN_OUTPUTS = new Set([
'unknown',
'i don\'t know',
'i do not know',
'cannot determine',
'cannot be determined',
'not specified',
'no information',
'insufficient information',
'no answer',
'n/a',
'na',
'none',
]);
const FORMAT_BULLET_PREFIXES = /^\s*([*\-•]|\d+[.)]\s)/m;
const FORMAT_JSON_PREFIXES = /^\s*[{[]/;
const FORMAT_FENCE_PREFIX = /^\s*```/;
// ─────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────
function asGoldArray(g: string | readonly string[]): readonly string[] {
return typeof g === 'string' ? [g] : g;
}
function lc(s: string): string {
return s.toLowerCase();
}
/** Strip punctuation + collapse whitespace + lowercase. */
function normalizeForCompare(s: string): string {
return s
.toLowerCase()
.replace(/[^\p{L}\p{N}\s]/gu, ' ')
.replace(/\s+/g, ' ')
.trim();
}
function tokenize(s: string): string[] {
return normalizeForCompare(s).split(/\s+/).filter(t => t.length > 0);
}
function tokenSet(s: string): Set<string> {
return new Set(tokenize(s));
}
/** Jaccard similarity over token sets. 0..1. */
function tokenOverlap(a: string, b: string): number {
const sa = tokenSet(a);
const sb = tokenSet(b);
if (sa.size === 0 && sb.size === 0) return 1;
if (sa.size === 0 || sb.size === 0) return 0;
let inter = 0;
for (const t of sa) if (sb.has(t)) inter += 1;
const union = sa.size + sb.size - inter;
return union === 0 ? 0 : inter / union;
}
function lowercaseSubstringMatch(output: string, gold: readonly string[]): boolean {
const lower = output.toLowerCase();
for (const g of gold) {
if (g && lower.includes(g.toLowerCase())) return true;
}
return false;
}
// ─────────────────────────────────────────────────────────────────────────
// Detector functions — each returns { matched, confidence, rationale, rule } | null
// ─────────────────────────────────────────────────────────────────────────
interface DetectorResult {
matched: boolean;
confidence: ConfidenceLevel;
rationale: string;
rule: string;
}
function detectUpstreamError(input: ClassifierInput): DetectorResult | null {
if (input.upstream_error && input.upstream_error.trim().length > 0) {
return {
matched: true,
confidence: 'high',
rationale: `upstream_error present: "${input.upstream_error.slice(0, 80)}"`,
rule: 'upstream-error',
};
}
return null;
}
function detectThinkingLeakage(input: ClassifierInput): DetectorResult | null {
const out = input.model_output;
if (THINKING_TAG_RE.test(out)) {
return {
matched: true,
confidence: 'high',
rationale: 'output contains literal <think> tags',
rule: 'thinking-tag',
};
}
if (THINKING_PREFIX_RE.test(out.trim())) {
return {
matched: true,
confidence: 'medium',
rationale: 'output starts with chain-of-thought prefix',
rule: 'thinking-prefix',
};
}
return null;
}
function detectMetadataCopy(input: ClassifierInput): DetectorResult | null {
const out = input.model_output;
for (const re of METADATA_LITERAL_PATTERNS) {
if (re.test(out)) {
return {
matched: true,
confidence: 'high',
rationale: `output contains literal substrate metadata pattern ${re}`,
rule: 'metadata-literal',
};
}
}
if (input.substrate) {
// Heuristic: long substring of substrate that's clearly metadata-shaped (timestamps,
// session ids) appearing verbatim in the output.
const ts = out.match(/session\s*\d{2,}|gop_id\s*[:=]\s*\S+|frame_id\s*[:=]/i);
if (ts) {
return {
matched: true,
confidence: 'medium',
rationale: `output contains substrate-shaped identifier "${ts[0]}"`,
rule: 'metadata-shaped-id',
};
}
}
return null;
}
function detectFormatViolation(input: ClassifierInput): DetectorResult | null {
const gold = asGoldArray(input.gold_answer);
const goldSample = gold[0] ?? '';
const out = input.model_output;
const outIsBullet = FORMAT_BULLET_PREFIXES.test(out);
const outIsJson = FORMAT_JSON_PREFIXES.test(out.trim());
const outIsFence = FORMAT_FENCE_PREFIX.test(out);
const goldIsBullet = FORMAT_BULLET_PREFIXES.test(goldSample);
const goldIsJson = FORMAT_JSON_PREFIXES.test(goldSample.trim());
if (outIsFence && !goldIsJson && !goldIsBullet) {
return {
matched: true,
confidence: 'high',
rationale: 'output wrapped in ``` code fence; gold is plain prose',
rule: 'format-fence-mismatch',
};
}
if (outIsJson && !goldIsJson) {
return {
matched: true,
confidence: 'medium',
rationale: 'output starts with { or [; gold is plain prose',
rule: 'format-json-mismatch',
};
}
if (outIsBullet && !goldIsBullet) {
return {
matched: true,
confidence: 'medium',
rationale: 'output starts with bullet/numbered list; gold is plain prose',
rule: 'format-bullet-mismatch',
};
}
return null;
}
function detectUnknownFalseNegative(input: ClassifierInput): DetectorResult | null {
const lcOut = input.model_output.toLowerCase().trim();
if (UNKNOWN_OUTPUTS.has(lcOut)) {
return {
matched: true,
confidence: 'high',
rationale: `output is exact "unknown"-class string ("${lcOut}")`,
rule: 'unknown-exact',
};
}
// Heuristic: output is short AND contains unknown-class phrase
if (lcOut.length < 80) {
for (const u of UNKNOWN_OUTPUTS) {
if (lcOut.includes(u)) {
return {
matched: true,
confidence: 'medium',
rationale: `output (short, ${lcOut.length} chars) contains "${u}"`,
rule: 'unknown-substring',
};
}
}
}
return null;
}
function detectPunctuationOrCaseOnly(input: ClassifierInput): DetectorResult | null {
const gold = asGoldArray(input.gold_answer);
const outNorm = normalizeForCompare(input.model_output);
const outRaw = input.model_output.trim();
for (const g of gold) {
if (!g) continue;
if (outNorm === normalizeForCompare(g) && outRaw !== g.trim()) {
return {
matched: true,
confidence: 'high',
rationale: 'output equals gold after normalize, but raw forms differ (punctuation/case)',
rule: 'punct-case-only',
};
}
}
return null;
}
function detectCorrectWithExtraText(input: ClassifierInput): DetectorResult | null {
const gold = asGoldArray(input.gold_answer);
if (!lowercaseSubstringMatch(input.model_output, gold)) return null;
const goldSample = gold.find(g => input.model_output.toLowerCase().includes(g.toLowerCase())) ?? '';
if (!goldSample) return null;
const goldLen = goldSample.length;
const outLen = input.model_output.trim().length;
if (outLen >= goldLen * 1.5 && outLen >= goldLen + 25) {
return {
matched: true,
confidence: outLen >= goldLen * 3 ? 'high' : 'medium',
rationale: `gold "${goldSample.slice(0, 40)}…" is contained in output, but output is ${outLen} chars vs gold ${goldLen} chars`,
rule: 'correct-with-extra',
};
}
return null;
}
function detectWrongSpan(input: ClassifierInput): DetectorResult | null {
const gold = asGoldArray(input.gold_answer);
if (lowercaseSubstringMatch(input.model_output, gold)) return null;
let bestOverlap = 0;
for (const g of gold) {
bestOverlap = Math.max(bestOverlap, tokenOverlap(input.model_output, g));
}
if (bestOverlap >= 0.5) {
return {
matched: true,
confidence: 'medium',
rationale: `Jaccard token overlap ${bestOverlap.toFixed(2)} ≥ 0.5 but no substring match`,
rule: 'wrong-span',
};
}
return null;
}
function detectWrongEntity(input: ClassifierInput): DetectorResult | null {
const gold = asGoldArray(input.gold_answer);
if (lowercaseSubstringMatch(input.model_output, gold)) return null;
let bestOverlap = 0;
for (const g of gold) {
bestOverlap = Math.max(bestOverlap, tokenOverlap(input.model_output, g));
}
// Definitive statement: output > 5 tokens AND not an unknown phrase.
const tokens = tokenize(input.model_output);
const definitive = tokens.length >= 5 && !Array.from(UNKNOWN_OUTPUTS).some(u => input.model_output.toLowerCase().includes(u));
if (bestOverlap < 0.3 && definitive) {
return {
matched: true,
confidence: bestOverlap < 0.1 ? 'high' : 'medium',
rationale: `Jaccard token overlap ${bestOverlap.toFixed(2)} < 0.3 with definitive output (${tokens.length} tokens)`,
rule: 'wrong-entity',
};
}
return null;
}
// ─────────────────────────────────────────────────────────────────────────
// LLM judge fallback
// ─────────────────────────────────────────────────────────────────────────
const LLM_JUDGE_PROMPT_TEMPLATE = (input: ClassifierInput): string => {
const gold = asGoldArray(input.gold_answer).join(' | ');
return [
'You are a failure-mode classifier for a benchmark agent\'s output.',
'',
`Question: ${input.question ?? '(not provided)'}`,
`Gold answer (any-of): ${gold}`,
`Model output: ${input.model_output}`,
'',
'Classify into exactly one of these categories:',
' - thinking_leakage (output contains chain-of-thought reasoning that should have been stripped)',
' - correct_answer_with_extra_text (output contains the gold answer but with verbose extra prose)',
' - unknown_false_negative (output abstained but the gold answer is definite)',
' - metadata_copy (output contains literal substrate metadata)',
' - format_violation (output uses wrong format — bullets/JSON/code fence vs expected prose)',
' - punctuation_or_case_only (output equals gold modulo punctuation or case)',
' - wrong_span (output gets the right entity but wrong span — too narrow or too broad)',
' - wrong_entity (output references the wrong entity entirely)',
' - hallucination (output contains content not in gold or substrate)',
' - retrieval_or_harness_error (failure caused by retrieval / infrastructure, not model reasoning)',
'',
'Output JSON only (no prose, no fences):',
'{"primary":"<category>","confidence":"high|medium|low","rationale":"<one short sentence>"}',
].join('\n');
};
async function invokeLlmJudge(
input: ClassifierInput,
opts: ClassifierOptions,
): Promise<{ primary: FailureCategory; confidence: ConfidenceLevel; rationale: string } | null> {
if (!opts.llmJudge || !opts.judgeModel) return null;
const prompt = LLM_JUDGE_PROMPT_TEMPLATE(input);
const r = await opts.llmJudge({
model: opts.judgeModel,
messages: [{ role: 'user', content: prompt }],
maxTokens: opts.judgeMaxTokens ?? 256,
temperature: 0,
});
if (r.error) return null;
const m = r.content.match(/\{[\s\S]*\}/);
if (!m) return null;
try {
const obj = JSON.parse(m[0]) as { primary?: string; confidence?: string; rationale?: string };
const primary = obj.primary as FailureCategory;
if (!FAILURE_CATEGORIES.includes(primary)) return null;
const confidence = (obj.confidence as ConfidenceLevel) ?? 'medium';
return {
primary,
confidence: confidence === 'high' || confidence === 'medium' || confidence === 'low' ? confidence : 'medium',
rationale: obj.rationale ?? '(no rationale)',
};
} catch {
return null;
}
}
// ─────────────────────────────────────────────────────────────────────────
// Main entry point
// ─────────────────────────────────────────────────────────────────────────
interface RuleHit {
category: FailureCategory;
result: DetectorResult;
}
function runRuleCascade(input: ClassifierInput): RuleHit[] {
const hits: RuleHit[] = [];
const detectors: Array<{ category: FailureCategory; fn: (i: ClassifierInput) => DetectorResult | null }> = [
{ category: 'retrieval_or_harness_error', fn: detectUpstreamError },
{ category: 'thinking_leakage', fn: detectThinkingLeakage },
{ category: 'metadata_copy', fn: detectMetadataCopy },
{ category: 'format_violation', fn: detectFormatViolation },
{ category: 'unknown_false_negative', fn: detectUnknownFalseNegative },
{ category: 'punctuation_or_case_only', fn: detectPunctuationOrCaseOnly },
{ category: 'correct_answer_with_extra_text', fn: detectCorrectWithExtraText },
{ category: 'wrong_span', fn: detectWrongSpan },
{ category: 'wrong_entity', fn: detectWrongEntity },
];
for (const { category, fn } of detectors) {
const r = fn(input);
if (r && r.matched) hits.push({ category, result: r });
}
return hits;
}
/**
* Classify a model output failure into one of the 10 pre-registered categories.
*
* Walks the rule cascade in priority order. If at least one rule fires, returns
* the highest-priority hit's verdict. If no rules fire, returns 'hallucination'
* with 'low' confidence (the catch-all default).
*
* If options.llmJudge is provided AND the rule-cascade verdict is 'low'
* confidence (or no rule fired), the LLM judge is invoked for potential
* upgrade. The judge's verdict overrides only if it returns a known category.
*/
export async function classifyFailure(
input: ClassifierInput,
opts: ClassifierOptions = {},
): Promise<FailureClassification> {
const hits = runRuleCascade(input);
let primary: FailureCategory;
let confidence: ConfidenceLevel;
let rationale: string;
let rulesFired: string[];
let secondary: FailureCategory | undefined;
if (hits.length > 0) {
const top = hits[0]!;
primary = top.category;
confidence = top.result.confidence;
rationale = top.result.rationale;
rulesFired = hits.map(h => h.result.rule);
if (hits.length > 1) secondary = hits[1]!.category;
} else {
primary = 'hallucination';
confidence = 'low';
rationale = 'no rule fired; defaulting to hallucination (catch-all)';
rulesFired = [];
}
// LLM judge fallback for low-confidence verdicts.
const shouldInvokeJudge =
Boolean(opts.llmJudge && opts.judgeModel) &&
confidence === 'low' &&
(opts.invokeJudgeOnLowConfidence ?? true);
if (shouldInvokeJudge) {
const judgeVerdict = await invokeLlmJudge(input, opts);
if (judgeVerdict) {
return {
primary: judgeVerdict.primary,
confidence: judgeVerdict.confidence,
secondary: primary === judgeVerdict.primary ? secondary : primary,
rationale: `[llm-judge] ${judgeVerdict.rationale}`,
rules_fired: rulesFired,
llm_judge_invoked: true,
};
}
}
return {
primary,
confidence,
secondary,
rationale,
rules_fired: rulesFired,
llm_judge_invoked: false,
};
}
/**
* Classify a batch of inputs. Convenience wrapper over classifyFailure.
*/
export async function classifyFailureBatch(
inputs: readonly ClassifierInput[],
opts: ClassifierOptions = {},
): Promise<readonly FailureClassification[]> {
const out: FailureClassification[] = [];
for (const inp of inputs) {
out.push(await classifyFailure(inp, opts));
}
return out;
}
/**
* Compute per-category counts from a list of classifications.
*/
export function failureDistribution(
classifications: readonly FailureClassification[],
): Readonly<Record<FailureCategory, number>> {
const dist: Record<FailureCategory, number> = {
thinking_leakage: 0,
correct_answer_with_extra_text: 0,
unknown_false_negative: 0,
metadata_copy: 0,
format_violation: 0,
punctuation_or_case_only: 0,
wrong_span: 0,
wrong_entity: 0,
hallucination: 0,
retrieval_or_harness_error: 0,
};
for (const c of classifications) {
dist[c.primary] += 1;
}
return dist;
}

View File

@@ -0,0 +1,326 @@
/**
* Messages-array compression — Phase 4.6 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.6)
*
* Addresses the Phase 3 acceptance gate finding: ContextManager (Phase 3.3)
* compresses ONLY accumulated_context (audit log, ~450 tokens by step 30 —
* never crosses 4K threshold). Real LLM cost is dominated by the messages
* array (~16K tokens by step 30 on Opus). Phase 4.6 closes the gap by adding
* a SEPARATE compression layer for the messages array, invoked at the top of
* each retrieval-agent-loop iteration.
*
* REUSE / NON-DUPLICATION (per CLAUDE.md §3.6 check-before-create):
* - estimateStringTokens (per-string version) from context-compressor.ts
* for content-aware token estimation
* - pruneToolResults from context-compressor.ts (drops tool-result blobs in
* older messages to reduce size before LLM summarization)
* - COMPACTION_PROMPT from behavioral-spec.ts (proven summarizer prompt)
*
* NEW (this module):
* - summarizeMiddleViaLlmCall: drop-in replacement for context-compressor's
* fetch-based summarizeMiddle that uses Phase 2.1's LlmCallFn abstraction.
* - Simple message-count-based split (head + middle + tail) — PM brief
* specified `retainRecentTurns` as a COUNT, not the token-budget surface
* used by context-compressor.ts's splitProtectedRegions.
*
* NOT REFACTORED:
* - context-compressor.ts itself stays fetch-based for its existing
* consumers (harness + server). Future Sprint 12 cleanup could unify on
* LlmCallFn but that's separate scope.
*/
import {
estimateStringTokens,
pruneToolResults,
type CompressibleMessage,
} from '../context-compressor.js';
import { COMPACTION_PROMPT } from '../behavioral-spec.js';
import type { LlmCallFn } from '../retrieval-agent-loop.js';
// ─────────────────────────────────────────────────────────────────────────
// Public types
// ─────────────────────────────────────────────────────────────────────────
export interface MessagesCompressionEvent {
type: 'messages_compressed';
before_tokens: number;
after_tokens: number;
messages_before: number;
messages_after: number;
cost_usd: number;
summary_generated: boolean;
}
export interface MessagesContextManagerConfig {
/** Token budget for the messages array (per LLM call). */
budgetTokens: number;
/** Fraction of budget that triggers compression. Default 0.7. */
threshold?: number;
/**
* Number of recent TURNS to retain verbatim. One turn ≈ 2 messages
* (assistant + user, or vice-versa). Default 5 turns = 10 messages.
*/
retainRecentTurns?: number;
/**
* Number of head messages (after the system prompt) to keep verbatim.
* Default 1 (typically the kickoff user prompt — system is always retained).
*/
protectedHeadMessages?: number;
/**
* Per-string token estimator override. Default = estimateStringTokens
* (content-aware: prose 4.0 / code 3.2 / json 3.5 / mixed 2.5 chars-per-token).
* Caller can inject tiktoken or Anthropic count_tokens for exact counts.
*/
estimateTokensFn?: (text: string) => number;
/**
* LLM call function for summarizing the middle region. If absent, the
* middle is replaced with a structural placeholder ("[N compressed
* messages]") — pruning still happens but no semantic summary.
*/
llmCall?: LlmCallFn;
/** Model alias for the summarizer. Required if llmCall is provided. */
summarizationModel?: string;
/** max_tokens for the summarizer LLM call. Default 2000. */
summarizationMaxTokens?: number;
/** Optional callback for messages-compression events. */
onCompressionEvent?: (event: MessagesCompressionEvent) => void;
}
export interface CompressMessagesResult {
/** The (possibly compressed) messages array. */
messages: Array<{ role: string; content: string }>;
/** Updated running summary (carries forward across compressions). */
summary: string | null;
/** Whether compression actually fired this call. */
compressed: boolean;
}
// ─────────────────────────────────────────────────────────────────────────
// Constants
// ─────────────────────────────────────────────────────────────────────────
const DEFAULT_THRESHOLD = 0.7;
const DEFAULT_RETAIN_RECENT_TURNS = 5;
const DEFAULT_PROTECTED_HEAD_MESSAGES = 1;
const DEFAULT_SUMMARIZATION_MAX_TOKENS = 2000;
const PER_MESSAGE_ROLE_OVERHEAD_TOKENS = 4;
// ─────────────────────────────────────────────────────────────────────────
// Internal helpers
// ─────────────────────────────────────────────────────────────────────────
function estimateMessagesTokens(
messages: ReadonlyArray<{ role: string; content: string }>,
perStringFn: (text: string) => number,
): number {
let sum = 0;
for (const m of messages) {
sum += PER_MESSAGE_ROLE_OVERHEAD_TOKENS;
sum += perStringFn(m.content ?? '');
}
return sum;
}
interface SplitRegions {
head: Array<{ role: string; content: string }>;
middle: Array<{ role: string; content: string }>;
tail: Array<{ role: string; content: string }>;
}
/**
* Split messages into head (system + first N messages) / middle (compressible)
* / tail (last K turns × 2 messages, retained verbatim).
*
* One turn ≈ 2 messages, so retainRecentTurns × 2 messages are kept in tail.
* Head + tail must not overlap; if the conversation is short, middle may be empty.
*/
function splitForCompress(
messages: ReadonlyArray<{ role: string; content: string }>,
protectedHead: number,
retainRecentTurns: number,
): SplitRegions {
const headEnd = Math.min(1 + protectedHead, messages.length);
const tailStart = Math.max(headEnd, messages.length - retainRecentTurns * 2);
return {
head: messages.slice(0, headEnd).map(m => ({ ...m })),
middle: messages.slice(headEnd, tailStart).map(m => ({ ...m })),
tail: messages.slice(tailStart).map(m => ({ ...m })),
};
}
// ─────────────────────────────────────────────────────────────────────────
// LLM-call-based summarizer (replaces fetch-based summarizeMiddle)
// ─────────────────────────────────────────────────────────────────────────
interface SummarizeResult {
summary: string;
costUsd: number;
generated: boolean;
}
async function summarizeMiddleViaLlmCall(
middle: ReadonlyArray<CompressibleMessage>,
llmCall: LlmCallFn,
model: string,
maxTokens: number,
previousSummary: string | null,
): Promise<SummarizeResult> {
if (middle.length === 0) {
return { summary: previousSummary ?? '', costUsd: 0, generated: false };
}
const summarizerMessages: Array<{ role: string; content: string }> = [];
if (previousSummary) {
summarizerMessages.push({
role: 'system',
content: `You are summarizing a conversation that has been compressed before. Here is the previous summary:\n\n${previousSummary}\n\nNow incorporate the new messages below into an updated summary.`,
});
}
for (const msg of middle) {
summarizerMessages.push({
role: msg.role === 'system' ? 'user' : msg.role,
content: msg.content ?? '',
});
}
summarizerMessages.push({ role: 'user', content: COMPACTION_PROMPT });
const r = await llmCall({
model,
messages: summarizerMessages,
maxTokens,
temperature: 0.1,
// Compaction is a known-lightweight internal call → route to the cheap model
// (model-class-router). Saves Waggle-funded proxy spend on every compaction.
class: 'lightweight',
});
if (r.error || !r.content) {
return { summary: buildFallbackSummary(middle, previousSummary), costUsd: r.costUsd, generated: false };
}
return { summary: r.content, costUsd: r.costUsd, generated: true };
}
function buildFallbackSummary(
middle: ReadonlyArray<CompressibleMessage>,
previousSummary: string | null,
): string {
const lines: string[] = [];
if (previousSummary) lines.push('## Previous summary\n' + previousSummary);
lines.push(`## Compressed region (${middle.length} messages, no LLM summary)`);
const userFirstLines = middle
.filter(m => m.role === 'user')
.map(m => (m.content ?? '').split('\n')[0]?.slice(0, 100).trim())
.filter((l): l is string => Boolean(l && l.length > 8))
.slice(0, 5);
if (userFirstLines.length > 0) {
lines.push('Topics covered: ' + userFirstLines.join(' → '));
}
return lines.join('\n\n');
}
// ─────────────────────────────────────────────────────────────────────────
// Main entry point
// ─────────────────────────────────────────────────────────────────────────
/**
* Compress the messages array if it exceeds threshold. Returns the (possibly
* compressed) messages, a running summary (carries forward), and a boolean
* indicating whether compression actually fired.
*
* Pipeline (when triggered):
* 1. Prune tool-result-shaped blobs from older messages (no LLM, free).
* 2. Split into head (system + first N) / middle (compressible) / tail
* (last retainRecentTurns × 2 messages verbatim).
* 3. Summarize the middle via LLM (or fallback placeholder if no LLM).
* 4. Replace middle with single summary message; reassemble [head, summary, tail].
*
* Idempotency: if called twice in a row without growth, the second call is a
* no-op (under-threshold).
*/
export async function maybeCompressMessages(
messages: ReadonlyArray<{ role: string; content: string }>,
config: MessagesContextManagerConfig,
previousSummary: string | null,
): Promise<CompressMessagesResult> {
const stringEstimator = config.estimateTokensFn ?? estimateStringTokens;
const threshold = config.threshold ?? DEFAULT_THRESHOLD;
const protectedHead = config.protectedHeadMessages ?? DEFAULT_PROTECTED_HEAD_MESSAGES;
const retainRecentTurns = config.retainRecentTurns ?? DEFAULT_RETAIN_RECENT_TURNS;
const tokensBefore = estimateMessagesTokens(messages, stringEstimator);
if (tokensBefore <= config.budgetTokens * threshold) {
return { messages: messages.map(m => ({ ...m })), summary: previousSummary, compressed: false };
}
// Step 1: prune tool-result blobs (free, no LLM).
const pruneTailCount = retainRecentTurns * 2;
const pruned = pruneToolResults(messages, pruneTailCount);
// Step 2: split into head / middle / tail.
const regions = splitForCompress(pruned, protectedHead, retainRecentTurns);
// If middle is empty (very short conversation, head + tail covers everything),
// no compression is possible — return as-is (compressed=false).
if (regions.middle.length === 0) {
return { messages: pruned.map(m => ({ ...m })), summary: previousSummary, compressed: false };
}
// Step 3: summarize middle.
let summaryText: string;
let summaryGenerated = false;
let costUsd = 0;
if (config.llmCall && config.summarizationModel) {
const r = await summarizeMiddleViaLlmCall(
regions.middle,
config.llmCall,
config.summarizationModel,
config.summarizationMaxTokens ?? DEFAULT_SUMMARIZATION_MAX_TOKENS,
previousSummary,
);
summaryText = r.summary;
summaryGenerated = r.generated;
costUsd = r.costUsd;
} else {
summaryText = buildFallbackSummary(regions.middle, previousSummary);
}
// Step 4: replace middle with summary message.
const summaryMessage = {
role: 'system',
content: `[Compressed: ${regions.middle.length} earlier messages]\n${summaryText}`,
};
const newMessages = [...regions.head, summaryMessage, ...regions.tail];
const tokensAfter = estimateMessagesTokens(newMessages, stringEstimator);
if (config.onCompressionEvent) {
config.onCompressionEvent({
type: 'messages_compressed',
before_tokens: tokensBefore,
after_tokens: tokensAfter,
messages_before: messages.length,
messages_after: newMessages.length,
cost_usd: costUsd,
summary_generated: summaryGenerated,
});
}
return { messages: newMessages, summary: summaryText, compressed: true };
}
/**
* Returns true if the given messages array would trigger compression under
* the given config. Exposed for callers that want to check without invoking.
*/
export function shouldCompressMessages(
messages: ReadonlyArray<{ role: string; content: string }>,
config: MessagesContextManagerConfig,
): boolean {
const stringEstimator = config.estimateTokensFn ?? estimateStringTokens;
const threshold = config.threshold ?? DEFAULT_THRESHOLD;
return estimateMessagesTokens(messages, stringEstimator) > config.budgetTokens * threshold;
}

View File

@@ -0,0 +1,464 @@
/**
* Long-task recovery — Phase 3.2 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.2)
*
* Wraps a single step's lifecycle (initial attempt + retries + fallback chain)
* around a CheckpointStore so the agent loop (Phase 3.4) can be crash-resilient
* and tool-failure-resilient. The state machine determines which step_index to
* attempt next based on the latest checkpoint:
*
* no prior checkpoint → fresh start (step 0, prior_state=undefined)
* prior checkpoint with error→ retry that step_index using step N-1's state
* prior checkpoint clean → advance to step_index + 1 using prior as state
*
* Within a single run() call:
* 1. Run stepFn with attempt=1.
* 2. If it throws: classify as 'retryable' or 'terminal'.
* - 'retryable' AND attempts left → exponential backoff + jitter, retry.
* - 'terminal' OR attempts exhausted → break to fallback chain.
* 3. Walk fallback chain (one shot per fallback tool). Each gets stepFn with
* fallback_tool set in StepFnInput.
* 4. On success at any stage: save success state to store, return {ok:true}.
* 5. If all retries + fallbacks fail: save error state, return {ok:false}.
*
* REPLAY DETERMINISM (per PM brief acceptance gate):
* Recovery DECISIONS (retry / fallback / exhaust / success) are pure functions
* of stepFn outcomes — given identical stepFn behavior, the resulting state
* chain is byte-identical. BACKOFF durations are non-deterministic by default
* (Math.random for jitter) but can be made deterministic via opts.rng for
* replay verification. Tests verify decisions, not durations.
*
* PERSISTENCE CADENCE:
* Saves happen only on (a) success and (b) exhaustion. Mid-retry failures do
* NOT save partial states — keeps the post-crash semantics simple: a fresh
* process either sees a clean prior step (and advances) or an exhaustion
* error checkpoint (and retries with full budget).
*/
import {
CheckpointStore,
CHECKPOINT_SCHEMA_VERSION,
nextStateFrom,
type CheckpointStepState,
type Decision,
} from './checkpoint.js';
// ─────────────────────────────────────────────────────────────────────────
// Public types
// ─────────────────────────────────────────────────────────────────────────
export type RecoveryEventType =
| 'fresh_start'
| 'resume_clean'
| 'resume_from_error'
| 'attempt'
| 'retry'
| 'fallback_attempted'
| 'success'
| 'exhausted';
export interface RecoveryEvent {
type: RecoveryEventType;
step_index: number;
/** 1-indexed attempt number (only set for attempt / retry / fallback / success). */
attempt?: number;
/** Set on retry / exhausted / resume_from_error. */
error?: string;
/** Set on fallback_attempted / success-via-fallback. */
fallback_tool?: string;
/** Set on retry: the duration the runner is about to sleep. */
backoff_ms?: number;
}
export interface StepFnInput {
/**
* State at the END of the previous step. Undefined for fresh start (step 0)
* and for resume_from_error at step 0.
*/
prior_state: CheckpointStepState | undefined;
/** Step index this attempt is for (0 for the first action). */
step_index: number;
/** 1-indexed attempt count (1 = first try, 2+ = retries, fallback ≥ retries+1). */
attempt: number;
/** Set when called via the fallback chain — the tool to use. */
fallback_tool?: string;
}
export interface StepFnResult {
/** Outputs the step produced (becomes step_output in the saved state). */
output: Record<string, unknown>;
/** Optional cost recorded with the step. */
cost_usd?: number;
/** Optional latency recorded with the step. */
latency_ms?: number;
/** Decisions made during this step — appended to decision_history. */
decisions?: readonly Decision[];
/** Retrieval cache additions — merged into retrieval_cache. */
retrieval_additions?: Record<string, unknown>;
/** Optional appended context (concatenated to accumulated_context). */
appended_context?: string;
}
export type StepFn = (input: StepFnInput) => Promise<StepFnResult>;
export type ErrorClass = 'retryable' | 'terminal';
export type ErrorClassifier = (err: unknown) => ErrorClass;
export interface RecoveryRunnerOptions {
/** Per-task checkpoint store. Recovery uses store.taskId for state lifecycle. */
store: CheckpointStore;
/** Run-level UUID written into every saved checkpoint. */
runId: string;
/** Default 3 retries (1 initial + 3 retries = 4 same-tool attempts). */
maxRetriesPerStep?: number;
/** Default 1000ms initial backoff (retry 1). */
baseBackoffMs?: number;
/** Default 30000ms cap on a single backoff. */
maxBackoffMs?: number;
/** Default 0.25 (±25% jitter on top of exponential). 0 = no jitter (deterministic). */
jitterFactor?: number;
/** Optional ordered tool fallback chain. One shot per tool. */
toolFallbackChain?: readonly string[];
/** Optional error classifier. Default: every error is 'retryable'. */
classifyError?: ErrorClassifier;
/** Optional injectable RNG for jitter. Default: Math.random. */
rng?: () => number;
/** Optional injectable sleep for tests. Default: native setTimeout. */
sleep?: (ms: number) => Promise<void>;
/** Optional event sink. Receives every state-machine event in order. */
onRecoveryEvent?: (event: RecoveryEvent) => void;
}
export interface RecoveryRunResult {
/** Final state for this run (success state OR error-tagged exhaustion state). */
state: CheckpointStepState;
/** True iff the step ultimately succeeded (no error in final state). */
ok: boolean;
}
export interface RecoveryRunOptions {
/** Label for this step (becomes step_action in the saved state). */
stepAction?: string;
}
// ─────────────────────────────────────────────────────────────────────────
// Internal helpers
// ─────────────────────────────────────────────────────────────────────────
const DEFAULT_MAX_RETRIES = 3;
const DEFAULT_BASE_BACKOFF_MS = 1000;
const DEFAULT_MAX_BACKOFF_MS = 30000;
const DEFAULT_JITTER_FACTOR = 0.25;
const ALWAYS_RETRYABLE: ErrorClassifier = () => 'retryable';
function defaultSleep(ms: number): Promise<void> {
return new Promise<void>((resolve) => {
setTimeout(resolve, ms);
});
}
function errMessage(err: unknown): string {
if (err instanceof Error) return err.message;
if (typeof err === 'string') return err;
try {
return JSON.stringify(err);
} catch {
return String(err);
}
}
/**
* Compute exponential backoff with optional ±jitterFactor jitter.
* retryNumber is 1-indexed (retry 1 = baseBackoffMs, retry 2 = 2*base, ...).
*/
function computeBackoff(opts: {
retryNumber: number;
baseMs: number;
maxMs: number;
jitterFactor: number;
rng: () => number;
}): number {
const exponential = Math.min(
opts.baseMs * 2 ** (opts.retryNumber - 1),
opts.maxMs,
);
if (opts.jitterFactor === 0) return exponential;
// jitter range: ±jitterFactor * exponential
const jitter = (opts.rng() * 2 - 1) * opts.jitterFactor * exponential;
return Math.max(0, exponential + jitter);
}
// ─────────────────────────────────────────────────────────────────────────
// RecoveryRunner
// ─────────────────────────────────────────────────────────────────────────
export class RecoveryRunner {
private readonly store: CheckpointStore;
private readonly runId: string;
private readonly maxRetries: number;
private readonly baseBackoffMs: number;
private readonly maxBackoffMs: number;
private readonly jitterFactor: number;
private readonly fallbackChain: readonly string[];
private readonly classify: ErrorClassifier;
private readonly rng: () => number;
private readonly sleep: (ms: number) => Promise<void>;
private readonly emit: (event: RecoveryEvent) => void;
constructor(opts: RecoveryRunnerOptions) {
if (!opts.store) throw new Error('RecoveryRunner: store is required');
if (!opts.runId) throw new Error('RecoveryRunner: runId is required');
const maxRetries = opts.maxRetriesPerStep ?? DEFAULT_MAX_RETRIES;
if (!Number.isInteger(maxRetries) || maxRetries < 0) {
throw new Error(`RecoveryRunner: maxRetriesPerStep must be a non-negative integer (got ${String(maxRetries)})`);
}
const baseMs = opts.baseBackoffMs ?? DEFAULT_BASE_BACKOFF_MS;
const maxMs = opts.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
if (baseMs < 0 || maxMs < 0 || maxMs < baseMs) {
throw new Error(`RecoveryRunner: invalid backoff bounds (base=${baseMs}, max=${maxMs})`);
}
const jitter = opts.jitterFactor ?? DEFAULT_JITTER_FACTOR;
if (jitter < 0 || jitter > 1) {
throw new Error(`RecoveryRunner: jitterFactor must be in [0, 1] (got ${jitter})`);
}
this.store = opts.store;
this.runId = opts.runId;
this.maxRetries = maxRetries;
this.baseBackoffMs = baseMs;
this.maxBackoffMs = maxMs;
this.jitterFactor = jitter;
this.fallbackChain = opts.toolFallbackChain ?? [];
this.classify = opts.classifyError ?? ALWAYS_RETRYABLE;
this.rng = opts.rng ?? Math.random;
this.sleep = opts.sleep ?? defaultSleep;
this.emit = opts.onRecoveryEvent ?? (() => {});
}
/**
* Runs ONE step's lifecycle (initial attempt → retries → fallback chain →
* success-or-exhaustion-save). The state machine inspects the store's
* latest checkpoint to decide which step to attempt and what prior state
* to feed into stepFn.
*/
async run(stepFn: StepFn, opts: RecoveryRunOptions = {}): Promise<RecoveryRunResult> {
const stepAction = opts.stepAction ?? 'step';
const { priorState, stepIndexToAttempt } = await this._resolveStartingPoint();
// Phase 1: same-tool attempts (1 initial + maxRetries retries).
let lastError: string | undefined;
for (let attempt = 1; attempt <= this.maxRetries + 1; attempt += 1) {
if (attempt > 1) {
const backoffMs = computeBackoff({
retryNumber: attempt - 1,
baseMs: this.baseBackoffMs,
maxMs: this.maxBackoffMs,
jitterFactor: this.jitterFactor,
rng: this.rng,
});
this.emit({ type: 'retry', step_index: stepIndexToAttempt, attempt, backoff_ms: backoffMs, error: lastError });
await this.sleep(backoffMs);
}
this.emit({ type: 'attempt', step_index: stepIndexToAttempt, attempt });
try {
const result = await stepFn({
prior_state: priorState,
step_index: stepIndexToAttempt,
attempt,
});
const successState = this._buildSuccessState({
priorState,
stepIndex: stepIndexToAttempt,
stepAction,
attempt,
fallbackTool: undefined,
result,
});
await this.store.save(successState);
this.emit({ type: 'success', step_index: stepIndexToAttempt, attempt });
return { state: successState, ok: true };
} catch (err) {
lastError = errMessage(err);
const cls = this.classify(err);
if (cls === 'terminal') break; // skip remaining same-tool retries
}
}
// Phase 2: fallback chain (one shot per fallback tool).
let nextAttemptCounter = this.maxRetries + 2;
for (const fallbackTool of this.fallbackChain) {
const attempt = nextAttemptCounter;
nextAttemptCounter += 1;
this.emit({
type: 'fallback_attempted',
step_index: stepIndexToAttempt,
attempt,
fallback_tool: fallbackTool,
error: lastError,
});
try {
const result = await stepFn({
prior_state: priorState,
step_index: stepIndexToAttempt,
attempt,
fallback_tool: fallbackTool,
});
const successState = this._buildSuccessState({
priorState,
stepIndex: stepIndexToAttempt,
stepAction,
attempt,
fallbackTool,
result,
});
await this.store.save(successState);
this.emit({
type: 'success',
step_index: stepIndexToAttempt,
attempt,
fallback_tool: fallbackTool,
});
return { state: successState, ok: true };
} catch (err) {
lastError = errMessage(err);
// continue to next fallback
}
}
// Phase 3: exhausted. Persist final error state.
this.emit({ type: 'exhausted', step_index: stepIndexToAttempt, error: lastError });
const errorState = this._buildErrorState({
priorState,
stepIndex: stepIndexToAttempt,
stepAction,
error: lastError ?? 'unknown error',
});
await this.store.save(errorState);
return { state: errorState, ok: false };
}
// ──────────────────────────────────────────────────────────────────────
// Private: state machine + state builders
// ──────────────────────────────────────────────────────────────────────
private async _resolveStartingPoint(): Promise<{
priorState: CheckpointStepState | undefined;
stepIndexToAttempt: number;
}> {
const latest = await this.store.loadLatest();
if (!latest) {
this.emit({ type: 'fresh_start', step_index: 0 });
return { priorState: undefined, stepIndexToAttempt: 0 };
}
if (latest.error) {
this.emit({
type: 'resume_from_error',
step_index: latest.step_index,
error: latest.error,
});
if (latest.step_index === 0) {
return { priorState: undefined, stepIndexToAttempt: 0 };
}
const prior = await this.store.load(latest.step_index - 1);
if (!prior) {
throw new Error(
`RecoveryRunner: step ${latest.step_index} has error but step ${latest.step_index - 1} is missing — checkpoint chain corrupt`,
);
}
if (prior.error) {
throw new Error(
`RecoveryRunner: prior step ${prior.step_index} also has error — checkpoint chain not clean`,
);
}
return { priorState: prior, stepIndexToAttempt: latest.step_index };
}
this.emit({ type: 'resume_clean', step_index: latest.step_index + 1 });
return { priorState: latest, stepIndexToAttempt: latest.step_index + 1 };
}
private _buildSuccessState(args: {
priorState: CheckpointStepState | undefined;
stepIndex: number;
stepAction: string;
attempt: number;
fallbackTool: string | undefined;
result: StepFnResult;
}): CheckpointStepState {
const stepInput: Record<string, unknown> = { attempt: args.attempt };
if (args.fallbackTool !== undefined) stepInput.fallback_tool = args.fallbackTool;
if (args.priorState) {
// Resume path: nextStateFrom advances by 1, but we want to OVERWRITE
// args.stepIndex (which equals priorState.step_index + 1 for clean
// resume, or priorState.step_index + 1 for error-resume after a
// rewind to step N-1). nextStateFrom does exactly this.
return nextStateFrom(args.priorState, {
step_action: args.stepAction,
step_input: stepInput,
step_output: args.result.output,
decisions: args.result.decisions,
retrieval_additions: args.result.retrieval_additions,
appended_context: args.result.appended_context,
cost_usd: args.result.cost_usd,
latency_ms: args.result.latency_ms,
});
}
// Fresh path (step 0 with no prior). Build directly — nextStateFrom
// would assume a prior at step -1 which doesn't exist.
return {
schema_version: CHECKPOINT_SCHEMA_VERSION,
task_id: this.store.taskId,
run_id: this.runId,
step_index: args.stepIndex,
timestamp_iso: new Date().toISOString(),
step_action: args.stepAction,
step_input: { ...stepInput },
step_output: { ...args.result.output },
accumulated_context: args.result.appended_context ?? '',
retrieval_cache: { ...(args.result.retrieval_additions ?? {}) },
decision_history: [...(args.result.decisions ?? [])],
cost_usd: args.result.cost_usd,
latency_ms: args.result.latency_ms,
};
}
private _buildErrorState(args: {
priorState: CheckpointStepState | undefined;
stepIndex: number;
stepAction: string;
error: string;
}): CheckpointStepState {
if (args.priorState) {
return nextStateFrom(args.priorState, {
step_action: args.stepAction,
step_input: { exhausted: true },
step_output: {},
error: args.error,
});
}
return {
schema_version: CHECKPOINT_SCHEMA_VERSION,
task_id: this.store.taskId,
run_id: this.runId,
step_index: args.stepIndex,
timestamp_iso: new Date().toISOString(),
step_action: args.stepAction,
step_input: { exhausted: true },
step_output: {},
accumulated_context: '',
retrieval_cache: {},
decision_history: [],
error: args.error,
};
}
}

View File

@@ -0,0 +1,599 @@
/**
* Reporting module — Phase 4.2 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.2)
*
* Consumes per-instance agent prediction records and emits four artifacts:
* - summary.json machine-readable per-run aggregate
* - summary.md human-readable per-run aggregate
* - predictions.jsonl one record per (model, cell, instance)
* - failures.jsonl one record per failed instance with Phase 4.1 classification
*
* Required metrics per (model, cell):
* - accuracy_raw + accuracy_normalized (substring-match against gold)
* - abstention rate
* - thinking_leakage rate, format_violation rate
* - avg output chars (raw + normalized), avg latency, total cost
* - 10-bucket failure distribution from Phase 4.1 classifier
* - Wilson 95% CI half-width
* - bootstrap 95% CI when N >= 30
* - win/loss vs baseline cell (in pp)
*
* Required overall metrics:
* - cross-model comparison matrix (cells × models)
* - best model per cell
* - regression notes (cell drop ≥ 5 pp vs baseline)
* - aggregate failure distribution
*
* NOTE on stats inlining: Wilson CI + cluster-free bootstrap are inlined here
* rather than imported from benchmarks/harness/src/stats/ because @waggle/agent
* cannot import from benchmarks/harness/ (wrong dependency direction). Future
* Sprint 12 cleanup should hoist the shared math to @waggle/shared and
* deduplicate.
*/
import * as fs from 'node:fs';
import * as fsp from 'node:fs/promises';
import * as path from 'node:path';
import {
classifyFailure,
failureDistribution,
FAILURE_CATEGORIES,
type FailureCategory,
type FailureClassification,
type ClassifierOptions,
} from './failure-classify.js';
// ─────────────────────────────────────────────────────────────────────────
// Public types
// ─────────────────────────────────────────────────────────────────────────
export interface AgentPredictionRecord {
model: string;
cell: string;
instance_id: string;
question?: string;
gold_answer: string | readonly string[];
output_raw: string;
output_normalized: string;
judge_verdict?: 'correct' | 'incorrect' | 'null' | string;
cost_usd?: number;
latency_ms?: number;
tokens_in?: number;
tokens_out?: number;
upstream_error?: string;
/** Optional pre-classified failure; classifier is skipped if set. */
failure_classification?: FailureClassification;
}
export interface ReportOptions {
run_id: string;
/** Baseline cell name for win/loss comparison. Default 'raw'. */
baseline_cell?: string;
/** Regression threshold in percentage points. Default 5.0 pp drop vs baseline. */
regression_threshold_pp?: number;
/** Bootstrap iterations for CI. Default 1000. Only computed when N >= 30. */
bootstrap_iterations?: number;
/** Optional LLM judge for the failure classifier on low-confidence cases. */
failure_classifier?: ClassifierOptions;
/** Optional seeded RNG for replay-deterministic bootstrap. Default Math.random. */
rng?: () => number;
}
export interface CellMetrics {
n: number;
accuracy_raw: number;
accuracy_normalized: number;
abstention_rate: number;
thinking_leakage_rate: number;
format_violation_rate: number;
avg_output_chars_raw: number;
avg_output_chars_normalized: number;
avg_latency_ms: number;
total_cost_usd: number;
failure_distribution: Record<FailureCategory, number>;
wilson_95ci_lower: number;
wilson_95ci_upper: number;
wilson_95ci_half_width_pp: number;
bootstrap_95ci_lower?: number;
bootstrap_95ci_upper?: number;
/** Delta vs baseline cell in percentage points (accuracy_normalized). */
win_loss_vs_baseline_pp?: number;
}
export interface ModelComparisonMatrix {
rows: readonly string[];
columns: readonly string[];
/** values[rowIdx][colIdx] = accuracy_normalized for (cell=rows[r], model=cols[c]); NaN if missing. */
values: readonly (readonly number[])[];
}
export interface RunSummary {
report_version: 1;
run_id: string;
generated_at_iso: string;
n_instances: number;
models: readonly string[];
cells: readonly string[];
baseline_cell?: string;
per_model_per_cell: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>;
cross_model_matrix: ModelComparisonMatrix;
best_per_cell: Readonly<Record<string, string>>;
regression_notes: readonly string[];
aggregate_failure_distribution: Readonly<Record<FailureCategory, number>>;
}
export interface ReportArtifacts {
summary: RunSummary;
summary_md: string;
predictions_jsonl: string;
failures_jsonl: string;
}
// ─────────────────────────────────────────────────────────────────────────
// Inline stats helpers
// ─────────────────────────────────────────────────────────────────────────
const Z_95 = 1.959964;
interface WilsonResult {
point_estimate: number;
lower: number;
upper: number;
half_width: number;
}
/** Wilson 95% binomial CI. Returns { lower, upper, half_width } in [0, 1]. */
function wilsonCI(successes: number, trials: number): WilsonResult {
if (trials === 0) return { point_estimate: 0, lower: 0, upper: 0, half_width: 0 };
const p = successes / trials;
const z = Z_95;
const z2 = z * z;
const denom = 1 + z2 / trials;
const center = (p + z2 / (2 * trials)) / denom;
const half = (z * Math.sqrt(p * (1 - p) / trials + z2 / (4 * trials * trials))) / denom;
return {
point_estimate: p,
lower: Math.max(0, center - half),
upper: Math.min(1, center + half),
half_width: half,
};
}
/** Non-clustered bootstrap 95% CI on a binary correctness array. */
function bootstrapCI(correctness: readonly number[], iterations: number, rng: () => number): { lower: number; upper: number } {
const n = correctness.length;
if (n === 0) return { lower: 0, upper: 0 };
const samples: number[] = [];
for (let i = 0; i < iterations; i += 1) {
let sum = 0;
for (let j = 0; j < n; j += 1) {
const idx = Math.floor(rng() * n);
sum += correctness[idx]!;
}
samples.push(sum / n);
}
samples.sort((a, b) => a - b);
const lo = samples[Math.floor(0.025 * iterations)] ?? 0;
const hi = samples[Math.floor(0.975 * iterations)] ?? 0;
return { lower: lo, upper: hi };
}
// ─────────────────────────────────────────────────────────────────────────
// Substring-match accuracy (v6 rule)
// ─────────────────────────────────────────────────────────────────────────
function asGolds(g: string | readonly string[]): readonly string[] {
return typeof g === 'string' ? [g] : g;
}
function scoreAccuracy(output: string, gold: string | readonly string[]): number {
const golds = asGolds(gold);
if (golds.length === 0) return 0;
const lower = output.toLowerCase();
for (const g of golds) {
if (g && lower.includes(g.toLowerCase())) return 1;
}
return 0;
}
// ─────────────────────────────────────────────────────────────────────────
// Per-cell metric computation
// ─────────────────────────────────────────────────────────────────────────
const UNKNOWN_OUTPUTS = new Set([
'unknown', 'i don\'t know', 'i do not know', 'cannot determine',
'cannot be determined', 'not specified', 'no information', 'insufficient information',
'no answer', 'n/a', 'na', 'none',
]);
const THINKING_RE = /<\s*\/?\s*think(?:ing)?\s*[>\s]/i;
function isAbstention(output: string): boolean {
const lc = output.toLowerCase().trim();
return UNKNOWN_OUTPUTS.has(lc);
}
function hasThinkingLeakage(rawOutput: string): boolean {
return THINKING_RE.test(rawOutput);
}
async function computeCellMetrics(
records: readonly AgentPredictionRecord[],
opts: ReportOptions,
): Promise<{ metrics: CellMetrics; failures: ReadonlyArray<AgentPredictionRecord & { classification: FailureClassification }> }> {
const n = records.length;
if (n === 0) {
const zeros = Object.fromEntries(FAILURE_CATEGORIES.map(c => [c, 0])) as Record<FailureCategory, number>;
return {
metrics: {
n: 0, accuracy_raw: 0, accuracy_normalized: 0,
abstention_rate: 0, thinking_leakage_rate: 0, format_violation_rate: 0,
avg_output_chars_raw: 0, avg_output_chars_normalized: 0,
avg_latency_ms: 0, total_cost_usd: 0,
failure_distribution: zeros,
wilson_95ci_lower: 0, wilson_95ci_upper: 0, wilson_95ci_half_width_pp: 0,
},
failures: [],
};
}
const correctnessRaw: number[] = [];
const correctnessNormalized: number[] = [];
let abstentions = 0;
let thinkingLeaks = 0;
let totalRawChars = 0;
let totalNormalizedChars = 0;
let totalLatency = 0;
let totalCost = 0;
for (const r of records) {
correctnessRaw.push(scoreAccuracy(r.output_raw, r.gold_answer));
correctnessNormalized.push(scoreAccuracy(r.output_normalized, r.gold_answer));
if (isAbstention(r.output_normalized)) abstentions += 1;
if (hasThinkingLeakage(r.output_raw)) thinkingLeaks += 1;
totalRawChars += r.output_raw.length;
totalNormalizedChars += r.output_normalized.length;
totalLatency += r.latency_ms ?? 0;
totalCost += r.cost_usd ?? 0;
}
// Classify failures (where accuracy_normalized = 0).
const classified: Array<AgentPredictionRecord & { classification: FailureClassification }> = [];
for (let i = 0; i < records.length; i += 1) {
const r = records[i]!;
if (correctnessNormalized[i] === 1) continue; // not a failure
const classification = r.failure_classification ?? await classifyFailure({
model_output: r.output_normalized,
gold_answer: r.gold_answer,
question: r.question,
upstream_error: r.upstream_error,
}, opts.failure_classifier ?? {});
classified.push({ ...r, classification });
}
const fdist = failureDistribution(classified.map(c => c.classification));
const formatViolations = fdist.format_violation;
const successesNorm = correctnessNormalized.reduce((a, b) => a + b, 0);
const wilson = wilsonCI(successesNorm, n);
let bootstrap: { lower: number; upper: number } | undefined;
const bootIter = opts.bootstrap_iterations ?? 1000;
if (n >= 30) {
bootstrap = bootstrapCI(correctnessNormalized, bootIter, opts.rng ?? Math.random);
}
const metrics: CellMetrics = {
n,
accuracy_raw: correctnessRaw.reduce((a, b) => a + b, 0) / n,
accuracy_normalized: successesNorm / n,
abstention_rate: abstentions / n,
thinking_leakage_rate: thinkingLeaks / n,
format_violation_rate: formatViolations / n,
avg_output_chars_raw: totalRawChars / n,
avg_output_chars_normalized: totalNormalizedChars / n,
avg_latency_ms: totalLatency / n,
total_cost_usd: totalCost,
failure_distribution: fdist,
wilson_95ci_lower: wilson.lower,
wilson_95ci_upper: wilson.upper,
wilson_95ci_half_width_pp: wilson.half_width * 100,
...(bootstrap ? { bootstrap_95ci_lower: bootstrap.lower, bootstrap_95ci_upper: bootstrap.upper } : {}),
};
return { metrics, failures: classified };
}
// ─────────────────────────────────────────────────────────────────────────
// Cross-model matrix + best per cell + regression notes
// ─────────────────────────────────────────────────────────────────────────
function buildCrossModelMatrix(
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
models: readonly string[],
cells: readonly string[],
): ModelComparisonMatrix {
const values: number[][] = cells.map(cell => models.map(model => {
const m = perMperC[model]?.[cell];
return m ? m.accuracy_normalized : Number.NaN;
}));
return { rows: cells, columns: models, values };
}
function bestModelPerCell(
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
models: readonly string[],
cells: readonly string[],
): Record<string, string> {
const out: Record<string, string> = {};
for (const cell of cells) {
let bestModel = '';
let bestScore = -Infinity;
for (const m of models) {
const score = perMperC[m]?.[cell]?.accuracy_normalized ?? -Infinity;
if (score > bestScore) {
bestScore = score;
bestModel = m;
}
}
if (bestModel) out[cell] = bestModel;
}
return out;
}
function regressionNotes(
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
models: readonly string[],
cells: readonly string[],
baselineCell: string,
thresholdPp: number,
): string[] {
const notes: string[] = [];
for (const m of models) {
const baseAcc = perMperC[m]?.[baselineCell]?.accuracy_normalized;
if (baseAcc === undefined) continue;
for (const c of cells) {
if (c === baselineCell) continue;
const acc = perMperC[m]?.[c]?.accuracy_normalized;
if (acc === undefined) continue;
const deltaPp = (acc - baseAcc) * 100;
if (-deltaPp >= thresholdPp) {
notes.push(
`Regression: ${m}/${c} (acc=${(acc * 100).toFixed(1)}%) is ${(-deltaPp).toFixed(1)}pp below baseline ${baselineCell} (acc=${(baseAcc * 100).toFixed(1)}%)`,
);
}
}
}
return notes;
}
function aggregateFailureDistribution(
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
): Record<FailureCategory, number> {
const agg = Object.fromEntries(FAILURE_CATEGORIES.map(c => [c, 0])) as Record<FailureCategory, number>;
for (const modelMap of Object.values(perMperC)) {
for (const cm of Object.values(modelMap)) {
for (const c of FAILURE_CATEGORIES) {
agg[c] += cm.failure_distribution[c];
}
}
}
return agg;
}
// ─────────────────────────────────────────────────────────────────────────
// Markdown rendering
// ─────────────────────────────────────────────────────────────────────────
function renderSummaryMd(summary: RunSummary): string {
const lines: string[] = [];
lines.push(`# Run Report — ${summary.run_id}`);
lines.push('');
lines.push(`Generated: ${summary.generated_at_iso}`);
lines.push(`N instances: ${summary.n_instances}`);
lines.push(`Models: ${summary.models.join(', ')}`);
lines.push(`Cells: ${summary.cells.join(', ')}`);
if (summary.baseline_cell) lines.push(`Baseline cell: \`${summary.baseline_cell}\``);
lines.push('');
// Per-model tables.
for (const m of summary.models) {
lines.push(`## ${m}`);
lines.push('');
lines.push('| cell | n | acc_raw | acc_norm | 95% CI | abstain | think_leak | fmt_viol | avg_lat_ms | cost_$ | win/loss vs baseline |');
lines.push('|---|---|---|---|---|---|---|---|---|---|---|');
for (const c of summary.cells) {
const cm = summary.per_model_per_cell[m]?.[c];
if (!cm) {
lines.push(`| ${c} | – | – | – | – | – | – | – | – | – | – |`);
continue;
}
const wlPp = cm.win_loss_vs_baseline_pp;
const wlStr = wlPp === undefined ? '–' : `${wlPp >= 0 ? '+' : ''}${wlPp.toFixed(1)}pp`;
lines.push([
c,
cm.n,
(cm.accuracy_raw * 100).toFixed(1) + '%',
(cm.accuracy_normalized * 100).toFixed(1) + '%',
`[${(cm.wilson_95ci_lower * 100).toFixed(1)}%, ${(cm.wilson_95ci_upper * 100).toFixed(1)}%]`,
(cm.abstention_rate * 100).toFixed(1) + '%',
(cm.thinking_leakage_rate * 100).toFixed(1) + '%',
(cm.format_violation_rate * 100).toFixed(1) + '%',
cm.avg_latency_ms.toFixed(0),
cm.total_cost_usd.toFixed(4),
wlStr,
].map(v => String(v)).join(' | ').replace(/^/, '| ').replace(/$/, ' |'));
}
lines.push('');
}
// Cross-model comparison matrix.
lines.push('## Cross-model comparison (accuracy_normalized %)');
lines.push('');
lines.push(`| cell | ${summary.cross_model_matrix.columns.join(' | ')} | best |`);
lines.push(`| --- | ${summary.cross_model_matrix.columns.map(() => '---').join(' | ')} | --- |`);
for (let r = 0; r < summary.cross_model_matrix.rows.length; r += 1) {
const cell = summary.cross_model_matrix.rows[r]!;
const row = summary.cross_model_matrix.values[r]!.map(v => Number.isNaN(v) ? '–' : (v * 100).toFixed(1) + '%');
const best = summary.best_per_cell[cell] ?? '–';
lines.push(`| ${cell} | ${row.join(' | ')} | **${best}** |`);
}
lines.push('');
// Aggregate failure distribution.
lines.push('## Aggregate failure distribution (across all model × cell)');
lines.push('');
lines.push('| category | count |');
lines.push('|---|---|');
for (const c of FAILURE_CATEGORIES) {
lines.push(`| ${c} | ${summary.aggregate_failure_distribution[c]} |`);
}
lines.push('');
// Regression notes.
if (summary.regression_notes.length > 0) {
lines.push('## Regression notes');
lines.push('');
for (const n of summary.regression_notes) lines.push(`- ${n}`);
lines.push('');
} else {
lines.push('## Regression notes');
lines.push('');
lines.push('_(none — no cell drops ≥ regression threshold vs baseline)_');
lines.push('');
}
return lines.join('\n');
}
// ─────────────────────────────────────────────────────────────────────────
// Main entry point
// ─────────────────────────────────────────────────────────────────────────
export async function generateReport(
records: readonly AgentPredictionRecord[],
opts: ReportOptions,
): Promise<ReportArtifacts> {
const baseline = opts.baseline_cell ?? 'raw';
const regressionThreshold = opts.regression_threshold_pp ?? 5.0;
const models = Array.from(new Set(records.map(r => r.model))).sort();
const cells = Array.from(new Set(records.map(r => r.cell))).sort();
const perMperC: Record<string, Record<string, CellMetrics>> = {};
const allFailures: Array<AgentPredictionRecord & { classification: FailureClassification }> = [];
for (const m of models) {
perMperC[m] = {};
for (const c of cells) {
const subset = records.filter(r => r.model === m && r.cell === c);
if (subset.length === 0) continue;
const { metrics, failures } = await computeCellMetrics(subset, opts);
perMperC[m]![c] = metrics;
for (const f of failures) allFailures.push(f);
}
// Compute win/loss vs baseline.
const baseAcc = perMperC[m]?.[baseline]?.accuracy_normalized;
if (baseAcc !== undefined) {
for (const c of cells) {
if (c === baseline) continue;
const cm = perMperC[m]?.[c];
if (!cm) continue;
perMperC[m]![c] = { ...cm, win_loss_vs_baseline_pp: (cm.accuracy_normalized - baseAcc) * 100 };
}
}
}
const summary: RunSummary = {
report_version: 1,
run_id: opts.run_id,
generated_at_iso: new Date().toISOString(),
n_instances: records.length,
models,
cells,
baseline_cell: baseline,
per_model_per_cell: perMperC,
cross_model_matrix: buildCrossModelMatrix(perMperC, models, cells),
best_per_cell: bestModelPerCell(perMperC, models, cells),
regression_notes: regressionNotes(perMperC, models, cells, baseline, regressionThreshold),
aggregate_failure_distribution: aggregateFailureDistribution(perMperC),
};
const summary_md = renderSummaryMd(summary);
const predictions_jsonl = records.map(r => JSON.stringify(r)).join('\n');
const failures_jsonl = allFailures.map(f => JSON.stringify(f)).join('\n');
return { summary, summary_md, predictions_jsonl, failures_jsonl };
}
export interface WrittenReportPaths {
summary_json_path: string;
summary_md_path: string;
predictions_jsonl_path: string;
failures_jsonl_path: string;
}
export async function writeReportToDisk(
records: readonly AgentPredictionRecord[],
opts: ReportOptions & { outputDir: string },
): Promise<WrittenReportPaths> {
const artifacts = await generateReport(records, opts);
await fsp.mkdir(opts.outputDir, { recursive: true });
const paths: WrittenReportPaths = {
summary_json_path: path.join(opts.outputDir, 'summary.json'),
summary_md_path: path.join(opts.outputDir, 'summary.md'),
predictions_jsonl_path: path.join(opts.outputDir, 'predictions.jsonl'),
failures_jsonl_path: path.join(opts.outputDir, 'failures.jsonl'),
};
await fsp.writeFile(paths.summary_json_path, JSON.stringify(artifacts.summary, null, 2), 'utf-8');
await fsp.writeFile(paths.summary_md_path, artifacts.summary_md, 'utf-8');
await fsp.writeFile(paths.predictions_jsonl_path, artifacts.predictions_jsonl + (artifacts.predictions_jsonl ? '\n' : ''), 'utf-8');
await fsp.writeFile(paths.failures_jsonl_path, artifacts.failures_jsonl + (artifacts.failures_jsonl ? '\n' : ''), 'utf-8');
return paths;
}
// ─────────────────────────────────────────────────────────────────────────
// Pilot-record adapter
// ─────────────────────────────────────────────────────────────────────────
export interface PilotJsonlRecord {
instance_id: string;
question: string;
gold_answer: string | string[];
subject: {
content: string;
in?: number;
out?: number;
cost?: number;
latency_ms?: number;
error?: string | null;
};
trio?: { majority?: string };
self_judge?: { verdict?: string };
}
/**
* Convert a pilot-format JSONL record into the canonical AgentPredictionRecord.
* Caller supplies the (model, cell) tuple — pilot records don't carry those
* fields explicitly (they live in the filename / external context).
*/
export function fromPilotRecord(
record: PilotJsonlRecord,
opts: { model: string; cell: string },
): AgentPredictionRecord {
return {
model: opts.model,
cell: opts.cell,
instance_id: record.instance_id,
question: record.question,
gold_answer: record.gold_answer,
output_raw: record.subject.content,
output_normalized: record.subject.content, // pilot records don't preserve a separate normalized form
judge_verdict: record.trio?.majority,
cost_usd: record.subject.cost,
latency_ms: record.subject.latency_ms,
tokens_in: record.subject.in,
tokens_out: record.subject.out,
upstream_error: record.subject.error ?? undefined,
};
}