This commit is contained in:
367
packages/agent/src/long-task/checkpoint.ts
Normal file
367
packages/agent/src/long-task/checkpoint.ts
Normal file
@@ -0,0 +1,367 @@
|
||||
/**
|
||||
* Long-task checkpointing — Phase 3.1 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.1)
|
||||
*
|
||||
* Per-step state serialization for long-task agent runs. Lets the recovery
|
||||
* layer (Phase 3.2) resume from the last successful step after a crash, a
|
||||
* mid-run kill, or a transient tool failure.
|
||||
*
|
||||
* Storage layout (filesystem JSON, atomic writes):
|
||||
* <rootDir>/<task-id>/step-000000.json
|
||||
* <rootDir>/<task-id>/step-000001.json
|
||||
* ...
|
||||
*
|
||||
* Each step file is written via tmp+rename so concurrent reads NEVER observe
|
||||
* a partial file. Per-task isolation via the directory prefix matches the
|
||||
* pilot 2026-04-26 amendment v1 §4 "per-task SessionStore" pattern (one
|
||||
* corpus per task, deleted after results are persisted).
|
||||
*
|
||||
* HARD RULES (from PM brief §3.1):
|
||||
* - schema_version is REQUIRED and validated on save (forward-compat)
|
||||
* - step_index is monotonic 0,1,2,... (verifyIntegrity flags gaps)
|
||||
* - dispose() recursively deletes the per-task directory
|
||||
* - Reads ignore *.tmp files (crash residue from a partial write)
|
||||
* - Per-task isolation: state.task_id MUST match the store's taskId
|
||||
*/
|
||||
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
export const CHECKPOINT_SCHEMA_VERSION = 1;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// State shape
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A single decision recorded in the agent's history. Used by the recovery
|
||||
* layer to inform retry strategy and by audit / replay tooling.
|
||||
*/
|
||||
export interface Decision {
|
||||
/** step_index where the decision was made. */
|
||||
step_index: number;
|
||||
/** What was decided (one short sentence). */
|
||||
decision: string;
|
||||
/** Optional rationale captured at decision time. */
|
||||
rationale?: string;
|
||||
/** Optional list of alternatives the agent considered. */
|
||||
alternatives_considered?: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-step agent state. JSON-serializable. One file per step on disk.
|
||||
*
|
||||
* Persistent fields (carried forward across steps by `nextStateFrom`):
|
||||
* accumulated_context · retrieval_cache · decision_history
|
||||
*
|
||||
* Per-step fields (overwritten each step):
|
||||
* step_action · step_input · step_output · cost_usd · latency_ms · error
|
||||
*/
|
||||
export interface CheckpointStepState {
|
||||
/** Forward-compat schema version. */
|
||||
schema_version: number;
|
||||
/** Logical task identifier (matches per-task directory name). */
|
||||
task_id: string;
|
||||
/** Run-level UUID grouping multiple tasks in the same run. */
|
||||
run_id: string;
|
||||
/** Monotonic 0-indexed step number. */
|
||||
step_index: number;
|
||||
/** ISO 8601 UTC timestamp at the moment this checkpoint was captured. */
|
||||
timestamp_iso: string;
|
||||
/** Action / tool / cell label that produced this step. */
|
||||
step_action: string;
|
||||
/** Inputs the step received. */
|
||||
step_input: Readonly<Record<string, unknown>>;
|
||||
/** Outputs the step produced. */
|
||||
step_output: Readonly<Record<string, unknown>>;
|
||||
/** Running scratchpad / synthesis built up across all steps. */
|
||||
accumulated_context: string;
|
||||
/** Cache of retrieval / search query → results, keyed by canonical query string. */
|
||||
retrieval_cache: Readonly<Record<string, unknown>>;
|
||||
/** History of decisions made up to and including this step. */
|
||||
decision_history: readonly Decision[];
|
||||
/** Optional per-step cost (USD). */
|
||||
cost_usd?: number;
|
||||
/** Optional per-step latency (ms). */
|
||||
latency_ms?: number;
|
||||
/** Optional error string if this step failed. Recovery layer reads this. */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Store
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const STEP_FILE_RE = /^step-(\d{6})\.json$/;
|
||||
const PADDING_WIDTH = 6;
|
||||
|
||||
function stepFilename(stepIndex: number): string {
|
||||
return `step-${String(stepIndex).padStart(PADDING_WIDTH, '0')}.json`;
|
||||
}
|
||||
|
||||
async function fileExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fsp.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CheckpointStoreOptions {
|
||||
/** Parent directory under which the per-task subfolder lives. */
|
||||
rootDir: string;
|
||||
/** Logical task ID — used as the per-task subfolder name. */
|
||||
taskId: string;
|
||||
}
|
||||
|
||||
export interface IntegrityReport {
|
||||
ok: boolean;
|
||||
issues: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-task checkpoint store. One instance corresponds to exactly one task —
|
||||
* mirrors the per-task SessionStore isolation pattern from the pilot
|
||||
* 2026-04-26 amendment v1 §4.
|
||||
*/
|
||||
export class CheckpointStore {
|
||||
private readonly taskIdInternal: string;
|
||||
private readonly taskDir: string;
|
||||
|
||||
constructor(opts: CheckpointStoreOptions) {
|
||||
if (!opts.rootDir) throw new Error('CheckpointStore: rootDir is required');
|
||||
if (!opts.taskId) throw new Error('CheckpointStore: taskId is required');
|
||||
if (opts.taskId.includes(path.sep) || opts.taskId.includes('/') || opts.taskId.includes('\\')) {
|
||||
throw new Error(`CheckpointStore: taskId must not contain path separators (got "${opts.taskId}")`);
|
||||
}
|
||||
if (opts.taskId === '.' || opts.taskId === '..') {
|
||||
throw new Error(`CheckpointStore: taskId must not be "." or ".."`);
|
||||
}
|
||||
this.taskIdInternal = opts.taskId;
|
||||
this.taskDir = path.join(opts.rootDir, opts.taskId);
|
||||
}
|
||||
|
||||
/** Per-task directory path (read-only accessor). */
|
||||
get directory(): string {
|
||||
return this.taskDir;
|
||||
}
|
||||
|
||||
/** Logical task ID (read-only accessor). */
|
||||
get taskId(): string {
|
||||
return this.taskIdInternal;
|
||||
}
|
||||
|
||||
/** Idempotent: ensures per-task directory exists. Safe to call repeatedly. */
|
||||
async init(): Promise<void> {
|
||||
await fsp.mkdir(this.taskDir, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically writes a step's state. Implementation: write to <file>.<rand>.tmp,
|
||||
* then rename to the final filename. Reads never observe partial files.
|
||||
*
|
||||
* Per-task directory is mkdir'd on every save, so callers don't strictly
|
||||
* need to call init() first.
|
||||
*
|
||||
* Returns the absolute path of the finalized step file.
|
||||
*/
|
||||
async save(state: CheckpointStepState): Promise<string> {
|
||||
if (state.task_id !== this.taskIdInternal) {
|
||||
throw new Error(
|
||||
`CheckpointStore: state.task_id="${state.task_id}" does not match store.taskId="${this.taskIdInternal}"`,
|
||||
);
|
||||
}
|
||||
if (!Number.isInteger(state.step_index) || state.step_index < 0) {
|
||||
throw new Error(
|
||||
`CheckpointStore: step_index must be a non-negative integer (got ${String(state.step_index)})`,
|
||||
);
|
||||
}
|
||||
if (state.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`CheckpointStore: schema_version mismatch — store expects ${CHECKPOINT_SCHEMA_VERSION}, state has ${String(state.schema_version)}`,
|
||||
);
|
||||
}
|
||||
|
||||
await fsp.mkdir(this.taskDir, { recursive: true });
|
||||
const finalPath = path.join(this.taskDir, stepFilename(state.step_index));
|
||||
// Randomized .tmp suffix so two concurrent writers at the same step_index
|
||||
// (e.g. retry races) don't collide on a shared tmp filename.
|
||||
const tmpPath = `${finalPath}.${crypto.randomBytes(4).toString('hex')}.tmp`;
|
||||
const json = JSON.stringify(state, null, 2);
|
||||
await fsp.writeFile(tmpPath, json, { encoding: 'utf-8' });
|
||||
try {
|
||||
await fsp.rename(tmpPath, finalPath);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
// Windows quirk: fs.rename refuses to overwrite an existing destination
|
||||
// (POSIX silently overwrites). When two writers race the same step_index
|
||||
// (e.g. a retry that overlapped its predecessor), the loser yields:
|
||||
// there is already a valid state at finalPath from the winner. Drop our
|
||||
// tmp file and report the destination path. Any other error is fatal.
|
||||
if ((code === 'EPERM' || code === 'EEXIST') && (await fileExists(finalPath))) {
|
||||
await fsp.rm(tmpPath, { force: true });
|
||||
return finalPath;
|
||||
}
|
||||
await fsp.rm(tmpPath, { force: true });
|
||||
throw err;
|
||||
}
|
||||
return finalPath;
|
||||
}
|
||||
|
||||
/** Loads a specific step. Returns undefined if no file exists at that index. */
|
||||
async load(stepIndex: number): Promise<CheckpointStepState | undefined> {
|
||||
const filePath = path.join(this.taskDir, stepFilename(stepIndex));
|
||||
let json: string;
|
||||
try {
|
||||
json = await fsp.readFile(filePath, 'utf-8');
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
|
||||
throw err;
|
||||
}
|
||||
return JSON.parse(json) as CheckpointStepState;
|
||||
}
|
||||
|
||||
/** Loads the highest-indexed step. Returns undefined if no steps. */
|
||||
async loadLatest(): Promise<CheckpointStepState | undefined> {
|
||||
const indices = await this.listSteps();
|
||||
if (indices.length === 0) return undefined;
|
||||
return this.load(indices[indices.length - 1]!);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns sorted-ascending step indices that have finalized files in the
|
||||
* per-task directory. Ignores *.tmp residue (crash artifact from a
|
||||
* partial save) and any non-matching filenames.
|
||||
*
|
||||
* Returns [] if the per-task directory does not exist (treated as
|
||||
* "no steps yet" rather than an error).
|
||||
*/
|
||||
async listSteps(): Promise<number[]> {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fsp.readdir(this.taskDir);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return [];
|
||||
throw err;
|
||||
}
|
||||
const indices: number[] = [];
|
||||
for (const name of entries) {
|
||||
const m = name.match(STEP_FILE_RE);
|
||||
if (!m) continue;
|
||||
indices.push(Number.parseInt(m[1]!, 10));
|
||||
}
|
||||
indices.sort((a, b) => a - b);
|
||||
return indices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks every step file and verifies:
|
||||
* 1. step_index sequence is monotonic 0,1,2,... with no gaps
|
||||
* 2. file's step_index matches its filename's step_index
|
||||
* 3. schema_version matches CHECKPOINT_SCHEMA_VERSION
|
||||
*
|
||||
* Returns ok=true iff all checks pass; otherwise returns the issues list.
|
||||
*/
|
||||
async verifyIntegrity(): Promise<IntegrityReport> {
|
||||
const indices = await this.listSteps();
|
||||
const issues: string[] = [];
|
||||
for (let expected = 0; expected < indices.length; expected += 1) {
|
||||
const observed = indices[expected]!;
|
||||
if (observed !== expected) {
|
||||
issues.push(`gap or out-of-order: expected step_index=${expected}, found ${observed}`);
|
||||
// Stop on first gap — the rest of the report would be noise.
|
||||
break;
|
||||
}
|
||||
const state = await this.load(observed);
|
||||
if (!state) {
|
||||
issues.push(`step ${observed}: file disappeared mid-verify`);
|
||||
continue;
|
||||
}
|
||||
if (state.step_index !== observed) {
|
||||
issues.push(`step ${observed}: file.step_index=${state.step_index} does not match filename`);
|
||||
}
|
||||
if (state.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
|
||||
issues.push(`step ${observed}: schema_version=${state.schema_version} (expected ${CHECKPOINT_SCHEMA_VERSION})`);
|
||||
}
|
||||
}
|
||||
return { ok: issues.length === 0, issues };
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively deletes the per-task directory. Per amendment v2 §4 pattern:
|
||||
* "deleted after each task once results are persisted". Idempotent —
|
||||
* no-ops if the directory does not exist.
|
||||
*/
|
||||
async dispose(): Promise<void> {
|
||||
await fsp.rm(this.taskDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers (used by Phase 3.2 recovery + Phase 3.4 agent-loop integration)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Builds an empty CheckpointStepState shell. Useful when no prior checkpoint
|
||||
* exists — Phase 3.2 recovery uses this to construct a fresh state.
|
||||
*/
|
||||
export function makeInitialState(opts: {
|
||||
task_id: string;
|
||||
run_id: string;
|
||||
step_action?: string;
|
||||
}): CheckpointStepState {
|
||||
return {
|
||||
schema_version: CHECKPOINT_SCHEMA_VERSION,
|
||||
task_id: opts.task_id,
|
||||
run_id: opts.run_id,
|
||||
step_index: 0,
|
||||
timestamp_iso: new Date().toISOString(),
|
||||
step_action: opts.step_action ?? 'init',
|
||||
step_input: {},
|
||||
step_output: {},
|
||||
accumulated_context: '',
|
||||
retrieval_cache: {},
|
||||
decision_history: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the next step state from the prior step's state. Increments
|
||||
* step_index, copies forward the persistent fields (accumulated_context,
|
||||
* retrieval_cache, decision_history) and merges in caller-supplied additions.
|
||||
*
|
||||
* Per-step fields (step_action / step_input / step_output / cost_usd /
|
||||
* latency_ms / error) are taken from `opts`, not carried forward.
|
||||
*/
|
||||
export function nextStateFrom(prior: CheckpointStepState, opts: {
|
||||
step_action: string;
|
||||
step_input?: Record<string, unknown>;
|
||||
step_output?: Record<string, unknown>;
|
||||
decisions?: readonly Decision[];
|
||||
retrieval_additions?: Record<string, unknown>;
|
||||
appended_context?: string;
|
||||
cost_usd?: number;
|
||||
latency_ms?: number;
|
||||
error?: string;
|
||||
}): CheckpointStepState {
|
||||
return {
|
||||
schema_version: CHECKPOINT_SCHEMA_VERSION,
|
||||
task_id: prior.task_id,
|
||||
run_id: prior.run_id,
|
||||
step_index: prior.step_index + 1,
|
||||
timestamp_iso: new Date().toISOString(),
|
||||
step_action: opts.step_action,
|
||||
step_input: { ...(opts.step_input ?? {}) },
|
||||
step_output: { ...(opts.step_output ?? {}) },
|
||||
accumulated_context: opts.appended_context !== undefined
|
||||
? prior.accumulated_context + opts.appended_context
|
||||
: prior.accumulated_context,
|
||||
retrieval_cache: { ...prior.retrieval_cache, ...(opts.retrieval_additions ?? {}) },
|
||||
decision_history: [...prior.decision_history, ...(opts.decisions ?? [])],
|
||||
cost_usd: opts.cost_usd,
|
||||
latency_ms: opts.latency_ms,
|
||||
error: opts.error,
|
||||
};
|
||||
}
|
||||
495
packages/agent/src/long-task/context-manager.ts
Normal file
495
packages/agent/src/long-task/context-manager.ts
Normal file
@@ -0,0 +1,495 @@
|
||||
/**
|
||||
* Long-task context management — Phase 3.3 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.3)
|
||||
*
|
||||
* Handles context-window exhaustion in multi-hour agent runs. Phase 3.1's
|
||||
* accumulated_context grows monotonically across steps; eventually exceeds
|
||||
* the model's context-window budget. ContextManager intelligently shrinks
|
||||
* accumulated_context, decision_history, and retrieval_cache while
|
||||
* preserving task-progress signal.
|
||||
*
|
||||
* THREE SHRINKING DIMENSIONS:
|
||||
* 1. accumulated_context (string) — compress via LLM summarization,
|
||||
* retrieve-only truncation, or hybrid (summarize + emit archive event
|
||||
* so caller can index for later retrieval).
|
||||
* 2. decision_history (Decision[]) — bucket-summarize older decisions
|
||||
* into one rolled-up Decision; keep last K verbatim. Optional archive
|
||||
* callback for audit-trail persistence.
|
||||
* 3. retrieval_cache (Record<string, unknown>) — LRU-evict to maxSize.
|
||||
* Defaults to insertion order; caller can supply accessOrder for true
|
||||
* LRU.
|
||||
*
|
||||
* REUSE / NON-DUPLICATION:
|
||||
* - estimateStringTokens from existing context-compressor.ts (content-aware:
|
||||
* prose 4.0 / code 3.2 / json 3.5 / mixed 2.5 chars/token).
|
||||
* - LlmCallFn / RetrievalSearchFn from Phase 2.1 (retrieval-agent-loop.ts).
|
||||
* - Decision type from Phase 3.1 (checkpoint.ts).
|
||||
*
|
||||
* HALT-AND-PING TRIGGER RESOLUTIONS (locked before code, per PM brief §3.3):
|
||||
* 1. Tokenizer model-class dependencies → injectable estimateTokensFn,
|
||||
* defaults to estimateStringTokens (no new package deps).
|
||||
* 2. Compression LLM-call cost → threshold-gated (default 0.7), caller
|
||||
* chooses summarizationModel, cost telemetry via onCompressionEvent.
|
||||
* 3. Hive-mind retrieval ↔ Memory Sync Repair conflict → ContextManager
|
||||
* does NOT touch hive-mind; consumes RetrievalSearchFn abstraction;
|
||||
* caller wires it to per-task SessionStore (scratch, NOT synced).
|
||||
* 4. Decision history compression dropping critical decisions → bucket-
|
||||
* summarize preserves Decision shape; optional archiveDecisionsTo
|
||||
* callback for lossless on-disk audit trail.
|
||||
*/
|
||||
|
||||
import { estimateStringTokens } from '../context-compressor.js';
|
||||
import type { LlmCallFn, RetrievalSearchFn } from '../retrieval-agent-loop.js';
|
||||
import {
|
||||
CHECKPOINT_SCHEMA_VERSION,
|
||||
type CheckpointStepState,
|
||||
type Decision,
|
||||
} from './checkpoint.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Public types
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type CompressionStrategy = 'summarize-only' | 'retrieve-only' | 'hybrid';
|
||||
|
||||
export interface ArchivedContextRange {
|
||||
/** The text that was archived/replaced. */
|
||||
archived_text: string;
|
||||
/** Char offset where the archived range began (0-indexed) in the original context. */
|
||||
range_start: number;
|
||||
/** Char offset where the archived range ended (exclusive) in the original context. */
|
||||
range_end: number;
|
||||
}
|
||||
|
||||
export interface ContextCompressionEvent {
|
||||
type: 'context-compressed' | 'context-compression-skipped';
|
||||
/** Tokens before the operation. */
|
||||
before_tokens: number;
|
||||
/** Tokens after the operation. */
|
||||
after_tokens: number;
|
||||
/** Strategy used. */
|
||||
strategy: CompressionStrategy;
|
||||
/** USD cost incurred (LLM-summarize calls). 0 for retrieve-only or skipped. */
|
||||
cost_usd: number;
|
||||
/** Archived range (set on retrieve-only and hybrid). */
|
||||
archived?: ArchivedContextRange;
|
||||
}
|
||||
|
||||
export interface DecisionsCompressionEvent {
|
||||
type: 'decisions-compressed';
|
||||
/** Number of decisions before. */
|
||||
before_count: number;
|
||||
/** Number of decisions after. */
|
||||
after_count: number;
|
||||
/** Number archived (= before - after - 1 summary entry, when summary added). */
|
||||
evicted_count: number;
|
||||
/** USD cost (0 if heuristic summary used). */
|
||||
cost_usd: number;
|
||||
}
|
||||
|
||||
export interface CacheEvictionEvent {
|
||||
type: 'cache-evicted';
|
||||
/** Cache size before eviction. */
|
||||
before_size: number;
|
||||
/** Cache size after eviction. */
|
||||
after_size: number;
|
||||
/** Number of entries dropped. */
|
||||
evicted_count: number;
|
||||
}
|
||||
|
||||
export type CompressionEvent = ContextCompressionEvent | DecisionsCompressionEvent | CacheEvictionEvent;
|
||||
|
||||
export interface ContextManagerOptions {
|
||||
/** Token budget for accumulated_context. Caller picks based on model. */
|
||||
contextTokenBudget: number;
|
||||
/** Threshold (fraction of budget) that triggers proactive compression. Default 0.7. */
|
||||
compressionThreshold?: number;
|
||||
/** Compression strategy. Default 'hybrid'. */
|
||||
strategy?: CompressionStrategy;
|
||||
|
||||
/** LLM call for summarization. Required for 'summarize-only' / 'hybrid'. */
|
||||
llmCall?: LlmCallFn;
|
||||
/** Optional retrieval function (consumed by Phase 3.4, not by ContextManager directly). */
|
||||
retrievalSearch?: RetrievalSearchFn;
|
||||
/** Optional override for token estimation. Default = estimateStringTokens. */
|
||||
estimateTokensFn?: (text: string) => number;
|
||||
|
||||
/** Model alias for summarization. Required if llmCall provided. */
|
||||
summarizationModel?: string;
|
||||
/** max_tokens for the summarization LLM call. Default 1500. */
|
||||
summarizationMaxTokens?: number;
|
||||
/** Recent verbatim chars to retain at the END of accumulated_context. Default 4000. */
|
||||
retainRecentChars?: number;
|
||||
/** Decision history compression: keep last K decisions verbatim. Default 10. */
|
||||
retainRecentDecisions?: number;
|
||||
/** Retrieval cache LRU max size. Default 500 entries. */
|
||||
retrievalCacheMaxSize?: number;
|
||||
|
||||
/** Optional callback for compression telemetry (any compression event type). */
|
||||
onCompressionEvent?: (event: CompressionEvent) => void;
|
||||
/** Optional async callback to persist evicted decisions (audit trail). */
|
||||
archiveDecisionsTo?: (decisions: readonly Decision[]) => Promise<void>;
|
||||
/** Optional async callback to persist evicted retrieval cache entries. */
|
||||
archiveCacheTo?: (entries: ReadonlyArray<readonly [string, unknown]>) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface CompressOptions {
|
||||
/**
|
||||
* Optional query string for retrieval-aware compression. Reserved for
|
||||
* future use; ContextManager does not currently call retrievalSearch
|
||||
* directly (Phase 3.4 will wire that into the agent loop). Stored on
|
||||
* the event for caller introspection.
|
||||
*/
|
||||
stepQuery?: string;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Constants + helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_COMPRESSION_THRESHOLD = 0.7;
|
||||
const DEFAULT_STRATEGY: CompressionStrategy = 'hybrid';
|
||||
const DEFAULT_RETAIN_RECENT_CHARS = 4000;
|
||||
const DEFAULT_RETAIN_RECENT_DECISIONS = 10;
|
||||
const DEFAULT_CACHE_MAX_SIZE = 500;
|
||||
const DEFAULT_SUMMARIZATION_MAX_TOKENS = 1500;
|
||||
|
||||
const CONTEXT_COMPACTION_SYSTEM_PROMPT = [
|
||||
'You compress an agent\'s scratchpad / running context.',
|
||||
'Preserve every material fact, decision, unresolved question, and open thread.',
|
||||
'Drop verbose narration, repetition, and intermediate reasoning that was already concluded.',
|
||||
'Output a tight, structured summary — no preamble, no apologies.',
|
||||
].join(' ');
|
||||
|
||||
const DECISION_COMPACTION_SYSTEM_PROMPT = [
|
||||
'You summarize an agent\'s history of decisions.',
|
||||
'Output a single concise paragraph capturing the gist of all decisions provided —',
|
||||
'what was chosen, what was rejected, and why.',
|
||||
'No preamble, no apologies.',
|
||||
].join(' ');
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// ContextManager
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export class ContextManager {
|
||||
private readonly contextTokenBudget: number;
|
||||
private readonly compressionThreshold: number;
|
||||
private readonly strategy: CompressionStrategy;
|
||||
private readonly llmCall: LlmCallFn | undefined;
|
||||
private readonly retrievalSearch: RetrievalSearchFn | undefined;
|
||||
private readonly estimateFn: (text: string) => number;
|
||||
private readonly summarizationModel: string | undefined;
|
||||
private readonly summarizationMaxTokens: number;
|
||||
private readonly retainRecentChars: number;
|
||||
private readonly retainRecentDecisions: number;
|
||||
private readonly retrievalCacheMaxSize: number;
|
||||
private readonly emit: (event: CompressionEvent) => void;
|
||||
private readonly archiveDecisionsTo: ((decisions: readonly Decision[]) => Promise<void>) | undefined;
|
||||
private readonly archiveCacheTo: ((entries: ReadonlyArray<readonly [string, unknown]>) => Promise<void>) | undefined;
|
||||
|
||||
constructor(opts: ContextManagerOptions) {
|
||||
if (opts.contextTokenBudget <= 0) {
|
||||
throw new Error(`ContextManager: contextTokenBudget must be > 0 (got ${opts.contextTokenBudget})`);
|
||||
}
|
||||
const threshold = opts.compressionThreshold ?? DEFAULT_COMPRESSION_THRESHOLD;
|
||||
if (threshold <= 0 || threshold > 1) {
|
||||
throw new Error(`ContextManager: compressionThreshold must be in (0, 1] (got ${threshold})`);
|
||||
}
|
||||
const strategy = opts.strategy ?? DEFAULT_STRATEGY;
|
||||
const needsLlm = strategy === 'summarize-only' || strategy === 'hybrid';
|
||||
if (needsLlm && (!opts.llmCall || !opts.summarizationModel)) {
|
||||
throw new Error(
|
||||
`ContextManager: strategy='${strategy}' requires both llmCall and summarizationModel`,
|
||||
);
|
||||
}
|
||||
|
||||
this.contextTokenBudget = opts.contextTokenBudget;
|
||||
this.compressionThreshold = threshold;
|
||||
this.strategy = strategy;
|
||||
this.llmCall = opts.llmCall;
|
||||
this.retrievalSearch = opts.retrievalSearch;
|
||||
this.estimateFn = opts.estimateTokensFn ?? estimateStringTokens;
|
||||
this.summarizationModel = opts.summarizationModel;
|
||||
this.summarizationMaxTokens = opts.summarizationMaxTokens ?? DEFAULT_SUMMARIZATION_MAX_TOKENS;
|
||||
this.retainRecentChars = opts.retainRecentChars ?? DEFAULT_RETAIN_RECENT_CHARS;
|
||||
this.retainRecentDecisions = opts.retainRecentDecisions ?? DEFAULT_RETAIN_RECENT_DECISIONS;
|
||||
this.retrievalCacheMaxSize = opts.retrievalCacheMaxSize ?? DEFAULT_CACHE_MAX_SIZE;
|
||||
this.emit = opts.onCompressionEvent ?? (() => {});
|
||||
this.archiveDecisionsTo = opts.archiveDecisionsTo;
|
||||
this.archiveCacheTo = opts.archiveCacheTo;
|
||||
|
||||
if (this.retainRecentChars < 0) {
|
||||
throw new Error(`ContextManager: retainRecentChars must be >= 0 (got ${this.retainRecentChars})`);
|
||||
}
|
||||
if (this.retainRecentDecisions < 0 || !Number.isInteger(this.retainRecentDecisions)) {
|
||||
throw new Error(`ContextManager: retainRecentDecisions must be a non-negative integer (got ${this.retainRecentDecisions})`);
|
||||
}
|
||||
if (this.retrievalCacheMaxSize < 0 || !Number.isInteger(this.retrievalCacheMaxSize)) {
|
||||
throw new Error(`ContextManager: retrievalCacheMaxSize must be a non-negative integer (got ${this.retrievalCacheMaxSize})`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Estimate token count of a single text string. */
|
||||
estimateTokens(text: string): number {
|
||||
return this.estimateFn(text);
|
||||
}
|
||||
|
||||
/** Returns true if the state's accumulated_context exceeds the configured threshold. */
|
||||
needsCompression(state: CheckpointStepState): boolean {
|
||||
const tokens = this.estimateTokens(state.accumulated_context);
|
||||
return tokens > this.contextTokenBudget * this.compressionThreshold;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress accumulated_context per the configured strategy. Returns a NEW
|
||||
* state object (does not mutate input). If state's context is below the
|
||||
* threshold OR shorter than retainRecentChars, returns input unchanged.
|
||||
*
|
||||
* Strategies:
|
||||
* - 'summarize-only': LLM-summarize older portion + retain recent
|
||||
* - 'retrieve-only': drop older portion + retain recent + emit archive event
|
||||
* - 'hybrid': summarize older + emit archive event for retrieval indexing
|
||||
*/
|
||||
async compress(state: CheckpointStepState, opts: CompressOptions = {}): Promise<CheckpointStepState> {
|
||||
void opts; // stepQuery reserved for Phase 3.4 retrieval-aware compression
|
||||
const beforeTokens = this.estimateTokens(state.accumulated_context);
|
||||
if (!this.needsCompression(state)) {
|
||||
this.emit({
|
||||
type: 'context-compression-skipped',
|
||||
before_tokens: beforeTokens,
|
||||
after_tokens: beforeTokens,
|
||||
strategy: this.strategy,
|
||||
cost_usd: 0,
|
||||
});
|
||||
return state;
|
||||
}
|
||||
if (state.accumulated_context.length <= this.retainRecentChars) {
|
||||
// Context exceeds token budget but the entire string fits in retainRecentChars.
|
||||
// Nothing to safely compress without dropping the recent verbatim region.
|
||||
this.emit({
|
||||
type: 'context-compression-skipped',
|
||||
before_tokens: beforeTokens,
|
||||
after_tokens: beforeTokens,
|
||||
strategy: this.strategy,
|
||||
cost_usd: 0,
|
||||
});
|
||||
return state;
|
||||
}
|
||||
|
||||
const splitAt = state.accumulated_context.length - this.retainRecentChars;
|
||||
const olderText = state.accumulated_context.slice(0, splitAt);
|
||||
const recentText = state.accumulated_context.slice(splitAt);
|
||||
const archivedRange: ArchivedContextRange = {
|
||||
archived_text: olderText,
|
||||
range_start: 0,
|
||||
range_end: splitAt,
|
||||
};
|
||||
|
||||
let newContext: string;
|
||||
let costUsd = 0;
|
||||
|
||||
if (this.strategy === 'retrieve-only') {
|
||||
newContext = `[archived: ${olderText.length} chars; see retrieval index]\n\n${recentText}`;
|
||||
} else {
|
||||
// summarize-only or hybrid — LLM-summarize the older portion.
|
||||
const summaryResult = await this._summarize(olderText);
|
||||
costUsd = summaryResult.cost_usd;
|
||||
const summaryText = summaryResult.summary;
|
||||
newContext = `[summary of ${olderText.length} archived chars]\n${summaryText}\n\n${recentText}`;
|
||||
}
|
||||
|
||||
const afterTokens = this.estimateTokens(newContext);
|
||||
const newState: CheckpointStepState = {
|
||||
...state,
|
||||
accumulated_context: newContext,
|
||||
};
|
||||
|
||||
const event: ContextCompressionEvent = {
|
||||
type: 'context-compressed',
|
||||
before_tokens: beforeTokens,
|
||||
after_tokens: afterTokens,
|
||||
strategy: this.strategy,
|
||||
cost_usd: costUsd,
|
||||
};
|
||||
if (this.strategy === 'retrieve-only' || this.strategy === 'hybrid') {
|
||||
event.archived = archivedRange;
|
||||
}
|
||||
this.emit(event);
|
||||
return newState;
|
||||
}
|
||||
|
||||
/**
|
||||
* LRU-evict retrieval_cache entries to fit retrievalCacheMaxSize. Returns
|
||||
* a NEW state object (does not mutate input). If cache size already fits,
|
||||
* returns input unchanged.
|
||||
*
|
||||
* If accessOrder is provided (keys ordered most-recent-LAST), entries
|
||||
* earlier in accessOrder are evicted first; keys not in accessOrder are
|
||||
* treated as oldest. Without accessOrder, evicts in insertion order
|
||||
* (JS spec preserves insertion order for string keys).
|
||||
*/
|
||||
evictRetrievalCache(state: CheckpointStepState, opts: { accessOrder?: readonly string[] } = {}): CheckpointStepState {
|
||||
const entries = Object.entries(state.retrieval_cache);
|
||||
const beforeSize = entries.length;
|
||||
if (beforeSize <= this.retrievalCacheMaxSize) {
|
||||
return state;
|
||||
}
|
||||
|
||||
let orderedKeys: string[];
|
||||
if (opts.accessOrder) {
|
||||
// accessOrder = recent-last → oldest at index 0.
|
||||
// Keys not present in accessOrder are oldest of all (prepend).
|
||||
const accessed = new Set(opts.accessOrder);
|
||||
const unaccessed = Object.keys(state.retrieval_cache).filter(k => !accessed.has(k));
|
||||
// Re-order accessOrder to only include keys actually in cache.
|
||||
const accessedInCache = opts.accessOrder.filter(k => k in state.retrieval_cache);
|
||||
orderedKeys = [...unaccessed, ...accessedInCache];
|
||||
} else {
|
||||
// Insertion order — already provided by Object.keys.
|
||||
orderedKeys = Object.keys(state.retrieval_cache);
|
||||
}
|
||||
|
||||
const evictCount = beforeSize - this.retrievalCacheMaxSize;
|
||||
const evictedKeys = orderedKeys.slice(0, evictCount);
|
||||
const evictedSet = new Set(evictedKeys);
|
||||
|
||||
const newCache: Record<string, unknown> = {};
|
||||
const evictedEntries: Array<[string, unknown]> = [];
|
||||
for (const [k, v] of entries) {
|
||||
if (evictedSet.has(k)) {
|
||||
evictedEntries.push([k, v]);
|
||||
} else {
|
||||
newCache[k] = v;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.archiveCacheTo) {
|
||||
void this.archiveCacheTo(evictedEntries);
|
||||
}
|
||||
|
||||
this.emit({
|
||||
type: 'cache-evicted',
|
||||
before_size: beforeSize,
|
||||
after_size: Object.keys(newCache).length,
|
||||
evicted_count: evictedEntries.length,
|
||||
});
|
||||
|
||||
return { ...state, retrieval_cache: newCache };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress decision_history to keep only the last K verbatim plus one
|
||||
* rolled-up summary Decision representing the older bucket. Returns a NEW
|
||||
* state object (does not mutate input). If history length ≤ retainRecentDecisions,
|
||||
* returns input unchanged.
|
||||
*
|
||||
* If llmCall is configured, the older bucket is summarized via LLM call.
|
||||
* Otherwise a heuristic summary (concatenation of decision strings) is used.
|
||||
* If archiveDecisionsTo is set, the original older bucket is persisted
|
||||
* before compression for lossless audit trail.
|
||||
*/
|
||||
async compressDecisionHistory(state: CheckpointStepState): Promise<CheckpointStepState> {
|
||||
const history = state.decision_history;
|
||||
const beforeCount = history.length;
|
||||
if (beforeCount <= this.retainRecentDecisions) {
|
||||
return state;
|
||||
}
|
||||
|
||||
const splitAt = beforeCount - this.retainRecentDecisions;
|
||||
const olderBucket = history.slice(0, splitAt);
|
||||
const recentBucket = history.slice(splitAt);
|
||||
|
||||
if (this.archiveDecisionsTo) {
|
||||
void this.archiveDecisionsTo(olderBucket);
|
||||
}
|
||||
|
||||
let summaryText: string;
|
||||
let costUsd = 0;
|
||||
if (this.llmCall && this.summarizationModel) {
|
||||
const result = await this._summarizeDecisions(olderBucket);
|
||||
summaryText = result.summary;
|
||||
costUsd = result.cost_usd;
|
||||
} else {
|
||||
summaryText = buildHeuristicDecisionSummary(olderBucket);
|
||||
}
|
||||
|
||||
const summaryDecision: Decision = {
|
||||
step_index: olderBucket[0]?.step_index ?? 0,
|
||||
decision: `[rolled-up summary of ${olderBucket.length} earlier decisions]`,
|
||||
rationale: summaryText,
|
||||
};
|
||||
|
||||
const newHistory = [summaryDecision, ...recentBucket];
|
||||
const afterCount = newHistory.length;
|
||||
|
||||
this.emit({
|
||||
type: 'decisions-compressed',
|
||||
before_count: beforeCount,
|
||||
after_count: afterCount,
|
||||
evicted_count: olderBucket.length,
|
||||
cost_usd: costUsd,
|
||||
});
|
||||
|
||||
void CHECKPOINT_SCHEMA_VERSION; // pin import for forward-compat use
|
||||
return { ...state, decision_history: newHistory };
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Private — LLM summarization helpers
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
private async _summarize(text: string): Promise<{ summary: string; cost_usd: number }> {
|
||||
if (!this.llmCall || !this.summarizationModel) {
|
||||
throw new Error('ContextManager: _summarize called without llmCall — constructor invariant violated');
|
||||
}
|
||||
const result = await this.llmCall({
|
||||
model: this.summarizationModel,
|
||||
messages: [
|
||||
{ role: 'system', content: CONTEXT_COMPACTION_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: text },
|
||||
],
|
||||
maxTokens: this.summarizationMaxTokens,
|
||||
temperature: 0.1,
|
||||
});
|
||||
return { summary: result.content, cost_usd: result.costUsd };
|
||||
}
|
||||
|
||||
private async _summarizeDecisions(decisions: readonly Decision[]): Promise<{ summary: string; cost_usd: number }> {
|
||||
if (!this.llmCall || !this.summarizationModel) {
|
||||
throw new Error('ContextManager: _summarizeDecisions called without llmCall — constructor invariant violated');
|
||||
}
|
||||
const formatted = decisions.map(d => {
|
||||
const parts = [`Step ${d.step_index}: ${d.decision}`];
|
||||
if (d.rationale) parts.push(` rationale: ${d.rationale}`);
|
||||
if (d.alternatives_considered && d.alternatives_considered.length > 0) {
|
||||
parts.push(` alternatives: ${d.alternatives_considered.join(' | ')}`);
|
||||
}
|
||||
return parts.join('\n');
|
||||
}).join('\n\n');
|
||||
|
||||
const result = await this.llmCall({
|
||||
model: this.summarizationModel,
|
||||
messages: [
|
||||
{ role: 'system', content: DECISION_COMPACTION_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: formatted },
|
||||
],
|
||||
maxTokens: this.summarizationMaxTokens,
|
||||
temperature: 0.1,
|
||||
});
|
||||
return { summary: result.content, cost_usd: result.costUsd };
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Heuristic summary fallback (no LLM)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function buildHeuristicDecisionSummary(decisions: readonly Decision[]): string {
|
||||
const lines = decisions.map(d => {
|
||||
const stepBit = `step ${d.step_index}: `;
|
||||
const decisionBit = d.decision.length > 80 ? d.decision.slice(0, 77) + '…' : d.decision;
|
||||
return stepBit + decisionBit;
|
||||
});
|
||||
return lines.join('; ');
|
||||
}
|
||||
594
packages/agent/src/long-task/failure-classify.ts
Normal file
594
packages/agent/src/long-task/failure-classify.ts
Normal file
@@ -0,0 +1,594 @@
|
||||
/**
|
||||
* Failure classifier — Phase 4.1 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.1)
|
||||
*
|
||||
* Classifies a model's failed output against a gold answer into one of ten
|
||||
* pre-registered failure categories. Rule-based first (fast, free, deterministic);
|
||||
* LLM-judge fallback only for ambiguous cases when the caller provides one.
|
||||
*
|
||||
* The ten categories are pre-registered in the sprint plan and reflect the
|
||||
* 2026-04-26 pilot's empirical failure-mode taxonomy. Phase 4.3 re-score uses
|
||||
* this classifier to bucket the 12-cell pilot's H2/H3/H4 failures into
|
||||
* Tier 1 (presentation / fixable by output-normalize layer) vs Tier 2
|
||||
* (real harness-design issues).
|
||||
*
|
||||
* The classifier walks rules in PRIORITY ORDER (most-specific first):
|
||||
* 1. retrieval_or_harness_error (upstream signal overrides everything)
|
||||
* 2. thinking_leakage (explicit <think> tags or chain-of-thought prefixes)
|
||||
* 3. metadata_copy (literal substrate metadata in output)
|
||||
* 4. format_violation (bullet list when prose expected, JSON when text, etc.)
|
||||
* 5. unknown_false_negative (model abstained but gold is answerable)
|
||||
* 6. punctuation_or_case_only (normalize-equal but raw-different)
|
||||
* 7. correct_answer_with_extra_text (gold-substring-match but output is verbose)
|
||||
* 8. wrong_span (high token overlap, but no substring match)
|
||||
* 9. wrong_entity (low token overlap, definitive statement)
|
||||
* 10. hallucination (default fallback for "wrong but uncategorized")
|
||||
*
|
||||
* Each category emits a `confidence` level:
|
||||
* - 'high': explicit signal (e.g., literal <think> tag, exact token-set equality)
|
||||
* - 'medium': heuristic signal (e.g., format-prefix mismatch, token-overlap threshold)
|
||||
* - 'low': no clear signal — defaults to hallucination
|
||||
*
|
||||
* Callers MUST decide whether to act on low-confidence classifications
|
||||
* (e.g., surface for human review, or invoke LLM judge fallback).
|
||||
*/
|
||||
|
||||
import type { LlmCallFn } from '../retrieval-agent-loop.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Types
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type FailureCategory =
|
||||
| 'thinking_leakage'
|
||||
| 'correct_answer_with_extra_text'
|
||||
| 'unknown_false_negative'
|
||||
| 'metadata_copy'
|
||||
| 'format_violation'
|
||||
| 'punctuation_or_case_only'
|
||||
| 'wrong_span'
|
||||
| 'wrong_entity'
|
||||
| 'hallucination'
|
||||
| 'retrieval_or_harness_error';
|
||||
|
||||
export const FAILURE_CATEGORIES: readonly FailureCategory[] = [
|
||||
'thinking_leakage',
|
||||
'correct_answer_with_extra_text',
|
||||
'unknown_false_negative',
|
||||
'metadata_copy',
|
||||
'format_violation',
|
||||
'punctuation_or_case_only',
|
||||
'wrong_span',
|
||||
'wrong_entity',
|
||||
'hallucination',
|
||||
'retrieval_or_harness_error',
|
||||
] as const;
|
||||
|
||||
export type ConfidenceLevel = 'high' | 'medium' | 'low';
|
||||
|
||||
export interface FailureClassification {
|
||||
primary: FailureCategory;
|
||||
confidence: ConfidenceLevel;
|
||||
secondary?: FailureCategory;
|
||||
rationale: string;
|
||||
/** Names of rules / heuristics that fired. Empty for LLM-judge-only verdicts. */
|
||||
rules_fired: readonly string[];
|
||||
/** Whether the LLM-judge fallback was invoked. */
|
||||
llm_judge_invoked: boolean;
|
||||
}
|
||||
|
||||
export interface ClassifierInput {
|
||||
model_output: string;
|
||||
/** Single string or list of acceptable gold answers (any-match = passing). */
|
||||
gold_answer: string | readonly string[];
|
||||
/** Optional question text — passed to LLM judge if invoked. */
|
||||
question?: string;
|
||||
/** Optional substrate / retrieved context — used to detect metadata leakage. */
|
||||
substrate?: string;
|
||||
/** Optional upstream error string (retrieval/harness). If set, overrides to category #1. */
|
||||
upstream_error?: string;
|
||||
}
|
||||
|
||||
export interface ClassifierOptions {
|
||||
/**
|
||||
* Optional LLM judge function for low-confidence rule cases. If absent, the
|
||||
* classifier returns the rule's verdict at the rule's confidence level.
|
||||
*/
|
||||
llmJudge?: LlmCallFn;
|
||||
/** Model alias for the LLM judge. Required if llmJudge is provided. */
|
||||
judgeModel?: string;
|
||||
/**
|
||||
* If true (default), the LLM judge is invoked for 'low' confidence verdicts
|
||||
* to potentially upgrade them. If false, never invoke the judge regardless.
|
||||
*/
|
||||
invokeJudgeOnLowConfidence?: boolean;
|
||||
/** Per-call max_tokens for the LLM judge. Default 256. */
|
||||
judgeMaxTokens?: number;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Detector regexes / constants
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const THINKING_TAG_RE = /<\s*\/?\s*think(?:ing)?\s*[>\s]/i;
|
||||
const THINKING_PREFIX_RE = /^(let me think|first[,\s]|step \d|i need to|i'?ll start|let's break|to answer this|let me analyze|let me work through)/i;
|
||||
|
||||
const METADATA_LITERAL_PATTERNS = [
|
||||
/\[memory:[^\]]*\]/i,
|
||||
/\[ref:[^\]]*\]/i,
|
||||
/\[source:[^\]]*\]/i,
|
||||
/from session\s*(?:#?\d+|"[^"]+")/i,
|
||||
/\[\s*context:[^\]]*\]/i,
|
||||
/^\s*<<\s*[a-z_]+\s*>>/i,
|
||||
];
|
||||
|
||||
const UNKNOWN_OUTPUTS = new Set([
|
||||
'unknown',
|
||||
'i don\'t know',
|
||||
'i do not know',
|
||||
'cannot determine',
|
||||
'cannot be determined',
|
||||
'not specified',
|
||||
'no information',
|
||||
'insufficient information',
|
||||
'no answer',
|
||||
'n/a',
|
||||
'na',
|
||||
'none',
|
||||
]);
|
||||
|
||||
const FORMAT_BULLET_PREFIXES = /^\s*([*\-•]|\d+[.)]\s)/m;
|
||||
const FORMAT_JSON_PREFIXES = /^\s*[{[]/;
|
||||
const FORMAT_FENCE_PREFIX = /^\s*```/;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function asGoldArray(g: string | readonly string[]): readonly string[] {
|
||||
return typeof g === 'string' ? [g] : g;
|
||||
}
|
||||
|
||||
function lc(s: string): string {
|
||||
return s.toLowerCase();
|
||||
}
|
||||
|
||||
/** Strip punctuation + collapse whitespace + lowercase. */
|
||||
function normalizeForCompare(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}\s]/gu, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tokenize(s: string): string[] {
|
||||
return normalizeForCompare(s).split(/\s+/).filter(t => t.length > 0);
|
||||
}
|
||||
|
||||
function tokenSet(s: string): Set<string> {
|
||||
return new Set(tokenize(s));
|
||||
}
|
||||
|
||||
/** Jaccard similarity over token sets. 0..1. */
|
||||
function tokenOverlap(a: string, b: string): number {
|
||||
const sa = tokenSet(a);
|
||||
const sb = tokenSet(b);
|
||||
if (sa.size === 0 && sb.size === 0) return 1;
|
||||
if (sa.size === 0 || sb.size === 0) return 0;
|
||||
let inter = 0;
|
||||
for (const t of sa) if (sb.has(t)) inter += 1;
|
||||
const union = sa.size + sb.size - inter;
|
||||
return union === 0 ? 0 : inter / union;
|
||||
}
|
||||
|
||||
function lowercaseSubstringMatch(output: string, gold: readonly string[]): boolean {
|
||||
const lower = output.toLowerCase();
|
||||
for (const g of gold) {
|
||||
if (g && lower.includes(g.toLowerCase())) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Detector functions — each returns { matched, confidence, rationale, rule } | null
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface DetectorResult {
|
||||
matched: boolean;
|
||||
confidence: ConfidenceLevel;
|
||||
rationale: string;
|
||||
rule: string;
|
||||
}
|
||||
|
||||
function detectUpstreamError(input: ClassifierInput): DetectorResult | null {
|
||||
if (input.upstream_error && input.upstream_error.trim().length > 0) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: `upstream_error present: "${input.upstream_error.slice(0, 80)}"`,
|
||||
rule: 'upstream-error',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectThinkingLeakage(input: ClassifierInput): DetectorResult | null {
|
||||
const out = input.model_output;
|
||||
if (THINKING_TAG_RE.test(out)) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: 'output contains literal <think> tags',
|
||||
rule: 'thinking-tag',
|
||||
};
|
||||
}
|
||||
if (THINKING_PREFIX_RE.test(out.trim())) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: 'output starts with chain-of-thought prefix',
|
||||
rule: 'thinking-prefix',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectMetadataCopy(input: ClassifierInput): DetectorResult | null {
|
||||
const out = input.model_output;
|
||||
for (const re of METADATA_LITERAL_PATTERNS) {
|
||||
if (re.test(out)) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: `output contains literal substrate metadata pattern ${re}`,
|
||||
rule: 'metadata-literal',
|
||||
};
|
||||
}
|
||||
}
|
||||
if (input.substrate) {
|
||||
// Heuristic: long substring of substrate that's clearly metadata-shaped (timestamps,
|
||||
// session ids) appearing verbatim in the output.
|
||||
const ts = out.match(/session\s*\d{2,}|gop_id\s*[:=]\s*\S+|frame_id\s*[:=]/i);
|
||||
if (ts) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: `output contains substrate-shaped identifier "${ts[0]}"`,
|
||||
rule: 'metadata-shaped-id',
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectFormatViolation(input: ClassifierInput): DetectorResult | null {
|
||||
const gold = asGoldArray(input.gold_answer);
|
||||
const goldSample = gold[0] ?? '';
|
||||
const out = input.model_output;
|
||||
const outIsBullet = FORMAT_BULLET_PREFIXES.test(out);
|
||||
const outIsJson = FORMAT_JSON_PREFIXES.test(out.trim());
|
||||
const outIsFence = FORMAT_FENCE_PREFIX.test(out);
|
||||
const goldIsBullet = FORMAT_BULLET_PREFIXES.test(goldSample);
|
||||
const goldIsJson = FORMAT_JSON_PREFIXES.test(goldSample.trim());
|
||||
|
||||
if (outIsFence && !goldIsJson && !goldIsBullet) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: 'output wrapped in ``` code fence; gold is plain prose',
|
||||
rule: 'format-fence-mismatch',
|
||||
};
|
||||
}
|
||||
if (outIsJson && !goldIsJson) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: 'output starts with { or [; gold is plain prose',
|
||||
rule: 'format-json-mismatch',
|
||||
};
|
||||
}
|
||||
if (outIsBullet && !goldIsBullet) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: 'output starts with bullet/numbered list; gold is plain prose',
|
||||
rule: 'format-bullet-mismatch',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectUnknownFalseNegative(input: ClassifierInput): DetectorResult | null {
|
||||
const lcOut = input.model_output.toLowerCase().trim();
|
||||
if (UNKNOWN_OUTPUTS.has(lcOut)) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: `output is exact "unknown"-class string ("${lcOut}")`,
|
||||
rule: 'unknown-exact',
|
||||
};
|
||||
}
|
||||
// Heuristic: output is short AND contains unknown-class phrase
|
||||
if (lcOut.length < 80) {
|
||||
for (const u of UNKNOWN_OUTPUTS) {
|
||||
if (lcOut.includes(u)) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: `output (short, ${lcOut.length} chars) contains "${u}"`,
|
||||
rule: 'unknown-substring',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectPunctuationOrCaseOnly(input: ClassifierInput): DetectorResult | null {
|
||||
const gold = asGoldArray(input.gold_answer);
|
||||
const outNorm = normalizeForCompare(input.model_output);
|
||||
const outRaw = input.model_output.trim();
|
||||
for (const g of gold) {
|
||||
if (!g) continue;
|
||||
if (outNorm === normalizeForCompare(g) && outRaw !== g.trim()) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'high',
|
||||
rationale: 'output equals gold after normalize, but raw forms differ (punctuation/case)',
|
||||
rule: 'punct-case-only',
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectCorrectWithExtraText(input: ClassifierInput): DetectorResult | null {
|
||||
const gold = asGoldArray(input.gold_answer);
|
||||
if (!lowercaseSubstringMatch(input.model_output, gold)) return null;
|
||||
const goldSample = gold.find(g => input.model_output.toLowerCase().includes(g.toLowerCase())) ?? '';
|
||||
if (!goldSample) return null;
|
||||
const goldLen = goldSample.length;
|
||||
const outLen = input.model_output.trim().length;
|
||||
if (outLen >= goldLen * 1.5 && outLen >= goldLen + 25) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: outLen >= goldLen * 3 ? 'high' : 'medium',
|
||||
rationale: `gold "${goldSample.slice(0, 40)}…" is contained in output, but output is ${outLen} chars vs gold ${goldLen} chars`,
|
||||
rule: 'correct-with-extra',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectWrongSpan(input: ClassifierInput): DetectorResult | null {
|
||||
const gold = asGoldArray(input.gold_answer);
|
||||
if (lowercaseSubstringMatch(input.model_output, gold)) return null;
|
||||
let bestOverlap = 0;
|
||||
for (const g of gold) {
|
||||
bestOverlap = Math.max(bestOverlap, tokenOverlap(input.model_output, g));
|
||||
}
|
||||
if (bestOverlap >= 0.5) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: 'medium',
|
||||
rationale: `Jaccard token overlap ${bestOverlap.toFixed(2)} ≥ 0.5 but no substring match`,
|
||||
rule: 'wrong-span',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectWrongEntity(input: ClassifierInput): DetectorResult | null {
|
||||
const gold = asGoldArray(input.gold_answer);
|
||||
if (lowercaseSubstringMatch(input.model_output, gold)) return null;
|
||||
let bestOverlap = 0;
|
||||
for (const g of gold) {
|
||||
bestOverlap = Math.max(bestOverlap, tokenOverlap(input.model_output, g));
|
||||
}
|
||||
// Definitive statement: output > 5 tokens AND not an unknown phrase.
|
||||
const tokens = tokenize(input.model_output);
|
||||
const definitive = tokens.length >= 5 && !Array.from(UNKNOWN_OUTPUTS).some(u => input.model_output.toLowerCase().includes(u));
|
||||
if (bestOverlap < 0.3 && definitive) {
|
||||
return {
|
||||
matched: true,
|
||||
confidence: bestOverlap < 0.1 ? 'high' : 'medium',
|
||||
rationale: `Jaccard token overlap ${bestOverlap.toFixed(2)} < 0.3 with definitive output (${tokens.length} tokens)`,
|
||||
rule: 'wrong-entity',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// LLM judge fallback
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const LLM_JUDGE_PROMPT_TEMPLATE = (input: ClassifierInput): string => {
|
||||
const gold = asGoldArray(input.gold_answer).join(' | ');
|
||||
return [
|
||||
'You are a failure-mode classifier for a benchmark agent\'s output.',
|
||||
'',
|
||||
`Question: ${input.question ?? '(not provided)'}`,
|
||||
`Gold answer (any-of): ${gold}`,
|
||||
`Model output: ${input.model_output}`,
|
||||
'',
|
||||
'Classify into exactly one of these categories:',
|
||||
' - thinking_leakage (output contains chain-of-thought reasoning that should have been stripped)',
|
||||
' - correct_answer_with_extra_text (output contains the gold answer but with verbose extra prose)',
|
||||
' - unknown_false_negative (output abstained but the gold answer is definite)',
|
||||
' - metadata_copy (output contains literal substrate metadata)',
|
||||
' - format_violation (output uses wrong format — bullets/JSON/code fence vs expected prose)',
|
||||
' - punctuation_or_case_only (output equals gold modulo punctuation or case)',
|
||||
' - wrong_span (output gets the right entity but wrong span — too narrow or too broad)',
|
||||
' - wrong_entity (output references the wrong entity entirely)',
|
||||
' - hallucination (output contains content not in gold or substrate)',
|
||||
' - retrieval_or_harness_error (failure caused by retrieval / infrastructure, not model reasoning)',
|
||||
'',
|
||||
'Output JSON only (no prose, no fences):',
|
||||
'{"primary":"<category>","confidence":"high|medium|low","rationale":"<one short sentence>"}',
|
||||
].join('\n');
|
||||
};
|
||||
|
||||
async function invokeLlmJudge(
|
||||
input: ClassifierInput,
|
||||
opts: ClassifierOptions,
|
||||
): Promise<{ primary: FailureCategory; confidence: ConfidenceLevel; rationale: string } | null> {
|
||||
if (!opts.llmJudge || !opts.judgeModel) return null;
|
||||
const prompt = LLM_JUDGE_PROMPT_TEMPLATE(input);
|
||||
const r = await opts.llmJudge({
|
||||
model: opts.judgeModel,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
maxTokens: opts.judgeMaxTokens ?? 256,
|
||||
temperature: 0,
|
||||
});
|
||||
if (r.error) return null;
|
||||
const m = r.content.match(/\{[\s\S]*\}/);
|
||||
if (!m) return null;
|
||||
try {
|
||||
const obj = JSON.parse(m[0]) as { primary?: string; confidence?: string; rationale?: string };
|
||||
const primary = obj.primary as FailureCategory;
|
||||
if (!FAILURE_CATEGORIES.includes(primary)) return null;
|
||||
const confidence = (obj.confidence as ConfidenceLevel) ?? 'medium';
|
||||
return {
|
||||
primary,
|
||||
confidence: confidence === 'high' || confidence === 'medium' || confidence === 'low' ? confidence : 'medium',
|
||||
rationale: obj.rationale ?? '(no rationale)',
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Main entry point
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface RuleHit {
|
||||
category: FailureCategory;
|
||||
result: DetectorResult;
|
||||
}
|
||||
|
||||
function runRuleCascade(input: ClassifierInput): RuleHit[] {
|
||||
const hits: RuleHit[] = [];
|
||||
const detectors: Array<{ category: FailureCategory; fn: (i: ClassifierInput) => DetectorResult | null }> = [
|
||||
{ category: 'retrieval_or_harness_error', fn: detectUpstreamError },
|
||||
{ category: 'thinking_leakage', fn: detectThinkingLeakage },
|
||||
{ category: 'metadata_copy', fn: detectMetadataCopy },
|
||||
{ category: 'format_violation', fn: detectFormatViolation },
|
||||
{ category: 'unknown_false_negative', fn: detectUnknownFalseNegative },
|
||||
{ category: 'punctuation_or_case_only', fn: detectPunctuationOrCaseOnly },
|
||||
{ category: 'correct_answer_with_extra_text', fn: detectCorrectWithExtraText },
|
||||
{ category: 'wrong_span', fn: detectWrongSpan },
|
||||
{ category: 'wrong_entity', fn: detectWrongEntity },
|
||||
];
|
||||
for (const { category, fn } of detectors) {
|
||||
const r = fn(input);
|
||||
if (r && r.matched) hits.push({ category, result: r });
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a model output failure into one of the 10 pre-registered categories.
|
||||
*
|
||||
* Walks the rule cascade in priority order. If at least one rule fires, returns
|
||||
* the highest-priority hit's verdict. If no rules fire, returns 'hallucination'
|
||||
* with 'low' confidence (the catch-all default).
|
||||
*
|
||||
* If options.llmJudge is provided AND the rule-cascade verdict is 'low'
|
||||
* confidence (or no rule fired), the LLM judge is invoked for potential
|
||||
* upgrade. The judge's verdict overrides only if it returns a known category.
|
||||
*/
|
||||
export async function classifyFailure(
|
||||
input: ClassifierInput,
|
||||
opts: ClassifierOptions = {},
|
||||
): Promise<FailureClassification> {
|
||||
const hits = runRuleCascade(input);
|
||||
let primary: FailureCategory;
|
||||
let confidence: ConfidenceLevel;
|
||||
let rationale: string;
|
||||
let rulesFired: string[];
|
||||
let secondary: FailureCategory | undefined;
|
||||
|
||||
if (hits.length > 0) {
|
||||
const top = hits[0]!;
|
||||
primary = top.category;
|
||||
confidence = top.result.confidence;
|
||||
rationale = top.result.rationale;
|
||||
rulesFired = hits.map(h => h.result.rule);
|
||||
if (hits.length > 1) secondary = hits[1]!.category;
|
||||
} else {
|
||||
primary = 'hallucination';
|
||||
confidence = 'low';
|
||||
rationale = 'no rule fired; defaulting to hallucination (catch-all)';
|
||||
rulesFired = [];
|
||||
}
|
||||
|
||||
// LLM judge fallback for low-confidence verdicts.
|
||||
const shouldInvokeJudge =
|
||||
Boolean(opts.llmJudge && opts.judgeModel) &&
|
||||
confidence === 'low' &&
|
||||
(opts.invokeJudgeOnLowConfidence ?? true);
|
||||
|
||||
if (shouldInvokeJudge) {
|
||||
const judgeVerdict = await invokeLlmJudge(input, opts);
|
||||
if (judgeVerdict) {
|
||||
return {
|
||||
primary: judgeVerdict.primary,
|
||||
confidence: judgeVerdict.confidence,
|
||||
secondary: primary === judgeVerdict.primary ? secondary : primary,
|
||||
rationale: `[llm-judge] ${judgeVerdict.rationale}`,
|
||||
rules_fired: rulesFired,
|
||||
llm_judge_invoked: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
primary,
|
||||
confidence,
|
||||
secondary,
|
||||
rationale,
|
||||
rules_fired: rulesFired,
|
||||
llm_judge_invoked: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a batch of inputs. Convenience wrapper over classifyFailure.
|
||||
*/
|
||||
export async function classifyFailureBatch(
|
||||
inputs: readonly ClassifierInput[],
|
||||
opts: ClassifierOptions = {},
|
||||
): Promise<readonly FailureClassification[]> {
|
||||
const out: FailureClassification[] = [];
|
||||
for (const inp of inputs) {
|
||||
out.push(await classifyFailure(inp, opts));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute per-category counts from a list of classifications.
|
||||
*/
|
||||
export function failureDistribution(
|
||||
classifications: readonly FailureClassification[],
|
||||
): Readonly<Record<FailureCategory, number>> {
|
||||
const dist: Record<FailureCategory, number> = {
|
||||
thinking_leakage: 0,
|
||||
correct_answer_with_extra_text: 0,
|
||||
unknown_false_negative: 0,
|
||||
metadata_copy: 0,
|
||||
format_violation: 0,
|
||||
punctuation_or_case_only: 0,
|
||||
wrong_span: 0,
|
||||
wrong_entity: 0,
|
||||
hallucination: 0,
|
||||
retrieval_or_harness_error: 0,
|
||||
};
|
||||
for (const c of classifications) {
|
||||
dist[c.primary] += 1;
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
326
packages/agent/src/long-task/messages-compressor.ts
Normal file
326
packages/agent/src/long-task/messages-compressor.ts
Normal file
@@ -0,0 +1,326 @@
|
||||
/**
|
||||
* Messages-array compression — Phase 4.6 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.6)
|
||||
*
|
||||
* Addresses the Phase 3 acceptance gate finding: ContextManager (Phase 3.3)
|
||||
* compresses ONLY accumulated_context (audit log, ~450 tokens by step 30 —
|
||||
* never crosses 4K threshold). Real LLM cost is dominated by the messages
|
||||
* array (~16K tokens by step 30 on Opus). Phase 4.6 closes the gap by adding
|
||||
* a SEPARATE compression layer for the messages array, invoked at the top of
|
||||
* each retrieval-agent-loop iteration.
|
||||
*
|
||||
* REUSE / NON-DUPLICATION (per CLAUDE.md §3.6 check-before-create):
|
||||
* - estimateStringTokens (per-string version) from context-compressor.ts
|
||||
* for content-aware token estimation
|
||||
* - pruneToolResults from context-compressor.ts (drops tool-result blobs in
|
||||
* older messages to reduce size before LLM summarization)
|
||||
* - COMPACTION_PROMPT from behavioral-spec.ts (proven summarizer prompt)
|
||||
*
|
||||
* NEW (this module):
|
||||
* - summarizeMiddleViaLlmCall: drop-in replacement for context-compressor's
|
||||
* fetch-based summarizeMiddle that uses Phase 2.1's LlmCallFn abstraction.
|
||||
* - Simple message-count-based split (head + middle + tail) — PM brief
|
||||
* specified `retainRecentTurns` as a COUNT, not the token-budget surface
|
||||
* used by context-compressor.ts's splitProtectedRegions.
|
||||
*
|
||||
* NOT REFACTORED:
|
||||
* - context-compressor.ts itself stays fetch-based for its existing
|
||||
* consumers (harness + server). Future Sprint 12 cleanup could unify on
|
||||
* LlmCallFn but that's separate scope.
|
||||
*/
|
||||
|
||||
import {
|
||||
estimateStringTokens,
|
||||
pruneToolResults,
|
||||
type CompressibleMessage,
|
||||
} from '../context-compressor.js';
|
||||
import { COMPACTION_PROMPT } from '../behavioral-spec.js';
|
||||
import type { LlmCallFn } from '../retrieval-agent-loop.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Public types
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface MessagesCompressionEvent {
|
||||
type: 'messages_compressed';
|
||||
before_tokens: number;
|
||||
after_tokens: number;
|
||||
messages_before: number;
|
||||
messages_after: number;
|
||||
cost_usd: number;
|
||||
summary_generated: boolean;
|
||||
}
|
||||
|
||||
export interface MessagesContextManagerConfig {
|
||||
/** Token budget for the messages array (per LLM call). */
|
||||
budgetTokens: number;
|
||||
/** Fraction of budget that triggers compression. Default 0.7. */
|
||||
threshold?: number;
|
||||
/**
|
||||
* Number of recent TURNS to retain verbatim. One turn ≈ 2 messages
|
||||
* (assistant + user, or vice-versa). Default 5 turns = 10 messages.
|
||||
*/
|
||||
retainRecentTurns?: number;
|
||||
/**
|
||||
* Number of head messages (after the system prompt) to keep verbatim.
|
||||
* Default 1 (typically the kickoff user prompt — system is always retained).
|
||||
*/
|
||||
protectedHeadMessages?: number;
|
||||
/**
|
||||
* Per-string token estimator override. Default = estimateStringTokens
|
||||
* (content-aware: prose 4.0 / code 3.2 / json 3.5 / mixed 2.5 chars-per-token).
|
||||
* Caller can inject tiktoken or Anthropic count_tokens for exact counts.
|
||||
*/
|
||||
estimateTokensFn?: (text: string) => number;
|
||||
/**
|
||||
* LLM call function for summarizing the middle region. If absent, the
|
||||
* middle is replaced with a structural placeholder ("[N compressed
|
||||
* messages]") — pruning still happens but no semantic summary.
|
||||
*/
|
||||
llmCall?: LlmCallFn;
|
||||
/** Model alias for the summarizer. Required if llmCall is provided. */
|
||||
summarizationModel?: string;
|
||||
/** max_tokens for the summarizer LLM call. Default 2000. */
|
||||
summarizationMaxTokens?: number;
|
||||
/** Optional callback for messages-compression events. */
|
||||
onCompressionEvent?: (event: MessagesCompressionEvent) => void;
|
||||
}
|
||||
|
||||
export interface CompressMessagesResult {
|
||||
/** The (possibly compressed) messages array. */
|
||||
messages: Array<{ role: string; content: string }>;
|
||||
/** Updated running summary (carries forward across compressions). */
|
||||
summary: string | null;
|
||||
/** Whether compression actually fired this call. */
|
||||
compressed: boolean;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Constants
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_THRESHOLD = 0.7;
|
||||
const DEFAULT_RETAIN_RECENT_TURNS = 5;
|
||||
const DEFAULT_PROTECTED_HEAD_MESSAGES = 1;
|
||||
const DEFAULT_SUMMARIZATION_MAX_TOKENS = 2000;
|
||||
const PER_MESSAGE_ROLE_OVERHEAD_TOKENS = 4;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Internal helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function estimateMessagesTokens(
|
||||
messages: ReadonlyArray<{ role: string; content: string }>,
|
||||
perStringFn: (text: string) => number,
|
||||
): number {
|
||||
let sum = 0;
|
||||
for (const m of messages) {
|
||||
sum += PER_MESSAGE_ROLE_OVERHEAD_TOKENS;
|
||||
sum += perStringFn(m.content ?? '');
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
interface SplitRegions {
|
||||
head: Array<{ role: string; content: string }>;
|
||||
middle: Array<{ role: string; content: string }>;
|
||||
tail: Array<{ role: string; content: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split messages into head (system + first N messages) / middle (compressible)
|
||||
* / tail (last K turns × 2 messages, retained verbatim).
|
||||
*
|
||||
* One turn ≈ 2 messages, so retainRecentTurns × 2 messages are kept in tail.
|
||||
* Head + tail must not overlap; if the conversation is short, middle may be empty.
|
||||
*/
|
||||
function splitForCompress(
|
||||
messages: ReadonlyArray<{ role: string; content: string }>,
|
||||
protectedHead: number,
|
||||
retainRecentTurns: number,
|
||||
): SplitRegions {
|
||||
const headEnd = Math.min(1 + protectedHead, messages.length);
|
||||
const tailStart = Math.max(headEnd, messages.length - retainRecentTurns * 2);
|
||||
return {
|
||||
head: messages.slice(0, headEnd).map(m => ({ ...m })),
|
||||
middle: messages.slice(headEnd, tailStart).map(m => ({ ...m })),
|
||||
tail: messages.slice(tailStart).map(m => ({ ...m })),
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// LLM-call-based summarizer (replaces fetch-based summarizeMiddle)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface SummarizeResult {
|
||||
summary: string;
|
||||
costUsd: number;
|
||||
generated: boolean;
|
||||
}
|
||||
|
||||
async function summarizeMiddleViaLlmCall(
|
||||
middle: ReadonlyArray<CompressibleMessage>,
|
||||
llmCall: LlmCallFn,
|
||||
model: string,
|
||||
maxTokens: number,
|
||||
previousSummary: string | null,
|
||||
): Promise<SummarizeResult> {
|
||||
if (middle.length === 0) {
|
||||
return { summary: previousSummary ?? '', costUsd: 0, generated: false };
|
||||
}
|
||||
|
||||
const summarizerMessages: Array<{ role: string; content: string }> = [];
|
||||
|
||||
if (previousSummary) {
|
||||
summarizerMessages.push({
|
||||
role: 'system',
|
||||
content: `You are summarizing a conversation that has been compressed before. Here is the previous summary:\n\n${previousSummary}\n\nNow incorporate the new messages below into an updated summary.`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const msg of middle) {
|
||||
summarizerMessages.push({
|
||||
role: msg.role === 'system' ? 'user' : msg.role,
|
||||
content: msg.content ?? '',
|
||||
});
|
||||
}
|
||||
|
||||
summarizerMessages.push({ role: 'user', content: COMPACTION_PROMPT });
|
||||
|
||||
const r = await llmCall({
|
||||
model,
|
||||
messages: summarizerMessages,
|
||||
maxTokens,
|
||||
temperature: 0.1,
|
||||
// Compaction is a known-lightweight internal call → route to the cheap model
|
||||
// (model-class-router). Saves Waggle-funded proxy spend on every compaction.
|
||||
class: 'lightweight',
|
||||
});
|
||||
|
||||
if (r.error || !r.content) {
|
||||
return { summary: buildFallbackSummary(middle, previousSummary), costUsd: r.costUsd, generated: false };
|
||||
}
|
||||
return { summary: r.content, costUsd: r.costUsd, generated: true };
|
||||
}
|
||||
|
||||
function buildFallbackSummary(
|
||||
middle: ReadonlyArray<CompressibleMessage>,
|
||||
previousSummary: string | null,
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
if (previousSummary) lines.push('## Previous summary\n' + previousSummary);
|
||||
lines.push(`## Compressed region (${middle.length} messages, no LLM summary)`);
|
||||
const userFirstLines = middle
|
||||
.filter(m => m.role === 'user')
|
||||
.map(m => (m.content ?? '').split('\n')[0]?.slice(0, 100).trim())
|
||||
.filter((l): l is string => Boolean(l && l.length > 8))
|
||||
.slice(0, 5);
|
||||
if (userFirstLines.length > 0) {
|
||||
lines.push('Topics covered: ' + userFirstLines.join(' → '));
|
||||
}
|
||||
return lines.join('\n\n');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Main entry point
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compress the messages array if it exceeds threshold. Returns the (possibly
|
||||
* compressed) messages, a running summary (carries forward), and a boolean
|
||||
* indicating whether compression actually fired.
|
||||
*
|
||||
* Pipeline (when triggered):
|
||||
* 1. Prune tool-result-shaped blobs from older messages (no LLM, free).
|
||||
* 2. Split into head (system + first N) / middle (compressible) / tail
|
||||
* (last retainRecentTurns × 2 messages verbatim).
|
||||
* 3. Summarize the middle via LLM (or fallback placeholder if no LLM).
|
||||
* 4. Replace middle with single summary message; reassemble [head, summary, tail].
|
||||
*
|
||||
* Idempotency: if called twice in a row without growth, the second call is a
|
||||
* no-op (under-threshold).
|
||||
*/
|
||||
export async function maybeCompressMessages(
|
||||
messages: ReadonlyArray<{ role: string; content: string }>,
|
||||
config: MessagesContextManagerConfig,
|
||||
previousSummary: string | null,
|
||||
): Promise<CompressMessagesResult> {
|
||||
const stringEstimator = config.estimateTokensFn ?? estimateStringTokens;
|
||||
const threshold = config.threshold ?? DEFAULT_THRESHOLD;
|
||||
const protectedHead = config.protectedHeadMessages ?? DEFAULT_PROTECTED_HEAD_MESSAGES;
|
||||
const retainRecentTurns = config.retainRecentTurns ?? DEFAULT_RETAIN_RECENT_TURNS;
|
||||
|
||||
const tokensBefore = estimateMessagesTokens(messages, stringEstimator);
|
||||
if (tokensBefore <= config.budgetTokens * threshold) {
|
||||
return { messages: messages.map(m => ({ ...m })), summary: previousSummary, compressed: false };
|
||||
}
|
||||
|
||||
// Step 1: prune tool-result blobs (free, no LLM).
|
||||
const pruneTailCount = retainRecentTurns * 2;
|
||||
const pruned = pruneToolResults(messages, pruneTailCount);
|
||||
|
||||
// Step 2: split into head / middle / tail.
|
||||
const regions = splitForCompress(pruned, protectedHead, retainRecentTurns);
|
||||
|
||||
// If middle is empty (very short conversation, head + tail covers everything),
|
||||
// no compression is possible — return as-is (compressed=false).
|
||||
if (regions.middle.length === 0) {
|
||||
return { messages: pruned.map(m => ({ ...m })), summary: previousSummary, compressed: false };
|
||||
}
|
||||
|
||||
// Step 3: summarize middle.
|
||||
let summaryText: string;
|
||||
let summaryGenerated = false;
|
||||
let costUsd = 0;
|
||||
|
||||
if (config.llmCall && config.summarizationModel) {
|
||||
const r = await summarizeMiddleViaLlmCall(
|
||||
regions.middle,
|
||||
config.llmCall,
|
||||
config.summarizationModel,
|
||||
config.summarizationMaxTokens ?? DEFAULT_SUMMARIZATION_MAX_TOKENS,
|
||||
previousSummary,
|
||||
);
|
||||
summaryText = r.summary;
|
||||
summaryGenerated = r.generated;
|
||||
costUsd = r.costUsd;
|
||||
} else {
|
||||
summaryText = buildFallbackSummary(regions.middle, previousSummary);
|
||||
}
|
||||
|
||||
// Step 4: replace middle with summary message.
|
||||
const summaryMessage = {
|
||||
role: 'system',
|
||||
content: `[Compressed: ${regions.middle.length} earlier messages]\n${summaryText}`,
|
||||
};
|
||||
const newMessages = [...regions.head, summaryMessage, ...regions.tail];
|
||||
|
||||
const tokensAfter = estimateMessagesTokens(newMessages, stringEstimator);
|
||||
|
||||
if (config.onCompressionEvent) {
|
||||
config.onCompressionEvent({
|
||||
type: 'messages_compressed',
|
||||
before_tokens: tokensBefore,
|
||||
after_tokens: tokensAfter,
|
||||
messages_before: messages.length,
|
||||
messages_after: newMessages.length,
|
||||
cost_usd: costUsd,
|
||||
summary_generated: summaryGenerated,
|
||||
});
|
||||
}
|
||||
|
||||
return { messages: newMessages, summary: summaryText, compressed: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given messages array would trigger compression under
|
||||
* the given config. Exposed for callers that want to check without invoking.
|
||||
*/
|
||||
export function shouldCompressMessages(
|
||||
messages: ReadonlyArray<{ role: string; content: string }>,
|
||||
config: MessagesContextManagerConfig,
|
||||
): boolean {
|
||||
const stringEstimator = config.estimateTokensFn ?? estimateStringTokens;
|
||||
const threshold = config.threshold ?? DEFAULT_THRESHOLD;
|
||||
return estimateMessagesTokens(messages, stringEstimator) > config.budgetTokens * threshold;
|
||||
}
|
||||
464
packages/agent/src/long-task/recovery.ts
Normal file
464
packages/agent/src/long-task/recovery.ts
Normal file
@@ -0,0 +1,464 @@
|
||||
/**
|
||||
* Long-task recovery — Phase 3.2 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §3.2)
|
||||
*
|
||||
* Wraps a single step's lifecycle (initial attempt + retries + fallback chain)
|
||||
* around a CheckpointStore so the agent loop (Phase 3.4) can be crash-resilient
|
||||
* and tool-failure-resilient. The state machine determines which step_index to
|
||||
* attempt next based on the latest checkpoint:
|
||||
*
|
||||
* no prior checkpoint → fresh start (step 0, prior_state=undefined)
|
||||
* prior checkpoint with error→ retry that step_index using step N-1's state
|
||||
* prior checkpoint clean → advance to step_index + 1 using prior as state
|
||||
*
|
||||
* Within a single run() call:
|
||||
* 1. Run stepFn with attempt=1.
|
||||
* 2. If it throws: classify as 'retryable' or 'terminal'.
|
||||
* - 'retryable' AND attempts left → exponential backoff + jitter, retry.
|
||||
* - 'terminal' OR attempts exhausted → break to fallback chain.
|
||||
* 3. Walk fallback chain (one shot per fallback tool). Each gets stepFn with
|
||||
* fallback_tool set in StepFnInput.
|
||||
* 4. On success at any stage: save success state to store, return {ok:true}.
|
||||
* 5. If all retries + fallbacks fail: save error state, return {ok:false}.
|
||||
*
|
||||
* REPLAY DETERMINISM (per PM brief acceptance gate):
|
||||
* Recovery DECISIONS (retry / fallback / exhaust / success) are pure functions
|
||||
* of stepFn outcomes — given identical stepFn behavior, the resulting state
|
||||
* chain is byte-identical. BACKOFF durations are non-deterministic by default
|
||||
* (Math.random for jitter) but can be made deterministic via opts.rng for
|
||||
* replay verification. Tests verify decisions, not durations.
|
||||
*
|
||||
* PERSISTENCE CADENCE:
|
||||
* Saves happen only on (a) success and (b) exhaustion. Mid-retry failures do
|
||||
* NOT save partial states — keeps the post-crash semantics simple: a fresh
|
||||
* process either sees a clean prior step (and advances) or an exhaustion
|
||||
* error checkpoint (and retries with full budget).
|
||||
*/
|
||||
|
||||
import {
|
||||
CheckpointStore,
|
||||
CHECKPOINT_SCHEMA_VERSION,
|
||||
nextStateFrom,
|
||||
type CheckpointStepState,
|
||||
type Decision,
|
||||
} from './checkpoint.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Public types
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type RecoveryEventType =
|
||||
| 'fresh_start'
|
||||
| 'resume_clean'
|
||||
| 'resume_from_error'
|
||||
| 'attempt'
|
||||
| 'retry'
|
||||
| 'fallback_attempted'
|
||||
| 'success'
|
||||
| 'exhausted';
|
||||
|
||||
export interface RecoveryEvent {
|
||||
type: RecoveryEventType;
|
||||
step_index: number;
|
||||
/** 1-indexed attempt number (only set for attempt / retry / fallback / success). */
|
||||
attempt?: number;
|
||||
/** Set on retry / exhausted / resume_from_error. */
|
||||
error?: string;
|
||||
/** Set on fallback_attempted / success-via-fallback. */
|
||||
fallback_tool?: string;
|
||||
/** Set on retry: the duration the runner is about to sleep. */
|
||||
backoff_ms?: number;
|
||||
}
|
||||
|
||||
export interface StepFnInput {
|
||||
/**
|
||||
* State at the END of the previous step. Undefined for fresh start (step 0)
|
||||
* and for resume_from_error at step 0.
|
||||
*/
|
||||
prior_state: CheckpointStepState | undefined;
|
||||
/** Step index this attempt is for (0 for the first action). */
|
||||
step_index: number;
|
||||
/** 1-indexed attempt count (1 = first try, 2+ = retries, fallback ≥ retries+1). */
|
||||
attempt: number;
|
||||
/** Set when called via the fallback chain — the tool to use. */
|
||||
fallback_tool?: string;
|
||||
}
|
||||
|
||||
export interface StepFnResult {
|
||||
/** Outputs the step produced (becomes step_output in the saved state). */
|
||||
output: Record<string, unknown>;
|
||||
/** Optional cost recorded with the step. */
|
||||
cost_usd?: number;
|
||||
/** Optional latency recorded with the step. */
|
||||
latency_ms?: number;
|
||||
/** Decisions made during this step — appended to decision_history. */
|
||||
decisions?: readonly Decision[];
|
||||
/** Retrieval cache additions — merged into retrieval_cache. */
|
||||
retrieval_additions?: Record<string, unknown>;
|
||||
/** Optional appended context (concatenated to accumulated_context). */
|
||||
appended_context?: string;
|
||||
}
|
||||
|
||||
export type StepFn = (input: StepFnInput) => Promise<StepFnResult>;
|
||||
|
||||
export type ErrorClass = 'retryable' | 'terminal';
|
||||
export type ErrorClassifier = (err: unknown) => ErrorClass;
|
||||
|
||||
export interface RecoveryRunnerOptions {
|
||||
/** Per-task checkpoint store. Recovery uses store.taskId for state lifecycle. */
|
||||
store: CheckpointStore;
|
||||
/** Run-level UUID written into every saved checkpoint. */
|
||||
runId: string;
|
||||
|
||||
/** Default 3 retries (1 initial + 3 retries = 4 same-tool attempts). */
|
||||
maxRetriesPerStep?: number;
|
||||
/** Default 1000ms initial backoff (retry 1). */
|
||||
baseBackoffMs?: number;
|
||||
/** Default 30000ms cap on a single backoff. */
|
||||
maxBackoffMs?: number;
|
||||
/** Default 0.25 (±25% jitter on top of exponential). 0 = no jitter (deterministic). */
|
||||
jitterFactor?: number;
|
||||
|
||||
/** Optional ordered tool fallback chain. One shot per tool. */
|
||||
toolFallbackChain?: readonly string[];
|
||||
|
||||
/** Optional error classifier. Default: every error is 'retryable'. */
|
||||
classifyError?: ErrorClassifier;
|
||||
|
||||
/** Optional injectable RNG for jitter. Default: Math.random. */
|
||||
rng?: () => number;
|
||||
/** Optional injectable sleep for tests. Default: native setTimeout. */
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
|
||||
/** Optional event sink. Receives every state-machine event in order. */
|
||||
onRecoveryEvent?: (event: RecoveryEvent) => void;
|
||||
}
|
||||
|
||||
export interface RecoveryRunResult {
|
||||
/** Final state for this run (success state OR error-tagged exhaustion state). */
|
||||
state: CheckpointStepState;
|
||||
/** True iff the step ultimately succeeded (no error in final state). */
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
export interface RecoveryRunOptions {
|
||||
/** Label for this step (becomes step_action in the saved state). */
|
||||
stepAction?: string;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Internal helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_MAX_RETRIES = 3;
|
||||
const DEFAULT_BASE_BACKOFF_MS = 1000;
|
||||
const DEFAULT_MAX_BACKOFF_MS = 30000;
|
||||
const DEFAULT_JITTER_FACTOR = 0.25;
|
||||
const ALWAYS_RETRYABLE: ErrorClassifier = () => 'retryable';
|
||||
|
||||
function defaultSleep(ms: number): Promise<void> {
|
||||
return new Promise<void>((resolve) => {
|
||||
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function errMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
if (typeof err === 'string') return err;
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
return String(err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute exponential backoff with optional ±jitterFactor jitter.
|
||||
* retryNumber is 1-indexed (retry 1 = baseBackoffMs, retry 2 = 2*base, ...).
|
||||
*/
|
||||
function computeBackoff(opts: {
|
||||
retryNumber: number;
|
||||
baseMs: number;
|
||||
maxMs: number;
|
||||
jitterFactor: number;
|
||||
rng: () => number;
|
||||
}): number {
|
||||
const exponential = Math.min(
|
||||
opts.baseMs * 2 ** (opts.retryNumber - 1),
|
||||
opts.maxMs,
|
||||
);
|
||||
if (opts.jitterFactor === 0) return exponential;
|
||||
// jitter range: ±jitterFactor * exponential
|
||||
const jitter = (opts.rng() * 2 - 1) * opts.jitterFactor * exponential;
|
||||
return Math.max(0, exponential + jitter);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// RecoveryRunner
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export class RecoveryRunner {
|
||||
private readonly store: CheckpointStore;
|
||||
private readonly runId: string;
|
||||
private readonly maxRetries: number;
|
||||
private readonly baseBackoffMs: number;
|
||||
private readonly maxBackoffMs: number;
|
||||
private readonly jitterFactor: number;
|
||||
private readonly fallbackChain: readonly string[];
|
||||
private readonly classify: ErrorClassifier;
|
||||
private readonly rng: () => number;
|
||||
private readonly sleep: (ms: number) => Promise<void>;
|
||||
private readonly emit: (event: RecoveryEvent) => void;
|
||||
|
||||
constructor(opts: RecoveryRunnerOptions) {
|
||||
if (!opts.store) throw new Error('RecoveryRunner: store is required');
|
||||
if (!opts.runId) throw new Error('RecoveryRunner: runId is required');
|
||||
const maxRetries = opts.maxRetriesPerStep ?? DEFAULT_MAX_RETRIES;
|
||||
if (!Number.isInteger(maxRetries) || maxRetries < 0) {
|
||||
throw new Error(`RecoveryRunner: maxRetriesPerStep must be a non-negative integer (got ${String(maxRetries)})`);
|
||||
}
|
||||
const baseMs = opts.baseBackoffMs ?? DEFAULT_BASE_BACKOFF_MS;
|
||||
const maxMs = opts.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
|
||||
if (baseMs < 0 || maxMs < 0 || maxMs < baseMs) {
|
||||
throw new Error(`RecoveryRunner: invalid backoff bounds (base=${baseMs}, max=${maxMs})`);
|
||||
}
|
||||
const jitter = opts.jitterFactor ?? DEFAULT_JITTER_FACTOR;
|
||||
if (jitter < 0 || jitter > 1) {
|
||||
throw new Error(`RecoveryRunner: jitterFactor must be in [0, 1] (got ${jitter})`);
|
||||
}
|
||||
|
||||
this.store = opts.store;
|
||||
this.runId = opts.runId;
|
||||
this.maxRetries = maxRetries;
|
||||
this.baseBackoffMs = baseMs;
|
||||
this.maxBackoffMs = maxMs;
|
||||
this.jitterFactor = jitter;
|
||||
this.fallbackChain = opts.toolFallbackChain ?? [];
|
||||
this.classify = opts.classifyError ?? ALWAYS_RETRYABLE;
|
||||
this.rng = opts.rng ?? Math.random;
|
||||
this.sleep = opts.sleep ?? defaultSleep;
|
||||
this.emit = opts.onRecoveryEvent ?? (() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs ONE step's lifecycle (initial attempt → retries → fallback chain →
|
||||
* success-or-exhaustion-save). The state machine inspects the store's
|
||||
* latest checkpoint to decide which step to attempt and what prior state
|
||||
* to feed into stepFn.
|
||||
*/
|
||||
async run(stepFn: StepFn, opts: RecoveryRunOptions = {}): Promise<RecoveryRunResult> {
|
||||
const stepAction = opts.stepAction ?? 'step';
|
||||
const { priorState, stepIndexToAttempt } = await this._resolveStartingPoint();
|
||||
|
||||
// Phase 1: same-tool attempts (1 initial + maxRetries retries).
|
||||
let lastError: string | undefined;
|
||||
for (let attempt = 1; attempt <= this.maxRetries + 1; attempt += 1) {
|
||||
if (attempt > 1) {
|
||||
const backoffMs = computeBackoff({
|
||||
retryNumber: attempt - 1,
|
||||
baseMs: this.baseBackoffMs,
|
||||
maxMs: this.maxBackoffMs,
|
||||
jitterFactor: this.jitterFactor,
|
||||
rng: this.rng,
|
||||
});
|
||||
this.emit({ type: 'retry', step_index: stepIndexToAttempt, attempt, backoff_ms: backoffMs, error: lastError });
|
||||
await this.sleep(backoffMs);
|
||||
}
|
||||
this.emit({ type: 'attempt', step_index: stepIndexToAttempt, attempt });
|
||||
try {
|
||||
const result = await stepFn({
|
||||
prior_state: priorState,
|
||||
step_index: stepIndexToAttempt,
|
||||
attempt,
|
||||
});
|
||||
const successState = this._buildSuccessState({
|
||||
priorState,
|
||||
stepIndex: stepIndexToAttempt,
|
||||
stepAction,
|
||||
attempt,
|
||||
fallbackTool: undefined,
|
||||
result,
|
||||
});
|
||||
await this.store.save(successState);
|
||||
this.emit({ type: 'success', step_index: stepIndexToAttempt, attempt });
|
||||
return { state: successState, ok: true };
|
||||
} catch (err) {
|
||||
lastError = errMessage(err);
|
||||
const cls = this.classify(err);
|
||||
if (cls === 'terminal') break; // skip remaining same-tool retries
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: fallback chain (one shot per fallback tool).
|
||||
let nextAttemptCounter = this.maxRetries + 2;
|
||||
for (const fallbackTool of this.fallbackChain) {
|
||||
const attempt = nextAttemptCounter;
|
||||
nextAttemptCounter += 1;
|
||||
this.emit({
|
||||
type: 'fallback_attempted',
|
||||
step_index: stepIndexToAttempt,
|
||||
attempt,
|
||||
fallback_tool: fallbackTool,
|
||||
error: lastError,
|
||||
});
|
||||
try {
|
||||
const result = await stepFn({
|
||||
prior_state: priorState,
|
||||
step_index: stepIndexToAttempt,
|
||||
attempt,
|
||||
fallback_tool: fallbackTool,
|
||||
});
|
||||
const successState = this._buildSuccessState({
|
||||
priorState,
|
||||
stepIndex: stepIndexToAttempt,
|
||||
stepAction,
|
||||
attempt,
|
||||
fallbackTool,
|
||||
result,
|
||||
});
|
||||
await this.store.save(successState);
|
||||
this.emit({
|
||||
type: 'success',
|
||||
step_index: stepIndexToAttempt,
|
||||
attempt,
|
||||
fallback_tool: fallbackTool,
|
||||
});
|
||||
return { state: successState, ok: true };
|
||||
} catch (err) {
|
||||
lastError = errMessage(err);
|
||||
// continue to next fallback
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 3: exhausted. Persist final error state.
|
||||
this.emit({ type: 'exhausted', step_index: stepIndexToAttempt, error: lastError });
|
||||
const errorState = this._buildErrorState({
|
||||
priorState,
|
||||
stepIndex: stepIndexToAttempt,
|
||||
stepAction,
|
||||
error: lastError ?? 'unknown error',
|
||||
});
|
||||
await this.store.save(errorState);
|
||||
return { state: errorState, ok: false };
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Private: state machine + state builders
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
private async _resolveStartingPoint(): Promise<{
|
||||
priorState: CheckpointStepState | undefined;
|
||||
stepIndexToAttempt: number;
|
||||
}> {
|
||||
const latest = await this.store.loadLatest();
|
||||
|
||||
if (!latest) {
|
||||
this.emit({ type: 'fresh_start', step_index: 0 });
|
||||
return { priorState: undefined, stepIndexToAttempt: 0 };
|
||||
}
|
||||
|
||||
if (latest.error) {
|
||||
this.emit({
|
||||
type: 'resume_from_error',
|
||||
step_index: latest.step_index,
|
||||
error: latest.error,
|
||||
});
|
||||
if (latest.step_index === 0) {
|
||||
return { priorState: undefined, stepIndexToAttempt: 0 };
|
||||
}
|
||||
const prior = await this.store.load(latest.step_index - 1);
|
||||
if (!prior) {
|
||||
throw new Error(
|
||||
`RecoveryRunner: step ${latest.step_index} has error but step ${latest.step_index - 1} is missing — checkpoint chain corrupt`,
|
||||
);
|
||||
}
|
||||
if (prior.error) {
|
||||
throw new Error(
|
||||
`RecoveryRunner: prior step ${prior.step_index} also has error — checkpoint chain not clean`,
|
||||
);
|
||||
}
|
||||
return { priorState: prior, stepIndexToAttempt: latest.step_index };
|
||||
}
|
||||
|
||||
this.emit({ type: 'resume_clean', step_index: latest.step_index + 1 });
|
||||
return { priorState: latest, stepIndexToAttempt: latest.step_index + 1 };
|
||||
}
|
||||
|
||||
private _buildSuccessState(args: {
|
||||
priorState: CheckpointStepState | undefined;
|
||||
stepIndex: number;
|
||||
stepAction: string;
|
||||
attempt: number;
|
||||
fallbackTool: string | undefined;
|
||||
result: StepFnResult;
|
||||
}): CheckpointStepState {
|
||||
const stepInput: Record<string, unknown> = { attempt: args.attempt };
|
||||
if (args.fallbackTool !== undefined) stepInput.fallback_tool = args.fallbackTool;
|
||||
|
||||
if (args.priorState) {
|
||||
// Resume path: nextStateFrom advances by 1, but we want to OVERWRITE
|
||||
// args.stepIndex (which equals priorState.step_index + 1 for clean
|
||||
// resume, or priorState.step_index + 1 for error-resume after a
|
||||
// rewind to step N-1). nextStateFrom does exactly this.
|
||||
return nextStateFrom(args.priorState, {
|
||||
step_action: args.stepAction,
|
||||
step_input: stepInput,
|
||||
step_output: args.result.output,
|
||||
decisions: args.result.decisions,
|
||||
retrieval_additions: args.result.retrieval_additions,
|
||||
appended_context: args.result.appended_context,
|
||||
cost_usd: args.result.cost_usd,
|
||||
latency_ms: args.result.latency_ms,
|
||||
});
|
||||
}
|
||||
|
||||
// Fresh path (step 0 with no prior). Build directly — nextStateFrom
|
||||
// would assume a prior at step -1 which doesn't exist.
|
||||
return {
|
||||
schema_version: CHECKPOINT_SCHEMA_VERSION,
|
||||
task_id: this.store.taskId,
|
||||
run_id: this.runId,
|
||||
step_index: args.stepIndex,
|
||||
timestamp_iso: new Date().toISOString(),
|
||||
step_action: args.stepAction,
|
||||
step_input: { ...stepInput },
|
||||
step_output: { ...args.result.output },
|
||||
accumulated_context: args.result.appended_context ?? '',
|
||||
retrieval_cache: { ...(args.result.retrieval_additions ?? {}) },
|
||||
decision_history: [...(args.result.decisions ?? [])],
|
||||
cost_usd: args.result.cost_usd,
|
||||
latency_ms: args.result.latency_ms,
|
||||
};
|
||||
}
|
||||
|
||||
private _buildErrorState(args: {
|
||||
priorState: CheckpointStepState | undefined;
|
||||
stepIndex: number;
|
||||
stepAction: string;
|
||||
error: string;
|
||||
}): CheckpointStepState {
|
||||
if (args.priorState) {
|
||||
return nextStateFrom(args.priorState, {
|
||||
step_action: args.stepAction,
|
||||
step_input: { exhausted: true },
|
||||
step_output: {},
|
||||
error: args.error,
|
||||
});
|
||||
}
|
||||
return {
|
||||
schema_version: CHECKPOINT_SCHEMA_VERSION,
|
||||
task_id: this.store.taskId,
|
||||
run_id: this.runId,
|
||||
step_index: args.stepIndex,
|
||||
timestamp_iso: new Date().toISOString(),
|
||||
step_action: args.stepAction,
|
||||
step_input: { exhausted: true },
|
||||
step_output: {},
|
||||
accumulated_context: '',
|
||||
retrieval_cache: {},
|
||||
decision_history: [],
|
||||
error: args.error,
|
||||
};
|
||||
}
|
||||
}
|
||||
599
packages/agent/src/long-task/report.ts
Normal file
599
packages/agent/src/long-task/report.ts
Normal file
@@ -0,0 +1,599 @@
|
||||
/**
|
||||
* Reporting module — Phase 4.2 of agent-fix sprint
|
||||
* (per decisions/2026-04-26-agent-fix-sprint-plan.md §4.2)
|
||||
*
|
||||
* Consumes per-instance agent prediction records and emits four artifacts:
|
||||
* - summary.json machine-readable per-run aggregate
|
||||
* - summary.md human-readable per-run aggregate
|
||||
* - predictions.jsonl one record per (model, cell, instance)
|
||||
* - failures.jsonl one record per failed instance with Phase 4.1 classification
|
||||
*
|
||||
* Required metrics per (model, cell):
|
||||
* - accuracy_raw + accuracy_normalized (substring-match against gold)
|
||||
* - abstention rate
|
||||
* - thinking_leakage rate, format_violation rate
|
||||
* - avg output chars (raw + normalized), avg latency, total cost
|
||||
* - 10-bucket failure distribution from Phase 4.1 classifier
|
||||
* - Wilson 95% CI half-width
|
||||
* - bootstrap 95% CI when N >= 30
|
||||
* - win/loss vs baseline cell (in pp)
|
||||
*
|
||||
* Required overall metrics:
|
||||
* - cross-model comparison matrix (cells × models)
|
||||
* - best model per cell
|
||||
* - regression notes (cell drop ≥ 5 pp vs baseline)
|
||||
* - aggregate failure distribution
|
||||
*
|
||||
* NOTE on stats inlining: Wilson CI + cluster-free bootstrap are inlined here
|
||||
* rather than imported from benchmarks/harness/src/stats/ because @waggle/agent
|
||||
* cannot import from benchmarks/harness/ (wrong dependency direction). Future
|
||||
* Sprint 12 cleanup should hoist the shared math to @waggle/shared and
|
||||
* deduplicate.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
classifyFailure,
|
||||
failureDistribution,
|
||||
FAILURE_CATEGORIES,
|
||||
type FailureCategory,
|
||||
type FailureClassification,
|
||||
type ClassifierOptions,
|
||||
} from './failure-classify.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Public types
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface AgentPredictionRecord {
|
||||
model: string;
|
||||
cell: string;
|
||||
instance_id: string;
|
||||
question?: string;
|
||||
gold_answer: string | readonly string[];
|
||||
output_raw: string;
|
||||
output_normalized: string;
|
||||
judge_verdict?: 'correct' | 'incorrect' | 'null' | string;
|
||||
cost_usd?: number;
|
||||
latency_ms?: number;
|
||||
tokens_in?: number;
|
||||
tokens_out?: number;
|
||||
upstream_error?: string;
|
||||
/** Optional pre-classified failure; classifier is skipped if set. */
|
||||
failure_classification?: FailureClassification;
|
||||
}
|
||||
|
||||
export interface ReportOptions {
|
||||
run_id: string;
|
||||
/** Baseline cell name for win/loss comparison. Default 'raw'. */
|
||||
baseline_cell?: string;
|
||||
/** Regression threshold in percentage points. Default 5.0 pp drop vs baseline. */
|
||||
regression_threshold_pp?: number;
|
||||
/** Bootstrap iterations for CI. Default 1000. Only computed when N >= 30. */
|
||||
bootstrap_iterations?: number;
|
||||
/** Optional LLM judge for the failure classifier on low-confidence cases. */
|
||||
failure_classifier?: ClassifierOptions;
|
||||
/** Optional seeded RNG for replay-deterministic bootstrap. Default Math.random. */
|
||||
rng?: () => number;
|
||||
}
|
||||
|
||||
export interface CellMetrics {
|
||||
n: number;
|
||||
accuracy_raw: number;
|
||||
accuracy_normalized: number;
|
||||
abstention_rate: number;
|
||||
thinking_leakage_rate: number;
|
||||
format_violation_rate: number;
|
||||
avg_output_chars_raw: number;
|
||||
avg_output_chars_normalized: number;
|
||||
avg_latency_ms: number;
|
||||
total_cost_usd: number;
|
||||
failure_distribution: Record<FailureCategory, number>;
|
||||
wilson_95ci_lower: number;
|
||||
wilson_95ci_upper: number;
|
||||
wilson_95ci_half_width_pp: number;
|
||||
bootstrap_95ci_lower?: number;
|
||||
bootstrap_95ci_upper?: number;
|
||||
/** Delta vs baseline cell in percentage points (accuracy_normalized). */
|
||||
win_loss_vs_baseline_pp?: number;
|
||||
}
|
||||
|
||||
export interface ModelComparisonMatrix {
|
||||
rows: readonly string[];
|
||||
columns: readonly string[];
|
||||
/** values[rowIdx][colIdx] = accuracy_normalized for (cell=rows[r], model=cols[c]); NaN if missing. */
|
||||
values: readonly (readonly number[])[];
|
||||
}
|
||||
|
||||
export interface RunSummary {
|
||||
report_version: 1;
|
||||
run_id: string;
|
||||
generated_at_iso: string;
|
||||
n_instances: number;
|
||||
models: readonly string[];
|
||||
cells: readonly string[];
|
||||
baseline_cell?: string;
|
||||
per_model_per_cell: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>;
|
||||
cross_model_matrix: ModelComparisonMatrix;
|
||||
best_per_cell: Readonly<Record<string, string>>;
|
||||
regression_notes: readonly string[];
|
||||
aggregate_failure_distribution: Readonly<Record<FailureCategory, number>>;
|
||||
}
|
||||
|
||||
export interface ReportArtifacts {
|
||||
summary: RunSummary;
|
||||
summary_md: string;
|
||||
predictions_jsonl: string;
|
||||
failures_jsonl: string;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Inline stats helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const Z_95 = 1.959964;
|
||||
|
||||
interface WilsonResult {
|
||||
point_estimate: number;
|
||||
lower: number;
|
||||
upper: number;
|
||||
half_width: number;
|
||||
}
|
||||
|
||||
/** Wilson 95% binomial CI. Returns { lower, upper, half_width } in [0, 1]. */
|
||||
function wilsonCI(successes: number, trials: number): WilsonResult {
|
||||
if (trials === 0) return { point_estimate: 0, lower: 0, upper: 0, half_width: 0 };
|
||||
const p = successes / trials;
|
||||
const z = Z_95;
|
||||
const z2 = z * z;
|
||||
const denom = 1 + z2 / trials;
|
||||
const center = (p + z2 / (2 * trials)) / denom;
|
||||
const half = (z * Math.sqrt(p * (1 - p) / trials + z2 / (4 * trials * trials))) / denom;
|
||||
return {
|
||||
point_estimate: p,
|
||||
lower: Math.max(0, center - half),
|
||||
upper: Math.min(1, center + half),
|
||||
half_width: half,
|
||||
};
|
||||
}
|
||||
|
||||
/** Non-clustered bootstrap 95% CI on a binary correctness array. */
|
||||
function bootstrapCI(correctness: readonly number[], iterations: number, rng: () => number): { lower: number; upper: number } {
|
||||
const n = correctness.length;
|
||||
if (n === 0) return { lower: 0, upper: 0 };
|
||||
const samples: number[] = [];
|
||||
for (let i = 0; i < iterations; i += 1) {
|
||||
let sum = 0;
|
||||
for (let j = 0; j < n; j += 1) {
|
||||
const idx = Math.floor(rng() * n);
|
||||
sum += correctness[idx]!;
|
||||
}
|
||||
samples.push(sum / n);
|
||||
}
|
||||
samples.sort((a, b) => a - b);
|
||||
const lo = samples[Math.floor(0.025 * iterations)] ?? 0;
|
||||
const hi = samples[Math.floor(0.975 * iterations)] ?? 0;
|
||||
return { lower: lo, upper: hi };
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Substring-match accuracy (v6 rule)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function asGolds(g: string | readonly string[]): readonly string[] {
|
||||
return typeof g === 'string' ? [g] : g;
|
||||
}
|
||||
|
||||
function scoreAccuracy(output: string, gold: string | readonly string[]): number {
|
||||
const golds = asGolds(gold);
|
||||
if (golds.length === 0) return 0;
|
||||
const lower = output.toLowerCase();
|
||||
for (const g of golds) {
|
||||
if (g && lower.includes(g.toLowerCase())) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Per-cell metric computation
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const UNKNOWN_OUTPUTS = new Set([
|
||||
'unknown', 'i don\'t know', 'i do not know', 'cannot determine',
|
||||
'cannot be determined', 'not specified', 'no information', 'insufficient information',
|
||||
'no answer', 'n/a', 'na', 'none',
|
||||
]);
|
||||
|
||||
const THINKING_RE = /<\s*\/?\s*think(?:ing)?\s*[>\s]/i;
|
||||
|
||||
function isAbstention(output: string): boolean {
|
||||
const lc = output.toLowerCase().trim();
|
||||
return UNKNOWN_OUTPUTS.has(lc);
|
||||
}
|
||||
|
||||
function hasThinkingLeakage(rawOutput: string): boolean {
|
||||
return THINKING_RE.test(rawOutput);
|
||||
}
|
||||
|
||||
async function computeCellMetrics(
|
||||
records: readonly AgentPredictionRecord[],
|
||||
opts: ReportOptions,
|
||||
): Promise<{ metrics: CellMetrics; failures: ReadonlyArray<AgentPredictionRecord & { classification: FailureClassification }> }> {
|
||||
const n = records.length;
|
||||
if (n === 0) {
|
||||
const zeros = Object.fromEntries(FAILURE_CATEGORIES.map(c => [c, 0])) as Record<FailureCategory, number>;
|
||||
return {
|
||||
metrics: {
|
||||
n: 0, accuracy_raw: 0, accuracy_normalized: 0,
|
||||
abstention_rate: 0, thinking_leakage_rate: 0, format_violation_rate: 0,
|
||||
avg_output_chars_raw: 0, avg_output_chars_normalized: 0,
|
||||
avg_latency_ms: 0, total_cost_usd: 0,
|
||||
failure_distribution: zeros,
|
||||
wilson_95ci_lower: 0, wilson_95ci_upper: 0, wilson_95ci_half_width_pp: 0,
|
||||
},
|
||||
failures: [],
|
||||
};
|
||||
}
|
||||
|
||||
const correctnessRaw: number[] = [];
|
||||
const correctnessNormalized: number[] = [];
|
||||
let abstentions = 0;
|
||||
let thinkingLeaks = 0;
|
||||
let totalRawChars = 0;
|
||||
let totalNormalizedChars = 0;
|
||||
let totalLatency = 0;
|
||||
let totalCost = 0;
|
||||
|
||||
for (const r of records) {
|
||||
correctnessRaw.push(scoreAccuracy(r.output_raw, r.gold_answer));
|
||||
correctnessNormalized.push(scoreAccuracy(r.output_normalized, r.gold_answer));
|
||||
if (isAbstention(r.output_normalized)) abstentions += 1;
|
||||
if (hasThinkingLeakage(r.output_raw)) thinkingLeaks += 1;
|
||||
totalRawChars += r.output_raw.length;
|
||||
totalNormalizedChars += r.output_normalized.length;
|
||||
totalLatency += r.latency_ms ?? 0;
|
||||
totalCost += r.cost_usd ?? 0;
|
||||
}
|
||||
|
||||
// Classify failures (where accuracy_normalized = 0).
|
||||
const classified: Array<AgentPredictionRecord & { classification: FailureClassification }> = [];
|
||||
for (let i = 0; i < records.length; i += 1) {
|
||||
const r = records[i]!;
|
||||
if (correctnessNormalized[i] === 1) continue; // not a failure
|
||||
const classification = r.failure_classification ?? await classifyFailure({
|
||||
model_output: r.output_normalized,
|
||||
gold_answer: r.gold_answer,
|
||||
question: r.question,
|
||||
upstream_error: r.upstream_error,
|
||||
}, opts.failure_classifier ?? {});
|
||||
classified.push({ ...r, classification });
|
||||
}
|
||||
|
||||
const fdist = failureDistribution(classified.map(c => c.classification));
|
||||
const formatViolations = fdist.format_violation;
|
||||
|
||||
const successesNorm = correctnessNormalized.reduce((a, b) => a + b, 0);
|
||||
const wilson = wilsonCI(successesNorm, n);
|
||||
|
||||
let bootstrap: { lower: number; upper: number } | undefined;
|
||||
const bootIter = opts.bootstrap_iterations ?? 1000;
|
||||
if (n >= 30) {
|
||||
bootstrap = bootstrapCI(correctnessNormalized, bootIter, opts.rng ?? Math.random);
|
||||
}
|
||||
|
||||
const metrics: CellMetrics = {
|
||||
n,
|
||||
accuracy_raw: correctnessRaw.reduce((a, b) => a + b, 0) / n,
|
||||
accuracy_normalized: successesNorm / n,
|
||||
abstention_rate: abstentions / n,
|
||||
thinking_leakage_rate: thinkingLeaks / n,
|
||||
format_violation_rate: formatViolations / n,
|
||||
avg_output_chars_raw: totalRawChars / n,
|
||||
avg_output_chars_normalized: totalNormalizedChars / n,
|
||||
avg_latency_ms: totalLatency / n,
|
||||
total_cost_usd: totalCost,
|
||||
failure_distribution: fdist,
|
||||
wilson_95ci_lower: wilson.lower,
|
||||
wilson_95ci_upper: wilson.upper,
|
||||
wilson_95ci_half_width_pp: wilson.half_width * 100,
|
||||
...(bootstrap ? { bootstrap_95ci_lower: bootstrap.lower, bootstrap_95ci_upper: bootstrap.upper } : {}),
|
||||
};
|
||||
|
||||
return { metrics, failures: classified };
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Cross-model matrix + best per cell + regression notes
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function buildCrossModelMatrix(
|
||||
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
|
||||
models: readonly string[],
|
||||
cells: readonly string[],
|
||||
): ModelComparisonMatrix {
|
||||
const values: number[][] = cells.map(cell => models.map(model => {
|
||||
const m = perMperC[model]?.[cell];
|
||||
return m ? m.accuracy_normalized : Number.NaN;
|
||||
}));
|
||||
return { rows: cells, columns: models, values };
|
||||
}
|
||||
|
||||
function bestModelPerCell(
|
||||
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
|
||||
models: readonly string[],
|
||||
cells: readonly string[],
|
||||
): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const cell of cells) {
|
||||
let bestModel = '';
|
||||
let bestScore = -Infinity;
|
||||
for (const m of models) {
|
||||
const score = perMperC[m]?.[cell]?.accuracy_normalized ?? -Infinity;
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestModel = m;
|
||||
}
|
||||
}
|
||||
if (bestModel) out[cell] = bestModel;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function regressionNotes(
|
||||
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
|
||||
models: readonly string[],
|
||||
cells: readonly string[],
|
||||
baselineCell: string,
|
||||
thresholdPp: number,
|
||||
): string[] {
|
||||
const notes: string[] = [];
|
||||
for (const m of models) {
|
||||
const baseAcc = perMperC[m]?.[baselineCell]?.accuracy_normalized;
|
||||
if (baseAcc === undefined) continue;
|
||||
for (const c of cells) {
|
||||
if (c === baselineCell) continue;
|
||||
const acc = perMperC[m]?.[c]?.accuracy_normalized;
|
||||
if (acc === undefined) continue;
|
||||
const deltaPp = (acc - baseAcc) * 100;
|
||||
if (-deltaPp >= thresholdPp) {
|
||||
notes.push(
|
||||
`Regression: ${m}/${c} (acc=${(acc * 100).toFixed(1)}%) is ${(-deltaPp).toFixed(1)}pp below baseline ${baselineCell} (acc=${(baseAcc * 100).toFixed(1)}%)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return notes;
|
||||
}
|
||||
|
||||
function aggregateFailureDistribution(
|
||||
perMperC: Readonly<Record<string, Readonly<Record<string, CellMetrics>>>>,
|
||||
): Record<FailureCategory, number> {
|
||||
const agg = Object.fromEntries(FAILURE_CATEGORIES.map(c => [c, 0])) as Record<FailureCategory, number>;
|
||||
for (const modelMap of Object.values(perMperC)) {
|
||||
for (const cm of Object.values(modelMap)) {
|
||||
for (const c of FAILURE_CATEGORIES) {
|
||||
agg[c] += cm.failure_distribution[c];
|
||||
}
|
||||
}
|
||||
}
|
||||
return agg;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Markdown rendering
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function renderSummaryMd(summary: RunSummary): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Run Report — ${summary.run_id}`);
|
||||
lines.push('');
|
||||
lines.push(`Generated: ${summary.generated_at_iso}`);
|
||||
lines.push(`N instances: ${summary.n_instances}`);
|
||||
lines.push(`Models: ${summary.models.join(', ')}`);
|
||||
lines.push(`Cells: ${summary.cells.join(', ')}`);
|
||||
if (summary.baseline_cell) lines.push(`Baseline cell: \`${summary.baseline_cell}\``);
|
||||
lines.push('');
|
||||
|
||||
// Per-model tables.
|
||||
for (const m of summary.models) {
|
||||
lines.push(`## ${m}`);
|
||||
lines.push('');
|
||||
lines.push('| cell | n | acc_raw | acc_norm | 95% CI | abstain | think_leak | fmt_viol | avg_lat_ms | cost_$ | win/loss vs baseline |');
|
||||
lines.push('|---|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const c of summary.cells) {
|
||||
const cm = summary.per_model_per_cell[m]?.[c];
|
||||
if (!cm) {
|
||||
lines.push(`| ${c} | – | – | – | – | – | – | – | – | – | – |`);
|
||||
continue;
|
||||
}
|
||||
const wlPp = cm.win_loss_vs_baseline_pp;
|
||||
const wlStr = wlPp === undefined ? '–' : `${wlPp >= 0 ? '+' : ''}${wlPp.toFixed(1)}pp`;
|
||||
lines.push([
|
||||
c,
|
||||
cm.n,
|
||||
(cm.accuracy_raw * 100).toFixed(1) + '%',
|
||||
(cm.accuracy_normalized * 100).toFixed(1) + '%',
|
||||
`[${(cm.wilson_95ci_lower * 100).toFixed(1)}%, ${(cm.wilson_95ci_upper * 100).toFixed(1)}%]`,
|
||||
(cm.abstention_rate * 100).toFixed(1) + '%',
|
||||
(cm.thinking_leakage_rate * 100).toFixed(1) + '%',
|
||||
(cm.format_violation_rate * 100).toFixed(1) + '%',
|
||||
cm.avg_latency_ms.toFixed(0),
|
||||
cm.total_cost_usd.toFixed(4),
|
||||
wlStr,
|
||||
].map(v => String(v)).join(' | ').replace(/^/, '| ').replace(/$/, ' |'));
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Cross-model comparison matrix.
|
||||
lines.push('## Cross-model comparison (accuracy_normalized %)');
|
||||
lines.push('');
|
||||
lines.push(`| cell | ${summary.cross_model_matrix.columns.join(' | ')} | best |`);
|
||||
lines.push(`| --- | ${summary.cross_model_matrix.columns.map(() => '---').join(' | ')} | --- |`);
|
||||
for (let r = 0; r < summary.cross_model_matrix.rows.length; r += 1) {
|
||||
const cell = summary.cross_model_matrix.rows[r]!;
|
||||
const row = summary.cross_model_matrix.values[r]!.map(v => Number.isNaN(v) ? '–' : (v * 100).toFixed(1) + '%');
|
||||
const best = summary.best_per_cell[cell] ?? '–';
|
||||
lines.push(`| ${cell} | ${row.join(' | ')} | **${best}** |`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Aggregate failure distribution.
|
||||
lines.push('## Aggregate failure distribution (across all model × cell)');
|
||||
lines.push('');
|
||||
lines.push('| category | count |');
|
||||
lines.push('|---|---|');
|
||||
for (const c of FAILURE_CATEGORIES) {
|
||||
lines.push(`| ${c} | ${summary.aggregate_failure_distribution[c]} |`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Regression notes.
|
||||
if (summary.regression_notes.length > 0) {
|
||||
lines.push('## Regression notes');
|
||||
lines.push('');
|
||||
for (const n of summary.regression_notes) lines.push(`- ${n}`);
|
||||
lines.push('');
|
||||
} else {
|
||||
lines.push('## Regression notes');
|
||||
lines.push('');
|
||||
lines.push('_(none — no cell drops ≥ regression threshold vs baseline)_');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Main entry point
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function generateReport(
|
||||
records: readonly AgentPredictionRecord[],
|
||||
opts: ReportOptions,
|
||||
): Promise<ReportArtifacts> {
|
||||
const baseline = opts.baseline_cell ?? 'raw';
|
||||
const regressionThreshold = opts.regression_threshold_pp ?? 5.0;
|
||||
|
||||
const models = Array.from(new Set(records.map(r => r.model))).sort();
|
||||
const cells = Array.from(new Set(records.map(r => r.cell))).sort();
|
||||
|
||||
const perMperC: Record<string, Record<string, CellMetrics>> = {};
|
||||
const allFailures: Array<AgentPredictionRecord & { classification: FailureClassification }> = [];
|
||||
|
||||
for (const m of models) {
|
||||
perMperC[m] = {};
|
||||
for (const c of cells) {
|
||||
const subset = records.filter(r => r.model === m && r.cell === c);
|
||||
if (subset.length === 0) continue;
|
||||
const { metrics, failures } = await computeCellMetrics(subset, opts);
|
||||
perMperC[m]![c] = metrics;
|
||||
for (const f of failures) allFailures.push(f);
|
||||
}
|
||||
// Compute win/loss vs baseline.
|
||||
const baseAcc = perMperC[m]?.[baseline]?.accuracy_normalized;
|
||||
if (baseAcc !== undefined) {
|
||||
for (const c of cells) {
|
||||
if (c === baseline) continue;
|
||||
const cm = perMperC[m]?.[c];
|
||||
if (!cm) continue;
|
||||
perMperC[m]![c] = { ...cm, win_loss_vs_baseline_pp: (cm.accuracy_normalized - baseAcc) * 100 };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const summary: RunSummary = {
|
||||
report_version: 1,
|
||||
run_id: opts.run_id,
|
||||
generated_at_iso: new Date().toISOString(),
|
||||
n_instances: records.length,
|
||||
models,
|
||||
cells,
|
||||
baseline_cell: baseline,
|
||||
per_model_per_cell: perMperC,
|
||||
cross_model_matrix: buildCrossModelMatrix(perMperC, models, cells),
|
||||
best_per_cell: bestModelPerCell(perMperC, models, cells),
|
||||
regression_notes: regressionNotes(perMperC, models, cells, baseline, regressionThreshold),
|
||||
aggregate_failure_distribution: aggregateFailureDistribution(perMperC),
|
||||
};
|
||||
|
||||
const summary_md = renderSummaryMd(summary);
|
||||
const predictions_jsonl = records.map(r => JSON.stringify(r)).join('\n');
|
||||
const failures_jsonl = allFailures.map(f => JSON.stringify(f)).join('\n');
|
||||
|
||||
return { summary, summary_md, predictions_jsonl, failures_jsonl };
|
||||
}
|
||||
|
||||
export interface WrittenReportPaths {
|
||||
summary_json_path: string;
|
||||
summary_md_path: string;
|
||||
predictions_jsonl_path: string;
|
||||
failures_jsonl_path: string;
|
||||
}
|
||||
|
||||
export async function writeReportToDisk(
|
||||
records: readonly AgentPredictionRecord[],
|
||||
opts: ReportOptions & { outputDir: string },
|
||||
): Promise<WrittenReportPaths> {
|
||||
const artifacts = await generateReport(records, opts);
|
||||
await fsp.mkdir(opts.outputDir, { recursive: true });
|
||||
const paths: WrittenReportPaths = {
|
||||
summary_json_path: path.join(opts.outputDir, 'summary.json'),
|
||||
summary_md_path: path.join(opts.outputDir, 'summary.md'),
|
||||
predictions_jsonl_path: path.join(opts.outputDir, 'predictions.jsonl'),
|
||||
failures_jsonl_path: path.join(opts.outputDir, 'failures.jsonl'),
|
||||
};
|
||||
await fsp.writeFile(paths.summary_json_path, JSON.stringify(artifacts.summary, null, 2), 'utf-8');
|
||||
await fsp.writeFile(paths.summary_md_path, artifacts.summary_md, 'utf-8');
|
||||
await fsp.writeFile(paths.predictions_jsonl_path, artifacts.predictions_jsonl + (artifacts.predictions_jsonl ? '\n' : ''), 'utf-8');
|
||||
await fsp.writeFile(paths.failures_jsonl_path, artifacts.failures_jsonl + (artifacts.failures_jsonl ? '\n' : ''), 'utf-8');
|
||||
return paths;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Pilot-record adapter
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PilotJsonlRecord {
|
||||
instance_id: string;
|
||||
question: string;
|
||||
gold_answer: string | string[];
|
||||
subject: {
|
||||
content: string;
|
||||
in?: number;
|
||||
out?: number;
|
||||
cost?: number;
|
||||
latency_ms?: number;
|
||||
error?: string | null;
|
||||
};
|
||||
trio?: { majority?: string };
|
||||
self_judge?: { verdict?: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a pilot-format JSONL record into the canonical AgentPredictionRecord.
|
||||
* Caller supplies the (model, cell) tuple — pilot records don't carry those
|
||||
* fields explicitly (they live in the filename / external context).
|
||||
*/
|
||||
export function fromPilotRecord(
|
||||
record: PilotJsonlRecord,
|
||||
opts: { model: string; cell: string },
|
||||
): AgentPredictionRecord {
|
||||
return {
|
||||
model: opts.model,
|
||||
cell: opts.cell,
|
||||
instance_id: record.instance_id,
|
||||
question: record.question,
|
||||
gold_answer: record.gold_answer,
|
||||
output_raw: record.subject.content,
|
||||
output_normalized: record.subject.content, // pilot records don't preserve a separate normalized form
|
||||
judge_verdict: record.trio?.majority,
|
||||
cost_usd: record.subject.cost,
|
||||
latency_ms: record.subject.latency_ms,
|
||||
tokens_in: record.subject.in,
|
||||
tokens_out: record.subject.out,
|
||||
upstream_error: record.subject.error ?? undefined,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user