moving

packages/agent/tests/long-task-recovery.test.ts

@@ -0,0 +1,820 @@
+/**
+ * Tests for long-task/recovery.ts (Phase 3.2 of agent-fix sprint).
+ *
+ * Coverage targets the PM brief acceptance gates:
+ *   - State machine: fresh_start / resume_clean / resume_from_error transitions
+ *   - Retry logic: maxRetriesPerStep semantics, classifyError, success-on-retry
+ *   - Backoff: exponential schedule, max-cap, jitter range, jitterFactor=0
+ *   - Fallback chain: walks in order, one shot each, success builds correct state
+ *   - Persistence: saves only on success/exhaustion, not between retries
+ *   - Events: ordering + payload contents per type
+ *   - Replay determinism: same stepFn outcomes → identical state chain
+ *   - Crash-resilience: kill mid-step → fresh runner → resume → identical final
+ *   - Edge cases: maxRetries=0, empty fallback, terminal-error skip, chain corrupt
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+
+import {
+  CheckpointStore,
+  CHECKPOINT_SCHEMA_VERSION,
+  type CheckpointStepState,
+  type Decision,
+} from '../src/long-task/checkpoint.js';
+import {
+  RecoveryRunner,
+  type RecoveryEvent,
+  type StepFn,
+  type StepFnResult,
+  type RecoveryRunnerOptions,
+} from '../src/long-task/recovery.js';
+
+// ─────────────────────────────────────────────────────────────────────────
+// Fixtures
+// ─────────────────────────────────────────────────────────────────────────
+
+let tmpRoot: string;
+
+beforeEach(async () => {
+  tmpRoot = await fsp.mkdtemp(path.join(os.tmpdir(), 'waggle-recovery-'));
+});
+
+afterEach(async () => {
+  await fsp.rm(tmpRoot, { recursive: true, force: true });
+});
+
+function makeStore(taskId = 'task-x'): CheckpointStore {
+  return new CheckpointStore({ rootDir: tmpRoot, taskId });
+}
+
+interface RunnerHandle {
+  store: CheckpointStore;
+  runner: RecoveryRunner;
+  events: RecoveryEvent[];
+  sleeps: number[];
+}
+
+function makeRunner(overrides: Partial<RecoveryRunnerOptions> = {}): RunnerHandle {
+  const events: RecoveryEvent[] = [];
+  const sleeps: number[] = [];
+  const store = overrides.store ?? makeStore();
+  const runner = new RecoveryRunner({
+    store,
+    runId: 'run-test',
+    onRecoveryEvent: (e) => events.push(e),
+    sleep: (ms) => {
+      sleeps.push(ms);
+      return Promise.resolve();
+    },
+    jitterFactor: 0, // deterministic by default in tests
+    ...overrides,
+  });
+  return { store, runner, events, sleeps };
+}
+
+function makeStepFn(impl: StepFn): { fn: StepFn; calls: number; inputs: Parameters<StepFn>[0][] } {
+  const inputs: Parameters<StepFn>[0][] = [];
+  let calls = 0;
+  const fn: StepFn = async (input) => {
+    calls += 1;
+    inputs.push(input);
+    return impl(input);
+  };
+  return {
+    get fn() { return fn; },
+    get calls() { return calls; },
+    get inputs() { return inputs; },
+  } as { fn: StepFn; calls: number; inputs: Parameters<StepFn>[0][] };
+}
+
+const okResult = (output: Record<string, unknown> = { ok: true }, extras: Partial<StepFnResult> = {}): StepFnResult => ({
+  output,
+  ...extras,
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Constructor validation
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — constructor', () => {
+  it('rejects missing store', () => {
+    expect(() => new RecoveryRunner({ store: undefined as unknown as CheckpointStore, runId: 'r' }))
+      .toThrow(/store is required/);
+  });
+
+  it('rejects missing runId', () => {
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: '' }))
+      .toThrow(/runId is required/);
+  });
+
+  it('rejects negative maxRetriesPerStep', () => {
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: 'r', maxRetriesPerStep: -1 }))
+      .toThrow(/non-negative integer/);
+  });
+
+  it('rejects non-integer maxRetriesPerStep', () => {
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: 'r', maxRetriesPerStep: 1.5 }))
+      .toThrow(/non-negative integer/);
+  });
+
+  it('rejects jitterFactor outside [0, 1]', () => {
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: 'r', jitterFactor: -0.1 })).toThrow();
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: 'r', jitterFactor: 1.5 })).toThrow();
+  });
+
+  it('rejects maxBackoffMs < baseBackoffMs', () => {
+    expect(() => new RecoveryRunner({ store: makeStore(), runId: 'r', baseBackoffMs: 5000, maxBackoffMs: 1000 }))
+      .toThrow(/invalid backoff bounds/);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// State machine — fresh_start
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — state machine: fresh_start', () => {
+  it('emits fresh_start when no prior checkpoint exists', async () => {
+    const { runner, events } = makeRunner();
+    const step = makeStepFn(async () => okResult({ x: 1 }));
+    await runner.run(step.fn, { stepAction: 'first' });
+    expect(events[0]).toEqual({ type: 'fresh_start', step_index: 0 });
+  });
+
+  it('passes prior_state=undefined to stepFn on fresh start', async () => {
+    const { runner } = makeRunner();
+    const step = makeStepFn(async () => okResult({ x: 1 }));
+    await runner.run(step.fn);
+    expect(step.inputs[0]?.prior_state).toBeUndefined();
+    expect(step.inputs[0]?.step_index).toBe(0);
+    expect(step.inputs[0]?.attempt).toBe(1);
+  });
+
+  it('saves step 0 with empty persistent fields on fresh start success', async () => {
+    const { runner, store } = makeRunner();
+    const step = makeStepFn(async () => okResult({ done: true }));
+    const result = await runner.run(step.fn, { stepAction: 'first' });
+    expect(result.ok).toBe(true);
+    expect(result.state.step_index).toBe(0);
+    expect(result.state.step_action).toBe('first');
+    expect(result.state.accumulated_context).toBe('');
+    expect(result.state.retrieval_cache).toEqual({});
+    expect(result.state.decision_history).toEqual([]);
+    expect(result.state.task_id).toBe('task-x');
+    expect(result.state.run_id).toBe('run-test');
+    const loaded = await store.load(0);
+    expect(loaded).toEqual(result.state);
+  });
+
+  it('captures appended_context / retrieval_additions / decisions on fresh start', async () => {
+    const { runner } = makeRunner();
+    const decisions: readonly Decision[] = [{ step_index: 0, decision: 'go top-3' }];
+    const step = makeStepFn(async () =>
+      okResult({ docs: ['a', 'b'] }, {
+        appended_context: 'context-piece',
+        retrieval_additions: { 'q:hello': ['a'] },
+        decisions,
+      }),
+    );
+    const result = await runner.run(step.fn);
+    expect(result.state.accumulated_context).toBe('context-piece');
+    expect(result.state.retrieval_cache).toEqual({ 'q:hello': ['a'] });
+    expect(result.state.decision_history).toEqual(decisions);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// State machine — resume_clean
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — state machine: resume_clean', () => {
+  it('emits resume_clean when latest is clean', async () => {
+    const { runner, events } = makeRunner();
+    // Run step 0 first.
+    await runner.run(makeStepFn(async () => okResult({ a: 1 })).fn);
+    events.length = 0;
+    // Run again — should resume cleanly to step 1.
+    await runner.run(makeStepFn(async () => okResult({ b: 2 })).fn);
+    expect(events[0]).toEqual({ type: 'resume_clean', step_index: 1 });
+  });
+
+  it('passes the previous state as prior_state', async () => {
+    const { runner } = makeRunner();
+    await runner.run(makeStepFn(async () => okResult({ first: true })).fn);
+    const step2 = makeStepFn(async () => okResult({ second: true }));
+    await runner.run(step2.fn);
+    expect(step2.inputs[0]?.prior_state?.step_index).toBe(0);
+    expect(step2.inputs[0]?.prior_state?.step_output).toEqual({ first: true });
+    expect(step2.inputs[0]?.step_index).toBe(1);
+  });
+
+  it('carries forward accumulated_context across resume_clean', async () => {
+    const { runner } = makeRunner();
+    await runner.run(makeStepFn(async () => okResult({}, { appended_context: 'A' })).fn);
+    const result = await runner.run(makeStepFn(async () => okResult({}, { appended_context: 'B' })).fn);
+    expect(result.state.accumulated_context).toBe('AB');
+  });
+
+  it('merges retrieval_additions across resume_clean', async () => {
+    const { runner } = makeRunner();
+    await runner.run(makeStepFn(async () => okResult({}, { retrieval_additions: { x: 1 } })).fn);
+    const result = await runner.run(makeStepFn(async () => okResult({}, { retrieval_additions: { y: 2 } })).fn);
+    expect(result.state.retrieval_cache).toEqual({ x: 1, y: 2 });
+  });
+
+  it('appends decisions across resume_clean', async () => {
+    const { runner } = makeRunner();
+    await runner.run(makeStepFn(async () => okResult({}, { decisions: [{ step_index: 0, decision: 'first' }] })).fn);
+    const result = await runner.run(makeStepFn(async () => okResult({}, { decisions: [{ step_index: 1, decision: 'second' }] })).fn);
+    expect(result.state.decision_history.map(d => d.decision)).toEqual(['first', 'second']);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// State machine — resume_from_error
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — state machine: resume_from_error', () => {
+  it('emits resume_from_error when latest has error', async () => {
+    const { runner, events, store } = makeRunner({ maxRetriesPerStep: 0 });
+    // Step 0: clean.
+    await runner.run(makeStepFn(async () => okResult({ s: 0 })).fn);
+    // Step 1: exhaust.
+    await runner.run(makeStepFn(async () => { throw new Error('boom'); }).fn);
+    expect((await store.load(1))?.error).toBe('boom');
+
+    // Resume should see error and retry step 1.
+    events.length = 0;
+    await runner.run(makeStepFn(async () => okResult({ s: 1 })).fn);
+    expect(events[0]).toMatchObject({ type: 'resume_from_error', step_index: 1, error: 'boom' });
+  });
+
+  it('rewinds to step N-1 as prior when latest has error at step N>0', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 0 });
+    await runner.run(makeStepFn(async () => okResult({ s: 0 })).fn);
+    await runner.run(makeStepFn(async () => { throw new Error('boom'); }).fn);
+
+    const retryStep = makeStepFn(async () => okResult({ retried: true }));
+    await runner.run(retryStep.fn);
+    expect(retryStep.inputs[0]?.prior_state?.step_index).toBe(0);
+    expect(retryStep.inputs[0]?.prior_state?.step_output).toEqual({ s: 0 });
+    expect(retryStep.inputs[0]?.step_index).toBe(1);
+  });
+
+  it('uses prior_state=undefined when error is at step 0', async () => {
+    const { runner } = makeRunner({ maxRetriesPerStep: 0 });
+    await runner.run(makeStepFn(async () => { throw new Error('first'); }).fn);
+
+    const retryStep = makeStepFn(async () => okResult({ ok: true }));
+    await runner.run(retryStep.fn);
+    expect(retryStep.inputs[0]?.prior_state).toBeUndefined();
+    expect(retryStep.inputs[0]?.step_index).toBe(0);
+  });
+
+  it('overwrites the error checkpoint with the success state', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 0 });
+    await runner.run(makeStepFn(async () => { throw new Error('first'); }).fn);
+    expect((await store.load(0))?.error).toBe('first');
+
+    const result = await runner.run(makeStepFn(async () => okResult({ recovered: true })).fn);
+    expect(result.ok).toBe(true);
+    expect((await store.load(0))?.error).toBeUndefined();
+    expect((await store.load(0))?.step_output).toEqual({ recovered: true });
+  });
+
+  it('throws when checkpoint chain is corrupt (step N-1 missing)', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 0 });
+    // Manually plant a step-3 error file with no step 0/1/2.
+    await store.init();
+    const errFile: CheckpointStepState = {
+      schema_version: CHECKPOINT_SCHEMA_VERSION,
+      task_id: 'task-x',
+      run_id: 'run-test',
+      step_index: 3,
+      timestamp_iso: new Date().toISOString(),
+      step_action: 'planted',
+      step_input: {},
+      step_output: {},
+      accumulated_context: '',
+      retrieval_cache: {},
+      decision_history: [],
+      error: 'planted error',
+    };
+    await fsp.writeFile(path.join(store.directory, 'step-000003.json'), JSON.stringify(errFile));
+
+    await expect(runner.run(makeStepFn(async () => okResult({})).fn))
+      .rejects.toThrow(/checkpoint chain corrupt/);
+  });
+
+  it('throws when prior step also has error (chain not clean)', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 0 });
+    await store.init();
+    // Plant step 0 error + step 1 error.
+    const mkErr = (step_index: number, err: string): CheckpointStepState => ({
+      schema_version: CHECKPOINT_SCHEMA_VERSION,
+      task_id: 'task-x',
+      run_id: 'run-test',
+      step_index,
+      timestamp_iso: new Date().toISOString(),
+      step_action: 'planted',
+      step_input: {}, step_output: {},
+      accumulated_context: '', retrieval_cache: {}, decision_history: [],
+      error: err,
+    });
+    await fsp.writeFile(path.join(store.directory, 'step-000000.json'), JSON.stringify(mkErr(0, 'first')));
+    await fsp.writeFile(path.join(store.directory, 'step-000001.json'), JSON.stringify(mkErr(1, 'second')));
+
+    await expect(runner.run(makeStepFn(async () => okResult({})).fn))
+      .rejects.toThrow(/chain not clean/);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Retry logic
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — retry logic', () => {
+  it('first attempt success → no retry events', async () => {
+    const { runner, events } = makeRunner();
+    await runner.run(makeStepFn(async () => okResult({})).fn);
+    const types = events.map(e => e.type);
+    expect(types).toContain('fresh_start');
+    expect(types).toContain('attempt');
+    expect(types).toContain('success');
+    expect(types).not.toContain('retry');
+  });
+
+  it('first fail then success → emits retry then success', async () => {
+    let n = 0;
+    const { runner, events } = makeRunner();
+    await runner.run(async () => {
+      n += 1;
+      if (n === 1) throw new Error('flaky');
+      return okResult({ ok: true });
+    });
+    const types = events.map(e => e.type);
+    expect(types).toEqual(['fresh_start', 'attempt', 'retry', 'attempt', 'success']);
+    expect(events.find(e => e.type === 'retry')?.error).toBe('flaky');
+  });
+
+  it('all retries fail (no fallback) → exhausted', async () => {
+    const { runner, events } = makeRunner({ maxRetriesPerStep: 2 });
+    const result = await runner.run(async () => { throw new Error('always-fails'); });
+    expect(result.ok).toBe(false);
+    expect(result.state.error).toBe('always-fails');
+    const types = events.map(e => e.type);
+    expect(types).toEqual([
+      'fresh_start', 'attempt', 'retry', 'attempt', 'retry', 'attempt', 'exhausted',
+    ]);
+  });
+
+  it('default classifyError treats every error as retryable', async () => {
+    const { runner } = makeRunner({ maxRetriesPerStep: 2 });
+    const step = makeStepFn(async () => { throw new Error('weird type'); });
+    await runner.run(step.fn);
+    expect(step.calls).toBe(3); // 1 initial + 2 retries
+  });
+
+  it('classifyError=terminal skips remaining same-tool retries', async () => {
+    const step = makeStepFn(async () => { throw new Error('auth-401'); });
+    const { runner, events } = makeRunner({
+      maxRetriesPerStep: 5,
+      classifyError: (err) => (errMessage(err).includes('auth-401') ? 'terminal' : 'retryable'),
+    });
+    await runner.run(step.fn);
+    expect(step.calls).toBe(1); // immediate stop
+    expect(events.some(e => e.type === 'retry')).toBe(false);
+    expect(events.some(e => e.type === 'exhausted')).toBe(true);
+  });
+
+  it('maxRetriesPerStep=0 → exactly one same-tool attempt', async () => {
+    const step = makeStepFn(async () => { throw new Error('x'); });
+    const { runner } = makeRunner({ maxRetriesPerStep: 0 });
+    await runner.run(step.fn);
+    expect(step.calls).toBe(1);
+  });
+
+  it('maxRetriesPerStep=3 → 4 same-tool attempts', async () => {
+    const step = makeStepFn(async () => { throw new Error('x'); });
+    const { runner } = makeRunner({ maxRetriesPerStep: 3 });
+    await runner.run(step.fn);
+    expect(step.calls).toBe(4); // 1 initial + 3 retries
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Backoff schedule
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — backoff schedule', () => {
+  it('jitterFactor=0 → exact exponential schedule (1×, 2×, 4×)', async () => {
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 3,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 60000,
+      jitterFactor: 0,
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps).toEqual([1000, 2000, 4000]);
+  });
+
+  it('caps backoff at maxBackoffMs', async () => {
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 5,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 3000,
+      jitterFactor: 0,
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps).toEqual([1000, 2000, 3000, 3000, 3000]);
+  });
+
+  it('with jitter, all sleeps fall within ±jitterFactor of base exponential', async () => {
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 3,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 60000,
+      jitterFactor: 0.25,
+      rng: () => 0.5, // jitter weight 0 (exact midpoint of [-0.25, +0.25])
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps).toEqual([1000, 2000, 4000]); // rng=0.5 → jitter delta 0
+  });
+
+  it('with rng=1 (max positive jitter), each sleep = 1.25× exponential', async () => {
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 3,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 60000,
+      jitterFactor: 0.25,
+      rng: () => 1, // (1*2-1)*0.25 = +0.25
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps).toEqual([1250, 2500, 5000]);
+  });
+
+  it('with rng=0 (max negative jitter), each sleep = 0.75× exponential', async () => {
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 3,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 60000,
+      jitterFactor: 0.25,
+      rng: () => 0, // (0*2-1)*0.25 = -0.25
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps).toEqual([750, 1500, 3000]);
+  });
+
+  it('clamps negative jitter to 0', async () => {
+    // If jitterFactor is huge AND rng=0, exponential + jitter could go negative.
+    const { runner, sleeps } = makeRunner({
+      maxRetriesPerStep: 1,
+      baseBackoffMs: 1000,
+      maxBackoffMs: 60000,
+      jitterFactor: 1.0,
+      rng: () => 0, // jitter delta = -1000
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    expect(sleeps[0]).toBe(0);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Fallback chain
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — fallback chain', () => {
+  it('walks fallback chain in order after retries exhaust', async () => {
+    const seen: Array<{ attempt: number; fallback?: string }> = [];
+    const step: StepFn = async (input) => {
+      seen.push({ attempt: input.attempt, fallback: input.fallback_tool });
+      throw new Error('always');
+    };
+    const { runner } = makeRunner({
+      maxRetriesPerStep: 1,
+      toolFallbackChain: ['toolA', 'toolB', 'toolC'],
+    });
+    await runner.run(step);
+    // 1 initial + 1 retry = 2 same-tool calls, then 3 fallbacks
+    expect(seen.length).toBe(5);
+    expect(seen[0]?.fallback).toBeUndefined();
+    expect(seen[1]?.fallback).toBeUndefined();
+    expect(seen[2]?.fallback).toBe('toolA');
+    expect(seen[3]?.fallback).toBe('toolB');
+    expect(seen[4]?.fallback).toBe('toolC');
+  });
+
+  it('first fallback success → state has fallback_tool in step_input', async () => {
+    let n = 0;
+    const { runner } = makeRunner({
+      maxRetriesPerStep: 1,
+      toolFallbackChain: ['toolA', 'toolB'],
+    });
+    const result = await runner.run(async (input) => {
+      n += 1;
+      if (n <= 2) throw new Error('same-tool fails');
+      return okResult({ via_fallback: input.fallback_tool });
+    });
+    expect(result.ok).toBe(true);
+    expect(result.state.step_input).toMatchObject({ fallback_tool: 'toolA' });
+    expect(result.state.step_output).toEqual({ via_fallback: 'toolA' });
+  });
+
+  it('all fallbacks fail → exhausts and returns ok=false', async () => {
+    const { runner } = makeRunner({
+      maxRetriesPerStep: 0,
+      toolFallbackChain: ['a', 'b'],
+    });
+    const result = await runner.run(async () => { throw new Error('all-fail'); });
+    expect(result.ok).toBe(false);
+    expect(result.state.error).toBe('all-fail');
+  });
+
+  it('emits fallback_attempted event for each fallback tried', async () => {
+    const { runner, events } = makeRunner({
+      maxRetriesPerStep: 0,
+      toolFallbackChain: ['a', 'b'],
+    });
+    await runner.run(async () => { throw new Error('x'); });
+    const fallbackEvents = events.filter(e => e.type === 'fallback_attempted');
+    expect(fallbackEvents.map(e => e.fallback_tool)).toEqual(['a', 'b']);
+  });
+
+  it('terminal error skips retries but still walks fallback chain', async () => {
+    let n = 0;
+    const { runner } = makeRunner({
+      maxRetriesPerStep: 5,
+      toolFallbackChain: ['rescue'],
+      classifyError: () => 'terminal',
+    });
+    const result = await runner.run(async (input) => {
+      n += 1;
+      if (input.fallback_tool) return okResult({ via: input.fallback_tool });
+      throw new Error('terminal');
+    });
+    expect(result.ok).toBe(true);
+    expect(n).toBe(2); // 1 initial + 1 fallback (no retries)
+    expect(result.state.step_output).toEqual({ via: 'rescue' });
+  });
+
+  it('empty fallback chain behaves like no chain', async () => {
+    const { runner, events } = makeRunner({
+      maxRetriesPerStep: 0,
+      toolFallbackChain: [],
+    });
+    const result = await runner.run(async () => { throw new Error('x'); });
+    expect(result.ok).toBe(false);
+    expect(events.some(e => e.type === 'fallback_attempted')).toBe(false);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Persistence cadence
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — persistence cadence', () => {
+  it('does NOT save between retries (in-memory only)', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 2 });
+    let n = 0;
+    await runner.run(async () => {
+      n += 1;
+      if (n < 3) throw new Error('flaky');
+      return okResult({ s: 'done' });
+    });
+    // Only the final success should be persisted, no intermediate error files.
+    const indices = await store.listSteps();
+    expect(indices).toEqual([0]);
+    const loaded = await store.load(0);
+    expect(loaded?.error).toBeUndefined();
+    expect(loaded?.step_output).toEqual({ s: 'done' });
+  });
+
+  it('saves error state on exhaustion with error field set', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 1 });
+    await runner.run(async () => { throw new Error('hard-fail'); });
+    const loaded = await store.load(0);
+    expect(loaded?.error).toBe('hard-fail');
+    expect(loaded?.step_output).toEqual({});
+    expect(loaded?.step_input).toEqual({ exhausted: true });
+  });
+
+  it('saves success state with attempt count in step_input', async () => {
+    const { runner, store } = makeRunner({ maxRetriesPerStep: 5 });
+    let n = 0;
+    await runner.run(async () => {
+      n += 1;
+      if (n < 3) throw new Error('flaky');
+      return okResult({ ok: true });
+    });
+    const loaded = await store.load(0);
+    expect(loaded?.step_input).toEqual({ attempt: 3 });
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Replay determinism
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — replay determinism', () => {
+  it('two runs with identical stepFn outcomes produce identical state chains', async () => {
+    // Build a deterministic stepFn: succeed always with predictable output.
+    const deterministicStep: StepFn = async (input) => okResult({
+      step: input.step_index,
+      attempt: input.attempt,
+    });
+
+    // Run A
+    const a = makeRunner({ store: makeStore('task-a'), jitterFactor: 0 });
+    for (let i = 0; i < 4; i += 1) {
+      await a.runner.run(deterministicStep, { stepAction: `s${i}` });
+    }
+    const aStates = await Promise.all((await a.store.listSteps()).map(i => a.store.load(i)));
+
+    // Run B with a separate store but identical stepFn behavior.
+    const b = makeRunner({ store: makeStore('task-b'), jitterFactor: 0 });
+    for (let i = 0; i < 4; i += 1) {
+      await b.runner.run(deterministicStep, { stepAction: `s${i}` });
+    }
+    const bStates = await Promise.all((await b.store.listSteps()).map(i => b.store.load(i)));
+
+    // Compare modulo task_id + timestamp_iso (always different).
+    const strip = (s: CheckpointStepState | undefined) => {
+      if (!s) return s;
+      const { task_id: _t, timestamp_iso: _ts, ...rest } = s;
+      return rest;
+    };
+    expect(aStates.map(strip)).toEqual(bStates.map(strip));
+  });
+
+  it('flaky stepFn with deterministic failures yields identical decision events', async () => {
+    // Same flaky pattern: fail attempt 1 of step 0, fail attempts 1+2 of step 1, success otherwise.
+    const buildFlakyStep = (): StepFn => {
+      const counts = new Map<number, number>();
+      return async (input) => {
+        const c = (counts.get(input.step_index) ?? 0) + 1;
+        counts.set(input.step_index, c);
+        if (input.step_index === 0 && c === 1) throw new Error('s0-flaky');
+        if (input.step_index === 1 && c <= 2) throw new Error('s1-flaky');
+        return okResult({ step: input.step_index, attempt: c });
+      };
+    };
+
+    const a = makeRunner({ store: makeStore('det-a'), jitterFactor: 0 });
+    const stepA = buildFlakyStep();
+    await a.runner.run(stepA);
+    await a.runner.run(stepA);
+
+    const b = makeRunner({ store: makeStore('det-b'), jitterFactor: 0 });
+    const stepB = buildFlakyStep();
+    await b.runner.run(stepB);
+    await b.runner.run(stepB);
+
+    const stripBackoff = (e: RecoveryEvent) => ({ ...e, backoff_ms: undefined });
+    expect(a.events.map(e => ({ type: e.type, step_index: e.step_index, attempt: e.attempt })))
+      .toEqual(b.events.map(e => ({ type: e.type, step_index: e.step_index, attempt: e.attempt })));
+    // backoff_ms identical too because jitterFactor=0
+    expect(a.events.map(stripBackoff)).toEqual(b.events.map(stripBackoff));
+  });
+
+  it('seeded rng yields identical jitter sequences across runs', async () => {
+    // Mulberry32-style seeded PRNG (small inline implementation for the test).
+    const seed = (s: number) => () => {
+      s |= 0; s = (s + 0x6D2B79F5) | 0;
+      let t = Math.imul(s ^ (s >>> 15), 1 | s);
+      t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+      return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+
+    const runOnce = async () => {
+      const { runner, sleeps } = makeRunner({
+        maxRetriesPerStep: 3,
+        baseBackoffMs: 1000,
+        maxBackoffMs: 60000,
+        jitterFactor: 0.25,
+        rng: seed(42),
+      });
+      await runner.run(async () => { throw new Error('x'); });
+      return sleeps;
+    };
+    const a = await runOnce();
+    const b = await runOnce();
+    expect(a).toEqual(b);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Crash-resilience scenario
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — crash resilience', () => {
+  it('exhausted error state can be resumed by a fresh runner', async () => {
+    const taskId = `task-${crypto.randomBytes(4).toString('hex')}`;
+
+    // Process A: run step 0 success, step 1 exhausts.
+    {
+      const a = makeRunner({ store: makeStore(taskId), maxRetriesPerStep: 0 });
+      await a.runner.run(makeStepFn(async () => okResult({ done: 0 })).fn);
+      const result = await a.runner.run(makeStepFn(async () => { throw new Error('boom'); }).fn);
+      expect(result.ok).toBe(false);
+    }
+
+    // Process B (simulated): brand-new runner on the same store.
+    const b = makeRunner({ store: makeStore(taskId), maxRetriesPerStep: 0 });
+    const retryStep = makeStepFn(async () => okResult({ done: 1, recovered: true }));
+    const result = await b.runner.run(retryStep.fn);
+
+    // Should have resumed the failed step, not advanced to step 2.
+    expect(result.ok).toBe(true);
+    expect(result.state.step_index).toBe(1);
+    expect(result.state.step_output).toEqual({ done: 1, recovered: true });
+    expect(b.events[0]).toMatchObject({ type: 'resume_from_error', step_index: 1 });
+  });
+
+  it('clean checkpoint chain is fully resumable', async () => {
+    const taskId = `task-${crypto.randomBytes(4).toString('hex')}`;
+
+    // Process A: 3 clean steps.
+    {
+      const a = makeRunner({ store: makeStore(taskId) });
+      await a.runner.run(makeStepFn(async () => okResult({ s: 0 }, { appended_context: 'A' })).fn);
+      await a.runner.run(makeStepFn(async () => okResult({ s: 1 }, { appended_context: 'B' })).fn);
+      await a.runner.run(makeStepFn(async () => okResult({ s: 2 }, { appended_context: 'C' })).fn);
+    }
+
+    // Process B: should resume to step 3.
+    const b = makeRunner({ store: makeStore(taskId) });
+    const step3 = makeStepFn(async () => okResult({ s: 3 }, { appended_context: 'D' }));
+    const result = await b.runner.run(step3.fn);
+    expect(result.state.step_index).toBe(3);
+    expect(result.state.accumulated_context).toBe('ABCD');
+    expect(step3.inputs[0]?.prior_state?.step_index).toBe(2);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Event ordering
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('RecoveryRunner — event ordering', () => {
+  it('orders events: state-machine → attempt → (retry → attempt)* → success', async () => {
+    let n = 0;
+    const { runner, events } = makeRunner({ maxRetriesPerStep: 2 });
+    await runner.run(async () => {
+      n += 1;
+      if (n < 3) throw new Error('flaky');
+      return okResult({});
+    });
+    expect(events.map(e => e.type)).toEqual([
+      'fresh_start', 'attempt', 'retry', 'attempt', 'retry', 'attempt', 'success',
+    ]);
+  });
+
+  it('orders events on fallback success: ... → fallback_attempted → success', async () => {
+    let n = 0;
+    const { runner, events } = makeRunner({
+      maxRetriesPerStep: 1,
+      toolFallbackChain: ['rescue'],
+    });
+    await runner.run(async (input) => {
+      n += 1;
+      if (input.fallback_tool) return okResult({});
+      throw new Error('x');
+    });
+    expect(events.map(e => e.type)).toEqual([
+      'fresh_start', 'attempt', 'retry', 'attempt', 'fallback_attempted', 'success',
+    ]);
+  });
+
+  it('attempt event payload: step_index + attempt set, no fallback_tool', async () => {
+    const { runner, events } = makeRunner();
+    await runner.run(async () => okResult({}));
+    const attemptEvent = events.find(e => e.type === 'attempt');
+    expect(attemptEvent).toEqual({ type: 'attempt', step_index: 0, attempt: 1 });
+  });
+
+  it('retry event payload: backoff_ms + error + attempt set', async () => {
+    let n = 0;
+    const { runner, events } = makeRunner({ maxRetriesPerStep: 1, jitterFactor: 0, baseBackoffMs: 500 });
+    await runner.run(async () => {
+      n += 1;
+      if (n === 1) throw new Error('flaky');
+      return okResult({});
+    });
+    const retry = events.find(e => e.type === 'retry');
+    expect(retry?.backoff_ms).toBe(500);
+    expect(retry?.error).toBe('flaky');
+    expect(retry?.attempt).toBe(2);
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────
+// Helper for tests
+// ─────────────────────────────────────────────────────────────────────────
+
+function errMessage(err: unknown): string {
+  if (err instanceof Error) return err.message;
+  return String(err);
+}