waggle-os/benchmarks/gaia2/adapter.ts

#!/usr/bin/env tsx
/**
 * Sesija C Phase 3b — Gaia2 ARE narrow-proxy adapter (IMPLEMENTATION).
 *
 * Wraps a Gaia2 task (HuggingFace dataset record) for execution against
 * `runRetrievalAgentLoop` from `@waggle/agent`. Selects a prompt shape per
 * config (`claude-gen1-v1`, `qwen-thinking-gen1-v1`, baselines).
 * Captures per-run telemetry to JSONL.
 *
 * Phase 3a was types-only (commit b93db4c). Phase 3b adds real function
 * bodies. See Phase 3a SCOPE NOTE below for narrow-proxy intent — this
 * adapter is NOT a full Gaia2 evaluation; it is cost-projection
 * ground-truth + GEPA-variant smoke + type-fit verification.
 *
 * ─── Reference anchors (unchanged from Phase 3a) ─────────────────────
 *
 * - Brief: briefs/2026-04-30-cc-sesija-C-gaia2-setup-dry-verification.md §2.2
 * - HF dataset: huggingface.co/datasets/meta-agents-research-environments/gaia2 (CC-BY-4.0)
 * - ARE pinned SHA: 0330191f (MIT, 2026-04-20)
 * - Phase 2 smoke: benchmarks/gaia2/smoke-evidence.md §3+§4
 * - GEPA shapes: packages/agent/src/prompt-shapes/gepa-evolved/
 * - Retrieval loop: packages/agent/src/retrieval-agent-loop.ts::runRetrievalAgentLoop (line 516)
 * - PM ratifications: 1B (branch base), γ (probe), A (narrow-proxy scope)
 * - Faza 1 LlmCallFn pattern: benchmarks/gepa/scripts/faza-1/run-gen-1.ts:230
 */

import * as fs from 'node:fs';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import {
  runRetrievalAgentLoop,
  type AgentRunResult,
  type LlmCallFn,
  type RetrievalSearchFn,
  type MultiStepAgentRunConfig,
} from '../../packages/agent/src/retrieval-agent-loop.js';
import {
  registerShape,
  REGISTRY,
} from '../../packages/agent/src/prompt-shapes/index.js';
import type { PromptShape } from '../../packages/agent/src/prompt-shapes/types.js';

// ─── Gaia2 dataset record types (HuggingFace schema) ──────────────────

export interface Gaia2HfTask {
  readonly id: string;
  readonly scenario_id: string;
  readonly split: string;
  /**
   * HF top-level `category` field (mini/standard/agent2agent/noise/...
   * routing tag — observed at first probe dump 2026-04-30, NOT
   * documented in HF dataset card sample). Treat as informational; not
   * consumed by the narrow-proxy adapter.
   */
  readonly category?: string;
  /**
   * NOTE: HF stores `data` on disk as a SERIALIZED JSON STRING (not a
   * nested object — HF dataset card sample shows the post-parse form).
   * dump-tasks.py parses the string at dump time so on-disk JSONL we
   * load here has data as the structured object.
   *
   * SCHEMA NOTES (verified against first probe dump 2026-04-30,
   * `mini/validation/0741_*`):
   *   - `apps` is an ARRAY of `{app_state, class_name, name}` objects
   *     (12 apps for `mini` config; HF dataset card showed it as a
   *     dict for documentation simplicity).
   *   - `events` is an ARRAY of `{action, class_name, dependencies,
   *     event_id, event_relative_time, event_time, event_type,
   *     metadata}` objects. `event_type` ∈ {'USER', 'AGENT', 'ENV', ...}.
   *     The user instruction lives in events with `event_type==='USER'`,
   *     `action.function==='send_message_to_agent'`, with the message
   *     text in `action.args[].value` where `args[].name==='content'`.
   *   - `metadata.definition` carries scenario-level metadata
   *     (`duration`, `hints`, `scenario_id`, `start_time`, `tags`) but
   *     NOT the user-facing instruction. Description extraction routes
   *     through events. See `extractTaskDescription` impl.
   *   - `data` may also carry `augmentation` and `version` siblings,
   *     not consumed by the narrow-proxy adapter.
   */
  readonly data: {
    readonly metadata: {
      readonly definition: Record<string, unknown>;
    };
    /** Array per real schema; HF card showed dict for doc simplicity. */
    readonly apps: readonly Record<string, unknown>[] | Record<string, unknown>;
    readonly events: readonly Record<string, unknown>[];
  };
}

export interface Gaia2TaskDefinition {
  readonly description: string;
  readonly capability_tags?: readonly string[];
}

export interface Gaia2AppSnapshot {
  readonly class_name: string;
  readonly state?: Record<string, unknown>;
}

// ─── Shape selection types ────────────────────────────────────────────

export type ShapeAlias =
  | 'claude'
  | 'claude-gen1-v1'
  | 'claude-gen1-v2'
  | 'qwen-thinking'
  | 'qwen-thinking-gen1-v1'
  | 'qwen-thinking-gen1-v2'
  | 'gpt'
  | 'gpt-gen1-v1'
  | 'gpt-gen1-v2'
  | 'qwen-non-thinking'
  | 'qwen-non-thinking-gen1-v1'
  | 'qwen-non-thinking-gen1-v2'
  | 'generic-simple'
  | 'generic-simple-gen1-v1'
  | 'generic-simple-gen1-v2';

export const BRIEF_DRY_RUN_SHAPES: readonly ShapeAlias[] = [
  'claude',
  'claude-gen1-v1',
  'qwen-thinking',
  'qwen-thinking-gen1-v1',
] as const;

// ─── Failure-mode taxonomy ────────────────────────────────────────────

export type FailureMode =
  | 'loop_exhausted'
  | 'timeout'
  | 'parse_fail'
  | 'judge_failure'
  | 'cost_halt'
  | 'llm_error';

// ─── Adapter run record (one row per JSONL output) ────────────────────

export interface Gaia2RunRecord {
  readonly task_id: string;
  readonly scenario_id: string;
  readonly shape: ShapeAlias;
  readonly run_number: number;
  readonly started_at: string;
  readonly ended_at: string;
  readonly raw_response: string;
  readonly normalized_response: string;
  readonly prompt_shape_name: string;
  readonly total_tokens_in: number;
  readonly total_tokens_out: number;
  readonly total_cost_usd: number;
  readonly total_latency_ms: number;
  readonly steps_taken: number;
  readonly retrieval_calls: number;
  readonly loop_exhausted: boolean;
  readonly pass: boolean | null;
  readonly failure_mode: FailureMode | null;
  readonly errors: readonly string[];
  readonly narrow_proxy_run: boolean;
}

// ─── Adapter config ───────────────────────────────────────────────────

export interface Gaia2AdapterConfig {
  readonly task_count_dry_run: number;
  readonly shapes: readonly ShapeAlias[];
  readonly baseline_shape: ShapeAlias;
  readonly judge_methodology:
    | 'self-judge-dry-run'
    | 'trio-strict'
    | 'write-action-verifier'
    | 'narrow-proxy-no-judge';
  readonly cost_cap_usd: number;
  readonly halt_trigger_usd: number;
  readonly per_call_halt_usd: number;
  readonly probe_invocation_count: number;
  readonly output_dir_root: string;
  readonly hf_dataset: string;
  readonly hf_config: string;
  readonly hf_split: string;
}

// ─── Shape → model mapping ────────────────────────────────────────────
// gen1-v1 variants run on the SAME model as their baseline (the prompt
// shape is the manipulated variable, not the model). Aligns with Faza 1
// methodology where shape-evolution effect is measured on a fixed model.

const SHAPE_TO_MODEL: Record<ShapeAlias, string> = {
  claude: 'claude-opus-4-7',
  'claude-gen1-v1': 'claude-opus-4-7',
  'claude-gen1-v2': 'claude-opus-4-7',
  'qwen-thinking': 'qwen3.6-35b-a3b',
  'qwen-thinking-gen1-v1': 'qwen3.6-35b-a3b',
  'qwen-thinking-gen1-v2': 'qwen3.6-35b-a3b',
  'qwen-non-thinking': 'qwen3.6-35b-a3b',
  'qwen-non-thinking-gen1-v1': 'qwen3.6-35b-a3b',
  'qwen-non-thinking-gen1-v2': 'qwen3.6-35b-a3b',
  gpt: 'gpt-5.4',
  'gpt-gen1-v1': 'gpt-5.4',
  'gpt-gen1-v2': 'gpt-5.4',
  'generic-simple': 'claude-haiku-4-5',
  'generic-simple-gen1-v1': 'claude-haiku-4-5',
  'generic-simple-gen1-v2': 'claude-haiku-4-5',
};

// ─── GEPA-evolved shape registration (lazy, idempotent) ───────────────
// Same pattern as benchmarks/gepa/scripts/faza-1/run-gen-1.ts:405-409.
// Shape `name` field is the REGISTRY key (hyphen format). Keep the set
// minimal — we register only the shapes referenced in BRIEF_DRY_RUN_SHAPES.

const GEPA_EVOLVED_FILES: Readonly<Record<ShapeAlias, string>> = Object.freeze({
  'claude-gen1-v1': '../../packages/agent/src/prompt-shapes/gepa-evolved/claude-gen1-v1.js',
  'claude-gen1-v2': '../../packages/agent/src/prompt-shapes/gepa-evolved/claude-gen1-v2.js',
  'qwen-thinking-gen1-v1': '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-thinking-gen1-v1.js',
  'qwen-thinking-gen1-v2': '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-thinking-gen1-v2.js',
  'gpt-gen1-v1': '../../packages/agent/src/prompt-shapes/gepa-evolved/gpt-gen1-v1.js',
  'gpt-gen1-v2': '../../packages/agent/src/prompt-shapes/gepa-evolved/gpt-gen1-v2.js',
  'qwen-non-thinking-gen1-v1': '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-non-thinking-gen1-v1.js',
  'qwen-non-thinking-gen1-v2': '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-non-thinking-gen1-v2.js',
  'generic-simple-gen1-v1': '../../packages/agent/src/prompt-shapes/gepa-evolved/generic-simple-gen1-v1.js',
  'generic-simple-gen1-v2': '../../packages/agent/src/prompt-shapes/gepa-evolved/generic-simple-gen1-v2.js',
  // Baselines are pre-registered via prompt-shapes/index.ts; map to empty path.
  claude: '',
  'qwen-thinking': '',
  'qwen-non-thinking': '',
  gpt: '',
  'generic-simple': '',
});

const registeredShapes = new Set<ShapeAlias>();

export async function ensureShapeRegistered(shape: ShapeAlias): Promise<void> {
  if (registeredShapes.has(shape)) return;
  if (REGISTRY[shape] !== undefined) {
    registeredShapes.add(shape);
    return;
  }
  const filePath = GEPA_EVOLVED_FILES[shape];
  if (!filePath) {
    throw new Error(
      `ensureShapeRegistered: shape "${shape}" not in REGISTRY and not a GEPA-evolved file (baseline shapes are pre-registered via prompt-shapes/index.ts).`,
    );
  }
  // Resolve relative path to absolute file:// URL for ESM dynamic import on Windows.
  const adapterDir = path.dirname(new URL(import.meta.url).pathname.replace(/^\//, ''));
  const absPath = path.resolve(adapterDir, filePath.replace(/\.js$/, '.ts'));
  const mod: Record<string, unknown> = await import(pathToFileURL(absPath).href);
  // Find the exported PromptShape (single export per file convention,
  // mirrors run-gen-1.ts:215).
  const promptShape = Object.values(mod).find(
    (v): v is PromptShape =>
      typeof v === 'object' &&
      v !== null &&
      'name' in v &&
      'systemPrompt' in v &&
      'soloUserPrompt' in v,
  );
  if (!promptShape) {
    throw new Error(`ensureShapeRegistered: no PromptShape export in ${absPath}`);
  }
  registerShape(promptShape.name, promptShape);
  registeredShapes.add(shape);
}

// ─── Loaders ──────────────────────────────────────────────────────────

export function loadGaia2TasksFromJsonl(
  filepath: string,
  limit: number,
): readonly Gaia2HfTask[] {
  if (!fs.existsSync(filepath)) {
    throw new Error(
      `loadGaia2TasksFromJsonl: file not found: ${filepath}\n` +
        `Run benchmarks/gaia2/scripts/dump-tasks.py first to produce this file.`,
    );
  }
  const raw = fs.readFileSync(filepath, 'utf-8');
  const lines = raw.split(/\r?\n/).filter((l) => l.trim().length > 0);
  const tasks: Gaia2HfTask[] = [];
  for (let i = 0; i < Math.min(limit, lines.length); i++) {
    try {
      tasks.push(JSON.parse(lines[i]) as Gaia2HfTask);
    } catch (err: unknown) {
      const msg = err instanceof Error ? err.message : String(err);
      throw new Error(`loadGaia2TasksFromJsonl: parse error on line ${i + 1}: ${msg}`);
    }
  }
  return tasks;
}

// ─── Task → corpus + question extraction ──────────────────────────────

export function flattenAppStateToCorpus(
  task: Gaia2HfTask,
): readonly { id: string; content: string }[] {
  const docs: { id: string; content: string }[] = [];

  // `apps` is an ARRAY of {app_state, class_name, name} per real schema
  // (HF card sample showed dict — doc simplification). Handle both for
  // forward-compat.
  const appsList: readonly Record<string, unknown>[] = Array.isArray(task.data.apps)
    ? (task.data.apps as readonly Record<string, unknown>[])
    : (Object.values(task.data.apps) as readonly Record<string, unknown>[]);

  for (const app of appsList) {
    const className = typeof app.class_name === 'string' ? app.class_name : 'UnknownApp';
    const name = typeof app.name === 'string' ? app.name : className;
    // Real format uses `app_state` key for the app's initial-state JSON;
    // HF card sample showed `state`. Try both.
    const state = app.app_state ?? app.state ?? app;
    let stateJson: string;
    try {
      stateJson = JSON.stringify(state, null, 2);
    } catch {
      stateJson = '(unserializable app state)';
    }
    docs.push({
      id: `app:${name}`,
      content: `[${className}] ${name}\n${stateJson}`,
    });
  }

  // Task definition itself goes into the corpus so the agent can
  // retrieve task-related metadata (tags, hints, duration).
  let defJson: string;
  try {
    defJson = JSON.stringify(task.data.metadata.definition, null, 2);
  } catch {
    defJson = '(unserializable task definition)';
  }
  docs.push({
    id: 'task:definition',
    content: `[Task Definition]\n${defJson}`,
  });

  return docs;
}

/**
 * Extracts the user instruction from a Gaia2 task.
 *
 * Real schema: instruction lives in events with `event_type === 'USER'`,
 * `action.function === 'send_message_to_agent'`, with the text in
 * `action.args[]` where `args[i].name === 'content'`. A scenario can have
 * multiple USER events injected over time (multi-turn). For the narrow
 * proxy we concatenate them into one prompt block — the agent doesn't
 * see the time evolution but does see the cumulative ask.
 *
 * Falls back through:
 *   1. definition.description (HF card schema; not present in mini/validation
 *      first probe but may be present in other configs)
 *   2. definition.instruction / definition.task (defensive)
 *   3. concatenated user-message events (real schema, primary path)
 *   4. stringified definition + tags (last-resort, never empty question)
 */
export function extractTaskDescription(task: Gaia2HfTask): string {
  const def = task.data.metadata.definition as Record<string, unknown>;

  // Strategy 1: HF card sample fields (defensive).
  if (typeof def.description === 'string' && def.description.trim().length > 0) {
    return def.description;
  }
  if (typeof def.instruction === 'string' && def.instruction.trim().length > 0) {
    return def.instruction;
  }
  if (typeof def.task === 'string' && def.task.trim().length > 0) {
    return def.task;
  }

  // Strategy 2: extract from USER events (primary path for real schema).
  const events = task.data.events;
  const userMessages: string[] = [];
  for (const ev of events) {
    if (ev.event_type !== 'USER') continue;
    const action = ev.action as Record<string, unknown> | undefined;
    if (!action) continue;
    const args = action.args;
    if (Array.isArray(args)) {
      for (const arg of args) {
        const argObj = arg as Record<string, unknown>;
        const name = argObj.name;
        const value = argObj.value;
        if (
          typeof value === 'string' &&
          (name === 'content' || name === 'message' || name === 'text')
        ) {
          userMessages.push(value);
        }
      }
    } else if (typeof args === 'object' && args !== null) {
      const argsObj = args as Record<string, unknown>;
      const candidate = argsObj.content ?? argsObj.message ?? argsObj.text;
      if (typeof candidate === 'string' && candidate.trim().length > 0) {
        userMessages.push(candidate);
      }
    }
  }
  if (userMessages.length > 0) {
    return userMessages.length === 1
      ? userMessages[0]
      : userMessages
          .map((m, i) => `--- User message ${i + 1}/${userMessages.length} ---\n${m}`)
          .join('\n\n');
  }

  // Strategy 3: fallback (never empty question).
  const tags = Array.isArray(def.tags) ? (def.tags as readonly string[]).join(', ') : '';
  return `Gaia2 scenario ${task.id} (tags: ${tags || 'none'}). Definition: ${JSON.stringify(def).slice(0, 500)}`;
}

// ─── Simple in-memory FTS RetrievalSearchFn ──────────────────────────
// Substring-match scoring is intentionally minimal — the narrow-proxy
// adapter is NOT trying to optimize retrieval quality. It's verifying
// the adapter contract end-to-end. Phase 4 Docker run uses ARE-native
// app interactions, no synthetic search.

export function buildSimpleSearch(
  corpus: readonly { id: string; content: string }[],
): RetrievalSearchFn {
  const lowercased = corpus.map((doc) => ({
    id: doc.id,
    content: doc.content,
    lower: doc.content.toLowerCase(),
  }));

  return async (input) => {
    const tokens = input.query
      .toLowerCase()
      .split(/\s+/)
      .filter((t) => t.length > 2);

    const scored = lowercased.map((doc) => {
      let score = 0;
      for (const tok of tokens) {
        if (doc.lower.includes(tok)) score += 1;
      }
      return { id: doc.id, content: doc.content, score };
    });

    scored.sort((a, b) => b.score - a.score);
    const top = scored.slice(0, input.limit).filter((d) => d.score > 0);

    if (top.length === 0) {
      return { formattedResults: '(no results matched the query in the available app state)', resultCount: 0 };
    }

    const formatted = top.map((d) => `## ${d.id}\n${d.content}`).join('\n\n---\n\n');
    return { formattedResults: formatted, resultCount: top.length };
  };
}

// ─── Failure-mode classifier (post-hoc, from AgentRunResult) ─────────

function classifyFailure(
  result: AgentRunResult | null,
  errMessage: string | null,
): FailureMode | null {
  if (!result && !errMessage) return null;
  if (errMessage) {
    const lower = errMessage.toLowerCase();
    if (lower.includes('cost') || lower.includes('halt')) return 'cost_halt';
    if (lower.includes('parse')) return 'parse_fail';
    if (lower.includes('judge')) return 'judge_failure';
    if (lower.includes('timeout')) return 'timeout';
    return 'llm_error';
  }
  if (result?.loopExhausted) return 'loop_exhausted';
  return null;
}

// ─── Main: run one task with one shape ────────────────────────────────

export async function runGaia2TaskWithShape(
  task: Gaia2HfTask,
  shape: ShapeAlias,
  config: Gaia2AdapterConfig,
  llmCall: LlmCallFn,
  runNumber: number = 1,
): Promise<Gaia2RunRecord> {
  await ensureShapeRegistered(shape);

  const startedAt = new Date().toISOString();
  const startMs = Date.now();

  const question = extractTaskDescription(task);
  const corpus = flattenAppStateToCorpus(task);
  const search = buildSimpleSearch(corpus);
  const model = SHAPE_TO_MODEL[shape];

  let result: AgentRunResult | null = null;
  let errMessage: string | null = null;

  const runConfig: MultiStepAgentRunConfig = {
    modelAlias: model,
    persona:
      'You are an AI agent operating within a simulated environment with multiple applications. ' +
      'Use the search tool to retrieve relevant app state, then answer the user task. ' +
      'Be concise and decisive.',
    question,
    llmCall,
    search,
    promptShapeOverride: shape,
    perCallHaltUsd: config.per_call_halt_usd,
    perCellHaltUsd: config.per_call_halt_usd * 5,
    maxSteps: 5,
    maxRetrievalsPerStep: 8,
    contextTag: `gaia2-narrow-proxy:${task.id}:${shape}:run${runNumber}`,
  };

  try {
    result = await runRetrievalAgentLoop(runConfig);
  } catch (err: unknown) {
    errMessage = err instanceof Error ? err.message : String(err);
  }

  const endedAt = new Date().toISOString();
  const errors: string[] = [];
  if (errMessage) errors.push(errMessage);
  if (result?.errors) errors.push(...result.errors);

  return {
    task_id: task.id,
    scenario_id: task.scenario_id,
    shape,
    run_number: runNumber,
    started_at: startedAt,
    ended_at: endedAt,
    raw_response: result?.rawResponse ?? '',
    normalized_response: result?.normalizedResponse ?? '',
    prompt_shape_name: result?.promptShapeName ?? shape,
    total_tokens_in: result?.totalTokensIn ?? 0,
    total_tokens_out: result?.totalTokensOut ?? 0,
    total_cost_usd: result?.totalCostUsd ?? 0,
    total_latency_ms: result?.totalLatencyMs ?? Date.now() - startMs,
    steps_taken: result?.stepsTaken ?? 0,
    retrieval_calls: result?.retrievalCalls ?? 0,
    loop_exhausted: result?.loopExhausted ?? false,
    pass: null, // Phase 4 Docker + ARE write-action verifier wires this.
    failure_mode: classifyFailure(result, errMessage),
    errors,
    narrow_proxy_run: true,
  };
}

// ─── Driver: γ probe-first sweep ──────────────────────────────────────

export interface DryRunSweepResult {
  readonly probe_records: readonly Gaia2RunRecord[];
  readonly probe_cost_usd: number;
  readonly projected_total_usd: number;
  readonly halt_triggered: boolean;
  readonly full_sweep_records: readonly Gaia2RunRecord[];
  readonly final_total_cost_usd: number;
}

export async function runDryRunSweep(
  config: Gaia2AdapterConfig,
  llmCall: LlmCallFn,
  options: {
    tasksFile: string;
    /** If true, halt after probe regardless of projection (for Phase 3b sample). */
    haltAfterProbe?: boolean;
  },
): Promise<DryRunSweepResult> {
  const tasks = loadGaia2TasksFromJsonl(options.tasksFile, config.task_count_dry_run);

  if (tasks.length === 0) {
    throw new Error(`runDryRunSweep: no tasks loaded from ${options.tasksFile}`);
  }

  // Probe: round-robin invocations across (task, shape) pairs until
  // probe_invocation_count is reached.
  const probe_records: Gaia2RunRecord[] = [];
  let probe_cost_usd = 0;
  let invocationsRun = 0;

  outer: for (const task of tasks) {
    for (const shape of config.shapes) {
      if (invocationsRun >= config.probe_invocation_count) break outer;
      const rec = await runGaia2TaskWithShape(task, shape, config, llmCall, 1);
      probe_records.push(rec);
      probe_cost_usd += rec.total_cost_usd;
      invocationsRun++;
    }
  }

  // Project full sweep cost from probe.
  const totalInvocations = config.shapes.length * config.task_count_dry_run;
  const avgCostPerInvocation =
    probe_records.length > 0 ? probe_cost_usd / probe_records.length : 0;
  const projected_total_usd = avgCostPerInvocation * totalInvocations;
  const halt_triggered =
    options.haltAfterProbe === true || projected_total_usd > config.halt_trigger_usd;

  if (halt_triggered) {
    return {
      probe_records,
      probe_cost_usd,
      projected_total_usd,
      halt_triggered: true,
      full_sweep_records: probe_records,
      final_total_cost_usd: probe_cost_usd,
    };
  }

  // Continue full sweep. Skip the (task, shape) pairs already covered
  // by the probe.
  const fullRecords: Gaia2RunRecord[] = [...probe_records];
  let totalCost = probe_cost_usd;
  const probeKeys = new Set(probe_records.map((r) => `${r.task_id}::${r.shape}`));

  for (const task of tasks) {
    for (const shape of config.shapes) {
      const key = `${task.id}::${shape}`;
      if (probeKeys.has(key)) continue;
      // Cost-cap check before each invocation (defensive — per-call halt
      // also enforced via runConfig).
      if (totalCost >= config.cost_cap_usd) {
        return {
          probe_records,
          probe_cost_usd,
          projected_total_usd,
          halt_triggered: true,
          full_sweep_records: fullRecords,
          final_total_cost_usd: totalCost,
        };
      }
      const rec = await runGaia2TaskWithShape(task, shape, config, llmCall, 1);
      fullRecords.push(rec);
      totalCost += rec.total_cost_usd;
    }
  }

  return {
    probe_records,
    probe_cost_usd,
    projected_total_usd,
    halt_triggered: false,
    full_sweep_records: fullRecords,
    final_total_cost_usd: totalCost,
  };
}

// ─── JSONL writer ─────────────────────────────────────────────────────

export function writeRecordsToJsonl(
  records: readonly Gaia2RunRecord[],
  outputPath: string,
): void {
  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
  const lines = records.map((r) => JSON.stringify(r)).join('\n');
  fs.writeFileSync(outputPath, lines + (lines.length > 0 ? '\n' : ''), 'utf-8');
}

// ─── Re-exports ───────────────────────────────────────────────────────

export type { AgentRunResult, LlmCallFn, RetrievalSearchFn, MultiStepAgentRunConfig };