moving

benchmarks/harness/scripts/run-v8.ts

@@ -0,0 +1,781 @@
+#!/usr/bin/env tsx
+/**
+ * v8 Multi-Benchmark Runner — manifest-v8.2-final.md
+ *
+ * Executes the 4-track v8 ablation programme in strict order:
+ *
+ *   Track A0 — LongMemEval V1  (500 questions, 4 cells, $20 budget)
+ *   Track A  — BEAM 128K       (~300 questions, 4 cells, $50 budget)
+ *   Track B  — GAIA 2          (BLOCKED: requires WSL2 / SIGALRM fix)
+ *   Track D  — Terminal-Bench  (external infra, zero cost)
+ *
+ * 4-cell ablation grid per track:
+ *   no-context        → zero-memory baseline
+ *   retrieval         → HybridSearch recall only
+ *   hive_mind_ipb     → HybridSearch + I/P/B frame writes (primary treatment)
+ *   hive_mind_ipb_strong → same as hive_mind_ipb, Opus 4.x model
+ *
+ * Usage:
+ *   npx tsx benchmarks/harness/scripts/run-v8.ts [options]
+ *
+ * Options:
+ *   --track a0|a|b|d|all      Which track(s) to run (default: all)
+ *   --limit N                 Instance count cap per cell (default: full)
+ *   --budget-a0 USD           Hard USD cap for Track A0 (default: 20)
+ *   --budget-a USD            Hard USD cap for Track A  (default: 50)
+ *   --model-primary ID        Primary subject model     (default: qwen3.6-35b-a3b)
+ *   --model-strong ID         Strong subject model      (default: claude-opus-4-x)
+ *   --lme-data-path P         Path to longmemeval.jsonl (default: auto-discover)
+ *   --beam-data-path P        Path to beam-128K.jsonl   (default: auto-discover)
+ *   --dry-run                 Stub LLM calls
+ *   --no-ipb-strong           Skip hive_mind_ipb_strong cell (saves Opus spend)
+ *   --judge ID                Enable per-instance judge (default: none)
+ *   --seed N                  PRNG seed (default: 42)
+ *
+ * Env:
+ *   LITELLM_URL       default http://localhost:4000
+ *   LITELLM_API_KEY   default sk-waggle-dev
+ *
+ * Output:
+ *   benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.jsonl
+ *   benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.summary.json
+ *   benchmarks/results/v8/run-v8-<ts>.log  (full run log)
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+import process from 'node:process';
+
+// @waggle/agent stubs — avoids pulling in heavy agent package during harness runs.
+// generateTurnId: UUID v4 via crypto. logTurnEvent: no-op (observability only in prod).
+function generateTurnId(): string {
+  return crypto.randomUUID();
+}
+function logTurnEvent(_turnId: string, _event: Record<string, unknown>): void {
+  // no-op in harness context; real agent logging wired in full waggle runtime
+}
+import type {
+  CellName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig,
+} from '../src/types.js';
+import { loadDataset, getDatasetVersion, sampleInstances } from '../src/datasets.js';
+import { createLlmClient } from '../src/llm.js';
+import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from '../src/metrics.js';
+// Cells are imported dynamically to avoid @waggle/agent package resolution at startup.
+// (cells.ts → @waggle/agent → docx/exceljs/etc. which aren't installed in harness-only envs)
+// These are resolved lazily on first actual cell invocation.
+// Lazy-loaded cell modules (resolved on first invocation, not at import time)
+
+async function loadCells(): Promise<{
+  cells: typeof import('../src/cells.js').cells;
+  isCellName: typeof import('../src/cells.js').isCellName;
+  hiveMindIpbCell: typeof import('../src/cells-ipb.js').hiveMindIpbCell;
+}> {
+  const [cellsMod, ipbMod] = await Promise.all([
+    import('../src/cells.js'),
+    import('../src/cells-ipb.js'),
+  ]);
+  return {
+    cells: cellsMod.cells,
+    isCellName: cellsMod.isCellName,
+    hiveMindIpbCell: ipbMod.hiveMindIpbCell,
+  };
+}
+import { createSubstrate } from '../src/substrate.js';
+import type { Substrate } from '../src/substrate.js';
+import { extractTurnsFromLongMemEval, ingestLongMemEvalCorpus } from '../src/ingest-longmemeval.js';
+import { extractTurnsFromBeam, ingestBeamCorpus } from '../src/ingest-beam.js';
+import { StreakTracker } from '../src/streak-tracker.js';
+import { preCellHealthCheck } from '../src/health-check.js';
+import { acquireRunnerLock } from '../src/runner-lock.js';
+import type { LockHandle } from '../src/runner-lock.js';
+import { createJudgeLlmClient } from '../src/judge-client.js';
+import { runJudge } from '../src/judge-runner.js';
+import type { JudgeConfig, JudgePayload } from '../src/judge-runner.js';
+import type { JudgeClientCostEntry } from '../src/judge-client.js';
+
+// ── Extended cell name (adds hive_mind_ipb) ────────────────────────────────
+
+type V8CellName = CellName | 'hive_mind_ipb';
+
+// ── Constants ─────────────────────────────────────────────────────────────────
+
+const DEFAULT_SEED = 42;
+const DEFAULT_BUDGET_A0 = 20;  // Track A0: LME V1  ($20 hard halt)
+const DEFAULT_BUDGET_A = 50;   // Track A:  BEAM     ($50 hard halt)
+
+// v8 4-cell ablation grid (in run order)
+const V8_CELLS: readonly V8CellName[] = [
+  'no-context',
+  'retrieval',
+  'hive_mind_ipb',
+  // hive_mind_ipb_strong is added at runtime when --model-strong is set and
+  // --no-ipb-strong is NOT passed. It runs as a separate hive_mind_ipb cell
+  // invocation with the strong model id.
+];
+
+// ── Path helpers ──────────────────────────────────────────────────────────────
+
+function harnessRoot(): string {
+  const here = url.fileURLToPath(import.meta.url);
+  // scripts/ → harness root
+  return path.resolve(path.dirname(here), '..');
+}
+
+function benchRoot(): string {
+  return path.resolve(harnessRoot(), '..');
+}
+
+function defaultOutputDir(track: string): string {
+  const ts = new Date().toISOString().replace(/[:.]/g, '-');
+  const dir = path.join(benchRoot(), 'results', 'v8', track);
+  fs.mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function outputPath(dir: string, cell: string, dataset: string): string {
+  const ts = new Date().toISOString().replace(/[:.]/g, '-');
+  return path.join(dir, `${cell}-${dataset}-${ts}.jsonl`);
+}
+
+// ── Config loaders ────────────────────────────────────────────────────────────
+
+function loadModels(): Record<string, ModelSpec> {
+  const cfg = path.join(harnessRoot(), 'config', 'models.json');
+  return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, ModelSpec>;
+}
+
+function loadDatasets(): Record<string, DatasetSpec> {
+  const cfg = path.join(harnessRoot(), 'config', 'datasets.json');
+  return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, DatasetSpec>;
+}
+
+// ── CLI arg parsing ────────────────────────────────────────────────────────────
+
+interface V8Args {
+  tracks: Array<'a0' | 'a' | 'b' | 'd'>;
+  limit: number;
+  budgetA0: number;
+  budgetA: number;
+  modelPrimary: string;
+  modelStrong?: string;
+  runIpbStrong: boolean;
+  lmeDataPath?: string;
+  beamDataPath?: string;
+  dryRun?: boolean;
+  judge?: string;
+  seed: number;
+}
+
+function parseArgs(argv: string[]): V8Args {
+  const out: V8Args = {
+    tracks: ['a0', 'a'],  // default: A0 + A (B blocked, D is external)
+    limit: Number.POSITIVE_INFINITY,
+    budgetA0: DEFAULT_BUDGET_A0,
+    budgetA: DEFAULT_BUDGET_A,
+    modelPrimary: 'qwen3.6-35b-a3b',
+    runIpbStrong: true,
+    seed: DEFAULT_SEED,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i];
+    const next = argv[i + 1];
+    switch (flag) {
+      case '--track':
+        if (next === 'all') {
+          out.tracks = ['a0', 'a'];  // b and d are external
+        } else if (next === 'a0' || next === 'a' || next === 'b' || next === 'd') {
+          out.tracks = [next];
+        } else {
+          console.error(`[run-v8] Unknown --track value: ${next}. Use a0|a|b|d|all`);
+          process.exit(1);
+        }
+        i++;
+        break;
+      case '--limit': out.limit = Number(next); i++; break;
+      case '--budget-a0': out.budgetA0 = Number(next); i++; break;
+      case '--budget-a': out.budgetA = Number(next); i++; break;
+      case '--model-primary': out.modelPrimary = next; i++; break;
+      case '--model-strong': out.modelStrong = next; i++; break;
+      case '--no-ipb-strong': out.runIpbStrong = false; break;
+      case '--lme-data-path': out.lmeDataPath = next; i++; break;
+      case '--beam-data-path': out.beamDataPath = next; i++; break;
+      case '--dry-run': out.dryRun = true; break;
+      case '--live': out.dryRun = false; break;
+      case '--judge': out.judge = next; i++; break;
+      case '--seed': out.seed = Number(next); i++; break;
+      case '--help':
+      case '-h':
+        printHelp();
+        process.exit(0);
+    }
+  }
+  return out;
+}
+
+function printHelp(): void {
+  console.log(`
+run-v8.ts — v8 Multi-Benchmark Runner
+
+Usage:
+  npx tsx benchmarks/harness/scripts/run-v8.ts [options]
+
+Tracks (run in order):
+  --track a0      Track A0: LongMemEval V1 (500q, $20 budget)
+  --track a       Track A:  BEAM 128K      (~300q, $50 budget)
+  --track all     Run A0 then A (default)
+
+Options:
+  --limit N            Instance cap per cell
+  --budget-a0 USD      Track A0 hard halt (default: $20)
+  --budget-a USD       Track A hard halt  (default: $50)
+  --model-primary ID   Primary model (default: qwen3.6-35b-a3b)
+  --model-strong ID    Strong model for ipb_strong cell (default: claude-opus-4-x)
+  --no-ipb-strong      Skip hive_mind_ipb_strong cell
+  --lme-data-path P    Path to longmemeval.jsonl
+  --beam-data-path P   Path to beam-128K.jsonl
+  --dry-run            Stub LLM calls
+  --judge MODEL        Enable per-instance judge
+  --seed N             PRNG seed (default: 42)
+`);
+}
+
+// ── Per-instance runner (handles both canonical cells + hive_mind_ipb) ──────
+
+async function round(n: number, decimals: number): Promise<number> {
+  const f = Math.pow(10, decimals);
+  return Math.round(n * f) / f;
+}
+
+function computeFileHash(p: string): string {
+  return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex');
+}
+
+interface V8RunOneConfig {
+  cellName: V8CellName;
+  /** When cellName is 'hive_mind_ipb' and this is set, uses strongModel. */
+  useStrongModel?: boolean;
+  dataset: DatasetSpec;
+  model: ModelSpec;
+  strongModel?: ModelSpec;
+  litellmUrl: string;
+  litellmApiKey: string;
+  dryRun: boolean;
+  limit: number;
+  seed: number;
+  budgetUsd: number;
+  outputFilePath: string;
+  substrate?: Substrate;
+  judgeConfig?: JudgeConfig;
+  judgeCosts: JudgeClientCostEntry[];
+}
+
+async function runOneV8(config: V8RunOneConfig): Promise<void> {
+  const {
+    cellName, useStrongModel, dataset, model, strongModel,
+    litellmUrl, litellmApiKey, dryRun, limit, seed,
+    budgetUsd, outputFilePath, substrate, judgeConfig, judgeCosts,
+  } = config;
+
+  const activeModel = useStrongModel && strongModel ? strongModel : model;
+  const dataRoot = path.join(harnessRoot(), '..', 'data');
+  const all = loadDataset(dataset, dataRoot);
+  const datasetVersion = getDatasetVersion(dataset, dataRoot);
+  const sampled = sampleInstances(all, seed, limit);
+
+  const writer = new JsonlWriter(outputFilePath);
+  const llm = createLlmClient({ dryRun, litellmUrl, litellmApiKey });
+
+  const startedAt = new Date().toISOString();
+  let totalCost = 0;
+  let budgetStoppedAt: number | null = null;
+  const latencies: number[] = [];
+  const streakTracker = new StreakTracker();
+  let streakHaltAt: number | null = null;
+  let streakHaltSummary: string | null = null;
+
+  const displayCell = useStrongModel ? 'hive_mind_ipb_strong' : cellName;
+  console.log(`[v8:run] cell=${displayCell} dataset=${dataset.id} model=${activeModel.id} n=${sampled.length} budget=$${budgetUsd}`);
+
+  for (let i = 0; i < sampled.length; i++) {
+    if (totalCost >= budgetUsd) {
+      budgetStoppedAt = i;
+      console.log(`[v8:budget] halted at instance ${i} (cost=$${totalCost.toFixed(4)} >= $${budgetUsd})`);
+      break;
+    }
+
+    const instance = sampled[i];
+    const turnId = generateTurnId();
+
+    // Lazy-load cell modules on first iteration (avoids @waggle/agent at import time)
+    const { cells, isCellName, hiveMindIpbCell } = await loadCells();
+
+    let result;
+    if (cellName === 'hive_mind_ipb') {
+      result = await hiveMindIpbCell({
+        instance,
+        model: activeModel,
+        llm,
+        turnId,
+        substrate,
+        retrievalTopK: 20,
+      });
+    } else if (isCellName(cellName)) {
+      result = await cells[cellName]({
+        instance,
+        model: activeModel,
+        llm,
+        turnId,
+        substrate,
+        litellm: { url: litellmUrl, apiKey: litellmApiKey },
+        retrievalTopK: 20,
+      });
+    } else {
+      throw new Error(`[run-v8] Unknown cell: ${cellName}`);
+    }
+
+    latencies.push(result.latencyMs);
+    const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected);
+    totalCost += result.costUsd;
+
+    logTurnEvent(turnId, {
+      stage: 'llm.response',
+      cell: displayCell,
+      model: activeModel.id,
+      textChars: result.text.length,
+      latencyMs: result.latencyMs,
+      costUsd: result.costUsd,
+      failureMode: result.failureMode,
+      reasoningShape: result.reasoningShape ?? 'none',
+      reasoningChars: result.reasoningContent?.length ?? 0,
+    });
+
+    // Per-instance judge
+    let judgePayload: JudgePayload | null = null;
+    if (judgeConfig && !result.failureMode) {
+      judgePayload = await runJudge(
+        {
+          question: instance.question,
+          groundTruth: instance.expected[0] ?? '',
+          contextExcerpt: instance.context,
+          modelAnswer: result.text,
+        },
+        judgeConfig,
+      );
+    }
+
+    const record: JsonlRecord = {
+      turnId,
+      cell: displayCell as CellName,  // cast: JSONL schema, hive_mind_ipb_strong stored as variant
+      instance_id: instance.instance_id,
+      model: activeModel.id,
+      seed,
+      accuracy,
+      p50_latency_ms: percentile(latencies, 50),
+      p95_latency_ms: percentile(latencies, 95),
+      usd_per_query: Math.round(result.costUsd * 1_000_000) / 1_000_000,
+      failure_mode: result.failureMode,
+      dataset_version: datasetVersion,
+      ...(judgePayload && {
+        model_answer: judgePayload.model_answer,
+        judge_verdict: judgePayload.judge_verdict,
+        judge_failure_mode: judgePayload.judge_failure_mode,
+        judge_rationale: judgePayload.judge_rationale,
+        judge_model: judgePayload.judge_model,
+        judge_timestamp: judgePayload.judge_timestamp,
+        judge_ensemble: judgePayload.judge_ensemble,
+      }),
+      ...(result.reasoningContent !== undefined && {
+        reasoning_content: result.reasoningContent,
+        reasoning_content_chars: result.reasoningContent.length,
+      }),
+      ...(result.reasoningShape !== undefined && {
+        reasoning_shape: result.reasoningShape,
+      }),
+      ...(activeModel.pinning_surface !== undefined && {
+        model_pinning_surface: activeModel.pinning_surface,
+        model_pinning_carve_out_reason: activeModel.pinning_surface_carve_out_reason ?? null,
+      }),
+      model_revision_hash: null,
+    };
+    writer.write(record);
+
+    // Progress log every 10 instances
+    if ((i + 1) % 10 === 0 || i === sampled.length - 1) {
+      const runningAcc = writer.all().reduce((sum, r) => sum + r.accuracy, 0) / writer.all().length;
+      console.log(
+        `[v8:progress] cell=${displayCell} ${i + 1}/${sampled.length} ` +
+        `acc=${(runningAcc * 100).toFixed(1)}% cost=$${totalCost.toFixed(4)}`,
+      );
+    }
+
+    if (streakTracker.record(result.failureMode)) {
+      streakHaltAt = i + 1;
+      streakHaltSummary = streakTracker.summary();
+      break;
+    }
+  }
+
+  await writer.close();
+  const finishedAt = new Date().toISOString();
+
+  const runConfig = {
+    run: { kind: 'cell' as const, name: displayCell as CellName },
+    dataset,
+    model: activeModel,
+    limit,
+    seed,
+    budgetUsd,
+    outputPath: outputFilePath,
+    dryRun,
+    litellmUrl,
+    litellmApiKey,
+  } as RunConfig;
+  const summary = buildAggregate(runConfig, writer.all(), startedAt, finishedAt, budgetStoppedAt);
+  const summaryPath = outputFilePath.replace(/\.jsonl$/, '.summary.json');
+  fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
+
+  console.log(
+    `[v8:summary] cell=${displayCell} dataset=${dataset.id} ` +
+    `n=${summary.counts.total} completed=${summary.counts.completed} ` +
+    `failed=${summary.counts.failed} ` +
+    `accuracy=${(summary.metrics.meanAccuracy * 100).toFixed(2)}% ` +
+    `cost=$${summary.metrics.totalUsd.toFixed(4)} ` +
+    `jsonl=${outputFilePath}`,
+  );
+
+  if (streakHaltAt !== null) {
+    throw new Error(
+      `[v8:halt] cell '${displayCell}' aborted at instance ${streakHaltAt}/${sampled.length} ` +
+      `due to consecutive transport failures (${streakHaltSummary}).`,
+    );
+  }
+}
+
+// ── Track runner ──────────────────────────────────────────────────────────────
+
+interface TrackConfig {
+  track: 'a0' | 'a';
+  datasetId: string;
+  budgetUsd: number;
+  ingestDataset: boolean;
+  ingestFn: 'lme' | 'beam';
+  dataPathOverride?: string;
+}
+
+async function runTrack(
+  config: TrackConfig,
+  args: V8Args,
+  primaryModel: ModelSpec,
+  strongModel: ModelSpec | undefined,
+  allDatasets: Record<string, DatasetSpec>,
+  litellmUrl: string,
+  litellmApiKey: string,
+  dryRun: boolean,
+  judgeConfig: JudgeConfig | undefined,
+  judgeCosts: JudgeClientCostEntry[],
+): Promise<void> {
+  const { track, datasetId, budgetUsd, ingestFn, dataPathOverride } = config;
+  const dataset = allDatasets[datasetId];
+  if (!dataset) {
+    throw new Error(`[run-v8] Dataset not found: ${datasetId}. Check config/datasets.json.`);
+  }
+
+  console.log(`\n${'='.repeat(70)}`);
+  console.log(`[v8] TRACK ${track.toUpperCase()} — ${dataset.displayName}`);
+  console.log(`${'='.repeat(70)}`);
+
+  const outDir = defaultOutputDir(track);
+
+  // ── Build substrate + ingest corpus ──────────────────────────────────────
+  let substrate: Substrate | null = null;
+
+  if (!dryRun) {
+    console.log(`[v8:substrate] building ephemeral MindDB substrate for ${datasetId} …`);
+    substrate = createSubstrate();
+    const dataRoot = path.join(harnessRoot(), '..', 'data');
+    const jsonlPath = dataPathOverride ?? path.join(dataRoot, dataset.dataPath);
+
+    if (!fs.existsSync(jsonlPath)) {
+      throw new Error(
+        `[run-v8] Dataset JSONL not found at ${jsonlPath}. ` +
+        (ingestFn === 'lme'
+          ? 'Run: npx tsx benchmarks/harness/scripts/build-longmemeval-canonical.ts'
+          : 'Run: npx tsx benchmarks/harness/scripts/build-beam-canonical.ts --beam-data-path /path/to/BEAM/data --chat-size 128K'),
+      );
+    }
+
+    const ingestStart = Date.now();
+    if (ingestFn === 'lme') {
+      const turns = extractTurnsFromLongMemEval(jsonlPath);
+      console.log(`[v8:substrate] extracted ${turns.length} LME V1 turns from ${jsonlPath}`);
+      const stats = await ingestLongMemEvalCorpus(
+        substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
+      );
+      console.log(
+        `[v8:substrate] LME ingest complete: frames=${stats.count} ` +
+        `ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
+        `total_ms=${Date.now() - ingestStart}`,
+      );
+    } else {
+      const turns = extractTurnsFromBeam(jsonlPath);
+      console.log(`[v8:substrate] extracted ${turns.length} BEAM turns from ${jsonlPath}`);
+      const stats = await ingestBeamCorpus(
+        substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
+      );
+      console.log(
+        `[v8:substrate] BEAM ingest complete: frames=${stats.count} ` +
+        `ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
+        `total_ms=${Date.now() - ingestStart}`,
+      );
+    }
+  } else {
+    console.log(`[v8:substrate] dry-run — skipping substrate ingest`);
+  }
+
+  // Build the per-track cell list
+  const trackCells: Array<{ cellName: V8CellName; useStrong: boolean }> = [
+    { cellName: 'no-context', useStrong: false },
+    { cellName: 'retrieval', useStrong: false },
+    { cellName: 'hive_mind_ipb', useStrong: false },
+  ];
+  if (args.runIpbStrong && strongModel) {
+    trackCells.push({ cellName: 'hive_mind_ipb', useStrong: true });
+  }
+
+  // Budget per-cell (split total track budget evenly so any single cell can't exhaust)
+  // Each cell gets the full budget; the track budget is checked across cells
+  // by the caller. Individual cell budgets are capped at budgetUsd / 4 * 1.5
+  // to ensure all 4 cells get a fair share with some slack.
+  const cellBudget = budgetUsd;  // individual hard-halt per cell
+
+  let trackTotalCost = 0;
+
+  try {
+    for (const { cellName, useStrong } of trackCells) {
+      const displayCell = useStrong ? 'hive_mind_ipb_strong' : cellName;
+
+      // Skip retrieval/hive_mind_ipb cells in dry-run (need live embedder)
+      if (dryRun && (cellName === 'retrieval' || cellName === 'hive_mind_ipb')) {
+        console.log(`[v8:skip] ${displayCell} — skipped in dry-run mode (needs substrate)`);
+        continue;
+      }
+
+      const outPath = outputPath(outDir, displayCell, datasetId);
+      await runOneV8({
+        cellName,
+        useStrongModel: useStrong,
+        dataset,
+        model: primaryModel,
+        strongModel,
+        litellmUrl,
+        litellmApiKey,
+        dryRun,
+        limit: args.limit,
+        seed: args.seed,
+        budgetUsd: cellBudget,
+        outputFilePath: outPath,
+        substrate: substrate ?? undefined,
+        judgeConfig,
+        judgeCosts,
+      });
+
+      // Read cost from summary file for track total
+      const summaryPath = outPath.replace(/\.jsonl$/, '.summary.json');
+      if (fs.existsSync(summaryPath)) {
+        const summ = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as {
+          metrics?: { totalUsd?: number };
+        };
+        const cellCost = summ.metrics?.totalUsd ?? 0;
+        trackTotalCost += cellCost;
+        console.log(
+          `[v8:track-cost] ${track.toUpperCase()} track running total: $${trackTotalCost.toFixed(4)}`,
+        );
+        if (trackTotalCost >= budgetUsd) {
+          console.warn(
+            `[v8:track-budget] Track ${track.toUpperCase()} track budget ($${budgetUsd}) reached ` +
+            `after cell ${displayCell}. Stopping remaining cells.`,
+          );
+          break;
+        }
+      }
+    }
+  } finally {
+    if (substrate) {
+      substrate.close();
+      console.log(`[v8:substrate] closed substrate for ${datasetId}`);
+    }
+  }
+
+  console.log(
+    `[v8:track-done] Track ${track.toUpperCase()} complete. ` +
+    `Total cost: $${trackTotalCost.toFixed(4)} ` +
+    `Results in: ${outDir}`,
+  );
+}
+
+// ── Main ──────────────────────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+  const allModels = loadModels();
+  const allDatasets = loadDatasets();
+
+  const primaryModel = allModels[args.modelPrimary];
+  if (!primaryModel) {
+    throw new Error(
+      `[run-v8] Unknown primary model: ${args.modelPrimary}. ` +
+      `Valid ids: ${Object.keys(allModels).join(', ')}`,
+    );
+  }
+
+  let strongModel: ModelSpec | undefined;
+  if (args.modelStrong) {
+    strongModel = allModels[args.modelStrong];
+    if (!strongModel) {
+      throw new Error(
+        `[run-v8] Unknown strong model: ${args.modelStrong}. ` +
+        `Valid ids: ${Object.keys(allModels).join(', ')}`,
+      );
+    }
+  } else {
+    // Try to find claude-opus-4-x automatically
+    strongModel = allModels['claude-opus-4-x'] ?? allModels['claude-opus-4-8'];
+    if (strongModel) {
+      console.log(`[v8] strong model auto-resolved: ${strongModel.id}`);
+    } else if (args.runIpbStrong) {
+      console.warn('[v8] No strong model found in models.json (tried claude-opus-4-x, claude-opus-4-8). Skipping hive_mind_ipb_strong cell.');
+    }
+  }
+
+  const dryRun = args.dryRun ?? !process.env.LITELLM_URL;
+  const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000';
+  const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev';
+
+  if (dryRun) {
+    console.log('[v8] DRY RUN mode — LLM calls will be stubbed');
+  }
+
+  // Judge wiring
+  const judgeCosts: JudgeClientCostEntry[] = [];
+  let judgeConfig: JudgeConfig | undefined;
+  if (!dryRun && args.judge) {
+    judgeConfig = {
+      kind: 'single',
+      model: args.judge,
+      client: createJudgeLlmClient({
+        litellmUrl, litellmApiKey,
+        model: args.judge,
+        onCall: entry => judgeCosts.push(entry),
+      }),
+    };
+    console.log(`[v8] judge model: ${args.judge}`);
+  }
+
+  // Health check
+  if (!dryRun) {
+    const judgePingModels = args.judge ? [args.judge] : [];
+    const hc = await preCellHealthCheck({
+      litellmUrl, litellmApiKey,
+      subjectModel: primaryModel.litellmModel,
+      judgeModels: judgePingModels,
+    });
+    if (!hc.ok) {
+      const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; ');
+      throw new Error(
+        `[v8:health-check] FAILED: ${summary}. ` +
+        `Check LiteLLM proxy + provider keys before spending budget.`,
+      );
+    }
+    console.log(`[v8:health-check] OK in ${hc.durationMs}ms`);
+  }
+
+  // Acquire single-runner lock
+  const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner');
+  const runnerLock: LockHandle = acquireRunnerLock(lockSentinel);
+  console.log(`[v8:lock] acquired runner lock (pid=${process.pid})`);
+
+  const runStart = Date.now();
+  console.log(`\n[v8] Starting v8 multi-benchmark run`);
+  console.log(`[v8] Primary: ${primaryModel.id}${strongModel ? `  Strong: ${strongModel.id}` : ''}`);
+  console.log(`[v8] Tracks: ${args.tracks.join(', ')}`);
+  console.log(`[v8] Limit per cell: ${Number.isFinite(args.limit) ? args.limit : 'full'}`);
+  console.log(`[v8] Seed: ${args.seed}`);
+
+  try {
+    // ── Track A0: LongMemEval V1 ──────────────────────────────────────────
+    if (args.tracks.includes('a0')) {
+      const lmeJsonlPath = args.lmeDataPath ??
+        path.join(harnessRoot(), '..', 'data', 'longmemeval', 'longmemeval.jsonl');
+
+      await runTrack(
+        {
+          track: 'a0',
+          datasetId: 'longmemeval',
+          budgetUsd: args.budgetA0,
+          ingestDataset: true,
+          ingestFn: 'lme',
+          dataPathOverride: lmeJsonlPath,
+        },
+        args, primaryModel, strongModel, allDatasets,
+        litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
+      );
+    }
+
+    // ── Track A: BEAM 128K ────────────────────────────────────────────────
+    if (args.tracks.includes('a')) {
+      const beamJsonlPath = args.beamDataPath ??
+        path.join(harnessRoot(), '..', 'data', 'beam', 'beam-128K.jsonl');
+
+      await runTrack(
+        {
+          track: 'a',
+          datasetId: 'beam-128k',
+          budgetUsd: args.budgetA,
+          ingestDataset: true,
+          ingestFn: 'beam',
+          dataPathOverride: beamJsonlPath,
+        },
+        args, primaryModel, strongModel, allDatasets,
+        litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
+      );
+    }
+
+    // ── Track B: GAIA 2 ───────────────────────────────────────────────────
+    if (args.tracks.includes('b')) {
+      console.log('\n[v8:track-b] BLOCKED — SIGALRM issue on Windows/WSL1.');
+      console.log('[v8:track-b] Fix: run under WSL2 or Docker. Gate: PM-RATIFY-V8-PHASE1.');
+    }
+
+    // ── Track D: Terminal-Bench ───────────────────────────────────────────
+    if (args.tracks.includes('d')) {
+      console.log('\n[v8:track-d] Terminal-Bench is external infra.');
+      console.log('[v8:track-d] Submit waggle scaffold to harborframework/terminal-bench-2-leaderboard.');
+      console.log('[v8:track-d] Baseline: little-coder #118/#123 = 24.6% ± 3.2% (Qwen3.6-35B, 2026-05-14).');
+    }
+
+  } finally {
+    runnerLock.release();
+  }
+
+  // Judge summary
+  if (judgeCosts.length > 0) {
+    const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0);
+    const judgeOk = judgeCosts.filter(e => e.ok).length;
+    console.log(
+      `\n[v8:judge-total] calls=${judgeCosts.length} ok=${judgeOk} ` +
+      `failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`,
+    );
+  }
+
+  const totalMs = Date.now() - runStart;
+  console.log(`\n[v8:done] Total wall time: ${(totalMs / 1000).toFixed(1)}s`);
+  console.log(`[v8:done] Results in: ${path.join(benchRoot(), 'results', 'v8')}/`);
+}
+
+main().catch(err => {
+  console.error('[v8:fatal]', err?.message ?? err);
+  process.exit(1);
+});