#!/usr/bin/env tsx
/**
 * v8 Multi-Benchmark Runner — manifest-v8.2-final.md
 *
 * Executes the 4-track v8 ablation programme in strict order:
 *
 *   Track A0 — LongMemEval V1  (500 questions, 4 cells, $20 budget)
 *   Track A  — BEAM 128K       (~300 questions, 4 cells, $50 budget)
 *   Track B  — GAIA 2          (BLOCKED: requires WSL2 / SIGALRM fix)
 *   Track D  — Terminal-Bench  (external infra, zero cost)
 *
 * 4-cell ablation grid per track:
 *   no-context        → zero-memory baseline
 *   retrieval         → HybridSearch recall only
 *   hive_mind_ipb     → HybridSearch + I/P/B frame writes (primary treatment)
 *   hive_mind_ipb_strong → same as hive_mind_ipb, Opus 4.x model
 *
 * Usage:
 *   npx tsx benchmarks/harness/scripts/run-v8.ts [options]
 *
 * Options:
 *   --track a0|a|b|d|all      Which track(s) to run (default: all)
 *   --limit N                 Instance count cap per cell (default: full)
 *   --budget-a0 USD           Hard USD cap for Track A0 (default: 20)
 *   --budget-a USD            Hard USD cap for Track A  (default: 50)
 *   --model-primary ID        Primary subject model     (default: qwen3.6-35b-a3b)
 *   --model-strong ID         Strong subject model      (default: claude-opus-4-x)
 *   --lme-data-path P         Path to longmemeval.jsonl (default: auto-discover)
 *   --beam-data-path P        Path to beam-128K.jsonl   (default: auto-discover)
 *   --dry-run                 Stub LLM calls
 *   --no-ipb-strong           Skip hive_mind_ipb_strong cell (saves Opus spend)
 *   --judge ID                Enable per-instance judge (default: none)
 *   --seed N                  PRNG seed (default: 42)
 *
 * Env:
 *   LITELLM_URL       default http://localhost:4000
 *   LITELLM_API_KEY   default sk-waggle-dev
 *
 * Output:
 *   benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.jsonl
 *   benchmarks/results/v8/<track>/<cell>-<dataset>-<ts>.summary.json
 *   benchmarks/results/v8/run-v8-<ts>.log  (full run log)
 */

import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import url from 'node:url';
import process from 'node:process';

// @waggle/agent stubs — avoids pulling in heavy agent package during harness runs.
// generateTurnId: UUID v4 via crypto. logTurnEvent: no-op (observability only in prod).
function generateTurnId(): string {
  return crypto.randomUUID();
}
function logTurnEvent(_turnId: string, _event: Record<string, unknown>): void {
  // no-op in harness context; real agent logging wired in full waggle runtime
}
import type {
  CellName, DatasetSpec, JsonlRecord, ModelSpec, RunConfig,
} from '../src/types.js';
import { loadDataset, getDatasetVersion, sampleInstances } from '../src/datasets.js';
import { createLlmClient } from '../src/llm.js';
import { JsonlWriter, buildAggregate, scoreAccuracy, percentile } from '../src/metrics.js';
// Cells are imported dynamically to avoid @waggle/agent package resolution at startup.
// (cells.ts → @waggle/agent → docx/exceljs/etc. which aren't installed in harness-only envs)
// These are resolved lazily on first actual cell invocation.
// Lazy-loaded cell modules (resolved on first invocation, not at import time)

async function loadCells(): Promise<{
  cells: typeof import('../src/cells.js').cells;
  isCellName: typeof import('../src/cells.js').isCellName;
  hiveMindIpbCell: typeof import('../src/cells-ipb.js').hiveMindIpbCell;
}> {
  const [cellsMod, ipbMod] = await Promise.all([
    import('../src/cells.js'),
    import('../src/cells-ipb.js'),
  ]);
  return {
    cells: cellsMod.cells,
    isCellName: cellsMod.isCellName,
    hiveMindIpbCell: ipbMod.hiveMindIpbCell,
  };
}
import { createSubstrate } from '../src/substrate.js';
import type { Substrate } from '../src/substrate.js';
import { extractTurnsFromLongMemEval, ingestLongMemEvalCorpus } from '../src/ingest-longmemeval.js';
import { extractTurnsFromBeam, ingestBeamCorpus } from '../src/ingest-beam.js';
import { StreakTracker } from '../src/streak-tracker.js';
import { preCellHealthCheck } from '../src/health-check.js';
import { acquireRunnerLock } from '../src/runner-lock.js';
import type { LockHandle } from '../src/runner-lock.js';
import { createJudgeLlmClient } from '../src/judge-client.js';
import { runJudge } from '../src/judge-runner.js';
import type { JudgeConfig, JudgePayload } from '../src/judge-runner.js';
import type { JudgeClientCostEntry } from '../src/judge-client.js';

// ── Extended cell name (adds hive_mind_ipb) ────────────────────────────────

type V8CellName = CellName | 'hive_mind_ipb';

// ── Constants ─────────────────────────────────────────────────────────────────

const DEFAULT_SEED = 42;
const DEFAULT_BUDGET_A0 = 20;  // Track A0: LME V1  ($20 hard halt)
const DEFAULT_BUDGET_A = 50;   // Track A:  BEAM     ($50 hard halt)

// v8 4-cell ablation grid (in run order)
const V8_CELLS: readonly V8CellName[] = [
  'no-context',
  'retrieval',
  'hive_mind_ipb',
  // hive_mind_ipb_strong is added at runtime when --model-strong is set and
  // --no-ipb-strong is NOT passed. It runs as a separate hive_mind_ipb cell
  // invocation with the strong model id.
];

// ── Path helpers ──────────────────────────────────────────────────────────────

function harnessRoot(): string {
  const here = url.fileURLToPath(import.meta.url);
  // scripts/ → harness root
  return path.resolve(path.dirname(here), '..');
}

function benchRoot(): string {
  return path.resolve(harnessRoot(), '..');
}

function defaultOutputDir(track: string): string {
  const ts = new Date().toISOString().replace(/[:.]/g, '-');
  const dir = path.join(benchRoot(), 'results', 'v8', track);
  fs.mkdirSync(dir, { recursive: true });
  return dir;
}

function outputPath(dir: string, cell: string, dataset: string): string {
  const ts = new Date().toISOString().replace(/[:.]/g, '-');
  return path.join(dir, `${cell}-${dataset}-${ts}.jsonl`);
}

// ── Config loaders ────────────────────────────────────────────────────────────

function loadModels(): Record<string, ModelSpec> {
  const cfg = path.join(harnessRoot(), 'config', 'models.json');
  return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, ModelSpec>;
}

function loadDatasets(): Record<string, DatasetSpec> {
  const cfg = path.join(harnessRoot(), 'config', 'datasets.json');
  return JSON.parse(fs.readFileSync(cfg, 'utf-8')) as Record<string, DatasetSpec>;
}

// ── CLI arg parsing ────────────────────────────────────────────────────────────

interface V8Args {
  tracks: Array<'a0' | 'a' | 'b' | 'd'>;
  limit: number;
  budgetA0: number;
  budgetA: number;
  modelPrimary: string;
  modelStrong?: string;
  runIpbStrong: boolean;
  lmeDataPath?: string;
  beamDataPath?: string;
  dryRun?: boolean;
  judge?: string;
  seed: number;
}

function parseArgs(argv: string[]): V8Args {
  const out: V8Args = {
    tracks: ['a0', 'a'],  // default: A0 + A (B blocked, D is external)
    limit: Number.POSITIVE_INFINITY,
    budgetA0: DEFAULT_BUDGET_A0,
    budgetA: DEFAULT_BUDGET_A,
    modelPrimary: 'qwen3.6-35b-a3b',
    runIpbStrong: true,
    seed: DEFAULT_SEED,
  };

  for (let i = 0; i < argv.length; i++) {
    const flag = argv[i];
    const next = argv[i + 1];
    switch (flag) {
      case '--track':
        if (next === 'all') {
          out.tracks = ['a0', 'a'];  // b and d are external
        } else if (next === 'a0' || next === 'a' || next === 'b' || next === 'd') {
          out.tracks = [next];
        } else {
          console.error(`[run-v8] Unknown --track value: ${next}. Use a0|a|b|d|all`);
          process.exit(1);
        }
        i++;
        break;
      case '--limit': out.limit = Number(next); i++; break;
      case '--budget-a0': out.budgetA0 = Number(next); i++; break;
      case '--budget-a': out.budgetA = Number(next); i++; break;
      case '--model-primary': out.modelPrimary = next; i++; break;
      case '--model-strong': out.modelStrong = next; i++; break;
      case '--no-ipb-strong': out.runIpbStrong = false; break;
      case '--lme-data-path': out.lmeDataPath = next; i++; break;
      case '--beam-data-path': out.beamDataPath = next; i++; break;
      case '--dry-run': out.dryRun = true; break;
      case '--live': out.dryRun = false; break;
      case '--judge': out.judge = next; i++; break;
      case '--seed': out.seed = Number(next); i++; break;
      case '--help':
      case '-h':
        printHelp();
        process.exit(0);
    }
  }
  return out;
}

function printHelp(): void {
  console.log(`
run-v8.ts — v8 Multi-Benchmark Runner

Usage:
  npx tsx benchmarks/harness/scripts/run-v8.ts [options]

Tracks (run in order):
  --track a0      Track A0: LongMemEval V1 (500q, $20 budget)
  --track a       Track A:  BEAM 128K      (~300q, $50 budget)
  --track all     Run A0 then A (default)

Options:
  --limit N            Instance cap per cell
  --budget-a0 USD      Track A0 hard halt (default: $20)
  --budget-a USD       Track A hard halt  (default: $50)
  --model-primary ID   Primary model (default: qwen3.6-35b-a3b)
  --model-strong ID    Strong model for ipb_strong cell (default: claude-opus-4-x)
  --no-ipb-strong      Skip hive_mind_ipb_strong cell
  --lme-data-path P    Path to longmemeval.jsonl
  --beam-data-path P   Path to beam-128K.jsonl
  --dry-run            Stub LLM calls
  --judge MODEL        Enable per-instance judge
  --seed N             PRNG seed (default: 42)
`);
}

// ── Per-instance runner (handles both canonical cells + hive_mind_ipb) ──────

async function round(n: number, decimals: number): Promise<number> {
  const f = Math.pow(10, decimals);
  return Math.round(n * f) / f;
}

function computeFileHash(p: string): string {
  return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex');
}

interface V8RunOneConfig {
  cellName: V8CellName;
  /** When cellName is 'hive_mind_ipb' and this is set, uses strongModel. */
  useStrongModel?: boolean;
  dataset: DatasetSpec;
  model: ModelSpec;
  strongModel?: ModelSpec;
  litellmUrl: string;
  litellmApiKey: string;
  dryRun: boolean;
  limit: number;
  seed: number;
  budgetUsd: number;
  outputFilePath: string;
  substrate?: Substrate;
  judgeConfig?: JudgeConfig;
  judgeCosts: JudgeClientCostEntry[];
}

async function runOneV8(config: V8RunOneConfig): Promise<void> {
  const {
    cellName, useStrongModel, dataset, model, strongModel,
    litellmUrl, litellmApiKey, dryRun, limit, seed,
    budgetUsd, outputFilePath, substrate, judgeConfig, judgeCosts,
  } = config;

  const activeModel = useStrongModel && strongModel ? strongModel : model;
  const dataRoot = path.join(harnessRoot(), '..', 'data');
  const all = loadDataset(dataset, dataRoot);
  const datasetVersion = getDatasetVersion(dataset, dataRoot);
  const sampled = sampleInstances(all, seed, limit);

  const writer = new JsonlWriter(outputFilePath);
  const llm = createLlmClient({ dryRun, litellmUrl, litellmApiKey });

  const startedAt = new Date().toISOString();
  let totalCost = 0;
  let budgetStoppedAt: number | null = null;
  const latencies: number[] = [];
  const streakTracker = new StreakTracker();
  let streakHaltAt: number | null = null;
  let streakHaltSummary: string | null = null;

  const displayCell = useStrongModel ? 'hive_mind_ipb_strong' : cellName;
  console.log(`[v8:run] cell=${displayCell} dataset=${dataset.id} model=${activeModel.id} n=${sampled.length} budget=$${budgetUsd}`);

  for (let i = 0; i < sampled.length; i++) {
    if (totalCost >= budgetUsd) {
      budgetStoppedAt = i;
      console.log(`[v8:budget] halted at instance ${i} (cost=$${totalCost.toFixed(4)} >= $${budgetUsd})`);
      break;
    }

    const instance = sampled[i];
    const turnId = generateTurnId();

    // Lazy-load cell modules on first iteration (avoids @waggle/agent at import time)
    const { cells, isCellName, hiveMindIpbCell } = await loadCells();

    let result;
    if (cellName === 'hive_mind_ipb') {
      result = await hiveMindIpbCell({
        instance,
        model: activeModel,
        llm,
        turnId,
        substrate,
        retrievalTopK: 20,
      });
    } else if (isCellName(cellName)) {
      result = await cells[cellName]({
        instance,
        model: activeModel,
        llm,
        turnId,
        substrate,
        litellm: { url: litellmUrl, apiKey: litellmApiKey },
        retrievalTopK: 20,
      });
    } else {
      throw new Error(`[run-v8] Unknown cell: ${cellName}`);
    }

    latencies.push(result.latencyMs);
    const accuracy = result.failureMode ? 0 : scoreAccuracy(result.text, instance.expected);
    totalCost += result.costUsd;

    logTurnEvent(turnId, {
      stage: 'llm.response',
      cell: displayCell,
      model: activeModel.id,
      textChars: result.text.length,
      latencyMs: result.latencyMs,
      costUsd: result.costUsd,
      failureMode: result.failureMode,
      reasoningShape: result.reasoningShape ?? 'none',
      reasoningChars: result.reasoningContent?.length ?? 0,
    });

    // Per-instance judge
    let judgePayload: JudgePayload | null = null;
    if (judgeConfig && !result.failureMode) {
      judgePayload = await runJudge(
        {
          question: instance.question,
          groundTruth: instance.expected[0] ?? '',
          contextExcerpt: instance.context,
          modelAnswer: result.text,
        },
        judgeConfig,
      );
    }

    const record: JsonlRecord = {
      turnId,
      cell: displayCell as CellName,  // cast: JSONL schema, hive_mind_ipb_strong stored as variant
      instance_id: instance.instance_id,
      model: activeModel.id,
      seed,
      accuracy,
      p50_latency_ms: percentile(latencies, 50),
      p95_latency_ms: percentile(latencies, 95),
      usd_per_query: Math.round(result.costUsd * 1_000_000) / 1_000_000,
      failure_mode: result.failureMode,
      dataset_version: datasetVersion,
      ...(judgePayload && {
        model_answer: judgePayload.model_answer,
        judge_verdict: judgePayload.judge_verdict,
        judge_failure_mode: judgePayload.judge_failure_mode,
        judge_rationale: judgePayload.judge_rationale,
        judge_model: judgePayload.judge_model,
        judge_timestamp: judgePayload.judge_timestamp,
        judge_ensemble: judgePayload.judge_ensemble,
      }),
      ...(result.reasoningContent !== undefined && {
        reasoning_content: result.reasoningContent,
        reasoning_content_chars: result.reasoningContent.length,
      }),
      ...(result.reasoningShape !== undefined && {
        reasoning_shape: result.reasoningShape,
      }),
      ...(activeModel.pinning_surface !== undefined && {
        model_pinning_surface: activeModel.pinning_surface,
        model_pinning_carve_out_reason: activeModel.pinning_surface_carve_out_reason ?? null,
      }),
      model_revision_hash: null,
    };
    writer.write(record);

    // Progress log every 10 instances
    if ((i + 1) % 10 === 0 || i === sampled.length - 1) {
      const runningAcc = writer.all().reduce((sum, r) => sum + r.accuracy, 0) / writer.all().length;
      console.log(
        `[v8:progress] cell=${displayCell} ${i + 1}/${sampled.length} ` +
        `acc=${(runningAcc * 100).toFixed(1)}% cost=$${totalCost.toFixed(4)}`,
      );
    }

    if (streakTracker.record(result.failureMode)) {
      streakHaltAt = i + 1;
      streakHaltSummary = streakTracker.summary();
      break;
    }
  }

  await writer.close();
  const finishedAt = new Date().toISOString();

  const runConfig = {
    run: { kind: 'cell' as const, name: displayCell as CellName },
    dataset,
    model: activeModel,
    limit,
    seed,
    budgetUsd,
    outputPath: outputFilePath,
    dryRun,
    litellmUrl,
    litellmApiKey,
  } as RunConfig;
  const summary = buildAggregate(runConfig, writer.all(), startedAt, finishedAt, budgetStoppedAt);
  const summaryPath = outputFilePath.replace(/\.jsonl$/, '.summary.json');
  fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');

  console.log(
    `[v8:summary] cell=${displayCell} dataset=${dataset.id} ` +
    `n=${summary.counts.total} completed=${summary.counts.completed} ` +
    `failed=${summary.counts.failed} ` +
    `accuracy=${(summary.metrics.meanAccuracy * 100).toFixed(2)}% ` +
    `cost=$${summary.metrics.totalUsd.toFixed(4)} ` +
    `jsonl=${outputFilePath}`,
  );

  if (streakHaltAt !== null) {
    throw new Error(
      `[v8:halt] cell '${displayCell}' aborted at instance ${streakHaltAt}/${sampled.length} ` +
      `due to consecutive transport failures (${streakHaltSummary}).`,
    );
  }
}

// ── Track runner ──────────────────────────────────────────────────────────────

interface TrackConfig {
  track: 'a0' | 'a';
  datasetId: string;
  budgetUsd: number;
  ingestDataset: boolean;
  ingestFn: 'lme' | 'beam';
  dataPathOverride?: string;
}

async function runTrack(
  config: TrackConfig,
  args: V8Args,
  primaryModel: ModelSpec,
  strongModel: ModelSpec | undefined,
  allDatasets: Record<string, DatasetSpec>,
  litellmUrl: string,
  litellmApiKey: string,
  dryRun: boolean,
  judgeConfig: JudgeConfig | undefined,
  judgeCosts: JudgeClientCostEntry[],
): Promise<void> {
  const { track, datasetId, budgetUsd, ingestFn, dataPathOverride } = config;
  const dataset = allDatasets[datasetId];
  if (!dataset) {
    throw new Error(`[run-v8] Dataset not found: ${datasetId}. Check config/datasets.json.`);
  }

  console.log(`\n${'='.repeat(70)}`);
  console.log(`[v8] TRACK ${track.toUpperCase()} — ${dataset.displayName}`);
  console.log(`${'='.repeat(70)}`);

  const outDir = defaultOutputDir(track);

  // ── Build substrate + ingest corpus ──────────────────────────────────────
  let substrate: Substrate | null = null;

  if (!dryRun) {
    console.log(`[v8:substrate] building ephemeral MindDB substrate for ${datasetId} …`);
    substrate = createSubstrate();
    const dataRoot = path.join(harnessRoot(), '..', 'data');
    const jsonlPath = dataPathOverride ?? path.join(dataRoot, dataset.dataPath);

    if (!fs.existsSync(jsonlPath)) {
      throw new Error(
        `[run-v8] Dataset JSONL not found at ${jsonlPath}. ` +
        (ingestFn === 'lme'
          ? 'Run: npx tsx benchmarks/harness/scripts/build-longmemeval-canonical.ts'
          : 'Run: npx tsx benchmarks/harness/scripts/build-beam-canonical.ts --beam-data-path /path/to/BEAM/data --chat-size 128K'),
      );
    }

    const ingestStart = Date.now();
    if (ingestFn === 'lme') {
      const turns = extractTurnsFromLongMemEval(jsonlPath);
      console.log(`[v8:substrate] extracted ${turns.length} LME V1 turns from ${jsonlPath}`);
      const stats = await ingestLongMemEvalCorpus(
        substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
      );
      console.log(
        `[v8:substrate] LME ingest complete: frames=${stats.count} ` +
        `ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
        `total_ms=${Date.now() - ingestStart}`,
      );
    } else {
      const turns = extractTurnsFromBeam(jsonlPath);
      console.log(`[v8:substrate] extracted ${turns.length} BEAM turns from ${jsonlPath}`);
      const stats = await ingestBeamCorpus(
        substrate.db, substrate.search, substrate.frames, substrate.sessions, turns,
      );
      console.log(
        `[v8:substrate] BEAM ingest complete: frames=${stats.count} ` +
        `ingest_ms=${stats.ingestMs} index_ms=${stats.indexMs} ` +
        `total_ms=${Date.now() - ingestStart}`,
      );
    }
  } else {
    console.log(`[v8:substrate] dry-run — skipping substrate ingest`);
  }

  // Build the per-track cell list
  const trackCells: Array<{ cellName: V8CellName; useStrong: boolean }> = [
    { cellName: 'no-context', useStrong: false },
    { cellName: 'retrieval', useStrong: false },
    { cellName: 'hive_mind_ipb', useStrong: false },
  ];
  if (args.runIpbStrong && strongModel) {
    trackCells.push({ cellName: 'hive_mind_ipb', useStrong: true });
  }

  // Budget per-cell (split total track budget evenly so any single cell can't exhaust)
  // Each cell gets the full budget; the track budget is checked across cells
  // by the caller. Individual cell budgets are capped at budgetUsd / 4 * 1.5
  // to ensure all 4 cells get a fair share with some slack.
  const cellBudget = budgetUsd;  // individual hard-halt per cell

  let trackTotalCost = 0;

  try {
    for (const { cellName, useStrong } of trackCells) {
      const displayCell = useStrong ? 'hive_mind_ipb_strong' : cellName;

      // Skip retrieval/hive_mind_ipb cells in dry-run (need live embedder)
      if (dryRun && (cellName === 'retrieval' || cellName === 'hive_mind_ipb')) {
        console.log(`[v8:skip] ${displayCell} — skipped in dry-run mode (needs substrate)`);
        continue;
      }

      const outPath = outputPath(outDir, displayCell, datasetId);
      await runOneV8({
        cellName,
        useStrongModel: useStrong,
        dataset,
        model: primaryModel,
        strongModel,
        litellmUrl,
        litellmApiKey,
        dryRun,
        limit: args.limit,
        seed: args.seed,
        budgetUsd: cellBudget,
        outputFilePath: outPath,
        substrate: substrate ?? undefined,
        judgeConfig,
        judgeCosts,
      });

      // Read cost from summary file for track total
      const summaryPath = outPath.replace(/\.jsonl$/, '.summary.json');
      if (fs.existsSync(summaryPath)) {
        const summ = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as {
          metrics?: { totalUsd?: number };
        };
        const cellCost = summ.metrics?.totalUsd ?? 0;
        trackTotalCost += cellCost;
        console.log(
          `[v8:track-cost] ${track.toUpperCase()} track running total: $${trackTotalCost.toFixed(4)}`,
        );
        if (trackTotalCost >= budgetUsd) {
          console.warn(
            `[v8:track-budget] Track ${track.toUpperCase()} track budget ($${budgetUsd}) reached ` +
            `after cell ${displayCell}. Stopping remaining cells.`,
          );
          break;
        }
      }
    }
  } finally {
    if (substrate) {
      substrate.close();
      console.log(`[v8:substrate] closed substrate for ${datasetId}`);
    }
  }

  console.log(
    `[v8:track-done] Track ${track.toUpperCase()} complete. ` +
    `Total cost: $${trackTotalCost.toFixed(4)} ` +
    `Results in: ${outDir}`,
  );
}

// ── Main ──────────────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  const args = parseArgs(process.argv.slice(2));
  const allModels = loadModels();
  const allDatasets = loadDatasets();

  const primaryModel = allModels[args.modelPrimary];
  if (!primaryModel) {
    throw new Error(
      `[run-v8] Unknown primary model: ${args.modelPrimary}. ` +
      `Valid ids: ${Object.keys(allModels).join(', ')}`,
    );
  }

  let strongModel: ModelSpec | undefined;
  if (args.modelStrong) {
    strongModel = allModels[args.modelStrong];
    if (!strongModel) {
      throw new Error(
        `[run-v8] Unknown strong model: ${args.modelStrong}. ` +
        `Valid ids: ${Object.keys(allModels).join(', ')}`,
      );
    }
  } else {
    // Try to find claude-opus-4-x automatically
    strongModel = allModels['claude-opus-4-x'] ?? allModels['claude-opus-4-8'];
    if (strongModel) {
      console.log(`[v8] strong model auto-resolved: ${strongModel.id}`);
    } else if (args.runIpbStrong) {
      console.warn('[v8] No strong model found in models.json (tried claude-opus-4-x, claude-opus-4-8). Skipping hive_mind_ipb_strong cell.');
    }
  }

  const dryRun = args.dryRun ?? !process.env.LITELLM_URL;
  const litellmUrl = process.env.LITELLM_URL ?? 'http://localhost:4000';
  const litellmApiKey = process.env.LITELLM_API_KEY ?? 'sk-waggle-dev';

  if (dryRun) {
    console.log('[v8] DRY RUN mode — LLM calls will be stubbed');
  }

  // Judge wiring
  const judgeCosts: JudgeClientCostEntry[] = [];
  let judgeConfig: JudgeConfig | undefined;
  if (!dryRun && args.judge) {
    judgeConfig = {
      kind: 'single',
      model: args.judge,
      client: createJudgeLlmClient({
        litellmUrl, litellmApiKey,
        model: args.judge,
        onCall: entry => judgeCosts.push(entry),
      }),
    };
    console.log(`[v8] judge model: ${args.judge}`);
  }

  // Health check
  if (!dryRun) {
    const judgePingModels = args.judge ? [args.judge] : [];
    const hc = await preCellHealthCheck({
      litellmUrl, litellmApiKey,
      subjectModel: primaryModel.litellmModel,
      judgeModels: judgePingModels,
    });
    if (!hc.ok) {
      const summary = hc.failures.map(f => `${f.endpoint} → ${f.error}`).join('; ');
      throw new Error(
        `[v8:health-check] FAILED: ${summary}. ` +
        `Check LiteLLM proxy + provider keys before spending budget.`,
      );
    }
    console.log(`[v8:health-check] OK in ${hc.durationMs}ms`);
  }

  // Acquire single-runner lock
  const lockSentinel = path.join(harnessRoot(), '..', 'results', '.benchmark-runner');
  const runnerLock: LockHandle = acquireRunnerLock(lockSentinel);
  console.log(`[v8:lock] acquired runner lock (pid=${process.pid})`);

  const runStart = Date.now();
  console.log(`\n[v8] Starting v8 multi-benchmark run`);
  console.log(`[v8] Primary: ${primaryModel.id}${strongModel ? `  Strong: ${strongModel.id}` : ''}`);
  console.log(`[v8] Tracks: ${args.tracks.join(', ')}`);
  console.log(`[v8] Limit per cell: ${Number.isFinite(args.limit) ? args.limit : 'full'}`);
  console.log(`[v8] Seed: ${args.seed}`);

  try {
    // ── Track A0: LongMemEval V1 ──────────────────────────────────────────
    if (args.tracks.includes('a0')) {
      const lmeJsonlPath = args.lmeDataPath ??
        path.join(harnessRoot(), '..', 'data', 'longmemeval', 'longmemeval.jsonl');

      await runTrack(
        {
          track: 'a0',
          datasetId: 'longmemeval',
          budgetUsd: args.budgetA0,
          ingestDataset: true,
          ingestFn: 'lme',
          dataPathOverride: lmeJsonlPath,
        },
        args, primaryModel, strongModel, allDatasets,
        litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
      );
    }

    // ── Track A: BEAM 128K ────────────────────────────────────────────────
    if (args.tracks.includes('a')) {
      const beamJsonlPath = args.beamDataPath ??
        path.join(harnessRoot(), '..', 'data', 'beam', 'beam-128K.jsonl');

      await runTrack(
        {
          track: 'a',
          datasetId: 'beam-128k',
          budgetUsd: args.budgetA,
          ingestDataset: true,
          ingestFn: 'beam',
          dataPathOverride: beamJsonlPath,
        },
        args, primaryModel, strongModel, allDatasets,
        litellmUrl, litellmApiKey, dryRun, judgeConfig, judgeCosts,
      );
    }

    // ── Track B: GAIA 2 ───────────────────────────────────────────────────
    if (args.tracks.includes('b')) {
      console.log('\n[v8:track-b] BLOCKED — SIGALRM issue on Windows/WSL1.');
      console.log('[v8:track-b] Fix: run under WSL2 or Docker. Gate: PM-RATIFY-V8-PHASE1.');
    }

    // ── Track D: Terminal-Bench ───────────────────────────────────────────
    if (args.tracks.includes('d')) {
      console.log('\n[v8:track-d] Terminal-Bench is external infra.');
      console.log('[v8:track-d] Submit waggle scaffold to harborframework/terminal-bench-2-leaderboard.');
      console.log('[v8:track-d] Baseline: little-coder #118/#123 = 24.6% ± 3.2% (Qwen3.6-35B, 2026-05-14).');
    }

  } finally {
    runnerLock.release();
  }

  // Judge summary
  if (judgeCosts.length > 0) {
    const judgeTotalUsd = judgeCosts.reduce((sum, e) => sum + e.usd, 0);
    const judgeOk = judgeCosts.filter(e => e.ok).length;
    console.log(
      `\n[v8:judge-total] calls=${judgeCosts.length} ok=${judgeOk} ` +
      `failed=${judgeCosts.length - judgeOk} total_usd=$${judgeTotalUsd.toFixed(6)}`,
    );
  }

  const totalMs = Date.now() - runStart;
  console.log(`\n[v8:done] Total wall time: ${(totalMs / 1000).toFixed(1)}s`);
  console.log(`[v8:done] Results in: ${path.join(benchRoot(), 'results', 'v8')}/`);
}

main().catch(err => {
  console.error('[v8:fatal]', err?.message ?? err);
  process.exit(1);
});