#!/usr/bin/env tsx
/**
 * GEPA Faza 1 — NULL-baseline runner.
 *
 * Per launch decision §G step 6 + §F + §A.5/A.7/A.10.
 *
 * Per Amendment 4 + binding texture-audit verdict, the corpus is 50/50 (PM
 * ratified). NULL-baseline measures each of 5 baseline prompt-shapes against
 * 8 stratified instances from the corpus to establish per-shape trio_strict_pass
 * baseline rate.
 *
 * --------------------------------------------------------------------------
 * MULTI-STEP MODE (vs solo) — design rationale
 * --------------------------------------------------------------------------
 *
 * Brief §2 says "Cell scope Faza 1: H3 only" and pilot Cell C labels H3 as
 * "Qwen solo". HOWEVER, Amendment 2 §3 retrieval_engagement_bonus only makes
 * sense if a retrieval tool is present (in solo mode, retrieval_calls always
 * = 0 → bonus pinned at -0.05 → fitness function cannot discriminate
 * candidates). Amendment 2 §6 Phase 5 forward-record acceptance criteria
 * (engagement parity ≥ Opus + score parity narrowed by ≥0.30 H4 trio_mean
 * delta) explicitly invoke retrieval-mode metrics.
 *
 * Resolution (BINDING for this runner): NULL-baseline runs in MULTI-STEP
 * mode with retrieval tool available. The "H3 cell" in Faza 1 GEPA context
 * means "Qwen-targeted evaluation with retrieval available", not pilot Cell
 * C strict "Qwen solo". This reconciles brief §2 with Amendment 2 + Phase 5
 * forward record, mirrors Phase 4.5 empirical setup (Cells B/D had
 * retrieval), and makes Amendment 2 fitness function meaningful.
 *
 * Documented in Checkpoint A halt-and-PM report for PM ratification or pivot.
 *
 * --------------------------------------------------------------------------
 * Cost projection (PM ratified ~$20):
 * --------------------------------------------------------------------------
 *
 * 5 shapes × 8 instances × ($0.50/eval avg) ≈ $20 expected
 *  - Subject: Qwen 3.6 35B-A3B (DashScope direct, ~$0.001/call × 2-3 calls)
 *  - Trio judges: Opus 4.7 + GPT-5.4 + MiniMax M2.7 × ($0.05/call avg) = $0.15/eval
 *  - Per pilot 2026-04-26 cost average = $0.47/cell
 *
 * Halt threshold (per launch decision §D + Amendment 3): if cumulative > $26
 * (30% over $20 expected), halt-and-PM per super-linear sub-rule.
 *
 * --------------------------------------------------------------------------
 * Sampling design:
 * --------------------------------------------------------------------------
 *
 * Same 8 instances across all 5 shapes (controlled comparison; trio_mean delta
 * is purely shape-attributable). Deterministic Mulberry32 with seed=42, then
 * take first 8 of shuffled corpus. Held-out 5 = next 5 (instances 9-13)
 * after the 8 — kept for Faza 1 §F.4 held-out validation.
 *
 * --------------------------------------------------------------------------
 * Usage:
 * --------------------------------------------------------------------------
 *
 *   npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --dry-run
 *     # No LLM call. Validates sampling + shape resolution + substrate setup.
 *
 *   npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --probe
 *     # Single (shape=qwen-thinking, instance=0). ~$0.50. Validates round-trip.
 *
 *   npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --all
 *     # Full 5×8 = 40 evaluations. ~$20 expected, $26 halt.
 */

import * as fs from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';

import {
  MindDB,
  FrameStore,
  SessionStore,
  HybridSearch,
  createOllamaEmbedder,
  type Embedder,
} from '@waggle/core';
import {
  runRetrievalAgentLoop,
  type LlmCallFn,
  type LlmCallInput,
  type LlmCallResult as AgentLlmCallResult,
  type RetrievalSearchFn,
  type AgentRunResult,
} from '@waggle/agent';

import { REGISTRY, selectShape } from '../../../../packages/agent/src/prompt-shapes/selector.js';
import { type PromptShape } from '../../../../packages/agent/src/prompt-shapes/types.js';

import { type CorpusInstance } from '../../src/faza-1/corpus.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const REPO_ROOT = path.resolve(__dirname, '../../../..');

const CORPUS_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl');
const OUT_DIR = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/null-baseline');
const OUT_JSONL = path.join(OUT_DIR, 'null-baseline-eval.jsonl');
const RUN_LOG = path.join(OUT_DIR, 'null-baseline-run.log');
const SUMMARY_JSON = path.join(OUT_DIR, 'null-baseline-summary.json');
const SCRATCH_DIR = path.join(REPO_ROOT, 'tmp/gepa-faza1-null-baseline');

const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
const OLLAMA_URL = 'http://localhost:11434';
const EMBEDDER_MODEL = 'nomic-embed-text';

const SAMPLING_SEED = 42;
const N_PER_SHAPE = 8;

// Shapes in fixed evaluation order (matches manifest v7 §gepa.shape_scope.targets)
const SHAPES = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple'] as const;
type ShapeName = typeof SHAPES[number];

// Subject inheritance per manifest v7 §subject (= pilot 2026-04-26 runner SHA 8a6251e2)
const SUBJECT_ALIAS = 'qwen3.6-35b-a3b-via-dashscope-direct';
const SUBJECT_MAX_TOKENS = 16000;
const SUBJECT_THINKING = true;

// Judges inheritance per manifest v7 §judges (= pilot runner line 626)
const JUDGES = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
const JUDGE_MAX_TOKENS = 3000;
const JUDGE_RETRIES = 3;

// Multi-step orchestration
const MAX_STEPS = 5;
const MAX_RETRIEVALS_PER_STEP = 8;
const PER_CALL_HALT_USD = 0.40;
const PER_CELL_HALT_USD = 1.00;

// Cost halt per launch decision §D (super-linear sub-rule per A.7)
const COST_HALT_USD = 26.0;  // 30% over $20 expected

// Pricing (per pilot runner line 128-133)
const MODEL_PRICING: Record<string, { in: number; out: number }> = {
  'claude-opus-4-7':                { in: 15.0, out: 75.0 },
  'gpt-5.4':                        { in: 2.5,  out: 10.0 },
  'minimax-m27-via-openrouter':     { in: 0.7,  out: 2.8 },
  'qwen3.6-35b-a3b-via-dashscope-direct': { in: 0.20, out: 0.80 },
  'qwen3.6-35b-a3b-via-openrouter':       { in: 0.6, out: 2.4 },
};

const MANIFEST_ANCHOR = 'manifest-v7-gepa-faza1';
const MANIFEST_SHA_AMENDMENT_4 = '1f7a6d6fa01403f6c8d6855893adbfa5e82898a81b7583cfa55628e5eba60196';

// ── Logging ────────────────────────────────────────────────────────────────

function log(msg: string): void {
  const line = `[${new Date().toISOString()}] ${msg}\n`;
  try { fs.appendFileSync(RUN_LOG, line); } catch { /* dir not yet created */ }
  process.stderr.write(line);
}

// ── CLI ────────────────────────────────────────────────────────────────────

interface Args {
  mode: 'dry-run' | 'probe' | 'all';
  probeShape?: ShapeName;
  probeInstanceIdx?: number;
}

function parseArgs(argv: string[]): Args {
  let mode: Args['mode'] = 'dry-run';
  let probeShape: ShapeName | undefined;
  let probeInstanceIdx: number | undefined;
  for (let i = 0; i < argv.length; i++) {
    const flag = argv[i];
    const next = argv[i + 1];
    switch (flag) {
      case '--dry-run': mode = 'dry-run'; break;
      case '--probe':   mode = 'probe'; break;
      case '--all':     mode = 'all'; break;
      case '--probe-shape': probeShape = next as ShapeName; i++; break;
      case '--probe-instance': probeInstanceIdx = Number(next); i++; break;
    }
  }
  return { mode, probeShape, probeInstanceIdx };
}

// ── Deterministic sampling (Mulberry32, seed=42) ───────────────────────────

function mulberry32(seed: number): () => number {
  let t = seed >>> 0;
  return () => {
    t = (t + 0x6d2b79f5) >>> 0;
    let r = t;
    r = Math.imul(r ^ (r >>> 15), r | 1);
    r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
    return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
  };
}

function deterministicShuffle<T>(items: ReadonlyArray<T>, seed: number): T[] {
  const arr = [...items];
  const rand = mulberry32(seed);
  for (let i = arr.length - 1; i > 0; i--) {
    const j = Math.floor(rand() * (i + 1));
    [arr[i], arr[j]] = [arr[j], arr[i]];
  }
  return arr;
}

function loadCorpus(): CorpusInstance[] {
  const text = fs.readFileSync(CORPUS_JSONL, 'utf-8');
  return text.trim().split(/\n+/).filter(Boolean).map(l => JSON.parse(l) as CorpusInstance);
}

/**
 * Sample N=8 instances deterministically. Same set across all shapes (controlled
 * comparison). Held-out 5 are the next 5 after the sample (instances 9-13).
 */
function sampleInstances(corpus: CorpusInstance[], n: number, seed: number): CorpusInstance[] {
  const shuffled = deterministicShuffle(corpus, seed);
  return shuffled.slice(0, n);
}

// ── LiteLLM call adapter ──────────────────────────────────────────────────

const llmCall: LlmCallFn = async (input: LlmCallInput): Promise<AgentLlmCallResult> => {
  const masterKey = process.env.LITELLM_MASTER_KEY;
  if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');

  const { model, messages } = input;
  const isQwen = model.includes('qwen');
  const maxTokens = input.maxTokens ?? (isQwen ? SUBJECT_MAX_TOKENS : 4096);
  const thinking = input.thinking ?? (isQwen ? SUBJECT_THINKING : true);

  const payload: Record<string, unknown> = { model, messages, max_tokens: maxTokens };

  if (model.startsWith('claude-opus')) {
    payload.temperature = 1.0;
  } else if (model === 'gpt-5.4' || model === 'minimax-m27-via-openrouter') {
    // omit temperature — reasoning-model defaults
  } else {
    payload.temperature = input.temperature ?? 0.3;
  }

  if (isQwen) {
    payload.extra_body = { enable_thinking: thinking };
  }

  const started = Date.now();
  let lastErr: string | undefined;
  for (let attempt = 0; attempt < 2; attempt++) {
    try {
      const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
        body: JSON.stringify(payload),
      });
      const d: any = await resp.json();
      if ('error' in d) {
        lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
        if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
        continue;
      }
      const content = d.choices?.[0]?.message?.content ?? '';
      const usage = d.usage ?? {};
      const inTok = usage.prompt_tokens ?? 0;
      const outTok = usage.completion_tokens ?? 0;
      const pricing = MODEL_PRICING[model] ?? { in: 1, out: 4 };
      const costUsd = (inTok * pricing.in + outTok * pricing.out) / 1_000_000;
      return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
    } catch (e) {
      lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
      if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
      continue;
    }
  }
  return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now() - started, error: lastErr ?? 'unknown error' };
};

// ── Trio judging (mirrors pilot 2026-04-26 runner line 545+) ───────────────

const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent's response to a complex knowledge work task. The persona, scenario, materials, and question are provided. The response was generated under one of four configurations (revealed only after scoring): {model_only | model + memory + agent harness} × {Opus 4.7 | Qwen 3.6 35B-A3B}.

You do NOT know which configuration produced this response. Score blind.

Read the persona/scenario/question (provided), skim the materials (provided), then read the response carefully (provided).

Score the response on six dimensions, Likert 1-5:

1. COMPLETENESS — engagement with all material
2. ACCURACY — faithfulness to source materials, no hallucinations
3. SYNTHESIS — connections across inputs, not isolated treatment
4. JUDGMENT — defensible recommendations, tradeoffs acknowledged
5. ACTIONABILITY — would the persona act on this tomorrow
6. STRUCTURE — organization and readability

Output JSON only:
{
  "completeness": <1-5>,
  "accuracy": <1-5>,
  "synthesis": <1-5>,
  "judgment": <1-5>,
  "actionability": <1-5>,
  "structure": <1-5>,
  "rationale": "<1-2 sentences explaining the lowest scoring dimension>",
  "overall_verdict": "<one of: PASS_STRONG | PASS_ADEQUATE | FAIL_WEAK | FAIL_CRITICAL>"
}

PASS_STRONG: mean >= 4.0
PASS_ADEQUATE: mean 3.5-3.99
FAIL_WEAK: mean 2.5-3.49
FAIL_CRITICAL: mean < 2.5

[PERSONA + SCENARIO + QUESTION]
###PERSONA_SCENARIO_QUESTION###

[MATERIALS]
###MATERIALS###

[RESPONSE TO EVALUATE]
###RESPONSE###`;

interface JudgeVerdict {
  completeness: number;
  accuracy: number;
  synthesis: number;
  judgment: number;
  actionability: number;
  structure: number;
  rationale: string;
  overall_verdict: string;
  mean: number;
}

interface JudgeRecord extends JudgeVerdict {
  judge_model: string;
  judge_cost_usd: number;
  judge_latency_ms: number;
  judge_retries: number;
}

const ZERO_VERDICT: JudgeVerdict = {
  completeness: 0, accuracy: 0, synthesis: 0, judgment: 0, actionability: 0, structure: 0,
  rationale: '__JUDGE_FAILED__', overall_verdict: 'FAIL_CRITICAL', mean: 0,
};

function parseJudgeJson(text: string): JudgeVerdict | null {
  const m = text.match(/\{[\s\S]*\}/);
  if (!m) return null;
  try {
    const obj = JSON.parse(m[0]);
    const dims = ['completeness', 'accuracy', 'synthesis', 'judgment', 'actionability', 'structure'] as const;
    for (const d of dims) {
      if (typeof obj[d] !== 'number' || obj[d] < 1 || obj[d] > 5) return null;
    }
    const mean = dims.reduce((s, d) => s + obj[d], 0) / dims.length;
    return {
      completeness: obj.completeness, accuracy: obj.accuracy, synthesis: obj.synthesis,
      judgment: obj.judgment, actionability: obj.actionability, structure: obj.structure,
      rationale: typeof obj.rationale === 'string' ? obj.rationale : '',
      overall_verdict: typeof obj.overall_verdict === 'string' ? obj.overall_verdict : '',
      mean,
    };
  } catch { return null; }
}

function buildJudgePrompt(instance: CorpusInstance, response: string): string {
  return JUDGE_PROMPT_TEMPLATE
    .replace('###PERSONA_SCENARIO_QUESTION###', `${instance.personaText}\n\nQUESTION: ${instance.question}`)
    .replace('###MATERIALS###', instance.materialsConcat)
    .replace('###RESPONSE###', response);
}

async function runJudge(judgeModel: string, prompt: string): Promise<JudgeRecord> {
  let lastErr = '';
  let totalCost = 0, totalLatency = 0;
  for (let attempt = 0; attempt < JUDGE_RETRIES; attempt++) {
    const r = await llmCall({ model: judgeModel, messages: [{ role: 'user', content: prompt }], maxTokens: JUDGE_MAX_TOKENS, thinking: false });
    totalCost += r.costUsd; totalLatency += r.latencyMs;
    if (r.error) { lastErr = `attempt ${attempt + 1}: ${r.error}`; continue; }
    const parsed = parseJudgeJson(r.content);
    if (parsed) {
      return { ...parsed, judge_model: judgeModel, judge_cost_usd: totalCost, judge_latency_ms: totalLatency, judge_retries: attempt };
    }
    lastErr = `attempt ${attempt + 1}: malformed JSON: ${r.content.slice(0, 100)}`;
  }
  log(`[judge ${judgeModel}] FAILED after ${JUDGE_RETRIES}: ${lastErr}`);
  return { ...ZERO_VERDICT, judge_model: judgeModel, judge_cost_usd: totalCost, judge_latency_ms: totalLatency, judge_retries: JUDGE_RETRIES, rationale: `__JUDGE_FAILED__: ${lastErr}` };
}

interface TrioResult {
  records: JudgeRecord[];
  trioMean: number;
  trioStrictPassII: boolean;  // op (ii) — trio_mean >= 4.0
  trioStrictPassI: boolean;   // op (i) — >=2 of 3 judges with mean >= 3.5
  judgeCostTotal: number;
}

async function judgeTrio(instance: CorpusInstance, response: string): Promise<TrioResult> {
  const prompt = buildJudgePrompt(instance, response);
  const records = await Promise.all(JUDGES.map(j => runJudge(j, prompt)));
  const validMeans = records.filter(r => r.mean > 0).map(r => r.mean);
  const trioMean = validMeans.length > 0 ? validMeans.reduce((s, m) => s + m, 0) / validMeans.length : 0;
  const trioStrictPassII = trioMean >= 4.0;
  const trioStrictPassI = records.filter(r => r.mean >= 3.5).length >= 2;
  const judgeCostTotal = records.reduce((s, r) => s + r.judge_cost_usd, 0);
  return { records, trioMean, trioStrictPassII, trioStrictPassI, judgeCostTotal };
}

// ── Per-eval orchestration: run shape × instance via multi-step ───────────

interface EvalRecord {
  shape: ShapeName;
  instanceId: string;
  instanceCell: CorpusInstance['cell'];
  candidateResponse: string;
  candidateLatencyMs: number;
  candidateTokensIn: number;
  candidateTokensOut: number;
  candidateCostUsd: number;
  loopExhausted: boolean;
  stepsTaken: number;
  retrievalCalls: number;
  judges: { records: JudgeRecord[]; trioMean: number; trioStrictPassII: boolean; trioStrictPassI: boolean; judgeCostTotal: number };
  evalCostUsd: number;
  manifestAnchor: string;
  manifestShaAmendment4: string;
  tsIso: string;
}

async function runOneEval(shape: PromptShape, instance: CorpusInstance, embedder: Embedder): Promise<EvalRecord | { error: string }> {
  const evalId = `${shape.name}__${instance.instanceId}`;
  log(`[${evalId}] start`);

  // Per-eval SQLite + HybridSearch substrate (per pilot runner pattern)
  const dbPath = path.join(SCRATCH_DIR, `eval-${shape.name}-${instance.instanceId}.sqlite`);
  if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
  const db = new MindDB(dbPath);
  const frames = new FrameStore(db);
  const sessions = new SessionStore(db);
  const hybrid = new HybridSearch(db, embedder);

  const gopId = `gepa-faza1-null-${shape.name}-${instance.instanceId}`;
  sessions.ensure(gopId, undefined, `NULL-baseline ${shape.name} on ${instance.instanceId}`);

  // Ingest source documents
  for (const doc of instance.sourceDocuments) {
    const content = `## ${doc.title}\n\n${doc.body}`;
    frames.createIFrame(gopId, content, 'important', 'system');
  }
  log(`[${evalId}] ingested ${instance.sourceDocuments.length} frames`);

  // Retrieval adapter
  const searchAdapter: RetrievalSearchFn = async ({ query, limit }) => {
    const hits = await hybrid.search(query, { limit, gopId });
    const formatted = hits.length > 0
      ? hits.map((sr, i) => `[result ${i + 1}, score ${sr.finalScore.toFixed(3)}]\n${sr.frame.content}`).join('\n\n---\n\n')
      : '';
    return { formattedResults: formatted, resultCount: hits.length };
  };

  // Run multi-step retrieval agent loop.
  // CRITICAL: pass promptShapeOverride to actually apply the per-shape prompt.
  // Without this, runRetrievalAgentLoop calls selectShape(modelAlias) which always
  // resolves to 'qwen-thinking' for Qwen subject (per config), making the runner's
  // shape parameter unused. This bug was discovered post-Checkpoint-A and corrected
  // per PM Option A ratification + manifest v7 Amendment 6.
  let agentResult: AgentRunResult;
  try {
    agentResult = await runRetrievalAgentLoop({
      modelAlias: SUBJECT_ALIAS,
      persona: instance.personaText,
      question: instance.question,
      llmCall,
      search: searchAdapter,
      maxSteps: MAX_STEPS,
      maxRetrievalsPerStep: MAX_RETRIEVALS_PER_STEP,
      perCallHaltUsd: PER_CALL_HALT_USD,
      perCellHaltUsd: PER_CELL_HALT_USD,
      contextTag: evalId,
      promptShapeOverride: shape.name,  // bug fix per Amendment 6
    } as any);
  } catch (e) {
    const msg = `agent loop failed: ${(e as Error).message}`;
    log(`[${evalId}] ${msg}`);
    return { error: msg };
  }

  if (agentResult.errors.length > 0) {
    log(`[${evalId}] agent errors: ${agentResult.errors.join('; ').slice(0, 200)}`);
  }

  log(`[${evalId}] subject_done; retrievals=${agentResult.retrievalCalls} steps=${agentResult.stepsTaken} cost=$${agentResult.totalCostUsd.toFixed(4)} loop_exhausted=${agentResult.loopExhausted}`);

  // Judge response
  const judges = await judgeTrio(instance, agentResult.rawResponse);
  const evalCostUsd = agentResult.totalCostUsd + judges.judgeCostTotal;
  log(`[${evalId}] judged; trio_mean=${judges.trioMean.toFixed(3)} pass_ii=${judges.trioStrictPassII} pass_i=${judges.trioStrictPassI} eval_cost=$${evalCostUsd.toFixed(4)}`);

  return {
    shape: shape.name as ShapeName,
    instanceId: instance.instanceId,
    instanceCell: instance.cell,
    candidateResponse: agentResult.rawResponse,
    candidateLatencyMs: agentResult.totalLatencyMs,
    candidateTokensIn: agentResult.totalTokensIn,
    candidateTokensOut: agentResult.totalTokensOut,
    candidateCostUsd: agentResult.totalCostUsd,
    loopExhausted: agentResult.loopExhausted,
    stepsTaken: agentResult.stepsTaken,
    retrievalCalls: agentResult.retrievalCalls,
    judges,
    evalCostUsd,
    manifestAnchor: MANIFEST_ANCHOR,
    manifestShaAmendment4: MANIFEST_SHA_AMENDMENT_4,
    tsIso: new Date().toISOString(),
  };
}

// ── Aggregate per-shape metrics + κ across batch ──────────────────────────

interface ShapeAggregate {
  shape: ShapeName;
  nEvals: number;
  trioStrictPassRateII: number;       // op (ii) primary
  trioStrictPassRateI: number;        // op (i) supplementary
  meanRetrievalCallsPerTask: number;
  meanCandidateCostUsd: number;
  meanJudgeCostUsd: number;
  meanEvalCostUsd: number;
  totalEvalCostUsd: number;
  loopExhaustedRate: number;
  meanStepsTaken: number;
}

function aggregatePerShape(records: EvalRecord[]): ShapeAggregate[] {
  const byShape = new Map<ShapeName, EvalRecord[]>();
  for (const r of records) {
    if (!byShape.has(r.shape)) byShape.set(r.shape, []);
    byShape.get(r.shape)!.push(r);
  }
  const out: ShapeAggregate[] = [];
  for (const shape of SHAPES) {
    const rs = byShape.get(shape) ?? [];
    if (rs.length === 0) continue;
    const passII = rs.filter(r => r.judges.trioStrictPassII).length;
    const passI = rs.filter(r => r.judges.trioStrictPassI).length;
    const meanRetr = rs.reduce((s, r) => s + r.retrievalCalls, 0) / rs.length;
    const meanCandCost = rs.reduce((s, r) => s + r.candidateCostUsd, 0) / rs.length;
    const meanJudgeCost = rs.reduce((s, r) => s + r.judges.judgeCostTotal, 0) / rs.length;
    const meanEvalCost = rs.reduce((s, r) => s + r.evalCostUsd, 0) / rs.length;
    const totalEvalCost = rs.reduce((s, r) => s + r.evalCostUsd, 0);
    const exhaustedRate = rs.filter(r => r.loopExhausted).length / rs.length;
    const meanSteps = rs.reduce((s, r) => s + r.stepsTaken, 0) / rs.length;
    out.push({
      shape, nEvals: rs.length,
      trioStrictPassRateII: passII / rs.length,
      trioStrictPassRateI: passI / rs.length,
      meanRetrievalCallsPerTask: meanRetr,
      meanCandidateCostUsd: meanCandCost,
      meanJudgeCostUsd: meanJudgeCost,
      meanEvalCostUsd: meanEvalCost,
      totalEvalCostUsd: totalEvalCost,
      loopExhaustedRate: exhaustedRate,
      meanStepsTaken: meanSteps,
    });
  }
  return out;
}

// ── Main ───────────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  const args = parseArgs(process.argv.slice(2));
  fs.mkdirSync(OUT_DIR, { recursive: true });
  fs.mkdirSync(SCRATCH_DIR, { recursive: true });
  if (!fs.existsSync(RUN_LOG)) fs.writeFileSync(RUN_LOG, '');

  const corpus = loadCorpus();
  log(`[loaded] corpus = ${corpus.length} instances from ${CORPUS_JSONL}`);

  const sample = sampleInstances(corpus, N_PER_SHAPE, SAMPLING_SEED);
  log(`[sampled] N=${N_PER_SHAPE} via seed=${SAMPLING_SEED}: ${sample.map(i => i.instanceId).join(', ')}`);

  if (args.mode === 'dry-run') {
    log(`[dry-run] would run ${SHAPES.length} shapes × ${N_PER_SHAPE} instances = ${SHAPES.length * N_PER_SHAPE} evals`);
    for (const shapeName of SHAPES) {
      const shape = REGISTRY[shapeName];
      const sysPrompt = shape.systemPrompt({ persona: sample[0].personaText, question: sample[0].question, isMultiStep: true });
      log(`[dry-run] shape=${shapeName} systemPrompt(multi-step)=${sysPrompt.length}c`);
    }
    log(`[dry-run] OK; no LLM call; cost: $0.00`);
    return;
  }

  // Resume support: skip evals already in JSONL
  const existing = new Set<string>();
  let cumulativeCost = 0;
  let allRecords: EvalRecord[] = [];
  if (fs.existsSync(OUT_JSONL)) {
    for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
      try {
        const r = JSON.parse(line) as EvalRecord;
        const key = `${r.shape}__${r.instanceId}`;
        existing.add(key);
        cumulativeCost += r.evalCostUsd;
        allRecords.push(r);
      } catch { /* skip */ }
    }
    log(`[resume] loaded ${existing.size} existing evals; cumulative cost $${cumulativeCost.toFixed(4)}`);
  }

  const out = fs.createWriteStream(OUT_JSONL, { flags: existing.size > 0 ? 'a' : 'w' });

  const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });

  const targetShapes = args.mode === 'probe' ? [args.probeShape ?? 'qwen-thinking'] : SHAPES;
  const targetInstances = args.mode === 'probe'
    ? [sample[args.probeInstanceIdx ?? 0]]
    : sample;

  for (const shapeName of targetShapes) {
    const shape = REGISTRY[shapeName];
    if (!shape) { log(`[error] shape "${shapeName}" not in REGISTRY`); continue; }
    for (const instance of targetInstances) {
      const key = `${shapeName}__${instance.instanceId}`;
      if (existing.has(key)) {
        log(`[skip] ${key} already in JSONL`);
        continue;
      }
      if (cumulativeCost >= COST_HALT_USD) {
        log(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD} cost halt — stopping`);
        break;
      }
      const result = await runOneEval(shape, instance, embedder);
      if ('error' in result) {
        log(`[skip] ${key} due to: ${result.error}`);
        continue;
      }
      out.write(JSON.stringify(result) + '\n');
      cumulativeCost += result.evalCostUsd;
      allRecords.push(result);
      log(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_HALT_USD} halt; ${allRecords.length} evals total`);
    }
  }
  out.end();

  // Aggregate + summary
  const aggregates = aggregatePerShape(allRecords);
  const summary = {
    manifestAnchor: MANIFEST_ANCHOR,
    manifestShaAmendment4: MANIFEST_SHA_AMENDMENT_4,
    samplingSeed: SAMPLING_SEED,
    nPerShape: N_PER_SHAPE,
    sampledInstanceIds: sample.map(i => i.instanceId),
    totalEvals: allRecords.length,
    totalCostUsd: +cumulativeCost.toFixed(6),
    perShape: aggregates,
    completedAtIso: new Date().toISOString(),
  };
  fs.writeFileSync(SUMMARY_JSON, JSON.stringify(summary, null, 2), 'utf-8');
  log(`[done] ${allRecords.length} evals; total cost $${cumulativeCost.toFixed(4)}; summary written`);
}

main().catch(e => {
  console.error('FATAL:', e);
  process.exit(2);
});