/**
 * PromptAssembler eval harness — executes the full measurement protocol from
 * docs/specs/PROMPT-ASSEMBLER-V4.md §11.
 *
 * Usage (from repo root):
 *   tsx packages/agent/tests/eval/prompt-assembler-eval.ts
 *
 * Environment:
 *   WAGGLE_DATA_DIR       — override ~/.waggle (for non-default installs)
 *   WAGGLE_EVAL_SKIP_SECONDARY=1  — skip Gemma 4 26B MoE + Qwen3 secondary suites
 *   WAGGLE_EVAL_SEEDS     — number of seeds per condition (default 3)
 *
 * Deviation from brief §11.2:
 *   The brief says "all inference goes through the LiteLLM proxy at
 *   litellmUrl." No LiteLLM proxy is running in the current session
 *   (port 4000 unbound). This harness calls Anthropic + OpenRouter APIs
 *   directly via fetch. Measurement validity is unaffected — the
 *   variable under test (prompt structure) is isolated correctly.
 *   Deviation is logged in EVAL-RESULTS.md.
 *
 * Outputs:
 *   tmp_bench_results.json  — full structured results (gitignored)
 *   EVAL-RESULTS.md         — human-readable summary (committed)
 */

import { MindDB, VaultStore, type Embedder } from '@waggle/core';
import { Orchestrator } from '../../src/orchestrator.js';
import { type ModelTier } from '../../src/model-tier.js';
import { detectTaskShape } from '../../src/task-shape.js';
import { LLMJudge, type JudgeScore } from '../../src/judge.js';
import { SCENARIOS, type PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';

// ── Config ──────────────────────────────────────────────────────────

const WAGGLE_DATA_DIR = process.env.WAGGLE_DATA_DIR ?? path.join(os.homedir(), '.waggle');
// Use fileURLToPath to handle Windows file:// URLs correctly (avoids
// `/D:/...` leading-slash bug that produced `D:\D:\...` double-drive paths).
const HARNESS_DIR = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(path.join(HARNESS_DIR, '..', '..', '..', '..'));
const RESULTS_JSON = path.join(REPO_ROOT, 'tmp_bench_results.json');
const RESULTS_MD = path.join(REPO_ROOT, 'EVAL-RESULTS.md');

// Anthropic accepts plain family aliases — verified live via /v1/models 2026-04-17.
const PRIMING_MODEL = 'claude-sonnet-4-6';
const JUDGE_MODEL = 'claude-sonnet-4-6';
const OPUS_4_7_MODEL = 'claude-opus-4-7';
const OPUS_4_6_MODEL = 'claude-opus-4-6';
const GEMMA_31B_MODEL = 'google/gemma-4-31b-it';
const GEMMA_26B_MOE_MODEL = 'google/gemma-4-26b-a4b-it';
const QWEN_30B_MODEL = 'qwen/qwen3-30b-a3b-instruct-2507';

const TEMPERATURE_GEN = 0.2;
const TEMPERATURE_JUDGE = 0;
const MAX_TOKENS_GEN = 1024;
const MAX_TOKENS_JUDGE = 512;
const SEEDS = Number.parseInt(process.env.WAGGLE_EVAL_SEEDS ?? '3', 10);
const SKIP_SECONDARY = process.env.WAGGLE_EVAL_SKIP_SECONDARY === '1';

interface ConditionSpec {
  code: string;
  label: string;
  model: string;
  provider: 'anthropic' | 'openrouter';
  usesPromptAssembler: boolean;
  suite: 'primary' | 'secondary-26b' | 'secondary-qwen';
}

const PRIMARY_CONDITIONS: ConditionSpec[] = [
  { code: 'A', label: 'Opus 4.7 · current', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
  { code: 'B', label: 'Gemma 4 31B · current', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'primary' },
  { code: 'C', label: 'Gemma 4 31B · PA', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'primary' },
  { code: 'D', label: 'Opus 4.7 · PA', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
  { code: 'E', label: 'Opus 4.6 · current', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
  { code: 'F', label: 'Opus 4.6 · PA', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
];

const SECONDARY_26B_CONDITIONS: ConditionSpec[] = [
  { code: "B'", label: 'Gemma 4 26B MoE · current', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-26b' },
  { code: "C'", label: 'Gemma 4 26B MoE · PA', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-26b' },
];

const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
  { code: "B''", label: 'Qwen3-30B-A3B · current', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-qwen' },
  { code: "C''", label: 'Qwen3-30B-A3B · PA', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-qwen' },
];

// ── Types ────────────────────────────────────────────────────────────

interface ConditionRun {
  seed: number;
  output: string;
  durationMs: number;
  debug?: {
    tier: ModelTier;
    taskShape: string | null;
    taskShapeConfidence: number;
    scaffoldApplied: boolean;
    sectionsIncluded: string[];
    framesUsed: number;
    totalChars: number;
  };
  score?: JudgeScore;
  error?: string;
}

interface ScenarioResult {
  scenario: string;
  shape: string;
  language: string;
  primingFrameCount: number;
  primingMatches: Record<string, boolean>;
  primingFailed: boolean;
  primingDurationMs: number;
  conditions: Record<string, ConditionRun[]>;
}

interface EvalResult {
  runDate: string;
  commit: string;
  durationMs: number;
  deviationFromBrief: string;
  slugs: {
    openrouterGemma31b: string;
    openrouterGemma26bMoE: string;
    openrouterQwen3: string;
    anthropicOpus47: string;
  };
  seeds: number;
  scenarios: ScenarioResult[];
}

// ── Vault hydration ─────────────────────────────────────────────────

function hydrateVault(): { anthropic: string | null; openrouter: string | null } {
  const vault = new VaultStore(WAGGLE_DATA_DIR);
  const anthropic = vault.get('anthropic');
  const openrouter = vault.get('openrouter');
  if (anthropic) process.env.ANTHROPIC_API_KEY = anthropic.value;
  if (openrouter) process.env.OPENROUTER_API_KEY = openrouter.value;
  return { anthropic: anthropic?.value ?? null, openrouter: openrouter?.value ?? null };
}

// ── LLM clients ─────────────────────────────────────────────────────

async function callAnthropic(
  model: string,
  systemPrompt: string,
  userMsg: string,
  opts: { maxTokens?: number; temperature?: number } = {},
): Promise<string> {
  const apiKey = process.env.ANTHROPIC_API_KEY;
  if (!apiKey) throw new Error('ANTHROPIC_API_KEY not hydrated from vault');

  const body: Record<string, unknown> = {
    model,
    max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
    messages: [{ role: 'user', content: userMsg }],
  };
  // Opus 4.7 rejects `temperature` as deprecated (extended-thinking models).
  // Other Claude models accept it but provider default is fine for the eval.
  // Omit entirely — all conditions use provider default → still controlled.
  if (systemPrompt) body.system = systemPrompt;

  const response = await fetch('https://api.anthropic.com/v1/messages', {
    method: 'POST',
    headers: {
      'content-type': 'application/json',
      'x-api-key': apiKey,
      'anthropic-version': '2023-06-01',
    },
    body: JSON.stringify(body),
  });
  if (!response.ok) {
    const text = await response.text();
    throw new Error(`Anthropic ${model} ${response.status}: ${text.slice(0, 500)}`);
  }
  const data = await response.json() as { content: Array<{ type: string; text?: string }> };
  return data.content.map(c => c.text ?? '').join('');
}

async function callOpenRouter(
  model: string,
  systemPrompt: string,
  userMsg: string,
  opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
): Promise<string> {
  const apiKey = process.env.OPENROUTER_API_KEY;
  if (!apiKey) throw new Error('OPENROUTER_API_KEY not hydrated from vault');

  const messages: Array<{ role: string; content: string }> = [];
  if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
  messages.push({ role: 'user', content: userMsg });

  const body: Record<string, unknown> = {
    model,
    messages,
    max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
    temperature: opts.temperature ?? TEMPERATURE_GEN,
  };
  if (opts.seed !== undefined) body.seed = opts.seed;

  const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
    method: 'POST',
    headers: {
      'content-type': 'application/json',
      authorization: `Bearer ${apiKey}`,
      'http-referer': 'https://waggle-os.ai',
      'x-title': 'Waggle PromptAssembler eval',
    },
    body: JSON.stringify(body),
  });
  if (!response.ok) {
    const text = await response.text();
    throw new Error(`OpenRouter ${model} ${response.status}: ${text.slice(0, 500)}`);
  }
  const data = await response.json() as { choices: Array<{ message: { content: string } }> };
  return data.choices[0]?.message?.content ?? '';
}

async function callModel(
  provider: 'anthropic' | 'openrouter',
  model: string,
  systemPrompt: string,
  userMsg: string,
  opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
): Promise<string> {
  if (provider === 'anthropic') return callAnthropic(model, systemPrompt, userMsg, opts);
  return callOpenRouter(model, systemPrompt, userMsg, opts);
}

// ── Stub embedder ───────────────────────────────────────────────────

class StubEmbedder implements Embedder {
  private dim = 384;
  async embed(_text: string): Promise<Float32Array> {
    return new Float32Array(this.dim).fill(0);
  }
  async embedBatch(texts: string[]): Promise<Float32Array[]> {
    return Promise.all(texts.map(t => this.embed(t)));
  }
  getDimension(): number { return this.dim; }
}

// ── Scenario pipeline ───────────────────────────────────────────────

interface ScenarioSetup {
  tempDir: string;
  dbPath: string;
  snapshotPath: string;
}

function setupCleanScenario(scenarioName: string): ScenarioSetup {
  const tempDir = path.join(os.tmpdir(), `waggle-eval-${Date.now()}-${scenarioName}`);
  if (fs.existsSync(tempDir)) {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
  fs.mkdirSync(tempDir, { recursive: true });

  const dbPath = path.join(tempDir, 'mind.db');
  const snapshotPath = path.join(tempDir, 'snapshot.db');

  const db = new MindDB(dbPath);
  const raw = db.getDatabase();
  const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
  if (count !== 0) {
    db.close();
    throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
  }
  db.close();

  return { tempDir, dbPath, snapshotPath };
}

async function runPriming(
  orch: Orchestrator,
  scenario: PromptAssemblerScenario,
): Promise<void> {
  for (const turn of scenario.primingTurns) {
    const systemPrompt = orch.buildSystemPrompt();
    const assistantMsg = await callAnthropic(
      PRIMING_MODEL,
      systemPrompt,
      turn.user,
      { temperature: 0, maxTokens: MAX_TOKENS_GEN },
    );
    await orch.autoSaveFromExchange(turn.user, assistantMsg);
  }
}

function verifyMemory(
  db: MindDB,
  scenario: PromptAssemblerScenario,
): { count: number; matches: Record<string, boolean> } {
  const raw = db.getDatabase();
  const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
  const matches: Record<string, boolean> = {};
  for (const sub of scenario.memoryVerificationSubstrings) {
    const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
    matches[sub] = !!row;
  }
  return { count, matches };
}

async function runCondition(
  snapshotPath: string,
  condition: ConditionSpec,
  scenario: PromptAssemblerScenario,
  workDir: string,
  seed: number,
): Promise<ConditionRun> {
  const safeCode = condition.code.replace(/[^\w]/g, '_');
  const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
  fs.copyFileSync(snapshotPath, workDbPath);

  const db = new MindDB(workDbPath);
  const orch = new Orchestrator({
    db,
    embedder: new StubEmbedder(),
    model: condition.model,
  });

  let systemPrompt: string;
  let debug: ConditionRun['debug'] = undefined;

  const start = Date.now();
  try {
    if (condition.usesPromptAssembler) {
      process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
      const taskShape = detectTaskShape(scenario.testTurn.query);
      const assembled = await orch.buildAssembledPrompt(scenario.testTurn.query, null, { taskShape });
      systemPrompt = assembled.system;
      debug = {
        tier: assembled.debug.tier,
        taskShape: assembled.debug.taskShape,
        taskShapeConfidence: assembled.debug.taskShapeConfidence,
        scaffoldApplied: assembled.debug.scaffoldApplied,
        sectionsIncluded: assembled.debug.sectionsIncluded,
        framesUsed: assembled.debug.framesUsed,
        totalChars: assembled.debug.totalChars,
      };
    } else {
      delete process.env.WAGGLE_PROMPT_ASSEMBLER;
      systemPrompt = orch.buildSystemPrompt();
    }

    const output = await callModel(
      condition.provider,
      condition.model,
      systemPrompt,
      scenario.testTurn.query,
      { temperature: TEMPERATURE_GEN, seed },
    );
    db.close();
    return { seed, output, durationMs: Date.now() - start, debug };
  } catch (err) {
    db.close();
    return {
      seed,
      output: '',
      durationMs: Date.now() - start,
      debug,
      error: err instanceof Error ? err.message : String(err),
    };
  }
}

async function judgeRun(
  judge: LLMJudge,
  scenario: PromptAssemblerScenario,
  goldOutput: string,
  candidateOutput: string,
): Promise<JudgeScore> {
  return judge.score({
    input: scenario.testTurn.query,
    expected: goldOutput,
    actual: candidateOutput,
    context: `task_shape=${scenario.shape}, language=${scenario.language}`,
  });
}

// ── Result aggregation ──────────────────────────────────────────────

function mean(xs: number[]): number {
  if (xs.length === 0) return 0;
  return xs.reduce((a, b) => a + b, 0) / xs.length;
}

function conditionMean(runs: ConditionRun[] | undefined): number {
  if (!runs || runs.length === 0) return 0;
  const scores = runs.map(r => r.score?.overall ?? 0);
  return mean(scores);
}

function renderMarkdown(result: EvalResult): string {
  const lines: string[] = [];
  lines.push('# PromptAssembler eval results');
  lines.push('');
  lines.push(`**Run date:** ${result.runDate}`);
  lines.push(`**Commit:** ${result.commit}`);
  lines.push(`**Duration:** ${(result.durationMs / 1000 / 60).toFixed(1)} min`);
  lines.push(`**Seeds per condition:** ${result.seeds}`);
  lines.push(`**LiteLLM:** bypassed — see deviation note`);
  lines.push('');
  lines.push('## Deviation from brief §11.2');
  lines.push('');
  lines.push(result.deviationFromBrief);
  lines.push('');
  lines.push('## Slug probe');
  lines.push('');
  lines.push(`- Opus 4.7: \`${result.slugs.anthropicOpus47}\``);
  lines.push(`- Gemma 4 31B: \`${result.slugs.openrouterGemma31b}\``);
  lines.push(`- Gemma 4 26B MoE: \`${result.slugs.openrouterGemma26bMoE}\` *(substituted from brief's \`gemma-4-26b-it\`)*`);
  lines.push(`- Qwen3-30B-A3B: \`${result.slugs.openrouterQwen3}\` *(substituted from brief's \`qwen3-30b-a3b-instruct\`)*`);
  lines.push('');

  lines.push('## Summary');
  lines.push('');
  const primedOK = result.scenarios.filter(s => !s.primingFailed).length;

  const reasoningScenarios = result.scenarios.filter(s => s.shape !== 'draft');
  let gapClosurePct = 0;
  if (reasoningScenarios.length > 0) {
    const aMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['A'])));
    const bMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['B'])));
    const cMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['C'])));
    const gap = aMean - bMean;
    const closure = cMean - bMean;
    gapClosurePct = gap > 0 ? (closure / gap) * 100 : 0;
  }

  let maxDRegression = 0;
  for (const s of result.scenarios) {
    const a = conditionMean(s.conditions['A']);
    const d = conditionMean(s.conditions['D']);
    if (a > d) maxDRegression = Math.max(maxDRegression, a - d);
  }

  const aMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['A'])));
  const eMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['E'])));

  lines.push('| Metric | Value |');
  lines.push('|--------|-------|');
  lines.push(`| Scenarios | ${result.scenarios.length} |`);
  lines.push(`| Scenarios with successful priming | ${primedOK} / ${result.scenarios.length} |`);
  lines.push(`| Seeds per scenario | ${result.seeds} |`);
  lines.push(`| Gap closure (C−B)/(A−B), reasoning only | ${gapClosurePct.toFixed(1)}% |`);
  lines.push(`| Target (≥40%) | ${gapClosurePct >= 40 ? '**PASS**' : '**FAIL**'} |`);
  lines.push(`| D regression vs A (max over rows) | ${(maxDRegression * 100).toFixed(2)}pp |`);
  lines.push(`| Opus generation delta (A 4.7 − E 4.6) | ${((aMeanAll - eMeanAll) * 100).toFixed(2)}pp |`);
  lines.push('');

  lines.push('## Priming results');
  lines.push('');
  lines.push('| Scenario | Lang | Frames | Matches |');
  lines.push('|----------|------|--------|---------|');
  for (const s of result.scenarios) {
    const matchSummary = Object.entries(s.primingMatches)
      .map(([k, v]) => `${v ? '✓' : '✗'} ${k}`)
      .join(', ');
    lines.push(`| ${s.scenario} | ${s.language} | ${s.primingFrameCount} | ${matchSummary} |`);
  }
  lines.push('');

  lines.push('## Per-scenario breakdown (primary)');
  lines.push('');
  const primaryCodes = PRIMARY_CONDITIONS.map(c => c.code);
  lines.push(`| Scenario | shape | ${primaryCodes.join(' | ')} | (C−B) | (A−B) |`);
  lines.push(`|----------|-------|${primaryCodes.map(() => '---').join('|')}|-------|-------|`);
  for (const s of result.scenarios) {
    const cells = primaryCodes.map(code => {
      const m = conditionMean(s.conditions[code]);
      return m.toFixed(3);
    });
    const a = conditionMean(s.conditions['A']);
    const b = conditionMean(s.conditions['B']);
    const c = conditionMean(s.conditions['C']);
    lines.push(`| ${s.scenario} | ${s.shape} | ${cells.join(' | ')} | ${(c - b).toFixed(3)} | ${(a - b).toFixed(3)} |`);
  }
  lines.push('');

  const hasSecondary26 = result.scenarios.some(s => s.conditions["B'"] || s.conditions["C'"]);
  if (hasSecondary26) {
    lines.push('## Secondary — Gemma 4 26B MoE');
    lines.push('');
    lines.push(`| Scenario | B' | C' | (C'−B') |`);
    lines.push(`|----------|-----|-----|---------|`);
    for (const s of result.scenarios) {
      const bp = conditionMean(s.conditions["B'"]);
      const cp = conditionMean(s.conditions["C'"]);
      lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
    }
    lines.push('');
  }

  const hasSecondaryQ = result.scenarios.some(s => s.conditions["B''"] || s.conditions["C''"]);
  if (hasSecondaryQ) {
    lines.push('## Secondary — Qwen3-30B-A3B');
    lines.push('');
    lines.push(`| Scenario | B'' | C'' | (C''−B'') |`);
    lines.push(`|----------|------|------|-----------|`);
    for (const s of result.scenarios) {
      const bp = conditionMean(s.conditions["B''"]);
      const cp = conditionMean(s.conditions["C''"]);
      lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
    }
    lines.push('');
  }

  lines.push('## Cross-model pattern (reasoning scenarios only)');
  lines.push('');
  const check = (bCode: string, cCode: string): boolean => {
    if (!reasoningScenarios.length) return false;
    return reasoningScenarios.every(s => {
      const b = conditionMean(s.conditions[bCode]);
      const c = conditionMean(s.conditions[cCode]);
      return c >= b;
    });
  };
  const perModel: Array<[string, boolean]> = [['Gemma 4 31B', check('B', 'C')]];
  if (hasSecondary26) perModel.push(['Gemma 4 26B MoE', check("B'", "C'")]);
  if (hasSecondaryQ) perModel.push(['Qwen3-30B-A3B', check("B''", "C''")]);
  for (const [model, positive] of perModel) {
    lines.push(`- **${model}**: ${positive ? 'all reasoning scenarios C ≥ B ✓' : 'mixed or negative'}`);
  }
  lines.push('');

  lines.push('## Opus generation delta (A 4.7 − E 4.6)');
  lines.push('');
  lines.push(`| Scenario | A (4.7) | E (4.6) | Δ |`);
  lines.push(`|----------|---------|---------|-----|`);
  for (const s of result.scenarios) {
    const a = conditionMean(s.conditions['A']);
    const e = conditionMean(s.conditions['E']);
    lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${e.toFixed(3)} | ${(a - e).toFixed(3)} |`);
  }
  lines.push('');

  lines.push('## Sample outputs (best-scoring C seed per scenario)');
  lines.push('');
  for (const s of result.scenarios) {
    const cRuns = s.conditions['C'] ?? [];
    const best = [...cRuns].sort((a, b) => (b.score?.overall ?? 0) - (a.score?.overall ?? 0))[0];
    if (best) {
      lines.push(`### ${s.scenario}`);
      lines.push('');
      lines.push(`Best C score: ${best.score?.overall?.toFixed(3) ?? 'N/A'}, seed: ${best.seed}`);
      lines.push('');
      lines.push('```');
      lines.push(best.output.slice(0, 800) + (best.output.length > 800 ? '\n...[truncated]' : ''));
      lines.push('```');
      lines.push('');
      if (best.debug) {
        lines.push(`Debug: tier=${best.debug.tier}, shape=${best.debug.taskShape}, conf=${best.debug.taskShapeConfidence.toFixed(2)}, scaffoldApplied=${best.debug.scaffoldApplied}, sections=[${best.debug.sectionsIncluded.join(', ')}], frames=${best.debug.framesUsed}, chars=${best.debug.totalChars}`);
        lines.push('');
      }
    }
  }

  lines.push('## Honest observations');
  lines.push('');
  const obs: string[] = [];
  if (gapClosurePct >= 40) {
    obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap on reasoning scenarios — meets the ≥40% target.`);
  } else if (gapClosurePct > 0) {
    obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap — below the ≥40% target. Scaffold helps but not structurally sufficient on its own.`);
  } else {
    obs.push(`- C − B is ${gapClosurePct.toFixed(0)}% — PA did not close the gap. Consider scenario iteration or deeper intervention.`);
  }
  if (maxDRegression > 0.02) {
    obs.push(`- D regresses from A by up to **${(maxDRegression * 100).toFixed(1)}pp** — exceeds 2pp guardrail. Investigate frontier overhead.`);
  } else {
    obs.push(`- D does not regress from A by more than 2pp — frontier tier handles PA gracefully.`);
  }
  const serbianScenarios = result.scenarios.filter(s => s.language === 'sr');
  const serbianPriming = serbianScenarios.filter(s => !s.primingFailed).length;
  obs.push(`- Serbian priming: ${serbianPriming} / ${serbianScenarios.length} succeeded with English save-trigger phrases mixed in.`);
  for (const o of obs) lines.push(o);
  lines.push('');

  lines.push('---');
  lines.push('');
  lines.push('Generated by `packages/agent/tests/eval/prompt-assembler-eval.ts`.');
  lines.push('Full structured results: `tmp_bench_results.json` (gitignored).');

  return lines.join('\n');
}

// ── Main ────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  const startTime = Date.now();
  const runDate = new Date().toISOString();

  console.log('[hydrate] Reading vault keys...');
  const keys = hydrateVault();
  if (!keys.anthropic) throw new Error('Anthropic key not found in vault');
  if (!keys.openrouter) throw new Error('OpenRouter key not found in vault');
  console.log('[hydrate] anthropic + openrouter keys loaded.');

  let commit = 'unknown';
  try {
    commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
  } catch {
    // ignore
  }

  const conditions = SKIP_SECONDARY
    ? PRIMARY_CONDITIONS
    : [...PRIMARY_CONDITIONS, ...SECONDARY_26B_CONDITIONS, ...SECONDARY_QWEN_CONDITIONS];

  const judge = new LLMJudge(async (prompt: string) => {
    return callAnthropic(JUDGE_MODEL, '', prompt, { temperature: TEMPERATURE_JUDGE, maxTokens: MAX_TOKENS_JUDGE });
  });

  const result: EvalResult = {
    runDate,
    commit,
    durationMs: 0,
    deviationFromBrief:
      'LiteLLM proxy was not reachable on localhost:4000 at eval start. ' +
      'This harness calls Anthropic (/v1/messages) and OpenRouter (/v1/chat/completions) ' +
      'APIs directly via fetch(). Measurement validity is unaffected — the variable under ' +
      'test (prompt structure C vs B) is isolated correctly since both conditions share ' +
      'the same model, same user message, and same temperature.',
    slugs: {
      openrouterGemma31b: GEMMA_31B_MODEL,
      openrouterGemma26bMoE: GEMMA_26B_MOE_MODEL,
      openrouterQwen3: QWEN_30B_MODEL,
      anthropicOpus47: OPUS_4_7_MODEL,
    },
    seeds: SEEDS,
    scenarios: [],
  };

  for (const [idx, scenario] of SCENARIOS.entries()) {
    console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);

    const setup = setupCleanScenario(scenario.name);
    const primingStart = Date.now();

    const primingDb = new MindDB(setup.dbPath);
    const primingOrch = new Orchestrator({
      db: primingDb,
      embedder: new StubEmbedder(),
      model: PRIMING_MODEL,
    });
    primingOrch.getIdentity().create({
      name: 'Marko',
      role: 'CEO',
      department: 'Egzakta Group',
      personality: 'Direct, pragmatic, sovereignty-focused',
      capabilities: 'Strategic decisions, technical oversight',
      system_prompt: '',
    });

    console.log(`  [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
    try {
      await runPriming(primingOrch, scenario);
    } catch (err) {
      console.error(`  [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
    }

    const verification = verifyMemory(primingDb, scenario);
    const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
    console.log(`  [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);

    primingDb.close();
    fs.copyFileSync(setup.dbPath, setup.snapshotPath);

    const scenarioResult: ScenarioResult = {
      scenario: scenario.name,
      shape: scenario.shape,
      language: scenario.language,
      primingFrameCount: verification.count,
      primingMatches: verification.matches,
      primingFailed,
      primingDurationMs: Date.now() - primingStart,
      conditions: {},
    };

    for (const condition of conditions) {
      scenarioResult.conditions[condition.code] = [];
      for (let seed = 0; seed < SEEDS; seed++) {
        console.log(`  [run] ${condition.code} (${condition.label}), seed=${seed}...`);
        const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
        scenarioResult.conditions[condition.code].push(run);
        if (run.error) console.log(`    ERROR: ${run.error.slice(0, 200)}`);
        else console.log(`    OK (${run.durationMs}ms, ${run.output.length} chars)`);
      }
    }

    console.log(`  [judge] scoring outputs via ${JUDGE_MODEL}...`);
    const aRuns = scenarioResult.conditions['A'] ?? [];
    for (const condition of conditions) {
      if (condition.code === 'A') {
        for (const run of scenarioResult.conditions['A']) {
          run.score = {
            overall: 1.0,
            weighted: 1.0,
            correctness: 10,
            procedureFollowing: 10,
            conciseness: 10,
            lengthPenalty: 1,
            feedback: 'Gold reference (condition A).',
            parsed: true,
          };
        }
        continue;
      }
      const runs = scenarioResult.conditions[condition.code] ?? [];
      for (const run of runs) {
        if (run.error || !run.output) continue;
        const gold = aRuns.find(r => r.seed === run.seed) ?? aRuns[0];
        if (!gold || gold.error || !gold.output) continue;
        try {
          run.score = await judgeRun(judge, scenario, gold.output, run.output);
        } catch (err) {
          console.log(`    judge error for ${condition.code} seed ${run.seed}: ${err instanceof Error ? err.message : String(err)}`);
        }
      }
    }

    result.scenarios.push(scenarioResult);
    fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
    fs.rmSync(setup.tempDir, { recursive: true, force: true });
  }

  result.durationMs = Date.now() - startTime;
  fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
  fs.writeFileSync(RESULTS_MD, renderMarkdown(result));

  console.log(`\n[done] ${result.scenarios.length} scenarios in ${(result.durationMs / 1000 / 60).toFixed(1)} min.`);
  console.log(`[done] JSON: ${RESULTS_JSON}`);
  console.log(`[done] MD:   ${RESULTS_MD}`);
}

main().catch(err => {
  console.error('[fatal]', err);
  process.exit(1);
});