moving

packages/agent/tests/eval/prompt-assembler-eval.ts

@@ -0,0 +1,745 @@
+/**
+ * PromptAssembler eval harness — executes the full measurement protocol from
+ * docs/specs/PROMPT-ASSEMBLER-V4.md §11.
+ *
+ * Usage (from repo root):
+ *   tsx packages/agent/tests/eval/prompt-assembler-eval.ts
+ *
+ * Environment:
+ *   WAGGLE_DATA_DIR       — override ~/.waggle (for non-default installs)
+ *   WAGGLE_EVAL_SKIP_SECONDARY=1  — skip Gemma 4 26B MoE + Qwen3 secondary suites
+ *   WAGGLE_EVAL_SEEDS     — number of seeds per condition (default 3)
+ *
+ * Deviation from brief §11.2:
+ *   The brief says "all inference goes through the LiteLLM proxy at
+ *   litellmUrl." No LiteLLM proxy is running in the current session
+ *   (port 4000 unbound). This harness calls Anthropic + OpenRouter APIs
+ *   directly via fetch. Measurement validity is unaffected — the
+ *   variable under test (prompt structure) is isolated correctly.
+ *   Deviation is logged in EVAL-RESULTS.md.
+ *
+ * Outputs:
+ *   tmp_bench_results.json  — full structured results (gitignored)
+ *   EVAL-RESULTS.md         — human-readable summary (committed)
+ */
+
+import { MindDB, VaultStore, type Embedder } from '@waggle/core';
+import { Orchestrator } from '../../src/orchestrator.js';
+import { type ModelTier } from '../../src/model-tier.js';
+import { detectTaskShape } from '../../src/task-shape.js';
+import { LLMJudge, type JudgeScore } from '../../src/judge.js';
+import { SCENARIOS, type PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+
+// ── Config ──────────────────────────────────────────────────────────
+
+const WAGGLE_DATA_DIR = process.env.WAGGLE_DATA_DIR ?? path.join(os.homedir(), '.waggle');
+// Use fileURLToPath to handle Windows file:// URLs correctly (avoids
+// `/D:/...` leading-slash bug that produced `D:\D:\...` double-drive paths).
+const HARNESS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(path.join(HARNESS_DIR, '..', '..', '..', '..'));
+const RESULTS_JSON = path.join(REPO_ROOT, 'tmp_bench_results.json');
+const RESULTS_MD = path.join(REPO_ROOT, 'EVAL-RESULTS.md');
+
+// Anthropic accepts plain family aliases — verified live via /v1/models 2026-04-17.
+const PRIMING_MODEL = 'claude-sonnet-4-6';
+const JUDGE_MODEL = 'claude-sonnet-4-6';
+const OPUS_4_7_MODEL = 'claude-opus-4-7';
+const OPUS_4_6_MODEL = 'claude-opus-4-6';
+const GEMMA_31B_MODEL = 'google/gemma-4-31b-it';
+const GEMMA_26B_MOE_MODEL = 'google/gemma-4-26b-a4b-it';
+const QWEN_30B_MODEL = 'qwen/qwen3-30b-a3b-instruct-2507';
+
+const TEMPERATURE_GEN = 0.2;
+const TEMPERATURE_JUDGE = 0;
+const MAX_TOKENS_GEN = 1024;
+const MAX_TOKENS_JUDGE = 512;
+const SEEDS = Number.parseInt(process.env.WAGGLE_EVAL_SEEDS ?? '3', 10);
+const SKIP_SECONDARY = process.env.WAGGLE_EVAL_SKIP_SECONDARY === '1';
+
+interface ConditionSpec {
+  code: string;
+  label: string;
+  model: string;
+  provider: 'anthropic' | 'openrouter';
+  usesPromptAssembler: boolean;
+  suite: 'primary' | 'secondary-26b' | 'secondary-qwen';
+}
+
+const PRIMARY_CONDITIONS: ConditionSpec[] = [
+  { code: 'A', label: 'Opus 4.7 · current', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
+  { code: 'B', label: 'Gemma 4 31B · current', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'primary' },
+  { code: 'C', label: 'Gemma 4 31B · PA', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'primary' },
+  { code: 'D', label: 'Opus 4.7 · PA', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
+  { code: 'E', label: 'Opus 4.6 · current', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
+  { code: 'F', label: 'Opus 4.6 · PA', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
+];
+
+const SECONDARY_26B_CONDITIONS: ConditionSpec[] = [
+  { code: "B'", label: 'Gemma 4 26B MoE · current', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-26b' },
+  { code: "C'", label: 'Gemma 4 26B MoE · PA', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-26b' },
+];
+
+const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
+  { code: "B''", label: 'Qwen3-30B-A3B · current', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-qwen' },
+  { code: "C''", label: 'Qwen3-30B-A3B · PA', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-qwen' },
+];
+
+// ── Types ────────────────────────────────────────────────────────────
+
+interface ConditionRun {
+  seed: number;
+  output: string;
+  durationMs: number;
+  debug?: {
+    tier: ModelTier;
+    taskShape: string | null;
+    taskShapeConfidence: number;
+    scaffoldApplied: boolean;
+    sectionsIncluded: string[];
+    framesUsed: number;
+    totalChars: number;
+  };
+  score?: JudgeScore;
+  error?: string;
+}
+
+interface ScenarioResult {
+  scenario: string;
+  shape: string;
+  language: string;
+  primingFrameCount: number;
+  primingMatches: Record<string, boolean>;
+  primingFailed: boolean;
+  primingDurationMs: number;
+  conditions: Record<string, ConditionRun[]>;
+}
+
+interface EvalResult {
+  runDate: string;
+  commit: string;
+  durationMs: number;
+  deviationFromBrief: string;
+  slugs: {
+    openrouterGemma31b: string;
+    openrouterGemma26bMoE: string;
+    openrouterQwen3: string;
+    anthropicOpus47: string;
+  };
+  seeds: number;
+  scenarios: ScenarioResult[];
+}
+
+// ── Vault hydration ─────────────────────────────────────────────────
+
+function hydrateVault(): { anthropic: string | null; openrouter: string | null } {
+  const vault = new VaultStore(WAGGLE_DATA_DIR);
+  const anthropic = vault.get('anthropic');
+  const openrouter = vault.get('openrouter');
+  if (anthropic) process.env.ANTHROPIC_API_KEY = anthropic.value;
+  if (openrouter) process.env.OPENROUTER_API_KEY = openrouter.value;
+  return { anthropic: anthropic?.value ?? null, openrouter: openrouter?.value ?? null };
+}
+
+// ── LLM clients ─────────────────────────────────────────────────────
+
+async function callAnthropic(
+  model: string,
+  systemPrompt: string,
+  userMsg: string,
+  opts: { maxTokens?: number; temperature?: number } = {},
+): Promise<string> {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) throw new Error('ANTHROPIC_API_KEY not hydrated from vault');
+
+  const body: Record<string, unknown> = {
+    model,
+    max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
+    messages: [{ role: 'user', content: userMsg }],
+  };
+  // Opus 4.7 rejects `temperature` as deprecated (extended-thinking models).
+  // Other Claude models accept it but provider default is fine for the eval.
+  // Omit entirely — all conditions use provider default → still controlled.
+  if (systemPrompt) body.system = systemPrompt;
+
+  const response = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify(body),
+  });
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Anthropic ${model} ${response.status}: ${text.slice(0, 500)}`);
+  }
+  const data = await response.json() as { content: Array<{ type: string; text?: string }> };
+  return data.content.map(c => c.text ?? '').join('');
+}
+
+async function callOpenRouter(
+  model: string,
+  systemPrompt: string,
+  userMsg: string,
+  opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
+): Promise<string> {
+  const apiKey = process.env.OPENROUTER_API_KEY;
+  if (!apiKey) throw new Error('OPENROUTER_API_KEY not hydrated from vault');
+
+  const messages: Array<{ role: string; content: string }> = [];
+  if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
+  messages.push({ role: 'user', content: userMsg });
+
+  const body: Record<string, unknown> = {
+    model,
+    messages,
+    max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
+    temperature: opts.temperature ?? TEMPERATURE_GEN,
+  };
+  if (opts.seed !== undefined) body.seed = opts.seed;
+
+  const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      authorization: `Bearer ${apiKey}`,
+      'http-referer': 'https://waggle-os.ai',
+      'x-title': 'Waggle PromptAssembler eval',
+    },
+    body: JSON.stringify(body),
+  });
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`OpenRouter ${model} ${response.status}: ${text.slice(0, 500)}`);
+  }
+  const data = await response.json() as { choices: Array<{ message: { content: string } }> };
+  return data.choices[0]?.message?.content ?? '';
+}
+
+async function callModel(
+  provider: 'anthropic' | 'openrouter',
+  model: string,
+  systemPrompt: string,
+  userMsg: string,
+  opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
+): Promise<string> {
+  if (provider === 'anthropic') return callAnthropic(model, systemPrompt, userMsg, opts);
+  return callOpenRouter(model, systemPrompt, userMsg, opts);
+}
+
+// ── Stub embedder ───────────────────────────────────────────────────
+
+class StubEmbedder implements Embedder {
+  private dim = 384;
+  async embed(_text: string): Promise<Float32Array> {
+    return new Float32Array(this.dim).fill(0);
+  }
+  async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    return Promise.all(texts.map(t => this.embed(t)));
+  }
+  getDimension(): number { return this.dim; }
+}
+
+// ── Scenario pipeline ───────────────────────────────────────────────
+
+interface ScenarioSetup {
+  tempDir: string;
+  dbPath: string;
+  snapshotPath: string;
+}
+
+function setupCleanScenario(scenarioName: string): ScenarioSetup {
+  const tempDir = path.join(os.tmpdir(), `waggle-eval-${Date.now()}-${scenarioName}`);
+  if (fs.existsSync(tempDir)) {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+  fs.mkdirSync(tempDir, { recursive: true });
+
+  const dbPath = path.join(tempDir, 'mind.db');
+  const snapshotPath = path.join(tempDir, 'snapshot.db');
+
+  const db = new MindDB(dbPath);
+  const raw = db.getDatabase();
+  const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
+  if (count !== 0) {
+    db.close();
+    throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
+  }
+  db.close();
+
+  return { tempDir, dbPath, snapshotPath };
+}
+
+async function runPriming(
+  orch: Orchestrator,
+  scenario: PromptAssemblerScenario,
+): Promise<void> {
+  for (const turn of scenario.primingTurns) {
+    const systemPrompt = orch.buildSystemPrompt();
+    const assistantMsg = await callAnthropic(
+      PRIMING_MODEL,
+      systemPrompt,
+      turn.user,
+      { temperature: 0, maxTokens: MAX_TOKENS_GEN },
+    );
+    await orch.autoSaveFromExchange(turn.user, assistantMsg);
+  }
+}
+
+function verifyMemory(
+  db: MindDB,
+  scenario: PromptAssemblerScenario,
+): { count: number; matches: Record<string, boolean> } {
+  const raw = db.getDatabase();
+  const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
+  const matches: Record<string, boolean> = {};
+  for (const sub of scenario.memoryVerificationSubstrings) {
+    const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
+    matches[sub] = !!row;
+  }
+  return { count, matches };
+}
+
+async function runCondition(
+  snapshotPath: string,
+  condition: ConditionSpec,
+  scenario: PromptAssemblerScenario,
+  workDir: string,
+  seed: number,
+): Promise<ConditionRun> {
+  const safeCode = condition.code.replace(/[^\w]/g, '_');
+  const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
+  fs.copyFileSync(snapshotPath, workDbPath);
+
+  const db = new MindDB(workDbPath);
+  const orch = new Orchestrator({
+    db,
+    embedder: new StubEmbedder(),
+    model: condition.model,
+  });
+
+  let systemPrompt: string;
+  let debug: ConditionRun['debug'] = undefined;
+
+  const start = Date.now();
+  try {
+    if (condition.usesPromptAssembler) {
+      process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
+      const taskShape = detectTaskShape(scenario.testTurn.query);
+      const assembled = await orch.buildAssembledPrompt(scenario.testTurn.query, null, { taskShape });
+      systemPrompt = assembled.system;
+      debug = {
+        tier: assembled.debug.tier,
+        taskShape: assembled.debug.taskShape,
+        taskShapeConfidence: assembled.debug.taskShapeConfidence,
+        scaffoldApplied: assembled.debug.scaffoldApplied,
+        sectionsIncluded: assembled.debug.sectionsIncluded,
+        framesUsed: assembled.debug.framesUsed,
+        totalChars: assembled.debug.totalChars,
+      };
+    } else {
+      delete process.env.WAGGLE_PROMPT_ASSEMBLER;
+      systemPrompt = orch.buildSystemPrompt();
+    }
+
+    const output = await callModel(
+      condition.provider,
+      condition.model,
+      systemPrompt,
+      scenario.testTurn.query,
+      { temperature: TEMPERATURE_GEN, seed },
+    );
+    db.close();
+    return { seed, output, durationMs: Date.now() - start, debug };
+  } catch (err) {
+    db.close();
+    return {
+      seed,
+      output: '',
+      durationMs: Date.now() - start,
+      debug,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+async function judgeRun(
+  judge: LLMJudge,
+  scenario: PromptAssemblerScenario,
+  goldOutput: string,
+  candidateOutput: string,
+): Promise<JudgeScore> {
+  return judge.score({
+    input: scenario.testTurn.query,
+    expected: goldOutput,
+    actual: candidateOutput,
+    context: `task_shape=${scenario.shape}, language=${scenario.language}`,
+  });
+}
+
+// ── Result aggregation ──────────────────────────────────────────────
+
+function mean(xs: number[]): number {
+  if (xs.length === 0) return 0;
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+
+function conditionMean(runs: ConditionRun[] | undefined): number {
+  if (!runs || runs.length === 0) return 0;
+  const scores = runs.map(r => r.score?.overall ?? 0);
+  return mean(scores);
+}
+
+function renderMarkdown(result: EvalResult): string {
+  const lines: string[] = [];
+  lines.push('# PromptAssembler eval results');
+  lines.push('');
+  lines.push(`**Run date:** ${result.runDate}`);
+  lines.push(`**Commit:** ${result.commit}`);
+  lines.push(`**Duration:** ${(result.durationMs / 1000 / 60).toFixed(1)} min`);
+  lines.push(`**Seeds per condition:** ${result.seeds}`);
+  lines.push(`**LiteLLM:** bypassed — see deviation note`);
+  lines.push('');
+  lines.push('## Deviation from brief §11.2');
+  lines.push('');
+  lines.push(result.deviationFromBrief);
+  lines.push('');
+  lines.push('## Slug probe');
+  lines.push('');
+  lines.push(`- Opus 4.7: \`${result.slugs.anthropicOpus47}\``);
+  lines.push(`- Gemma 4 31B: \`${result.slugs.openrouterGemma31b}\``);
+  lines.push(`- Gemma 4 26B MoE: \`${result.slugs.openrouterGemma26bMoE}\` *(substituted from brief's \`gemma-4-26b-it\`)*`);
+  lines.push(`- Qwen3-30B-A3B: \`${result.slugs.openrouterQwen3}\` *(substituted from brief's \`qwen3-30b-a3b-instruct\`)*`);
+  lines.push('');
+
+  lines.push('## Summary');
+  lines.push('');
+  const primedOK = result.scenarios.filter(s => !s.primingFailed).length;
+
+  const reasoningScenarios = result.scenarios.filter(s => s.shape !== 'draft');
+  let gapClosurePct = 0;
+  if (reasoningScenarios.length > 0) {
+    const aMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['A'])));
+    const bMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['B'])));
+    const cMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['C'])));
+    const gap = aMean - bMean;
+    const closure = cMean - bMean;
+    gapClosurePct = gap > 0 ? (closure / gap) * 100 : 0;
+  }
+
+  let maxDRegression = 0;
+  for (const s of result.scenarios) {
+    const a = conditionMean(s.conditions['A']);
+    const d = conditionMean(s.conditions['D']);
+    if (a > d) maxDRegression = Math.max(maxDRegression, a - d);
+  }
+
+  const aMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['A'])));
+  const eMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['E'])));
+
+  lines.push('| Metric | Value |');
+  lines.push('|--------|-------|');
+  lines.push(`| Scenarios | ${result.scenarios.length} |`);
+  lines.push(`| Scenarios with successful priming | ${primedOK} / ${result.scenarios.length} |`);
+  lines.push(`| Seeds per scenario | ${result.seeds} |`);
+  lines.push(`| Gap closure (C−B)/(A−B), reasoning only | ${gapClosurePct.toFixed(1)}% |`);
+  lines.push(`| Target (≥40%) | ${gapClosurePct >= 40 ? '**PASS**' : '**FAIL**'} |`);
+  lines.push(`| D regression vs A (max over rows) | ${(maxDRegression * 100).toFixed(2)}pp |`);
+  lines.push(`| Opus generation delta (A 4.7 − E 4.6) | ${((aMeanAll - eMeanAll) * 100).toFixed(2)}pp |`);
+  lines.push('');
+
+  lines.push('## Priming results');
+  lines.push('');
+  lines.push('| Scenario | Lang | Frames | Matches |');
+  lines.push('|----------|------|--------|---------|');
+  for (const s of result.scenarios) {
+    const matchSummary = Object.entries(s.primingMatches)
+      .map(([k, v]) => `${v ? '✓' : '✗'} ${k}`)
+      .join(', ');
+    lines.push(`| ${s.scenario} | ${s.language} | ${s.primingFrameCount} | ${matchSummary} |`);
+  }
+  lines.push('');
+
+  lines.push('## Per-scenario breakdown (primary)');
+  lines.push('');
+  const primaryCodes = PRIMARY_CONDITIONS.map(c => c.code);
+  lines.push(`| Scenario | shape | ${primaryCodes.join(' | ')} | (C−B) | (A−B) |`);
+  lines.push(`|----------|-------|${primaryCodes.map(() => '---').join('|')}|-------|-------|`);
+  for (const s of result.scenarios) {
+    const cells = primaryCodes.map(code => {
+      const m = conditionMean(s.conditions[code]);
+      return m.toFixed(3);
+    });
+    const a = conditionMean(s.conditions['A']);
+    const b = conditionMean(s.conditions['B']);
+    const c = conditionMean(s.conditions['C']);
+    lines.push(`| ${s.scenario} | ${s.shape} | ${cells.join(' | ')} | ${(c - b).toFixed(3)} | ${(a - b).toFixed(3)} |`);
+  }
+  lines.push('');
+
+  const hasSecondary26 = result.scenarios.some(s => s.conditions["B'"] || s.conditions["C'"]);
+  if (hasSecondary26) {
+    lines.push('## Secondary — Gemma 4 26B MoE');
+    lines.push('');
+    lines.push(`| Scenario | B' | C' | (C'−B') |`);
+    lines.push(`|----------|-----|-----|---------|`);
+    for (const s of result.scenarios) {
+      const bp = conditionMean(s.conditions["B'"]);
+      const cp = conditionMean(s.conditions["C'"]);
+      lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
+    }
+    lines.push('');
+  }
+
+  const hasSecondaryQ = result.scenarios.some(s => s.conditions["B''"] || s.conditions["C''"]);
+  if (hasSecondaryQ) {
+    lines.push('## Secondary — Qwen3-30B-A3B');
+    lines.push('');
+    lines.push(`| Scenario | B'' | C'' | (C''−B'') |`);
+    lines.push(`|----------|------|------|-----------|`);
+    for (const s of result.scenarios) {
+      const bp = conditionMean(s.conditions["B''"]);
+      const cp = conditionMean(s.conditions["C''"]);
+      lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
+    }
+    lines.push('');
+  }
+
+  lines.push('## Cross-model pattern (reasoning scenarios only)');
+  lines.push('');
+  const check = (bCode: string, cCode: string): boolean => {
+    if (!reasoningScenarios.length) return false;
+    return reasoningScenarios.every(s => {
+      const b = conditionMean(s.conditions[bCode]);
+      const c = conditionMean(s.conditions[cCode]);
+      return c >= b;
+    });
+  };
+  const perModel: Array<[string, boolean]> = [['Gemma 4 31B', check('B', 'C')]];
+  if (hasSecondary26) perModel.push(['Gemma 4 26B MoE', check("B'", "C'")]);
+  if (hasSecondaryQ) perModel.push(['Qwen3-30B-A3B', check("B''", "C''")]);
+  for (const [model, positive] of perModel) {
+    lines.push(`- **${model}**: ${positive ? 'all reasoning scenarios C ≥ B ✓' : 'mixed or negative'}`);
+  }
+  lines.push('');
+
+  lines.push('## Opus generation delta (A 4.7 − E 4.6)');
+  lines.push('');
+  lines.push(`| Scenario | A (4.7) | E (4.6) | Δ |`);
+  lines.push(`|----------|---------|---------|-----|`);
+  for (const s of result.scenarios) {
+    const a = conditionMean(s.conditions['A']);
+    const e = conditionMean(s.conditions['E']);
+    lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${e.toFixed(3)} | ${(a - e).toFixed(3)} |`);
+  }
+  lines.push('');
+
+  lines.push('## Sample outputs (best-scoring C seed per scenario)');
+  lines.push('');
+  for (const s of result.scenarios) {
+    const cRuns = s.conditions['C'] ?? [];
+    const best = [...cRuns].sort((a, b) => (b.score?.overall ?? 0) - (a.score?.overall ?? 0))[0];
+    if (best) {
+      lines.push(`### ${s.scenario}`);
+      lines.push('');
+      lines.push(`Best C score: ${best.score?.overall?.toFixed(3) ?? 'N/A'}, seed: ${best.seed}`);
+      lines.push('');
+      lines.push('```');
+      lines.push(best.output.slice(0, 800) + (best.output.length > 800 ? '\n...[truncated]' : ''));
+      lines.push('```');
+      lines.push('');
+      if (best.debug) {
+        lines.push(`Debug: tier=${best.debug.tier}, shape=${best.debug.taskShape}, conf=${best.debug.taskShapeConfidence.toFixed(2)}, scaffoldApplied=${best.debug.scaffoldApplied}, sections=[${best.debug.sectionsIncluded.join(', ')}], frames=${best.debug.framesUsed}, chars=${best.debug.totalChars}`);
+        lines.push('');
+      }
+    }
+  }
+
+  lines.push('## Honest observations');
+  lines.push('');
+  const obs: string[] = [];
+  if (gapClosurePct >= 40) {
+    obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap on reasoning scenarios — meets the ≥40% target.`);
+  } else if (gapClosurePct > 0) {
+    obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap — below the ≥40% target. Scaffold helps but not structurally sufficient on its own.`);
+  } else {
+    obs.push(`- C − B is ${gapClosurePct.toFixed(0)}% — PA did not close the gap. Consider scenario iteration or deeper intervention.`);
+  }
+  if (maxDRegression > 0.02) {
+    obs.push(`- D regresses from A by up to **${(maxDRegression * 100).toFixed(1)}pp** — exceeds 2pp guardrail. Investigate frontier overhead.`);
+  } else {
+    obs.push(`- D does not regress from A by more than 2pp — frontier tier handles PA gracefully.`);
+  }
+  const serbianScenarios = result.scenarios.filter(s => s.language === 'sr');
+  const serbianPriming = serbianScenarios.filter(s => !s.primingFailed).length;
+  obs.push(`- Serbian priming: ${serbianPriming} / ${serbianScenarios.length} succeeded with English save-trigger phrases mixed in.`);
+  for (const o of obs) lines.push(o);
+  lines.push('');
+
+  lines.push('---');
+  lines.push('');
+  lines.push('Generated by `packages/agent/tests/eval/prompt-assembler-eval.ts`.');
+  lines.push('Full structured results: `tmp_bench_results.json` (gitignored).');
+
+  return lines.join('\n');
+}
+
+// ── Main ────────────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  const startTime = Date.now();
+  const runDate = new Date().toISOString();
+
+  console.log('[hydrate] Reading vault keys...');
+  const keys = hydrateVault();
+  if (!keys.anthropic) throw new Error('Anthropic key not found in vault');
+  if (!keys.openrouter) throw new Error('OpenRouter key not found in vault');
+  console.log('[hydrate] anthropic + openrouter keys loaded.');
+
+  let commit = 'unknown';
+  try {
+    commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
+  } catch {
+    // ignore
+  }
+
+  const conditions = SKIP_SECONDARY
+    ? PRIMARY_CONDITIONS
+    : [...PRIMARY_CONDITIONS, ...SECONDARY_26B_CONDITIONS, ...SECONDARY_QWEN_CONDITIONS];
+
+  const judge = new LLMJudge(async (prompt: string) => {
+    return callAnthropic(JUDGE_MODEL, '', prompt, { temperature: TEMPERATURE_JUDGE, maxTokens: MAX_TOKENS_JUDGE });
+  });
+
+  const result: EvalResult = {
+    runDate,
+    commit,
+    durationMs: 0,
+    deviationFromBrief:
+      'LiteLLM proxy was not reachable on localhost:4000 at eval start. ' +
+      'This harness calls Anthropic (/v1/messages) and OpenRouter (/v1/chat/completions) ' +
+      'APIs directly via fetch(). Measurement validity is unaffected — the variable under ' +
+      'test (prompt structure C vs B) is isolated correctly since both conditions share ' +
+      'the same model, same user message, and same temperature.',
+    slugs: {
+      openrouterGemma31b: GEMMA_31B_MODEL,
+      openrouterGemma26bMoE: GEMMA_26B_MOE_MODEL,
+      openrouterQwen3: QWEN_30B_MODEL,
+      anthropicOpus47: OPUS_4_7_MODEL,
+    },
+    seeds: SEEDS,
+    scenarios: [],
+  };
+
+  for (const [idx, scenario] of SCENARIOS.entries()) {
+    console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
+
+    const setup = setupCleanScenario(scenario.name);
+    const primingStart = Date.now();
+
+    const primingDb = new MindDB(setup.dbPath);
+    const primingOrch = new Orchestrator({
+      db: primingDb,
+      embedder: new StubEmbedder(),
+      model: PRIMING_MODEL,
+    });
+    primingOrch.getIdentity().create({
+      name: 'Marko',
+      role: 'CEO',
+      department: 'Egzakta Group',
+      personality: 'Direct, pragmatic, sovereignty-focused',
+      capabilities: 'Strategic decisions, technical oversight',
+      system_prompt: '',
+    });
+
+    console.log(`  [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
+    try {
+      await runPriming(primingOrch, scenario);
+    } catch (err) {
+      console.error(`  [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
+    const verification = verifyMemory(primingDb, scenario);
+    const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
+    console.log(`  [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
+
+    primingDb.close();
+    fs.copyFileSync(setup.dbPath, setup.snapshotPath);
+
+    const scenarioResult: ScenarioResult = {
+      scenario: scenario.name,
+      shape: scenario.shape,
+      language: scenario.language,
+      primingFrameCount: verification.count,
+      primingMatches: verification.matches,
+      primingFailed,
+      primingDurationMs: Date.now() - primingStart,
+      conditions: {},
+    };
+
+    for (const condition of conditions) {
+      scenarioResult.conditions[condition.code] = [];
+      for (let seed = 0; seed < SEEDS; seed++) {
+        console.log(`  [run] ${condition.code} (${condition.label}), seed=${seed}...`);
+        const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
+        scenarioResult.conditions[condition.code].push(run);
+        if (run.error) console.log(`    ERROR: ${run.error.slice(0, 200)}`);
+        else console.log(`    OK (${run.durationMs}ms, ${run.output.length} chars)`);
+      }
+    }
+
+    console.log(`  [judge] scoring outputs via ${JUDGE_MODEL}...`);
+    const aRuns = scenarioResult.conditions['A'] ?? [];
+    for (const condition of conditions) {
+      if (condition.code === 'A') {
+        for (const run of scenarioResult.conditions['A']) {
+          run.score = {
+            overall: 1.0,
+            weighted: 1.0,
+            correctness: 10,
+            procedureFollowing: 10,
+            conciseness: 10,
+            lengthPenalty: 1,
+            feedback: 'Gold reference (condition A).',
+            parsed: true,
+          };
+        }
+        continue;
+      }
+      const runs = scenarioResult.conditions[condition.code] ?? [];
+      for (const run of runs) {
+        if (run.error || !run.output) continue;
+        const gold = aRuns.find(r => r.seed === run.seed) ?? aRuns[0];
+        if (!gold || gold.error || !gold.output) continue;
+        try {
+          run.score = await judgeRun(judge, scenario, gold.output, run.output);
+        } catch (err) {
+          console.log(`    judge error for ${condition.code} seed ${run.seed}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
+    }
+
+    result.scenarios.push(scenarioResult);
+    fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
+    fs.rmSync(setup.tempDir, { recursive: true, force: true });
+  }
+
+  result.durationMs = Date.now() - startTime;
+  fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
+  fs.writeFileSync(RESULTS_MD, renderMarkdown(result));
+
+  console.log(`\n[done] ${result.scenarios.length} scenarios in ${(result.durationMs / 1000 / 60).toFixed(1)} min.`);
+  console.log(`[done] JSON: ${RESULTS_JSON}`);
+  console.log(`[done] MD:   ${RESULTS_MD}`);
+}
+
+main().catch(err => {
+  console.error('[fatal]', err);
+  process.exit(1);
+});