Files
waggle-os/packages/agent/tests/eval/prompt-assembler-eval.ts
shumeiko.it f94331c202 test(agent): move agent tests off the legacy memory engine (P1.5)
- prompt assembler harnesses (v4 and v5): prime an in-memory contract store,
  snapshot its memories and replay them into a fresh store per condition
  instead of copying a MindDB file; the identity card is created per
  orchestrator. They had also drifted from the current Orchestrator API
  (config takes memory, buildSystemPrompt is async).
- prompt assembler unit test: build RecallFrame fixtures, the type the
  assembler consumes, instead of legacy MemoryFrame rows.
- M3c integration: drop the knowledge-graph Ontology case; the knowledge
  graph is not part of v1 and core no longer exports it.
2026-10-24 17:43:27 +02:00

719 lines
28 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* PromptAssembler eval harness — executes the full measurement protocol from
* docs/specs/PROMPT-ASSEMBLER-V4.md §11.
*
* Usage (from repo root):
* tsx packages/agent/tests/eval/prompt-assembler-eval.ts
*
* Environment:
* WAGGLE_DATA_DIR — override ~/.waggle (for non-default installs)
* WAGGLE_EVAL_SKIP_SECONDARY=1 — skip Gemma 4 26B MoE + Qwen3 secondary suites
* WAGGLE_EVAL_SEEDS — number of seeds per condition (default 3)
*
* Deviation from brief §11.2:
* The brief says "all inference goes through the LiteLLM proxy at
* litellmUrl." No LiteLLM proxy is running in the current session
* (port 4000 unbound). This harness calls Anthropic + OpenRouter APIs
* directly via fetch. Measurement validity is unaffected — the
* variable under test (prompt structure) is isolated correctly.
* Deviation is logged in EVAL-RESULTS.md.
*
* Outputs:
* tmp_bench_results.json — full structured results (gitignored)
* EVAL-RESULTS.md — human-readable summary (committed)
*/
import { VaultStore } from '@waggle/core';
import {
createInMemoryMemoryStore,
PERSONAL_SCOPE,
type MemoryRecord,
type MemoryStore,
} from '@waggle/memory-contract';
import { Orchestrator } from '../../src/orchestrator.js';
import { type ModelTier } from '../../src/model-tier.js';
import { detectTaskShape } from '../../src/task-shape.js';
import { LLMJudge, type JudgeScore } from '../../src/judge.js';
import { SCENARIOS, type PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
// ── Config ──────────────────────────────────────────────────────────
const WAGGLE_DATA_DIR = process.env.WAGGLE_DATA_DIR ?? path.join(os.homedir(), '.waggle');
// Use fileURLToPath to handle Windows file:// URLs correctly (avoids
// `/D:/...` leading-slash bug that produced `D:\D:\...` double-drive paths).
const HARNESS_DIR = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(path.join(HARNESS_DIR, '..', '..', '..', '..'));
const RESULTS_JSON = path.join(REPO_ROOT, 'tmp_bench_results.json');
const RESULTS_MD = path.join(REPO_ROOT, 'EVAL-RESULTS.md');
// Anthropic accepts plain family aliases — verified live via /v1/models 2026-04-17.
const PRIMING_MODEL = 'claude-sonnet-4-6';
const JUDGE_MODEL = 'claude-sonnet-4-6';
const OPUS_4_7_MODEL = 'claude-opus-4-7';
const OPUS_4_6_MODEL = 'claude-opus-4-6';
const GEMMA_31B_MODEL = 'google/gemma-4-31b-it';
const GEMMA_26B_MOE_MODEL = 'google/gemma-4-26b-a4b-it';
const QWEN_30B_MODEL = 'qwen/qwen3-30b-a3b-instruct-2507';
const TEMPERATURE_GEN = 0.2;
const TEMPERATURE_JUDGE = 0;
const MAX_TOKENS_GEN = 1024;
const MAX_TOKENS_JUDGE = 512;
const SEEDS = Number.parseInt(process.env.WAGGLE_EVAL_SEEDS ?? '3', 10);
const SKIP_SECONDARY = process.env.WAGGLE_EVAL_SKIP_SECONDARY === '1';
interface ConditionSpec {
code: string;
label: string;
model: string;
provider: 'anthropic' | 'openrouter';
usesPromptAssembler: boolean;
suite: 'primary' | 'secondary-26b' | 'secondary-qwen';
}
const PRIMARY_CONDITIONS: ConditionSpec[] = [
{ code: 'A', label: 'Opus 4.7 · current', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
{ code: 'B', label: 'Gemma 4 31B · current', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'primary' },
{ code: 'C', label: 'Gemma 4 31B · PA', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'primary' },
{ code: 'D', label: 'Opus 4.7 · PA', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
{ code: 'E', label: 'Opus 4.6 · current', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
{ code: 'F', label: 'Opus 4.6 · PA', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
];
const SECONDARY_26B_CONDITIONS: ConditionSpec[] = [
{ code: "B'", label: 'Gemma 4 26B MoE · current', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-26b' },
{ code: "C'", label: 'Gemma 4 26B MoE · PA', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-26b' },
];
const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
{ code: "B''", label: 'Qwen3-30B-A3B · current', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-qwen' },
{ code: "C''", label: 'Qwen3-30B-A3B · PA', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-qwen' },
];
// ── Types ────────────────────────────────────────────────────────────
interface ConditionRun {
seed: number;
output: string;
durationMs: number;
debug?: {
tier: ModelTier;
taskShape: string | null;
taskShapeConfidence: number;
scaffoldApplied: boolean;
sectionsIncluded: string[];
framesUsed: number;
totalChars: number;
};
score?: JudgeScore;
error?: string;
}
interface ScenarioResult {
scenario: string;
shape: string;
language: string;
primingFrameCount: number;
primingMatches: Record<string, boolean>;
primingFailed: boolean;
primingDurationMs: number;
conditions: Record<string, ConditionRun[]>;
}
interface EvalResult {
runDate: string;
commit: string;
durationMs: number;
deviationFromBrief: string;
slugs: {
openrouterGemma31b: string;
openrouterGemma26bMoE: string;
openrouterQwen3: string;
anthropicOpus47: string;
};
seeds: number;
scenarios: ScenarioResult[];
}
// ── Vault hydration ─────────────────────────────────────────────────
function hydrateVault(): { anthropic: string | null; openrouter: string | null } {
const vault = new VaultStore(WAGGLE_DATA_DIR);
const anthropic = vault.get('anthropic');
const openrouter = vault.get('openrouter');
if (anthropic) process.env.ANTHROPIC_API_KEY = anthropic.value;
if (openrouter) process.env.OPENROUTER_API_KEY = openrouter.value;
return { anthropic: anthropic?.value ?? null, openrouter: openrouter?.value ?? null };
}
// ── LLM clients ─────────────────────────────────────────────────────
async function callAnthropic(
model: string,
systemPrompt: string,
userMsg: string,
opts: { maxTokens?: number; temperature?: number } = {},
): Promise<string> {
const apiKey = process.env.ANTHROPIC_API_KEY;
if (!apiKey) throw new Error('ANTHROPIC_API_KEY not hydrated from vault');
const body: Record<string, unknown> = {
model,
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
messages: [{ role: 'user', content: userMsg }],
};
// Opus 4.7 rejects `temperature` as deprecated (extended-thinking models).
// Other Claude models accept it but provider default is fine for the eval.
// Omit entirely — all conditions use provider default → still controlled.
if (systemPrompt) body.system = systemPrompt;
const response = await fetch('https://api.anthropic.com/v1/messages', {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
},
body: JSON.stringify(body),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Anthropic ${model} ${response.status}: ${text.slice(0, 500)}`);
}
const data = await response.json() as { content: Array<{ type: string; text?: string }> };
return data.content.map(c => c.text ?? '').join('');
}
async function callOpenRouter(
model: string,
systemPrompt: string,
userMsg: string,
opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
): Promise<string> {
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) throw new Error('OPENROUTER_API_KEY not hydrated from vault');
const messages: Array<{ role: string; content: string }> = [];
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
messages.push({ role: 'user', content: userMsg });
const body: Record<string, unknown> = {
model,
messages,
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
temperature: opts.temperature ?? TEMPERATURE_GEN,
};
if (opts.seed !== undefined) body.seed = opts.seed;
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${apiKey}`,
'http-referer': 'https://waggle-os.ai',
'x-title': 'Waggle PromptAssembler eval',
},
body: JSON.stringify(body),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`OpenRouter ${model} ${response.status}: ${text.slice(0, 500)}`);
}
const data = await response.json() as { choices: Array<{ message: { content: string } }> };
return data.choices[0]?.message?.content ?? '';
}
async function callModel(
provider: 'anthropic' | 'openrouter',
model: string,
systemPrompt: string,
userMsg: string,
opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
): Promise<string> {
if (provider === 'anthropic') return callAnthropic(model, systemPrompt, userMsg, opts);
return callOpenRouter(model, systemPrompt, userMsg, opts);
}
// ── Scenario pipeline ───────────────────────────────────────────────
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
type MemorySnapshot = readonly MemoryRecord[];
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
const orch = new Orchestrator({ memory, model });
orch.getIdentity().create({
name: 'Marko',
role: 'CEO',
department: 'Egzakta Group',
personality: 'Direct, pragmatic, sovereignty-focused',
capabilities: 'Strategic decisions, technical oversight',
system_prompt: '',
});
return orch;
}
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
const filter = { scopes: [PERSONAL_SCOPE] };
return memory.list({ ...filter, limit: await memory.count(filter) });
}
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
const memory = createInMemoryMemoryStore();
// list() returns newest first; replay oldest first.
for (const record of [...snapshot].reverse()) {
await memory.remember({
scope: record.scope,
text: record.text,
kind: record.kind,
importance: record.importance,
source: record.source,
origin: record.origin ?? undefined,
sessionId: record.sessionId ?? undefined,
metadata: record.metadata,
});
}
return memory;
}
async function runPriming(
orch: Orchestrator,
scenario: PromptAssemblerScenario,
): Promise<void> {
for (const turn of scenario.primingTurns) {
const systemPrompt = await orch.buildSystemPrompt();
const assistantMsg = await callAnthropic(
PRIMING_MODEL,
systemPrompt,
turn.user,
{ temperature: 0, maxTokens: MAX_TOKENS_GEN },
);
await orch.autoSaveFromExchange(turn.user, assistantMsg);
}
}
function verifyMemory(
snapshot: MemorySnapshot,
scenario: PromptAssemblerScenario,
): { count: number; matches: Record<string, boolean> } {
const matches: Record<string, boolean> = {};
for (const sub of scenario.memoryVerificationSubstrings) {
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
}
return { count: snapshot.length, matches };
}
async function runCondition(
snapshot: MemorySnapshot,
condition: ConditionSpec,
scenario: PromptAssemblerScenario,
seed: number,
): Promise<ConditionRun> {
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
let systemPrompt: string;
let debug: ConditionRun['debug'] = undefined;
const start = Date.now();
try {
if (condition.usesPromptAssembler) {
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
const taskShape = detectTaskShape(scenario.testTurn.query);
const assembled = await orch.buildAssembledPrompt(scenario.testTurn.query, null, { taskShape });
systemPrompt = assembled.system;
debug = {
tier: assembled.debug.tier,
taskShape: assembled.debug.taskShape,
taskShapeConfidence: assembled.debug.taskShapeConfidence,
scaffoldApplied: assembled.debug.scaffoldApplied,
sectionsIncluded: assembled.debug.sectionsIncluded,
framesUsed: assembled.debug.framesUsed,
totalChars: assembled.debug.totalChars,
};
} else {
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
systemPrompt = await orch.buildSystemPrompt();
}
const output = await callModel(
condition.provider,
condition.model,
systemPrompt,
scenario.testTurn.query,
{ temperature: TEMPERATURE_GEN, seed },
);
return { seed, output, durationMs: Date.now() - start, debug };
} catch (err) {
return {
seed,
output: '',
durationMs: Date.now() - start,
debug,
error: err instanceof Error ? err.message : String(err),
};
}
}
async function judgeRun(
judge: LLMJudge,
scenario: PromptAssemblerScenario,
goldOutput: string,
candidateOutput: string,
): Promise<JudgeScore> {
return judge.score({
input: scenario.testTurn.query,
expected: goldOutput,
actual: candidateOutput,
context: `task_shape=${scenario.shape}, language=${scenario.language}`,
});
}
// ── Result aggregation ──────────────────────────────────────────────
function mean(xs: number[]): number {
if (xs.length === 0) return 0;
return xs.reduce((a, b) => a + b, 0) / xs.length;
}
function conditionMean(runs: ConditionRun[] | undefined): number {
if (!runs || runs.length === 0) return 0;
const scores = runs.map(r => r.score?.overall ?? 0);
return mean(scores);
}
function renderMarkdown(result: EvalResult): string {
const lines: string[] = [];
lines.push('# PromptAssembler eval results');
lines.push('');
lines.push(`**Run date:** ${result.runDate}`);
lines.push(`**Commit:** ${result.commit}`);
lines.push(`**Duration:** ${(result.durationMs / 1000 / 60).toFixed(1)} min`);
lines.push(`**Seeds per condition:** ${result.seeds}`);
lines.push(`**LiteLLM:** bypassed — see deviation note`);
lines.push('');
lines.push('## Deviation from brief §11.2');
lines.push('');
lines.push(result.deviationFromBrief);
lines.push('');
lines.push('## Slug probe');
lines.push('');
lines.push(`- Opus 4.7: \`${result.slugs.anthropicOpus47}\``);
lines.push(`- Gemma 4 31B: \`${result.slugs.openrouterGemma31b}\``);
lines.push(`- Gemma 4 26B MoE: \`${result.slugs.openrouterGemma26bMoE}\` *(substituted from brief's \`gemma-4-26b-it\`)*`);
lines.push(`- Qwen3-30B-A3B: \`${result.slugs.openrouterQwen3}\` *(substituted from brief's \`qwen3-30b-a3b-instruct\`)*`);
lines.push('');
lines.push('## Summary');
lines.push('');
const primedOK = result.scenarios.filter(s => !s.primingFailed).length;
const reasoningScenarios = result.scenarios.filter(s => s.shape !== 'draft');
let gapClosurePct = 0;
if (reasoningScenarios.length > 0) {
const aMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['A'])));
const bMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['B'])));
const cMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['C'])));
const gap = aMean - bMean;
const closure = cMean - bMean;
gapClosurePct = gap > 0 ? (closure / gap) * 100 : 0;
}
let maxDRegression = 0;
for (const s of result.scenarios) {
const a = conditionMean(s.conditions['A']);
const d = conditionMean(s.conditions['D']);
if (a > d) maxDRegression = Math.max(maxDRegression, a - d);
}
const aMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['A'])));
const eMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['E'])));
lines.push('| Metric | Value |');
lines.push('|--------|-------|');
lines.push(`| Scenarios | ${result.scenarios.length} |`);
lines.push(`| Scenarios with successful priming | ${primedOK} / ${result.scenarios.length} |`);
lines.push(`| Seeds per scenario | ${result.seeds} |`);
lines.push(`| Gap closure (C−B)/(A−B), reasoning only | ${gapClosurePct.toFixed(1)}% |`);
lines.push(`| Target (≥40%) | ${gapClosurePct >= 40 ? '**PASS**' : '**FAIL**'} |`);
lines.push(`| D regression vs A (max over rows) | ${(maxDRegression * 100).toFixed(2)}pp |`);
lines.push(`| Opus generation delta (A 4.7 − E 4.6) | ${((aMeanAll - eMeanAll) * 100).toFixed(2)}pp |`);
lines.push('');
lines.push('## Priming results');
lines.push('');
lines.push('| Scenario | Lang | Frames | Matches |');
lines.push('|----------|------|--------|---------|');
for (const s of result.scenarios) {
const matchSummary = Object.entries(s.primingMatches)
.map(([k, v]) => `${v ? '✓' : '✗'} ${k}`)
.join(', ');
lines.push(`| ${s.scenario} | ${s.language} | ${s.primingFrameCount} | ${matchSummary} |`);
}
lines.push('');
lines.push('## Per-scenario breakdown (primary)');
lines.push('');
const primaryCodes = PRIMARY_CONDITIONS.map(c => c.code);
lines.push(`| Scenario | shape | ${primaryCodes.join(' | ')} | (C−B) | (A−B) |`);
lines.push(`|----------|-------|${primaryCodes.map(() => '---').join('|')}|-------|-------|`);
for (const s of result.scenarios) {
const cells = primaryCodes.map(code => {
const m = conditionMean(s.conditions[code]);
return m.toFixed(3);
});
const a = conditionMean(s.conditions['A']);
const b = conditionMean(s.conditions['B']);
const c = conditionMean(s.conditions['C']);
lines.push(`| ${s.scenario} | ${s.shape} | ${cells.join(' | ')} | ${(c - b).toFixed(3)} | ${(a - b).toFixed(3)} |`);
}
lines.push('');
const hasSecondary26 = result.scenarios.some(s => s.conditions["B'"] || s.conditions["C'"]);
if (hasSecondary26) {
lines.push('## Secondary — Gemma 4 26B MoE');
lines.push('');
lines.push(`| Scenario | B' | C' | (C'−B') |`);
lines.push(`|----------|-----|-----|---------|`);
for (const s of result.scenarios) {
const bp = conditionMean(s.conditions["B'"]);
const cp = conditionMean(s.conditions["C'"]);
lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
}
lines.push('');
}
const hasSecondaryQ = result.scenarios.some(s => s.conditions["B''"] || s.conditions["C''"]);
if (hasSecondaryQ) {
lines.push('## Secondary — Qwen3-30B-A3B');
lines.push('');
lines.push(`| Scenario | B'' | C'' | (C''−B'') |`);
lines.push(`|----------|------|------|-----------|`);
for (const s of result.scenarios) {
const bp = conditionMean(s.conditions["B''"]);
const cp = conditionMean(s.conditions["C''"]);
lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
}
lines.push('');
}
lines.push('## Cross-model pattern (reasoning scenarios only)');
lines.push('');
const check = (bCode: string, cCode: string): boolean => {
if (!reasoningScenarios.length) return false;
return reasoningScenarios.every(s => {
const b = conditionMean(s.conditions[bCode]);
const c = conditionMean(s.conditions[cCode]);
return c >= b;
});
};
const perModel: Array<[string, boolean]> = [['Gemma 4 31B', check('B', 'C')]];
if (hasSecondary26) perModel.push(['Gemma 4 26B MoE', check("B'", "C'")]);
if (hasSecondaryQ) perModel.push(['Qwen3-30B-A3B', check("B''", "C''")]);
for (const [model, positive] of perModel) {
lines.push(`- **${model}**: ${positive ? 'all reasoning scenarios C ≥ B ✓' : 'mixed or negative'}`);
}
lines.push('');
lines.push('## Opus generation delta (A 4.7 − E 4.6)');
lines.push('');
lines.push(`| Scenario | A (4.7) | E (4.6) | Δ |`);
lines.push(`|----------|---------|---------|-----|`);
for (const s of result.scenarios) {
const a = conditionMean(s.conditions['A']);
const e = conditionMean(s.conditions['E']);
lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${e.toFixed(3)} | ${(a - e).toFixed(3)} |`);
}
lines.push('');
lines.push('## Sample outputs (best-scoring C seed per scenario)');
lines.push('');
for (const s of result.scenarios) {
const cRuns = s.conditions['C'] ?? [];
const best = [...cRuns].sort((a, b) => (b.score?.overall ?? 0) - (a.score?.overall ?? 0))[0];
if (best) {
lines.push(`### ${s.scenario}`);
lines.push('');
lines.push(`Best C score: ${best.score?.overall?.toFixed(3) ?? 'N/A'}, seed: ${best.seed}`);
lines.push('');
lines.push('```');
lines.push(best.output.slice(0, 800) + (best.output.length > 800 ? '\n...[truncated]' : ''));
lines.push('```');
lines.push('');
if (best.debug) {
lines.push(`Debug: tier=${best.debug.tier}, shape=${best.debug.taskShape}, conf=${best.debug.taskShapeConfidence.toFixed(2)}, scaffoldApplied=${best.debug.scaffoldApplied}, sections=[${best.debug.sectionsIncluded.join(', ')}], frames=${best.debug.framesUsed}, chars=${best.debug.totalChars}`);
lines.push('');
}
}
}
lines.push('## Honest observations');
lines.push('');
const obs: string[] = [];
if (gapClosurePct >= 40) {
obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap on reasoning scenarios — meets the ≥40% target.`);
} else if (gapClosurePct > 0) {
obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap — below the ≥40% target. Scaffold helps but not structurally sufficient on its own.`);
} else {
obs.push(`- C − B is ${gapClosurePct.toFixed(0)}% — PA did not close the gap. Consider scenario iteration or deeper intervention.`);
}
if (maxDRegression > 0.02) {
obs.push(`- D regresses from A by up to **${(maxDRegression * 100).toFixed(1)}pp** — exceeds 2pp guardrail. Investigate frontier overhead.`);
} else {
obs.push(`- D does not regress from A by more than 2pp — frontier tier handles PA gracefully.`);
}
const serbianScenarios = result.scenarios.filter(s => s.language === 'sr');
const serbianPriming = serbianScenarios.filter(s => !s.primingFailed).length;
obs.push(`- Serbian priming: ${serbianPriming} / ${serbianScenarios.length} succeeded with English save-trigger phrases mixed in.`);
for (const o of obs) lines.push(o);
lines.push('');
lines.push('---');
lines.push('');
lines.push('Generated by `packages/agent/tests/eval/prompt-assembler-eval.ts`.');
lines.push('Full structured results: `tmp_bench_results.json` (gitignored).');
return lines.join('\n');
}
// ── Main ────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const startTime = Date.now();
const runDate = new Date().toISOString();
console.log('[hydrate] Reading vault keys...');
const keys = hydrateVault();
if (!keys.anthropic) throw new Error('Anthropic key not found in vault');
if (!keys.openrouter) throw new Error('OpenRouter key not found in vault');
console.log('[hydrate] anthropic + openrouter keys loaded.');
let commit = 'unknown';
try {
commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
} catch {
// ignore
}
const conditions = SKIP_SECONDARY
? PRIMARY_CONDITIONS
: [...PRIMARY_CONDITIONS, ...SECONDARY_26B_CONDITIONS, ...SECONDARY_QWEN_CONDITIONS];
const judge = new LLMJudge(async (prompt: string) => {
return callAnthropic(JUDGE_MODEL, '', prompt, { temperature: TEMPERATURE_JUDGE, maxTokens: MAX_TOKENS_JUDGE });
});
const result: EvalResult = {
runDate,
commit,
durationMs: 0,
deviationFromBrief:
'LiteLLM proxy was not reachable on localhost:4000 at eval start. ' +
'This harness calls Anthropic (/v1/messages) and OpenRouter (/v1/chat/completions) ' +
'APIs directly via fetch(). Measurement validity is unaffected — the variable under ' +
'test (prompt structure C vs B) is isolated correctly since both conditions share ' +
'the same model, same user message, and same temperature.',
slugs: {
openrouterGemma31b: GEMMA_31B_MODEL,
openrouterGemma26bMoE: GEMMA_26B_MOE_MODEL,
openrouterQwen3: QWEN_30B_MODEL,
anthropicOpus47: OPUS_4_7_MODEL,
},
seeds: SEEDS,
scenarios: [],
};
for (const [idx, scenario] of SCENARIOS.entries()) {
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
const primingStart = Date.now();
const primingMemory = createInMemoryMemoryStore();
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
try {
await runPriming(primingOrch, scenario);
} catch (err) {
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
}
const snapshot = await snapshotMemory(primingMemory);
const verification = verifyMemory(snapshot, scenario);
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
const scenarioResult: ScenarioResult = {
scenario: scenario.name,
shape: scenario.shape,
language: scenario.language,
primingFrameCount: verification.count,
primingMatches: verification.matches,
primingFailed,
primingDurationMs: Date.now() - primingStart,
conditions: {},
};
for (const condition of conditions) {
scenarioResult.conditions[condition.code] = [];
for (let seed = 0; seed < SEEDS; seed++) {
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
const run = await runCondition(snapshot, condition, scenario, seed);
scenarioResult.conditions[condition.code].push(run);
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
}
}
console.log(` [judge] scoring outputs via ${JUDGE_MODEL}...`);
const aRuns = scenarioResult.conditions['A'] ?? [];
for (const condition of conditions) {
if (condition.code === 'A') {
for (const run of scenarioResult.conditions['A']) {
run.score = {
overall: 1.0,
weighted: 1.0,
correctness: 10,
procedureFollowing: 10,
conciseness: 10,
lengthPenalty: 1,
feedback: 'Gold reference (condition A).',
parsed: true,
};
}
continue;
}
const runs = scenarioResult.conditions[condition.code] ?? [];
for (const run of runs) {
if (run.error || !run.output) continue;
const gold = aRuns.find(r => r.seed === run.seed) ?? aRuns[0];
if (!gold || gold.error || !gold.output) continue;
try {
run.score = await judgeRun(judge, scenario, gold.output, run.output);
} catch (err) {
console.log(` judge error for ${condition.code} seed ${run.seed}: ${err instanceof Error ? err.message : String(err)}`);
}
}
}
result.scenarios.push(scenarioResult);
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
}
result.durationMs = Date.now() - startTime;
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
fs.writeFileSync(RESULTS_MD, renderMarkdown(result));
console.log(`\n[done] ${result.scenarios.length} scenarios in ${(result.durationMs / 1000 / 60).toFixed(1)} min.`);
console.log(`[done] JSON: ${RESULTS_JSON}`);
console.log(`[done] MD: ${RESULTS_MD}`);
}
main().catch(err => {
console.error('[fatal]', err);
process.exit(1);
});