The community edition defaulted to ~/.waggle, the data directory of the commercial Waggle, so both editions would open the same config, vault, waggle.db, marketplace catalog and skills. Default to ~/.waggle-community instead (ADR-0011). WAGGLE_DATA_DIR still overrides it wherever it did; nothing is migrated from ~/.waggle. The name is defined once as DEFAULT_DATA_DIR_NAME and DEFAULT_DATA_DIR_LABEL in @waggle/shared, with defaultDataDir() in @waggle/core for the sidecar, the CLI, the agent and the model caches. The marketplace installs on its own without workspace packages, so it keeps a copy in its data-dir.ts; the CLI help, which loads no dependency, spells the path out and its runtime test checks it. The marketplace security scan also flags skills that reference the new skills folder. Tests that asserted the old default now use the constant, and comments name the new directory.
719 lines
28 KiB
TypeScript
719 lines
28 KiB
TypeScript
/**
|
||
* PromptAssembler eval harness — executes the full measurement protocol from
|
||
* docs/specs/PROMPT-ASSEMBLER-V4.md §11.
|
||
*
|
||
* Usage (from repo root):
|
||
* tsx packages/agent/tests/eval/prompt-assembler-eval.ts
|
||
*
|
||
* Environment:
|
||
* WAGGLE_DATA_DIR — override ~/.waggle-community (for non-default installs)
|
||
* WAGGLE_EVAL_SKIP_SECONDARY=1 — skip Gemma 4 26B MoE + Qwen3 secondary suites
|
||
* WAGGLE_EVAL_SEEDS — number of seeds per condition (default 3)
|
||
*
|
||
* Deviation from brief §11.2:
|
||
* The brief says "all inference goes through the LiteLLM proxy at
|
||
* litellmUrl." No LiteLLM proxy is running in the current session
|
||
* (port 4000 unbound). This harness calls Anthropic + OpenRouter APIs
|
||
* directly via fetch. Measurement validity is unaffected — the
|
||
* variable under test (prompt structure) is isolated correctly.
|
||
* Deviation is logged in EVAL-RESULTS.md.
|
||
*
|
||
* Outputs:
|
||
* tmp_bench_results.json — full structured results (gitignored)
|
||
* EVAL-RESULTS.md — human-readable summary (committed)
|
||
*/
|
||
|
||
import { VaultStore, defaultDataDir } from '@waggle/core';
|
||
import {
|
||
createInMemoryMemoryStore,
|
||
PERSONAL_SCOPE,
|
||
type MemoryRecord,
|
||
type MemoryStore,
|
||
} from '@waggle/memory-contract';
|
||
import { Orchestrator } from '../../src/orchestrator.js';
|
||
import { type ModelTier } from '../../src/model-tier.js';
|
||
import { detectTaskShape } from '../../src/task-shape.js';
|
||
import { LLMJudge, type JudgeScore } from '../../src/judge.js';
|
||
import { SCENARIOS, type PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
|
||
import * as fs from 'node:fs';
|
||
import * as path from 'node:path';
|
||
import * as os from 'node:os';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { execFileSync } from 'node:child_process';
|
||
|
||
// ── Config ──────────────────────────────────────────────────────────
|
||
|
||
const WAGGLE_DATA_DIR = process.env.WAGGLE_DATA_DIR ?? defaultDataDir();
|
||
// Use fileURLToPath to handle Windows file:// URLs correctly (avoids
|
||
// `/D:/...` leading-slash bug that produced `D:\D:\...` double-drive paths).
|
||
const HARNESS_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||
const REPO_ROOT = path.resolve(path.join(HARNESS_DIR, '..', '..', '..', '..'));
|
||
const RESULTS_JSON = path.join(REPO_ROOT, 'tmp_bench_results.json');
|
||
const RESULTS_MD = path.join(REPO_ROOT, 'EVAL-RESULTS.md');
|
||
|
||
// Anthropic accepts plain family aliases — verified live via /v1/models 2026-04-17.
|
||
const PRIMING_MODEL = 'claude-sonnet-4-6';
|
||
const JUDGE_MODEL = 'claude-sonnet-4-6';
|
||
const OPUS_4_7_MODEL = 'claude-opus-4-7';
|
||
const OPUS_4_6_MODEL = 'claude-opus-4-6';
|
||
const GEMMA_31B_MODEL = 'google/gemma-4-31b-it';
|
||
const GEMMA_26B_MOE_MODEL = 'google/gemma-4-26b-a4b-it';
|
||
const QWEN_30B_MODEL = 'qwen/qwen3-30b-a3b-instruct-2507';
|
||
|
||
const TEMPERATURE_GEN = 0.2;
|
||
const TEMPERATURE_JUDGE = 0;
|
||
const MAX_TOKENS_GEN = 1024;
|
||
const MAX_TOKENS_JUDGE = 512;
|
||
const SEEDS = Number.parseInt(process.env.WAGGLE_EVAL_SEEDS ?? '3', 10);
|
||
const SKIP_SECONDARY = process.env.WAGGLE_EVAL_SKIP_SECONDARY === '1';
|
||
|
||
interface ConditionSpec {
|
||
code: string;
|
||
label: string;
|
||
model: string;
|
||
provider: 'anthropic' | 'openrouter';
|
||
usesPromptAssembler: boolean;
|
||
suite: 'primary' | 'secondary-26b' | 'secondary-qwen';
|
||
}
|
||
|
||
const PRIMARY_CONDITIONS: ConditionSpec[] = [
|
||
{ code: 'A', label: 'Opus 4.7 · current', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'B', label: 'Gemma 4 31B · current', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'C', label: 'Gemma 4 31B · PA', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'primary' },
|
||
{ code: 'D', label: 'Opus 4.7 · PA', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
|
||
{ code: 'E', label: 'Opus 4.6 · current', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'F', label: 'Opus 4.6 · PA', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: true, suite: 'primary' },
|
||
];
|
||
|
||
const SECONDARY_26B_CONDITIONS: ConditionSpec[] = [
|
||
{ code: "B'", label: 'Gemma 4 26B MoE · current', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-26b' },
|
||
{ code: "C'", label: 'Gemma 4 26B MoE · PA', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-26b' },
|
||
];
|
||
|
||
const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
|
||
{ code: "B''", label: 'Qwen3-30B-A3B · current', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-qwen' },
|
||
{ code: "C''", label: 'Qwen3-30B-A3B · PA', model: QWEN_30B_MODEL, provider: 'openrouter', usesPromptAssembler: true, suite: 'secondary-qwen' },
|
||
];
|
||
|
||
// ── Types ────────────────────────────────────────────────────────────
|
||
|
||
interface ConditionRun {
|
||
seed: number;
|
||
output: string;
|
||
durationMs: number;
|
||
debug?: {
|
||
tier: ModelTier;
|
||
taskShape: string | null;
|
||
taskShapeConfidence: number;
|
||
scaffoldApplied: boolean;
|
||
sectionsIncluded: string[];
|
||
framesUsed: number;
|
||
totalChars: number;
|
||
};
|
||
score?: JudgeScore;
|
||
error?: string;
|
||
}
|
||
|
||
interface ScenarioResult {
|
||
scenario: string;
|
||
shape: string;
|
||
language: string;
|
||
primingFrameCount: number;
|
||
primingMatches: Record<string, boolean>;
|
||
primingFailed: boolean;
|
||
primingDurationMs: number;
|
||
conditions: Record<string, ConditionRun[]>;
|
||
}
|
||
|
||
interface EvalResult {
|
||
runDate: string;
|
||
commit: string;
|
||
durationMs: number;
|
||
deviationFromBrief: string;
|
||
slugs: {
|
||
openrouterGemma31b: string;
|
||
openrouterGemma26bMoE: string;
|
||
openrouterQwen3: string;
|
||
anthropicOpus47: string;
|
||
};
|
||
seeds: number;
|
||
scenarios: ScenarioResult[];
|
||
}
|
||
|
||
// ── Vault hydration ─────────────────────────────────────────────────
|
||
|
||
function hydrateVault(): { anthropic: string | null; openrouter: string | null } {
|
||
const vault = new VaultStore(WAGGLE_DATA_DIR);
|
||
const anthropic = vault.get('anthropic');
|
||
const openrouter = vault.get('openrouter');
|
||
if (anthropic) process.env.ANTHROPIC_API_KEY = anthropic.value;
|
||
if (openrouter) process.env.OPENROUTER_API_KEY = openrouter.value;
|
||
return { anthropic: anthropic?.value ?? null, openrouter: openrouter?.value ?? null };
|
||
}
|
||
|
||
// ── LLM clients ─────────────────────────────────────────────────────
|
||
|
||
async function callAnthropic(
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: { maxTokens?: number; temperature?: number } = {},
|
||
): Promise<string> {
|
||
const apiKey = process.env.ANTHROPIC_API_KEY;
|
||
if (!apiKey) throw new Error('ANTHROPIC_API_KEY not hydrated from vault');
|
||
|
||
const body: Record<string, unknown> = {
|
||
model,
|
||
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
|
||
messages: [{ role: 'user', content: userMsg }],
|
||
};
|
||
// Opus 4.7 rejects `temperature` as deprecated (extended-thinking models).
|
||
// Other Claude models accept it but provider default is fine for the eval.
|
||
// Omit entirely — all conditions use provider default → still controlled.
|
||
if (systemPrompt) body.system = systemPrompt;
|
||
|
||
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
||
method: 'POST',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
'x-api-key': apiKey,
|
||
'anthropic-version': '2023-06-01',
|
||
},
|
||
body: JSON.stringify(body),
|
||
});
|
||
if (!response.ok) {
|
||
const text = await response.text();
|
||
throw new Error(`Anthropic ${model} ${response.status}: ${text.slice(0, 500)}`);
|
||
}
|
||
const data = await response.json() as { content: Array<{ type: string; text?: string }> };
|
||
return data.content.map(c => c.text ?? '').join('');
|
||
}
|
||
|
||
async function callOpenRouter(
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
|
||
): Promise<string> {
|
||
const apiKey = process.env.OPENROUTER_API_KEY;
|
||
if (!apiKey) throw new Error('OPENROUTER_API_KEY not hydrated from vault');
|
||
|
||
const messages: Array<{ role: string; content: string }> = [];
|
||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||
messages.push({ role: 'user', content: userMsg });
|
||
|
||
const body: Record<string, unknown> = {
|
||
model,
|
||
messages,
|
||
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
|
||
temperature: opts.temperature ?? TEMPERATURE_GEN,
|
||
};
|
||
if (opts.seed !== undefined) body.seed = opts.seed;
|
||
|
||
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||
method: 'POST',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
authorization: `Bearer ${apiKey}`,
|
||
'http-referer': 'https://waggle-os.ai',
|
||
'x-title': 'Waggle PromptAssembler eval',
|
||
},
|
||
body: JSON.stringify(body),
|
||
});
|
||
if (!response.ok) {
|
||
const text = await response.text();
|
||
throw new Error(`OpenRouter ${model} ${response.status}: ${text.slice(0, 500)}`);
|
||
}
|
||
const data = await response.json() as { choices: Array<{ message: { content: string } }> };
|
||
return data.choices[0]?.message?.content ?? '';
|
||
}
|
||
|
||
async function callModel(
|
||
provider: 'anthropic' | 'openrouter',
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: { maxTokens?: number; temperature?: number; seed?: number } = {},
|
||
): Promise<string> {
|
||
if (provider === 'anthropic') return callAnthropic(model, systemPrompt, userMsg, opts);
|
||
return callOpenRouter(model, systemPrompt, userMsg, opts);
|
||
}
|
||
|
||
// ── Scenario pipeline ───────────────────────────────────────────────
|
||
|
||
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
|
||
type MemorySnapshot = readonly MemoryRecord[];
|
||
|
||
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
|
||
const orch = new Orchestrator({ memory, model });
|
||
orch.getIdentity().create({
|
||
name: 'Alex',
|
||
role: 'CEO',
|
||
department: 'Egzakta Group',
|
||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||
capabilities: 'Strategic decisions, technical oversight',
|
||
system_prompt: '',
|
||
});
|
||
return orch;
|
||
}
|
||
|
||
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
|
||
const filter = { scopes: [PERSONAL_SCOPE] };
|
||
return memory.list({ ...filter, limit: await memory.count(filter) });
|
||
}
|
||
|
||
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
|
||
const memory = createInMemoryMemoryStore();
|
||
// list() returns newest first; replay oldest first.
|
||
for (const record of [...snapshot].reverse()) {
|
||
await memory.remember({
|
||
scope: record.scope,
|
||
text: record.text,
|
||
kind: record.kind,
|
||
importance: record.importance,
|
||
source: record.source,
|
||
origin: record.origin ?? undefined,
|
||
sessionId: record.sessionId ?? undefined,
|
||
metadata: record.metadata,
|
||
});
|
||
}
|
||
return memory;
|
||
}
|
||
|
||
async function runPriming(
|
||
orch: Orchestrator,
|
||
scenario: PromptAssemblerScenario,
|
||
): Promise<void> {
|
||
for (const turn of scenario.primingTurns) {
|
||
const systemPrompt = await orch.buildSystemPrompt();
|
||
const assistantMsg = await callAnthropic(
|
||
PRIMING_MODEL,
|
||
systemPrompt,
|
||
turn.user,
|
||
{ temperature: 0, maxTokens: MAX_TOKENS_GEN },
|
||
);
|
||
await orch.autoSaveFromExchange(turn.user, assistantMsg);
|
||
}
|
||
}
|
||
|
||
function verifyMemory(
|
||
snapshot: MemorySnapshot,
|
||
scenario: PromptAssemblerScenario,
|
||
): { count: number; matches: Record<string, boolean> } {
|
||
const matches: Record<string, boolean> = {};
|
||
for (const sub of scenario.memoryVerificationSubstrings) {
|
||
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
|
||
}
|
||
return { count: snapshot.length, matches };
|
||
}
|
||
|
||
async function runCondition(
|
||
snapshot: MemorySnapshot,
|
||
condition: ConditionSpec,
|
||
scenario: PromptAssemblerScenario,
|
||
seed: number,
|
||
): Promise<ConditionRun> {
|
||
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
|
||
|
||
let systemPrompt: string;
|
||
let debug: ConditionRun['debug'] = undefined;
|
||
|
||
const start = Date.now();
|
||
try {
|
||
if (condition.usesPromptAssembler) {
|
||
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
||
const taskShape = detectTaskShape(scenario.testTurn.query);
|
||
const assembled = await orch.buildAssembledPrompt(scenario.testTurn.query, null, { taskShape });
|
||
systemPrompt = assembled.system;
|
||
debug = {
|
||
tier: assembled.debug.tier,
|
||
taskShape: assembled.debug.taskShape,
|
||
taskShapeConfidence: assembled.debug.taskShapeConfidence,
|
||
scaffoldApplied: assembled.debug.scaffoldApplied,
|
||
sectionsIncluded: assembled.debug.sectionsIncluded,
|
||
framesUsed: assembled.debug.framesUsed,
|
||
totalChars: assembled.debug.totalChars,
|
||
};
|
||
} else {
|
||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||
systemPrompt = await orch.buildSystemPrompt();
|
||
}
|
||
|
||
const output = await callModel(
|
||
condition.provider,
|
||
condition.model,
|
||
systemPrompt,
|
||
scenario.testTurn.query,
|
||
{ temperature: TEMPERATURE_GEN, seed },
|
||
);
|
||
return { seed, output, durationMs: Date.now() - start, debug };
|
||
} catch (err) {
|
||
return {
|
||
seed,
|
||
output: '',
|
||
durationMs: Date.now() - start,
|
||
debug,
|
||
error: err instanceof Error ? err.message : String(err),
|
||
};
|
||
}
|
||
}
|
||
|
||
async function judgeRun(
|
||
judge: LLMJudge,
|
||
scenario: PromptAssemblerScenario,
|
||
goldOutput: string,
|
||
candidateOutput: string,
|
||
): Promise<JudgeScore> {
|
||
return judge.score({
|
||
input: scenario.testTurn.query,
|
||
expected: goldOutput,
|
||
actual: candidateOutput,
|
||
context: `task_shape=${scenario.shape}, language=${scenario.language}`,
|
||
});
|
||
}
|
||
|
||
// ── Result aggregation ──────────────────────────────────────────────
|
||
|
||
function mean(xs: number[]): number {
|
||
if (xs.length === 0) return 0;
|
||
return xs.reduce((a, b) => a + b, 0) / xs.length;
|
||
}
|
||
|
||
function conditionMean(runs: ConditionRun[] | undefined): number {
|
||
if (!runs || runs.length === 0) return 0;
|
||
const scores = runs.map(r => r.score?.overall ?? 0);
|
||
return mean(scores);
|
||
}
|
||
|
||
function renderMarkdown(result: EvalResult): string {
|
||
const lines: string[] = [];
|
||
lines.push('# PromptAssembler eval results');
|
||
lines.push('');
|
||
lines.push(`**Run date:** ${result.runDate}`);
|
||
lines.push(`**Commit:** ${result.commit}`);
|
||
lines.push(`**Duration:** ${(result.durationMs / 1000 / 60).toFixed(1)} min`);
|
||
lines.push(`**Seeds per condition:** ${result.seeds}`);
|
||
lines.push(`**LiteLLM:** bypassed — see deviation note`);
|
||
lines.push('');
|
||
lines.push('## Deviation from brief §11.2');
|
||
lines.push('');
|
||
lines.push(result.deviationFromBrief);
|
||
lines.push('');
|
||
lines.push('## Slug probe');
|
||
lines.push('');
|
||
lines.push(`- Opus 4.7: \`${result.slugs.anthropicOpus47}\``);
|
||
lines.push(`- Gemma 4 31B: \`${result.slugs.openrouterGemma31b}\``);
|
||
lines.push(`- Gemma 4 26B MoE: \`${result.slugs.openrouterGemma26bMoE}\` *(substituted from brief's \`gemma-4-26b-it\`)*`);
|
||
lines.push(`- Qwen3-30B-A3B: \`${result.slugs.openrouterQwen3}\` *(substituted from brief's \`qwen3-30b-a3b-instruct\`)*`);
|
||
lines.push('');
|
||
|
||
lines.push('## Summary');
|
||
lines.push('');
|
||
const primedOK = result.scenarios.filter(s => !s.primingFailed).length;
|
||
|
||
const reasoningScenarios = result.scenarios.filter(s => s.shape !== 'draft');
|
||
let gapClosurePct = 0;
|
||
if (reasoningScenarios.length > 0) {
|
||
const aMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['A'])));
|
||
const bMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['B'])));
|
||
const cMean = mean(reasoningScenarios.map(s => conditionMean(s.conditions['C'])));
|
||
const gap = aMean - bMean;
|
||
const closure = cMean - bMean;
|
||
gapClosurePct = gap > 0 ? (closure / gap) * 100 : 0;
|
||
}
|
||
|
||
let maxDRegression = 0;
|
||
for (const s of result.scenarios) {
|
||
const a = conditionMean(s.conditions['A']);
|
||
const d = conditionMean(s.conditions['D']);
|
||
if (a > d) maxDRegression = Math.max(maxDRegression, a - d);
|
||
}
|
||
|
||
const aMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['A'])));
|
||
const eMeanAll = mean(result.scenarios.map(s => conditionMean(s.conditions['E'])));
|
||
|
||
lines.push('| Metric | Value |');
|
||
lines.push('|--------|-------|');
|
||
lines.push(`| Scenarios | ${result.scenarios.length} |`);
|
||
lines.push(`| Scenarios with successful priming | ${primedOK} / ${result.scenarios.length} |`);
|
||
lines.push(`| Seeds per scenario | ${result.seeds} |`);
|
||
lines.push(`| Gap closure (C−B)/(A−B), reasoning only | ${gapClosurePct.toFixed(1)}% |`);
|
||
lines.push(`| Target (≥40%) | ${gapClosurePct >= 40 ? '**PASS**' : '**FAIL**'} |`);
|
||
lines.push(`| D regression vs A (max over rows) | ${(maxDRegression * 100).toFixed(2)}pp |`);
|
||
lines.push(`| Opus generation delta (A 4.7 − E 4.6) | ${((aMeanAll - eMeanAll) * 100).toFixed(2)}pp |`);
|
||
lines.push('');
|
||
|
||
lines.push('## Priming results');
|
||
lines.push('');
|
||
lines.push('| Scenario | Lang | Frames | Matches |');
|
||
lines.push('|----------|------|--------|---------|');
|
||
for (const s of result.scenarios) {
|
||
const matchSummary = Object.entries(s.primingMatches)
|
||
.map(([k, v]) => `${v ? '✓' : '✗'} ${k}`)
|
||
.join(', ');
|
||
lines.push(`| ${s.scenario} | ${s.language} | ${s.primingFrameCount} | ${matchSummary} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
lines.push('## Per-scenario breakdown (primary)');
|
||
lines.push('');
|
||
const primaryCodes = PRIMARY_CONDITIONS.map(c => c.code);
|
||
lines.push(`| Scenario | shape | ${primaryCodes.join(' | ')} | (C−B) | (A−B) |`);
|
||
lines.push(`|----------|-------|${primaryCodes.map(() => '---').join('|')}|-------|-------|`);
|
||
for (const s of result.scenarios) {
|
||
const cells = primaryCodes.map(code => {
|
||
const m = conditionMean(s.conditions[code]);
|
||
return m.toFixed(3);
|
||
});
|
||
const a = conditionMean(s.conditions['A']);
|
||
const b = conditionMean(s.conditions['B']);
|
||
const c = conditionMean(s.conditions['C']);
|
||
lines.push(`| ${s.scenario} | ${s.shape} | ${cells.join(' | ')} | ${(c - b).toFixed(3)} | ${(a - b).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
const hasSecondary26 = result.scenarios.some(s => s.conditions["B'"] || s.conditions["C'"]);
|
||
if (hasSecondary26) {
|
||
lines.push('## Secondary — Gemma 4 26B MoE');
|
||
lines.push('');
|
||
lines.push(`| Scenario | B' | C' | (C'−B') |`);
|
||
lines.push(`|----------|-----|-----|---------|`);
|
||
for (const s of result.scenarios) {
|
||
const bp = conditionMean(s.conditions["B'"]);
|
||
const cp = conditionMean(s.conditions["C'"]);
|
||
lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
|
||
const hasSecondaryQ = result.scenarios.some(s => s.conditions["B''"] || s.conditions["C''"]);
|
||
if (hasSecondaryQ) {
|
||
lines.push('## Secondary — Qwen3-30B-A3B');
|
||
lines.push('');
|
||
lines.push(`| Scenario | B'' | C'' | (C''−B'') |`);
|
||
lines.push(`|----------|------|------|-----------|`);
|
||
for (const s of result.scenarios) {
|
||
const bp = conditionMean(s.conditions["B''"]);
|
||
const cp = conditionMean(s.conditions["C''"]);
|
||
lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${cp.toFixed(3)} | ${(cp - bp).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
|
||
lines.push('## Cross-model pattern (reasoning scenarios only)');
|
||
lines.push('');
|
||
const check = (bCode: string, cCode: string): boolean => {
|
||
if (!reasoningScenarios.length) return false;
|
||
return reasoningScenarios.every(s => {
|
||
const b = conditionMean(s.conditions[bCode]);
|
||
const c = conditionMean(s.conditions[cCode]);
|
||
return c >= b;
|
||
});
|
||
};
|
||
const perModel: Array<[string, boolean]> = [['Gemma 4 31B', check('B', 'C')]];
|
||
if (hasSecondary26) perModel.push(['Gemma 4 26B MoE', check("B'", "C'")]);
|
||
if (hasSecondaryQ) perModel.push(['Qwen3-30B-A3B', check("B''", "C''")]);
|
||
for (const [model, positive] of perModel) {
|
||
lines.push(`- **${model}**: ${positive ? 'all reasoning scenarios C ≥ B ✓' : 'mixed or negative'}`);
|
||
}
|
||
lines.push('');
|
||
|
||
lines.push('## Opus generation delta (A 4.7 − E 4.6)');
|
||
lines.push('');
|
||
lines.push(`| Scenario | A (4.7) | E (4.6) | Δ |`);
|
||
lines.push(`|----------|---------|---------|-----|`);
|
||
for (const s of result.scenarios) {
|
||
const a = conditionMean(s.conditions['A']);
|
||
const e = conditionMean(s.conditions['E']);
|
||
lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${e.toFixed(3)} | ${(a - e).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
lines.push('## Sample outputs (best-scoring C seed per scenario)');
|
||
lines.push('');
|
||
for (const s of result.scenarios) {
|
||
const cRuns = s.conditions['C'] ?? [];
|
||
const best = [...cRuns].sort((a, b) => (b.score?.overall ?? 0) - (a.score?.overall ?? 0))[0];
|
||
if (best) {
|
||
lines.push(`### ${s.scenario}`);
|
||
lines.push('');
|
||
lines.push(`Best C score: ${best.score?.overall?.toFixed(3) ?? 'N/A'}, seed: ${best.seed}`);
|
||
lines.push('');
|
||
lines.push('```');
|
||
lines.push(best.output.slice(0, 800) + (best.output.length > 800 ? '\n...[truncated]' : ''));
|
||
lines.push('```');
|
||
lines.push('');
|
||
if (best.debug) {
|
||
lines.push(`Debug: tier=${best.debug.tier}, shape=${best.debug.taskShape}, conf=${best.debug.taskShapeConfidence.toFixed(2)}, scaffoldApplied=${best.debug.scaffoldApplied}, sections=[${best.debug.sectionsIncluded.join(', ')}], frames=${best.debug.framesUsed}, chars=${best.debug.totalChars}`);
|
||
lines.push('');
|
||
}
|
||
}
|
||
}
|
||
|
||
lines.push('## Honest observations');
|
||
lines.push('');
|
||
const obs: string[] = [];
|
||
if (gapClosurePct >= 40) {
|
||
obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap on reasoning scenarios — meets the ≥40% target.`);
|
||
} else if (gapClosurePct > 0) {
|
||
obs.push(`- C closes **${gapClosurePct.toFixed(0)}%** of the A−B gap — below the ≥40% target. Scaffold helps but not structurally sufficient on its own.`);
|
||
} else {
|
||
obs.push(`- C − B is ${gapClosurePct.toFixed(0)}% — PA did not close the gap. Consider scenario iteration or deeper intervention.`);
|
||
}
|
||
if (maxDRegression > 0.02) {
|
||
obs.push(`- D regresses from A by up to **${(maxDRegression * 100).toFixed(1)}pp** — exceeds 2pp guardrail. Investigate frontier overhead.`);
|
||
} else {
|
||
obs.push(`- D does not regress from A by more than 2pp — frontier tier handles PA gracefully.`);
|
||
}
|
||
const serbianScenarios = result.scenarios.filter(s => s.language === 'sr');
|
||
const serbianPriming = serbianScenarios.filter(s => !s.primingFailed).length;
|
||
obs.push(`- Serbian priming: ${serbianPriming} / ${serbianScenarios.length} succeeded with English save-trigger phrases mixed in.`);
|
||
for (const o of obs) lines.push(o);
|
||
lines.push('');
|
||
|
||
lines.push('---');
|
||
lines.push('');
|
||
lines.push('Generated by `packages/agent/tests/eval/prompt-assembler-eval.ts`.');
|
||
lines.push('Full structured results: `tmp_bench_results.json` (gitignored).');
|
||
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ── Main ────────────────────────────────────────────────────────────
|
||
|
||
async function main(): Promise<void> {
|
||
const startTime = Date.now();
|
||
const runDate = new Date().toISOString();
|
||
|
||
console.log('[hydrate] Reading vault keys...');
|
||
const keys = hydrateVault();
|
||
if (!keys.anthropic) throw new Error('Anthropic key not found in vault');
|
||
if (!keys.openrouter) throw new Error('OpenRouter key not found in vault');
|
||
console.log('[hydrate] anthropic + openrouter keys loaded.');
|
||
|
||
let commit = 'unknown';
|
||
try {
|
||
commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
|
||
const conditions = SKIP_SECONDARY
|
||
? PRIMARY_CONDITIONS
|
||
: [...PRIMARY_CONDITIONS, ...SECONDARY_26B_CONDITIONS, ...SECONDARY_QWEN_CONDITIONS];
|
||
|
||
const judge = new LLMJudge(async (prompt: string) => {
|
||
return callAnthropic(JUDGE_MODEL, '', prompt, { temperature: TEMPERATURE_JUDGE, maxTokens: MAX_TOKENS_JUDGE });
|
||
});
|
||
|
||
const result: EvalResult = {
|
||
runDate,
|
||
commit,
|
||
durationMs: 0,
|
||
deviationFromBrief:
|
||
'LiteLLM proxy was not reachable on localhost:4000 at eval start. ' +
|
||
'This harness calls Anthropic (/v1/messages) and OpenRouter (/v1/chat/completions) ' +
|
||
'APIs directly via fetch(). Measurement validity is unaffected — the variable under ' +
|
||
'test (prompt structure C vs B) is isolated correctly since both conditions share ' +
|
||
'the same model, same user message, and same temperature.',
|
||
slugs: {
|
||
openrouterGemma31b: GEMMA_31B_MODEL,
|
||
openrouterGemma26bMoE: GEMMA_26B_MOE_MODEL,
|
||
openrouterQwen3: QWEN_30B_MODEL,
|
||
anthropicOpus47: OPUS_4_7_MODEL,
|
||
},
|
||
seeds: SEEDS,
|
||
scenarios: [],
|
||
};
|
||
|
||
for (const [idx, scenario] of SCENARIOS.entries()) {
|
||
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||
|
||
const primingStart = Date.now();
|
||
|
||
const primingMemory = createInMemoryMemoryStore();
|
||
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
|
||
|
||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
|
||
try {
|
||
await runPriming(primingOrch, scenario);
|
||
} catch (err) {
|
||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
const snapshot = await snapshotMemory(primingMemory);
|
||
const verification = verifyMemory(snapshot, scenario);
|
||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
|
||
|
||
const scenarioResult: ScenarioResult = {
|
||
scenario: scenario.name,
|
||
shape: scenario.shape,
|
||
language: scenario.language,
|
||
primingFrameCount: verification.count,
|
||
primingMatches: verification.matches,
|
||
primingFailed,
|
||
primingDurationMs: Date.now() - primingStart,
|
||
conditions: {},
|
||
};
|
||
|
||
for (const condition of conditions) {
|
||
scenarioResult.conditions[condition.code] = [];
|
||
for (let seed = 0; seed < SEEDS; seed++) {
|
||
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
|
||
const run = await runCondition(snapshot, condition, scenario, seed);
|
||
scenarioResult.conditions[condition.code].push(run);
|
||
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
|
||
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
|
||
}
|
||
}
|
||
|
||
console.log(` [judge] scoring outputs via ${JUDGE_MODEL}...`);
|
||
const aRuns = scenarioResult.conditions['A'] ?? [];
|
||
for (const condition of conditions) {
|
||
if (condition.code === 'A') {
|
||
for (const run of scenarioResult.conditions['A']) {
|
||
run.score = {
|
||
overall: 1.0,
|
||
weighted: 1.0,
|
||
correctness: 10,
|
||
procedureFollowing: 10,
|
||
conciseness: 10,
|
||
lengthPenalty: 1,
|
||
feedback: 'Gold reference (condition A).',
|
||
parsed: true,
|
||
};
|
||
}
|
||
continue;
|
||
}
|
||
const runs = scenarioResult.conditions[condition.code] ?? [];
|
||
for (const run of runs) {
|
||
if (run.error || !run.output) continue;
|
||
const gold = aRuns.find(r => r.seed === run.seed) ?? aRuns[0];
|
||
if (!gold || gold.error || !gold.output) continue;
|
||
try {
|
||
run.score = await judgeRun(judge, scenario, gold.output, run.output);
|
||
} catch (err) {
|
||
console.log(` judge error for ${condition.code} seed ${run.seed}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
result.scenarios.push(scenarioResult);
|
||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||
}
|
||
|
||
result.durationMs = Date.now() - startTime;
|
||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||
fs.writeFileSync(RESULTS_MD, renderMarkdown(result));
|
||
|
||
console.log(`\n[done] ${result.scenarios.length} scenarios in ${(result.durationMs / 1000 / 60).toFixed(1)} min.`);
|
||
console.log(`[done] JSON: ${RESULTS_JSON}`);
|
||
console.log(`[done] MD: ${RESULTS_MD}`);
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error('[fatal]', err);
|
||
process.exit(1);
|
||
});
|