1248 lines
51 KiB
TypeScript
1248 lines
51 KiB
TypeScript
/**
|
||
* PromptAssembler v5 eval harness.
|
||
*
|
||
* Executes the §11 measurement protocol from the v5 brief:
|
||
* - 12 conditions (7 primary + 3 secondary-26b + 2 secondary-qwen)
|
||
* - 6 scenarios from SCENARIOS_V5 (revised priming per §11.4)
|
||
* - 3 seeds per condition, with variance-triggered retry (§11.5)
|
||
* - 4-model frontier judge ensemble (§11.2): Gemini 3.1 Pro, GPT-5.4,
|
||
* Grok 4.20, MiniMax M2.7 — no Claude judge to avoid v4's
|
||
* same-family bias
|
||
* - Per-run ensemble mean + disagreement tracking
|
||
* - WAGGLE_EVAL_MODE=1 tier bypass throughout (§11.3)
|
||
* - Pre-eval slug probe across all 11 slugs (§17 step 7)
|
||
*
|
||
* Usage (from repo root):
|
||
* tsx packages/agent/tests/eval/prompt-assembler-v5-eval.ts
|
||
*
|
||
* Environment:
|
||
* WAGGLE_DATA_DIR — override ~/.waggle (for non-default installs)
|
||
* WAGGLE_EVAL_SKIP_SECONDARY=1 — skip Gemma 4 26B MoE + Qwen thinking
|
||
* WAGGLE_EVAL_SEEDS — base seeds per condition (default 3)
|
||
* WAGGLE_EVAL_SKIP_SLUG_PROBE=1 — skip the pre-eval slug probe
|
||
* WAGGLE_EVAL_MAX_JUDGE_MS=180000 — per-judge-call timeout (ms, default 3 min)
|
||
*
|
||
* Outputs:
|
||
* tmp_bench_results-v5.json — full structured results (gitignored)
|
||
* EVAL-RESULTS-V5.md — rendered results per §11.8 (committed)
|
||
*
|
||
* Deviation policy: if any judge is unavailable at eval time, continue
|
||
* with the remaining judges (minimum 3). Do NOT substitute Sonnet 4.6
|
||
* — the ensemble must stay outside the Claude family. If fewer than 3
|
||
* judges are reachable, the eval halts and reports partial state.
|
||
*/
|
||
|
||
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
|
||
import { Orchestrator } from '../../src/orchestrator.js';
|
||
import { type ModelTier } from '../../src/model-tier.js';
|
||
import { detectTaskShape } from '../../src/task-shape.js';
|
||
import { LLMJudge, type JudgeScore } from '../../src/judge.js';
|
||
import type { ScaffoldStyle } from '../../src/prompt-assembler.js';
|
||
import { SCENARIOS_V5, type PromptAssemblerScenario } from './scenarios-prompt-assembler-v5.js';
|
||
import * as fs from 'node:fs';
|
||
import * as path from 'node:path';
|
||
import * as os from 'node:os';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { execFileSync } from 'node:child_process';
|
||
|
||
// ── Eval-mode flag — MUST be set before Orchestrator construction so the
|
||
// embedding-provider tier gate (and any future gates) are bypassed. ──
|
||
process.env.WAGGLE_EVAL_MODE = '1';
|
||
|
||
// ── Config ──────────────────────────────────────────────────────────
|
||
|
||
const WAGGLE_DATA_DIR = process.env.WAGGLE_DATA_DIR ?? path.join(os.homedir(), '.waggle');
|
||
const HARNESS_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||
const REPO_ROOT = path.resolve(path.join(HARNESS_DIR, '..', '..', '..', '..'));
|
||
const RESULTS_JSON = path.join(REPO_ROOT, 'tmp_bench_results-v5.json');
|
||
const RESULTS_MD = path.join(REPO_ROOT, 'EVAL-RESULTS-V5.md');
|
||
|
||
// Candidate + priming slugs (verified live 2026-04-17 via v4 eval run).
|
||
const PRIMING_MODEL = 'claude-sonnet-4-6';
|
||
const OPUS_4_7_MODEL = 'claude-opus-4-7';
|
||
const OPUS_4_6_MODEL = 'claude-opus-4-6';
|
||
const GEMMA_31B_MODEL = 'google/gemma-4-31b-it';
|
||
const GEMMA_26B_MOE_MODEL = 'google/gemma-4-26b-a4b-it';
|
||
// v5 brief §7.1: Qwen thinking variant is the v5 Qwen candidate.
|
||
const QWEN_THINKING_MODEL = 'qwen/qwen3-30b-a3b-thinking-2507';
|
||
// Kept only for legacy slug probe per §14 — not used in any v5 condition.
|
||
const QWEN_INSTRUCT_LEGACY_MODEL = 'qwen/qwen3-30b-a3b-instruct-2507';
|
||
|
||
// Judge slugs (4 labs, none in Claude family). Verified live 2026-04-17.
|
||
const JUDGE_GEMINI = 'gemini-3.1-pro-preview';
|
||
const JUDGE_GPT = 'gpt-5.4';
|
||
const JUDGE_GROK = 'grok-4.20';
|
||
const JUDGE_MINIMAX = 'MiniMax-M2.7';
|
||
|
||
const TEMPERATURE_GEN = 0.2;
|
||
const TEMPERATURE_JUDGE = 0;
|
||
const MAX_TOKENS_GEN = 1024;
|
||
// Raised from 512 after dry run: thinking-mode judges (Gemini 3.1 Pro,
|
||
// MiniMax M2.7) consume significant output budget for internal reasoning
|
||
// BEFORE emitting the rubric JSON. 512 caused Gemini to truncate mid-JSON
|
||
// ("{"correctness": 7, "procedure": 10, "conciseness") and MiniMax never
|
||
// reached the JSON payload at all. 4096 gives headroom for reasoning +
|
||
// a full rubric response including the 3-sentence feedback field.
|
||
const MAX_TOKENS_JUDGE = 4096;
|
||
const BASE_SEEDS = Number.parseInt(process.env.WAGGLE_EVAL_SEEDS ?? '3', 10);
|
||
const SKIP_SECONDARY = process.env.WAGGLE_EVAL_SKIP_SECONDARY === '1';
|
||
const SKIP_SLUG_PROBE = process.env.WAGGLE_EVAL_SKIP_SLUG_PROBE === '1';
|
||
const PROBE_ONLY = process.env.WAGGLE_EVAL_PROBE_ONLY === '1';
|
||
const MAX_SCENARIOS = Number.parseInt(process.env.WAGGLE_EVAL_MAX_SCENARIOS ?? '0', 10);
|
||
const MAX_JUDGE_MS = Number.parseInt(process.env.WAGGLE_EVAL_MAX_JUDGE_MS ?? '180000', 10);
|
||
|
||
// Variance-retry threshold — if max-min of per-seed ensemble means
|
||
// exceeds this on a condition, add 2 more seeds (total 5).
|
||
const VARIANCE_RETRY_THRESHOLD = 0.15;
|
||
const RETRY_EXTRA_SEEDS = 2;
|
||
const MIN_JUDGES_REACHABLE = 3;
|
||
|
||
// ── Types ────────────────────────────────────────────────────────────
|
||
|
||
type JudgeName = typeof JUDGE_GEMINI | typeof JUDGE_GPT | typeof JUDGE_GROK | typeof JUDGE_MINIMAX;
|
||
type Provider = 'anthropic' | 'openrouter' | 'openai' | 'xai' | 'minimax' | 'gemini';
|
||
|
||
interface ConditionSpec {
|
||
code: string;
|
||
label: string;
|
||
model: string;
|
||
provider: Extract<Provider, 'anthropic' | 'openrouter'>;
|
||
usesPromptAssembler: boolean;
|
||
scaffoldStyle?: ScaffoldStyle;
|
||
suite: 'primary' | 'secondary-26b' | 'secondary-qwen';
|
||
}
|
||
|
||
interface JudgeCallResult {
|
||
judge: JudgeName;
|
||
score: JudgeScore | null;
|
||
error?: string;
|
||
durationMs: number;
|
||
}
|
||
|
||
interface EnsembleScore {
|
||
judgeResults: JudgeCallResult[];
|
||
judgesAvailable: number;
|
||
mean: number; // mean over judges that returned a parsed score
|
||
min: number;
|
||
max: number;
|
||
disagreement: number; // max - min across available judges
|
||
}
|
||
|
||
interface ConditionRun {
|
||
seed: number;
|
||
output: string;
|
||
durationMs: number;
|
||
debug?: {
|
||
tier: ModelTier;
|
||
taskShape: string | null;
|
||
taskShapeConfidence: number;
|
||
scaffoldApplied: boolean;
|
||
scaffoldStyle: ScaffoldStyle;
|
||
sectionsIncluded: string[];
|
||
framesUsed: number;
|
||
totalChars: number;
|
||
};
|
||
ensemble?: EnsembleScore;
|
||
error?: string;
|
||
}
|
||
|
||
interface ScenarioResult {
|
||
scenario: string;
|
||
shape: string;
|
||
language: string;
|
||
primingFrameCount: number;
|
||
primingMatches: Record<string, boolean>;
|
||
primingFailed: boolean;
|
||
primingPartial: boolean;
|
||
primingDurationMs: number;
|
||
effectiveTier: string;
|
||
conditions: Record<string, ConditionRun[]>;
|
||
retriedConditions: string[];
|
||
}
|
||
|
||
interface SlugProbeResult {
|
||
slug: string;
|
||
provider: Provider;
|
||
role: 'priming' | 'candidate' | 'candidate-legacy' | 'judge';
|
||
ok: boolean;
|
||
error?: string;
|
||
durationMs: number;
|
||
}
|
||
|
||
interface EvalResult {
|
||
runDate: string;
|
||
commit: string;
|
||
durationMs: number;
|
||
slugProbe: SlugProbeResult[];
|
||
judgesAvailable: JudgeName[];
|
||
tierBypass: boolean;
|
||
seeds: number;
|
||
scenarios: ScenarioResult[];
|
||
}
|
||
|
||
// ── Vault hydration ─────────────────────────────────────────────────
|
||
|
||
function hydrateVault(): Record<string, string | null> {
|
||
const vault = new VaultStore(WAGGLE_DATA_DIR);
|
||
const keys: Record<string, string | null> = {
|
||
anthropic: vault.get('anthropic')?.value ?? null,
|
||
openrouter: vault.get('openrouter')?.value ?? null,
|
||
openai: vault.get('openai')?.value ?? null,
|
||
gemini: vault.get('gemini')?.value ?? null,
|
||
xai: vault.get('xai')?.value ?? null,
|
||
minimax: vault.get('minimax')?.value ?? null,
|
||
};
|
||
if (keys.anthropic) process.env.ANTHROPIC_API_KEY = keys.anthropic;
|
||
if (keys.openrouter) process.env.OPENROUTER_API_KEY = keys.openrouter;
|
||
if (keys.openai) process.env.OPENAI_API_KEY = keys.openai;
|
||
if (keys.gemini) process.env.GEMINI_API_KEY = keys.gemini;
|
||
if (keys.xai) process.env.XAI_API_KEY = keys.xai;
|
||
if (keys.minimax) process.env.MINIMAX_API_KEY = keys.minimax;
|
||
return keys;
|
||
}
|
||
|
||
// ── LLM clients ─────────────────────────────────────────────────────
|
||
|
||
interface CallOpts {
|
||
maxTokens?: number;
|
||
temperature?: number;
|
||
seed?: number;
|
||
/** Judge-only: per-call timeout. */
|
||
timeoutMs?: number;
|
||
}
|
||
|
||
async function fetchWithTimeout(url: string, init: RequestInit, timeoutMs: number): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
return await fetch(url, { ...init, signal: controller.signal });
|
||
} finally {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
async function callAnthropic(
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: CallOpts = {},
|
||
): Promise<string> {
|
||
const apiKey = process.env.ANTHROPIC_API_KEY;
|
||
if (!apiKey) throw new Error('ANTHROPIC_API_KEY not hydrated from vault');
|
||
const body: Record<string, unknown> = {
|
||
model,
|
||
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
|
||
messages: [{ role: 'user', content: userMsg }],
|
||
};
|
||
if (systemPrompt) body.system = systemPrompt;
|
||
const response = await fetchWithTimeout(
|
||
'https://api.anthropic.com/v1/messages',
|
||
{
|
||
method: 'POST',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
'x-api-key': apiKey,
|
||
'anthropic-version': '2023-06-01',
|
||
},
|
||
body: JSON.stringify(body),
|
||
},
|
||
opts.timeoutMs ?? 120_000,
|
||
);
|
||
if (!response.ok) {
|
||
const text = await response.text();
|
||
throw new Error(`Anthropic ${model} ${response.status}: ${text.slice(0, 500)}`);
|
||
}
|
||
const data = (await response.json()) as { content: Array<{ type: string; text?: string }> };
|
||
return data.content.map(c => c.text ?? '').join('');
|
||
}
|
||
|
||
async function callOpenRouter(
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: CallOpts = {},
|
||
): Promise<string> {
|
||
const apiKey = process.env.OPENROUTER_API_KEY;
|
||
if (!apiKey) throw new Error('OPENROUTER_API_KEY not hydrated from vault');
|
||
const messages: Array<{ role: string; content: string }> = [];
|
||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||
messages.push({ role: 'user', content: userMsg });
|
||
const body: Record<string, unknown> = {
|
||
model,
|
||
messages,
|
||
max_tokens: opts.maxTokens ?? MAX_TOKENS_GEN,
|
||
temperature: opts.temperature ?? TEMPERATURE_GEN,
|
||
};
|
||
if (opts.seed !== undefined) body.seed = opts.seed;
|
||
const response = await fetchWithTimeout(
|
||
'https://openrouter.ai/api/v1/chat/completions',
|
||
{
|
||
method: 'POST',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
authorization: `Bearer ${apiKey}`,
|
||
'http-referer': 'https://waggle-os.ai',
|
||
'x-title': 'Waggle PromptAssembler v5 eval',
|
||
},
|
||
body: JSON.stringify(body),
|
||
},
|
||
opts.timeoutMs ?? 180_000,
|
||
);
|
||
if (!response.ok) {
|
||
const text = await response.text();
|
||
throw new Error(`OpenRouter ${model} ${response.status}: ${text.slice(0, 500)}`);
|
||
}
|
||
const data = (await response.json()) as { choices: Array<{ message: { content: string } }> };
|
||
return data.choices[0]?.message?.content ?? '';
|
||
}
|
||
|
||
/**
|
||
* Body-shape flags for OpenAI-compatible endpoints.
|
||
*
|
||
* OpenAI's reasoning-class models (o-series, gpt-5.x) require
|
||
* `max_completion_tokens` instead of `max_tokens` and reject
|
||
* non-default `temperature`. xAI, MiniMax, and Gemini's shim still
|
||
* accept the classic shape. Use these flags to adapt per-endpoint.
|
||
*/
|
||
interface OpenAICompatBodyShape {
|
||
/** Use `max_completion_tokens` instead of `max_tokens`. Default false. */
|
||
useCompletionTokensParam?: boolean;
|
||
/** Omit `temperature` entirely. Required for OpenAI reasoning models. */
|
||
omitTemperature?: boolean;
|
||
}
|
||
|
||
async function callOpenAICompat(
|
||
baseUrl: string,
|
||
apiKey: string,
|
||
apiKeyName: string,
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: CallOpts = {},
|
||
extraBody: Record<string, unknown> = {},
|
||
shape: OpenAICompatBodyShape = {},
|
||
): Promise<string> {
|
||
if (!apiKey) throw new Error(`${apiKeyName} not hydrated from vault`);
|
||
const messages: Array<{ role: string; content: string }> = [];
|
||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||
messages.push({ role: 'user', content: userMsg });
|
||
const body: Record<string, unknown> = { model, messages, ...extraBody };
|
||
const tokenCount = opts.maxTokens ?? MAX_TOKENS_JUDGE;
|
||
if (shape.useCompletionTokensParam) body.max_completion_tokens = tokenCount;
|
||
else body.max_tokens = tokenCount;
|
||
if (!shape.omitTemperature) body.temperature = opts.temperature ?? TEMPERATURE_JUDGE;
|
||
const response = await fetchWithTimeout(
|
||
`${baseUrl}/chat/completions`,
|
||
{
|
||
method: 'POST',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(body),
|
||
},
|
||
opts.timeoutMs ?? MAX_JUDGE_MS,
|
||
);
|
||
if (!response.ok) {
|
||
const text = await response.text();
|
||
throw new Error(`${apiKeyName} ${model} ${response.status}: ${text.slice(0, 500)}`);
|
||
}
|
||
const data = (await response.json()) as { choices: Array<{ message: { content: string } }> };
|
||
return data.choices[0]?.message?.content ?? '';
|
||
}
|
||
|
||
/** GPT-5.4 and OpenAI o-series body shape (completion_tokens, no temperature). */
|
||
const OPENAI_REASONING_SHAPE: OpenAICompatBodyShape = {
|
||
useCompletionTokensParam: true,
|
||
omitTemperature: true,
|
||
};
|
||
|
||
async function callGemini(
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: CallOpts = {},
|
||
): Promise<string> {
|
||
const apiKey = process.env.GEMINI_API_KEY;
|
||
if (!apiKey) throw new Error('GEMINI_API_KEY not hydrated from vault');
|
||
// Use Gemini's OpenAI-compatible shim — same body shape as OpenAI.
|
||
return callOpenAICompat(
|
||
'https://generativelanguage.googleapis.com/v1beta/openai',
|
||
apiKey,
|
||
'GEMINI_API_KEY',
|
||
model,
|
||
systemPrompt,
|
||
userMsg,
|
||
opts,
|
||
);
|
||
}
|
||
|
||
async function callGenerationModel(
|
||
provider: 'anthropic' | 'openrouter',
|
||
model: string,
|
||
systemPrompt: string,
|
||
userMsg: string,
|
||
opts: CallOpts = {},
|
||
): Promise<string> {
|
||
if (provider === 'anthropic') return callAnthropic(model, systemPrompt, userMsg, opts);
|
||
return callOpenRouter(model, systemPrompt, userMsg, opts);
|
||
}
|
||
|
||
// ── Judge wiring ────────────────────────────────────────────────────
|
||
|
||
interface JudgeWiring {
|
||
name: JudgeName;
|
||
call: (prompt: string) => Promise<string>;
|
||
/** True after a successful slug probe. False → excluded from ensemble. */
|
||
available: boolean;
|
||
}
|
||
|
||
/**
|
||
* Strip interleaved-thinking blocks from a judge response.
|
||
*
|
||
* MiniMax M2.7 wraps its output in `<think>...</think>` blocks by default.
|
||
* Gemini 3.1 Pro sometimes emits similar blocks under its thinking modes.
|
||
* The LLMJudge parser expects clean JSON, so we unwrap before returning.
|
||
* Defensive: also handles nested or unclosed think blocks by stripping
|
||
* any remaining opening tags after close-pairing. No-op on outputs that
|
||
* don't contain think tags (Grok, GPT-5.4 under OpenAI-compat).
|
||
*
|
||
* Dry run on 2026-04-17 showed MiniMax at 15/18 parse failures entirely
|
||
* due to this wrapping; fix validated by re-probe.
|
||
*/
|
||
function stripThinkingBlocks(raw: string): string {
|
||
// Strip balanced <think>...</think> (non-greedy, multi-line).
|
||
let out = raw.replace(/<think>[\s\S]*?<\/think>\s*/gi, '');
|
||
// Defensive: if an opening <think> survived without a close (budget
|
||
// cutoff mid-reasoning), drop everything up to and including it.
|
||
const stray = out.lastIndexOf('<think>');
|
||
if (stray >= 0) out = out.slice(stray + '<think>'.length);
|
||
return out.trim();
|
||
}
|
||
|
||
/** Wrap a judge call to strip thinking tags from the response. */
|
||
function withThinkingStripper(
|
||
fn: (prompt: string) => Promise<string>,
|
||
): (prompt: string) => Promise<string> {
|
||
return async (prompt: string) => stripThinkingBlocks(await fn(prompt));
|
||
}
|
||
|
||
function buildJudgeWirings(): JudgeWiring[] {
|
||
// All judges get the thinking-block stripper applied uniformly.
|
||
// For Grok and GPT-5.4 it's a no-op; for MiniMax and Gemini it's
|
||
// the difference between a parsed score and a silent drop.
|
||
return [
|
||
{
|
||
name: JUDGE_GEMINI,
|
||
available: true,
|
||
call: withThinkingStripper((p) =>
|
||
callGemini(JUDGE_GEMINI, '', p, { maxTokens: MAX_TOKENS_JUDGE, temperature: TEMPERATURE_JUDGE }),
|
||
),
|
||
},
|
||
{
|
||
name: JUDGE_GPT,
|
||
available: true,
|
||
// GPT-5.4 is a reasoning-class model — must use max_completion_tokens
|
||
// and omit temperature per OpenAI's current API contract (verified
|
||
// via 400 response during slug probe 2026-04-17).
|
||
call: withThinkingStripper((p) =>
|
||
callOpenAICompat(
|
||
'https://api.openai.com/v1',
|
||
process.env.OPENAI_API_KEY ?? '',
|
||
'OPENAI_API_KEY',
|
||
JUDGE_GPT,
|
||
'',
|
||
p,
|
||
{ maxTokens: MAX_TOKENS_JUDGE },
|
||
{},
|
||
OPENAI_REASONING_SHAPE,
|
||
),
|
||
),
|
||
},
|
||
{
|
||
name: JUDGE_GROK,
|
||
available: true,
|
||
call: withThinkingStripper((p) =>
|
||
callOpenAICompat(
|
||
'https://api.x.ai/v1',
|
||
process.env.XAI_API_KEY ?? '',
|
||
'XAI_API_KEY',
|
||
JUDGE_GROK,
|
||
'',
|
||
p,
|
||
{ maxTokens: MAX_TOKENS_JUDGE, temperature: TEMPERATURE_JUDGE },
|
||
// Per §11.2: explicitly enable Grok's reasoning mode for judging.
|
||
{ reasoning: { enabled: true } },
|
||
),
|
||
),
|
||
},
|
||
{
|
||
name: JUDGE_MINIMAX,
|
||
available: true,
|
||
call: withThinkingStripper((p) =>
|
||
callOpenAICompat(
|
||
'https://api.minimaxi.chat/v1',
|
||
process.env.MINIMAX_API_KEY ?? '',
|
||
'MINIMAX_API_KEY',
|
||
JUDGE_MINIMAX,
|
||
'',
|
||
p,
|
||
{ maxTokens: MAX_TOKENS_JUDGE, temperature: TEMPERATURE_JUDGE },
|
||
),
|
||
),
|
||
},
|
||
];
|
||
}
|
||
|
||
// ── Slug probe ──────────────────────────────────────────────────────
|
||
|
||
interface SlugProbeSpec {
|
||
slug: string;
|
||
provider: Provider;
|
||
role: SlugProbeResult['role'];
|
||
probe: () => Promise<string>;
|
||
}
|
||
|
||
function buildSlugProbeSpecs(): SlugProbeSpec[] {
|
||
const trivial = 'Reply with exactly: OK';
|
||
return [
|
||
{ slug: PRIMING_MODEL, provider: 'anthropic', role: 'priming', probe: () => callAnthropic(PRIMING_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: OPUS_4_7_MODEL, provider: 'anthropic', role: 'candidate', probe: () => callAnthropic(OPUS_4_7_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: OPUS_4_6_MODEL, provider: 'anthropic', role: 'candidate', probe: () => callAnthropic(OPUS_4_6_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: GEMMA_31B_MODEL, provider: 'openrouter', role: 'candidate', probe: () => callOpenRouter(GEMMA_31B_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: GEMMA_26B_MOE_MODEL, provider: 'openrouter', role: 'candidate', probe: () => callOpenRouter(GEMMA_26B_MOE_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: QWEN_THINKING_MODEL, provider: 'openrouter', role: 'candidate', probe: () => callOpenRouter(QWEN_THINKING_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: QWEN_INSTRUCT_LEGACY_MODEL, provider: 'openrouter', role: 'candidate-legacy', probe: () => callOpenRouter(QWEN_INSTRUCT_LEGACY_MODEL, '', trivial, { maxTokens: 20 }) },
|
||
{ slug: JUDGE_GEMINI, provider: 'gemini', role: 'judge', probe: () => callGemini(JUDGE_GEMINI, '', trivial, { maxTokens: 20 }) },
|
||
{
|
||
slug: JUDGE_GPT,
|
||
provider: 'openai',
|
||
role: 'judge',
|
||
probe: () =>
|
||
callOpenAICompat(
|
||
'https://api.openai.com/v1',
|
||
process.env.OPENAI_API_KEY ?? '',
|
||
'OPENAI_API_KEY',
|
||
JUDGE_GPT,
|
||
'',
|
||
trivial,
|
||
// Reasoning models need ≥ ~20 tokens minimum to produce any
|
||
// visible output after internal chain-of-thought.
|
||
{ maxTokens: 200 },
|
||
{},
|
||
OPENAI_REASONING_SHAPE,
|
||
),
|
||
},
|
||
{
|
||
slug: JUDGE_GROK,
|
||
provider: 'xai',
|
||
role: 'judge',
|
||
probe: () =>
|
||
callOpenAICompat(
|
||
'https://api.x.ai/v1',
|
||
process.env.XAI_API_KEY ?? '',
|
||
'XAI_API_KEY',
|
||
JUDGE_GROK,
|
||
'',
|
||
trivial,
|
||
{ maxTokens: 20 },
|
||
),
|
||
},
|
||
{
|
||
slug: JUDGE_MINIMAX,
|
||
provider: 'minimax',
|
||
role: 'judge',
|
||
probe: () =>
|
||
callOpenAICompat(
|
||
'https://api.minimaxi.chat/v1',
|
||
process.env.MINIMAX_API_KEY ?? '',
|
||
'MINIMAX_API_KEY',
|
||
JUDGE_MINIMAX,
|
||
'',
|
||
trivial,
|
||
{ maxTokens: 20 },
|
||
),
|
||
},
|
||
];
|
||
}
|
||
|
||
async function runSlugProbe(): Promise<SlugProbeResult[]> {
|
||
const specs = buildSlugProbeSpecs();
|
||
const results: SlugProbeResult[] = [];
|
||
for (const spec of specs) {
|
||
const t0 = Date.now();
|
||
try {
|
||
await spec.probe();
|
||
results.push({ slug: spec.slug, provider: spec.provider, role: spec.role, ok: true, durationMs: Date.now() - t0 });
|
||
console.log(` [probe] ✓ ${spec.role.padEnd(18)} ${spec.slug}`);
|
||
} catch (err) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
results.push({ slug: spec.slug, provider: spec.provider, role: spec.role, ok: false, error: msg.slice(0, 200), durationMs: Date.now() - t0 });
|
||
console.log(` [probe] ✗ ${spec.role.padEnd(18)} ${spec.slug} [${msg.slice(0, 100)}]`);
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ── Conditions ──────────────────────────────────────────────────────
|
||
|
||
const PRIMARY_CONDITIONS: ConditionSpec[] = [
|
||
{ code: 'A', label: 'Opus 4.7 · current', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'B', label: 'Gemma 4 31B · current', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'C1', label: 'Gemma 4 31B · PA (compression)', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'primary' },
|
||
{ code: 'C2', label: 'Gemma 4 31B · PA (expansion)', model: GEMMA_31B_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'expansion', suite: 'primary' },
|
||
{ code: 'D', label: 'Opus 4.7 · PA (compression)', model: OPUS_4_7_MODEL, provider: 'anthropic', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'primary' },
|
||
{ code: 'E', label: 'Opus 4.6 · current', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: false, suite: 'primary' },
|
||
{ code: 'F', label: 'Opus 4.6 · PA (compression)', model: OPUS_4_6_MODEL, provider: 'anthropic', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'primary' },
|
||
];
|
||
|
||
const SECONDARY_26B_CONDITIONS: ConditionSpec[] = [
|
||
{ code: "B'", label: 'Gemma 4 26B MoE · current', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-26b' },
|
||
{ code: "C1'", label: 'Gemma 4 26B MoE · PA (compression)', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-26b' },
|
||
{ code: "C2'", label: 'Gemma 4 26B MoE · PA (expansion)', model: GEMMA_26B_MOE_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'expansion', suite: 'secondary-26b' },
|
||
];
|
||
|
||
const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
|
||
{ code: 'G', label: 'Qwen3-30B thinking · current', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: false, suite: 'secondary-qwen' },
|
||
{ code: 'H', label: 'Qwen3-30B thinking · PA (compression)', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-qwen' },
|
||
];
|
||
|
||
// ── Stub embedder (reuse v4 pattern) ────────────────────────────────
|
||
|
||
class StubEmbedder implements Embedder {
|
||
private dim = 384;
|
||
async embed(_text: string): Promise<Float32Array> { return new Float32Array(this.dim).fill(0); }
|
||
async embedBatch(texts: string[]): Promise<Float32Array[]> { return Promise.all(texts.map(t => this.embed(t))); }
|
||
getDimension(): number { return this.dim; }
|
||
}
|
||
|
||
// ── Scenario pipeline ───────────────────────────────────────────────
|
||
|
||
interface ScenarioSetup {
|
||
tempDir: string;
|
||
dbPath: string;
|
||
snapshotPath: string;
|
||
}
|
||
|
||
function setupCleanScenario(scenarioName: string): ScenarioSetup {
|
||
const tempDir = path.join(os.tmpdir(), `waggle-v5eval-${Date.now()}-${scenarioName}`);
|
||
if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true });
|
||
fs.mkdirSync(tempDir, { recursive: true });
|
||
const dbPath = path.join(tempDir, 'mind.db');
|
||
const snapshotPath = path.join(tempDir, 'snapshot.db');
|
||
const db = new MindDB(dbPath);
|
||
const raw = db.getDatabase();
|
||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||
if (count !== 0) {
|
||
db.close();
|
||
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
|
||
}
|
||
db.close();
|
||
return { tempDir, dbPath, snapshotPath };
|
||
}
|
||
|
||
async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario): Promise<void> {
|
||
for (const turn of scenario.primingTurns) {
|
||
const systemPrompt = orch.buildSystemPrompt();
|
||
const assistantMsg = await callAnthropic(PRIMING_MODEL, systemPrompt, turn.user, {
|
||
temperature: 0,
|
||
maxTokens: MAX_TOKENS_GEN,
|
||
});
|
||
await orch.autoSaveFromExchange(turn.user, assistantMsg);
|
||
}
|
||
}
|
||
|
||
function verifyMemory(db: MindDB, scenario: PromptAssemblerScenario): { count: number; matches: Record<string, boolean> } {
|
||
const raw = db.getDatabase();
|
||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||
const matches: Record<string, boolean> = {};
|
||
for (const sub of scenario.memoryVerificationSubstrings) {
|
||
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
|
||
matches[sub] = !!row;
|
||
}
|
||
return { count, matches };
|
||
}
|
||
|
||
async function runCondition(
|
||
snapshotPath: string,
|
||
condition: ConditionSpec,
|
||
scenario: PromptAssemblerScenario,
|
||
workDir: string,
|
||
seed: number,
|
||
): Promise<ConditionRun> {
|
||
const safeCode = condition.code.replace(/[^\w]/g, '_');
|
||
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
|
||
fs.copyFileSync(snapshotPath, workDbPath);
|
||
|
||
const db = new MindDB(workDbPath);
|
||
const orch = new Orchestrator({ db, embedder: new StubEmbedder(), model: condition.model });
|
||
|
||
let systemPrompt: string;
|
||
let debug: ConditionRun['debug'] = undefined;
|
||
const start = Date.now();
|
||
|
||
try {
|
||
if (condition.usesPromptAssembler) {
|
||
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
||
const taskShape = detectTaskShape(scenario.testTurn.query);
|
||
const assembled = await orch.buildAssembledPrompt(scenario.testTurn.query, null, {
|
||
taskShape,
|
||
scaffoldStyle: condition.scaffoldStyle ?? 'compression',
|
||
});
|
||
systemPrompt = assembled.system;
|
||
debug = {
|
||
tier: assembled.debug.tier,
|
||
taskShape: assembled.debug.taskShape,
|
||
taskShapeConfidence: assembled.debug.taskShapeConfidence,
|
||
scaffoldApplied: assembled.debug.scaffoldApplied,
|
||
scaffoldStyle: assembled.debug.scaffoldStyle,
|
||
sectionsIncluded: assembled.debug.sectionsIncluded,
|
||
framesUsed: assembled.debug.framesUsed,
|
||
totalChars: assembled.debug.totalChars,
|
||
};
|
||
} else {
|
||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||
systemPrompt = orch.buildSystemPrompt();
|
||
}
|
||
|
||
const output = await callGenerationModel(
|
||
condition.provider,
|
||
condition.model,
|
||
systemPrompt,
|
||
scenario.testTurn.query,
|
||
{ temperature: TEMPERATURE_GEN, seed },
|
||
);
|
||
db.close();
|
||
return { seed, output, durationMs: Date.now() - start, debug };
|
||
} catch (err) {
|
||
db.close();
|
||
return {
|
||
seed,
|
||
output: '',
|
||
durationMs: Date.now() - start,
|
||
debug,
|
||
error: err instanceof Error ? err.message : String(err),
|
||
};
|
||
}
|
||
}
|
||
|
||
// ── Ensemble judging ────────────────────────────────────────────────
|
||
|
||
async function judgeWithEnsemble(
|
||
judges: JudgeWiring[],
|
||
scenario: PromptAssemblerScenario,
|
||
goldOutput: string,
|
||
candidateOutput: string,
|
||
): Promise<EnsembleScore> {
|
||
// Fire all available judges in parallel; Promise.allSettled so a single
|
||
// rate-limit doesn't lose the whole ensemble row.
|
||
const active = judges.filter(j => j.available);
|
||
const judgeInput = {
|
||
input: scenario.testTurn.query,
|
||
expected: goldOutput,
|
||
actual: candidateOutput,
|
||
context: `task_shape=${scenario.shape}, language=${scenario.language}`,
|
||
};
|
||
|
||
const results = await Promise.allSettled(
|
||
active.map(async (judge) => {
|
||
const t0 = Date.now();
|
||
const llmJudge = new LLMJudge(judge.call);
|
||
try {
|
||
const score = await llmJudge.score(judgeInput);
|
||
return { judge: judge.name, score, durationMs: Date.now() - t0 } satisfies JudgeCallResult;
|
||
} catch (err) {
|
||
return {
|
||
judge: judge.name,
|
||
score: null,
|
||
error: err instanceof Error ? err.message.slice(0, 200) : String(err).slice(0, 200),
|
||
durationMs: Date.now() - t0,
|
||
} satisfies JudgeCallResult;
|
||
}
|
||
}),
|
||
);
|
||
|
||
const judgeResults: JudgeCallResult[] = results.map((r, i) =>
|
||
r.status === 'fulfilled'
|
||
? r.value
|
||
: { judge: active[i].name, score: null, error: String(r.reason).slice(0, 200), durationMs: 0 },
|
||
);
|
||
|
||
const validScores = judgeResults.filter(r => r.score !== null && r.score.parsed).map(r => r.score!.overall);
|
||
const mean = validScores.length ? validScores.reduce((a, b) => a + b, 0) / validScores.length : 0;
|
||
const min = validScores.length ? Math.min(...validScores) : 0;
|
||
const max = validScores.length ? Math.max(...validScores) : 0;
|
||
return {
|
||
judgeResults,
|
||
judgesAvailable: validScores.length,
|
||
mean,
|
||
min,
|
||
max,
|
||
disagreement: max - min,
|
||
};
|
||
}
|
||
|
||
function conditionMeanFromRuns(runs: ConditionRun[] | undefined): number {
|
||
if (!runs || runs.length === 0) return 0;
|
||
const validMeans = runs.map(r => r.ensemble?.mean ?? 0).filter(x => x > 0);
|
||
if (!validMeans.length) return 0;
|
||
return validMeans.reduce((a, b) => a + b, 0) / validMeans.length;
|
||
}
|
||
|
||
function perSeedDisagreement(runs: ConditionRun[]): number {
|
||
const means = runs.map(r => r.ensemble?.mean ?? 0).filter(x => x > 0);
|
||
if (means.length < 2) return 0;
|
||
return Math.max(...means) - Math.min(...means);
|
||
}
|
||
|
||
// ── Markdown renderer (§11.8) ───────────────────────────────────────
|
||
|
||
function mean(xs: number[]): number {
|
||
if (!xs.length) return 0;
|
||
return xs.reduce((a, b) => a + b, 0) / xs.length;
|
||
}
|
||
|
||
function renderMarkdown(result: EvalResult): string {
|
||
const lines: string[] = [];
|
||
lines.push('# PromptAssembler v5 eval results');
|
||
lines.push('');
|
||
lines.push(`**Run date:** ${result.runDate}`);
|
||
lines.push(`**Commit:** ${result.commit}`);
|
||
lines.push(`**Duration:** ${(result.durationMs / 1000 / 60).toFixed(1)} min`);
|
||
lines.push(`**Base seeds per condition:** ${result.seeds} (5 on variance retry)`);
|
||
lines.push(`**Judge ensemble:** ${result.judgesAvailable.join(', ')} (mean across ${result.judgesAvailable.length} judges = primary score)`);
|
||
lines.push(`**Tier bypass:** ${result.tierBypass ? 'WAGGLE_EVAL_MODE=1 (confirmed enterprise-equivalent throughout)' : 'NOT ACTIVE — results may be tier-confounded'}`);
|
||
lines.push('');
|
||
|
||
// Slug probe summary
|
||
lines.push('## Slug probe');
|
||
lines.push('');
|
||
lines.push('| Slug | Role | Result |');
|
||
lines.push('|------|------|--------|');
|
||
for (const p of result.slugProbe) {
|
||
lines.push(`| \`${p.slug}\` | ${p.role} | ${p.ok ? '✓' : `✗ ${(p.error ?? '').slice(0, 60)}`} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Aggregate disagreement
|
||
const allRuns: ConditionRun[] = [];
|
||
for (const s of result.scenarios) for (const runs of Object.values(s.conditions)) allRuns.push(...runs);
|
||
const allDisagreements = allRuns.map(r => r.ensemble?.disagreement ?? 0).filter(x => x > 0);
|
||
const meanDisagreement = mean(allDisagreements);
|
||
|
||
lines.push(`**Mean inter-judge disagreement (max − min across ${result.judgesAvailable.length} judges, per output):** ${meanDisagreement.toFixed(3)}`);
|
||
lines.push(meanDisagreement > 0.25
|
||
? '⚠️ FLAG: mean disagreement > 0.25 — findings reported but not claimed as robust.'
|
||
: '✓ Disagreement below 0.25 threshold — findings are methodologically sound.');
|
||
lines.push('');
|
||
|
||
const reasoningScenarios = result.scenarios.filter(s => s.shape !== 'draft' && s.shape !== 'mixed');
|
||
|
||
// ── Hypothesis outcomes ──
|
||
lines.push('## Hypothesis outcomes');
|
||
lines.push('');
|
||
|
||
// H1: PA helps frontier (F > E)
|
||
lines.push('### H1 — PA helps frontier (F > E)');
|
||
lines.push('');
|
||
lines.push('| Scenario | F | E | F − E |');
|
||
lines.push('|----------|-----|-----|-------|');
|
||
let h1Wins = 0;
|
||
const h1Regressions: number[] = [];
|
||
for (const s of result.scenarios) {
|
||
const f = conditionMeanFromRuns(s.conditions['F']);
|
||
const e = conditionMeanFromRuns(s.conditions['E']);
|
||
const delta = f - e;
|
||
if (delta > 0) h1Wins++;
|
||
else if (delta < -0.05) h1Regressions.push(delta);
|
||
lines.push(`| ${s.scenario} | ${f.toFixed(3)} | ${e.toFixed(3)} | ${delta >= 0 ? '+' : ''}${delta.toFixed(3)} |`);
|
||
}
|
||
const h1Pass = h1Wins >= 4 && h1Regressions.length === 0;
|
||
lines.push(`Result: **${h1Pass ? 'PASS' : 'FAIL'}** — F > E on ${h1Wins}/${result.scenarios.length} scenarios; ${h1Regressions.length} regressions > 5pp.`);
|
||
lines.push('');
|
||
|
||
// H2: PA helps reasoning-tuned small on analytical (H > G)
|
||
lines.push('### H2 — PA helps reasoning-tuned small (H > G on analytical)');
|
||
lines.push('');
|
||
const analyticalShapes = ['compare', 'decide', 'review'] as const;
|
||
const analytical = result.scenarios.filter(s => (analyticalShapes as readonly string[]).includes(s.shape));
|
||
lines.push('| Scenario (analytical) | H | G | H − G |');
|
||
lines.push('|----------|-----|-----|-------|');
|
||
let h2Wins = 0;
|
||
let h2MaxGain = 0;
|
||
for (const s of analytical) {
|
||
const h = conditionMeanFromRuns(s.conditions['H']);
|
||
const g = conditionMeanFromRuns(s.conditions['G']);
|
||
const delta = h - g;
|
||
if (delta > 0) h2Wins++;
|
||
if (delta > h2MaxGain) h2MaxGain = delta;
|
||
lines.push(`| ${s.scenario} | ${h.toFixed(3)} | ${g.toFixed(3)} | ${delta >= 0 ? '+' : ''}${delta.toFixed(3)} |`);
|
||
}
|
||
const h2Pass = h2Wins >= 3 && h2MaxGain >= 0.1;
|
||
lines.push(`Result: **${h2Pass ? 'PASS' : analytical.length ? 'FAIL' : 'N/A (no analytical scenarios)'}** — H > G on ${h2Wins}/${analytical.length} analytical scenarios; max gain ${(h2MaxGain * 100).toFixed(1)}pp.`);
|
||
lines.push('');
|
||
|
||
// H3: expansion closes ≥40% for Gemma (C2 vs B)
|
||
lines.push('### H3 — Expansion scaffold helps Gemma (C2 closes ≥40% of A − B on reasoning)');
|
||
lines.push('');
|
||
lines.push('| Scenario (reasoning) | A | B | C2 | (C2 − B) | (A − B) | closure |');
|
||
lines.push('|----------|-----|-----|-----|----------|---------|---------|');
|
||
let h3ClosureSum = 0;
|
||
let h3Count = 0;
|
||
for (const s of reasoningScenarios) {
|
||
const a = conditionMeanFromRuns(s.conditions['A']);
|
||
const b = conditionMeanFromRuns(s.conditions['B']);
|
||
const c2 = conditionMeanFromRuns(s.conditions['C2']);
|
||
const gap = a - b;
|
||
const closure = c2 - b;
|
||
const closurePct = gap > 0 ? (closure / gap) * 100 : 0;
|
||
h3ClosureSum += closurePct;
|
||
h3Count++;
|
||
lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${b.toFixed(3)} | ${c2.toFixed(3)} | ${closure >= 0 ? '+' : ''}${closure.toFixed(3)} | ${gap.toFixed(3)} | ${closurePct.toFixed(1)}% |`);
|
||
}
|
||
const h3MeanClosure = h3Count ? h3ClosureSum / h3Count : 0;
|
||
const h3Pass = h3MeanClosure >= 40;
|
||
lines.push(`Result: **${h3Pass ? 'PASS' : 'FAIL'}** — mean closure ${h3MeanClosure.toFixed(1)}% of the A−B gap (target ≥40%).`);
|
||
lines.push('');
|
||
|
||
// H4: compression regresses Gemma (C1 vs B)
|
||
lines.push('### H4 — Compression scaffold regresses Gemma (replication of v4)');
|
||
lines.push('');
|
||
lines.push('| Scenario (reasoning) | B | C1 | (C1 − B) |');
|
||
lines.push('|----------|-----|-----|----------|');
|
||
let h4Neg = 0;
|
||
for (const s of reasoningScenarios) {
|
||
const b = conditionMeanFromRuns(s.conditions['B']);
|
||
const c1 = conditionMeanFromRuns(s.conditions['C1']);
|
||
const delta = c1 - b;
|
||
if (delta <= 0) h4Neg++;
|
||
lines.push(`| ${s.scenario} | ${b.toFixed(3)} | ${c1.toFixed(3)} | ${delta >= 0 ? '+' : ''}${delta.toFixed(3)} |`);
|
||
}
|
||
const h4Pass = reasoningScenarios.length ? h4Neg >= reasoningScenarios.length - 1 : false;
|
||
lines.push(`Result: **${h4Pass ? 'REPLICATED' : 'NOT REPLICATED'}** — C1 ≤ B on ${h4Neg}/${reasoningScenarios.length} reasoning scenarios.`);
|
||
lines.push('');
|
||
|
||
// Priming verification
|
||
lines.push('## Priming verification');
|
||
lines.push('');
|
||
lines.push('| Scenario | Lang | Frames | Substrings | Failed | Partial |');
|
||
lines.push('|----------|------|--------|------------|--------|---------|');
|
||
for (const s of result.scenarios) {
|
||
const matchStr = Object.entries(s.primingMatches)
|
||
.map(([k, v]) => `${v ? '✓' : '✗'} ${k}`)
|
||
.join(', ');
|
||
lines.push(`| ${s.scenario} | ${s.language} | ${s.primingFrameCount} | ${matchStr} | ${s.primingFailed ? '✗' : '—'} | ${s.primingPartial ? '⚠' : '—'} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Full condition grid — primary
|
||
lines.push('## Full condition grid — Gemma 4 31B (primary)');
|
||
lines.push('');
|
||
const primaryCodes = PRIMARY_CONDITIONS.map(c => c.code);
|
||
lines.push(`| Scenario | shape | ${primaryCodes.join(' | ')} |`);
|
||
lines.push(`|----------|-------|${primaryCodes.map(() => '---').join('|')}|`);
|
||
for (const s of result.scenarios) {
|
||
const cells = primaryCodes.map(code => conditionMeanFromRuns(s.conditions[code]).toFixed(3));
|
||
lines.push(`| ${s.scenario} | ${s.shape} | ${cells.join(' | ')} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Secondary 26B
|
||
const hasSecondary26 = result.scenarios.some(s => s.conditions["B'"] || s.conditions["C1'"] || s.conditions["C2'"]);
|
||
if (hasSecondary26) {
|
||
lines.push('## Secondary — Gemma 4 26B MoE');
|
||
lines.push('');
|
||
lines.push(`| Scenario | B' | C1' | C2' | (C2'−B') |`);
|
||
lines.push(`|----------|------|------|------|----------|`);
|
||
for (const s of result.scenarios) {
|
||
const bp = conditionMeanFromRuns(s.conditions["B'"]);
|
||
const c1p = conditionMeanFromRuns(s.conditions["C1'"]);
|
||
const c2p = conditionMeanFromRuns(s.conditions["C2'"]);
|
||
lines.push(`| ${s.scenario} | ${bp.toFixed(3)} | ${c1p.toFixed(3)} | ${c2p.toFixed(3)} | ${(c2p - bp >= 0 ? '+' : '') + (c2p - bp).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
|
||
// Secondary Qwen
|
||
const hasSecondaryQ = result.scenarios.some(s => s.conditions['G'] || s.conditions['H']);
|
||
if (hasSecondaryQ) {
|
||
lines.push('## Secondary — Qwen3-30B thinking');
|
||
lines.push('');
|
||
lines.push('| Scenario | G | H | (H − G) |');
|
||
lines.push('|----------|-----|-----|---------|');
|
||
for (const s of result.scenarios) {
|
||
const g = conditionMeanFromRuns(s.conditions['G']);
|
||
const h = conditionMeanFromRuns(s.conditions['H']);
|
||
lines.push(`| ${s.scenario} | ${g.toFixed(3)} | ${h.toFixed(3)} | ${(h - g >= 0 ? '+' : '') + (h - g).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
|
||
// Opus generation delta
|
||
lines.push('## Opus generation delta (A 4.7 − E 4.6)');
|
||
lines.push('');
|
||
lines.push('| Scenario | A (4.7) | E (4.6) | Δ |');
|
||
lines.push('|----------|---------|---------|-----|');
|
||
for (const s of result.scenarios) {
|
||
const a = conditionMeanFromRuns(s.conditions['A']);
|
||
const e = conditionMeanFromRuns(s.conditions['E']);
|
||
lines.push(`| ${s.scenario} | ${a.toFixed(3)} | ${e.toFixed(3)} | ${(a - e >= 0 ? '+' : '') + (a - e).toFixed(3)} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Judge disagreement analysis
|
||
lines.push('## Judge ensemble disagreement analysis');
|
||
lines.push('');
|
||
lines.push('| Judge | Mean score (all outputs) | Scores parsed / total |');
|
||
lines.push('|-------|--------------------------|-----------------------|');
|
||
for (const judgeName of result.judgesAvailable) {
|
||
const all = allRuns.flatMap(r => r.ensemble?.judgeResults.filter(j => j.judge === judgeName) ?? []);
|
||
const parsed = all.filter(r => r.score && r.score.parsed);
|
||
const m = parsed.length ? mean(parsed.map(r => r.score!.overall)) : 0;
|
||
lines.push(`| ${judgeName} | ${m.toFixed(3)} | ${parsed.length} / ${all.length} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
// Retry-triggered conditions
|
||
const retryScenarios = result.scenarios.filter(s => s.retriedConditions.length > 0);
|
||
if (retryScenarios.length > 0) {
|
||
lines.push('## Variance-retry-triggered conditions');
|
||
lines.push('');
|
||
for (const s of retryScenarios) {
|
||
lines.push(`- **${s.scenario}**: ${s.retriedConditions.join(', ')} (5-seed run due to max-min > ${VARIANCE_RETRY_THRESHOLD * 100}pp across first 3 seeds)`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
|
||
lines.push('## Honest observations');
|
||
lines.push('');
|
||
lines.push('<!-- Fill in after reviewing the full grid. -->');
|
||
lines.push('');
|
||
lines.push('## What v5 does NOT tell us');
|
||
lines.push('');
|
||
lines.push('- Whether expansion scaffolds help Qwen thinking (not tested; G/H use compression only).');
|
||
lines.push('- Whether the pattern generalizes to Llama or other open families (not tested).');
|
||
lines.push('- Whether GEPA-evolved scaffolds would close remaining gaps (phase 3+ work).');
|
||
lines.push('- Statistical significance beyond 3–5 seeds (would need dozens).');
|
||
lines.push('');
|
||
lines.push('---');
|
||
lines.push('');
|
||
lines.push('Generated by `packages/agent/tests/eval/prompt-assembler-v5-eval.ts`.');
|
||
lines.push('Full structured results: `tmp_bench_results-v5.json` (gitignored).');
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ── Main ────────────────────────────────────────────────────────────
|
||
|
||
async function main(): Promise<void> {
|
||
const startTime = Date.now();
|
||
const runDate = new Date().toISOString();
|
||
|
||
console.log('[setup] WAGGLE_EVAL_MODE=1 set for tier bypass.');
|
||
console.log('[hydrate] Reading vault keys...');
|
||
const keys = hydrateVault();
|
||
const hydrated = Object.entries(keys).filter(([, v]) => v).map(([k]) => k);
|
||
const missing = Object.entries(keys).filter(([, v]) => !v).map(([k]) => k);
|
||
console.log(`[hydrate] loaded: ${hydrated.join(', ') || '(none)'}`);
|
||
if (missing.length) console.log(`[hydrate] missing: ${missing.join(', ')}`);
|
||
if (!keys.anthropic) throw new Error('Anthropic key is required (priming + A/D/E/F conditions).');
|
||
if (!keys.openrouter) throw new Error('OpenRouter key is required (B/C/secondary Gemma/Qwen).');
|
||
|
||
let commit = 'unknown';
|
||
try {
|
||
commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: REPO_ROOT }).toString().trim();
|
||
} catch { /* ignore */ }
|
||
|
||
// ── Slug probe ──
|
||
let slugProbe: SlugProbeResult[] = [];
|
||
if (!SKIP_SLUG_PROBE) {
|
||
console.log('\n[probe] Running pre-eval slug probe (11 slugs)...');
|
||
slugProbe = await runSlugProbe();
|
||
} else {
|
||
console.log('\n[probe] SKIPPED (WAGGLE_EVAL_SKIP_SLUG_PROBE=1).');
|
||
}
|
||
|
||
// Determine which judges are available.
|
||
const judges = buildJudgeWirings();
|
||
if (!SKIP_SLUG_PROBE) {
|
||
for (const j of judges) {
|
||
const probed = slugProbe.find(p => p.slug === j.name);
|
||
if (probed && !probed.ok) j.available = false;
|
||
}
|
||
}
|
||
const judgesAvailable = judges.filter(j => j.available).map(j => j.name);
|
||
console.log(`\n[judges] available: ${judgesAvailable.join(', ')} (${judgesAvailable.length}/4)`);
|
||
|
||
if (judgesAvailable.length < MIN_JUDGES_REACHABLE) {
|
||
throw new Error(
|
||
`Only ${judgesAvailable.length} judges reachable; minimum ${MIN_JUDGES_REACHABLE} required. Halting.`,
|
||
);
|
||
}
|
||
|
||
// ── Probe-only mode: exit after vault hydration + slug probe ──
|
||
if (PROBE_ONLY) {
|
||
const okCount = slugProbe.filter(p => p.ok).length;
|
||
console.log(`\n[probe-only] ${okCount}/${slugProbe.length} slugs reachable.`);
|
||
console.log(`[probe-only] judges available: ${judgesAvailable.length}/4 (need ≥${MIN_JUDGES_REACHABLE}).`);
|
||
console.log('[probe-only] Exiting before any scenario generation.');
|
||
return;
|
||
}
|
||
|
||
// ── Condition list ──
|
||
const conditions: ConditionSpec[] = SKIP_SECONDARY
|
||
? PRIMARY_CONDITIONS
|
||
: [...PRIMARY_CONDITIONS, ...SECONDARY_26B_CONDITIONS, ...SECONDARY_QWEN_CONDITIONS];
|
||
|
||
const scenariosToRun = MAX_SCENARIOS > 0 ? SCENARIOS_V5.slice(0, MAX_SCENARIOS) : SCENARIOS_V5;
|
||
console.log(`\n[plan] ${scenariosToRun.length} scenarios × ${conditions.length} conditions × ${BASE_SEEDS} seeds (+5 on retry)`);
|
||
console.log(`[plan] Output: ${RESULTS_JSON} + ${RESULTS_MD}`);
|
||
|
||
const result: EvalResult = {
|
||
runDate,
|
||
commit,
|
||
durationMs: 0,
|
||
slugProbe,
|
||
judgesAvailable,
|
||
tierBypass: process.env.WAGGLE_EVAL_MODE === '1',
|
||
seeds: BASE_SEEDS,
|
||
scenarios: [],
|
||
};
|
||
|
||
for (const [idx, scenario] of scenariosToRun.entries()) {
|
||
console.log(`\n[${idx + 1}/${scenariosToRun.length}] === ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||
|
||
const setup = setupCleanScenario(scenario.name);
|
||
const primingStart = Date.now();
|
||
|
||
const primingDb = new MindDB(setup.dbPath);
|
||
const primingOrch = new Orchestrator({
|
||
db: primingDb,
|
||
embedder: new StubEmbedder(),
|
||
model: PRIMING_MODEL,
|
||
});
|
||
primingOrch.getIdentity().create({
|
||
name: 'Marko',
|
||
role: 'CEO',
|
||
department: 'Egzakta Group',
|
||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||
capabilities: 'Strategic decisions, technical oversight',
|
||
system_prompt: '',
|
||
});
|
||
|
||
const effectiveTier = process.env.WAGGLE_EVAL_MODE === '1' ? 'bypassed (enterprise-equivalent)' : 'default (tier-enforced)';
|
||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL} | tier: ${effectiveTier}`);
|
||
|
||
try {
|
||
await runPriming(primingOrch, scenario);
|
||
} catch (err) {
|
||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
const verification = verifyMemory(primingDb, scenario);
|
||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||
const primingPartial = !primingFailed && Object.values(verification.matches).some(v => !v);
|
||
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
|
||
|
||
primingDb.close();
|
||
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
|
||
|
||
const scenarioResult: ScenarioResult = {
|
||
scenario: scenario.name,
|
||
shape: scenario.shape,
|
||
language: scenario.language,
|
||
primingFrameCount: verification.count,
|
||
primingMatches: verification.matches,
|
||
primingFailed,
|
||
primingPartial,
|
||
primingDurationMs: Date.now() - primingStart,
|
||
effectiveTier,
|
||
conditions: {},
|
||
retriedConditions: [],
|
||
};
|
||
|
||
// ── Per-condition generation + ensemble judging ──
|
||
for (const condition of conditions) {
|
||
scenarioResult.conditions[condition.code] = [];
|
||
|
||
// Initial 3 seeds.
|
||
for (let seed = 0; seed < BASE_SEEDS; seed++) {
|
||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
||
scenarioResult.conditions[condition.code].push(run);
|
||
if (run.error) console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||
else console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
|
||
}
|
||
}
|
||
|
||
// ── Judge the non-gold runs (A is gold; skip judging) ──
|
||
const aRuns = scenarioResult.conditions['A'] ?? [];
|
||
for (const condition of conditions) {
|
||
if (condition.code === 'A') {
|
||
// Perfect gold reference.
|
||
for (const run of scenarioResult.conditions['A']) {
|
||
run.ensemble = {
|
||
judgeResults: [],
|
||
judgesAvailable: 0,
|
||
mean: 1.0,
|
||
min: 1.0,
|
||
max: 1.0,
|
||
disagreement: 0,
|
||
};
|
||
}
|
||
continue;
|
||
}
|
||
const runs = scenarioResult.conditions[condition.code] ?? [];
|
||
for (const run of runs) {
|
||
if (run.error || !run.output) continue;
|
||
const gold = aRuns.find(r => r.seed === run.seed) ?? aRuns[0];
|
||
if (!gold || gold.error || !gold.output) continue;
|
||
process.stdout.write(` [judge] ${condition.code.padEnd(4)} seed=${run.seed} ensemble... `);
|
||
run.ensemble = await judgeWithEnsemble(judges, scenario, gold.output, run.output);
|
||
console.log(`μ=${run.ensemble.mean.toFixed(3)} Δ=${run.ensemble.disagreement.toFixed(3)} (${run.ensemble.judgesAvailable}/${judgesAvailable.length})`);
|
||
}
|
||
|
||
// ── Variance-triggered retry (§11.5) ──
|
||
if (condition.code !== 'A' && runs.length >= BASE_SEEDS) {
|
||
const disagree = perSeedDisagreement(runs);
|
||
if (disagree > VARIANCE_RETRY_THRESHOLD) {
|
||
console.log(` [retry] ${condition.code} seed disagreement ${disagree.toFixed(3)} > ${VARIANCE_RETRY_THRESHOLD}; running ${RETRY_EXTRA_SEEDS} more seeds...`);
|
||
scenarioResult.retriedConditions.push(condition.code);
|
||
for (let seed = BASE_SEEDS; seed < BASE_SEEDS + RETRY_EXTRA_SEEDS; seed++) {
|
||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
||
scenarioResult.conditions[condition.code].push(run);
|
||
if (run.error) {
|
||
console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||
continue;
|
||
}
|
||
console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
|
||
const gold = aRuns[0];
|
||
if (gold && gold.output) {
|
||
process.stdout.write(` [judge] ${condition.code.padEnd(4)} seed=${seed} ensemble... `);
|
||
run.ensemble = await judgeWithEnsemble(judges, scenario, gold.output, run.output);
|
||
console.log(`μ=${run.ensemble.mean.toFixed(3)} Δ=${run.ensemble.disagreement.toFixed(3)}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
result.scenarios.push(scenarioResult);
|
||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||
fs.rmSync(setup.tempDir, { recursive: true, force: true });
|
||
}
|
||
|
||
result.durationMs = Date.now() - startTime;
|
||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||
fs.writeFileSync(RESULTS_MD, renderMarkdown(result));
|
||
|
||
console.log(`\n[done] ${result.scenarios.length} scenarios in ${(result.durationMs / 1000 / 60).toFixed(1)} min.`);
|
||
console.log(`[done] JSON: ${RESULTS_JSON}`);
|
||
console.log(`[done] MD: ${RESULTS_MD}`);
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error('[fatal]', err);
|
||
process.exit(1);
|
||
});
|