782 lines
33 KiB
JavaScript
782 lines
33 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Evolution Hypothesis Test — the mission's north-star experiment.
|
||
*
|
||
* Tests whether Waggle's self-evolution loop can close the quality gap
|
||
* between a small model (Gemma 4) and a large model (Opus 4.6).
|
||
*
|
||
* THREE ARMS (all evaluated on the same eval set):
|
||
* A) Opus 4.6 + baseline prompt ← strong-model upper bound
|
||
* B) Gemma 4 + baseline prompt ← weak-model lower bound
|
||
* C) Gemma 4 + Waggle-evolved prompt ← the claim
|
||
*
|
||
* BLIND JUDGING:
|
||
* Each arm's output is scored independently by multiple judges via
|
||
* OpenRouter (Opus, GPT, Gemini, Grok). Per-judge ratios are aggregated
|
||
* so a lenient/strict judge doesn't bias the comparison.
|
||
*
|
||
* VERDICT:
|
||
* Mean(arm_C / arm_A) across judges ≥ 0.95 → hypothesis CONFIRMED.
|
||
*
|
||
* USAGE:
|
||
* # 1. Create .env.hypothesis.local with:
|
||
* # OPENROUTER_API_KEY=sk-or-v1-...
|
||
* # ANTHROPIC_API_KEY=sk-ant-...
|
||
* # 2. Dry run (no API calls, just previews the plan + cost):
|
||
* node --env-file=.env.hypothesis.local scripts/evolution-hypothesis.mjs --dry-run
|
||
* # 3. Live run:
|
||
* node --env-file=.env.hypothesis.local scripts/evolution-hypothesis.mjs --confirm
|
||
*
|
||
* FLAGS:
|
||
* --dry-run Preview plan + cost estimate, no API calls.
|
||
* --confirm Required to make actual API calls.
|
||
* --eval-size=N Number of eval questions (default: 10, max: 15).
|
||
* --gepa-gens=N GEPA generations (default: 2).
|
||
* --gepa-pop=N GEPA population size (default: 3).
|
||
* --skip-evolution Use baseline prompt for Arm C too (sanity check).
|
||
* --judges=a,b,c Comma-separated OpenRouter judge model IDs.
|
||
* --arm-a=ID OpenRouter model ID for Arm A (default: opus).
|
||
* --arm-b=ID OpenRouter model ID for Arms B/C (default: gemma).
|
||
*
|
||
* SECURITY:
|
||
* - API keys are read ONLY from process.env (supply via --env-file).
|
||
* - Script NEVER writes keys to disk or logs.
|
||
* - Intermediate state is persisted to docs/.evolution-hypothesis-{ts}/
|
||
* — safe to inspect, commit, or delete.
|
||
*/
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
// Import directly from built dist/ to avoid the package.json `.ts` entry
|
||
// (package.json points at src/ for the in-repo test runner, not Node).
|
||
import { LLMJudge } from '../packages/agent/dist/judge.js';
|
||
import { IterativeGEPA } from '../packages/agent/dist/iterative-optimizer.js';
|
||
import { EvolveSchema } from '../packages/agent/dist/evolve-schema.js';
|
||
import {
|
||
buildReflectiveMutationPrompt,
|
||
buildSchemaFillPrompt,
|
||
makeRunningJudge,
|
||
} from '../packages/agent/dist/evolution-llm-wiring.js';
|
||
import { filterJudgeFeedback } from '../packages/agent/dist/compose-evolution.js';
|
||
|
||
// ── Args ─────────────────────────────────────────────────────────
|
||
|
||
const args = parseArgs(process.argv.slice(2));
|
||
const DRY_RUN = args.has('dry-run');
|
||
const CONFIRMED = args.has('confirm');
|
||
const SKIP_EVOLUTION = args.has('skip-evolution');
|
||
const EVAL_SIZE = Math.min(15, Math.max(1, Number(args.get('eval-size') ?? 10)));
|
||
const GEPA_GENS = Math.max(1, Number(args.get('gepa-gens') ?? 2));
|
||
const GEPA_POP = Math.max(1, Number(args.get('gepa-pop') ?? 3));
|
||
|
||
// Default OpenRouter model IDs. These are replaced at runtime by the
|
||
// latest matching model pulled from /api/v1/models if --auto-model is set.
|
||
// Keeping explicit defaults makes the run reproducible.
|
||
const ARM_A_MODEL = args.get('arm-a') ?? 'anthropic/claude-opus-4.6';
|
||
const ARM_B_MODEL = args.get('arm-b') ?? 'google/gemma-4-31b-it';
|
||
const JUDGE_MODELS = (args.get('judges') ?? [
|
||
'anthropic/claude-opus-4.6',
|
||
'openai/gpt-5.4',
|
||
'google/gemini-2.5-pro',
|
||
'x-ai/grok-4.20',
|
||
].join(',')).split(',').map(s => s.trim()).filter(Boolean);
|
||
|
||
// ── Safety gate ─────────────────────────────────────────────────
|
||
|
||
if (!DRY_RUN && !CONFIRMED) {
|
||
console.error(`
|
||
✋ This script makes real API calls and costs real money.
|
||
Pass --dry-run to preview, or --confirm to actually run.
|
||
`);
|
||
process.exit(1);
|
||
}
|
||
|
||
const OPENROUTER_KEY = process.env.OPENROUTER_API_KEY;
|
||
const ANTHROPIC_KEY = process.env.ANTHROPIC_API_KEY;
|
||
if (!DRY_RUN) {
|
||
if (!OPENROUTER_KEY) {
|
||
console.error('OPENROUTER_API_KEY not set. Use --env-file=.env.hypothesis.local.');
|
||
process.exit(1);
|
||
}
|
||
if (!ANTHROPIC_KEY) {
|
||
console.error('ANTHROPIC_API_KEY not set. Use --env-file=.env.hypothesis.local.');
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
// ── Eval dataset — 15 curated coder questions ───────────────────
|
||
|
||
const EVAL_EXAMPLES_ALL = [
|
||
{
|
||
id: 'js-map-vs-foreach',
|
||
input: 'What is the key difference between Array.prototype.map and Array.prototype.forEach in JavaScript?',
|
||
expected: 'map returns a new array of transformed values; forEach returns undefined and is used for side effects. Use map when you need the transformed results, forEach when you only care about iteration.',
|
||
},
|
||
{
|
||
id: 'py-list-tuple',
|
||
input: 'In Python, when should I use a tuple instead of a list?',
|
||
expected: 'Use a tuple when the collection is fixed/immutable — like coordinates, record fields, or dictionary keys. Use a list when the collection will be mutated (append, remove, reorder). Tuples are also slightly faster and hashable.',
|
||
},
|
||
{
|
||
id: 'sql-inner-vs-left',
|
||
input: 'Explain the difference between INNER JOIN and LEFT JOIN in SQL.',
|
||
expected: 'INNER JOIN returns only rows with matches in both tables. LEFT JOIN returns all rows from the left table, with NULL for columns from the right table when no match exists. Use LEFT JOIN when you need to preserve all left-side rows.',
|
||
},
|
||
{
|
||
id: 'ts-type-vs-interface',
|
||
input: 'When should I use a TypeScript type alias vs an interface?',
|
||
expected: 'Use interface for object shapes that may be extended or implemented; they support declaration merging and are more idiomatic for class contracts. Use type for unions, intersections, tuples, mapped types, or other non-object shapes.',
|
||
},
|
||
{
|
||
id: 'regex-bug',
|
||
input: "What is wrong with this regex used to validate lowercase letters only: /^[a-z]+$/ in JavaScript, when applied against non-ASCII lowercase letters like 'é'?",
|
||
expected: 'The character class [a-z] only matches ASCII a through z. Non-ASCII lowercase letters like é, ü, ñ, or Cyrillic letters will fail to match. Use Unicode property escapes: /^\\p{Ll}+$/u, or explicitly include the expected characters.',
|
||
},
|
||
{
|
||
id: 'async-race',
|
||
input: 'In JavaScript, what happens if I await two promises sequentially vs with Promise.all?',
|
||
expected: 'Sequential awaits run the promises one after another: total time = sum of durations. Promise.all runs them concurrently: total time = max of durations. Use Promise.all when the awaits are independent, sequential when one depends on the previous result.',
|
||
},
|
||
{
|
||
id: 'go-slice-append',
|
||
input: 'Why might appending to a Go slice sometimes unexpectedly modify other slices sharing the same underlying array?',
|
||
expected: 'Slices share an underlying array. If a slice has capacity beyond its length, append writes into that shared memory without reallocating. Other slices viewing the same region see the change. When capacity is exceeded, a new array is allocated and the sharing breaks. To always get a fresh array, use make+copy or append to a slice of length and capacity equal.',
|
||
},
|
||
{
|
||
id: 'rust-lifetime',
|
||
input: 'In Rust, why does the compiler complain about a function that returns a reference with no explicit lifetime?',
|
||
expected: 'When a function returns a reference, the compiler needs to know which input that reference is borrowed from (its lifetime). If there is ambiguity — multiple reference inputs, or a reference unrelated to inputs — you must annotate lifetimes explicitly. The compiler cannot infer which input the returned reference is tied to.',
|
||
},
|
||
{
|
||
id: 'python-gil',
|
||
input: 'Does the Python GIL prevent all concurrency?',
|
||
expected: 'No. The GIL only prevents multiple threads from executing Python bytecode simultaneously within one process. I/O-bound work releases the GIL (so threading still helps for I/O-bound code). CPU-bound work does not parallelize across threads but can use multiprocessing to bypass the GIL entirely.',
|
||
},
|
||
{
|
||
id: 'react-key',
|
||
input: 'Why does React require a unique key prop when rendering a list?',
|
||
expected: 'React uses keys to identify elements across renders so it can match old and new children efficiently, preserving component state and avoiding unnecessary unmount/remount cycles. Without stable keys, React falls back to index-based matching which reorders or remounts components incorrectly when the list changes.',
|
||
},
|
||
{
|
||
id: 'tcp-vs-udp',
|
||
input: 'When would you choose UDP over TCP for network communication?',
|
||
expected: 'Use UDP when low latency and minimal overhead matter more than reliability — real-time video/voice, online games, DNS queries, high-frequency telemetry. UDP skips connection setup and retransmission, so occasional packet loss is acceptable. Use TCP for anything needing guaranteed delivery and ordering.',
|
||
},
|
||
{
|
||
id: 'git-rebase-vs-merge',
|
||
input: 'What is the practical difference between git merge and git rebase when integrating a feature branch?',
|
||
expected: 'merge creates a new commit that preserves the branch history — non-destructive, easier for others to follow, but creates merge commits. rebase rewrites the feature branch commits on top of the target branch — linear history, cleaner log, but rewrites hashes so it is dangerous on shared branches. Merge for integration, rebase for local cleanup.',
|
||
},
|
||
{
|
||
id: 'docker-copy-vs-add',
|
||
input: 'What is the difference between COPY and ADD in a Dockerfile?',
|
||
expected: 'COPY only copies local files into the image. ADD also copies files but additionally can auto-extract local tar archives and download files from URLs. Best practice: prefer COPY because it is explicit and predictable. Use ADD only when you need its extraction or URL-fetch behavior.',
|
||
},
|
||
{
|
||
id: 'hash-vs-encrypt',
|
||
input: 'Why should you hash passwords instead of encrypting them?',
|
||
expected: 'Hashing is one-way — you verify a password by hashing the input and comparing, never by reversing. Encryption is two-way and requires a key, which itself becomes a critical secret that, if leaked, exposes all passwords. Use a slow, salted hash like bcrypt, scrypt, or argon2 so even if the hashed-password table leaks, attackers face expensive per-guess work.',
|
||
},
|
||
{
|
||
id: 'big-o-quicksort',
|
||
input: 'What is the average and worst-case time complexity of quicksort, and why do they differ?',
|
||
expected: 'Average: O(n log n) — pivots split the array into roughly equal halves, so depth is log n and each level does n work. Worst case: O(n²) — pivots always produce extremely unbalanced splits (e.g. already-sorted input with a naive first-element pivot). Good implementations use random or median-of-three pivots to make the worst case practically unreachable.',
|
||
},
|
||
];
|
||
|
||
const EVAL_EXAMPLES = EVAL_EXAMPLES_ALL.slice(0, EVAL_SIZE);
|
||
|
||
// ── Baseline prompt being evolved ───────────────────────────────
|
||
|
||
const BASELINE_PROMPT = `You are a coding assistant. Answer the user's coding question clearly.`;
|
||
|
||
// ── OpenRouter client ───────────────────────────────────────────
|
||
|
||
const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions';
|
||
let _openrouterCalls = 0;
|
||
let _openrouterTokens = { in: 0, out: 0 };
|
||
|
||
async function openrouter(model, prompt, { temperature = 0.2, maxTokens = 1024 } = {}) {
|
||
_openrouterCalls++;
|
||
const res = await fetch(OPENROUTER_URL, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Authorization': `Bearer ${OPENROUTER_KEY}`,
|
||
'Content-Type': 'application/json',
|
||
'HTTP-Referer': 'https://waggle-os.ai',
|
||
'X-Title': 'Waggle Evolution Hypothesis',
|
||
},
|
||
body: JSON.stringify({
|
||
model,
|
||
messages: [{ role: 'user', content: prompt }],
|
||
temperature,
|
||
max_tokens: maxTokens,
|
||
}),
|
||
});
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`OpenRouter ${model} failed: HTTP ${res.status} ${text.slice(0, 200)}`);
|
||
}
|
||
const body = await res.json();
|
||
const usage = body.usage ?? {};
|
||
_openrouterTokens.in += usage.prompt_tokens ?? 0;
|
||
_openrouterTokens.out += usage.completion_tokens ?? 0;
|
||
const content = body.choices?.[0]?.message?.content ?? '';
|
||
return content;
|
||
}
|
||
|
||
async function openrouterChat(model, systemPrompt, userInput, opts = {}) {
|
||
_openrouterCalls++;
|
||
const res = await fetch(OPENROUTER_URL, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Authorization': `Bearer ${OPENROUTER_KEY}`,
|
||
'Content-Type': 'application/json',
|
||
'HTTP-Referer': 'https://waggle-os.ai',
|
||
'X-Title': 'Waggle Evolution Hypothesis',
|
||
},
|
||
body: JSON.stringify({
|
||
model,
|
||
messages: [
|
||
{ role: 'system', content: systemPrompt },
|
||
{ role: 'user', content: userInput },
|
||
],
|
||
temperature: opts.temperature ?? 0.2,
|
||
max_tokens: opts.maxTokens ?? 1024,
|
||
}),
|
||
});
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`OpenRouter ${model} failed: HTTP ${res.status} ${text.slice(0, 200)}`);
|
||
}
|
||
const body = await res.json();
|
||
const usage = body.usage ?? {};
|
||
_openrouterTokens.in += usage.prompt_tokens ?? 0;
|
||
_openrouterTokens.out += usage.completion_tokens ?? 0;
|
||
return body.choices?.[0]?.message?.content ?? '';
|
||
}
|
||
|
||
// ── Anthropic direct (for Haiku judge/mutate in evolution run) ──
|
||
|
||
const ANTHROPIC_URL = 'https://api.anthropic.com/v1/messages';
|
||
const HAIKU_MODEL = 'claude-haiku-4-5-20251001';
|
||
let _anthropicCalls = 0;
|
||
let _anthropicTokens = { in: 0, out: 0 };
|
||
|
||
async function anthropic(prompt, { temperature = 0.3, maxTokens = 512 } = {}) {
|
||
_anthropicCalls++;
|
||
const res = await fetch(ANTHROPIC_URL, {
|
||
method: 'POST',
|
||
headers: {
|
||
'x-api-key': ANTHROPIC_KEY,
|
||
'anthropic-version': '2023-06-01',
|
||
'Content-Type': 'application/json',
|
||
},
|
||
body: JSON.stringify({
|
||
model: HAIKU_MODEL,
|
||
max_tokens: maxTokens,
|
||
temperature,
|
||
messages: [{ role: 'user', content: prompt }],
|
||
}),
|
||
});
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`Anthropic ${HAIKU_MODEL} failed: HTTP ${res.status} ${text.slice(0, 200)}`);
|
||
}
|
||
const body = await res.json();
|
||
const usage = body.usage ?? {};
|
||
_anthropicTokens.in += usage.input_tokens ?? 0;
|
||
_anthropicTokens.out += usage.output_tokens ?? 0;
|
||
const content = body.content?.[0]?.text ?? '';
|
||
return content;
|
||
}
|
||
|
||
// ── Workspace setup ─────────────────────────────────────────────
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const repoRoot = path.resolve(__dirname, '..');
|
||
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||
const workDir = path.join(repoRoot, 'docs', `.evolution-hypothesis-${ts}`);
|
||
if (!DRY_RUN) fs.mkdirSync(workDir, { recursive: true });
|
||
|
||
function saveCheckpoint(name, data) {
|
||
if (DRY_RUN) return;
|
||
fs.writeFileSync(
|
||
path.join(workDir, `${name}.json`),
|
||
JSON.stringify(data, null, 2),
|
||
'utf-8',
|
||
);
|
||
}
|
||
|
||
// ── Plan summary ────────────────────────────────────────────────
|
||
|
||
function printPlan() {
|
||
const evolveCalls = SKIP_EVOLUTION
|
||
? 0
|
||
: (GEPA_POP * GEPA_GENS * Math.min(8, EVAL_SIZE)) + 20; // rough
|
||
const armExecCalls = 3 * EVAL_SIZE;
|
||
const judgeCalls = JUDGE_MODELS.length * 3 * EVAL_SIZE;
|
||
|
||
console.log('');
|
||
console.log('╭─ Evolution Hypothesis Test — Plan ─────────────────────────╮');
|
||
console.log(`│ Timestamp: ${ts}`);
|
||
console.log(`│ Eval size: ${EVAL_SIZE} questions (coder domain)`);
|
||
console.log(`│ Skip evolution: ${SKIP_EVOLUTION}`);
|
||
console.log(`│ GEPA: pop=${GEPA_POP}, gens=${GEPA_GENS}`);
|
||
console.log(`│`);
|
||
console.log(`│ Arm A (upper bound): ${ARM_A_MODEL}`);
|
||
console.log(`│ Arm B (lower bound): ${ARM_B_MODEL}`);
|
||
console.log(`│ Arm C (Gemma evolved): ${ARM_B_MODEL} + evolved prompt`);
|
||
console.log(`│`);
|
||
console.log(`│ Judges (blind, ${JUDGE_MODELS.length}):`);
|
||
for (const j of JUDGE_MODELS) console.log(`│ · ${j}`);
|
||
console.log(`│`);
|
||
console.log(`│ Estimated API calls:`);
|
||
console.log(`│ Evolution (Haiku): ~${evolveCalls}`);
|
||
console.log(`│ Arm execution (OpenRouter): ${armExecCalls}`);
|
||
console.log(`│ Judges (OpenRouter): ${judgeCalls}`);
|
||
console.log(`│ TOTAL OpenRouter: ~${armExecCalls + judgeCalls}`);
|
||
console.log(`│`);
|
||
console.log(`│ Rough cost estimate: $3-8 USD`);
|
||
console.log(`│ Work dir: ${workDir}`);
|
||
console.log('╰────────────────────────────────────────────────────────────╯');
|
||
console.log('');
|
||
}
|
||
|
||
printPlan();
|
||
|
||
if (DRY_RUN) {
|
||
console.log('✅ Dry run complete. Re-run with --confirm to execute.');
|
||
process.exit(0);
|
||
}
|
||
|
||
// ── Build LLM wrappers ──────────────────────────────────────────
|
||
|
||
const armALLM = {
|
||
complete(prompt) { return openrouter(ARM_A_MODEL, prompt); },
|
||
};
|
||
const armBLLM = {
|
||
complete(prompt) { return openrouter(ARM_B_MODEL, prompt); },
|
||
};
|
||
const haikuLLM = {
|
||
complete(prompt) { return anthropic(prompt); },
|
||
};
|
||
|
||
// LLMJudge's JudgeLLMCall contract is `(prompt: string) => Promise<string>`.
|
||
const haikuJudgeCall = (prompt) => anthropic(prompt, { maxTokens: 400 });
|
||
|
||
// ── Phase 1: Evolve the prompt (Arm C preparation) ─────────────
|
||
|
||
async function evolvePrompt() {
|
||
if (SKIP_EVOLUTION) {
|
||
console.log('⏭ Skipping evolution — Arm C will use baseline prompt.');
|
||
return BASELINE_PROMPT;
|
||
}
|
||
|
||
console.log('🧬 Phase 1 — Evolving the prompt...');
|
||
const startedAt = Date.now();
|
||
|
||
// The judge that GEPA sees. Judge scores candidate prompts by running
|
||
// them via Gemma and comparing Gemma's output to the expected answer.
|
||
const baseJudge = new LLMJudge(haikuJudgeCall);
|
||
const runningJudge = makeRunningJudge(baseJudge, armBLLM);
|
||
|
||
// Reflective mutation with Haiku.
|
||
const mutate = async ({ parent, strategy, weaknessFeedback, targetKind, generation }) => {
|
||
const prompt = buildReflectiveMutationPrompt({
|
||
parent: parent.prompt,
|
||
strategy,
|
||
weaknessFeedback,
|
||
targetKind,
|
||
generation,
|
||
});
|
||
try {
|
||
const raw = await anthropic(prompt, { maxTokens: 600, temperature: 0.6 });
|
||
const cleaned = raw.replace(/```\w*\s*/g, '').replace(/```\s*/g, '').trim();
|
||
if (cleaned.length === 0) return parent.prompt;
|
||
return cleaned;
|
||
} catch {
|
||
return parent.prompt;
|
||
}
|
||
};
|
||
|
||
// Feed the GEPA loop a small sample of eval examples for scoring.
|
||
// Use a FRESH set separate from the main eval set to reduce leakage,
|
||
// but we don't have enough examples to fully separate — warn and proceed.
|
||
const gepaExamples = EVAL_EXAMPLES.map(e => ({
|
||
input: e.input,
|
||
expected_output: e.expected,
|
||
source: 'curated',
|
||
metadata: {},
|
||
}));
|
||
|
||
const gepa = new IterativeGEPA();
|
||
const result = await gepa.run({
|
||
baseline: BASELINE_PROMPT,
|
||
examples: gepaExamples,
|
||
judge: runningJudge,
|
||
mutate,
|
||
targetKind: 'persona-system-prompt',
|
||
populationSize: GEPA_POP,
|
||
generations: GEPA_GENS,
|
||
miniEvalSize: Math.min(5, EVAL_SIZE),
|
||
microScreenSize: Math.min(3, EVAL_SIZE),
|
||
anchorEvalSize: Math.min(EVAL_SIZE, 10),
|
||
seed: 42,
|
||
onProgress: (e) => {
|
||
console.log(` [${e.phase}] gen ${e.generation} best=${e.best.toFixed(3)} ${e.message ?? ''}`);
|
||
},
|
||
});
|
||
|
||
const evolved = result.winner.prompt;
|
||
const delta = result.delta;
|
||
const elapsed = ((Date.now() - startedAt) / 1000).toFixed(1);
|
||
|
||
console.log(` Winner: ${result.winner.id} (gen ${result.winner.generation})`);
|
||
console.log(` Score: ${(result.winner.score?.overall ?? 0).toFixed(3)}`);
|
||
console.log(` Delta vs baseline: ${(delta * 100).toFixed(1)}pp`);
|
||
console.log(` Elapsed: ${elapsed}s`);
|
||
|
||
saveCheckpoint('01-evolved-prompt', {
|
||
baseline: BASELINE_PROMPT,
|
||
evolved,
|
||
delta,
|
||
winnerId: result.winner.id,
|
||
winnerScore: result.winner.score,
|
||
historyCount: result.history.length,
|
||
});
|
||
|
||
return evolved;
|
||
}
|
||
|
||
// ── Phase 2: Run the three arms on the eval set ────────────────
|
||
|
||
async function runArm(name, model, systemPrompt) {
|
||
console.log(` Running Arm ${name} (${model})...`);
|
||
const outputs = [];
|
||
for (let i = 0; i < EVAL_EXAMPLES.length; i++) {
|
||
const ex = EVAL_EXAMPLES[i];
|
||
let output = '';
|
||
try {
|
||
output = await openrouterChat(model, systemPrompt, ex.input, { maxTokens: 600 });
|
||
} catch (err) {
|
||
console.error(` ✖ ${ex.id}: ${err.message}`);
|
||
output = '[ERROR: execution failed]';
|
||
}
|
||
outputs.push({ id: ex.id, output });
|
||
process.stdout.write('.');
|
||
}
|
||
console.log('');
|
||
return outputs;
|
||
}
|
||
|
||
async function runAllArms(evolvedPrompt) {
|
||
console.log('🏁 Phase 2 — Running three arms...');
|
||
const startedAt = Date.now();
|
||
|
||
const armAOutputs = await runArm('A', ARM_A_MODEL, BASELINE_PROMPT);
|
||
saveCheckpoint('02a-arm-a-outputs', armAOutputs);
|
||
const armBOutputs = await runArm('B', ARM_B_MODEL, BASELINE_PROMPT);
|
||
saveCheckpoint('02b-arm-b-outputs', armBOutputs);
|
||
const armCOutputs = await runArm('C', ARM_B_MODEL, evolvedPrompt);
|
||
saveCheckpoint('02c-arm-c-outputs', armCOutputs);
|
||
|
||
console.log(` Elapsed: ${((Date.now() - startedAt) / 1000).toFixed(1)}s`);
|
||
return { armAOutputs, armBOutputs, armCOutputs };
|
||
}
|
||
|
||
// ── Phase 3: Blind judging ─────────────────────────────────────
|
||
|
||
const RUBRIC = `You are a strict, fair evaluator scoring an AI assistant's response to a coding question.
|
||
|
||
You will be given:
|
||
- The user's INSTRUCTION (a coding question)
|
||
- The EXPECTED output (the reference answer — the ground truth)
|
||
- The ACTUAL output from the AI assistant
|
||
|
||
Score the ACTUAL response on three dimensions, each on a 0-10 integer scale:
|
||
1. CORRECTNESS (0-10): Does it match the expected answer semantically? Correct facts / steps?
|
||
2. PROCEDURE_FOLLOWING (0-10): Did it answer the question asked, in an appropriate format?
|
||
3. CONCISENESS (0-10): Tight and on-point, or padded / verbose?
|
||
|
||
Then write a short, ACTIONABLE FEEDBACK (max 2 sentences) explaining the biggest issue (if any).
|
||
|
||
Return ONLY a JSON object on a single line, no markdown:
|
||
{"correctness": <0-10>, "procedure": <0-10>, "conciseness": <0-10>, "feedback": "<string>"}`;
|
||
|
||
async function judgeOnce(judgeModel, example, actual) {
|
||
const prompt = `${RUBRIC}
|
||
|
||
INSTRUCTION:
|
||
${example.input}
|
||
|
||
EXPECTED:
|
||
${example.expected}
|
||
|
||
ACTUAL:
|
||
${actual}
|
||
|
||
Return the JSON now.`;
|
||
const raw = await openrouter(judgeModel, prompt, { temperature: 0.0, maxTokens: 300 });
|
||
return parseJudgeJSON(raw);
|
||
}
|
||
|
||
function parseJudgeJSON(raw) {
|
||
if (!raw) return null;
|
||
const cleaned = raw.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
|
||
// Find the first balanced { ... } block
|
||
let depth = 0, start = -1, inStr = false, esc = false;
|
||
for (let i = 0; i < cleaned.length; i++) {
|
||
const c = cleaned[i];
|
||
if (inStr) {
|
||
if (esc) esc = false;
|
||
else if (c === '\\') esc = true;
|
||
else if (c === '"') inStr = false;
|
||
continue;
|
||
}
|
||
if (c === '"') inStr = true;
|
||
else if (c === '{') { if (depth === 0) start = i; depth++; }
|
||
else if (c === '}') {
|
||
depth--;
|
||
if (depth === 0 && start >= 0) {
|
||
const candidate = cleaned.slice(start, i + 1);
|
||
try {
|
||
const obj = JSON.parse(candidate);
|
||
if (typeof obj.correctness === 'number' && typeof obj.procedure === 'number' && typeof obj.conciseness === 'number') {
|
||
return {
|
||
correctness: obj.correctness,
|
||
procedure: obj.procedure,
|
||
conciseness: obj.conciseness,
|
||
feedback: String(obj.feedback ?? ''),
|
||
};
|
||
}
|
||
} catch { /* keep searching */ }
|
||
start = -1;
|
||
}
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function overallFromParsed(p) {
|
||
if (!p) return 0;
|
||
// Same weighting as LLMJudge's defaults
|
||
return (p.correctness * 0.5 + p.procedure * 0.3 + p.conciseness * 0.2) / 10;
|
||
}
|
||
|
||
async function judgeAllOutputs(outputs) {
|
||
console.log('⚖️ Phase 3 — Blind judging...');
|
||
const startedAt = Date.now();
|
||
/** shape: [judgeModel][armName][exampleId] = {parsed, overall} */
|
||
const scores = {};
|
||
for (const judge of JUDGE_MODELS) scores[judge] = { A: {}, B: {}, C: {} };
|
||
|
||
for (const judge of JUDGE_MODELS) {
|
||
console.log(` Judge: ${judge}`);
|
||
for (const { arm, armOutputs } of [
|
||
{ arm: 'A', armOutputs: outputs.armAOutputs },
|
||
{ arm: 'B', armOutputs: outputs.armBOutputs },
|
||
{ arm: 'C', armOutputs: outputs.armCOutputs },
|
||
]) {
|
||
for (let i = 0; i < EVAL_EXAMPLES.length; i++) {
|
||
const ex = EVAL_EXAMPLES[i];
|
||
const actual = armOutputs[i]?.output ?? '';
|
||
try {
|
||
const parsed = await judgeOnce(judge, ex, actual);
|
||
scores[judge][arm][ex.id] = {
|
||
parsed,
|
||
overall: overallFromParsed(parsed),
|
||
};
|
||
} catch (err) {
|
||
console.error(` ✖ ${judge} / ${arm} / ${ex.id}: ${err.message}`);
|
||
scores[judge][arm][ex.id] = { parsed: null, overall: 0 };
|
||
}
|
||
process.stdout.write('.');
|
||
}
|
||
console.log(` [${arm} done]`);
|
||
}
|
||
}
|
||
|
||
console.log(` Elapsed: ${((Date.now() - startedAt) / 1000).toFixed(1)}s`);
|
||
saveCheckpoint('03-judge-scores', scores);
|
||
return scores;
|
||
}
|
||
|
||
// ── Phase 4: Aggregate + report ─────────────────────────────────
|
||
|
||
function aggregate(scores) {
|
||
const perJudge = {};
|
||
for (const judge of JUDGE_MODELS) {
|
||
const armMeans = {};
|
||
for (const arm of ['A', 'B', 'C']) {
|
||
const vals = EVAL_EXAMPLES.map(e => scores[judge][arm][e.id]?.overall ?? 0);
|
||
armMeans[arm] = mean(vals);
|
||
}
|
||
const ratioCA = armMeans.A > 0 ? armMeans.C / armMeans.A : 0;
|
||
const ratioBA = armMeans.A > 0 ? armMeans.B / armMeans.A : 0;
|
||
perJudge[judge] = { ...armMeans, ratioCA, ratioBA };
|
||
}
|
||
const overall = {
|
||
A: mean(JUDGE_MODELS.map(j => perJudge[j].A)),
|
||
B: mean(JUDGE_MODELS.map(j => perJudge[j].B)),
|
||
C: mean(JUDGE_MODELS.map(j => perJudge[j].C)),
|
||
ratioCA: mean(JUDGE_MODELS.map(j => perJudge[j].ratioCA)),
|
||
ratioBA: mean(JUDGE_MODELS.map(j => perJudge[j].ratioBA)),
|
||
};
|
||
return { perJudge, overall };
|
||
}
|
||
|
||
function mean(vals) {
|
||
if (vals.length === 0) return 0;
|
||
return vals.reduce((a, b) => a + b, 0) / vals.length;
|
||
}
|
||
|
||
function writeReport({ evolvedPrompt, outputs, scores, agg }) {
|
||
const { perJudge, overall } = agg;
|
||
const verdict = overall.ratioCA >= 0.95
|
||
? '✅ **HYPOTHESIS CONFIRMED** — Arm C reached ≥ 95% of Opus 4.6 quality.'
|
||
: overall.ratioCA >= 0.85
|
||
? '⚠️ **PARTIAL** — Arm C closed the gap substantially (85–95%) but not to 95%.'
|
||
: '❌ **NOT CONFIRMED** — Arm C is still noticeably below Opus quality.';
|
||
|
||
const lines = [];
|
||
lines.push(`# Evolution Hypothesis Report — ${ts}`);
|
||
lines.push('');
|
||
lines.push('> **Hypothesis:** Waggle\u0027s self-evolution loop can close the quality');
|
||
lines.push('> gap between a weak model (Gemma 4) and a strong model (Opus 4.6).');
|
||
lines.push('');
|
||
lines.push('## Verdict');
|
||
lines.push('');
|
||
lines.push(verdict);
|
||
lines.push('');
|
||
lines.push(`- Arm C / Arm A mean ratio: **${(overall.ratioCA * 100).toFixed(1)}%**`);
|
||
lines.push(`- Arm B / Arm A mean ratio: ${(overall.ratioBA * 100).toFixed(1)}% (weak-model floor)`);
|
||
lines.push(`- Gap closed by evolution: ${((overall.ratioCA - overall.ratioBA) * 100).toFixed(1)}pp`);
|
||
lines.push('');
|
||
lines.push('## Setup');
|
||
lines.push('');
|
||
lines.push(`- **Eval size:** ${EVAL_SIZE} coder questions (curated, with reference answers)`);
|
||
lines.push(`- **GEPA config:** population=${GEPA_POP}, generations=${GEPA_GENS}`);
|
||
lines.push(`- **Skip evolution:** ${SKIP_EVOLUTION}`);
|
||
lines.push('');
|
||
lines.push('| Arm | Description | Model |');
|
||
lines.push('|---|---|---|');
|
||
lines.push(`| A | Strong-model upper bound | ${ARM_A_MODEL} |`);
|
||
lines.push(`| B | Weak-model lower bound | ${ARM_B_MODEL} (baseline prompt) |`);
|
||
lines.push(`| C | Weak + Waggle evolution | ${ARM_B_MODEL} (evolved prompt) |`);
|
||
lines.push('');
|
||
lines.push('### Judges (blind, independent scoring)');
|
||
lines.push('');
|
||
for (const j of JUDGE_MODELS) lines.push(`- ${j}`);
|
||
lines.push('');
|
||
|
||
// Aggregate table
|
||
lines.push('## Results by Judge');
|
||
lines.push('');
|
||
lines.push('| Judge | Arm A | Arm B | Arm C | C / A | B / A |');
|
||
lines.push('|---|---|---|---|---|---|');
|
||
for (const judge of JUDGE_MODELS) {
|
||
const p = perJudge[judge];
|
||
lines.push(`| ${judge} | ${p.A.toFixed(3)} | ${p.B.toFixed(3)} | ${p.C.toFixed(3)} | ${(p.ratioCA * 100).toFixed(1)}% | ${(p.ratioBA * 100).toFixed(1)}% |`);
|
||
}
|
||
lines.push(`| **Mean** | **${overall.A.toFixed(3)}** | **${overall.B.toFixed(3)}** | **${overall.C.toFixed(3)}** | **${(overall.ratioCA * 100).toFixed(1)}%** | **${(overall.ratioBA * 100).toFixed(1)}%** |`);
|
||
lines.push('');
|
||
|
||
// Evolved prompt
|
||
lines.push('## Evolved Prompt (Arm C)');
|
||
lines.push('');
|
||
lines.push('### Baseline');
|
||
lines.push('```');
|
||
lines.push(BASELINE_PROMPT);
|
||
lines.push('```');
|
||
lines.push('');
|
||
lines.push('### Evolved');
|
||
lines.push('```');
|
||
lines.push(evolvedPrompt);
|
||
lines.push('```');
|
||
lines.push('');
|
||
|
||
// Per-example
|
||
lines.push('## Per-Example Mean Scores (averaged across judges)');
|
||
lines.push('');
|
||
lines.push('| Example | Arm A | Arm B | Arm C |');
|
||
lines.push('|---|---|---|---|');
|
||
for (const ex of EVAL_EXAMPLES) {
|
||
const armVal = (arm) => mean(JUDGE_MODELS.map(j => scores[j][arm][ex.id]?.overall ?? 0));
|
||
lines.push(`| ${ex.id} | ${armVal('A').toFixed(2)} | ${armVal('B').toFixed(2)} | ${armVal('C').toFixed(2)} |`);
|
||
}
|
||
lines.push('');
|
||
|
||
lines.push('## Cost Summary');
|
||
lines.push('');
|
||
lines.push(`- OpenRouter calls: ${_openrouterCalls} (tokens: in ${_openrouterTokens.in.toLocaleString()}, out ${_openrouterTokens.out.toLocaleString()})`);
|
||
lines.push(`- Anthropic direct calls: ${_anthropicCalls} (tokens: in ${_anthropicTokens.in.toLocaleString()}, out ${_anthropicTokens.out.toLocaleString()})`);
|
||
lines.push('');
|
||
lines.push('## Intermediate Artifacts');
|
||
lines.push('');
|
||
lines.push(`All checkpoints: \`${path.relative(repoRoot, workDir)}\``);
|
||
lines.push('');
|
||
lines.push('---');
|
||
lines.push('');
|
||
lines.push(`Generated by \`scripts/evolution-hypothesis.mjs\` at ${new Date().toISOString()}.`);
|
||
|
||
const reportPath = path.join(repoRoot, 'docs', `evolution-hypothesis-report-${ts}.md`);
|
||
fs.writeFileSync(reportPath, lines.join('\n'), 'utf-8');
|
||
return reportPath;
|
||
}
|
||
|
||
// ── Main ────────────────────────────────────────────────────────
|
||
|
||
(async () => {
|
||
try {
|
||
const evolvedPrompt = await evolvePrompt();
|
||
const outputs = await runAllArms(evolvedPrompt);
|
||
const scores = await judgeAllOutputs(outputs);
|
||
const agg = aggregate(scores);
|
||
|
||
console.log('');
|
||
console.log('📊 Final Results:');
|
||
console.log(` Arm A (${ARM_A_MODEL}): ${agg.overall.A.toFixed(3)}`);
|
||
console.log(` Arm B (${ARM_B_MODEL} raw): ${agg.overall.B.toFixed(3)}`);
|
||
console.log(` Arm C (${ARM_B_MODEL} evolved): ${agg.overall.C.toFixed(3)}`);
|
||
console.log(` C / A ratio: ${(agg.overall.ratioCA * 100).toFixed(1)}%`);
|
||
console.log('');
|
||
|
||
const reportPath = writeReport({ evolvedPrompt, outputs, scores, agg });
|
||
console.log(`📄 Report: ${path.relative(repoRoot, reportPath)}`);
|
||
console.log(`📁 Checkpoints: ${path.relative(repoRoot, workDir)}`);
|
||
} catch (err) {
|
||
console.error('');
|
||
console.error('💥 Hypothesis run failed:');
|
||
console.error(err?.stack ?? err);
|
||
process.exit(2);
|
||
}
|
||
})();
|
||
|
||
// ── Utilities ───────────────────────────────────────────────────
|
||
|
||
function parseArgs(argv) {
|
||
const m = new Map();
|
||
const flags = new Set();
|
||
for (const arg of argv) {
|
||
if (arg.startsWith('--')) {
|
||
const [key, ...val] = arg.slice(2).split('=');
|
||
if (val.length > 0) m.set(key, val.join('='));
|
||
else flags.add(key);
|
||
}
|
||
}
|
||
return {
|
||
has: (key) => flags.has(key) || m.has(key),
|
||
get: (key) => m.get(key),
|
||
};
|
||
}
|
||
|
||
// Keep the reference so eslint doesn't complain (filterJudgeFeedback is imported
|
||
// but only used if we enable structural-feedback filtering on GEPA — the simpler
|
||
// running-judge approach doesn't need it, but keeping the import documents the
|
||
// pattern for a future reviewer). Use it as a no-op to avoid unused-import lint.
|
||
void filterJudgeFeedback;
|
||
void EvolveSchema;
|
||
void buildSchemaFillPrompt;
|