This commit is contained in:
Oleg Maslov
142
benchmarks/gepa/scripts/faza-1/analyze-checkpoint-a.py
Normal file
142
benchmarks/gepa/scripts/faza-1/analyze-checkpoint-a.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Compute Checkpoint A v2 aggregates + classify per pre-registered bands."""
|
||||
import json, sys, io, math, os
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
JSONL = 'D:/Projects/waggle-os-faza1-wt/benchmarks/results/gepa-faza1/null-baseline/null-baseline-eval.jsonl'
|
||||
with open(JSONL, encoding='utf-8') as f:
|
||||
records = [json.loads(l) for l in f if l.strip()]
|
||||
|
||||
print(f'TOTAL records: {len(records)}')
|
||||
total_cost = sum(r['evalCostUsd'] for r in records)
|
||||
total_subject = sum(r['candidateCostUsd'] for r in records)
|
||||
total_judge = sum(r['judges']['judgeCostTotal'] for r in records)
|
||||
print(f'cumulative cost: ${total_cost:.4f} (subject ${total_subject:.4f} + judge ${total_judge:.4f})')
|
||||
|
||||
SHAPES = ['claude','qwen-thinking','qwen-non-thinking','gpt','generic-simple']
|
||||
shape_aggs = {}
|
||||
for s in SHAPES:
|
||||
rs = [r for r in records if r['shape'] == s]
|
||||
if not rs: continue
|
||||
n = len(rs)
|
||||
pii = sum(1 for r in rs if r['judges']['trioStrictPassII'])
|
||||
pi = sum(1 for r in rs if r['judges']['trioStrictPassI'])
|
||||
mt = sum(r['judges']['trioMean'] for r in rs)/n
|
||||
mr = sum(r['retrievalCalls'] for r in rs)/n
|
||||
mc = sum(r['evalCostUsd'] for r in rs)/n
|
||||
le = sum(1 for r in rs if r['loopExhausted'])/n
|
||||
ms = sum(r['stepsTaken'] for r in rs)/n
|
||||
rate = pii/n
|
||||
z = 1.96
|
||||
denom = 1 + z*z/n
|
||||
center = (rate + z*z/(2*n)) / denom
|
||||
margin = z * math.sqrt(rate*(1-rate)/n + z*z/(4*n*n)) / denom
|
||||
ci_low = max(0, center - margin)
|
||||
ci_hi = min(1, center + margin)
|
||||
shape_aggs[s] = dict(n=n, pii=pii, pi=pi, rate=rate, mt=mt, mr=mr, mc=mc, le=le, ms=ms, ci_low=ci_low, ci_hi=ci_hi)
|
||||
|
||||
print('\n=== PER-SHAPE AGGREGATES (REAL) ===')
|
||||
for s in SHAPES:
|
||||
a = shape_aggs[s]
|
||||
print(f' {s:<22} pass_II={a["pii"]}/{a["n"]} ({a["rate"]:.1%}) pass_I={a["pi"]}/{a["n"]} trio={a["mt"]:.3f} retr={a["mr"]:.2f} cost=${a["mc"]:.4f} CI95=[{a["ci_low"]:.3f},{a["ci_hi"]:.3f}]')
|
||||
|
||||
artifactual = {'claude': 0.50, 'qwen-thinking': 1.0, 'qwen-non-thinking': 0.75, 'gpt': 0.875, 'generic-simple': 0.875}
|
||||
print('\n=== DELTA vs artifactual ===')
|
||||
deltas = {}
|
||||
for s in SHAPES:
|
||||
d = (shape_aggs[s]['rate'] - artifactual[s]) * 100
|
||||
deltas[s] = d
|
||||
print(f' {s:<22} artifactual={artifactual[s]:.1%} real={shape_aggs[s]["rate"]:.1%} delta={d:+.1f}pp')
|
||||
|
||||
print('\n=== PRE-REGISTERED BAND CLASSIFICATION (LOCKED §C) ===')
|
||||
expected_bounds = {'claude':(35,65),'qwen-thinking':(85,100),'qwen-non-thinking':(60,90),'gpt':(73,100),'generic-simple':(73,100)}
|
||||
in_per_shape_band = True
|
||||
for s in SHAPES:
|
||||
pct = shape_aggs[s]['rate'] * 100
|
||||
lo, hi = expected_bounds[s]
|
||||
ok = lo <= pct <= hi
|
||||
print(f' {s:<22} real {pct:>5.1f}% expected band [{lo}, {hi}] {"IN" if ok else "OUT"}')
|
||||
if not ok: in_per_shape_band = False
|
||||
|
||||
max_abs = max(abs(d) for d in deltas.values())
|
||||
sign_flips = sum(1 for s in SHAPES if (artifactual[s] >= 0.5) != (shape_aggs[s]['rate'] >= 0.5))
|
||||
all_neg = all(d <= 0 for d in deltas.values())
|
||||
all_pos = all(d >= 0 for d in deltas.values())
|
||||
uniform_shift_ok = (all_neg or all_pos) and (max(deltas.values()) - min(deltas.values()) <= 25)
|
||||
|
||||
print('\n=== RAW AGREEMENT + KAPPA ===')
|
||||
def per_judge(model, t=4.0):
|
||||
return [next(j['mean']>=t for j in r['judges']['records'] if j['judge_model']==model) for r in records]
|
||||
opus = per_judge('claude-opus-4-7')
|
||||
gpt_j = per_judge('gpt-5.4')
|
||||
mm = per_judge('minimax-m27-via-openrouter')
|
||||
|
||||
def raw_agree(a,b): return sum(1 for x,y in zip(a,b) if x==y)/len(a)
|
||||
def kappa(a,b):
|
||||
n=len(a); cc=sum(1 for x,y in zip(a,b) if x and y); ii=sum(1 for x,y in zip(a,b) if not x and not y)
|
||||
ci=sum(1 for x,y in zip(a,b) if x and not y); ic=sum(1 for x,y in zip(a,b) if not x and y)
|
||||
po=(cc+ii)/n; pa=(cc+ci)/n; pb=(cc+ic)/n; pe=pa*pb+(1-pa)*(1-pb)
|
||||
return (po-pe)/(1-pe) if pe<1 else 1.0
|
||||
|
||||
ra={'opus_gpt':raw_agree(opus,gpt_j),'opus_minimax':raw_agree(opus,mm),'gpt_minimax':raw_agree(gpt_j,mm)}
|
||||
k={'opus_gpt':kappa(opus,gpt_j),'opus_minimax':kappa(opus,mm),'gpt_minimax':kappa(gpt_j,mm)}
|
||||
for pair in ['opus_gpt','opus_minimax','gpt_minimax']:
|
||||
print(f' {pair:<16} raw={ra[pair]:.1%} kappa={k[pair]:+.3f}')
|
||||
min_ra = min(ra.values())
|
||||
min_k = min(k.values())
|
||||
print(f' MIN raw agreement: {min_ra:.1%} (threshold 65%: {"PASS" if min_ra>=0.65 else "FAIL"})')
|
||||
print(f' MIN kappa: {min_k:+.3f}')
|
||||
|
||||
print('\nPer-judge pass rates at 4.0:')
|
||||
print(f' Opus: {sum(opus)}/{len(opus)} = {sum(opus)/len(opus):.1%}')
|
||||
print(f' GPT: {sum(gpt_j)}/{len(gpt_j)} = {sum(gpt_j)/len(gpt_j):.1%}')
|
||||
print(f' MiniMax: {sum(mm)}/{len(mm)} = {sum(mm)/len(mm):.1%}')
|
||||
|
||||
artif_min_ra = 0.70
|
||||
ra_collapse_pp = (artif_min_ra - min_ra) * 100
|
||||
anomalous_max = max_abs > 30
|
||||
anomalous_flips = sign_flips > 2
|
||||
anomalous_ra = ra_collapse_pp > 20
|
||||
|
||||
print(f'\n=== ANOMALY CHECK ===')
|
||||
print(f' max |delta| > 30pp: {anomalous_max} (max={max_abs:.1f}pp)')
|
||||
print(f' sign flips > 2: {anomalous_flips} (count={sign_flips})')
|
||||
print(f' raw agreement collapse > 20pp: {anomalous_ra} (artifactual {artif_min_ra:.1%} -> real {min_ra:.1%}, delta {ra_collapse_pp:+.1f}pp)')
|
||||
|
||||
is_anomalous = anomalous_max or anomalous_flips or anomalous_ra
|
||||
is_expected = (in_per_shape_band or uniform_shift_ok) and not is_anomalous
|
||||
print(f'\nPER-SHAPE BANDS: {"all IN" if in_per_shape_band else "some OUT"}')
|
||||
print(f'UNIFORM SHIFT: ok={uniform_shift_ok} (all_neg={all_neg}, all_pos={all_pos}, spread={max(deltas.values())-min(deltas.values()):.1f}pp)')
|
||||
print(f'EXPECTED met: {is_expected}')
|
||||
print(f'ANOMALOUS triggered: {is_anomalous}')
|
||||
print(f'CLASSIFICATION: {"EXPECTED -> Gen 1 GO" if is_expected else "ANOMALOUS -> INVESTIGATE"}')
|
||||
|
||||
real_per_eval = total_cost / len(records)
|
||||
gen1_proj = 5 * 3 * 8 * real_per_eval
|
||||
artif_per_eval = 0.124
|
||||
sens_pct = (real_per_eval - artif_per_eval) / artif_per_eval * 100
|
||||
print(f'\n=== COST SENSITIVITY ===')
|
||||
print(f' artifactual: ${artif_per_eval:.4f}/eval real: ${real_per_eval:.4f}/eval delta: {sens_pct:+.1f}%')
|
||||
print(f' Gen 1 proj: ${gen1_proj:.4f} $78 halt: {"PASS" if gen1_proj<=78 else "HALT"}')
|
||||
|
||||
print('\n=== F-SATURATED PER-SHAPE ===')
|
||||
n_qual = 0
|
||||
for s in SHAPES:
|
||||
a = shape_aggs[s]
|
||||
qual = a['ci_low'] >= 0.88
|
||||
if qual: n_qual += 1
|
||||
print(f' {s:<22} pii={a["pii"]}/{a["n"]} CI_low={a["ci_low"]:.3f} >=0.88? {"Y" if qual else "N"} policy={"F-sat" if qual else "F.1 (>=+5pp)"}')
|
||||
if n_qual == 0: decision = 'GLOBAL: revoke F-saturated, apply F.1 to all 5'
|
||||
elif n_qual == 5: decision = 'GLOBAL: re-instate F-saturated for all 5'
|
||||
else: decision = f'MIXED: {n_qual}/5 qualify (per-shape policy, pre-authorized)'
|
||||
print(f'Decision: {decision}')
|
||||
|
||||
out = {
|
||||
'shape_aggregates': shape_aggs, 'deltas_pp': deltas, 'classification': 'EXPECTED' if is_expected else 'ANOMALOUS',
|
||||
'raw_agreement': ra, 'min_raw_agreement': min_ra, 'kappa': k, 'min_kappa': min_k,
|
||||
'per_judge_pass_rate': {'opus': sum(opus)/len(opus), 'gpt': sum(gpt_j)/len(gpt_j), 'minimax': sum(mm)/len(mm)},
|
||||
'cost': {'total': total_cost, 'per_eval': real_per_eval, 'sensitivity_pct': sens_pct, 'gen1_projected': gen1_proj},
|
||||
'F_saturated_n_qualifying': n_qual, 'F_saturated_decision': decision,
|
||||
}
|
||||
with open('D:/Projects/waggle-os-faza1-wt/benchmarks/results/gepa-faza1/null-baseline/checkpoint-a-v2-aggregates.json', 'w') as f:
|
||||
json.dump(out, f, indent=2)
|
||||
print('\n=> aggregates JSON written')
|
||||
194
benchmarks/gepa/scripts/faza-1/compute-final-kappa.ts
Normal file
194
benchmarks/gepa/scripts/faza-1/compute-final-kappa.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — final κ_trio recompute on combined 135 evals.
|
||||
*
|
||||
* Per launch decision §F condition_3 + manifest v7 §canonical_kappa_anchor.
|
||||
*
|
||||
* Combines:
|
||||
* - Gen 1 eval JSONL (120 records)
|
||||
* - Checkpoint C eval JSONL (15 records)
|
||||
*
|
||||
* For each eval, derives per-judge binary pass (mean ≥ 4.0) and computes
|
||||
* pairwise Cohen's κ for the 3 judge pairs (Opus↔GPT, Opus↔MiniMax,
|
||||
* GPT↔MiniMax). Reports κ_trio (= min of pairs) + audit verdict.
|
||||
*
|
||||
* Per Amendment 5 §judge_metric_design: also reports raw agreement rate
|
||||
* (primary metric for synthesis Likert).
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
auditKappa,
|
||||
computeCohensKappa,
|
||||
CANONICAL_KAPPA,
|
||||
KAPPA_DRIFT_BAND_LOW,
|
||||
KAPPA_DRIFT_BAND_HIGH,
|
||||
} from '../../src/faza-1/kappa-audit.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const GEN_1_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/gen-1-eval.jsonl');
|
||||
const CHECKPOINT_C_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/checkpoint-c/checkpoint-c-eval.jsonl');
|
||||
const OUT_JSON = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/checkpoint-c/final-kappa-audit.json');
|
||||
|
||||
const PASS_THRESHOLD = 4.0;
|
||||
const JUDGE_ORDER = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
|
||||
|
||||
interface JudgeRec { judge_model: string; mean: number; raw: any }
|
||||
interface EvalRec {
|
||||
candidateId: string;
|
||||
judgeRecords: JudgeRec[];
|
||||
evalSet?: 'held-out';
|
||||
}
|
||||
|
||||
function loadJsonl(filepath: string): EvalRec[] {
|
||||
if (!fs.existsSync(filepath)) {
|
||||
console.error(`MISSING: ${filepath}`);
|
||||
return [];
|
||||
}
|
||||
return fs.readFileSync(filepath, 'utf-8').trim().split(/\n+/).filter(Boolean).map(l => JSON.parse(l));
|
||||
}
|
||||
|
||||
function judgePassPerEval(rec: EvalRec): { opus: boolean | null; gpt: boolean | null; minimax: boolean | null } {
|
||||
const findBy = (model: string) => rec.judgeRecords.find(j => j.judge_model === model);
|
||||
const opus = findBy('claude-opus-4-7');
|
||||
const gpt = findBy('gpt-5.4');
|
||||
const minimax = findBy('minimax-m27-via-openrouter');
|
||||
return {
|
||||
opus: opus && opus.mean > 0 ? opus.mean >= PASS_THRESHOLD : null,
|
||||
gpt: gpt && gpt.mean > 0 ? gpt.mean >= PASS_THRESHOLD : null,
|
||||
minimax: minimax && minimax.mean > 0 ? minimax.mean >= PASS_THRESHOLD : null,
|
||||
};
|
||||
}
|
||||
|
||||
interface ConfusionMatrix {
|
||||
bothCorrect: number;
|
||||
bothIncorrect: number;
|
||||
firstCorrectSecondIncorrect: number;
|
||||
firstIncorrectSecondCorrect: number;
|
||||
}
|
||||
|
||||
function buildConfusion(pairs: Array<[boolean, boolean]>): ConfusionMatrix {
|
||||
const m: ConfusionMatrix = { bothCorrect: 0, bothIncorrect: 0, firstCorrectSecondIncorrect: 0, firstIncorrectSecondCorrect: 0 };
|
||||
for (const [a, b] of pairs) {
|
||||
if (a && b) m.bothCorrect++;
|
||||
else if (!a && !b) m.bothIncorrect++;
|
||||
else if (a && !b) m.firstCorrectSecondIncorrect++;
|
||||
else m.firstIncorrectSecondCorrect++;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
function rawAgreement(pairs: Array<[boolean, boolean]>): number {
|
||||
if (pairs.length === 0) return NaN;
|
||||
let agree = 0;
|
||||
for (const [a, b] of pairs) if (a === b) agree++;
|
||||
return agree / pairs.length;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const gen1 = loadJsonl(GEN_1_JSONL);
|
||||
const cpc = loadJsonl(CHECKPOINT_C_JSONL);
|
||||
const combined = [...gen1, ...cpc];
|
||||
console.log(`Loaded: gen-1=${gen1.length}, checkpoint-c=${cpc.length}, combined=${combined.length}`);
|
||||
|
||||
// Build per-judge pass arrays (skip evals where any judge failed parse)
|
||||
const opusGpt: Array<[boolean, boolean]> = [];
|
||||
const opusMinimax: Array<[boolean, boolean]> = [];
|
||||
const gptMinimax: Array<[boolean, boolean]> = [];
|
||||
let droppedDueToFailedJudge = 0;
|
||||
|
||||
for (const rec of combined) {
|
||||
const v = judgePassPerEval(rec);
|
||||
if (v.opus === null || v.gpt === null || v.minimax === null) {
|
||||
droppedDueToFailedJudge++;
|
||||
continue;
|
||||
}
|
||||
opusGpt.push([v.opus, v.gpt]);
|
||||
opusMinimax.push([v.opus, v.minimax]);
|
||||
gptMinimax.push([v.gpt, v.minimax]);
|
||||
}
|
||||
console.log(`Effective N (after dropping failed-judge evals): ${opusGpt.length} (dropped: ${droppedDueToFailedJudge})`);
|
||||
|
||||
const kOpusGpt = computeCohensKappa(buildConfusion(opusGpt));
|
||||
const kOpusMinimax = computeCohensKappa(buildConfusion(opusMinimax));
|
||||
const kGptMinimax = computeCohensKappa(buildConfusion(gptMinimax));
|
||||
|
||||
const audit = auditKappa({ kOpusGpt, kOpusMinimax, kGptMinimax });
|
||||
|
||||
const rawOG = rawAgreement(opusGpt);
|
||||
const rawOM = rawAgreement(opusMinimax);
|
||||
const rawGM = rawAgreement(gptMinimax);
|
||||
const rawMin = Math.min(rawOG, rawOM, rawGM);
|
||||
|
||||
const passRates = {
|
||||
opus: opusGpt.filter(p => p[0]).length / opusGpt.length,
|
||||
gpt: opusGpt.filter(p => p[1]).length / opusGpt.length,
|
||||
minimax: opusMinimax.filter(p => p[1]).length / opusMinimax.length,
|
||||
};
|
||||
|
||||
const result = {
|
||||
generated_at: new Date().toISOString(),
|
||||
inputs: {
|
||||
gen_1_jsonl_records: gen1.length,
|
||||
checkpoint_c_jsonl_records: cpc.length,
|
||||
combined_records: combined.length,
|
||||
effective_n_after_judge_failures: opusGpt.length,
|
||||
dropped_due_to_failed_judge: droppedDueToFailedJudge,
|
||||
threshold: PASS_THRESHOLD,
|
||||
},
|
||||
pairwise_kappa: {
|
||||
opus_gpt: kOpusGpt,
|
||||
opus_minimax: kOpusMinimax,
|
||||
gpt_minimax: kGptMinimax,
|
||||
},
|
||||
pairwise_raw_agreement: {
|
||||
opus_gpt: rawOG,
|
||||
opus_minimax: rawOM,
|
||||
gpt_minimax: rawGM,
|
||||
min: rawMin,
|
||||
},
|
||||
per_judge_pass_rates: passRates,
|
||||
canonical_kappa: CANONICAL_KAPPA,
|
||||
drift_band: { low: KAPPA_DRIFT_BAND_LOW, high: KAPPA_DRIFT_BAND_HIGH },
|
||||
audit: audit,
|
||||
};
|
||||
|
||||
fs.writeFileSync(OUT_JSON, JSON.stringify(result, null, 2));
|
||||
|
||||
console.log('');
|
||||
console.log('━'.repeat(76));
|
||||
console.log(' Faza 1 final κ audit on combined 135 evals');
|
||||
console.log('━'.repeat(76));
|
||||
console.log(` N effective : ${opusGpt.length}`);
|
||||
console.log(` Per-judge pass rate@4.0 : Opus=${(passRates.opus*100).toFixed(1)}% GPT=${(passRates.gpt*100).toFixed(1)}% MiniMax=${(passRates.minimax*100).toFixed(1)}%`);
|
||||
console.log('');
|
||||
console.log(' PAIRWISE Cohen\'s κ:');
|
||||
console.log(` Opus↔GPT : ${kOpusGpt.toFixed(4)}`);
|
||||
console.log(` Opus↔MiniMax : ${kOpusMinimax.toFixed(4)}`);
|
||||
console.log(` GPT↔MiniMax : ${kGptMinimax.toFixed(4)}`);
|
||||
console.log('');
|
||||
console.log(' PAIRWISE raw agreement (Amendment 5 PRIMARY for synthesis Likert):');
|
||||
console.log(` Opus↔GPT : ${(rawOG*100).toFixed(1)}%`);
|
||||
console.log(` Opus↔MiniMax : ${(rawOM*100).toFixed(1)}%`);
|
||||
console.log(` GPT↔MiniMax : ${(rawGM*100).toFixed(1)}%`);
|
||||
console.log(` MIN raw : ${(rawMin*100).toFixed(1)}%`);
|
||||
console.log('');
|
||||
console.log(` κ_conservative_trio : ${audit.kConservativeTrio.toFixed(4)}`);
|
||||
console.log(` Canonical κ : ${CANONICAL_KAPPA.toFixed(4)}`);
|
||||
console.log(` Drift band : [${KAPPA_DRIFT_BAND_LOW.toFixed(4)}, ${KAPPA_DRIFT_BAND_HIGH.toFixed(4)}]`);
|
||||
console.log(` §F.3 verdict : ${audit.verdict}`);
|
||||
console.log(` v6 policy floor (≥0.70): ${audit.v6PolicyFloorPass ? 'PASS' : 'FAIL'}`);
|
||||
console.log(` Amendment 5 raw 65% min: ${rawMin >= 0.65 ? 'PASS' : 'FAIL'} (observed ${(rawMin*100).toFixed(1)}%)`);
|
||||
console.log('');
|
||||
console.log(` Audit log: ${audit.auditLogLine}`);
|
||||
console.log('');
|
||||
console.log(`Wrote: ${OUT_JSON}`);
|
||||
}
|
||||
|
||||
main();
|
||||
488
benchmarks/gepa/scripts/faza-1/generate-h3-corpus.ts
Normal file
488
benchmarks/gepa/scripts/faza-1/generate-h3-corpus.ts
Normal file
@@ -0,0 +1,488 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — H3 NorthLane CFO synthesis corpus generator.
|
||||
*
|
||||
* Per launch decision §G step 4 + manifest v7 §corpus_design + Amendment 1 Ask A Option C.
|
||||
*
|
||||
* Generates 50 stratified synthesis-task instances via Opus 4.7 oracle.
|
||||
*
|
||||
* Cost projection: ~$5 (50 × $0.10/instance avg).
|
||||
* Halt threshold: $7 (40% buffer per manifest v7 §corpus_design.expected_generation_cost_usd).
|
||||
*
|
||||
* Output: benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/generate-h3-corpus.ts --dry-run
|
||||
* # No LLM call. Validates stratification + prompt build only.
|
||||
*
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/generate-h3-corpus.ts --probe
|
||||
* # Single instance (cell index 0). ~$0.10. Validates LiteLLM connection + JSON parse.
|
||||
*
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/generate-h3-corpus.ts --all
|
||||
* # All 50 instances. ~$5. Halt at $7. Spot-audit + Pre-A report afterwards.
|
||||
*
|
||||
* Authority: launch decision LOCK at decisions/2026-04-28-gepa-faza1-launch.md (PM-Waggle-OS)
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
TOTAL_INSTANCES,
|
||||
STRATIFICATION_SEED,
|
||||
type CorpusInstance,
|
||||
type StratificationCell,
|
||||
listStratificationCells,
|
||||
buildInstanceId,
|
||||
validateInstance,
|
||||
runSpotAudit,
|
||||
corpusSha256,
|
||||
} from '../../src/faza-1/corpus.js';
|
||||
import { buildCorpusInstancePrompt } from '../../src/faza-1/corpus-prompt.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const OUT_DIR = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/corpus');
|
||||
const OUT_JSONL = path.join(OUT_DIR, 'h3-northlane-cfo-50-instances.jsonl');
|
||||
const RUN_LOG = path.join(OUT_DIR, 'generation-run.log');
|
||||
const SPOT_AUDIT_REPORT = path.join(OUT_DIR, 'h3-spot-audit-pre-a-report.md');
|
||||
|
||||
// ── LLM config (per launch decision §A.1 inheritance) ─────────────────────
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const ORACLE_MODEL = 'claude-opus-4-7';
|
||||
const ORACLE_MAX_TOKENS = 8000;
|
||||
const ORACLE_TEMPERATURE = 0.7; // higher for instance variation per manifest v7
|
||||
const ORACLE_THINKING = true;
|
||||
const MANIFEST_ANCHOR = 'manifest-v7-gepa-faza1';
|
||||
|
||||
// Pricing per pilot runner SHA 8a6251e2 line 129
|
||||
const PRICE_INPUT_PER_M = 15.0;
|
||||
const PRICE_OUTPUT_PER_M = 75.0;
|
||||
|
||||
// Cost halt per manifest v7 Amendment 3: $15 = 40% buffer over $13.58 actual expected
|
||||
// (was $7 pre-Amendment-3; raised after probe revealed inherited $0.10/instance estimate
|
||||
// was 170% off vs actual Opus 4.7 generation cost of $0.27/instance).
|
||||
const COST_HALT_USD = 15.0;
|
||||
|
||||
// ── Logging ────────────────────────────────────────────────────────────────
|
||||
|
||||
function log(msg: string): void {
|
||||
const line = `[${new Date().toISOString()}] ${msg}\n`;
|
||||
try { fs.appendFileSync(RUN_LOG, line); } catch { /* dir may not exist yet */ }
|
||||
process.stderr.write(line);
|
||||
}
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface Args {
|
||||
mode: 'dry-run' | 'probe' | 'all' | 'retry-failed';
|
||||
startIdx?: number;
|
||||
endIdx?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The 3 cells that failed in the 2026-04-28 first-pass generation due to
|
||||
* Opus emitting unescaped quotation marks in long doc bodies. Per
|
||||
* Amendment 4 retry methodology, these cells are re-run with JSON-mode
|
||||
* response_format + lowered temperature + reduced max_tokens.
|
||||
*/
|
||||
const RETRY_FAILED_CELLS: ReadonlyArray<{ family: string; persona: string; stage: string }> = [
|
||||
{ family: 'F4', persona: 'p2_cfo', stage: 'stage_a_series_b_growth_burning' },
|
||||
{ family: 'F4', persona: 'p2_cfo', stage: 'stage_b_post_profitable_consolidation' },
|
||||
{ family: 'F5', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' },
|
||||
];
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
let mode: Args['mode'] = 'dry-run';
|
||||
let startIdx: number | undefined;
|
||||
let endIdx: number | undefined;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
const next = argv[i + 1];
|
||||
switch (flag) {
|
||||
case '--dry-run': mode = 'dry-run'; break;
|
||||
case '--probe': mode = 'probe'; break;
|
||||
case '--all': mode = 'all'; break;
|
||||
case '--retry-failed': mode = 'retry-failed'; break;
|
||||
case '--start': startIdx = Number(next); i++; break;
|
||||
case '--end': endIdx = Number(next); i++; break;
|
||||
}
|
||||
}
|
||||
return { mode, startIdx, endIdx };
|
||||
}
|
||||
|
||||
// ── LiteLLM call ───────────────────────────────────────────────────────────
|
||||
|
||||
interface LlmResult {
|
||||
content: string;
|
||||
inTokens: number;
|
||||
outTokens: number;
|
||||
costUsd: number;
|
||||
latencyMs: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call Opus oracle options. The default (temperature 1.0, max_tokens 8000,
|
||||
* no response_format) matches the original generation. JSON-mode retry uses
|
||||
* temperature 0.3 + max_tokens 6000 + response_format json_object per
|
||||
* Amendment 4 retry methodology.
|
||||
*/
|
||||
interface OpusOracleOptions {
|
||||
maxTokens?: number;
|
||||
temperature?: number;
|
||||
responseFormatJsonObject?: boolean;
|
||||
}
|
||||
|
||||
async function callOpusOracle(
|
||||
prompt: string,
|
||||
options: OpusOracleOptions = {},
|
||||
): Promise<LlmResult> {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY;
|
||||
if (!masterKey) {
|
||||
throw new Error('LITELLM_MASTER_KEY env not set; cannot call Opus oracle');
|
||||
}
|
||||
|
||||
const payload: Record<string, unknown> = {
|
||||
model: ORACLE_MODEL,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: options.maxTokens ?? ORACLE_MAX_TOKENS,
|
||||
};
|
||||
// Anthropic Opus 4.7 + response_format=json_object rejects `temperature`
|
||||
// as deprecated for that mode. Omit temperature when JSON-mode is requested
|
||||
// (matches the pilot runner's "reasoning model omit temperature" precedent
|
||||
// for GPT-5.4 + MiniMax). Standard mode keeps temperature.
|
||||
if (options.responseFormatJsonObject) {
|
||||
payload.response_format = { type: 'json_object' };
|
||||
} else {
|
||||
payload.temperature = options.temperature ?? 1.0;
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastErr: string | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const d: any = await resp.json();
|
||||
if ('error' in d) {
|
||||
lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
const content = d.choices?.[0]?.message?.content ?? '';
|
||||
const usage = d.usage ?? {};
|
||||
const inTok = usage.prompt_tokens ?? 0;
|
||||
const outTok = usage.completion_tokens ?? 0;
|
||||
const costUsd = (inTok * PRICE_INPUT_PER_M + outTok * PRICE_OUTPUT_PER_M) / 1_000_000;
|
||||
return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
|
||||
} catch (e) {
|
||||
lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return {
|
||||
content: '', inTokens: 0, outTokens: 0, costUsd: 0,
|
||||
latencyMs: Date.now() - started,
|
||||
error: lastErr ?? 'unknown error',
|
||||
};
|
||||
}
|
||||
|
||||
// ── JSON extraction ────────────────────────────────────────────────────────
|
||||
|
||||
interface ParsedInstance {
|
||||
personaText: string;
|
||||
scenario: string; // optional — some prompts may put scenario inside personaText
|
||||
sourceDocuments: Array<{ title: string; body: string }>;
|
||||
question: string;
|
||||
}
|
||||
|
||||
function parseInstanceJson(content: string): ParsedInstance | { error: string } {
|
||||
// Strip code fence wrappers if present (Opus sometimes adds them despite instructions)
|
||||
let s = content.trim();
|
||||
if (s.startsWith('```')) {
|
||||
s = s.replace(/^```[a-z]*\n?/, '').replace(/```\s*$/, '');
|
||||
}
|
||||
// Find first { and last } for tolerant parsing
|
||||
const firstBrace = s.indexOf('{');
|
||||
const lastBrace = s.lastIndexOf('}');
|
||||
if (firstBrace < 0 || lastBrace < 0) {
|
||||
return { error: `no JSON object found in oracle response (length=${content.length})` };
|
||||
}
|
||||
const jsonStr = s.slice(firstBrace, lastBrace + 1);
|
||||
try {
|
||||
const parsed = JSON.parse(jsonStr);
|
||||
// Tolerant schema: accept either a "scenario" field or scenario embedded in personaText
|
||||
const personaText: string = parsed.personaText ?? '';
|
||||
const scenario: string = parsed.scenario ?? '';
|
||||
const sourceDocuments = Array.isArray(parsed.sourceDocuments) ? parsed.sourceDocuments : [];
|
||||
const question: string = parsed.question ?? '';
|
||||
|
||||
if (!personaText || !sourceDocuments.length || !question) {
|
||||
return { error: `parsed JSON missing required fields (personaText/sourceDocuments/question)` };
|
||||
}
|
||||
return { personaText, scenario, sourceDocuments, question };
|
||||
} catch (e) {
|
||||
return { error: `JSON parse failed: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Build full CorpusInstance from parsed oracle output ────────────────────
|
||||
|
||||
function assembleInstance(
|
||||
cell: StratificationCell,
|
||||
instanceId: string,
|
||||
parsed: ParsedInstance,
|
||||
llm: LlmResult,
|
||||
): CorpusInstance {
|
||||
const sourceDocuments = parsed.sourceDocuments.map(d => ({
|
||||
title: d.title,
|
||||
body: d.body,
|
||||
charCount: d.body.length,
|
||||
}));
|
||||
// If oracle put scenario in personaText, just use personaText as-is.
|
||||
// Otherwise concatenate "personaText\n\nScenario: scenario".
|
||||
const fullPersonaText = parsed.scenario
|
||||
? `${parsed.personaText}\n\nScenario: ${parsed.scenario}`
|
||||
: parsed.personaText;
|
||||
const materialsConcat = sourceDocuments
|
||||
.map(d => `## ${d.title}\n\n${d.body}`)
|
||||
.join('\n\n---\n\n');
|
||||
return {
|
||||
instanceId,
|
||||
cell,
|
||||
personaText: fullPersonaText,
|
||||
scenario: parsed.scenario || extractScenarioFromPersonaText(parsed.personaText),
|
||||
sourceDocuments,
|
||||
question: parsed.question,
|
||||
materialsConcat,
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
generatedBy: ORACLE_MODEL,
|
||||
generatedAtIso: new Date().toISOString(),
|
||||
generationCostUsd: llm.costUsd,
|
||||
};
|
||||
}
|
||||
|
||||
function extractScenarioFromPersonaText(personaText: string): string {
|
||||
const m = personaText.match(/Scenario:\s*([\s\S]*)/i);
|
||||
return m ? m[1].trim() : '';
|
||||
}
|
||||
|
||||
// ── Generate one cell ──────────────────────────────────────────────────────
|
||||
|
||||
async function generateOneCell(
|
||||
cell: StratificationCell,
|
||||
ordinal: number,
|
||||
options: OpusOracleOptions & { retryNote?: string } = {},
|
||||
): Promise<CorpusInstance | { error: string }> {
|
||||
const instanceId = buildInstanceId(cell, ordinal);
|
||||
const prompt = buildCorpusInstancePrompt({ cell, instanceId });
|
||||
const noteSuffix = options.retryNote ? ` [${options.retryNote}]` : '';
|
||||
log(`[${instanceId}] generating via ${ORACLE_MODEL} (prompt ${prompt.length}c)${noteSuffix}`);
|
||||
const llm = await callOpusOracle(prompt, options);
|
||||
if (llm.error) {
|
||||
log(`[${instanceId}] LLM error: ${llm.error}`);
|
||||
return { error: `LLM error: ${llm.error}` };
|
||||
}
|
||||
const parsed = parseInstanceJson(llm.content);
|
||||
if ('error' in parsed) {
|
||||
log(`[${instanceId}] parse error: ${parsed.error}; raw content first 200c: ${llm.content.slice(0, 200)}`);
|
||||
return { error: parsed.error };
|
||||
}
|
||||
const instance = assembleInstance(cell, instanceId, parsed, llm);
|
||||
const validation = validateInstance(instance);
|
||||
if (!validation.valid) {
|
||||
log(`[${instanceId}] validation failed: ${validation.violations.join('; ')}`);
|
||||
return { error: `validation failed: ${validation.violations.join('; ')}` };
|
||||
}
|
||||
log(`[${instanceId}] OK; cost=$${llm.costUsd.toFixed(4)}; ${instance.sourceDocuments.length} docs; latency=${llm.latencyMs}ms`);
|
||||
return instance;
|
||||
}
|
||||
|
||||
// ── Spot-audit report writer (Pre-A halt-and-PM artifact) ──────────────────
|
||||
|
||||
function writeSpotAuditReport(instances: CorpusInstance[], totalCostUsd: number): void {
|
||||
const audit = runSpotAudit(instances);
|
||||
const sha = corpusSha256(instances);
|
||||
const md: string[] = [];
|
||||
md.push('---');
|
||||
md.push('report_id: 2026-04-28-gepa-faza1-pre-a-corpus-audit');
|
||||
md.push('date: 2026-04-28');
|
||||
md.push('checkpoint: Pre-A (corpus quality + NULL kick auth)');
|
||||
md.push('manifest_anchor: manifest-v7-gepa-faza1');
|
||||
md.push(`corpus_sha256: ${sha}`);
|
||||
md.push(`total_instances: ${instances.length}`);
|
||||
md.push(`total_generation_cost_usd: ${totalCostUsd.toFixed(4)}`);
|
||||
md.push(`spot_audit_sample_size: ${audit.sampleSize}`);
|
||||
md.push(`spot_audit_seed: ${STRATIFICATION_SEED}`);
|
||||
md.push(`halt_on_failure: ${audit.haltOnFailure}`);
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('# Pre-A Halt-and-PM Report — H3 Corpus Quality Audit');
|
||||
md.push('');
|
||||
md.push('## TL;DR');
|
||||
md.push('');
|
||||
md.push(`Generated **${instances.length}/${TOTAL_INSTANCES}** instances at total cost **$${totalCostUsd.toFixed(2)}** (vs $5 expected, $7 halt). Spot-audit sample of ${audit.sampleSize} random instances (seed=${STRATIFICATION_SEED}) ${audit.haltOnFailure ? 'FAILED — corpus regeneration required.' : 'PASSED — NULL-baseline kick authorized pending PM ratify.'}`);
|
||||
md.push('');
|
||||
md.push('## Spot-audit results (per-instance)');
|
||||
md.push('');
|
||||
md.push('| Instance ID | Result | Violations |');
|
||||
md.push('|---|---|---|');
|
||||
for (const a of audit.perInstance) {
|
||||
md.push(`| \`${a.instanceId}\` | ${a.result.valid ? '✓ PASS' : '✗ FAIL'} | ${a.result.valid ? '—' : a.result.violations.join('; ')} |`);
|
||||
}
|
||||
md.push('');
|
||||
md.push('## Stratification coverage');
|
||||
md.push('');
|
||||
md.push(`All 50 (5 task families × 5 personas × 2 company stages) cells generated in canonical order. Each (family, persona) pair appears exactly twice (once per stage). Stratification verified via library tests (\`corpus.test.ts\` 34 tests passing).`);
|
||||
md.push('');
|
||||
md.push('## Audit chain');
|
||||
md.push('');
|
||||
md.push(`- Corpus JSONL: \`benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl\``);
|
||||
md.push(`- Corpus SHA256: \`${sha}\``);
|
||||
md.push(`- Generation log: \`benchmarks/results/gepa-faza1/corpus/generation-run.log\``);
|
||||
md.push(`- Manifest v7 SHA: \`583712dde139ffc87fb1ab21643f68d52c56469ded9e8090a624980b05969beb\``);
|
||||
md.push(`- Substrate: c9bda3d (Phase 4.7) via worktree D:/Projects/waggle-os-faza1-wt`);
|
||||
md.push('');
|
||||
md.push('## PM ratification ask');
|
||||
md.push('');
|
||||
md.push(audit.haltOnFailure
|
||||
? 'CORPUS FAILED spot-audit. **Do NOT authorize NULL-baseline kick.** Recommended action: review failed instances above + re-run generation for failed cells (cost ~$0.20 per re-gen).'
|
||||
: 'CORPUS PASSED spot-audit. **Authorize NULL-baseline kick** (5 shapes × 8 instances per shape, expected cost ~$20).');
|
||||
md.push('');
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('**End of Pre-A halt-and-PM report. Standing AWAITING PM ratification.**');
|
||||
|
||||
fs.writeFileSync(SPOT_AUDIT_REPORT, md.join('\n'), 'utf-8');
|
||||
log(`[pre-a] spot-audit report written to ${SPOT_AUDIT_REPORT}`);
|
||||
}
|
||||
|
||||
// ── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
const cells = listStratificationCells();
|
||||
log(`[generator] mode=${args.mode}; total cells=${cells.length}`);
|
||||
|
||||
if (args.mode === 'dry-run') {
|
||||
log(`[dry-run] validating ${cells.length} stratification cells + prompt builds`);
|
||||
for (let i = 0; i < cells.length; i++) {
|
||||
const cell = cells[i];
|
||||
const id = buildInstanceId(cell, 1);
|
||||
const prompt = buildCorpusInstancePrompt({ cell, instanceId: id });
|
||||
if (i < 3 || i === cells.length - 1) {
|
||||
log(`[dry-run] cell[${i}] = ${id}; prompt = ${prompt.length}c`);
|
||||
}
|
||||
}
|
||||
log(`[dry-run] OK — all ${cells.length} cells produce valid prompts; no LLM call made`);
|
||||
log(`[dry-run] cost: $0.00`);
|
||||
return;
|
||||
}
|
||||
|
||||
let targetCells: StratificationCell[];
|
||||
if (args.mode === 'probe') {
|
||||
targetCells = cells.slice(args.startIdx ?? 0, (args.startIdx ?? 0) + 1);
|
||||
} else if (args.mode === 'retry-failed') {
|
||||
// Filter stratification to only the cells listed in RETRY_FAILED_CELLS.
|
||||
const lookup = new Set(RETRY_FAILED_CELLS.map(c => `${c.family}|${c.persona}|${c.stage}`));
|
||||
targetCells = cells.filter(c => lookup.has(`${c.family}|${c.persona}|${c.stage}`));
|
||||
if (targetCells.length !== RETRY_FAILED_CELLS.length) {
|
||||
log(`[retry-failed] FATAL: expected ${RETRY_FAILED_CELLS.length} cells, found ${targetCells.length}`);
|
||||
process.exit(2);
|
||||
}
|
||||
} else {
|
||||
targetCells = cells.slice(args.startIdx ?? 0, args.endIdx ?? cells.length);
|
||||
}
|
||||
|
||||
log(`[${args.mode}] generating ${targetCells.length} instance(s)`);
|
||||
|
||||
const instances: CorpusInstance[] = [];
|
||||
let cumulativeCost = 0;
|
||||
// Resume support: always load existing JSONL on startup so we never truncate
|
||||
// an existing corpus. Originally guarded on mode==='all' which broke retry-failed
|
||||
// mode (corpus was truncated to 3 retry instances; recovery via git checkout
|
||||
// restored 47 originals; this fix prevents recurrence).
|
||||
if (fs.existsSync(OUT_JSONL) && args.mode !== 'dry-run') {
|
||||
const lines = fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean);
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const inst = JSON.parse(line) as CorpusInstance;
|
||||
instances.push(inst);
|
||||
cumulativeCost += inst.generationCostUsd;
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
log(`[resume] loaded ${instances.length} existing instances; cumulative cost = $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
// Open JSONL for append
|
||||
const out = fs.createWriteStream(OUT_JSONL, { flags: instances.length > 0 ? 'a' : 'w' });
|
||||
const existingIds = new Set(instances.map(i => i.instanceId));
|
||||
|
||||
// Per Amendment 4: retry-failed mode uses JSON-mode response_format +
|
||||
// lower temperature + reduced max_tokens to mitigate the "Opus emits
|
||||
// unescaped quotes in long doc bodies" failure class observed on first run.
|
||||
const isRetryMode = args.mode === 'retry-failed';
|
||||
const cellOptions: OpusOracleOptions & { retryNote?: string } = isRetryMode
|
||||
? {
|
||||
responseFormatJsonObject: true,
|
||||
temperature: 0.3,
|
||||
maxTokens: 6000,
|
||||
retryNote: 'JSON-mode retry per Amendment 4',
|
||||
}
|
||||
: {};
|
||||
|
||||
for (let i = 0; i < targetCells.length; i++) {
|
||||
const cell = targetCells[i];
|
||||
const id = buildInstanceId(cell, 1);
|
||||
if (existingIds.has(id) && !isRetryMode) {
|
||||
log(`[skip] ${id} already in JSONL`);
|
||||
continue;
|
||||
}
|
||||
if (existingIds.has(id) && isRetryMode) {
|
||||
// In retry mode, this should not happen (retry targets only failed cells)
|
||||
log(`[retry-failed] WARNING: ${id} already in JSONL — skipping`);
|
||||
continue;
|
||||
}
|
||||
if (cumulativeCost >= COST_HALT_USD) {
|
||||
log(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD} cost halt — stopping generation`);
|
||||
break;
|
||||
}
|
||||
const result = await generateOneCell(cell, 1, cellOptions);
|
||||
if ('error' in result) {
|
||||
log(`[error] cell ${id} skipped due to: ${result.error}`);
|
||||
continue;
|
||||
}
|
||||
instances.push(result);
|
||||
cumulativeCost += result.generationCostUsd;
|
||||
out.write(JSON.stringify(result) + '\n');
|
||||
log(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_HALT_USD} halt; ${instances.length}/${TOTAL_INSTANCES} instances`);
|
||||
}
|
||||
out.end();
|
||||
|
||||
log(`[done] generated ${instances.length} instances; total cost $${cumulativeCost.toFixed(4)}`);
|
||||
|
||||
// Spot-audit + Pre-A report (only meaningful if we have a full or near-full corpus).
|
||||
// Skipped in retry-failed mode — the corrected Pre-A addendum is authored manually
|
||||
// by the orchestrating session per Amendment 4 §texture_audit_methodology.
|
||||
if (args.mode === 'all' && instances.length > 0) {
|
||||
writeSpotAuditReport(instances, cumulativeCost);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('FATAL:', e);
|
||||
process.exit(2);
|
||||
});
|
||||
161
benchmarks/gepa/scripts/faza-1/probe-registry-injection.ts
Normal file
161
benchmarks/gepa/scripts/faza-1/probe-registry-injection.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — REGISTRY-injection root-cause diagnostic probe.
|
||||
*
|
||||
* Per investigate-report.md §C.3, this probe distinguishes:
|
||||
* H1 — ESM module-identity mismatch (deep relative path vs package import)
|
||||
* H2 — registry mutation timing / silent freeze
|
||||
* H3 — other (tsx loader, vite-node interop, etc.)
|
||||
*
|
||||
* Method: import REGISTRY via BOTH paths the failing runner uses, then
|
||||
* mutate via the script's path and probe via both. Also exercise the
|
||||
* exact selectShape() call site the agent-loop uses.
|
||||
*
|
||||
* Cost: ~$0 (no LLM calls).
|
||||
*
|
||||
* Verdicts:
|
||||
* - "Same object?" === true → mutations propagate; H2/H3 candidates
|
||||
* - "Same object?" === false → H1 confirmed (module-identity mismatch)
|
||||
* - selectShape from package returns the mutated shape → fix is unnecessary
|
||||
* - selectShape from package throws → bug is on the read path
|
||||
*/
|
||||
|
||||
// ── Path A: deep relative import (matches the failing runner's import) ──
|
||||
import { REGISTRY as RegistryFromScriptDeepPath } from '../../../../packages/agent/src/prompt-shapes/selector.js';
|
||||
|
||||
// ── Path B: package import (matches what runRetrievalAgentLoop's call to
|
||||
// selectShape() goes through internally) ─────────────────────────
|
||||
import {
|
||||
REGISTRY as RegistryFromPackage,
|
||||
selectShape as selectShapeFromPackage,
|
||||
} from '@waggle/agent';
|
||||
|
||||
// ── Path C: package-internal prompt-shapes export (parallel re-export check) ──
|
||||
import { REGISTRY as RegistryFromPromptShapes } from '@waggle/agent';
|
||||
|
||||
const PROBE_SHAPE_NAME = 'claude-gen1-v1-probe';
|
||||
|
||||
// Build a minimal valid PromptShape stub matching the type contract.
|
||||
// We only care that it gets registered + retrieved; method bodies are not
|
||||
// invoked in this probe.
|
||||
const probeShape = {
|
||||
name: PROBE_SHAPE_NAME,
|
||||
metadata: {
|
||||
description: 'Probe shape for REGISTRY-injection diagnostic',
|
||||
modelClass: 'probe',
|
||||
defaultThinking: false,
|
||||
defaultMaxTokens: 100,
|
||||
evidence_link: 'benchmarks/results/gepa-faza1/gen-1/investigate-report.md',
|
||||
},
|
||||
systemPrompt: () => 'probe',
|
||||
soloUserPrompt: () => 'probe',
|
||||
multiStepKickoffUserPrompt: () => 'probe',
|
||||
retrievalInjectionUserPrompt: () => 'probe',
|
||||
};
|
||||
|
||||
function log(line: string): void {
|
||||
process.stdout.write(line + '\n');
|
||||
}
|
||||
|
||||
function header(t: string): void {
|
||||
log('');
|
||||
log('━'.repeat(76));
|
||||
log(` ${t}`);
|
||||
log('━'.repeat(76));
|
||||
}
|
||||
|
||||
header('GEPA Faza 1 — REGISTRY-injection diagnostic probe');
|
||||
|
||||
log(`Probe shape name: ${PROBE_SHAPE_NAME}`);
|
||||
log(`Node version: ${process.version}`);
|
||||
log(`tsx : (running via tsx if argv[0] is node + script)`);
|
||||
|
||||
// ── Step 1 — pre-mutation snapshot ────────────────────────────────────────
|
||||
|
||||
header('STEP 1 — Pre-mutation snapshot (3 import paths)');
|
||||
|
||||
const keysScriptPath = Object.keys(RegistryFromScriptDeepPath).sort();
|
||||
const keysPackagePath = Object.keys(RegistryFromPackage).sort();
|
||||
const keysPromptShapesPath = Object.keys(RegistryFromPromptShapes).sort();
|
||||
|
||||
log(`A) Script deep-relative-path REGISTRY: ${keysScriptPath.length} keys: [${keysScriptPath.join(', ')}]`);
|
||||
log(`B) Package @waggle/agent REGISTRY: ${keysPackagePath.length} keys: [${keysPackagePath.join(', ')}]`);
|
||||
log(`C) Package @waggle/agent (2nd import): ${keysPromptShapesPath.length} keys: [${keysPromptShapesPath.join(', ')}]`);
|
||||
log('');
|
||||
log(`Object identity A === B: ${RegistryFromScriptDeepPath === RegistryFromPackage}`);
|
||||
log(`Object identity A === C: ${RegistryFromScriptDeepPath === RegistryFromPromptShapes}`);
|
||||
log(`Object identity B === C: ${RegistryFromPackage === RegistryFromPromptShapes}`);
|
||||
|
||||
// ── Step 2 — mutate via script's path (the failing pattern) ───────────────
|
||||
|
||||
header('STEP 2 — Mutate REGISTRY via script deep-relative-path');
|
||||
|
||||
(RegistryFromScriptDeepPath as any)[PROBE_SHAPE_NAME] = probeShape;
|
||||
|
||||
log(`Mutation via Path A: REGISTRY[${PROBE_SHAPE_NAME}] = probeShape`);
|
||||
|
||||
// ── Step 3 — read-back from all 3 paths ───────────────────────────────────
|
||||
|
||||
header('STEP 3 — Read-back probe-shape via each path');
|
||||
|
||||
const seenInA = RegistryFromScriptDeepPath[PROBE_SHAPE_NAME] !== undefined;
|
||||
const seenInB = RegistryFromPackage[PROBE_SHAPE_NAME] !== undefined;
|
||||
const seenInC = RegistryFromPromptShapes[PROBE_SHAPE_NAME] !== undefined;
|
||||
|
||||
log(`A) Script-import sees probe-shape: ${seenInA}`);
|
||||
log(`B) Package-import sees probe-shape: ${seenInB}`);
|
||||
log(`C) Re-import sees probe-shape: ${seenInC}`);
|
||||
|
||||
// ── Step 4 — call selectShape via package (the agent-loop path) ───────────
|
||||
|
||||
header('STEP 4 — selectShape({override}) via @waggle/agent (agent-loop path)');
|
||||
|
||||
let selectShapeVerdict: 'FOUND' | 'NOT_FOUND' | 'OTHER_ERROR';
|
||||
let selectShapeError: string | null = null;
|
||||
|
||||
try {
|
||||
const found = selectShapeFromPackage('any-alias-not-relevant', { override: PROBE_SHAPE_NAME });
|
||||
log(`selectShape returned shape with name="${(found as any).name ?? '<missing>'}"`);
|
||||
selectShapeVerdict = 'FOUND';
|
||||
} catch (e) {
|
||||
selectShapeError = (e as Error).message;
|
||||
log(`selectShape THREW: ${selectShapeError}`);
|
||||
selectShapeVerdict = selectShapeError.includes('not in REGISTRY') ? 'NOT_FOUND' : 'OTHER_ERROR';
|
||||
}
|
||||
|
||||
// ── Step 5 — verdict ───────────────────────────────────────────────────────
|
||||
|
||||
header('STEP 5 — VERDICT');
|
||||
|
||||
const sameObjectAB = RegistryFromScriptDeepPath === RegistryFromPackage;
|
||||
|
||||
if (!sameObjectAB) {
|
||||
log('VERDICT: H1 CONFIRMED — ESM module-identity mismatch');
|
||||
log(' Script deep-relative-path REGISTRY and @waggle/agent REGISTRY are');
|
||||
log(' separate object instances. Mutations to one do not propagate.');
|
||||
log('');
|
||||
log('Fix: add registerShape(name, shape) API in selector.ts, called from');
|
||||
log(' a single canonical entry point. Avoid direct REGISTRY mutation.');
|
||||
} else if (sameObjectAB && seenInA && !seenInB) {
|
||||
log('VERDICT: H2/H3 — Same object but read mismatch');
|
||||
log(' This should not happen: identical objects with different key sets.');
|
||||
log(' Investigate JS engine optimization, hidden Proxy, or freeze-on-read.');
|
||||
} else if (sameObjectAB && seenInA && seenInB && selectShapeVerdict === 'FOUND') {
|
||||
log('VERDICT: NOT REPRODUCED — REGISTRY mutation works in this probe.');
|
||||
log(' The runner-time failure must be due to a different cause (timing,');
|
||||
log(' loader, dynamic-import side effect on candidate shape). Investigate');
|
||||
log(' loadCandidates() dynamic import return shape vs static import.');
|
||||
} else if (sameObjectAB && seenInA && seenInB && selectShapeVerdict !== 'FOUND') {
|
||||
log('VERDICT: H2/H3 — Read found but selectShape rejected');
|
||||
log(` Direct read sees probe-shape, but selectShape() error: ${selectShapeError}`);
|
||||
log(' Investigate selectShape() implementation for hidden state or guard.');
|
||||
} else {
|
||||
log('VERDICT: UNKNOWN');
|
||||
log(` sameObjectAB=${sameObjectAB} seenInA=${seenInA} seenInB=${seenInB} seenInC=${seenInC} selectShape=${selectShapeVerdict}`);
|
||||
}
|
||||
|
||||
log('');
|
||||
log('Probe complete. Cost: $0 (no LLM calls). Halt-and-PM with verdict above.');
|
||||
|
||||
// Exit 0 in all cases — we want PM to read the full output regardless of verdict
|
||||
process.exit(0);
|
||||
722
benchmarks/gepa/scripts/faza-1/run-checkpoint-c.ts
Normal file
722
benchmarks/gepa/scripts/faza-1/run-checkpoint-c.ts
Normal file
@@ -0,0 +1,722 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — Checkpoint C held-out validation runner.
|
||||
*
|
||||
* Per launch decision §F + §G step 9 + PM brief 2026-04-29 Checkpoint C ratify.
|
||||
*
|
||||
* Validates §F.1-passing candidates on 5 held-out instances per candidate
|
||||
* (NOT in original Gen 1 8-instance sample). Confirms §F.2 PASS isn't
|
||||
* overfit per §F.5 condition_2 (held-out Pass II within ±15pp of in-sample).
|
||||
*
|
||||
* Pre-registered candidates per PM brief 2026-04-29:
|
||||
* - claude::gen1-v1
|
||||
* - qwen-thinking::gen1-v1
|
||||
* - gpt::gen1-v2
|
||||
*
|
||||
* Usage:
|
||||
* --candidates <id,id,id> comma-separated candidate IDs (required)
|
||||
* --held-out-instances <N> default 5
|
||||
* --dry-run list planned evals without executing
|
||||
*
|
||||
* Held-out sample: deterministicShuffle(corpus, seed=42).slice(8, 8 + N_HELD_OUT)
|
||||
* — instances 8..12 of the same shuffled order Gen 1 used (Gen 1 used 0..7).
|
||||
*
|
||||
* Manifest binding: Amendment 11 (manifest_sha256_post_amendment_11 = fa716ff90a...).
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
import {
|
||||
MindDB,
|
||||
FrameStore,
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
createOllamaEmbedder,
|
||||
type Embedder,
|
||||
} from '@waggle/core';
|
||||
import {
|
||||
runRetrievalAgentLoop,
|
||||
type LlmCallFn,
|
||||
type LlmCallInput,
|
||||
type LlmCallResult as AgentLlmCallResult,
|
||||
type RetrievalSearchFn,
|
||||
type AgentRunResult,
|
||||
// Amendment 8 §canonical_mutation_api: registerShape MUST be imported from
|
||||
// '@waggle/agent' (same path the agent-loop uses internally) so the mutation
|
||||
// hits the SAME REGISTRY instance.
|
||||
REGISTRY,
|
||||
registerShape,
|
||||
type PromptShape,
|
||||
} from '@waggle/agent';
|
||||
|
||||
import { type CorpusInstance } from '../../src/faza-1/corpus.js';
|
||||
import {
|
||||
NULL_BASELINE_PER_SHAPE,
|
||||
NULL_BASELINE_AGGREGATE,
|
||||
type TieredFitnessComponents,
|
||||
type ShapeName,
|
||||
} from '../../src/faza-1/types.js';
|
||||
import {
|
||||
computeTieredFitness,
|
||||
} from '../../src/faza-1/fitness.js';
|
||||
import {
|
||||
validateCandidate,
|
||||
} from '../../src/faza-1/mutation-validator.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const CORPUS_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl');
|
||||
const PROMPT_SHAPES_DIR = path.join(REPO_ROOT, 'packages/agent/src/prompt-shapes');
|
||||
const GEPA_EVOLVED_DIR = path.join(PROMPT_SHAPES_DIR, 'gepa-evolved');
|
||||
const OUT_DIR = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/checkpoint-c');
|
||||
const OUT_JSONL = path.join(OUT_DIR, 'checkpoint-c-eval.jsonl');
|
||||
const RUN_LOG = path.join(OUT_DIR, 'checkpoint-c-run.log');
|
||||
const SUMMARY_JSON = path.join(OUT_DIR, 'checkpoint-c-summary.json');
|
||||
const SCRATCH_DIR = path.join(REPO_ROOT, 'tmp/gepa-faza1-checkpoint-c');
|
||||
const GEN_1_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/gen-1-eval.jsonl');
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const EMBEDDER_MODEL = 'nomic-embed-text';
|
||||
|
||||
const SAMPLING_SEED = 42;
|
||||
const N_GEN_1_SAMPLE = 8;
|
||||
const DEFAULT_HELD_OUT = 5;
|
||||
|
||||
const SUBJECT_ALIAS = 'qwen3.6-35b-a3b-via-dashscope-direct';
|
||||
const SUBJECT_MAX_TOKENS = 16000;
|
||||
const SUBJECT_THINKING = true;
|
||||
|
||||
const JUDGES = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
|
||||
const JUDGE_MAX_TOKENS = 3000;
|
||||
const JUDGE_RETRIES = 3;
|
||||
|
||||
const MAX_STEPS = 5;
|
||||
const MAX_RETRIEVALS_PER_STEP = 8;
|
||||
const PER_CALL_HALT_USD = 0.40;
|
||||
const PER_CELL_HALT_USD = 1.00;
|
||||
|
||||
const COST_HALT_USD = 8.0; // generous budget for held-out (3 candidates × 5 evals × ~$0.13 = $1.95 expected)
|
||||
|
||||
const MODEL_PRICING: Record<string, { in: number; out: number }> = {
|
||||
'claude-opus-4-7': { in: 15.0, out: 75.0 },
|
||||
'gpt-5.4': { in: 2.5, out: 10.0 },
|
||||
'minimax-m27-via-openrouter': { in: 0.7, out: 2.8 },
|
||||
'qwen3.6-35b-a3b-via-dashscope-direct': { in: 0.20, out: 0.80 },
|
||||
'qwen3.6-35b-a3b-via-openrouter': { in: 0.6, out: 2.4 },
|
||||
};
|
||||
|
||||
const MANIFEST_ANCHOR = 'manifest-v7-gepa-faza1';
|
||||
const MANIFEST_SHA_AMENDMENT_11 = 'fa716ff90a4345eb87962789f3a2ab3d54994edc93964f850ad64cf6fbf6d227';
|
||||
|
||||
// §F.5 condition_2 overfitting bound: held-out Pass II must be within ±15pp of in-sample Pass II
|
||||
const F5_OVERFITTING_BOUND_PP = 15;
|
||||
|
||||
function log(msg: string): void {
|
||||
const line = `[${new Date().toISOString()}] ${msg}\n`;
|
||||
try { fs.appendFileSync(RUN_LOG, line); } catch { /* dir may not exist */ }
|
||||
process.stderr.write(line);
|
||||
}
|
||||
|
||||
interface Args {
|
||||
mode: 'dry-run' | 'execute';
|
||||
candidateIds: string[];
|
||||
heldOutCount: number;
|
||||
}
|
||||
function parseArgs(argv: string[]): Args {
|
||||
let mode: Args['mode'] = 'execute';
|
||||
let candidateIds: string[] = [];
|
||||
let heldOutCount = DEFAULT_HELD_OUT;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const f = argv[i];
|
||||
if (f === '--dry-run') mode = 'dry-run';
|
||||
else if (f === '--candidates' && i + 1 < argv.length) {
|
||||
candidateIds = argv[i + 1].split(',').map(s => s.trim()).filter(Boolean);
|
||||
i++;
|
||||
} else if (f === '--held-out-instances' && i + 1 < argv.length) {
|
||||
heldOutCount = parseInt(argv[i + 1], 10);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (candidateIds.length === 0) {
|
||||
throw new Error('--candidates flag required (comma-separated candidate IDs e.g. claude::gen1-v1,qwen-thinking::gen1-v1,gpt::gen1-v2)');
|
||||
}
|
||||
return { mode, candidateIds, heldOutCount };
|
||||
}
|
||||
|
||||
// ── Mulberry32 sampling (same as Gen 1 / NULL-baseline) ──────────────────
|
||||
|
||||
function mulberry32(seed: number): () => number {
|
||||
let t = seed >>> 0;
|
||||
return () => {
|
||||
t = (t + 0x6d2b79f5) >>> 0;
|
||||
let r = t;
|
||||
r = Math.imul(r ^ (r >>> 15), r | 1);
|
||||
r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
|
||||
return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
function deterministicShuffle<T>(items: ReadonlyArray<T>, seed: number): T[] {
|
||||
const arr = [...items];
|
||||
const rand = mulberry32(seed);
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rand() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
function loadCorpus(): CorpusInstance[] {
|
||||
return fs.readFileSync(CORPUS_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean).map(l => JSON.parse(l));
|
||||
}
|
||||
|
||||
// ── Candidate loader (subset by ID) ────────────────────────────────────────
|
||||
|
||||
interface Candidate {
|
||||
candidateId: string;
|
||||
shape: ShapeName;
|
||||
variant: 'baseline' | 'gen1-v1' | 'gen1-v2';
|
||||
promptShape: PromptShape;
|
||||
}
|
||||
|
||||
async function loadCandidatesById(candidateIds: string[]): Promise<Candidate[]> {
|
||||
const out: Candidate[] = [];
|
||||
for (const id of candidateIds) {
|
||||
// Format: <shape>::<variant>
|
||||
const m = id.match(/^([a-z-]+)::([a-z0-9-]+)$/);
|
||||
if (!m) throw new Error(`Invalid candidate ID format "${id}" — expected "<shape>::<variant>" (e.g., "claude::gen1-v1")`);
|
||||
const shape = m[1] as ShapeName;
|
||||
const variant = m[2] as Candidate['variant'];
|
||||
|
||||
if (variant === 'baseline') {
|
||||
const baseline = REGISTRY[shape];
|
||||
if (!baseline) throw new Error(`baseline shape "${shape}" not in REGISTRY`);
|
||||
out.push({ candidateId: id, shape, variant: 'baseline', promptShape: baseline });
|
||||
continue;
|
||||
}
|
||||
|
||||
const filename = `${shape}-${variant}.ts`;
|
||||
const filepath = path.join(GEPA_EVOLVED_DIR, filename);
|
||||
if (!fs.existsSync(filepath)) {
|
||||
throw new Error(`mutation file missing: ${filepath}`);
|
||||
}
|
||||
const mod: any = await import(pathToFileURL(filepath).href);
|
||||
const promptShape = Object.values(mod).find(
|
||||
(v: any) => v && typeof v === 'object' && 'name' in v && 'systemPrompt' in v && 'soloUserPrompt' in v,
|
||||
) as PromptShape | undefined;
|
||||
if (!promptShape) throw new Error(`no PromptShape export found in ${filepath}`);
|
||||
out.push({ candidateId: id, shape, variant, promptShape });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── LLM call adapter (same as Gen 1) ──────────────────────────────────────
|
||||
|
||||
const llmCall: LlmCallFn = async (input: LlmCallInput): Promise<AgentLlmCallResult> => {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY;
|
||||
if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
|
||||
const { model, messages } = input;
|
||||
const isQwen = model.includes('qwen');
|
||||
const maxTokens = input.maxTokens ?? (isQwen ? SUBJECT_MAX_TOKENS : 4096);
|
||||
const thinking = input.thinking ?? (isQwen ? SUBJECT_THINKING : true);
|
||||
const payload: Record<string, unknown> = { model, messages, max_tokens: maxTokens };
|
||||
if (model.startsWith('claude-opus')) payload.temperature = 1.0;
|
||||
else if (model === 'gpt-5.4' || model === 'minimax-m27-via-openrouter') {/* omit */}
|
||||
else payload.temperature = input.temperature ?? 0.3;
|
||||
if (isQwen) payload.extra_body = { enable_thinking: thinking };
|
||||
|
||||
const started = Date.now();
|
||||
let lastErr: string | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const d: any = await resp.json();
|
||||
if ('error' in d) {
|
||||
lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
const content = d.choices?.[0]?.message?.content ?? '';
|
||||
const usage = d.usage ?? {};
|
||||
const inTok = usage.prompt_tokens ?? 0;
|
||||
const outTok = usage.completion_tokens ?? 0;
|
||||
const pricing = MODEL_PRICING[model] ?? { in: 1, out: 4 };
|
||||
return { content, inTokens: inTok, outTokens: outTok, costUsd: (inTok*pricing.in + outTok*pricing.out)/1_000_000, latencyMs: Date.now()-started };
|
||||
} catch (e) {
|
||||
lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
}
|
||||
}
|
||||
return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now()-started, error: lastErr };
|
||||
};
|
||||
|
||||
// ── Trio judging (same as Gen 1) ──────────────────────────────────────────
|
||||
|
||||
const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent's response to a complex knowledge work task. The persona, scenario, materials, and question are provided. The response was generated under one of four configurations (revealed only after scoring): {model_only | model + memory + agent harness} × {Opus 4.7 | Qwen 3.6 35B-A3B}.
|
||||
|
||||
You do NOT know which configuration produced this response. Score blind.
|
||||
|
||||
Read the persona/scenario/question (provided), skim the materials (provided), then read the response carefully (provided).
|
||||
|
||||
Score the response on six dimensions, Likert 1-5:
|
||||
|
||||
1. COMPLETENESS — engagement with all material
|
||||
2. ACCURACY — faithfulness to source materials, no hallucinations
|
||||
3. SYNTHESIS — connections across inputs, not isolated treatment
|
||||
4. JUDGMENT — defensible recommendations, tradeoffs acknowledged
|
||||
5. ACTIONABILITY — would the persona act on this tomorrow
|
||||
6. STRUCTURE — organization and readability
|
||||
|
||||
Output JSON only:
|
||||
{"completeness":<1-5>,"accuracy":<1-5>,"synthesis":<1-5>,"judgment":<1-5>,"actionability":<1-5>,"structure":<1-5>,"rationale":"<1-2 sentences explaining the lowest scoring dimension>","overall_verdict":"<one of: PASS_STRONG | PASS_ADEQUATE | FAIL_WEAK | FAIL_CRITICAL>"}
|
||||
|
||||
PASS_STRONG: mean >= 4.0
|
||||
PASS_ADEQUATE: mean 3.5-3.99
|
||||
FAIL_WEAK: mean 2.5-3.49
|
||||
FAIL_CRITICAL: mean < 2.5
|
||||
|
||||
[PERSONA + SCENARIO + QUESTION]
|
||||
###PSQ###
|
||||
|
||||
[MATERIALS]
|
||||
###MAT###
|
||||
|
||||
[RESPONSE TO EVALUATE]
|
||||
###RES###`;
|
||||
|
||||
interface JudgeRecord { judge_model: string; mean: number; cost: number; latency_ms: number; raw: any; retries: number }
|
||||
interface TrioResult { records: JudgeRecord[]; trioMean: number; trioStrictPassII: boolean; trioStrictPassI: boolean; cost: number }
|
||||
|
||||
function parseJudgeJson(text: string): { mean: number; raw: any } | null {
|
||||
const m = text.match(/\{[\s\S]*\}/);
|
||||
if (!m) return null;
|
||||
try {
|
||||
const obj = JSON.parse(m[0]);
|
||||
const dims = ['completeness','accuracy','synthesis','judgment','actionability','structure'];
|
||||
for (const d of dims) if (typeof obj[d] !== 'number' || obj[d] < 1 || obj[d] > 5) return null;
|
||||
const mean = dims.reduce((s, d) => s + obj[d], 0) / dims.length;
|
||||
return { mean, raw: obj };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
async function runJudge(model: string, prompt: string): Promise<JudgeRecord> {
|
||||
let totalCost = 0, totalLat = 0;
|
||||
for (let attempt = 0; attempt < JUDGE_RETRIES; attempt++) {
|
||||
const r = await llmCall({ model, messages: [{ role: 'user', content: prompt }], maxTokens: JUDGE_MAX_TOKENS, thinking: false });
|
||||
totalCost += r.costUsd; totalLat += r.latencyMs;
|
||||
if (r.error) continue;
|
||||
const parsed = parseJudgeJson(r.content);
|
||||
if (parsed) return { judge_model: model, mean: parsed.mean, cost: totalCost, latency_ms: totalLat, raw: parsed.raw, retries: attempt };
|
||||
}
|
||||
return { judge_model: model, mean: 0, cost: totalCost, latency_ms: totalLat, raw: null, retries: JUDGE_RETRIES };
|
||||
}
|
||||
|
||||
async function judgeTrio(instance: CorpusInstance, response: string): Promise<TrioResult> {
|
||||
const prompt = JUDGE_PROMPT_TEMPLATE
|
||||
.replace('###PSQ###', `${instance.personaText}\n\nQUESTION: ${instance.question}`)
|
||||
.replace('###MAT###', instance.materialsConcat)
|
||||
.replace('###RES###', response);
|
||||
const records = await Promise.all(JUDGES.map(j => runJudge(j, prompt)));
|
||||
const valid = records.filter(r => r.mean > 0).map(r => r.mean);
|
||||
const trioMean = valid.length > 0 ? valid.reduce((a,b)=>a+b,0)/valid.length : 0;
|
||||
return {
|
||||
records,
|
||||
trioMean,
|
||||
trioStrictPassII: trioMean >= 4.0,
|
||||
trioStrictPassI: records.filter(r => r.mean >= 3.5).length >= 2,
|
||||
cost: records.reduce((s,r) => s + r.cost, 0),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Per-eval orchestration (mirrors Gen 1; uses registerShape for canonical injection) ──
|
||||
|
||||
interface EvalRecord {
|
||||
shape: ShapeName;
|
||||
candidateId: string;
|
||||
variant: Candidate['variant'];
|
||||
instanceId: string;
|
||||
instanceCell: CorpusInstance['cell'];
|
||||
candidateResponse: string;
|
||||
candidateLatencyMs: number;
|
||||
candidateCostUsd: number;
|
||||
loopExhausted: boolean;
|
||||
stepsTaken: number;
|
||||
retrievalCalls: number;
|
||||
trioMean: number;
|
||||
trioStrictPassII: boolean;
|
||||
trioStrictPassI: boolean;
|
||||
judgeRecords: JudgeRecord[];
|
||||
evalCostUsd: number;
|
||||
manifestAnchor: string;
|
||||
manifestShaAmendment11: string;
|
||||
tsIso: string;
|
||||
evalSet: 'held-out'; // marks this record as Checkpoint C held-out (vs Gen 1 in-sample)
|
||||
}
|
||||
|
||||
async function runOneEval(cand: Candidate, instance: CorpusInstance, embedder: Embedder): Promise<EvalRecord | { error: string }> {
|
||||
const evalId = `${cand.candidateId}__${instance.instanceId}`;
|
||||
log(`[${evalId}] start`);
|
||||
const dbPath = path.join(SCRATCH_DIR, `eval-${cand.candidateId.replace(/[:]/g, '_')}-${instance.instanceId}.sqlite`);
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
const db = new MindDB(dbPath);
|
||||
const frames = new FrameStore(db);
|
||||
const sessions = new SessionStore(db);
|
||||
const hybrid = new HybridSearch(db, embedder);
|
||||
const gopId = `cp-c-${cand.candidateId.replace(/[:]/g,'_')}-${instance.instanceId}`;
|
||||
sessions.ensure(gopId, undefined, `Checkpoint C ${cand.candidateId} on ${instance.instanceId}`);
|
||||
for (const doc of instance.sourceDocuments) frames.createIFrame(gopId, `## ${doc.title}\n\n${doc.body}`, 'important', 'system');
|
||||
|
||||
const search: RetrievalSearchFn = async ({ query, limit }) => {
|
||||
const hits = await hybrid.search(query, { limit, gopId });
|
||||
return {
|
||||
formattedResults: hits.length > 0 ? hits.map((s,i)=>`[result ${i+1}, score ${s.finalScore.toFixed(3)}]\n${s.frame.content}`).join('\n\n---\n\n') : '',
|
||||
resultCount: hits.length,
|
||||
};
|
||||
};
|
||||
|
||||
let agentResult: AgentRunResult;
|
||||
try {
|
||||
// Amendment 8 §canonical_mutation_api: register via @waggle/agent's registerShape
|
||||
registerShape(cand.promptShape.name, cand.promptShape);
|
||||
agentResult = await runRetrievalAgentLoop({
|
||||
modelAlias: SUBJECT_ALIAS,
|
||||
persona: instance.personaText,
|
||||
question: instance.question,
|
||||
llmCall,
|
||||
search,
|
||||
maxSteps: MAX_STEPS,
|
||||
maxRetrievalsPerStep: MAX_RETRIEVALS_PER_STEP,
|
||||
perCallHaltUsd: PER_CALL_HALT_USD,
|
||||
perCellHaltUsd: PER_CELL_HALT_USD,
|
||||
contextTag: evalId,
|
||||
promptShapeOverride: cand.promptShape.name,
|
||||
} as any);
|
||||
} catch (e) {
|
||||
return { error: `agent loop failed: ${(e as Error).message}` };
|
||||
}
|
||||
|
||||
log(`[${evalId}] subject_done; retrievals=${agentResult.retrievalCalls} steps=${agentResult.stepsTaken} cost=$${agentResult.totalCostUsd.toFixed(4)}`);
|
||||
const judges = await judgeTrio(instance, agentResult.rawResponse);
|
||||
const evalCostUsd = agentResult.totalCostUsd + judges.cost;
|
||||
log(`[${evalId}] judged; trio_mean=${judges.trioMean.toFixed(3)} pass_ii=${judges.trioStrictPassII} retrievals=${agentResult.retrievalCalls} eval_cost=$${evalCostUsd.toFixed(4)}`);
|
||||
|
||||
return {
|
||||
shape: cand.shape, candidateId: cand.candidateId, variant: cand.variant,
|
||||
instanceId: instance.instanceId, instanceCell: instance.cell,
|
||||
candidateResponse: agentResult.rawResponse,
|
||||
candidateLatencyMs: agentResult.totalLatencyMs,
|
||||
candidateCostUsd: agentResult.totalCostUsd,
|
||||
loopExhausted: agentResult.loopExhausted,
|
||||
stepsTaken: agentResult.stepsTaken,
|
||||
retrievalCalls: agentResult.retrievalCalls,
|
||||
trioMean: judges.trioMean,
|
||||
trioStrictPassII: judges.trioStrictPassII,
|
||||
trioStrictPassI: judges.trioStrictPassI,
|
||||
judgeRecords: judges.records,
|
||||
evalCostUsd,
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment11: MANIFEST_SHA_AMENDMENT_11,
|
||||
tsIso: new Date().toISOString(),
|
||||
evalSet: 'held-out',
|
||||
};
|
||||
}
|
||||
|
||||
// ── In-sample lookup from Gen 1 JSONL ─────────────────────────────────────
|
||||
|
||||
interface InSampleStats {
|
||||
candidateId: string;
|
||||
shape: ShapeName;
|
||||
evalCount: number;
|
||||
passIICount: number;
|
||||
passIIRate: number;
|
||||
meanRetrieval: number;
|
||||
}
|
||||
|
||||
function loadInSampleStats(candidateIds: string[]): Map<string, InSampleStats> {
|
||||
const out = new Map<string, InSampleStats>();
|
||||
if (!fs.existsSync(GEN_1_JSONL)) {
|
||||
log(`[in-sample] WARN: Gen 1 JSONL not found at ${GEN_1_JSONL}; in-sample stats unavailable`);
|
||||
return out;
|
||||
}
|
||||
const records: EvalRecord[] = [];
|
||||
for (const line of fs.readFileSync(GEN_1_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
|
||||
try { records.push(JSON.parse(line) as EvalRecord); } catch { /* skip */ }
|
||||
}
|
||||
for (const id of candidateIds) {
|
||||
const candRecs = records.filter(r => r.candidateId === id);
|
||||
if (candRecs.length === 0) continue;
|
||||
const passII = candRecs.filter(r => r.trioStrictPassII).length;
|
||||
const totalRetr = candRecs.reduce((s, r) => s + r.retrievalCalls, 0);
|
||||
out.set(id, {
|
||||
candidateId: id,
|
||||
shape: candRecs[0].shape,
|
||||
evalCount: candRecs.length,
|
||||
passIICount: passII,
|
||||
passIIRate: passII / candRecs.length,
|
||||
meanRetrieval: totalRetr / candRecs.length,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Summary writer ─────────────────────────────────────────────────────────
|
||||
|
||||
interface PerCandidateSummary {
|
||||
candidateId: string;
|
||||
shape: ShapeName;
|
||||
variant: string;
|
||||
inSample: { evalCount: number; passIIRate: number; meanRetrieval: number } | null;
|
||||
heldOut: { evalCount: number; passIIRate: number; meanRetrieval: number; tieredFitness: TieredFitnessComponents };
|
||||
passIIGapPP: number; // (in-sample - held-out) × 100; positive = held-out worse than in-sample
|
||||
retrievalGapAbsolute: number; // (in-sample - held-out); positive = held-out lower retrieval
|
||||
f5_condition_2_verdict: 'PASS' | 'FAIL';
|
||||
f5_condition_2_detail: string;
|
||||
phase_5_deployment_authorized: boolean;
|
||||
}
|
||||
|
||||
interface CheckpointCSummary {
|
||||
manifestAnchor: string;
|
||||
manifestShaAmendment11: string;
|
||||
generated_at: string;
|
||||
candidateIds: string[];
|
||||
heldOutInstanceIds: string[];
|
||||
totalEvals: number;
|
||||
totalCostUsd: number;
|
||||
perCandidate: PerCandidateSummary[];
|
||||
f2_verdict_confirmation: 'CONFIRMED' | 'REVERTED' | 'MIXED';
|
||||
f2_verdict_detail: string;
|
||||
f5_overfitting_bound_pp: number;
|
||||
faza_2_deployment_authorization: 'AUTHORIZED' | 'WITHHELD' | 'PARTIAL';
|
||||
next_steps: string[];
|
||||
}
|
||||
|
||||
function buildCheckpointCSummary(
|
||||
args: Args,
|
||||
candidates: Candidate[],
|
||||
heldOutInstances: CorpusInstance[],
|
||||
recordsByCandidate: Map<string, EvalRecord[]>,
|
||||
inSampleStats: Map<string, InSampleStats>,
|
||||
totalCost: number,
|
||||
): CheckpointCSummary {
|
||||
const perCandidate: PerCandidateSummary[] = [];
|
||||
for (const cand of candidates) {
|
||||
const recs = recordsByCandidate.get(cand.candidateId) ?? [];
|
||||
if (recs.length === 0) continue;
|
||||
const passIICount = recs.filter(r => r.trioStrictPassII).length;
|
||||
const passIIRate = passIICount / recs.length;
|
||||
const meanRetrieval = recs.reduce((s, r) => s + r.retrievalCalls, 0) / recs.length;
|
||||
|
||||
const inSample = inSampleStats.get(cand.candidateId);
|
||||
const passIIGapPP = inSample ? (inSample.passIIRate - passIIRate) * 100 : 0;
|
||||
const retrievalGapAbsolute = inSample ? (inSample.meanRetrieval - meanRetrieval) : 0;
|
||||
|
||||
const candidateMetrics = {
|
||||
candidateId: cand.candidateId,
|
||||
shape: cand.shape,
|
||||
evaluations: [],
|
||||
trioStrictPassRateII: passIIRate,
|
||||
trioStrictPassRateI: 0,
|
||||
meanRetrievalCallsPerTask: meanRetrieval,
|
||||
meanCostUsd: recs.reduce((s, r) => s + r.evalCostUsd, 0) / recs.length,
|
||||
};
|
||||
const tieredFitness = computeTieredFitness({
|
||||
candidate: candidateMetrics,
|
||||
nullBaselinePassRateII: NULL_BASELINE_PER_SHAPE[cand.shape].trioStrictPassRateII,
|
||||
nullBaselineMeanRetrievalCallsPerTask: NULL_BASELINE_PER_SHAPE[cand.shape].meanRetrievalCallsPerTask,
|
||||
mutationValidatorPassed: true, // all candidates validated upstream
|
||||
saturatedRegime: true,
|
||||
});
|
||||
|
||||
// §F.5 condition_2 verdict: held-out Pass II within ±15pp of in-sample
|
||||
let f5_condition_2_verdict: 'PASS' | 'FAIL' = 'PASS';
|
||||
let f5_condition_2_detail = '';
|
||||
if (!inSample) {
|
||||
f5_condition_2_verdict = 'FAIL';
|
||||
f5_condition_2_detail = 'in-sample stats unavailable (Gen 1 JSONL missing or candidate not in Gen 1)';
|
||||
} else {
|
||||
const absGapPP = Math.abs(passIIGapPP);
|
||||
if (absGapPP <= F5_OVERFITTING_BOUND_PP) {
|
||||
f5_condition_2_verdict = 'PASS';
|
||||
f5_condition_2_detail = `held-out ${(passIIRate * 100).toFixed(1)}% within ±${F5_OVERFITTING_BOUND_PP}pp of in-sample ${(inSample.passIIRate * 100).toFixed(1)}% (gap=${passIIGapPP.toFixed(1)}pp)`;
|
||||
} else {
|
||||
f5_condition_2_verdict = 'FAIL';
|
||||
f5_condition_2_detail = `held-out ${(passIIRate * 100).toFixed(1)}% diverges from in-sample ${(inSample.passIIRate * 100).toFixed(1)}% by ${absGapPP.toFixed(1)}pp > ${F5_OVERFITTING_BOUND_PP}pp threshold`;
|
||||
}
|
||||
}
|
||||
|
||||
perCandidate.push({
|
||||
candidateId: cand.candidateId,
|
||||
shape: cand.shape,
|
||||
variant: cand.variant,
|
||||
inSample: inSample ? { evalCount: inSample.evalCount, passIIRate: inSample.passIIRate, meanRetrieval: inSample.meanRetrieval } : null,
|
||||
heldOut: { evalCount: recs.length, passIIRate, meanRetrieval, tieredFitness },
|
||||
passIIGapPP,
|
||||
retrievalGapAbsolute,
|
||||
f5_condition_2_verdict,
|
||||
f5_condition_2_detail,
|
||||
phase_5_deployment_authorized: f5_condition_2_verdict === 'PASS',
|
||||
});
|
||||
}
|
||||
|
||||
// §F.2 confirmation: if all candidates PASS §F.5 condition_2 → CONFIRMED;
|
||||
// if all FAIL → REVERTED; else MIXED
|
||||
const passCount = perCandidate.filter(c => c.f5_condition_2_verdict === 'PASS').length;
|
||||
const total = perCandidate.length;
|
||||
let f2_verdict_confirmation: 'CONFIRMED' | 'REVERTED' | 'MIXED';
|
||||
let f2_verdict_detail = '';
|
||||
if (passCount === total) {
|
||||
f2_verdict_confirmation = 'CONFIRMED';
|
||||
f2_verdict_detail = `all ${total} held-out candidates PASS §F.5 condition_2 (within ±${F5_OVERFITTING_BOUND_PP}pp)`;
|
||||
} else if (passCount === 0) {
|
||||
f2_verdict_confirmation = 'REVERTED';
|
||||
f2_verdict_detail = `all ${total} held-out candidates FAIL §F.5 condition_2 — Gen 1 §F.2 PASS suspected overfit`;
|
||||
} else {
|
||||
f2_verdict_confirmation = 'MIXED';
|
||||
f2_verdict_detail = `${passCount}/${total} held-out candidates PASS §F.5 condition_2`;
|
||||
}
|
||||
|
||||
let faza_2_deployment_authorization: 'AUTHORIZED' | 'WITHHELD' | 'PARTIAL';
|
||||
if (f2_verdict_confirmation === 'CONFIRMED') faza_2_deployment_authorization = 'AUTHORIZED';
|
||||
else if (f2_verdict_confirmation === 'REVERTED') faza_2_deployment_authorization = 'WITHHELD';
|
||||
else faza_2_deployment_authorization = 'PARTIAL';
|
||||
|
||||
const next_steps = [
|
||||
'Compute κ_trio on combined Gen 1 (120) + Checkpoint C (15) sample for §F.3 verdict',
|
||||
`Author Faza 1 final summary memo per ${faza_2_deployment_authorization} authorization status`,
|
||||
'PM ratify final Faza 1 closure decision (decisions/2026-04-XX-gepa-faza1-results.md)',
|
||||
];
|
||||
|
||||
return {
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment11: MANIFEST_SHA_AMENDMENT_11,
|
||||
generated_at: new Date().toISOString(),
|
||||
candidateIds: args.candidateIds,
|
||||
heldOutInstanceIds: heldOutInstances.map(i => i.instanceId),
|
||||
totalEvals: perCandidate.reduce((s, c) => s + c.heldOut.evalCount, 0),
|
||||
totalCostUsd: totalCost,
|
||||
perCandidate,
|
||||
f2_verdict_confirmation,
|
||||
f2_verdict_detail,
|
||||
f5_overfitting_bound_pp: F5_OVERFITTING_BOUND_PP,
|
||||
faza_2_deployment_authorization,
|
||||
next_steps,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
fs.mkdirSync(SCRATCH_DIR, { recursive: true });
|
||||
if (!fs.existsSync(RUN_LOG)) fs.writeFileSync(RUN_LOG, '');
|
||||
|
||||
const corpus = loadCorpus();
|
||||
const allShuffled = deterministicShuffle(corpus, SAMPLING_SEED);
|
||||
const heldOutSample = allShuffled.slice(N_GEN_1_SAMPLE, N_GEN_1_SAMPLE + args.heldOutCount);
|
||||
log(`[loaded] corpus=${corpus.length}; held-out sample=${args.heldOutCount} via seed=${SAMPLING_SEED} offset=${N_GEN_1_SAMPLE}`);
|
||||
|
||||
const candidates = await loadCandidatesById(args.candidateIds);
|
||||
log(`[loaded] ${candidates.length} candidates: ${candidates.map(c => c.candidateId).join(', ')}`);
|
||||
|
||||
// Pre-validate mutation candidates
|
||||
const TYPES_FILE_PATH = path.join(PROMPT_SHAPES_DIR, 'types.ts');
|
||||
for (const cand of candidates) {
|
||||
if (cand.variant === 'baseline') continue;
|
||||
const filename = `${cand.shape}-${cand.variant}.ts`;
|
||||
const candPath = path.join(GEPA_EVOLVED_DIR, filename);
|
||||
try {
|
||||
const verdict = validateCandidate({
|
||||
candidateShapeFilePath: candPath,
|
||||
baselineShapeName: `${cand.shape}.ts`,
|
||||
typesFilePath: TYPES_FILE_PATH,
|
||||
expectShapeDiff: true,
|
||||
});
|
||||
log(`[validator] ${cand.candidateId} valid=${verdict.valid} violations=${verdict.violations.length}`);
|
||||
} catch (e) {
|
||||
log(`[validator] ${cand.candidateId} ERROR ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (args.mode === 'dry-run') {
|
||||
log(`[dry-run] would run ${candidates.length} candidates × ${heldOutSample.length} instances = ${candidates.length * heldOutSample.length} evals`);
|
||||
for (const cand of candidates) {
|
||||
for (const inst of heldOutSample) {
|
||||
log(`[dry-run] ${cand.candidateId}__${inst.instanceId}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume support
|
||||
const existing = new Set<string>();
|
||||
let cumulativeCost = 0;
|
||||
if (fs.existsSync(OUT_JSONL)) {
|
||||
for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
|
||||
try {
|
||||
const r = JSON.parse(line) as EvalRecord;
|
||||
existing.add(`${r.candidateId}__${r.instanceId}`);
|
||||
cumulativeCost += r.evalCostUsd;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
log(`[resume] loaded ${existing.size} existing evals; cumulative $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
const out = fs.createWriteStream(OUT_JSONL, { flags: existing.size > 0 ? 'a' : 'w' });
|
||||
const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });
|
||||
|
||||
const recordsByCandidate = new Map<string, EvalRecord[]>();
|
||||
for (const cand of candidates) recordsByCandidate.set(cand.candidateId, []);
|
||||
|
||||
// Re-load existing records into recordsByCandidate
|
||||
if (fs.existsSync(OUT_JSONL)) {
|
||||
for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
|
||||
try {
|
||||
const r = JSON.parse(line) as EvalRecord;
|
||||
const list = recordsByCandidate.get(r.candidateId);
|
||||
if (list) list.push(r);
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
|
||||
let nDone = existing.size;
|
||||
outer: for (const cand of candidates) {
|
||||
for (const inst of heldOutSample) {
|
||||
const key = `${cand.candidateId}__${inst.instanceId}`;
|
||||
if (existing.has(key)) { log(`[skip] ${key} already in JSONL`); continue; }
|
||||
if (cumulativeCost >= COST_HALT_USD) { log(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD}`); break outer; }
|
||||
const r = await runOneEval(cand, inst, embedder);
|
||||
if ('error' in r) { log(`[skip] ${key}: ${r.error}`); continue; }
|
||||
out.write(JSON.stringify(r) + '\n');
|
||||
cumulativeCost += r.evalCostUsd;
|
||||
nDone++;
|
||||
const list = recordsByCandidate.get(cand.candidateId);
|
||||
if (list) list.push(r);
|
||||
log(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_HALT_USD} halt; ${nDone} evals total`);
|
||||
}
|
||||
}
|
||||
out.end();
|
||||
|
||||
// Write summary
|
||||
const inSampleStats = loadInSampleStats(args.candidateIds);
|
||||
const summary = buildCheckpointCSummary(args, candidates, heldOutSample, recordsByCandidate, inSampleStats, cumulativeCost);
|
||||
fs.writeFileSync(SUMMARY_JSON, JSON.stringify(summary, null, 2));
|
||||
log(`[summary] wrote ${SUMMARY_JSON}`);
|
||||
log(`[F.2] verdict_confirmation=${summary.f2_verdict_confirmation} (${summary.f2_verdict_detail})`);
|
||||
log(`[F.5] overfitting_bound=±${F5_OVERFITTING_BOUND_PP}pp`);
|
||||
log(`[Faza 2] deployment_authorization=${summary.faza_2_deployment_authorization}`);
|
||||
for (const c of summary.perCandidate) {
|
||||
log(`[F.5] ${c.candidateId}: in-sample=${c.inSample ? (c.inSample.passIIRate * 100).toFixed(1) : 'n/a'}% held-out=${(c.heldOut.passIIRate * 100).toFixed(1)}% gap=${c.passIIGapPP.toFixed(1)}pp verdict=${c.f5_condition_2_verdict}`);
|
||||
}
|
||||
|
||||
log(`[done] ${nDone} evals; total cost $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('FATAL:', e); process.exit(2); });
|
||||
912
benchmarks/gepa/scripts/faza-1/run-gen-1.ts
Normal file
912
benchmarks/gepa/scripts/faza-1/run-gen-1.ts
Normal file
@@ -0,0 +1,912 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — Gen 1 evaluation runner.
|
||||
*
|
||||
* Per launch decision §G step 7+ + manifest v7 §gepa + Amendment 5.
|
||||
*
|
||||
* For each of 5 shapes, evaluate 3 candidates (baseline + 2 mutations) × 8
|
||||
* instances = 120 total evaluations. Same instances as NULL-baseline (seed=42)
|
||||
* for direct shape-vs-shape comparison.
|
||||
*
|
||||
* Halt at:
|
||||
* - 30 evaluations (Checkpoint B per launch decision §E)
|
||||
* - $26 cumulative (cost halt per Amendment 3 + super-linear)
|
||||
* - 2 consecutive cell-semantic violations (per brief §5)
|
||||
*
|
||||
* Mode: MULTI-STEP (retrieval available) per Amendment 5 Ask 1 ratification.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
import {
|
||||
MindDB,
|
||||
FrameStore,
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
createOllamaEmbedder,
|
||||
type Embedder,
|
||||
} from '@waggle/core';
|
||||
import {
|
||||
runRetrievalAgentLoop,
|
||||
type LlmCallFn,
|
||||
type LlmCallInput,
|
||||
type LlmCallResult as AgentLlmCallResult,
|
||||
type RetrievalSearchFn,
|
||||
type AgentRunResult,
|
||||
// Amendment 8 §canonical_mutation_api: REGISTRY + registerShape MUST be imported from
|
||||
// '@waggle/agent' (same path the agent-loop uses internally). Importing via deep
|
||||
// relative paths produces a separate module instance under tsx + Node ESM workspace
|
||||
// resolution → mutations would not propagate. Diagnostic probe + Gen 1 partial
|
||||
// b5avslp51 confirmed empirically.
|
||||
REGISTRY,
|
||||
registerShape,
|
||||
type PromptShape,
|
||||
} from '@waggle/agent';
|
||||
|
||||
import { type CorpusInstance } from '../../src/faza-1/corpus.js';
|
||||
import {
|
||||
NULL_BASELINE_PER_SHAPE,
|
||||
NULL_BASELINE_AGGREGATE,
|
||||
type DeltaFloorVerdict,
|
||||
type TieredFitnessComponents,
|
||||
} from '../../src/faza-1/types.js';
|
||||
import {
|
||||
computeTieredFitness,
|
||||
computeDeltaFloorVerdict,
|
||||
computeTier2RetrievalBonus,
|
||||
} from '../../src/faza-1/fitness.js';
|
||||
import {
|
||||
validateCandidate,
|
||||
type ValidatorVerdict,
|
||||
} from '../../src/faza-1/mutation-validator.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const CORPUS_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl');
|
||||
const PROMPT_SHAPES_DIR = path.join(REPO_ROOT, 'packages/agent/src/prompt-shapes');
|
||||
const GEPA_EVOLVED_DIR = path.join(PROMPT_SHAPES_DIR, 'gepa-evolved');
|
||||
const OUT_DIR = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1');
|
||||
const OUT_JSONL = path.join(OUT_DIR, 'gen-1-eval.jsonl');
|
||||
const RUN_LOG = path.join(OUT_DIR, 'gen-1-run.log');
|
||||
const SUMMARY_JSON = path.join(OUT_DIR, 'gen-1-summary.json');
|
||||
const SCRATCH_DIR = path.join(REPO_ROOT, 'tmp/gepa-faza1-gen-1');
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const EMBEDDER_MODEL = 'nomic-embed-text';
|
||||
|
||||
const SAMPLING_SEED = 42;
|
||||
const N_PER_SHAPE = 8;
|
||||
const N_CANDIDATES_PER_SHAPE = 3; // 1 baseline + 2 mutations
|
||||
|
||||
const SHAPES = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple'] as const;
|
||||
type ShapeName = typeof SHAPES[number];
|
||||
|
||||
const SUBJECT_ALIAS = 'qwen3.6-35b-a3b-via-dashscope-direct';
|
||||
const SUBJECT_MAX_TOKENS = 16000;
|
||||
const SUBJECT_THINKING = true;
|
||||
|
||||
const JUDGES = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
|
||||
const JUDGE_MAX_TOKENS = 3000;
|
||||
const JUDGE_RETRIES = 3;
|
||||
|
||||
const MAX_STEPS = 5;
|
||||
const MAX_RETRIEVALS_PER_STEP = 8;
|
||||
const PER_CALL_HALT_USD = 0.40;
|
||||
const PER_CELL_HALT_USD = 1.00;
|
||||
|
||||
// Halt at 30 evals (Checkpoint B) unless --full
|
||||
const CHECKPOINT_B_HALT_EVALS = 30;
|
||||
const COST_HALT_USD = 26.0; // 30% over $20 NULL projection (Amendment 3 envelope basis)
|
||||
|
||||
const MODEL_PRICING: Record<string, { in: number; out: number }> = {
|
||||
'claude-opus-4-7': { in: 15.0, out: 75.0 },
|
||||
'gpt-5.4': { in: 2.5, out: 10.0 },
|
||||
'minimax-m27-via-openrouter': { in: 0.7, out: 2.8 },
|
||||
'qwen3.6-35b-a3b-via-dashscope-direct': { in: 0.20, out: 0.80 },
|
||||
'qwen3.6-35b-a3b-via-openrouter': { in: 0.6, out: 2.4 },
|
||||
};
|
||||
|
||||
const MANIFEST_ANCHOR = 'manifest-v7-gepa-faza1';
|
||||
const MANIFEST_SHA_AMENDMENT_5 = '062dfc4935aaa89f0b25595c5dc3ce4af06c95c4c261075a1f0226d8af3f3dee';
|
||||
const MANIFEST_SHA_AMENDMENT_6 = '0b55d8e353299594254e1a4a76f26f53014d726315dc6a0e5d6dc1a3a44a368a';
|
||||
const MANIFEST_SHA_AMENDMENT_7 = 'bc0bcf9bd8b0c8344b25e5f8ab15b0475039ba28a1f782ebffe4cc1c4ff7d1de';
|
||||
const MANIFEST_SHA_AMENDMENT_8 = '85858f12f1270da28277dd4d98e454d1dae8ef970537cb8c561f484599c4e2e9';
|
||||
const MANIFEST_SHA_AMENDMENT_9 = '5e3ad831c61beb19ccb4ff42b455b4c3964d830808944d4915189c5e9b1709b8';
|
||||
const MANIFEST_SHA_AMENDMENT_10 = '7fb2fb930670b5a28e417a76c64ca1a556f05afb9cf0761aba9f83f0c5de1c9b';
|
||||
|
||||
// ── Amendment 7 — mid-run halt thresholds (binding) ───────────────────────
|
||||
// Per manifest v7 Amendment 7 §checkpoint_b_tightened.mid_run_halt_thresholds.
|
||||
|
||||
/** Per-eval cost projection from Checkpoint A v2 §E (USD). */
|
||||
const PER_EVAL_COST_PROJECTION_USD = 0.1243;
|
||||
/** Mid-run halt: per-candidate cost overshoot threshold (>25% over projection = >$0.156/eval). */
|
||||
const PER_CANDIDATE_COST_OVERSHOOT_THRESHOLD_USD = PER_EVAL_COST_PROJECTION_USD * 1.25; // 0.155375
|
||||
/** Mid-run halt: count of candidates with overshoot that triggers halt (>3). */
|
||||
const MID_RUN_HALT_OVERSHOOT_CANDIDATE_COUNT = 3;
|
||||
/** Mid-run halt: per-shape variance widens (max-min trio_strict_pass_rate_II range across candidates) >40pp. */
|
||||
const PER_SHAPE_VARIANCE_HALT_PP = 40;
|
||||
/** Minimum evals before per-shape variance check runs (avoid noise on N<3). */
|
||||
const PER_SHAPE_VARIANCE_MIN_EVALS = 3;
|
||||
/**
|
||||
* Minimum Qwen evals before retrieval regression check runs.
|
||||
*
|
||||
* Amendment 10 §10.1 calibration_fix: raised from 3 → 5 based on empirical
|
||||
* evidence from 2 prior halt firings (b5avslp51 sunk + b1t474yqd full Gen 1)
|
||||
* where halt fired on baseline-only data within ±0.10 absolute noise band.
|
||||
* Each candidate must have 5+ evals to enter the per-shape aggregate check;
|
||||
* reduces N=3 binomial-tail noise sensitivity.
|
||||
*/
|
||||
const QWEN_RETRIEVAL_REGRESSION_MIN_EVALS = 5; // Amendment 10 §10.1 (was 3 per Amendment 7)
|
||||
|
||||
function log(msg: string): void {
|
||||
const line = `[${new Date().toISOString()}] ${msg}\n`;
|
||||
try { fs.appendFileSync(RUN_LOG, line); } catch { /* dir may not exist */ }
|
||||
process.stderr.write(line);
|
||||
}
|
||||
|
||||
interface Args {
|
||||
mode: 'dry-run' | 'checkpoint-b' | 'full';
|
||||
}
|
||||
function parseArgs(argv: string[]): Args {
|
||||
let mode: Args['mode'] = 'dry-run';
|
||||
for (const f of argv) {
|
||||
if (f === '--dry-run') mode = 'dry-run';
|
||||
else if (f === '--checkpoint-b') mode = 'checkpoint-b';
|
||||
else if (f === '--full') mode = 'full';
|
||||
}
|
||||
return { mode };
|
||||
}
|
||||
|
||||
// ── Mulberry32 sampling (same as NULL-baseline) ───────────────────────────
|
||||
|
||||
function mulberry32(seed: number): () => number {
|
||||
let t = seed >>> 0;
|
||||
return () => {
|
||||
t = (t + 0x6d2b79f5) >>> 0;
|
||||
let r = t;
|
||||
r = Math.imul(r ^ (r >>> 15), r | 1);
|
||||
r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
|
||||
return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
function deterministicShuffle<T>(items: ReadonlyArray<T>, seed: number): T[] {
|
||||
const arr = [...items];
|
||||
const rand = mulberry32(seed);
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rand() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
function loadCorpus(): CorpusInstance[] {
|
||||
return fs.readFileSync(CORPUS_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean).map(l => JSON.parse(l));
|
||||
}
|
||||
|
||||
// ── Candidate loader (baseline from REGISTRY + 2 mutations dynamic import) ─
|
||||
|
||||
interface Candidate {
|
||||
candidateId: string; // e.g., qwen-thinking::baseline | qwen-thinking::gen1-v1
|
||||
shape: ShapeName;
|
||||
variant: 'baseline' | 'gen1-v1' | 'gen1-v2';
|
||||
promptShape: PromptShape;
|
||||
}
|
||||
|
||||
async function loadCandidates(): Promise<Map<ShapeName, Candidate[]>> {
|
||||
const out = new Map<ShapeName, Candidate[]>();
|
||||
for (const shape of SHAPES) {
|
||||
const cands: Candidate[] = [];
|
||||
const baseline = REGISTRY[shape];
|
||||
if (!baseline) throw new Error(`shape "${shape}" not in REGISTRY`);
|
||||
cands.push({ candidateId: `${shape}::baseline`, shape, variant: 'baseline', promptShape: baseline });
|
||||
|
||||
for (let v = 1; v <= 2; v++) {
|
||||
const filename = `${shape}-gen1-v${v}.ts`;
|
||||
const filepath = path.join(GEPA_EVOLVED_DIR, filename);
|
||||
if (!fs.existsSync(filepath)) {
|
||||
throw new Error(`mutation file missing: ${filepath}`);
|
||||
}
|
||||
// Windows ESM requires file:// URL for absolute paths
|
||||
const mod: any = await import(pathToFileURL(filepath).href);
|
||||
// Find the exported PromptShape (single export per file convention)
|
||||
const promptShape = Object.values(mod).find(
|
||||
(v: any) => v && typeof v === 'object' && 'name' in v && 'systemPrompt' in v && 'soloUserPrompt' in v,
|
||||
) as PromptShape | undefined;
|
||||
if (!promptShape) throw new Error(`no PromptShape export found in ${filepath}`);
|
||||
cands.push({
|
||||
candidateId: `${shape}::gen1-v${v}`, shape,
|
||||
variant: `gen1-v${v}` as 'gen1-v1' | 'gen1-v2', promptShape,
|
||||
});
|
||||
}
|
||||
out.set(shape, cands);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── LLM call adapter (same as NULL-baseline) ──────────────────────────────
|
||||
|
||||
const llmCall: LlmCallFn = async (input: LlmCallInput): Promise<AgentLlmCallResult> => {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY;
|
||||
if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
|
||||
const { model, messages } = input;
|
||||
const isQwen = model.includes('qwen');
|
||||
const maxTokens = input.maxTokens ?? (isQwen ? SUBJECT_MAX_TOKENS : 4096);
|
||||
const thinking = input.thinking ?? (isQwen ? SUBJECT_THINKING : true);
|
||||
const payload: Record<string, unknown> = { model, messages, max_tokens: maxTokens };
|
||||
if (model.startsWith('claude-opus')) payload.temperature = 1.0;
|
||||
else if (model === 'gpt-5.4' || model === 'minimax-m27-via-openrouter') {/* omit */}
|
||||
else payload.temperature = input.temperature ?? 0.3;
|
||||
if (isQwen) payload.extra_body = { enable_thinking: thinking };
|
||||
|
||||
const started = Date.now();
|
||||
let lastErr: string | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const d: any = await resp.json();
|
||||
if ('error' in d) {
|
||||
lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
const content = d.choices?.[0]?.message?.content ?? '';
|
||||
const usage = d.usage ?? {};
|
||||
const inTok = usage.prompt_tokens ?? 0;
|
||||
const outTok = usage.completion_tokens ?? 0;
|
||||
const pricing = MODEL_PRICING[model] ?? { in: 1, out: 4 };
|
||||
return { content, inTokens: inTok, outTokens: outTok, costUsd: (inTok*pricing.in + outTok*pricing.out)/1_000_000, latencyMs: Date.now()-started };
|
||||
} catch (e) {
|
||||
lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
}
|
||||
}
|
||||
return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now()-started, error: lastErr };
|
||||
};
|
||||
|
||||
// ── Trio judging (mirror NULL-baseline) ───────────────────────────────────
|
||||
|
||||
const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent's response to a complex knowledge work task. The persona, scenario, materials, and question are provided. The response was generated under one of four configurations (revealed only after scoring): {model_only | model + memory + agent harness} × {Opus 4.7 | Qwen 3.6 35B-A3B}.
|
||||
|
||||
You do NOT know which configuration produced this response. Score blind.
|
||||
|
||||
Read the persona/scenario/question (provided), skim the materials (provided), then read the response carefully (provided).
|
||||
|
||||
Score the response on six dimensions, Likert 1-5:
|
||||
|
||||
1. COMPLETENESS — engagement with all material
|
||||
2. ACCURACY — faithfulness to source materials, no hallucinations
|
||||
3. SYNTHESIS — connections across inputs, not isolated treatment
|
||||
4. JUDGMENT — defensible recommendations, tradeoffs acknowledged
|
||||
5. ACTIONABILITY — would the persona act on this tomorrow
|
||||
6. STRUCTURE — organization and readability
|
||||
|
||||
Output JSON only:
|
||||
{"completeness":<1-5>,"accuracy":<1-5>,"synthesis":<1-5>,"judgment":<1-5>,"actionability":<1-5>,"structure":<1-5>,"rationale":"<1-2 sentences explaining the lowest scoring dimension>","overall_verdict":"<one of: PASS_STRONG | PASS_ADEQUATE | FAIL_WEAK | FAIL_CRITICAL>"}
|
||||
|
||||
PASS_STRONG: mean >= 4.0
|
||||
PASS_ADEQUATE: mean 3.5-3.99
|
||||
FAIL_WEAK: mean 2.5-3.49
|
||||
FAIL_CRITICAL: mean < 2.5
|
||||
|
||||
[PERSONA + SCENARIO + QUESTION]
|
||||
###PSQ###
|
||||
|
||||
[MATERIALS]
|
||||
###MAT###
|
||||
|
||||
[RESPONSE TO EVALUATE]
|
||||
###RES###`;
|
||||
|
||||
interface JudgeRecord { judge_model: string; mean: number; cost: number; latency_ms: number; raw: any; retries: number }
|
||||
interface TrioResult { records: JudgeRecord[]; trioMean: number; trioStrictPassII: boolean; trioStrictPassI: boolean; cost: number }
|
||||
|
||||
function parseJudgeJson(text: string): { mean: number; raw: any } | null {
|
||||
const m = text.match(/\{[\s\S]*\}/);
|
||||
if (!m) return null;
|
||||
try {
|
||||
const obj = JSON.parse(m[0]);
|
||||
const dims = ['completeness','accuracy','synthesis','judgment','actionability','structure'];
|
||||
for (const d of dims) if (typeof obj[d] !== 'number' || obj[d] < 1 || obj[d] > 5) return null;
|
||||
const mean = dims.reduce((s, d) => s + obj[d], 0) / dims.length;
|
||||
return { mean, raw: obj };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
async function runJudge(model: string, prompt: string): Promise<JudgeRecord> {
|
||||
let totalCost = 0, totalLat = 0;
|
||||
for (let attempt = 0; attempt < JUDGE_RETRIES; attempt++) {
|
||||
const r = await llmCall({ model, messages: [{ role: 'user', content: prompt }], maxTokens: JUDGE_MAX_TOKENS, thinking: false });
|
||||
totalCost += r.costUsd; totalLat += r.latencyMs;
|
||||
if (r.error) continue;
|
||||
const parsed = parseJudgeJson(r.content);
|
||||
if (parsed) return { judge_model: model, mean: parsed.mean, cost: totalCost, latency_ms: totalLat, raw: parsed.raw, retries: attempt };
|
||||
}
|
||||
return { judge_model: model, mean: 0, cost: totalCost, latency_ms: totalLat, raw: null, retries: JUDGE_RETRIES };
|
||||
}
|
||||
|
||||
async function judgeTrio(instance: CorpusInstance, response: string): Promise<TrioResult> {
|
||||
const prompt = JUDGE_PROMPT_TEMPLATE
|
||||
.replace('###PSQ###', `${instance.personaText}\n\nQUESTION: ${instance.question}`)
|
||||
.replace('###MAT###', instance.materialsConcat)
|
||||
.replace('###RES###', response);
|
||||
const records = await Promise.all(JUDGES.map(j => runJudge(j, prompt)));
|
||||
const valid = records.filter(r => r.mean > 0).map(r => r.mean);
|
||||
const trioMean = valid.length > 0 ? valid.reduce((a,b)=>a+b,0)/valid.length : 0;
|
||||
return {
|
||||
records,
|
||||
trioMean,
|
||||
trioStrictPassII: trioMean >= 4.0,
|
||||
trioStrictPassI: records.filter(r => r.mean >= 3.5).length >= 2,
|
||||
cost: records.reduce((s,r) => s + r.cost, 0),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Per-eval orchestration ────────────────────────────────────────────────
|
||||
|
||||
interface EvalRecord {
|
||||
shape: ShapeName;
|
||||
candidateId: string;
|
||||
variant: Candidate['variant'];
|
||||
instanceId: string;
|
||||
instanceCell: CorpusInstance['cell'];
|
||||
candidateResponse: string;
|
||||
candidateLatencyMs: number;
|
||||
candidateCostUsd: number;
|
||||
loopExhausted: boolean;
|
||||
stepsTaken: number;
|
||||
retrievalCalls: number;
|
||||
trioMean: number;
|
||||
trioStrictPassII: boolean;
|
||||
trioStrictPassI: boolean;
|
||||
judgeRecords: JudgeRecord[];
|
||||
evalCostUsd: number;
|
||||
manifestAnchor: string;
|
||||
manifestShaAmendment5: string;
|
||||
tsIso: string;
|
||||
}
|
||||
|
||||
async function runOneEval(cand: Candidate, instance: CorpusInstance, embedder: Embedder): Promise<EvalRecord | { error: string }> {
|
||||
const evalId = `${cand.candidateId}__${instance.instanceId}`;
|
||||
log(`[${evalId}] start`);
|
||||
const dbPath = path.join(SCRATCH_DIR, `eval-${cand.candidateId.replace(/[:]/g, '_')}-${instance.instanceId}.sqlite`);
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
const db = new MindDB(dbPath);
|
||||
const frames = new FrameStore(db);
|
||||
const sessions = new SessionStore(db);
|
||||
const hybrid = new HybridSearch(db, embedder);
|
||||
const gopId = `gen1-${cand.candidateId.replace(/[:]/g,'_')}-${instance.instanceId}`;
|
||||
sessions.ensure(gopId, undefined, `Gen 1 ${cand.candidateId} on ${instance.instanceId}`);
|
||||
for (const doc of instance.sourceDocuments) frames.createIFrame(gopId, `## ${doc.title}\n\n${doc.body}`, 'important', 'system');
|
||||
|
||||
const search: RetrievalSearchFn = async ({ query, limit }) => {
|
||||
const hits = await hybrid.search(query, { limit, gopId });
|
||||
return {
|
||||
formattedResults: hits.length > 0 ? hits.map((s,i)=>`[result ${i+1}, score ${s.finalScore.toFixed(3)}]\n${s.frame.content}`).join('\n\n---\n\n') : '',
|
||||
resultCount: hits.length,
|
||||
};
|
||||
};
|
||||
|
||||
// Inject the candidate's prompt shape via custom orchestration: we use runRetrievalAgentLoop
|
||||
// with the candidate's modelAlias + prompt-shape. The agent loop internally selects shape via
|
||||
// selector; we override by passing the candidate's shape directly. Since runRetrievalAgentLoop
|
||||
// uses selectShape internally, we override REGISTRY at runtime by name match.
|
||||
// For Faza 1 simplicity, we register candidate as override under its unique name:
|
||||
let agentResult: AgentRunResult;
|
||||
try {
|
||||
// Amendment 8 §canonical_mutation_api: register the candidate via the sanctioned
|
||||
// mutation path. registerShape() is imported from '@waggle/agent' so it mutates
|
||||
// the SAME REGISTRY instance the agent-loop's selectShape() reads from. Direct
|
||||
// (REGISTRY as any)[name] = shape is forbidden post-Amendment-8 (would mutate a
|
||||
// separate module instance under tsx + Node ESM workspace resolution).
|
||||
registerShape(cand.promptShape.name, cand.promptShape);
|
||||
agentResult = await runRetrievalAgentLoop({
|
||||
modelAlias: SUBJECT_ALIAS,
|
||||
persona: instance.personaText,
|
||||
question: instance.question,
|
||||
llmCall,
|
||||
search,
|
||||
maxSteps: MAX_STEPS,
|
||||
maxRetrievalsPerStep: MAX_RETRIEVALS_PER_STEP,
|
||||
perCallHaltUsd: PER_CALL_HALT_USD,
|
||||
perCellHaltUsd: PER_CELL_HALT_USD,
|
||||
contextTag: evalId,
|
||||
promptShapeOverride: cand.promptShape.name, // if supported
|
||||
} as any);
|
||||
} catch (e) {
|
||||
return { error: `agent loop failed: ${(e as Error).message}` };
|
||||
}
|
||||
|
||||
log(`[${evalId}] subject_done; retrievals=${agentResult.retrievalCalls} steps=${agentResult.stepsTaken} cost=$${agentResult.totalCostUsd.toFixed(4)}`);
|
||||
const judges = await judgeTrio(instance, agentResult.rawResponse);
|
||||
const evalCostUsd = agentResult.totalCostUsd + judges.cost;
|
||||
log(`[${evalId}] judged; trio_mean=${judges.trioMean.toFixed(3)} pass_ii=${judges.trioStrictPassII} retrievals=${agentResult.retrievalCalls} eval_cost=$${evalCostUsd.toFixed(4)}`);
|
||||
|
||||
return {
|
||||
shape: cand.shape, candidateId: cand.candidateId, variant: cand.variant,
|
||||
instanceId: instance.instanceId, instanceCell: instance.cell,
|
||||
candidateResponse: agentResult.rawResponse,
|
||||
candidateLatencyMs: agentResult.totalLatencyMs,
|
||||
candidateCostUsd: agentResult.totalCostUsd,
|
||||
loopExhausted: agentResult.loopExhausted,
|
||||
stepsTaken: agentResult.stepsTaken,
|
||||
retrievalCalls: agentResult.retrievalCalls,
|
||||
trioMean: judges.trioMean,
|
||||
trioStrictPassII: judges.trioStrictPassII,
|
||||
trioStrictPassI: judges.trioStrictPassI,
|
||||
judgeRecords: judges.records,
|
||||
evalCostUsd,
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment5: MANIFEST_SHA_AMENDMENT_5,
|
||||
tsIso: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// ── Amendment 7 — per-candidate accumulator ───────────────────────────────
|
||||
|
||||
interface CandidateAcc {
|
||||
candidateId: string;
|
||||
shape: ShapeName;
|
||||
variant: 'baseline' | 'gen1-v1' | 'gen1-v2';
|
||||
evalCount: number;
|
||||
passIICount: number; // count of trioStrictPassII = true
|
||||
totalCostUsd: number;
|
||||
totalRetrievalCalls: number;
|
||||
trioMeans: number[]; // per-eval trioMean for variance + audit
|
||||
retrievalCalls: number[]; // per-eval retrieval calls for audit
|
||||
mutationValidatorPassed: boolean; // computed from validateCandidate at startup
|
||||
}
|
||||
|
||||
function makeCandidateAcc(cand: Candidate, validatorPassed: boolean): CandidateAcc {
|
||||
return {
|
||||
candidateId: cand.candidateId,
|
||||
shape: cand.shape,
|
||||
variant: cand.variant,
|
||||
evalCount: 0,
|
||||
passIICount: 0,
|
||||
totalCostUsd: 0,
|
||||
totalRetrievalCalls: 0,
|
||||
trioMeans: [],
|
||||
retrievalCalls: [],
|
||||
mutationValidatorPassed: validatorPassed,
|
||||
};
|
||||
}
|
||||
|
||||
function ingestEvalIntoAcc(acc: CandidateAcc, r: EvalRecord): void {
|
||||
acc.evalCount++;
|
||||
if (r.trioStrictPassII) acc.passIICount++;
|
||||
acc.totalCostUsd += r.evalCostUsd;
|
||||
acc.totalRetrievalCalls += r.retrievalCalls;
|
||||
acc.trioMeans.push(r.trioMean);
|
||||
acc.retrievalCalls.push(r.retrievalCalls);
|
||||
}
|
||||
|
||||
function accMeanCostPerEval(acc: CandidateAcc): number {
|
||||
return acc.evalCount > 0 ? acc.totalCostUsd / acc.evalCount : 0;
|
||||
}
|
||||
function accPassRateII(acc: CandidateAcc): number {
|
||||
return acc.evalCount > 0 ? acc.passIICount / acc.evalCount : 0;
|
||||
}
|
||||
function accMeanRetrievalCallsPerTask(acc: CandidateAcc): number {
|
||||
return acc.evalCount > 0 ? acc.totalRetrievalCalls / acc.evalCount : 0;
|
||||
}
|
||||
|
||||
// ── Amendment 7 — mid-run halt check (binding) ────────────────────────────
|
||||
|
||||
interface MidRunHaltCheckResult {
|
||||
shouldHalt: boolean;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
function checkMidRunHalts(accs: Map<string, CandidateAcc>): MidRunHaltCheckResult {
|
||||
// Threshold A — per-candidate cost overshoot >25% on >3 candidates
|
||||
let overshootCount = 0;
|
||||
const overshootCandidates: string[] = [];
|
||||
for (const acc of accs.values()) {
|
||||
if (acc.evalCount === 0) continue;
|
||||
if (accMeanCostPerEval(acc) > PER_CANDIDATE_COST_OVERSHOOT_THRESHOLD_USD) {
|
||||
overshootCount++;
|
||||
overshootCandidates.push(`${acc.candidateId}=$${accMeanCostPerEval(acc).toFixed(4)}/eval`);
|
||||
}
|
||||
}
|
||||
if (overshootCount > MID_RUN_HALT_OVERSHOOT_CANDIDATE_COUNT) {
|
||||
return {
|
||||
shouldHalt: true,
|
||||
reason: `Amendment 7 §checkpoint_b_tightened.per_candidate_cost_overshoot: ${overshootCount} candidates >$${PER_CANDIDATE_COST_OVERSHOOT_THRESHOLD_USD.toFixed(4)}/eval (threshold >${MID_RUN_HALT_OVERSHOOT_CANDIDATE_COUNT}); offenders=[${overshootCandidates.join(', ')}]`,
|
||||
};
|
||||
}
|
||||
|
||||
// Threshold B — per-shape variance widens >40pp range (max-min trio_strict_pass_rate_II) on any shape
|
||||
for (const shape of SHAPES) {
|
||||
const shapeAccs = [...accs.values()].filter(a => a.shape === shape && a.evalCount >= PER_SHAPE_VARIANCE_MIN_EVALS);
|
||||
if (shapeAccs.length < 2) continue;
|
||||
const passRates = shapeAccs.map(accPassRateII);
|
||||
const max = Math.max(...passRates);
|
||||
const min = Math.min(...passRates);
|
||||
const rangePP = (max - min) * 100;
|
||||
if (rangePP > PER_SHAPE_VARIANCE_HALT_PP) {
|
||||
return {
|
||||
shouldHalt: true,
|
||||
reason: `Amendment 7 §checkpoint_b_tightened.per_shape_variance_widens: shape=${shape} range=${rangePP.toFixed(1)}pp > ${PER_SHAPE_VARIANCE_HALT_PP}pp; rates=${passRates.map(r => r.toFixed(2)).join(',')}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Threshold C — Qwen-targeted retrieval engagement drops below per-shape NULL baseline
|
||||
//
|
||||
// Amendment 10 §10.1 mutation_execution_gate (binding): halt only fires when at
|
||||
// least one mutation candidate has been evaluated for this shape. Baseline-only
|
||||
// data does NOT trigger halt. This matches Amendment 9 §qwen_evolution_verdict_capture
|
||||
// .mid_run_halt_binding intent that the halt represents direction_2 verdict
|
||||
// (mutations regress retrieval), not baseline-running stochastic variance.
|
||||
// Empirical basis: 2 prior halt firings (b5avslp51 + b1t474yqd) on baseline-only
|
||||
// data within ±0.10 absolute noise band were both per-Amendment-9 NOT direction_2
|
||||
// verdicts.
|
||||
for (const shape of ['qwen-thinking', 'qwen-non-thinking'] as const) {
|
||||
const allShapeAccs = [...accs.values()].filter(a => a.shape === shape);
|
||||
|
||||
// Amendment 11 §11.1 second_order_calibration_patch (binding):
|
||||
// mutation_execution_gate threshold tightened from ≥1 eval to ≥MIN_EVALS evals.
|
||||
// Halt only fires when at least one mutation candidate (variant !== 'baseline')
|
||||
// has STATISTICALLY MEANINGFUL sample size (≥QWEN_RETRIEVAL_REGRESSION_MIN_EVALS=5
|
||||
// evals). This guarantees the mutation IS in the per-shape aggregate (not
|
||||
// excluded by the individual-candidate MIN_EVALS filter), eliminating the
|
||||
// second-order false-positive class where halt fired on baseline-only
|
||||
// aggregate while gate was mechanically met by a single mutation eval.
|
||||
//
|
||||
// Per Amendment 11 §11.2 terminal_calibration_clause (BINDING): if halt
|
||||
// fires AGAIN with this calibration ACTIVE, that IS Phase 4.5 direction_2
|
||||
// verdict. No further calibration patches; escalate to Option C (Amendment 12
|
||||
// interface refactor).
|
||||
const hasStatisticallyMeaningfulMutationForShape = allShapeAccs.some(
|
||||
a => a.variant !== 'baseline' && a.evalCount >= QWEN_RETRIEVAL_REGRESSION_MIN_EVALS,
|
||||
);
|
||||
if (!hasStatisticallyMeaningfulMutationForShape) continue;
|
||||
|
||||
const baseline = NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask;
|
||||
const shapeAccs = allShapeAccs.filter(a => a.evalCount >= QWEN_RETRIEVAL_REGRESSION_MIN_EVALS);
|
||||
if (shapeAccs.length === 0) continue;
|
||||
const totalRetr = shapeAccs.reduce((s, a) => s + a.totalRetrievalCalls, 0);
|
||||
const totalEvals = shapeAccs.reduce((s, a) => s + a.evalCount, 0);
|
||||
if (totalEvals === 0) continue;
|
||||
const aggMean = totalRetr / totalEvals;
|
||||
if (aggMean < baseline) {
|
||||
return {
|
||||
shouldHalt: true,
|
||||
reason: `Amendment 7 §checkpoint_b_tightened.qwen_retrieval_engagement_regression (post Amendment 11 §11.1 second_order_calibration_patch): shape=${shape} mean=${aggMean.toFixed(3)} < NULL baseline ${baseline.toFixed(3)} (n=${totalEvals}; ≥1 mutation candidate with ≥${QWEN_RETRIEVAL_REGRESSION_MIN_EVALS} evals evaluated for shape; per Amendment 11 §11.2 terminal_calibration_clause, this IS Phase 4.5 direction_2 verdict)`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { shouldHalt: false, reason: null };
|
||||
}
|
||||
|
||||
// ── Amendment 7 — Checkpoint B summary writer (binding extensions) ────────
|
||||
|
||||
interface PerCandidateTierBreakdown {
|
||||
candidateId: string;
|
||||
shape: ShapeName;
|
||||
variant: string;
|
||||
evalCount: number;
|
||||
trioStrictPassRateII: number;
|
||||
meanRetrievalCallsPerTask: number;
|
||||
meanEvalCostUsd: number;
|
||||
costOvershoot: boolean;
|
||||
tieredFitness: TieredFitnessComponents;
|
||||
}
|
||||
|
||||
interface CheckpointBSummary {
|
||||
manifestAnchor: string;
|
||||
manifestShaAmendment7: string;
|
||||
generated_at: string;
|
||||
mode: string;
|
||||
totalEvals: number;
|
||||
totalCostUsd: number;
|
||||
haltReason: string | null;
|
||||
perCandidateTierBreakdown: PerCandidateTierBreakdown[];
|
||||
retrievalEngagementDeltasPerQwenShape: {
|
||||
'qwen-thinking': { nullBaselineMean: number; gen1PartialMean: number | null; deltaAbsolute: number | null };
|
||||
'qwen-non-thinking': { nullBaselineMean: number; gen1PartialMean: number | null; deltaAbsolute: number | null };
|
||||
};
|
||||
cellSemanticAnchorInvarianceCountPerCandidate: Record<string, number>;
|
||||
preRegisteredDeltaFloorVerdict: DeltaFloorVerdict;
|
||||
midRunHaltsBindingThresholds: {
|
||||
perCandidateCostOvershoot: { threshold: number; candidatesOvershoot: number };
|
||||
perShapeVariance: { thresholdPP: number; maxRangeObservedPP: number };
|
||||
qwenRetrievalRegression: { triggered: boolean; details: string };
|
||||
};
|
||||
}
|
||||
|
||||
function buildCheckpointBSummary(
|
||||
args: ReturnType<typeof parseArgs>,
|
||||
accs: Map<string, CandidateAcc>,
|
||||
totalEvals: number,
|
||||
totalCostUsd: number,
|
||||
haltReason: string | null,
|
||||
): CheckpointBSummary {
|
||||
const perCandidate: PerCandidateTierBreakdown[] = [];
|
||||
for (const acc of accs.values()) {
|
||||
if (acc.evalCount === 0) continue;
|
||||
const passRate = accPassRateII(acc);
|
||||
const meanRetr = accMeanRetrievalCallsPerTask(acc);
|
||||
const meanCost = accMeanCostPerEval(acc);
|
||||
const candidateMetrics = {
|
||||
candidateId: acc.candidateId,
|
||||
shape: acc.shape,
|
||||
evaluations: [],
|
||||
trioStrictPassRateII: passRate,
|
||||
trioStrictPassRateI: 0, // not tracked here; reported in JSONL
|
||||
meanRetrievalCallsPerTask: meanRetr,
|
||||
meanCostUsd: meanCost,
|
||||
};
|
||||
const tieredFitness = computeTieredFitness({
|
||||
candidate: candidateMetrics,
|
||||
nullBaselinePassRateII: NULL_BASELINE_PER_SHAPE[acc.shape].trioStrictPassRateII,
|
||||
nullBaselineMeanRetrievalCallsPerTask: NULL_BASELINE_PER_SHAPE[acc.shape].meanRetrievalCallsPerTask,
|
||||
mutationValidatorPassed: acc.mutationValidatorPassed,
|
||||
saturatedRegime: true, // 5/5 shapes ≥75% per Checkpoint A v2 §B.2
|
||||
});
|
||||
perCandidate.push({
|
||||
candidateId: acc.candidateId,
|
||||
shape: acc.shape,
|
||||
variant: acc.variant,
|
||||
evalCount: acc.evalCount,
|
||||
trioStrictPassRateII: passRate,
|
||||
meanRetrievalCallsPerTask: meanRetr,
|
||||
meanEvalCostUsd: meanCost,
|
||||
costOvershoot: meanCost > PER_CANDIDATE_COST_OVERSHOOT_THRESHOLD_USD,
|
||||
tieredFitness,
|
||||
});
|
||||
}
|
||||
|
||||
// Aggregate Tier 1: mean trio_strict_pass_rate_II across all evals
|
||||
const totalEvalsAcc = perCandidate.reduce((s, c) => s + c.evalCount, 0);
|
||||
const aggregateTrioStrictPassRateII =
|
||||
totalEvalsAcc > 0
|
||||
? perCandidate.reduce((s, c) => s + c.trioStrictPassRateII * c.evalCount, 0) / totalEvalsAcc
|
||||
: 0;
|
||||
|
||||
// Per-shape Qwen retrieval means (across that shape's candidates)
|
||||
function qwenShapeAggregate(shape: 'qwen-thinking' | 'qwen-non-thinking'):
|
||||
{ gen1PartialMean: number | null; deltaAbsolute: number | null } {
|
||||
const shapeAccs = [...accs.values()].filter(a => a.shape === shape && a.evalCount > 0);
|
||||
if (shapeAccs.length === 0) return { gen1PartialMean: null, deltaAbsolute: null };
|
||||
const totalRetr = shapeAccs.reduce((s, a) => s + a.totalRetrievalCalls, 0);
|
||||
const totalEvalsLocal = shapeAccs.reduce((s, a) => s + a.evalCount, 0);
|
||||
const mean = totalEvalsLocal > 0 ? totalRetr / totalEvalsLocal : null;
|
||||
const baseline = NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask;
|
||||
return { gen1PartialMean: mean, deltaAbsolute: mean === null ? null : mean - baseline };
|
||||
}
|
||||
const qwenThinkingAgg = qwenShapeAggregate('qwen-thinking');
|
||||
const qwenNonThinkingAgg = qwenShapeAggregate('qwen-non-thinking');
|
||||
|
||||
const qwenShapeRetrievalMeans: Partial<Record<ShapeName, number>> = {};
|
||||
if (qwenThinkingAgg.gen1PartialMean !== null) qwenShapeRetrievalMeans['qwen-thinking'] = qwenThinkingAgg.gen1PartialMean;
|
||||
if (qwenNonThinkingAgg.gen1PartialMean !== null) qwenShapeRetrievalMeans['qwen-non-thinking'] = qwenNonThinkingAgg.gen1PartialMean;
|
||||
|
||||
// Aggregate Tier 2 bonus across Qwen-targeted candidates (mean across qwen candidates with data)
|
||||
const qwenCandidates = perCandidate.filter(c => c.shape === 'qwen-thinking' || c.shape === 'qwen-non-thinking');
|
||||
const qwenAggregateTier2Bonus =
|
||||
qwenCandidates.length > 0
|
||||
? qwenCandidates.reduce((s, c) => s + c.tieredFitness.tier2RetrievalBonus, 0) / qwenCandidates.length
|
||||
: 0;
|
||||
|
||||
const deltaFloorVerdict = computeDeltaFloorVerdict({
|
||||
aggregateTrioStrictPassRateII,
|
||||
aggregateNullBaselinePassRateII: NULL_BASELINE_AGGREGATE.trioStrictPassRateII,
|
||||
qwenShapeRetrievalMeans,
|
||||
qwenShapeNullBaselineRetrievalMeans: {
|
||||
'qwen-thinking': NULL_BASELINE_PER_SHAPE['qwen-thinking'].meanRetrievalCallsPerTask,
|
||||
'qwen-non-thinking': NULL_BASELINE_PER_SHAPE['qwen-non-thinking'].meanRetrievalCallsPerTask,
|
||||
},
|
||||
qwenAggregateTier2Bonus,
|
||||
});
|
||||
|
||||
// Per-shape variance maxRange snapshot
|
||||
let maxRangeObservedPP = 0;
|
||||
for (const shape of SHAPES) {
|
||||
const shapeAccs = perCandidate.filter(c => c.shape === shape);
|
||||
if (shapeAccs.length < 2) continue;
|
||||
const rates = shapeAccs.map(c => c.trioStrictPassRateII);
|
||||
const range = (Math.max(...rates) - Math.min(...rates)) * 100;
|
||||
if (range > maxRangeObservedPP) maxRangeObservedPP = range;
|
||||
}
|
||||
const overshootCount = perCandidate.filter(c => c.costOvershoot).length;
|
||||
|
||||
// Qwen retrieval regression check (binary informational; halt logic in checkMidRunHalts)
|
||||
let qwenRegressionDetails = 'no_regression';
|
||||
let qwenRegressionTriggered = false;
|
||||
for (const shape of ['qwen-thinking', 'qwen-non-thinking'] as const) {
|
||||
const agg = shape === 'qwen-thinking' ? qwenThinkingAgg : qwenNonThinkingAgg;
|
||||
if (agg.gen1PartialMean !== null && agg.gen1PartialMean < NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask) {
|
||||
qwenRegressionTriggered = true;
|
||||
qwenRegressionDetails = `${shape} mean=${agg.gen1PartialMean.toFixed(3)} < NULL ${NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask}`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const cellSemanticAnchorInvarianceCountPerCandidate: Record<string, number> = {};
|
||||
for (const c of perCandidate) {
|
||||
cellSemanticAnchorInvarianceCountPerCandidate[c.candidateId] = c.tieredFitness.cellSemanticAnchorInvarianceCount;
|
||||
}
|
||||
|
||||
return {
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment7: MANIFEST_SHA_AMENDMENT_7,
|
||||
generated_at: new Date().toISOString(),
|
||||
mode: args.mode,
|
||||
totalEvals,
|
||||
totalCostUsd,
|
||||
haltReason,
|
||||
perCandidateTierBreakdown: perCandidate,
|
||||
retrievalEngagementDeltasPerQwenShape: {
|
||||
'qwen-thinking': {
|
||||
nullBaselineMean: NULL_BASELINE_PER_SHAPE['qwen-thinking'].meanRetrievalCallsPerTask,
|
||||
gen1PartialMean: qwenThinkingAgg.gen1PartialMean,
|
||||
deltaAbsolute: qwenThinkingAgg.deltaAbsolute,
|
||||
},
|
||||
'qwen-non-thinking': {
|
||||
nullBaselineMean: NULL_BASELINE_PER_SHAPE['qwen-non-thinking'].meanRetrievalCallsPerTask,
|
||||
gen1PartialMean: qwenNonThinkingAgg.gen1PartialMean,
|
||||
deltaAbsolute: qwenNonThinkingAgg.deltaAbsolute,
|
||||
},
|
||||
},
|
||||
cellSemanticAnchorInvarianceCountPerCandidate,
|
||||
preRegisteredDeltaFloorVerdict: deltaFloorVerdict,
|
||||
midRunHaltsBindingThresholds: {
|
||||
perCandidateCostOvershoot: {
|
||||
threshold: PER_CANDIDATE_COST_OVERSHOOT_THRESHOLD_USD,
|
||||
candidatesOvershoot: overshootCount,
|
||||
},
|
||||
perShapeVariance: {
|
||||
thresholdPP: PER_SHAPE_VARIANCE_HALT_PP,
|
||||
maxRangeObservedPP,
|
||||
},
|
||||
qwenRetrievalRegression: {
|
||||
triggered: qwenRegressionTriggered,
|
||||
details: qwenRegressionDetails,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
fs.mkdirSync(SCRATCH_DIR, { recursive: true });
|
||||
if (!fs.existsSync(RUN_LOG)) fs.writeFileSync(RUN_LOG, '');
|
||||
|
||||
const corpus = loadCorpus();
|
||||
const sample = deterministicShuffle(corpus, SAMPLING_SEED).slice(0, N_PER_SHAPE);
|
||||
log(`[loaded] corpus=${corpus.length}; sample=${N_PER_SHAPE} via seed=${SAMPLING_SEED}`);
|
||||
|
||||
const candidates = await loadCandidates();
|
||||
log(`[loaded] ${SHAPES.length} shapes × ${N_CANDIDATES_PER_SHAPE} candidates each`);
|
||||
|
||||
// Amendment 7 — pre-validate all candidates against cell-semantic anchors (Tier 3 input)
|
||||
const candidateValidatorVerdicts = new Map<string, ValidatorVerdict | null>();
|
||||
const TYPES_FILE_PATH = path.join(PROMPT_SHAPES_DIR, 'types.ts');
|
||||
for (const shape of SHAPES) {
|
||||
for (const cand of candidates.get(shape)!) {
|
||||
if (cand.variant === 'baseline') {
|
||||
// Baselines pass by definition (they ARE the pinned shape file)
|
||||
candidateValidatorVerdicts.set(cand.candidateId, null); // null = baseline (Tier 3 = 0.10 by anchor invariance)
|
||||
continue;
|
||||
}
|
||||
const filename = `${shape}-${cand.variant}.ts`;
|
||||
const candPath = path.join(GEPA_EVOLVED_DIR, filename);
|
||||
try {
|
||||
const verdict = validateCandidate({
|
||||
candidateShapeFilePath: candPath,
|
||||
baselineShapeName: `${shape}.ts`,
|
||||
typesFilePath: TYPES_FILE_PATH,
|
||||
expectShapeDiff: true,
|
||||
});
|
||||
candidateValidatorVerdicts.set(cand.candidateId, verdict);
|
||||
log(`[validator] ${cand.candidateId} valid=${verdict.valid} violations=${verdict.violations.length}`);
|
||||
} catch (e) {
|
||||
log(`[validator] ${cand.candidateId} ERROR ${(e as Error).message}`);
|
||||
candidateValidatorVerdicts.set(cand.candidateId, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.mode === 'dry-run') {
|
||||
log(`[dry-run] would run 5×3×8 = 120 evals (or halt at 30 = Checkpoint B)`);
|
||||
for (const shape of SHAPES) {
|
||||
for (const cand of candidates.get(shape)!) {
|
||||
log(`[dry-run] candidate=${cand.candidateId} variant=${cand.variant} shape.name=${cand.promptShape.name}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume support
|
||||
const existing = new Set<string>();
|
||||
let cumulativeCost = 0;
|
||||
|
||||
// Amendment 7 — per-candidate accumulator (rebuilt from JSONL on resume)
|
||||
const accs = new Map<string, CandidateAcc>();
|
||||
for (const shape of SHAPES) {
|
||||
for (const cand of candidates.get(shape)!) {
|
||||
const verdict = candidateValidatorVerdicts.get(cand.candidateId);
|
||||
// Baselines: validatorPassed = true (anchor invariant by definition).
|
||||
// Mutations: validatorPassed = verdict.valid (or false if validator threw).
|
||||
const validatorPassed = cand.variant === 'baseline' ? true : verdict?.valid ?? false;
|
||||
accs.set(cand.candidateId, makeCandidateAcc(cand, validatorPassed));
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.existsSync(OUT_JSONL)) {
|
||||
for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
|
||||
try {
|
||||
const r = JSON.parse(line) as EvalRecord;
|
||||
existing.add(`${r.candidateId}__${r.instanceId}`);
|
||||
cumulativeCost += r.evalCostUsd;
|
||||
const acc = accs.get(r.candidateId);
|
||||
if (acc) ingestEvalIntoAcc(acc, r);
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
log(`[resume] loaded ${existing.size} existing evals; cumulative $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
const out = fs.createWriteStream(OUT_JSONL, { flags: existing.size > 0 ? 'a' : 'w' });
|
||||
const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });
|
||||
|
||||
const haltAt = args.mode === 'checkpoint-b' ? CHECKPOINT_B_HALT_EVALS : 120;
|
||||
log(`[mode=${args.mode}] target eval count: ${haltAt}`);
|
||||
|
||||
let nDone = existing.size;
|
||||
let amendment7HaltReason: string | null = null;
|
||||
|
||||
outer: for (const shape of SHAPES) {
|
||||
for (const cand of candidates.get(shape)!) {
|
||||
for (const inst of sample) {
|
||||
const key = `${cand.candidateId}__${inst.instanceId}`;
|
||||
if (existing.has(key)) { log(`[skip] ${key} already in JSONL`); continue; }
|
||||
if (nDone >= haltAt) { log(`[HALT] reached ${haltAt} evals (Checkpoint B)`); break outer; }
|
||||
if (cumulativeCost >= COST_HALT_USD) { log(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD}`); break outer; }
|
||||
const r = await runOneEval(cand, inst, embedder);
|
||||
if ('error' in r) { log(`[skip] ${key}: ${r.error}`); continue; }
|
||||
out.write(JSON.stringify(r) + '\n');
|
||||
cumulativeCost += r.evalCostUsd;
|
||||
nDone++;
|
||||
// Amendment 7 — update accumulator + check mid-run halts
|
||||
const acc = accs.get(cand.candidateId);
|
||||
if (acc) ingestEvalIntoAcc(acc, r);
|
||||
const haltCheck = checkMidRunHalts(accs);
|
||||
if (haltCheck.shouldHalt) {
|
||||
amendment7HaltReason = haltCheck.reason;
|
||||
log(`[HALT-A7] ${haltCheck.reason}`);
|
||||
break outer;
|
||||
}
|
||||
log(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_HALT_USD} halt; ${nDone} evals total`);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.end();
|
||||
|
||||
// Amendment 7 — write Checkpoint B summary (binding extension per §checkpoint_b_tightened.report_extensions)
|
||||
const summary = buildCheckpointBSummary(args, accs, nDone, cumulativeCost, amendment7HaltReason);
|
||||
fs.writeFileSync(SUMMARY_JSON, JSON.stringify(summary, null, 2));
|
||||
log(`[summary] wrote ${SUMMARY_JSON}`);
|
||||
log(`[delta-floor] verdict=${summary.preRegisteredDeltaFloorVerdict.overallVerdict}`);
|
||||
log(`[delta-floor] threshold_1_aggregate_tier_1=${summary.preRegisteredDeltaFloorVerdict.threshold1AggregateTier1} (value=${summary.preRegisteredDeltaFloorVerdict.threshold1ValuePP.toFixed(2)}pp)`);
|
||||
log(`[delta-floor] threshold_2_qwen_retrieval_absolute=${summary.preRegisteredDeltaFloorVerdict.threshold2QwenRetrievalAbsolute} (max_delta=${summary.preRegisteredDeltaFloorVerdict.threshold2MaxDeltaAbsolute.toFixed(3)})`);
|
||||
log(`[delta-floor] threshold_3_compound_tier_1_plus_tier_2=${summary.preRegisteredDeltaFloorVerdict.threshold3CompoundTier1PlusTier2} (tier1=${summary.preRegisteredDeltaFloorVerdict.threshold3Tier1ValuePP.toFixed(2)}pp tier2_agg=${summary.preRegisteredDeltaFloorVerdict.threshold3Tier2Aggregate.toFixed(3)})`);
|
||||
|
||||
log(`[done] ${nDone} evals; total cost $${cumulativeCost.toFixed(4)}; halt_reason=${amendment7HaltReason ?? 'none (Checkpoint B reached or completed)'}`);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('FATAL:', e); process.exit(2); });
|
||||
438
benchmarks/gepa/scripts/faza-1/run-mutation-oracle.ts
Normal file
438
benchmarks/gepa/scripts/faza-1/run-mutation-oracle.ts
Normal file
@@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — mutation oracle runner.
|
||||
*
|
||||
* Per launch decision §G step 7 + manifest v7 §gepa.mutation_oracle +
|
||||
* §mutation_oracle_design + Amendment 2 §4 (forked Qwen vs non-Qwen templates).
|
||||
*
|
||||
* For each of 5 shapes, generate 2 mutation candidates via Opus 4.7. Each
|
||||
* candidate is validated via mutation-validator.ts (cell-semantic preservation).
|
||||
* Output: 10 candidate files at packages/agent/src/prompt-shapes/gepa-evolved/<shape>-gen1-v<N>.ts
|
||||
*
|
||||
* Approach:
|
||||
* - Use JSON-mode response_format (Amendment 4 lesson) for reliable parsing
|
||||
* - Opus outputs JSON with the 5 method body strings + new evidence_link
|
||||
* - Runner assembles TS file from fixed template (preserves cell semantics by construction)
|
||||
* - Validator confirms types.ts + MULTI_STEP_ACTION_CONTRACT SHAs unchanged
|
||||
*
|
||||
* Cost projection: 10 calls × ~$0.15 = ~$1.50
|
||||
*
|
||||
* Failure handling per brief §5: 2 consecutive invalid mutations from oracle
|
||||
* → halt-and-PM.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { validateCandidate, type ValidatorVerdict } from '../../src/faza-1/mutation-validator.js';
|
||||
import { classifyShape, type TemplateClass } from '../../src/faza-1/mutation-oracle-fork.js';
|
||||
import { type ShapeName, QWEN_TARGETED_SHAPES } from '../../src/faza-1/types.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const PROMPT_SHAPES_DIR = path.join(REPO_ROOT, 'packages/agent/src/prompt-shapes');
|
||||
const TYPES_FILE = path.join(PROMPT_SHAPES_DIR, 'types.ts');
|
||||
const GEPA_EVOLVED_DIR = path.join(PROMPT_SHAPES_DIR, 'gepa-evolved');
|
||||
const ORACLE_LOG = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/mutation-oracle-run.log');
|
||||
const OUT_MANIFEST = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/gen-1/mutation-oracle-manifest.json');
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const ORACLE_MODEL = 'claude-opus-4-7';
|
||||
const ORACLE_MAX_TOKENS = 8000;
|
||||
const N_MUTATIONS_PER_SHAPE = 2;
|
||||
|
||||
const SHAPES: ShapeName[] = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple'];
|
||||
|
||||
const PHASE_4_3_FAILURE_MODES = `Top T2 categories from Phase 4.3 verdict (decisions/2026-04-28-phase-4-3-rescore-delta-report.md):
|
||||
1. unsupported-specifics (10 of 26 T2 hits): hallucinated specifics or overreach beyond materials
|
||||
2. missed / didn't-consider / shallow (9 of 26 T2 hits): incomplete coverage of source documents
|
||||
3. conflation / weak-synthesis (5 of 26 T2 hits): risks blended together rather than separated
|
||||
4. wrong-entity / off-topic (sparse): minor framing errors
|
||||
|
||||
Phase 4.5 retrieval-engagement signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task on byte-identical tool surface. H3 corpus NULL-baseline replicated this (mean retrievals = 1.05).`;
|
||||
|
||||
// ── Logging ────────────────────────────────────────────────────────────────
|
||||
|
||||
function log(msg: string): void {
|
||||
const line = `[${new Date().toISOString()}] ${msg}\n`;
|
||||
try { fs.appendFileSync(ORACLE_LOG, line); } catch { /* dir may not exist yet */ }
|
||||
process.stderr.write(line);
|
||||
}
|
||||
|
||||
// ── Baseline shape inspection (extract metadata for prompt) ────────────────
|
||||
|
||||
interface BaselineShapeMetadata {
|
||||
description: string;
|
||||
modelClass: string;
|
||||
defaultThinking: boolean;
|
||||
defaultMaxTokens: number;
|
||||
shapeFileContent: string; // full file text
|
||||
}
|
||||
|
||||
function loadBaselineShape(shapeName: ShapeName): BaselineShapeMetadata {
|
||||
const shapeFile = path.join(PROMPT_SHAPES_DIR, `${shapeName}.ts`);
|
||||
const content = fs.readFileSync(shapeFile, 'utf-8');
|
||||
|
||||
const description = content.match(/description: '([^']+)'/)?.[1] ?? '';
|
||||
const modelClass = content.match(/modelClass: '([^']+)'/)?.[1] ?? shapeName;
|
||||
const defaultThinking = content.match(/defaultThinking: (true|false|undefined)/)?.[1] === 'true';
|
||||
const defaultMaxTokens = Number(content.match(/defaultMaxTokens: (\d+)/)?.[1] ?? 4096);
|
||||
|
||||
return { description, modelClass, defaultThinking, defaultMaxTokens, shapeFileContent: content };
|
||||
}
|
||||
|
||||
// ── Oracle prompt (JSON-mode) ──────────────────────────────────────────────
|
||||
|
||||
function buildOraclePrompt(shapeName: ShapeName, baseline: BaselineShapeMetadata, mutationIdx: number): string {
|
||||
const cls: TemplateClass = classifyShape(shapeName);
|
||||
const isQwen = QWEN_TARGETED_SHAPES.has(shapeName);
|
||||
|
||||
const qwenGuidance = `For Qwen-targeted shape mutation:
|
||||
- Emphasize multi-turn retrieval over single-shot retrieval. Phrase like "Continue retrieving until you have evidence from at least 2 distinct queries before finalizing."
|
||||
- Add anti-premature-finalization scaffolding. Phrase like "Before finalizing, ask: what gap in evidence remains? Issue another retrieval if any gap exists."
|
||||
- Encourage iterative refinement of retrieval queries based on prior turn results.
|
||||
- Goal: push mean retrieval_calls per task from current 1.0 baseline toward >= 1.5 (escape Amendment 2 penalty zone) and ideally >= 2.0 (Opus parity proxy).`;
|
||||
|
||||
const nonQwenGuidance = `For non-Qwen shape mutation:
|
||||
- Standard mutation guidance per brief §3.3 — evolve reasoning scaffold, planning step structure, chain-of-thought triggers.
|
||||
- Restructure implicit reasoning prompts (e.g., "think step by step" variants, planning bullets).
|
||||
- Refine where the model is prompted to articulate reasoning before producing output.
|
||||
- Improve multi-step task decomposition explicitness.`;
|
||||
|
||||
return `You are an expert prompt engineer. Generate ONE mutated variant of the prompt-shape below, evolving reasoning scaffold + retrieval-engagement guidance while preserving cell semantics.
|
||||
|
||||
## Target shape
|
||||
- Name: ${shapeName}
|
||||
- Class: ${cls}
|
||||
- Mutation variant index: ${mutationIdx} (you are generating mutation #${mutationIdx} of 2 for this shape)
|
||||
|
||||
## Baseline metadata (LOCKED — do NOT change these)
|
||||
- description: ${baseline.description}
|
||||
- modelClass: ${baseline.modelClass}
|
||||
- defaultThinking: ${baseline.defaultThinking}
|
||||
- defaultMaxTokens: ${baseline.defaultMaxTokens}
|
||||
|
||||
## Phase 4.3 + Phase 4.5 failure modes to address
|
||||
${PHASE_4_3_FAILURE_MODES}
|
||||
|
||||
## Mutation guidance
|
||||
${isQwen ? qwenGuidance : nonQwenGuidance}
|
||||
|
||||
## Cell semantic boundaries (LOCKED — violation = REJECTED candidate)
|
||||
You may NOT modify:
|
||||
- The MULTI_STEP_ACTION_CONTRACT constant (lives in types.ts; bytes are SHA-pinned)
|
||||
- The JSON action contract format ({"action": "retrieve" | "finalize", ...})
|
||||
- Task framing (persona/question/materials section labels)
|
||||
- Imports block
|
||||
- Locked metadata fields above
|
||||
|
||||
You MAY modify:
|
||||
- The 5 method bodies (string-building only): systemPromptSolo, systemPromptMultiStep, soloUserPrompt, multiStepKickoffUserPrompt, retrievalInjectionUserPrompt
|
||||
- The evidence_link metadata (you MUST update to point to GEPA Gen 1 results: "benchmarks/results/gepa-faza1/gen-1/mutation-oracle-run.log + Phase 4.5 + Amendment 2 §3 retrieval-engagement bonus")
|
||||
|
||||
## Baseline shape file (your input)
|
||||
|
||||
\`\`\`typescript
|
||||
${baseline.shapeFileContent}
|
||||
\`\`\`
|
||||
|
||||
## Your output: JSON object
|
||||
|
||||
Output a single JSON object with these fields. Each method body field should be a TypeScript expression that evaluates to a string (the prompt text). Use the same approach as the baseline (e.g., array.join('\\n')). Variable references like \${persona}, \${question}, \${maxSteps}, \${maxRetrievalsPerStep}, \${input.persona}, etc. must be preserved verbatim where the baseline used them.
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"evidenceLink": "<updated evidence_link string referencing Gen 1 results + Phase 4.5/Amendment 2>",
|
||||
"systemPromptSolo": "<TypeScript expression returning system prompt for !isMultiStep — typically a join of strings or template literal; reference {persona}>",
|
||||
"systemPromptMultiStep": "<TypeScript expression returning system prompt for isMultiStep — must reference MULTI_STEP_ACTION_CONTRACT verbatim, {persona}, {question}, {maxSteps}, {maxRetrievalsPerStep}>",
|
||||
"soloUserPrompt": "<TypeScript expression returning user prompt with {input.persona}, {input.materials}, {input.question}>",
|
||||
"multiStepKickoffUserPrompt": "<TypeScript expression returning kickoff user message; baseline uses 'Begin. Output your first action JSON now.' — your variant should request engagement scaffolding for Qwen shapes>",
|
||||
"retrievalInjectionUserPrompt": "<TypeScript expression returning retrieval-injection user message with {input.query}, {input.resultCount}, {input.results}>"
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
CRITICAL: each field's value must be a STRING containing valid TypeScript code that, when wrapped in \`return (\${value})\`, would compile + return a string. The simplest valid pattern is template literals or .join('\\n') over an array of strings.
|
||||
|
||||
Output ONLY the JSON object. No prose. No code fences.`;
|
||||
}
|
||||
|
||||
// ── LLM call (JSON-mode, no temperature per Amendment 4 lesson) ────────────
|
||||
|
||||
interface OracleCallResult {
|
||||
content: string;
|
||||
inTokens: number;
|
||||
outTokens: number;
|
||||
costUsd: number;
|
||||
latencyMs: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
async function callOpusOracle(prompt: string): Promise<OracleCallResult> {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY;
|
||||
if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
|
||||
|
||||
const payload = {
|
||||
model: ORACLE_MODEL,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: ORACLE_MAX_TOKENS,
|
||||
response_format: { type: 'json_object' },
|
||||
// temperature omitted per Amendment 4 (Anthropic deprecates with JSON mode)
|
||||
};
|
||||
|
||||
const started = Date.now();
|
||||
let lastErr: string | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const d: any = await resp.json();
|
||||
if ('error' in d) {
|
||||
lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
const content = d.choices?.[0]?.message?.content ?? '';
|
||||
const usage = d.usage ?? {};
|
||||
const inTok = usage.prompt_tokens ?? 0;
|
||||
const outTok = usage.completion_tokens ?? 0;
|
||||
const costUsd = (inTok * 15.0 + outTok * 75.0) / 1_000_000;
|
||||
return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
|
||||
} catch (e) {
|
||||
lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
}
|
||||
}
|
||||
return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now() - started, error: lastErr };
|
||||
}
|
||||
|
||||
// ── JSON parsing + TS file assembly ────────────────────────────────────────
|
||||
|
||||
interface MutationFields {
|
||||
evidenceLink: string;
|
||||
systemPromptSolo: string;
|
||||
systemPromptMultiStep: string;
|
||||
soloUserPrompt: string;
|
||||
multiStepKickoffUserPrompt: string;
|
||||
retrievalInjectionUserPrompt: string;
|
||||
}
|
||||
|
||||
function parseOracleOutput(content: string): MutationFields | { error: string } {
|
||||
let s = content.trim();
|
||||
if (s.startsWith('```')) s = s.replace(/^```[a-z]*\n?/, '').replace(/```\s*$/, '');
|
||||
const firstBrace = s.indexOf('{');
|
||||
const lastBrace = s.lastIndexOf('}');
|
||||
if (firstBrace < 0 || lastBrace < 0) {
|
||||
return { error: `no JSON object found in output (length=${content.length})` };
|
||||
}
|
||||
try {
|
||||
const obj = JSON.parse(s.slice(firstBrace, lastBrace + 1));
|
||||
const required = ['evidenceLink', 'systemPromptSolo', 'systemPromptMultiStep', 'soloUserPrompt', 'multiStepKickoffUserPrompt', 'retrievalInjectionUserPrompt'];
|
||||
for (const k of required) {
|
||||
if (typeof obj[k] !== 'string' || obj[k].length === 0) {
|
||||
return { error: `field "${k}" missing or empty` };
|
||||
}
|
||||
}
|
||||
return obj as MutationFields;
|
||||
} catch (e) {
|
||||
return { error: `JSON parse failed: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
function buildShapeFile(shapeName: ShapeName, baseline: BaselineShapeMetadata, fields: MutationFields, mutationIdx: number): string {
|
||||
const exportName = `${shapeName.replace(/-([a-z])/g, (_, c) => c.toUpperCase())}Gen1V${mutationIdx}Shape`;
|
||||
const tsName = `${shapeName}-gen1-v${mutationIdx}`;
|
||||
return `/**
|
||||
* GEPA Faza 1 — Gen 1 mutation #${mutationIdx} of ${shapeName}.
|
||||
*
|
||||
* Generated by Opus 4.7 mutation oracle per Amendment 2 §4 forked template
|
||||
* (${classifyShape(shapeName)} branch). Cell-semantic boundaries preserved
|
||||
* via mutation-validator.ts SHA pins.
|
||||
*
|
||||
* Evidence: ${fields.evidenceLink}
|
||||
*
|
||||
* Baseline anchor: packages/agent/src/prompt-shapes/${shapeName}.ts (manifest v7
|
||||
* §gepa.mutation_validator.baseline_shape_shas[${shapeName}.ts]).
|
||||
*/
|
||||
|
||||
import {
|
||||
type PromptShape,
|
||||
type SystemPromptInput,
|
||||
type SoloUserPromptInput,
|
||||
type MultiStepKickoffInput,
|
||||
type RetrievalInjectionInput,
|
||||
MULTI_STEP_ACTION_CONTRACT,
|
||||
} from '../types.js';
|
||||
|
||||
export const ${exportName}: PromptShape = {
|
||||
name: '${tsName}',
|
||||
metadata: {
|
||||
description: '${baseline.description.replace(/'/g, "\\'")}',
|
||||
modelClass: '${baseline.modelClass}',
|
||||
evidence_link: ${JSON.stringify(fields.evidenceLink)},
|
||||
defaultThinking: ${baseline.defaultThinking},
|
||||
defaultMaxTokens: ${baseline.defaultMaxTokens},
|
||||
},
|
||||
|
||||
systemPrompt(input: SystemPromptInput): string {
|
||||
const { persona, question, isMultiStep, maxSteps = 5, maxRetrievalsPerStep = 8 } = input;
|
||||
if (!isMultiStep) {
|
||||
return ${fields.systemPromptSolo};
|
||||
}
|
||||
return ${fields.systemPromptMultiStep};
|
||||
},
|
||||
|
||||
soloUserPrompt(input: SoloUserPromptInput): string {
|
||||
return ${fields.soloUserPrompt};
|
||||
},
|
||||
|
||||
multiStepKickoffUserPrompt(_input: MultiStepKickoffInput): string {
|
||||
return ${fields.multiStepKickoffUserPrompt};
|
||||
},
|
||||
|
||||
retrievalInjectionUserPrompt(input: RetrievalInjectionInput): string {
|
||||
return ${fields.retrievalInjectionUserPrompt};
|
||||
},
|
||||
};
|
||||
`;
|
||||
}
|
||||
|
||||
// ── Validate via runtime sanity (compile check) + structural check ─────────
|
||||
|
||||
function validateAssembledFile(filepath: string, baselineShapeName: ShapeName): { valid: boolean; reason?: string } {
|
||||
// Check file parses as TypeScript by attempting a require-style import
|
||||
// For Faza 1 simplicity, just check that the file:
|
||||
// 1. Imports from types.js (mutation-validator's import preservation check)
|
||||
// 2. Contains the required locked metadata fields
|
||||
// 3. Exports a single PromptShape
|
||||
// 4. Doesn't break the cell-semantic boundary (types.ts/MULTI_STEP_ACTION_CONTRACT SHAs unchanged)
|
||||
// The mutation-validator.ts does the SHA checks; we use it via validateCandidate.
|
||||
|
||||
const verdict: ValidatorVerdict = validateCandidate({
|
||||
candidateShapeFilePath: filepath,
|
||||
baselineShapeName: `${baselineShapeName}.ts` as any,
|
||||
typesFilePath: TYPES_FILE,
|
||||
expectShapeDiff: true,
|
||||
});
|
||||
|
||||
if (!verdict.valid) {
|
||||
return { valid: false, reason: `validator violations: ${verdict.violations.map(v => `${v.category}: ${v.detail}`).join('; ')}` };
|
||||
}
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
// ── Main: generate 10 mutations ────────────────────────────────────────────
|
||||
|
||||
interface MutationManifestEntry {
|
||||
shape: ShapeName;
|
||||
mutationIdx: number;
|
||||
filename: string;
|
||||
costUsd: number;
|
||||
latencyMs: number;
|
||||
tsName: string;
|
||||
validatorVerdict: 'valid' | 'invalid_after_retry';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
fs.mkdirSync(GEPA_EVOLVED_DIR, { recursive: true });
|
||||
fs.mkdirSync(path.dirname(ORACLE_LOG), { recursive: true });
|
||||
if (!fs.existsSync(ORACLE_LOG)) fs.writeFileSync(ORACLE_LOG, '');
|
||||
|
||||
log(`[start] mutation oracle for ${SHAPES.length} shapes × ${N_MUTATIONS_PER_SHAPE} mutations`);
|
||||
|
||||
const manifest: MutationManifestEntry[] = [];
|
||||
let cumulativeCost = 0;
|
||||
let consecutiveInvalidGlobal = 0;
|
||||
|
||||
for (const shapeName of SHAPES) {
|
||||
const baseline = loadBaselineShape(shapeName);
|
||||
log(`[${shapeName}] baseline loaded; modelClass=${baseline.modelClass} description="${baseline.description.slice(0, 60)}..."`);
|
||||
|
||||
for (let mutIdx = 1; mutIdx <= N_MUTATIONS_PER_SHAPE; mutIdx++) {
|
||||
const tsName = `${shapeName}-gen1-v${mutIdx}`;
|
||||
const outFile = path.join(GEPA_EVOLVED_DIR, `${tsName}.ts`);
|
||||
|
||||
if (fs.existsSync(outFile)) {
|
||||
log(`[${tsName}] already exists; skipping`);
|
||||
manifest.push({ shape: shapeName, mutationIdx: mutIdx, filename: outFile, costUsd: 0, latencyMs: 0, tsName, validatorVerdict: 'valid' });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Try once; if invalid, retry once with structural feedback. After 2 fails → mark invalid_after_retry.
|
||||
let valid = false;
|
||||
let totalCost = 0;
|
||||
let totalLatency = 0;
|
||||
let errMsg: string | undefined;
|
||||
|
||||
for (let attempt = 0; attempt < 2 && !valid; attempt++) {
|
||||
const prompt = buildOraclePrompt(shapeName, baseline, mutIdx);
|
||||
log(`[${tsName}] oracle call attempt ${attempt + 1}; prompt_len=${prompt.length}c`);
|
||||
const llm = await callOpusOracle(prompt);
|
||||
totalCost += llm.costUsd;
|
||||
totalLatency += llm.latencyMs;
|
||||
|
||||
if (llm.error) {
|
||||
errMsg = `LLM error: ${llm.error}`;
|
||||
log(`[${tsName}] ${errMsg}`);
|
||||
continue;
|
||||
}
|
||||
const parsed = parseOracleOutput(llm.content);
|
||||
if ('error' in parsed) {
|
||||
errMsg = `parse error: ${parsed.error}`;
|
||||
log(`[${tsName}] ${errMsg}; first 200c: ${llm.content.slice(0, 200)}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const tsContent = buildShapeFile(shapeName, baseline, parsed, mutIdx);
|
||||
fs.writeFileSync(outFile, tsContent, 'utf-8');
|
||||
|
||||
const v = validateAssembledFile(outFile, shapeName);
|
||||
if (v.valid) {
|
||||
valid = true;
|
||||
log(`[${tsName}] OK; cost=$${llm.costUsd.toFixed(4)}; latency=${llm.latencyMs}ms; file=${path.basename(outFile)}`);
|
||||
} else {
|
||||
errMsg = `validator failed: ${v.reason}`;
|
||||
log(`[${tsName}] ${errMsg}`);
|
||||
fs.unlinkSync(outFile);
|
||||
}
|
||||
}
|
||||
|
||||
cumulativeCost += totalCost;
|
||||
manifest.push({
|
||||
shape: shapeName, mutationIdx: mutIdx, filename: outFile,
|
||||
costUsd: totalCost, latencyMs: totalLatency, tsName,
|
||||
validatorVerdict: valid ? 'valid' : 'invalid_after_retry',
|
||||
error: valid ? undefined : errMsg,
|
||||
});
|
||||
|
||||
if (valid) {
|
||||
consecutiveInvalidGlobal = 0;
|
||||
} else {
|
||||
consecutiveInvalidGlobal++;
|
||||
if (consecutiveInvalidGlobal >= 2) {
|
||||
log(`[HALT] 2 consecutive invalid mutations from oracle (per brief §5) — stopping`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (consecutiveInvalidGlobal >= 2) break;
|
||||
}
|
||||
|
||||
fs.writeFileSync(OUT_MANIFEST, JSON.stringify({
|
||||
totalMutationsAttempted: manifest.length,
|
||||
totalValid: manifest.filter(m => m.validatorVerdict === 'valid').length,
|
||||
totalInvalid: manifest.filter(m => m.validatorVerdict !== 'valid').length,
|
||||
cumulativeCostUsd: +cumulativeCost.toFixed(6),
|
||||
entries: manifest,
|
||||
completedAtIso: new Date().toISOString(),
|
||||
}, null, 2));
|
||||
log(`[done] ${manifest.length} mutations attempted; ${manifest.filter(m => m.validatorVerdict === 'valid').length} valid; cumulative $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('FATAL:', e); process.exit(2); });
|
||||
662
benchmarks/gepa/scripts/faza-1/run-null-baseline.ts
Normal file
662
benchmarks/gepa/scripts/faza-1/run-null-baseline.ts
Normal file
@@ -0,0 +1,662 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* GEPA Faza 1 — NULL-baseline runner.
|
||||
*
|
||||
* Per launch decision §G step 6 + §F + §A.5/A.7/A.10.
|
||||
*
|
||||
* Per Amendment 4 + binding texture-audit verdict, the corpus is 50/50 (PM
|
||||
* ratified). NULL-baseline measures each of 5 baseline prompt-shapes against
|
||||
* 8 stratified instances from the corpus to establish per-shape trio_strict_pass
|
||||
* baseline rate.
|
||||
*
|
||||
* --------------------------------------------------------------------------
|
||||
* MULTI-STEP MODE (vs solo) — design rationale
|
||||
* --------------------------------------------------------------------------
|
||||
*
|
||||
* Brief §2 says "Cell scope Faza 1: H3 only" and pilot Cell C labels H3 as
|
||||
* "Qwen solo". HOWEVER, Amendment 2 §3 retrieval_engagement_bonus only makes
|
||||
* sense if a retrieval tool is present (in solo mode, retrieval_calls always
|
||||
* = 0 → bonus pinned at -0.05 → fitness function cannot discriminate
|
||||
* candidates). Amendment 2 §6 Phase 5 forward-record acceptance criteria
|
||||
* (engagement parity ≥ Opus + score parity narrowed by ≥0.30 H4 trio_mean
|
||||
* delta) explicitly invoke retrieval-mode metrics.
|
||||
*
|
||||
* Resolution (BINDING for this runner): NULL-baseline runs in MULTI-STEP
|
||||
* mode with retrieval tool available. The "H3 cell" in Faza 1 GEPA context
|
||||
* means "Qwen-targeted evaluation with retrieval available", not pilot Cell
|
||||
* C strict "Qwen solo". This reconciles brief §2 with Amendment 2 + Phase 5
|
||||
* forward record, mirrors Phase 4.5 empirical setup (Cells B/D had
|
||||
* retrieval), and makes Amendment 2 fitness function meaningful.
|
||||
*
|
||||
* Documented in Checkpoint A halt-and-PM report for PM ratification or pivot.
|
||||
*
|
||||
* --------------------------------------------------------------------------
|
||||
* Cost projection (PM ratified ~$20):
|
||||
* --------------------------------------------------------------------------
|
||||
*
|
||||
* 5 shapes × 8 instances × ($0.50/eval avg) ≈ $20 expected
|
||||
* - Subject: Qwen 3.6 35B-A3B (DashScope direct, ~$0.001/call × 2-3 calls)
|
||||
* - Trio judges: Opus 4.7 + GPT-5.4 + MiniMax M2.7 × ($0.05/call avg) = $0.15/eval
|
||||
* - Per pilot 2026-04-26 cost average = $0.47/cell
|
||||
*
|
||||
* Halt threshold (per launch decision §D + Amendment 3): if cumulative > $26
|
||||
* (30% over $20 expected), halt-and-PM per super-linear sub-rule.
|
||||
*
|
||||
* --------------------------------------------------------------------------
|
||||
* Sampling design:
|
||||
* --------------------------------------------------------------------------
|
||||
*
|
||||
* Same 8 instances across all 5 shapes (controlled comparison; trio_mean delta
|
||||
* is purely shape-attributable). Deterministic Mulberry32 with seed=42, then
|
||||
* take first 8 of shuffled corpus. Held-out 5 = next 5 (instances 9-13)
|
||||
* after the 8 — kept for Faza 1 §F.4 held-out validation.
|
||||
*
|
||||
* --------------------------------------------------------------------------
|
||||
* Usage:
|
||||
* --------------------------------------------------------------------------
|
||||
*
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --dry-run
|
||||
* # No LLM call. Validates sampling + shape resolution + substrate setup.
|
||||
*
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --probe
|
||||
* # Single (shape=qwen-thinking, instance=0). ~$0.50. Validates round-trip.
|
||||
*
|
||||
* npx tsx benchmarks/gepa/scripts/faza-1/run-null-baseline.ts --all
|
||||
* # Full 5×8 = 40 evaluations. ~$20 expected, $26 halt.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
MindDB,
|
||||
FrameStore,
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
createOllamaEmbedder,
|
||||
type Embedder,
|
||||
} from '@waggle/core';
|
||||
import {
|
||||
runRetrievalAgentLoop,
|
||||
type LlmCallFn,
|
||||
type LlmCallInput,
|
||||
type LlmCallResult as AgentLlmCallResult,
|
||||
type RetrievalSearchFn,
|
||||
type AgentRunResult,
|
||||
} from '@waggle/agent';
|
||||
|
||||
import { REGISTRY, selectShape } from '../../../../packages/agent/src/prompt-shapes/selector.js';
|
||||
import { type PromptShape } from '../../../../packages/agent/src/prompt-shapes/types.js';
|
||||
|
||||
import { type CorpusInstance } from '../../src/faza-1/corpus.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const CORPUS_JSONL = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/corpus/h3-northlane-cfo-50-instances.jsonl');
|
||||
const OUT_DIR = path.join(REPO_ROOT, 'benchmarks/results/gepa-faza1/null-baseline');
|
||||
const OUT_JSONL = path.join(OUT_DIR, 'null-baseline-eval.jsonl');
|
||||
const RUN_LOG = path.join(OUT_DIR, 'null-baseline-run.log');
|
||||
const SUMMARY_JSON = path.join(OUT_DIR, 'null-baseline-summary.json');
|
||||
const SCRATCH_DIR = path.join(REPO_ROOT, 'tmp/gepa-faza1-null-baseline');
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const EMBEDDER_MODEL = 'nomic-embed-text';
|
||||
|
||||
const SAMPLING_SEED = 42;
|
||||
const N_PER_SHAPE = 8;
|
||||
|
||||
// Shapes in fixed evaluation order (matches manifest v7 §gepa.shape_scope.targets)
|
||||
const SHAPES = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple'] as const;
|
||||
type ShapeName = typeof SHAPES[number];
|
||||
|
||||
// Subject inheritance per manifest v7 §subject (= pilot 2026-04-26 runner SHA 8a6251e2)
|
||||
const SUBJECT_ALIAS = 'qwen3.6-35b-a3b-via-dashscope-direct';
|
||||
const SUBJECT_MAX_TOKENS = 16000;
|
||||
const SUBJECT_THINKING = true;
|
||||
|
||||
// Judges inheritance per manifest v7 §judges (= pilot runner line 626)
|
||||
const JUDGES = ['claude-opus-4-7', 'gpt-5.4', 'minimax-m27-via-openrouter'] as const;
|
||||
const JUDGE_MAX_TOKENS = 3000;
|
||||
const JUDGE_RETRIES = 3;
|
||||
|
||||
// Multi-step orchestration
|
||||
const MAX_STEPS = 5;
|
||||
const MAX_RETRIEVALS_PER_STEP = 8;
|
||||
const PER_CALL_HALT_USD = 0.40;
|
||||
const PER_CELL_HALT_USD = 1.00;
|
||||
|
||||
// Cost halt per launch decision §D (super-linear sub-rule per A.7)
|
||||
const COST_HALT_USD = 26.0; // 30% over $20 expected
|
||||
|
||||
// Pricing (per pilot runner line 128-133)
|
||||
const MODEL_PRICING: Record<string, { in: number; out: number }> = {
|
||||
'claude-opus-4-7': { in: 15.0, out: 75.0 },
|
||||
'gpt-5.4': { in: 2.5, out: 10.0 },
|
||||
'minimax-m27-via-openrouter': { in: 0.7, out: 2.8 },
|
||||
'qwen3.6-35b-a3b-via-dashscope-direct': { in: 0.20, out: 0.80 },
|
||||
'qwen3.6-35b-a3b-via-openrouter': { in: 0.6, out: 2.4 },
|
||||
};
|
||||
|
||||
const MANIFEST_ANCHOR = 'manifest-v7-gepa-faza1';
|
||||
const MANIFEST_SHA_AMENDMENT_4 = '1f7a6d6fa01403f6c8d6855893adbfa5e82898a81b7583cfa55628e5eba60196';
|
||||
|
||||
// ── Logging ────────────────────────────────────────────────────────────────
|
||||
|
||||
function log(msg: string): void {
|
||||
const line = `[${new Date().toISOString()}] ${msg}\n`;
|
||||
try { fs.appendFileSync(RUN_LOG, line); } catch { /* dir not yet created */ }
|
||||
process.stderr.write(line);
|
||||
}
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface Args {
|
||||
mode: 'dry-run' | 'probe' | 'all';
|
||||
probeShape?: ShapeName;
|
||||
probeInstanceIdx?: number;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
let mode: Args['mode'] = 'dry-run';
|
||||
let probeShape: ShapeName | undefined;
|
||||
let probeInstanceIdx: number | undefined;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
const next = argv[i + 1];
|
||||
switch (flag) {
|
||||
case '--dry-run': mode = 'dry-run'; break;
|
||||
case '--probe': mode = 'probe'; break;
|
||||
case '--all': mode = 'all'; break;
|
||||
case '--probe-shape': probeShape = next as ShapeName; i++; break;
|
||||
case '--probe-instance': probeInstanceIdx = Number(next); i++; break;
|
||||
}
|
||||
}
|
||||
return { mode, probeShape, probeInstanceIdx };
|
||||
}
|
||||
|
||||
// ── Deterministic sampling (Mulberry32, seed=42) ───────────────────────────
|
||||
|
||||
function mulberry32(seed: number): () => number {
|
||||
let t = seed >>> 0;
|
||||
return () => {
|
||||
t = (t + 0x6d2b79f5) >>> 0;
|
||||
let r = t;
|
||||
r = Math.imul(r ^ (r >>> 15), r | 1);
|
||||
r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
|
||||
return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function deterministicShuffle<T>(items: ReadonlyArray<T>, seed: number): T[] {
|
||||
const arr = [...items];
|
||||
const rand = mulberry32(seed);
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rand() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function loadCorpus(): CorpusInstance[] {
|
||||
const text = fs.readFileSync(CORPUS_JSONL, 'utf-8');
|
||||
return text.trim().split(/\n+/).filter(Boolean).map(l => JSON.parse(l) as CorpusInstance);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample N=8 instances deterministically. Same set across all shapes (controlled
|
||||
* comparison). Held-out 5 are the next 5 after the sample (instances 9-13).
|
||||
*/
|
||||
function sampleInstances(corpus: CorpusInstance[], n: number, seed: number): CorpusInstance[] {
|
||||
const shuffled = deterministicShuffle(corpus, seed);
|
||||
return shuffled.slice(0, n);
|
||||
}
|
||||
|
||||
// ── LiteLLM call adapter ──────────────────────────────────────────────────
|
||||
|
||||
const llmCall: LlmCallFn = async (input: LlmCallInput): Promise<AgentLlmCallResult> => {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY;
|
||||
if (!masterKey) throw new Error('LITELLM_MASTER_KEY env not set');
|
||||
|
||||
const { model, messages } = input;
|
||||
const isQwen = model.includes('qwen');
|
||||
const maxTokens = input.maxTokens ?? (isQwen ? SUBJECT_MAX_TOKENS : 4096);
|
||||
const thinking = input.thinking ?? (isQwen ? SUBJECT_THINKING : true);
|
||||
|
||||
const payload: Record<string, unknown> = { model, messages, max_tokens: maxTokens };
|
||||
|
||||
if (model.startsWith('claude-opus')) {
|
||||
payload.temperature = 1.0;
|
||||
} else if (model === 'gpt-5.4' || model === 'minimax-m27-via-openrouter') {
|
||||
// omit temperature — reasoning-model defaults
|
||||
} else {
|
||||
payload.temperature = input.temperature ?? 0.3;
|
||||
}
|
||||
|
||||
if (isQwen) {
|
||||
payload.extra_body = { enable_thinking: thinking };
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastErr: string | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${masterKey}` },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const d: any = await resp.json();
|
||||
if ('error' in d) {
|
||||
lastErr = String(d.error?.message ?? JSON.stringify(d.error)).slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
const content = d.choices?.[0]?.message?.content ?? '';
|
||||
const usage = d.usage ?? {};
|
||||
const inTok = usage.prompt_tokens ?? 0;
|
||||
const outTok = usage.completion_tokens ?? 0;
|
||||
const pricing = MODEL_PRICING[model] ?? { in: 1, out: 4 };
|
||||
const costUsd = (inTok * pricing.in + outTok * pricing.out) / 1_000_000;
|
||||
return { content, inTokens: inTok, outTokens: outTok, costUsd, latencyMs: Date.now() - started };
|
||||
} catch (e) {
|
||||
lastErr = `${(e as Error).name}: ${(e as Error).message}`.slice(0, 200);
|
||||
if (attempt < 1) await new Promise(r => setTimeout(r, 1500));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return { content: '', inTokens: 0, outTokens: 0, costUsd: 0, latencyMs: Date.now() - started, error: lastErr ?? 'unknown error' };
|
||||
};
|
||||
|
||||
// ── Trio judging (mirrors pilot 2026-04-26 runner line 545+) ───────────────
|
||||
|
||||
const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent's response to a complex knowledge work task. The persona, scenario, materials, and question are provided. The response was generated under one of four configurations (revealed only after scoring): {model_only | model + memory + agent harness} × {Opus 4.7 | Qwen 3.6 35B-A3B}.
|
||||
|
||||
You do NOT know which configuration produced this response. Score blind.
|
||||
|
||||
Read the persona/scenario/question (provided), skim the materials (provided), then read the response carefully (provided).
|
||||
|
||||
Score the response on six dimensions, Likert 1-5:
|
||||
|
||||
1. COMPLETENESS — engagement with all material
|
||||
2. ACCURACY — faithfulness to source materials, no hallucinations
|
||||
3. SYNTHESIS — connections across inputs, not isolated treatment
|
||||
4. JUDGMENT — defensible recommendations, tradeoffs acknowledged
|
||||
5. ACTIONABILITY — would the persona act on this tomorrow
|
||||
6. STRUCTURE — organization and readability
|
||||
|
||||
Output JSON only:
|
||||
{
|
||||
"completeness": <1-5>,
|
||||
"accuracy": <1-5>,
|
||||
"synthesis": <1-5>,
|
||||
"judgment": <1-5>,
|
||||
"actionability": <1-5>,
|
||||
"structure": <1-5>,
|
||||
"rationale": "<1-2 sentences explaining the lowest scoring dimension>",
|
||||
"overall_verdict": "<one of: PASS_STRONG | PASS_ADEQUATE | FAIL_WEAK | FAIL_CRITICAL>"
|
||||
}
|
||||
|
||||
PASS_STRONG: mean >= 4.0
|
||||
PASS_ADEQUATE: mean 3.5-3.99
|
||||
FAIL_WEAK: mean 2.5-3.49
|
||||
FAIL_CRITICAL: mean < 2.5
|
||||
|
||||
[PERSONA + SCENARIO + QUESTION]
|
||||
###PERSONA_SCENARIO_QUESTION###
|
||||
|
||||
[MATERIALS]
|
||||
###MATERIALS###
|
||||
|
||||
[RESPONSE TO EVALUATE]
|
||||
###RESPONSE###`;
|
||||
|
||||
interface JudgeVerdict {
|
||||
completeness: number;
|
||||
accuracy: number;
|
||||
synthesis: number;
|
||||
judgment: number;
|
||||
actionability: number;
|
||||
structure: number;
|
||||
rationale: string;
|
||||
overall_verdict: string;
|
||||
mean: number;
|
||||
}
|
||||
|
||||
interface JudgeRecord extends JudgeVerdict {
|
||||
judge_model: string;
|
||||
judge_cost_usd: number;
|
||||
judge_latency_ms: number;
|
||||
judge_retries: number;
|
||||
}
|
||||
|
||||
const ZERO_VERDICT: JudgeVerdict = {
|
||||
completeness: 0, accuracy: 0, synthesis: 0, judgment: 0, actionability: 0, structure: 0,
|
||||
rationale: '__JUDGE_FAILED__', overall_verdict: 'FAIL_CRITICAL', mean: 0,
|
||||
};
|
||||
|
||||
function parseJudgeJson(text: string): JudgeVerdict | null {
|
||||
const m = text.match(/\{[\s\S]*\}/);
|
||||
if (!m) return null;
|
||||
try {
|
||||
const obj = JSON.parse(m[0]);
|
||||
const dims = ['completeness', 'accuracy', 'synthesis', 'judgment', 'actionability', 'structure'] as const;
|
||||
for (const d of dims) {
|
||||
if (typeof obj[d] !== 'number' || obj[d] < 1 || obj[d] > 5) return null;
|
||||
}
|
||||
const mean = dims.reduce((s, d) => s + obj[d], 0) / dims.length;
|
||||
return {
|
||||
completeness: obj.completeness, accuracy: obj.accuracy, synthesis: obj.synthesis,
|
||||
judgment: obj.judgment, actionability: obj.actionability, structure: obj.structure,
|
||||
rationale: typeof obj.rationale === 'string' ? obj.rationale : '',
|
||||
overall_verdict: typeof obj.overall_verdict === 'string' ? obj.overall_verdict : '',
|
||||
mean,
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function buildJudgePrompt(instance: CorpusInstance, response: string): string {
|
||||
return JUDGE_PROMPT_TEMPLATE
|
||||
.replace('###PERSONA_SCENARIO_QUESTION###', `${instance.personaText}\n\nQUESTION: ${instance.question}`)
|
||||
.replace('###MATERIALS###', instance.materialsConcat)
|
||||
.replace('###RESPONSE###', response);
|
||||
}
|
||||
|
||||
async function runJudge(judgeModel: string, prompt: string): Promise<JudgeRecord> {
|
||||
let lastErr = '';
|
||||
let totalCost = 0, totalLatency = 0;
|
||||
for (let attempt = 0; attempt < JUDGE_RETRIES; attempt++) {
|
||||
const r = await llmCall({ model: judgeModel, messages: [{ role: 'user', content: prompt }], maxTokens: JUDGE_MAX_TOKENS, thinking: false });
|
||||
totalCost += r.costUsd; totalLatency += r.latencyMs;
|
||||
if (r.error) { lastErr = `attempt ${attempt + 1}: ${r.error}`; continue; }
|
||||
const parsed = parseJudgeJson(r.content);
|
||||
if (parsed) {
|
||||
return { ...parsed, judge_model: judgeModel, judge_cost_usd: totalCost, judge_latency_ms: totalLatency, judge_retries: attempt };
|
||||
}
|
||||
lastErr = `attempt ${attempt + 1}: malformed JSON: ${r.content.slice(0, 100)}`;
|
||||
}
|
||||
log(`[judge ${judgeModel}] FAILED after ${JUDGE_RETRIES}: ${lastErr}`);
|
||||
return { ...ZERO_VERDICT, judge_model: judgeModel, judge_cost_usd: totalCost, judge_latency_ms: totalLatency, judge_retries: JUDGE_RETRIES, rationale: `__JUDGE_FAILED__: ${lastErr}` };
|
||||
}
|
||||
|
||||
interface TrioResult {
|
||||
records: JudgeRecord[];
|
||||
trioMean: number;
|
||||
trioStrictPassII: boolean; // op (ii) — trio_mean >= 4.0
|
||||
trioStrictPassI: boolean; // op (i) — >=2 of 3 judges with mean >= 3.5
|
||||
judgeCostTotal: number;
|
||||
}
|
||||
|
||||
async function judgeTrio(instance: CorpusInstance, response: string): Promise<TrioResult> {
|
||||
const prompt = buildJudgePrompt(instance, response);
|
||||
const records = await Promise.all(JUDGES.map(j => runJudge(j, prompt)));
|
||||
const validMeans = records.filter(r => r.mean > 0).map(r => r.mean);
|
||||
const trioMean = validMeans.length > 0 ? validMeans.reduce((s, m) => s + m, 0) / validMeans.length : 0;
|
||||
const trioStrictPassII = trioMean >= 4.0;
|
||||
const trioStrictPassI = records.filter(r => r.mean >= 3.5).length >= 2;
|
||||
const judgeCostTotal = records.reduce((s, r) => s + r.judge_cost_usd, 0);
|
||||
return { records, trioMean, trioStrictPassII, trioStrictPassI, judgeCostTotal };
|
||||
}
|
||||
|
||||
// ── Per-eval orchestration: run shape × instance via multi-step ───────────
|
||||
|
||||
interface EvalRecord {
|
||||
shape: ShapeName;
|
||||
instanceId: string;
|
||||
instanceCell: CorpusInstance['cell'];
|
||||
candidateResponse: string;
|
||||
candidateLatencyMs: number;
|
||||
candidateTokensIn: number;
|
||||
candidateTokensOut: number;
|
||||
candidateCostUsd: number;
|
||||
loopExhausted: boolean;
|
||||
stepsTaken: number;
|
||||
retrievalCalls: number;
|
||||
judges: { records: JudgeRecord[]; trioMean: number; trioStrictPassII: boolean; trioStrictPassI: boolean; judgeCostTotal: number };
|
||||
evalCostUsd: number;
|
||||
manifestAnchor: string;
|
||||
manifestShaAmendment4: string;
|
||||
tsIso: string;
|
||||
}
|
||||
|
||||
async function runOneEval(shape: PromptShape, instance: CorpusInstance, embedder: Embedder): Promise<EvalRecord | { error: string }> {
|
||||
const evalId = `${shape.name}__${instance.instanceId}`;
|
||||
log(`[${evalId}] start`);
|
||||
|
||||
// Per-eval SQLite + HybridSearch substrate (per pilot runner pattern)
|
||||
const dbPath = path.join(SCRATCH_DIR, `eval-${shape.name}-${instance.instanceId}.sqlite`);
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
const db = new MindDB(dbPath);
|
||||
const frames = new FrameStore(db);
|
||||
const sessions = new SessionStore(db);
|
||||
const hybrid = new HybridSearch(db, embedder);
|
||||
|
||||
const gopId = `gepa-faza1-null-${shape.name}-${instance.instanceId}`;
|
||||
sessions.ensure(gopId, undefined, `NULL-baseline ${shape.name} on ${instance.instanceId}`);
|
||||
|
||||
// Ingest source documents
|
||||
for (const doc of instance.sourceDocuments) {
|
||||
const content = `## ${doc.title}\n\n${doc.body}`;
|
||||
frames.createIFrame(gopId, content, 'important', 'system');
|
||||
}
|
||||
log(`[${evalId}] ingested ${instance.sourceDocuments.length} frames`);
|
||||
|
||||
// Retrieval adapter
|
||||
const searchAdapter: RetrievalSearchFn = async ({ query, limit }) => {
|
||||
const hits = await hybrid.search(query, { limit, gopId });
|
||||
const formatted = hits.length > 0
|
||||
? hits.map((sr, i) => `[result ${i + 1}, score ${sr.finalScore.toFixed(3)}]\n${sr.frame.content}`).join('\n\n---\n\n')
|
||||
: '';
|
||||
return { formattedResults: formatted, resultCount: hits.length };
|
||||
};
|
||||
|
||||
// Run multi-step retrieval agent loop.
|
||||
// CRITICAL: pass promptShapeOverride to actually apply the per-shape prompt.
|
||||
// Without this, runRetrievalAgentLoop calls selectShape(modelAlias) which always
|
||||
// resolves to 'qwen-thinking' for Qwen subject (per config), making the runner's
|
||||
// shape parameter unused. This bug was discovered post-Checkpoint-A and corrected
|
||||
// per PM Option A ratification + manifest v7 Amendment 6.
|
||||
let agentResult: AgentRunResult;
|
||||
try {
|
||||
agentResult = await runRetrievalAgentLoop({
|
||||
modelAlias: SUBJECT_ALIAS,
|
||||
persona: instance.personaText,
|
||||
question: instance.question,
|
||||
llmCall,
|
||||
search: searchAdapter,
|
||||
maxSteps: MAX_STEPS,
|
||||
maxRetrievalsPerStep: MAX_RETRIEVALS_PER_STEP,
|
||||
perCallHaltUsd: PER_CALL_HALT_USD,
|
||||
perCellHaltUsd: PER_CELL_HALT_USD,
|
||||
contextTag: evalId,
|
||||
promptShapeOverride: shape.name, // bug fix per Amendment 6
|
||||
} as any);
|
||||
} catch (e) {
|
||||
const msg = `agent loop failed: ${(e as Error).message}`;
|
||||
log(`[${evalId}] ${msg}`);
|
||||
return { error: msg };
|
||||
}
|
||||
|
||||
if (agentResult.errors.length > 0) {
|
||||
log(`[${evalId}] agent errors: ${agentResult.errors.join('; ').slice(0, 200)}`);
|
||||
}
|
||||
|
||||
log(`[${evalId}] subject_done; retrievals=${agentResult.retrievalCalls} steps=${agentResult.stepsTaken} cost=$${agentResult.totalCostUsd.toFixed(4)} loop_exhausted=${agentResult.loopExhausted}`);
|
||||
|
||||
// Judge response
|
||||
const judges = await judgeTrio(instance, agentResult.rawResponse);
|
||||
const evalCostUsd = agentResult.totalCostUsd + judges.judgeCostTotal;
|
||||
log(`[${evalId}] judged; trio_mean=${judges.trioMean.toFixed(3)} pass_ii=${judges.trioStrictPassII} pass_i=${judges.trioStrictPassI} eval_cost=$${evalCostUsd.toFixed(4)}`);
|
||||
|
||||
return {
|
||||
shape: shape.name as ShapeName,
|
||||
instanceId: instance.instanceId,
|
||||
instanceCell: instance.cell,
|
||||
candidateResponse: agentResult.rawResponse,
|
||||
candidateLatencyMs: agentResult.totalLatencyMs,
|
||||
candidateTokensIn: agentResult.totalTokensIn,
|
||||
candidateTokensOut: agentResult.totalTokensOut,
|
||||
candidateCostUsd: agentResult.totalCostUsd,
|
||||
loopExhausted: agentResult.loopExhausted,
|
||||
stepsTaken: agentResult.stepsTaken,
|
||||
retrievalCalls: agentResult.retrievalCalls,
|
||||
judges,
|
||||
evalCostUsd,
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment4: MANIFEST_SHA_AMENDMENT_4,
|
||||
tsIso: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Aggregate per-shape metrics + κ across batch ──────────────────────────
|
||||
|
||||
interface ShapeAggregate {
|
||||
shape: ShapeName;
|
||||
nEvals: number;
|
||||
trioStrictPassRateII: number; // op (ii) primary
|
||||
trioStrictPassRateI: number; // op (i) supplementary
|
||||
meanRetrievalCallsPerTask: number;
|
||||
meanCandidateCostUsd: number;
|
||||
meanJudgeCostUsd: number;
|
||||
meanEvalCostUsd: number;
|
||||
totalEvalCostUsd: number;
|
||||
loopExhaustedRate: number;
|
||||
meanStepsTaken: number;
|
||||
}
|
||||
|
||||
function aggregatePerShape(records: EvalRecord[]): ShapeAggregate[] {
|
||||
const byShape = new Map<ShapeName, EvalRecord[]>();
|
||||
for (const r of records) {
|
||||
if (!byShape.has(r.shape)) byShape.set(r.shape, []);
|
||||
byShape.get(r.shape)!.push(r);
|
||||
}
|
||||
const out: ShapeAggregate[] = [];
|
||||
for (const shape of SHAPES) {
|
||||
const rs = byShape.get(shape) ?? [];
|
||||
if (rs.length === 0) continue;
|
||||
const passII = rs.filter(r => r.judges.trioStrictPassII).length;
|
||||
const passI = rs.filter(r => r.judges.trioStrictPassI).length;
|
||||
const meanRetr = rs.reduce((s, r) => s + r.retrievalCalls, 0) / rs.length;
|
||||
const meanCandCost = rs.reduce((s, r) => s + r.candidateCostUsd, 0) / rs.length;
|
||||
const meanJudgeCost = rs.reduce((s, r) => s + r.judges.judgeCostTotal, 0) / rs.length;
|
||||
const meanEvalCost = rs.reduce((s, r) => s + r.evalCostUsd, 0) / rs.length;
|
||||
const totalEvalCost = rs.reduce((s, r) => s + r.evalCostUsd, 0);
|
||||
const exhaustedRate = rs.filter(r => r.loopExhausted).length / rs.length;
|
||||
const meanSteps = rs.reduce((s, r) => s + r.stepsTaken, 0) / rs.length;
|
||||
out.push({
|
||||
shape, nEvals: rs.length,
|
||||
trioStrictPassRateII: passII / rs.length,
|
||||
trioStrictPassRateI: passI / rs.length,
|
||||
meanRetrievalCallsPerTask: meanRetr,
|
||||
meanCandidateCostUsd: meanCandCost,
|
||||
meanJudgeCostUsd: meanJudgeCost,
|
||||
meanEvalCostUsd: meanEvalCost,
|
||||
totalEvalCostUsd: totalEvalCost,
|
||||
loopExhaustedRate: exhaustedRate,
|
||||
meanStepsTaken: meanSteps,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
fs.mkdirSync(SCRATCH_DIR, { recursive: true });
|
||||
if (!fs.existsSync(RUN_LOG)) fs.writeFileSync(RUN_LOG, '');
|
||||
|
||||
const corpus = loadCorpus();
|
||||
log(`[loaded] corpus = ${corpus.length} instances from ${CORPUS_JSONL}`);
|
||||
|
||||
const sample = sampleInstances(corpus, N_PER_SHAPE, SAMPLING_SEED);
|
||||
log(`[sampled] N=${N_PER_SHAPE} via seed=${SAMPLING_SEED}: ${sample.map(i => i.instanceId).join(', ')}`);
|
||||
|
||||
if (args.mode === 'dry-run') {
|
||||
log(`[dry-run] would run ${SHAPES.length} shapes × ${N_PER_SHAPE} instances = ${SHAPES.length * N_PER_SHAPE} evals`);
|
||||
for (const shapeName of SHAPES) {
|
||||
const shape = REGISTRY[shapeName];
|
||||
const sysPrompt = shape.systemPrompt({ persona: sample[0].personaText, question: sample[0].question, isMultiStep: true });
|
||||
log(`[dry-run] shape=${shapeName} systemPrompt(multi-step)=${sysPrompt.length}c`);
|
||||
}
|
||||
log(`[dry-run] OK; no LLM call; cost: $0.00`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume support: skip evals already in JSONL
|
||||
const existing = new Set<string>();
|
||||
let cumulativeCost = 0;
|
||||
let allRecords: EvalRecord[] = [];
|
||||
if (fs.existsSync(OUT_JSONL)) {
|
||||
for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').trim().split(/\n+/).filter(Boolean)) {
|
||||
try {
|
||||
const r = JSON.parse(line) as EvalRecord;
|
||||
const key = `${r.shape}__${r.instanceId}`;
|
||||
existing.add(key);
|
||||
cumulativeCost += r.evalCostUsd;
|
||||
allRecords.push(r);
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
log(`[resume] loaded ${existing.size} existing evals; cumulative cost $${cumulativeCost.toFixed(4)}`);
|
||||
}
|
||||
|
||||
const out = fs.createWriteStream(OUT_JSONL, { flags: existing.size > 0 ? 'a' : 'w' });
|
||||
|
||||
const embedder = createOllamaEmbedder({ baseUrl: OLLAMA_URL, model: EMBEDDER_MODEL });
|
||||
|
||||
const targetShapes = args.mode === 'probe' ? [args.probeShape ?? 'qwen-thinking'] : SHAPES;
|
||||
const targetInstances = args.mode === 'probe'
|
||||
? [sample[args.probeInstanceIdx ?? 0]]
|
||||
: sample;
|
||||
|
||||
for (const shapeName of targetShapes) {
|
||||
const shape = REGISTRY[shapeName];
|
||||
if (!shape) { log(`[error] shape "${shapeName}" not in REGISTRY`); continue; }
|
||||
for (const instance of targetInstances) {
|
||||
const key = `${shapeName}__${instance.instanceId}`;
|
||||
if (existing.has(key)) {
|
||||
log(`[skip] ${key} already in JSONL`);
|
||||
continue;
|
||||
}
|
||||
if (cumulativeCost >= COST_HALT_USD) {
|
||||
log(`[HALT] cumulative $${cumulativeCost.toFixed(4)} >= $${COST_HALT_USD} cost halt — stopping`);
|
||||
break;
|
||||
}
|
||||
const result = await runOneEval(shape, instance, embedder);
|
||||
if ('error' in result) {
|
||||
log(`[skip] ${key} due to: ${result.error}`);
|
||||
continue;
|
||||
}
|
||||
out.write(JSON.stringify(result) + '\n');
|
||||
cumulativeCost += result.evalCostUsd;
|
||||
allRecords.push(result);
|
||||
log(`[cumulative] $${cumulativeCost.toFixed(4)} / $${COST_HALT_USD} halt; ${allRecords.length} evals total`);
|
||||
}
|
||||
}
|
||||
out.end();
|
||||
|
||||
// Aggregate + summary
|
||||
const aggregates = aggregatePerShape(allRecords);
|
||||
const summary = {
|
||||
manifestAnchor: MANIFEST_ANCHOR,
|
||||
manifestShaAmendment4: MANIFEST_SHA_AMENDMENT_4,
|
||||
samplingSeed: SAMPLING_SEED,
|
||||
nPerShape: N_PER_SHAPE,
|
||||
sampledInstanceIds: sample.map(i => i.instanceId),
|
||||
totalEvals: allRecords.length,
|
||||
totalCostUsd: +cumulativeCost.toFixed(6),
|
||||
perShape: aggregates,
|
||||
completedAtIso: new Date().toISOString(),
|
||||
};
|
||||
fs.writeFileSync(SUMMARY_JSON, JSON.stringify(summary, null, 2), 'utf-8');
|
||||
log(`[done] ${allRecords.length} evals; total cost $${cumulativeCost.toFixed(4)}; summary written`);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error('FATAL:', e);
|
||||
process.exit(2);
|
||||
});
|
||||
Reference in New Issue
Block a user