138 lines
6.2 KiB
TypeScript
138 lines
6.2 KiB
TypeScript
#!/usr/bin/env tsx
|
||
/**
|
||
* E1 aggregation — Eywa answers @ OUR gpt-5 judge vs Eywa self-judge.
|
||
* Reads e1-eywa-ourjudge.jsonl + the original Eywa JSON, emits console + markdown.
|
||
*/
|
||
import fs from 'node:fs';
|
||
|
||
const EYWA_JSON =
|
||
'D:/Projects/KorroResearch/benchmarks/eywa-artifacts/eywa-beam-sonnet46-answers.json';
|
||
const OUT_JSONL =
|
||
'D:/Projects/KorroResearch/benchmarks/eywa-artifacts/e1-eywa-ourjudge.jsonl';
|
||
|
||
interface EywaRecord { id: string; cat: string; eywaNugget: number[]; }
|
||
interface OutRecord {
|
||
id: string; cat: string; our_raw_score: number;
|
||
our_nugget_scores: number[]; eywa_raw: number;
|
||
}
|
||
|
||
const OUR_OWN_HEADLINE = 0.6482; // our own answers @ our gpt-5 judge
|
||
|
||
function mean(xs: number[]): number { return xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0; }
|
||
function pct(x: number): string { return (x * 100).toFixed(2) + '%'; }
|
||
|
||
const eywaRecords = JSON.parse(fs.readFileSync(EYWA_JSON, 'utf-8')) as EywaRecord[];
|
||
const eywaById = new Map(eywaRecords.map(r => [r.id, r]));
|
||
|
||
const out: OutRecord[] = [];
|
||
for (const line of fs.readFileSync(OUT_JSONL, 'utf-8').split('\n')) {
|
||
const t = line.trim(); if (!t) continue;
|
||
out.push(JSON.parse(t) as OutRecord);
|
||
}
|
||
// dedup by id (last wins)
|
||
const outById = new Map(out.map(r => [r.id, r]));
|
||
const rows = [...outById.values()];
|
||
|
||
console.log(`[agg] scored records: ${rows.length}/700`);
|
||
|
||
// Overall
|
||
const ourOverall = mean(rows.map(r => r.our_raw_score));
|
||
const eywaOverall = mean(rows.map(r => r.eywa_raw));
|
||
|
||
// Per-category
|
||
const cats = [...new Set(rows.map(r => r.cat))].sort();
|
||
interface CatStat { cat: string; n: number; our: number; eywa: number; delta: number; }
|
||
const catStats: CatStat[] = cats.map(cat => {
|
||
const rs = rows.filter(r => r.cat === cat);
|
||
const our = mean(rs.map(r => r.our_raw_score));
|
||
const eywa = mean(rs.map(r => r.eywa_raw));
|
||
return { cat, n: rs.length, our, eywa, delta: our - eywa };
|
||
});
|
||
|
||
// Per-nugget agreement (align our_nugget_scores to eywaNugget by index)
|
||
let nTotal = 0, nOurLess = 0, nOurMore = 0, nEqual = 0;
|
||
let sumOur = 0, sumEywa = 0;
|
||
for (const r of rows) {
|
||
const e = eywaById.get(r.id);
|
||
if (!e) continue;
|
||
const our = r.our_nugget_scores;
|
||
const eyw = e.eywaNugget;
|
||
const k = Math.min(our.length, eyw.length);
|
||
for (let i = 0; i < k; i++) {
|
||
nTotal++;
|
||
sumOur += our[i]; sumEywa += eyw[i];
|
||
if (our[i] < eyw[i]) nOurLess++;
|
||
else if (our[i] > eyw[i]) nOurMore++;
|
||
else nEqual++;
|
||
}
|
||
}
|
||
|
||
// ── Console report ──
|
||
console.log('\n=== OVERALL ===');
|
||
console.log(`Eywa answers @ OUR gpt-5 judge : ${pct(ourOverall)}`);
|
||
console.log(`Eywa self-judge (Sonnet 4.6) : ${pct(eywaOverall)} (published headline 81.45%)`);
|
||
console.log(`Self-judge inflation delta : ${((eywaOverall - ourOverall) * 100).toFixed(2)} pp`);
|
||
console.log(`\nHONEST SAME-JUDGE LEADERBOARD (gpt-5 nugget judge):`);
|
||
console.log(` Eywa answers @ gpt-5 judge : ${pct(ourOverall)}`);
|
||
console.log(` OUR answers @ gpt-5 judge : ${pct(OUR_OWN_HEADLINE)}`);
|
||
console.log(` Eywa - ours = ${((ourOverall - OUR_OWN_HEADLINE) * 100).toFixed(2)} pp`);
|
||
|
||
console.log('\n=== PER CATEGORY ===');
|
||
console.log('cat'.padEnd(28), 'n'.padStart(4), 'ourJ%'.padStart(8), 'eywaSelf%'.padStart(10), 'delta_pp'.padStart(9));
|
||
for (const c of catStats.sort((a, b) => a.delta - b.delta)) {
|
||
console.log(
|
||
c.cat.padEnd(28), String(c.n).padStart(4),
|
||
pct(c.our).padStart(8), pct(c.eywa).padStart(10),
|
||
((c.delta) * 100).toFixed(2).padStart(9),
|
||
);
|
||
}
|
||
|
||
console.log('\n=== PER-NUGGET AGREEMENT ===');
|
||
console.log(`total nuggets compared: ${nTotal}`);
|
||
console.log(`our < eywa (we stricter): ${nOurLess} (${pct(nOurLess / nTotal)})`);
|
||
console.log(`our > eywa (we lenient) : ${nOurMore} (${pct(nOurMore / nTotal)})`);
|
||
console.log(`equal : ${nEqual} (${pct(nEqual / nTotal)})`);
|
||
console.log(`mean(our) - mean(eywa) per nugget: ${((sumOur - sumEywa) / nTotal).toFixed(4)}`);
|
||
|
||
// ── Markdown ──
|
||
const md: string[] = [];
|
||
md.push('# E1 — Eywa BEAM answers re-judged with OUR canonical gpt-5 nugget judge\n');
|
||
md.push(`_Judge: gpt-5 (model id \`gpt-5\`), verbatim mem0 graded nugget prompt from \`src/beam-nugget-judge.ts\` — the SAME judge behind our published 0.6482 headline. Per-question score = mean of 0/0.5/1 nugget scores._\n`);
|
||
md.push(`_Scored ${rows.length}/700 Eywa Sonnet-4.6 answers over ${nTotal} rubric nuggets._\n`);
|
||
md.push('## Headline\n');
|
||
md.push('| Metric | Value |');
|
||
md.push('|---|---|');
|
||
md.push(`| Eywa answers @ **our gpt-5 judge** | **${pct(ourOverall)}** |`);
|
||
md.push(`| Eywa self-judge (Sonnet 4.6), our recompute | ${pct(eywaOverall)} |`);
|
||
md.push(`| Eywa published headline | 81.45% |`);
|
||
md.push(`| **Self-judge inflation** (eywa-self − our-judge) | **${((eywaOverall - ourOverall) * 100).toFixed(2)} pp** |`);
|
||
md.push('');
|
||
md.push('## Honest same-judge leaderboard (both @ gpt-5 nugget judge)\n');
|
||
md.push('| System | Answers @ gpt-5 judge |');
|
||
md.push('|---|---|');
|
||
md.push(`| Eywa (Sonnet 4.6 answers) | **${pct(ourOverall)}** |`);
|
||
md.push(`| Ours (published) | ${pct(OUR_OWN_HEADLINE)} |`);
|
||
md.push(`| Gap (Eywa − ours) | ${((ourOverall - OUR_OWN_HEADLINE) * 100).toFixed(2)} pp |`);
|
||
md.push('');
|
||
md.push('## Per-category (n=70 each)\n');
|
||
md.push('| Category | n | Our gpt-5 judge % | Eywa self-judge % | Delta (pp) |');
|
||
md.push('|---|---|---|---|---|');
|
||
for (const c of catStats.sort((a, b) => a.delta - b.delta)) {
|
||
md.push(`| ${c.cat} | ${c.n} | ${pct(c.our)} | ${pct(c.eywa)} | ${(c.delta * 100).toFixed(2)} |`);
|
||
}
|
||
md.push('');
|
||
md.push('_Delta = our-judge − eywa-self-judge. Large negative delta = Eywa self-judge inflated that category. Small delta = self-judge was honest there._\n');
|
||
md.push('## Per-nugget judge agreement\n');
|
||
md.push(`- Total nuggets compared: ${nTotal}`);
|
||
md.push(`- Our score **< **Eywa (we stricter / leniency in their favor): ${nOurLess} (${pct(nOurLess / nTotal)})`);
|
||
md.push(`- Our score **>** Eywa (we more lenient): ${nOurMore} (${pct(nOurMore / nTotal)})`);
|
||
md.push(`- Equal: ${nEqual} (${pct(nEqual / nTotal)})`);
|
||
md.push(`- mean(our) − mean(eywa) per nugget: **${((sumOur - sumEywa) / nTotal).toFixed(4)}**`);
|
||
md.push('');
|
||
|
||
fs.writeFileSync(
|
||
'D:/Projects/KorroResearch/benchmarks/eywa-artifacts/E1-eywa-answers-ourjudge.md',
|
||
md.join('\n'),
|
||
);
|
||
console.log('\n[agg] wrote E1-eywa-answers-ourjudge.md');
|