This commit is contained in:
Oleg Maslov
317
scripts/analyze-ensemble-baseline.mjs
Normal file
317
scripts/analyze-ensemble-baseline.mjs
Normal file
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env node
|
||||
// Sprint 10 Task 2.1 — Ensemble baseline analyzer.
|
||||
//
|
||||
// Reads a judge-calibration ensemble JSON artifact and reports:
|
||||
// 1. Per-vendor match rate vs PM ground-truth labels (verdict + failure_mode).
|
||||
// 2. Per-pair Cohen-style agreement (verdict+failure_mode joint category).
|
||||
// 3. Dataset-wide Fleiss' kappa across all instances × 3 raters.
|
||||
// 4. Cost + latency per vendor (verdict-specific breakdown).
|
||||
// 5. Instance-level disagreement log.
|
||||
//
|
||||
// This is the analysis half of Task 2.1. The calibration artifact itself
|
||||
// is produced by judge-calibration.mjs --ensemble ... — this script
|
||||
// consumes that artifact and produces the baseline markdown report.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/analyze-ensemble-baseline.mjs \
|
||||
// --in preflight-results/judge-calibration-ensemble-<ISO>.json \
|
||||
// [--out docs/reports/multi-vendor-ensemble-baseline-<ISO>.md]
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const args = (() => {
|
||||
const out = { inPath: undefined, outPath: undefined };
|
||||
const argv = process.argv.slice(2);
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--in') { out.inPath = argv[++i]; }
|
||||
else if (argv[i] === '--out') { out.outPath = argv[++i]; }
|
||||
}
|
||||
if (!out.inPath) throw new Error('--in <path-to-ensemble-json> required');
|
||||
if (!out.outPath) {
|
||||
const base = path.basename(out.inPath, '.json').replace('judge-calibration-ensemble-', 'multi-vendor-ensemble-baseline-');
|
||||
out.outPath = `docs/reports/${base}.md`;
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
|
||||
// ── Fleiss' kappa across N items × k raters × c categories ────────────────
|
||||
|
||||
/**
|
||||
* Fleiss' kappa for multiple raters on categorical ratings.
|
||||
* Input: n×c matrix where cell [i][j] = number of raters who assigned
|
||||
* category j to item i. Sum of each row = number of raters (constant k).
|
||||
*
|
||||
* Formula per Fleiss (1971):
|
||||
* P_j = sum_i(n_ij) / (N * k) # overall proportion of cat j
|
||||
* P_i = (1/(k*(k-1))) * (sum_j n_ij^2 - k) # agreement on item i
|
||||
* P_bar = mean(P_i)
|
||||
* P_e = sum_j P_j^2
|
||||
* kappa = (P_bar - P_e) / (1 - P_e)
|
||||
*/
|
||||
function fleissKappa(matrix) {
|
||||
const N = matrix.length;
|
||||
if (N === 0) return NaN;
|
||||
const c = matrix[0].length;
|
||||
const k = matrix[0].reduce((s, v) => s + v, 0);
|
||||
if (k < 2) return NaN;
|
||||
|
||||
// P_j
|
||||
const totalRatings = N * k;
|
||||
const P_j = new Array(c).fill(0);
|
||||
for (const row of matrix) {
|
||||
for (let j = 0; j < c; j++) P_j[j] += row[j];
|
||||
}
|
||||
for (let j = 0; j < c; j++) P_j[j] /= totalRatings;
|
||||
|
||||
// P_i and mean
|
||||
let P_bar = 0;
|
||||
for (const row of matrix) {
|
||||
let sumSq = 0;
|
||||
for (let j = 0; j < c; j++) sumSq += row[j] * row[j];
|
||||
const Pi = (sumSq - k) / (k * (k - 1));
|
||||
P_bar += Pi;
|
||||
}
|
||||
P_bar /= N;
|
||||
|
||||
// P_e
|
||||
let P_e = 0;
|
||||
for (let j = 0; j < c; j++) P_e += P_j[j] * P_j[j];
|
||||
|
||||
if (P_e >= 1) return NaN;
|
||||
return (P_bar - P_e) / (1 - P_e);
|
||||
}
|
||||
|
||||
/** Cohen's kappa between exactly two raters on categorical ratings. */
|
||||
function cohensKappa(rater1, rater2) {
|
||||
if (rater1.length !== rater2.length || rater1.length === 0) return NaN;
|
||||
const n = rater1.length;
|
||||
const categories = new Set([...rater1, ...rater2]);
|
||||
let agree = 0;
|
||||
for (let i = 0; i < n; i++) if (rater1[i] === rater2[i]) agree++;
|
||||
const P_o = agree / n;
|
||||
let P_e = 0;
|
||||
for (const cat of categories) {
|
||||
const p1 = rater1.filter(c => c === cat).length / n;
|
||||
const p2 = rater2.filter(c => c === cat).length / n;
|
||||
P_e += p1 * p2;
|
||||
}
|
||||
if (P_e >= 1) return NaN;
|
||||
return (P_o - P_e) / (1 - P_e);
|
||||
}
|
||||
|
||||
function interpretKappa(k) {
|
||||
if (isNaN(k)) return 'undefined';
|
||||
if (k < 0) return 'worse than chance';
|
||||
if (k < 0.20) return 'slight';
|
||||
if (k < 0.40) return 'fair';
|
||||
if (k < 0.60) return 'moderate';
|
||||
if (k < 0.80) return 'substantial';
|
||||
return 'strong';
|
||||
}
|
||||
|
||||
// ── Joint category builder ───────────────────────────────────────────────
|
||||
|
||||
/** Joint category key for (verdict, failure_mode). Matches PM label space. */
|
||||
function joinLabel(verdict, failureMode) {
|
||||
const fm = failureMode === null || failureMode === undefined ? 'null' : failureMode;
|
||||
return `${verdict}/${fm}`;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(args.inPath, 'utf-8'));
|
||||
const instances = data.perInstance;
|
||||
const ensembleModels = data.ensemble;
|
||||
const majorityMatches = data.matchRate.matches;
|
||||
const total = data.matchRate.total;
|
||||
|
||||
// Per-vendor match rate vs PM
|
||||
const perVendor = ensembleModels.map(model => {
|
||||
let matches = 0;
|
||||
const disagreements = [];
|
||||
for (const inst of instances) {
|
||||
const vendorVerdict = inst.judgeOutput.ensemble.find(e => e.model === model);
|
||||
if (!vendorVerdict) continue;
|
||||
const pmLabel = joinLabel(inst.humanVerdict, inst.humanFailureMode);
|
||||
const vendorLabel = joinLabel(vendorVerdict.verdict, vendorVerdict.failure_mode);
|
||||
if (pmLabel === vendorLabel) matches++;
|
||||
else disagreements.push({
|
||||
index: inst.index,
|
||||
instanceId: inst.instanceId,
|
||||
pm: pmLabel,
|
||||
vendor: vendorLabel,
|
||||
rationale: vendorVerdict.rationale,
|
||||
});
|
||||
}
|
||||
return { model, matches, total: instances.length, disagreements };
|
||||
});
|
||||
|
||||
// Per-pair Cohen's kappa (joint labels across vendors only, not PM)
|
||||
const pairKappas = [];
|
||||
for (let i = 0; i < ensembleModels.length; i++) {
|
||||
for (let j = i + 1; j < ensembleModels.length; j++) {
|
||||
const labels1 = instances.map(inst => {
|
||||
const v = inst.judgeOutput.ensemble.find(e => e.model === ensembleModels[i]);
|
||||
return joinLabel(v.verdict, v.failure_mode);
|
||||
});
|
||||
const labels2 = instances.map(inst => {
|
||||
const v = inst.judgeOutput.ensemble.find(e => e.model === ensembleModels[j]);
|
||||
return joinLabel(v.verdict, v.failure_mode);
|
||||
});
|
||||
const agree = labels1.filter((l, k) => l === labels2[k]).length;
|
||||
const k = cohensKappa(labels1, labels2);
|
||||
pairKappas.push({ a: ensembleModels[i], b: ensembleModels[j], kappa: k, agreePct: (agree / labels1.length) * 100 });
|
||||
}
|
||||
}
|
||||
|
||||
// Dataset-wide Fleiss' kappa across 3 vendor raters on joint labels
|
||||
const allCategories = new Set();
|
||||
for (const inst of instances) {
|
||||
for (const v of inst.judgeOutput.ensemble) allCategories.add(joinLabel(v.verdict, v.failure_mode));
|
||||
}
|
||||
const cats = [...allCategories].sort();
|
||||
const fleissMatrix = instances.map(inst => {
|
||||
const row = new Array(cats.length).fill(0);
|
||||
for (const v of inst.judgeOutput.ensemble) {
|
||||
const lbl = joinLabel(v.verdict, v.failure_mode);
|
||||
row[cats.indexOf(lbl)]++;
|
||||
}
|
||||
return row;
|
||||
});
|
||||
const fleissK = fleissKappa(fleissMatrix);
|
||||
|
||||
// Dataset-wide Fleiss' kappa INCLUDING PM as a 4th rater (agreement with ground truth)
|
||||
const fleissMatrixWithPM = instances.map(inst => {
|
||||
const row = new Array(cats.length).fill(0);
|
||||
for (const v of inst.judgeOutput.ensemble) {
|
||||
const lbl = joinLabel(v.verdict, v.failure_mode);
|
||||
row[cats.indexOf(lbl)]++;
|
||||
}
|
||||
const pmLbl = joinLabel(inst.humanVerdict, inst.humanFailureMode);
|
||||
const pmIdx = cats.indexOf(pmLbl);
|
||||
if (pmIdx >= 0) {
|
||||
row[pmIdx]++;
|
||||
} else {
|
||||
// PM's label isn't in the vendors' category set — extend
|
||||
cats.push(pmLbl);
|
||||
row.push(1);
|
||||
}
|
||||
return row;
|
||||
});
|
||||
// Pad all rows to match new cats length
|
||||
for (const row of fleissMatrixWithPM) while (row.length < cats.length) row.push(0);
|
||||
const fleissKWithPM = fleissKappa(fleissMatrixWithPM);
|
||||
|
||||
// Cost by vendor
|
||||
const costByVendor = Object.fromEntries(ensembleModels.map(m => [m, { calls: 0, usd: 0, latencyMsTotal: 0 }]));
|
||||
for (const entry of data.cost.entries) {
|
||||
if (costByVendor[entry.model]) {
|
||||
costByVendor[entry.model].calls++;
|
||||
costByVendor[entry.model].usd += entry.usd;
|
||||
costByVendor[entry.model].latencyMsTotal += entry.latencyMs;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Emit markdown report ──────────────────────────────────────────────────
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const md = [];
|
||||
md.push(`# Multi-Vendor Ensemble Baseline — Sprint 10 Task 2.1`);
|
||||
md.push('');
|
||||
md.push(`**Generated:** ${now}`);
|
||||
md.push(`**Calibration artifact:** \`${args.inPath}\``);
|
||||
md.push(`**Labels source:** \`${data.labelsSource}\``);
|
||||
md.push(`**Ensemble vendors:** ${ensembleModels.join(', ')}`);
|
||||
md.push(`**Instances:** ${instances.length}`);
|
||||
md.push('');
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('## 1. Per-vendor match rate vs PM ground truth');
|
||||
md.push('');
|
||||
md.push('| Vendor | Match rate | Spend | Avg latency | Disagreements |');
|
||||
md.push('|---|---|---|---|---|');
|
||||
for (const pv of perVendor) {
|
||||
const cost = costByVendor[pv.model];
|
||||
const avgLatMs = cost.calls > 0 ? Math.round(cost.latencyMsTotal / cost.calls) : 0;
|
||||
md.push(`| \`${pv.model}\` | ${pv.matches}/${pv.total} | $${cost.usd.toFixed(6)} | ${avgLatMs}ms | ${pv.disagreements.length} |`);
|
||||
}
|
||||
md.push('');
|
||||
md.push(`**Ensemble majority match rate:** ${majorityMatches}/${total}`);
|
||||
md.push(`**Total ensemble spend:** $${data.cost.totalUsd.toFixed(6)} (${data.cost.judgeCalls} calls across ${ensembleModels.length} vendors × ${instances.length} instances)`);
|
||||
md.push('');
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('## 2. Pair-wise Cohen\'s kappa (inter-rater, vendors only)');
|
||||
md.push('');
|
||||
md.push('| Pair | Kappa | Band | Agree% |');
|
||||
md.push('|---|---|---|---|');
|
||||
for (const p of pairKappas) {
|
||||
md.push(`| \`${p.a}\` ↔ \`${p.b}\` | ${p.kappa.toFixed(4)} | ${interpretKappa(p.kappa)} | ${p.agreePct.toFixed(1)}% |`);
|
||||
}
|
||||
md.push('');
|
||||
md.push(`### Dataset-wide Fleiss' kappa`);
|
||||
md.push('');
|
||||
md.push(`- **Vendors only (3 raters):** κ = **${fleissK.toFixed(4)}** → ${interpretKappa(fleissK)}`);
|
||||
md.push(`- **Vendors + PM (4 raters):** κ = **${fleissKWithPM.toFixed(4)}** → ${interpretKappa(fleissKWithPM)}`);
|
||||
md.push('');
|
||||
md.push('### Interpretation band (brief §2.2)');
|
||||
md.push('');
|
||||
md.push('| κ range | Band | Stage 2 implication |');
|
||||
md.push('|---|---|---|');
|
||||
md.push('| ≥ 0.80 | strong | ensemble ready; ensemble verdict primary |');
|
||||
md.push('| 0.60 — 0.80 | substantial | ensemble ready + tie-breaker policy documented |');
|
||||
md.push('| 0.40 — 0.60 | moderate | **PM review required before Stage 2 kickoff** |');
|
||||
md.push('| < 0.40 | fair or worse | **go/no-go review**; scope pivot to single-judge Opus + rubric refinement |');
|
||||
md.push('');
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('## 3. Disagreement log (vendor vs PM)');
|
||||
md.push('');
|
||||
for (const pv of perVendor) {
|
||||
if (pv.disagreements.length === 0) continue;
|
||||
md.push(`### \`${pv.model}\` — ${pv.disagreements.length} disagreement(s)`);
|
||||
md.push('');
|
||||
for (const d of pv.disagreements) {
|
||||
md.push(`**Instance ${d.index}** (${d.instanceId})`);
|
||||
md.push(`- PM: \`${d.pm}\``);
|
||||
md.push(`- Vendor: \`${d.vendor}\``);
|
||||
md.push(`- Rationale: ${d.rationale}`);
|
||||
md.push('');
|
||||
}
|
||||
}
|
||||
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('## 4. Notes for Stage 2 primary-judge selection');
|
||||
md.push('');
|
||||
md.push('Sprint 10 brief §1.3 + §2.1 decision tree:');
|
||||
md.push('- Task 1.3 Sonnet calibration produced 8/10 match → borderline 7-8 band → **multi-vendor kappa required before Stage 2 primary lock**.');
|
||||
md.push(`- Task 2.1 ensemble majority produced ${majorityMatches}/${total} match → ${majorityMatches === total ? 'unanimous with PM' : majorityMatches >= Math.floor(total * 0.9) ? 'near-unanimous' : 'moderate agreement'} with PM on the current 10-instance dataset.`);
|
||||
md.push(`- Fleiss' κ (vendors only) = ${fleissK.toFixed(3)} → **${interpretKappa(fleissK)}** band.`);
|
||||
md.push('');
|
||||
md.push(`**Task 2.2 scope:** brief §2.2 calls for 15 triples (10 Sprint-9 + 5 new PM-authored) to extend this baseline. The 10-instance result above is INDICATIVE, not final — full band assessment requires the additional 5 triples to avoid small-sample bias.`);
|
||||
md.push('');
|
||||
md.push('**Open signal (flagged to PM):**');
|
||||
md.push('- Instance 9 (`locomo_conv-50_q037`) — PM labeled `correct/null`; Haiku (Sprint 9 Task 4), Sonnet (Sprint 10 Task 1.3), and 2-of-3 ensemble vendors (GPT-5.4 + Gemini 3.1 Pro) flag `incorrect/F4` (fabrication: "touring with Frank Ocean" + "Tokyo stage").');
|
||||
md.push('- Sprint 9 Task 4 Opus 4.7 solo agreed with PM. Today\'s ensemble Opus 4.7 also agrees with PM.');
|
||||
md.push('- Signal: one PM label may warrant re-review. Not a judge weakness; a consistent-across-3-vendor-families disagreement on a specific instance.');
|
||||
md.push('');
|
||||
md.push('---');
|
||||
md.push('');
|
||||
md.push('*End of Task 2.1 baseline report. Task 2.2 (full 15-triple Fleiss\' kappa) opens next after PM authors the 5 additional ground-truth triples.*');
|
||||
|
||||
fs.mkdirSync(path.dirname(args.outPath), { recursive: true });
|
||||
fs.writeFileSync(args.outPath, md.join('\n') + '\n', 'utf-8');
|
||||
console.log(`[ensemble-baseline] wrote ${args.outPath}`);
|
||||
|
||||
// Also print the key numbers to stdout for quick eyeballing
|
||||
console.log(`[ensemble-baseline:summary] majority=${majorityMatches}/${total}`);
|
||||
for (const pv of perVendor) {
|
||||
console.log(` ${pv.model.padEnd(22,' ')} match=${pv.matches}/${pv.total} spend=$${costByVendor[pv.model].usd.toFixed(6)}`);
|
||||
}
|
||||
console.log(` fleiss_kappa_vendors_only=${fleissK.toFixed(4)} (${interpretKappa(fleissK)})`);
|
||||
console.log(` fleiss_kappa_vendors_plus_pm=${fleissKWithPM.toFixed(4)} (${interpretKappa(fleissKWithPM)})`);
|
||||
for (const p of pairKappas) {
|
||||
console.log(` cohen_kappa ${p.a} x ${p.b} = ${p.kappa.toFixed(4)} (${interpretKappa(p.kappa)}, ${p.agreePct.toFixed(1)}% agree)`);
|
||||
}
|
||||
Reference in New Issue
Block a user