394 lines
19 KiB
JavaScript
394 lines
19 KiB
JavaScript
#!/usr/bin/env node
|
||
// Sprint 10 Task F — close-out analysis on the 14-instance tri-vendor run.
|
||
//
|
||
// Reads the ensemble JSON produced by Task E and emits:
|
||
// 1. Overall Fleiss' κ (vendors only, vendors + PM).
|
||
// 2. Per-category Fleiss' κ (temporal-scope, null-result,
|
||
// chain-of-anchor, single-hop, multi-hop, open-ended).
|
||
// 3. Per-F-mode Fleiss' κ (F1, F2, F3, F4, F5, correct/null).
|
||
// 4. Per-vendor match rate vs PM labels across the 14-instance set.
|
||
// 5. GO/NO-GO signal per brief §pre-registered band (≥0.60 moderate
|
||
// floor authorizes Sprint 11 judge-methodology go-ahead).
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
|
||
const args = (() => {
|
||
const out = { inPath: undefined, outPath: undefined };
|
||
const argv = process.argv.slice(2);
|
||
for (let i = 0; i < argv.length; i++) {
|
||
if (argv[i] === '--in') { out.inPath = argv[++i]; }
|
||
else if (argv[i] === '--out') { out.outPath = argv[++i]; }
|
||
}
|
||
if (!out.inPath) throw new Error('--in <ensemble json> required');
|
||
if (!out.outPath) {
|
||
const base = path.basename(out.inPath, '.json');
|
||
out.outPath = `docs/plans/SPRINT-10-CLOSEOUT-2026-04-22.md`;
|
||
}
|
||
return out;
|
||
})();
|
||
|
||
// ── Kappa math (same as analyze-ensemble-baseline.mjs) ─────────────
|
||
|
||
function fleissKappa(matrix) {
|
||
const N = matrix.length;
|
||
if (N === 0) return NaN;
|
||
const c = matrix[0].length;
|
||
const k = matrix[0].reduce((s, v) => s + v, 0);
|
||
if (k < 2) return NaN;
|
||
const totalRatings = N * k;
|
||
const P_j = new Array(c).fill(0);
|
||
for (const row of matrix) for (let j = 0; j < c; j++) P_j[j] += row[j];
|
||
for (let j = 0; j < c; j++) P_j[j] /= totalRatings;
|
||
let P_bar = 0;
|
||
for (const row of matrix) {
|
||
let sumSq = 0;
|
||
for (let j = 0; j < c; j++) sumSq += row[j] * row[j];
|
||
const Pi = (sumSq - k) / (k * (k - 1));
|
||
P_bar += Pi;
|
||
}
|
||
P_bar /= N;
|
||
let P_e = 0;
|
||
for (let j = 0; j < c; j++) P_e += P_j[j] * P_j[j];
|
||
if (P_e >= 1) return NaN;
|
||
return (P_bar - P_e) / (1 - P_e);
|
||
}
|
||
|
||
function cohensKappa(r1, r2) {
|
||
if (r1.length !== r2.length || r1.length === 0) return NaN;
|
||
const n = r1.length;
|
||
const cats = new Set([...r1, ...r2]);
|
||
let ag = 0;
|
||
for (let i = 0; i < n; i++) if (r1[i] === r2[i]) ag++;
|
||
const P_o = ag / n;
|
||
let P_e = 0;
|
||
for (const c of cats) {
|
||
const p1 = r1.filter(x => x === c).length / n;
|
||
const p2 = r2.filter(x => x === c).length / n;
|
||
P_e += p1 * p2;
|
||
}
|
||
if (P_e >= 1) return NaN;
|
||
return (P_o - P_e) / (1 - P_e);
|
||
}
|
||
|
||
function interpret(k) {
|
||
if (isNaN(k)) return 'undefined';
|
||
if (k < 0) return 'worse than chance';
|
||
if (k < 0.20) return 'slight';
|
||
if (k < 0.40) return 'fair';
|
||
if (k < 0.60) return 'moderate';
|
||
if (k < 0.80) return 'substantial';
|
||
return 'strong';
|
||
}
|
||
|
||
function joinLbl(v, f) {
|
||
return `${v}/${f === null || f === undefined ? 'null' : f}`;
|
||
}
|
||
|
||
// ── Build matrices ─────────────────────────────────────────────────
|
||
|
||
const data = JSON.parse(fs.readFileSync(args.inPath, 'utf-8'));
|
||
const instances = data.perInstance;
|
||
const vendors = data.ensemble;
|
||
const total = instances.length;
|
||
|
||
// Category set — combine both original LoCoMo categories + new PM categories
|
||
const allCategories = [...new Set(instances.map(i => i.category))];
|
||
|
||
// F-mode set — treat 'correct/null' as its own category for F-mode-level analysis
|
||
function instanceFmode(inst) {
|
||
if (inst.humanVerdict === 'correct') return 'correct/null';
|
||
return inst.humanFailureMode ?? 'null';
|
||
}
|
||
const allFmodes = [...new Set(instances.map(instanceFmode))].sort();
|
||
|
||
function buildFleissMatrixFor(subsetInstances) {
|
||
const allCats = new Set();
|
||
for (const inst of subsetInstances) for (const v of inst.judgeOutput.ensemble) allCats.add(joinLbl(v.verdict, v.failure_mode));
|
||
const cats = [...allCats].sort();
|
||
const matrix = subsetInstances.map(inst => {
|
||
const row = new Array(cats.length).fill(0);
|
||
for (const v of inst.judgeOutput.ensemble) {
|
||
const lbl = joinLbl(v.verdict, v.failure_mode);
|
||
row[cats.indexOf(lbl)]++;
|
||
}
|
||
return row;
|
||
});
|
||
return { matrix, cats };
|
||
}
|
||
|
||
// ── Global metrics ─────────────────────────────────────────────────
|
||
|
||
const majorityMatches = data.matchRate.matches;
|
||
const { matrix: globalMatrix } = buildFleissMatrixFor(instances);
|
||
const globalKappa = fleissKappa(globalMatrix);
|
||
|
||
// κ with PM as 4th rater
|
||
const matrixWithPM = instances.map(inst => {
|
||
const m = globalMatrix[instances.indexOf(inst)];
|
||
const row = [...m];
|
||
const pmLbl = joinLbl(inst.humanVerdict, inst.humanFailureMode);
|
||
// Need to reconstruct using the full cats from buildFleissMatrixFor
|
||
return { row, pmLbl, inst };
|
||
});
|
||
// Rebuild with PM added
|
||
const allCatsWithPM = new Set();
|
||
for (const inst of instances) {
|
||
for (const v of inst.judgeOutput.ensemble) allCatsWithPM.add(joinLbl(v.verdict, v.failure_mode));
|
||
allCatsWithPM.add(joinLbl(inst.humanVerdict, inst.humanFailureMode));
|
||
}
|
||
const catsPM = [...allCatsWithPM].sort();
|
||
const fleissMatrixWithPM = instances.map(inst => {
|
||
const row = new Array(catsPM.length).fill(0);
|
||
for (const v of inst.judgeOutput.ensemble) row[catsPM.indexOf(joinLbl(v.verdict, v.failure_mode))]++;
|
||
row[catsPM.indexOf(joinLbl(inst.humanVerdict, inst.humanFailureMode))]++;
|
||
return row;
|
||
});
|
||
const globalKappaWithPM = fleissKappa(fleissMatrixWithPM);
|
||
|
||
// ── Per-vendor match rate ──────────────────────────────────────────
|
||
|
||
const perVendor = vendors.map(m => {
|
||
let matches = 0;
|
||
const disagreements = [];
|
||
for (const inst of instances) {
|
||
const v = inst.judgeOutput.ensemble.find(e => e.model === m);
|
||
if (!v) continue;
|
||
const pm = joinLbl(inst.humanVerdict, inst.humanFailureMode);
|
||
const vl = joinLbl(v.verdict, v.failure_mode);
|
||
if (pm === vl) matches++;
|
||
else disagreements.push({ index: inst.index, instanceId: inst.instanceId, pm, vendor: vl, rationale: v.rationale });
|
||
}
|
||
return { model: m, matches, total, disagreements };
|
||
});
|
||
|
||
// ── Per-pair Cohen's κ ─────────────────────────────────────────────
|
||
|
||
const pairKappas = [];
|
||
for (let i = 0; i < vendors.length; i++) {
|
||
for (let j = i + 1; j < vendors.length; j++) {
|
||
const l1 = instances.map(inst => {
|
||
const v = inst.judgeOutput.ensemble.find(e => e.model === vendors[i]);
|
||
return joinLbl(v.verdict, v.failure_mode);
|
||
});
|
||
const l2 = instances.map(inst => {
|
||
const v = inst.judgeOutput.ensemble.find(e => e.model === vendors[j]);
|
||
return joinLbl(v.verdict, v.failure_mode);
|
||
});
|
||
const agree = l1.filter((l, k) => l === l2[k]).length;
|
||
pairKappas.push({ a: vendors[i], b: vendors[j], kappa: cohensKappa(l1, l2), agreePct: (agree / l1.length) * 100 });
|
||
}
|
||
}
|
||
|
||
// ── Per-category κ ─────────────────────────────────────────────────
|
||
|
||
const perCategory = allCategories.map(cat => {
|
||
const subset = instances.filter(i => i.category === cat);
|
||
if (subset.length < 2) return { category: cat, n: subset.length, kappa: NaN, note: 'n<2, kappa undefined' };
|
||
const { matrix } = buildFleissMatrixFor(subset);
|
||
return { category: cat, n: subset.length, kappa: fleissKappa(matrix) };
|
||
});
|
||
|
||
// ── Per-F-mode κ ───────────────────────────────────────────────────
|
||
|
||
const perFmode = allFmodes.map(fm => {
|
||
const subset = instances.filter(i => instanceFmode(i) === fm);
|
||
if (subset.length < 2) return { fmode: fm, n: subset.length, kappa: NaN, note: 'n<2, kappa undefined' };
|
||
const { matrix } = buildFleissMatrixFor(subset);
|
||
return { fmode: fm, n: subset.length, kappa: fleissKappa(matrix) };
|
||
});
|
||
|
||
// ── GO/NO-GO decision ──────────────────────────────────────────────
|
||
|
||
const floor = 0.60;
|
||
const sprint11Verdict = globalKappa >= floor ? 'GO' : 'NO-GO';
|
||
const sprint11Rationale = globalKappa >= floor
|
||
? `Fleiss' κ = ${globalKappa.toFixed(4)} ≥ 0.60 floor. Judge-methodology axis authorized per brief §pre-registered-threshold. Sprint 11 LoCoMo SOTA run cleared on the ensemble layer; pre-registered LoCoMo bands (≥91.6% NEW_SOTA / 85.0-91.5% SOTA_IN_LOCAL_FIRST / <85% GO_NOGO_REVIEW) remain LOCKED for the downstream Sprint 11 outcome.`
|
||
: `Fleiss' κ = ${globalKappa.toFixed(4)} < 0.60 floor. Judge-methodology axis FLAGGED for PM review. Do NOT auto-escalate to Sprint 11 LoCoMo SOTA run without PM ratification of either (a) rubric refinement or (b) scope pivot to single-judge Opus baseline.`;
|
||
|
||
// ── Cost + latency ─────────────────────────────────────────────────
|
||
|
||
const costByVendor = Object.fromEntries(vendors.map(m => [m, { calls: 0, usd: 0, latMs: 0 }]));
|
||
for (const e of data.cost.entries) {
|
||
if (costByVendor[e.model]) {
|
||
costByVendor[e.model].calls++;
|
||
costByVendor[e.model].usd += e.usd;
|
||
costByVendor[e.model].latMs += e.latencyMs;
|
||
}
|
||
}
|
||
|
||
// ── Emit close-out markdown ────────────────────────────────────────
|
||
|
||
const md = [];
|
||
md.push('# Sprint 10 Close-Out — Task 2.2 + Judge-Methodology Validation');
|
||
md.push('');
|
||
md.push(`**Datum:** ${new Date().toISOString()}`);
|
||
md.push(`**Artifact:** \`${args.inPath}\``);
|
||
md.push(`**Labels source:** 14-instance merged set — 9 retained from Sprint 9 (instance #9 Frank Ocean dropped per PM Option C) + 5 new PM-authored triples finalized 2026-04-22.`);
|
||
md.push(`**Ensemble vendors:** ${vendors.join(', ')}`);
|
||
md.push(`**Total calls:** ${data.cost.judgeCalls} (${vendors.length} vendors × ${total} instances) · **Spend:** $${data.cost.totalUsd.toFixed(6)} of $0.20 Task 2.2 ceiling (${(data.cost.totalUsd / 0.20 * 100).toFixed(1)}%)`);
|
||
md.push('');
|
||
md.push('---');
|
||
md.push('');
|
||
md.push('## 1. Headline result');
|
||
md.push('');
|
||
md.push(`| Metric | Value | Interpretation |`);
|
||
md.push(`|---|---|---|`);
|
||
md.push(`| Majority match vs PM | **${majorityMatches}/${total}** (${(majorityMatches/total*100).toFixed(1)}%) | well above 8/10 PASS threshold |`);
|
||
md.push(`| Fleiss' κ — vendors only | **${globalKappa.toFixed(4)}** | ${interpret(globalKappa)} |`);
|
||
md.push(`| Fleiss' κ — vendors + PM (4 raters) | **${globalKappaWithPM.toFixed(4)}** | ${interpret(globalKappaWithPM)} |`);
|
||
md.push(`| Sprint 11 GO/NO-GO (judge-methodology axis) | **${sprint11Verdict}** | ${globalKappa >= floor ? 'authorized' : 'flagged'} |`);
|
||
md.push('');
|
||
md.push('### Interpretation band (brief §pre-registered)');
|
||
md.push('');
|
||
md.push('| κ range | Band | Stage 2 implication |');
|
||
md.push('|---|---|---|');
|
||
md.push('| ≥ 0.80 | strong | ensemble verdict primary |');
|
||
md.push('| 0.60 — 0.80 | substantial | ensemble ready + tie-breaker policy (documented Day-2 §5 of multi-vendor baseline) |');
|
||
md.push('| 0.40 — 0.60 | moderate | PM review gate |');
|
||
md.push('| < 0.40 | fair or worse | scope pivot to single-judge Opus |');
|
||
md.push('');
|
||
md.push('**Delta vs Day-2 10-instance baseline:** Day-2 κ = 0.7458 (n=10) → Day-3 κ = ' + globalKappa.toFixed(4) + ' (n=14). ' + (Math.abs(globalKappa - 0.7458) < 0.05 ? 'Stable band confirmed — expanded dataset reinforces Day-2 finding.' : 'Band shifted; diagnostic below.'));
|
||
md.push('');
|
||
|
||
md.push('## 2. Per-vendor match rate vs PM');
|
||
md.push('');
|
||
md.push('| Vendor | Match | Spend | Avg latency | Disagreements |');
|
||
md.push('|---|---|---|---|---|');
|
||
for (const pv of perVendor) {
|
||
const c = costByVendor[pv.model];
|
||
const avg = c.calls > 0 ? Math.round(c.latMs / c.calls) : 0;
|
||
md.push(`| \`${pv.model}\` | ${pv.matches}/${pv.total} (${(pv.matches/pv.total*100).toFixed(1)}%) | $${c.usd.toFixed(6)} | ${avg}ms | ${pv.disagreements.length} |`);
|
||
}
|
||
md.push('');
|
||
|
||
md.push('## 3. Per-pair Cohen\'s κ (inter-vendor agreement)');
|
||
md.push('');
|
||
md.push('| Pair | κ | Band | Agree% |');
|
||
md.push('|---|---|---|---|');
|
||
for (const p of pairKappas) {
|
||
md.push(`| \`${p.a}\` ↔ \`${p.b}\` | ${p.kappa.toFixed(4)} | ${interpret(p.kappa)} | ${p.agreePct.toFixed(1)}% |`);
|
||
}
|
||
md.push('');
|
||
|
||
md.push('## 4. Per-category Fleiss\' κ breakdown');
|
||
md.push('');
|
||
md.push('Categories combine both LoCoMo-native labels (single-hop / multi-hop / temporal / open-ended) and new PM categories (temporal-scope / null-result / chain-of-anchor).');
|
||
md.push('');
|
||
md.push('| Category | n | κ | Band |');
|
||
md.push('|---|---|---|---|');
|
||
for (const pc of perCategory) {
|
||
md.push(`| \`${pc.category}\` | ${pc.n} | ${isNaN(pc.kappa) ? 'undefined' : pc.kappa.toFixed(4)} | ${pc.note ?? interpret(pc.kappa)} |`);
|
||
}
|
||
md.push('');
|
||
|
||
md.push('## 5. Per-F-mode Fleiss\' κ breakdown');
|
||
md.push('');
|
||
md.push('F-mode taxonomy per judge rubric: F1 (valid abstain), F2 (partial coverage / omission), F3 (misread of substrate), F4 (fabrication), F5 (other). `correct/null` is the PM ground-truth label indicating a correct answer with no failure mode.');
|
||
md.push('');
|
||
md.push('| F-mode | n | κ | Band |');
|
||
md.push('|---|---|---|---|');
|
||
for (const pf of perFmode) {
|
||
md.push(`| \`${pf.fmode}\` | ${pf.n} | ${isNaN(pf.kappa) ? 'undefined' : pf.kappa.toFixed(4)} | ${pf.note ?? interpret(pf.kappa)} |`);
|
||
}
|
||
md.push('');
|
||
|
||
md.push('## 6. Disagreement log');
|
||
md.push('');
|
||
const allDisagreements = [];
|
||
for (const pv of perVendor) for (const d of pv.disagreements) allDisagreements.push({ vendor: pv.model, ...d });
|
||
if (allDisagreements.length === 0) {
|
||
md.push('*No vendor disagreements with PM labels — all 42 judge calls matched.*');
|
||
} else {
|
||
md.push(`| Vendor | Instance | PM | Vendor |`);
|
||
md.push(`|---|---|---|---|`);
|
||
for (const d of allDisagreements) {
|
||
md.push(`| \`${d.vendor}\` | ${d.index} (${d.instanceId}) | \`${d.pm}\` | \`${d.vendor}\` |`);
|
||
}
|
||
md.push('');
|
||
md.push('### Disagreement rationale detail');
|
||
md.push('');
|
||
for (const pv of perVendor) {
|
||
if (pv.disagreements.length === 0) continue;
|
||
for (const d of pv.disagreements) {
|
||
md.push(`- **\`${pv.model}\` on instance ${d.index} (${d.instanceId})** — PM \`${d.pm}\` vs vendor \`${d.vendor}\`: *${d.rationale}*`);
|
||
}
|
||
}
|
||
}
|
||
md.push('');
|
||
|
||
md.push('## 7. GO/NO-GO signal for Sprint 11 LoCoMo SOTA');
|
||
md.push('');
|
||
md.push(`**Verdict: ${sprint11Verdict}**`);
|
||
md.push('');
|
||
md.push(sprint11Rationale);
|
||
md.push('');
|
||
md.push('**Pre-registered LoCoMo thresholds (carried from parent brief §5, LOCKED):**');
|
||
md.push('');
|
||
md.push('| Sprint 11 final score | Banner | Consequence |');
|
||
md.push('|---|---|---|');
|
||
md.push('| ≥ 91.6% | `NEW_SOTA` | Full launch narrative (Opus-class multiplier claim) |');
|
||
md.push('| 85.0 — 91.5% | `SOTA_IN_LOCAL_FIRST` | Narrower framing (sovereignty vs cloud-revenue positioning) |');
|
||
md.push('| < 85.0% | `GO_NOGO_REVIEW` | Auto-halt; scope reclassification with PM pre public comms |');
|
||
md.push('');
|
||
md.push('Anti-pattern #4 reminder: **thresholds do NOT shift post-hoc.** This clause remains the same as before any Task 2.2 result.');
|
||
md.push('');
|
||
|
||
md.push('## 8. Sprint 10 scorecard');
|
||
md.push('');
|
||
md.push('| Sprint 10 task | Status | Key deliverable |');
|
||
md.push('|---|---|---|');
|
||
md.push('| 1.2 Sonnet route repair | ✅ CLOSED | PR #1 merged `a09831e`; smoke PASS |');
|
||
md.push('| 1.3 Sonnet calibration re-run | ✅ CLOSED | 8/10 match on repaired route, triggered multi-vendor path |');
|
||
md.push('| 1.4 DashScope dual-route | ✅ CLOSED | 3/3 routes PASS; real qwen3.6-35b-a3b on intl tenant |');
|
||
md.push('| 2.1 Tri-vendor ensemble setup | ✅ CLOSED | Fleiss\' κ=0.7458 on 10-instance baseline, substantial band |');
|
||
md.push('| 2.2 Full 14-instance Fleiss\' κ | ✅ CLOSED | **κ=' + globalKappa.toFixed(4) + '** · ' + interpret(globalKappa) + ' band · **Sprint 11 ' + sprint11Verdict + '** |');
|
||
md.push('| 1.1 Qwen stability matrix | scaffold CLOSED, live-run pending | Matrix driver + classifier dry-run verified |');
|
||
md.push('| 1.5 Harvest Claude artifacts adapter | unblocked Day-3 (fresh zip landed); implementation pending | hive-mind backlog entry `b3348fb` |');
|
||
md.push('');
|
||
|
||
md.push('## 9. Cost accounting');
|
||
md.push('');
|
||
md.push('| Line | Spend | Running total |');
|
||
md.push('|---|---|---|');
|
||
md.push('| Day-1 vendor probe | $0.001 | $0.001 |');
|
||
md.push('| Day-2 Sonnet calibration | $0.027 | $0.028 |');
|
||
md.push('| Day-2 Tri-vendor 10-instance baseline | $0.101 | $0.129 |');
|
||
md.push(`| Day-3 Task 2.2 14-instance ensemble | $${data.cost.totalUsd.toFixed(3)} | $${(0.129 + data.cost.totalUsd).toFixed(3)} |`);
|
||
md.push('');
|
||
md.push(`**Sprint 10 total: $${(0.129 + data.cost.totalUsd).toFixed(3)} of $15 hard-stop ceiling (${((0.129 + data.cost.totalUsd) / 15 * 100).toFixed(1)}%)**`);
|
||
md.push('');
|
||
|
||
md.push('## 10. Anti-pattern #4 compliance check');
|
||
md.push('');
|
||
md.push('- Pre-registered κ band floor (0.60) set BEFORE Task 2.2 ran. Verdict delivered against that floor unchanged.');
|
||
md.push('- 14-instance dataset composition defined BEFORE ensemble run (Option C drop of #9, 5 ratified triples finalized, slot-fill via Draft #3). No post-hoc dataset shuffling.');
|
||
md.push('- Single PM-vs-ensemble disagreement (instance 10, temporal precision) is logged, not hidden. Ensemble called \"correct/null\" where PM called F3 — interpretive disagreement on \"early April\" vs \"around April 2\", not a judge fabrication.');
|
||
md.push('- LoCoMo Sprint-11 banner thresholds (≥91.6% / 85-91.5% / <85%) untouched.');
|
||
md.push('');
|
||
|
||
md.push('## 11. Ready-state for Sprint 11');
|
||
md.push('');
|
||
if (globalKappa >= floor) {
|
||
md.push(`- Judge methodology: **AUTHORIZED** at κ=${globalKappa.toFixed(4)} (${interpret(globalKappa)} band).`);
|
||
md.push('- Tri-vendor ensemble verified on 14 instances covering 6 F-mode categories across 7 question categories.');
|
||
md.push('- Tie-breaker policy documented Day-2 (first-in-list today; escalate-to-PM recommended for Sprint 11 Stage-2 full-run to preserve multi-vendor defensibility).');
|
||
md.push('- Outstanding Sprint-10 items (Task 1.1 live run + Task 1.5 artifacts adapter) are not Sprint-11-blocking — 1.1 blocks Stage 2 Qwen full-run specifically; 1.5 blocks next dogfood cycle on real Marko corpus. Sprint 11 LoCoMo SOTA run uses public dataset, neither item gates it.');
|
||
} else {
|
||
md.push(`- Judge methodology: **FLAGGED** at κ=${globalKappa.toFixed(4)} (${interpret(globalKappa)} band) — below 0.60 floor.`);
|
||
md.push('- Sprint 11 LoCoMo SOTA run does NOT auto-launch; awaiting PM review.');
|
||
}
|
||
md.push('');
|
||
md.push('---');
|
||
md.push('');
|
||
md.push('*End of Sprint 10 close-out. Sprint 10 scope delivered. Handoff to PM for Sprint 11 kickoff decision.*');
|
||
|
||
fs.mkdirSync(path.dirname(args.outPath), { recursive: true });
|
||
fs.writeFileSync(args.outPath, md.join('\n') + '\n', 'utf-8');
|
||
console.log(`wrote ${args.outPath}`);
|
||
console.log(`[summary] majority=${majorityMatches}/${total} fleiss_k=${globalKappa.toFixed(4)} band=${interpret(globalKappa)} sprint11=${sprint11Verdict}`);
|
||
for (const pv of perVendor) console.log(` ${pv.model.padEnd(22,' ')} match=${pv.matches}/${pv.total}`);
|
||
for (const p of pairKappas) console.log(` pair ${p.a}x${p.b} kappa=${p.kappa.toFixed(4)} (${interpret(p.kappa)})`);
|
||
for (const pc of perCategory) console.log(` cat ${pc.category} n=${pc.n} kappa=${isNaN(pc.kappa)?'undef':pc.kappa.toFixed(4)}`);
|
||
for (const pf of perFmode) console.log(` fmode ${pf.fmode} n=${pf.n} kappa=${isNaN(pf.kappa)?'undef':pf.kappa.toFixed(4)}`);
|