chore(server): remove the memory benchmark judge scaffolding (P2.2)
packages/server/src/benchmarks held a LoCoMo failure-mode aggregator and an LLM judge ensemble for the old memory engine benchmarks. Nothing in the sidecar imports it and no npm script runs it; its inputs lived in the root benchmarks/ directory that P2.2 already removed. Judge and eval loops are also out of the community edition (AGENTS.md, rule 4).
This commit is contained in:
@@ -1,432 +0,0 @@
|
||||
/**
|
||||
* Aggregate.ts — failure-mode distribution rollup for Stage 1 / Stage 2
|
||||
* preflight + Week 1/2 main runs.
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3
|
||||
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
|
||||
*
|
||||
* Consumes JSONL output from `benchmarks/harness` (one record per
|
||||
* instance per cell) and emits a structured report covering:
|
||||
* 1. Per-cell verdict distribution + weighted quality score
|
||||
* 2. Per-LoCoMo-category distribution (single/multi-hop/temporal/
|
||||
* open-ended) with a hallucination rate flag for PM review
|
||||
* 3. Cross-cell delta matrix (full-context vs raw: correct / F4 /
|
||||
* F1 lift per taxonomy §5)
|
||||
* 4. Judge cost summary when the run included judge calls
|
||||
*
|
||||
* Input shape: each JSONL line has the fields declared in
|
||||
* `benchmarks/harness/src/types.ts` (optional `judge_verdict`,
|
||||
* `judge_failure_mode`, `model_answer`, etc.). Pre-judge records — rows
|
||||
* without `judge_verdict` — are counted under an `unjudged` bucket so
|
||||
* partial-run reports stay honest instead of silently skipping data.
|
||||
*/
|
||||
|
||||
export type Verdict6 =
|
||||
| 'correct'
|
||||
| 'F1_abstain'
|
||||
| 'F2_partial'
|
||||
| 'F3_incorrect'
|
||||
| 'F4_hallucinated'
|
||||
| 'F5_offtopic'
|
||||
| 'unjudged';
|
||||
|
||||
export type CellName = 'raw' | 'filtered' | 'compressed' | 'full-context';
|
||||
const CELL_NAMES: readonly CellName[] = ['raw', 'filtered', 'compressed', 'full-context'];
|
||||
|
||||
/** Categories match `preflight-locomo-50.json` `_meta.locomo_category_map`
|
||||
* minus the excluded adversarial bucket. Unknown categories fall into
|
||||
* `other` so the report still tallies them instead of dropping. */
|
||||
export type LocomoCategory = 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended' | 'other';
|
||||
const LOCOMO_CATEGORIES: readonly LocomoCategory[] = [
|
||||
'single-hop', 'multi-hop', 'temporal', 'open-ended', 'other',
|
||||
];
|
||||
|
||||
/** Inputs can carry either the explicit category column or the
|
||||
* pre-lock instance_id (`locomo_<sample_id>_q<qindex>`) from which
|
||||
* we can recover the category via a sibling lookup. This module only
|
||||
* consumes the already-tagged JsonlRecord; category recovery lives in
|
||||
* the caller (harness runner attaches `category` when loading from
|
||||
* the sample lock). */
|
||||
export interface JudgedJsonlRecord {
|
||||
turnId: string;
|
||||
cell: string;
|
||||
instance_id: string;
|
||||
model: string;
|
||||
seed: number;
|
||||
accuracy: number;
|
||||
p50_latency_ms: number;
|
||||
p95_latency_ms: number;
|
||||
usd_per_query: number;
|
||||
failure_mode: string | null;
|
||||
model_answer?: string;
|
||||
judge_verdict?: 'correct' | 'incorrect';
|
||||
judge_failure_mode?: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
|
||||
judge_rationale?: string;
|
||||
judge_model?: string;
|
||||
judge_timestamp?: string;
|
||||
judge_confidence?: number;
|
||||
judge_ensemble?: Array<{
|
||||
model: string;
|
||||
verdict: 'correct' | 'incorrect';
|
||||
failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
|
||||
rationale?: string;
|
||||
latency_ms?: number;
|
||||
}>;
|
||||
/** Optional category tag attached by the runner when the sample came
|
||||
* from preflight-locomo-50.json. When absent, rows fall into
|
||||
* `other` for the per-category rollup. */
|
||||
category?: LocomoCategory;
|
||||
}
|
||||
|
||||
/** Projection from the taxonomy §9 binary `verdict` + `failure_mode`
|
||||
* slot pair to the 6-value Verdict6 enum the brief Task-1 spec asked
|
||||
* for. Centralising this here means JsonlRecord keeps the canonical
|
||||
* taxonomy shape while the aggregator surfaces the denser report
|
||||
* form. */
|
||||
export function projectVerdict6(r: JudgedJsonlRecord): Verdict6 {
|
||||
if (r.judge_verdict === undefined) return 'unjudged';
|
||||
if (r.judge_verdict === 'correct') return 'correct';
|
||||
switch (r.judge_failure_mode) {
|
||||
case 'F1': return 'F1_abstain';
|
||||
case 'F2': return 'F2_partial';
|
||||
case 'F3': return 'F3_incorrect';
|
||||
case 'F4': return 'F4_hallucinated';
|
||||
case 'F5': return 'F5_offtopic';
|
||||
default:
|
||||
// Contract violation — the taxonomy schema (judge module's Zod)
|
||||
// prevents `incorrect` with a null failure_mode. If it sneaks
|
||||
// through a malformed JSONL row, bucket as F3 by convention so
|
||||
// the row doesn't silently drop.
|
||||
return 'F3_incorrect';
|
||||
}
|
||||
}
|
||||
|
||||
export const VERDICT6_VALUES: readonly Verdict6[] = [
|
||||
'correct', 'F1_abstain', 'F2_partial', 'F3_incorrect', 'F4_hallucinated', 'F5_offtopic', 'unjudged',
|
||||
];
|
||||
|
||||
// ── Per-cell distribution + weighted score ─────────────────────────────
|
||||
|
||||
export interface PerCellRow {
|
||||
cell: string;
|
||||
total: number;
|
||||
counts: Record<Verdict6, number>;
|
||||
percents: Record<Verdict6, number>;
|
||||
/** Weighted quality score per taxonomy §5 rubric. `unjudged` is
|
||||
* excluded from the denominator so the score is computed over the
|
||||
* set of instances actually graded. */
|
||||
weightedScore: number;
|
||||
}
|
||||
|
||||
/** Taxonomy §5 coefficient table. Positive weights for correct / F2
|
||||
* (partial credit), zero for F1 (abstain neutral), negatives for
|
||||
* F3 / F4 / F5. Exported so Task-3 unit tests can reproduce the
|
||||
* hand-check number by name. */
|
||||
export const WEIGHTS: Record<Exclude<Verdict6, 'unjudged'>, number> = {
|
||||
correct: 1.0,
|
||||
F2_partial: 0.30,
|
||||
F1_abstain: 0.0,
|
||||
F3_incorrect: -0.15,
|
||||
F4_hallucinated: -0.35,
|
||||
F5_offtopic: -0.10,
|
||||
};
|
||||
|
||||
function emptyVerdictMap(fill = 0): Record<Verdict6, number> {
|
||||
const out = {} as Record<Verdict6, number>;
|
||||
for (const v of VERDICT6_VALUES) out[v] = fill;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function perCellRollup(records: readonly JudgedJsonlRecord[]): PerCellRow[] {
|
||||
const byCell = new Map<string, JudgedJsonlRecord[]>();
|
||||
for (const r of records) {
|
||||
const arr = byCell.get(r.cell) ?? [];
|
||||
arr.push(r);
|
||||
byCell.set(r.cell, arr);
|
||||
}
|
||||
const rows: PerCellRow[] = [];
|
||||
// Emit in CELL_NAMES order when present, then any controls / extras.
|
||||
const orderedCellKeys = [
|
||||
...CELL_NAMES.filter(c => byCell.has(c)),
|
||||
...[...byCell.keys()].filter(c => !CELL_NAMES.includes(c as CellName)),
|
||||
];
|
||||
for (const cell of orderedCellKeys) {
|
||||
const rowRecords = byCell.get(cell)!;
|
||||
const counts = emptyVerdictMap();
|
||||
for (const r of rowRecords) counts[projectVerdict6(r)]++;
|
||||
const total = rowRecords.length;
|
||||
const judgedTotal = total - counts['unjudged'];
|
||||
const percents = emptyVerdictMap();
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
percents[v] = total === 0 ? 0 : counts[v] / total;
|
||||
}
|
||||
let weightedScore = 0;
|
||||
if (judgedTotal > 0) {
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
if (v === 'unjudged') continue;
|
||||
weightedScore += (counts[v] / judgedTotal) * WEIGHTS[v];
|
||||
}
|
||||
}
|
||||
rows.push({ cell, total, counts, percents, weightedScore });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── Per-LoCoMo-category distribution ───────────────────────────────────
|
||||
|
||||
export interface PerCategoryRow {
|
||||
category: LocomoCategory;
|
||||
total: number;
|
||||
counts: Record<Verdict6, number>;
|
||||
percents: Record<Verdict6, number>;
|
||||
/** True when F4 (hallucinated) share exceeds 20% — flags the
|
||||
* category for PM spot-review per brief §Task-3 rule 2. */
|
||||
hallucinationFlag: boolean;
|
||||
}
|
||||
|
||||
const HALLUCINATION_FLAG_THRESHOLD = 0.20;
|
||||
|
||||
export function perCategoryRollup(records: readonly JudgedJsonlRecord[]): PerCategoryRow[] {
|
||||
const byCat = new Map<LocomoCategory, JudgedJsonlRecord[]>();
|
||||
for (const r of records) {
|
||||
const cat = r.category ?? 'other';
|
||||
const arr = byCat.get(cat) ?? [];
|
||||
arr.push(r);
|
||||
byCat.set(cat, arr);
|
||||
}
|
||||
const rows: PerCategoryRow[] = [];
|
||||
for (const cat of LOCOMO_CATEGORIES) {
|
||||
const recs = byCat.get(cat) ?? [];
|
||||
if (recs.length === 0) continue;
|
||||
const counts = emptyVerdictMap();
|
||||
for (const r of recs) counts[projectVerdict6(r)]++;
|
||||
const total = recs.length;
|
||||
const percents = emptyVerdictMap();
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
percents[v] = total === 0 ? 0 : counts[v] / total;
|
||||
}
|
||||
rows.push({
|
||||
category: cat,
|
||||
total,
|
||||
counts,
|
||||
percents,
|
||||
hallucinationFlag: percents['F4_hallucinated'] > HALLUCINATION_FLAG_THRESHOLD,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── Cross-cell delta matrix (full-context vs raw) ──────────────────────
|
||||
|
||||
export interface CrossCellDelta {
|
||||
verdict: Verdict6;
|
||||
rawPercent: number;
|
||||
fullContextPercent: number;
|
||||
/** fullContext − raw. Positive = full-context raised this verdict's share;
|
||||
* negative = full-context lowered it. The brief's headline expectation:
|
||||
* `correct` delta > 0, `F4_hallucinated` delta < 0, `F1_abstain`
|
||||
* may go either way (more abstains is sometimes an OK signal). */
|
||||
delta: number;
|
||||
}
|
||||
|
||||
export function crossCellDeltaMatrix(perCell: readonly PerCellRow[]): CrossCellDelta[] | null {
|
||||
const raw = perCell.find(r => r.cell === 'raw');
|
||||
const full = perCell.find(r => r.cell === 'full-context');
|
||||
if (!raw || !full) return null;
|
||||
const out: CrossCellDelta[] = [];
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
out.push({
|
||||
verdict: v,
|
||||
rawPercent: raw.percents[v],
|
||||
fullContextPercent: full.percents[v],
|
||||
delta: full.percents[v] - raw.percents[v],
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Cost summary ───────────────────────────────────────────────────────
|
||||
|
||||
export interface CostSummary {
|
||||
totalUsd: number;
|
||||
perCellUsd: Record<string, number>;
|
||||
judgeTotalUsd: number;
|
||||
judgeCallCount: number;
|
||||
/** Median judge call latency across the run; `null` when no judge
|
||||
* call recorded a latency. */
|
||||
medianJudgeMs: number | null;
|
||||
/** Brief §Task-3 rule 4: flag when total judge spend would exceed
|
||||
* ~$20 projected for a full 4-cell × 50-instance run. Computed as
|
||||
* `perInstanceJudgeUsd × 200`. */
|
||||
week1WarningProjectedUsd: number;
|
||||
week1WarningFired: boolean;
|
||||
}
|
||||
|
||||
const WEEK1_PROJECTION_INSTANCE_COUNT = 200; // 4 cells × 50 instances
|
||||
|
||||
export function costSummary(records: readonly JudgedJsonlRecord[]): CostSummary {
|
||||
const perCellUsd: Record<string, number> = {};
|
||||
let total = 0;
|
||||
const judgeLatencies: number[] = [];
|
||||
const judgeTotal = 0;
|
||||
let judgeCalls = 0;
|
||||
for (const r of records) {
|
||||
total += r.usd_per_query;
|
||||
perCellUsd[r.cell] = (perCellUsd[r.cell] ?? 0) + r.usd_per_query;
|
||||
if (r.judge_ensemble) {
|
||||
for (const entry of r.judge_ensemble) {
|
||||
if (typeof entry.latency_ms === 'number') judgeLatencies.push(entry.latency_ms);
|
||||
}
|
||||
judgeCalls += r.judge_ensemble.length;
|
||||
} else if (r.judge_verdict !== undefined) {
|
||||
// Single-judge call — we don't store per-judge latency in the
|
||||
// JsonlRecord shape (no judge_latency_ms field), so the median
|
||||
// is computed only over ensemble entries that carry it. If no
|
||||
// ensemble row is present, medianJudgeMs stays null.
|
||||
judgeCalls += 1;
|
||||
}
|
||||
}
|
||||
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
|
||||
// Per-instance judge spend is not stored directly — the aggregator
|
||||
// doesn't know judge USD without the onCall sink. When the caller
|
||||
// passes judgeTotalUsd separately (see buildReport), we populate
|
||||
// this from the authoritative source. Here we return zero; buildReport
|
||||
// overwrites with the real total.
|
||||
const medianJudgeMs = judgeLatencies.length === 0
|
||||
? null
|
||||
: (() => {
|
||||
const s = [...judgeLatencies].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 === 0 ? (s[mid - 1] + s[mid]) / 2 : s[mid];
|
||||
})();
|
||||
const perInstanceJudgeUsd = judgedCount > 0 ? judgeTotal / judgedCount : 0;
|
||||
const projected = perInstanceJudgeUsd * WEEK1_PROJECTION_INSTANCE_COUNT;
|
||||
return {
|
||||
totalUsd: total,
|
||||
perCellUsd,
|
||||
judgeTotalUsd: judgeTotal,
|
||||
judgeCallCount: judgeCalls,
|
||||
medianJudgeMs,
|
||||
week1WarningProjectedUsd: projected,
|
||||
week1WarningFired: projected > 20,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Top-level report + markdown renderer ───────────────────────────────
|
||||
|
||||
export interface AggregateReport {
|
||||
generatedAt: string;
|
||||
totalRecords: number;
|
||||
perCell: PerCellRow[];
|
||||
perCategory: PerCategoryRow[];
|
||||
crossCellDelta: CrossCellDelta[] | null;
|
||||
cost: CostSummary;
|
||||
}
|
||||
|
||||
export function buildReport(
|
||||
records: readonly JudgedJsonlRecord[],
|
||||
opts: { judgeTotalUsd?: number } = {},
|
||||
): AggregateReport {
|
||||
const perCell = perCellRollup(records);
|
||||
const perCategory = perCategoryRollup(records);
|
||||
const crossCellDelta = crossCellDeltaMatrix(perCell);
|
||||
const cost = costSummary(records);
|
||||
// Callers that run the harness end-to-end know the authoritative
|
||||
// judge spend because judge-client emits `onCall` entries. Pass the
|
||||
// aggregated total here so the report carries real dollars instead
|
||||
// of the placeholder zero computed from JSONL alone.
|
||||
if (opts.judgeTotalUsd !== undefined) {
|
||||
cost.judgeTotalUsd = opts.judgeTotalUsd;
|
||||
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
|
||||
const perInstance = judgedCount > 0 ? opts.judgeTotalUsd / judgedCount : 0;
|
||||
cost.week1WarningProjectedUsd = perInstance * WEEK1_PROJECTION_INSTANCE_COUNT;
|
||||
cost.week1WarningFired = cost.week1WarningProjectedUsd > 20;
|
||||
}
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalRecords: records.length,
|
||||
perCell,
|
||||
perCategory,
|
||||
crossCellDelta,
|
||||
cost,
|
||||
};
|
||||
}
|
||||
|
||||
function pct(n: number): string {
|
||||
return `${(n * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function fmtUsd(n: number): string {
|
||||
return `$${n.toFixed(n < 1 ? 6 : 2)}`;
|
||||
}
|
||||
|
||||
export function renderMarkdown(report: AggregateReport): string {
|
||||
const lines: string[] = [];
|
||||
lines.push('# Benchmark Aggregate Report');
|
||||
lines.push('');
|
||||
lines.push(`**Generated at:** ${report.generatedAt}`);
|
||||
lines.push(`**Total records:** ${report.totalRecords}`);
|
||||
lines.push('');
|
||||
lines.push('## Per-cell verdict distribution');
|
||||
lines.push('');
|
||||
lines.push('| Cell | Total | Correct | F1 abstain | F2 partial | F3 incorrect | F4 hallucinated | F5 off-topic | Unjudged | Weighted score |');
|
||||
lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|');
|
||||
for (const row of report.perCell) {
|
||||
lines.push(
|
||||
`| ${row.cell} | ${row.total} | ${row.counts.correct} (${pct(row.percents.correct)}) | ` +
|
||||
`${row.counts.F1_abstain} (${pct(row.percents.F1_abstain)}) | ` +
|
||||
`${row.counts.F2_partial} (${pct(row.percents.F2_partial)}) | ` +
|
||||
`${row.counts.F3_incorrect} (${pct(row.percents.F3_incorrect)}) | ` +
|
||||
`${row.counts.F4_hallucinated} (${pct(row.percents.F4_hallucinated)}) | ` +
|
||||
`${row.counts.F5_offtopic} (${pct(row.percents.F5_offtopic)}) | ` +
|
||||
`${row.counts.unjudged} (${pct(row.percents.unjudged)}) | ` +
|
||||
`${row.weightedScore.toFixed(3)} |`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
if (report.perCategory.length > 0) {
|
||||
lines.push('## Per-LoCoMo-category distribution');
|
||||
lines.push('');
|
||||
lines.push('| Category | Total | Correct% | F4 (hallucinated)% | Flagged? |');
|
||||
lines.push('|---|---:|---:|---:|:---:|');
|
||||
for (const row of report.perCategory) {
|
||||
lines.push(
|
||||
`| ${row.category} | ${row.total} | ${pct(row.percents.correct)} | ` +
|
||||
`${pct(row.percents.F4_hallucinated)} | ` +
|
||||
`${row.hallucinationFlag ? '⚠️ PM review' : '✓'} |`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
if (report.crossCellDelta) {
|
||||
lines.push('## Full-context vs raw delta');
|
||||
lines.push('');
|
||||
lines.push('| Verdict | raw % | full-context % | Δ (full − raw) |');
|
||||
lines.push('|---|---:|---:|---:|');
|
||||
for (const r of report.crossCellDelta) {
|
||||
const sign = r.delta > 0 ? '+' : '';
|
||||
lines.push(`| ${r.verdict} | ${pct(r.rawPercent)} | ${pct(r.fullContextPercent)} | ${sign}${pct(r.delta)} |`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
lines.push('## Cost summary');
|
||||
lines.push('');
|
||||
lines.push(`- Total cell spend: **${fmtUsd(report.cost.totalUsd)}**`);
|
||||
lines.push(`- Judge spend: **${fmtUsd(report.cost.judgeTotalUsd)}** across ${report.cost.judgeCallCount} call(s)`);
|
||||
if (report.cost.medianJudgeMs !== null) {
|
||||
lines.push(`- Median judge latency: ${report.cost.medianJudgeMs.toFixed(0)} ms`);
|
||||
}
|
||||
if (Object.keys(report.cost.perCellUsd).length > 0) {
|
||||
lines.push('- Per-cell cell spend:');
|
||||
for (const [cell, usd] of Object.entries(report.cost.perCellUsd)) {
|
||||
lines.push(` - ${cell}: ${fmtUsd(usd)}`);
|
||||
}
|
||||
}
|
||||
if (report.cost.week1WarningFired) {
|
||||
lines.push(
|
||||
`- ⚠️ **Week-1 scale-up warning:** projected judge spend for a full 4-cell × 50-instance run is ` +
|
||||
`${fmtUsd(report.cost.week1WarningProjectedUsd)} (threshold $20). Flagged per brief §Task-3 rule 4.`,
|
||||
);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -1,314 +0,0 @@
|
||||
/**
|
||||
* Sprint 11 Task B2 — Tie-break policy for multi-vendor judge ensemble.
|
||||
*
|
||||
* Authority:
|
||||
* - PM-Waggle-OS/briefs/2026-04-22-cc-sprint-11-kickoff.md §3 Track B B2
|
||||
* - PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
|
||||
*
|
||||
* Context — what this resolves:
|
||||
*
|
||||
* The Sprint 10 Task 2.2 ratified primary judge ensemble is a THREE-vendor
|
||||
* panel (Anthropic Opus 4.7 + OpenAI GPT-5.4 + Google Gemini 3.1-Pro). Three
|
||||
* votes means three possible distributions:
|
||||
*
|
||||
* - 3-0 consensus → trivial, no tie-break
|
||||
* - 2-1 majority → majority wins, no tie-break
|
||||
* - 1-1-1 split → three different verdicts, no primary majority
|
||||
*
|
||||
* On 1-1-1, we escalate to a FOURTH vendor from a lineage disjoint from the
|
||||
* primary trio: xai/grok-4.20 per Marko's 2026-04-22 LOCK. After the 4th
|
||||
* vote arrives we have four votes total:
|
||||
*
|
||||
* - 2-1-1 / 1-1-2 → plurality winner (2 votes)
|
||||
* - 1-1-1-1 → four-way split → PM escalation
|
||||
*
|
||||
* The fourth vendor is SPECIFICALLY xai/grok-4.20 — not Sonnet 4.6 (which
|
||||
* would give Anthropic 2-of-4 weight → homogeneous-bias risk — rejected per
|
||||
* LOCK doc §2) and not Opus 4.7 (already Judge 1).
|
||||
*
|
||||
* Observability: pino structured log emits `tie_break.path` ∈ { 'none',
|
||||
* 'majority', 'quadri-vendor', 'pm-escalation' } plus
|
||||
* `tie_break.fourth_vendor_slug` on quadri paths (field is future-proof
|
||||
* even though Sprint 11 scope is grok-4.20 only).
|
||||
*/
|
||||
|
||||
import type { JudgeResult } from './failure-mode-judge.js';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A single judge vote. Reusing JudgeResult from failure-mode-judge.ts keeps
|
||||
* the tie-break input shape aligned with what judgeEnsemble already produces.
|
||||
*/
|
||||
export type Vote = JudgeResult;
|
||||
|
||||
export type TieBreakPath = 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
|
||||
|
||||
/**
|
||||
* The canonical sentinel verdict returned on a 1-1-1-1 four-way split.
|
||||
* Callers (aggregators, report generators) MUST treat this as an operator
|
||||
* action signal, not as a normal verdict string. The shape is intentionally
|
||||
* unambiguous so no accidental inclusion into recall/accuracy math.
|
||||
*/
|
||||
export const PM_ESCALATION_VERDICT = '__PM_ESCALATION__';
|
||||
|
||||
/** Default fourth vendor for Sprint 11 — per LOCK doc §1. */
|
||||
export const DEFAULT_FOURTH_VENDOR = 'xai/grok-4.20';
|
||||
|
||||
export interface TieBreakResult {
|
||||
/**
|
||||
* The resolved verdict string. Encoded as `<verdict>|<failure_mode_or_NA>`
|
||||
* matching the aggregation key used in computeMajority of failure-mode-judge.ts,
|
||||
* so downstream consumers can uniformly decompose.
|
||||
* On pm-escalation path: `__PM_ESCALATION__`.
|
||||
*/
|
||||
verdict: string;
|
||||
path: TieBreakPath;
|
||||
/** The full vote list after resolution — 3 votes for none/majority, 4 for quadri/pm-escalation. */
|
||||
votes: Vote[];
|
||||
/** The fourth vendor's vote when a quadri path was taken. Undefined on none/majority. */
|
||||
fourthVendorVote?: Vote;
|
||||
/** The fourth-vendor slug that was used, for observability. Undefined on none/majority. */
|
||||
fourthVendorSlug?: string;
|
||||
}
|
||||
|
||||
export interface FourthVendorCallPayload {
|
||||
/** The three primary votes that produced the 1-1-1 split. */
|
||||
primaryVotes: Vote[];
|
||||
/** Model slug to invoke — e.g. 'xai/grok-4.20'. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export type CallFourthVendor = (payload: FourthVendorCallPayload) => Promise<Vote>;
|
||||
|
||||
export interface TieBreakLogger {
|
||||
info(event: string, fields: Record<string, unknown>): void;
|
||||
warn?(event: string, fields: Record<string, unknown>): void;
|
||||
}
|
||||
|
||||
export interface ResolveTieBreakOptions {
|
||||
/**
|
||||
* Invoked only on 1-1-1 splits. Required when primaryVotes.length === 3
|
||||
* and the distribution is 1-1-1; optional otherwise. Tests mock this.
|
||||
*/
|
||||
callFourthVendor?: CallFourthVendor;
|
||||
/** Override the fourth vendor slug. Defaults to `DEFAULT_FOURTH_VENDOR`. */
|
||||
fourthVendorModel?: string;
|
||||
/** Structured logger. Silently no-ops when omitted. */
|
||||
logger?: TieBreakLogger;
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Canonical key for a vote — combines verdict + failure_mode so identical
|
||||
* answers with different failure codes don't accidentally tie.
|
||||
*/
|
||||
function voteKey(v: Vote): string {
|
||||
return `${v.verdict}|${v.failure_mode ?? 'NA'}`;
|
||||
}
|
||||
|
||||
function countByKey(votes: Vote[]): Map<string, { count: number; first: Vote }> {
|
||||
const tally = new Map<string, { count: number; first: Vote }>();
|
||||
for (const v of votes) {
|
||||
const key = voteKey(v);
|
||||
const existing = tally.get(key);
|
||||
if (existing) existing.count += 1;
|
||||
else tally.set(key, { count: 1, first: v });
|
||||
}
|
||||
return tally;
|
||||
}
|
||||
|
||||
function pluralityTop(tally: Map<string, { count: number; first: Vote }>): {
|
||||
topCount: number;
|
||||
topKeys: string[];
|
||||
} {
|
||||
let topCount = 0;
|
||||
for (const { count } of tally.values()) if (count > topCount) topCount = count;
|
||||
const topKeys: string[] = [];
|
||||
for (const [key, { count }] of tally) if (count === topCount) topKeys.push(key);
|
||||
return { topCount, topKeys };
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolve a multi-vendor judge ensemble vote.
|
||||
*
|
||||
* Input shape:
|
||||
* - 3 primary votes (Sprint 10 Task 2.2 trio) — the normal call path.
|
||||
* - 4 votes also accepted, in which case the function treats the vector
|
||||
* as already-quadri-resolved and returns the plurality winner (or
|
||||
* pm-escalation on 1-1-1-1). This shape exists for tests and for
|
||||
* caller-side recomposition.
|
||||
*
|
||||
* On 1-1-1 with 3 primary votes, `callFourthVendor` MUST be provided.
|
||||
* The function invokes it with the three primary votes and the canonical
|
||||
* model slug, then recurses with the four-vote list.
|
||||
*
|
||||
* Returns:
|
||||
* - `{ path: 'none' }` for 3-0 consensus (3 primary votes).
|
||||
* - `{ path: 'majority' }` for 2-1 majority (3 votes) OR for a plurality
|
||||
* win on 4 votes when the caller supplied the vector directly.
|
||||
* - `{ path: 'quadri-vendor' }` when we called the 4th vendor ourselves
|
||||
* and got a 2-1-1 or 1-1-2 distribution.
|
||||
* - `{ path: 'pm-escalation' }` on a 1-1-1-1 four-way split; verdict is
|
||||
* the `PM_ESCALATION_VERDICT` sentinel.
|
||||
*/
|
||||
export async function resolveTieBreak(
|
||||
votes: Vote[],
|
||||
options: ResolveTieBreakOptions = {},
|
||||
): Promise<TieBreakResult> {
|
||||
if (votes.length !== 3 && votes.length !== 4) {
|
||||
throw new Error(
|
||||
`resolveTieBreak: votes.length must be 3 (primary ensemble) or 4 (post-quadri). Got ${votes.length}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const logger = options.logger;
|
||||
|
||||
if (votes.length === 3) {
|
||||
const tally = countByKey(votes);
|
||||
const { topCount, topKeys } = pluralityTop(tally);
|
||||
|
||||
// 3-0 consensus — one bucket holds all three.
|
||||
if (topCount === 3) {
|
||||
const entry = tally.get(topKeys[0])!;
|
||||
logger?.info('tie_break', { path: 'none' as TieBreakPath, verdict: topKeys[0] });
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'none',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// 2-1 majority — one bucket holds two, another holds one.
|
||||
if (topCount === 2 && topKeys.length === 1) {
|
||||
logger?.info('tie_break', { path: 'majority' as TieBreakPath, verdict: topKeys[0] });
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// 1-1-1 split — three buckets of one each. Escalate to fourth vendor.
|
||||
if (topCount === 1 && topKeys.length === 3) {
|
||||
if (!options.callFourthVendor) {
|
||||
throw new Error(
|
||||
'resolveTieBreak: 1-1-1 three-way split requires a callFourthVendor implementation.',
|
||||
);
|
||||
}
|
||||
const fourthVendorModel = options.fourthVendorModel ?? DEFAULT_FOURTH_VENDOR;
|
||||
|
||||
logger?.info('tie_break.quadri-vendor.invoke', {
|
||||
path: 'quadri-vendor' as TieBreakPath,
|
||||
fourth_vendor_slug: fourthVendorModel,
|
||||
primary_keys: topKeys,
|
||||
});
|
||||
|
||||
const fourthVote = await options.callFourthVendor({
|
||||
primaryVotes: votes,
|
||||
model: fourthVendorModel,
|
||||
});
|
||||
|
||||
// Recurse with four votes. The 4-vote branch returns quadri-vendor or pm-escalation.
|
||||
const resolved = await resolveTieBreak([...votes, fourthVote], options);
|
||||
|
||||
// Re-tag the path so "quadri-vendor" sticks on success cases and
|
||||
// "pm-escalation" stays on 1-1-1-1 after escalation.
|
||||
const path: TieBreakPath =
|
||||
resolved.path === 'pm-escalation' ? 'pm-escalation' : 'quadri-vendor';
|
||||
|
||||
logger?.info('tie_break.quadri-vendor.resolved', {
|
||||
path,
|
||||
fourth_vendor_slug: fourthVendorModel,
|
||||
verdict: resolved.verdict,
|
||||
});
|
||||
|
||||
return {
|
||||
verdict: resolved.verdict,
|
||||
path,
|
||||
votes: resolved.votes,
|
||||
fourthVendorVote: fourthVote,
|
||||
fourthVendorSlug: fourthVendorModel,
|
||||
};
|
||||
}
|
||||
|
||||
// Should be unreachable under correct input (any 3-vote distribution is
|
||||
// 3-0, 2-1, or 1-1-1), but stay defensive.
|
||||
throw new Error(
|
||||
`resolveTieBreak: unexpected 3-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
// votes.length === 4 — post-quadri-vendor resolution path.
|
||||
const tally = countByKey(votes);
|
||||
const { topCount, topKeys } = pluralityTop(tally);
|
||||
|
||||
// 1-1-1-1 four-way split — every bucket is one. PM escalation.
|
||||
if (topCount === 1) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'pm-escalation' as TieBreakPath,
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
four_way_keys: topKeys,
|
||||
});
|
||||
return {
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
path: 'pm-escalation',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// Plurality winner: 2-1-1 / 1-1-2 (unique top). 2-2 ties with 4 votes are
|
||||
// structurally impossible from our flow (3 primary votes make 2-2
|
||||
// impossible after the +1 fourth vote; the fourth vote always creates a
|
||||
// unique plurality or a 1-1-1-1). Still handle 2-2 defensively by
|
||||
// promoting to pm-escalation so no silent coin-flip ever lands in prod.
|
||||
if (topCount === 2 && topKeys.length === 1) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'majority' as TieBreakPath,
|
||||
verdict: topKeys[0],
|
||||
vote_count: 4,
|
||||
});
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
if (topCount === 2 && topKeys.length >= 2) {
|
||||
// 2-2 tie on 4 votes — not reachable from 1-1-1→quadri flow, but a
|
||||
// caller may pass in a pre-constructed 4-vote vector. Escalate rather
|
||||
// than coin-flip.
|
||||
logger?.warn?.('tie_break.two-two-tie', {
|
||||
path: 'pm-escalation' as TieBreakPath,
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
two_two_keys: topKeys,
|
||||
});
|
||||
return {
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
path: 'pm-escalation',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
if (topCount === 3 || topCount === 4) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'majority' as TieBreakPath,
|
||||
verdict: topKeys[0],
|
||||
vote_count: 4,
|
||||
});
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// Truly unreachable given 4 input votes; defensive.
|
||||
throw new Error(
|
||||
`resolveTieBreak: unexpected 4-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
|
||||
);
|
||||
}
|
||||
@@ -1,383 +0,0 @@
|
||||
/**
|
||||
* Failure-mode judge — scaffold.
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
|
||||
* Spec: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §4 (prompt),
|
||||
* §6 (ensemble), §8 (inter-judge agreement / Fleiss' kappa thresholds)
|
||||
* LOCKED: PM-Waggle-OS/decisions/2026-04-20-failure-mode-oq-resolutions-locked.md
|
||||
*
|
||||
* This module is deliberately *not wired* into the harness runner in this
|
||||
* sprint (explicitly out of scope per the brief). It exists so that Sprint 9
|
||||
* can drop in the judge call on top of a tested foundation.
|
||||
*
|
||||
* Design decisions worth knowing:
|
||||
*
|
||||
* — Zod validates strictly. The Step-3 schema contract between the model
|
||||
* and us is enforced with a `.refine()` that captures the two
|
||||
* cross-field invariants from the spec §4:
|
||||
* verdict === 'correct' ⇒ failure_mode MUST be null
|
||||
* verdict === 'incorrect' ⇒ failure_mode MUST be one of F1..F5
|
||||
* Any violation → retry once with the reminder → JudgeParseError.
|
||||
*
|
||||
* — JSON extraction is best-effort. Models often wrap JSON in ```json
|
||||
* fences, leading prose, or trailing chatter. We strip the obvious
|
||||
* wrappers before parsing; if both attempts fail we surface the raw
|
||||
* response in the thrown error so a debugger can reproduce.
|
||||
*
|
||||
* — Ensemble tie-break is explicit: on a 2-2 split the ensemble uses the
|
||||
* first judge in the `judgeModels` list. By convention the caller puts
|
||||
* Sonnet first so "2-2 tie broken by Sonnet" holds per the spec §6.
|
||||
*
|
||||
* — Fleiss' kappa is the standard multi-rater formulation (Fleiss 1971).
|
||||
* Raters per subject must be constant; we throw if any subject has a
|
||||
* different rater count, since the formula is undefined otherwise.
|
||||
* Categories are the 6-class vocabulary from §8 (`correct` treated as
|
||||
* a separate class from F1..F5 — the same convention §8 specifies).
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────────────
|
||||
|
||||
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
|
||||
export type Verdict = 'correct' | 'incorrect';
|
||||
|
||||
export interface JudgeResult {
|
||||
verdict: Verdict;
|
||||
failure_mode: null | FailureMode;
|
||||
rationale: string;
|
||||
judge_model: string;
|
||||
}
|
||||
|
||||
export interface LlmClient {
|
||||
/** One-shot completion. Returns the raw text produced by the judge LLM. */
|
||||
complete(prompt: string): Promise<string>;
|
||||
}
|
||||
|
||||
export class JudgeParseError extends Error {
|
||||
readonly lastResponse: string;
|
||||
readonly lastParseError?: string;
|
||||
readonly judgeModel: string;
|
||||
constructor(message: string, details: { lastResponse: string; judgeModel: string; lastParseError?: string }) {
|
||||
super(message);
|
||||
this.name = 'JudgeParseError';
|
||||
this.lastResponse = details.lastResponse;
|
||||
this.judgeModel = details.judgeModel;
|
||||
this.lastParseError = details.lastParseError;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Zod schema (enforces spec §4 Step-3 contract) ──────────────────────
|
||||
|
||||
const judgeSchema = z
|
||||
.object({
|
||||
verdict: z.enum(['correct', 'incorrect']),
|
||||
failure_mode: z.union([z.null(), z.enum(['F1', 'F2', 'F3', 'F4', 'F5'])]),
|
||||
rationale: z.string().min(1, 'rationale must be a non-empty sentence'),
|
||||
})
|
||||
.strict()
|
||||
.refine(
|
||||
r => (r.verdict === 'correct' ? r.failure_mode === null : r.failure_mode !== null),
|
||||
{
|
||||
message:
|
||||
'verdict/failure_mode invariant violated: ' +
|
||||
'verdict="correct" requires failure_mode=null; verdict="incorrect" requires one of F1..F5',
|
||||
},
|
||||
);
|
||||
|
||||
// ── Prompt builder (exact text from strategy §4) ───────────────────────
|
||||
|
||||
/** Exact judge prompt text from `strategy/2026-04-20-failure-mode-taxonomy.md` §4.
|
||||
* Do not modify. The spec requires cell-independent scoring — changing the
|
||||
* prompt between cells, or even between invocations, invalidates the ensemble. */
|
||||
export function buildJudgePrompt(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
}): string {
|
||||
return [
|
||||
"You are evaluating whether an LLM's answer is correct against ground truth.",
|
||||
'',
|
||||
'## Question',
|
||||
params.question,
|
||||
'',
|
||||
'## Ground-truth answer',
|
||||
params.groundTruth,
|
||||
'',
|
||||
'## Ground-truth supporting context (excerpt shown to the model)',
|
||||
params.contextExcerpt,
|
||||
'',
|
||||
"## Model's answer",
|
||||
params.modelAnswer,
|
||||
'',
|
||||
'## Your task',
|
||||
'',
|
||||
"Step 1: Determine if the model's answer is correct.",
|
||||
'- "correct" means the model\'s answer contains all required facts from ground truth, with no additional incorrect claims.',
|
||||
'- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.',
|
||||
'- Extra detail is acceptable ONLY if it is factually correct.',
|
||||
'',
|
||||
'Step 2: If incorrect, assign exactly one failure mode using this decision tree:',
|
||||
'',
|
||||
'1. Does the model explicitly refuse or say it does not know? → F1 (ABSTAIN)',
|
||||
'2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? → F5 (OFF-TOPIC)',
|
||||
'3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? → F4 (HALLUCINATED)',
|
||||
'4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? → F2 (PARTIAL)',
|
||||
'5. Otherwise (model states facts derived from the context but gets them wrong): → F3 (INCORRECT)',
|
||||
'',
|
||||
'Step 3: Return JSON only, no prose, in this exact schema:',
|
||||
'',
|
||||
'{',
|
||||
' "verdict": "correct" | "incorrect",',
|
||||
' "failure_mode": null | "F1" | "F2" | "F3" | "F4" | "F5",',
|
||||
' "rationale": "one sentence explaining the verdict"',
|
||||
'}',
|
||||
'',
|
||||
'If verdict is "correct", failure_mode MUST be null.',
|
||||
'If verdict is "incorrect", failure_mode MUST be one of F1-F5.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Retry reminder prepended to the original prompt when the first response
|
||||
* fails to parse. Text is verbatim from the Task-4 acceptance spec. */
|
||||
export const RETRY_REMINDER =
|
||||
'Your previous response was not valid JSON. Return only the JSON object, no prose.';
|
||||
|
||||
// ── JSON extraction ────────────────────────────────────────────────────
|
||||
|
||||
/** Best-effort extraction of the JSON body from a model response. Handles:
|
||||
* - bare JSON
|
||||
* - `{ ... }` embedded in prose (first `{` to matching last `}`)
|
||||
* - markdown code fences ```json ... ``` or ``` ... ``` */
|
||||
export function extractJsonBody(raw: string): string | null {
|
||||
if (!raw) return null;
|
||||
const trimmed = raw.trim();
|
||||
// Code-fence strip first.
|
||||
const fenceMatch = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i);
|
||||
const afterFence = fenceMatch ? fenceMatch[1].trim() : trimmed;
|
||||
if (afterFence.startsWith('{') && afterFence.endsWith('}')) return afterFence;
|
||||
// Fallback — greedy `{ ... }` scan, stopping at the last `}`.
|
||||
const first = afterFence.indexOf('{');
|
||||
const last = afterFence.lastIndexOf('}');
|
||||
if (first >= 0 && last > first) return afterFence.slice(first, last + 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
function tryParse(raw: string): { ok: true; value: JudgeResult } | { ok: false; error: string } {
|
||||
const body = extractJsonBody(raw);
|
||||
if (!body) return { ok: false, error: 'no JSON object found in response' };
|
||||
let parsedJson: unknown;
|
||||
try {
|
||||
parsedJson = JSON.parse(body);
|
||||
} catch (err) {
|
||||
return { ok: false, error: `JSON.parse failed: ${(err as Error).message}` };
|
||||
}
|
||||
const result = judgeSchema.safeParse(parsedJson);
|
||||
if (!result.success) {
|
||||
return { ok: false, error: `schema validation failed: ${result.error.issues.map(i => i.message).join('; ')}` };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
value: {
|
||||
verdict: result.data.verdict,
|
||||
failure_mode: result.data.failure_mode,
|
||||
rationale: result.data.rationale,
|
||||
judge_model: '', // filled by judgeAnswer from params.judgeModel
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
export async function judgeAnswer(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
judgeModel: string;
|
||||
llmClient: LlmClient;
|
||||
}): Promise<JudgeResult> {
|
||||
const prompt = buildJudgePrompt({
|
||||
question: params.question,
|
||||
groundTruth: params.groundTruth,
|
||||
contextExcerpt: params.contextExcerpt,
|
||||
modelAnswer: params.modelAnswer,
|
||||
});
|
||||
|
||||
const first = await params.llmClient.complete(prompt);
|
||||
const firstParsed = tryParse(first);
|
||||
if (firstParsed.ok) {
|
||||
return { ...firstParsed.value, judge_model: params.judgeModel };
|
||||
}
|
||||
|
||||
// Retry once with the reminder prefixed. A fresh LLM call — callers can
|
||||
// wire conversation-state preservation later; spec §4 does not require it.
|
||||
const retryPrompt = `${RETRY_REMINDER}\n\n${prompt}`;
|
||||
const second = await params.llmClient.complete(retryPrompt);
|
||||
const secondParsed = tryParse(second);
|
||||
if (secondParsed.ok) {
|
||||
return { ...secondParsed.value, judge_model: params.judgeModel };
|
||||
}
|
||||
|
||||
throw new JudgeParseError(
|
||||
`Judge ${params.judgeModel} produced unparseable output after one retry`,
|
||||
{
|
||||
lastResponse: second,
|
||||
judgeModel: params.judgeModel,
|
||||
lastParseError: secondParsed.error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function judgeEnsemble(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
judgeModels: string[];
|
||||
llmClients: Map<string, LlmClient>;
|
||||
}): Promise<{ ensemble: JudgeResult[]; majority: JudgeResult; fleissKappa: number }> {
|
||||
if (params.judgeModels.length === 0) {
|
||||
throw new Error('judgeEnsemble requires at least one judgeModel');
|
||||
}
|
||||
|
||||
const results: JudgeResult[] = [];
|
||||
for (const model of params.judgeModels) {
|
||||
const client = params.llmClients.get(model);
|
||||
if (!client) throw new Error(`judgeEnsemble: no LlmClient registered for model ${model}`);
|
||||
const result = await judgeAnswer({
|
||||
question: params.question,
|
||||
groundTruth: params.groundTruth,
|
||||
contextExcerpt: params.contextExcerpt,
|
||||
modelAnswer: params.modelAnswer,
|
||||
judgeModel: model,
|
||||
llmClient: client,
|
||||
});
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
const majority = computeMajority(results, params.judgeModels[0]);
|
||||
// For a single-subject ensemble, Fleiss' kappa is computed over the same
|
||||
// one-subject matrix — degenerate but well-defined (NaN gets clamped to 0
|
||||
// when the subject has only one category). In production, this function
|
||||
// is typically called per-instance and kappa is aggregated at the batch
|
||||
// layer; we expose the single-subject value for diagnostic continuity.
|
||||
const fleissKappa = computeFleissKappa([results]);
|
||||
|
||||
return { ensemble: results, majority, fleissKappa };
|
||||
}
|
||||
|
||||
function computeMajority(results: readonly JudgeResult[], tieBreakerModel: string): JudgeResult {
|
||||
// Count verdict × failure_mode pairs (null collapses to 'NA' for keying).
|
||||
const tally = new Map<string, number>();
|
||||
const first = new Map<string, JudgeResult>();
|
||||
for (const r of results) {
|
||||
const key = `${r.verdict}|${r.failure_mode ?? 'NA'}`;
|
||||
tally.set(key, (tally.get(key) ?? 0) + 1);
|
||||
if (!first.has(key)) first.set(key, r);
|
||||
}
|
||||
let winnerKey = '';
|
||||
let winnerCount = -1;
|
||||
for (const [key, count] of tally) {
|
||||
if (count > winnerCount) {
|
||||
winnerKey = key;
|
||||
winnerCount = count;
|
||||
}
|
||||
}
|
||||
// Tie detection: if any other key has equal count, the tie-breaker model
|
||||
// wins — its verdict becomes the majority.
|
||||
const tied = Array.from(tally.entries()).filter(([, c]) => c === winnerCount);
|
||||
if (tied.length > 1) {
|
||||
const tb = results.find(r => r.judge_model === tieBreakerModel);
|
||||
if (tb) return tb;
|
||||
// Fallback: keep the first deterministic winner (stable map iteration).
|
||||
}
|
||||
const winner = first.get(winnerKey);
|
||||
if (!winner) {
|
||||
// Unreachable under normal flow; defensive.
|
||||
return results[0];
|
||||
}
|
||||
return {
|
||||
verdict: winner.verdict,
|
||||
failure_mode: winner.failure_mode,
|
||||
rationale: winner.rationale,
|
||||
judge_model: 'ensemble_majority',
|
||||
};
|
||||
}
|
||||
|
||||
// ── Fleiss' kappa ──────────────────────────────────────────────────────
|
||||
|
||||
/** The 6-class rating space used by Fleiss' kappa: correct + F1..F5.
|
||||
* §8: "standardni Fleiss' kappa preko 4 raters × N instanci × 6 klasa". */
|
||||
const KAPPA_CATEGORIES = ['correct', 'F1', 'F2', 'F3', 'F4', 'F5'] as const;
|
||||
type KappaCategory = typeof KAPPA_CATEGORIES[number];
|
||||
|
||||
function categoryOf(r: JudgeResult): KappaCategory {
|
||||
if (r.verdict === 'correct') return 'correct';
|
||||
return r.failure_mode ?? 'F3'; // should never hit fallback — schema guard
|
||||
}
|
||||
|
||||
/** Computes Fleiss' kappa on an N-subjects × n-raters matrix.
|
||||
*
|
||||
* Input shape: `ratings[i]` is the list of judge results for subject i —
|
||||
* all subjects must have the same number of raters. Categories are the
|
||||
* fixed 6-class vocabulary.
|
||||
*
|
||||
* Returns: κ in [-1, 1]. If all subjects are unanimous on the same
|
||||
* category (expected-agreement = 1), returns 1 (the (1-1)/(1-1) limit).
|
||||
*/
|
||||
export function computeFleissKappa(ratings: readonly (readonly JudgeResult[])[]): number {
|
||||
if (ratings.length === 0) return 0;
|
||||
const n = ratings[0].length;
|
||||
if (n < 2) {
|
||||
// Fleiss' kappa is undefined for a single rater. Degenerate → 0.
|
||||
return 0;
|
||||
}
|
||||
for (const row of ratings) {
|
||||
if (row.length !== n) {
|
||||
throw new Error(`Fleiss' kappa requires a constant rater count; saw ${row.length} and ${n}`);
|
||||
}
|
||||
}
|
||||
|
||||
const N = ratings.length;
|
||||
const k = KAPPA_CATEGORIES.length;
|
||||
const catIndex: Record<KappaCategory, number> = { correct: 0, F1: 1, F2: 2, F3: 3, F4: 4, F5: 5 };
|
||||
|
||||
// nij[i][j] = count of raters who assigned subject i to category j
|
||||
const nij: number[][] = Array.from({ length: N }, () => new Array<number>(k).fill(0));
|
||||
for (let i = 0; i < N; i++) {
|
||||
for (const r of ratings[i]) {
|
||||
const col = catIndex[categoryOf(r)];
|
||||
nij[i][col] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Pj = (1/(N*n)) * sum_i nij[i][j]
|
||||
const Pj: number[] = new Array<number>(k).fill(0);
|
||||
for (let j = 0; j < k; j++) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < N; i++) sum += nij[i][j];
|
||||
Pj[j] = sum / (N * n);
|
||||
}
|
||||
|
||||
// Pi = (1/(n(n-1))) * ( sum_j nij[i][j]^2 - n )
|
||||
let PbarSum = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
let sqSum = 0;
|
||||
for (let j = 0; j < k; j++) sqSum += nij[i][j] * nij[i][j];
|
||||
const Pi = (sqSum - n) / (n * (n - 1));
|
||||
PbarSum += Pi;
|
||||
}
|
||||
const Pbar = PbarSum / N;
|
||||
|
||||
// Pebar = sum_j Pj^2
|
||||
let Pebar = 0;
|
||||
for (let j = 0; j < k; j++) Pebar += Pj[j] * Pj[j];
|
||||
|
||||
if (Pebar >= 1 - 1e-12) {
|
||||
// Every rating in one category → expected = observed = 1 → κ = 1 by convention.
|
||||
return 1;
|
||||
}
|
||||
return (Pbar - Pebar) / (1 - Pebar);
|
||||
}
|
||||
Reference in New Issue
Block a user