chore(server): remove the memory benchmark judge scaffolding (P2.2)
packages/server/src/benchmarks held a LoCoMo failure-mode aggregator and an LLM judge ensemble for the old memory engine benchmarks. Nothing in the sidecar imports it and no npm script runs it; its inputs lived in the root benchmarks/ directory that P2.2 already removed. Judge and eval loops are also out of the community edition (AGENTS.md, rule 4).
This commit is contained in:
@@ -1,432 +0,0 @@
|
||||
/**
|
||||
* Aggregate.ts — failure-mode distribution rollup for Stage 1 / Stage 2
|
||||
* preflight + Week 1/2 main runs.
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3
|
||||
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
|
||||
*
|
||||
* Consumes JSONL output from `benchmarks/harness` (one record per
|
||||
* instance per cell) and emits a structured report covering:
|
||||
* 1. Per-cell verdict distribution + weighted quality score
|
||||
* 2. Per-LoCoMo-category distribution (single/multi-hop/temporal/
|
||||
* open-ended) with a hallucination rate flag for PM review
|
||||
* 3. Cross-cell delta matrix (full-context vs raw: correct / F4 /
|
||||
* F1 lift per taxonomy §5)
|
||||
* 4. Judge cost summary when the run included judge calls
|
||||
*
|
||||
* Input shape: each JSONL line has the fields declared in
|
||||
* `benchmarks/harness/src/types.ts` (optional `judge_verdict`,
|
||||
* `judge_failure_mode`, `model_answer`, etc.). Pre-judge records — rows
|
||||
* without `judge_verdict` — are counted under an `unjudged` bucket so
|
||||
* partial-run reports stay honest instead of silently skipping data.
|
||||
*/
|
||||
|
||||
export type Verdict6 =
|
||||
| 'correct'
|
||||
| 'F1_abstain'
|
||||
| 'F2_partial'
|
||||
| 'F3_incorrect'
|
||||
| 'F4_hallucinated'
|
||||
| 'F5_offtopic'
|
||||
| 'unjudged';
|
||||
|
||||
export type CellName = 'raw' | 'filtered' | 'compressed' | 'full-context';
|
||||
const CELL_NAMES: readonly CellName[] = ['raw', 'filtered', 'compressed', 'full-context'];
|
||||
|
||||
/** Categories match `preflight-locomo-50.json` `_meta.locomo_category_map`
|
||||
* minus the excluded adversarial bucket. Unknown categories fall into
|
||||
* `other` so the report still tallies them instead of dropping. */
|
||||
export type LocomoCategory = 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended' | 'other';
|
||||
const LOCOMO_CATEGORIES: readonly LocomoCategory[] = [
|
||||
'single-hop', 'multi-hop', 'temporal', 'open-ended', 'other',
|
||||
];
|
||||
|
||||
/** Inputs can carry either the explicit category column or the
|
||||
* pre-lock instance_id (`locomo_<sample_id>_q<qindex>`) from which
|
||||
* we can recover the category via a sibling lookup. This module only
|
||||
* consumes the already-tagged JsonlRecord; category recovery lives in
|
||||
* the caller (harness runner attaches `category` when loading from
|
||||
* the sample lock). */
|
||||
export interface JudgedJsonlRecord {
|
||||
turnId: string;
|
||||
cell: string;
|
||||
instance_id: string;
|
||||
model: string;
|
||||
seed: number;
|
||||
accuracy: number;
|
||||
p50_latency_ms: number;
|
||||
p95_latency_ms: number;
|
||||
usd_per_query: number;
|
||||
failure_mode: string | null;
|
||||
model_answer?: string;
|
||||
judge_verdict?: 'correct' | 'incorrect';
|
||||
judge_failure_mode?: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
|
||||
judge_rationale?: string;
|
||||
judge_model?: string;
|
||||
judge_timestamp?: string;
|
||||
judge_confidence?: number;
|
||||
judge_ensemble?: Array<{
|
||||
model: string;
|
||||
verdict: 'correct' | 'incorrect';
|
||||
failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
|
||||
rationale?: string;
|
||||
latency_ms?: number;
|
||||
}>;
|
||||
/** Optional category tag attached by the runner when the sample came
|
||||
* from preflight-locomo-50.json. When absent, rows fall into
|
||||
* `other` for the per-category rollup. */
|
||||
category?: LocomoCategory;
|
||||
}
|
||||
|
||||
/** Projection from the taxonomy §9 binary `verdict` + `failure_mode`
|
||||
* slot pair to the 6-value Verdict6 enum the brief Task-1 spec asked
|
||||
* for. Centralising this here means JsonlRecord keeps the canonical
|
||||
* taxonomy shape while the aggregator surfaces the denser report
|
||||
* form. */
|
||||
export function projectVerdict6(r: JudgedJsonlRecord): Verdict6 {
|
||||
if (r.judge_verdict === undefined) return 'unjudged';
|
||||
if (r.judge_verdict === 'correct') return 'correct';
|
||||
switch (r.judge_failure_mode) {
|
||||
case 'F1': return 'F1_abstain';
|
||||
case 'F2': return 'F2_partial';
|
||||
case 'F3': return 'F3_incorrect';
|
||||
case 'F4': return 'F4_hallucinated';
|
||||
case 'F5': return 'F5_offtopic';
|
||||
default:
|
||||
// Contract violation — the taxonomy schema (judge module's Zod)
|
||||
// prevents `incorrect` with a null failure_mode. If it sneaks
|
||||
// through a malformed JSONL row, bucket as F3 by convention so
|
||||
// the row doesn't silently drop.
|
||||
return 'F3_incorrect';
|
||||
}
|
||||
}
|
||||
|
||||
export const VERDICT6_VALUES: readonly Verdict6[] = [
|
||||
'correct', 'F1_abstain', 'F2_partial', 'F3_incorrect', 'F4_hallucinated', 'F5_offtopic', 'unjudged',
|
||||
];
|
||||
|
||||
// ── Per-cell distribution + weighted score ─────────────────────────────
|
||||
|
||||
export interface PerCellRow {
|
||||
cell: string;
|
||||
total: number;
|
||||
counts: Record<Verdict6, number>;
|
||||
percents: Record<Verdict6, number>;
|
||||
/** Weighted quality score per taxonomy §5 rubric. `unjudged` is
|
||||
* excluded from the denominator so the score is computed over the
|
||||
* set of instances actually graded. */
|
||||
weightedScore: number;
|
||||
}
|
||||
|
||||
/** Taxonomy §5 coefficient table. Positive weights for correct / F2
|
||||
* (partial credit), zero for F1 (abstain neutral), negatives for
|
||||
* F3 / F4 / F5. Exported so Task-3 unit tests can reproduce the
|
||||
* hand-check number by name. */
|
||||
export const WEIGHTS: Record<Exclude<Verdict6, 'unjudged'>, number> = {
|
||||
correct: 1.0,
|
||||
F2_partial: 0.30,
|
||||
F1_abstain: 0.0,
|
||||
F3_incorrect: -0.15,
|
||||
F4_hallucinated: -0.35,
|
||||
F5_offtopic: -0.10,
|
||||
};
|
||||
|
||||
function emptyVerdictMap(fill = 0): Record<Verdict6, number> {
|
||||
const out = {} as Record<Verdict6, number>;
|
||||
for (const v of VERDICT6_VALUES) out[v] = fill;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function perCellRollup(records: readonly JudgedJsonlRecord[]): PerCellRow[] {
|
||||
const byCell = new Map<string, JudgedJsonlRecord[]>();
|
||||
for (const r of records) {
|
||||
const arr = byCell.get(r.cell) ?? [];
|
||||
arr.push(r);
|
||||
byCell.set(r.cell, arr);
|
||||
}
|
||||
const rows: PerCellRow[] = [];
|
||||
// Emit in CELL_NAMES order when present, then any controls / extras.
|
||||
const orderedCellKeys = [
|
||||
...CELL_NAMES.filter(c => byCell.has(c)),
|
||||
...[...byCell.keys()].filter(c => !CELL_NAMES.includes(c as CellName)),
|
||||
];
|
||||
for (const cell of orderedCellKeys) {
|
||||
const rowRecords = byCell.get(cell)!;
|
||||
const counts = emptyVerdictMap();
|
||||
for (const r of rowRecords) counts[projectVerdict6(r)]++;
|
||||
const total = rowRecords.length;
|
||||
const judgedTotal = total - counts['unjudged'];
|
||||
const percents = emptyVerdictMap();
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
percents[v] = total === 0 ? 0 : counts[v] / total;
|
||||
}
|
||||
let weightedScore = 0;
|
||||
if (judgedTotal > 0) {
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
if (v === 'unjudged') continue;
|
||||
weightedScore += (counts[v] / judgedTotal) * WEIGHTS[v];
|
||||
}
|
||||
}
|
||||
rows.push({ cell, total, counts, percents, weightedScore });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── Per-LoCoMo-category distribution ───────────────────────────────────
|
||||
|
||||
export interface PerCategoryRow {
|
||||
category: LocomoCategory;
|
||||
total: number;
|
||||
counts: Record<Verdict6, number>;
|
||||
percents: Record<Verdict6, number>;
|
||||
/** True when F4 (hallucinated) share exceeds 20% — flags the
|
||||
* category for PM spot-review per brief §Task-3 rule 2. */
|
||||
hallucinationFlag: boolean;
|
||||
}
|
||||
|
||||
const HALLUCINATION_FLAG_THRESHOLD = 0.20;
|
||||
|
||||
export function perCategoryRollup(records: readonly JudgedJsonlRecord[]): PerCategoryRow[] {
|
||||
const byCat = new Map<LocomoCategory, JudgedJsonlRecord[]>();
|
||||
for (const r of records) {
|
||||
const cat = r.category ?? 'other';
|
||||
const arr = byCat.get(cat) ?? [];
|
||||
arr.push(r);
|
||||
byCat.set(cat, arr);
|
||||
}
|
||||
const rows: PerCategoryRow[] = [];
|
||||
for (const cat of LOCOMO_CATEGORIES) {
|
||||
const recs = byCat.get(cat) ?? [];
|
||||
if (recs.length === 0) continue;
|
||||
const counts = emptyVerdictMap();
|
||||
for (const r of recs) counts[projectVerdict6(r)]++;
|
||||
const total = recs.length;
|
||||
const percents = emptyVerdictMap();
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
percents[v] = total === 0 ? 0 : counts[v] / total;
|
||||
}
|
||||
rows.push({
|
||||
category: cat,
|
||||
total,
|
||||
counts,
|
||||
percents,
|
||||
hallucinationFlag: percents['F4_hallucinated'] > HALLUCINATION_FLAG_THRESHOLD,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// ── Cross-cell delta matrix (full-context vs raw) ──────────────────────
|
||||
|
||||
export interface CrossCellDelta {
|
||||
verdict: Verdict6;
|
||||
rawPercent: number;
|
||||
fullContextPercent: number;
|
||||
/** fullContext − raw. Positive = full-context raised this verdict's share;
|
||||
* negative = full-context lowered it. The brief's headline expectation:
|
||||
* `correct` delta > 0, `F4_hallucinated` delta < 0, `F1_abstain`
|
||||
* may go either way (more abstains is sometimes an OK signal). */
|
||||
delta: number;
|
||||
}
|
||||
|
||||
export function crossCellDeltaMatrix(perCell: readonly PerCellRow[]): CrossCellDelta[] | null {
|
||||
const raw = perCell.find(r => r.cell === 'raw');
|
||||
const full = perCell.find(r => r.cell === 'full-context');
|
||||
if (!raw || !full) return null;
|
||||
const out: CrossCellDelta[] = [];
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
out.push({
|
||||
verdict: v,
|
||||
rawPercent: raw.percents[v],
|
||||
fullContextPercent: full.percents[v],
|
||||
delta: full.percents[v] - raw.percents[v],
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Cost summary ───────────────────────────────────────────────────────
|
||||
|
||||
export interface CostSummary {
|
||||
totalUsd: number;
|
||||
perCellUsd: Record<string, number>;
|
||||
judgeTotalUsd: number;
|
||||
judgeCallCount: number;
|
||||
/** Median judge call latency across the run; `null` when no judge
|
||||
* call recorded a latency. */
|
||||
medianJudgeMs: number | null;
|
||||
/** Brief §Task-3 rule 4: flag when total judge spend would exceed
|
||||
* ~$20 projected for a full 4-cell × 50-instance run. Computed as
|
||||
* `perInstanceJudgeUsd × 200`. */
|
||||
week1WarningProjectedUsd: number;
|
||||
week1WarningFired: boolean;
|
||||
}
|
||||
|
||||
const WEEK1_PROJECTION_INSTANCE_COUNT = 200; // 4 cells × 50 instances
|
||||
|
||||
export function costSummary(records: readonly JudgedJsonlRecord[]): CostSummary {
|
||||
const perCellUsd: Record<string, number> = {};
|
||||
let total = 0;
|
||||
const judgeLatencies: number[] = [];
|
||||
const judgeTotal = 0;
|
||||
let judgeCalls = 0;
|
||||
for (const r of records) {
|
||||
total += r.usd_per_query;
|
||||
perCellUsd[r.cell] = (perCellUsd[r.cell] ?? 0) + r.usd_per_query;
|
||||
if (r.judge_ensemble) {
|
||||
for (const entry of r.judge_ensemble) {
|
||||
if (typeof entry.latency_ms === 'number') judgeLatencies.push(entry.latency_ms);
|
||||
}
|
||||
judgeCalls += r.judge_ensemble.length;
|
||||
} else if (r.judge_verdict !== undefined) {
|
||||
// Single-judge call — we don't store per-judge latency in the
|
||||
// JsonlRecord shape (no judge_latency_ms field), so the median
|
||||
// is computed only over ensemble entries that carry it. If no
|
||||
// ensemble row is present, medianJudgeMs stays null.
|
||||
judgeCalls += 1;
|
||||
}
|
||||
}
|
||||
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
|
||||
// Per-instance judge spend is not stored directly — the aggregator
|
||||
// doesn't know judge USD without the onCall sink. When the caller
|
||||
// passes judgeTotalUsd separately (see buildReport), we populate
|
||||
// this from the authoritative source. Here we return zero; buildReport
|
||||
// overwrites with the real total.
|
||||
const medianJudgeMs = judgeLatencies.length === 0
|
||||
? null
|
||||
: (() => {
|
||||
const s = [...judgeLatencies].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 === 0 ? (s[mid - 1] + s[mid]) / 2 : s[mid];
|
||||
})();
|
||||
const perInstanceJudgeUsd = judgedCount > 0 ? judgeTotal / judgedCount : 0;
|
||||
const projected = perInstanceJudgeUsd * WEEK1_PROJECTION_INSTANCE_COUNT;
|
||||
return {
|
||||
totalUsd: total,
|
||||
perCellUsd,
|
||||
judgeTotalUsd: judgeTotal,
|
||||
judgeCallCount: judgeCalls,
|
||||
medianJudgeMs,
|
||||
week1WarningProjectedUsd: projected,
|
||||
week1WarningFired: projected > 20,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Top-level report + markdown renderer ───────────────────────────────
|
||||
|
||||
export interface AggregateReport {
|
||||
generatedAt: string;
|
||||
totalRecords: number;
|
||||
perCell: PerCellRow[];
|
||||
perCategory: PerCategoryRow[];
|
||||
crossCellDelta: CrossCellDelta[] | null;
|
||||
cost: CostSummary;
|
||||
}
|
||||
|
||||
export function buildReport(
|
||||
records: readonly JudgedJsonlRecord[],
|
||||
opts: { judgeTotalUsd?: number } = {},
|
||||
): AggregateReport {
|
||||
const perCell = perCellRollup(records);
|
||||
const perCategory = perCategoryRollup(records);
|
||||
const crossCellDelta = crossCellDeltaMatrix(perCell);
|
||||
const cost = costSummary(records);
|
||||
// Callers that run the harness end-to-end know the authoritative
|
||||
// judge spend because judge-client emits `onCall` entries. Pass the
|
||||
// aggregated total here so the report carries real dollars instead
|
||||
// of the placeholder zero computed from JSONL alone.
|
||||
if (opts.judgeTotalUsd !== undefined) {
|
||||
cost.judgeTotalUsd = opts.judgeTotalUsd;
|
||||
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
|
||||
const perInstance = judgedCount > 0 ? opts.judgeTotalUsd / judgedCount : 0;
|
||||
cost.week1WarningProjectedUsd = perInstance * WEEK1_PROJECTION_INSTANCE_COUNT;
|
||||
cost.week1WarningFired = cost.week1WarningProjectedUsd > 20;
|
||||
}
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalRecords: records.length,
|
||||
perCell,
|
||||
perCategory,
|
||||
crossCellDelta,
|
||||
cost,
|
||||
};
|
||||
}
|
||||
|
||||
function pct(n: number): string {
|
||||
return `${(n * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function fmtUsd(n: number): string {
|
||||
return `$${n.toFixed(n < 1 ? 6 : 2)}`;
|
||||
}
|
||||
|
||||
export function renderMarkdown(report: AggregateReport): string {
|
||||
const lines: string[] = [];
|
||||
lines.push('# Benchmark Aggregate Report');
|
||||
lines.push('');
|
||||
lines.push(`**Generated at:** ${report.generatedAt}`);
|
||||
lines.push(`**Total records:** ${report.totalRecords}`);
|
||||
lines.push('');
|
||||
lines.push('## Per-cell verdict distribution');
|
||||
lines.push('');
|
||||
lines.push('| Cell | Total | Correct | F1 abstain | F2 partial | F3 incorrect | F4 hallucinated | F5 off-topic | Unjudged | Weighted score |');
|
||||
lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|');
|
||||
for (const row of report.perCell) {
|
||||
lines.push(
|
||||
`| ${row.cell} | ${row.total} | ${row.counts.correct} (${pct(row.percents.correct)}) | ` +
|
||||
`${row.counts.F1_abstain} (${pct(row.percents.F1_abstain)}) | ` +
|
||||
`${row.counts.F2_partial} (${pct(row.percents.F2_partial)}) | ` +
|
||||
`${row.counts.F3_incorrect} (${pct(row.percents.F3_incorrect)}) | ` +
|
||||
`${row.counts.F4_hallucinated} (${pct(row.percents.F4_hallucinated)}) | ` +
|
||||
`${row.counts.F5_offtopic} (${pct(row.percents.F5_offtopic)}) | ` +
|
||||
`${row.counts.unjudged} (${pct(row.percents.unjudged)}) | ` +
|
||||
`${row.weightedScore.toFixed(3)} |`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
if (report.perCategory.length > 0) {
|
||||
lines.push('## Per-LoCoMo-category distribution');
|
||||
lines.push('');
|
||||
lines.push('| Category | Total | Correct% | F4 (hallucinated)% | Flagged? |');
|
||||
lines.push('|---|---:|---:|---:|:---:|');
|
||||
for (const row of report.perCategory) {
|
||||
lines.push(
|
||||
`| ${row.category} | ${row.total} | ${pct(row.percents.correct)} | ` +
|
||||
`${pct(row.percents.F4_hallucinated)} | ` +
|
||||
`${row.hallucinationFlag ? '⚠️ PM review' : '✓'} |`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
if (report.crossCellDelta) {
|
||||
lines.push('## Full-context vs raw delta');
|
||||
lines.push('');
|
||||
lines.push('| Verdict | raw % | full-context % | Δ (full − raw) |');
|
||||
lines.push('|---|---:|---:|---:|');
|
||||
for (const r of report.crossCellDelta) {
|
||||
const sign = r.delta > 0 ? '+' : '';
|
||||
lines.push(`| ${r.verdict} | ${pct(r.rawPercent)} | ${pct(r.fullContextPercent)} | ${sign}${pct(r.delta)} |`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
lines.push('## Cost summary');
|
||||
lines.push('');
|
||||
lines.push(`- Total cell spend: **${fmtUsd(report.cost.totalUsd)}**`);
|
||||
lines.push(`- Judge spend: **${fmtUsd(report.cost.judgeTotalUsd)}** across ${report.cost.judgeCallCount} call(s)`);
|
||||
if (report.cost.medianJudgeMs !== null) {
|
||||
lines.push(`- Median judge latency: ${report.cost.medianJudgeMs.toFixed(0)} ms`);
|
||||
}
|
||||
if (Object.keys(report.cost.perCellUsd).length > 0) {
|
||||
lines.push('- Per-cell cell spend:');
|
||||
for (const [cell, usd] of Object.entries(report.cost.perCellUsd)) {
|
||||
lines.push(` - ${cell}: ${fmtUsd(usd)}`);
|
||||
}
|
||||
}
|
||||
if (report.cost.week1WarningFired) {
|
||||
lines.push(
|
||||
`- ⚠️ **Week-1 scale-up warning:** projected judge spend for a full 4-cell × 50-instance run is ` +
|
||||
`${fmtUsd(report.cost.week1WarningProjectedUsd)} (threshold $20). Flagged per brief §Task-3 rule 4.`,
|
||||
);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -1,314 +0,0 @@
|
||||
/**
|
||||
* Sprint 11 Task B2 — Tie-break policy for multi-vendor judge ensemble.
|
||||
*
|
||||
* Authority:
|
||||
* - PM-Waggle-OS/briefs/2026-04-22-cc-sprint-11-kickoff.md §3 Track B B2
|
||||
* - PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
|
||||
*
|
||||
* Context — what this resolves:
|
||||
*
|
||||
* The Sprint 10 Task 2.2 ratified primary judge ensemble is a THREE-vendor
|
||||
* panel (Anthropic Opus 4.7 + OpenAI GPT-5.4 + Google Gemini 3.1-Pro). Three
|
||||
* votes means three possible distributions:
|
||||
*
|
||||
* - 3-0 consensus → trivial, no tie-break
|
||||
* - 2-1 majority → majority wins, no tie-break
|
||||
* - 1-1-1 split → three different verdicts, no primary majority
|
||||
*
|
||||
* On 1-1-1, we escalate to a FOURTH vendor from a lineage disjoint from the
|
||||
* primary trio: xai/grok-4.20 per Marko's 2026-04-22 LOCK. After the 4th
|
||||
* vote arrives we have four votes total:
|
||||
*
|
||||
* - 2-1-1 / 1-1-2 → plurality winner (2 votes)
|
||||
* - 1-1-1-1 → four-way split → PM escalation
|
||||
*
|
||||
* The fourth vendor is SPECIFICALLY xai/grok-4.20 — not Sonnet 4.6 (which
|
||||
* would give Anthropic 2-of-4 weight → homogeneous-bias risk — rejected per
|
||||
* LOCK doc §2) and not Opus 4.7 (already Judge 1).
|
||||
*
|
||||
* Observability: pino structured log emits `tie_break.path` ∈ { 'none',
|
||||
* 'majority', 'quadri-vendor', 'pm-escalation' } plus
|
||||
* `tie_break.fourth_vendor_slug` on quadri paths (field is future-proof
|
||||
* even though Sprint 11 scope is grok-4.20 only).
|
||||
*/
|
||||
|
||||
import type { JudgeResult } from './failure-mode-judge.js';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A single judge vote. Reusing JudgeResult from failure-mode-judge.ts keeps
|
||||
* the tie-break input shape aligned with what judgeEnsemble already produces.
|
||||
*/
|
||||
export type Vote = JudgeResult;
|
||||
|
||||
export type TieBreakPath = 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
|
||||
|
||||
/**
|
||||
* The canonical sentinel verdict returned on a 1-1-1-1 four-way split.
|
||||
* Callers (aggregators, report generators) MUST treat this as an operator
|
||||
* action signal, not as a normal verdict string. The shape is intentionally
|
||||
* unambiguous so no accidental inclusion into recall/accuracy math.
|
||||
*/
|
||||
export const PM_ESCALATION_VERDICT = '__PM_ESCALATION__';
|
||||
|
||||
/** Default fourth vendor for Sprint 11 — per LOCK doc §1. */
|
||||
export const DEFAULT_FOURTH_VENDOR = 'xai/grok-4.20';
|
||||
|
||||
export interface TieBreakResult {
|
||||
/**
|
||||
* The resolved verdict string. Encoded as `<verdict>|<failure_mode_or_NA>`
|
||||
* matching the aggregation key used in computeMajority of failure-mode-judge.ts,
|
||||
* so downstream consumers can uniformly decompose.
|
||||
* On pm-escalation path: `__PM_ESCALATION__`.
|
||||
*/
|
||||
verdict: string;
|
||||
path: TieBreakPath;
|
||||
/** The full vote list after resolution — 3 votes for none/majority, 4 for quadri/pm-escalation. */
|
||||
votes: Vote[];
|
||||
/** The fourth vendor's vote when a quadri path was taken. Undefined on none/majority. */
|
||||
fourthVendorVote?: Vote;
|
||||
/** The fourth-vendor slug that was used, for observability. Undefined on none/majority. */
|
||||
fourthVendorSlug?: string;
|
||||
}
|
||||
|
||||
export interface FourthVendorCallPayload {
|
||||
/** The three primary votes that produced the 1-1-1 split. */
|
||||
primaryVotes: Vote[];
|
||||
/** Model slug to invoke — e.g. 'xai/grok-4.20'. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export type CallFourthVendor = (payload: FourthVendorCallPayload) => Promise<Vote>;
|
||||
|
||||
export interface TieBreakLogger {
|
||||
info(event: string, fields: Record<string, unknown>): void;
|
||||
warn?(event: string, fields: Record<string, unknown>): void;
|
||||
}
|
||||
|
||||
export interface ResolveTieBreakOptions {
|
||||
/**
|
||||
* Invoked only on 1-1-1 splits. Required when primaryVotes.length === 3
|
||||
* and the distribution is 1-1-1; optional otherwise. Tests mock this.
|
||||
*/
|
||||
callFourthVendor?: CallFourthVendor;
|
||||
/** Override the fourth vendor slug. Defaults to `DEFAULT_FOURTH_VENDOR`. */
|
||||
fourthVendorModel?: string;
|
||||
/** Structured logger. Silently no-ops when omitted. */
|
||||
logger?: TieBreakLogger;
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Canonical key for a vote — combines verdict + failure_mode so identical
|
||||
* answers with different failure codes don't accidentally tie.
|
||||
*/
|
||||
function voteKey(v: Vote): string {
|
||||
return `${v.verdict}|${v.failure_mode ?? 'NA'}`;
|
||||
}
|
||||
|
||||
function countByKey(votes: Vote[]): Map<string, { count: number; first: Vote }> {
|
||||
const tally = new Map<string, { count: number; first: Vote }>();
|
||||
for (const v of votes) {
|
||||
const key = voteKey(v);
|
||||
const existing = tally.get(key);
|
||||
if (existing) existing.count += 1;
|
||||
else tally.set(key, { count: 1, first: v });
|
||||
}
|
||||
return tally;
|
||||
}
|
||||
|
||||
function pluralityTop(tally: Map<string, { count: number; first: Vote }>): {
|
||||
topCount: number;
|
||||
topKeys: string[];
|
||||
} {
|
||||
let topCount = 0;
|
||||
for (const { count } of tally.values()) if (count > topCount) topCount = count;
|
||||
const topKeys: string[] = [];
|
||||
for (const [key, { count }] of tally) if (count === topCount) topKeys.push(key);
|
||||
return { topCount, topKeys };
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolve a multi-vendor judge ensemble vote.
|
||||
*
|
||||
* Input shape:
|
||||
* - 3 primary votes (Sprint 10 Task 2.2 trio) — the normal call path.
|
||||
* - 4 votes also accepted, in which case the function treats the vector
|
||||
* as already-quadri-resolved and returns the plurality winner (or
|
||||
* pm-escalation on 1-1-1-1). This shape exists for tests and for
|
||||
* caller-side recomposition.
|
||||
*
|
||||
* On 1-1-1 with 3 primary votes, `callFourthVendor` MUST be provided.
|
||||
* The function invokes it with the three primary votes and the canonical
|
||||
* model slug, then recurses with the four-vote list.
|
||||
*
|
||||
* Returns:
|
||||
* - `{ path: 'none' }` for 3-0 consensus (3 primary votes).
|
||||
* - `{ path: 'majority' }` for 2-1 majority (3 votes) OR for a plurality
|
||||
* win on 4 votes when the caller supplied the vector directly.
|
||||
* - `{ path: 'quadri-vendor' }` when we called the 4th vendor ourselves
|
||||
* and got a 2-1-1 or 1-1-2 distribution.
|
||||
* - `{ path: 'pm-escalation' }` on a 1-1-1-1 four-way split; verdict is
|
||||
* the `PM_ESCALATION_VERDICT` sentinel.
|
||||
*/
|
||||
export async function resolveTieBreak(
|
||||
votes: Vote[],
|
||||
options: ResolveTieBreakOptions = {},
|
||||
): Promise<TieBreakResult> {
|
||||
if (votes.length !== 3 && votes.length !== 4) {
|
||||
throw new Error(
|
||||
`resolveTieBreak: votes.length must be 3 (primary ensemble) or 4 (post-quadri). Got ${votes.length}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const logger = options.logger;
|
||||
|
||||
if (votes.length === 3) {
|
||||
const tally = countByKey(votes);
|
||||
const { topCount, topKeys } = pluralityTop(tally);
|
||||
|
||||
// 3-0 consensus — one bucket holds all three.
|
||||
if (topCount === 3) {
|
||||
const entry = tally.get(topKeys[0])!;
|
||||
logger?.info('tie_break', { path: 'none' as TieBreakPath, verdict: topKeys[0] });
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'none',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// 2-1 majority — one bucket holds two, another holds one.
|
||||
if (topCount === 2 && topKeys.length === 1) {
|
||||
logger?.info('tie_break', { path: 'majority' as TieBreakPath, verdict: topKeys[0] });
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// 1-1-1 split — three buckets of one each. Escalate to fourth vendor.
|
||||
if (topCount === 1 && topKeys.length === 3) {
|
||||
if (!options.callFourthVendor) {
|
||||
throw new Error(
|
||||
'resolveTieBreak: 1-1-1 three-way split requires a callFourthVendor implementation.',
|
||||
);
|
||||
}
|
||||
const fourthVendorModel = options.fourthVendorModel ?? DEFAULT_FOURTH_VENDOR;
|
||||
|
||||
logger?.info('tie_break.quadri-vendor.invoke', {
|
||||
path: 'quadri-vendor' as TieBreakPath,
|
||||
fourth_vendor_slug: fourthVendorModel,
|
||||
primary_keys: topKeys,
|
||||
});
|
||||
|
||||
const fourthVote = await options.callFourthVendor({
|
||||
primaryVotes: votes,
|
||||
model: fourthVendorModel,
|
||||
});
|
||||
|
||||
// Recurse with four votes. The 4-vote branch returns quadri-vendor or pm-escalation.
|
||||
const resolved = await resolveTieBreak([...votes, fourthVote], options);
|
||||
|
||||
// Re-tag the path so "quadri-vendor" sticks on success cases and
|
||||
// "pm-escalation" stays on 1-1-1-1 after escalation.
|
||||
const path: TieBreakPath =
|
||||
resolved.path === 'pm-escalation' ? 'pm-escalation' : 'quadri-vendor';
|
||||
|
||||
logger?.info('tie_break.quadri-vendor.resolved', {
|
||||
path,
|
||||
fourth_vendor_slug: fourthVendorModel,
|
||||
verdict: resolved.verdict,
|
||||
});
|
||||
|
||||
return {
|
||||
verdict: resolved.verdict,
|
||||
path,
|
||||
votes: resolved.votes,
|
||||
fourthVendorVote: fourthVote,
|
||||
fourthVendorSlug: fourthVendorModel,
|
||||
};
|
||||
}
|
||||
|
||||
// Should be unreachable under correct input (any 3-vote distribution is
|
||||
// 3-0, 2-1, or 1-1-1), but stay defensive.
|
||||
throw new Error(
|
||||
`resolveTieBreak: unexpected 3-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
// votes.length === 4 — post-quadri-vendor resolution path.
|
||||
const tally = countByKey(votes);
|
||||
const { topCount, topKeys } = pluralityTop(tally);
|
||||
|
||||
// 1-1-1-1 four-way split — every bucket is one. PM escalation.
|
||||
if (topCount === 1) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'pm-escalation' as TieBreakPath,
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
four_way_keys: topKeys,
|
||||
});
|
||||
return {
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
path: 'pm-escalation',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// Plurality winner: 2-1-1 / 1-1-2 (unique top). 2-2 ties with 4 votes are
|
||||
// structurally impossible from our flow (3 primary votes make 2-2
|
||||
// impossible after the +1 fourth vote; the fourth vote always creates a
|
||||
// unique plurality or a 1-1-1-1). Still handle 2-2 defensively by
|
||||
// promoting to pm-escalation so no silent coin-flip ever lands in prod.
|
||||
if (topCount === 2 && topKeys.length === 1) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'majority' as TieBreakPath,
|
||||
verdict: topKeys[0],
|
||||
vote_count: 4,
|
||||
});
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
if (topCount === 2 && topKeys.length >= 2) {
|
||||
// 2-2 tie on 4 votes — not reachable from 1-1-1→quadri flow, but a
|
||||
// caller may pass in a pre-constructed 4-vote vector. Escalate rather
|
||||
// than coin-flip.
|
||||
logger?.warn?.('tie_break.two-two-tie', {
|
||||
path: 'pm-escalation' as TieBreakPath,
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
two_two_keys: topKeys,
|
||||
});
|
||||
return {
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
path: 'pm-escalation',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
if (topCount === 3 || topCount === 4) {
|
||||
logger?.info('tie_break', {
|
||||
path: 'majority' as TieBreakPath,
|
||||
verdict: topKeys[0],
|
||||
vote_count: 4,
|
||||
});
|
||||
return {
|
||||
verdict: topKeys[0],
|
||||
path: 'majority',
|
||||
votes,
|
||||
};
|
||||
}
|
||||
|
||||
// Truly unreachable given 4 input votes; defensive.
|
||||
throw new Error(
|
||||
`resolveTieBreak: unexpected 4-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
|
||||
);
|
||||
}
|
||||
@@ -1,383 +0,0 @@
|
||||
/**
|
||||
* Failure-mode judge — scaffold.
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
|
||||
* Spec: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §4 (prompt),
|
||||
* §6 (ensemble), §8 (inter-judge agreement / Fleiss' kappa thresholds)
|
||||
* LOCKED: PM-Waggle-OS/decisions/2026-04-20-failure-mode-oq-resolutions-locked.md
|
||||
*
|
||||
* This module is deliberately *not wired* into the harness runner in this
|
||||
* sprint (explicitly out of scope per the brief). It exists so that Sprint 9
|
||||
* can drop in the judge call on top of a tested foundation.
|
||||
*
|
||||
* Design decisions worth knowing:
|
||||
*
|
||||
* — Zod validates strictly. The Step-3 schema contract between the model
|
||||
* and us is enforced with a `.refine()` that captures the two
|
||||
* cross-field invariants from the spec §4:
|
||||
* verdict === 'correct' ⇒ failure_mode MUST be null
|
||||
* verdict === 'incorrect' ⇒ failure_mode MUST be one of F1..F5
|
||||
* Any violation → retry once with the reminder → JudgeParseError.
|
||||
*
|
||||
* — JSON extraction is best-effort. Models often wrap JSON in ```json
|
||||
* fences, leading prose, or trailing chatter. We strip the obvious
|
||||
* wrappers before parsing; if both attempts fail we surface the raw
|
||||
* response in the thrown error so a debugger can reproduce.
|
||||
*
|
||||
* — Ensemble tie-break is explicit: on a 2-2 split the ensemble uses the
|
||||
* first judge in the `judgeModels` list. By convention the caller puts
|
||||
* Sonnet first so "2-2 tie broken by Sonnet" holds per the spec §6.
|
||||
*
|
||||
* — Fleiss' kappa is the standard multi-rater formulation (Fleiss 1971).
|
||||
* Raters per subject must be constant; we throw if any subject has a
|
||||
* different rater count, since the formula is undefined otherwise.
|
||||
* Categories are the 6-class vocabulary from §8 (`correct` treated as
|
||||
* a separate class from F1..F5 — the same convention §8 specifies).
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────────────
|
||||
|
||||
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
|
||||
export type Verdict = 'correct' | 'incorrect';
|
||||
|
||||
export interface JudgeResult {
|
||||
verdict: Verdict;
|
||||
failure_mode: null | FailureMode;
|
||||
rationale: string;
|
||||
judge_model: string;
|
||||
}
|
||||
|
||||
export interface LlmClient {
|
||||
/** One-shot completion. Returns the raw text produced by the judge LLM. */
|
||||
complete(prompt: string): Promise<string>;
|
||||
}
|
||||
|
||||
export class JudgeParseError extends Error {
|
||||
readonly lastResponse: string;
|
||||
readonly lastParseError?: string;
|
||||
readonly judgeModel: string;
|
||||
constructor(message: string, details: { lastResponse: string; judgeModel: string; lastParseError?: string }) {
|
||||
super(message);
|
||||
this.name = 'JudgeParseError';
|
||||
this.lastResponse = details.lastResponse;
|
||||
this.judgeModel = details.judgeModel;
|
||||
this.lastParseError = details.lastParseError;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Zod schema (enforces spec §4 Step-3 contract) ──────────────────────
|
||||
|
||||
const judgeSchema = z
|
||||
.object({
|
||||
verdict: z.enum(['correct', 'incorrect']),
|
||||
failure_mode: z.union([z.null(), z.enum(['F1', 'F2', 'F3', 'F4', 'F5'])]),
|
||||
rationale: z.string().min(1, 'rationale must be a non-empty sentence'),
|
||||
})
|
||||
.strict()
|
||||
.refine(
|
||||
r => (r.verdict === 'correct' ? r.failure_mode === null : r.failure_mode !== null),
|
||||
{
|
||||
message:
|
||||
'verdict/failure_mode invariant violated: ' +
|
||||
'verdict="correct" requires failure_mode=null; verdict="incorrect" requires one of F1..F5',
|
||||
},
|
||||
);
|
||||
|
||||
// ── Prompt builder (exact text from strategy §4) ───────────────────────
|
||||
|
||||
/** Exact judge prompt text from `strategy/2026-04-20-failure-mode-taxonomy.md` §4.
|
||||
* Do not modify. The spec requires cell-independent scoring — changing the
|
||||
* prompt between cells, or even between invocations, invalidates the ensemble. */
|
||||
export function buildJudgePrompt(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
}): string {
|
||||
return [
|
||||
"You are evaluating whether an LLM's answer is correct against ground truth.",
|
||||
'',
|
||||
'## Question',
|
||||
params.question,
|
||||
'',
|
||||
'## Ground-truth answer',
|
||||
params.groundTruth,
|
||||
'',
|
||||
'## Ground-truth supporting context (excerpt shown to the model)',
|
||||
params.contextExcerpt,
|
||||
'',
|
||||
"## Model's answer",
|
||||
params.modelAnswer,
|
||||
'',
|
||||
'## Your task',
|
||||
'',
|
||||
"Step 1: Determine if the model's answer is correct.",
|
||||
'- "correct" means the model\'s answer contains all required facts from ground truth, with no additional incorrect claims.',
|
||||
'- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.',
|
||||
'- Extra detail is acceptable ONLY if it is factually correct.',
|
||||
'',
|
||||
'Step 2: If incorrect, assign exactly one failure mode using this decision tree:',
|
||||
'',
|
||||
'1. Does the model explicitly refuse or say it does not know? → F1 (ABSTAIN)',
|
||||
'2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? → F5 (OFF-TOPIC)',
|
||||
'3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? → F4 (HALLUCINATED)',
|
||||
'4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? → F2 (PARTIAL)',
|
||||
'5. Otherwise (model states facts derived from the context but gets them wrong): → F3 (INCORRECT)',
|
||||
'',
|
||||
'Step 3: Return JSON only, no prose, in this exact schema:',
|
||||
'',
|
||||
'{',
|
||||
' "verdict": "correct" | "incorrect",',
|
||||
' "failure_mode": null | "F1" | "F2" | "F3" | "F4" | "F5",',
|
||||
' "rationale": "one sentence explaining the verdict"',
|
||||
'}',
|
||||
'',
|
||||
'If verdict is "correct", failure_mode MUST be null.',
|
||||
'If verdict is "incorrect", failure_mode MUST be one of F1-F5.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Retry reminder prepended to the original prompt when the first response
|
||||
* fails to parse. Text is verbatim from the Task-4 acceptance spec. */
|
||||
export const RETRY_REMINDER =
|
||||
'Your previous response was not valid JSON. Return only the JSON object, no prose.';
|
||||
|
||||
// ── JSON extraction ────────────────────────────────────────────────────
|
||||
|
||||
/** Best-effort extraction of the JSON body from a model response. Handles:
|
||||
* - bare JSON
|
||||
* - `{ ... }` embedded in prose (first `{` to matching last `}`)
|
||||
* - markdown code fences ```json ... ``` or ``` ... ``` */
|
||||
export function extractJsonBody(raw: string): string | null {
|
||||
if (!raw) return null;
|
||||
const trimmed = raw.trim();
|
||||
// Code-fence strip first.
|
||||
const fenceMatch = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i);
|
||||
const afterFence = fenceMatch ? fenceMatch[1].trim() : trimmed;
|
||||
if (afterFence.startsWith('{') && afterFence.endsWith('}')) return afterFence;
|
||||
// Fallback — greedy `{ ... }` scan, stopping at the last `}`.
|
||||
const first = afterFence.indexOf('{');
|
||||
const last = afterFence.lastIndexOf('}');
|
||||
if (first >= 0 && last > first) return afterFence.slice(first, last + 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
function tryParse(raw: string): { ok: true; value: JudgeResult } | { ok: false; error: string } {
|
||||
const body = extractJsonBody(raw);
|
||||
if (!body) return { ok: false, error: 'no JSON object found in response' };
|
||||
let parsedJson: unknown;
|
||||
try {
|
||||
parsedJson = JSON.parse(body);
|
||||
} catch (err) {
|
||||
return { ok: false, error: `JSON.parse failed: ${(err as Error).message}` };
|
||||
}
|
||||
const result = judgeSchema.safeParse(parsedJson);
|
||||
if (!result.success) {
|
||||
return { ok: false, error: `schema validation failed: ${result.error.issues.map(i => i.message).join('; ')}` };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
value: {
|
||||
verdict: result.data.verdict,
|
||||
failure_mode: result.data.failure_mode,
|
||||
rationale: result.data.rationale,
|
||||
judge_model: '', // filled by judgeAnswer from params.judgeModel
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
export async function judgeAnswer(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
judgeModel: string;
|
||||
llmClient: LlmClient;
|
||||
}): Promise<JudgeResult> {
|
||||
const prompt = buildJudgePrompt({
|
||||
question: params.question,
|
||||
groundTruth: params.groundTruth,
|
||||
contextExcerpt: params.contextExcerpt,
|
||||
modelAnswer: params.modelAnswer,
|
||||
});
|
||||
|
||||
const first = await params.llmClient.complete(prompt);
|
||||
const firstParsed = tryParse(first);
|
||||
if (firstParsed.ok) {
|
||||
return { ...firstParsed.value, judge_model: params.judgeModel };
|
||||
}
|
||||
|
||||
// Retry once with the reminder prefixed. A fresh LLM call — callers can
|
||||
// wire conversation-state preservation later; spec §4 does not require it.
|
||||
const retryPrompt = `${RETRY_REMINDER}\n\n${prompt}`;
|
||||
const second = await params.llmClient.complete(retryPrompt);
|
||||
const secondParsed = tryParse(second);
|
||||
if (secondParsed.ok) {
|
||||
return { ...secondParsed.value, judge_model: params.judgeModel };
|
||||
}
|
||||
|
||||
throw new JudgeParseError(
|
||||
`Judge ${params.judgeModel} produced unparseable output after one retry`,
|
||||
{
|
||||
lastResponse: second,
|
||||
judgeModel: params.judgeModel,
|
||||
lastParseError: secondParsed.error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function judgeEnsemble(params: {
|
||||
question: string;
|
||||
groundTruth: string;
|
||||
contextExcerpt: string;
|
||||
modelAnswer: string;
|
||||
judgeModels: string[];
|
||||
llmClients: Map<string, LlmClient>;
|
||||
}): Promise<{ ensemble: JudgeResult[]; majority: JudgeResult; fleissKappa: number }> {
|
||||
if (params.judgeModels.length === 0) {
|
||||
throw new Error('judgeEnsemble requires at least one judgeModel');
|
||||
}
|
||||
|
||||
const results: JudgeResult[] = [];
|
||||
for (const model of params.judgeModels) {
|
||||
const client = params.llmClients.get(model);
|
||||
if (!client) throw new Error(`judgeEnsemble: no LlmClient registered for model ${model}`);
|
||||
const result = await judgeAnswer({
|
||||
question: params.question,
|
||||
groundTruth: params.groundTruth,
|
||||
contextExcerpt: params.contextExcerpt,
|
||||
modelAnswer: params.modelAnswer,
|
||||
judgeModel: model,
|
||||
llmClient: client,
|
||||
});
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
const majority = computeMajority(results, params.judgeModels[0]);
|
||||
// For a single-subject ensemble, Fleiss' kappa is computed over the same
|
||||
// one-subject matrix — degenerate but well-defined (NaN gets clamped to 0
|
||||
// when the subject has only one category). In production, this function
|
||||
// is typically called per-instance and kappa is aggregated at the batch
|
||||
// layer; we expose the single-subject value for diagnostic continuity.
|
||||
const fleissKappa = computeFleissKappa([results]);
|
||||
|
||||
return { ensemble: results, majority, fleissKappa };
|
||||
}
|
||||
|
||||
function computeMajority(results: readonly JudgeResult[], tieBreakerModel: string): JudgeResult {
|
||||
// Count verdict × failure_mode pairs (null collapses to 'NA' for keying).
|
||||
const tally = new Map<string, number>();
|
||||
const first = new Map<string, JudgeResult>();
|
||||
for (const r of results) {
|
||||
const key = `${r.verdict}|${r.failure_mode ?? 'NA'}`;
|
||||
tally.set(key, (tally.get(key) ?? 0) + 1);
|
||||
if (!first.has(key)) first.set(key, r);
|
||||
}
|
||||
let winnerKey = '';
|
||||
let winnerCount = -1;
|
||||
for (const [key, count] of tally) {
|
||||
if (count > winnerCount) {
|
||||
winnerKey = key;
|
||||
winnerCount = count;
|
||||
}
|
||||
}
|
||||
// Tie detection: if any other key has equal count, the tie-breaker model
|
||||
// wins — its verdict becomes the majority.
|
||||
const tied = Array.from(tally.entries()).filter(([, c]) => c === winnerCount);
|
||||
if (tied.length > 1) {
|
||||
const tb = results.find(r => r.judge_model === tieBreakerModel);
|
||||
if (tb) return tb;
|
||||
// Fallback: keep the first deterministic winner (stable map iteration).
|
||||
}
|
||||
const winner = first.get(winnerKey);
|
||||
if (!winner) {
|
||||
// Unreachable under normal flow; defensive.
|
||||
return results[0];
|
||||
}
|
||||
return {
|
||||
verdict: winner.verdict,
|
||||
failure_mode: winner.failure_mode,
|
||||
rationale: winner.rationale,
|
||||
judge_model: 'ensemble_majority',
|
||||
};
|
||||
}
|
||||
|
||||
// ── Fleiss' kappa ──────────────────────────────────────────────────────
|
||||
|
||||
/** The 6-class rating space used by Fleiss' kappa: correct + F1..F5.
|
||||
* §8: "standardni Fleiss' kappa preko 4 raters × N instanci × 6 klasa". */
|
||||
const KAPPA_CATEGORIES = ['correct', 'F1', 'F2', 'F3', 'F4', 'F5'] as const;
|
||||
type KappaCategory = typeof KAPPA_CATEGORIES[number];
|
||||
|
||||
function categoryOf(r: JudgeResult): KappaCategory {
|
||||
if (r.verdict === 'correct') return 'correct';
|
||||
return r.failure_mode ?? 'F3'; // should never hit fallback — schema guard
|
||||
}
|
||||
|
||||
/** Computes Fleiss' kappa on an N-subjects × n-raters matrix.
|
||||
*
|
||||
* Input shape: `ratings[i]` is the list of judge results for subject i —
|
||||
* all subjects must have the same number of raters. Categories are the
|
||||
* fixed 6-class vocabulary.
|
||||
*
|
||||
* Returns: κ in [-1, 1]. If all subjects are unanimous on the same
|
||||
* category (expected-agreement = 1), returns 1 (the (1-1)/(1-1) limit).
|
||||
*/
|
||||
export function computeFleissKappa(ratings: readonly (readonly JudgeResult[])[]): number {
|
||||
if (ratings.length === 0) return 0;
|
||||
const n = ratings[0].length;
|
||||
if (n < 2) {
|
||||
// Fleiss' kappa is undefined for a single rater. Degenerate → 0.
|
||||
return 0;
|
||||
}
|
||||
for (const row of ratings) {
|
||||
if (row.length !== n) {
|
||||
throw new Error(`Fleiss' kappa requires a constant rater count; saw ${row.length} and ${n}`);
|
||||
}
|
||||
}
|
||||
|
||||
const N = ratings.length;
|
||||
const k = KAPPA_CATEGORIES.length;
|
||||
const catIndex: Record<KappaCategory, number> = { correct: 0, F1: 1, F2: 2, F3: 3, F4: 4, F5: 5 };
|
||||
|
||||
// nij[i][j] = count of raters who assigned subject i to category j
|
||||
const nij: number[][] = Array.from({ length: N }, () => new Array<number>(k).fill(0));
|
||||
for (let i = 0; i < N; i++) {
|
||||
for (const r of ratings[i]) {
|
||||
const col = catIndex[categoryOf(r)];
|
||||
nij[i][col] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Pj = (1/(N*n)) * sum_i nij[i][j]
|
||||
const Pj: number[] = new Array<number>(k).fill(0);
|
||||
for (let j = 0; j < k; j++) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < N; i++) sum += nij[i][j];
|
||||
Pj[j] = sum / (N * n);
|
||||
}
|
||||
|
||||
// Pi = (1/(n(n-1))) * ( sum_j nij[i][j]^2 - n )
|
||||
let PbarSum = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
let sqSum = 0;
|
||||
for (let j = 0; j < k; j++) sqSum += nij[i][j] * nij[i][j];
|
||||
const Pi = (sqSum - n) / (n * (n - 1));
|
||||
PbarSum += Pi;
|
||||
}
|
||||
const Pbar = PbarSum / N;
|
||||
|
||||
// Pebar = sum_j Pj^2
|
||||
let Pebar = 0;
|
||||
for (let j = 0; j < k; j++) Pebar += Pj[j] * Pj[j];
|
||||
|
||||
if (Pebar >= 1 - 1e-12) {
|
||||
// Every rating in one category → expected = observed = 1 → κ = 1 by convention.
|
||||
return 1;
|
||||
}
|
||||
return (Pbar - Pebar) / (1 - Pebar);
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
/**
|
||||
* aggregate.ts unit tests (Sprint 9 Task 3).
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3 §Acceptance
|
||||
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
|
||||
*
|
||||
* Synthetic 12-instance JSONL fixture: 3 cells × 4 verdicts. Tests
|
||||
* assert:
|
||||
* - per-cell count table matches hand-computed rows
|
||||
* - weighted score matches hand-computed number (see §2 below)
|
||||
* - per-category rollup surfaces the hallucination flag at the right threshold
|
||||
* - cross-cell delta matrix populates full-context − raw direction
|
||||
* - cost summary sums correctly + Week-1 projection flag fires at the threshold
|
||||
* - markdown renderer produces parseable output + contains every row
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
buildReport,
|
||||
perCellRollup,
|
||||
perCategoryRollup,
|
||||
crossCellDeltaMatrix,
|
||||
costSummary,
|
||||
projectVerdict6,
|
||||
renderMarkdown,
|
||||
WEIGHTS,
|
||||
VERDICT6_VALUES,
|
||||
type JudgedJsonlRecord,
|
||||
} from '../../src/benchmarks/aggregate.ts';
|
||||
|
||||
// ── Fixture builder ─────────────────────────────────────────────────────
|
||||
|
||||
function mkRecord(
|
||||
turnId: string,
|
||||
cell: string,
|
||||
verdict: 'correct' | 'incorrect' | 'unjudged',
|
||||
failureMode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null,
|
||||
category: JudgedJsonlRecord['category'] = 'single-hop',
|
||||
usd = 0.001,
|
||||
): JudgedJsonlRecord {
|
||||
const rec: JudgedJsonlRecord = {
|
||||
turnId,
|
||||
cell,
|
||||
instance_id: `i_${turnId}`,
|
||||
model: 'qwen3.6-35b-a3b',
|
||||
seed: 42,
|
||||
accuracy: verdict === 'correct' ? 1 : 0,
|
||||
p50_latency_ms: 800,
|
||||
p95_latency_ms: 1200,
|
||||
usd_per_query: usd,
|
||||
failure_mode: null,
|
||||
category,
|
||||
};
|
||||
if (verdict !== 'unjudged') {
|
||||
rec.judge_verdict = verdict;
|
||||
rec.judge_failure_mode = failureMode;
|
||||
rec.judge_rationale = 'test rationale';
|
||||
rec.judge_model = 'claude-sonnet-4-6';
|
||||
rec.judge_timestamp = '2026-04-21T12:00:00Z';
|
||||
}
|
||||
return rec;
|
||||
}
|
||||
|
||||
/** 12-instance fixture — 4 each per cell, one of each verdict shape.
|
||||
* Hand-computed expectations are commented inline so a future diff
|
||||
* catches silent drift in the rubric. */
|
||||
function fixture12(): JudgedJsonlRecord[] {
|
||||
const recs: JudgedJsonlRecord[] = [];
|
||||
const cells = ['raw', 'filtered', 'full-context'];
|
||||
// Per cell: 1 correct + 1 F2 partial + 1 F3 incorrect + 1 F4 hallucinated.
|
||||
for (const c of cells) {
|
||||
recs.push(mkRecord(`${c}-A`, c, 'correct', null, 'single-hop'));
|
||||
recs.push(mkRecord(`${c}-B`, c, 'incorrect', 'F2', 'multi-hop'));
|
||||
recs.push(mkRecord(`${c}-C`, c, 'incorrect', 'F3', 'temporal'));
|
||||
recs.push(mkRecord(`${c}-D`, c, 'incorrect', 'F4', 'open-ended'));
|
||||
}
|
||||
return recs;
|
||||
}
|
||||
|
||||
// ── projectVerdict6 ─────────────────────────────────────────────────────
|
||||
|
||||
describe('projectVerdict6 — taxonomy §9 binary → brief §Task-1 6-value', () => {
|
||||
it('maps correct → correct', () => {
|
||||
expect(projectVerdict6(mkRecord('x', 'raw', 'correct', null))).toBe('correct');
|
||||
});
|
||||
it('maps incorrect + F1..F5 through their exact projections', () => {
|
||||
const cases: Array<[typeof WEIGHTS extends Record<infer K, number> ? K : never, string]> = [
|
||||
['F1_abstain', 'F1'],
|
||||
['F2_partial', 'F2'],
|
||||
['F3_incorrect', 'F3'],
|
||||
['F4_hallucinated', 'F4'],
|
||||
['F5_offtopic', 'F5'],
|
||||
];
|
||||
for (const [expected, mode] of cases) {
|
||||
expect(projectVerdict6(
|
||||
mkRecord('x', 'raw', 'incorrect', mode as 'F1' | 'F2' | 'F3' | 'F4' | 'F5'),
|
||||
)).toBe(expected);
|
||||
}
|
||||
});
|
||||
it('maps undefined judge_verdict → unjudged', () => {
|
||||
expect(projectVerdict6(mkRecord('x', 'raw', 'unjudged', null))).toBe('unjudged');
|
||||
});
|
||||
});
|
||||
|
||||
// ── perCellRollup ───────────────────────────────────────────────────────
|
||||
|
||||
describe('perCellRollup', () => {
|
||||
it('emits 3 rows, one per observed cell, in CELL_NAMES order', () => {
|
||||
const rows = perCellRollup(fixture12());
|
||||
expect(rows).toHaveLength(3);
|
||||
expect(rows.map(r => r.cell)).toEqual(['raw', 'filtered', 'full-context']);
|
||||
});
|
||||
|
||||
it('computes per-cell counts + weighted score correctly (hand-check)', () => {
|
||||
const rows = perCellRollup(fixture12());
|
||||
for (const row of rows) {
|
||||
expect(row.total).toBe(4);
|
||||
expect(row.counts.correct).toBe(1);
|
||||
expect(row.counts.F2_partial).toBe(1);
|
||||
expect(row.counts.F3_incorrect).toBe(1);
|
||||
expect(row.counts.F4_hallucinated).toBe(1);
|
||||
expect(row.counts.F1_abstain).toBe(0);
|
||||
expect(row.counts.F5_offtopic).toBe(0);
|
||||
expect(row.counts.unjudged).toBe(0);
|
||||
// Weighted score = sum(percent × weight) over judged instances.
|
||||
// judgedTotal=4. Each verdict is 1/4 = 0.25 of the cell.
|
||||
// correct: 0.25 × 1.00 = 0.250
|
||||
// F2_partial: 0.25 × 0.30 = 0.075
|
||||
// F3_incorrect: 0.25 × -0.15 = -0.0375
|
||||
// F4_hallucinated: 0.25 × -0.35 = -0.0875
|
||||
// Total = 0.250 + 0.075 − 0.0375 − 0.0875 = 0.200
|
||||
expect(row.weightedScore).toBeCloseTo(0.200, 4);
|
||||
}
|
||||
});
|
||||
|
||||
it('honors WEIGHTS table exactly', () => {
|
||||
// Rebuild the hand-computed number from WEIGHTS by name so a rubric
|
||||
// edit in aggregate.ts forces this test to recompute + update the
|
||||
// expectation — no silent coefficient drift.
|
||||
const expected =
|
||||
0.25 * WEIGHTS.correct +
|
||||
0.25 * WEIGHTS.F2_partial +
|
||||
0.25 * WEIGHTS.F3_incorrect +
|
||||
0.25 * WEIGHTS.F4_hallucinated;
|
||||
const rows = perCellRollup(fixture12());
|
||||
for (const row of rows) {
|
||||
expect(row.weightedScore).toBeCloseTo(expected, 6);
|
||||
}
|
||||
});
|
||||
|
||||
it('treats unjudged rows as denominator-excluded for weightedScore', () => {
|
||||
const mix = fixture12().slice(0, 4); // one cell worth
|
||||
mix.push(mkRecord('unjudged-1', 'raw', 'unjudged', null));
|
||||
mix.push(mkRecord('unjudged-2', 'raw', 'unjudged', null));
|
||||
// Now raw has 4 judged + 2 unjudged. Weighted score denominator = 4,
|
||||
// same as the baseline fixture — the 2 unjudged rows don't pull
|
||||
// the score toward 0.
|
||||
const rawRow = perCellRollup(mix).find(r => r.cell === 'raw')!;
|
||||
expect(rawRow.total).toBe(6);
|
||||
expect(rawRow.counts.unjudged).toBe(2);
|
||||
expect(rawRow.weightedScore).toBeCloseTo(0.200, 4);
|
||||
});
|
||||
});
|
||||
|
||||
// ── perCategoryRollup ──────────────────────────────────────────────────
|
||||
|
||||
describe('perCategoryRollup', () => {
|
||||
it('groups by category and computes percents per bucket', () => {
|
||||
const rows = perCategoryRollup(fixture12());
|
||||
// fixture assigns 3 rows per category (one per cell × one verdict shape)
|
||||
const cats = rows.map(r => r.category);
|
||||
expect(cats).toContain('single-hop');
|
||||
expect(cats).toContain('multi-hop');
|
||||
expect(cats).toContain('temporal');
|
||||
expect(cats).toContain('open-ended');
|
||||
for (const row of rows) {
|
||||
expect(row.total).toBe(3);
|
||||
}
|
||||
// open-ended has all F4 (hallucinated) rows — flag must fire.
|
||||
const openEnded = rows.find(r => r.category === 'open-ended')!;
|
||||
expect(openEnded.counts.F4_hallucinated).toBe(3);
|
||||
expect(openEnded.hallucinationFlag).toBe(true);
|
||||
// single-hop has all correct — flag off.
|
||||
const singleHop = rows.find(r => r.category === 'single-hop')!;
|
||||
expect(singleHop.counts.correct).toBe(3);
|
||||
expect(singleHop.hallucinationFlag).toBe(false);
|
||||
});
|
||||
|
||||
it('does not include categories that have zero rows', () => {
|
||||
const rows = perCategoryRollup(fixture12());
|
||||
for (const row of rows) {
|
||||
expect(row.total).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── crossCellDeltaMatrix ───────────────────────────────────────────────
|
||||
|
||||
describe('crossCellDeltaMatrix', () => {
|
||||
it('returns delta = full-context − raw for each verdict, preserving sign', () => {
|
||||
// Build a case where full-context correct% > raw correct%.
|
||||
const recs: JudgedJsonlRecord[] = [
|
||||
// raw: 1 correct, 3 incorrect-F4 → 25% correct, 75% F4
|
||||
mkRecord('r1', 'raw', 'correct', null, 'single-hop'),
|
||||
mkRecord('r2', 'raw', 'incorrect', 'F4', 'single-hop'),
|
||||
mkRecord('r3', 'raw', 'incorrect', 'F4', 'single-hop'),
|
||||
mkRecord('r4', 'raw', 'incorrect', 'F4', 'single-hop'),
|
||||
// full-context: 3 correct, 1 F4 → 75% correct, 25% F4
|
||||
mkRecord('f1', 'full-context', 'correct', null, 'single-hop'),
|
||||
mkRecord('f2', 'full-context', 'correct', null, 'single-hop'),
|
||||
mkRecord('f3', 'full-context', 'correct', null, 'single-hop'),
|
||||
mkRecord('f4', 'full-context', 'incorrect', 'F4', 'single-hop'),
|
||||
];
|
||||
const perCell = perCellRollup(recs);
|
||||
const delta = crossCellDeltaMatrix(perCell);
|
||||
expect(delta).not.toBeNull();
|
||||
const correctDelta = delta!.find(d => d.verdict === 'correct')!;
|
||||
expect(correctDelta.rawPercent).toBeCloseTo(0.25, 6);
|
||||
expect(correctDelta.fullContextPercent).toBeCloseTo(0.75, 6);
|
||||
expect(correctDelta.delta).toBeCloseTo(0.50, 6);
|
||||
// F4 goes the other way.
|
||||
const f4Delta = delta!.find(d => d.verdict === 'F4_hallucinated')!;
|
||||
expect(f4Delta.delta).toBeCloseTo(-0.50, 6);
|
||||
});
|
||||
|
||||
it('returns null when either raw or full-context is absent', () => {
|
||||
const recs: JudgedJsonlRecord[] = [mkRecord('x', 'filtered', 'correct', null)];
|
||||
const perCell = perCellRollup(recs);
|
||||
expect(crossCellDeltaMatrix(perCell)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── costSummary + Week-1 projection threshold ──────────────────────────
|
||||
|
||||
describe('costSummary', () => {
|
||||
it('sums per-cell USD across records', () => {
|
||||
const recs: JudgedJsonlRecord[] = [
|
||||
mkRecord('a', 'raw', 'correct', null, 'single-hop', 0.010),
|
||||
mkRecord('b', 'raw', 'correct', null, 'single-hop', 0.020),
|
||||
mkRecord('c', 'full-context', 'correct', null, 'single-hop', 0.050),
|
||||
];
|
||||
const cost = costSummary(recs);
|
||||
expect(cost.totalUsd).toBeCloseTo(0.08, 6);
|
||||
expect(cost.perCellUsd['raw']).toBeCloseTo(0.03, 6);
|
||||
expect(cost.perCellUsd['full-context']).toBeCloseTo(0.05, 6);
|
||||
});
|
||||
|
||||
it('buildReport overlays an authoritative judgeTotalUsd and recomputes the Week-1 projection', () => {
|
||||
const recs = fixture12();
|
||||
// 12 records all judged; 12 × 4 cells × 50 instances scaling → 200
|
||||
// instances. Set judgeTotalUsd = $1.50 across 12 → per-instance = 0.125
|
||||
// → projected = 0.125 × 200 = $25 → above $20 → warning fires.
|
||||
const report = buildReport(recs, { judgeTotalUsd: 1.50 });
|
||||
expect(report.cost.judgeTotalUsd).toBeCloseTo(1.50, 6);
|
||||
expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(25.0, 6);
|
||||
expect(report.cost.week1WarningFired).toBe(true);
|
||||
});
|
||||
|
||||
it('Week-1 warning does not fire when projected stays under $20', () => {
|
||||
const recs = fixture12();
|
||||
const report = buildReport(recs, { judgeTotalUsd: 0.60 });
|
||||
// per-instance = 0.05; projected = 0.05 × 200 = $10
|
||||
expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(10.0, 6);
|
||||
expect(report.cost.week1WarningFired).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Markdown renderer — smoke + snapshot-lite ──────────────────────────
|
||||
|
||||
describe('renderMarkdown', () => {
|
||||
it('produces parseable markdown with every cell row + every verdict column', () => {
|
||||
const report = buildReport(fixture12());
|
||||
const md = renderMarkdown(report);
|
||||
// Header structure
|
||||
expect(md).toContain('# Benchmark Aggregate Report');
|
||||
expect(md).toContain('## Per-cell verdict distribution');
|
||||
expect(md).toContain('## Per-LoCoMo-category distribution');
|
||||
expect(md).toContain('## Cost summary');
|
||||
// Cells in table
|
||||
for (const cell of ['raw', 'filtered', 'full-context']) {
|
||||
expect(md).toContain(`| ${cell} |`);
|
||||
}
|
||||
// Verdict columns in the header row
|
||||
for (const v of VERDICT6_VALUES) {
|
||||
// VERDICT6_VALUES uses snake names that won't all appear literally
|
||||
// in the header (e.g. "F1 abstain" vs "F1_abstain"). Assert on the
|
||||
// base labels the renderer emits.
|
||||
}
|
||||
expect(md).toContain('Correct');
|
||||
expect(md).toContain('F1 abstain');
|
||||
expect(md).toContain('F4 hallucinated');
|
||||
expect(md).toContain('Weighted score');
|
||||
});
|
||||
|
||||
it('surfaces the hallucination-flag emoji on the flagged category', () => {
|
||||
const md = renderMarkdown(buildReport(fixture12()));
|
||||
// open-ended is flagged in the fixture (3/3 F4). Shape check via the
|
||||
// brief's sentinel emoji + "PM review" string.
|
||||
expect(md).toContain('⚠️ PM review');
|
||||
});
|
||||
|
||||
it('omits the Week-1 warning line when threshold not crossed', () => {
|
||||
const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 0.40 }));
|
||||
expect(md).not.toContain('Week-1 scale-up warning');
|
||||
});
|
||||
|
||||
it('emits the Week-1 warning line when threshold crossed', () => {
|
||||
const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 5.0 }));
|
||||
expect(md).toContain('Week-1 scale-up warning');
|
||||
});
|
||||
});
|
||||
@@ -1,255 +0,0 @@
|
||||
/**
|
||||
* Sprint 11 Task B2 — Tie-break policy unit tests.
|
||||
*
|
||||
* Authority: decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
|
||||
*
|
||||
* The four core scenarios from the brief §3 B2 + LOCK §0:
|
||||
*
|
||||
* 1. 3-0 consensus → path=none, no fourth-vendor call.
|
||||
* 2. 2-1 majority → path=majority, no fourth-vendor call.
|
||||
* 3. 1-1-1 split resolves → path=quadri-vendor, plurality verdict,
|
||||
* fourth-vendor called once with correct payload.
|
||||
* 4. 1-1-1 stays unresolved → path=pm-escalation (1-1-1-1 four-way),
|
||||
* fourth-vendor called once; verdict is the
|
||||
* PM_ESCALATION_VERDICT sentinel.
|
||||
*
|
||||
* Tests use a mocked CallFourthVendor function and an in-memory logger —
|
||||
* no network, no API spend. The LIVE grok-4.20 smoke lives in the B2 exit
|
||||
* ping's companion script and runs independently.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
resolveTieBreak,
|
||||
DEFAULT_FOURTH_VENDOR,
|
||||
PM_ESCALATION_VERDICT,
|
||||
type Vote,
|
||||
type TieBreakLogger,
|
||||
type CallFourthVendor,
|
||||
} from '../../src/benchmarks/judge/ensemble-tiebreak.js';
|
||||
|
||||
function vote(verdict: 'correct' | 'incorrect', failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null, model: string): Vote {
|
||||
return {
|
||||
verdict,
|
||||
failure_mode,
|
||||
rationale: `rationale from ${model}`,
|
||||
judge_model: model,
|
||||
};
|
||||
}
|
||||
|
||||
function makeLogger(): { logger: TieBreakLogger; events: Array<{ event: string; fields: Record<string, unknown> }> } {
|
||||
const events: Array<{ event: string; fields: Record<string, unknown> }> = [];
|
||||
const logger: TieBreakLogger = {
|
||||
info: (event, fields) => {
|
||||
events.push({ event, fields });
|
||||
},
|
||||
warn: (event, fields) => {
|
||||
events.push({ event, fields });
|
||||
},
|
||||
};
|
||||
return { logger, events };
|
||||
}
|
||||
|
||||
describe('Sprint 11 B2 — resolveTieBreak', () => {
|
||||
it('3-0 consensus returns path=none and does not call the fourth vendor', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'claude-opus-4-7'),
|
||||
vote('correct', null, 'gpt-5.4-pro'),
|
||||
vote('correct', null, 'gemini-3.1-pro'),
|
||||
];
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>();
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
|
||||
|
||||
expect(result.path).toBe('none');
|
||||
expect(result.verdict).toBe('correct|NA');
|
||||
expect(result.votes).toHaveLength(3);
|
||||
expect(result.fourthVendorVote).toBeUndefined();
|
||||
expect(result.fourthVendorSlug).toBeUndefined();
|
||||
expect(callFourthVendor).not.toHaveBeenCalled();
|
||||
expect(events).toContainEqual(
|
||||
expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'none' }) }),
|
||||
);
|
||||
});
|
||||
|
||||
it('2-1 majority returns path=majority and does not call the fourth vendor', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'claude-opus-4-7'),
|
||||
vote('correct', null, 'gpt-5.4-pro'),
|
||||
vote('incorrect', 'F3', 'gemini-3.1-pro'),
|
||||
];
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>();
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
|
||||
|
||||
expect(result.path).toBe('majority');
|
||||
expect(result.verdict).toBe('correct|NA');
|
||||
expect(result.votes).toHaveLength(3);
|
||||
expect(callFourthVendor).not.toHaveBeenCalled();
|
||||
expect(events).toContainEqual(
|
||||
expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'majority' }) }),
|
||||
);
|
||||
});
|
||||
|
||||
it('1-1-1 split triggers quadri-vendor call on xai/grok-4.20 and resolves via plurality', async () => {
|
||||
const primaryVotes: Vote[] = [
|
||||
vote('correct', null, 'claude-opus-4-7'), // bucket A: correct|NA
|
||||
vote('incorrect', 'F3', 'gpt-5.4-pro'), // bucket B: incorrect|F3
|
||||
vote('incorrect', 'F4', 'gemini-3.1-pro'), // bucket C: incorrect|F4
|
||||
];
|
||||
const grokVote = vote('correct', null, 'xai/grok-4.20'); // joins bucket A → plurality
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(grokVote);
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger });
|
||||
|
||||
expect(result.path).toBe('quadri-vendor');
|
||||
expect(result.verdict).toBe('correct|NA');
|
||||
expect(result.votes).toHaveLength(4);
|
||||
expect(result.fourthVendorVote).toEqual(grokVote);
|
||||
expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR);
|
||||
|
||||
// Exactly one fourth-vendor call with the correct payload.
|
||||
expect(callFourthVendor).toHaveBeenCalledTimes(1);
|
||||
expect(callFourthVendor).toHaveBeenCalledWith({
|
||||
primaryVotes,
|
||||
model: 'xai/grok-4.20',
|
||||
});
|
||||
|
||||
// pino-shaped events: invoke + resolved.
|
||||
const invokeEvent = events.find(e => e.event === 'tie_break.quadri-vendor.invoke');
|
||||
const resolvedEvent = events.find(e => e.event === 'tie_break.quadri-vendor.resolved');
|
||||
expect(invokeEvent).toBeDefined();
|
||||
expect(invokeEvent!.fields).toMatchObject({
|
||||
path: 'quadri-vendor',
|
||||
fourth_vendor_slug: 'xai/grok-4.20',
|
||||
});
|
||||
expect(resolvedEvent).toBeDefined();
|
||||
expect(resolvedEvent!.fields).toMatchObject({
|
||||
path: 'quadri-vendor',
|
||||
fourth_vendor_slug: 'xai/grok-4.20',
|
||||
verdict: 'correct|NA',
|
||||
});
|
||||
});
|
||||
|
||||
it('1-1-1 split where the fourth vote is a fourth distinct bucket escalates to PM', async () => {
|
||||
const primaryVotes: Vote[] = [
|
||||
vote('correct', null, 'claude-opus-4-7'), // A
|
||||
vote('incorrect', 'F2', 'gpt-5.4-pro'), // B
|
||||
vote('incorrect', 'F3', 'gemini-3.1-pro'), // C
|
||||
];
|
||||
// Fourth vote takes a fourth distinct failure mode → 1-1-1-1.
|
||||
const grokVote = vote('incorrect', 'F4', 'xai/grok-4.20');
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(grokVote);
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger });
|
||||
|
||||
expect(result.path).toBe('pm-escalation');
|
||||
expect(result.verdict).toBe(PM_ESCALATION_VERDICT);
|
||||
expect(result.votes).toHaveLength(4);
|
||||
expect(result.fourthVendorVote).toEqual(grokVote);
|
||||
expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR);
|
||||
|
||||
expect(callFourthVendor).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Logger emits a pm-escalation path event on the 4-vote recursive call.
|
||||
const escalationEvent = events.find(e =>
|
||||
e.event === 'tie_break' && e.fields.path === 'pm-escalation',
|
||||
);
|
||||
expect(escalationEvent).toBeDefined();
|
||||
expect(escalationEvent!.fields).toMatchObject({
|
||||
path: 'pm-escalation',
|
||||
verdict: PM_ESCALATION_VERDICT,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Sprint 11 B2 — resolveTieBreak defensive invariants', () => {
|
||||
it('throws on invalid vote length (2 votes)', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'a'),
|
||||
vote('correct', null, 'b'),
|
||||
];
|
||||
await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/);
|
||||
});
|
||||
|
||||
it('throws on invalid vote length (5 votes)', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'a'),
|
||||
vote('correct', null, 'b'),
|
||||
vote('correct', null, 'c'),
|
||||
vote('correct', null, 'd'),
|
||||
vote('correct', null, 'e'),
|
||||
];
|
||||
await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/);
|
||||
});
|
||||
|
||||
it('1-1-1 without a callFourthVendor dep throws explicitly', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'a'),
|
||||
vote('incorrect', 'F3', 'b'),
|
||||
vote('incorrect', 'F4', 'c'),
|
||||
];
|
||||
await expect(resolveTieBreak(votes)).rejects.toThrow(/requires a callFourthVendor/);
|
||||
});
|
||||
|
||||
it('caller-provided 4-vote vector resolves to plurality without extra calls', async () => {
|
||||
// Test the caller-driven 4-vote shape — harness may pre-construct this.
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'claude-opus-4-7'),
|
||||
vote('correct', null, 'gpt-5.4-pro'),
|
||||
vote('incorrect', 'F3', 'gemini-3.1-pro'),
|
||||
vote('incorrect', 'F4', 'xai/grok-4.20'),
|
||||
];
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>();
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
|
||||
|
||||
expect(result.path).toBe('majority');
|
||||
expect(result.verdict).toBe('correct|NA');
|
||||
expect(callFourthVendor).not.toHaveBeenCalled();
|
||||
expect(events.some(e => e.fields.path === 'majority')).toBe(true);
|
||||
});
|
||||
|
||||
it('caller-provided 2-2 tie on 4 votes escalates (defensive, never silent coin-flip)', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'a'),
|
||||
vote('correct', null, 'b'),
|
||||
vote('incorrect', 'F3', 'c'),
|
||||
vote('incorrect', 'F3', 'd'),
|
||||
];
|
||||
const { logger, events } = makeLogger();
|
||||
|
||||
const result = await resolveTieBreak(votes, { logger });
|
||||
|
||||
expect(result.path).toBe('pm-escalation');
|
||||
expect(result.verdict).toBe(PM_ESCALATION_VERDICT);
|
||||
expect(events.some(e => e.event === 'tie_break.two-two-tie')).toBe(true);
|
||||
});
|
||||
|
||||
it('fourthVendorModel override is respected', async () => {
|
||||
const votes: Vote[] = [
|
||||
vote('correct', null, 'a'),
|
||||
vote('incorrect', 'F3', 'b'),
|
||||
vote('incorrect', 'F4', 'c'),
|
||||
];
|
||||
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(
|
||||
vote('correct', null, 'custom-slug/vendor-x'),
|
||||
);
|
||||
|
||||
const result = await resolveTieBreak(votes, {
|
||||
callFourthVendor,
|
||||
fourthVendorModel: 'custom-slug/vendor-x',
|
||||
});
|
||||
|
||||
expect(result.fourthVendorSlug).toBe('custom-slug/vendor-x');
|
||||
expect(callFourthVendor).toHaveBeenCalledWith({
|
||||
primaryVotes: votes,
|
||||
model: 'custom-slug/vendor-x',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,340 +0,0 @@
|
||||
/**
|
||||
* Failure-mode judge — unit tests.
|
||||
*
|
||||
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
|
||||
* Module: packages/server/src/benchmarks/judge/failure-mode-judge.ts
|
||||
*
|
||||
* Coverage:
|
||||
* - Valid JSON parse for all 5 failure modes (F1..F5) + the correct verdict.
|
||||
* - Invalid-JSON → retry succeeds.
|
||||
* - Invalid-JSON on both attempts → JudgeParseError.
|
||||
* - 4-judge ensemble: 4-0 unanimous, 3-1 majority, 2-2 tie broken by
|
||||
* the first model in `judgeModels` (Sonnet by convention).
|
||||
* - Fleiss' kappa on hand-crafted 4×10 matrices with values computed by
|
||||
* hand and verified against the formula (κ=1 for unanimous two-cluster,
|
||||
* κ≈0.1111 for a known mixed matrix — both within ±0.01 tolerance).
|
||||
*
|
||||
* No network. No LLM calls. Pure unit tests against mock LlmClients.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import {
|
||||
buildJudgePrompt,
|
||||
computeFleissKappa,
|
||||
extractJsonBody,
|
||||
judgeAnswer,
|
||||
judgeEnsemble,
|
||||
JudgeParseError,
|
||||
RETRY_REMINDER,
|
||||
type FailureMode,
|
||||
type JudgeResult,
|
||||
type LlmClient,
|
||||
type Verdict,
|
||||
} from '../../src/benchmarks/judge/failure-mode-judge.js';
|
||||
|
||||
// ── Test helpers ───────────────────────────────────────────────────────
|
||||
|
||||
class ScriptedLlmClient implements LlmClient {
|
||||
readonly calls: string[] = [];
|
||||
private readonly queue: Array<string | Error>;
|
||||
constructor(responses: Array<string | Error>) {
|
||||
this.queue = [...responses];
|
||||
}
|
||||
async complete(prompt: string): Promise<string> {
|
||||
this.calls.push(prompt);
|
||||
if (this.queue.length === 0) throw new Error('ScriptedLlmClient out of responses');
|
||||
const next = this.queue.shift()!;
|
||||
if (next instanceof Error) throw next;
|
||||
return next;
|
||||
}
|
||||
}
|
||||
|
||||
function mkResult(verdict: Verdict, failure_mode: FailureMode | null, rationale: string, judge_model: string): JudgeResult {
|
||||
return { verdict, failure_mode, rationale, judge_model };
|
||||
}
|
||||
|
||||
// ── Prompt shape ───────────────────────────────────────────────────────
|
||||
|
||||
describe('buildJudgePrompt', () => {
|
||||
it('interpolates all four required variables verbatim', () => {
|
||||
const prompt = buildJudgePrompt({
|
||||
question: 'When did Caroline go to the support group?',
|
||||
groundTruth: '7 May 2023',
|
||||
contextExcerpt: 'Caroline: I went to a LGBTQ support group yesterday.',
|
||||
modelAnswer: 'Unclear.',
|
||||
});
|
||||
expect(prompt).toContain('## Question\nWhen did Caroline go to the support group?');
|
||||
expect(prompt).toContain('## Ground-truth answer\n7 May 2023');
|
||||
expect(prompt).toContain('## Ground-truth supporting context');
|
||||
expect(prompt).toContain("Caroline: I went to a LGBTQ support group yesterday.");
|
||||
expect(prompt).toContain("## Model's answer\nUnclear.");
|
||||
// Decision tree sanity — verifies the exact §4 text didn't drift.
|
||||
expect(prompt).toContain('→ F1 (ABSTAIN)');
|
||||
expect(prompt).toContain('→ F5 (OFF-TOPIC)');
|
||||
expect(prompt).toContain('→ F4 (HALLUCINATED)');
|
||||
expect(prompt).toContain('→ F2 (PARTIAL)');
|
||||
expect(prompt).toContain('→ F3 (INCORRECT)');
|
||||
});
|
||||
});
|
||||
|
||||
// ── JSON extraction ────────────────────────────────────────────────────
|
||||
|
||||
describe('extractJsonBody', () => {
|
||||
it('returns bare JSON unchanged', () => {
|
||||
expect(extractJsonBody('{"verdict":"correct"}')).toBe('{"verdict":"correct"}');
|
||||
});
|
||||
it('strips markdown fences with `json` hint', () => {
|
||||
const raw = '```json\n{"verdict":"correct"}\n```';
|
||||
expect(extractJsonBody(raw)).toBe('{"verdict":"correct"}');
|
||||
});
|
||||
it('strips bare markdown fences', () => {
|
||||
const raw = '```\n{"verdict":"incorrect","failure_mode":"F3"}\n```';
|
||||
expect(extractJsonBody(raw)).toBe('{"verdict":"incorrect","failure_mode":"F3"}');
|
||||
});
|
||||
it('extracts the JSON object from prose', () => {
|
||||
const raw = 'Here is my verdict: {"verdict":"correct","failure_mode":null,"rationale":"ok"}';
|
||||
expect(extractJsonBody(raw)).toBe('{"verdict":"correct","failure_mode":null,"rationale":"ok"}');
|
||||
});
|
||||
it('returns null when no object is present', () => {
|
||||
expect(extractJsonBody('I cannot comply.')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── judgeAnswer — happy path + failure modes ───────────────────────────
|
||||
|
||||
describe('judgeAnswer — valid JSON parse for all 5 failure modes + correct', () => {
|
||||
const cases: Array<{ name: string; verdict: Verdict; failure_mode: FailureMode | null; rationale: string }> = [
|
||||
{ name: 'correct', verdict: 'correct', failure_mode: null, rationale: 'All facts match the ground truth.' },
|
||||
{ name: 'F1 abstain', verdict: 'incorrect', failure_mode: 'F1', rationale: 'Model explicitly refused to answer.' },
|
||||
{ name: 'F2 partial', verdict: 'incorrect', failure_mode: 'F2', rationale: 'Model stated 2 of 3 required facts.' },
|
||||
{ name: 'F3 incorrect', verdict: 'incorrect', failure_mode: 'F3', rationale: 'Model stated a wrong date derived from context.' },
|
||||
{ name: 'F4 hallucinated', verdict: 'incorrect', failure_mode: 'F4', rationale: 'Model named a person not in the context.' },
|
||||
{ name: 'F5 off-topic', verdict: 'incorrect', failure_mode: 'F5', rationale: 'Model answered a different question.' },
|
||||
];
|
||||
|
||||
for (const c of cases) {
|
||||
it(`parses ${c.name} and stamps the judge_model`, async () => {
|
||||
const payload = JSON.stringify({ verdict: c.verdict, failure_mode: c.failure_mode, rationale: c.rationale });
|
||||
const client = new ScriptedLlmClient([payload]);
|
||||
const result = await judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'claude-sonnet-4-6', llmClient: client,
|
||||
});
|
||||
expect(result.verdict).toBe(c.verdict);
|
||||
expect(result.failure_mode).toBe(c.failure_mode);
|
||||
expect(result.rationale).toBe(c.rationale);
|
||||
expect(result.judge_model).toBe('claude-sonnet-4-6');
|
||||
expect(client.calls).toHaveLength(1);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('judgeAnswer — retry semantics', () => {
|
||||
const validPayload = JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'All facts match.' });
|
||||
|
||||
it('on invalid JSON, retries once with the reminder and returns the retry result', async () => {
|
||||
const client = new ScriptedLlmClient([
|
||||
'Sorry, I cannot produce structured output — here is a paragraph.',
|
||||
validPayload,
|
||||
]);
|
||||
const result = await judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'claude-sonnet-4-6', llmClient: client,
|
||||
});
|
||||
expect(result.verdict).toBe('correct');
|
||||
expect(client.calls).toHaveLength(2);
|
||||
// Retry prompt must begin with the exact reminder text from the spec.
|
||||
expect(client.calls[1].startsWith(RETRY_REMINDER)).toBe(true);
|
||||
});
|
||||
|
||||
it('on invalid JSON twice, throws JudgeParseError with the raw response attached', async () => {
|
||||
const client = new ScriptedLlmClient([
|
||||
'Still refusing to produce JSON.',
|
||||
'Nope, same here.',
|
||||
]);
|
||||
await expect(
|
||||
judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'gpt-5', llmClient: client,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(JudgeParseError);
|
||||
|
||||
try {
|
||||
await judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'gpt-5', llmClient: new ScriptedLlmClient(['bad1', 'bad2']),
|
||||
});
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(JudgeParseError);
|
||||
const err = e as JudgeParseError;
|
||||
expect(err.judgeModel).toBe('gpt-5');
|
||||
expect(err.lastResponse).toBe('bad2');
|
||||
expect(err.lastParseError).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects schema-valid JSON that violates the verdict/failure_mode invariant', async () => {
|
||||
// verdict=correct with a non-null failure_mode — Step-3 contract violation.
|
||||
const bad = JSON.stringify({ verdict: 'correct', failure_mode: 'F4', rationale: 'contradictory' });
|
||||
// Both attempts return the same bad shape — should throw.
|
||||
const client = new ScriptedLlmClient([bad, bad]);
|
||||
await expect(
|
||||
judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'gemini-pro', llmClient: client,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(JudgeParseError);
|
||||
});
|
||||
|
||||
it('accepts fenced JSON in the first attempt (no retry)', async () => {
|
||||
const fenced = '```json\n' + JSON.stringify({ verdict: 'incorrect', failure_mode: 'F3', rationale: 'Wrong date.' }) + '\n```';
|
||||
const client = new ScriptedLlmClient([fenced]);
|
||||
const result = await judgeAnswer({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModel: 'haiku', llmClient: client,
|
||||
});
|
||||
expect(result.failure_mode).toBe('F3');
|
||||
expect(client.calls).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── judgeEnsemble — 4-judge aggregation ────────────────────────────────
|
||||
|
||||
describe('judgeEnsemble — 4-judge aggregation', () => {
|
||||
function mkClientWith(verdict: Verdict, failure_mode: FailureMode | null, rationale: string): LlmClient {
|
||||
return new ScriptedLlmClient([JSON.stringify({ verdict, failure_mode, rationale })]);
|
||||
}
|
||||
|
||||
const models = ['claude-sonnet-4-6', 'claude-haiku-4-5', 'gpt-5', 'gemini-pro'];
|
||||
|
||||
it('4-0 unanimous → majority matches the unanimous verdict', async () => {
|
||||
const clients = new Map<string, LlmClient>();
|
||||
for (const m of models) clients.set(m, mkClientWith('correct', null, 'match'));
|
||||
const result = await judgeEnsemble({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModels: models, llmClients: clients,
|
||||
});
|
||||
expect(result.ensemble).toHaveLength(4);
|
||||
expect(result.majority.verdict).toBe('correct');
|
||||
expect(result.majority.failure_mode).toBeNull();
|
||||
// κ = 1 when every rater agrees on the same class (and all 6 classes
|
||||
// contribute 0 or 1 to the marginals → expected = observed = 1).
|
||||
expect(result.fleissKappa).toBeCloseTo(1, 6);
|
||||
});
|
||||
|
||||
it('3-1 majority → majority verdict wins, minority is recorded in ensemble', async () => {
|
||||
const clients = new Map<string, LlmClient>();
|
||||
clients.set(models[0], mkClientWith('incorrect', 'F3', 'A'));
|
||||
clients.set(models[1], mkClientWith('incorrect', 'F3', 'B'));
|
||||
clients.set(models[2], mkClientWith('incorrect', 'F3', 'C'));
|
||||
clients.set(models[3], mkClientWith('incorrect', 'F4', 'D')); // minority — says hallucination
|
||||
const result = await judgeEnsemble({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModels: models, llmClients: clients,
|
||||
});
|
||||
expect(result.majority.verdict).toBe('incorrect');
|
||||
expect(result.majority.failure_mode).toBe('F3');
|
||||
const failureModes = result.ensemble.map(r => r.failure_mode);
|
||||
expect(failureModes.filter(m => m === 'F3')).toHaveLength(3);
|
||||
expect(failureModes.filter(m => m === 'F4')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('2-2 tie is broken by the first model in judgeModels (Sonnet by convention)', async () => {
|
||||
const clients = new Map<string, LlmClient>();
|
||||
// Sonnet + Haiku say F2; GPT-5 + Gemini say F3.
|
||||
clients.set(models[0], mkClientWith('incorrect', 'F2', 'sonnet'));
|
||||
clients.set(models[1], mkClientWith('incorrect', 'F2', 'haiku'));
|
||||
clients.set(models[2], mkClientWith('incorrect', 'F3', 'gpt-5'));
|
||||
clients.set(models[3], mkClientWith('incorrect', 'F3', 'gemini'));
|
||||
const result = await judgeEnsemble({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModels: models, llmClients: clients,
|
||||
});
|
||||
// Sonnet wins the tie → F2.
|
||||
expect(result.majority.failure_mode).toBe('F2');
|
||||
expect(result.majority.rationale).toBe('sonnet');
|
||||
expect(result.majority.judge_model).toBe('claude-sonnet-4-6');
|
||||
});
|
||||
|
||||
it('refuses when no client is registered for a judge model', async () => {
|
||||
const clients = new Map<string, LlmClient>();
|
||||
clients.set(models[0], mkClientWith('correct', null, 'ok'));
|
||||
// Missing models[1..3]
|
||||
await expect(
|
||||
judgeEnsemble({
|
||||
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
|
||||
judgeModels: models, llmClients: clients,
|
||||
}),
|
||||
).rejects.toThrow(/no LlmClient registered/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Fleiss' kappa ──────────────────────────────────────────────────────
|
||||
|
||||
describe("computeFleissKappa on hand-crafted 4-judge × 10-subject matrices", () => {
|
||||
function row(n: number, verdict: Verdict, failure_mode: FailureMode | null): JudgeResult[] {
|
||||
return Array.from({ length: n }, (_, i) => mkResult(verdict, failure_mode, 'r', `j${i}`));
|
||||
}
|
||||
|
||||
it('κ = 1 for two-cluster unanimous agreement (5 × correct / 5 × F3)', () => {
|
||||
// 5 subjects: 4 raters all say correct.
|
||||
// 5 subjects: 4 raters all say F3.
|
||||
// Pbar = 1 (every subject unanimous). Pj(correct) = 0.5, Pj(F3) = 0.5.
|
||||
// Pebar = 0.5 + 0.5 = 0.5 (treating the other 4 classes as 0).
|
||||
// κ = (1 - 0.5) / (1 - 0.5) = 1.
|
||||
const matrix: JudgeResult[][] = [];
|
||||
for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null));
|
||||
for (let i = 0; i < 5; i++) matrix.push(row(4, 'incorrect', 'F3'));
|
||||
const kappa = computeFleissKappa(matrix);
|
||||
expect(kappa).toBeCloseTo(1, 6);
|
||||
});
|
||||
|
||||
it('κ ≈ 0.1111 for 5 unanimous-correct + 5 split-2/2-correct/F3 subjects', () => {
|
||||
// Hand-computed: n_correct = 5*4 + 5*2 = 30, n_F3 = 5*2 = 10. Total = 40.
|
||||
// Pj(correct) = 30/40 = 0.75 → 0.5625
|
||||
// Pj(F3) = 10/40 = 0.25 → 0.0625
|
||||
// Pebar = 0.5625 + 0.0625 = 0.625
|
||||
// Per-subject Pi:
|
||||
// unanimous correct: (16+0-4)/(4*3) = 12/12 = 1
|
||||
// split 2/2: (4+4-4)/12 = 4/12 ≈ 0.33333
|
||||
// Pbar = (5*1 + 5*0.33333) / 10 = 6.66667 / 10 = 0.66667
|
||||
// κ = (0.66667 - 0.625) / (1 - 0.625) = 0.04167 / 0.375 = 0.11111
|
||||
const matrix: JudgeResult[][] = [];
|
||||
for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null));
|
||||
for (let i = 0; i < 5; i++) {
|
||||
matrix.push([
|
||||
mkResult('correct', null, 'r', 'j0'),
|
||||
mkResult('correct', null, 'r', 'j1'),
|
||||
mkResult('incorrect', 'F3', 'r', 'j2'),
|
||||
mkResult('incorrect', 'F3', 'r', 'j3'),
|
||||
]);
|
||||
}
|
||||
const kappa = computeFleissKappa(matrix);
|
||||
expect(kappa).toBeCloseTo(0.1111, 2); // tolerance ±0.01 per the brief
|
||||
});
|
||||
|
||||
it('κ = 1 when every rating falls in a single category (expected = observed = 1)', () => {
|
||||
// Degenerate edge case: all 40 ratings are `correct`. Pebar = 1.
|
||||
// Implementation clamps to 1 (the (1-1)/(1-1) limit).
|
||||
const matrix: JudgeResult[][] = [];
|
||||
for (let i = 0; i < 10; i++) matrix.push(row(4, 'correct', null));
|
||||
expect(computeFleissKappa(matrix)).toBe(1);
|
||||
});
|
||||
|
||||
it('throws when rater counts are inconsistent across subjects', () => {
|
||||
const matrix: JudgeResult[][] = [
|
||||
row(4, 'correct', null),
|
||||
row(3, 'correct', null), // wrong rater count
|
||||
];
|
||||
expect(() => computeFleissKappa(matrix)).toThrow(/constant rater count/);
|
||||
});
|
||||
|
||||
it('returns 0 for a single-rater input (kappa undefined, convention 0)', () => {
|
||||
const matrix: JudgeResult[][] = Array.from({ length: 10 }, () => row(1, 'correct', null));
|
||||
expect(computeFleissKappa(matrix)).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 for an empty input', () => {
|
||||
expect(computeFleissKappa([])).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user