chore(server): remove the memory benchmark judge scaffolding (P2.2)

packages/server/src/benchmarks held a LoCoMo failure-mode aggregator and
an LLM judge ensemble for the old memory engine benchmarks. Nothing in
the sidecar imports it and no npm script runs it; its inputs lived in
the root benchmarks/ directory that P2.2 already removed. Judge and eval
loops are also out of the community edition (AGENTS.md, rule 4).
This commit is contained in:
shumeiko.it
2026-10-24 16:39:23 +02:00
parent 76c7102123
commit 93d73be7b4
6 changed files with 0 additions and 2035 deletions

View File

@@ -1,432 +0,0 @@
/**
* Aggregate.ts — failure-mode distribution rollup for Stage 1 / Stage 2
* preflight + Week 1/2 main runs.
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
*
* Consumes JSONL output from `benchmarks/harness` (one record per
* instance per cell) and emits a structured report covering:
* 1. Per-cell verdict distribution + weighted quality score
* 2. Per-LoCoMo-category distribution (single/multi-hop/temporal/
* open-ended) with a hallucination rate flag for PM review
* 3. Cross-cell delta matrix (full-context vs raw: correct / F4 /
* F1 lift per taxonomy §5)
* 4. Judge cost summary when the run included judge calls
*
* Input shape: each JSONL line has the fields declared in
* `benchmarks/harness/src/types.ts` (optional `judge_verdict`,
* `judge_failure_mode`, `model_answer`, etc.). Pre-judge records — rows
* without `judge_verdict` — are counted under an `unjudged` bucket so
* partial-run reports stay honest instead of silently skipping data.
*/
export type Verdict6 =
| 'correct'
| 'F1_abstain'
| 'F2_partial'
| 'F3_incorrect'
| 'F4_hallucinated'
| 'F5_offtopic'
| 'unjudged';
export type CellName = 'raw' | 'filtered' | 'compressed' | 'full-context';
const CELL_NAMES: readonly CellName[] = ['raw', 'filtered', 'compressed', 'full-context'];
/** Categories match `preflight-locomo-50.json` `_meta.locomo_category_map`
* minus the excluded adversarial bucket. Unknown categories fall into
* `other` so the report still tallies them instead of dropping. */
export type LocomoCategory = 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended' | 'other';
const LOCOMO_CATEGORIES: readonly LocomoCategory[] = [
'single-hop', 'multi-hop', 'temporal', 'open-ended', 'other',
];
/** Inputs can carry either the explicit category column or the
* pre-lock instance_id (`locomo_<sample_id>_q<qindex>`) from which
* we can recover the category via a sibling lookup. This module only
* consumes the already-tagged JsonlRecord; category recovery lives in
* the caller (harness runner attaches `category` when loading from
* the sample lock). */
export interface JudgedJsonlRecord {
turnId: string;
cell: string;
instance_id: string;
model: string;
seed: number;
accuracy: number;
p50_latency_ms: number;
p95_latency_ms: number;
usd_per_query: number;
failure_mode: string | null;
model_answer?: string;
judge_verdict?: 'correct' | 'incorrect';
judge_failure_mode?: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
judge_rationale?: string;
judge_model?: string;
judge_timestamp?: string;
judge_confidence?: number;
judge_ensemble?: Array<{
model: string;
verdict: 'correct' | 'incorrect';
failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
rationale?: string;
latency_ms?: number;
}>;
/** Optional category tag attached by the runner when the sample came
* from preflight-locomo-50.json. When absent, rows fall into
* `other` for the per-category rollup. */
category?: LocomoCategory;
}
/** Projection from the taxonomy §9 binary `verdict` + `failure_mode`
* slot pair to the 6-value Verdict6 enum the brief Task-1 spec asked
* for. Centralising this here means JsonlRecord keeps the canonical
* taxonomy shape while the aggregator surfaces the denser report
* form. */
export function projectVerdict6(r: JudgedJsonlRecord): Verdict6 {
if (r.judge_verdict === undefined) return 'unjudged';
if (r.judge_verdict === 'correct') return 'correct';
switch (r.judge_failure_mode) {
case 'F1': return 'F1_abstain';
case 'F2': return 'F2_partial';
case 'F3': return 'F3_incorrect';
case 'F4': return 'F4_hallucinated';
case 'F5': return 'F5_offtopic';
default:
// Contract violation — the taxonomy schema (judge module's Zod)
// prevents `incorrect` with a null failure_mode. If it sneaks
// through a malformed JSONL row, bucket as F3 by convention so
// the row doesn't silently drop.
return 'F3_incorrect';
}
}
export const VERDICT6_VALUES: readonly Verdict6[] = [
'correct', 'F1_abstain', 'F2_partial', 'F3_incorrect', 'F4_hallucinated', 'F5_offtopic', 'unjudged',
];
// ── Per-cell distribution + weighted score ─────────────────────────────
export interface PerCellRow {
cell: string;
total: number;
counts: Record<Verdict6, number>;
percents: Record<Verdict6, number>;
/** Weighted quality score per taxonomy §5 rubric. `unjudged` is
* excluded from the denominator so the score is computed over the
* set of instances actually graded. */
weightedScore: number;
}
/** Taxonomy §5 coefficient table. Positive weights for correct / F2
* (partial credit), zero for F1 (abstain neutral), negatives for
* F3 / F4 / F5. Exported so Task-3 unit tests can reproduce the
* hand-check number by name. */
export const WEIGHTS: Record<Exclude<Verdict6, 'unjudged'>, number> = {
correct: 1.0,
F2_partial: 0.30,
F1_abstain: 0.0,
F3_incorrect: -0.15,
F4_hallucinated: -0.35,
F5_offtopic: -0.10,
};
function emptyVerdictMap(fill = 0): Record<Verdict6, number> {
const out = {} as Record<Verdict6, number>;
for (const v of VERDICT6_VALUES) out[v] = fill;
return out;
}
export function perCellRollup(records: readonly JudgedJsonlRecord[]): PerCellRow[] {
const byCell = new Map<string, JudgedJsonlRecord[]>();
for (const r of records) {
const arr = byCell.get(r.cell) ?? [];
arr.push(r);
byCell.set(r.cell, arr);
}
const rows: PerCellRow[] = [];
// Emit in CELL_NAMES order when present, then any controls / extras.
const orderedCellKeys = [
...CELL_NAMES.filter(c => byCell.has(c)),
...[...byCell.keys()].filter(c => !CELL_NAMES.includes(c as CellName)),
];
for (const cell of orderedCellKeys) {
const rowRecords = byCell.get(cell)!;
const counts = emptyVerdictMap();
for (const r of rowRecords) counts[projectVerdict6(r)]++;
const total = rowRecords.length;
const judgedTotal = total - counts['unjudged'];
const percents = emptyVerdictMap();
for (const v of VERDICT6_VALUES) {
percents[v] = total === 0 ? 0 : counts[v] / total;
}
let weightedScore = 0;
if (judgedTotal > 0) {
for (const v of VERDICT6_VALUES) {
if (v === 'unjudged') continue;
weightedScore += (counts[v] / judgedTotal) * WEIGHTS[v];
}
}
rows.push({ cell, total, counts, percents, weightedScore });
}
return rows;
}
// ── Per-LoCoMo-category distribution ───────────────────────────────────
export interface PerCategoryRow {
category: LocomoCategory;
total: number;
counts: Record<Verdict6, number>;
percents: Record<Verdict6, number>;
/** True when F4 (hallucinated) share exceeds 20% — flags the
* category for PM spot-review per brief §Task-3 rule 2. */
hallucinationFlag: boolean;
}
const HALLUCINATION_FLAG_THRESHOLD = 0.20;
export function perCategoryRollup(records: readonly JudgedJsonlRecord[]): PerCategoryRow[] {
const byCat = new Map<LocomoCategory, JudgedJsonlRecord[]>();
for (const r of records) {
const cat = r.category ?? 'other';
const arr = byCat.get(cat) ?? [];
arr.push(r);
byCat.set(cat, arr);
}
const rows: PerCategoryRow[] = [];
for (const cat of LOCOMO_CATEGORIES) {
const recs = byCat.get(cat) ?? [];
if (recs.length === 0) continue;
const counts = emptyVerdictMap();
for (const r of recs) counts[projectVerdict6(r)]++;
const total = recs.length;
const percents = emptyVerdictMap();
for (const v of VERDICT6_VALUES) {
percents[v] = total === 0 ? 0 : counts[v] / total;
}
rows.push({
category: cat,
total,
counts,
percents,
hallucinationFlag: percents['F4_hallucinated'] > HALLUCINATION_FLAG_THRESHOLD,
});
}
return rows;
}
// ── Cross-cell delta matrix (full-context vs raw) ──────────────────────
export interface CrossCellDelta {
verdict: Verdict6;
rawPercent: number;
fullContextPercent: number;
/** fullContext − raw. Positive = full-context raised this verdict's share;
* negative = full-context lowered it. The brief's headline expectation:
* `correct` delta > 0, `F4_hallucinated` delta < 0, `F1_abstain`
* may go either way (more abstains is sometimes an OK signal). */
delta: number;
}
export function crossCellDeltaMatrix(perCell: readonly PerCellRow[]): CrossCellDelta[] | null {
const raw = perCell.find(r => r.cell === 'raw');
const full = perCell.find(r => r.cell === 'full-context');
if (!raw || !full) return null;
const out: CrossCellDelta[] = [];
for (const v of VERDICT6_VALUES) {
out.push({
verdict: v,
rawPercent: raw.percents[v],
fullContextPercent: full.percents[v],
delta: full.percents[v] - raw.percents[v],
});
}
return out;
}
// ── Cost summary ───────────────────────────────────────────────────────
export interface CostSummary {
totalUsd: number;
perCellUsd: Record<string, number>;
judgeTotalUsd: number;
judgeCallCount: number;
/** Median judge call latency across the run; `null` when no judge
* call recorded a latency. */
medianJudgeMs: number | null;
/** Brief §Task-3 rule 4: flag when total judge spend would exceed
* ~$20 projected for a full 4-cell × 50-instance run. Computed as
* `perInstanceJudgeUsd × 200`. */
week1WarningProjectedUsd: number;
week1WarningFired: boolean;
}
const WEEK1_PROJECTION_INSTANCE_COUNT = 200; // 4 cells × 50 instances
export function costSummary(records: readonly JudgedJsonlRecord[]): CostSummary {
const perCellUsd: Record<string, number> = {};
let total = 0;
const judgeLatencies: number[] = [];
const judgeTotal = 0;
let judgeCalls = 0;
for (const r of records) {
total += r.usd_per_query;
perCellUsd[r.cell] = (perCellUsd[r.cell] ?? 0) + r.usd_per_query;
if (r.judge_ensemble) {
for (const entry of r.judge_ensemble) {
if (typeof entry.latency_ms === 'number') judgeLatencies.push(entry.latency_ms);
}
judgeCalls += r.judge_ensemble.length;
} else if (r.judge_verdict !== undefined) {
// Single-judge call — we don't store per-judge latency in the
// JsonlRecord shape (no judge_latency_ms field), so the median
// is computed only over ensemble entries that carry it. If no
// ensemble row is present, medianJudgeMs stays null.
judgeCalls += 1;
}
}
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
// Per-instance judge spend is not stored directly — the aggregator
// doesn't know judge USD without the onCall sink. When the caller
// passes judgeTotalUsd separately (see buildReport), we populate
// this from the authoritative source. Here we return zero; buildReport
// overwrites with the real total.
const medianJudgeMs = judgeLatencies.length === 0
? null
: (() => {
const s = [...judgeLatencies].sort((a, b) => a - b);
const mid = Math.floor(s.length / 2);
return s.length % 2 === 0 ? (s[mid - 1] + s[mid]) / 2 : s[mid];
})();
const perInstanceJudgeUsd = judgedCount > 0 ? judgeTotal / judgedCount : 0;
const projected = perInstanceJudgeUsd * WEEK1_PROJECTION_INSTANCE_COUNT;
return {
totalUsd: total,
perCellUsd,
judgeTotalUsd: judgeTotal,
judgeCallCount: judgeCalls,
medianJudgeMs,
week1WarningProjectedUsd: projected,
week1WarningFired: projected > 20,
};
}
// ── Top-level report + markdown renderer ───────────────────────────────
export interface AggregateReport {
generatedAt: string;
totalRecords: number;
perCell: PerCellRow[];
perCategory: PerCategoryRow[];
crossCellDelta: CrossCellDelta[] | null;
cost: CostSummary;
}
export function buildReport(
records: readonly JudgedJsonlRecord[],
opts: { judgeTotalUsd?: number } = {},
): AggregateReport {
const perCell = perCellRollup(records);
const perCategory = perCategoryRollup(records);
const crossCellDelta = crossCellDeltaMatrix(perCell);
const cost = costSummary(records);
// Callers that run the harness end-to-end know the authoritative
// judge spend because judge-client emits `onCall` entries. Pass the
// aggregated total here so the report carries real dollars instead
// of the placeholder zero computed from JSONL alone.
if (opts.judgeTotalUsd !== undefined) {
cost.judgeTotalUsd = opts.judgeTotalUsd;
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
const perInstance = judgedCount > 0 ? opts.judgeTotalUsd / judgedCount : 0;
cost.week1WarningProjectedUsd = perInstance * WEEK1_PROJECTION_INSTANCE_COUNT;
cost.week1WarningFired = cost.week1WarningProjectedUsd > 20;
}
return {
generatedAt: new Date().toISOString(),
totalRecords: records.length,
perCell,
perCategory,
crossCellDelta,
cost,
};
}
function pct(n: number): string {
return `${(n * 100).toFixed(1)}%`;
}
function fmtUsd(n: number): string {
return `$${n.toFixed(n < 1 ? 6 : 2)}`;
}
export function renderMarkdown(report: AggregateReport): string {
const lines: string[] = [];
lines.push('# Benchmark Aggregate Report');
lines.push('');
lines.push(`**Generated at:** ${report.generatedAt}`);
lines.push(`**Total records:** ${report.totalRecords}`);
lines.push('');
lines.push('## Per-cell verdict distribution');
lines.push('');
lines.push('| Cell | Total | Correct | F1 abstain | F2 partial | F3 incorrect | F4 hallucinated | F5 off-topic | Unjudged | Weighted score |');
lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|');
for (const row of report.perCell) {
lines.push(
`| ${row.cell} | ${row.total} | ${row.counts.correct} (${pct(row.percents.correct)}) | ` +
`${row.counts.F1_abstain} (${pct(row.percents.F1_abstain)}) | ` +
`${row.counts.F2_partial} (${pct(row.percents.F2_partial)}) | ` +
`${row.counts.F3_incorrect} (${pct(row.percents.F3_incorrect)}) | ` +
`${row.counts.F4_hallucinated} (${pct(row.percents.F4_hallucinated)}) | ` +
`${row.counts.F5_offtopic} (${pct(row.percents.F5_offtopic)}) | ` +
`${row.counts.unjudged} (${pct(row.percents.unjudged)}) | ` +
`${row.weightedScore.toFixed(3)} |`,
);
}
lines.push('');
if (report.perCategory.length > 0) {
lines.push('## Per-LoCoMo-category distribution');
lines.push('');
lines.push('| Category | Total | Correct% | F4 (hallucinated)% | Flagged? |');
lines.push('|---|---:|---:|---:|:---:|');
for (const row of report.perCategory) {
lines.push(
`| ${row.category} | ${row.total} | ${pct(row.percents.correct)} | ` +
`${pct(row.percents.F4_hallucinated)} | ` +
`${row.hallucinationFlag ? '⚠️ PM review' : '✓'} |`,
);
}
lines.push('');
}
if (report.crossCellDelta) {
lines.push('## Full-context vs raw delta');
lines.push('');
lines.push('| Verdict | raw % | full-context % | Δ (full − raw) |');
lines.push('|---|---:|---:|---:|');
for (const r of report.crossCellDelta) {
const sign = r.delta > 0 ? '+' : '';
lines.push(`| ${r.verdict} | ${pct(r.rawPercent)} | ${pct(r.fullContextPercent)} | ${sign}${pct(r.delta)} |`);
}
lines.push('');
}
lines.push('## Cost summary');
lines.push('');
lines.push(`- Total cell spend: **${fmtUsd(report.cost.totalUsd)}**`);
lines.push(`- Judge spend: **${fmtUsd(report.cost.judgeTotalUsd)}** across ${report.cost.judgeCallCount} call(s)`);
if (report.cost.medianJudgeMs !== null) {
lines.push(`- Median judge latency: ${report.cost.medianJudgeMs.toFixed(0)} ms`);
}
if (Object.keys(report.cost.perCellUsd).length > 0) {
lines.push('- Per-cell cell spend:');
for (const [cell, usd] of Object.entries(report.cost.perCellUsd)) {
lines.push(` - ${cell}: ${fmtUsd(usd)}`);
}
}
if (report.cost.week1WarningFired) {
lines.push(
`- ⚠️ **Week-1 scale-up warning:** projected judge spend for a full 4-cell × 50-instance run is ` +
`${fmtUsd(report.cost.week1WarningProjectedUsd)} (threshold $20). Flagged per brief §Task-3 rule 4.`,
);
}
return lines.join('\n');
}

View File

@@ -1,314 +0,0 @@
/**
* Sprint 11 Task B2 — Tie-break policy for multi-vendor judge ensemble.
*
* Authority:
* - PM-Waggle-OS/briefs/2026-04-22-cc-sprint-11-kickoff.md §3 Track B B2
* - PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
*
* Context — what this resolves:
*
* The Sprint 10 Task 2.2 ratified primary judge ensemble is a THREE-vendor
* panel (Anthropic Opus 4.7 + OpenAI GPT-5.4 + Google Gemini 3.1-Pro). Three
* votes means three possible distributions:
*
* - 3-0 consensus → trivial, no tie-break
* - 2-1 majority → majority wins, no tie-break
* - 1-1-1 split → three different verdicts, no primary majority
*
* On 1-1-1, we escalate to a FOURTH vendor from a lineage disjoint from the
* primary trio: xai/grok-4.20 per Marko's 2026-04-22 LOCK. After the 4th
* vote arrives we have four votes total:
*
* - 2-1-1 / 1-1-2 → plurality winner (2 votes)
* - 1-1-1-1 → four-way split → PM escalation
*
* The fourth vendor is SPECIFICALLY xai/grok-4.20 — not Sonnet 4.6 (which
* would give Anthropic 2-of-4 weight → homogeneous-bias risk — rejected per
* LOCK doc §2) and not Opus 4.7 (already Judge 1).
*
* Observability: pino structured log emits `tie_break.path` ∈ { 'none',
* 'majority', 'quadri-vendor', 'pm-escalation' } plus
* `tie_break.fourth_vendor_slug` on quadri paths (field is future-proof
* even though Sprint 11 scope is grok-4.20 only).
*/
import type { JudgeResult } from './failure-mode-judge.js';
// ── Public types ───────────────────────────────────────────────────────────
/**
* A single judge vote. Reusing JudgeResult from failure-mode-judge.ts keeps
* the tie-break input shape aligned with what judgeEnsemble already produces.
*/
export type Vote = JudgeResult;
export type TieBreakPath = 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
/**
* The canonical sentinel verdict returned on a 1-1-1-1 four-way split.
* Callers (aggregators, report generators) MUST treat this as an operator
* action signal, not as a normal verdict string. The shape is intentionally
* unambiguous so no accidental inclusion into recall/accuracy math.
*/
export const PM_ESCALATION_VERDICT = '__PM_ESCALATION__';
/** Default fourth vendor for Sprint 11 — per LOCK doc §1. */
export const DEFAULT_FOURTH_VENDOR = 'xai/grok-4.20';
export interface TieBreakResult {
/**
* The resolved verdict string. Encoded as `<verdict>|<failure_mode_or_NA>`
* matching the aggregation key used in computeMajority of failure-mode-judge.ts,
* so downstream consumers can uniformly decompose.
* On pm-escalation path: `__PM_ESCALATION__`.
*/
verdict: string;
path: TieBreakPath;
/** The full vote list after resolution — 3 votes for none/majority, 4 for quadri/pm-escalation. */
votes: Vote[];
/** The fourth vendor's vote when a quadri path was taken. Undefined on none/majority. */
fourthVendorVote?: Vote;
/** The fourth-vendor slug that was used, for observability. Undefined on none/majority. */
fourthVendorSlug?: string;
}
export interface FourthVendorCallPayload {
/** The three primary votes that produced the 1-1-1 split. */
primaryVotes: Vote[];
/** Model slug to invoke — e.g. 'xai/grok-4.20'. */
model: string;
}
export type CallFourthVendor = (payload: FourthVendorCallPayload) => Promise<Vote>;
export interface TieBreakLogger {
info(event: string, fields: Record<string, unknown>): void;
warn?(event: string, fields: Record<string, unknown>): void;
}
export interface ResolveTieBreakOptions {
/**
* Invoked only on 1-1-1 splits. Required when primaryVotes.length === 3
* and the distribution is 1-1-1; optional otherwise. Tests mock this.
*/
callFourthVendor?: CallFourthVendor;
/** Override the fourth vendor slug. Defaults to `DEFAULT_FOURTH_VENDOR`. */
fourthVendorModel?: string;
/** Structured logger. Silently no-ops when omitted. */
logger?: TieBreakLogger;
}
// ── Internal helpers ───────────────────────────────────────────────────────
/**
* Canonical key for a vote — combines verdict + failure_mode so identical
* answers with different failure codes don't accidentally tie.
*/
function voteKey(v: Vote): string {
return `${v.verdict}|${v.failure_mode ?? 'NA'}`;
}
function countByKey(votes: Vote[]): Map<string, { count: number; first: Vote }> {
const tally = new Map<string, { count: number; first: Vote }>();
for (const v of votes) {
const key = voteKey(v);
const existing = tally.get(key);
if (existing) existing.count += 1;
else tally.set(key, { count: 1, first: v });
}
return tally;
}
function pluralityTop(tally: Map<string, { count: number; first: Vote }>): {
topCount: number;
topKeys: string[];
} {
let topCount = 0;
for (const { count } of tally.values()) if (count > topCount) topCount = count;
const topKeys: string[] = [];
for (const [key, { count }] of tally) if (count === topCount) topKeys.push(key);
return { topCount, topKeys };
}
// ── Public API ─────────────────────────────────────────────────────────────
/**
* Resolve a multi-vendor judge ensemble vote.
*
* Input shape:
* - 3 primary votes (Sprint 10 Task 2.2 trio) — the normal call path.
* - 4 votes also accepted, in which case the function treats the vector
* as already-quadri-resolved and returns the plurality winner (or
* pm-escalation on 1-1-1-1). This shape exists for tests and for
* caller-side recomposition.
*
* On 1-1-1 with 3 primary votes, `callFourthVendor` MUST be provided.
* The function invokes it with the three primary votes and the canonical
* model slug, then recurses with the four-vote list.
*
* Returns:
* - `{ path: 'none' }` for 3-0 consensus (3 primary votes).
* - `{ path: 'majority' }` for 2-1 majority (3 votes) OR for a plurality
* win on 4 votes when the caller supplied the vector directly.
* - `{ path: 'quadri-vendor' }` when we called the 4th vendor ourselves
* and got a 2-1-1 or 1-1-2 distribution.
* - `{ path: 'pm-escalation' }` on a 1-1-1-1 four-way split; verdict is
* the `PM_ESCALATION_VERDICT` sentinel.
*/
export async function resolveTieBreak(
votes: Vote[],
options: ResolveTieBreakOptions = {},
): Promise<TieBreakResult> {
if (votes.length !== 3 && votes.length !== 4) {
throw new Error(
`resolveTieBreak: votes.length must be 3 (primary ensemble) or 4 (post-quadri). Got ${votes.length}.`,
);
}
const logger = options.logger;
if (votes.length === 3) {
const tally = countByKey(votes);
const { topCount, topKeys } = pluralityTop(tally);
// 3-0 consensus — one bucket holds all three.
if (topCount === 3) {
const entry = tally.get(topKeys[0])!;
logger?.info('tie_break', { path: 'none' as TieBreakPath, verdict: topKeys[0] });
return {
verdict: topKeys[0],
path: 'none',
votes,
};
}
// 2-1 majority — one bucket holds two, another holds one.
if (topCount === 2 && topKeys.length === 1) {
logger?.info('tie_break', { path: 'majority' as TieBreakPath, verdict: topKeys[0] });
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
// 1-1-1 split — three buckets of one each. Escalate to fourth vendor.
if (topCount === 1 && topKeys.length === 3) {
if (!options.callFourthVendor) {
throw new Error(
'resolveTieBreak: 1-1-1 three-way split requires a callFourthVendor implementation.',
);
}
const fourthVendorModel = options.fourthVendorModel ?? DEFAULT_FOURTH_VENDOR;
logger?.info('tie_break.quadri-vendor.invoke', {
path: 'quadri-vendor' as TieBreakPath,
fourth_vendor_slug: fourthVendorModel,
primary_keys: topKeys,
});
const fourthVote = await options.callFourthVendor({
primaryVotes: votes,
model: fourthVendorModel,
});
// Recurse with four votes. The 4-vote branch returns quadri-vendor or pm-escalation.
const resolved = await resolveTieBreak([...votes, fourthVote], options);
// Re-tag the path so "quadri-vendor" sticks on success cases and
// "pm-escalation" stays on 1-1-1-1 after escalation.
const path: TieBreakPath =
resolved.path === 'pm-escalation' ? 'pm-escalation' : 'quadri-vendor';
logger?.info('tie_break.quadri-vendor.resolved', {
path,
fourth_vendor_slug: fourthVendorModel,
verdict: resolved.verdict,
});
return {
verdict: resolved.verdict,
path,
votes: resolved.votes,
fourthVendorVote: fourthVote,
fourthVendorSlug: fourthVendorModel,
};
}
// Should be unreachable under correct input (any 3-vote distribution is
// 3-0, 2-1, or 1-1-1), but stay defensive.
throw new Error(
`resolveTieBreak: unexpected 3-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
);
}
// votes.length === 4 — post-quadri-vendor resolution path.
const tally = countByKey(votes);
const { topCount, topKeys } = pluralityTop(tally);
// 1-1-1-1 four-way split — every bucket is one. PM escalation.
if (topCount === 1) {
logger?.info('tie_break', {
path: 'pm-escalation' as TieBreakPath,
verdict: PM_ESCALATION_VERDICT,
four_way_keys: topKeys,
});
return {
verdict: PM_ESCALATION_VERDICT,
path: 'pm-escalation',
votes,
};
}
// Plurality winner: 2-1-1 / 1-1-2 (unique top). 2-2 ties with 4 votes are
// structurally impossible from our flow (3 primary votes make 2-2
// impossible after the +1 fourth vote; the fourth vote always creates a
// unique plurality or a 1-1-1-1). Still handle 2-2 defensively by
// promoting to pm-escalation so no silent coin-flip ever lands in prod.
if (topCount === 2 && topKeys.length === 1) {
logger?.info('tie_break', {
path: 'majority' as TieBreakPath,
verdict: topKeys[0],
vote_count: 4,
});
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
if (topCount === 2 && topKeys.length >= 2) {
// 2-2 tie on 4 votes — not reachable from 1-1-1→quadri flow, but a
// caller may pass in a pre-constructed 4-vote vector. Escalate rather
// than coin-flip.
logger?.warn?.('tie_break.two-two-tie', {
path: 'pm-escalation' as TieBreakPath,
verdict: PM_ESCALATION_VERDICT,
two_two_keys: topKeys,
});
return {
verdict: PM_ESCALATION_VERDICT,
path: 'pm-escalation',
votes,
};
}
if (topCount === 3 || topCount === 4) {
logger?.info('tie_break', {
path: 'majority' as TieBreakPath,
verdict: topKeys[0],
vote_count: 4,
});
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
// Truly unreachable given 4 input votes; defensive.
throw new Error(
`resolveTieBreak: unexpected 4-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
);
}

View File

@@ -1,383 +0,0 @@
/**
* Failure-mode judge — scaffold.
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
* Spec: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §4 (prompt),
* §6 (ensemble), §8 (inter-judge agreement / Fleiss' kappa thresholds)
* LOCKED: PM-Waggle-OS/decisions/2026-04-20-failure-mode-oq-resolutions-locked.md
*
* This module is deliberately *not wired* into the harness runner in this
* sprint (explicitly out of scope per the brief). It exists so that Sprint 9
* can drop in the judge call on top of a tested foundation.
*
* Design decisions worth knowing:
*
* — Zod validates strictly. The Step-3 schema contract between the model
* and us is enforced with a `.refine()` that captures the two
* cross-field invariants from the spec §4:
* verdict === 'correct' ⇒ failure_mode MUST be null
* verdict === 'incorrect' ⇒ failure_mode MUST be one of F1..F5
* Any violation → retry once with the reminder → JudgeParseError.
*
* — JSON extraction is best-effort. Models often wrap JSON in ```json
* fences, leading prose, or trailing chatter. We strip the obvious
* wrappers before parsing; if both attempts fail we surface the raw
* response in the thrown error so a debugger can reproduce.
*
* — Ensemble tie-break is explicit: on a 2-2 split the ensemble uses the
* first judge in the `judgeModels` list. By convention the caller puts
* Sonnet first so "2-2 tie broken by Sonnet" holds per the spec §6.
*
* — Fleiss' kappa is the standard multi-rater formulation (Fleiss 1971).
* Raters per subject must be constant; we throw if any subject has a
* different rater count, since the formula is undefined otherwise.
* Categories are the 6-class vocabulary from §8 (`correct` treated as
* a separate class from F1..F5 — the same convention §8 specifies).
*/
import { z } from 'zod';
// ── Public types ───────────────────────────────────────────────────────
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
export type Verdict = 'correct' | 'incorrect';
export interface JudgeResult {
verdict: Verdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
}
export interface LlmClient {
/** One-shot completion. Returns the raw text produced by the judge LLM. */
complete(prompt: string): Promise<string>;
}
export class JudgeParseError extends Error {
readonly lastResponse: string;
readonly lastParseError?: string;
readonly judgeModel: string;
constructor(message: string, details: { lastResponse: string; judgeModel: string; lastParseError?: string }) {
super(message);
this.name = 'JudgeParseError';
this.lastResponse = details.lastResponse;
this.judgeModel = details.judgeModel;
this.lastParseError = details.lastParseError;
}
}
// ── Zod schema (enforces spec §4 Step-3 contract) ──────────────────────
const judgeSchema = z
.object({
verdict: z.enum(['correct', 'incorrect']),
failure_mode: z.union([z.null(), z.enum(['F1', 'F2', 'F3', 'F4', 'F5'])]),
rationale: z.string().min(1, 'rationale must be a non-empty sentence'),
})
.strict()
.refine(
r => (r.verdict === 'correct' ? r.failure_mode === null : r.failure_mode !== null),
{
message:
'verdict/failure_mode invariant violated: ' +
'verdict="correct" requires failure_mode=null; verdict="incorrect" requires one of F1..F5',
},
);
// ── Prompt builder (exact text from strategy §4) ───────────────────────
/** Exact judge prompt text from `strategy/2026-04-20-failure-mode-taxonomy.md` §4.
* Do not modify. The spec requires cell-independent scoring — changing the
* prompt between cells, or even between invocations, invalidates the ensemble. */
export function buildJudgePrompt(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
}): string {
return [
"You are evaluating whether an LLM's answer is correct against ground truth.",
'',
'## Question',
params.question,
'',
'## Ground-truth answer',
params.groundTruth,
'',
'## Ground-truth supporting context (excerpt shown to the model)',
params.contextExcerpt,
'',
"## Model's answer",
params.modelAnswer,
'',
'## Your task',
'',
"Step 1: Determine if the model's answer is correct.",
'- "correct" means the model\'s answer contains all required facts from ground truth, with no additional incorrect claims.',
'- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.',
'- Extra detail is acceptable ONLY if it is factually correct.',
'',
'Step 2: If incorrect, assign exactly one failure mode using this decision tree:',
'',
'1. Does the model explicitly refuse or say it does not know? → F1 (ABSTAIN)',
'2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? → F5 (OFF-TOPIC)',
'3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? → F4 (HALLUCINATED)',
'4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? → F2 (PARTIAL)',
'5. Otherwise (model states facts derived from the context but gets them wrong): → F3 (INCORRECT)',
'',
'Step 3: Return JSON only, no prose, in this exact schema:',
'',
'{',
' "verdict": "correct" | "incorrect",',
' "failure_mode": null | "F1" | "F2" | "F3" | "F4" | "F5",',
' "rationale": "one sentence explaining the verdict"',
'}',
'',
'If verdict is "correct", failure_mode MUST be null.',
'If verdict is "incorrect", failure_mode MUST be one of F1-F5.',
].join('\n');
}
/** Retry reminder prepended to the original prompt when the first response
* fails to parse. Text is verbatim from the Task-4 acceptance spec. */
export const RETRY_REMINDER =
'Your previous response was not valid JSON. Return only the JSON object, no prose.';
// ── JSON extraction ────────────────────────────────────────────────────
/** Best-effort extraction of the JSON body from a model response. Handles:
* - bare JSON
* - `{ ... }` embedded in prose (first `{` to matching last `}`)
* - markdown code fences ```json ... ``` or ``` ... ``` */
export function extractJsonBody(raw: string): string | null {
if (!raw) return null;
const trimmed = raw.trim();
// Code-fence strip first.
const fenceMatch = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i);
const afterFence = fenceMatch ? fenceMatch[1].trim() : trimmed;
if (afterFence.startsWith('{') && afterFence.endsWith('}')) return afterFence;
// Fallback — greedy `{ ... }` scan, stopping at the last `}`.
const first = afterFence.indexOf('{');
const last = afterFence.lastIndexOf('}');
if (first >= 0 && last > first) return afterFence.slice(first, last + 1);
return null;
}
function tryParse(raw: string): { ok: true; value: JudgeResult } | { ok: false; error: string } {
const body = extractJsonBody(raw);
if (!body) return { ok: false, error: 'no JSON object found in response' };
let parsedJson: unknown;
try {
parsedJson = JSON.parse(body);
} catch (err) {
return { ok: false, error: `JSON.parse failed: ${(err as Error).message}` };
}
const result = judgeSchema.safeParse(parsedJson);
if (!result.success) {
return { ok: false, error: `schema validation failed: ${result.error.issues.map(i => i.message).join('; ')}` };
}
return {
ok: true,
value: {
verdict: result.data.verdict,
failure_mode: result.data.failure_mode,
rationale: result.data.rationale,
judge_model: '', // filled by judgeAnswer from params.judgeModel
},
};
}
// ── Public API ─────────────────────────────────────────────────────────
export async function judgeAnswer(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModel: string;
llmClient: LlmClient;
}): Promise<JudgeResult> {
const prompt = buildJudgePrompt({
question: params.question,
groundTruth: params.groundTruth,
contextExcerpt: params.contextExcerpt,
modelAnswer: params.modelAnswer,
});
const first = await params.llmClient.complete(prompt);
const firstParsed = tryParse(first);
if (firstParsed.ok) {
return { ...firstParsed.value, judge_model: params.judgeModel };
}
// Retry once with the reminder prefixed. A fresh LLM call — callers can
// wire conversation-state preservation later; spec §4 does not require it.
const retryPrompt = `${RETRY_REMINDER}\n\n${prompt}`;
const second = await params.llmClient.complete(retryPrompt);
const secondParsed = tryParse(second);
if (secondParsed.ok) {
return { ...secondParsed.value, judge_model: params.judgeModel };
}
throw new JudgeParseError(
`Judge ${params.judgeModel} produced unparseable output after one retry`,
{
lastResponse: second,
judgeModel: params.judgeModel,
lastParseError: secondParsed.error,
},
);
}
export async function judgeEnsemble(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModels: string[];
llmClients: Map<string, LlmClient>;
}): Promise<{ ensemble: JudgeResult[]; majority: JudgeResult; fleissKappa: number }> {
if (params.judgeModels.length === 0) {
throw new Error('judgeEnsemble requires at least one judgeModel');
}
const results: JudgeResult[] = [];
for (const model of params.judgeModels) {
const client = params.llmClients.get(model);
if (!client) throw new Error(`judgeEnsemble: no LlmClient registered for model ${model}`);
const result = await judgeAnswer({
question: params.question,
groundTruth: params.groundTruth,
contextExcerpt: params.contextExcerpt,
modelAnswer: params.modelAnswer,
judgeModel: model,
llmClient: client,
});
results.push(result);
}
const majority = computeMajority(results, params.judgeModels[0]);
// For a single-subject ensemble, Fleiss' kappa is computed over the same
// one-subject matrix — degenerate but well-defined (NaN gets clamped to 0
// when the subject has only one category). In production, this function
// is typically called per-instance and kappa is aggregated at the batch
// layer; we expose the single-subject value for diagnostic continuity.
const fleissKappa = computeFleissKappa([results]);
return { ensemble: results, majority, fleissKappa };
}
function computeMajority(results: readonly JudgeResult[], tieBreakerModel: string): JudgeResult {
// Count verdict × failure_mode pairs (null collapses to 'NA' for keying).
const tally = new Map<string, number>();
const first = new Map<string, JudgeResult>();
for (const r of results) {
const key = `${r.verdict}|${r.failure_mode ?? 'NA'}`;
tally.set(key, (tally.get(key) ?? 0) + 1);
if (!first.has(key)) first.set(key, r);
}
let winnerKey = '';
let winnerCount = -1;
for (const [key, count] of tally) {
if (count > winnerCount) {
winnerKey = key;
winnerCount = count;
}
}
// Tie detection: if any other key has equal count, the tie-breaker model
// wins — its verdict becomes the majority.
const tied = Array.from(tally.entries()).filter(([, c]) => c === winnerCount);
if (tied.length > 1) {
const tb = results.find(r => r.judge_model === tieBreakerModel);
if (tb) return tb;
// Fallback: keep the first deterministic winner (stable map iteration).
}
const winner = first.get(winnerKey);
if (!winner) {
// Unreachable under normal flow; defensive.
return results[0];
}
return {
verdict: winner.verdict,
failure_mode: winner.failure_mode,
rationale: winner.rationale,
judge_model: 'ensemble_majority',
};
}
// ── Fleiss' kappa ──────────────────────────────────────────────────────
/** The 6-class rating space used by Fleiss' kappa: correct + F1..F5.
* §8: "standardni Fleiss' kappa preko 4 raters × N instanci × 6 klasa". */
const KAPPA_CATEGORIES = ['correct', 'F1', 'F2', 'F3', 'F4', 'F5'] as const;
type KappaCategory = typeof KAPPA_CATEGORIES[number];
function categoryOf(r: JudgeResult): KappaCategory {
if (r.verdict === 'correct') return 'correct';
return r.failure_mode ?? 'F3'; // should never hit fallback — schema guard
}
/** Computes Fleiss' kappa on an N-subjects × n-raters matrix.
*
* Input shape: `ratings[i]` is the list of judge results for subject i —
* all subjects must have the same number of raters. Categories are the
* fixed 6-class vocabulary.
*
* Returns: κ in [-1, 1]. If all subjects are unanimous on the same
* category (expected-agreement = 1), returns 1 (the (1-1)/(1-1) limit).
*/
export function computeFleissKappa(ratings: readonly (readonly JudgeResult[])[]): number {
if (ratings.length === 0) return 0;
const n = ratings[0].length;
if (n < 2) {
// Fleiss' kappa is undefined for a single rater. Degenerate → 0.
return 0;
}
for (const row of ratings) {
if (row.length !== n) {
throw new Error(`Fleiss' kappa requires a constant rater count; saw ${row.length} and ${n}`);
}
}
const N = ratings.length;
const k = KAPPA_CATEGORIES.length;
const catIndex: Record<KappaCategory, number> = { correct: 0, F1: 1, F2: 2, F3: 3, F4: 4, F5: 5 };
// nij[i][j] = count of raters who assigned subject i to category j
const nij: number[][] = Array.from({ length: N }, () => new Array<number>(k).fill(0));
for (let i = 0; i < N; i++) {
for (const r of ratings[i]) {
const col = catIndex[categoryOf(r)];
nij[i][col] += 1;
}
}
// Pj = (1/(N*n)) * sum_i nij[i][j]
const Pj: number[] = new Array<number>(k).fill(0);
for (let j = 0; j < k; j++) {
let sum = 0;
for (let i = 0; i < N; i++) sum += nij[i][j];
Pj[j] = sum / (N * n);
}
// Pi = (1/(n(n-1))) * ( sum_j nij[i][j]^2 - n )
let PbarSum = 0;
for (let i = 0; i < N; i++) {
let sqSum = 0;
for (let j = 0; j < k; j++) sqSum += nij[i][j] * nij[i][j];
const Pi = (sqSum - n) / (n * (n - 1));
PbarSum += Pi;
}
const Pbar = PbarSum / N;
// Pebar = sum_j Pj^2
let Pebar = 0;
for (let j = 0; j < k; j++) Pebar += Pj[j] * Pj[j];
if (Pebar >= 1 - 1e-12) {
// Every rating in one category → expected = observed = 1 → κ = 1 by convention.
return 1;
}
return (Pbar - Pebar) / (1 - Pebar);
}

View File

@@ -1,311 +0,0 @@
/**
* aggregate.ts unit tests (Sprint 9 Task 3).
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3 §Acceptance
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
*
* Synthetic 12-instance JSONL fixture: 3 cells × 4 verdicts. Tests
* assert:
* - per-cell count table matches hand-computed rows
* - weighted score matches hand-computed number (see §2 below)
* - per-category rollup surfaces the hallucination flag at the right threshold
* - cross-cell delta matrix populates full-context − raw direction
* - cost summary sums correctly + Week-1 projection flag fires at the threshold
* - markdown renderer produces parseable output + contains every row
*/
import { describe, it, expect } from 'vitest';
import {
buildReport,
perCellRollup,
perCategoryRollup,
crossCellDeltaMatrix,
costSummary,
projectVerdict6,
renderMarkdown,
WEIGHTS,
VERDICT6_VALUES,
type JudgedJsonlRecord,
} from '../../src/benchmarks/aggregate.ts';
// ── Fixture builder ─────────────────────────────────────────────────────
function mkRecord(
turnId: string,
cell: string,
verdict: 'correct' | 'incorrect' | 'unjudged',
failureMode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null,
category: JudgedJsonlRecord['category'] = 'single-hop',
usd = 0.001,
): JudgedJsonlRecord {
const rec: JudgedJsonlRecord = {
turnId,
cell,
instance_id: `i_${turnId}`,
model: 'qwen3.6-35b-a3b',
seed: 42,
accuracy: verdict === 'correct' ? 1 : 0,
p50_latency_ms: 800,
p95_latency_ms: 1200,
usd_per_query: usd,
failure_mode: null,
category,
};
if (verdict !== 'unjudged') {
rec.judge_verdict = verdict;
rec.judge_failure_mode = failureMode;
rec.judge_rationale = 'test rationale';
rec.judge_model = 'claude-sonnet-4-6';
rec.judge_timestamp = '2026-04-21T12:00:00Z';
}
return rec;
}
/** 12-instance fixture — 4 each per cell, one of each verdict shape.
* Hand-computed expectations are commented inline so a future diff
* catches silent drift in the rubric. */
function fixture12(): JudgedJsonlRecord[] {
const recs: JudgedJsonlRecord[] = [];
const cells = ['raw', 'filtered', 'full-context'];
// Per cell: 1 correct + 1 F2 partial + 1 F3 incorrect + 1 F4 hallucinated.
for (const c of cells) {
recs.push(mkRecord(`${c}-A`, c, 'correct', null, 'single-hop'));
recs.push(mkRecord(`${c}-B`, c, 'incorrect', 'F2', 'multi-hop'));
recs.push(mkRecord(`${c}-C`, c, 'incorrect', 'F3', 'temporal'));
recs.push(mkRecord(`${c}-D`, c, 'incorrect', 'F4', 'open-ended'));
}
return recs;
}
// ── projectVerdict6 ─────────────────────────────────────────────────────
describe('projectVerdict6 — taxonomy §9 binary → brief §Task-1 6-value', () => {
it('maps correct → correct', () => {
expect(projectVerdict6(mkRecord('x', 'raw', 'correct', null))).toBe('correct');
});
it('maps incorrect + F1..F5 through their exact projections', () => {
const cases: Array<[typeof WEIGHTS extends Record<infer K, number> ? K : never, string]> = [
['F1_abstain', 'F1'],
['F2_partial', 'F2'],
['F3_incorrect', 'F3'],
['F4_hallucinated', 'F4'],
['F5_offtopic', 'F5'],
];
for (const [expected, mode] of cases) {
expect(projectVerdict6(
mkRecord('x', 'raw', 'incorrect', mode as 'F1' | 'F2' | 'F3' | 'F4' | 'F5'),
)).toBe(expected);
}
});
it('maps undefined judge_verdict → unjudged', () => {
expect(projectVerdict6(mkRecord('x', 'raw', 'unjudged', null))).toBe('unjudged');
});
});
// ── perCellRollup ───────────────────────────────────────────────────────
describe('perCellRollup', () => {
it('emits 3 rows, one per observed cell, in CELL_NAMES order', () => {
const rows = perCellRollup(fixture12());
expect(rows).toHaveLength(3);
expect(rows.map(r => r.cell)).toEqual(['raw', 'filtered', 'full-context']);
});
it('computes per-cell counts + weighted score correctly (hand-check)', () => {
const rows = perCellRollup(fixture12());
for (const row of rows) {
expect(row.total).toBe(4);
expect(row.counts.correct).toBe(1);
expect(row.counts.F2_partial).toBe(1);
expect(row.counts.F3_incorrect).toBe(1);
expect(row.counts.F4_hallucinated).toBe(1);
expect(row.counts.F1_abstain).toBe(0);
expect(row.counts.F5_offtopic).toBe(0);
expect(row.counts.unjudged).toBe(0);
// Weighted score = sum(percent × weight) over judged instances.
// judgedTotal=4. Each verdict is 1/4 = 0.25 of the cell.
// correct: 0.25 × 1.00 = 0.250
// F2_partial: 0.25 × 0.30 = 0.075
// F3_incorrect: 0.25 × -0.15 = -0.0375
// F4_hallucinated: 0.25 × -0.35 = -0.0875
// Total = 0.250 + 0.075 − 0.0375 − 0.0875 = 0.200
expect(row.weightedScore).toBeCloseTo(0.200, 4);
}
});
it('honors WEIGHTS table exactly', () => {
// Rebuild the hand-computed number from WEIGHTS by name so a rubric
// edit in aggregate.ts forces this test to recompute + update the
// expectation — no silent coefficient drift.
const expected =
0.25 * WEIGHTS.correct +
0.25 * WEIGHTS.F2_partial +
0.25 * WEIGHTS.F3_incorrect +
0.25 * WEIGHTS.F4_hallucinated;
const rows = perCellRollup(fixture12());
for (const row of rows) {
expect(row.weightedScore).toBeCloseTo(expected, 6);
}
});
it('treats unjudged rows as denominator-excluded for weightedScore', () => {
const mix = fixture12().slice(0, 4); // one cell worth
mix.push(mkRecord('unjudged-1', 'raw', 'unjudged', null));
mix.push(mkRecord('unjudged-2', 'raw', 'unjudged', null));
// Now raw has 4 judged + 2 unjudged. Weighted score denominator = 4,
// same as the baseline fixture — the 2 unjudged rows don't pull
// the score toward 0.
const rawRow = perCellRollup(mix).find(r => r.cell === 'raw')!;
expect(rawRow.total).toBe(6);
expect(rawRow.counts.unjudged).toBe(2);
expect(rawRow.weightedScore).toBeCloseTo(0.200, 4);
});
});
// ── perCategoryRollup ──────────────────────────────────────────────────
describe('perCategoryRollup', () => {
it('groups by category and computes percents per bucket', () => {
const rows = perCategoryRollup(fixture12());
// fixture assigns 3 rows per category (one per cell × one verdict shape)
const cats = rows.map(r => r.category);
expect(cats).toContain('single-hop');
expect(cats).toContain('multi-hop');
expect(cats).toContain('temporal');
expect(cats).toContain('open-ended');
for (const row of rows) {
expect(row.total).toBe(3);
}
// open-ended has all F4 (hallucinated) rows — flag must fire.
const openEnded = rows.find(r => r.category === 'open-ended')!;
expect(openEnded.counts.F4_hallucinated).toBe(3);
expect(openEnded.hallucinationFlag).toBe(true);
// single-hop has all correct — flag off.
const singleHop = rows.find(r => r.category === 'single-hop')!;
expect(singleHop.counts.correct).toBe(3);
expect(singleHop.hallucinationFlag).toBe(false);
});
it('does not include categories that have zero rows', () => {
const rows = perCategoryRollup(fixture12());
for (const row of rows) {
expect(row.total).toBeGreaterThan(0);
}
});
});
// ── crossCellDeltaMatrix ───────────────────────────────────────────────
describe('crossCellDeltaMatrix', () => {
it('returns delta = full-context − raw for each verdict, preserving sign', () => {
// Build a case where full-context correct% > raw correct%.
const recs: JudgedJsonlRecord[] = [
// raw: 1 correct, 3 incorrect-F4 → 25% correct, 75% F4
mkRecord('r1', 'raw', 'correct', null, 'single-hop'),
mkRecord('r2', 'raw', 'incorrect', 'F4', 'single-hop'),
mkRecord('r3', 'raw', 'incorrect', 'F4', 'single-hop'),
mkRecord('r4', 'raw', 'incorrect', 'F4', 'single-hop'),
// full-context: 3 correct, 1 F4 → 75% correct, 25% F4
mkRecord('f1', 'full-context', 'correct', null, 'single-hop'),
mkRecord('f2', 'full-context', 'correct', null, 'single-hop'),
mkRecord('f3', 'full-context', 'correct', null, 'single-hop'),
mkRecord('f4', 'full-context', 'incorrect', 'F4', 'single-hop'),
];
const perCell = perCellRollup(recs);
const delta = crossCellDeltaMatrix(perCell);
expect(delta).not.toBeNull();
const correctDelta = delta!.find(d => d.verdict === 'correct')!;
expect(correctDelta.rawPercent).toBeCloseTo(0.25, 6);
expect(correctDelta.fullContextPercent).toBeCloseTo(0.75, 6);
expect(correctDelta.delta).toBeCloseTo(0.50, 6);
// F4 goes the other way.
const f4Delta = delta!.find(d => d.verdict === 'F4_hallucinated')!;
expect(f4Delta.delta).toBeCloseTo(-0.50, 6);
});
it('returns null when either raw or full-context is absent', () => {
const recs: JudgedJsonlRecord[] = [mkRecord('x', 'filtered', 'correct', null)];
const perCell = perCellRollup(recs);
expect(crossCellDeltaMatrix(perCell)).toBeNull();
});
});
// ── costSummary + Week-1 projection threshold ──────────────────────────
describe('costSummary', () => {
it('sums per-cell USD across records', () => {
const recs: JudgedJsonlRecord[] = [
mkRecord('a', 'raw', 'correct', null, 'single-hop', 0.010),
mkRecord('b', 'raw', 'correct', null, 'single-hop', 0.020),
mkRecord('c', 'full-context', 'correct', null, 'single-hop', 0.050),
];
const cost = costSummary(recs);
expect(cost.totalUsd).toBeCloseTo(0.08, 6);
expect(cost.perCellUsd['raw']).toBeCloseTo(0.03, 6);
expect(cost.perCellUsd['full-context']).toBeCloseTo(0.05, 6);
});
it('buildReport overlays an authoritative judgeTotalUsd and recomputes the Week-1 projection', () => {
const recs = fixture12();
// 12 records all judged; 12 × 4 cells × 50 instances scaling → 200
// instances. Set judgeTotalUsd = $1.50 across 12 → per-instance = 0.125
// → projected = 0.125 × 200 = $25 → above $20 → warning fires.
const report = buildReport(recs, { judgeTotalUsd: 1.50 });
expect(report.cost.judgeTotalUsd).toBeCloseTo(1.50, 6);
expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(25.0, 6);
expect(report.cost.week1WarningFired).toBe(true);
});
it('Week-1 warning does not fire when projected stays under $20', () => {
const recs = fixture12();
const report = buildReport(recs, { judgeTotalUsd: 0.60 });
// per-instance = 0.05; projected = 0.05 × 200 = $10
expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(10.0, 6);
expect(report.cost.week1WarningFired).toBe(false);
});
});
// ── Markdown renderer — smoke + snapshot-lite ──────────────────────────
describe('renderMarkdown', () => {
it('produces parseable markdown with every cell row + every verdict column', () => {
const report = buildReport(fixture12());
const md = renderMarkdown(report);
// Header structure
expect(md).toContain('# Benchmark Aggregate Report');
expect(md).toContain('## Per-cell verdict distribution');
expect(md).toContain('## Per-LoCoMo-category distribution');
expect(md).toContain('## Cost summary');
// Cells in table
for (const cell of ['raw', 'filtered', 'full-context']) {
expect(md).toContain(`| ${cell} |`);
}
// Verdict columns in the header row
for (const v of VERDICT6_VALUES) {
// VERDICT6_VALUES uses snake names that won't all appear literally
// in the header (e.g. "F1 abstain" vs "F1_abstain"). Assert on the
// base labels the renderer emits.
}
expect(md).toContain('Correct');
expect(md).toContain('F1 abstain');
expect(md).toContain('F4 hallucinated');
expect(md).toContain('Weighted score');
});
it('surfaces the hallucination-flag emoji on the flagged category', () => {
const md = renderMarkdown(buildReport(fixture12()));
// open-ended is flagged in the fixture (3/3 F4). Shape check via the
// brief's sentinel emoji + "PM review" string.
expect(md).toContain('⚠️ PM review');
});
it('omits the Week-1 warning line when threshold not crossed', () => {
const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 0.40 }));
expect(md).not.toContain('Week-1 scale-up warning');
});
it('emits the Week-1 warning line when threshold crossed', () => {
const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 5.0 }));
expect(md).toContain('Week-1 scale-up warning');
});
});

View File

@@ -1,255 +0,0 @@
/**
* Sprint 11 Task B2 — Tie-break policy unit tests.
*
* Authority: decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
*
* The four core scenarios from the brief §3 B2 + LOCK §0:
*
* 1. 3-0 consensus → path=none, no fourth-vendor call.
* 2. 2-1 majority → path=majority, no fourth-vendor call.
* 3. 1-1-1 split resolves → path=quadri-vendor, plurality verdict,
* fourth-vendor called once with correct payload.
* 4. 1-1-1 stays unresolved → path=pm-escalation (1-1-1-1 four-way),
* fourth-vendor called once; verdict is the
* PM_ESCALATION_VERDICT sentinel.
*
* Tests use a mocked CallFourthVendor function and an in-memory logger —
* no network, no API spend. The LIVE grok-4.20 smoke lives in the B2 exit
* ping's companion script and runs independently.
*/
import { describe, it, expect, vi } from 'vitest';
import {
resolveTieBreak,
DEFAULT_FOURTH_VENDOR,
PM_ESCALATION_VERDICT,
type Vote,
type TieBreakLogger,
type CallFourthVendor,
} from '../../src/benchmarks/judge/ensemble-tiebreak.js';
function vote(verdict: 'correct' | 'incorrect', failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null, model: string): Vote {
return {
verdict,
failure_mode,
rationale: `rationale from ${model}`,
judge_model: model,
};
}
function makeLogger(): { logger: TieBreakLogger; events: Array<{ event: string; fields: Record<string, unknown> }> } {
const events: Array<{ event: string; fields: Record<string, unknown> }> = [];
const logger: TieBreakLogger = {
info: (event, fields) => {
events.push({ event, fields });
},
warn: (event, fields) => {
events.push({ event, fields });
},
};
return { logger, events };
}
describe('Sprint 11 B2 — resolveTieBreak', () => {
it('3-0 consensus returns path=none and does not call the fourth vendor', async () => {
const votes: Vote[] = [
vote('correct', null, 'claude-opus-4-7'),
vote('correct', null, 'gpt-5.4-pro'),
vote('correct', null, 'gemini-3.1-pro'),
];
const callFourthVendor = vi.fn<CallFourthVendor>();
const { logger, events } = makeLogger();
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
expect(result.path).toBe('none');
expect(result.verdict).toBe('correct|NA');
expect(result.votes).toHaveLength(3);
expect(result.fourthVendorVote).toBeUndefined();
expect(result.fourthVendorSlug).toBeUndefined();
expect(callFourthVendor).not.toHaveBeenCalled();
expect(events).toContainEqual(
expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'none' }) }),
);
});
it('2-1 majority returns path=majority and does not call the fourth vendor', async () => {
const votes: Vote[] = [
vote('correct', null, 'claude-opus-4-7'),
vote('correct', null, 'gpt-5.4-pro'),
vote('incorrect', 'F3', 'gemini-3.1-pro'),
];
const callFourthVendor = vi.fn<CallFourthVendor>();
const { logger, events } = makeLogger();
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
expect(result.path).toBe('majority');
expect(result.verdict).toBe('correct|NA');
expect(result.votes).toHaveLength(3);
expect(callFourthVendor).not.toHaveBeenCalled();
expect(events).toContainEqual(
expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'majority' }) }),
);
});
it('1-1-1 split triggers quadri-vendor call on xai/grok-4.20 and resolves via plurality', async () => {
const primaryVotes: Vote[] = [
vote('correct', null, 'claude-opus-4-7'), // bucket A: correct|NA
vote('incorrect', 'F3', 'gpt-5.4-pro'), // bucket B: incorrect|F3
vote('incorrect', 'F4', 'gemini-3.1-pro'), // bucket C: incorrect|F4
];
const grokVote = vote('correct', null, 'xai/grok-4.20'); // joins bucket A → plurality
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(grokVote);
const { logger, events } = makeLogger();
const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger });
expect(result.path).toBe('quadri-vendor');
expect(result.verdict).toBe('correct|NA');
expect(result.votes).toHaveLength(4);
expect(result.fourthVendorVote).toEqual(grokVote);
expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR);
// Exactly one fourth-vendor call with the correct payload.
expect(callFourthVendor).toHaveBeenCalledTimes(1);
expect(callFourthVendor).toHaveBeenCalledWith({
primaryVotes,
model: 'xai/grok-4.20',
});
// pino-shaped events: invoke + resolved.
const invokeEvent = events.find(e => e.event === 'tie_break.quadri-vendor.invoke');
const resolvedEvent = events.find(e => e.event === 'tie_break.quadri-vendor.resolved');
expect(invokeEvent).toBeDefined();
expect(invokeEvent!.fields).toMatchObject({
path: 'quadri-vendor',
fourth_vendor_slug: 'xai/grok-4.20',
});
expect(resolvedEvent).toBeDefined();
expect(resolvedEvent!.fields).toMatchObject({
path: 'quadri-vendor',
fourth_vendor_slug: 'xai/grok-4.20',
verdict: 'correct|NA',
});
});
it('1-1-1 split where the fourth vote is a fourth distinct bucket escalates to PM', async () => {
const primaryVotes: Vote[] = [
vote('correct', null, 'claude-opus-4-7'), // A
vote('incorrect', 'F2', 'gpt-5.4-pro'), // B
vote('incorrect', 'F3', 'gemini-3.1-pro'), // C
];
// Fourth vote takes a fourth distinct failure mode → 1-1-1-1.
const grokVote = vote('incorrect', 'F4', 'xai/grok-4.20');
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(grokVote);
const { logger, events } = makeLogger();
const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger });
expect(result.path).toBe('pm-escalation');
expect(result.verdict).toBe(PM_ESCALATION_VERDICT);
expect(result.votes).toHaveLength(4);
expect(result.fourthVendorVote).toEqual(grokVote);
expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR);
expect(callFourthVendor).toHaveBeenCalledTimes(1);
// Logger emits a pm-escalation path event on the 4-vote recursive call.
const escalationEvent = events.find(e =>
e.event === 'tie_break' && e.fields.path === 'pm-escalation',
);
expect(escalationEvent).toBeDefined();
expect(escalationEvent!.fields).toMatchObject({
path: 'pm-escalation',
verdict: PM_ESCALATION_VERDICT,
});
});
});
describe('Sprint 11 B2 — resolveTieBreak defensive invariants', () => {
it('throws on invalid vote length (2 votes)', async () => {
const votes: Vote[] = [
vote('correct', null, 'a'),
vote('correct', null, 'b'),
];
await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/);
});
it('throws on invalid vote length (5 votes)', async () => {
const votes: Vote[] = [
vote('correct', null, 'a'),
vote('correct', null, 'b'),
vote('correct', null, 'c'),
vote('correct', null, 'd'),
vote('correct', null, 'e'),
];
await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/);
});
it('1-1-1 without a callFourthVendor dep throws explicitly', async () => {
const votes: Vote[] = [
vote('correct', null, 'a'),
vote('incorrect', 'F3', 'b'),
vote('incorrect', 'F4', 'c'),
];
await expect(resolveTieBreak(votes)).rejects.toThrow(/requires a callFourthVendor/);
});
it('caller-provided 4-vote vector resolves to plurality without extra calls', async () => {
// Test the caller-driven 4-vote shape — harness may pre-construct this.
const votes: Vote[] = [
vote('correct', null, 'claude-opus-4-7'),
vote('correct', null, 'gpt-5.4-pro'),
vote('incorrect', 'F3', 'gemini-3.1-pro'),
vote('incorrect', 'F4', 'xai/grok-4.20'),
];
const callFourthVendor = vi.fn<CallFourthVendor>();
const { logger, events } = makeLogger();
const result = await resolveTieBreak(votes, { callFourthVendor, logger });
expect(result.path).toBe('majority');
expect(result.verdict).toBe('correct|NA');
expect(callFourthVendor).not.toHaveBeenCalled();
expect(events.some(e => e.fields.path === 'majority')).toBe(true);
});
it('caller-provided 2-2 tie on 4 votes escalates (defensive, never silent coin-flip)', async () => {
const votes: Vote[] = [
vote('correct', null, 'a'),
vote('correct', null, 'b'),
vote('incorrect', 'F3', 'c'),
vote('incorrect', 'F3', 'd'),
];
const { logger, events } = makeLogger();
const result = await resolveTieBreak(votes, { logger });
expect(result.path).toBe('pm-escalation');
expect(result.verdict).toBe(PM_ESCALATION_VERDICT);
expect(events.some(e => e.event === 'tie_break.two-two-tie')).toBe(true);
});
it('fourthVendorModel override is respected', async () => {
const votes: Vote[] = [
vote('correct', null, 'a'),
vote('incorrect', 'F3', 'b'),
vote('incorrect', 'F4', 'c'),
];
const callFourthVendor = vi.fn<CallFourthVendor>().mockResolvedValue(
vote('correct', null, 'custom-slug/vendor-x'),
);
const result = await resolveTieBreak(votes, {
callFourthVendor,
fourthVendorModel: 'custom-slug/vendor-x',
});
expect(result.fourthVendorSlug).toBe('custom-slug/vendor-x');
expect(callFourthVendor).toHaveBeenCalledWith({
primaryVotes: votes,
model: 'custom-slug/vendor-x',
});
});
});

View File

@@ -1,340 +0,0 @@
/**
* Failure-mode judge — unit tests.
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
* Module: packages/server/src/benchmarks/judge/failure-mode-judge.ts
*
* Coverage:
* - Valid JSON parse for all 5 failure modes (F1..F5) + the correct verdict.
* - Invalid-JSON → retry succeeds.
* - Invalid-JSON on both attempts → JudgeParseError.
* - 4-judge ensemble: 4-0 unanimous, 3-1 majority, 2-2 tie broken by
* the first model in `judgeModels` (Sonnet by convention).
* - Fleiss' kappa on hand-crafted 4×10 matrices with values computed by
* hand and verified against the formula (κ=1 for unanimous two-cluster,
* κ≈0.1111 for a known mixed matrix — both within ±0.01 tolerance).
*
* No network. No LLM calls. Pure unit tests against mock LlmClients.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import {
buildJudgePrompt,
computeFleissKappa,
extractJsonBody,
judgeAnswer,
judgeEnsemble,
JudgeParseError,
RETRY_REMINDER,
type FailureMode,
type JudgeResult,
type LlmClient,
type Verdict,
} from '../../src/benchmarks/judge/failure-mode-judge.js';
// ── Test helpers ───────────────────────────────────────────────────────
class ScriptedLlmClient implements LlmClient {
readonly calls: string[] = [];
private readonly queue: Array<string | Error>;
constructor(responses: Array<string | Error>) {
this.queue = [...responses];
}
async complete(prompt: string): Promise<string> {
this.calls.push(prompt);
if (this.queue.length === 0) throw new Error('ScriptedLlmClient out of responses');
const next = this.queue.shift()!;
if (next instanceof Error) throw next;
return next;
}
}
function mkResult(verdict: Verdict, failure_mode: FailureMode | null, rationale: string, judge_model: string): JudgeResult {
return { verdict, failure_mode, rationale, judge_model };
}
// ── Prompt shape ───────────────────────────────────────────────────────
describe('buildJudgePrompt', () => {
it('interpolates all four required variables verbatim', () => {
const prompt = buildJudgePrompt({
question: 'When did Caroline go to the support group?',
groundTruth: '7 May 2023',
contextExcerpt: 'Caroline: I went to a LGBTQ support group yesterday.',
modelAnswer: 'Unclear.',
});
expect(prompt).toContain('## Question\nWhen did Caroline go to the support group?');
expect(prompt).toContain('## Ground-truth answer\n7 May 2023');
expect(prompt).toContain('## Ground-truth supporting context');
expect(prompt).toContain("Caroline: I went to a LGBTQ support group yesterday.");
expect(prompt).toContain("## Model's answer\nUnclear.");
// Decision tree sanity — verifies the exact §4 text didn't drift.
expect(prompt).toContain('→ F1 (ABSTAIN)');
expect(prompt).toContain('→ F5 (OFF-TOPIC)');
expect(prompt).toContain('→ F4 (HALLUCINATED)');
expect(prompt).toContain('→ F2 (PARTIAL)');
expect(prompt).toContain('→ F3 (INCORRECT)');
});
});
// ── JSON extraction ────────────────────────────────────────────────────
describe('extractJsonBody', () => {
it('returns bare JSON unchanged', () => {
expect(extractJsonBody('{"verdict":"correct"}')).toBe('{"verdict":"correct"}');
});
it('strips markdown fences with `json` hint', () => {
const raw = '```json\n{"verdict":"correct"}\n```';
expect(extractJsonBody(raw)).toBe('{"verdict":"correct"}');
});
it('strips bare markdown fences', () => {
const raw = '```\n{"verdict":"incorrect","failure_mode":"F3"}\n```';
expect(extractJsonBody(raw)).toBe('{"verdict":"incorrect","failure_mode":"F3"}');
});
it('extracts the JSON object from prose', () => {
const raw = 'Here is my verdict: {"verdict":"correct","failure_mode":null,"rationale":"ok"}';
expect(extractJsonBody(raw)).toBe('{"verdict":"correct","failure_mode":null,"rationale":"ok"}');
});
it('returns null when no object is present', () => {
expect(extractJsonBody('I cannot comply.')).toBeNull();
});
});
// ── judgeAnswer — happy path + failure modes ───────────────────────────
describe('judgeAnswer — valid JSON parse for all 5 failure modes + correct', () => {
const cases: Array<{ name: string; verdict: Verdict; failure_mode: FailureMode | null; rationale: string }> = [
{ name: 'correct', verdict: 'correct', failure_mode: null, rationale: 'All facts match the ground truth.' },
{ name: 'F1 abstain', verdict: 'incorrect', failure_mode: 'F1', rationale: 'Model explicitly refused to answer.' },
{ name: 'F2 partial', verdict: 'incorrect', failure_mode: 'F2', rationale: 'Model stated 2 of 3 required facts.' },
{ name: 'F3 incorrect', verdict: 'incorrect', failure_mode: 'F3', rationale: 'Model stated a wrong date derived from context.' },
{ name: 'F4 hallucinated', verdict: 'incorrect', failure_mode: 'F4', rationale: 'Model named a person not in the context.' },
{ name: 'F5 off-topic', verdict: 'incorrect', failure_mode: 'F5', rationale: 'Model answered a different question.' },
];
for (const c of cases) {
it(`parses ${c.name} and stamps the judge_model`, async () => {
const payload = JSON.stringify({ verdict: c.verdict, failure_mode: c.failure_mode, rationale: c.rationale });
const client = new ScriptedLlmClient([payload]);
const result = await judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'claude-sonnet-4-6', llmClient: client,
});
expect(result.verdict).toBe(c.verdict);
expect(result.failure_mode).toBe(c.failure_mode);
expect(result.rationale).toBe(c.rationale);
expect(result.judge_model).toBe('claude-sonnet-4-6');
expect(client.calls).toHaveLength(1);
});
}
});
describe('judgeAnswer — retry semantics', () => {
const validPayload = JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'All facts match.' });
it('on invalid JSON, retries once with the reminder and returns the retry result', async () => {
const client = new ScriptedLlmClient([
'Sorry, I cannot produce structured output — here is a paragraph.',
validPayload,
]);
const result = await judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'claude-sonnet-4-6', llmClient: client,
});
expect(result.verdict).toBe('correct');
expect(client.calls).toHaveLength(2);
// Retry prompt must begin with the exact reminder text from the spec.
expect(client.calls[1].startsWith(RETRY_REMINDER)).toBe(true);
});
it('on invalid JSON twice, throws JudgeParseError with the raw response attached', async () => {
const client = new ScriptedLlmClient([
'Still refusing to produce JSON.',
'Nope, same here.',
]);
await expect(
judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'gpt-5', llmClient: client,
}),
).rejects.toBeInstanceOf(JudgeParseError);
try {
await judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'gpt-5', llmClient: new ScriptedLlmClient(['bad1', 'bad2']),
});
} catch (e) {
expect(e).toBeInstanceOf(JudgeParseError);
const err = e as JudgeParseError;
expect(err.judgeModel).toBe('gpt-5');
expect(err.lastResponse).toBe('bad2');
expect(err.lastParseError).toBeTruthy();
}
});
it('rejects schema-valid JSON that violates the verdict/failure_mode invariant', async () => {
// verdict=correct with a non-null failure_mode — Step-3 contract violation.
const bad = JSON.stringify({ verdict: 'correct', failure_mode: 'F4', rationale: 'contradictory' });
// Both attempts return the same bad shape — should throw.
const client = new ScriptedLlmClient([bad, bad]);
await expect(
judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'gemini-pro', llmClient: client,
}),
).rejects.toBeInstanceOf(JudgeParseError);
});
it('accepts fenced JSON in the first attempt (no retry)', async () => {
const fenced = '```json\n' + JSON.stringify({ verdict: 'incorrect', failure_mode: 'F3', rationale: 'Wrong date.' }) + '\n```';
const client = new ScriptedLlmClient([fenced]);
const result = await judgeAnswer({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModel: 'haiku', llmClient: client,
});
expect(result.failure_mode).toBe('F3');
expect(client.calls).toHaveLength(1);
});
});
// ── judgeEnsemble — 4-judge aggregation ────────────────────────────────
describe('judgeEnsemble — 4-judge aggregation', () => {
function mkClientWith(verdict: Verdict, failure_mode: FailureMode | null, rationale: string): LlmClient {
return new ScriptedLlmClient([JSON.stringify({ verdict, failure_mode, rationale })]);
}
const models = ['claude-sonnet-4-6', 'claude-haiku-4-5', 'gpt-5', 'gemini-pro'];
it('4-0 unanimous → majority matches the unanimous verdict', async () => {
const clients = new Map<string, LlmClient>();
for (const m of models) clients.set(m, mkClientWith('correct', null, 'match'));
const result = await judgeEnsemble({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModels: models, llmClients: clients,
});
expect(result.ensemble).toHaveLength(4);
expect(result.majority.verdict).toBe('correct');
expect(result.majority.failure_mode).toBeNull();
// κ = 1 when every rater agrees on the same class (and all 6 classes
// contribute 0 or 1 to the marginals → expected = observed = 1).
expect(result.fleissKappa).toBeCloseTo(1, 6);
});
it('3-1 majority → majority verdict wins, minority is recorded in ensemble', async () => {
const clients = new Map<string, LlmClient>();
clients.set(models[0], mkClientWith('incorrect', 'F3', 'A'));
clients.set(models[1], mkClientWith('incorrect', 'F3', 'B'));
clients.set(models[2], mkClientWith('incorrect', 'F3', 'C'));
clients.set(models[3], mkClientWith('incorrect', 'F4', 'D')); // minority — says hallucination
const result = await judgeEnsemble({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModels: models, llmClients: clients,
});
expect(result.majority.verdict).toBe('incorrect');
expect(result.majority.failure_mode).toBe('F3');
const failureModes = result.ensemble.map(r => r.failure_mode);
expect(failureModes.filter(m => m === 'F3')).toHaveLength(3);
expect(failureModes.filter(m => m === 'F4')).toHaveLength(1);
});
it('2-2 tie is broken by the first model in judgeModels (Sonnet by convention)', async () => {
const clients = new Map<string, LlmClient>();
// Sonnet + Haiku say F2; GPT-5 + Gemini say F3.
clients.set(models[0], mkClientWith('incorrect', 'F2', 'sonnet'));
clients.set(models[1], mkClientWith('incorrect', 'F2', 'haiku'));
clients.set(models[2], mkClientWith('incorrect', 'F3', 'gpt-5'));
clients.set(models[3], mkClientWith('incorrect', 'F3', 'gemini'));
const result = await judgeEnsemble({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModels: models, llmClients: clients,
});
// Sonnet wins the tie → F2.
expect(result.majority.failure_mode).toBe('F2');
expect(result.majority.rationale).toBe('sonnet');
expect(result.majority.judge_model).toBe('claude-sonnet-4-6');
});
it('refuses when no client is registered for a judge model', async () => {
const clients = new Map<string, LlmClient>();
clients.set(models[0], mkClientWith('correct', null, 'ok'));
// Missing models[1..3]
await expect(
judgeEnsemble({
question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma',
judgeModels: models, llmClients: clients,
}),
).rejects.toThrow(/no LlmClient registered/);
});
});
// ── Fleiss' kappa ──────────────────────────────────────────────────────
describe("computeFleissKappa on hand-crafted 4-judge × 10-subject matrices", () => {
function row(n: number, verdict: Verdict, failure_mode: FailureMode | null): JudgeResult[] {
return Array.from({ length: n }, (_, i) => mkResult(verdict, failure_mode, 'r', `j${i}`));
}
it('κ = 1 for two-cluster unanimous agreement (5 × correct / 5 × F3)', () => {
// 5 subjects: 4 raters all say correct.
// 5 subjects: 4 raters all say F3.
// Pbar = 1 (every subject unanimous). Pj(correct) = 0.5, Pj(F3) = 0.5.
// Pebar = 0.5 + 0.5 = 0.5 (treating the other 4 classes as 0).
// κ = (1 - 0.5) / (1 - 0.5) = 1.
const matrix: JudgeResult[][] = [];
for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null));
for (let i = 0; i < 5; i++) matrix.push(row(4, 'incorrect', 'F3'));
const kappa = computeFleissKappa(matrix);
expect(kappa).toBeCloseTo(1, 6);
});
it('κ ≈ 0.1111 for 5 unanimous-correct + 5 split-2/2-correct/F3 subjects', () => {
// Hand-computed: n_correct = 5*4 + 5*2 = 30, n_F3 = 5*2 = 10. Total = 40.
// Pj(correct) = 30/40 = 0.75 → 0.5625
// Pj(F3) = 10/40 = 0.25 → 0.0625
// Pebar = 0.5625 + 0.0625 = 0.625
// Per-subject Pi:
// unanimous correct: (16+0-4)/(4*3) = 12/12 = 1
// split 2/2: (4+4-4)/12 = 4/12 ≈ 0.33333
// Pbar = (5*1 + 5*0.33333) / 10 = 6.66667 / 10 = 0.66667
// κ = (0.66667 - 0.625) / (1 - 0.625) = 0.04167 / 0.375 = 0.11111
const matrix: JudgeResult[][] = [];
for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null));
for (let i = 0; i < 5; i++) {
matrix.push([
mkResult('correct', null, 'r', 'j0'),
mkResult('correct', null, 'r', 'j1'),
mkResult('incorrect', 'F3', 'r', 'j2'),
mkResult('incorrect', 'F3', 'r', 'j3'),
]);
}
const kappa = computeFleissKappa(matrix);
expect(kappa).toBeCloseTo(0.1111, 2); // tolerance ±0.01 per the brief
});
it('κ = 1 when every rating falls in a single category (expected = observed = 1)', () => {
// Degenerate edge case: all 40 ratings are `correct`. Pebar = 1.
// Implementation clamps to 1 (the (1-1)/(1-1) limit).
const matrix: JudgeResult[][] = [];
for (let i = 0; i < 10; i++) matrix.push(row(4, 'correct', null));
expect(computeFleissKappa(matrix)).toBe(1);
});
it('throws when rater counts are inconsistent across subjects', () => {
const matrix: JudgeResult[][] = [
row(4, 'correct', null),
row(3, 'correct', null), // wrong rater count
];
expect(() => computeFleissKappa(matrix)).toThrow(/constant rater count/);
});
it('returns 0 for a single-rater input (kappa undefined, convention 0)', () => {
const matrix: JudgeResult[][] = Array.from({ length: 10 }, () => row(1, 'correct', null));
expect(computeFleissKappa(matrix)).toBe(0);
});
it('returns 0 for an empty input', () => {
expect(computeFleissKappa([])).toBe(0);
});
});