diff --git a/packages/server/src/benchmarks/aggregate.ts b/packages/server/src/benchmarks/aggregate.ts deleted file mode 100644 index faec5ba..0000000 --- a/packages/server/src/benchmarks/aggregate.ts +++ /dev/null @@ -1,432 +0,0 @@ -/** - * Aggregate.ts — failure-mode distribution rollup for Stage 1 / Stage 2 - * preflight + Week 1/2 main runs. - * - * Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3 - * Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5 - * - * Consumes JSONL output from `benchmarks/harness` (one record per - * instance per cell) and emits a structured report covering: - * 1. Per-cell verdict distribution + weighted quality score - * 2. Per-LoCoMo-category distribution (single/multi-hop/temporal/ - * open-ended) with a hallucination rate flag for PM review - * 3. Cross-cell delta matrix (full-context vs raw: correct / F4 / - * F1 lift per taxonomy §5) - * 4. Judge cost summary when the run included judge calls - * - * Input shape: each JSONL line has the fields declared in - * `benchmarks/harness/src/types.ts` (optional `judge_verdict`, - * `judge_failure_mode`, `model_answer`, etc.). Pre-judge records — rows - * without `judge_verdict` — are counted under an `unjudged` bucket so - * partial-run reports stay honest instead of silently skipping data. - */ - -export type Verdict6 = - | 'correct' - | 'F1_abstain' - | 'F2_partial' - | 'F3_incorrect' - | 'F4_hallucinated' - | 'F5_offtopic' - | 'unjudged'; - -export type CellName = 'raw' | 'filtered' | 'compressed' | 'full-context'; -const CELL_NAMES: readonly CellName[] = ['raw', 'filtered', 'compressed', 'full-context']; - -/** Categories match `preflight-locomo-50.json` `_meta.locomo_category_map` - * minus the excluded adversarial bucket. Unknown categories fall into - * `other` so the report still tallies them instead of dropping. */ -export type LocomoCategory = 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended' | 'other'; -const LOCOMO_CATEGORIES: readonly LocomoCategory[] = [ - 'single-hop', 'multi-hop', 'temporal', 'open-ended', 'other', -]; - -/** Inputs can carry either the explicit category column or the - * pre-lock instance_id (`locomo__q`) from which - * we can recover the category via a sibling lookup. This module only - * consumes the already-tagged JsonlRecord; category recovery lives in - * the caller (harness runner attaches `category` when loading from - * the sample lock). */ -export interface JudgedJsonlRecord { - turnId: string; - cell: string; - instance_id: string; - model: string; - seed: number; - accuracy: number; - p50_latency_ms: number; - p95_latency_ms: number; - usd_per_query: number; - failure_mode: string | null; - model_answer?: string; - judge_verdict?: 'correct' | 'incorrect'; - judge_failure_mode?: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null; - judge_rationale?: string; - judge_model?: string; - judge_timestamp?: string; - judge_confidence?: number; - judge_ensemble?: Array<{ - model: string; - verdict: 'correct' | 'incorrect'; - failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null; - rationale?: string; - latency_ms?: number; - }>; - /** Optional category tag attached by the runner when the sample came - * from preflight-locomo-50.json. When absent, rows fall into - * `other` for the per-category rollup. */ - category?: LocomoCategory; -} - -/** Projection from the taxonomy §9 binary `verdict` + `failure_mode` - * slot pair to the 6-value Verdict6 enum the brief Task-1 spec asked - * for. Centralising this here means JsonlRecord keeps the canonical - * taxonomy shape while the aggregator surfaces the denser report - * form. */ -export function projectVerdict6(r: JudgedJsonlRecord): Verdict6 { - if (r.judge_verdict === undefined) return 'unjudged'; - if (r.judge_verdict === 'correct') return 'correct'; - switch (r.judge_failure_mode) { - case 'F1': return 'F1_abstain'; - case 'F2': return 'F2_partial'; - case 'F3': return 'F3_incorrect'; - case 'F4': return 'F4_hallucinated'; - case 'F5': return 'F5_offtopic'; - default: - // Contract violation — the taxonomy schema (judge module's Zod) - // prevents `incorrect` with a null failure_mode. If it sneaks - // through a malformed JSONL row, bucket as F3 by convention so - // the row doesn't silently drop. - return 'F3_incorrect'; - } -} - -export const VERDICT6_VALUES: readonly Verdict6[] = [ - 'correct', 'F1_abstain', 'F2_partial', 'F3_incorrect', 'F4_hallucinated', 'F5_offtopic', 'unjudged', -]; - -// ── Per-cell distribution + weighted score ───────────────────────────── - -export interface PerCellRow { - cell: string; - total: number; - counts: Record; - percents: Record; - /** Weighted quality score per taxonomy §5 rubric. `unjudged` is - * excluded from the denominator so the score is computed over the - * set of instances actually graded. */ - weightedScore: number; -} - -/** Taxonomy §5 coefficient table. Positive weights for correct / F2 - * (partial credit), zero for F1 (abstain neutral), negatives for - * F3 / F4 / F5. Exported so Task-3 unit tests can reproduce the - * hand-check number by name. */ -export const WEIGHTS: Record, number> = { - correct: 1.0, - F2_partial: 0.30, - F1_abstain: 0.0, - F3_incorrect: -0.15, - F4_hallucinated: -0.35, - F5_offtopic: -0.10, -}; - -function emptyVerdictMap(fill = 0): Record { - const out = {} as Record; - for (const v of VERDICT6_VALUES) out[v] = fill; - return out; -} - -export function perCellRollup(records: readonly JudgedJsonlRecord[]): PerCellRow[] { - const byCell = new Map(); - for (const r of records) { - const arr = byCell.get(r.cell) ?? []; - arr.push(r); - byCell.set(r.cell, arr); - } - const rows: PerCellRow[] = []; - // Emit in CELL_NAMES order when present, then any controls / extras. - const orderedCellKeys = [ - ...CELL_NAMES.filter(c => byCell.has(c)), - ...[...byCell.keys()].filter(c => !CELL_NAMES.includes(c as CellName)), - ]; - for (const cell of orderedCellKeys) { - const rowRecords = byCell.get(cell)!; - const counts = emptyVerdictMap(); - for (const r of rowRecords) counts[projectVerdict6(r)]++; - const total = rowRecords.length; - const judgedTotal = total - counts['unjudged']; - const percents = emptyVerdictMap(); - for (const v of VERDICT6_VALUES) { - percents[v] = total === 0 ? 0 : counts[v] / total; - } - let weightedScore = 0; - if (judgedTotal > 0) { - for (const v of VERDICT6_VALUES) { - if (v === 'unjudged') continue; - weightedScore += (counts[v] / judgedTotal) * WEIGHTS[v]; - } - } - rows.push({ cell, total, counts, percents, weightedScore }); - } - return rows; -} - -// ── Per-LoCoMo-category distribution ─────────────────────────────────── - -export interface PerCategoryRow { - category: LocomoCategory; - total: number; - counts: Record; - percents: Record; - /** True when F4 (hallucinated) share exceeds 20% — flags the - * category for PM spot-review per brief §Task-3 rule 2. */ - hallucinationFlag: boolean; -} - -const HALLUCINATION_FLAG_THRESHOLD = 0.20; - -export function perCategoryRollup(records: readonly JudgedJsonlRecord[]): PerCategoryRow[] { - const byCat = new Map(); - for (const r of records) { - const cat = r.category ?? 'other'; - const arr = byCat.get(cat) ?? []; - arr.push(r); - byCat.set(cat, arr); - } - const rows: PerCategoryRow[] = []; - for (const cat of LOCOMO_CATEGORIES) { - const recs = byCat.get(cat) ?? []; - if (recs.length === 0) continue; - const counts = emptyVerdictMap(); - for (const r of recs) counts[projectVerdict6(r)]++; - const total = recs.length; - const percents = emptyVerdictMap(); - for (const v of VERDICT6_VALUES) { - percents[v] = total === 0 ? 0 : counts[v] / total; - } - rows.push({ - category: cat, - total, - counts, - percents, - hallucinationFlag: percents['F4_hallucinated'] > HALLUCINATION_FLAG_THRESHOLD, - }); - } - return rows; -} - -// ── Cross-cell delta matrix (full-context vs raw) ────────────────────── - -export interface CrossCellDelta { - verdict: Verdict6; - rawPercent: number; - fullContextPercent: number; - /** fullContext − raw. Positive = full-context raised this verdict's share; - * negative = full-context lowered it. The brief's headline expectation: - * `correct` delta > 0, `F4_hallucinated` delta < 0, `F1_abstain` - * may go either way (more abstains is sometimes an OK signal). */ - delta: number; -} - -export function crossCellDeltaMatrix(perCell: readonly PerCellRow[]): CrossCellDelta[] | null { - const raw = perCell.find(r => r.cell === 'raw'); - const full = perCell.find(r => r.cell === 'full-context'); - if (!raw || !full) return null; - const out: CrossCellDelta[] = []; - for (const v of VERDICT6_VALUES) { - out.push({ - verdict: v, - rawPercent: raw.percents[v], - fullContextPercent: full.percents[v], - delta: full.percents[v] - raw.percents[v], - }); - } - return out; -} - -// ── Cost summary ─────────────────────────────────────────────────────── - -export interface CostSummary { - totalUsd: number; - perCellUsd: Record; - judgeTotalUsd: number; - judgeCallCount: number; - /** Median judge call latency across the run; `null` when no judge - * call recorded a latency. */ - medianJudgeMs: number | null; - /** Brief §Task-3 rule 4: flag when total judge spend would exceed - * ~$20 projected for a full 4-cell × 50-instance run. Computed as - * `perInstanceJudgeUsd × 200`. */ - week1WarningProjectedUsd: number; - week1WarningFired: boolean; -} - -const WEEK1_PROJECTION_INSTANCE_COUNT = 200; // 4 cells × 50 instances - -export function costSummary(records: readonly JudgedJsonlRecord[]): CostSummary { - const perCellUsd: Record = {}; - let total = 0; - const judgeLatencies: number[] = []; - const judgeTotal = 0; - let judgeCalls = 0; - for (const r of records) { - total += r.usd_per_query; - perCellUsd[r.cell] = (perCellUsd[r.cell] ?? 0) + r.usd_per_query; - if (r.judge_ensemble) { - for (const entry of r.judge_ensemble) { - if (typeof entry.latency_ms === 'number') judgeLatencies.push(entry.latency_ms); - } - judgeCalls += r.judge_ensemble.length; - } else if (r.judge_verdict !== undefined) { - // Single-judge call — we don't store per-judge latency in the - // JsonlRecord shape (no judge_latency_ms field), so the median - // is computed only over ensemble entries that carry it. If no - // ensemble row is present, medianJudgeMs stays null. - judgeCalls += 1; - } - } - const judgedCount = records.filter(r => r.judge_verdict !== undefined).length; - // Per-instance judge spend is not stored directly — the aggregator - // doesn't know judge USD without the onCall sink. When the caller - // passes judgeTotalUsd separately (see buildReport), we populate - // this from the authoritative source. Here we return zero; buildReport - // overwrites with the real total. - const medianJudgeMs = judgeLatencies.length === 0 - ? null - : (() => { - const s = [...judgeLatencies].sort((a, b) => a - b); - const mid = Math.floor(s.length / 2); - return s.length % 2 === 0 ? (s[mid - 1] + s[mid]) / 2 : s[mid]; - })(); - const perInstanceJudgeUsd = judgedCount > 0 ? judgeTotal / judgedCount : 0; - const projected = perInstanceJudgeUsd * WEEK1_PROJECTION_INSTANCE_COUNT; - return { - totalUsd: total, - perCellUsd, - judgeTotalUsd: judgeTotal, - judgeCallCount: judgeCalls, - medianJudgeMs, - week1WarningProjectedUsd: projected, - week1WarningFired: projected > 20, - }; -} - -// ── Top-level report + markdown renderer ─────────────────────────────── - -export interface AggregateReport { - generatedAt: string; - totalRecords: number; - perCell: PerCellRow[]; - perCategory: PerCategoryRow[]; - crossCellDelta: CrossCellDelta[] | null; - cost: CostSummary; -} - -export function buildReport( - records: readonly JudgedJsonlRecord[], - opts: { judgeTotalUsd?: number } = {}, -): AggregateReport { - const perCell = perCellRollup(records); - const perCategory = perCategoryRollup(records); - const crossCellDelta = crossCellDeltaMatrix(perCell); - const cost = costSummary(records); - // Callers that run the harness end-to-end know the authoritative - // judge spend because judge-client emits `onCall` entries. Pass the - // aggregated total here so the report carries real dollars instead - // of the placeholder zero computed from JSONL alone. - if (opts.judgeTotalUsd !== undefined) { - cost.judgeTotalUsd = opts.judgeTotalUsd; - const judgedCount = records.filter(r => r.judge_verdict !== undefined).length; - const perInstance = judgedCount > 0 ? opts.judgeTotalUsd / judgedCount : 0; - cost.week1WarningProjectedUsd = perInstance * WEEK1_PROJECTION_INSTANCE_COUNT; - cost.week1WarningFired = cost.week1WarningProjectedUsd > 20; - } - return { - generatedAt: new Date().toISOString(), - totalRecords: records.length, - perCell, - perCategory, - crossCellDelta, - cost, - }; -} - -function pct(n: number): string { - return `${(n * 100).toFixed(1)}%`; -} - -function fmtUsd(n: number): string { - return `$${n.toFixed(n < 1 ? 6 : 2)}`; -} - -export function renderMarkdown(report: AggregateReport): string { - const lines: string[] = []; - lines.push('# Benchmark Aggregate Report'); - lines.push(''); - lines.push(`**Generated at:** ${report.generatedAt}`); - lines.push(`**Total records:** ${report.totalRecords}`); - lines.push(''); - lines.push('## Per-cell verdict distribution'); - lines.push(''); - lines.push('| Cell | Total | Correct | F1 abstain | F2 partial | F3 incorrect | F4 hallucinated | F5 off-topic | Unjudged | Weighted score |'); - lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|'); - for (const row of report.perCell) { - lines.push( - `| ${row.cell} | ${row.total} | ${row.counts.correct} (${pct(row.percents.correct)}) | ` + - `${row.counts.F1_abstain} (${pct(row.percents.F1_abstain)}) | ` + - `${row.counts.F2_partial} (${pct(row.percents.F2_partial)}) | ` + - `${row.counts.F3_incorrect} (${pct(row.percents.F3_incorrect)}) | ` + - `${row.counts.F4_hallucinated} (${pct(row.percents.F4_hallucinated)}) | ` + - `${row.counts.F5_offtopic} (${pct(row.percents.F5_offtopic)}) | ` + - `${row.counts.unjudged} (${pct(row.percents.unjudged)}) | ` + - `${row.weightedScore.toFixed(3)} |`, - ); - } - lines.push(''); - if (report.perCategory.length > 0) { - lines.push('## Per-LoCoMo-category distribution'); - lines.push(''); - lines.push('| Category | Total | Correct% | F4 (hallucinated)% | Flagged? |'); - lines.push('|---|---:|---:|---:|:---:|'); - for (const row of report.perCategory) { - lines.push( - `| ${row.category} | ${row.total} | ${pct(row.percents.correct)} | ` + - `${pct(row.percents.F4_hallucinated)} | ` + - `${row.hallucinationFlag ? '⚠️ PM review' : '✓'} |`, - ); - } - lines.push(''); - } - if (report.crossCellDelta) { - lines.push('## Full-context vs raw delta'); - lines.push(''); - lines.push('| Verdict | raw % | full-context % | Δ (full − raw) |'); - lines.push('|---|---:|---:|---:|'); - for (const r of report.crossCellDelta) { - const sign = r.delta > 0 ? '+' : ''; - lines.push(`| ${r.verdict} | ${pct(r.rawPercent)} | ${pct(r.fullContextPercent)} | ${sign}${pct(r.delta)} |`); - } - lines.push(''); - } - lines.push('## Cost summary'); - lines.push(''); - lines.push(`- Total cell spend: **${fmtUsd(report.cost.totalUsd)}**`); - lines.push(`- Judge spend: **${fmtUsd(report.cost.judgeTotalUsd)}** across ${report.cost.judgeCallCount} call(s)`); - if (report.cost.medianJudgeMs !== null) { - lines.push(`- Median judge latency: ${report.cost.medianJudgeMs.toFixed(0)} ms`); - } - if (Object.keys(report.cost.perCellUsd).length > 0) { - lines.push('- Per-cell cell spend:'); - for (const [cell, usd] of Object.entries(report.cost.perCellUsd)) { - lines.push(` - ${cell}: ${fmtUsd(usd)}`); - } - } - if (report.cost.week1WarningFired) { - lines.push( - `- ⚠️ **Week-1 scale-up warning:** projected judge spend for a full 4-cell × 50-instance run is ` + - `${fmtUsd(report.cost.week1WarningProjectedUsd)} (threshold $20). Flagged per brief §Task-3 rule 4.`, - ); - } - return lines.join('\n'); -} diff --git a/packages/server/src/benchmarks/judge/ensemble-tiebreak.ts b/packages/server/src/benchmarks/judge/ensemble-tiebreak.ts deleted file mode 100644 index 3a0ad33..0000000 --- a/packages/server/src/benchmarks/judge/ensemble-tiebreak.ts +++ /dev/null @@ -1,314 +0,0 @@ -/** - * Sprint 11 Task B2 — Tie-break policy for multi-vendor judge ensemble. - * - * Authority: - * - PM-Waggle-OS/briefs/2026-04-22-cc-sprint-11-kickoff.md §3 Track B B2 - * - PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md (LOCKED) - * - * Context — what this resolves: - * - * The Sprint 10 Task 2.2 ratified primary judge ensemble is a THREE-vendor - * panel (Anthropic Opus 4.7 + OpenAI GPT-5.4 + Google Gemini 3.1-Pro). Three - * votes means three possible distributions: - * - * - 3-0 consensus → trivial, no tie-break - * - 2-1 majority → majority wins, no tie-break - * - 1-1-1 split → three different verdicts, no primary majority - * - * On 1-1-1, we escalate to a FOURTH vendor from a lineage disjoint from the - * primary trio: xai/grok-4.20 per Marko's 2026-04-22 LOCK. After the 4th - * vote arrives we have four votes total: - * - * - 2-1-1 / 1-1-2 → plurality winner (2 votes) - * - 1-1-1-1 → four-way split → PM escalation - * - * The fourth vendor is SPECIFICALLY xai/grok-4.20 — not Sonnet 4.6 (which - * would give Anthropic 2-of-4 weight → homogeneous-bias risk — rejected per - * LOCK doc §2) and not Opus 4.7 (already Judge 1). - * - * Observability: pino structured log emits `tie_break.path` ∈ { 'none', - * 'majority', 'quadri-vendor', 'pm-escalation' } plus - * `tie_break.fourth_vendor_slug` on quadri paths (field is future-proof - * even though Sprint 11 scope is grok-4.20 only). - */ - -import type { JudgeResult } from './failure-mode-judge.js'; - -// ── Public types ─────────────────────────────────────────────────────────── - -/** - * A single judge vote. Reusing JudgeResult from failure-mode-judge.ts keeps - * the tie-break input shape aligned with what judgeEnsemble already produces. - */ -export type Vote = JudgeResult; - -export type TieBreakPath = 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation'; - -/** - * The canonical sentinel verdict returned on a 1-1-1-1 four-way split. - * Callers (aggregators, report generators) MUST treat this as an operator - * action signal, not as a normal verdict string. The shape is intentionally - * unambiguous so no accidental inclusion into recall/accuracy math. - */ -export const PM_ESCALATION_VERDICT = '__PM_ESCALATION__'; - -/** Default fourth vendor for Sprint 11 — per LOCK doc §1. */ -export const DEFAULT_FOURTH_VENDOR = 'xai/grok-4.20'; - -export interface TieBreakResult { - /** - * The resolved verdict string. Encoded as `|` - * matching the aggregation key used in computeMajority of failure-mode-judge.ts, - * so downstream consumers can uniformly decompose. - * On pm-escalation path: `__PM_ESCALATION__`. - */ - verdict: string; - path: TieBreakPath; - /** The full vote list after resolution — 3 votes for none/majority, 4 for quadri/pm-escalation. */ - votes: Vote[]; - /** The fourth vendor's vote when a quadri path was taken. Undefined on none/majority. */ - fourthVendorVote?: Vote; - /** The fourth-vendor slug that was used, for observability. Undefined on none/majority. */ - fourthVendorSlug?: string; -} - -export interface FourthVendorCallPayload { - /** The three primary votes that produced the 1-1-1 split. */ - primaryVotes: Vote[]; - /** Model slug to invoke — e.g. 'xai/grok-4.20'. */ - model: string; -} - -export type CallFourthVendor = (payload: FourthVendorCallPayload) => Promise; - -export interface TieBreakLogger { - info(event: string, fields: Record): void; - warn?(event: string, fields: Record): void; -} - -export interface ResolveTieBreakOptions { - /** - * Invoked only on 1-1-1 splits. Required when primaryVotes.length === 3 - * and the distribution is 1-1-1; optional otherwise. Tests mock this. - */ - callFourthVendor?: CallFourthVendor; - /** Override the fourth vendor slug. Defaults to `DEFAULT_FOURTH_VENDOR`. */ - fourthVendorModel?: string; - /** Structured logger. Silently no-ops when omitted. */ - logger?: TieBreakLogger; -} - -// ── Internal helpers ─────────────────────────────────────────────────────── - -/** - * Canonical key for a vote — combines verdict + failure_mode so identical - * answers with different failure codes don't accidentally tie. - */ -function voteKey(v: Vote): string { - return `${v.verdict}|${v.failure_mode ?? 'NA'}`; -} - -function countByKey(votes: Vote[]): Map { - const tally = new Map(); - for (const v of votes) { - const key = voteKey(v); - const existing = tally.get(key); - if (existing) existing.count += 1; - else tally.set(key, { count: 1, first: v }); - } - return tally; -} - -function pluralityTop(tally: Map): { - topCount: number; - topKeys: string[]; -} { - let topCount = 0; - for (const { count } of tally.values()) if (count > topCount) topCount = count; - const topKeys: string[] = []; - for (const [key, { count }] of tally) if (count === topCount) topKeys.push(key); - return { topCount, topKeys }; -} - -// ── Public API ───────────────────────────────────────────────────────────── - -/** - * Resolve a multi-vendor judge ensemble vote. - * - * Input shape: - * - 3 primary votes (Sprint 10 Task 2.2 trio) — the normal call path. - * - 4 votes also accepted, in which case the function treats the vector - * as already-quadri-resolved and returns the plurality winner (or - * pm-escalation on 1-1-1-1). This shape exists for tests and for - * caller-side recomposition. - * - * On 1-1-1 with 3 primary votes, `callFourthVendor` MUST be provided. - * The function invokes it with the three primary votes and the canonical - * model slug, then recurses with the four-vote list. - * - * Returns: - * - `{ path: 'none' }` for 3-0 consensus (3 primary votes). - * - `{ path: 'majority' }` for 2-1 majority (3 votes) OR for a plurality - * win on 4 votes when the caller supplied the vector directly. - * - `{ path: 'quadri-vendor' }` when we called the 4th vendor ourselves - * and got a 2-1-1 or 1-1-2 distribution. - * - `{ path: 'pm-escalation' }` on a 1-1-1-1 four-way split; verdict is - * the `PM_ESCALATION_VERDICT` sentinel. - */ -export async function resolveTieBreak( - votes: Vote[], - options: ResolveTieBreakOptions = {}, -): Promise { - if (votes.length !== 3 && votes.length !== 4) { - throw new Error( - `resolveTieBreak: votes.length must be 3 (primary ensemble) or 4 (post-quadri). Got ${votes.length}.`, - ); - } - - const logger = options.logger; - - if (votes.length === 3) { - const tally = countByKey(votes); - const { topCount, topKeys } = pluralityTop(tally); - - // 3-0 consensus — one bucket holds all three. - if (topCount === 3) { - const entry = tally.get(topKeys[0])!; - logger?.info('tie_break', { path: 'none' as TieBreakPath, verdict: topKeys[0] }); - return { - verdict: topKeys[0], - path: 'none', - votes, - }; - } - - // 2-1 majority — one bucket holds two, another holds one. - if (topCount === 2 && topKeys.length === 1) { - logger?.info('tie_break', { path: 'majority' as TieBreakPath, verdict: topKeys[0] }); - return { - verdict: topKeys[0], - path: 'majority', - votes, - }; - } - - // 1-1-1 split — three buckets of one each. Escalate to fourth vendor. - if (topCount === 1 && topKeys.length === 3) { - if (!options.callFourthVendor) { - throw new Error( - 'resolveTieBreak: 1-1-1 three-way split requires a callFourthVendor implementation.', - ); - } - const fourthVendorModel = options.fourthVendorModel ?? DEFAULT_FOURTH_VENDOR; - - logger?.info('tie_break.quadri-vendor.invoke', { - path: 'quadri-vendor' as TieBreakPath, - fourth_vendor_slug: fourthVendorModel, - primary_keys: topKeys, - }); - - const fourthVote = await options.callFourthVendor({ - primaryVotes: votes, - model: fourthVendorModel, - }); - - // Recurse with four votes. The 4-vote branch returns quadri-vendor or pm-escalation. - const resolved = await resolveTieBreak([...votes, fourthVote], options); - - // Re-tag the path so "quadri-vendor" sticks on success cases and - // "pm-escalation" stays on 1-1-1-1 after escalation. - const path: TieBreakPath = - resolved.path === 'pm-escalation' ? 'pm-escalation' : 'quadri-vendor'; - - logger?.info('tie_break.quadri-vendor.resolved', { - path, - fourth_vendor_slug: fourthVendorModel, - verdict: resolved.verdict, - }); - - return { - verdict: resolved.verdict, - path, - votes: resolved.votes, - fourthVendorVote: fourthVote, - fourthVendorSlug: fourthVendorModel, - }; - } - - // Should be unreachable under correct input (any 3-vote distribution is - // 3-0, 2-1, or 1-1-1), but stay defensive. - throw new Error( - `resolveTieBreak: unexpected 3-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`, - ); - } - - // votes.length === 4 — post-quadri-vendor resolution path. - const tally = countByKey(votes); - const { topCount, topKeys } = pluralityTop(tally); - - // 1-1-1-1 four-way split — every bucket is one. PM escalation. - if (topCount === 1) { - logger?.info('tie_break', { - path: 'pm-escalation' as TieBreakPath, - verdict: PM_ESCALATION_VERDICT, - four_way_keys: topKeys, - }); - return { - verdict: PM_ESCALATION_VERDICT, - path: 'pm-escalation', - votes, - }; - } - - // Plurality winner: 2-1-1 / 1-1-2 (unique top). 2-2 ties with 4 votes are - // structurally impossible from our flow (3 primary votes make 2-2 - // impossible after the +1 fourth vote; the fourth vote always creates a - // unique plurality or a 1-1-1-1). Still handle 2-2 defensively by - // promoting to pm-escalation so no silent coin-flip ever lands in prod. - if (topCount === 2 && topKeys.length === 1) { - logger?.info('tie_break', { - path: 'majority' as TieBreakPath, - verdict: topKeys[0], - vote_count: 4, - }); - return { - verdict: topKeys[0], - path: 'majority', - votes, - }; - } - - if (topCount === 2 && topKeys.length >= 2) { - // 2-2 tie on 4 votes — not reachable from 1-1-1→quadri flow, but a - // caller may pass in a pre-constructed 4-vote vector. Escalate rather - // than coin-flip. - logger?.warn?.('tie_break.two-two-tie', { - path: 'pm-escalation' as TieBreakPath, - verdict: PM_ESCALATION_VERDICT, - two_two_keys: topKeys, - }); - return { - verdict: PM_ESCALATION_VERDICT, - path: 'pm-escalation', - votes, - }; - } - - if (topCount === 3 || topCount === 4) { - logger?.info('tie_break', { - path: 'majority' as TieBreakPath, - verdict: topKeys[0], - vote_count: 4, - }); - return { - verdict: topKeys[0], - path: 'majority', - votes, - }; - } - - // Truly unreachable given 4 input votes; defensive. - throw new Error( - `resolveTieBreak: unexpected 4-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`, - ); -} diff --git a/packages/server/src/benchmarks/judge/failure-mode-judge.ts b/packages/server/src/benchmarks/judge/failure-mode-judge.ts deleted file mode 100644 index eadb786..0000000 --- a/packages/server/src/benchmarks/judge/failure-mode-judge.ts +++ /dev/null @@ -1,383 +0,0 @@ -/** - * Failure-mode judge — scaffold. - * - * Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4 - * Spec: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §4 (prompt), - * §6 (ensemble), §8 (inter-judge agreement / Fleiss' kappa thresholds) - * LOCKED: PM-Waggle-OS/decisions/2026-04-20-failure-mode-oq-resolutions-locked.md - * - * This module is deliberately *not wired* into the harness runner in this - * sprint (explicitly out of scope per the brief). It exists so that Sprint 9 - * can drop in the judge call on top of a tested foundation. - * - * Design decisions worth knowing: - * - * — Zod validates strictly. The Step-3 schema contract between the model - * and us is enforced with a `.refine()` that captures the two - * cross-field invariants from the spec §4: - * verdict === 'correct' ⇒ failure_mode MUST be null - * verdict === 'incorrect' ⇒ failure_mode MUST be one of F1..F5 - * Any violation → retry once with the reminder → JudgeParseError. - * - * — JSON extraction is best-effort. Models often wrap JSON in ```json - * fences, leading prose, or trailing chatter. We strip the obvious - * wrappers before parsing; if both attempts fail we surface the raw - * response in the thrown error so a debugger can reproduce. - * - * — Ensemble tie-break is explicit: on a 2-2 split the ensemble uses the - * first judge in the `judgeModels` list. By convention the caller puts - * Sonnet first so "2-2 tie broken by Sonnet" holds per the spec §6. - * - * — Fleiss' kappa is the standard multi-rater formulation (Fleiss 1971). - * Raters per subject must be constant; we throw if any subject has a - * different rater count, since the formula is undefined otherwise. - * Categories are the 6-class vocabulary from §8 (`correct` treated as - * a separate class from F1..F5 — the same convention §8 specifies). - */ - -import { z } from 'zod'; - -// ── Public types ─────────────────────────────────────────────────────── - -export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5'; -export type Verdict = 'correct' | 'incorrect'; - -export interface JudgeResult { - verdict: Verdict; - failure_mode: null | FailureMode; - rationale: string; - judge_model: string; -} - -export interface LlmClient { - /** One-shot completion. Returns the raw text produced by the judge LLM. */ - complete(prompt: string): Promise; -} - -export class JudgeParseError extends Error { - readonly lastResponse: string; - readonly lastParseError?: string; - readonly judgeModel: string; - constructor(message: string, details: { lastResponse: string; judgeModel: string; lastParseError?: string }) { - super(message); - this.name = 'JudgeParseError'; - this.lastResponse = details.lastResponse; - this.judgeModel = details.judgeModel; - this.lastParseError = details.lastParseError; - } -} - -// ── Zod schema (enforces spec §4 Step-3 contract) ────────────────────── - -const judgeSchema = z - .object({ - verdict: z.enum(['correct', 'incorrect']), - failure_mode: z.union([z.null(), z.enum(['F1', 'F2', 'F3', 'F4', 'F5'])]), - rationale: z.string().min(1, 'rationale must be a non-empty sentence'), - }) - .strict() - .refine( - r => (r.verdict === 'correct' ? r.failure_mode === null : r.failure_mode !== null), - { - message: - 'verdict/failure_mode invariant violated: ' + - 'verdict="correct" requires failure_mode=null; verdict="incorrect" requires one of F1..F5', - }, - ); - -// ── Prompt builder (exact text from strategy §4) ─────────────────────── - -/** Exact judge prompt text from `strategy/2026-04-20-failure-mode-taxonomy.md` §4. - * Do not modify. The spec requires cell-independent scoring — changing the - * prompt between cells, or even between invocations, invalidates the ensemble. */ -export function buildJudgePrompt(params: { - question: string; - groundTruth: string; - contextExcerpt: string; - modelAnswer: string; -}): string { - return [ - "You are evaluating whether an LLM's answer is correct against ground truth.", - '', - '## Question', - params.question, - '', - '## Ground-truth answer', - params.groundTruth, - '', - '## Ground-truth supporting context (excerpt shown to the model)', - params.contextExcerpt, - '', - "## Model's answer", - params.modelAnswer, - '', - '## Your task', - '', - "Step 1: Determine if the model's answer is correct.", - '- "correct" means the model\'s answer contains all required facts from ground truth, with no additional incorrect claims.', - '- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.', - '- Extra detail is acceptable ONLY if it is factually correct.', - '', - 'Step 2: If incorrect, assign exactly one failure mode using this decision tree:', - '', - '1. Does the model explicitly refuse or say it does not know? → F1 (ABSTAIN)', - '2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? → F5 (OFF-TOPIC)', - '3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? → F4 (HALLUCINATED)', - '4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? → F2 (PARTIAL)', - '5. Otherwise (model states facts derived from the context but gets them wrong): → F3 (INCORRECT)', - '', - 'Step 3: Return JSON only, no prose, in this exact schema:', - '', - '{', - ' "verdict": "correct" | "incorrect",', - ' "failure_mode": null | "F1" | "F2" | "F3" | "F4" | "F5",', - ' "rationale": "one sentence explaining the verdict"', - '}', - '', - 'If verdict is "correct", failure_mode MUST be null.', - 'If verdict is "incorrect", failure_mode MUST be one of F1-F5.', - ].join('\n'); -} - -/** Retry reminder prepended to the original prompt when the first response - * fails to parse. Text is verbatim from the Task-4 acceptance spec. */ -export const RETRY_REMINDER = - 'Your previous response was not valid JSON. Return only the JSON object, no prose.'; - -// ── JSON extraction ──────────────────────────────────────────────────── - -/** Best-effort extraction of the JSON body from a model response. Handles: - * - bare JSON - * - `{ ... }` embedded in prose (first `{` to matching last `}`) - * - markdown code fences ```json ... ``` or ``` ... ``` */ -export function extractJsonBody(raw: string): string | null { - if (!raw) return null; - const trimmed = raw.trim(); - // Code-fence strip first. - const fenceMatch = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i); - const afterFence = fenceMatch ? fenceMatch[1].trim() : trimmed; - if (afterFence.startsWith('{') && afterFence.endsWith('}')) return afterFence; - // Fallback — greedy `{ ... }` scan, stopping at the last `}`. - const first = afterFence.indexOf('{'); - const last = afterFence.lastIndexOf('}'); - if (first >= 0 && last > first) return afterFence.slice(first, last + 1); - return null; -} - -function tryParse(raw: string): { ok: true; value: JudgeResult } | { ok: false; error: string } { - const body = extractJsonBody(raw); - if (!body) return { ok: false, error: 'no JSON object found in response' }; - let parsedJson: unknown; - try { - parsedJson = JSON.parse(body); - } catch (err) { - return { ok: false, error: `JSON.parse failed: ${(err as Error).message}` }; - } - const result = judgeSchema.safeParse(parsedJson); - if (!result.success) { - return { ok: false, error: `schema validation failed: ${result.error.issues.map(i => i.message).join('; ')}` }; - } - return { - ok: true, - value: { - verdict: result.data.verdict, - failure_mode: result.data.failure_mode, - rationale: result.data.rationale, - judge_model: '', // filled by judgeAnswer from params.judgeModel - }, - }; -} - -// ── Public API ───────────────────────────────────────────────────────── - -export async function judgeAnswer(params: { - question: string; - groundTruth: string; - contextExcerpt: string; - modelAnswer: string; - judgeModel: string; - llmClient: LlmClient; -}): Promise { - const prompt = buildJudgePrompt({ - question: params.question, - groundTruth: params.groundTruth, - contextExcerpt: params.contextExcerpt, - modelAnswer: params.modelAnswer, - }); - - const first = await params.llmClient.complete(prompt); - const firstParsed = tryParse(first); - if (firstParsed.ok) { - return { ...firstParsed.value, judge_model: params.judgeModel }; - } - - // Retry once with the reminder prefixed. A fresh LLM call — callers can - // wire conversation-state preservation later; spec §4 does not require it. - const retryPrompt = `${RETRY_REMINDER}\n\n${prompt}`; - const second = await params.llmClient.complete(retryPrompt); - const secondParsed = tryParse(second); - if (secondParsed.ok) { - return { ...secondParsed.value, judge_model: params.judgeModel }; - } - - throw new JudgeParseError( - `Judge ${params.judgeModel} produced unparseable output after one retry`, - { - lastResponse: second, - judgeModel: params.judgeModel, - lastParseError: secondParsed.error, - }, - ); -} - -export async function judgeEnsemble(params: { - question: string; - groundTruth: string; - contextExcerpt: string; - modelAnswer: string; - judgeModels: string[]; - llmClients: Map; -}): Promise<{ ensemble: JudgeResult[]; majority: JudgeResult; fleissKappa: number }> { - if (params.judgeModels.length === 0) { - throw new Error('judgeEnsemble requires at least one judgeModel'); - } - - const results: JudgeResult[] = []; - for (const model of params.judgeModels) { - const client = params.llmClients.get(model); - if (!client) throw new Error(`judgeEnsemble: no LlmClient registered for model ${model}`); - const result = await judgeAnswer({ - question: params.question, - groundTruth: params.groundTruth, - contextExcerpt: params.contextExcerpt, - modelAnswer: params.modelAnswer, - judgeModel: model, - llmClient: client, - }); - results.push(result); - } - - const majority = computeMajority(results, params.judgeModels[0]); - // For a single-subject ensemble, Fleiss' kappa is computed over the same - // one-subject matrix — degenerate but well-defined (NaN gets clamped to 0 - // when the subject has only one category). In production, this function - // is typically called per-instance and kappa is aggregated at the batch - // layer; we expose the single-subject value for diagnostic continuity. - const fleissKappa = computeFleissKappa([results]); - - return { ensemble: results, majority, fleissKappa }; -} - -function computeMajority(results: readonly JudgeResult[], tieBreakerModel: string): JudgeResult { - // Count verdict × failure_mode pairs (null collapses to 'NA' for keying). - const tally = new Map(); - const first = new Map(); - for (const r of results) { - const key = `${r.verdict}|${r.failure_mode ?? 'NA'}`; - tally.set(key, (tally.get(key) ?? 0) + 1); - if (!first.has(key)) first.set(key, r); - } - let winnerKey = ''; - let winnerCount = -1; - for (const [key, count] of tally) { - if (count > winnerCount) { - winnerKey = key; - winnerCount = count; - } - } - // Tie detection: if any other key has equal count, the tie-breaker model - // wins — its verdict becomes the majority. - const tied = Array.from(tally.entries()).filter(([, c]) => c === winnerCount); - if (tied.length > 1) { - const tb = results.find(r => r.judge_model === tieBreakerModel); - if (tb) return tb; - // Fallback: keep the first deterministic winner (stable map iteration). - } - const winner = first.get(winnerKey); - if (!winner) { - // Unreachable under normal flow; defensive. - return results[0]; - } - return { - verdict: winner.verdict, - failure_mode: winner.failure_mode, - rationale: winner.rationale, - judge_model: 'ensemble_majority', - }; -} - -// ── Fleiss' kappa ────────────────────────────────────────────────────── - -/** The 6-class rating space used by Fleiss' kappa: correct + F1..F5. - * §8: "standardni Fleiss' kappa preko 4 raters × N instanci × 6 klasa". */ -const KAPPA_CATEGORIES = ['correct', 'F1', 'F2', 'F3', 'F4', 'F5'] as const; -type KappaCategory = typeof KAPPA_CATEGORIES[number]; - -function categoryOf(r: JudgeResult): KappaCategory { - if (r.verdict === 'correct') return 'correct'; - return r.failure_mode ?? 'F3'; // should never hit fallback — schema guard -} - -/** Computes Fleiss' kappa on an N-subjects × n-raters matrix. - * - * Input shape: `ratings[i]` is the list of judge results for subject i — - * all subjects must have the same number of raters. Categories are the - * fixed 6-class vocabulary. - * - * Returns: κ in [-1, 1]. If all subjects are unanimous on the same - * category (expected-agreement = 1), returns 1 (the (1-1)/(1-1) limit). - */ -export function computeFleissKappa(ratings: readonly (readonly JudgeResult[])[]): number { - if (ratings.length === 0) return 0; - const n = ratings[0].length; - if (n < 2) { - // Fleiss' kappa is undefined for a single rater. Degenerate → 0. - return 0; - } - for (const row of ratings) { - if (row.length !== n) { - throw new Error(`Fleiss' kappa requires a constant rater count; saw ${row.length} and ${n}`); - } - } - - const N = ratings.length; - const k = KAPPA_CATEGORIES.length; - const catIndex: Record = { correct: 0, F1: 1, F2: 2, F3: 3, F4: 4, F5: 5 }; - - // nij[i][j] = count of raters who assigned subject i to category j - const nij: number[][] = Array.from({ length: N }, () => new Array(k).fill(0)); - for (let i = 0; i < N; i++) { - for (const r of ratings[i]) { - const col = catIndex[categoryOf(r)]; - nij[i][col] += 1; - } - } - - // Pj = (1/(N*n)) * sum_i nij[i][j] - const Pj: number[] = new Array(k).fill(0); - for (let j = 0; j < k; j++) { - let sum = 0; - for (let i = 0; i < N; i++) sum += nij[i][j]; - Pj[j] = sum / (N * n); - } - - // Pi = (1/(n(n-1))) * ( sum_j nij[i][j]^2 - n ) - let PbarSum = 0; - for (let i = 0; i < N; i++) { - let sqSum = 0; - for (let j = 0; j < k; j++) sqSum += nij[i][j] * nij[i][j]; - const Pi = (sqSum - n) / (n * (n - 1)); - PbarSum += Pi; - } - const Pbar = PbarSum / N; - - // Pebar = sum_j Pj^2 - let Pebar = 0; - for (let j = 0; j < k; j++) Pebar += Pj[j] * Pj[j]; - - if (Pebar >= 1 - 1e-12) { - // Every rating in one category → expected = observed = 1 → κ = 1 by convention. - return 1; - } - return (Pbar - Pebar) / (1 - Pebar); -} diff --git a/packages/server/tests/benchmarks/aggregate.test.ts b/packages/server/tests/benchmarks/aggregate.test.ts deleted file mode 100644 index a2a095b..0000000 --- a/packages/server/tests/benchmarks/aggregate.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -/** - * aggregate.ts unit tests (Sprint 9 Task 3). - * - * Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3 §Acceptance - * Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5 - * - * Synthetic 12-instance JSONL fixture: 3 cells × 4 verdicts. Tests - * assert: - * - per-cell count table matches hand-computed rows - * - weighted score matches hand-computed number (see §2 below) - * - per-category rollup surfaces the hallucination flag at the right threshold - * - cross-cell delta matrix populates full-context − raw direction - * - cost summary sums correctly + Week-1 projection flag fires at the threshold - * - markdown renderer produces parseable output + contains every row - */ - -import { describe, it, expect } from 'vitest'; -import { - buildReport, - perCellRollup, - perCategoryRollup, - crossCellDeltaMatrix, - costSummary, - projectVerdict6, - renderMarkdown, - WEIGHTS, - VERDICT6_VALUES, - type JudgedJsonlRecord, -} from '../../src/benchmarks/aggregate.ts'; - -// ── Fixture builder ───────────────────────────────────────────────────── - -function mkRecord( - turnId: string, - cell: string, - verdict: 'correct' | 'incorrect' | 'unjudged', - failureMode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null, - category: JudgedJsonlRecord['category'] = 'single-hop', - usd = 0.001, -): JudgedJsonlRecord { - const rec: JudgedJsonlRecord = { - turnId, - cell, - instance_id: `i_${turnId}`, - model: 'qwen3.6-35b-a3b', - seed: 42, - accuracy: verdict === 'correct' ? 1 : 0, - p50_latency_ms: 800, - p95_latency_ms: 1200, - usd_per_query: usd, - failure_mode: null, - category, - }; - if (verdict !== 'unjudged') { - rec.judge_verdict = verdict; - rec.judge_failure_mode = failureMode; - rec.judge_rationale = 'test rationale'; - rec.judge_model = 'claude-sonnet-4-6'; - rec.judge_timestamp = '2026-04-21T12:00:00Z'; - } - return rec; -} - -/** 12-instance fixture — 4 each per cell, one of each verdict shape. - * Hand-computed expectations are commented inline so a future diff - * catches silent drift in the rubric. */ -function fixture12(): JudgedJsonlRecord[] { - const recs: JudgedJsonlRecord[] = []; - const cells = ['raw', 'filtered', 'full-context']; - // Per cell: 1 correct + 1 F2 partial + 1 F3 incorrect + 1 F4 hallucinated. - for (const c of cells) { - recs.push(mkRecord(`${c}-A`, c, 'correct', null, 'single-hop')); - recs.push(mkRecord(`${c}-B`, c, 'incorrect', 'F2', 'multi-hop')); - recs.push(mkRecord(`${c}-C`, c, 'incorrect', 'F3', 'temporal')); - recs.push(mkRecord(`${c}-D`, c, 'incorrect', 'F4', 'open-ended')); - } - return recs; -} - -// ── projectVerdict6 ───────────────────────────────────────────────────── - -describe('projectVerdict6 — taxonomy §9 binary → brief §Task-1 6-value', () => { - it('maps correct → correct', () => { - expect(projectVerdict6(mkRecord('x', 'raw', 'correct', null))).toBe('correct'); - }); - it('maps incorrect + F1..F5 through their exact projections', () => { - const cases: Array<[typeof WEIGHTS extends Record ? K : never, string]> = [ - ['F1_abstain', 'F1'], - ['F2_partial', 'F2'], - ['F3_incorrect', 'F3'], - ['F4_hallucinated', 'F4'], - ['F5_offtopic', 'F5'], - ]; - for (const [expected, mode] of cases) { - expect(projectVerdict6( - mkRecord('x', 'raw', 'incorrect', mode as 'F1' | 'F2' | 'F3' | 'F4' | 'F5'), - )).toBe(expected); - } - }); - it('maps undefined judge_verdict → unjudged', () => { - expect(projectVerdict6(mkRecord('x', 'raw', 'unjudged', null))).toBe('unjudged'); - }); -}); - -// ── perCellRollup ─────────────────────────────────────────────────────── - -describe('perCellRollup', () => { - it('emits 3 rows, one per observed cell, in CELL_NAMES order', () => { - const rows = perCellRollup(fixture12()); - expect(rows).toHaveLength(3); - expect(rows.map(r => r.cell)).toEqual(['raw', 'filtered', 'full-context']); - }); - - it('computes per-cell counts + weighted score correctly (hand-check)', () => { - const rows = perCellRollup(fixture12()); - for (const row of rows) { - expect(row.total).toBe(4); - expect(row.counts.correct).toBe(1); - expect(row.counts.F2_partial).toBe(1); - expect(row.counts.F3_incorrect).toBe(1); - expect(row.counts.F4_hallucinated).toBe(1); - expect(row.counts.F1_abstain).toBe(0); - expect(row.counts.F5_offtopic).toBe(0); - expect(row.counts.unjudged).toBe(0); - // Weighted score = sum(percent × weight) over judged instances. - // judgedTotal=4. Each verdict is 1/4 = 0.25 of the cell. - // correct: 0.25 × 1.00 = 0.250 - // F2_partial: 0.25 × 0.30 = 0.075 - // F3_incorrect: 0.25 × -0.15 = -0.0375 - // F4_hallucinated: 0.25 × -0.35 = -0.0875 - // Total = 0.250 + 0.075 − 0.0375 − 0.0875 = 0.200 - expect(row.weightedScore).toBeCloseTo(0.200, 4); - } - }); - - it('honors WEIGHTS table exactly', () => { - // Rebuild the hand-computed number from WEIGHTS by name so a rubric - // edit in aggregate.ts forces this test to recompute + update the - // expectation — no silent coefficient drift. - const expected = - 0.25 * WEIGHTS.correct + - 0.25 * WEIGHTS.F2_partial + - 0.25 * WEIGHTS.F3_incorrect + - 0.25 * WEIGHTS.F4_hallucinated; - const rows = perCellRollup(fixture12()); - for (const row of rows) { - expect(row.weightedScore).toBeCloseTo(expected, 6); - } - }); - - it('treats unjudged rows as denominator-excluded for weightedScore', () => { - const mix = fixture12().slice(0, 4); // one cell worth - mix.push(mkRecord('unjudged-1', 'raw', 'unjudged', null)); - mix.push(mkRecord('unjudged-2', 'raw', 'unjudged', null)); - // Now raw has 4 judged + 2 unjudged. Weighted score denominator = 4, - // same as the baseline fixture — the 2 unjudged rows don't pull - // the score toward 0. - const rawRow = perCellRollup(mix).find(r => r.cell === 'raw')!; - expect(rawRow.total).toBe(6); - expect(rawRow.counts.unjudged).toBe(2); - expect(rawRow.weightedScore).toBeCloseTo(0.200, 4); - }); -}); - -// ── perCategoryRollup ────────────────────────────────────────────────── - -describe('perCategoryRollup', () => { - it('groups by category and computes percents per bucket', () => { - const rows = perCategoryRollup(fixture12()); - // fixture assigns 3 rows per category (one per cell × one verdict shape) - const cats = rows.map(r => r.category); - expect(cats).toContain('single-hop'); - expect(cats).toContain('multi-hop'); - expect(cats).toContain('temporal'); - expect(cats).toContain('open-ended'); - for (const row of rows) { - expect(row.total).toBe(3); - } - // open-ended has all F4 (hallucinated) rows — flag must fire. - const openEnded = rows.find(r => r.category === 'open-ended')!; - expect(openEnded.counts.F4_hallucinated).toBe(3); - expect(openEnded.hallucinationFlag).toBe(true); - // single-hop has all correct — flag off. - const singleHop = rows.find(r => r.category === 'single-hop')!; - expect(singleHop.counts.correct).toBe(3); - expect(singleHop.hallucinationFlag).toBe(false); - }); - - it('does not include categories that have zero rows', () => { - const rows = perCategoryRollup(fixture12()); - for (const row of rows) { - expect(row.total).toBeGreaterThan(0); - } - }); -}); - -// ── crossCellDeltaMatrix ─────────────────────────────────────────────── - -describe('crossCellDeltaMatrix', () => { - it('returns delta = full-context − raw for each verdict, preserving sign', () => { - // Build a case where full-context correct% > raw correct%. - const recs: JudgedJsonlRecord[] = [ - // raw: 1 correct, 3 incorrect-F4 → 25% correct, 75% F4 - mkRecord('r1', 'raw', 'correct', null, 'single-hop'), - mkRecord('r2', 'raw', 'incorrect', 'F4', 'single-hop'), - mkRecord('r3', 'raw', 'incorrect', 'F4', 'single-hop'), - mkRecord('r4', 'raw', 'incorrect', 'F4', 'single-hop'), - // full-context: 3 correct, 1 F4 → 75% correct, 25% F4 - mkRecord('f1', 'full-context', 'correct', null, 'single-hop'), - mkRecord('f2', 'full-context', 'correct', null, 'single-hop'), - mkRecord('f3', 'full-context', 'correct', null, 'single-hop'), - mkRecord('f4', 'full-context', 'incorrect', 'F4', 'single-hop'), - ]; - const perCell = perCellRollup(recs); - const delta = crossCellDeltaMatrix(perCell); - expect(delta).not.toBeNull(); - const correctDelta = delta!.find(d => d.verdict === 'correct')!; - expect(correctDelta.rawPercent).toBeCloseTo(0.25, 6); - expect(correctDelta.fullContextPercent).toBeCloseTo(0.75, 6); - expect(correctDelta.delta).toBeCloseTo(0.50, 6); - // F4 goes the other way. - const f4Delta = delta!.find(d => d.verdict === 'F4_hallucinated')!; - expect(f4Delta.delta).toBeCloseTo(-0.50, 6); - }); - - it('returns null when either raw or full-context is absent', () => { - const recs: JudgedJsonlRecord[] = [mkRecord('x', 'filtered', 'correct', null)]; - const perCell = perCellRollup(recs); - expect(crossCellDeltaMatrix(perCell)).toBeNull(); - }); -}); - -// ── costSummary + Week-1 projection threshold ────────────────────────── - -describe('costSummary', () => { - it('sums per-cell USD across records', () => { - const recs: JudgedJsonlRecord[] = [ - mkRecord('a', 'raw', 'correct', null, 'single-hop', 0.010), - mkRecord('b', 'raw', 'correct', null, 'single-hop', 0.020), - mkRecord('c', 'full-context', 'correct', null, 'single-hop', 0.050), - ]; - const cost = costSummary(recs); - expect(cost.totalUsd).toBeCloseTo(0.08, 6); - expect(cost.perCellUsd['raw']).toBeCloseTo(0.03, 6); - expect(cost.perCellUsd['full-context']).toBeCloseTo(0.05, 6); - }); - - it('buildReport overlays an authoritative judgeTotalUsd and recomputes the Week-1 projection', () => { - const recs = fixture12(); - // 12 records all judged; 12 × 4 cells × 50 instances scaling → 200 - // instances. Set judgeTotalUsd = $1.50 across 12 → per-instance = 0.125 - // → projected = 0.125 × 200 = $25 → above $20 → warning fires. - const report = buildReport(recs, { judgeTotalUsd: 1.50 }); - expect(report.cost.judgeTotalUsd).toBeCloseTo(1.50, 6); - expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(25.0, 6); - expect(report.cost.week1WarningFired).toBe(true); - }); - - it('Week-1 warning does not fire when projected stays under $20', () => { - const recs = fixture12(); - const report = buildReport(recs, { judgeTotalUsd: 0.60 }); - // per-instance = 0.05; projected = 0.05 × 200 = $10 - expect(report.cost.week1WarningProjectedUsd).toBeCloseTo(10.0, 6); - expect(report.cost.week1WarningFired).toBe(false); - }); -}); - -// ── Markdown renderer — smoke + snapshot-lite ────────────────────────── - -describe('renderMarkdown', () => { - it('produces parseable markdown with every cell row + every verdict column', () => { - const report = buildReport(fixture12()); - const md = renderMarkdown(report); - // Header structure - expect(md).toContain('# Benchmark Aggregate Report'); - expect(md).toContain('## Per-cell verdict distribution'); - expect(md).toContain('## Per-LoCoMo-category distribution'); - expect(md).toContain('## Cost summary'); - // Cells in table - for (const cell of ['raw', 'filtered', 'full-context']) { - expect(md).toContain(`| ${cell} |`); - } - // Verdict columns in the header row - for (const v of VERDICT6_VALUES) { - // VERDICT6_VALUES uses snake names that won't all appear literally - // in the header (e.g. "F1 abstain" vs "F1_abstain"). Assert on the - // base labels the renderer emits. - } - expect(md).toContain('Correct'); - expect(md).toContain('F1 abstain'); - expect(md).toContain('F4 hallucinated'); - expect(md).toContain('Weighted score'); - }); - - it('surfaces the hallucination-flag emoji on the flagged category', () => { - const md = renderMarkdown(buildReport(fixture12())); - // open-ended is flagged in the fixture (3/3 F4). Shape check via the - // brief's sentinel emoji + "PM review" string. - expect(md).toContain('⚠️ PM review'); - }); - - it('omits the Week-1 warning line when threshold not crossed', () => { - const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 0.40 })); - expect(md).not.toContain('Week-1 scale-up warning'); - }); - - it('emits the Week-1 warning line when threshold crossed', () => { - const md = renderMarkdown(buildReport(fixture12(), { judgeTotalUsd: 5.0 })); - expect(md).toContain('Week-1 scale-up warning'); - }); -}); diff --git a/packages/server/tests/benchmarks/ensemble-tiebreak.test.ts b/packages/server/tests/benchmarks/ensemble-tiebreak.test.ts deleted file mode 100644 index 8a0b110..0000000 --- a/packages/server/tests/benchmarks/ensemble-tiebreak.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Sprint 11 Task B2 — Tie-break policy unit tests. - * - * Authority: decisions/2026-04-22-tie-break-policy-locked.md (LOCKED) - * - * The four core scenarios from the brief §3 B2 + LOCK §0: - * - * 1. 3-0 consensus → path=none, no fourth-vendor call. - * 2. 2-1 majority → path=majority, no fourth-vendor call. - * 3. 1-1-1 split resolves → path=quadri-vendor, plurality verdict, - * fourth-vendor called once with correct payload. - * 4. 1-1-1 stays unresolved → path=pm-escalation (1-1-1-1 four-way), - * fourth-vendor called once; verdict is the - * PM_ESCALATION_VERDICT sentinel. - * - * Tests use a mocked CallFourthVendor function and an in-memory logger — - * no network, no API spend. The LIVE grok-4.20 smoke lives in the B2 exit - * ping's companion script and runs independently. - */ - -import { describe, it, expect, vi } from 'vitest'; -import { - resolveTieBreak, - DEFAULT_FOURTH_VENDOR, - PM_ESCALATION_VERDICT, - type Vote, - type TieBreakLogger, - type CallFourthVendor, -} from '../../src/benchmarks/judge/ensemble-tiebreak.js'; - -function vote(verdict: 'correct' | 'incorrect', failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null, model: string): Vote { - return { - verdict, - failure_mode, - rationale: `rationale from ${model}`, - judge_model: model, - }; -} - -function makeLogger(): { logger: TieBreakLogger; events: Array<{ event: string; fields: Record }> } { - const events: Array<{ event: string; fields: Record }> = []; - const logger: TieBreakLogger = { - info: (event, fields) => { - events.push({ event, fields }); - }, - warn: (event, fields) => { - events.push({ event, fields }); - }, - }; - return { logger, events }; -} - -describe('Sprint 11 B2 — resolveTieBreak', () => { - it('3-0 consensus returns path=none and does not call the fourth vendor', async () => { - const votes: Vote[] = [ - vote('correct', null, 'claude-opus-4-7'), - vote('correct', null, 'gpt-5.4-pro'), - vote('correct', null, 'gemini-3.1-pro'), - ]; - const callFourthVendor = vi.fn(); - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(votes, { callFourthVendor, logger }); - - expect(result.path).toBe('none'); - expect(result.verdict).toBe('correct|NA'); - expect(result.votes).toHaveLength(3); - expect(result.fourthVendorVote).toBeUndefined(); - expect(result.fourthVendorSlug).toBeUndefined(); - expect(callFourthVendor).not.toHaveBeenCalled(); - expect(events).toContainEqual( - expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'none' }) }), - ); - }); - - it('2-1 majority returns path=majority and does not call the fourth vendor', async () => { - const votes: Vote[] = [ - vote('correct', null, 'claude-opus-4-7'), - vote('correct', null, 'gpt-5.4-pro'), - vote('incorrect', 'F3', 'gemini-3.1-pro'), - ]; - const callFourthVendor = vi.fn(); - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(votes, { callFourthVendor, logger }); - - expect(result.path).toBe('majority'); - expect(result.verdict).toBe('correct|NA'); - expect(result.votes).toHaveLength(3); - expect(callFourthVendor).not.toHaveBeenCalled(); - expect(events).toContainEqual( - expect.objectContaining({ event: 'tie_break', fields: expect.objectContaining({ path: 'majority' }) }), - ); - }); - - it('1-1-1 split triggers quadri-vendor call on xai/grok-4.20 and resolves via plurality', async () => { - const primaryVotes: Vote[] = [ - vote('correct', null, 'claude-opus-4-7'), // bucket A: correct|NA - vote('incorrect', 'F3', 'gpt-5.4-pro'), // bucket B: incorrect|F3 - vote('incorrect', 'F4', 'gemini-3.1-pro'), // bucket C: incorrect|F4 - ]; - const grokVote = vote('correct', null, 'xai/grok-4.20'); // joins bucket A → plurality - const callFourthVendor = vi.fn().mockResolvedValue(grokVote); - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger }); - - expect(result.path).toBe('quadri-vendor'); - expect(result.verdict).toBe('correct|NA'); - expect(result.votes).toHaveLength(4); - expect(result.fourthVendorVote).toEqual(grokVote); - expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR); - - // Exactly one fourth-vendor call with the correct payload. - expect(callFourthVendor).toHaveBeenCalledTimes(1); - expect(callFourthVendor).toHaveBeenCalledWith({ - primaryVotes, - model: 'xai/grok-4.20', - }); - - // pino-shaped events: invoke + resolved. - const invokeEvent = events.find(e => e.event === 'tie_break.quadri-vendor.invoke'); - const resolvedEvent = events.find(e => e.event === 'tie_break.quadri-vendor.resolved'); - expect(invokeEvent).toBeDefined(); - expect(invokeEvent!.fields).toMatchObject({ - path: 'quadri-vendor', - fourth_vendor_slug: 'xai/grok-4.20', - }); - expect(resolvedEvent).toBeDefined(); - expect(resolvedEvent!.fields).toMatchObject({ - path: 'quadri-vendor', - fourth_vendor_slug: 'xai/grok-4.20', - verdict: 'correct|NA', - }); - }); - - it('1-1-1 split where the fourth vote is a fourth distinct bucket escalates to PM', async () => { - const primaryVotes: Vote[] = [ - vote('correct', null, 'claude-opus-4-7'), // A - vote('incorrect', 'F2', 'gpt-5.4-pro'), // B - vote('incorrect', 'F3', 'gemini-3.1-pro'), // C - ]; - // Fourth vote takes a fourth distinct failure mode → 1-1-1-1. - const grokVote = vote('incorrect', 'F4', 'xai/grok-4.20'); - const callFourthVendor = vi.fn().mockResolvedValue(grokVote); - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(primaryVotes, { callFourthVendor, logger }); - - expect(result.path).toBe('pm-escalation'); - expect(result.verdict).toBe(PM_ESCALATION_VERDICT); - expect(result.votes).toHaveLength(4); - expect(result.fourthVendorVote).toEqual(grokVote); - expect(result.fourthVendorSlug).toBe(DEFAULT_FOURTH_VENDOR); - - expect(callFourthVendor).toHaveBeenCalledTimes(1); - - // Logger emits a pm-escalation path event on the 4-vote recursive call. - const escalationEvent = events.find(e => - e.event === 'tie_break' && e.fields.path === 'pm-escalation', - ); - expect(escalationEvent).toBeDefined(); - expect(escalationEvent!.fields).toMatchObject({ - path: 'pm-escalation', - verdict: PM_ESCALATION_VERDICT, - }); - }); -}); - -describe('Sprint 11 B2 — resolveTieBreak defensive invariants', () => { - it('throws on invalid vote length (2 votes)', async () => { - const votes: Vote[] = [ - vote('correct', null, 'a'), - vote('correct', null, 'b'), - ]; - await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/); - }); - - it('throws on invalid vote length (5 votes)', async () => { - const votes: Vote[] = [ - vote('correct', null, 'a'), - vote('correct', null, 'b'), - vote('correct', null, 'c'), - vote('correct', null, 'd'), - vote('correct', null, 'e'), - ]; - await expect(resolveTieBreak(votes)).rejects.toThrow(/must be 3 .* or 4/); - }); - - it('1-1-1 without a callFourthVendor dep throws explicitly', async () => { - const votes: Vote[] = [ - vote('correct', null, 'a'), - vote('incorrect', 'F3', 'b'), - vote('incorrect', 'F4', 'c'), - ]; - await expect(resolveTieBreak(votes)).rejects.toThrow(/requires a callFourthVendor/); - }); - - it('caller-provided 4-vote vector resolves to plurality without extra calls', async () => { - // Test the caller-driven 4-vote shape — harness may pre-construct this. - const votes: Vote[] = [ - vote('correct', null, 'claude-opus-4-7'), - vote('correct', null, 'gpt-5.4-pro'), - vote('incorrect', 'F3', 'gemini-3.1-pro'), - vote('incorrect', 'F4', 'xai/grok-4.20'), - ]; - const callFourthVendor = vi.fn(); - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(votes, { callFourthVendor, logger }); - - expect(result.path).toBe('majority'); - expect(result.verdict).toBe('correct|NA'); - expect(callFourthVendor).not.toHaveBeenCalled(); - expect(events.some(e => e.fields.path === 'majority')).toBe(true); - }); - - it('caller-provided 2-2 tie on 4 votes escalates (defensive, never silent coin-flip)', async () => { - const votes: Vote[] = [ - vote('correct', null, 'a'), - vote('correct', null, 'b'), - vote('incorrect', 'F3', 'c'), - vote('incorrect', 'F3', 'd'), - ]; - const { logger, events } = makeLogger(); - - const result = await resolveTieBreak(votes, { logger }); - - expect(result.path).toBe('pm-escalation'); - expect(result.verdict).toBe(PM_ESCALATION_VERDICT); - expect(events.some(e => e.event === 'tie_break.two-two-tie')).toBe(true); - }); - - it('fourthVendorModel override is respected', async () => { - const votes: Vote[] = [ - vote('correct', null, 'a'), - vote('incorrect', 'F3', 'b'), - vote('incorrect', 'F4', 'c'), - ]; - const callFourthVendor = vi.fn().mockResolvedValue( - vote('correct', null, 'custom-slug/vendor-x'), - ); - - const result = await resolveTieBreak(votes, { - callFourthVendor, - fourthVendorModel: 'custom-slug/vendor-x', - }); - - expect(result.fourthVendorSlug).toBe('custom-slug/vendor-x'); - expect(callFourthVendor).toHaveBeenCalledWith({ - primaryVotes: votes, - model: 'custom-slug/vendor-x', - }); - }); -}); diff --git a/packages/server/tests/benchmarks/failure-mode-judge.test.ts b/packages/server/tests/benchmarks/failure-mode-judge.test.ts deleted file mode 100644 index 503972c..0000000 --- a/packages/server/tests/benchmarks/failure-mode-judge.test.ts +++ /dev/null @@ -1,340 +0,0 @@ -/** - * Failure-mode judge — unit tests. - * - * Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4 - * Module: packages/server/src/benchmarks/judge/failure-mode-judge.ts - * - * Coverage: - * - Valid JSON parse for all 5 failure modes (F1..F5) + the correct verdict. - * - Invalid-JSON → retry succeeds. - * - Invalid-JSON on both attempts → JudgeParseError. - * - 4-judge ensemble: 4-0 unanimous, 3-1 majority, 2-2 tie broken by - * the first model in `judgeModels` (Sonnet by convention). - * - Fleiss' kappa on hand-crafted 4×10 matrices with values computed by - * hand and verified against the formula (κ=1 for unanimous two-cluster, - * κ≈0.1111 for a known mixed matrix — both within ±0.01 tolerance). - * - * No network. No LLM calls. Pure unit tests against mock LlmClients. - */ - -import { describe, it, expect, beforeEach } from 'vitest'; -import { - buildJudgePrompt, - computeFleissKappa, - extractJsonBody, - judgeAnswer, - judgeEnsemble, - JudgeParseError, - RETRY_REMINDER, - type FailureMode, - type JudgeResult, - type LlmClient, - type Verdict, -} from '../../src/benchmarks/judge/failure-mode-judge.js'; - -// ── Test helpers ─────────────────────────────────────────────────────── - -class ScriptedLlmClient implements LlmClient { - readonly calls: string[] = []; - private readonly queue: Array; - constructor(responses: Array) { - this.queue = [...responses]; - } - async complete(prompt: string): Promise { - this.calls.push(prompt); - if (this.queue.length === 0) throw new Error('ScriptedLlmClient out of responses'); - const next = this.queue.shift()!; - if (next instanceof Error) throw next; - return next; - } -} - -function mkResult(verdict: Verdict, failure_mode: FailureMode | null, rationale: string, judge_model: string): JudgeResult { - return { verdict, failure_mode, rationale, judge_model }; -} - -// ── Prompt shape ─────────────────────────────────────────────────────── - -describe('buildJudgePrompt', () => { - it('interpolates all four required variables verbatim', () => { - const prompt = buildJudgePrompt({ - question: 'When did Caroline go to the support group?', - groundTruth: '7 May 2023', - contextExcerpt: 'Caroline: I went to a LGBTQ support group yesterday.', - modelAnswer: 'Unclear.', - }); - expect(prompt).toContain('## Question\nWhen did Caroline go to the support group?'); - expect(prompt).toContain('## Ground-truth answer\n7 May 2023'); - expect(prompt).toContain('## Ground-truth supporting context'); - expect(prompt).toContain("Caroline: I went to a LGBTQ support group yesterday."); - expect(prompt).toContain("## Model's answer\nUnclear."); - // Decision tree sanity — verifies the exact §4 text didn't drift. - expect(prompt).toContain('→ F1 (ABSTAIN)'); - expect(prompt).toContain('→ F5 (OFF-TOPIC)'); - expect(prompt).toContain('→ F4 (HALLUCINATED)'); - expect(prompt).toContain('→ F2 (PARTIAL)'); - expect(prompt).toContain('→ F3 (INCORRECT)'); - }); -}); - -// ── JSON extraction ──────────────────────────────────────────────────── - -describe('extractJsonBody', () => { - it('returns bare JSON unchanged', () => { - expect(extractJsonBody('{"verdict":"correct"}')).toBe('{"verdict":"correct"}'); - }); - it('strips markdown fences with `json` hint', () => { - const raw = '```json\n{"verdict":"correct"}\n```'; - expect(extractJsonBody(raw)).toBe('{"verdict":"correct"}'); - }); - it('strips bare markdown fences', () => { - const raw = '```\n{"verdict":"incorrect","failure_mode":"F3"}\n```'; - expect(extractJsonBody(raw)).toBe('{"verdict":"incorrect","failure_mode":"F3"}'); - }); - it('extracts the JSON object from prose', () => { - const raw = 'Here is my verdict: {"verdict":"correct","failure_mode":null,"rationale":"ok"}'; - expect(extractJsonBody(raw)).toBe('{"verdict":"correct","failure_mode":null,"rationale":"ok"}'); - }); - it('returns null when no object is present', () => { - expect(extractJsonBody('I cannot comply.')).toBeNull(); - }); -}); - -// ── judgeAnswer — happy path + failure modes ─────────────────────────── - -describe('judgeAnswer — valid JSON parse for all 5 failure modes + correct', () => { - const cases: Array<{ name: string; verdict: Verdict; failure_mode: FailureMode | null; rationale: string }> = [ - { name: 'correct', verdict: 'correct', failure_mode: null, rationale: 'All facts match the ground truth.' }, - { name: 'F1 abstain', verdict: 'incorrect', failure_mode: 'F1', rationale: 'Model explicitly refused to answer.' }, - { name: 'F2 partial', verdict: 'incorrect', failure_mode: 'F2', rationale: 'Model stated 2 of 3 required facts.' }, - { name: 'F3 incorrect', verdict: 'incorrect', failure_mode: 'F3', rationale: 'Model stated a wrong date derived from context.' }, - { name: 'F4 hallucinated', verdict: 'incorrect', failure_mode: 'F4', rationale: 'Model named a person not in the context.' }, - { name: 'F5 off-topic', verdict: 'incorrect', failure_mode: 'F5', rationale: 'Model answered a different question.' }, - ]; - - for (const c of cases) { - it(`parses ${c.name} and stamps the judge_model`, async () => { - const payload = JSON.stringify({ verdict: c.verdict, failure_mode: c.failure_mode, rationale: c.rationale }); - const client = new ScriptedLlmClient([payload]); - const result = await judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'claude-sonnet-4-6', llmClient: client, - }); - expect(result.verdict).toBe(c.verdict); - expect(result.failure_mode).toBe(c.failure_mode); - expect(result.rationale).toBe(c.rationale); - expect(result.judge_model).toBe('claude-sonnet-4-6'); - expect(client.calls).toHaveLength(1); - }); - } -}); - -describe('judgeAnswer — retry semantics', () => { - const validPayload = JSON.stringify({ verdict: 'correct', failure_mode: null, rationale: 'All facts match.' }); - - it('on invalid JSON, retries once with the reminder and returns the retry result', async () => { - const client = new ScriptedLlmClient([ - 'Sorry, I cannot produce structured output — here is a paragraph.', - validPayload, - ]); - const result = await judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'claude-sonnet-4-6', llmClient: client, - }); - expect(result.verdict).toBe('correct'); - expect(client.calls).toHaveLength(2); - // Retry prompt must begin with the exact reminder text from the spec. - expect(client.calls[1].startsWith(RETRY_REMINDER)).toBe(true); - }); - - it('on invalid JSON twice, throws JudgeParseError with the raw response attached', async () => { - const client = new ScriptedLlmClient([ - 'Still refusing to produce JSON.', - 'Nope, same here.', - ]); - await expect( - judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'gpt-5', llmClient: client, - }), - ).rejects.toBeInstanceOf(JudgeParseError); - - try { - await judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'gpt-5', llmClient: new ScriptedLlmClient(['bad1', 'bad2']), - }); - } catch (e) { - expect(e).toBeInstanceOf(JudgeParseError); - const err = e as JudgeParseError; - expect(err.judgeModel).toBe('gpt-5'); - expect(err.lastResponse).toBe('bad2'); - expect(err.lastParseError).toBeTruthy(); - } - }); - - it('rejects schema-valid JSON that violates the verdict/failure_mode invariant', async () => { - // verdict=correct with a non-null failure_mode — Step-3 contract violation. - const bad = JSON.stringify({ verdict: 'correct', failure_mode: 'F4', rationale: 'contradictory' }); - // Both attempts return the same bad shape — should throw. - const client = new ScriptedLlmClient([bad, bad]); - await expect( - judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'gemini-pro', llmClient: client, - }), - ).rejects.toBeInstanceOf(JudgeParseError); - }); - - it('accepts fenced JSON in the first attempt (no retry)', async () => { - const fenced = '```json\n' + JSON.stringify({ verdict: 'incorrect', failure_mode: 'F3', rationale: 'Wrong date.' }) + '\n```'; - const client = new ScriptedLlmClient([fenced]); - const result = await judgeAnswer({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModel: 'haiku', llmClient: client, - }); - expect(result.failure_mode).toBe('F3'); - expect(client.calls).toHaveLength(1); - }); -}); - -// ── judgeEnsemble — 4-judge aggregation ──────────────────────────────── - -describe('judgeEnsemble — 4-judge aggregation', () => { - function mkClientWith(verdict: Verdict, failure_mode: FailureMode | null, rationale: string): LlmClient { - return new ScriptedLlmClient([JSON.stringify({ verdict, failure_mode, rationale })]); - } - - const models = ['claude-sonnet-4-6', 'claude-haiku-4-5', 'gpt-5', 'gemini-pro']; - - it('4-0 unanimous → majority matches the unanimous verdict', async () => { - const clients = new Map(); - for (const m of models) clients.set(m, mkClientWith('correct', null, 'match')); - const result = await judgeEnsemble({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModels: models, llmClients: clients, - }); - expect(result.ensemble).toHaveLength(4); - expect(result.majority.verdict).toBe('correct'); - expect(result.majority.failure_mode).toBeNull(); - // κ = 1 when every rater agrees on the same class (and all 6 classes - // contribute 0 or 1 to the marginals → expected = observed = 1). - expect(result.fleissKappa).toBeCloseTo(1, 6); - }); - - it('3-1 majority → majority verdict wins, minority is recorded in ensemble', async () => { - const clients = new Map(); - clients.set(models[0], mkClientWith('incorrect', 'F3', 'A')); - clients.set(models[1], mkClientWith('incorrect', 'F3', 'B')); - clients.set(models[2], mkClientWith('incorrect', 'F3', 'C')); - clients.set(models[3], mkClientWith('incorrect', 'F4', 'D')); // minority — says hallucination - const result = await judgeEnsemble({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModels: models, llmClients: clients, - }); - expect(result.majority.verdict).toBe('incorrect'); - expect(result.majority.failure_mode).toBe('F3'); - const failureModes = result.ensemble.map(r => r.failure_mode); - expect(failureModes.filter(m => m === 'F3')).toHaveLength(3); - expect(failureModes.filter(m => m === 'F4')).toHaveLength(1); - }); - - it('2-2 tie is broken by the first model in judgeModels (Sonnet by convention)', async () => { - const clients = new Map(); - // Sonnet + Haiku say F2; GPT-5 + Gemini say F3. - clients.set(models[0], mkClientWith('incorrect', 'F2', 'sonnet')); - clients.set(models[1], mkClientWith('incorrect', 'F2', 'haiku')); - clients.set(models[2], mkClientWith('incorrect', 'F3', 'gpt-5')); - clients.set(models[3], mkClientWith('incorrect', 'F3', 'gemini')); - const result = await judgeEnsemble({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModels: models, llmClients: clients, - }); - // Sonnet wins the tie → F2. - expect(result.majority.failure_mode).toBe('F2'); - expect(result.majority.rationale).toBe('sonnet'); - expect(result.majority.judge_model).toBe('claude-sonnet-4-6'); - }); - - it('refuses when no client is registered for a judge model', async () => { - const clients = new Map(); - clients.set(models[0], mkClientWith('correct', null, 'ok')); - // Missing models[1..3] - await expect( - judgeEnsemble({ - question: 'q', groundTruth: 'gt', contextExcerpt: 'ctx', modelAnswer: 'ma', - judgeModels: models, llmClients: clients, - }), - ).rejects.toThrow(/no LlmClient registered/); - }); -}); - -// ── Fleiss' kappa ────────────────────────────────────────────────────── - -describe("computeFleissKappa on hand-crafted 4-judge × 10-subject matrices", () => { - function row(n: number, verdict: Verdict, failure_mode: FailureMode | null): JudgeResult[] { - return Array.from({ length: n }, (_, i) => mkResult(verdict, failure_mode, 'r', `j${i}`)); - } - - it('κ = 1 for two-cluster unanimous agreement (5 × correct / 5 × F3)', () => { - // 5 subjects: 4 raters all say correct. - // 5 subjects: 4 raters all say F3. - // Pbar = 1 (every subject unanimous). Pj(correct) = 0.5, Pj(F3) = 0.5. - // Pebar = 0.5 + 0.5 = 0.5 (treating the other 4 classes as 0). - // κ = (1 - 0.5) / (1 - 0.5) = 1. - const matrix: JudgeResult[][] = []; - for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null)); - for (let i = 0; i < 5; i++) matrix.push(row(4, 'incorrect', 'F3')); - const kappa = computeFleissKappa(matrix); - expect(kappa).toBeCloseTo(1, 6); - }); - - it('κ ≈ 0.1111 for 5 unanimous-correct + 5 split-2/2-correct/F3 subjects', () => { - // Hand-computed: n_correct = 5*4 + 5*2 = 30, n_F3 = 5*2 = 10. Total = 40. - // Pj(correct) = 30/40 = 0.75 → 0.5625 - // Pj(F3) = 10/40 = 0.25 → 0.0625 - // Pebar = 0.5625 + 0.0625 = 0.625 - // Per-subject Pi: - // unanimous correct: (16+0-4)/(4*3) = 12/12 = 1 - // split 2/2: (4+4-4)/12 = 4/12 ≈ 0.33333 - // Pbar = (5*1 + 5*0.33333) / 10 = 6.66667 / 10 = 0.66667 - // κ = (0.66667 - 0.625) / (1 - 0.625) = 0.04167 / 0.375 = 0.11111 - const matrix: JudgeResult[][] = []; - for (let i = 0; i < 5; i++) matrix.push(row(4, 'correct', null)); - for (let i = 0; i < 5; i++) { - matrix.push([ - mkResult('correct', null, 'r', 'j0'), - mkResult('correct', null, 'r', 'j1'), - mkResult('incorrect', 'F3', 'r', 'j2'), - mkResult('incorrect', 'F3', 'r', 'j3'), - ]); - } - const kappa = computeFleissKappa(matrix); - expect(kappa).toBeCloseTo(0.1111, 2); // tolerance ±0.01 per the brief - }); - - it('κ = 1 when every rating falls in a single category (expected = observed = 1)', () => { - // Degenerate edge case: all 40 ratings are `correct`. Pebar = 1. - // Implementation clamps to 1 (the (1-1)/(1-1) limit). - const matrix: JudgeResult[][] = []; - for (let i = 0; i < 10; i++) matrix.push(row(4, 'correct', null)); - expect(computeFleissKappa(matrix)).toBe(1); - }); - - it('throws when rater counts are inconsistent across subjects', () => { - const matrix: JudgeResult[][] = [ - row(4, 'correct', null), - row(3, 'correct', null), // wrong rater count - ]; - expect(() => computeFleissKappa(matrix)).toThrow(/constant rater count/); - }); - - it('returns 0 for a single-rater input (kappa undefined, convention 0)', () => { - const matrix: JudgeResult[][] = Array.from({ length: 10 }, () => row(1, 'correct', null)); - expect(computeFleissKappa(matrix)).toBe(0); - }); - - it('returns 0 for an empty input', () => { - expect(computeFleissKappa([])).toBe(0); - }); -});