chore(server): remove the memory benchmark judge scaffolding (P2.2)

packages/server/src/benchmarks held a LoCoMo failure-mode aggregator and
an LLM judge ensemble for the old memory engine benchmarks. Nothing in
the sidecar imports it and no npm script runs it; its inputs lived in
the root benchmarks/ directory that P2.2 already removed. Judge and eval
loops are also out of the community edition (AGENTS.md, rule 4).
This commit is contained in:
shumeiko.it
2026-10-24 16:39:23 +02:00
parent 76c7102123
commit 93d73be7b4
6 changed files with 0 additions and 2035 deletions

View File

@@ -1,432 +0,0 @@
/**
* Aggregate.ts — failure-mode distribution rollup for Stage 1 / Stage 2
* preflight + Week 1/2 main runs.
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md Task 3
* Rubric: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §5
*
* Consumes JSONL output from `benchmarks/harness` (one record per
* instance per cell) and emits a structured report covering:
* 1. Per-cell verdict distribution + weighted quality score
* 2. Per-LoCoMo-category distribution (single/multi-hop/temporal/
* open-ended) with a hallucination rate flag for PM review
* 3. Cross-cell delta matrix (full-context vs raw: correct / F4 /
* F1 lift per taxonomy §5)
* 4. Judge cost summary when the run included judge calls
*
* Input shape: each JSONL line has the fields declared in
* `benchmarks/harness/src/types.ts` (optional `judge_verdict`,
* `judge_failure_mode`, `model_answer`, etc.). Pre-judge records — rows
* without `judge_verdict` — are counted under an `unjudged` bucket so
* partial-run reports stay honest instead of silently skipping data.
*/
export type Verdict6 =
| 'correct'
| 'F1_abstain'
| 'F2_partial'
| 'F3_incorrect'
| 'F4_hallucinated'
| 'F5_offtopic'
| 'unjudged';
export type CellName = 'raw' | 'filtered' | 'compressed' | 'full-context';
const CELL_NAMES: readonly CellName[] = ['raw', 'filtered', 'compressed', 'full-context'];
/** Categories match `preflight-locomo-50.json` `_meta.locomo_category_map`
* minus the excluded adversarial bucket. Unknown categories fall into
* `other` so the report still tallies them instead of dropping. */
export type LocomoCategory = 'single-hop' | 'multi-hop' | 'temporal' | 'open-ended' | 'other';
const LOCOMO_CATEGORIES: readonly LocomoCategory[] = [
'single-hop', 'multi-hop', 'temporal', 'open-ended', 'other',
];
/** Inputs can carry either the explicit category column or the
* pre-lock instance_id (`locomo_<sample_id>_q<qindex>`) from which
* we can recover the category via a sibling lookup. This module only
* consumes the already-tagged JsonlRecord; category recovery lives in
* the caller (harness runner attaches `category` when loading from
* the sample lock). */
export interface JudgedJsonlRecord {
turnId: string;
cell: string;
instance_id: string;
model: string;
seed: number;
accuracy: number;
p50_latency_ms: number;
p95_latency_ms: number;
usd_per_query: number;
failure_mode: string | null;
model_answer?: string;
judge_verdict?: 'correct' | 'incorrect';
judge_failure_mode?: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
judge_rationale?: string;
judge_model?: string;
judge_timestamp?: string;
judge_confidence?: number;
judge_ensemble?: Array<{
model: string;
verdict: 'correct' | 'incorrect';
failure_mode: 'F1' | 'F2' | 'F3' | 'F4' | 'F5' | null;
rationale?: string;
latency_ms?: number;
}>;
/** Optional category tag attached by the runner when the sample came
* from preflight-locomo-50.json. When absent, rows fall into
* `other` for the per-category rollup. */
category?: LocomoCategory;
}
/** Projection from the taxonomy §9 binary `verdict` + `failure_mode`
* slot pair to the 6-value Verdict6 enum the brief Task-1 spec asked
* for. Centralising this here means JsonlRecord keeps the canonical
* taxonomy shape while the aggregator surfaces the denser report
* form. */
export function projectVerdict6(r: JudgedJsonlRecord): Verdict6 {
if (r.judge_verdict === undefined) return 'unjudged';
if (r.judge_verdict === 'correct') return 'correct';
switch (r.judge_failure_mode) {
case 'F1': return 'F1_abstain';
case 'F2': return 'F2_partial';
case 'F3': return 'F3_incorrect';
case 'F4': return 'F4_hallucinated';
case 'F5': return 'F5_offtopic';
default:
// Contract violation — the taxonomy schema (judge module's Zod)
// prevents `incorrect` with a null failure_mode. If it sneaks
// through a malformed JSONL row, bucket as F3 by convention so
// the row doesn't silently drop.
return 'F3_incorrect';
}
}
export const VERDICT6_VALUES: readonly Verdict6[] = [
'correct', 'F1_abstain', 'F2_partial', 'F3_incorrect', 'F4_hallucinated', 'F5_offtopic', 'unjudged',
];
// ── Per-cell distribution + weighted score ─────────────────────────────
export interface PerCellRow {
cell: string;
total: number;
counts: Record<Verdict6, number>;
percents: Record<Verdict6, number>;
/** Weighted quality score per taxonomy §5 rubric. `unjudged` is
* excluded from the denominator so the score is computed over the
* set of instances actually graded. */
weightedScore: number;
}
/** Taxonomy §5 coefficient table. Positive weights for correct / F2
* (partial credit), zero for F1 (abstain neutral), negatives for
* F3 / F4 / F5. Exported so Task-3 unit tests can reproduce the
* hand-check number by name. */
export const WEIGHTS: Record<Exclude<Verdict6, 'unjudged'>, number> = {
correct: 1.0,
F2_partial: 0.30,
F1_abstain: 0.0,
F3_incorrect: -0.15,
F4_hallucinated: -0.35,
F5_offtopic: -0.10,
};
function emptyVerdictMap(fill = 0): Record<Verdict6, number> {
const out = {} as Record<Verdict6, number>;
for (const v of VERDICT6_VALUES) out[v] = fill;
return out;
}
export function perCellRollup(records: readonly JudgedJsonlRecord[]): PerCellRow[] {
const byCell = new Map<string, JudgedJsonlRecord[]>();
for (const r of records) {
const arr = byCell.get(r.cell) ?? [];
arr.push(r);
byCell.set(r.cell, arr);
}
const rows: PerCellRow[] = [];
// Emit in CELL_NAMES order when present, then any controls / extras.
const orderedCellKeys = [
...CELL_NAMES.filter(c => byCell.has(c)),
...[...byCell.keys()].filter(c => !CELL_NAMES.includes(c as CellName)),
];
for (const cell of orderedCellKeys) {
const rowRecords = byCell.get(cell)!;
const counts = emptyVerdictMap();
for (const r of rowRecords) counts[projectVerdict6(r)]++;
const total = rowRecords.length;
const judgedTotal = total - counts['unjudged'];
const percents = emptyVerdictMap();
for (const v of VERDICT6_VALUES) {
percents[v] = total === 0 ? 0 : counts[v] / total;
}
let weightedScore = 0;
if (judgedTotal > 0) {
for (const v of VERDICT6_VALUES) {
if (v === 'unjudged') continue;
weightedScore += (counts[v] / judgedTotal) * WEIGHTS[v];
}
}
rows.push({ cell, total, counts, percents, weightedScore });
}
return rows;
}
// ── Per-LoCoMo-category distribution ───────────────────────────────────
export interface PerCategoryRow {
category: LocomoCategory;
total: number;
counts: Record<Verdict6, number>;
percents: Record<Verdict6, number>;
/** True when F4 (hallucinated) share exceeds 20% — flags the
* category for PM spot-review per brief §Task-3 rule 2. */
hallucinationFlag: boolean;
}
const HALLUCINATION_FLAG_THRESHOLD = 0.20;
export function perCategoryRollup(records: readonly JudgedJsonlRecord[]): PerCategoryRow[] {
const byCat = new Map<LocomoCategory, JudgedJsonlRecord[]>();
for (const r of records) {
const cat = r.category ?? 'other';
const arr = byCat.get(cat) ?? [];
arr.push(r);
byCat.set(cat, arr);
}
const rows: PerCategoryRow[] = [];
for (const cat of LOCOMO_CATEGORIES) {
const recs = byCat.get(cat) ?? [];
if (recs.length === 0) continue;
const counts = emptyVerdictMap();
for (const r of recs) counts[projectVerdict6(r)]++;
const total = recs.length;
const percents = emptyVerdictMap();
for (const v of VERDICT6_VALUES) {
percents[v] = total === 0 ? 0 : counts[v] / total;
}
rows.push({
category: cat,
total,
counts,
percents,
hallucinationFlag: percents['F4_hallucinated'] > HALLUCINATION_FLAG_THRESHOLD,
});
}
return rows;
}
// ── Cross-cell delta matrix (full-context vs raw) ──────────────────────
export interface CrossCellDelta {
verdict: Verdict6;
rawPercent: number;
fullContextPercent: number;
/** fullContext − raw. Positive = full-context raised this verdict's share;
* negative = full-context lowered it. The brief's headline expectation:
* `correct` delta > 0, `F4_hallucinated` delta < 0, `F1_abstain`
* may go either way (more abstains is sometimes an OK signal). */
delta: number;
}
export function crossCellDeltaMatrix(perCell: readonly PerCellRow[]): CrossCellDelta[] | null {
const raw = perCell.find(r => r.cell === 'raw');
const full = perCell.find(r => r.cell === 'full-context');
if (!raw || !full) return null;
const out: CrossCellDelta[] = [];
for (const v of VERDICT6_VALUES) {
out.push({
verdict: v,
rawPercent: raw.percents[v],
fullContextPercent: full.percents[v],
delta: full.percents[v] - raw.percents[v],
});
}
return out;
}
// ── Cost summary ───────────────────────────────────────────────────────
export interface CostSummary {
totalUsd: number;
perCellUsd: Record<string, number>;
judgeTotalUsd: number;
judgeCallCount: number;
/** Median judge call latency across the run; `null` when no judge
* call recorded a latency. */
medianJudgeMs: number | null;
/** Brief §Task-3 rule 4: flag when total judge spend would exceed
* ~$20 projected for a full 4-cell × 50-instance run. Computed as
* `perInstanceJudgeUsd × 200`. */
week1WarningProjectedUsd: number;
week1WarningFired: boolean;
}
const WEEK1_PROJECTION_INSTANCE_COUNT = 200; // 4 cells × 50 instances
export function costSummary(records: readonly JudgedJsonlRecord[]): CostSummary {
const perCellUsd: Record<string, number> = {};
let total = 0;
const judgeLatencies: number[] = [];
const judgeTotal = 0;
let judgeCalls = 0;
for (const r of records) {
total += r.usd_per_query;
perCellUsd[r.cell] = (perCellUsd[r.cell] ?? 0) + r.usd_per_query;
if (r.judge_ensemble) {
for (const entry of r.judge_ensemble) {
if (typeof entry.latency_ms === 'number') judgeLatencies.push(entry.latency_ms);
}
judgeCalls += r.judge_ensemble.length;
} else if (r.judge_verdict !== undefined) {
// Single-judge call — we don't store per-judge latency in the
// JsonlRecord shape (no judge_latency_ms field), so the median
// is computed only over ensemble entries that carry it. If no
// ensemble row is present, medianJudgeMs stays null.
judgeCalls += 1;
}
}
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
// Per-instance judge spend is not stored directly — the aggregator
// doesn't know judge USD without the onCall sink. When the caller
// passes judgeTotalUsd separately (see buildReport), we populate
// this from the authoritative source. Here we return zero; buildReport
// overwrites with the real total.
const medianJudgeMs = judgeLatencies.length === 0
? null
: (() => {
const s = [...judgeLatencies].sort((a, b) => a - b);
const mid = Math.floor(s.length / 2);
return s.length % 2 === 0 ? (s[mid - 1] + s[mid]) / 2 : s[mid];
})();
const perInstanceJudgeUsd = judgedCount > 0 ? judgeTotal / judgedCount : 0;
const projected = perInstanceJudgeUsd * WEEK1_PROJECTION_INSTANCE_COUNT;
return {
totalUsd: total,
perCellUsd,
judgeTotalUsd: judgeTotal,
judgeCallCount: judgeCalls,
medianJudgeMs,
week1WarningProjectedUsd: projected,
week1WarningFired: projected > 20,
};
}
// ── Top-level report + markdown renderer ───────────────────────────────
export interface AggregateReport {
generatedAt: string;
totalRecords: number;
perCell: PerCellRow[];
perCategory: PerCategoryRow[];
crossCellDelta: CrossCellDelta[] | null;
cost: CostSummary;
}
export function buildReport(
records: readonly JudgedJsonlRecord[],
opts: { judgeTotalUsd?: number } = {},
): AggregateReport {
const perCell = perCellRollup(records);
const perCategory = perCategoryRollup(records);
const crossCellDelta = crossCellDeltaMatrix(perCell);
const cost = costSummary(records);
// Callers that run the harness end-to-end know the authoritative
// judge spend because judge-client emits `onCall` entries. Pass the
// aggregated total here so the report carries real dollars instead
// of the placeholder zero computed from JSONL alone.
if (opts.judgeTotalUsd !== undefined) {
cost.judgeTotalUsd = opts.judgeTotalUsd;
const judgedCount = records.filter(r => r.judge_verdict !== undefined).length;
const perInstance = judgedCount > 0 ? opts.judgeTotalUsd / judgedCount : 0;
cost.week1WarningProjectedUsd = perInstance * WEEK1_PROJECTION_INSTANCE_COUNT;
cost.week1WarningFired = cost.week1WarningProjectedUsd > 20;
}
return {
generatedAt: new Date().toISOString(),
totalRecords: records.length,
perCell,
perCategory,
crossCellDelta,
cost,
};
}
function pct(n: number): string {
return `${(n * 100).toFixed(1)}%`;
}
function fmtUsd(n: number): string {
return `$${n.toFixed(n < 1 ? 6 : 2)}`;
}
export function renderMarkdown(report: AggregateReport): string {
const lines: string[] = [];
lines.push('# Benchmark Aggregate Report');
lines.push('');
lines.push(`**Generated at:** ${report.generatedAt}`);
lines.push(`**Total records:** ${report.totalRecords}`);
lines.push('');
lines.push('## Per-cell verdict distribution');
lines.push('');
lines.push('| Cell | Total | Correct | F1 abstain | F2 partial | F3 incorrect | F4 hallucinated | F5 off-topic | Unjudged | Weighted score |');
lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|');
for (const row of report.perCell) {
lines.push(
`| ${row.cell} | ${row.total} | ${row.counts.correct} (${pct(row.percents.correct)}) | ` +
`${row.counts.F1_abstain} (${pct(row.percents.F1_abstain)}) | ` +
`${row.counts.F2_partial} (${pct(row.percents.F2_partial)}) | ` +
`${row.counts.F3_incorrect} (${pct(row.percents.F3_incorrect)}) | ` +
`${row.counts.F4_hallucinated} (${pct(row.percents.F4_hallucinated)}) | ` +
`${row.counts.F5_offtopic} (${pct(row.percents.F5_offtopic)}) | ` +
`${row.counts.unjudged} (${pct(row.percents.unjudged)}) | ` +
`${row.weightedScore.toFixed(3)} |`,
);
}
lines.push('');
if (report.perCategory.length > 0) {
lines.push('## Per-LoCoMo-category distribution');
lines.push('');
lines.push('| Category | Total | Correct% | F4 (hallucinated)% | Flagged? |');
lines.push('|---|---:|---:|---:|:---:|');
for (const row of report.perCategory) {
lines.push(
`| ${row.category} | ${row.total} | ${pct(row.percents.correct)} | ` +
`${pct(row.percents.F4_hallucinated)} | ` +
`${row.hallucinationFlag ? '⚠️ PM review' : '✓'} |`,
);
}
lines.push('');
}
if (report.crossCellDelta) {
lines.push('## Full-context vs raw delta');
lines.push('');
lines.push('| Verdict | raw % | full-context % | Δ (full − raw) |');
lines.push('|---|---:|---:|---:|');
for (const r of report.crossCellDelta) {
const sign = r.delta > 0 ? '+' : '';
lines.push(`| ${r.verdict} | ${pct(r.rawPercent)} | ${pct(r.fullContextPercent)} | ${sign}${pct(r.delta)} |`);
}
lines.push('');
}
lines.push('## Cost summary');
lines.push('');
lines.push(`- Total cell spend: **${fmtUsd(report.cost.totalUsd)}**`);
lines.push(`- Judge spend: **${fmtUsd(report.cost.judgeTotalUsd)}** across ${report.cost.judgeCallCount} call(s)`);
if (report.cost.medianJudgeMs !== null) {
lines.push(`- Median judge latency: ${report.cost.medianJudgeMs.toFixed(0)} ms`);
}
if (Object.keys(report.cost.perCellUsd).length > 0) {
lines.push('- Per-cell cell spend:');
for (const [cell, usd] of Object.entries(report.cost.perCellUsd)) {
lines.push(` - ${cell}: ${fmtUsd(usd)}`);
}
}
if (report.cost.week1WarningFired) {
lines.push(
`- ⚠️ **Week-1 scale-up warning:** projected judge spend for a full 4-cell × 50-instance run is ` +
`${fmtUsd(report.cost.week1WarningProjectedUsd)} (threshold $20). Flagged per brief §Task-3 rule 4.`,
);
}
return lines.join('\n');
}

View File

@@ -1,314 +0,0 @@
/**
* Sprint 11 Task B2 — Tie-break policy for multi-vendor judge ensemble.
*
* Authority:
* - PM-Waggle-OS/briefs/2026-04-22-cc-sprint-11-kickoff.md §3 Track B B2
* - PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md (LOCKED)
*
* Context — what this resolves:
*
* The Sprint 10 Task 2.2 ratified primary judge ensemble is a THREE-vendor
* panel (Anthropic Opus 4.7 + OpenAI GPT-5.4 + Google Gemini 3.1-Pro). Three
* votes means three possible distributions:
*
* - 3-0 consensus → trivial, no tie-break
* - 2-1 majority → majority wins, no tie-break
* - 1-1-1 split → three different verdicts, no primary majority
*
* On 1-1-1, we escalate to a FOURTH vendor from a lineage disjoint from the
* primary trio: xai/grok-4.20 per Marko's 2026-04-22 LOCK. After the 4th
* vote arrives we have four votes total:
*
* - 2-1-1 / 1-1-2 → plurality winner (2 votes)
* - 1-1-1-1 → four-way split → PM escalation
*
* The fourth vendor is SPECIFICALLY xai/grok-4.20 — not Sonnet 4.6 (which
* would give Anthropic 2-of-4 weight → homogeneous-bias risk — rejected per
* LOCK doc §2) and not Opus 4.7 (already Judge 1).
*
* Observability: pino structured log emits `tie_break.path` ∈ { 'none',
* 'majority', 'quadri-vendor', 'pm-escalation' } plus
* `tie_break.fourth_vendor_slug` on quadri paths (field is future-proof
* even though Sprint 11 scope is grok-4.20 only).
*/
import type { JudgeResult } from './failure-mode-judge.js';
// ── Public types ───────────────────────────────────────────────────────────
/**
* A single judge vote. Reusing JudgeResult from failure-mode-judge.ts keeps
* the tie-break input shape aligned with what judgeEnsemble already produces.
*/
export type Vote = JudgeResult;
export type TieBreakPath = 'none' | 'majority' | 'quadri-vendor' | 'pm-escalation';
/**
* The canonical sentinel verdict returned on a 1-1-1-1 four-way split.
* Callers (aggregators, report generators) MUST treat this as an operator
* action signal, not as a normal verdict string. The shape is intentionally
* unambiguous so no accidental inclusion into recall/accuracy math.
*/
export const PM_ESCALATION_VERDICT = '__PM_ESCALATION__';
/** Default fourth vendor for Sprint 11 — per LOCK doc §1. */
export const DEFAULT_FOURTH_VENDOR = 'xai/grok-4.20';
export interface TieBreakResult {
/**
* The resolved verdict string. Encoded as `<verdict>|<failure_mode_or_NA>`
* matching the aggregation key used in computeMajority of failure-mode-judge.ts,
* so downstream consumers can uniformly decompose.
* On pm-escalation path: `__PM_ESCALATION__`.
*/
verdict: string;
path: TieBreakPath;
/** The full vote list after resolution — 3 votes for none/majority, 4 for quadri/pm-escalation. */
votes: Vote[];
/** The fourth vendor's vote when a quadri path was taken. Undefined on none/majority. */
fourthVendorVote?: Vote;
/** The fourth-vendor slug that was used, for observability. Undefined on none/majority. */
fourthVendorSlug?: string;
}
export interface FourthVendorCallPayload {
/** The three primary votes that produced the 1-1-1 split. */
primaryVotes: Vote[];
/** Model slug to invoke — e.g. 'xai/grok-4.20'. */
model: string;
}
export type CallFourthVendor = (payload: FourthVendorCallPayload) => Promise<Vote>;
export interface TieBreakLogger {
info(event: string, fields: Record<string, unknown>): void;
warn?(event: string, fields: Record<string, unknown>): void;
}
export interface ResolveTieBreakOptions {
/**
* Invoked only on 1-1-1 splits. Required when primaryVotes.length === 3
* and the distribution is 1-1-1; optional otherwise. Tests mock this.
*/
callFourthVendor?: CallFourthVendor;
/** Override the fourth vendor slug. Defaults to `DEFAULT_FOURTH_VENDOR`. */
fourthVendorModel?: string;
/** Structured logger. Silently no-ops when omitted. */
logger?: TieBreakLogger;
}
// ── Internal helpers ───────────────────────────────────────────────────────
/**
* Canonical key for a vote — combines verdict + failure_mode so identical
* answers with different failure codes don't accidentally tie.
*/
function voteKey(v: Vote): string {
return `${v.verdict}|${v.failure_mode ?? 'NA'}`;
}
function countByKey(votes: Vote[]): Map<string, { count: number; first: Vote }> {
const tally = new Map<string, { count: number; first: Vote }>();
for (const v of votes) {
const key = voteKey(v);
const existing = tally.get(key);
if (existing) existing.count += 1;
else tally.set(key, { count: 1, first: v });
}
return tally;
}
function pluralityTop(tally: Map<string, { count: number; first: Vote }>): {
topCount: number;
topKeys: string[];
} {
let topCount = 0;
for (const { count } of tally.values()) if (count > topCount) topCount = count;
const topKeys: string[] = [];
for (const [key, { count }] of tally) if (count === topCount) topKeys.push(key);
return { topCount, topKeys };
}
// ── Public API ─────────────────────────────────────────────────────────────
/**
* Resolve a multi-vendor judge ensemble vote.
*
* Input shape:
* - 3 primary votes (Sprint 10 Task 2.2 trio) — the normal call path.
* - 4 votes also accepted, in which case the function treats the vector
* as already-quadri-resolved and returns the plurality winner (or
* pm-escalation on 1-1-1-1). This shape exists for tests and for
* caller-side recomposition.
*
* On 1-1-1 with 3 primary votes, `callFourthVendor` MUST be provided.
* The function invokes it with the three primary votes and the canonical
* model slug, then recurses with the four-vote list.
*
* Returns:
* - `{ path: 'none' }` for 3-0 consensus (3 primary votes).
* - `{ path: 'majority' }` for 2-1 majority (3 votes) OR for a plurality
* win on 4 votes when the caller supplied the vector directly.
* - `{ path: 'quadri-vendor' }` when we called the 4th vendor ourselves
* and got a 2-1-1 or 1-1-2 distribution.
* - `{ path: 'pm-escalation' }` on a 1-1-1-1 four-way split; verdict is
* the `PM_ESCALATION_VERDICT` sentinel.
*/
export async function resolveTieBreak(
votes: Vote[],
options: ResolveTieBreakOptions = {},
): Promise<TieBreakResult> {
if (votes.length !== 3 && votes.length !== 4) {
throw new Error(
`resolveTieBreak: votes.length must be 3 (primary ensemble) or 4 (post-quadri). Got ${votes.length}.`,
);
}
const logger = options.logger;
if (votes.length === 3) {
const tally = countByKey(votes);
const { topCount, topKeys } = pluralityTop(tally);
// 3-0 consensus — one bucket holds all three.
if (topCount === 3) {
const entry = tally.get(topKeys[0])!;
logger?.info('tie_break', { path: 'none' as TieBreakPath, verdict: topKeys[0] });
return {
verdict: topKeys[0],
path: 'none',
votes,
};
}
// 2-1 majority — one bucket holds two, another holds one.
if (topCount === 2 && topKeys.length === 1) {
logger?.info('tie_break', { path: 'majority' as TieBreakPath, verdict: topKeys[0] });
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
// 1-1-1 split — three buckets of one each. Escalate to fourth vendor.
if (topCount === 1 && topKeys.length === 3) {
if (!options.callFourthVendor) {
throw new Error(
'resolveTieBreak: 1-1-1 three-way split requires a callFourthVendor implementation.',
);
}
const fourthVendorModel = options.fourthVendorModel ?? DEFAULT_FOURTH_VENDOR;
logger?.info('tie_break.quadri-vendor.invoke', {
path: 'quadri-vendor' as TieBreakPath,
fourth_vendor_slug: fourthVendorModel,
primary_keys: topKeys,
});
const fourthVote = await options.callFourthVendor({
primaryVotes: votes,
model: fourthVendorModel,
});
// Recurse with four votes. The 4-vote branch returns quadri-vendor or pm-escalation.
const resolved = await resolveTieBreak([...votes, fourthVote], options);
// Re-tag the path so "quadri-vendor" sticks on success cases and
// "pm-escalation" stays on 1-1-1-1 after escalation.
const path: TieBreakPath =
resolved.path === 'pm-escalation' ? 'pm-escalation' : 'quadri-vendor';
logger?.info('tie_break.quadri-vendor.resolved', {
path,
fourth_vendor_slug: fourthVendorModel,
verdict: resolved.verdict,
});
return {
verdict: resolved.verdict,
path,
votes: resolved.votes,
fourthVendorVote: fourthVote,
fourthVendorSlug: fourthVendorModel,
};
}
// Should be unreachable under correct input (any 3-vote distribution is
// 3-0, 2-1, or 1-1-1), but stay defensive.
throw new Error(
`resolveTieBreak: unexpected 3-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
);
}
// votes.length === 4 — post-quadri-vendor resolution path.
const tally = countByKey(votes);
const { topCount, topKeys } = pluralityTop(tally);
// 1-1-1-1 four-way split — every bucket is one. PM escalation.
if (topCount === 1) {
logger?.info('tie_break', {
path: 'pm-escalation' as TieBreakPath,
verdict: PM_ESCALATION_VERDICT,
four_way_keys: topKeys,
});
return {
verdict: PM_ESCALATION_VERDICT,
path: 'pm-escalation',
votes,
};
}
// Plurality winner: 2-1-1 / 1-1-2 (unique top). 2-2 ties with 4 votes are
// structurally impossible from our flow (3 primary votes make 2-2
// impossible after the +1 fourth vote; the fourth vote always creates a
// unique plurality or a 1-1-1-1). Still handle 2-2 defensively by
// promoting to pm-escalation so no silent coin-flip ever lands in prod.
if (topCount === 2 && topKeys.length === 1) {
logger?.info('tie_break', {
path: 'majority' as TieBreakPath,
verdict: topKeys[0],
vote_count: 4,
});
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
if (topCount === 2 && topKeys.length >= 2) {
// 2-2 tie on 4 votes — not reachable from 1-1-1→quadri flow, but a
// caller may pass in a pre-constructed 4-vote vector. Escalate rather
// than coin-flip.
logger?.warn?.('tie_break.two-two-tie', {
path: 'pm-escalation' as TieBreakPath,
verdict: PM_ESCALATION_VERDICT,
two_two_keys: topKeys,
});
return {
verdict: PM_ESCALATION_VERDICT,
path: 'pm-escalation',
votes,
};
}
if (topCount === 3 || topCount === 4) {
logger?.info('tie_break', {
path: 'majority' as TieBreakPath,
verdict: topKeys[0],
vote_count: 4,
});
return {
verdict: topKeys[0],
path: 'majority',
votes,
};
}
// Truly unreachable given 4 input votes; defensive.
throw new Error(
`resolveTieBreak: unexpected 4-vote distribution — topCount=${topCount} topKeys=${topKeys.length}`,
);
}

View File

@@ -1,383 +0,0 @@
/**
* Failure-mode judge — scaffold.
*
* Brief: PM-Waggle-OS/briefs/2026-04-20-cc-preflight-prep-tasks.md Task 4
* Spec: PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md §4 (prompt),
* §6 (ensemble), §8 (inter-judge agreement / Fleiss' kappa thresholds)
* LOCKED: PM-Waggle-OS/decisions/2026-04-20-failure-mode-oq-resolutions-locked.md
*
* This module is deliberately *not wired* into the harness runner in this
* sprint (explicitly out of scope per the brief). It exists so that Sprint 9
* can drop in the judge call on top of a tested foundation.
*
* Design decisions worth knowing:
*
* — Zod validates strictly. The Step-3 schema contract between the model
* and us is enforced with a `.refine()` that captures the two
* cross-field invariants from the spec §4:
* verdict === 'correct' ⇒ failure_mode MUST be null
* verdict === 'incorrect' ⇒ failure_mode MUST be one of F1..F5
* Any violation → retry once with the reminder → JudgeParseError.
*
* — JSON extraction is best-effort. Models often wrap JSON in ```json
* fences, leading prose, or trailing chatter. We strip the obvious
* wrappers before parsing; if both attempts fail we surface the raw
* response in the thrown error so a debugger can reproduce.
*
* — Ensemble tie-break is explicit: on a 2-2 split the ensemble uses the
* first judge in the `judgeModels` list. By convention the caller puts
* Sonnet first so "2-2 tie broken by Sonnet" holds per the spec §6.
*
* — Fleiss' kappa is the standard multi-rater formulation (Fleiss 1971).
* Raters per subject must be constant; we throw if any subject has a
* different rater count, since the formula is undefined otherwise.
* Categories are the 6-class vocabulary from §8 (`correct` treated as
* a separate class from F1..F5 — the same convention §8 specifies).
*/
import { z } from 'zod';
// ── Public types ───────────────────────────────────────────────────────
export type FailureMode = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
export type Verdict = 'correct' | 'incorrect';
export interface JudgeResult {
verdict: Verdict;
failure_mode: null | FailureMode;
rationale: string;
judge_model: string;
}
export interface LlmClient {
/** One-shot completion. Returns the raw text produced by the judge LLM. */
complete(prompt: string): Promise<string>;
}
export class JudgeParseError extends Error {
readonly lastResponse: string;
readonly lastParseError?: string;
readonly judgeModel: string;
constructor(message: string, details: { lastResponse: string; judgeModel: string; lastParseError?: string }) {
super(message);
this.name = 'JudgeParseError';
this.lastResponse = details.lastResponse;
this.judgeModel = details.judgeModel;
this.lastParseError = details.lastParseError;
}
}
// ── Zod schema (enforces spec §4 Step-3 contract) ──────────────────────
const judgeSchema = z
.object({
verdict: z.enum(['correct', 'incorrect']),
failure_mode: z.union([z.null(), z.enum(['F1', 'F2', 'F3', 'F4', 'F5'])]),
rationale: z.string().min(1, 'rationale must be a non-empty sentence'),
})
.strict()
.refine(
r => (r.verdict === 'correct' ? r.failure_mode === null : r.failure_mode !== null),
{
message:
'verdict/failure_mode invariant violated: ' +
'verdict="correct" requires failure_mode=null; verdict="incorrect" requires one of F1..F5',
},
);
// ── Prompt builder (exact text from strategy §4) ───────────────────────
/** Exact judge prompt text from `strategy/2026-04-20-failure-mode-taxonomy.md` §4.
* Do not modify. The spec requires cell-independent scoring — changing the
* prompt between cells, or even between invocations, invalidates the ensemble. */
export function buildJudgePrompt(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
}): string {
return [
"You are evaluating whether an LLM's answer is correct against ground truth.",
'',
'## Question',
params.question,
'',
'## Ground-truth answer',
params.groundTruth,
'',
'## Ground-truth supporting context (excerpt shown to the model)',
params.contextExcerpt,
'',
"## Model's answer",
params.modelAnswer,
'',
'## Your task',
'',
"Step 1: Determine if the model's answer is correct.",
'- "correct" means the model\'s answer contains all required facts from ground truth, with no additional incorrect claims.',
'- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.',
'- Extra detail is acceptable ONLY if it is factually correct.',
'',
'Step 2: If incorrect, assign exactly one failure mode using this decision tree:',
'',
'1. Does the model explicitly refuse or say it does not know? → F1 (ABSTAIN)',
'2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? → F5 (OFF-TOPIC)',
'3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? → F4 (HALLUCINATED)',
'4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? → F2 (PARTIAL)',
'5. Otherwise (model states facts derived from the context but gets them wrong): → F3 (INCORRECT)',
'',
'Step 3: Return JSON only, no prose, in this exact schema:',
'',
'{',
' "verdict": "correct" | "incorrect",',
' "failure_mode": null | "F1" | "F2" | "F3" | "F4" | "F5",',
' "rationale": "one sentence explaining the verdict"',
'}',
'',
'If verdict is "correct", failure_mode MUST be null.',
'If verdict is "incorrect", failure_mode MUST be one of F1-F5.',
].join('\n');
}
/** Retry reminder prepended to the original prompt when the first response
* fails to parse. Text is verbatim from the Task-4 acceptance spec. */
export const RETRY_REMINDER =
'Your previous response was not valid JSON. Return only the JSON object, no prose.';
// ── JSON extraction ────────────────────────────────────────────────────
/** Best-effort extraction of the JSON body from a model response. Handles:
* - bare JSON
* - `{ ... }` embedded in prose (first `{` to matching last `}`)
* - markdown code fences ```json ... ``` or ``` ... ``` */
export function extractJsonBody(raw: string): string | null {
if (!raw) return null;
const trimmed = raw.trim();
// Code-fence strip first.
const fenceMatch = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/i);
const afterFence = fenceMatch ? fenceMatch[1].trim() : trimmed;
if (afterFence.startsWith('{') && afterFence.endsWith('}')) return afterFence;
// Fallback — greedy `{ ... }` scan, stopping at the last `}`.
const first = afterFence.indexOf('{');
const last = afterFence.lastIndexOf('}');
if (first >= 0 && last > first) return afterFence.slice(first, last + 1);
return null;
}
function tryParse(raw: string): { ok: true; value: JudgeResult } | { ok: false; error: string } {
const body = extractJsonBody(raw);
if (!body) return { ok: false, error: 'no JSON object found in response' };
let parsedJson: unknown;
try {
parsedJson = JSON.parse(body);
} catch (err) {
return { ok: false, error: `JSON.parse failed: ${(err as Error).message}` };
}
const result = judgeSchema.safeParse(parsedJson);
if (!result.success) {
return { ok: false, error: `schema validation failed: ${result.error.issues.map(i => i.message).join('; ')}` };
}
return {
ok: true,
value: {
verdict: result.data.verdict,
failure_mode: result.data.failure_mode,
rationale: result.data.rationale,
judge_model: '', // filled by judgeAnswer from params.judgeModel
},
};
}
// ── Public API ─────────────────────────────────────────────────────────
export async function judgeAnswer(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModel: string;
llmClient: LlmClient;
}): Promise<JudgeResult> {
const prompt = buildJudgePrompt({
question: params.question,
groundTruth: params.groundTruth,
contextExcerpt: params.contextExcerpt,
modelAnswer: params.modelAnswer,
});
const first = await params.llmClient.complete(prompt);
const firstParsed = tryParse(first);
if (firstParsed.ok) {
return { ...firstParsed.value, judge_model: params.judgeModel };
}
// Retry once with the reminder prefixed. A fresh LLM call — callers can
// wire conversation-state preservation later; spec §4 does not require it.
const retryPrompt = `${RETRY_REMINDER}\n\n${prompt}`;
const second = await params.llmClient.complete(retryPrompt);
const secondParsed = tryParse(second);
if (secondParsed.ok) {
return { ...secondParsed.value, judge_model: params.judgeModel };
}
throw new JudgeParseError(
`Judge ${params.judgeModel} produced unparseable output after one retry`,
{
lastResponse: second,
judgeModel: params.judgeModel,
lastParseError: secondParsed.error,
},
);
}
export async function judgeEnsemble(params: {
question: string;
groundTruth: string;
contextExcerpt: string;
modelAnswer: string;
judgeModels: string[];
llmClients: Map<string, LlmClient>;
}): Promise<{ ensemble: JudgeResult[]; majority: JudgeResult; fleissKappa: number }> {
if (params.judgeModels.length === 0) {
throw new Error('judgeEnsemble requires at least one judgeModel');
}
const results: JudgeResult[] = [];
for (const model of params.judgeModels) {
const client = params.llmClients.get(model);
if (!client) throw new Error(`judgeEnsemble: no LlmClient registered for model ${model}`);
const result = await judgeAnswer({
question: params.question,
groundTruth: params.groundTruth,
contextExcerpt: params.contextExcerpt,
modelAnswer: params.modelAnswer,
judgeModel: model,
llmClient: client,
});
results.push(result);
}
const majority = computeMajority(results, params.judgeModels[0]);
// For a single-subject ensemble, Fleiss' kappa is computed over the same
// one-subject matrix — degenerate but well-defined (NaN gets clamped to 0
// when the subject has only one category). In production, this function
// is typically called per-instance and kappa is aggregated at the batch
// layer; we expose the single-subject value for diagnostic continuity.
const fleissKappa = computeFleissKappa([results]);
return { ensemble: results, majority, fleissKappa };
}
function computeMajority(results: readonly JudgeResult[], tieBreakerModel: string): JudgeResult {
// Count verdict × failure_mode pairs (null collapses to 'NA' for keying).
const tally = new Map<string, number>();
const first = new Map<string, JudgeResult>();
for (const r of results) {
const key = `${r.verdict}|${r.failure_mode ?? 'NA'}`;
tally.set(key, (tally.get(key) ?? 0) + 1);
if (!first.has(key)) first.set(key, r);
}
let winnerKey = '';
let winnerCount = -1;
for (const [key, count] of tally) {
if (count > winnerCount) {
winnerKey = key;
winnerCount = count;
}
}
// Tie detection: if any other key has equal count, the tie-breaker model
// wins — its verdict becomes the majority.
const tied = Array.from(tally.entries()).filter(([, c]) => c === winnerCount);
if (tied.length > 1) {
const tb = results.find(r => r.judge_model === tieBreakerModel);
if (tb) return tb;
// Fallback: keep the first deterministic winner (stable map iteration).
}
const winner = first.get(winnerKey);
if (!winner) {
// Unreachable under normal flow; defensive.
return results[0];
}
return {
verdict: winner.verdict,
failure_mode: winner.failure_mode,
rationale: winner.rationale,
judge_model: 'ensemble_majority',
};
}
// ── Fleiss' kappa ──────────────────────────────────────────────────────
/** The 6-class rating space used by Fleiss' kappa: correct + F1..F5.
* §8: "standardni Fleiss' kappa preko 4 raters × N instanci × 6 klasa". */
const KAPPA_CATEGORIES = ['correct', 'F1', 'F2', 'F3', 'F4', 'F5'] as const;
type KappaCategory = typeof KAPPA_CATEGORIES[number];
function categoryOf(r: JudgeResult): KappaCategory {
if (r.verdict === 'correct') return 'correct';
return r.failure_mode ?? 'F3'; // should never hit fallback — schema guard
}
/** Computes Fleiss' kappa on an N-subjects × n-raters matrix.
*
* Input shape: `ratings[i]` is the list of judge results for subject i —
* all subjects must have the same number of raters. Categories are the
* fixed 6-class vocabulary.
*
* Returns: κ in [-1, 1]. If all subjects are unanimous on the same
* category (expected-agreement = 1), returns 1 (the (1-1)/(1-1) limit).
*/
export function computeFleissKappa(ratings: readonly (readonly JudgeResult[])[]): number {
if (ratings.length === 0) return 0;
const n = ratings[0].length;
if (n < 2) {
// Fleiss' kappa is undefined for a single rater. Degenerate → 0.
return 0;
}
for (const row of ratings) {
if (row.length !== n) {
throw new Error(`Fleiss' kappa requires a constant rater count; saw ${row.length} and ${n}`);
}
}
const N = ratings.length;
const k = KAPPA_CATEGORIES.length;
const catIndex: Record<KappaCategory, number> = { correct: 0, F1: 1, F2: 2, F3: 3, F4: 4, F5: 5 };
// nij[i][j] = count of raters who assigned subject i to category j
const nij: number[][] = Array.from({ length: N }, () => new Array<number>(k).fill(0));
for (let i = 0; i < N; i++) {
for (const r of ratings[i]) {
const col = catIndex[categoryOf(r)];
nij[i][col] += 1;
}
}
// Pj = (1/(N*n)) * sum_i nij[i][j]
const Pj: number[] = new Array<number>(k).fill(0);
for (let j = 0; j < k; j++) {
let sum = 0;
for (let i = 0; i < N; i++) sum += nij[i][j];
Pj[j] = sum / (N * n);
}
// Pi = (1/(n(n-1))) * ( sum_j nij[i][j]^2 - n )
let PbarSum = 0;
for (let i = 0; i < N; i++) {
let sqSum = 0;
for (let j = 0; j < k; j++) sqSum += nij[i][j] * nij[i][j];
const Pi = (sqSum - n) / (n * (n - 1));
PbarSum += Pi;
}
const Pbar = PbarSum / N;
// Pebar = sum_j Pj^2
let Pebar = 0;
for (let j = 0; j < k; j++) Pebar += Pj[j] * Pj[j];
if (Pebar >= 1 - 1e-12) {
// Every rating in one category → expected = observed = 1 → κ = 1 by convention.
return 1;
}
return (Pbar - Pebar) / (1 - Pebar);
}