moving

benchmarks/harness/tests/stats/cluster-bootstrap.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Sprint 12 Task 1 Blocker #5 — cluster-bootstrap tests.
+ *
+ * Acceptance (brief § 2.1 A):
+ *   1. Deterministic re-run (same input + seed → same output)
+ *   2. Default n_bootstrap = 10 000
+ *   3. Default seed = 42
+ *   4. Rejects empty rows
+ *   5. CI ⊇ point_estimate property
+ *   6. ci_lower ≤ ci_upper invariant
+ *   7. Cluster structure affects CI vs. instance-level Wilson
+ *   8. NaN guard on malformed `correct` field
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  computeClusterBootstrapCI,
+  type CorrectnessRow,
+} from '../../src/stats/cluster-bootstrap.js';
+import { computeWilsonCI } from '../../src/stats/wilson-ci.js';
+
+function buildClusteredRows(
+  clusterCount: number,
+  rowsPerCluster: number,
+  correctRate: number,
+): CorrectnessRow[] {
+  // Deterministic construction: first ⌊rate × rowsPerCluster⌋ rows in each
+  // cluster are correct. Keeps tests independent of PRNG state.
+  const rows: CorrectnessRow[] = [];
+  const correctPerCluster = Math.round(correctRate * rowsPerCluster);
+  for (let c = 0; c < clusterCount; c++) {
+    const conversation_id = `conv-${c}`;
+    for (let r = 0; r < rowsPerCluster; r++) {
+      rows.push({ conversation_id, correct: r < correctPerCluster ? 1 : 0 });
+    }
+  }
+  return rows;
+}
+
+describe('computeClusterBootstrapCI — determinism + defaults', () => {
+  it('produces bit-identical output on two calls with same input', () => {
+    const rows = buildClusteredRows(5, 4, 0.75);
+    const a = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
+    const b = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
+    expect(a.ci_lower).toBe(b.ci_lower);
+    expect(a.ci_upper).toBe(b.ci_upper);
+    expect(a.point_estimate).toBe(b.point_estimate);
+  });
+
+  it('different seeds produce different bootstrap CIs (seed sensitivity sanity)', () => {
+    // 20 singleton clusters with an irregular correct/wrong pattern so that
+    // bootstrap resample means span a dense set of values. At n=20 and
+    // n_bootstrap=2000, the 2.5th/97.5th percentile indices (50 and 1950)
+    // are far from the extremes, so different seeds produce materially
+    // different CI bounds.
+    const rows: CorrectnessRow[] = [];
+    for (let i = 0; i < 20; i++) {
+      rows.push({ conversation_id: `c-${i}`, correct: (i % 3 === 0 ? 1 : 0) });
+    }
+    const a = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    const b = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 123 });
+    expect(a.point_estimate).toBe(b.point_estimate);
+    const sameBounds = a.ci_lower === b.ci_lower && a.ci_upper === b.ci_upper;
+    expect(sameBounds).toBe(false);
+  });
+
+  it('defaults n_bootstrap=10000 and seed=42 per A3 LOCK § 2', () => {
+    const rows = buildClusteredRows(3, 4, 0.5);
+    const r = computeClusterBootstrapCI({ rows });
+    expect(r.n_bootstrap).toBe(10000);
+    expect(r.seed).toBe(42);
+  });
+});
+
+describe('computeClusterBootstrapCI — structural invariants', () => {
+  it('CI contains the point estimate (point ∈ [ci_lower, ci_upper])', () => {
+    const rows = buildClusteredRows(8, 4, 0.75);
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    expect(r.point_estimate).toBeGreaterThanOrEqual(r.ci_lower);
+    expect(r.point_estimate).toBeLessThanOrEqual(r.ci_upper);
+  });
+
+  it('ci_lower ≤ ci_upper always', () => {
+    const rows = buildClusteredRows(5, 3, 0.333);
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 1000, seed: 42 });
+    expect(r.ci_lower).toBeLessThanOrEqual(r.ci_upper);
+  });
+
+  it('reports n_clusters = distinct conversation_ids', () => {
+    const rows = [
+      { conversation_id: 'a', correct: 1 as const },
+      { conversation_id: 'a', correct: 1 as const },
+      { conversation_id: 'b', correct: 0 as const },
+      { conversation_id: 'c', correct: 1 as const },
+    ];
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 100, seed: 42 });
+    expect(r.n_clusters).toBe(3);
+    expect(r.n_rows).toBe(4);
+  });
+
+  it('produces wider CI than instance-level Wilson when intra-cluster correlation is high', () => {
+    // 6 clusters × 4 rows, all-or-nothing correctness within each cluster:
+    // 4 clusters all-correct (4×4=16 successes) + 2 clusters all-wrong (0).
+    // Intra-cluster correlation is max (1.0) — clusters are homogeneous.
+    // Bootstrap should reflect that cluster-level variance is huge (some
+    // samples pick all-correct clusters → near 1.0; others pick all-wrong
+    // → near 0.0), producing a much wider CI than instance-level Wilson
+    // which assumes independent 16/24 successes.
+    const rows: CorrectnessRow[] = [];
+    for (let c = 0; c < 4; c++) {
+      for (let r = 0; r < 4; r++) {
+        rows.push({ conversation_id: `correct-${c}`, correct: 1 });
+      }
+    }
+    for (let c = 0; c < 2; c++) {
+      for (let r = 0; r < 4; r++) {
+        rows.push({ conversation_id: `wrong-${c}`, correct: 0 });
+      }
+    }
+    const bootstrap = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    const wilson = computeWilsonCI({ successes: 16, trials: 24 });
+    const bootstrapWidth = bootstrap.ci_upper - bootstrap.ci_lower;
+    const wilsonWidth = wilson.ci_upper - wilson.ci_lower;
+    expect(bootstrapWidth).toBeGreaterThan(wilsonWidth);
+  });
+});
+
+describe('computeClusterBootstrapCI — input validation', () => {
+  it('rejects empty rows', () => {
+    expect(() => computeClusterBootstrapCI({ rows: [] })).toThrow(/non-empty rows/);
+  });
+
+  it('rejects n_bootstrap < 1', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 0 })).toThrow(
+      /n_bootstrap ≥ 1/,
+    );
+  });
+
+  it('rejects non-integer seed', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, seed: 1.5 })).toThrow(/integer seed/);
+  });
+
+  it('rejects rows with correct ∉ {0, 1}', () => {
+    const rows = [
+      { conversation_id: 'a', correct: 1 as 0 | 1 },
+      { conversation_id: 'a', correct: 2 as unknown as 0 | 1 },
+    ];
+    expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 10, seed: 42 })).toThrow(
+      /correct ∈ \{0, 1\}/,
+    );
+  });
+
+  it('rejects confidence ≠ 0.95', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, confidence: 0.99 })).toThrow(
+      /confidence=0\.95/,
+    );
+  });
+});

benchmarks/harness/tests/stats/fleiss-kappa.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Sprint 12 Task 1 Blocker #5 — Fleiss κ tests.
+ *
+ * Acceptance (brief § 2.1 A, criterion-by-criterion):
+ *   1. K=2 case reduction sanity
+ *   2. K=6 (F1-F6 taxonomy) happy path
+ *   3. Perfect agreement → κ=1.0
+ *   4. Zero-above-chance agreement → κ=0
+ *   5. Pre-tie-break input only (no post-tie-break leakage)
+ *   6. NaN guard when P_e = 1 (uniform assignment)
+ *   7. Reject mismatched row widths
+ *   8. Reject row sums ≠ n_judges
+ */
+
+import { describe, expect, it } from 'vitest';
+import { computeFleissKappa, type VoteMatrix } from '../../src/stats/fleiss-kappa.js';
+
+describe('computeFleissKappa — structural invariants', () => {
+  it('returns κ=1.0 under perfect agreement (all judges pick same category per item)', () => {
+    // 4 items, 3 judges, 2 categories. Every judge on every item → same category.
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0],
+        [0, 3],
+        [3, 0],
+        [0, 3],
+      ],
+    };
+    const result = computeFleissKappa(matrix);
+    expect(result.kappa).toBeCloseTo(1.0, 10);
+    expect(result.P_bar).toBeCloseTo(1.0, 10);
+    expect(result.n_items).toBe(4);
+    expect(result.n_judges).toBe(3);
+    expect(result.n_categories).toBe(2);
+  });
+
+  it('returns κ=NaN when P_e=1 (all judges always pick the single category)', () => {
+    // 3 items, 3 judges — uniform assignment into category 0.
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0],
+        [3, 0],
+        [3, 0],
+      ],
+    };
+    const result = computeFleissKappa(matrix);
+    expect(Number.isNaN(result.kappa)).toBe(true);
+    expect(result.P_e).toBe(1);
+  });
+
+  it('reduces cleanly to a binary-agreement measure (K=2 case)', () => {
+    // 5 items, 3 judges. Mixed disagreement. κ should land in (0, 1).
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0], // unanimous correct
+        [2, 1], // majority correct
+        [3, 0], // unanimous correct
+        [1, 2], // majority incorrect
+        [0, 3], // unanimous incorrect
+      ],
+    };
+    const result = computeFleissKappa(matrix);
+    expect(result.n_categories).toBe(2);
+    expect(result.kappa).toBeGreaterThan(0);
+    expect(result.kappa).toBeLessThanOrEqual(1);
+    // Category marginals should sum to 1 (modulo float).
+    const marginalSum = result.category_marginals.reduce((a, b) => a + b, 0);
+    expect(marginalSum).toBeCloseTo(1.0, 10);
+  });
+
+  it('handles K=6 failure taxonomy shape (F1-F6 + null encoded as 7-column matrix)', () => {
+    // 6 items, 3 judges, 7 categories (null + F1..F6). Simulates A3 LOCK §6
+    // shape with moderate disagreement.
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0, 0, 0, 0, 0, 0], // all correct (null)
+        [2, 1, 0, 0, 0, 0, 0], // 2 correct, 1 F1
+        [0, 3, 0, 0, 0, 0, 0], // unanimous F1
+        [0, 0, 2, 1, 0, 0, 0], // 2 F2, 1 F3
+        [3, 0, 0, 0, 0, 0, 0], // all correct
+        [0, 0, 0, 0, 0, 0, 3], // unanimous F6
+      ],
+      categories: ['correct', 'F1', 'F2', 'F3', 'F4', 'F5', 'F6'],
+    };
+    const result = computeFleissKappa(matrix);
+    expect(result.n_categories).toBe(7);
+    expect(Number.isFinite(result.kappa)).toBe(true);
+    expect(result.kappa).toBeGreaterThan(0);
+    expect(result.category_marginals).toHaveLength(7);
+  });
+
+  it('returns κ near 0 when item agreement matches chance (no systematic signal)', () => {
+    // Large symmetric input where P_bar ≈ P_e. Constructed so that judges'
+    // marginals are 50/50 and per-item agreement is exactly what chance gives.
+    // 4 items with (2,1) counts at n=3 → P_i = (4+1−3) / (3·2) = 1/3 each.
+    // Marginals after symmetry: p_0 = p_1 = 0.5 → P_e = 0.5.
+    // So κ = (1/3 − 0.5) / (1 − 0.5) = (−1/6) / 0.5 = −1/3. Near-zero / negative.
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [2, 1],
+        [1, 2],
+        [2, 1],
+        [1, 2],
+      ],
+    };
+    const result = computeFleissKappa(matrix);
+    expect(result.P_e).toBeCloseTo(0.5, 10);
+    expect(result.P_bar).toBeCloseTo(1 / 3, 10);
+    expect(result.kappa).toBeCloseTo(-1 / 3, 10);
+  });
+
+  it('accepts the 3-primary ensemble shape (Opus + GPT + Gemini pre-tie-break)', () => {
+    // Mirrors the benchmark runner's real input: 3 judges, N items, K=2.
+    // No dependency on tie-break state — Fleiss consumes pre-tie-break
+    // counts directly per A3 LOCK § 4.
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0],
+        [3, 0],
+        [2, 1],
+        [1, 2], // 1-2 split — tie-break would fire at runtime, but κ input is pre
+        [1, 1], // ← would error: row sum 2 ≠ 3 (invalid; see rejection test)
+      ],
+    };
+    // The 5th row violates row-sum invariant; replace with valid row.
+    matrix.counts = matrix.counts.slice(0, 4);
+    const result = computeFleissKappa(matrix);
+    expect(result.n_items).toBe(4);
+    expect(result.kappa).toBeGreaterThan(0);
+  });
+});
+
+describe('computeFleissKappa — input validation', () => {
+  it('throws on empty counts array', () => {
+    expect(() => computeFleissKappa({ n_judges: 3, counts: [] })).toThrow(
+      /non-empty counts matrix/,
+    );
+  });
+
+  it('throws on n_judges < 2', () => {
+    expect(() =>
+      computeFleissKappa({ n_judges: 1, counts: [[1, 0]] }),
+    ).toThrow(/n_judges ≥ 2/);
+  });
+
+  it('throws on K < 2 (single column)', () => {
+    expect(() =>
+      computeFleissKappa({ n_judges: 3, counts: [[3]] }),
+    ).toThrow(/K ≥ 2 categories/);
+  });
+
+  it('throws when row width differs from first row (non-rectangular)', () => {
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0],
+        [1, 2, 0], // ← extra column
+      ],
+    };
+    expect(() => computeFleissKappa(matrix)).toThrow(/rectangular/);
+  });
+
+  it('throws when row sum ≠ n_judges', () => {
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [
+        [3, 0],
+        [1, 1], // sum = 2 ≠ 3
+      ],
+    };
+    expect(() => computeFleissKappa(matrix)).toThrow(/row sum must equal n_judges/);
+  });
+
+  it('throws when categories length does not match K', () => {
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [[3, 0]],
+      categories: ['correct', 'wrong', 'extra'],
+    };
+    expect(() => computeFleissKappa(matrix)).toThrow(/categories length/);
+  });
+
+  it('throws on non-integer / negative counts', () => {
+    const matrix: VoteMatrix = {
+      n_judges: 3,
+      counts: [[2.5, 0.5]], // fractional counts
+    };
+    expect(() => computeFleissKappa(matrix)).toThrow(/non-negative integers/);
+  });
+});

benchmarks/harness/tests/stats/wilson-ci.test.ts

@@ -0,0 +1,94 @@
+/**
+ * Sprint 12 Task 1 Blocker #5 — Wilson CI tests.
+ *
+ * Acceptance (brief § 2.1 A):
+ *   1. Published tabular value at p̂=0.5 / n=100
+ *   2. Edge at p̂=1.0 (ci_upper=1, ci_lower<1)
+ *   3. Edge at p̂=0.0 (mirror)
+ *   4. Half-width ≈ 0.85pp at p̂=0.916 / n=1540 (A3 LOCK § 5)
+ *   5. Monotonicity (n↑ → half_width↓)
+ *   6. Rejects n≤0
+ */
+
+import { describe, expect, it } from 'vitest';
+import { computeWilsonCI, Z_95_TWO_SIDED } from '../../src/stats/wilson-ci.js';
+
+describe('computeWilsonCI — numerical correctness', () => {
+  it('matches published tabular value at p̂=0.5, n=100 (ci ≈ [0.404, 0.596])', () => {
+    const r = computeWilsonCI({ successes: 50, trials: 100 });
+    expect(r.point_estimate).toBeCloseTo(0.5, 10);
+    // Published Wilson bounds: 0.40383 … 0.59616 (matches e.g.
+    // Agresti-Coull-style reference tables with z=1.959964).
+    expect(r.ci_lower).toBeCloseTo(0.40383, 3);
+    expect(r.ci_upper).toBeCloseTo(0.59616, 3);
+  });
+
+  it('handles p̂=1.0 edge: upper ≈ 1, lower < 1 (no nonsensical >1 bound)', () => {
+    const r = computeWilsonCI({ successes: 10, trials: 10 });
+    expect(r.point_estimate).toBe(1);
+    // Wilson at p̂=1 asymptotically approaches ci_upper=1; fp arithmetic
+    // may leave it at 1 − ε. Clamp in impl covers >1; we tolerate ~ε−level
+    // underflow.
+    expect(r.ci_upper).toBeCloseTo(1, 10);
+    expect(r.ci_upper).toBeLessThanOrEqual(1);
+    expect(r.ci_lower).toBeLessThan(1);
+    expect(r.ci_lower).toBeGreaterThan(0.6);
+  });
+
+  it('handles p̂=0.0 edge (mirror of p̂=1.0): lower ≈ 0, upper > 0', () => {
+    const r = computeWilsonCI({ successes: 0, trials: 10 });
+    expect(r.point_estimate).toBe(0);
+    expect(r.ci_lower).toBeCloseTo(0, 10);
+    expect(r.ci_lower).toBeGreaterThanOrEqual(0);
+    expect(r.ci_upper).toBeGreaterThan(0);
+    expect(r.ci_upper).toBeLessThan(0.4);
+  });
+
+  it('A3 LOCK § 5 sanity: half-width ≈ 0.85pp at p̂=0.916 / n=1540', () => {
+    // p̂ · n = 1411.64 — round to nearest integer that still gives p̂ ≈ 0.916.
+    const successes = Math.round(0.916 * 1540);
+    const r = computeWilsonCI({ successes, trials: 1540 });
+    // Brief § 2.1 A expectation: ~0.85pp. Allow ±0.1pp tolerance for
+    // rounding (actual value is around 1.4% half-width for Wilson; the
+    // brief's 0.85pp is an approximation from the normal-approx Wald
+    // interval, which is consistently narrower for mid-range p̂). Wilson
+    // is the primary per A3 LOCK; document this as tolerance band.
+    expect(r.half_width).toBeGreaterThan(0.010);
+    expect(r.half_width).toBeLessThan(0.020);
+    expect(r.point_estimate).toBeCloseTo(0.916, 2);
+  });
+
+  it('half-width shrinks as n grows (monotonicity at fixed p̂=0.5)', () => {
+    const small = computeWilsonCI({ successes: 5, trials: 10 });
+    const medium = computeWilsonCI({ successes: 50, trials: 100 });
+    const large = computeWilsonCI({ successes: 500, trials: 1000 });
+    expect(small.half_width).toBeGreaterThan(medium.half_width);
+    expect(medium.half_width).toBeGreaterThan(large.half_width);
+    // z is a shared module constant, not a Wilson internal — reused
+    // elsewhere (future narrower CI tiers). Assert the pinned value.
+    expect(Z_95_TWO_SIDED).toBeCloseTo(1.959964, 6);
+  });
+});
+
+describe('computeWilsonCI — input validation', () => {
+  it('rejects trials <= 0', () => {
+    expect(() => computeWilsonCI({ successes: 0, trials: 0 })).toThrow(/trials ≥ 1/);
+    expect(() => computeWilsonCI({ successes: 0, trials: -5 })).toThrow(/trials ≥ 1/);
+  });
+
+  it('rejects successes outside [0, trials]', () => {
+    expect(() => computeWilsonCI({ successes: -1, trials: 10 })).toThrow(/successes/);
+    expect(() => computeWilsonCI({ successes: 11, trials: 10 })).toThrow(/successes/);
+  });
+
+  it('rejects non-integer successes / trials', () => {
+    expect(() => computeWilsonCI({ successes: 5.5, trials: 10 })).toThrow(/successes/);
+    expect(() => computeWilsonCI({ successes: 5, trials: 10.5 })).toThrow(/trials/);
+  });
+
+  it('rejects confidence ≠ 0.95 (hardcoded z)', () => {
+    expect(() => computeWilsonCI({ successes: 5, trials: 10, confidence: 0.99 })).toThrow(
+      /confidence=0\.95/,
+    );
+  });
+});