moving

benchmarks/harness/tests/stats/cluster-bootstrap.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Sprint 12 Task 1 Blocker #5 — cluster-bootstrap tests.
+ *
+ * Acceptance (brief § 2.1 A):
+ *   1. Deterministic re-run (same input + seed → same output)
+ *   2. Default n_bootstrap = 10 000
+ *   3. Default seed = 42
+ *   4. Rejects empty rows
+ *   5. CI ⊇ point_estimate property
+ *   6. ci_lower ≤ ci_upper invariant
+ *   7. Cluster structure affects CI vs. instance-level Wilson
+ *   8. NaN guard on malformed `correct` field
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  computeClusterBootstrapCI,
+  type CorrectnessRow,
+} from '../../src/stats/cluster-bootstrap.js';
+import { computeWilsonCI } from '../../src/stats/wilson-ci.js';
+
+function buildClusteredRows(
+  clusterCount: number,
+  rowsPerCluster: number,
+  correctRate: number,
+): CorrectnessRow[] {
+  // Deterministic construction: first ⌊rate × rowsPerCluster⌋ rows in each
+  // cluster are correct. Keeps tests independent of PRNG state.
+  const rows: CorrectnessRow[] = [];
+  const correctPerCluster = Math.round(correctRate * rowsPerCluster);
+  for (let c = 0; c < clusterCount; c++) {
+    const conversation_id = `conv-${c}`;
+    for (let r = 0; r < rowsPerCluster; r++) {
+      rows.push({ conversation_id, correct: r < correctPerCluster ? 1 : 0 });
+    }
+  }
+  return rows;
+}
+
+describe('computeClusterBootstrapCI — determinism + defaults', () => {
+  it('produces bit-identical output on two calls with same input', () => {
+    const rows = buildClusteredRows(5, 4, 0.75);
+    const a = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
+    const b = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
+    expect(a.ci_lower).toBe(b.ci_lower);
+    expect(a.ci_upper).toBe(b.ci_upper);
+    expect(a.point_estimate).toBe(b.point_estimate);
+  });
+
+  it('different seeds produce different bootstrap CIs (seed sensitivity sanity)', () => {
+    // 20 singleton clusters with an irregular correct/wrong pattern so that
+    // bootstrap resample means span a dense set of values. At n=20 and
+    // n_bootstrap=2000, the 2.5th/97.5th percentile indices (50 and 1950)
+    // are far from the extremes, so different seeds produce materially
+    // different CI bounds.
+    const rows: CorrectnessRow[] = [];
+    for (let i = 0; i < 20; i++) {
+      rows.push({ conversation_id: `c-${i}`, correct: (i % 3 === 0 ? 1 : 0) });
+    }
+    const a = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    const b = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 123 });
+    expect(a.point_estimate).toBe(b.point_estimate);
+    const sameBounds = a.ci_lower === b.ci_lower && a.ci_upper === b.ci_upper;
+    expect(sameBounds).toBe(false);
+  });
+
+  it('defaults n_bootstrap=10000 and seed=42 per A3 LOCK § 2', () => {
+    const rows = buildClusteredRows(3, 4, 0.5);
+    const r = computeClusterBootstrapCI({ rows });
+    expect(r.n_bootstrap).toBe(10000);
+    expect(r.seed).toBe(42);
+  });
+});
+
+describe('computeClusterBootstrapCI — structural invariants', () => {
+  it('CI contains the point estimate (point ∈ [ci_lower, ci_upper])', () => {
+    const rows = buildClusteredRows(8, 4, 0.75);
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    expect(r.point_estimate).toBeGreaterThanOrEqual(r.ci_lower);
+    expect(r.point_estimate).toBeLessThanOrEqual(r.ci_upper);
+  });
+
+  it('ci_lower ≤ ci_upper always', () => {
+    const rows = buildClusteredRows(5, 3, 0.333);
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 1000, seed: 42 });
+    expect(r.ci_lower).toBeLessThanOrEqual(r.ci_upper);
+  });
+
+  it('reports n_clusters = distinct conversation_ids', () => {
+    const rows = [
+      { conversation_id: 'a', correct: 1 as const },
+      { conversation_id: 'a', correct: 1 as const },
+      { conversation_id: 'b', correct: 0 as const },
+      { conversation_id: 'c', correct: 1 as const },
+    ];
+    const r = computeClusterBootstrapCI({ rows, n_bootstrap: 100, seed: 42 });
+    expect(r.n_clusters).toBe(3);
+    expect(r.n_rows).toBe(4);
+  });
+
+  it('produces wider CI than instance-level Wilson when intra-cluster correlation is high', () => {
+    // 6 clusters × 4 rows, all-or-nothing correctness within each cluster:
+    // 4 clusters all-correct (4×4=16 successes) + 2 clusters all-wrong (0).
+    // Intra-cluster correlation is max (1.0) — clusters are homogeneous.
+    // Bootstrap should reflect that cluster-level variance is huge (some
+    // samples pick all-correct clusters → near 1.0; others pick all-wrong
+    // → near 0.0), producing a much wider CI than instance-level Wilson
+    // which assumes independent 16/24 successes.
+    const rows: CorrectnessRow[] = [];
+    for (let c = 0; c < 4; c++) {
+      for (let r = 0; r < 4; r++) {
+        rows.push({ conversation_id: `correct-${c}`, correct: 1 });
+      }
+    }
+    for (let c = 0; c < 2; c++) {
+      for (let r = 0; r < 4; r++) {
+        rows.push({ conversation_id: `wrong-${c}`, correct: 0 });
+      }
+    }
+    const bootstrap = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
+    const wilson = computeWilsonCI({ successes: 16, trials: 24 });
+    const bootstrapWidth = bootstrap.ci_upper - bootstrap.ci_lower;
+    const wilsonWidth = wilson.ci_upper - wilson.ci_lower;
+    expect(bootstrapWidth).toBeGreaterThan(wilsonWidth);
+  });
+});
+
+describe('computeClusterBootstrapCI — input validation', () => {
+  it('rejects empty rows', () => {
+    expect(() => computeClusterBootstrapCI({ rows: [] })).toThrow(/non-empty rows/);
+  });
+
+  it('rejects n_bootstrap < 1', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 0 })).toThrow(
+      /n_bootstrap ≥ 1/,
+    );
+  });
+
+  it('rejects non-integer seed', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, seed: 1.5 })).toThrow(/integer seed/);
+  });
+
+  it('rejects rows with correct ∉ {0, 1}', () => {
+    const rows = [
+      { conversation_id: 'a', correct: 1 as 0 | 1 },
+      { conversation_id: 'a', correct: 2 as unknown as 0 | 1 },
+    ];
+    expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 10, seed: 42 })).toThrow(
+      /correct ∈ \{0, 1\}/,
+    );
+  });
+
+  it('rejects confidence ≠ 0.95', () => {
+    const rows = buildClusteredRows(2, 2, 0.5);
+    expect(() => computeClusterBootstrapCI({ rows, confidence: 0.99 })).toThrow(
+      /confidence=0\.95/,
+    );
+  });
+});