162 lines
6.3 KiB
TypeScript
162 lines
6.3 KiB
TypeScript
/**
|
||
* Sprint 12 Task 1 Blocker #5 — cluster-bootstrap tests.
|
||
*
|
||
* Acceptance (brief § 2.1 A):
|
||
* 1. Deterministic re-run (same input + seed → same output)
|
||
* 2. Default n_bootstrap = 10 000
|
||
* 3. Default seed = 42
|
||
* 4. Rejects empty rows
|
||
* 5. CI ⊇ point_estimate property
|
||
* 6. ci_lower ≤ ci_upper invariant
|
||
* 7. Cluster structure affects CI vs. instance-level Wilson
|
||
* 8. NaN guard on malformed `correct` field
|
||
*/
|
||
|
||
import { describe, expect, it } from 'vitest';
|
||
import {
|
||
computeClusterBootstrapCI,
|
||
type CorrectnessRow,
|
||
} from '../../src/stats/cluster-bootstrap.js';
|
||
import { computeWilsonCI } from '../../src/stats/wilson-ci.js';
|
||
|
||
function buildClusteredRows(
|
||
clusterCount: number,
|
||
rowsPerCluster: number,
|
||
correctRate: number,
|
||
): CorrectnessRow[] {
|
||
// Deterministic construction: first ⌊rate × rowsPerCluster⌋ rows in each
|
||
// cluster are correct. Keeps tests independent of PRNG state.
|
||
const rows: CorrectnessRow[] = [];
|
||
const correctPerCluster = Math.round(correctRate * rowsPerCluster);
|
||
for (let c = 0; c < clusterCount; c++) {
|
||
const conversation_id = `conv-${c}`;
|
||
for (let r = 0; r < rowsPerCluster; r++) {
|
||
rows.push({ conversation_id, correct: r < correctPerCluster ? 1 : 0 });
|
||
}
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
describe('computeClusterBootstrapCI — determinism + defaults', () => {
|
||
it('produces bit-identical output on two calls with same input', () => {
|
||
const rows = buildClusteredRows(5, 4, 0.75);
|
||
const a = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
|
||
const b = computeClusterBootstrapCI({ rows, n_bootstrap: 500, seed: 42 });
|
||
expect(a.ci_lower).toBe(b.ci_lower);
|
||
expect(a.ci_upper).toBe(b.ci_upper);
|
||
expect(a.point_estimate).toBe(b.point_estimate);
|
||
});
|
||
|
||
it('different seeds produce different bootstrap CIs (seed sensitivity sanity)', () => {
|
||
// 20 singleton clusters with an irregular correct/wrong pattern so that
|
||
// bootstrap resample means span a dense set of values. At n=20 and
|
||
// n_bootstrap=2000, the 2.5th/97.5th percentile indices (50 and 1950)
|
||
// are far from the extremes, so different seeds produce materially
|
||
// different CI bounds.
|
||
const rows: CorrectnessRow[] = [];
|
||
for (let i = 0; i < 20; i++) {
|
||
rows.push({ conversation_id: `c-${i}`, correct: (i % 3 === 0 ? 1 : 0) });
|
||
}
|
||
const a = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
|
||
const b = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 123 });
|
||
expect(a.point_estimate).toBe(b.point_estimate);
|
||
const sameBounds = a.ci_lower === b.ci_lower && a.ci_upper === b.ci_upper;
|
||
expect(sameBounds).toBe(false);
|
||
});
|
||
|
||
it('defaults n_bootstrap=10000 and seed=42 per A3 LOCK § 2', () => {
|
||
const rows = buildClusteredRows(3, 4, 0.5);
|
||
const r = computeClusterBootstrapCI({ rows });
|
||
expect(r.n_bootstrap).toBe(10000);
|
||
expect(r.seed).toBe(42);
|
||
});
|
||
});
|
||
|
||
describe('computeClusterBootstrapCI — structural invariants', () => {
|
||
it('CI contains the point estimate (point ∈ [ci_lower, ci_upper])', () => {
|
||
const rows = buildClusteredRows(8, 4, 0.75);
|
||
const r = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
|
||
expect(r.point_estimate).toBeGreaterThanOrEqual(r.ci_lower);
|
||
expect(r.point_estimate).toBeLessThanOrEqual(r.ci_upper);
|
||
});
|
||
|
||
it('ci_lower ≤ ci_upper always', () => {
|
||
const rows = buildClusteredRows(5, 3, 0.333);
|
||
const r = computeClusterBootstrapCI({ rows, n_bootstrap: 1000, seed: 42 });
|
||
expect(r.ci_lower).toBeLessThanOrEqual(r.ci_upper);
|
||
});
|
||
|
||
it('reports n_clusters = distinct conversation_ids', () => {
|
||
const rows = [
|
||
{ conversation_id: 'a', correct: 1 as const },
|
||
{ conversation_id: 'a', correct: 1 as const },
|
||
{ conversation_id: 'b', correct: 0 as const },
|
||
{ conversation_id: 'c', correct: 1 as const },
|
||
];
|
||
const r = computeClusterBootstrapCI({ rows, n_bootstrap: 100, seed: 42 });
|
||
expect(r.n_clusters).toBe(3);
|
||
expect(r.n_rows).toBe(4);
|
||
});
|
||
|
||
it('produces wider CI than instance-level Wilson when intra-cluster correlation is high', () => {
|
||
// 6 clusters × 4 rows, all-or-nothing correctness within each cluster:
|
||
// 4 clusters all-correct (4×4=16 successes) + 2 clusters all-wrong (0).
|
||
// Intra-cluster correlation is max (1.0) — clusters are homogeneous.
|
||
// Bootstrap should reflect that cluster-level variance is huge (some
|
||
// samples pick all-correct clusters → near 1.0; others pick all-wrong
|
||
// → near 0.0), producing a much wider CI than instance-level Wilson
|
||
// which assumes independent 16/24 successes.
|
||
const rows: CorrectnessRow[] = [];
|
||
for (let c = 0; c < 4; c++) {
|
||
for (let r = 0; r < 4; r++) {
|
||
rows.push({ conversation_id: `correct-${c}`, correct: 1 });
|
||
}
|
||
}
|
||
for (let c = 0; c < 2; c++) {
|
||
for (let r = 0; r < 4; r++) {
|
||
rows.push({ conversation_id: `wrong-${c}`, correct: 0 });
|
||
}
|
||
}
|
||
const bootstrap = computeClusterBootstrapCI({ rows, n_bootstrap: 2000, seed: 42 });
|
||
const wilson = computeWilsonCI({ successes: 16, trials: 24 });
|
||
const bootstrapWidth = bootstrap.ci_upper - bootstrap.ci_lower;
|
||
const wilsonWidth = wilson.ci_upper - wilson.ci_lower;
|
||
expect(bootstrapWidth).toBeGreaterThan(wilsonWidth);
|
||
});
|
||
});
|
||
|
||
describe('computeClusterBootstrapCI — input validation', () => {
|
||
it('rejects empty rows', () => {
|
||
expect(() => computeClusterBootstrapCI({ rows: [] })).toThrow(/non-empty rows/);
|
||
});
|
||
|
||
it('rejects n_bootstrap < 1', () => {
|
||
const rows = buildClusteredRows(2, 2, 0.5);
|
||
expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 0 })).toThrow(
|
||
/n_bootstrap ≥ 1/,
|
||
);
|
||
});
|
||
|
||
it('rejects non-integer seed', () => {
|
||
const rows = buildClusteredRows(2, 2, 0.5);
|
||
expect(() => computeClusterBootstrapCI({ rows, seed: 1.5 })).toThrow(/integer seed/);
|
||
});
|
||
|
||
it('rejects rows with correct ∉ {0, 1}', () => {
|
||
const rows = [
|
||
{ conversation_id: 'a', correct: 1 as 0 | 1 },
|
||
{ conversation_id: 'a', correct: 2 as unknown as 0 | 1 },
|
||
];
|
||
expect(() => computeClusterBootstrapCI({ rows, n_bootstrap: 10, seed: 42 })).toThrow(
|
||
/correct ∈ \{0, 1\}/,
|
||
);
|
||
});
|
||
|
||
it('rejects confidence ≠ 0.95', () => {
|
||
const rows = buildClusteredRows(2, 2, 0.5);
|
||
expect(() => computeClusterBootstrapCI({ rows, confidence: 0.99 })).toThrow(
|
||
/confidence=0\.95/,
|
||
);
|
||
});
|
||
});
|