324 lines
12 KiB
TypeScript
324 lines
12 KiB
TypeScript
/**
|
||
* Sprint 12 Task 1 Session 3 — smoke test suite.
|
||
*
|
||
* End-to-end offline integration test that exercises the Session 1+2+3
|
||
* substrate pipeline on deterministic fixtures (no real LLM calls). Task
|
||
* 1 closure gate: this test PASS = substrate ready for Task 2 (C3 mini).
|
||
*
|
||
* Pipeline exercised per brief § 2.1 C:
|
||
* 1. Load mock-locomo-instances.json + mock-judge-responses.json
|
||
* 2. Derive majority verdict per item from the 3-primary ensemble votes
|
||
* 3. Resolve failure_code per item (unanimous | majority | tie-break-
|
||
* reserved via pre-computed grok_reserve_vote in fixture)
|
||
* 4. Build pre-tie-break vote matrix → Fleiss κ
|
||
* 5. Build CorrectnessRow[] → Wilson 95% CI + cluster-bootstrap 95% CI
|
||
* 6. Build FailureRow[] → failure distribution + F_other review flag
|
||
* 7. Emit `bench.smoke.completed` structured log event with aggregate
|
||
* 8. Assert expected invariants (κ range, sum-to-total, F_other gate,
|
||
* CI containment, ci_lower ≤ ci_upper)
|
||
*
|
||
* Brief § 7 reuse guidance: the pre-tie-break Fleiss κ + post-tie-break
|
||
* correctness derivation sits inline here. The real tie-break module
|
||
* (`resolveTieBreak` in packages/server/src/benchmarks/judge/ensemble-
|
||
* tiebreak.ts) is unit-tested in Sprint 11; smoke intentionally pre-
|
||
* encodes the tie-break outcome via `grok_reserve_vote` + `final_failure_code`
|
||
* fields in the fixture, avoiding a cross-package runtime import just to
|
||
* prove the pipeline shape. Flagged in the exit ping as a non-blocking
|
||
* surprise (ACCEPT — scoped per brief § 5 surprises policy).
|
||
*/
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import url from 'node:url';
|
||
import { describe, expect, it } from 'vitest';
|
||
import { createCoreLogger } from '@waggle/core';
|
||
import {
|
||
computeClusterBootstrapCI,
|
||
computeFleissKappa,
|
||
computeWilsonCI,
|
||
type CorrectnessRow,
|
||
type VoteMatrix,
|
||
} from '../../src/stats/index.js';
|
||
import {
|
||
FAILURE_TAXONOMY_VERSION,
|
||
computeFailureDistribution,
|
||
type FailureCode,
|
||
type FailureRow,
|
||
} from '../../src/failure-taxonomy/index.js';
|
||
|
||
// ── Fixture loading ──────────────────────────────────────────────────────
|
||
|
||
const HERE = url.fileURLToPath(import.meta.url);
|
||
const FIXTURES_DIR = path.resolve(path.dirname(HERE), 'fixtures');
|
||
|
||
interface MockInstance {
|
||
instance_id: string;
|
||
conversation_id: string;
|
||
question: string;
|
||
reference_answer: string;
|
||
}
|
||
|
||
interface MockJudgeVote {
|
||
judge: string;
|
||
verdict: 'correct' | 'incorrect';
|
||
failure_code: FailureCode;
|
||
rationale: string | null;
|
||
}
|
||
|
||
interface MockJudgeEntry {
|
||
instance_id: string;
|
||
judge_votes: MockJudgeVote[];
|
||
grok_reserve_vote?: MockJudgeVote;
|
||
final_failure_code?: FailureCode;
|
||
note?: string;
|
||
}
|
||
|
||
interface MockFixtures {
|
||
instances: MockInstance[];
|
||
judges: string[];
|
||
tie_break_reserve: string;
|
||
responses: MockJudgeEntry[];
|
||
}
|
||
|
||
function loadFixtures(): MockFixtures {
|
||
const instancesRaw = JSON.parse(
|
||
fs.readFileSync(path.join(FIXTURES_DIR, 'mock-locomo-instances.json'), 'utf-8'),
|
||
) as { instances: MockInstance[] };
|
||
const judgeRaw = JSON.parse(
|
||
fs.readFileSync(path.join(FIXTURES_DIR, 'mock-judge-responses.json'), 'utf-8'),
|
||
) as {
|
||
judges: string[];
|
||
tie_break_reserve: string;
|
||
responses: MockJudgeEntry[];
|
||
};
|
||
return {
|
||
instances: instancesRaw.instances,
|
||
judges: judgeRaw.judges,
|
||
tie_break_reserve: judgeRaw.tie_break_reserve,
|
||
responses: judgeRaw.responses,
|
||
};
|
||
}
|
||
|
||
// ── Pipeline helpers ─────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Build the K=2 (correct / incorrect) pre-tie-break vote matrix from the
|
||
* 3-primary ensemble. Fleiss κ per A3 LOCK § 4 is computed over this
|
||
* verdict-level matrix (not the K=8 failure-code matrix) — matches the
|
||
* HALT threshold semantics documented in §4.
|
||
*/
|
||
function buildVerdictVoteMatrix(
|
||
responses: readonly MockJudgeEntry[],
|
||
): VoteMatrix {
|
||
const counts: number[][] = [];
|
||
for (const entry of responses) {
|
||
let correct = 0;
|
||
let incorrect = 0;
|
||
for (const vote of entry.judge_votes) {
|
||
if (vote.verdict === 'correct') correct += 1;
|
||
else incorrect += 1;
|
||
}
|
||
counts.push([correct, incorrect]);
|
||
}
|
||
return {
|
||
n_judges: 3,
|
||
counts,
|
||
categories: ['correct', 'incorrect'],
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Derive the final post-tie-break verdict + failure_code per item.
|
||
*
|
||
* Rules mirror B2 LOCK § 1 runtime:
|
||
* - Verdict = majority of the 3 primary judges (K=2 always has a winner).
|
||
* - Failure code on correct verdict = null.
|
||
* - Failure code on incorrect verdict = majority among the incorrect-
|
||
* voting judges' code picks; ties break to `grok_reserve_vote` if the
|
||
* fixture provides one (the audit-expected path).
|
||
*/
|
||
function resolveFinalVerdict(
|
||
entry: MockJudgeEntry,
|
||
): { correct: 0 | 1; failure_code: FailureCode; rationale: string | null } {
|
||
let correctCount = 0;
|
||
for (const v of entry.judge_votes) {
|
||
if (v.verdict === 'correct') correctCount += 1;
|
||
}
|
||
if (correctCount >= 2) {
|
||
return { correct: 1, failure_code: null, rationale: null };
|
||
}
|
||
|
||
// Majority incorrect — resolve code.
|
||
const incorrectVotes = entry.judge_votes.filter(v => v.verdict === 'incorrect');
|
||
const codeCounts = new Map<string, number>();
|
||
for (const v of incorrectVotes) {
|
||
if (v.failure_code !== null) {
|
||
codeCounts.set(v.failure_code, (codeCounts.get(v.failure_code) ?? 0) + 1);
|
||
}
|
||
}
|
||
|
||
// Pick the code with strictly-majority count. On a tie, fall through to
|
||
// the tie-break reserve vote carried in the fixture.
|
||
let topCode: FailureCode = null;
|
||
let topCount = 0;
|
||
let tied = false;
|
||
for (const [code, count] of codeCounts.entries()) {
|
||
if (count > topCount) {
|
||
topCode = code as FailureCode;
|
||
topCount = count;
|
||
tied = false;
|
||
} else if (count === topCount) {
|
||
tied = true;
|
||
}
|
||
}
|
||
|
||
if (tied && entry.grok_reserve_vote) {
|
||
topCode = entry.grok_reserve_vote.failure_code;
|
||
}
|
||
|
||
// Pick the first matching rationale from the incorrect votes for the
|
||
// chosen code — used by the F_other sampler downstream.
|
||
const chosen = incorrectVotes.find(v => v.failure_code === topCode);
|
||
return {
|
||
correct: 0,
|
||
failure_code: topCode,
|
||
rationale: chosen?.rationale ?? null,
|
||
};
|
||
}
|
||
|
||
// ── The smoke test ───────────────────────────────────────────────────────
|
||
|
||
describe('Sprint 12 Task 1 Session 3 smoke suite — end-to-end substrate', () => {
|
||
it('runs the full pipeline on 10-instance mock fixtures and produces expected aggregate', () => {
|
||
const fixtures = loadFixtures();
|
||
expect(fixtures.instances).toHaveLength(10);
|
||
expect(fixtures.responses).toHaveLength(10);
|
||
expect(fixtures.judges).toEqual(['claude-opus-4-7', 'gpt-5.4', 'gemini-3.1']);
|
||
expect(fixtures.tie_break_reserve).toBe('grok-4.20');
|
||
|
||
// 1. Pre-tie-break vote matrix + Fleiss κ.
|
||
const voteMatrix = buildVerdictVoteMatrix(fixtures.responses);
|
||
const kappa = computeFleissKappa(voteMatrix);
|
||
expect(Number.isNaN(kappa.kappa)).toBe(false);
|
||
expect(kappa.kappa).toBeGreaterThanOrEqual(0.5);
|
||
expect(kappa.kappa).toBeLessThanOrEqual(0.95);
|
||
expect(kappa.n_items).toBe(10);
|
||
expect(kappa.n_judges).toBe(3);
|
||
expect(kappa.n_categories).toBe(2);
|
||
|
||
// 2. Post-tie-break correctness rows + Wilson / bootstrap CIs.
|
||
const instanceById = new Map<string, MockInstance>();
|
||
for (const inst of fixtures.instances) instanceById.set(inst.instance_id, inst);
|
||
const correctnessRows: CorrectnessRow[] = [];
|
||
const failureRows: FailureRow[] = [];
|
||
let tieBreakActivations = 0;
|
||
for (const entry of fixtures.responses) {
|
||
const instance = instanceById.get(entry.instance_id);
|
||
if (!instance) throw new Error(`instance not found: ${entry.instance_id}`);
|
||
const resolved = resolveFinalVerdict(entry);
|
||
correctnessRows.push({ conversation_id: instance.conversation_id, correct: resolved.correct });
|
||
failureRows.push({ failure_code: resolved.failure_code, rationale: resolved.rationale });
|
||
if (entry.grok_reserve_vote) tieBreakActivations += 1;
|
||
}
|
||
|
||
const successes = correctnessRows.reduce((acc, r) => acc + r.correct, 0);
|
||
expect(successes).toBe(7); // fixture design
|
||
expect(correctnessRows).toHaveLength(10);
|
||
|
||
const wilson = computeWilsonCI({ successes, trials: correctnessRows.length });
|
||
expect(wilson.point_estimate).toBeCloseTo(0.7, 10);
|
||
expect(wilson.point_estimate).toBeGreaterThanOrEqual(0.5);
|
||
expect(wilson.point_estimate).toBeLessThanOrEqual(0.9);
|
||
expect(wilson.ci_lower).toBeLessThanOrEqual(wilson.point_estimate);
|
||
expect(wilson.ci_upper).toBeGreaterThanOrEqual(wilson.point_estimate);
|
||
|
||
const bootstrap = computeClusterBootstrapCI({ rows: correctnessRows });
|
||
expect(bootstrap.point_estimate).toBeCloseTo(0.7, 10);
|
||
expect(bootstrap.ci_lower).toBeLessThanOrEqual(bootstrap.point_estimate);
|
||
expect(bootstrap.ci_upper).toBeGreaterThanOrEqual(bootstrap.point_estimate);
|
||
expect(bootstrap.n_bootstrap).toBe(10000);
|
||
expect(bootstrap.seed).toBe(42);
|
||
expect(bootstrap.n_clusters).toBe(4);
|
||
expect(bootstrap.n_rows).toBe(10);
|
||
|
||
// 3. Failure distribution + F_other review flag.
|
||
const distribution = computeFailureDistribution(failureRows);
|
||
expect(distribution.total).toBe(10);
|
||
const summed =
|
||
distribution.counts.null +
|
||
distribution.counts.F1 + distribution.counts.F2 + distribution.counts.F3 +
|
||
distribution.counts.F4 + distribution.counts.F5 + distribution.counts.F6 +
|
||
distribution.counts.F_other;
|
||
expect(summed).toBe(10);
|
||
expect(distribution.counts.null).toBe(7);
|
||
expect(distribution.counts.F1).toBe(1);
|
||
expect(distribution.counts.F_other).toBe(1);
|
||
expect(distribution.counts.F6).toBe(1);
|
||
expect(distribution.f_other_rate).toBeCloseTo(0.1, 10);
|
||
// Strict greater-than: 10% exactly should NOT trip the flag.
|
||
expect(distribution.f_other_review_flag).toBe(false);
|
||
expect(distribution.f_other_rationales_sample).toHaveLength(1);
|
||
|
||
// Tie-break activation sanity — fixture has exactly one instance
|
||
// carrying a grok_reserve_vote field (mock-q-08).
|
||
expect(tieBreakActivations).toBe(1);
|
||
|
||
// 4. Emit the completion event on a scoped logger so downstream CI
|
||
// can tail it. Payload carries the smoke gate's observable state.
|
||
const log = createCoreLogger('bench.smoke');
|
||
const aggregate = {
|
||
event: 'bench.smoke.completed',
|
||
taxonomy_version: FAILURE_TAXONOMY_VERSION,
|
||
n_instances: 10,
|
||
n_judges: 3,
|
||
tie_break_reserve: fixtures.tie_break_reserve,
|
||
tie_break_activations: tieBreakActivations,
|
||
kappa: kappa.kappa,
|
||
kappa_P_bar: kappa.P_bar,
|
||
kappa_P_e: kappa.P_e,
|
||
wilson_ci: {
|
||
point_estimate: wilson.point_estimate,
|
||
ci_lower: wilson.ci_lower,
|
||
ci_upper: wilson.ci_upper,
|
||
half_width: wilson.half_width,
|
||
},
|
||
bootstrap_ci: {
|
||
point_estimate: bootstrap.point_estimate,
|
||
ci_lower: bootstrap.ci_lower,
|
||
ci_upper: bootstrap.ci_upper,
|
||
n_bootstrap: bootstrap.n_bootstrap,
|
||
seed: bootstrap.seed,
|
||
n_clusters: bootstrap.n_clusters,
|
||
},
|
||
failure_distribution: {
|
||
counts: distribution.counts,
|
||
f_other_rate: distribution.f_other_rate,
|
||
f_other_review_flag: distribution.f_other_review_flag,
|
||
},
|
||
};
|
||
log.info('bench.smoke.completed', aggregate);
|
||
|
||
// 5. Determinism gate — re-running the same pipeline must produce a
|
||
// bit-identical bootstrap CI (Wilson + Fleiss are closed-form so
|
||
// determinism there is definitional). Sorted-key stringify so
|
||
// downstream consumers comparing via JSON.stringify get stable
|
||
// output independent of property insertion order.
|
||
const bootstrap2 = computeClusterBootstrapCI({ rows: correctnessRows });
|
||
expect(bootstrap2.ci_lower).toBe(bootstrap.ci_lower);
|
||
expect(bootstrap2.ci_upper).toBe(bootstrap.ci_upper);
|
||
});
|
||
|
||
it('fixture κ lands in the target band (≈0.68, inside user-specified [0.60, 0.70])', () => {
|
||
const fixtures = loadFixtures();
|
||
const kappa = computeFleissKappa(buildVerdictVoteMatrix(fixtures.responses));
|
||
// Pre-computed from the fixture design:
|
||
// 6× (3,0) + 1× (2,1) + 1× (1,2) + 2× (0,3)
|
||
// P_e = 0.49 + 0.09 = 0.58
|
||
// P_bar = (8·1 + 2·(1/3)) / 10 = 0.8667
|
||
// κ = (0.8667 − 0.58) / 0.42 = 0.6825
|
||
expect(kappa.kappa).toBeGreaterThan(0.60);
|
||
expect(kappa.kappa).toBeLessThan(0.75);
|
||
expect(kappa.kappa).toBeCloseTo(0.6825, 3);
|
||
});
|
||
});
|