moving

benchmarks/harness/tests/smoke/fixtures/mock-judge-responses.json

@@ -0,0 +1,119 @@
+{
+  "_meta": {
+    "description": "Sprint 12 Task 1 Session 3 smoke test — mock judge responses for the 3-primary ensemble (Opus 4.7, GPT-5.4, Gemini 3.1) across 10 mock instances. One entry per (instance_id, judge_model) pair. Tie-break fourth-vendor (Grok 4.20) reserve vote is captured as a top-level `grok_reserve_vote` field on the single instance that triggers the 1-1 code split among incorrect judges (mock-q-08) — avoids pulling in the runtime resolveTieBreak module since smoke test is pipeline proof, not tie-break unit test.",
+    "kappa_target_band": "[0.60, 0.70]",
+    "kappa_predicted": 0.682,
+    "correctness_target": "7 of 10 final verdicts correct",
+    "brief": "PM-Waggle-OS/briefs/2026-04-22-cc-sprint-12-task1-session3-brief.md §2.1 C"
+  },
+  "judges": ["claude-opus-4-7", "gpt-5.4", "gemini-3.1"],
+  "tie_break_reserve": "grok-4.20",
+  "responses": [
+    {
+      "instance_id": "mock-q-01",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-02",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-03",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-04",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-05",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-06",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct", "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "correct", "failure_code": null, "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-07",
+      "note": "(2,1) majority correct — 1 dissenting judge picks F3 off-topic. Final verdict = correct, no tie-break needed.",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct",   "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "correct",   "failure_code": null, "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "incorrect", "failure_code": "F3", "rationale": null }
+      ]
+    },
+    {
+      "instance_id": "mock-q-08",
+      "note": "(1,2) majority incorrect — 1 correct + 2 incorrect judges split codes (F1 vs F_other). Tie-break reserve (Grok 4.20) votes F1 → final verdict = incorrect, failure_code = F1.",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "correct",   "failure_code": null, "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "incorrect", "failure_code": "F1", "rationale": null },
+        {
+          "judge": "gemini-3.1",
+          "verdict": "incorrect",
+          "failure_code": "F_other",
+          "rationale": "model answered with a completely different year and also attributed the move to the wrong city entirely"
+        }
+      ],
+      "grok_reserve_vote": { "verdict": "incorrect", "failure_code": "F1", "rationale": null },
+      "final_failure_code": "F1"
+    },
+    {
+      "instance_id": "mock-q-09",
+      "note": "(0,3) unanimous F_other — all three judges agree the failure does not fit F1..F6 and each provides a ≥10-word rationale. Takes the sole F_other slot in the fixture so aggregate f_other_rate = 1/10 = 10% (not > 10%, review_flag stays off per A3 LOCK §6 strict-gt semantic).",
+      "judge_votes": [
+        {
+          "judge": "claude-opus-4-7",
+          "verdict": "incorrect",
+          "failure_code": "F_other",
+          "rationale": "model returned a long musical digression about unrelated string instruments rather than naming the one played"
+        },
+        {
+          "judge": "gpt-5.4",
+          "verdict": "incorrect",
+          "failure_code": "F_other",
+          "rationale": "the answer drifts into a tangential essay about orchestra sections and never actually states the instrument"
+        },
+        {
+          "judge": "gemini-3.1",
+          "verdict": "incorrect",
+          "failure_code": "F_other",
+          "rationale": "response compares cello and viola tonal range but fails to commit to a single instrument name"
+        }
+      ]
+    },
+    {
+      "instance_id": "mock-q-10",
+      "note": "(0,3) unanimous F6 format-violation — all three judges agree the content is correct but formatted wrong (e.g. JSON envelope violated).",
+      "judge_votes": [
+        { "judge": "claude-opus-4-7", "verdict": "incorrect", "failure_code": "F6", "rationale": null },
+        { "judge": "gpt-5.4",         "verdict": "incorrect", "failure_code": "F6", "rationale": null },
+        { "judge": "gemini-3.1",      "verdict": "incorrect", "failure_code": "F6", "rationale": null }
+      ]
+    }
+  ]
+}

benchmarks/harness/tests/smoke/fixtures/mock-locomo-instances.json

@@ -0,0 +1,75 @@
+{
+  "_meta": {
+    "description": "Sprint 12 Task 1 Session 3 smoke test — 10-instance synthetic LoCoMo-shaped fixture. Content is synthetic, structure mirrors the real locomo-1540.jsonl row shape at the fields the smoke pipeline exercises (instance_id, conversation_id, question, reference_answer). Prefix 'mock-conv-*' + 'mock-q-*' makes the synthetic-ness explicit so the fixture cannot be confused for a real LoCoMo slice.",
+    "conversation_distribution": {
+      "mock-conv-A": 3,
+      "mock-conv-B": 3,
+      "mock-conv-C": 2,
+      "mock-conv-D": 2
+    },
+    "total_instances": 10,
+    "brief": "PM-Waggle-OS/briefs/2026-04-22-cc-sprint-12-task1-session3-brief.md §2.1 C"
+  },
+  "instances": [
+    {
+      "instance_id": "mock-q-01",
+      "conversation_id": "mock-conv-A",
+      "question": "What day did Alice meet Bob?",
+      "reference_answer": "Tuesday"
+    },
+    {
+      "instance_id": "mock-q-02",
+      "conversation_id": "mock-conv-A",
+      "question": "Where did they go on the weekend?",
+      "reference_answer": "the lake"
+    },
+    {
+      "instance_id": "mock-q-03",
+      "conversation_id": "mock-conv-A",
+      "question": "What did Carol bring to the picnic?",
+      "reference_answer": "potato salad"
+    },
+    {
+      "instance_id": "mock-q-04",
+      "conversation_id": "mock-conv-B",
+      "question": "How many guests came to the party?",
+      "reference_answer": "twelve"
+    },
+    {
+      "instance_id": "mock-q-05",
+      "conversation_id": "mock-conv-B",
+      "question": "What year did Dave move to Berlin?",
+      "reference_answer": "2021"
+    },
+    {
+      "instance_id": "mock-q-06",
+      "conversation_id": "mock-conv-B",
+      "question": "Who hosted the book club?",
+      "reference_answer": "Emma"
+    },
+    {
+      "instance_id": "mock-q-07",
+      "conversation_id": "mock-conv-C",
+      "question": "What kind of car does Frank drive?",
+      "reference_answer": "a blue Subaru"
+    },
+    {
+      "instance_id": "mock-q-08",
+      "conversation_id": "mock-conv-C",
+      "question": "When did Grace finish her PhD?",
+      "reference_answer": "May 2022"
+    },
+    {
+      "instance_id": "mock-q-09",
+      "conversation_id": "mock-conv-D",
+      "question": "What instrument does Henry play?",
+      "reference_answer": "cello"
+    },
+    {
+      "instance_id": "mock-q-10",
+      "conversation_id": "mock-conv-D",
+      "question": "Where does Ivy's sister live?",
+      "reference_answer": "Lisbon"
+    }
+  ]
+}

benchmarks/harness/tests/smoke/outputs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

benchmarks/harness/tests/smoke/smoke-run.test.ts

@@ -0,0 +1,323 @@
+/**
+ * Sprint 12 Task 1 Session 3 — smoke test suite.
+ *
+ * End-to-end offline integration test that exercises the Session 1+2+3
+ * substrate pipeline on deterministic fixtures (no real LLM calls). Task
+ * 1 closure gate: this test PASS = substrate ready for Task 2 (C3 mini).
+ *
+ * Pipeline exercised per brief § 2.1 C:
+ *   1. Load mock-locomo-instances.json + mock-judge-responses.json
+ *   2. Derive majority verdict per item from the 3-primary ensemble votes
+ *   3. Resolve failure_code per item (unanimous | majority | tie-break-
+ *      reserved via pre-computed grok_reserve_vote in fixture)
+ *   4. Build pre-tie-break vote matrix → Fleiss κ
+ *   5. Build CorrectnessRow[] → Wilson 95% CI + cluster-bootstrap 95% CI
+ *   6. Build FailureRow[] → failure distribution + F_other review flag
+ *   7. Emit `bench.smoke.completed` structured log event with aggregate
+ *   8. Assert expected invariants (κ range, sum-to-total, F_other gate,
+ *      CI containment, ci_lower ≤ ci_upper)
+ *
+ * Brief § 7 reuse guidance: the pre-tie-break Fleiss κ + post-tie-break
+ * correctness derivation sits inline here. The real tie-break module
+ * (`resolveTieBreak` in packages/server/src/benchmarks/judge/ensemble-
+ * tiebreak.ts) is unit-tested in Sprint 11; smoke intentionally pre-
+ * encodes the tie-break outcome via `grok_reserve_vote` + `final_failure_code`
+ * fields in the fixture, avoiding a cross-package runtime import just to
+ * prove the pipeline shape. Flagged in the exit ping as a non-blocking
+ * surprise (ACCEPT — scoped per brief § 5 surprises policy).
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { createCoreLogger } from '@waggle/core';
+import {
+  computeClusterBootstrapCI,
+  computeFleissKappa,
+  computeWilsonCI,
+  type CorrectnessRow,
+  type VoteMatrix,
+} from '../../src/stats/index.js';
+import {
+  FAILURE_TAXONOMY_VERSION,
+  computeFailureDistribution,
+  type FailureCode,
+  type FailureRow,
+} from '../../src/failure-taxonomy/index.js';
+
+// ── Fixture loading ──────────────────────────────────────────────────────
+
+const HERE = url.fileURLToPath(import.meta.url);
+const FIXTURES_DIR = path.resolve(path.dirname(HERE), 'fixtures');
+
+interface MockInstance {
+  instance_id: string;
+  conversation_id: string;
+  question: string;
+  reference_answer: string;
+}
+
+interface MockJudgeVote {
+  judge: string;
+  verdict: 'correct' | 'incorrect';
+  failure_code: FailureCode;
+  rationale: string | null;
+}
+
+interface MockJudgeEntry {
+  instance_id: string;
+  judge_votes: MockJudgeVote[];
+  grok_reserve_vote?: MockJudgeVote;
+  final_failure_code?: FailureCode;
+  note?: string;
+}
+
+interface MockFixtures {
+  instances: MockInstance[];
+  judges: string[];
+  tie_break_reserve: string;
+  responses: MockJudgeEntry[];
+}
+
+function loadFixtures(): MockFixtures {
+  const instancesRaw = JSON.parse(
+    fs.readFileSync(path.join(FIXTURES_DIR, 'mock-locomo-instances.json'), 'utf-8'),
+  ) as { instances: MockInstance[] };
+  const judgeRaw = JSON.parse(
+    fs.readFileSync(path.join(FIXTURES_DIR, 'mock-judge-responses.json'), 'utf-8'),
+  ) as {
+    judges: string[];
+    tie_break_reserve: string;
+    responses: MockJudgeEntry[];
+  };
+  return {
+    instances: instancesRaw.instances,
+    judges: judgeRaw.judges,
+    tie_break_reserve: judgeRaw.tie_break_reserve,
+    responses: judgeRaw.responses,
+  };
+}
+
+// ── Pipeline helpers ─────────────────────────────────────────────────────
+
+/**
+ * Build the K=2 (correct / incorrect) pre-tie-break vote matrix from the
+ * 3-primary ensemble. Fleiss κ per A3 LOCK § 4 is computed over this
+ * verdict-level matrix (not the K=8 failure-code matrix) — matches the
+ * HALT threshold semantics documented in §4.
+ */
+function buildVerdictVoteMatrix(
+  responses: readonly MockJudgeEntry[],
+): VoteMatrix {
+  const counts: number[][] = [];
+  for (const entry of responses) {
+    let correct = 0;
+    let incorrect = 0;
+    for (const vote of entry.judge_votes) {
+      if (vote.verdict === 'correct') correct += 1;
+      else incorrect += 1;
+    }
+    counts.push([correct, incorrect]);
+  }
+  return {
+    n_judges: 3,
+    counts,
+    categories: ['correct', 'incorrect'],
+  };
+}
+
+/**
+ * Derive the final post-tie-break verdict + failure_code per item.
+ *
+ * Rules mirror B2 LOCK § 1 runtime:
+ *   - Verdict = majority of the 3 primary judges (K=2 always has a winner).
+ *   - Failure code on correct verdict = null.
+ *   - Failure code on incorrect verdict = majority among the incorrect-
+ *     voting judges' code picks; ties break to `grok_reserve_vote` if the
+ *     fixture provides one (the audit-expected path).
+ */
+function resolveFinalVerdict(
+  entry: MockJudgeEntry,
+): { correct: 0 | 1; failure_code: FailureCode; rationale: string | null } {
+  let correctCount = 0;
+  for (const v of entry.judge_votes) {
+    if (v.verdict === 'correct') correctCount += 1;
+  }
+  if (correctCount >= 2) {
+    return { correct: 1, failure_code: null, rationale: null };
+  }
+
+  // Majority incorrect — resolve code.
+  const incorrectVotes = entry.judge_votes.filter(v => v.verdict === 'incorrect');
+  const codeCounts = new Map<string, number>();
+  for (const v of incorrectVotes) {
+    if (v.failure_code !== null) {
+      codeCounts.set(v.failure_code, (codeCounts.get(v.failure_code) ?? 0) + 1);
+    }
+  }
+
+  // Pick the code with strictly-majority count. On a tie, fall through to
+  // the tie-break reserve vote carried in the fixture.
+  let topCode: FailureCode = null;
+  let topCount = 0;
+  let tied = false;
+  for (const [code, count] of codeCounts.entries()) {
+    if (count > topCount) {
+      topCode = code as FailureCode;
+      topCount = count;
+      tied = false;
+    } else if (count === topCount) {
+      tied = true;
+    }
+  }
+
+  if (tied && entry.grok_reserve_vote) {
+    topCode = entry.grok_reserve_vote.failure_code;
+  }
+
+  // Pick the first matching rationale from the incorrect votes for the
+  // chosen code — used by the F_other sampler downstream.
+  const chosen = incorrectVotes.find(v => v.failure_code === topCode);
+  return {
+    correct: 0,
+    failure_code: topCode,
+    rationale: chosen?.rationale ?? null,
+  };
+}
+
+// ── The smoke test ───────────────────────────────────────────────────────
+
+describe('Sprint 12 Task 1 Session 3 smoke suite — end-to-end substrate', () => {
+  it('runs the full pipeline on 10-instance mock fixtures and produces expected aggregate', () => {
+    const fixtures = loadFixtures();
+    expect(fixtures.instances).toHaveLength(10);
+    expect(fixtures.responses).toHaveLength(10);
+    expect(fixtures.judges).toEqual(['claude-opus-4-7', 'gpt-5.4', 'gemini-3.1']);
+    expect(fixtures.tie_break_reserve).toBe('grok-4.20');
+
+    // 1. Pre-tie-break vote matrix + Fleiss κ.
+    const voteMatrix = buildVerdictVoteMatrix(fixtures.responses);
+    const kappa = computeFleissKappa(voteMatrix);
+    expect(Number.isNaN(kappa.kappa)).toBe(false);
+    expect(kappa.kappa).toBeGreaterThanOrEqual(0.5);
+    expect(kappa.kappa).toBeLessThanOrEqual(0.95);
+    expect(kappa.n_items).toBe(10);
+    expect(kappa.n_judges).toBe(3);
+    expect(kappa.n_categories).toBe(2);
+
+    // 2. Post-tie-break correctness rows + Wilson / bootstrap CIs.
+    const instanceById = new Map<string, MockInstance>();
+    for (const inst of fixtures.instances) instanceById.set(inst.instance_id, inst);
+    const correctnessRows: CorrectnessRow[] = [];
+    const failureRows: FailureRow[] = [];
+    let tieBreakActivations = 0;
+    for (const entry of fixtures.responses) {
+      const instance = instanceById.get(entry.instance_id);
+      if (!instance) throw new Error(`instance not found: ${entry.instance_id}`);
+      const resolved = resolveFinalVerdict(entry);
+      correctnessRows.push({ conversation_id: instance.conversation_id, correct: resolved.correct });
+      failureRows.push({ failure_code: resolved.failure_code, rationale: resolved.rationale });
+      if (entry.grok_reserve_vote) tieBreakActivations += 1;
+    }
+
+    const successes = correctnessRows.reduce((acc, r) => acc + r.correct, 0);
+    expect(successes).toBe(7); // fixture design
+    expect(correctnessRows).toHaveLength(10);
+
+    const wilson = computeWilsonCI({ successes, trials: correctnessRows.length });
+    expect(wilson.point_estimate).toBeCloseTo(0.7, 10);
+    expect(wilson.point_estimate).toBeGreaterThanOrEqual(0.5);
+    expect(wilson.point_estimate).toBeLessThanOrEqual(0.9);
+    expect(wilson.ci_lower).toBeLessThanOrEqual(wilson.point_estimate);
+    expect(wilson.ci_upper).toBeGreaterThanOrEqual(wilson.point_estimate);
+
+    const bootstrap = computeClusterBootstrapCI({ rows: correctnessRows });
+    expect(bootstrap.point_estimate).toBeCloseTo(0.7, 10);
+    expect(bootstrap.ci_lower).toBeLessThanOrEqual(bootstrap.point_estimate);
+    expect(bootstrap.ci_upper).toBeGreaterThanOrEqual(bootstrap.point_estimate);
+    expect(bootstrap.n_bootstrap).toBe(10000);
+    expect(bootstrap.seed).toBe(42);
+    expect(bootstrap.n_clusters).toBe(4);
+    expect(bootstrap.n_rows).toBe(10);
+
+    // 3. Failure distribution + F_other review flag.
+    const distribution = computeFailureDistribution(failureRows);
+    expect(distribution.total).toBe(10);
+    const summed =
+      distribution.counts.null +
+      distribution.counts.F1 + distribution.counts.F2 + distribution.counts.F3 +
+      distribution.counts.F4 + distribution.counts.F5 + distribution.counts.F6 +
+      distribution.counts.F_other;
+    expect(summed).toBe(10);
+    expect(distribution.counts.null).toBe(7);
+    expect(distribution.counts.F1).toBe(1);
+    expect(distribution.counts.F_other).toBe(1);
+    expect(distribution.counts.F6).toBe(1);
+    expect(distribution.f_other_rate).toBeCloseTo(0.1, 10);
+    // Strict greater-than: 10% exactly should NOT trip the flag.
+    expect(distribution.f_other_review_flag).toBe(false);
+    expect(distribution.f_other_rationales_sample).toHaveLength(1);
+
+    // Tie-break activation sanity — fixture has exactly one instance
+    // carrying a grok_reserve_vote field (mock-q-08).
+    expect(tieBreakActivations).toBe(1);
+
+    // 4. Emit the completion event on a scoped logger so downstream CI
+    //    can tail it. Payload carries the smoke gate's observable state.
+    const log = createCoreLogger('bench.smoke');
+    const aggregate = {
+      event: 'bench.smoke.completed',
+      taxonomy_version: FAILURE_TAXONOMY_VERSION,
+      n_instances: 10,
+      n_judges: 3,
+      tie_break_reserve: fixtures.tie_break_reserve,
+      tie_break_activations: tieBreakActivations,
+      kappa: kappa.kappa,
+      kappa_P_bar: kappa.P_bar,
+      kappa_P_e: kappa.P_e,
+      wilson_ci: {
+        point_estimate: wilson.point_estimate,
+        ci_lower: wilson.ci_lower,
+        ci_upper: wilson.ci_upper,
+        half_width: wilson.half_width,
+      },
+      bootstrap_ci: {
+        point_estimate: bootstrap.point_estimate,
+        ci_lower: bootstrap.ci_lower,
+        ci_upper: bootstrap.ci_upper,
+        n_bootstrap: bootstrap.n_bootstrap,
+        seed: bootstrap.seed,
+        n_clusters: bootstrap.n_clusters,
+      },
+      failure_distribution: {
+        counts: distribution.counts,
+        f_other_rate: distribution.f_other_rate,
+        f_other_review_flag: distribution.f_other_review_flag,
+      },
+    };
+    log.info('bench.smoke.completed', aggregate);
+
+    // 5. Determinism gate — re-running the same pipeline must produce a
+    //    bit-identical bootstrap CI (Wilson + Fleiss are closed-form so
+    //    determinism there is definitional). Sorted-key stringify so
+    //    downstream consumers comparing via JSON.stringify get stable
+    //    output independent of property insertion order.
+    const bootstrap2 = computeClusterBootstrapCI({ rows: correctnessRows });
+    expect(bootstrap2.ci_lower).toBe(bootstrap.ci_lower);
+    expect(bootstrap2.ci_upper).toBe(bootstrap.ci_upper);
+  });
+
+  it('fixture κ lands in the target band (≈0.68, inside user-specified [0.60, 0.70])', () => {
+    const fixtures = loadFixtures();
+    const kappa = computeFleissKappa(buildVerdictVoteMatrix(fixtures.responses));
+    // Pre-computed from the fixture design:
+    //   6× (3,0) + 1× (2,1) + 1× (1,2) + 2× (0,3)
+    //   P_e = 0.49 + 0.09 = 0.58
+    //   P_bar = (8·1 + 2·(1/3)) / 10 = 0.8667
+    //   κ = (0.8667 − 0.58) / 0.42 = 0.6825
+    expect(kappa.kappa).toBeGreaterThan(0.60);
+    expect(kappa.kappa).toBeLessThan(0.75);
+    expect(kappa.kappa).toBeCloseTo(0.6825, 3);
+  });
+});