moving

benchmarks/harness/tests/jsonl-record-schema.test.ts

@@ -0,0 +1,265 @@
+/**
+ * JsonlRecord extension tests (Sprint 9 Task 1).
+ *
+ * Extension spec: `PM-Waggle-OS/strategy/2026-04-20-failure-mode-taxonomy.md` §9
+ * Brief: `PM-Waggle-OS/briefs/2026-04-20-cc-sprint-9-tasks.md` Task 1
+ *
+ * These tests prove three acceptance criteria from the brief:
+ *   (i)  A pre-judge JSONL record (Sprint-7/8 shape, no judge fields)
+ *        parses as a JsonlRecord without error — backward compatibility.
+ *   (ii) A post-judge JSONL record (Task-1 extended shape) parses and
+ *        preserves every judge field.
+ *   (iii) The `JudgeVerdict` union is a closed enum — downstream code that
+ *        exhaustive-switches on it caught at compile time. We enforce this
+ *        with a compile-time never-check so TypeScript proves the coverage
+ *        rather than depending on runtime validation (no Zod dep on the
+ *        harness side yet; TypeScript is the check as per the brief:
+ *        "Ako postoji schema validator (Zod ili sl.), dopuniti; ako ne,
+ *        preskoči i ostavi TypeScript checking").
+ */
+
+import { describe, it, expect } from 'vitest';
+import type {
+  JsonlRecord,
+  FailureMode,
+  JudgeVerdict,
+  JudgeEnsembleEntry,
+  PinningSurface,
+} from '../src/types.js';
+
+// Fixture: pre-judge record exactly as Sprint-7/8 runners emitted it.
+const PRE_JUDGE_JSONL = JSON.stringify({
+  turnId: '5a02a79a-0a56-4e1d-a6e5-bc5a7b80b19f',
+  cell: 'raw',
+  instance_id: 'locomo_conv-26_q000',
+  model: 'qwen3.6-35b-a3b',
+  seed: 42,
+  accuracy: 1,
+  p50_latency_ms: 820,
+  p95_latency_ms: 1240,
+  usd_per_query: 0.000017,
+  failure_mode: null,
+});
+
+// Fixture: post-judge record with every new field populated, ensemble shape.
+const POST_JUDGE_JSONL = JSON.stringify({
+  turnId: '9f16c4b2-e831-4a23-8a97-cd8b1e4c7210',
+  cell: 'full-context',
+  instance_id: 'locomo_conv-26_q001',
+  model: 'qwen3.6-35b-a3b',
+  seed: 42,
+  accuracy: 1,
+  p50_latency_ms: 1145,
+  p95_latency_ms: 1540,
+  usd_per_query: 0.000087,
+  failure_mode: null,
+  model_answer: '7 May 2023',
+  judge_verdict: 'correct',
+  judge_failure_mode: null,
+  judge_rationale: 'Answer matches ground truth date precisely.',
+  judge_model: 'claude-sonnet-4-6',
+  judge_timestamp: '2026-04-21T14:32:00.000Z',
+  judge_confidence: 0.98,
+  judge_ensemble: [
+    { model: 'claude-sonnet-4-6', verdict: 'correct', failure_mode: null, latency_ms: 920 },
+    { model: 'claude-haiku-4-5', verdict: 'correct', failure_mode: null, latency_ms: 410 },
+  ],
+});
+
+// Fixture: incorrect verdict with a specific failure mode — asserts the
+// binary verdict + failure_mode pairing that replaces the brief Task-1
+// combined 6-value enum.
+const INCORRECT_JSONL = JSON.stringify({
+  turnId: 'c7d54b11-a2e8-4c50-8f96-1a3b00c4ff70',
+  cell: 'raw',
+  instance_id: 'locomo_conv-26_q002',
+  model: 'qwen3.6-35b-a3b',
+  seed: 42,
+  accuracy: 0,
+  p50_latency_ms: 890,
+  p95_latency_ms: 1300,
+  usd_per_query: 0.000021,
+  failure_mode: null,
+  model_answer: 'The event took place on 12 December 2024.',
+  judge_verdict: 'incorrect',
+  judge_failure_mode: 'F3',
+  judge_rationale: 'Model states a date that contradicts the ground-truth context.',
+  judge_model: 'claude-sonnet-4-6',
+  judge_timestamp: '2026-04-21T14:33:05.000Z',
+});
+
+describe('JsonlRecord backward compatibility (Task 1 acceptance)', () => {
+  it('parses a pre-judge Sprint-7/8 record without error', () => {
+    const parsed = JSON.parse(PRE_JUDGE_JSONL) as JsonlRecord;
+    expect(parsed.turnId).toMatch(/^[0-9a-f-]{36}$/);
+    expect(parsed.cell).toBe('raw');
+    expect(parsed.failure_mode).toBeNull();
+    // All judge fields must be absent — treated as "not judged yet".
+    expect(parsed.judge_verdict).toBeUndefined();
+    expect(parsed.judge_failure_mode).toBeUndefined();
+    expect(parsed.judge_rationale).toBeUndefined();
+    expect(parsed.judge_model).toBeUndefined();
+    expect(parsed.judge_timestamp).toBeUndefined();
+    expect(parsed.judge_confidence).toBeUndefined();
+    expect(parsed.judge_ensemble).toBeUndefined();
+    expect(parsed.model_answer).toBeUndefined();
+  });
+
+  it('parses a post-judge record with ensemble and preserves every field', () => {
+    const parsed = JSON.parse(POST_JUDGE_JSONL) as JsonlRecord;
+    expect(parsed.model_answer).toBe('7 May 2023');
+    expect(parsed.judge_verdict).toBe('correct');
+    expect(parsed.judge_failure_mode).toBeNull();
+    expect(parsed.judge_rationale).toBe('Answer matches ground truth date precisely.');
+    expect(parsed.judge_model).toBe('claude-sonnet-4-6');
+    expect(parsed.judge_timestamp).toBe('2026-04-21T14:32:00.000Z');
+    expect(parsed.judge_confidence).toBeCloseTo(0.98, 3);
+    expect(parsed.judge_ensemble).toHaveLength(2);
+    expect(parsed.judge_ensemble?.[0].model).toBe('claude-sonnet-4-6');
+    expect(parsed.judge_ensemble?.[0].latency_ms).toBe(920);
+  });
+
+  it('parses an incorrect record with a populated failure_mode code', () => {
+    const parsed = JSON.parse(INCORRECT_JSONL) as JsonlRecord;
+    expect(parsed.judge_verdict).toBe('incorrect');
+    expect(parsed.judge_failure_mode).toBe('F3');
+    expect(parsed.judge_ensemble).toBeUndefined(); // single-judge run
+  });
+});
+
+describe('JsonlRecord judge-field type closedness (Task 1 acceptance)', () => {
+  // Compile-time never-check: any new value in the JudgeVerdict union
+  // will produce a TypeScript error here, forcing the author to update
+  // the aggregator and schema consumers. Serves as the "invalid string"
+  // gate the brief specified: TS catches at tsc time instead of at
+  // runtime via Zod.
+  it('JudgeVerdict is an exhaustive closed union', () => {
+    const verdicts: JudgeVerdict[] = ['correct', 'incorrect'];
+    for (const v of verdicts) {
+      switch (v) {
+        case 'correct':
+          expect(v).toBe('correct');
+          break;
+        case 'incorrect':
+          expect(v).toBe('incorrect');
+          break;
+        default: {
+          const _exhaustive: never = v;
+          throw new Error(`unreachable: ${_exhaustive as string}`);
+        }
+      }
+    }
+  });
+
+  it('FailureMode is exactly F1..F5 — no extras or aliases', () => {
+    const codes: FailureMode[] = ['F1', 'F2', 'F3', 'F4', 'F5'];
+    expect(codes).toHaveLength(5);
+    for (const code of codes) {
+      expect(code).toMatch(/^F[1-5]$/);
+    }
+  });
+
+  it('JudgeEnsembleEntry carries model + verdict + failure_mode at minimum', () => {
+    const entry: JudgeEnsembleEntry = {
+      model: 'claude-sonnet-4-6',
+      verdict: 'incorrect',
+      failure_mode: 'F4',
+    };
+    expect(entry.model).toBe('claude-sonnet-4-6');
+    expect(entry.verdict).toBe('incorrect');
+    expect(entry.failure_mode).toBe('F4');
+    // Optional fields are assignable without being required.
+    const withOptionals: JudgeEnsembleEntry = {
+      ...entry,
+      rationale: 'hallucinated a name',
+      latency_ms: 540,
+    };
+    expect(withOptionals.rationale).toBeTruthy();
+    expect(withOptionals.latency_ms).toBe(540);
+  });
+});
+
+// ── Sprint 12 Task 1 / B3 addendum § 4 — pinning surface fields ──────────
+
+describe('JsonlRecord B3 addendum pinning fields (Sub-deliverable C)', () => {
+  it('accepts an anthropic_immutable target row with null carve-out + null revision', () => {
+    const row: JsonlRecord = {
+      turnId: '11111111-1111-1111-1111-111111111111',
+      cell: 'raw',
+      instance_id: 'locomo_conv-26_q000',
+      model: 'claude-opus-4-7',
+      seed: 42,
+      accuracy: 1,
+      p50_latency_ms: 900,
+      p95_latency_ms: 1500,
+      usd_per_query: 0.0006,
+      failure_mode: null,
+      model_pinning_surface: 'anthropic_immutable',
+      model_pinning_carve_out_reason: null,
+      model_revision_hash: null,
+    };
+    expect(row.model_pinning_surface).toBe('anthropic_immutable');
+    expect(row.model_pinning_carve_out_reason).toBeNull();
+    expect(row.model_revision_hash).toBeNull();
+  });
+
+  it('accepts a floating_alias target row with non-null carve-out reason', () => {
+    const row: JsonlRecord = {
+      turnId: '22222222-2222-2222-2222-222222222222',
+      cell: 'filtered',
+      instance_id: 'locomo_conv-26_q001',
+      model: 'qwen3.6-35b-a3b-stage2',
+      seed: 42,
+      accuracy: 0,
+      p50_latency_ms: 1100,
+      p95_latency_ms: 1700,
+      usd_per_query: 0.0012,
+      failure_mode: null,
+      model_pinning_surface: 'floating_alias',
+      model_pinning_carve_out_reason:
+        'DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5',
+      model_revision_hash: null,
+    };
+    expect(row.model_pinning_surface).toBe('floating_alias');
+    expect(typeof row.model_pinning_carve_out_reason).toBe('string');
+    expect((row.model_pinning_carve_out_reason as string).length).toBeGreaterThan(10);
+  });
+
+  it('parses a pre-Sprint-12 row (pinning fields absent) without error — backward compat', () => {
+    const raw = JSON.stringify({
+      turnId: '33333333-3333-3333-3333-333333333333',
+      cell: 'raw',
+      instance_id: 'locomo_conv-26_q002',
+      model: 'qwen3.6-35b-a3b',
+      seed: 42,
+      accuracy: 1,
+      p50_latency_ms: 800,
+      p95_latency_ms: 1200,
+      usd_per_query: 0.0008,
+      failure_mode: null,
+    });
+    const parsed = JSON.parse(raw) as JsonlRecord;
+    expect(parsed.model_pinning_surface).toBeUndefined();
+    expect(parsed.model_pinning_carve_out_reason).toBeUndefined();
+    expect(parsed.model_revision_hash).toBeUndefined();
+    expect(parsed.turnId).toMatch(/^[0-9a-f-]{36}$/);
+  });
+
+  it('PinningSurface union is exactly the B3 addendum § 4 three-value enum', () => {
+    const surfaces: PinningSurface[] = ['anthropic_immutable', 'floating_alias', 'revision_hash_pinned'];
+    expect(surfaces).toHaveLength(3);
+    for (const s of surfaces) {
+      switch (s) {
+        case 'anthropic_immutable':
+        case 'floating_alias':
+        case 'revision_hash_pinned':
+          expect(s).toBeTruthy();
+          break;
+        default: {
+          const _exhaustive: never = s;
+          throw new Error(`unreachable: ${_exhaustive as string}`);
+        }
+      }
+    }
+  });
+});