moving

benchmarks/harness/tests/failure-taxonomy/aggregate.test.ts

@@ -0,0 +1,123 @@
+/**
+ * Sprint 12 Task 1 Blocker #6 — failure distribution aggregator tests.
+ *
+ * Acceptance (brief § 2.1 B):
+ *   1. counts sum equals total
+ *   2. f_other_rate computation correct
+ *   3. review_flag at 11% triggers
+ *   4. review_flag at 10% does NOT trigger (strict greater-than)
+ *   5. sample captures first 10 F_other rationales in order
+ *   6. zero-F_other input yields empty sample
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  computeFailureDistribution,
+  type FailureRow,
+} from '../../src/failure-taxonomy/aggregate.js';
+
+describe('computeFailureDistribution — structural invariants', () => {
+  it('counts sum equals total', () => {
+    const rows: FailureRow[] = [
+      { failure_code: null },
+      { failure_code: null },
+      { failure_code: 'F1' },
+      { failure_code: 'F3' },
+      { failure_code: 'F6' },
+    ];
+    const dist = computeFailureDistribution(rows);
+    expect(dist.total).toBe(5);
+    const summed =
+      dist.counts.null +
+      dist.counts.F1 + dist.counts.F2 + dist.counts.F3 +
+      dist.counts.F4 + dist.counts.F5 + dist.counts.F6 +
+      dist.counts.F_other;
+    expect(summed).toBe(dist.total);
+  });
+
+  it('f_other_rate = F_other count / total', () => {
+    const rows: FailureRow[] = [
+      { failure_code: 'F_other', rationale: 'rationale one two three four five six seven eight nine' },
+      { failure_code: 'F_other', rationale: 'another rationale two three four five six seven eight nine' },
+      { failure_code: null },
+      { failure_code: 'F1' },
+      { failure_code: null },
+    ];
+    const dist = computeFailureDistribution(rows);
+    expect(dist.counts.F_other).toBe(2);
+    expect(dist.total).toBe(5);
+    expect(dist.f_other_rate).toBeCloseTo(2 / 5, 10);
+  });
+});
+
+describe('computeFailureDistribution — F_other review flag threshold', () => {
+  function buildRows(fOtherCount: number, total: number): FailureRow[] {
+    const rows: FailureRow[] = [];
+    for (let i = 0; i < fOtherCount; i++) {
+      rows.push({
+        failure_code: 'F_other',
+        rationale: `rationale ${i} padded padded padded padded padded padded padded padded padded`,
+      });
+    }
+    for (let i = 0; i < total - fOtherCount; i++) {
+      rows.push({ failure_code: null });
+    }
+    return rows;
+  }
+
+  it('flag triggers at 11% (11/100 > 10%)', () => {
+    const dist = computeFailureDistribution(buildRows(11, 100));
+    expect(dist.f_other_rate).toBeCloseTo(0.11, 10);
+    expect(dist.f_other_review_flag).toBe(true);
+  });
+
+  it('flag does NOT trigger at 10% (strict greater-than: 10/100 not > 10%)', () => {
+    const dist = computeFailureDistribution(buildRows(10, 100));
+    expect(dist.f_other_rate).toBeCloseTo(0.10, 10);
+    expect(dist.f_other_review_flag).toBe(false);
+  });
+
+  it('flag does not trigger on empty input', () => {
+    const dist = computeFailureDistribution([]);
+    expect(dist.total).toBe(0);
+    expect(dist.f_other_rate).toBe(0);
+    expect(dist.f_other_review_flag).toBe(false);
+  });
+});
+
+describe('computeFailureDistribution — F_other rationale sample', () => {
+  it('captures first 10 F_other rationales in input order', () => {
+    const rows: FailureRow[] = [];
+    for (let i = 0; i < 15; i++) {
+      rows.push({
+        failure_code: 'F_other',
+        rationale: `rationale-${i} padded padded padded padded padded padded padded padded padded`,
+      });
+    }
+    const dist = computeFailureDistribution(rows);
+    expect(dist.f_other_rationales_sample).toHaveLength(10);
+    expect(dist.f_other_rationales_sample[0]).toMatch(/^rationale-0 /);
+    expect(dist.f_other_rationales_sample[9]).toMatch(/^rationale-9 /);
+  });
+
+  it('zero-F_other input yields empty sample array', () => {
+    const rows: FailureRow[] = [
+      { failure_code: null },
+      { failure_code: 'F1' },
+      { failure_code: 'F2' },
+    ];
+    const dist = computeFailureDistribution(rows);
+    expect(dist.counts.F_other).toBe(0);
+    expect(dist.f_other_rationales_sample).toEqual([]);
+  });
+
+  it('skips F_other rows without a rationale string in the sample (robustness)', () => {
+    const rows: FailureRow[] = [
+      { failure_code: 'F_other', rationale: null },
+      { failure_code: 'F_other', rationale: 'valid rationale one two three four five six seven eight' },
+    ];
+    const dist = computeFailureDistribution(rows);
+    expect(dist.counts.F_other).toBe(2);
+    expect(dist.f_other_rationales_sample).toHaveLength(1);
+  });
+});

benchmarks/harness/tests/failure-taxonomy/codes.test.ts

@@ -0,0 +1,84 @@
+/**
+ * Sprint 12 Task 1 Blocker #6 — failure-code enum + definitions tests.
+ *
+ * Acceptance (brief § 2.1 B):
+ *   1. FAILURE_CODES length === 7 (F1..F6 + F_other)
+ *   2. All definitions present (F1..F6 + F_other)
+ *   3. FailureCode type compiles as the 8-value union (compile-time
+ *      proof via exhaustive switch)
+ *   4. No duplicate code entries
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  FAILURE_CODE_DEFINITIONS,
+  FAILURE_CODES,
+  FAILURE_TAXONOMY_VERSION,
+  F_OTHER_REVIEW_THRESHOLD,
+  type FailureCode,
+} from '../../src/failure-taxonomy/codes.js';
+
+describe('FAILURE_CODES constant', () => {
+  it('lists exactly 7 non-null codes in A3 LOCK §6 order', () => {
+    expect(FAILURE_CODES).toHaveLength(7);
+    expect(FAILURE_CODES).toEqual(['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F_other']);
+  });
+
+  it('has no duplicates', () => {
+    const unique = new Set(FAILURE_CODES);
+    expect(unique.size).toBe(FAILURE_CODES.length);
+  });
+});
+
+describe('FAILURE_CODE_DEFINITIONS record', () => {
+  it('provides a non-empty definition for every non-null code', () => {
+    for (const code of FAILURE_CODES) {
+      const def = FAILURE_CODE_DEFINITIONS[code];
+      expect(typeof def).toBe('string');
+      expect(def.length).toBeGreaterThan(10);
+    }
+  });
+
+  it('definitions match A3 LOCK §6 short-form taxonomy labels', () => {
+    expect(FAILURE_CODE_DEFINITIONS.F1).toContain('contradicts-ground-truth');
+    expect(FAILURE_CODE_DEFINITIONS.F2).toContain('partial-answer');
+    expect(FAILURE_CODE_DEFINITIONS.F3).toContain('off-topic');
+    expect(FAILURE_CODE_DEFINITIONS.F4).toContain('refusal');
+    expect(FAILURE_CODE_DEFINITIONS.F5).toContain('tool-use-error');
+    expect(FAILURE_CODE_DEFINITIONS.F6).toContain('format-violation');
+    expect(FAILURE_CODE_DEFINITIONS.F_other).toContain('F-other');
+  });
+});
+
+describe('FailureCode union shape', () => {
+  it('FailureCode is the exhaustive 8-value union (null + F1..F6 + F_other)', () => {
+    // Compile-time + runtime coverage: every case must be handled, else
+    // TS flags the `never` arm and the test fails at compile.
+    const all: FailureCode[] = [null, 'F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F_other'];
+    expect(all).toHaveLength(8);
+    for (const code of all) {
+      switch (code) {
+        case null:
+        case 'F1':
+        case 'F2':
+        case 'F3':
+        case 'F4':
+        case 'F5':
+        case 'F6':
+        case 'F_other':
+          // Exhaustive — no default needed.
+          break;
+      }
+    }
+  });
+});
+
+describe('taxonomy version + review threshold constants', () => {
+  it('FAILURE_TAXONOMY_VERSION pinned to "F1-F6+other v1"', () => {
+    expect(FAILURE_TAXONOMY_VERSION).toBe('F1-F6+other v1');
+  });
+
+  it('F_OTHER_REVIEW_THRESHOLD = 0.10 (A3 LOCK §6 strict-greater-than gate)', () => {
+    expect(F_OTHER_REVIEW_THRESHOLD).toBe(0.10);
+  });
+});

benchmarks/harness/tests/failure-taxonomy/rubric.test.ts

@@ -0,0 +1,43 @@
+/**
+ * Sprint 12 Task 1 Blocker #6 — judge rubric block tests.
+ *
+ * Acceptance (brief § 2.1 B):
+ *   1. Block contains verbatim "F1 — contradicts-ground-truth"
+ *   2. Block contains verbatim "F6 — format-violation"
+ *   3. Block contains "F-other" escape clause (≥10-word rationale directive)
+ *   4. Block contains taxonomy version tag "F1-F6+other v1"
+ *
+ * Plus: determinism (same bytes on two calls).
+ */
+
+import { describe, expect, it } from 'vitest';
+import { buildJudgeRubricBlock } from '../../src/failure-taxonomy/rubric.js';
+
+describe('buildJudgeRubricBlock', () => {
+  it('contains the F1 — contradicts-ground-truth label verbatim', () => {
+    const block = buildJudgeRubricBlock();
+    expect(block).toContain('F1 — contradicts-ground-truth');
+  });
+
+  it('contains the F6 — format-violation label verbatim', () => {
+    const block = buildJudgeRubricBlock();
+    expect(block).toContain('F6 — format-violation');
+  });
+
+  it('contains the F-other escape clause with the ≥10-word rationale directive', () => {
+    const block = buildJudgeRubricBlock();
+    expect(block).toContain('F-other');
+    expect(block).toContain('≥10-word rationale');
+  });
+
+  it('carries the taxonomy version tag "F1-F6+other v1"', () => {
+    const block = buildJudgeRubricBlock();
+    expect(block).toContain('F1-F6+other v1');
+  });
+
+  it('is deterministic — two successive calls return byte-identical strings', () => {
+    const a = buildJudgeRubricBlock();
+    const b = buildJudgeRubricBlock();
+    expect(a).toBe(b);
+  });
+});

benchmarks/harness/tests/failure-taxonomy/validator.test.ts

@@ -0,0 +1,123 @@
+/**
+ * Sprint 12 Task 1 Blocker #6 — failure-code entry validator tests.
+ *
+ * Acceptance (brief § 2.1 B, 10 tests):
+ *   1. null code + null rationale passes
+ *   2. null code + non-null rationale rejects
+ *   3. F1 + no rationale passes
+ *   4. F_other + 15-word rationale passes
+ *   5. F_other + 5-word rationale rejects (F_other_rationale_too_short)
+ *   6. F_other + null rationale rejects (F_other_rationale_missing)
+ *   7. F_other + whitespace-only rationale rejects
+ *   8. F_other + exactly-10-word rationale passes (boundary)
+ *   9. Invalid code enum rejects
+ *  10. F_other + newline-separated 10-word rationale passes
+ */
+
+import { describe, expect, it } from 'vitest';
+import { validateFailureCodeEntry } from '../../src/failure-taxonomy/validator.js';
+
+describe('validateFailureCodeEntry — null code (correct verdict)', () => {
+  it('null code + null rationale passes', () => {
+    const r = validateFailureCodeEntry({ failure_code: null, rationale: null });
+    expect(r.ok).toBe(true);
+  });
+
+  it('null code + undefined rationale passes', () => {
+    const r = validateFailureCodeEntry({ failure_code: null });
+    expect(r.ok).toBe(true);
+  });
+
+  it('null code + non-null rationale rejects (null_code_with_rationale)', () => {
+    const r = validateFailureCodeEntry({
+      failure_code: null,
+      rationale: 'model was correct but here is a comment',
+    });
+    expect(r.ok).toBe(false);
+    if (!r.ok) {
+      expect(r.code).toBe('null_code_with_rationale');
+    }
+  });
+});
+
+describe('validateFailureCodeEntry — F1..F6 codes', () => {
+  it('F1 + no rationale passes (rationale optional for F1..F6)', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F1' });
+    expect(r.ok).toBe(true);
+  });
+
+  it('F3 + short rationale passes (no length constraint outside F_other)', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F3', rationale: 'bad' });
+    expect(r.ok).toBe(true);
+  });
+
+  it('F6 + null rationale passes', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F6', rationale: null });
+    expect(r.ok).toBe(true);
+  });
+});
+
+describe('validateFailureCodeEntry — F_other code rationale enforcement', () => {
+  it('F_other + 15-word rationale passes', () => {
+    const r = validateFailureCodeEntry({
+      failure_code: 'F_other',
+      rationale:
+        'the model produced a mostly-correct answer but reversed one subject pronoun in the middle which is confusing',
+    });
+    expect(r.ok).toBe(true);
+  });
+
+  it('F_other + 5-word rationale rejects (F_other_rationale_too_short)', () => {
+    const r = validateFailureCodeEntry({
+      failure_code: 'F_other',
+      rationale: 'model hallucinated extra facts wrong',
+    });
+    expect(r.ok).toBe(false);
+    if (!r.ok) {
+      expect(r.code).toBe('F_other_rationale_too_short');
+      expect(r.message).toContain('10');
+    }
+  });
+
+  it('F_other + null rationale rejects (F_other_rationale_missing)', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F_other', rationale: null });
+    expect(r.ok).toBe(false);
+    if (!r.ok) {
+      expect(r.code).toBe('F_other_rationale_missing');
+    }
+  });
+
+  it('F_other + whitespace-only rationale rejects', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F_other', rationale: '    \t\n  ' });
+    expect(r.ok).toBe(false);
+    if (!r.ok) {
+      expect(r.code).toBe('F_other_rationale_missing');
+    }
+  });
+
+  it('F_other + exactly-10-word rationale passes (boundary)', () => {
+    const r = validateFailureCodeEntry({
+      failure_code: 'F_other',
+      rationale: 'one two three four five six seven eight nine ten',
+    });
+    expect(r.ok).toBe(true);
+  });
+
+  it('F_other + newline-separated 10-word rationale passes (tokenize on any whitespace)', () => {
+    const r = validateFailureCodeEntry({
+      failure_code: 'F_other',
+      rationale: 'alpha\nbeta\ngamma\ndelta\nepsilon\nzeta\neta\ntheta\niota\nkappa',
+    });
+    expect(r.ok).toBe(true);
+  });
+});
+
+describe('validateFailureCodeEntry — invalid input', () => {
+  it('rejects a code outside the enum', () => {
+    const r = validateFailureCodeEntry({ failure_code: 'F99' });
+    expect(r.ok).toBe(false);
+    if (!r.ok) {
+      expect(r.code).toBe('invalid_failure_code');
+    }
+  });
+});