moving

benchmarks/harness/tests/preregistration.test.ts

@@ -0,0 +1,342 @@
+/**
+ * Sprint 12 Task 1 Blocker #3 — pre-registration emitter tests.
+ *
+ * Acceptance criteria (per brief § 4.1):
+ *   1. Payload schema valid — every required field present and typed right.
+ *   2. Emitter fires exactly once when called.
+ *   3. Payload includes canonical dataset SHA-256.
+ *   4. Payload includes manifest hash (CLI override OR auto-computed).
+ *   5. Manifest hash deterministic across 3 reads.
+ *   6. `resolveManifestPath` honors `BENCH_SPEC_MANIFEST_PATH` env var.
+ *   7. `ManifestNotFoundError` thrown when path absent + no override.
+ *   8. `readManifestLockedDate` normalises YAML `locked_date: YYYY-MM-DD`
+ *      to ISO-8601 `YYYY-MM-DDT00:00:00Z`.
+ *   9. `sanitizeArgv` redacts API-key-shaped arguments.
+ *   10. Event name matches the canonical `bench.preregistration.manifest_hash`.
+ *
+ * No LLM calls. No real manifest — uses tmp-dir fixture YAML.
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+  type MockInstance,
+} from 'vitest';
+import {
+  CANONICAL_MANIFEST_PATH,
+  ManifestNotFoundError,
+  PREREGISTRATION_EVENT_NAME,
+  RUNNER_VERSION_FALLBACK,
+  computeBenchSpecManifestHash,
+  emitPreregistrationManifest,
+  getRunnerVersion,
+  readManifestLockedDate,
+  resolveManifestPath,
+  sanitizeArgv,
+  type PreregistrationManifestPayload,
+} from '../src/preregistration.js';
+
+// Fixture YAML that mirrors the A3 LOCK v1 `locked_date:` line exactly
+// so the regex extractor + hash functions get real-shape input. Kept
+// minimal — tests don't need the full 250-line canonical doc.
+const FIXTURE_YAML = `# Bench-Spec LOCK v1 — machine-readable twin (test fixture)
+manifest_version: v1.0.0
+manifest_type: bench_spec_lock_parent
+locked_date: 2026-04-22
+authority: PM (Marko Marković) — A3 interview 7/7 closed 2026-04-22
+sprint: 11
+track: A
+task: A3
+`;
+
+// Known-good SHA-256 of FIXTURE_YAML bytes. Computed once here and
+// asserted in determinism tests — any accidental fixture mutation
+// surfaces as a test break, not silent drift.
+const FIXTURE_HASH = crypto.createHash('sha256').update(FIXTURE_YAML, 'utf-8').digest('hex');
+
+function makeValidPayload(overrides: Partial<PreregistrationManifestPayload> = {}): PreregistrationManifestPayload {
+  return {
+    manifest_hash: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+    manifest_path: CANONICAL_MANIFEST_PATH,
+    manifest_locked_at: '2026-04-22T00:00:00Z',
+    dataset_version: '39e415e2f3a0fa1bd3cb1804a58d0b440b50d3070b2100698437e4ec402a5b24',
+    dataset_path: 'locomo/locomo-1540.jsonl',
+    dataset_instance_count: 1531,
+    per_cell: ['raw', 'filtered', 'compressed', 'full-context'],
+    judge_tiebreak: 'quadri-vendor',
+    judge_models: [],
+    emitted_at: '2026-04-22T12:00:00.000Z',
+    runner_version: 'abc1234',
+    runner_invocation: { argv: ['node', 'runner.ts'], cwd: '/tmp/test' },
+    ...overrides,
+  };
+}
+
+describe('PreregistrationManifestPayload schema (criterion 1)', () => {
+  it('accepts a fully-populated payload', () => {
+    const payload = makeValidPayload();
+    // Compile-time proof: TS picks up the interface. Runtime proof: every
+    // required field is a string/number/array of the right shape.
+    expect(typeof payload.manifest_hash).toBe('string');
+    expect(payload.manifest_hash).toMatch(/^[0-9a-f]{64}$/);
+    expect(typeof payload.manifest_path).toBe('string');
+    expect(typeof payload.manifest_locked_at).toBe('string');
+    expect(typeof payload.dataset_version).toBe('string');
+    expect(typeof payload.dataset_path).toBe('string');
+    expect(typeof payload.dataset_instance_count).toBe('number');
+    expect(Array.isArray(payload.per_cell)).toBe(true);
+    expect(typeof payload.judge_tiebreak).toBe('string');
+    expect(Array.isArray(payload.judge_models)).toBe(true);
+    expect(typeof payload.emitted_at).toBe('string');
+    expect(typeof payload.runner_version).toBe('string');
+    expect(typeof payload.runner_invocation.cwd).toBe('string');
+    expect(Array.isArray(payload.runner_invocation.argv)).toBe(true);
+  });
+
+  it('carries the canonical manifest path constant', () => {
+    expect(CANONICAL_MANIFEST_PATH).toBe('decisions/2026-04-22-bench-spec-locked.manifest.yaml');
+  });
+});
+
+describe('emitPreregistrationManifest (criteria 2, 10)', () => {
+  let infoSpy: MockInstance;
+
+  beforeEach(() => {
+    // createCoreLogger(...).info() routes to console.error (stderr) so library
+    // log lines never corrupt stdout machine-consumers (hive-mind-core/src/logger.ts).
+    infoSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+  });
+  afterEach(() => {
+    infoSpy.mockRestore();
+  });
+
+  it('emits exactly once per call with the canonical event name', () => {
+    emitPreregistrationManifest(makeValidPayload());
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+    const call = infoSpy.mock.calls[0];
+    // createCoreLogger('bench.preregistration') emits
+    //   `[waggle:bench.preregistration] <msg>` as first arg, then payload.
+    expect(String(call[0])).toContain('[waggle:bench.preregistration]');
+    expect(String(call[0])).toContain(PREREGISTRATION_EVENT_NAME);
+    const payload = call[1] as Record<string, unknown>;
+    expect(payload.event).toBe(PREREGISTRATION_EVENT_NAME);
+    expect(PREREGISTRATION_EVENT_NAME).toBe('bench.preregistration.manifest_hash');
+  });
+
+  it('payload carries canonical dataset SHA + instance count verbatim', () => {
+    emitPreregistrationManifest(makeValidPayload({
+      dataset_version: 'abc123',
+      dataset_instance_count: 1531,
+    }));
+    const payload = infoSpy.mock.calls[0][1] as Record<string, unknown>;
+    expect(payload.dataset_version).toBe('abc123');
+    expect(payload.dataset_instance_count).toBe(1531);
+  });
+});
+
+describe('resolveManifestPath (criteria 6, 7)', () => {
+  let tmp: string;
+  const savedEnv = process.env.BENCH_SPEC_MANIFEST_PATH;
+
+  beforeEach(() => {
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-prereg-'));
+    delete process.env.BENCH_SPEC_MANIFEST_PATH;
+  });
+  afterEach(() => {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    if (savedEnv === undefined) {
+      delete process.env.BENCH_SPEC_MANIFEST_PATH;
+    } else {
+      process.env.BENCH_SPEC_MANIFEST_PATH = savedEnv;
+    }
+  });
+
+  it('resolves explicit override path when it exists', () => {
+    const fixturePath = path.join(tmp, 'bench-spec.manifest.yaml');
+    fs.writeFileSync(fixturePath, FIXTURE_YAML, 'utf-8');
+    expect(resolveManifestPath(fixturePath)).toBe(fixturePath);
+  });
+
+  it('throws ManifestNotFoundError when explicit override is absent', () => {
+    const missing = path.join(tmp, 'does-not-exist.yaml');
+    expect(() => resolveManifestPath(missing)).toThrow(ManifestNotFoundError);
+  });
+
+  it('honors BENCH_SPEC_MANIFEST_PATH env var when set', () => {
+    const fixturePath = path.join(tmp, 'env-driven.yaml');
+    fs.writeFileSync(fixturePath, FIXTURE_YAML, 'utf-8');
+    process.env.BENCH_SPEC_MANIFEST_PATH = fixturePath;
+    expect(resolveManifestPath()).toBe(fixturePath);
+  });
+
+  it('ManifestNotFoundError exposes the attempted paths', () => {
+    const missing = path.join(tmp, 'missing.yaml');
+    try {
+      resolveManifestPath(missing);
+      throw new Error('expected throw');
+    } catch (err) {
+      expect(err).toBeInstanceOf(ManifestNotFoundError);
+      const typed = err as ManifestNotFoundError;
+      expect(typed.attemptedPaths.length).toBeGreaterThan(0);
+      expect(typed.message).toContain('BENCH_SPEC_MANIFEST_PATH');
+      expect(typed.message).toContain('--manifest-hash');
+    }
+  });
+});
+
+describe('computeBenchSpecManifestHash (criteria 4, 5)', () => {
+  let tmp: string;
+  let fixturePath: string;
+
+  beforeEach(() => {
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-hash-'));
+    fixturePath = path.join(tmp, 'manifest.yaml');
+    fs.writeFileSync(fixturePath, FIXTURE_YAML, 'utf-8');
+  });
+  afterEach(() => {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it('returns deterministic SHA-256 hex (matches byte hash)', () => {
+    const h = computeBenchSpecManifestHash(fixturePath);
+    expect(h).toBe(FIXTURE_HASH);
+    expect(h).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('is deterministic across 3 consecutive calls', () => {
+    const a = computeBenchSpecManifestHash(fixturePath);
+    const b = computeBenchSpecManifestHash(fixturePath);
+    const c = computeBenchSpecManifestHash(fixturePath);
+    expect(a).toBe(b);
+    expect(b).toBe(c);
+  });
+
+  it('changes when the manifest YAML bytes change', () => {
+    const h1 = computeBenchSpecManifestHash(fixturePath);
+    fs.writeFileSync(fixturePath, FIXTURE_YAML + '# a single-byte change\n', 'utf-8');
+    const h2 = computeBenchSpecManifestHash(fixturePath);
+    expect(h1).not.toBe(h2);
+  });
+});
+
+describe('readManifestLockedDate (criterion 8)', () => {
+  let tmp: string;
+  let fixturePath: string;
+
+  beforeEach(() => {
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-locked-date-'));
+    fixturePath = path.join(tmp, 'manifest.yaml');
+  });
+  afterEach(() => {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it('normalises YYYY-MM-DD to ISO-8601 midnight UTC', () => {
+    fs.writeFileSync(fixturePath, FIXTURE_YAML, 'utf-8');
+    expect(readManifestLockedDate(fixturePath)).toBe('2026-04-22T00:00:00Z');
+  });
+
+  it('passes through full ISO-8601 timestamps unchanged', () => {
+    fs.writeFileSync(fixturePath, 'locked_date: 2026-04-22T14:30:00Z\n', 'utf-8');
+    expect(readManifestLockedDate(fixturePath)).toBe('2026-04-22T14:30:00Z');
+  });
+
+  it('returns "unknown" when the field is absent', () => {
+    fs.writeFileSync(fixturePath, 'other_field: value\n', 'utf-8');
+    expect(readManifestLockedDate(fixturePath)).toBe('unknown');
+  });
+});
+
+describe('sanitizeArgv (criterion 9)', () => {
+  it('redacts values after --api-key / --bearer / --token / --key', () => {
+    const argv = ['node', 'runner.ts', '--api-key', 'secret-key-123', '--token', 'bearer-abc'];
+    const result = sanitizeArgv(argv);
+    expect(result).toEqual([
+      'node', 'runner.ts', '--api-key', '[REDACTED]', '--token', '[REDACTED]',
+    ]);
+  });
+
+  it('redacts sk-* and Bearer * tokens inline', () => {
+    const argv = ['node', 'runner.ts', 'sk-proj-abcdef0123', 'Bearer xyz789'];
+    const result = sanitizeArgv(argv);
+    expect(result).toEqual(['node', 'runner.ts', '[REDACTED]', '[REDACTED]']);
+  });
+
+  it('passes through normal arguments unchanged', () => {
+    const argv = ['node', 'runner.ts', '--cell', 'raw', '--seed', '42'];
+    expect(sanitizeArgv(argv)).toEqual(argv);
+  });
+});
+
+describe('getRunnerVersion', () => {
+  it('returns a string — either git short SHA or the fallback', () => {
+    const v = getRunnerVersion();
+    expect(typeof v).toBe('string');
+    expect(v.length).toBeGreaterThan(0);
+    // Either looks like a short SHA (7-40 hex chars) or is the fallback.
+    expect(/^[0-9a-f]{7,40}$/.test(v) || v === RUNNER_VERSION_FALLBACK).toBe(true);
+  });
+});
+
+// ── Sub-deliverable C — per-judge pinning in manifest payload ────────────
+
+describe('judge_models B3 addendum pinning (Sub-deliverable C)', () => {
+  let infoSpy: MockInstance;
+
+  beforeEach(() => {
+    // createCoreLogger(...).info() routes to console.error (stderr) so library
+    // log lines never corrupt stdout machine-consumers (hive-mind-core/src/logger.ts).
+    infoSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+  });
+  afterEach(() => {
+    infoSpy.mockRestore();
+  });
+
+  it('payload.judge_models carries per-model pinning_surface + carve_out_reason', () => {
+    const payload = makeValidPayload({
+      judge_models: [
+        {
+          model_id: 'claude-opus-4-7',
+          provider: 'anthropic',
+          judge_role: 'primary',
+          pinning_surface: 'anthropic_immutable',
+          pinning_surface_carve_out_reason: null,
+        },
+        {
+          model_id: 'gemini-3.1',
+          provider: 'google_via_openrouter',
+          judge_role: 'secondary',
+          pinning_surface: 'floating_alias',
+          pinning_surface_carve_out_reason:
+            'Google does not expose immutable model snapshots through OpenRouter routing layer; floating alias mandated by B3 addendum § 5',
+        },
+      ],
+    });
+    emitPreregistrationManifest(payload);
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+    const emitted = infoSpy.mock.calls[0][1] as { judge_models: Array<Record<string, unknown>> };
+    expect(emitted.judge_models).toHaveLength(2);
+    expect(emitted.judge_models[0].model_id).toBe('claude-opus-4-7');
+    expect(emitted.judge_models[0].pinning_surface).toBe('anthropic_immutable');
+    expect(emitted.judge_models[0].pinning_surface_carve_out_reason).toBeNull();
+    expect(emitted.judge_models[1].model_id).toBe('gemini-3.1');
+    expect(emitted.judge_models[1].pinning_surface).toBe('floating_alias');
+    expect((emitted.judge_models[1].pinning_surface_carve_out_reason as string)).toMatch(/B3 addendum/);
+  });
+
+  it('payload.judge_models[] is empty when judging is disabled (schema-valid)', () => {
+    const payload = makeValidPayload({ judge_models: [] });
+    emitPreregistrationManifest(payload);
+    const emitted = infoSpy.mock.calls[0][1] as { judge_models: unknown[] };
+    expect(Array.isArray(emitted.judge_models)).toBe(true);
+    expect(emitted.judge_models).toHaveLength(0);
+  });
+});