/** * Sprint 12 Task 1 Blocker #1 — dataset loader smoke tests. * * Acceptance criteria (per brief §1): * 1. Canonical LoCoMo archive is present at the expected path, loads, * and contains a stable number of instances. (Paper claim: 1540. * Actual non-adversarial-with-evidence count: 1531. Delta documented * in locomo-1540.meta.json.) * 2. `getDatasetVersion` returns a deterministic SHA-256 hex across 3 * consecutive calls against the same archive. * 3. `loadDataset` throws `DatasetMissingError` when the archive path * is absent. * 4. `BENCH_SYNTHETIC_DATASET=1` env flag re-enables the synthetic * fallback (dev convenience only). * 5. `getDatasetVersion` for synthetic specs returns the static * `synthetic-scaffold-v1` string. * 6. The computed hash matches the one written into * `locomo-1540.meta.json` by the build script. * * Zero LLM calls. Pure loader + hash verification. */ import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import url from 'node:url'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { DatasetMissingError, SYNTHETIC_DATASET_VERSION, getDatasetVersion, loadDataset, } from '../src/datasets.js'; import type { DatasetSpec } from '../src/types.js'; const HERE = url.fileURLToPath(import.meta.url); const HARNESS_ROOT = path.resolve(path.dirname(HERE), '..'); const DATA_ROOT = path.resolve(HARNESS_ROOT, '..', 'data'); const LOCOMO_ARCHIVE = path.join(DATA_ROOT, 'locomo', 'locomo-1540.jsonl'); const LOCOMO_META = path.join(DATA_ROOT, 'locomo', 'locomo-1540.meta.json'); const LOCOMO_SPEC: DatasetSpec = { id: 'locomo', displayName: 'LoCoMo canonical', dataPath: 'locomo/locomo-1540.jsonl', source: 'external', }; const SYNTHETIC_SPEC: DatasetSpec = { id: 'synthetic', displayName: 'Synthetic scaffold', dataPath: 'synthetic/placeholder.jsonl', source: 'synthetic', }; describe('canonical LoCoMo archive', () => { it('exists at the expected path', () => { expect(fs.existsSync(LOCOMO_ARCHIVE)).toBe(true); }); it('loads via loadDataset with a positive instance count', () => { const instances = loadDataset(LOCOMO_SPEC, DATA_ROOT); // Actual count is 1531 at build time (paper claim 1540 minus 9 edge // cases with no resolvable evidence). Assert the known-good number so // silent drift is caught; update deliberately if the upstream source // is replaced. expect(instances.length).toBe(1531); for (const inst of instances) { expect(inst.instance_id).toMatch(/^locomo_conv-\d+_q\d{3}$/); expect(typeof inst.question).toBe('string'); expect(inst.question.length).toBeGreaterThan(0); expect(Array.isArray(inst.expected)).toBe(true); expect(inst.expected.length).toBeGreaterThan(0); } }); it('sidecar meta.json records the same count and a pinned hash', () => { expect(fs.existsSync(LOCOMO_META)).toBe(true); const meta = JSON.parse(fs.readFileSync(LOCOMO_META, 'utf-8')) as { dataset_version: string; instance_count: number; paper_total_claim: number; actual_count: number; }; expect(meta.instance_count).toBe(1531); expect(meta.actual_count).toBe(1531); expect(meta.paper_total_claim).toBe(1540); expect(meta.dataset_version).toMatch(/^[0-9a-f]{64}$/); }); }); describe('getDatasetVersion determinism', () => { it('returns identical SHA-256 hex across 3 consecutive calls', () => { const a = getDatasetVersion(LOCOMO_SPEC, DATA_ROOT); const b = getDatasetVersion(LOCOMO_SPEC, DATA_ROOT); const c = getDatasetVersion(LOCOMO_SPEC, DATA_ROOT); expect(a).toBe(b); expect(b).toBe(c); expect(a).toMatch(/^[0-9a-f]{64}$/); }); it('matches the hash written to locomo-1540.meta.json', () => { const version = getDatasetVersion(LOCOMO_SPEC, DATA_ROOT); const meta = JSON.parse(fs.readFileSync(LOCOMO_META, 'utf-8')) as { dataset_version: string; }; expect(version).toBe(meta.dataset_version); }); it('reproduces the hash when computed externally from the same bytes', () => { const version = getDatasetVersion(LOCOMO_SPEC, DATA_ROOT); const buf = fs.readFileSync(LOCOMO_ARCHIVE); const manual = crypto.createHash('sha256').update(buf).digest('hex'); expect(version).toBe(manual); }); it('returns the static string for synthetic specs', () => { expect(getDatasetVersion(SYNTHETIC_SPEC, DATA_ROOT)).toBe(SYNTHETIC_DATASET_VERSION); expect(SYNTHETIC_DATASET_VERSION).toBe('synthetic-scaffold-v1'); }); }); describe('missing-archive behaviour', () => { let tmpRoot: string; const previousEnv = process.env.BENCH_SYNTHETIC_DATASET; beforeEach(() => { tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-bench-missing-')); delete process.env.BENCH_SYNTHETIC_DATASET; }); afterEach(() => { fs.rmSync(tmpRoot, { recursive: true, force: true }); if (previousEnv === undefined) { delete process.env.BENCH_SYNTHETIC_DATASET; } else { process.env.BENCH_SYNTHETIC_DATASET = previousEnv; } }); it('loadDataset throws DatasetMissingError when archive is absent', () => { expect(() => loadDataset(LOCOMO_SPEC, tmpRoot)).toThrow(DatasetMissingError); }); it('getDatasetVersion throws DatasetMissingError when archive is absent', () => { expect(() => getDatasetVersion(LOCOMO_SPEC, tmpRoot)).toThrow(DatasetMissingError); }); it('DatasetMissingError exposes dataset id + resolved path', () => { try { loadDataset(LOCOMO_SPEC, tmpRoot); throw new Error('expected throw'); } catch (err) { expect(err).toBeInstanceOf(DatasetMissingError); const typed = err as DatasetMissingError; expect(typed.datasetId).toBe('locomo'); expect(typed.resolvedPath.endsWith('locomo-1540.jsonl')).toBe(true); expect(typed.message).toContain('build-locomo-canonical'); expect(typed.message).toContain('BENCH_SYNTHETIC_DATASET=1'); } }); it('BENCH_SYNTHETIC_DATASET=1 re-enables synthetic fallback for loadDataset', () => { process.env.BENCH_SYNTHETIC_DATASET = '1'; const instances = loadDataset(LOCOMO_SPEC, tmpRoot); // Synthetic scaffold is 60 deterministic instances. expect(instances.length).toBe(60); for (const inst of instances) { expect(inst.instance_id).toMatch(/^synthetic_\d{3}$/); } }); it('BENCH_SYNTHETIC_DATASET=1 returns the synthetic version string', () => { process.env.BENCH_SYNTHETIC_DATASET = '1'; expect(getDatasetVersion(LOCOMO_SPEC, tmpRoot)).toBe(SYNTHETIC_DATASET_VERSION); }); });