350 lines
16 KiB
TypeScript
350 lines
16 KiB
TypeScript
/**
|
||
* GEPA Faza 1 — H3 corpus library tests.
|
||
*
|
||
* Coverage targets per manifest v7 §corpus_design + §amendment_2_integration:
|
||
* - Stratification: exactly 50 unique cells (5 × 5 × 2)
|
||
* - Deterministic enumeration order (canonical)
|
||
* - Instance validation per quality floor
|
||
* - Spot-audit sampler determinism (same seed → same sample)
|
||
* - Spot-audit halt-on-failure semantics
|
||
*/
|
||
|
||
import { describe, expect, it } from 'vitest';
|
||
import {
|
||
TASK_FAMILIES,
|
||
PERSONAS,
|
||
COMPANY_STAGES,
|
||
TOTAL_INSTANCES,
|
||
DOCS_PER_INSTANCE_MIN,
|
||
DOCS_PER_INSTANCE_MAX,
|
||
SPOT_AUDIT_SAMPLE_SIZE,
|
||
STRATIFICATION_SEED,
|
||
TASK_FAMILY_DESCRIPTORS,
|
||
type StratificationCell,
|
||
type CorpusInstance,
|
||
iterateStratificationCells,
|
||
listStratificationCells,
|
||
buildInstanceId,
|
||
validateInstance,
|
||
deterministicSample,
|
||
selectSpotAuditSample,
|
||
runSpotAudit,
|
||
corpusSha256,
|
||
} from '../../src/faza-1/corpus.js';
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// Helpers
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
function makeValidInstance(cell: StratificationCell, ordinal: number = 1): CorpusInstance {
|
||
const docs = Array.from({ length: 7 }, (_, i) => ({
|
||
title: `DOC ${i + 1} — Sample`,
|
||
body: 'x'.repeat(800),
|
||
charCount: 800,
|
||
}));
|
||
return {
|
||
instanceId: buildInstanceId(cell, ordinal),
|
||
cell,
|
||
personaText: 'p'.repeat(200),
|
||
scenario: 's'.repeat(400),
|
||
sourceDocuments: docs,
|
||
question: 'q'.repeat(200),
|
||
materialsConcat: docs.map(d => `## ${d.title}\n\n${d.body}`).join('\n\n---\n\n'),
|
||
manifestAnchor: 'manifest-v7-gepa-faza1',
|
||
generatedBy: 'claude-opus-4-7',
|
||
generatedAtIso: '2026-04-28T00:00:00.000Z',
|
||
generationCostUsd: 0.10,
|
||
};
|
||
}
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// Constants exposed for auditing
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('manifest v7 §corpus_design constants', () => {
|
||
it('TOTAL_INSTANCES = 50', () => {
|
||
expect(TOTAL_INSTANCES).toBe(50);
|
||
});
|
||
it('TASK_FAMILIES has 5 entries', () => {
|
||
expect(TASK_FAMILIES).toHaveLength(5);
|
||
});
|
||
it('PERSONAS has 5 entries', () => {
|
||
expect(PERSONAS).toHaveLength(5);
|
||
});
|
||
it('COMPANY_STAGES has 2 entries', () => {
|
||
expect(COMPANY_STAGES).toHaveLength(2);
|
||
});
|
||
it('5 × 5 × 2 = 50 (stratification yields TOTAL_INSTANCES)', () => {
|
||
expect(TASK_FAMILIES.length * PERSONAS.length * COMPANY_STAGES.length).toBe(TOTAL_INSTANCES);
|
||
});
|
||
it('SPOT_AUDIT_SAMPLE_SIZE = 5 per Amendment 1', () => {
|
||
expect(SPOT_AUDIT_SAMPLE_SIZE).toBe(5);
|
||
});
|
||
it('STRATIFICATION_SEED = 42', () => {
|
||
expect(STRATIFICATION_SEED).toBe(42);
|
||
});
|
||
it('DOCS_PER_INSTANCE bounds', () => {
|
||
expect(DOCS_PER_INSTANCE_MIN).toBe(6);
|
||
expect(DOCS_PER_INSTANCE_MAX).toBe(8);
|
||
});
|
||
it('TASK_FAMILY_DESCRIPTORS covers all 5 families', () => {
|
||
for (const f of TASK_FAMILIES) {
|
||
expect(TASK_FAMILY_DESCRIPTORS[f]).toBeDefined();
|
||
expect(TASK_FAMILY_DESCRIPTORS[f].docsPerInstance).toBeGreaterThanOrEqual(DOCS_PER_INSTANCE_MIN);
|
||
expect(TASK_FAMILY_DESCRIPTORS[f].docsPerInstance).toBeLessThanOrEqual(DOCS_PER_INSTANCE_MAX);
|
||
}
|
||
});
|
||
it('F1-F3 mirror pilot tasks; F4-F5 are net-new', () => {
|
||
expect(TASK_FAMILY_DESCRIPTORS.F1.mirrorPilotTask).toBe('task-1');
|
||
expect(TASK_FAMILY_DESCRIPTORS.F2.mirrorPilotTask).toBe('task-2');
|
||
expect(TASK_FAMILY_DESCRIPTORS.F3.mirrorPilotTask).toBe('task-3');
|
||
expect(TASK_FAMILY_DESCRIPTORS.F4.mirrorPilotTask).toBeNull();
|
||
expect(TASK_FAMILY_DESCRIPTORS.F5.mirrorPilotTask).toBeNull();
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// Stratification enumeration
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('iterateStratificationCells / listStratificationCells', () => {
|
||
it('yields exactly 50 cells (5 × 5 × 2)', () => {
|
||
const cells = listStratificationCells();
|
||
expect(cells).toHaveLength(50);
|
||
});
|
||
|
||
it('all 50 cells are unique', () => {
|
||
const cells = listStratificationCells();
|
||
const keys = new Set(cells.map(c => `${c.family}|${c.persona}|${c.stage}`));
|
||
expect(keys.size).toBe(50);
|
||
});
|
||
|
||
it('canonical ordering: F1 first, F5 last', () => {
|
||
const cells = listStratificationCells();
|
||
expect(cells[0].family).toBe('F1');
|
||
expect(cells[cells.length - 1].family).toBe('F5');
|
||
});
|
||
|
||
it('persona enumeration is the inner loop after family', () => {
|
||
const cells = listStratificationCells();
|
||
// First 10 cells should all be family F1
|
||
for (let i = 0; i < 10; i++) {
|
||
expect(cells[i].family).toBe('F1');
|
||
}
|
||
// Cells 10-19 should all be F2
|
||
for (let i = 10; i < 20; i++) {
|
||
expect(cells[i].family).toBe('F2');
|
||
}
|
||
});
|
||
|
||
it('each (family, persona) pair appears exactly twice (once per stage)', () => {
|
||
const cells = listStratificationCells();
|
||
const pairCounts = new Map<string, number>();
|
||
for (const c of cells) {
|
||
const key = `${c.family}|${c.persona}`;
|
||
pairCounts.set(key, (pairCounts.get(key) ?? 0) + 1);
|
||
}
|
||
expect(pairCounts.size).toBe(25); // 5 × 5 family-persona pairs
|
||
for (const count of pairCounts.values()) {
|
||
expect(count).toBe(2); // once per stage
|
||
}
|
||
});
|
||
|
||
it('iterator is deterministic (same yield order across calls)', () => {
|
||
const a = listStratificationCells();
|
||
const b = listStratificationCells();
|
||
expect(a).toEqual(b);
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// buildInstanceId
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('buildInstanceId', () => {
|
||
it('produces stable IDs with h3- prefix + zero-padded ordinal', () => {
|
||
const id = buildInstanceId({ family: 'F1', persona: 'p2_cfo', stage: 'stage_a_series_b_growth_burning' }, 7);
|
||
expect(id).toBe('h3-F1-p2_cfo-stage_a_series_b_growth_burning-007');
|
||
});
|
||
|
||
it('default ordinal is 1', () => {
|
||
const id = buildInstanceId({ family: 'F3', persona: 'p4_vp_finance', stage: 'stage_b_post_profitable_consolidation' });
|
||
expect(id).toContain('-001');
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// validateInstance
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('validateInstance — quality floor', () => {
|
||
it('valid synthetic instance passes', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(true);
|
||
expect(r.violations).toHaveLength(0);
|
||
});
|
||
|
||
it('FAIL: too few docs', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.sourceDocuments = inst.sourceDocuments.slice(0, 3);
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations[0]).toContain('docs count');
|
||
expect(r.violations[0]).toContain('< 6 min');
|
||
});
|
||
|
||
it('FAIL: too many docs', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.sourceDocuments = [...inst.sourceDocuments, ...inst.sourceDocuments, ...inst.sourceDocuments];
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations[0]).toContain('> 8 max');
|
||
});
|
||
|
||
it('FAIL: persona too short', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.personaText = 'short';
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations.some(v => v.includes('persona length'))).toBe(true);
|
||
});
|
||
|
||
it('PASS: rich persona up to 1500 chars (post-probe loosening)', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.personaText = 'p'.repeat(1400);
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(true);
|
||
});
|
||
|
||
it('FAIL: persona above 1500 cap', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.personaText = 'p'.repeat(1600);
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations.some(v => v.includes('persona length'))).toBe(true);
|
||
});
|
||
|
||
it('PASS: empty scenario (oracle embedded it in personaText)', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.scenario = ''; // embedded case
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(true);
|
||
});
|
||
|
||
it('FAIL: doc charCount mismatch with body length', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.sourceDocuments[0].charCount = 999; // intentional mismatch
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations.some(v => v.includes('charCount'))).toBe(true);
|
||
});
|
||
|
||
it('FAIL: instanceId missing h3- prefix', () => {
|
||
const inst = makeValidInstance({ family: 'F1', persona: 'p1_founder_ceo', stage: 'stage_a_series_b_growth_burning' });
|
||
inst.instanceId = 'wrong-prefix-001';
|
||
const r = validateInstance(inst);
|
||
expect(r.valid).toBe(false);
|
||
expect(r.violations.some(v => v.includes('h3- prefix'))).toBe(true);
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// deterministicSample / selectSpotAuditSample
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('deterministicSample', () => {
|
||
it('returns sample of requested size', () => {
|
||
const items = Array.from({ length: 50 }, (_, i) => `item-${i}`);
|
||
const sample = deterministicSample(items, 5);
|
||
expect(sample).toHaveLength(5);
|
||
});
|
||
|
||
it('same seed produces identical sample (reproducibility)', () => {
|
||
const items = Array.from({ length: 50 }, (_, i) => `item-${i}`);
|
||
const a = deterministicSample(items, 5, 42);
|
||
const b = deterministicSample(items, 5, 42);
|
||
expect(a).toEqual(b);
|
||
});
|
||
|
||
it('different seeds produce different samples', () => {
|
||
const items = Array.from({ length: 50 }, (_, i) => `item-${i}`);
|
||
const a = deterministicSample(items, 5, 42);
|
||
const c = deterministicSample(items, 5, 100);
|
||
expect(a).not.toEqual(c);
|
||
});
|
||
|
||
it('sampleSize >= items.length returns full list copy', () => {
|
||
const items = ['a', 'b', 'c'];
|
||
const sample = deterministicSample(items, 5);
|
||
expect(sample).toHaveLength(3);
|
||
expect(sample).toEqual(items);
|
||
expect(sample).not.toBe(items); // copy, not reference
|
||
});
|
||
});
|
||
|
||
describe('selectSpotAuditSample', () => {
|
||
it('returns 5 instances by default (manifest v7 spot_audit.sample_size)', () => {
|
||
const cells = listStratificationCells();
|
||
const instances = cells.map(c => makeValidInstance(c));
|
||
const sample = selectSpotAuditSample(instances);
|
||
expect(sample).toHaveLength(5);
|
||
});
|
||
|
||
it('reproducible with seed=42 (same instances picked across runs)', () => {
|
||
const cells = listStratificationCells();
|
||
const instances = cells.map(c => makeValidInstance(c));
|
||
const a = selectSpotAuditSample(instances);
|
||
const b = selectSpotAuditSample(instances);
|
||
expect(a.map(i => i.instanceId)).toEqual(b.map(i => i.instanceId));
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// runSpotAudit aggregate
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('runSpotAudit', () => {
|
||
it('PASS when all sampled instances valid', () => {
|
||
const cells = listStratificationCells();
|
||
const instances = cells.map(c => makeValidInstance(c));
|
||
const report = runSpotAudit(instances);
|
||
expect(report.sampleSize).toBe(5);
|
||
expect(report.haltOnFailure).toBe(false);
|
||
expect(report.haltReason).toBeUndefined();
|
||
});
|
||
|
||
it('HALT when any sampled instance invalid (manifest v7 spot_audit.halt_on)', () => {
|
||
const cells = listStratificationCells();
|
||
const instances = cells.map(c => makeValidInstance(c));
|
||
// Corrupt every instance (so sample will definitely include corrupted ones)
|
||
for (const inst of instances) {
|
||
inst.sourceDocuments = inst.sourceDocuments.slice(0, 2); // below 6 min
|
||
}
|
||
const report = runSpotAudit(instances);
|
||
expect(report.haltOnFailure).toBe(true);
|
||
expect(report.haltReason).toMatch(/spot-audit instances failed validation/);
|
||
});
|
||
});
|
||
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
// corpusSha256
|
||
// ───────────────────────────────────────────────────────────────────────────
|
||
|
||
describe('corpusSha256', () => {
|
||
it('produces deterministic SHA across runs', () => {
|
||
const cells = listStratificationCells().slice(0, 5);
|
||
const instances = cells.map(c => makeValidInstance(c));
|
||
const a = corpusSha256(instances);
|
||
const b = corpusSha256(instances);
|
||
expect(a).toBe(b);
|
||
});
|
||
|
||
it('different corpora produce different SHAs', () => {
|
||
const cells = listStratificationCells();
|
||
const a = cells.slice(0, 5).map(c => makeValidInstance(c));
|
||
const b = cells.slice(5, 10).map(c => makeValidInstance(c));
|
||
expect(corpusSha256(a)).not.toBe(corpusSha256(b));
|
||
});
|
||
});
|