/** * GEPA Faza 1 — acceptance validator tests. * * Coverage targets per manifest v7 §amendment_2_integration.scaffold_test_coverage_NEW_requirements * mandatory_acceptance_tests: * - §F.5 FAIL: Qwen candidate with trio_strict delta = +6pp AND mean retrieval = 1.4 → REJECTED * - §F.5 PASS path: Qwen candidate with trio_strict delta = +6pp AND mean retrieval = 1.7 → ACCEPTED * * Plus comprehensive coverage of §F condition 1 (third update) for both Qwen and non-Qwen branches. */ import { describe, expect, it } from 'vitest'; import { evaluateCandidate, TRIO_STRICT_DELTA_THRESHOLD_PP, QWEN_RETRIEVAL_ENGAGEMENT_FLOOR, QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR, } from '../../src/faza-1/acceptance.js'; import { type CandidateMetrics, type ShapeName } from '../../src/faza-1/types.js'; function makeCandidate(overrides: Partial & Pick): CandidateMetrics { return { candidateId: overrides.candidateId ?? `${overrides.shape}-test-candidate`, shape: overrides.shape, evaluations: overrides.evaluations ?? [], trioStrictPassRateII: overrides.trioStrictPassRateII ?? 0.5, trioStrictPassRateI: overrides.trioStrictPassRateI ?? 0.5, meanRetrievalCallsPerTask: overrides.meanRetrievalCallsPerTask ?? 1.5, meanCostUsd: overrides.meanCostUsd ?? 0.5, }; } // ─────────────────────────────────────────────────────────────────────────── // MANDATORY: §F.5 false-positive guard tests per Amendment 2 §5 // ─────────────────────────────────────────────────────────────────────────── describe('§F.5 false-positive evolution guard — mandatory Amendment 2 acceptance tests', () => { it('Qwen candidate, trio_strict delta = +6pp, mean retrieval = 1.4 → REJECTED', () => { // baseline = 0.20, candidate = 0.26 → delta = +6pp ≥ 5pp threshold // retrieval = 1.4 < 1.5 false-positive floor → §F.5 triggers const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.26, meanRetrievalCallsPerTask: 1.4, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.trioStrictDeltaPP).toBeCloseTo(6.0, 6); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(true); expect(verdict.accepted).toBe(false); expect(verdict.reason).toContain('REJECTED §F.5 false-positive guard'); }); it('Qwen candidate, trio_strict delta = +6pp, mean retrieval = 1.7 → ACCEPTED', () => { // baseline = 0.20, candidate = 0.26 → delta = +6pp // retrieval = 1.7 ≥ 1.7 floor (engagement gap closed) → §F.5 does NOT trigger const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.26, meanRetrievalCallsPerTask: 1.7, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.trioStrictDeltaPP).toBeCloseTo(6.0, 6); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.condition1Pass).toBe(true); expect(verdict.accepted).toBe(true); expect(verdict.reason).toContain('PASS §F.1'); }); it('§F.5 boundary: retrieval = 1.49 → REJECTED (just below floor)', () => { const candidate = makeCandidate({ shape: 'qwen-non-thinking', trioStrictPassRateII: 0.30, // delta = +10pp meanRetrievalCallsPerTask: 1.49, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(true); expect(verdict.accepted).toBe(false); }); it('§F.5 boundary: retrieval = 1.50 → §F.5 NOT triggered (exact floor inclusive)', () => { // 1.50 ≥ 1.50 false-positive floor → guard does not fire // But 1.50 < 1.70 §F.1 Qwen floor → condition 1 still fails on different criterion const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.30, // delta = +10pp meanRetrievalCallsPerTask: 1.50, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.condition1Pass).toBe(false); // fails Qwen retrieval floor 1.7 expect(verdict.accepted).toBe(false); expect(verdict.reason).toContain('FAIL §F.1 Qwen retrieval floor'); }); it('§F.5 does NOT trigger when delta < threshold even if retrieval low (Qwen)', () => { // delta = +3pp < 5pp threshold → guard not even evaluated const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.23, meanRetrievalCallsPerTask: 1.0, // low, but delta below threshold }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.condition1Pass).toBe(false); expect(verdict.accepted).toBe(false); expect(verdict.reason).toContain('FAIL §F.1 trio_strict delta'); }); }); // ─────────────────────────────────────────────────────────────────────────── // §F.5 does NOT apply to non-Qwen shapes // ─────────────────────────────────────────────────────────────────────────── describe('§F.5 scoping — only applies to Qwen-targeted shapes', () => { it('claude shape with delta = +6pp + retrieval = 0.5 → ACCEPTED (no false-positive guard)', () => { const candidate = makeCandidate({ shape: 'claude', trioStrictPassRateII: 0.26, meanRetrievalCallsPerTask: 0.5, // would trigger §F.5 if Qwen, but doesn't apply here }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.condition1Pass).toBe(true); expect(verdict.accepted).toBe(true); }); it('gpt shape with delta = +5pp + retrieval = 0 → ACCEPTED', () => { const candidate = makeCandidate({ shape: 'gpt', trioStrictPassRateII: 0.25, meanRetrievalCallsPerTask: 0, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.accepted).toBe(true); }); it('generic-simple shape: only trio_strict delta matters, no retrieval requirement', () => { const candidate = makeCandidate({ shape: 'generic-simple', trioStrictPassRateII: 0.30, meanRetrievalCallsPerTask: 1.0, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.accepted).toBe(true); }); }); // ─────────────────────────────────────────────────────────────────────────── // §F condition 1 trio_strict delta — basic threshold tests // ─────────────────────────────────────────────────────────────────────────── describe('§F condition 1 — trio_strict delta threshold (≥+5pp)', () => { it('delta = +5pp exactly (boundary inclusive) → condition 1 PASS for non-Qwen', () => { const candidate = makeCandidate({ shape: 'claude', trioStrictPassRateII: 0.25, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.trioStrictDeltaPP).toBeCloseTo(5.0, 6); expect(verdict.condition1Pass).toBe(true); expect(verdict.accepted).toBe(true); }); it('delta = +4.99pp (just below boundary) → condition 1 FAIL', () => { const candidate = makeCandidate({ shape: 'claude', trioStrictPassRateII: 0.2499, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition1Pass).toBe(false); expect(verdict.accepted).toBe(false); }); it('negative delta (regression) → condition 1 FAIL', () => { const candidate = makeCandidate({ shape: 'gpt', trioStrictPassRateII: 0.10, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.trioStrictDeltaPP).toBeCloseTo(-10.0, 6); expect(verdict.condition1Pass).toBe(false); }); it('exposes binding constants for external auditing', () => { expect(TRIO_STRICT_DELTA_THRESHOLD_PP).toBe(5); expect(QWEN_RETRIEVAL_ENGAGEMENT_FLOOR).toBe(1.7); expect(QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR).toBe(1.5); }); }); // ─────────────────────────────────────────────────────────────────────────── // §F condition 1 — Qwen-only retrieval engagement floor (1.7) // ─────────────────────────────────────────────────────────────────────────── describe('§F condition 1 — Qwen retrieval engagement floor (1.7) sub-criterion', () => { it('Qwen candidate, delta = +5pp, retrieval = 1.69 → FAIL (below 1.7 floor)', () => { // 1.69 ≥ 1.5 false-positive floor (so §F.5 does not trigger) // but 1.69 < 1.7 §F.1 Qwen floor (so condition 1 fails) const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.25, meanRetrievalCallsPerTask: 1.69, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition5FalsePositiveGuardTriggered).toBe(false); expect(verdict.condition1Pass).toBe(false); expect(verdict.accepted).toBe(false); expect(verdict.reason).toContain('FAIL §F.1 Qwen retrieval floor'); }); it('Qwen candidate, delta = +5pp, retrieval = 1.70 → PASS (exact floor inclusive)', () => { const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.25, meanRetrievalCallsPerTask: 1.70, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.condition1Pass).toBe(true); expect(verdict.accepted).toBe(true); }); it('Qwen candidate, delta = +5pp, retrieval = 2.5 → PASS (well above floor)', () => { const candidate = makeCandidate({ shape: 'qwen-non-thinking', trioStrictPassRateII: 0.25, meanRetrievalCallsPerTask: 2.5, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.accepted).toBe(true); }); }); // ─────────────────────────────────────────────────────────────────────────── // Audit log invariants // ─────────────────────────────────────────────────────────────────────────── describe('audit log invariants', () => { it('reason string includes shape, delta, retrieval, accepted flag', () => { const candidate = makeCandidate({ shape: 'qwen-thinking', trioStrictPassRateII: 0.30, meanRetrievalCallsPerTask: 2.0, }); const verdict = evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.20 }); expect(verdict.reason).toContain('shape=qwen-thinking'); expect(verdict.reason).toContain('trio_strict_delta=10.00pp'); expect(verdict.reason).toContain('mean_retrieval_calls=2.00'); expect(verdict.reason).toContain('accepted=true'); }); it('every shape produces a verdict (no exceptions)', () => { const shapes: ShapeName[] = ['claude', 'qwen-thinking', 'qwen-non-thinking', 'gpt', 'generic-simple']; for (const shape of shapes) { const candidate = makeCandidate({ shape, trioStrictPassRateII: 0.5 }); expect(() => evaluateCandidate({ candidate, baselineTrioStrictPassRateII: 0.4 }), ).not.toThrow(); } }); });