/** * Standing end-to-end premium-contract fixture. * * The per-mechanism locks (verification-gate-loop, skill-distillation- * loop) each test D3 / D1 in ISOLATION (the latter even disables D3 to * isolate). Nothing tests them BOTH default-on, composed in one real * session — which is the realistic premium scenario and the exact * interaction introduced when both gates were added at the same * agent-loop completion boundary. A future loop change that reorders, * double-fires, or lets one gate swallow the other would pass every * unit lock and still break premium. This fixture fails if the composed * contract regresses. Deterministic (mock fetch) → CI-safe, no spend. */ import { describe, it, expect, vi } from 'vitest'; import { runAgentLoop, type AgentLoopConfig } from '../src/agent-loop.js'; import { VERIFICATION_GATE_DIRECTIVE, VERIFICATION_NO_TOOL_DISCLOSURE, } from '../src/verification-gate.js'; import { planSkillDistillation } from '../src/skill-distillation.js'; import type { ToolDefinition } from '../src/tools.js'; type Turn = { content: string | null; tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> }; function mockFetch(turns: Turn[]) { let i = 0; return vi.fn(async (_u: string, _i?: RequestInit) => { const t = turns[i++]; return { ok: true, status: 200, json: async () => ({ choices: [{ message: { role: 'assistant', content: t.content, tool_calls: t.tool_calls }, finish_reason: t.tool_calls ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 10, completion_tokens: 5 }, }), } as unknown as Response; }); } const probe: ToolDefinition = { name: 'probe', description: 'noop', parameters: { type: 'object', properties: {}, required: [] }, execute: async () => 'ok', }; const runTests: ToolDefinition = { name: 'run_tests', description: 'run the relevant test suite', parameters: { type: 'object', properties: {}, required: [] }, execute: async () => 'tests passed', }; // Distinct args — identical calls would (correctly) trip the LoopGuard. const fiveCalls = [1, 2, 3, 4, 5].map(n => ({ id: `c${n}`, function: { name: 'probe', arguments: JSON.stringify({ step: n }) } })); function cfg(fetch: ReturnType, over: Partial = {}): AgentLoopConfig { // BOTH gates default-on — the whole point of this fixture. return { litellmUrl: 'http://x', litellmApiKey: 'k', model: 'm', systemPrompt: 's', tools: [probe, runTests], messages: [{ role: 'user', content: 'do the multi-step task' }], fetch: fetch as unknown as typeof globalThis.fetch, ...over, }; } describe('premium contract — D3 + D1 compose at the completion boundary (standing fixture)', () => { it('a ≥5-tool run that first claims unverified success: D3 corrects, THEN D1 distils, both one-shot, in order', async () => { const unverified = 'All tests pass and the build succeeds.'; const honest = 'UNVERIFIED — I did not run the suite. Across the run I traced the full pipeline.'; const fetch = mockFetch([ { content: null, tool_calls: fiveCalls }, // ≥5 tools { content: unverified }, // D3 must fire (no verify tool ran) { content: honest }, // D3 one-shot done → D1 must fire (≥5 tools, not a refusal) { content: 'Distilled the reusable skill.' }, // both gates spent → loop returns ]); const emitted: string[] = []; const result = await runAgentLoop(cfg(fetch, { onToken: token => emitted.push(token) })); expect(fetch).toHaveBeenCalledTimes(4); const body3 = JSON.parse((fetch.mock.calls[2][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>; const body4 = JSON.parse((fetch.mock.calls[3][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>; // D3 fired before turn 3 (verification corrective injected)… expect(body3.some(m => m.role === 'system' && m.content.includes(VERIFICATION_GATE_DIRECTIVE))).toBe(true); // …and D1 fired before turn 4 (the real distillation directive injected), // i.e. ordering preserved and D3 did NOT swallow D1. const expectedDistill = planSkillDistillation(['probe', 'probe', 'probe', 'probe', 'probe'], honest)!; expect(expectedDistill).not.toBeNull(); expect(body4.some(m => m.role === 'user' && m.content === expectedDistill.directive)).toBe(true); // D3 directive must NOT reappear in turn 4 (one-shot, not re-fired). expect(body4.filter(m => m.content.includes(VERIFICATION_GATE_DIRECTIVE)).length).toBe(1); // Issue #4 — the D3-corrected honest answer is what the caller gets; // D1's distillation runs as a side-effect that does NOT overwrite the // delivered answer with the skill summary. expect(result.content).toBe(honest); expect(emitted).toEqual([honest]); expect(result.toolsUsed.length).toBe(5); }); it('adds local disclosure and still runs D1 when no verification tool is available', async () => { const unverified = 'All tests pass and the build succeeds.'; const onSkillDistillationFire = vi.fn(); const fetch = mockFetch([ { content: null, tool_calls: fiveCalls }, { content: unverified }, { content: 'Distilled the reusable skill.' }, ]); const result = await runAgentLoop(cfg(fetch, { tools: [probe], onSkillDistillationFire, })); expect(fetch).toHaveBeenCalledTimes(3); const body3 = JSON.parse((fetch.mock.calls[2][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>; const accepted = `${unverified}${VERIFICATION_NO_TOOL_DISCLOSURE}`; expect(body3.some(message => message.role === 'assistant' && message.content === accepted)).toBe(true); expect(onSkillDistillationFire).toHaveBeenCalledTimes(1); expect(result.content).toBe(accepted); expect(result.toolsUsed.length).toBe(5); }); it('the common case is untouched — neither default-on gate perturbs a normal short run', async () => { const fetch = mockFetch([ { content: null, tool_calls: [{ id: 'c1', function: { name: 'probe', arguments: '{}' } }] }, { content: 'I updated the config as requested.' }, // neutral, <5 tools ]); const result = await runAgentLoop(cfg(fetch)); expect(fetch).toHaveBeenCalledTimes(2); // returned immediately — no spurious gate turn expect(result.content).toBe('I updated the config as requested.'); }); });