waggle-os/packages/agent/tests/premium-contract-e2e.test.ts

/**
 * Standing end-to-end premium-contract fixture.
 *
 * The per-mechanism locks (verification-gate-loop, skill-distillation-
 * loop) each test D3 / D1 in ISOLATION (the latter even disables D3 to
 * isolate). Nothing tests them BOTH default-on, composed in one real
 * session — which is the realistic premium scenario and the exact
 * interaction introduced when both gates were added at the same
 * agent-loop completion boundary. A future loop change that reorders,
 * double-fires, or lets one gate swallow the other would pass every
 * unit lock and still break premium. This fixture fails if the composed
 * contract regresses. Deterministic (mock fetch) → CI-safe, no spend.
 */
import { describe, it, expect, vi } from 'vitest';
import { runAgentLoop, type AgentLoopConfig } from '../src/agent-loop.js';
import {
  VERIFICATION_GATE_DIRECTIVE,
  VERIFICATION_NO_TOOL_DISCLOSURE,
} from '../src/verification-gate.js';
import { planSkillDistillation } from '../src/skill-distillation.js';
import type { ToolDefinition } from '../src/tools.js';

type Turn = { content: string | null; tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> };

function mockFetch(turns: Turn[]) {
  let i = 0;
  return vi.fn(async (_u: string, _i?: RequestInit) => {
    const t = turns[i++];
    return {
      ok: true, status: 200,
      json: async () => ({
        choices: [{ message: { role: 'assistant', content: t.content, tool_calls: t.tool_calls }, finish_reason: t.tool_calls ? 'tool_calls' : 'stop' }],
        usage: { prompt_tokens: 10, completion_tokens: 5 },
      }),
    } as unknown as Response;
  });
}

const probe: ToolDefinition = {
  name: 'probe', description: 'noop',
  parameters: { type: 'object', properties: {}, required: [] },
  execute: async () => 'ok',
};
const runTests: ToolDefinition = {
  name: 'run_tests', description: 'run the relevant test suite',
  parameters: { type: 'object', properties: {}, required: [] },
  execute: async () => 'tests passed',
};
// Distinct args — identical calls would (correctly) trip the LoopGuard.
const fiveCalls = [1, 2, 3, 4, 5].map(n => ({ id: `c${n}`, function: { name: 'probe', arguments: JSON.stringify({ step: n }) } }));

function cfg(fetch: ReturnType<typeof mockFetch>, over: Partial<AgentLoopConfig> = {}): AgentLoopConfig {
  // BOTH gates default-on — the whole point of this fixture.
  return {
    litellmUrl: 'http://x', litellmApiKey: 'k', model: 'm', systemPrompt: 's',
    tools: [probe, runTests], messages: [{ role: 'user', content: 'do the multi-step task' }],
    fetch: fetch as unknown as typeof globalThis.fetch,
    ...over,
  };
}

describe('premium contract — D3 + D1 compose at the completion boundary (standing fixture)', () => {
  it('a ≥5-tool run that first claims unverified success: D3 corrects, THEN D1 distils, both one-shot, in order', async () => {
    const unverified = 'All tests pass and the build succeeds.';
    const honest = 'UNVERIFIED — I did not run the suite. Across the run I traced the full pipeline.';
    const fetch = mockFetch([
      { content: null, tool_calls: fiveCalls },   // ≥5 tools
      { content: unverified },                    // D3 must fire (no verify tool ran)
      { content: honest },                        // D3 one-shot done → D1 must fire (≥5 tools, not a refusal)
      { content: 'Distilled the reusable skill.' }, // both gates spent → loop returns
    ]);

    const emitted: string[] = [];
    const result = await runAgentLoop(cfg(fetch, { onToken: token => emitted.push(token) }));

    expect(fetch).toHaveBeenCalledTimes(4);
    const body3 = JSON.parse((fetch.mock.calls[2][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>;
    const body4 = JSON.parse((fetch.mock.calls[3][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>;

    // D3 fired before turn 3 (verification corrective injected)…
    expect(body3.some(m => m.role === 'system' && m.content.includes(VERIFICATION_GATE_DIRECTIVE))).toBe(true);
    // …and D1 fired before turn 4 (the real distillation directive injected),
    // i.e. ordering preserved and D3 did NOT swallow D1.
    const expectedDistill = planSkillDistillation(['probe', 'probe', 'probe', 'probe', 'probe'], honest)!;
    expect(expectedDistill).not.toBeNull();
    expect(body4.some(m => m.role === 'user' && m.content === expectedDistill.directive)).toBe(true);
    // D3 directive must NOT reappear in turn 4 (one-shot, not re-fired).
    expect(body4.filter(m => m.content.includes(VERIFICATION_GATE_DIRECTIVE)).length).toBe(1);

    // Issue #4 — the D3-corrected honest answer is what the caller gets;
    // D1's distillation runs as a side-effect that does NOT overwrite the
    // delivered answer with the skill summary.
    expect(result.content).toBe(honest);
    expect(emitted).toEqual([honest]);
    expect(result.toolsUsed.length).toBe(5);
  });

  it('adds local disclosure and still runs D1 when no verification tool is available', async () => {
    const unverified = 'All tests pass and the build succeeds.';
    const onSkillDistillationFire = vi.fn();
    const fetch = mockFetch([
      { content: null, tool_calls: fiveCalls },
      { content: unverified },
      { content: 'Distilled the reusable skill.' },
    ]);

    const result = await runAgentLoop(cfg(fetch, {
      tools: [probe],
      onSkillDistillationFire,
    }));

    expect(fetch).toHaveBeenCalledTimes(3);
    const body3 = JSON.parse((fetch.mock.calls[2][1] as RequestInit).body as string).messages as Array<{ role: string; content: string }>;
    const accepted = `${unverified}${VERIFICATION_NO_TOOL_DISCLOSURE}`;
    expect(body3.some(message => message.role === 'assistant' && message.content === accepted)).toBe(true);
    expect(onSkillDistillationFire).toHaveBeenCalledTimes(1);
    expect(result.content).toBe(accepted);
    expect(result.toolsUsed.length).toBe(5);
  });

  it('the common case is untouched — neither default-on gate perturbs a normal short run', async () => {
    const fetch = mockFetch([
      { content: null, tool_calls: [{ id: 'c1', function: { name: 'probe', arguments: '{}' } }] },
      { content: 'I updated the config as requested.' }, // neutral, <5 tools
    ]);
    const result = await runAgentLoop(cfg(fetch));
    expect(fetch).toHaveBeenCalledTimes(2); // returned immediately — no spurious gate turn
    expect(result.content).toBe('I updated the config as requested.');
  });
});