/** * Integration tests for POST /api/evolution/run. * * The endpoint needs an Anthropic key and an LLM. We stub the LLM by * installing a global factory override (`__waggleEvolutionLlmFactory`) so * every call goes through an in-memory responder — no network, no @ax-llm/ax. */ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { MindDB, VaultStore } from '@waggle/core'; import { buildLocalServer } from '../src/local/index.js'; import { injectWithAuth } from './test-utils.js'; import type { FastifyInstance } from 'fastify'; import type { EvolutionLLM } from '@waggle/agent'; // ── Global LLM override hook ───────────────────────────────────── type LLMFactory = (apiKey: string) => EvolutionLLM | Promise; function installLLMFactory(factory: LLMFactory): void { (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory }) .__waggleEvolutionLlmFactory = factory; } function clearLLMFactory(): void { delete (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory }) .__waggleEvolutionLlmFactory; } /** * Stub LLM that returns a canned answer based on a pattern match on the * prompt. Falls back to echoing the input so the judge still produces a * score. Every call increments a counter so tests can assert the LLM was * actually invoked. */ function makeStubLLM(responses: Array<{ match: RegExp; reply: string }>): { llm: EvolutionLLM; callCount: () => number; calls: string[]; } { const calls: string[] = []; return { callCount: () => calls.length, calls, llm: { async complete(prompt: string): Promise { calls.push(prompt); for (const r of responses) { if (r.match.test(prompt)) return r.reply; } // Default response that parses as a valid judge verdict. return '{"correctness":8,"procedure":8,"conciseness":7,"feedback":"ok"}'; }, }, }; } // ── Test scaffolding ───────────────────────────────────────────── describe('POST /api/evolution/run', () => { let server: FastifyInstance; let tmpDir: string; beforeAll(async () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-run-')); // Fresh mind database. const personalPath = path.join(tmpDir, 'personal.mind'); const mind = new MindDB(personalPath); mind.close(); // Seed the vault with an Anthropic key before the server boots — the // server loads the vault from disk on startup. const vault = new VaultStore(tmpDir); vault.set('anthropic', 'sk-ant-stub-for-tests', { models: ['claude-haiku-4-5-20251001'] }); server = await buildLocalServer({ dataDir: tmpDir }); }, 60_000); afterAll(async () => { await server?.close(); try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ } clearLLMFactory(); }, 60_000); beforeEach(() => { server.evolutionStore.clear(); clearLLMFactory(); }); afterEach(() => { clearLLMFactory(); }); /** Seed a handful of finalized traces so the orchestrator's dataset miner has material. */ function seedTraces(count = 8): void { for (let i = 0; i < count; i++) { const id = server.traceStore.start({ input: `What is the capital of country ${i}?`, personaId: 'coder', model: 'claude-haiku-4-5-20251001', taskShape: 'qa', }); server.traceStore.finalize(id, { outcome: i % 3 === 0 ? 'verified' : 'success', output: `The capital is City${i}.`, tokens: { input: 40, output: 20 }, }); } } const baseBody = { targetKind: 'behavioral-spec-section' as const, targetName: 'coreLoop', baseline: 'You are a careful assistant. Follow instructions.', schemaBaseline: { name: 'answer', version: 1, fields: [ { name: 'answer', type: 'string', description: 'the answer', required: true, constraints: [] }, ], }, // Keep populations tiny so the test stays fast even though we use a stub LLM. schema: { populationSize: 1, generations: 1, evalSize: 2, anchorEvalSize: 2, seed: 1 }, gepa: { populationSize: 1, generations: 1, miniEvalSize: 2, anchorEvalSize: 2, seed: 1 }, }; // ── 422 when no API key ──────────────────────────────────────── it('returns 422 when no Anthropic key is in the vault', async () => { // Temporarily hide the key for just this request. const saved = server.vault?.get('anthropic'); server.vault?.delete('anthropic'); try { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, }); expect(res.statusCode).toBe(422); const body = JSON.parse(res.body); expect(body.error).toMatch(/Anthropic API key/i); } finally { if (saved) { server.vault?.set('anthropic', saved.value, saved.metadata); } } }); // ── 400 on validation errors ─────────────────────────────────── describe('validation', () => { it('rejects missing body', async () => { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', }); expect(res.statusCode).toBe(400); }); it('rejects invalid targetKind', async () => { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: { ...baseBody, targetKind: 'nope' }, }); expect(res.statusCode).toBe(400); expect(JSON.parse(res.body).error).toMatch(/targetKind/); }); it('rejects empty targetName', async () => { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: { ...baseBody, targetName: '' }, }); expect(res.statusCode).toBe(400); expect(JSON.parse(res.body).error).toMatch(/targetName/); }); it('rejects empty baseline', async () => { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: { ...baseBody, baseline: '' }, }); expect(res.statusCode).toBe(400); expect(JSON.parse(res.body).error).toMatch(/baseline/); }); it('rejects malformed schemaBaseline', async () => { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: { ...baseBody, schemaBaseline: { name: 'x' } }, }); expect(res.statusCode).toBe(400); expect(JSON.parse(res.body).error).toMatch(/schemaBaseline/); }); }); // ── 503 when LLM factory fails ───────────────────────────────── it('returns 503 when the LLM factory returns null', async () => { installLLMFactory(() => null as unknown as EvolutionLLM); const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, }); expect(res.statusCode).toBe(503); expect(JSON.parse(res.body).error).toMatch(/ax-llm/); }); // ── Skip path: no traces ─────────────────────────────────────── it('returns outcome=skipped-trigger when the trace store has no eligible traces', async () => { const { llm } = makeStubLLM([]); installLLMFactory(() => llm); const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.outcome).toBe('skipped-trigger'); expect(body.reason).toMatch(/eligible traces/); }); // ── Happy path: runs + persists ──────────────────────────────── it('runs the full orchestrator when traces exist and returns a proposal summary', async () => { seedTraces(8); const stub = makeStubLLM([ // Mutation requests get a slightly different prompt text { match: /reflective|evolving an AI prompt/i, reply: 'You are an extremely careful assistant. Follow all instructions step by step.' }, // Schema fill requests get valid JSON { match: /schema/i, reply: '{"answer":"City-X"}' }, // Running-judge execution requests get a short answer { match: /USER INPUT:/i, reply: 'The capital is City-X.' }, ]); installLLMFactory(() => stub.llm); const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); // Any terminal outcome is acceptable — the point is that the orchestrator // ran end-to-end, talked to our stub LLM, and wrote something to the store. expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(body.outcome); expect(body.composeSummary).toBeTruthy(); // Every run (including skipped-delta) returns a composeSummary; only // 'proposed' and 'skipped-gates' attach a persisted run. expect(stub.callCount()).toBeGreaterThan(0); // If a run was persisted, check it shows up in the list endpoint. if (body.run) { const listRes = await injectWithAuth(server, { method: 'GET', url: '/api/evolution/runs', }); const listed = JSON.parse(listRes.body); expect(listed.count).toBeGreaterThan(0); const matching = listed.runs.find((r: { run_uuid: string }) => r.run_uuid === body.run.run_uuid); expect(matching).toBeDefined(); } }, 30_000); // ── SSE streaming path ───────────────────────────────────────── describe('SSE streaming (Accept: text/event-stream)', () => { /** Tiny SSE parser — takes the raw event-stream body and emits parsed events. */ function parseSseEvents(raw: string): Array<{ event: string; data: unknown }> { const out: Array<{ event: string; data: unknown }> = []; for (const block of raw.split('\n\n')) { if (!block.trim()) continue; let event = 'message'; const dataLines: string[] = []; for (const line of block.split('\n')) { if (line.startsWith('event:')) event = line.slice(6).trim(); else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim()); } if (dataLines.length > 0) { try { out.push({ event, data: JSON.parse(dataLines.join('\n')) }); } catch { out.push({ event, data: dataLines.join('\n') }); } } } return out; } it('emits open → progress* → done sequence for a successful run', async () => { seedTraces(8); const stub = makeStubLLM([ { match: /reflective|evolving an AI prompt/i, reply: 'A careful assistant.' }, { match: /schema/i, reply: '{"answer":"City-X"}' }, { match: /USER INPUT:/i, reply: 'The capital is City-X.' }, ]); installLLMFactory(() => stub.llm); const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, headers: { accept: 'text/event-stream' }, }); expect(res.statusCode).toBe(200); expect(res.headers['content-type']).toMatch(/text\/event-stream/); const events = parseSseEvents(res.body); const types = events.map(e => e.event); expect(types[0]).toBe('open'); expect(types).toContain('done'); // Terminal event carries the same payload shape as the JSON path. const done = events.find(e => e.event === 'done'); expect(done).toBeDefined(); const donePayload = done!.data as { outcome: string; composeSummary: unknown }; expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(donePayload.outcome); expect(donePayload.composeSummary).toBeTruthy(); // There should be at least one progress event reporting an orchestrator phase. const progresses = events.filter(e => e.event === 'progress'); expect(progresses.length).toBeGreaterThan(0); const phases = progresses.map(p => (p.data as { phase: string }).phase); // Orchestrator emits at least 'dataset' before skipping or compose. expect(phases.some(p => ['dataset', 'compose', 'gates', 'persist', 'skipped', 'done'].includes(p))).toBe(true); }, 30_000); it('emits open → error when the orchestrator throws', async () => { seedTraces(4); installLLMFactory(() => ({ async complete() { throw new Error('stub-llm-always-fails'); }, })); // Force a real error path like the JSON test does. const originalQuery = server.traceStore.queryParsed.bind(server.traceStore); server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); }; try { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, headers: { accept: 'text/event-stream' }, }); expect(res.statusCode).toBe(200); // SSE streams return 200 even on internal errors; the error is an SSE event const events = parseSseEvents(res.body); const types = events.map(e => e.event); expect(types[0]).toBe('open'); expect(types).toContain('error'); const errEvent = events.find(e => e.event === 'error'); expect((errEvent!.data as { error: string }).error).toMatch(/stub-trace-store-explodes/); } finally { server.traceStore.queryParsed = originalQuery; } }); it('falls back to JSON when Accept header is absent', async () => { seedTraces(8); const stub = makeStubLLM([]); installLLMFactory(() => stub.llm); const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, // No accept header — should return JSON. }); expect(res.statusCode).toBe(200); expect(res.headers['content-type']).toMatch(/application\/json/); expect(() => JSON.parse(res.body)).not.toThrow(); }); }); // ── Orchestrator error path ──────────────────────────────────── it('returns 500 when the orchestrator throws', async () => { seedTraces(4); installLLMFactory(() => ({ async complete() { throw new Error('stub-llm-always-fails'); }, })); // Judge swallows its own LLM errors with a zero score, so we need to // make the mutate + execute paths noisy enough that *something* escapes. // Easiest: swap out the trace store to blow up during buildExamples. const originalQuery = server.traceStore.queryParsed.bind(server.traceStore); server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); }; try { const res = await injectWithAuth(server, { method: 'POST', url: '/api/evolution/run', payload: baseBody, }); expect(res.statusCode).toBe(500); expect(JSON.parse(res.body).error).toMatch(/stub-trace-store-explodes/); } finally { server.traceStore.queryParsed = originalQuery; } }); });