398 lines
15 KiB
TypeScript
398 lines
15 KiB
TypeScript
/**
|
|
* Integration tests for POST /api/evolution/run.
|
|
*
|
|
* The endpoint needs an Anthropic key and an LLM. We stub the LLM by
|
|
* installing a global factory override (`__waggleEvolutionLlmFactory`) so
|
|
* every call goes through an in-memory responder — no network, no @ax-llm/ax.
|
|
*/
|
|
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { MindDB, VaultStore } from '@waggle/core';
|
|
import { buildLocalServer } from '../src/local/index.js';
|
|
import { injectWithAuth } from './test-utils.js';
|
|
import type { FastifyInstance } from 'fastify';
|
|
import type { EvolutionLLM } from '@waggle/agent';
|
|
|
|
// ── Global LLM override hook ─────────────────────────────────────
|
|
|
|
type LLMFactory = (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
|
|
|
function installLLMFactory(factory: LLMFactory): void {
|
|
(globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
|
.__waggleEvolutionLlmFactory = factory;
|
|
}
|
|
|
|
function clearLLMFactory(): void {
|
|
delete (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
|
.__waggleEvolutionLlmFactory;
|
|
}
|
|
|
|
/**
|
|
* Stub LLM that returns a canned answer based on a pattern match on the
|
|
* prompt. Falls back to echoing the input so the judge still produces a
|
|
* score. Every call increments a counter so tests can assert the LLM was
|
|
* actually invoked.
|
|
*/
|
|
function makeStubLLM(responses: Array<{ match: RegExp; reply: string }>): {
|
|
llm: EvolutionLLM;
|
|
callCount: () => number;
|
|
calls: string[];
|
|
} {
|
|
const calls: string[] = [];
|
|
return {
|
|
callCount: () => calls.length,
|
|
calls,
|
|
llm: {
|
|
async complete(prompt: string): Promise<string> {
|
|
calls.push(prompt);
|
|
for (const r of responses) {
|
|
if (r.match.test(prompt)) return r.reply;
|
|
}
|
|
// Default response that parses as a valid judge verdict.
|
|
return '{"correctness":8,"procedure":8,"conciseness":7,"feedback":"ok"}';
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
// ── Test scaffolding ─────────────────────────────────────────────
|
|
|
|
describe('POST /api/evolution/run', () => {
|
|
let server: FastifyInstance;
|
|
let tmpDir: string;
|
|
|
|
beforeAll(async () => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-run-'));
|
|
|
|
// Fresh mind database.
|
|
const personalPath = path.join(tmpDir, 'personal.mind');
|
|
const mind = new MindDB(personalPath);
|
|
mind.close();
|
|
|
|
// Seed the vault with an Anthropic key before the server boots — the
|
|
// server loads the vault from disk on startup.
|
|
const vault = new VaultStore(tmpDir);
|
|
vault.set('anthropic', 'sk-ant-stub-for-tests', { models: ['claude-haiku-4-5-20251001'] });
|
|
|
|
server = await buildLocalServer({ dataDir: tmpDir });
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
await server?.close();
|
|
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
clearLLMFactory();
|
|
}, 60_000);
|
|
|
|
beforeEach(() => {
|
|
server.evolutionStore.clear();
|
|
clearLLMFactory();
|
|
});
|
|
|
|
afterEach(() => {
|
|
clearLLMFactory();
|
|
});
|
|
|
|
/** Seed a handful of finalized traces so the orchestrator's dataset miner has material. */
|
|
function seedTraces(count = 8): void {
|
|
for (let i = 0; i < count; i++) {
|
|
const id = server.traceStore.start({
|
|
input: `What is the capital of country ${i}?`,
|
|
personaId: 'coder',
|
|
model: 'claude-haiku-4-5-20251001',
|
|
taskShape: 'qa',
|
|
});
|
|
server.traceStore.finalize(id, {
|
|
outcome: i % 3 === 0 ? 'verified' : 'success',
|
|
output: `The capital is City${i}.`,
|
|
tokens: { input: 40, output: 20 },
|
|
});
|
|
}
|
|
}
|
|
|
|
const baseBody = {
|
|
targetKind: 'behavioral-spec-section' as const,
|
|
targetName: 'coreLoop',
|
|
baseline: 'You are a careful assistant. Follow instructions.',
|
|
schemaBaseline: {
|
|
name: 'answer',
|
|
version: 1,
|
|
fields: [
|
|
{ name: 'answer', type: 'string', description: 'the answer', required: true, constraints: [] },
|
|
],
|
|
},
|
|
// Keep populations tiny so the test stays fast even though we use a stub LLM.
|
|
schema: { populationSize: 1, generations: 1, evalSize: 2, anchorEvalSize: 2, seed: 1 },
|
|
gepa: { populationSize: 1, generations: 1, miniEvalSize: 2, anchorEvalSize: 2, seed: 1 },
|
|
};
|
|
|
|
// ── 422 when no API key ────────────────────────────────────────
|
|
|
|
it('returns 422 when no Anthropic key is in the vault', async () => {
|
|
// Temporarily hide the key for just this request.
|
|
const saved = server.vault?.get('anthropic');
|
|
server.vault?.delete('anthropic');
|
|
try {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
});
|
|
expect(res.statusCode).toBe(422);
|
|
const body = JSON.parse(res.body);
|
|
expect(body.error).toMatch(/Anthropic API key/i);
|
|
} finally {
|
|
if (saved) {
|
|
server.vault?.set('anthropic', saved.value, saved.metadata);
|
|
}
|
|
}
|
|
});
|
|
|
|
// ── 400 on validation errors ───────────────────────────────────
|
|
|
|
describe('validation', () => {
|
|
it('rejects missing body', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run',
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
});
|
|
|
|
it('rejects invalid targetKind', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run',
|
|
payload: { ...baseBody, targetKind: 'nope' },
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
expect(JSON.parse(res.body).error).toMatch(/targetKind/);
|
|
});
|
|
|
|
it('rejects empty targetName', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run',
|
|
payload: { ...baseBody, targetName: '' },
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
expect(JSON.parse(res.body).error).toMatch(/targetName/);
|
|
});
|
|
|
|
it('rejects empty baseline', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run',
|
|
payload: { ...baseBody, baseline: '' },
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
expect(JSON.parse(res.body).error).toMatch(/baseline/);
|
|
});
|
|
|
|
it('rejects malformed schemaBaseline', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run',
|
|
payload: { ...baseBody, schemaBaseline: { name: 'x' } },
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
expect(JSON.parse(res.body).error).toMatch(/schemaBaseline/);
|
|
});
|
|
});
|
|
|
|
// ── 503 when LLM factory fails ─────────────────────────────────
|
|
|
|
it('returns 503 when the LLM factory returns null', async () => {
|
|
installLLMFactory(() => null as unknown as EvolutionLLM);
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
});
|
|
expect(res.statusCode).toBe(503);
|
|
expect(JSON.parse(res.body).error).toMatch(/ax-llm/);
|
|
});
|
|
|
|
// ── Skip path: no traces ───────────────────────────────────────
|
|
|
|
it('returns outcome=skipped-trigger when the trace store has no eligible traces', async () => {
|
|
const { llm } = makeStubLLM([]);
|
|
installLLMFactory(() => llm);
|
|
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
});
|
|
|
|
expect(res.statusCode).toBe(200);
|
|
const body = JSON.parse(res.body);
|
|
expect(body.outcome).toBe('skipped-trigger');
|
|
expect(body.reason).toMatch(/eligible traces/);
|
|
});
|
|
|
|
// ── Happy path: runs + persists ────────────────────────────────
|
|
|
|
it('runs the full orchestrator when traces exist and returns a proposal summary', async () => {
|
|
seedTraces(8);
|
|
|
|
const stub = makeStubLLM([
|
|
// Mutation requests get a slightly different prompt text
|
|
{ match: /reflective|evolving an AI prompt/i, reply: 'You are an extremely careful assistant. Follow all instructions step by step.' },
|
|
// Schema fill requests get valid JSON
|
|
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
|
// Running-judge execution requests get a short answer
|
|
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
|
]);
|
|
installLLMFactory(() => stub.llm);
|
|
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
});
|
|
|
|
expect(res.statusCode).toBe(200);
|
|
const body = JSON.parse(res.body);
|
|
// Any terminal outcome is acceptable — the point is that the orchestrator
|
|
// ran end-to-end, talked to our stub LLM, and wrote something to the store.
|
|
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(body.outcome);
|
|
expect(body.composeSummary).toBeTruthy();
|
|
// Every run (including skipped-delta) returns a composeSummary; only
|
|
// 'proposed' and 'skipped-gates' attach a persisted run.
|
|
expect(stub.callCount()).toBeGreaterThan(0);
|
|
|
|
// If a run was persisted, check it shows up in the list endpoint.
|
|
if (body.run) {
|
|
const listRes = await injectWithAuth(server, {
|
|
method: 'GET', url: '/api/evolution/runs',
|
|
});
|
|
const listed = JSON.parse(listRes.body);
|
|
expect(listed.count).toBeGreaterThan(0);
|
|
const matching = listed.runs.find((r: { run_uuid: string }) => r.run_uuid === body.run.run_uuid);
|
|
expect(matching).toBeDefined();
|
|
}
|
|
}, 30_000);
|
|
|
|
// ── SSE streaming path ─────────────────────────────────────────
|
|
|
|
describe('SSE streaming (Accept: text/event-stream)', () => {
|
|
/** Tiny SSE parser — takes the raw event-stream body and emits parsed events. */
|
|
function parseSseEvents(raw: string): Array<{ event: string; data: unknown }> {
|
|
const out: Array<{ event: string; data: unknown }> = [];
|
|
for (const block of raw.split('\n\n')) {
|
|
if (!block.trim()) continue;
|
|
let event = 'message';
|
|
const dataLines: string[] = [];
|
|
for (const line of block.split('\n')) {
|
|
if (line.startsWith('event:')) event = line.slice(6).trim();
|
|
else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
|
|
}
|
|
if (dataLines.length > 0) {
|
|
try {
|
|
out.push({ event, data: JSON.parse(dataLines.join('\n')) });
|
|
} catch {
|
|
out.push({ event, data: dataLines.join('\n') });
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
it('emits open → progress* → done sequence for a successful run', async () => {
|
|
seedTraces(8);
|
|
const stub = makeStubLLM([
|
|
{ match: /reflective|evolving an AI prompt/i, reply: 'A careful assistant.' },
|
|
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
|
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
|
]);
|
|
installLLMFactory(() => stub.llm);
|
|
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/evolution/run',
|
|
payload: baseBody,
|
|
headers: { accept: 'text/event-stream' },
|
|
});
|
|
|
|
expect(res.statusCode).toBe(200);
|
|
expect(res.headers['content-type']).toMatch(/text\/event-stream/);
|
|
|
|
const events = parseSseEvents(res.body);
|
|
const types = events.map(e => e.event);
|
|
|
|
expect(types[0]).toBe('open');
|
|
expect(types).toContain('done');
|
|
|
|
// Terminal event carries the same payload shape as the JSON path.
|
|
const done = events.find(e => e.event === 'done');
|
|
expect(done).toBeDefined();
|
|
const donePayload = done!.data as { outcome: string; composeSummary: unknown };
|
|
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(donePayload.outcome);
|
|
expect(donePayload.composeSummary).toBeTruthy();
|
|
|
|
// There should be at least one progress event reporting an orchestrator phase.
|
|
const progresses = events.filter(e => e.event === 'progress');
|
|
expect(progresses.length).toBeGreaterThan(0);
|
|
const phases = progresses.map(p => (p.data as { phase: string }).phase);
|
|
// Orchestrator emits at least 'dataset' before skipping or compose.
|
|
expect(phases.some(p => ['dataset', 'compose', 'gates', 'persist', 'skipped', 'done'].includes(p))).toBe(true);
|
|
}, 30_000);
|
|
|
|
it('emits open → error when the orchestrator throws', async () => {
|
|
seedTraces(4);
|
|
installLLMFactory(() => ({
|
|
async complete() { throw new Error('stub-llm-always-fails'); },
|
|
}));
|
|
// Force a real error path like the JSON test does.
|
|
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
|
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/evolution/run',
|
|
payload: baseBody,
|
|
headers: { accept: 'text/event-stream' },
|
|
});
|
|
expect(res.statusCode).toBe(200); // SSE streams return 200 even on internal errors; the error is an SSE event
|
|
const events = parseSseEvents(res.body);
|
|
const types = events.map(e => e.event);
|
|
expect(types[0]).toBe('open');
|
|
expect(types).toContain('error');
|
|
const errEvent = events.find(e => e.event === 'error');
|
|
expect((errEvent!.data as { error: string }).error).toMatch(/stub-trace-store-explodes/);
|
|
} finally {
|
|
server.traceStore.queryParsed = originalQuery;
|
|
}
|
|
});
|
|
|
|
it('falls back to JSON when Accept header is absent', async () => {
|
|
seedTraces(8);
|
|
const stub = makeStubLLM([]);
|
|
installLLMFactory(() => stub.llm);
|
|
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
// No accept header — should return JSON.
|
|
});
|
|
expect(res.statusCode).toBe(200);
|
|
expect(res.headers['content-type']).toMatch(/application\/json/);
|
|
expect(() => JSON.parse(res.body)).not.toThrow();
|
|
});
|
|
});
|
|
|
|
// ── Orchestrator error path ────────────────────────────────────
|
|
|
|
it('returns 500 when the orchestrator throws', async () => {
|
|
seedTraces(4);
|
|
installLLMFactory(() => ({
|
|
async complete() { throw new Error('stub-llm-always-fails'); },
|
|
}));
|
|
|
|
// Judge swallows its own LLM errors with a zero score, so we need to
|
|
// make the mutate + execute paths noisy enough that *something* escapes.
|
|
// Easiest: swap out the trace store to blow up during buildExamples.
|
|
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
|
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
|
});
|
|
expect(res.statusCode).toBe(500);
|
|
expect(JSON.parse(res.body).error).toMatch(/stub-trace-store-explodes/);
|
|
} finally {
|
|
server.traceStore.queryParsed = originalQuery;
|
|
}
|
|
});
|
|
});
|