Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
325 lines
11 KiB
TypeScript
325 lines
11 KiB
TypeScript
import { describe, it, expect, vi } from 'vitest';
|
|
import {
|
|
buildJudgeLLMCall,
|
|
retryWithBackoff,
|
|
wrapWithRetry,
|
|
isRetryableEvolutionError,
|
|
computeRetryDelay,
|
|
DEFAULT_RETRY_OPTIONS,
|
|
type EvolutionLLM,
|
|
type RetryOptions,
|
|
type RetryInfo,
|
|
} from '../src/evolution-llm-wiring.js';
|
|
|
|
// ── Fixtures ──────────────────────────────────────────────────────
|
|
|
|
function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
|
|
llm: EvolutionLLM;
|
|
calls: string[];
|
|
} {
|
|
const calls: string[] = [];
|
|
return {
|
|
calls,
|
|
llm: {
|
|
async complete(prompt: string) {
|
|
calls.push(prompt);
|
|
return await handler(prompt);
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
// ── buildJudgeLLMCall ─────────────────────────────────────────────
|
|
|
|
describe('buildJudgeLLMCall', () => {
|
|
it('forwards the prompt unchanged and returns the raw completion', async () => {
|
|
const { llm, calls } = makeMockLLM(() => '{"correctness":8,"procedure":7,"conciseness":6,"feedback":"ok"}');
|
|
const judgeCall = buildJudgeLLMCall(llm);
|
|
|
|
const result = await judgeCall('SCORE THIS: foo');
|
|
|
|
expect(calls).toEqual(['SCORE THIS: foo']);
|
|
expect(result).toContain('correctness');
|
|
});
|
|
|
|
it('propagates errors from the underlying LLM', async () => {
|
|
const llm: EvolutionLLM = {
|
|
async complete() { throw new Error('network down'); },
|
|
};
|
|
const judgeCall = buildJudgeLLMCall(llm);
|
|
|
|
await expect(judgeCall('anything')).rejects.toThrow('network down');
|
|
});
|
|
});
|
|
|
|
// ── isRetryableEvolutionError ─────────────────────────────────────
|
|
|
|
describe('isRetryableEvolutionError', () => {
|
|
it('returns true for retryable HTTP statuses (429/502/503/504/529/500/408/425)', () => {
|
|
for (const status of [408, 425, 429, 500, 502, 503, 504, 529]) {
|
|
expect(isRetryableEvolutionError({ status })).toBe(true);
|
|
expect(isRetryableEvolutionError({ statusCode: status })).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('returns false for non-retryable HTTP statuses', () => {
|
|
for (const status of [400, 401, 403, 404, 422]) {
|
|
expect(isRetryableEvolutionError({ status })).toBe(false);
|
|
}
|
|
});
|
|
|
|
it('returns true for transient network error codes', () => {
|
|
for (const code of ['ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN']) {
|
|
expect(isRetryableEvolutionError({ code })).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('returns true for rate-limit / overloaded / timeout message patterns', () => {
|
|
const patterns = [
|
|
new Error('Rate limit exceeded'),
|
|
new Error('Too many requests, slow down'),
|
|
new Error('Anthropic: server overloaded — retry shortly'),
|
|
new Error('connection reset by peer'),
|
|
new Error('fetch failed'),
|
|
new Error('socket hang up'),
|
|
new Error('HTTP 429: rate limited'),
|
|
new Error('HTTP 503 Service Unavailable'),
|
|
];
|
|
for (const err of patterns) {
|
|
expect(isRetryableEvolutionError(err)).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('returns false for null, undefined, and empty errors', () => {
|
|
expect(isRetryableEvolutionError(null)).toBe(false);
|
|
expect(isRetryableEvolutionError(undefined)).toBe(false);
|
|
expect(isRetryableEvolutionError({})).toBe(false);
|
|
expect(isRetryableEvolutionError(new Error(''))).toBe(false);
|
|
});
|
|
|
|
it('returns false for deterministic logic errors', () => {
|
|
expect(isRetryableEvolutionError(new Error('Invalid input schema'))).toBe(false);
|
|
expect(isRetryableEvolutionError(new Error('Permission denied'))).toBe(false);
|
|
expect(isRetryableEvolutionError(new Error('Not found'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ── computeRetryDelay ─────────────────────────────────────────────
|
|
|
|
describe('computeRetryDelay', () => {
|
|
it('produces the 5s → 15s → 45s → 135s → 150s (capped) schedule with zero jitter', () => {
|
|
const opts: RetryOptions = { jitterMs: 0 };
|
|
expect(computeRetryDelay(1, opts)).toBe(5_000);
|
|
expect(computeRetryDelay(2, opts)).toBe(15_000);
|
|
expect(computeRetryDelay(3, opts)).toBe(45_000);
|
|
expect(computeRetryDelay(4, opts)).toBe(135_000);
|
|
expect(computeRetryDelay(5, opts)).toBe(150_000);
|
|
expect(computeRetryDelay(6, opts)).toBe(150_000);
|
|
});
|
|
|
|
it('clamps attempt < 1 to attempt 1', () => {
|
|
expect(computeRetryDelay(0, { jitterMs: 0 })).toBe(5_000);
|
|
expect(computeRetryDelay(-5, { jitterMs: 0 })).toBe(5_000);
|
|
});
|
|
|
|
it('applies jitter within [0, jitterMs)', () => {
|
|
// Stub Math.random to isolate jitter behavior.
|
|
const spy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
|
|
try {
|
|
expect(computeRetryDelay(1, { jitterMs: 2_000 })).toBe(5_000 + 1_000);
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('honors custom baseMs / capMs / factor', () => {
|
|
const opts: RetryOptions = { baseMs: 100, capMs: 1_000, factor: 2, jitterMs: 0 };
|
|
expect(computeRetryDelay(1, opts)).toBe(100);
|
|
expect(computeRetryDelay(2, opts)).toBe(200);
|
|
expect(computeRetryDelay(3, opts)).toBe(400);
|
|
expect(computeRetryDelay(4, opts)).toBe(800);
|
|
// Cap engages at attempt 5: 1600 → 1000.
|
|
expect(computeRetryDelay(5, opts)).toBe(1_000);
|
|
});
|
|
});
|
|
|
|
// ── retryWithBackoff ──────────────────────────────────────────────
|
|
|
|
describe('retryWithBackoff', () => {
|
|
/** Instant sleep — tests run in ~microseconds. */
|
|
const noSleep = async (): Promise<void> => { /* noop */ };
|
|
|
|
it('returns the first-try result without retry', async () => {
|
|
const op = vi.fn(async () => 'ok');
|
|
const result = await retryWithBackoff(op, { sleep: noSleep });
|
|
expect(result).toBe('ok');
|
|
expect(op).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('retries on retryable errors and eventually succeeds', async () => {
|
|
let callCount = 0;
|
|
const op = vi.fn(async () => {
|
|
callCount++;
|
|
if (callCount < 3) {
|
|
const err = Object.assign(new Error('rate limit'), { status: 429 });
|
|
throw err;
|
|
}
|
|
return 'recovered';
|
|
});
|
|
|
|
const sleeps: number[] = [];
|
|
const result = await retryWithBackoff(op, {
|
|
sleep: async (ms) => { sleeps.push(ms); },
|
|
jitterMs: 0,
|
|
});
|
|
|
|
expect(result).toBe('recovered');
|
|
expect(op).toHaveBeenCalledTimes(3);
|
|
expect(sleeps).toEqual([5_000, 15_000]);
|
|
});
|
|
|
|
it('propagates non-retryable errors on first occurrence without retrying', async () => {
|
|
const err = Object.assign(new Error('bad request'), { status: 400 });
|
|
const op = vi.fn(async () => { throw err; });
|
|
|
|
await expect(retryWithBackoff(op, { sleep: noSleep })).rejects.toBe(err);
|
|
expect(op).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('throws the last error after maxAttempts retryable failures', async () => {
|
|
const err = Object.assign(new Error('always 503'), { status: 503 });
|
|
const op = vi.fn(async () => { throw err; });
|
|
|
|
await expect(
|
|
retryWithBackoff(op, { sleep: noSleep, maxAttempts: 3 }),
|
|
).rejects.toBe(err);
|
|
expect(op).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it('fires onRetry hook before each sleep with attempt + delay + error', async () => {
|
|
let callCount = 0;
|
|
const op = vi.fn(async () => {
|
|
callCount++;
|
|
if (callCount < 3) {
|
|
throw Object.assign(new Error('rl'), { status: 429 });
|
|
}
|
|
return 'ok';
|
|
});
|
|
|
|
const retries: RetryInfo[] = [];
|
|
await retryWithBackoff(op, {
|
|
sleep: noSleep,
|
|
jitterMs: 0,
|
|
onRetry: (info) => retries.push(info),
|
|
});
|
|
|
|
expect(retries).toHaveLength(2);
|
|
expect(retries[0].attempt).toBe(1);
|
|
expect(retries[0].delayMs).toBe(5_000);
|
|
expect(retries[1].attempt).toBe(2);
|
|
expect(retries[1].delayMs).toBe(15_000);
|
|
expect(retries.every(r => r.error instanceof Error)).toBe(true);
|
|
});
|
|
|
|
it('honors a custom isRetryable predicate', async () => {
|
|
const err = new Error('custom: transient');
|
|
const op = vi.fn(async () => { throw err; });
|
|
|
|
// Default predicate would reject this — but custom says "always retry then give up".
|
|
await expect(retryWithBackoff(op, {
|
|
sleep: noSleep,
|
|
maxAttempts: 2,
|
|
isRetryable: (e) => e === err,
|
|
})).rejects.toBe(err);
|
|
expect(op).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('throws immediately when signal is pre-aborted', async () => {
|
|
const controller = new AbortController();
|
|
controller.abort(new Error('user cancelled'));
|
|
const op = vi.fn(async () => 'never');
|
|
|
|
await expect(retryWithBackoff(op, {
|
|
sleep: noSleep,
|
|
signal: controller.signal,
|
|
})).rejects.toThrow(/user cancelled|aborted/i);
|
|
expect(op).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('uses DEFAULT_RETRY_OPTIONS when nothing is supplied', async () => {
|
|
// Smoke test: a successful op should return without touching defaults.
|
|
const result = await retryWithBackoff(async () => 42);
|
|
expect(result).toBe(42);
|
|
expect(DEFAULT_RETRY_OPTIONS.maxAttempts).toBe(6);
|
|
expect(DEFAULT_RETRY_OPTIONS.baseMs).toBe(5_000);
|
|
expect(DEFAULT_RETRY_OPTIONS.capMs).toBe(150_000);
|
|
expect(DEFAULT_RETRY_OPTIONS.factor).toBe(3);
|
|
});
|
|
|
|
it('provides the 1-based attempt number to the operation', async () => {
|
|
const attempts: number[] = [];
|
|
let n = 0;
|
|
const op = vi.fn(async (attempt: number) => {
|
|
attempts.push(attempt);
|
|
n++;
|
|
if (n < 3) throw Object.assign(new Error('rl'), { status: 429 });
|
|
return 'done';
|
|
});
|
|
|
|
await retryWithBackoff(op, { sleep: noSleep });
|
|
expect(attempts).toEqual([1, 2, 3]);
|
|
});
|
|
});
|
|
|
|
// ── wrapWithRetry ─────────────────────────────────────────────────
|
|
|
|
describe('wrapWithRetry', () => {
|
|
it('wraps an EvolutionLLM so complete() retries on retryable errors', async () => {
|
|
let n = 0;
|
|
const base: EvolutionLLM = {
|
|
async complete() {
|
|
n++;
|
|
if (n < 3) throw Object.assign(new Error('429'), { status: 429 });
|
|
return 'finally';
|
|
},
|
|
};
|
|
const wrapped = wrapWithRetry(base, { sleep: async () => {}, jitterMs: 0 });
|
|
|
|
const result = await wrapped.complete('anything');
|
|
expect(result).toBe('finally');
|
|
expect(n).toBe(3);
|
|
});
|
|
|
|
it('does not retry non-retryable errors', async () => {
|
|
let n = 0;
|
|
const err = Object.assign(new Error('bad auth'), { status: 401 });
|
|
const base: EvolutionLLM = {
|
|
async complete() { n++; throw err; },
|
|
};
|
|
const wrapped = wrapWithRetry(base, { sleep: async () => {} });
|
|
|
|
await expect(wrapped.complete('x')).rejects.toBe(err);
|
|
expect(n).toBe(1);
|
|
});
|
|
|
|
it('threads retry options through to each call (independent per complete)', async () => {
|
|
const calls: number[] = [];
|
|
const base: EvolutionLLM = {
|
|
async complete(prompt: string) {
|
|
calls.push(prompt.length);
|
|
throw Object.assign(new Error('503'), { status: 503 });
|
|
},
|
|
};
|
|
const wrapped = wrapWithRetry(base, {
|
|
sleep: async () => {},
|
|
maxAttempts: 2,
|
|
});
|
|
|
|
await expect(wrapped.complete('a')).rejects.toThrow();
|
|
await expect(wrapped.complete('bb')).rejects.toThrow();
|
|
|
|
// Two separate complete() calls, each retried maxAttempts=2 times.
|
|
expect(calls).toEqual([1, 1, 2, 2]);
|
|
});
|
|
});
|