refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -1,11 +1,6 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
buildReflectiveMutationPrompt,
|
||||
buildSchemaFillPrompt,
|
||||
retryWithBackoff,
|
||||
wrapWithRetry,
|
||||
isRetryableEvolutionError,
|
||||
@@ -15,9 +10,6 @@ import {
|
||||
type RetryOptions,
|
||||
type RetryInfo,
|
||||
} from '../src/evolution-llm-wiring.js';
|
||||
import type { MutateArgs, GEPACandidate } from '../src/index.js';
|
||||
import type { Schema } from '../src/evolve-schema.js';
|
||||
import type { JudgeInput } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
|
||||
return {
|
||||
id: 'g1-m0',
|
||||
prompt: 'You are a tester. Respond with the exact expected word.',
|
||||
generation: 1,
|
||||
parent: 'g0-baseline',
|
||||
strategy: 'expand-edge-cases',
|
||||
score: null,
|
||||
perExample: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(overrides: Partial<Schema> = {}): Schema {
|
||||
return {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
|
||||
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
|
||||
],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── buildJudgeLLMCall ─────────────────────────────────────────────
|
||||
|
||||
describe('buildJudgeLLMCall', () => {
|
||||
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildGEPAMutateFn ─────────────────────────────────────────────
|
||||
|
||||
describe('buildGEPAMutateFn', () => {
|
||||
const mutateArgs: MutateArgs = {
|
||||
parent: makeCandidate(),
|
||||
strategy: 'expand-edge-cases',
|
||||
weaknessFeedback: [
|
||||
'Fails to handle empty input',
|
||||
'Sometimes forgets to include the final answer',
|
||||
],
|
||||
targetKind: 'persona-system-prompt',
|
||||
generation: 2,
|
||||
};
|
||||
|
||||
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe('EVOLVED SYSTEM PROMPT');
|
||||
expect(calls).toHaveLength(1);
|
||||
const prompt = calls[0];
|
||||
expect(prompt).toContain(mutateArgs.parent.prompt);
|
||||
expect(prompt).toContain(mutateArgs.strategy);
|
||||
expect(prompt).toContain('Fails to handle empty input');
|
||||
expect(prompt).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
|
||||
const { llm } = makeMockLLM(() => ' \n ');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('strips markdown code fences from the LLM response', async () => {
|
||||
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result.trim()).toBe('HIDDEN PROMPT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildReflectiveMutationPrompt', () => {
|
||||
it('includes the generation number and weakness bullets', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'tighten-format',
|
||||
weaknessFeedback: ['a', 'b'],
|
||||
targetKind: 'behavioral-spec-section',
|
||||
generation: 3,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('generation 3');
|
||||
expect(prompt).toContain('- a');
|
||||
expect(prompt).toContain('- b');
|
||||
expect(prompt).toContain('tighten-format');
|
||||
expect(prompt).toContain('PARENT');
|
||||
});
|
||||
|
||||
it('handles missing weakness feedback gracefully', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'add-examples',
|
||||
weaknessFeedback: [],
|
||||
targetKind: 'generic',
|
||||
generation: 0,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('PARENT');
|
||||
expect(prompt).toContain('(no specific weakness signals yet)');
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildSchemaExecuteFn ──────────────────────────────────────────
|
||||
|
||||
describe('buildSchemaExecuteFn', () => {
|
||||
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
|
||||
expect(calls[0]).toContain('What is the answer?');
|
||||
expect(calls[0]).toContain('"reasoning"');
|
||||
expect(calls[0]).toContain('"answer"');
|
||||
});
|
||||
|
||||
it('returns parsed=false when LLM returns non-JSON', async () => {
|
||||
const { llm } = makeMockLLM(() => 'just some text');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('just some text');
|
||||
});
|
||||
|
||||
it('extracts JSON from markdown fences', async () => {
|
||||
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toContain('"reasoning"');
|
||||
});
|
||||
|
||||
it('returns parsed=false and empty actual when LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSchemaFillPrompt', () => {
|
||||
it('serializes each field with name + type + description', () => {
|
||||
const prompt = buildSchemaFillPrompt({
|
||||
schema: makeSchema(),
|
||||
input: 'What is 2+2?',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('"reasoning"');
|
||||
expect(prompt).toContain('"answer"');
|
||||
expect(prompt).toContain('step-by-step thinking');
|
||||
expect(prompt).toContain('the final answer');
|
||||
expect(prompt).toContain('What is 2+2?');
|
||||
});
|
||||
|
||||
it('notes required constraints', () => {
|
||||
const schema: Schema = {
|
||||
name: 'x', version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'score',
|
||||
type: 'number',
|
||||
description: 'a rating',
|
||||
required: true,
|
||||
constraints: [{ kind: 'range', value: '0-10' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
|
||||
expect(prompt).toContain('score');
|
||||
expect(prompt).toContain('number');
|
||||
expect(prompt).toContain('range');
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeRunningJudge ──────────────────────────────────────────────
|
||||
|
||||
describe('makeRunningJudge', () => {
|
||||
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
|
||||
const { llm, calls } = makeMockLLM((prompt) => {
|
||||
// Simulate running the candidate prompt and returning a model response.
|
||||
return `MODEL OUTPUT for: ${prompt}`;
|
||||
});
|
||||
|
||||
const baseScore = vi.fn(async (args: JudgeInput) => ({
|
||||
overall: 0.8, weighted: 0.8,
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
lengthPenalty: 1, feedback: 'passed through', parsed: true,
|
||||
}));
|
||||
const baseJudge = { score: baseScore };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'What is love?',
|
||||
expected: 'baby don\'t hurt me',
|
||||
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
|
||||
});
|
||||
|
||||
// Model was asked to run the candidate prompt against the example input.
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
|
||||
expect(calls[0]).toContain('What is love?');
|
||||
|
||||
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
|
||||
expect(baseScore).toHaveBeenCalledTimes(1);
|
||||
const passedArgs = baseScore.mock.calls[0][0];
|
||||
expect(passedArgs.input).toBe('What is love?');
|
||||
expect(passedArgs.expected).toBe('baby don\'t hurt me');
|
||||
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
|
||||
|
||||
expect(result.overall).toBe(0.8);
|
||||
});
|
||||
|
||||
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('rate limited'); },
|
||||
};
|
||||
const baseJudge = { score: vi.fn() };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'any', expected: 'any', actual: 'prompt',
|
||||
});
|
||||
|
||||
expect(result.overall).toBe(0);
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.feedback).toMatch(/execution failed/i);
|
||||
expect(baseJudge.score).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes context through to the base judge', async () => {
|
||||
const { llm } = makeMockLLM(() => 'EXECUTED');
|
||||
const baseScore = vi.fn(async () => ({
|
||||
overall: 0.5, weighted: 0.5,
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
lengthPenalty: 1, feedback: '', parsed: true,
|
||||
}));
|
||||
|
||||
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
|
||||
|
||||
await runningJudge.score({
|
||||
input: 'q', expected: 'a', actual: 'prompt',
|
||||
context: 'persona:coder',
|
||||
});
|
||||
|
||||
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
|
||||
});
|
||||
});
|
||||
|
||||
// ── isRetryableEvolutionError ─────────────────────────────────────
|
||||
|
||||
describe('isRetryableEvolutionError', () => {
|
||||
|
||||
Reference in New Issue
Block a user