refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -1,373 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
ComposeEvolution,
|
||||
defaultFeedbackFilter,
|
||||
filterJudgeFeedback,
|
||||
stripStructuralLines,
|
||||
schemaExecutorFromInstructionRunner,
|
||||
} from '../src/compose-evolution.js';
|
||||
import type {
|
||||
Schema,
|
||||
SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(name => ({
|
||||
name, type: 'string',
|
||||
description: `description of ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback: string): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFakeRunner(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ── defaultFeedbackFilter ──────────────────────────────────────
|
||||
|
||||
describe('defaultFeedbackFilter', () => {
|
||||
it('classifies "missing reasoning field" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Missing reasoning field.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "wrong field type" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Wrong field type for answer.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "schema mismatch" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Schema mismatch — expected 3 fields.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "should reorder fields" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Should reorder fields — put reasoning first.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies value-level complaints as value', () => {
|
||||
expect(defaultFeedbackFilter('The answer is too terse')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Response is too verbose')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Incorrect calculation')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Wrong tone — should be more formal')).toBe('value');
|
||||
});
|
||||
|
||||
it('classifies empty feedback as value (safe default)', () => {
|
||||
expect(defaultFeedbackFilter('')).toBe('value');
|
||||
});
|
||||
});
|
||||
|
||||
// ── stripStructuralLines ──────────────────────────────────────
|
||||
|
||||
describe('stripStructuralLines', () => {
|
||||
it('returns empty for empty input', () => {
|
||||
expect(stripStructuralLines('')).toBe('');
|
||||
});
|
||||
|
||||
it('drops only structural lines from multi-line feedback', () => {
|
||||
const input = [
|
||||
'Missing reasoning field.',
|
||||
'The answer is too terse.',
|
||||
'Wrong field type for confidence.',
|
||||
'Needs more detail.',
|
||||
].join('\n');
|
||||
const out = stripStructuralLines(input);
|
||||
expect(out).not.toContain('Missing reasoning field');
|
||||
expect(out).not.toContain('Wrong field type');
|
||||
expect(out).toContain('too terse');
|
||||
expect(out).toContain('Needs more detail');
|
||||
});
|
||||
|
||||
it('keeps all lines when none are structural', () => {
|
||||
const input = 'Too terse.\nNeeds more examples.';
|
||||
expect(stripStructuralLines(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('returns empty when all lines are structural', () => {
|
||||
const input = 'Missing reasoning field.\nSchema mismatch.';
|
||||
expect(stripStructuralLines(input)).toBe('');
|
||||
});
|
||||
|
||||
it('accepts a custom filter', () => {
|
||||
const custom = (line: string) => line.includes('DROP') ? 'structural' : 'value';
|
||||
const input = 'keep me\nDROP me\nkeep too';
|
||||
expect(stripStructuralLines(input, custom)).toBe('keep me\nkeep too');
|
||||
});
|
||||
});
|
||||
|
||||
// ── filterJudgeFeedback ───────────────────────────────────────
|
||||
|
||||
describe('filterJudgeFeedback', () => {
|
||||
it('strips structural feedback but preserves numeric scores', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.7, 'Missing reasoning field.\nResponse is too terse.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
|
||||
expect(score.overall).toBe(0.7);
|
||||
expect(score.correctness).toBe(0.7);
|
||||
expect(score.feedback).not.toContain('Missing reasoning field');
|
||||
expect(score.feedback).toContain('too terse');
|
||||
});
|
||||
|
||||
it('returns empty feedback when all lines are structural', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nSchema mismatch.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
expect(score.feedback).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
// ── schemaExecutorFromInstructionRunner ───────────────────────
|
||||
|
||||
describe('schemaExecutorFromInstructionRunner', () => {
|
||||
it('builds a schema prefix and delegates to the runner', async () => {
|
||||
let captured: { prompt: string; input: string } | null = null;
|
||||
const runner = async (args: { prompt: string; input: string }) => {
|
||||
captured = args;
|
||||
return '{"answer":"42"}';
|
||||
};
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(runner);
|
||||
const result = await schemaRunner({
|
||||
schema: makeSchema(['reasoning', 'answer']),
|
||||
input: 'What is 6*7?',
|
||||
});
|
||||
|
||||
expect(result.actual).toBe('{"answer":"42"}');
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(captured!.prompt).toContain('reasoning');
|
||||
expect(captured!.prompt).toContain('answer');
|
||||
expect(captured!.input).toBe('What is 6*7?');
|
||||
});
|
||||
|
||||
it('reports parsed=false for non-JSON output', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => 'just prose');
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
|
||||
it('swallows runner errors and reports parsed=false', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.actual).toBe('');
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── ComposeEvolution end-to-end ────────────────────────────────
|
||||
|
||||
describe('ComposeEvolution.run', () => {
|
||||
it('runs schema stage then instruction stage', async () => {
|
||||
const stages: string[] = [];
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
mutate: async ({ parent }) => `${parent.prompt} v2`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
onProgress: (e) => stages.push(e.stage),
|
||||
});
|
||||
|
||||
expect(stages).toContain('schema');
|
||||
expect(stages).toContain('instructions');
|
||||
expect(stages).toContain('done');
|
||||
expect(result.schema.winner).toBeDefined();
|
||||
expect(result.instructions.winner).toBeDefined();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
});
|
||||
|
||||
it('feedback separation: GEPA cannot see structural feedback from the judge', async () => {
|
||||
const capturedFeedbacks: string[] = [];
|
||||
|
||||
// Judge always emits structural + value feedback
|
||||
const mixedJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nResponse too terse.');
|
||||
},
|
||||
};
|
||||
|
||||
const mutateSpy = vi.fn(async ({ weaknessFeedback }: { parent: unknown; weaknessFeedback: string[]; strategy: string; targetKind: string; generation: number }) => {
|
||||
capturedFeedbacks.push(...weaknessFeedback);
|
||||
return 'mutated instruction prompt is a reasonable length';
|
||||
});
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: mixedJudge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: mixedJudge,
|
||||
mutate: mutateSpy,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
});
|
||||
|
||||
// GEPA must never receive "Missing reasoning field" in weaknessFeedback.
|
||||
for (const fb of capturedFeedbacks) {
|
||||
expect(fb.toLowerCase()).not.toContain('missing reasoning field');
|
||||
expect(fb.toLowerCase()).not.toContain('wrong field type');
|
||||
}
|
||||
// But it should see the value-level complaint
|
||||
expect(capturedFeedbacks.join(' ')).toContain('too terse');
|
||||
});
|
||||
|
||||
it('accepts a custom feedback filter', async () => {
|
||||
let filterCalls = 0;
|
||||
const customFilter = (feedback: string): 'structural' | 'value' => {
|
||||
filterCalls++;
|
||||
return feedback.includes('CUSTOM-DROP') ? 'structural' : 'value';
|
||||
};
|
||||
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'CUSTOM-DROP this line\nkeep this line');
|
||||
},
|
||||
};
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt text is long enough',
|
||||
examples: makeExamples(3),
|
||||
judge,
|
||||
mutate: async () => 'new prompt text that is long enough',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
feedbackFilter: customFilter,
|
||||
});
|
||||
|
||||
expect(filterCalls).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns a stable shape when aborted before instruction stage', async () => {
|
||||
const ctrl = new AbortController();
|
||||
let progressCount = 0;
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: async () => {
|
||||
// Abort during the schema stage
|
||||
if (progressCount === 1) ctrl.abort();
|
||||
progressCount++;
|
||||
return { actual: '', parsed: false };
|
||||
},
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt',
|
||||
examples: makeExamples(3),
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
mutate: async () => 'should not run',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
expect(result.schema).toBeDefined();
|
||||
expect(result.instructions).toBeDefined();
|
||||
expect(result.instructions.winner.prompt).toBe('baseline prompt');
|
||||
});
|
||||
|
||||
it('combinedDelta is a number; frozenSchema matches ES winner', async () => {
|
||||
const judge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(Math.min(1, args.actual.length / 50), 'ok');
|
||||
},
|
||||
};
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(8),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 4, anchorEvalSize: 4,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'short',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate: async ({ parent }) => `${parent.prompt} more tokens here`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 2,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 6,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.instructions.winner.score).not.toBeNull();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
expect(typeof result.combinedDelta).toBe('number');
|
||||
});
|
||||
});
|
||||
@@ -1,204 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
detectCorrection,
|
||||
detectCorrectionsInHistory,
|
||||
} from '../src/correction-detector.js';
|
||||
|
||||
describe('correction-detector', () => {
|
||||
// ── Basic detection ────────────────────────────────────────
|
||||
|
||||
describe('detectCorrection', () => {
|
||||
it('returns null for normal messages', () => {
|
||||
expect(detectCorrection('What is the weather today?')).toBeNull();
|
||||
expect(detectCorrection('Please write a function that sorts an array')).toBeNull();
|
||||
expect(detectCorrection('Thanks, that looks good')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for very short messages', () => {
|
||||
expect(detectCorrection('ok')).toBeNull();
|
||||
expect(detectCorrection('yes')).toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "no, not that"', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a bullet list, not paragraphs.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeGreaterThanOrEqual(0.3);
|
||||
});
|
||||
|
||||
it('detects strong correction: "I said / I told you"', () => {
|
||||
const result = detectCorrection('I said to use markdown headers, not plain text.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "that\'s wrong"', () => {
|
||||
const result = detectCorrection("That's wrong. The API endpoint should be POST, not GET.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects moderate correction: "actually, use X instead"', () => {
|
||||
const result = detectCorrection('Actually, instead use TypeScript for this module.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects tone correction', () => {
|
||||
const result = detectCorrection('No, that response was too formal. Keep it casual.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('detects format correction', () => {
|
||||
const result = detectCorrection("No, don't use those headers. Use bullet formatting instead.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects length correction', () => {
|
||||
const result = detectCorrection("That's not what I wanted. It's too long, make it shorter.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('length');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Durability classification ──────────────────────────────
|
||||
|
||||
describe('durability classification', () => {
|
||||
it('classifies "always" as durable', () => {
|
||||
const result = detectCorrection('No, always use bullet points instead of numbered lists.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
expect(result!.durability).toBe('durable');
|
||||
});
|
||||
|
||||
it('classifies "never" as durable', () => {
|
||||
const result = detectCorrection("No, never use code blocks for simple text. That's wrong.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "from now on" as durable', () => {
|
||||
const result = detectCorrection('From now on, please don\'t include timestamps in summaries.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "I prefer" as durable', () => {
|
||||
const result = detectCorrection("No, I prefer shorter responses. That's not what I wanted.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "this time" as task_local', () => {
|
||||
const result = detectCorrection("No, not that. This time just use a simple list.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(false);
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('classifies "for now" as task_local', () => {
|
||||
const result = detectCorrection("That's wrong for now. Instead use the old API endpoint.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('defaults to task_local when no explicit signal', () => {
|
||||
const result = detectCorrection('No, not that approach. Use the factory pattern.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pattern key extraction ─────────────────────────────────
|
||||
|
||||
describe('pattern key extraction', () => {
|
||||
it('extracts tone pattern key', () => {
|
||||
const result = detectCorrection("No, that's too verbose. Stop doing that.");
|
||||
expect(result).not.toBeNull();
|
||||
// "too verbose" maps to tone category
|
||||
// Note: might also match length depending on exact wording
|
||||
});
|
||||
|
||||
it('extracts accuracy pattern key for "wrong"', () => {
|
||||
const result = detectCorrection("That's incorrect. I said the opposite.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('accuracy');
|
||||
});
|
||||
|
||||
it('falls back to general:correction for unrecognized patterns', () => {
|
||||
const result = detectCorrection("No, not that. I told you to use something else entirely.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBe('general:correction');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Detail extraction ──────────────────────────────────────
|
||||
|
||||
describe('detail extraction', () => {
|
||||
it('extracts first sentence as detail', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a completely different structure. Also fix the naming.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail).toBe('No, not that.');
|
||||
});
|
||||
|
||||
it('truncates long messages', () => {
|
||||
const longMessage = 'No, not that. ' + 'A'.repeat(200);
|
||||
const result = detectCorrection(longMessage);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail.length).toBeLessThanOrEqual(120);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Confidence scoring ─────────────────────────────────────
|
||||
|
||||
describe('confidence', () => {
|
||||
it('gives higher confidence to strong corrections', () => {
|
||||
const strong = detectCorrection("No, that's wrong. I told you not to do that. Stop.");
|
||||
const moderate = detectCorrection('Actually, let\'s try a different approach instead.');
|
||||
expect(strong).not.toBeNull();
|
||||
expect(moderate).not.toBeNull();
|
||||
expect(strong!.confidence).toBeGreaterThan(moderate!.confidence);
|
||||
});
|
||||
|
||||
it('caps confidence at 1.0', () => {
|
||||
const result = detectCorrection(
|
||||
"No, that's wrong. I said don't do that. Not what I wanted. Please stop. Instead use something else.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeLessThanOrEqual(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── History analysis ───────────────────────────────────────
|
||||
|
||||
describe('detectCorrectionsInHistory', () => {
|
||||
it('detects corrections in a message sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Here is a very detailed formal report...' },
|
||||
{ role: 'user', content: "No, that's too formal. I said keep it casual." },
|
||||
{ role: 'assistant', content: 'Got it. Here is a casual version...' },
|
||||
{ role: 'user', content: 'Thanks, that looks better.' },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections).toHaveLength(1);
|
||||
expect(corrections[0].patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('returns empty array when no corrections found', () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Write a function to sort an array' },
|
||||
{ role: 'assistant', content: 'Here you go...' },
|
||||
{ role: 'user', content: 'Thanks!' },
|
||||
];
|
||||
expect(detectCorrectionsInHistory(messages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles multiple corrections in sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Draft version 1...' },
|
||||
{ role: 'user', content: "No, not that. That's wrong. Use a different approach." },
|
||||
{ role: 'assistant', content: 'Draft version 2...' },
|
||||
{ role: 'user', content: "I said don't use bullet points. Instead use paragraphs." },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,348 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore } from '@waggle/core';
|
||||
import {
|
||||
EvalDatasetBuilder,
|
||||
detectSecrets,
|
||||
SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL,
|
||||
fromJSONL as evalFromJSONL,
|
||||
type EvalExample,
|
||||
} from '../src/eval-dataset.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JSONL round-trip', () => {
|
||||
const examples: EvalExample[] = [
|
||||
{
|
||||
input: 'write a haiku',
|
||||
expected_output: 'Cherry blossoms fall / ...',
|
||||
metadata: { source: 'trace', personaId: 'writer', outcome: 'success' },
|
||||
},
|
||||
{
|
||||
input: 'explain closures',
|
||||
expected_output: 'A closure captures variables from its enclosing scope.',
|
||||
metadata: { source: 'trace', personaId: 'coder', outcome: 'verified' },
|
||||
},
|
||||
];
|
||||
|
||||
it('round-trips through JSONL', () => {
|
||||
const jsonl = evalToJSONL(examples);
|
||||
expect(jsonl.split('\n')).toHaveLength(2);
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toEqual(examples);
|
||||
});
|
||||
|
||||
it('skips malformed lines silently', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify(examples[0]),
|
||||
'{ this is not valid json',
|
||||
'',
|
||||
JSON.stringify(examples[1]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips lines missing required fields', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify({ input: 'x' }), // missing expected_output + metadata
|
||||
JSON.stringify(examples[0]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('EvalDatasetBuilder', () => {
|
||||
let db: MindDB;
|
||||
let store: ExecutionTraceStore;
|
||||
let builder: EvalDatasetBuilder;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ExecutionTraceStore(db);
|
||||
builder = new EvalDatasetBuilder(store);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function seedTrace(input: string, output: string, outcome: 'success' | 'verified' | 'corrected' | 'abandoned' = 'success', personaId = 'coder'): number {
|
||||
const id = store.start({ input, personaId });
|
||||
store.finalize(id, { outcome, output });
|
||||
return id;
|
||||
}
|
||||
|
||||
// ── sourceFromTraces ──
|
||||
|
||||
describe('sourceFromTraces', () => {
|
||||
it('pulls only traces matching the requested outcomes', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'verified');
|
||||
seedTrace('c?', 'C.', 'corrected');
|
||||
seedTrace('d?', 'D.', 'abandoned');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success', 'verified'], false);
|
||||
expect(examples).toHaveLength(2);
|
||||
expect(examples.map(e => e.input).sort()).toEqual(['a?', 'b?']);
|
||||
});
|
||||
|
||||
it('includes corrections when flag set', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'corrected');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
expect(examples).toHaveLength(2);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.metadata.source).toBe('correction');
|
||||
});
|
||||
|
||||
it('maps trace fields onto metadata', () => {
|
||||
seedTrace('hello', 'world', 'success', 'writer');
|
||||
const [example] = builder.sourceFromTraces(['success'], false);
|
||||
expect(example.metadata.personaId).toBe('writer');
|
||||
expect(example.metadata.traceId).toBeGreaterThan(0);
|
||||
expect(example.metadata.source).toBe('trace');
|
||||
});
|
||||
});
|
||||
|
||||
// ── build (end-to-end) ──
|
||||
|
||||
describe('build', () => {
|
||||
it('returns deterministic split with same seed', async () => {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
seedTrace(`question ${i}?`, `answer ${i}.`, 'success');
|
||||
}
|
||||
|
||||
const a = await builder.build({ seed: 42 });
|
||||
const b = await builder.build({ seed: 42 });
|
||||
|
||||
expect(a.train.map(e => e.input)).toEqual(b.train.map(e => e.input));
|
||||
expect(a.val.map(e => e.input)).toEqual(b.val.map(e => e.input));
|
||||
expect(a.holdout.map(e => e.input)).toEqual(b.holdout.map(e => e.input));
|
||||
});
|
||||
|
||||
it('different seeds produce different orderings', async () => {
|
||||
for (let i = 0; i < 30; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const a = await builder.build({ seed: 1 });
|
||||
const b = await builder.build({ seed: 999 });
|
||||
|
||||
const aIds = a.train.map(e => e.input).join('|');
|
||||
const bIds = b.train.map(e => e.input).join('|');
|
||||
expect(aIds).not.toBe(bIds);
|
||||
});
|
||||
|
||||
it('splits 60/20/20 when enough examples', async () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length).toBe(60);
|
||||
expect(split.val.length).toBe(20);
|
||||
expect(split.holdout.length).toBe(20);
|
||||
expect(split.stats.total).toBe(100);
|
||||
});
|
||||
|
||||
it('honors custom split ratios', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1, splitRatios: [0.8, 0.1, 0.1] });
|
||||
expect(split.train.length).toBe(40);
|
||||
expect(split.val.length).toBe(5);
|
||||
expect(split.holdout.length).toBe(5);
|
||||
});
|
||||
|
||||
it('rejects ratios that do not sum to 1', async () => {
|
||||
await expect(builder.build({ splitRatios: [0.5, 0.3, 0.3] })).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('filters out examples containing secrets', async () => {
|
||||
seedTrace('clean input', 'clean output', 'success');
|
||||
seedTrace('here is a key: AKIAIOSFODNN7EXAMPLE', 'sure', 'success');
|
||||
seedTrace('what is your token?', 'ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.sourced).toBe(3);
|
||||
expect(split.stats.acceptedAfterSecretScan).toBe(1);
|
||||
expect(split.rejected.filter(r => r.reason.startsWith('secret:'))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('filters examples by min length', async () => {
|
||||
seedTrace('?', 'ok', 'success'); // below both mins
|
||||
seedTrace('a proper question', 'a proper answer', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, minInputChars: 5, minOutputChars: 3 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-short-input')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters examples over max combined length', async () => {
|
||||
seedTrace('this is an ok prompt', 'this is an ok answer', 'success');
|
||||
seedTrace('x'.repeat(5000), 'y'.repeat(5000), 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, maxCombinedChars: 1000 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-long')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters low-signal garbage', async () => {
|
||||
seedTrace('.......................', '!!!!!!!!!!!!!!!!', 'success');
|
||||
seedTrace('real question?', 'real answer.', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'low-signal')).toBe(true);
|
||||
});
|
||||
|
||||
it('dedupes by input hash', async () => {
|
||||
seedTrace('same prompt', 'answer a', 'success');
|
||||
seedTrace('same prompt', 'answer b', 'success');
|
||||
seedTrace('different prompt', 'answer c', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.unique).toBe(2);
|
||||
expect(split.rejected.some(r => r.reason === 'duplicate')).toBe(true);
|
||||
});
|
||||
|
||||
it('applies optional LLM judge', async () => {
|
||||
seedTrace('keep this example please', 'kept response', 'success');
|
||||
seedTrace('drop this example please', 'dropped response', 'success');
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async (ex) => ({
|
||||
keep: !ex.input.includes('drop'),
|
||||
reason: ex.input.includes('drop') ? 'contains drop' : undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason.startsWith('judge:'))).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps example when judge throws (fail-open)', async () => {
|
||||
seedTrace('some prompt text', 'some answer text', 'success');
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async () => {
|
||||
throw new Error('llm down');
|
||||
},
|
||||
});
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.stats.total).toBe(1);
|
||||
});
|
||||
|
||||
it('accepts external harvest/correction examples', async () => {
|
||||
const harvest: EvalExample = {
|
||||
input: 'harvest prompt',
|
||||
expected_output: 'harvest answer',
|
||||
metadata: { source: 'harvest' },
|
||||
};
|
||||
const correction: EvalExample = {
|
||||
input: 'correction prompt',
|
||||
expected_output: 'correction answer',
|
||||
metadata: { source: 'correction' },
|
||||
};
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
harvestExamples: [harvest],
|
||||
correctionExamples: [correction],
|
||||
});
|
||||
|
||||
const allInputs = [...split.train, ...split.val, ...split.holdout].map(e => e.input);
|
||||
expect(allInputs).toContain('harvest prompt');
|
||||
expect(allInputs).toContain('correction prompt');
|
||||
});
|
||||
|
||||
it('maps correction trace output to correctionFeedback when available', async () => {
|
||||
const id = store.start({ input: 'wrong approach', personaId: 'coder' });
|
||||
store.finalize(id, { outcome: 'success', output: 'first try' });
|
||||
store.markCorrected(id, 'Should use async/await here');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.expected_output).toBe('Should use async/await here');
|
||||
});
|
||||
|
||||
it('returns empty splits when no traces match', async () => {
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(0);
|
||||
expect(split.train).toEqual([]);
|
||||
expect(split.val).toEqual([]);
|
||||
expect(split.holdout).toEqual([]);
|
||||
});
|
||||
|
||||
it('tiny datasets assign leftover to holdout', async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
// Ratios 0.6/0.2/0.2 with n=3 → floor(1.8)=1, floor(0.6)=0, remainder=2 → holdout
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length + split.val.length + split.holdout.length).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,372 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
deployPersonaOverride,
|
||||
rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride,
|
||||
rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides,
|
||||
applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
} from '../src/evolution-deploy.js';
|
||||
import { getPersona, listPersonas, setPersonaDataDir } from '../src/personas.js';
|
||||
import { loadCustomPersonas } from '../src/custom-personas.js';
|
||||
|
||||
// ── Test fixture: temp dir per-test ────────────────────────────
|
||||
|
||||
describe('evolution-deploy', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-deploy-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch { /* Windows lock cleanup racy; ignore */ }
|
||||
});
|
||||
|
||||
// ── deployPersonaOverride ──
|
||||
|
||||
describe('deployPersonaOverride', () => {
|
||||
it('writes a JSON file named {id}.json in personas/', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'You are the evolved coder.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('personas', 'coder.json'))).toBe(true);
|
||||
expect(result.backupPath).toBeNull();
|
||||
expect(result.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('inherits built-in persona fields when targeting a known id', () => {
|
||||
// 'coder' is a built-in persona
|
||||
const builtin = getPersona('coder');
|
||||
expect(builtin).not.toBeNull();
|
||||
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED coder prompt — tighter format.',
|
||||
});
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.id).toBe('coder');
|
||||
expect(loaded.name).toBe(builtin!.name);
|
||||
expect(loaded.icon).toBe(builtin!.icon);
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder prompt — tighter format.');
|
||||
expect(loaded.tools).toEqual(builtin!.tools);
|
||||
});
|
||||
|
||||
it('creates a backup on subsequent writes', () => {
|
||||
const first = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
|
||||
const second = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(fs.existsSync(second.backupPath!)).toBe(true);
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).systemPrompt).toBe('v1');
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).systemPrompt).toBe('v2');
|
||||
});
|
||||
|
||||
it('creates a minimal shell for unknown persona ids', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up-persona',
|
||||
systemPrompt: 'fresh prompt',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.id).toBe('made-up-persona');
|
||||
expect(loaded.name).toBe('made-up-persona');
|
||||
expect(loaded.systemPrompt).toBe('fresh prompt');
|
||||
expect(Array.isArray(loaded.tools)).toBe(true);
|
||||
});
|
||||
|
||||
it('applies caller-supplied overrides (description, icon, tools)', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up',
|
||||
systemPrompt: 'x',
|
||||
overrides: {
|
||||
description: 'custom description',
|
||||
icon: 'brain',
|
||||
tools: ['tool_a', 'tool_b'],
|
||||
},
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.description).toBe('custom description');
|
||||
expect(loaded.icon).toBe('brain');
|
||||
expect(loaded.tools).toEqual(['tool_a', 'tool_b']);
|
||||
});
|
||||
|
||||
it('is picked up by listPersonas after setPersonaDataDir', () => {
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED ROUTING',
|
||||
});
|
||||
|
||||
setPersonaDataDir(tmpDir);
|
||||
try {
|
||||
const personas = listPersonas();
|
||||
const evolved = personas.find(p => p.id === 'coder' && p.systemPrompt.includes('EVOLVED ROUTING'));
|
||||
expect(evolved).toBeDefined();
|
||||
} finally {
|
||||
setPersonaDataDir(''); // reset so other tests aren't polluted
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackPersonaOverride ──
|
||||
|
||||
describe('rollbackPersonaOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const second = deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v2' });
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.systemPrompt).toBe('v1');
|
||||
});
|
||||
|
||||
it('deletes the override entirely when no .bak exists', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadCustomPersonas(tmpDir)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('returns false for non-existent personas', () => {
|
||||
expect(rollbackPersonaOverride(tmpDir, 'ghost')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── deployBehavioralSpecOverride ──
|
||||
|
||||
describe('deployBehavioralSpecOverride', () => {
|
||||
it('writes a JSON file named {section}.json in behavioral-overrides/', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop',
|
||||
text: 'Evolved core loop instructions.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('behavioral-overrides', 'coreLoop.json'))).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.section).toBe('coreLoop');
|
||||
expect(loaded.text).toBe('Evolved core loop instructions.');
|
||||
expect(loaded.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('stores the originating run uuid for audit', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'qualityRules',
|
||||
text: 'new quality rules',
|
||||
runUuid: 'test-run-uuid-123',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.runUuid).toBe('test-run-uuid-123');
|
||||
});
|
||||
|
||||
it('creates backup on subsequent writes', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
});
|
||||
|
||||
it('retries transient Windows rename locks before replacing an override', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync')
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('busy'), { code: 'EBUSY' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked again'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('still busy'), { code: 'EBUSY' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(5);
|
||||
expect(wait).toHaveBeenCalledTimes(4);
|
||||
expect(JSON.parse(fs.readFileSync(second.path, 'utf-8')).text).toBe('v2');
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves the current override when transient rename retries are exhausted', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('still locked'), { code: 'EACCES' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/still locked/);
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(10);
|
||||
expect(wait).toHaveBeenCalledTimes(9);
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('does not retry non-transient rename failures', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('disk error'), { code: 'EIO' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/disk error/);
|
||||
|
||||
expect(rename).toHaveBeenCalledOnce();
|
||||
expect(wait).not.toHaveBeenCalled();
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects unknown sections', () => {
|
||||
expect(() =>
|
||||
deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'notASection' as BehavioralSpecSection,
|
||||
text: 'x',
|
||||
}),
|
||||
).toThrow(/Unknown behavioral-spec section/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackBehavioralSpecOverride ──
|
||||
|
||||
describe('rollbackBehavioralSpecOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v2' });
|
||||
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('v1');
|
||||
});
|
||||
|
||||
it('removes the override when no backup exists', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── loadBehavioralSpecOverrides ──
|
||||
|
||||
describe('loadBehavioralSpecOverrides', () => {
|
||||
it('returns empty object when no overrides exist', () => {
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('returns a section → text map for all written overrides', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'evolved core' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'qualityRules', text: 'evolved quality' });
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('evolved core');
|
||||
expect(overrides.qualityRules).toBe('evolved quality');
|
||||
expect(overrides.behavioralRules).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores malformed override files', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), '{ not valid json');
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('ignores files with empty text', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), JSON.stringify({
|
||||
section: 'coreLoop', text: '', deployedAt: '2026-04-14',
|
||||
}));
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── applyBehavioralSpecOverrides ──
|
||||
|
||||
describe('applyBehavioralSpecOverrides', () => {
|
||||
const baseline: Record<BehavioralSpecSection, string> = {
|
||||
coreLoop: 'base core',
|
||||
qualityRules: 'base quality',
|
||||
behavioralRules: 'base behavioral',
|
||||
workPatterns: 'base patterns',
|
||||
intelligenceDefaults: 'base intelligence',
|
||||
};
|
||||
|
||||
it('returns baseline unchanged when no overrides', () => {
|
||||
expect(applyBehavioralSpecOverrides(baseline, {})).toEqual(baseline);
|
||||
});
|
||||
|
||||
it('overlays provided sections', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: 'EVOLVED core',
|
||||
qualityRules: 'EVOLVED quality',
|
||||
});
|
||||
expect(merged.coreLoop).toBe('EVOLVED core');
|
||||
expect(merged.qualityRules).toBe('EVOLVED quality');
|
||||
expect(merged.behavioralRules).toBe('base behavioral');
|
||||
});
|
||||
|
||||
it('ignores empty/undefined overrides without clobbering baseline', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: '',
|
||||
qualityRules: undefined,
|
||||
});
|
||||
expect(merged.coreLoop).toBe('base core');
|
||||
expect(merged.qualityRules).toBe('base quality');
|
||||
});
|
||||
});
|
||||
|
||||
// ── BEHAVIORAL_SPEC_SECTIONS constant ──
|
||||
|
||||
describe('BEHAVIORAL_SPEC_SECTIONS', () => {
|
||||
it('contains the expected 5 sections in expected order', () => {
|
||||
expect(BEHAVIORAL_SPEC_SECTIONS).toEqual([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules',
|
||||
'workPatterns', 'intelligenceDefaults',
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,245 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
} from '../src/evolution-gates.js';
|
||||
|
||||
describe('checkNonEmpty', () => {
|
||||
it('fails on empty string', () => {
|
||||
expect(checkNonEmpty('').verdict).toBe('fail');
|
||||
});
|
||||
it('fails on whitespace-only string', () => {
|
||||
expect(checkNonEmpty(' \n\n ').verdict).toBe('fail');
|
||||
});
|
||||
it('passes on real content', () => {
|
||||
expect(checkNonEmpty('hello').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkSize', () => {
|
||||
it('passes within the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(2000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.current).toBe(2000);
|
||||
expect(result.detail?.max).toBe(DEFAULT_SIZE_LIMITS.personaSystemPrompt);
|
||||
});
|
||||
|
||||
it('fails over the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(4000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.reason).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('uses tool-description cap (500)', () => {
|
||||
expect(checkSize('x'.repeat(500), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(501), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses skill-body cap (15k)', () => {
|
||||
expect(checkSize('x'.repeat(15_000), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(15_001), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses generic cap for unknown target', () => {
|
||||
const result = checkSize('x'.repeat(5000), 'generic', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('accepts custom size limits', () => {
|
||||
const limits = { ...DEFAULT_SIZE_LIMITS, generic: 100 };
|
||||
const result = checkSize('x'.repeat(200), 'generic', limits);
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkGrowth', () => {
|
||||
it('passes when candidate is smaller than baseline', () => {
|
||||
const result = checkGrowth('abc', 'abcdefgh', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes when growth is within budget', () => {
|
||||
const result = checkGrowth('x'.repeat(110), 'x'.repeat(100), 0.2); // +10%
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('fails when growth exceeds budget', () => {
|
||||
const result = checkGrowth('x'.repeat(140), 'x'.repeat(100), 0.2); // +40%
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.ratio).toBeCloseTo(0.4, 5);
|
||||
});
|
||||
|
||||
it('passes when baseline is empty (cannot compute ratio)', () => {
|
||||
const result = checkGrowth('anything', '', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('boundary: exactly 20% growth passes with default budget', () => {
|
||||
// 100 → 120 is exactly 20%, not greater
|
||||
const result = checkGrowth('x'.repeat(120), 'x'.repeat(100), 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkBalancedFences', () => {
|
||||
it('passes no fences', () => {
|
||||
expect(checkBalancedFences('plain prose').verdict).toBe('pass');
|
||||
});
|
||||
it('passes two fences (one code block)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\n```\ndone';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
it('fails one fence (unbalanced)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\nstill going';
|
||||
const result = checkBalancedFences(text);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.fences).toBe(1);
|
||||
});
|
||||
it('passes four fences (two blocks)', () => {
|
||||
const text = '```a\nfoo\n```\n```b\nbar\n```';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoPlaceholders', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoPlaceholders('A prompt with normal prose.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails [TODO] bracket placeholder', () => {
|
||||
expect(checkNoPlaceholders('Write about [TODO] here.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [PLACEHOLDER]', () => {
|
||||
expect(checkNoPlaceholders('Fill in [PLACEHOLDER].').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [YOUR NAME] style', () => {
|
||||
expect(checkNoPlaceholders('Hi [YOUR NAME HERE]!').verdict).toBe('fail');
|
||||
});
|
||||
it('fails angle-brackets <placeholder>', () => {
|
||||
expect(checkNoPlaceholders('Set <placeholder> before sending.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails handlebars {{var}}', () => {
|
||||
expect(checkNoPlaceholders('Hello {{name}}.').verdict).toBe('fail');
|
||||
});
|
||||
it('does not false-positive normal bracketed citations', () => {
|
||||
expect(checkNoPlaceholders('See [1] and [Smith 2024] for details.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoObviousTodos', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoObviousTodos('Write clear documentation.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails lines starting with TODO:', () => {
|
||||
expect(checkNoObviousTodos('Intro paragraph.\nTODO: finish this').verdict).toBe('fail');
|
||||
});
|
||||
it('fails lines starting with FIXME:', () => {
|
||||
expect(checkNoObviousTodos('FIXME: tighten this up').verdict).toBe('fail');
|
||||
});
|
||||
it('does not fail mid-sentence "todo" mention', () => {
|
||||
expect(checkNoObviousTodos('Build a todo list UI.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkRegression', () => {
|
||||
it('passes when delta is within tolerance', () => {
|
||||
const result = checkRegression(0.8, 0.79, -0.02); // -1pp, floor -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes on improvement', () => {
|
||||
const result = checkRegression(0.7, 0.85, -0.02);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.delta).toBeCloseTo(0.15, 5);
|
||||
});
|
||||
|
||||
it('fails when delta exceeds tolerance', () => {
|
||||
const result = checkRegression(0.9, 0.7, -0.02); // -20pp
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('boundary: equal to tolerance passes', () => {
|
||||
const result = checkRegression(0.8, 0.78, -0.02); // exactly -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
// ── runGates end-to-end ────────────────────────────────────────
|
||||
|
||||
describe('runGates', () => {
|
||||
it('passes a clean candidate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'A concise persona prompt for the researcher.',
|
||||
baseline: 'A concise persona prompt for the researcher.',
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('pass');
|
||||
expect(res.firstFailure).toBeNull();
|
||||
});
|
||||
|
||||
it('fails an empty candidate on the first gate', () => {
|
||||
const res = runGates({ candidate: '', baseline: 'whatever' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('non-empty');
|
||||
});
|
||||
|
||||
it('fails oversized candidate on size gate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(4000),
|
||||
baseline: 'x'.repeat(3000),
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('size');
|
||||
});
|
||||
|
||||
it('fails runaway growth', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('growth');
|
||||
});
|
||||
|
||||
it('fails placeholder leakage', () => {
|
||||
const res = runGates({
|
||||
candidate: 'You are {{persona.name}} — answer questions.',
|
||||
baseline: 'You are the researcher — answer questions.',
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('no-placeholders');
|
||||
});
|
||||
|
||||
it('fails unbalanced markdown fences', () => {
|
||||
// Pad baseline so the growth gate doesn't trip before the fence gate.
|
||||
const padding = ' '.repeat(100);
|
||||
const res = runGates({
|
||||
candidate: `Instructions:\n\`\`\`js\nconst x = 1;\nmore prose${padding}`,
|
||||
baseline: `Instructions: write code clearly.${padding}`,
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('balanced-fences');
|
||||
});
|
||||
|
||||
it('applies the regression gate when scores are supplied', () => {
|
||||
const res = runGates({
|
||||
candidate: 'new prompt',
|
||||
baseline: 'old prompt',
|
||||
scores: { baseline: 0.85, candidate: 0.6 },
|
||||
}, { maxRegression: -0.02 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('regression');
|
||||
});
|
||||
|
||||
it('skips the regression gate when scores are omitted', () => {
|
||||
const res = runGates({ candidate: 'new', baseline: 'old' });
|
||||
expect(res.results.find(r => r.gate === 'regression')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns results for every gate even when one fails', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.results.length).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,6 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
buildReflectiveMutationPrompt,
|
||||
buildSchemaFillPrompt,
|
||||
retryWithBackoff,
|
||||
wrapWithRetry,
|
||||
isRetryableEvolutionError,
|
||||
@@ -15,9 +10,6 @@ import {
|
||||
type RetryOptions,
|
||||
type RetryInfo,
|
||||
} from '../src/evolution-llm-wiring.js';
|
||||
import type { MutateArgs, GEPACandidate } from '../src/index.js';
|
||||
import type { Schema } from '../src/evolve-schema.js';
|
||||
import type { JudgeInput } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
|
||||
return {
|
||||
id: 'g1-m0',
|
||||
prompt: 'You are a tester. Respond with the exact expected word.',
|
||||
generation: 1,
|
||||
parent: 'g0-baseline',
|
||||
strategy: 'expand-edge-cases',
|
||||
score: null,
|
||||
perExample: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(overrides: Partial<Schema> = {}): Schema {
|
||||
return {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
|
||||
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
|
||||
],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── buildJudgeLLMCall ─────────────────────────────────────────────
|
||||
|
||||
describe('buildJudgeLLMCall', () => {
|
||||
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildGEPAMutateFn ─────────────────────────────────────────────
|
||||
|
||||
describe('buildGEPAMutateFn', () => {
|
||||
const mutateArgs: MutateArgs = {
|
||||
parent: makeCandidate(),
|
||||
strategy: 'expand-edge-cases',
|
||||
weaknessFeedback: [
|
||||
'Fails to handle empty input',
|
||||
'Sometimes forgets to include the final answer',
|
||||
],
|
||||
targetKind: 'persona-system-prompt',
|
||||
generation: 2,
|
||||
};
|
||||
|
||||
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe('EVOLVED SYSTEM PROMPT');
|
||||
expect(calls).toHaveLength(1);
|
||||
const prompt = calls[0];
|
||||
expect(prompt).toContain(mutateArgs.parent.prompt);
|
||||
expect(prompt).toContain(mutateArgs.strategy);
|
||||
expect(prompt).toContain('Fails to handle empty input');
|
||||
expect(prompt).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
|
||||
const { llm } = makeMockLLM(() => ' \n ');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('strips markdown code fences from the LLM response', async () => {
|
||||
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result.trim()).toBe('HIDDEN PROMPT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildReflectiveMutationPrompt', () => {
|
||||
it('includes the generation number and weakness bullets', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'tighten-format',
|
||||
weaknessFeedback: ['a', 'b'],
|
||||
targetKind: 'behavioral-spec-section',
|
||||
generation: 3,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('generation 3');
|
||||
expect(prompt).toContain('- a');
|
||||
expect(prompt).toContain('- b');
|
||||
expect(prompt).toContain('tighten-format');
|
||||
expect(prompt).toContain('PARENT');
|
||||
});
|
||||
|
||||
it('handles missing weakness feedback gracefully', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'add-examples',
|
||||
weaknessFeedback: [],
|
||||
targetKind: 'generic',
|
||||
generation: 0,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('PARENT');
|
||||
expect(prompt).toContain('(no specific weakness signals yet)');
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildSchemaExecuteFn ──────────────────────────────────────────
|
||||
|
||||
describe('buildSchemaExecuteFn', () => {
|
||||
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
|
||||
expect(calls[0]).toContain('What is the answer?');
|
||||
expect(calls[0]).toContain('"reasoning"');
|
||||
expect(calls[0]).toContain('"answer"');
|
||||
});
|
||||
|
||||
it('returns parsed=false when LLM returns non-JSON', async () => {
|
||||
const { llm } = makeMockLLM(() => 'just some text');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('just some text');
|
||||
});
|
||||
|
||||
it('extracts JSON from markdown fences', async () => {
|
||||
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toContain('"reasoning"');
|
||||
});
|
||||
|
||||
it('returns parsed=false and empty actual when LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSchemaFillPrompt', () => {
|
||||
it('serializes each field with name + type + description', () => {
|
||||
const prompt = buildSchemaFillPrompt({
|
||||
schema: makeSchema(),
|
||||
input: 'What is 2+2?',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('"reasoning"');
|
||||
expect(prompt).toContain('"answer"');
|
||||
expect(prompt).toContain('step-by-step thinking');
|
||||
expect(prompt).toContain('the final answer');
|
||||
expect(prompt).toContain('What is 2+2?');
|
||||
});
|
||||
|
||||
it('notes required constraints', () => {
|
||||
const schema: Schema = {
|
||||
name: 'x', version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'score',
|
||||
type: 'number',
|
||||
description: 'a rating',
|
||||
required: true,
|
||||
constraints: [{ kind: 'range', value: '0-10' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
|
||||
expect(prompt).toContain('score');
|
||||
expect(prompt).toContain('number');
|
||||
expect(prompt).toContain('range');
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeRunningJudge ──────────────────────────────────────────────
|
||||
|
||||
describe('makeRunningJudge', () => {
|
||||
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
|
||||
const { llm, calls } = makeMockLLM((prompt) => {
|
||||
// Simulate running the candidate prompt and returning a model response.
|
||||
return `MODEL OUTPUT for: ${prompt}`;
|
||||
});
|
||||
|
||||
const baseScore = vi.fn(async (args: JudgeInput) => ({
|
||||
overall: 0.8, weighted: 0.8,
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
lengthPenalty: 1, feedback: 'passed through', parsed: true,
|
||||
}));
|
||||
const baseJudge = { score: baseScore };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'What is love?',
|
||||
expected: 'baby don\'t hurt me',
|
||||
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
|
||||
});
|
||||
|
||||
// Model was asked to run the candidate prompt against the example input.
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
|
||||
expect(calls[0]).toContain('What is love?');
|
||||
|
||||
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
|
||||
expect(baseScore).toHaveBeenCalledTimes(1);
|
||||
const passedArgs = baseScore.mock.calls[0][0];
|
||||
expect(passedArgs.input).toBe('What is love?');
|
||||
expect(passedArgs.expected).toBe('baby don\'t hurt me');
|
||||
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
|
||||
|
||||
expect(result.overall).toBe(0.8);
|
||||
});
|
||||
|
||||
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('rate limited'); },
|
||||
};
|
||||
const baseJudge = { score: vi.fn() };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'any', expected: 'any', actual: 'prompt',
|
||||
});
|
||||
|
||||
expect(result.overall).toBe(0);
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.feedback).toMatch(/execution failed/i);
|
||||
expect(baseJudge.score).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes context through to the base judge', async () => {
|
||||
const { llm } = makeMockLLM(() => 'EXECUTED');
|
||||
const baseScore = vi.fn(async () => ({
|
||||
overall: 0.5, weighted: 0.5,
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
lengthPenalty: 1, feedback: '', parsed: true,
|
||||
}));
|
||||
|
||||
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
|
||||
|
||||
await runningJudge.score({
|
||||
input: 'q', expected: 'a', actual: 'prompt',
|
||||
context: 'persona:coder',
|
||||
});
|
||||
|
||||
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
|
||||
});
|
||||
});
|
||||
|
||||
// ── isRetryableEvolutionError ─────────────────────────────────────
|
||||
|
||||
describe('isRetryableEvolutionError', () => {
|
||||
|
||||
@@ -1,447 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
eligibleForEvolution,
|
||||
summarizeRuns,
|
||||
} from '../src/evolution-orchestrator.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
|
||||
import type { MutateFn } from '../src/iterative-optimizer.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(n => ({
|
||||
name: n, type: 'string' as const,
|
||||
description: `${n} field`,
|
||||
required: true, constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Executor that produces longer output for bigger schemas.
|
||||
function makeExec(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Judge that prefers candidates containing the "plus" token so the
|
||||
// mutateAppend helper ("... plus") wins reliably. Creates a measurable
|
||||
// delta between baseline and winner regardless of absolute lengths.
|
||||
function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
|
||||
return {
|
||||
async score(args) {
|
||||
const base = Math.min(1, args.actual.length / 100) * 0.6;
|
||||
const bonus = args.actual.includes('plus') ? 0.3 : 0;
|
||||
return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Mutate appends ~10 chars so growth ratio stays below the default +20%.
|
||||
const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
|
||||
|
||||
// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
|
||||
const BASELINE =
|
||||
'Return a concise answer that directly addresses the question. ' +
|
||||
'Provide a brief explanation and the final answer. Be clear and accurate.';
|
||||
|
||||
function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
|
||||
for (let i = 0; i < n; i++) {
|
||||
const id = store.start({
|
||||
personaId,
|
||||
input: `test question ${i} with enough length to pass filters`,
|
||||
workspaceId: 'ws-1',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
store.finalize(id, {
|
||||
outcome: 'success',
|
||||
output: `a full answer for question ${i} that is long enough for use`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function baseComposeOptions() {
|
||||
return {
|
||||
schema: {
|
||||
examples: [] as never[],
|
||||
execute: makeExec(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
examples: [] as never[],
|
||||
judge: makeJudge(),
|
||||
mutate: mutateAppend,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── eligibleForEvolution ──
|
||||
|
||||
describe('eligibleForEvolution', () => {
|
||||
it('keeps finalized traces with non-empty input and output', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, session_id: 's', persona_id: null, workspace_id: null,
|
||||
model: null, task_shape: null, outcome: 'success',
|
||||
cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
|
||||
payload: {
|
||||
input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('drops pending traces', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'pending',
|
||||
payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops traces with empty input', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'success',
|
||||
payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'corrected',
|
||||
payload: {
|
||||
input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
correctionFeedback: 'use bullets',
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── summarizeRuns ──
|
||||
|
||||
describe('summarizeRuns', () => {
|
||||
it('returns zero-aggregates on empty input', () => {
|
||||
const s = summarizeRuns([]);
|
||||
expect(s.total).toBe(0);
|
||||
expect(s.byStatus.proposed).toBe(0);
|
||||
expect(s.bestDelta).toBe(0);
|
||||
});
|
||||
|
||||
it('counts by status and target kind, tracks best delta', () => {
|
||||
const runs = [
|
||||
{ status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
|
||||
{ status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
|
||||
{ status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
|
||||
{ status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
|
||||
] as unknown as EvolutionRun[];
|
||||
const s = summarizeRuns(runs);
|
||||
expect(s.total).toBe(4);
|
||||
expect(s.byStatus.proposed).toBe(1);
|
||||
expect(s.byStatus.accepted).toBe(1);
|
||||
expect(s.byStatus.rejected).toBe(1);
|
||||
expect(s.byStatus.deployed).toBe(1);
|
||||
expect(s.byTargetKind['persona-system-prompt']).toBe(3);
|
||||
expect(s.byTargetKind['tool-description']).toBe(1);
|
||||
expect(s.bestDelta).toBeCloseTo(0.12, 5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolutionOrchestrator end-to-end ───────────────────────────
|
||||
|
||||
describe('EvolutionOrchestrator', () => {
|
||||
let db: MindDB;
|
||||
let traceStore: ExecutionTraceStore;
|
||||
let runStore: EvolutionRunStore;
|
||||
let orchestrator: EvolutionOrchestrator;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
traceStore = new ExecutionTraceStore(db);
|
||||
runStore = new EvolutionRunStore(db);
|
||||
orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── runOnce ──
|
||||
|
||||
describe('runOnce', () => {
|
||||
it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.run?.status).toBe('proposed');
|
||||
expect(result.run?.delta_accuracy).toBeGreaterThan(0);
|
||||
expect(result.compose).toBeDefined();
|
||||
expect(result.gateResults).toBeDefined();
|
||||
});
|
||||
|
||||
it('persists compose artifacts (schema JSON, gates, delta)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const run = result.run!;
|
||||
expect(run.winner_schema_json).toBeTruthy();
|
||||
const schema = JSON.parse(run.winner_schema_json!);
|
||||
expect(schema.name).toBe('test');
|
||||
expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('skips with skipped-delta when improvement is below threshold', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'a reasonably detailed baseline prompt that already scores well',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
// Mutate returns the same text → zero delta
|
||||
mutate: async ({ parent }) => parent.prompt,
|
||||
},
|
||||
},
|
||||
minDelta: 0.5, // artificially high
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-delta');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: skips when too few traces', async () => {
|
||||
seedSuccessfulTraces(traceStore, 2);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 50 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-trigger');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: proceeds when threshold is met', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 5 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
});
|
||||
|
||||
it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
// Judge that prefers the mutated "y" text — ensures the GEPA winner
|
||||
// is the oversized candidate, not the baseline.
|
||||
const yPreferringJudge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
|
||||
},
|
||||
};
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'tool-description',
|
||||
baseline: 'x'.repeat(30),
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
judge: yPreferringJudge,
|
||||
// Mutate produces candidate that blows past tool-description cap (500)
|
||||
mutate: async () => 'y'.repeat(2000),
|
||||
},
|
||||
},
|
||||
minDelta: 0, // allow any delta so we reach the gate stage
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-gates');
|
||||
expect(result.run?.status).toBe('rejected');
|
||||
expect(result.run?.gate_verdict).toBe('fail');
|
||||
expect(result.run?.user_note).toMatch(/gate failure/);
|
||||
});
|
||||
|
||||
it('aborts cleanly when signal fires before start', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.outcome).toBe('aborted');
|
||||
});
|
||||
|
||||
it('emits progress events', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const phases: string[] = [];
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('compose');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
});
|
||||
|
||||
// ── accept ──
|
||||
|
||||
describe('accept', () => {
|
||||
it('invokes deploy callback and marks run deployed on success', async () => {
|
||||
const deploy = vi.fn(async () => { /* success */ });
|
||||
const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await deployedOrch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
expect(run.outcome).toBe('proposed');
|
||||
|
||||
const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
|
||||
expect(accepted?.status).toBe('deployed');
|
||||
expect(deploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy callback throws', async () => {
|
||||
const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
|
||||
const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await orch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const result = await orch.accept(run.run!.run_uuid);
|
||||
expect(result?.status).toBe('failed');
|
||||
expect(result?.failure_reason).toContain('persona write failed');
|
||||
});
|
||||
|
||||
it('leaves run as accepted when no deploy hook configured', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const accepted = await orchestrator.accept(run.run!.run_uuid);
|
||||
expect(accepted?.status).toBe('accepted');
|
||||
});
|
||||
|
||||
it('returns undefined for unknown uuid', async () => {
|
||||
expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── reject ──
|
||||
|
||||
describe('reject', () => {
|
||||
it('marks proposed run as rejected', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
|
||||
expect(rejected?.status).toBe('rejected');
|
||||
expect(rejected?.user_note).toBe('regressed on custom cases');
|
||||
});
|
||||
});
|
||||
|
||||
// ── list / get ──
|
||||
|
||||
describe('list / get', () => {
|
||||
it('returns runs in reverse-chronological order', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'a',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'b',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const list = orchestrator.list();
|
||||
expect(list.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('get returns a specific run by uuid', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const created = await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'short',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const fetched = orchestrator.get(created.run!.run_uuid);
|
||||
expect(fetched?.id).toBe(created.run!.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,545 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
EvolveSchema,
|
||||
addOutputField,
|
||||
removeField,
|
||||
editFieldDescription,
|
||||
changeFieldType,
|
||||
addConstraint,
|
||||
removeConstraint,
|
||||
reorderFields,
|
||||
replaceOutputFields,
|
||||
schemaComplexity,
|
||||
aggregateSchemaScores,
|
||||
paretoFrontSchema,
|
||||
pickSchemaWinner,
|
||||
generateStructureMutations,
|
||||
generateOrderMutations,
|
||||
generateRefinementMutations,
|
||||
pickSample,
|
||||
type Schema,
|
||||
type SchemaField,
|
||||
type SchemaCandidate,
|
||||
type SchemaCandidateScore,
|
||||
type SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeField(name: string, partial: Partial<SchemaField> = {}): SchemaField {
|
||||
return {
|
||||
name,
|
||||
type: 'string',
|
||||
description: `description for ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(f => makeField(f)),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, schema: Schema, score: SchemaCandidateScore | null): SchemaCandidate {
|
||||
return {
|
||||
id,
|
||||
schema,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake executor: returns a deterministic output string that scores higher
|
||||
// when schema has more fields (up to a cap) and scores lower when schema
|
||||
// is bloated beyond 5 fields.
|
||||
function makeFakeExecutor(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more'): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => `${f.name}=mock`).join('; '),
|
||||
parsed: schema.fields.length > 0,
|
||||
});
|
||||
}
|
||||
|
||||
function makeJudge(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const fieldCount = (args.actual.match(/=/g) || []).length;
|
||||
let overall: number;
|
||||
if (bias === 'prefers-more') {
|
||||
// More fields = better, up to 6 fields, then flat
|
||||
overall = Math.min(1, fieldCount / 6);
|
||||
} else if (bias === 'prefers-less') {
|
||||
overall = Math.max(0, 1 - fieldCount / 10);
|
||||
} else {
|
||||
overall = 0.5;
|
||||
}
|
||||
return makeJudgeScore(overall);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ────────────────────────────────────
|
||||
|
||||
describe('addOutputField', () => {
|
||||
it('appends when no position given', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = addOutputField(s, makeField('c'));
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('inserts at given position', () => {
|
||||
const s = makeSchema(['a', 'c']);
|
||||
const next = addOutputField(s, makeField('b'), 1);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('does not mutate source', () => {
|
||||
const s = makeSchema(['a']);
|
||||
addOutputField(s, makeField('b'));
|
||||
expect(s.fields.map(f => f.name)).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeField', () => {
|
||||
it('drops the named field', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = removeField(s, 'b');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = removeField(s, 'z');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('editFieldDescription', () => {
|
||||
it('updates the description of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = editFieldDescription(s, 'a', 'new');
|
||||
expect(next.fields[0].description).toBe('new');
|
||||
});
|
||||
});
|
||||
|
||||
describe('changeFieldType', () => {
|
||||
it('updates the type of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = changeFieldType(s, 'a', 'number');
|
||||
expect(next.fields[0].type).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('addConstraint / removeConstraint', () => {
|
||||
it('adds a constraint', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'a', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes the given constraint index', () => {
|
||||
const s = addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 });
|
||||
const next = removeConstraint(s, 'a', 0);
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field when adding', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'z', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reorderFields', () => {
|
||||
it('reorders by name', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['c', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
|
||||
it('appends fields omitted from newOrder', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['b']);
|
||||
// 'b' first, then the rest in original order
|
||||
expect(next.fields.map(f => f.name)).toEqual(['b', 'a', 'c']);
|
||||
});
|
||||
|
||||
it('ignores unknown field names in newOrder', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = reorderFields(s, ['unknown', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('replaceOutputFields', () => {
|
||||
it('replaces all fields', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = replaceOutputFields(s, [makeField('x'), makeField('y')]);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['x', 'y']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Complexity + scoring ───────────────────────────────────────
|
||||
|
||||
describe('schemaComplexity', () => {
|
||||
it('returns 0 for empty schema', () => {
|
||||
expect(schemaComplexity(makeSchema([]))).toBe(0);
|
||||
});
|
||||
|
||||
it('scales with field count, constraint count, and description length', () => {
|
||||
const simple = schemaComplexity(makeSchema(['a']));
|
||||
const twoField = schemaComplexity(makeSchema(['a', 'b']));
|
||||
expect(twoField).toBeGreaterThan(simple);
|
||||
|
||||
const withConstraint = schemaComplexity(
|
||||
addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 }),
|
||||
);
|
||||
expect(withConstraint).toBeGreaterThan(simple);
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregateSchemaScores', () => {
|
||||
it('returns zero aggregate on empty results', () => {
|
||||
const agg = aggregateSchemaScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.accuracy).toBe(0);
|
||||
expect(agg.parseRate).toBe(0);
|
||||
});
|
||||
|
||||
it('averages accuracy + computes parse rate', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.8), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.6), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'X', score: makeJudgeScore(0.2), parsed: false },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.n).toBe(3);
|
||||
expect(agg.accuracy).toBeCloseTo((0.8 + 0.6 + 0.2) / 3, 5);
|
||||
expect(agg.parseRate).toBeCloseTo(2 / 3, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 example feedback', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.9, 'best'), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.1, 'worst'), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'C', score: makeJudgeScore(0.5, 'middle'), parsed: true },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('middle');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pareto ─────────────────────────────────────────────────────
|
||||
|
||||
describe('paretoFrontSchema', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('removes strictly dominated candidates', () => {
|
||||
const dominated = makeCandidate('d', makeSchema(['a']), makeScore(0.5, 5));
|
||||
const better = makeCandidate('b', makeSchema(['a']), makeScore(0.8, 3));
|
||||
const front = paretoFrontSchema([dominated, better]);
|
||||
expect(front.map(c => c.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates (accurate vs simple)', () => {
|
||||
const accurate = makeCandidate('acc', makeSchema(['a', 'b']), makeScore(0.9, 10));
|
||||
const simple = makeCandidate('sim', makeSchema(['a']), makeScore(0.7, 2));
|
||||
const front = paretoFrontSchema([accurate, simple]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores unscored candidates', () => {
|
||||
const scored = makeCandidate('s', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const unscored = makeCandidate('u', makeSchema(['a']), null);
|
||||
const front = paretoFrontSchema([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSchemaWinner', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('picks highest accuracy', () => {
|
||||
const lo = makeCandidate('lo', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const hi = makeCandidate('hi', makeSchema(['a']), makeScore(0.9, 10));
|
||||
expect(pickSchemaWinner([lo, hi]).id).toBe('hi');
|
||||
});
|
||||
|
||||
it('ties on accuracy → prefers lower complexity', () => {
|
||||
const complex = makeCandidate('cx', makeSchema(['a', 'b']), makeScore(0.8, 10));
|
||||
const simple = makeCandidate('sm', makeSchema(['a']), makeScore(0.8, 3));
|
||||
expect(pickSchemaWinner([complex, simple]).id).toBe('sm');
|
||||
});
|
||||
|
||||
it('throws on empty', () => {
|
||||
expect(() => pickSchemaWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Mutation generators ────────────────────────────────────────
|
||||
|
||||
describe('generateStructureMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('suggests adding reasoning if absent', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.description.includes('reasoning'))).toBe(true);
|
||||
});
|
||||
|
||||
it('does not re-suggest reasoning if already present', () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
const hasReasoning = muts.some(m => m.description.includes('reasoning field'));
|
||||
expect(hasReasoning).toBe(false);
|
||||
});
|
||||
|
||||
it('suggests drop when schema is large', () => {
|
||||
const s = makeSchema(['a', 'b', 'c', 'd', 'e']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.kind === 'remove_field')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns at most n mutations', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 1, rng);
|
||||
expect(muts.length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateOrderMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty for 1-field schemas', () => {
|
||||
expect(generateOrderMutations(makeSchema(['a']), 3, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('moves a reasoning field to the front', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('answer'), makeField('reasoning'), makeField('confidence')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const applied = muts[0].apply(s);
|
||||
expect(applied.fields[0].name).toBe('reasoning');
|
||||
});
|
||||
|
||||
it('moves a confidence field to the end', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('confidence'), makeField('reasoning'), makeField('answer')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const last = muts[0].apply(s);
|
||||
// One of the generated mutations should put 'confidence' at the end
|
||||
const someMovesConfidenceToEnd = muts.some(m => {
|
||||
const applied = m.apply(s);
|
||||
return applied.fields[applied.fields.length - 1].name === 'confidence';
|
||||
});
|
||||
expect(someMovesConfidenceToEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateRefinementMutations', () => {
|
||||
it('adds maxLength when feedback mentions verbosity', async () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['too verbose, output wordy'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'add_constraint')).toBe(true);
|
||||
});
|
||||
|
||||
it('adds minLength when feedback mentions incompleteness', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['response is too brief, missing detail'], 5,
|
||||
);
|
||||
expect(muts.some(m =>
|
||||
m.description.includes('minLength'),
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
it('edits field descriptions when feedback mentions format issues', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format, could not parse'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'edit_field_desc')).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back to a single description rewrite when no heuristic matches', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(s, ['something unrelated'], 5);
|
||||
expect(muts.length).toBeGreaterThan(0);
|
||||
expect(muts[0].kind).toBe('edit_field_desc');
|
||||
});
|
||||
|
||||
it('uses LLM-provided editor when supplied', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format'], 5,
|
||||
async ({ field }) => `LLM-REWRITE of ${field.name}`,
|
||||
);
|
||||
const edit = muts.find(m => m.kind === 'edit_field_desc');
|
||||
const applied = edit!.apply(s);
|
||||
expect(applied.fields[0].description).toBe('LLM-REWRITE of answer');
|
||||
});
|
||||
});
|
||||
|
||||
// ── pickSample ─────────────────────────────────────────────────
|
||||
|
||||
describe('pickSample', () => {
|
||||
it('returns up to k items', () => {
|
||||
expect(pickSample(makeExamples(5), 3, () => 0.5)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(5), 0, () => 0.5)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolveSchema.run end-to-end ───────────────────────────────
|
||||
|
||||
describe('EvolveSchema.run', () => {
|
||||
it('runs through all phases and returns a winner', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('winner accuracy >= baseline accuracy when judge prefers richer schemas', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner.score!.accuracy).toBeGreaterThanOrEqual(
|
||||
result.history[0].score!.accuracy,
|
||||
);
|
||||
});
|
||||
|
||||
it('emits progress events for each phase', async () => {
|
||||
const phases: string[] = [];
|
||||
await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('start');
|
||||
expect(phases).toContain('structure');
|
||||
expect(phases).toContain('order');
|
||||
expect(phases).toContain('refinement');
|
||||
expect(phases).toContain('anchor');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const cfg = () => ({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 8,
|
||||
seed: 7,
|
||||
});
|
||||
const a = await new EvolveSchema().run(cfg());
|
||||
const b = await new EvolveSchema().run(cfg());
|
||||
expect(a.winner.schema.fields.map(f => f.name)).toEqual(
|
||||
b.winner.schema.fields.map(f => f.name),
|
||||
);
|
||||
});
|
||||
|
||||
it('respects abort signal', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
|
||||
it('records all candidates in history with generation numbers', async () => {
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
mutationMix: { structure: 2, order: 0, refinement: 0 },
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
const gen0 = result.history.filter(c => c.generation === 0);
|
||||
const gen1 = result.history.filter(c => c.generation === 1);
|
||||
expect(gen0.length).toBe(1);
|
||||
expect(gen1.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -1,226 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
recordCapabilityGap,
|
||||
analyzeAndRecordCorrection,
|
||||
recordWorkflowPattern,
|
||||
buildAwarenessSummary,
|
||||
formatAwarenessPrompt,
|
||||
markSummarySurfaced,
|
||||
} from '../src/improvement-detector.js';
|
||||
|
||||
describe('improvement-detector', () => {
|
||||
let db: MindDB;
|
||||
let store: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── recordCapabilityGap ────────────────────────────────────
|
||||
|
||||
describe('recordCapabilityGap', () => {
|
||||
it('records a capability gap signal', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader', 'user asked to read a PDF');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].pattern_key).toBe('missing:pdf_reader');
|
||||
expect(gaps[0].count).toBe(1);
|
||||
});
|
||||
|
||||
it('increments count on repeated gap', () => {
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].count).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── analyzeAndRecordCorrection ─────────────────────────────
|
||||
|
||||
describe('analyzeAndRecordCorrection', () => {
|
||||
it('returns null for non-correction messages', () => {
|
||||
const result = analyzeAndRecordCorrection(store, 'Please write a sorting function');
|
||||
expect(result).toBeNull();
|
||||
expect(store.getByCategory('correction')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('detects and records a correction', () => {
|
||||
const result = analyzeAndRecordCorrection(
|
||||
store,
|
||||
"No, that's wrong. I told you to use markdown headers.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBeDefined();
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records both durable and task-local corrections', () => {
|
||||
analyzeAndRecordCorrection(store, "No, always use shorter responses. I said keep it brief.");
|
||||
analyzeAndRecordCorrection(store, "No, not that. Just use a list this time.");
|
||||
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordWorkflowPattern ──────────────────────────────────
|
||||
|
||||
describe('recordWorkflowPattern', () => {
|
||||
it('records a workflow pattern', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].pattern_key).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('increments on repeated task shapes', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
recordWorkflowPattern(store, 'research', 'investigate market trends');
|
||||
recordWorkflowPattern(store, 'research', 'review latest papers');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].count).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildAwarenessSummary ──────────────────────────────────
|
||||
|
||||
describe('buildAwarenessSummary', () => {
|
||||
it('returns empty summary when no actionable signals', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBe(0);
|
||||
expect(summary.capabilityGaps).toEqual([]);
|
||||
expect(summary.corrections).toEqual([]);
|
||||
expect(summary.workflowPatterns).toEqual([]);
|
||||
});
|
||||
|
||||
it('includes capability gaps above threshold', () => {
|
||||
// Default threshold for capability_gap is 2
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
expect(summary.capabilityGaps[0].toolName).toBe('pdf_reader');
|
||||
expect(summary.capabilityGaps[0].occurrences).toBe(2);
|
||||
expect(summary.capabilityGaps[0].suggestion).toContain('pdf_reader');
|
||||
});
|
||||
|
||||
it('includes corrections above threshold', () => {
|
||||
// Default threshold for correction is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep responses casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.corrections).toHaveLength(1);
|
||||
expect(summary.corrections[0].patternKey).toBe('tone:too_formal');
|
||||
expect(summary.corrections[0].occurrences).toBe(3);
|
||||
});
|
||||
|
||||
it('includes recent workflow patterns above threshold', () => {
|
||||
// Default threshold for workflow_pattern is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'research', 'investigate topic');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(1);
|
||||
expect(summary.workflowPatterns[0].patternKey).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('excludes stale workflow patterns', () => {
|
||||
// Manually insert a workflow_pattern with old last_seen
|
||||
const raw = db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO improvement_signals (category, pattern_key, count, last_seen)
|
||||
VALUES ('workflow_pattern', 'shape:old_task', 5, datetime('now', '-30 days'))
|
||||
`).run();
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('respects total cap of 3 actionable signals', () => {
|
||||
// Create many signals above threshold
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const key = `missing:tool_${i}`;
|
||||
store.record('capability_gap', key);
|
||||
store.record('capability_gap', key);
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── formatAwarenessPrompt ──────────────────────────────────
|
||||
|
||||
describe('formatAwarenessPrompt', () => {
|
||||
it('returns null for empty summary', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(formatAwarenessPrompt(summary)).toBeNull();
|
||||
});
|
||||
|
||||
it('formats capability gaps into prompt text', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).not.toBeNull();
|
||||
expect(prompt).toContain('## Improvement Signals');
|
||||
expect(prompt).toContain('pdf_reader');
|
||||
expect(prompt).toContain('Missing capabilities');
|
||||
});
|
||||
|
||||
it('formats corrections into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep it casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Behavioral adjustments');
|
||||
expect(prompt).toContain('casual');
|
||||
});
|
||||
|
||||
it('formats workflow patterns into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'compare', 'compare options');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Recurring workflows');
|
||||
expect(prompt).toContain('compare');
|
||||
});
|
||||
});
|
||||
|
||||
// ── markSummarySurfaced ────────────────────────────────────
|
||||
|
||||
describe('markSummarySurfaced', () => {
|
||||
it('marks all signals in summary as surfaced', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
|
||||
markSummarySurfaced(store, summary);
|
||||
|
||||
// Should not appear in next summary
|
||||
const summary2 = buildAwarenessSummary(store);
|
||||
expect(summary2.capabilityGaps).toHaveLength(0);
|
||||
expect(summary2.totalActionable).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,153 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { processInteractionForImprovement } from '../src/improvement-wiring.js';
|
||||
|
||||
describe('improvement-wiring', () => {
|
||||
const baseParams = {
|
||||
workspaceId: 'ws-1',
|
||||
sessionId: 'sess-1',
|
||||
toolsUsed: [],
|
||||
agentResponse: 'Here is the result.',
|
||||
userMessage: '',
|
||||
};
|
||||
|
||||
// ── Correction detection ─────────────────────────────────────
|
||||
|
||||
describe('correction detection', () => {
|
||||
it('detects corrections in user messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use bullet points, not paragraphs.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.correctionDetail).toBeDefined();
|
||||
expect(result.correctionDetail!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for normal messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you help me write a function to sort an array?',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.correctionDetail).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for very short messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'ok',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
});
|
||||
|
||||
it('detects "I said" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'I said to use markdown headers, not plain text.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
|
||||
it('detects "actually instead use" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Actually, instead use TypeScript for this module.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Capability gap detection ─────────────────────────────────
|
||||
|
||||
describe('capability gap detection', () => {
|
||||
it('identifies capability gaps from tool-not-found in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Read this PDF for me.',
|
||||
agentResponse: 'Tool "pdf_reader" not found. I cannot read PDFs directly.',
|
||||
});
|
||||
expect(result.capabilityGap).toBe('pdf_reader');
|
||||
});
|
||||
|
||||
it('identifies capability gaps from "I don\'t have a tool" in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send this to Slack?',
|
||||
agentResponse: "I don't have a tool for sending Slack messages.",
|
||||
});
|
||||
expect(result.capabilityGap).toBe('sending Slack messages');
|
||||
});
|
||||
|
||||
it('identifies gaps from user requesting integration', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send to Slack the summary?',
|
||||
agentResponse: 'Let me try to do that.',
|
||||
toolsUsed: [],
|
||||
});
|
||||
expect(result.capabilityGap).toBe('slack');
|
||||
});
|
||||
|
||||
it('returns no gap for normal interactions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the weather today?',
|
||||
agentResponse: 'The weather today is sunny and 72F.',
|
||||
});
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Workflow pattern detection ───────────────────────────────
|
||||
|
||||
describe('workflow pattern detection', () => {
|
||||
it('detects multi-step workflow patterns with "then"', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'First research the competitors, then compare their features, then draft a summary report.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeDefined();
|
||||
});
|
||||
|
||||
it('does not flag single-step requests as workflow patterns', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Write a summary of this document.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for simple questions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the status of the project?',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Combined signals ─────────────────────────────────────────
|
||||
|
||||
describe('combined signals', () => {
|
||||
it('can detect correction and gap simultaneously', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use the Jira tool.",
|
||||
agentResponse: 'Tool "jira_create" not found.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.capabilityGap).toBe('jira_create');
|
||||
});
|
||||
|
||||
it('returns all-false for benign exchange', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Thanks, that looks great!',
|
||||
agentResponse: "You're welcome! Let me know if you need anything else.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,655 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
paretoFront,
|
||||
scoreCandidate,
|
||||
aggregateScores,
|
||||
pickWinner,
|
||||
pickSample,
|
||||
type Candidate,
|
||||
type CandidateScore,
|
||||
type IterativeGEPAOptions,
|
||||
type MutateFn,
|
||||
} from '../src/iterative-optimizer.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
/**
|
||||
* H-09 G3 — unit tests mock the judge, so they opt into the running-judge
|
||||
* safety-check bypass. Production code paths must still wrap with
|
||||
* makeRunningJudge(); the check in IterativeGEPA enforces that.
|
||||
*/
|
||||
function runGEPA(options: Omit<IterativeGEPAOptions, 'allowBareJudge'>) {
|
||||
const gepa = new IterativeGEPA();
|
||||
return gepa.run({ ...options, allowBareJudge: true });
|
||||
}
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `question ${i}`,
|
||||
expected_output: `answer ${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeScore(overall: number, extra: Partial<JudgeScore> = {}): JudgeScore {
|
||||
return {
|
||||
overall,
|
||||
weighted: overall,
|
||||
correctness: extra.correctness ?? overall,
|
||||
procedureFollowing: extra.procedureFollowing ?? overall,
|
||||
conciseness: extra.conciseness ?? overall,
|
||||
lengthPenalty: extra.lengthPenalty ?? 1,
|
||||
feedback: extra.feedback ?? `feedback for score ${overall}`,
|
||||
parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, score: CandidateScore | null): Candidate {
|
||||
return {
|
||||
id,
|
||||
prompt: `prompt-${id}`,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidateScore(
|
||||
overall: number,
|
||||
dims: Partial<{ correctness: number; procedureFollowing: number; conciseness: number; lengthPenalty: number; n: number; weaknessFeedback: string[] }> = {},
|
||||
): CandidateScore {
|
||||
return {
|
||||
overall,
|
||||
correctness: dims.correctness ?? overall,
|
||||
procedureFollowing: dims.procedureFollowing ?? overall,
|
||||
conciseness: dims.conciseness ?? overall,
|
||||
lengthPenalty: dims.lengthPenalty ?? 1,
|
||||
n: dims.n ?? 10,
|
||||
weaknessFeedback: dims.weaknessFeedback ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake judge that computes a deterministic score based on candidate text length.
|
||||
// Longer prompts → higher correctness, shorter → higher conciseness.
|
||||
function makeFakeJudge(bias: 'long' | 'short' | 'uniform' = 'uniform') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const len = args.actual.length;
|
||||
const corr = bias === 'long' ? Math.min(1, len / 50) : bias === 'short' ? Math.max(0, 1 - len / 100) : 0.5;
|
||||
const proc = 0.6;
|
||||
const conc = bias === 'short' ? 0.9 : 0.5;
|
||||
const overall = 0.5 * corr + 0.3 * proc + 0.2 * conc;
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: corr,
|
||||
procedureFollowing: proc,
|
||||
conciseness: conc,
|
||||
lengthPenalty: 1,
|
||||
feedback: `len=${len}`,
|
||||
parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────
|
||||
|
||||
describe('aggregateScores', () => {
|
||||
it('handles empty input', () => {
|
||||
const agg = aggregateScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.overall).toBe(0);
|
||||
expect(agg.weaknessFeedback).toEqual([]);
|
||||
});
|
||||
|
||||
it('averages component dimensions', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.8, { correctness: 0.9, procedureFollowing: 0.7, conciseness: 0.8 }),
|
||||
makeScore(0.6, { correctness: 0.7, procedureFollowing: 0.5, conciseness: 0.6 }),
|
||||
]);
|
||||
expect(agg.n).toBe(2);
|
||||
expect(agg.correctness).toBeCloseTo(0.8, 5);
|
||||
expect(agg.procedureFollowing).toBeCloseTo(0.6, 5);
|
||||
expect(agg.conciseness).toBeCloseTo(0.7, 5);
|
||||
expect(agg.overall).toBeCloseTo(0.7, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 feedback lines in weaknessFeedback', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.9, { feedback: 'best' }),
|
||||
makeScore(0.1, { feedback: 'worst' }),
|
||||
makeScore(0.2, { feedback: 'second-worst' }),
|
||||
makeScore(0.3, { feedback: 'third-worst' }),
|
||||
makeScore(0.8, { feedback: 'good' }),
|
||||
]);
|
||||
// Worst 3 should contain 'worst', 'second-worst', 'third-worst' in some order
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('second-worst');
|
||||
expect(agg.weaknessFeedback).toContain('third-worst');
|
||||
expect(agg.weaknessFeedback).not.toContain('best');
|
||||
});
|
||||
|
||||
it('drops empty feedback from weakness list', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.1, { feedback: '' }),
|
||||
makeScore(0.2, { feedback: 'real feedback' }),
|
||||
]);
|
||||
expect(agg.weaknessFeedback).toEqual(['real feedback']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('paretoFront', () => {
|
||||
it('keeps a single member when only one has a score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.8));
|
||||
const b = makeCandidate('b', null);
|
||||
expect(paretoFront([a, b])).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes strictly-dominated candidates', () => {
|
||||
const weak = makeCandidate('weak', makeCandidateScore(0.5, {
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
}));
|
||||
const strong = makeCandidate('strong', makeCandidateScore(0.8, {
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
}));
|
||||
const front = paretoFront([weak, strong]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('strong');
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates on the front', () => {
|
||||
const accurate = makeCandidate('accurate', makeCandidateScore(0.7, {
|
||||
correctness: 0.9, procedureFollowing: 0.6, conciseness: 0.5,
|
||||
}));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, {
|
||||
correctness: 0.6, procedureFollowing: 0.6, conciseness: 0.9,
|
||||
}));
|
||||
const front = paretoFront([accurate, concise]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('returns all candidates when they have identical scores', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.5));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.5));
|
||||
const front = paretoFront([a, b]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores candidates with null scores entirely', () => {
|
||||
const scored = makeCandidate('scored', makeCandidateScore(0.5));
|
||||
const unscored = makeCandidate('unscored', null);
|
||||
const front = paretoFront([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('scored');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickWinner', () => {
|
||||
it('picks highest overall score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.6));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.9));
|
||||
expect(pickWinner([a, b]).id).toBe('b');
|
||||
});
|
||||
|
||||
it('breaks ties using length penalty (higher = more concise, preferred)', () => {
|
||||
const verbose = makeCandidate('verbose', makeCandidateScore(0.7, { lengthPenalty: 0.6 }));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, { lengthPenalty: 0.9 }));
|
||||
expect(pickWinner([verbose, concise]).id).toBe('concise');
|
||||
});
|
||||
|
||||
it('breaks further ties by earliest generation', () => {
|
||||
const early = makeCandidate('early', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
const late = makeCandidate('late', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
late.generation = 3;
|
||||
expect(pickWinner([early, late]).id).toBe('early');
|
||||
});
|
||||
|
||||
it('throws on empty input', () => {
|
||||
expect(() => pickWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSample', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(10), 0, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns full set (shuffled) when k >= length', () => {
|
||||
const sample = pickSample(makeExamples(5), 10, rng);
|
||||
expect(sample).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('is deterministic with a fixed rng', () => {
|
||||
const rngA = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const rngB = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const a = pickSample(makeExamples(20), 5, rngA).map(e => e.input);
|
||||
const b = pickSample(makeExamples(20), 5, rngB).map(e => e.input);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreCandidate', () => {
|
||||
it('scores candidate against examples and populates the score field', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), makeFakeJudge('uniform'));
|
||||
expect(score.n).toBe(3);
|
||||
expect(cand.score).not.toBeNull();
|
||||
expect(cand.perExample).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns zero-score aggregation when examples list is empty', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, [], makeFakeJudge());
|
||||
expect(score.n).toBe(0);
|
||||
expect(score.overall).toBe(0);
|
||||
});
|
||||
|
||||
it('continues iterating if one score call throws', async () => {
|
||||
let callCount = 0;
|
||||
const flakyJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
callCount++;
|
||||
if (callCount === 2) throw new Error('transient');
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), flakyJudge);
|
||||
expect(score.n).toBe(2); // 2 succeeded, 1 failed
|
||||
});
|
||||
|
||||
it('stops early when abort signal fires', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const judge = makeFakeJudge();
|
||||
// Abort before any call
|
||||
ctrl.abort();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(10), judge, ctrl.signal);
|
||||
expect(cand.perExample.length).toBeLessThan(10);
|
||||
});
|
||||
|
||||
// ── Concurrency ────────────────────────────────────────────────
|
||||
|
||||
it('accepts an options object (new API) equivalently to AbortSignal (legacy)', async () => {
|
||||
const judge = makeFakeJudge('uniform');
|
||||
const candA = makeCandidate('a', null);
|
||||
const candB = makeCandidate('b', null);
|
||||
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
|
||||
await scoreCandidate(candA, makeExamples(5), judge, ctrl.signal);
|
||||
await scoreCandidate(candB, makeExamples(5), judge, { signal: ctrl.signal });
|
||||
|
||||
expect(candA.perExample.length).toBe(candB.perExample.length);
|
||||
});
|
||||
|
||||
it('concurrency=1 produces identical aggregate to no option (sequential baseline)', async () => {
|
||||
const examples = makeExamples(6);
|
||||
const candSeq = makeCandidate('seq', null);
|
||||
const candPar = makeCandidate('par', null);
|
||||
|
||||
const seqScore = await scoreCandidate(candSeq, examples, makeFakeJudge('uniform'));
|
||||
const parScore = await scoreCandidate(candPar, examples, makeFakeJudge('uniform'), { concurrency: 1 });
|
||||
|
||||
expect(parScore.overall).toBeCloseTo(seqScore.overall);
|
||||
expect(parScore.n).toBe(seqScore.n);
|
||||
});
|
||||
|
||||
it('with concurrency > 1, runs scores in parallel (observable via in-flight counter)', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
// Await a microtask + a real delay so parallel workers can accumulate.
|
||||
await new Promise(r => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(8), judge, { concurrency: 4 });
|
||||
|
||||
expect(peakInFlight).toBe(4);
|
||||
expect(cand.perExample).toHaveLength(8);
|
||||
});
|
||||
|
||||
it('with concurrency=1, exactly 1 in-flight call at a time', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 3));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(5), judge, { concurrency: 1 });
|
||||
expect(peakInFlight).toBe(1);
|
||||
});
|
||||
|
||||
it('concurrency is capped at examples.length (does not start idle workers)', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(3), judge, { concurrency: 100 });
|
||||
expect(peakInFlight).toBeLessThanOrEqual(3);
|
||||
});
|
||||
|
||||
it('parallel mode still filters out thrown-error results without corrupting the batch', async () => {
|
||||
let call = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
call++;
|
||||
if (call % 3 === 0) throw new Error('flaky');
|
||||
return makeScore(0.7);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(9), judge, { concurrency: 3 });
|
||||
// 9 total, every 3rd throws → 6 succeed.
|
||||
expect(score.n).toBe(6);
|
||||
});
|
||||
|
||||
it('parallel mode respects abort signal by not dispatching further workers', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const judge = makeFakeJudge();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(20), judge, {
|
||||
signal: ctrl.signal,
|
||||
concurrency: 4,
|
||||
});
|
||||
expect(cand.perExample.length).toBeLessThan(20);
|
||||
});
|
||||
});
|
||||
|
||||
// ── IterativeGEPA end-to-end with concurrency ────────────────────
|
||||
|
||||
describe('IterativeGEPA with concurrency', () => {
|
||||
it('threads options.concurrency into every scoreCandidate call', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt} :: ${strategy}`;
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 4,
|
||||
miniEvalSize: 4,
|
||||
anchorEvalSize: 4,
|
||||
concurrency: 3,
|
||||
});
|
||||
|
||||
expect(peakInFlight).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-09 G3 · running-judge guard ──────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run · running-judge guard (H-09 G3)', () => {
|
||||
it('throws when a bare judge is passed without allowBareJudge', async () => {
|
||||
const bareJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.5, weighted: 0.5, correctness: 0.5,
|
||||
procedureFollowing: 0.5, conciseness: 0.5, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} mutated`;
|
||||
await expect(
|
||||
new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: bareJudge,
|
||||
mutate,
|
||||
}),
|
||||
).rejects.toThrow(/not a running judge|makeRunningJudge/);
|
||||
});
|
||||
|
||||
it('accepts a judge wrapped with makeRunningJudge', async () => {
|
||||
const { makeRunningJudge } = await import('../src/evolution-llm-wiring.js');
|
||||
const baseJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.8, weighted: 0.8, correctness: 0.8,
|
||||
procedureFollowing: 0.8, conciseness: 0.8, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const fakeLLM = { complete: async () => 'LLM output' };
|
||||
const wrapped = makeRunningJudge(baseJudge, fakeLLM);
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} v2`;
|
||||
// Should not throw — the wrapped judge carries the brand.
|
||||
const result = await new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: wrapped,
|
||||
mutate,
|
||||
populationSize: 1,
|
||||
generations: 1,
|
||||
microScreenSize: 1, miniEvalSize: 1, anchorEvalSize: 1,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── End-to-end run ─────────────────────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run', () => {
|
||||
it('runs through all phases and produces a winner', async () => {
|
||||
const baseline = 'short baseline prompt';
|
||||
const examples = makeExamples(10);
|
||||
const judge = makeFakeJudge('uniform');
|
||||
|
||||
// Mutate appends text — simulates generation of children
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => {
|
||||
return `${parent.prompt} :: ${strategy}`;
|
||||
};
|
||||
|
||||
const progress: string[] = [];
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
onProgress: (e) => progress.push(`${e.phase}@g${e.generation}`),
|
||||
});
|
||||
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
expect(progress).toContain('start@g0');
|
||||
expect(progress.some(p => p.startsWith('anchor'))).toBe(true);
|
||||
expect(progress.some(p => p.startsWith('done'))).toBe(true);
|
||||
});
|
||||
|
||||
it('winner dominates or matches baseline on overall score', async () => {
|
||||
const baseline = 'x'; // very short
|
||||
const examples = makeExamples(20);
|
||||
// Long-biased judge: longer prompts score higher
|
||||
const judge = makeFakeJudge('long');
|
||||
|
||||
// Mutate doubles length each time
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} ${parent.prompt}more`;
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 15,
|
||||
});
|
||||
|
||||
// Winner should have evolved (not baseline prompt).
|
||||
expect(result.winner.prompt.length).toBeGreaterThan(baseline.length);
|
||||
expect(result.improved).toBe(true);
|
||||
expect(result.delta).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('passes weakness feedback from parent into mutate()', async () => {
|
||||
const captured: Array<{ strategy: string; feedbacks: string[] }> = [];
|
||||
const mutate: MutateFn = async ({ parent, strategy, weaknessFeedback }) => {
|
||||
captured.push({ strategy, feedbacks: weaknessFeedback });
|
||||
return `${parent.prompt} mutated`;
|
||||
};
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(10),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 5,
|
||||
});
|
||||
|
||||
// 2 mutations were spawned — each received the parent's feedback
|
||||
expect(captured.length).toBe(2);
|
||||
for (const c of captured) {
|
||||
expect(Array.isArray(c.feedbacks)).toBe(true);
|
||||
// Feedback array may be empty on first gen if baseline had no weaknesses;
|
||||
// we just assert the prop is present and is an array.
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers when mutate throws (falls back to parent prompt)', async () => {
|
||||
let mutations = 0;
|
||||
const mutate: MutateFn = async ({ parent }) => {
|
||||
mutations++;
|
||||
if (mutations === 1) throw new Error('boom');
|
||||
return `${parent.prompt} ok`;
|
||||
};
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
// Winner must be defined even though one mutation failed
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
});
|
||||
|
||||
it('respects abort signal and exits early without throwing', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const mutate: MutateFn = vi.fn(async () => 'should never run');
|
||||
|
||||
ctrl.abort(); // abort before starting generations
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
microScreenSize: 2,
|
||||
miniEvalSize: 2,
|
||||
anchorEvalSize: 2,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
// With abort before baseline scoring, no winner should evolve
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(mutate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const examples = makeExamples(20);
|
||||
const judge = makeFakeJudge();
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt}::${strategy}`;
|
||||
|
||||
const run = () => runGEPA({
|
||||
baseline: 'seed',
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
seed: 42,
|
||||
});
|
||||
|
||||
const r1 = await run();
|
||||
const r2 = await run();
|
||||
|
||||
expect(r1.winner.prompt).toBe(r2.winner.prompt);
|
||||
expect(r1.history.length).toBe(r2.history.length);
|
||||
});
|
||||
|
||||
it('produces history containing baseline and all mutated children', async () => {
|
||||
const result = await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate: async ({ parent, strategy }) => `${parent.prompt}_${strategy}`,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
expect(result.history[0].id).toBe('g0-baseline');
|
||||
expect(result.history[0].strategy).toBe('baseline');
|
||||
// 2 generations * 3 mutations + 1 baseline = 7 total
|
||||
expect(result.history.length).toBe(7);
|
||||
});
|
||||
});
|
||||
@@ -1,280 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import {
|
||||
captureInteraction,
|
||||
getRecentLogs,
|
||||
getWorkspaceLogs,
|
||||
isWithinBudget,
|
||||
} from '../src/optimization-capture.js';
|
||||
import type { AgentResponse } from '../src/agent-loop.js';
|
||||
|
||||
describe('optimization-capture', () => {
|
||||
let db: MindDB;
|
||||
let store: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function makeResponse(overrides?: Partial<AgentResponse>): AgentResponse {
|
||||
return {
|
||||
content: 'Hello, I can help with that.',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
usage: { inputTokens: 500, outputTokens: 200 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── captureInteraction ─────────────────────────────────────
|
||||
|
||||
describe('captureInteraction', () => {
|
||||
it('stores a log entry with correct fields', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'You are a helpful assistant.',
|
||||
response: makeResponse(),
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.id).toBeGreaterThan(0);
|
||||
expect(entry.session_id).toBe('session-1');
|
||||
expect(entry.workspace_id).toBe('ws-abc');
|
||||
expect(entry.system_prompt).toBe('You are a helpful assistant.');
|
||||
expect(JSON.parse(entry.tools_used)).toEqual(['search_memory', 'save_memory']);
|
||||
expect(entry.turn_count).toBe(3);
|
||||
expect(entry.was_correction).toBe(0);
|
||||
expect(entry.input_tokens).toBe(500);
|
||||
expect(entry.output_tokens).toBe(200);
|
||||
expect(entry.timestamp).toBeTruthy();
|
||||
});
|
||||
|
||||
it('records wasCorrection as 1 when true', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse(),
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
expect(entry.was_correction).toBe(1);
|
||||
});
|
||||
|
||||
it('handles empty toolsUsed', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse({ toolsUsed: [] }),
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(JSON.parse(entry.tools_used)).toEqual([]);
|
||||
});
|
||||
|
||||
it('defaults token counts to 0 when missing', () => {
|
||||
const response: AgentResponse = {
|
||||
content: 'response',
|
||||
toolsUsed: [],
|
||||
usage: { inputTokens: 0, outputTokens: 0 },
|
||||
};
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 's1',
|
||||
workspaceId: 'ws1',
|
||||
systemPrompt: 'prompt',
|
||||
response,
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.input_tokens).toBe(0);
|
||||
expect(entry.output_tokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getRecentLogs ──────────────────────────────────────────
|
||||
|
||||
describe('getRecentLogs', () => {
|
||||
it('retrieves stored entries ordered by most recent first', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p1',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p2',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const logs = getRecentLogs(store, 10);
|
||||
expect(logs).toHaveLength(2);
|
||||
// Ordered by timestamp DESC, id DESC — s2 was inserted second so it comes first
|
||||
expect(logs[0].session_id).toBe('s2');
|
||||
expect(logs[1].session_id).toBe('s1');
|
||||
});
|
||||
|
||||
it('respects the limit parameter', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
captureInteraction(store, {
|
||||
sessionId: `s${i}`, workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(store, 3);
|
||||
expect(logs).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty array when no entries exist', () => {
|
||||
const logs = getRecentLogs(store);
|
||||
expect(logs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getWorkspaceLogs ───────────────────────────────────────
|
||||
|
||||
describe('getWorkspaceLogs', () => {
|
||||
it('filters by workspace ID', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws-b', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's3', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const wsALogs = getWorkspaceLogs(store, 'ws-a');
|
||||
expect(wsALogs).toHaveLength(2);
|
||||
expect(wsALogs.every(l => l.workspace_id === 'ws-a')).toBe(true);
|
||||
|
||||
const wsBLogs = getWorkspaceLogs(store, 'ws-b');
|
||||
expect(wsBLogs).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Entries have correct shape ─────────────────────────────
|
||||
|
||||
describe('entry shape', () => {
|
||||
it('has all required fields with correct types', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-x',
|
||||
workspaceId: 'workspace-y',
|
||||
systemPrompt: 'You are helpful.',
|
||||
response: makeResponse({ toolsUsed: ['bash', 'read_file'] }),
|
||||
turnCount: 7,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
// Check all fields exist and are of expected types
|
||||
expect(typeof entry.id).toBe('number');
|
||||
expect(typeof entry.session_id).toBe('string');
|
||||
expect(typeof entry.workspace_id).toBe('string');
|
||||
expect(typeof entry.system_prompt).toBe('string');
|
||||
expect(typeof entry.tools_used).toBe('string');
|
||||
expect(typeof entry.turn_count).toBe('number');
|
||||
expect(typeof entry.was_correction).toBe('number');
|
||||
expect(typeof entry.input_tokens).toBe('number');
|
||||
expect(typeof entry.output_tokens).toBe('number');
|
||||
expect(typeof entry.timestamp).toBe('string');
|
||||
|
||||
// tools_used is valid JSON array
|
||||
const parsed = JSON.parse(entry.tools_used);
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed).toEqual(['bash', 'read_file']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Budget enforcement ─────────────────────────────────────
|
||||
|
||||
describe('isWithinBudget', () => {
|
||||
it('returns true when no entries exist (under budget)', () => {
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when token cost is under budget', () => {
|
||||
// Insert entry with small token count
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 100 input * $3/M + 50 output * $15/M = $0.0003 + $0.00075 = ~$0.00105 = ~0.1 cents
|
||||
// Budget is 100 cents = $1, so well within budget
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when token cost exceeds budget', () => {
|
||||
// Insert entry with large token count to exceed a tiny budget
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 1_000_000, outputTokens: 500_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 1M input * $3/M + 500K output * $15/M = $3 + $7.5 = $10.50 = 1050 cents
|
||||
// Budget is 1 cent, so way over budget
|
||||
expect(isWithinBudget(store, 1)).toBe(false);
|
||||
});
|
||||
|
||||
it('respects custom cost-per-token rates', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 10_000, outputTokens: 5_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// With very cheap rates: 10K * $0.0001/M + 5K * $0.0001/M ≈ ~0 cents
|
||||
expect(isWithinBudget(store, 1, 0.0001 / 1_000_000, 0.0001 / 1_000_000)).toBe(true);
|
||||
|
||||
// With very expensive rates: 10K * $100/M + 5K * $100/M = $1 + $0.5 = 150 cents
|
||||
expect(isWithinBudget(store, 1, 100 / 1_000_000, 100 / 1_000_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Store stats ────────────────────────────────────────────
|
||||
|
||||
describe('getStats (via store)', () => {
|
||||
it('computes correct aggregate stats', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 3, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 200, outputTokens: 100 } }),
|
||||
turnCount: 5, wasCorrection: true,
|
||||
});
|
||||
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.correctionRate).toBe(0.5); // 1/2
|
||||
expect(stats.avgTurnCount).toBe(4); // (3+5)/2
|
||||
expect(stats.totalInputTokens).toBe(300); // 100+200
|
||||
expect(stats.totalOutputTokens).toBe(150); // 50+100
|
||||
});
|
||||
|
||||
it('returns zeros when empty', () => {
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(0);
|
||||
expect(stats.correctionRate).toBe(0);
|
||||
expect(stats.avgTurnCount).toBe(0);
|
||||
expect(stats.totalInputTokens).toBe(0);
|
||||
expect(stats.totalOutputTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,10 +2,9 @@ import { describe, it, expect } from 'vitest';
|
||||
import { PERSONAS, getPersona, listPersonas, composePersonaPrompt } from '../src/personas.js';
|
||||
|
||||
describe('Agent Personas', () => {
|
||||
it('PERSONAS catalog has 23 entries', () => {
|
||||
// 22 user-facing (8 universal + 14 specialists) + 1 internal review persona
|
||||
// (session-reviewer) used only by the IdleSessionWatcher self-evolution loop.
|
||||
expect(PERSONAS).toHaveLength(23);
|
||||
it('PERSONAS catalog has 22 entries', () => {
|
||||
// 8 universal + 14 specialists.
|
||||
expect(PERSONAS).toHaveLength(22);
|
||||
});
|
||||
|
||||
it('each persona has required fields', () => {
|
||||
@@ -40,9 +39,9 @@ describe('Agent Personas', () => {
|
||||
expect(getPersona('')).toBeNull();
|
||||
});
|
||||
|
||||
it('listPersonas() returns all 23 personas', () => {
|
||||
it('listPersonas() returns all 22 personas', () => {
|
||||
const list = listPersonas();
|
||||
expect(list).toHaveLength(23);
|
||||
expect(list).toHaveLength(22);
|
||||
expect(list).not.toBe(PERSONAS); // Returns a copy
|
||||
});
|
||||
|
||||
@@ -208,19 +207,6 @@ describe('Soft tool model', () => {
|
||||
expect(v!.disallowedTools).toContain('write_file');
|
||||
expect(v!.isReadOnly).toBe(true);
|
||||
});
|
||||
|
||||
it('session-reviewer is a trust-boundary persona: reads + create_skill only, no writes/exec/memory', () => {
|
||||
const r = getPersona('session-reviewer');
|
||||
expect(r, 'session-reviewer not found').toBeTruthy();
|
||||
// create_skill is the proposal vehicle and must stay in the allowlist.
|
||||
expect(r!.tools).toContain('create_skill');
|
||||
// isReadOnly would strip create_skill, so it must be false.
|
||||
expect(r!.isReadOnly).toBe(false);
|
||||
// The critical tools ALWAYS_AVAILABLE_TOOLS re-adds must be explicitly denied.
|
||||
for (const t of ['save_memory', 'delete_skill', 'install_capability', 'acquire_capability', 'spawn_agent', 'bash']) {
|
||||
expect(r!.disallowedTools, `${t} must be disallowed`).toContain(t);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('New domain personas', () => {
|
||||
@@ -239,8 +225,7 @@ describe('New domain personas', () => {
|
||||
});
|
||||
}
|
||||
|
||||
it('total persona count is 23', () => {
|
||||
// 22 tiered + session-reviewer (internal self-evolution reviewer).
|
||||
expect(PERSONAS.length).toBe(23);
|
||||
it('total persona count is 22', () => {
|
||||
expect(PERSONAS.length).toBe(22);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
*
|
||||
* Tests the one-step promotion personal → workspace, the refusal of team and
|
||||
* enterprise scopes in this edition, frontmatter rewrite with promoted_from
|
||||
* history, file move, and skill_promotion improvement signal emission.
|
||||
* history and file move.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { createSkillTools, getSkillDirForScope } from '../src/skill-tools.js';
|
||||
import { parseSkillFrontmatter } from '../src/skill-frontmatter.js';
|
||||
|
||||
@@ -22,8 +21,6 @@ function seedSkill(dir: string, name: string, scope: string | null, body = 'body
|
||||
|
||||
describe('promote_skill — gap E', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
let reloadCount: number;
|
||||
|
||||
function createTools(overrides: Partial<Parameters<typeof createSkillTools>[0]> = {}) {
|
||||
@@ -32,7 +29,6 @@ describe('promote_skill — gap E', () => {
|
||||
onSkillsChanged: () => { reloadCount++; },
|
||||
getWorkspaceId: () => 'ws-alpha',
|
||||
getTeamId: () => 'team-bravo',
|
||||
improvementSignals: signals,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -45,18 +41,15 @@ describe('promote_skill — gap E', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-promote-test-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
reloadCount = 0;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('happy path', () => {
|
||||
it('promotes personal → workspace, moves file, appends history, records signal', async () => {
|
||||
it('promotes personal → workspace, moves file, appends history', async () => {
|
||||
const personalDir = getSkillDirForScope(tmpDir, 'personal', {})!;
|
||||
seedSkill(personalDir, 'deploy-helper', 'personal');
|
||||
const tools = createTools();
|
||||
@@ -64,7 +57,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(tools, 'promote_skill', { skill_name: 'deploy-helper' });
|
||||
|
||||
expect(out).toContain('personal → workspace');
|
||||
expect(out).toContain('Signal recorded');
|
||||
|
||||
// File moved
|
||||
expect(fs.existsSync(path.join(personalDir, 'deploy-helper.md'))).toBe(false);
|
||||
@@ -77,11 +69,6 @@ describe('promote_skill — gap E', () => {
|
||||
expect(parsed.frontmatter.scope).toBe('workspace');
|
||||
expect(parsed.frontmatter.promoted_from).toEqual(['personal']);
|
||||
|
||||
// Signal emitted
|
||||
const recorded = signals.getByCategory('skill_promotion');
|
||||
expect(recorded.length).toBe(1);
|
||||
expect(recorded[0].pattern_key).toBe('promote:deploy-helper:personal->workspace');
|
||||
|
||||
// onSkillsChanged fired
|
||||
expect(reloadCount).toBe(1);
|
||||
});
|
||||
@@ -125,7 +112,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(createTools(), 'promote_skill', { skill_name: 'shared-helper' });
|
||||
expect(out).toContain('not available in this edition');
|
||||
expect(fs.existsSync(path.join(wsDir, 'shared-helper.md'))).toBe(true);
|
||||
expect(signals.getByCategory('skill_promotion')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('refuses enterprise promotion', async () => {
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
} from '../src/prompt-loader.js';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -68,7 +67,6 @@ describe('Prompt Loader', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-08 G2 — override-aware loader ──────────────────────────
|
||||
describe('loadSystemPromptWithOverrides', () => {
|
||||
it('returns baseline when no overrides exist', () => {
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
@@ -86,55 +84,17 @@ describe('Prompt Loader', () => {
|
||||
expect(composed.userPromptMd).toBe('hello from user');
|
||||
});
|
||||
|
||||
it('applies behavioral-spec overrides into the active spec', () => {
|
||||
it('ignores override files left on disk by earlier versions', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'CUSTOM CORE LOOP CONTENT FOR TEST',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
JSON.stringify({ section: 'coreLoop', text: 'STALE OVERRIDE TEXT', deployedAt: new Date().toISOString() }),
|
||||
);
|
||||
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
const flattened = Object.values(composed.behavioralSpec).join('\n');
|
||||
expect(flattened).toContain('CUSTOM CORE LOOP CONTENT FOR TEST');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertOverridesReachActiveSpec', () => {
|
||||
it('is quiet when no overrides are deployed', () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
assertOverridesReachActiveSpec(tmpDir, composed.behavioralSpec);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('warns when disk has overrides but activeSpec does not contain them', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'UNIQUE MARKER STRING FOR G2 TEST — LONG ENOUGH TO MATCH',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
// Simulate using the BARE loader — activeSpec is the compile-time
|
||||
// baseline with no overrides folded in.
|
||||
const fakeActiveSpec = loadSystemPromptWithOverrides(path.join(tmpDir, 'nothing')).behavioralSpec;
|
||||
assertOverridesReachActiveSpec(tmpDir, fakeActiveSpec);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
const message = warnSpy.mock.calls[0]?.[0] as string;
|
||||
expect(message).toContain('coreLoop');
|
||||
warnSpy.mockRestore();
|
||||
expect(flattened).not.toContain('STALE OVERRIDE TEXT');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,13 +2,8 @@
|
||||
* Skills 2.0 gap K — write-path conflict detection.
|
||||
*
|
||||
* When save_memory writes a new frame that contradicts an existing frame
|
||||
* (detectContradiction), the save still succeeds but:
|
||||
* - the return string carries a [flag: contradicts_existing …] marker
|
||||
* - a correction-category improvement signal is recorded
|
||||
*
|
||||
* Previously detectContradiction existed but was never called on the write
|
||||
* path — write-path conflicts silently overwrote reality, degrading the
|
||||
* evolution-signal loop.
|
||||
* (detectContradiction), the save still succeeds but the return string
|
||||
* carries a [flag: contradicts_existing …] marker.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
@@ -19,7 +14,6 @@ import {
|
||||
SessionStore,
|
||||
KnowledgeGraph,
|
||||
HybridSearch,
|
||||
ImprovementSignalStore,
|
||||
} from '@waggle/core';
|
||||
import { createMindTools } from '../src/tools.js';
|
||||
import type { ToolDefinition } from '../src/tools.js';
|
||||
@@ -30,7 +24,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
let frames: FrameStore;
|
||||
let sessions: SessionStore;
|
||||
let tools: ToolDefinition[];
|
||||
let improvementSignals: ImprovementSignalStore;
|
||||
|
||||
function getSaveMemory(): ToolDefinition {
|
||||
const tool = tools.find(t => t.name === 'save_memory');
|
||||
@@ -45,7 +38,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions = new SessionStore(db);
|
||||
const knowledge = new KnowledgeGraph(db);
|
||||
const search = new HybridSearch(db, embedder);
|
||||
improvementSignals = new ImprovementSignalStore(db);
|
||||
|
||||
tools = createMindTools({
|
||||
db,
|
||||
@@ -55,7 +47,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions,
|
||||
search,
|
||||
knowledge,
|
||||
improvementSignals,
|
||||
// Intentionally NO cognify → exercises the raw-frame fallback path
|
||||
// (no LLM-dependent entity extraction in unit tests).
|
||||
});
|
||||
@@ -94,17 +85,9 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
expect(second).toContain('Memory saved');
|
||||
// But the return message is annotated
|
||||
expect(second).toContain('[flag: contradicts_existing');
|
||||
|
||||
// And a correction signal is recorded
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.length).toBeGreaterThan(0);
|
||||
const writeConflict = top.find(s => s.pattern_key.startsWith('write-conflict:'));
|
||||
expect(writeConflict).toBeDefined();
|
||||
expect(writeConflict!.detail).toMatch(/contradicts existing frame/i);
|
||||
expect(writeConflict!.category).toBe('correction');
|
||||
});
|
||||
|
||||
it('non-contradicting saves do not flag or emit a signal', async () => {
|
||||
it('non-contradicting saves are not flagged', async () => {
|
||||
const save = getSaveMemory();
|
||||
await save.execute({
|
||||
content: 'Decision: we chose PostgreSQL for analytics; approved and confirmed.',
|
||||
@@ -117,34 +100,5 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
source: 'user_stated',
|
||||
});
|
||||
expect(result).not.toContain('contradicts_existing');
|
||||
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.find(s => s.pattern_key.startsWith('write-conflict:'))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('works without improvementSignals dep (backwards-compatible)', async () => {
|
||||
// Rebuild tools without signals dep
|
||||
const db2 = new MindDB(':memory:');
|
||||
const embedder = new MockEmbedder();
|
||||
const frames2 = new FrameStore(db2);
|
||||
const sessions2 = new SessionStore(db2);
|
||||
const knowledge2 = new KnowledgeGraph(db2);
|
||||
const search2 = new HybridSearch(db2, embedder);
|
||||
const tools2 = createMindTools({
|
||||
db: db2,
|
||||
identity: new IdentityLayer(db2),
|
||||
awareness: new AwarenessLayer(db2),
|
||||
frames: frames2,
|
||||
sessions: sessions2,
|
||||
search: search2,
|
||||
knowledge: knowledge2,
|
||||
// No improvementSignals — should not throw
|
||||
});
|
||||
const save = tools2.find(t => t.name === 'save_memory')!;
|
||||
await save.execute({ content: 'Decision: enable feature flag rollout for users; proceed.', target: 'personal' });
|
||||
const result = await save.execute({ content: 'Decision: disable feature flag rollout for users; cancel.', target: 'personal' });
|
||||
// Flag still appears (detectContradiction doesn't need the signal store)
|
||||
expect(result).toContain('contradicts_existing');
|
||||
db2.close();
|
||||
});
|
||||
});
|
||||
|
||||
77
packages/agent/tests/secret-patterns.test.ts
Normal file
77
packages/agent/tests/secret-patterns.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from '../src/secret-patterns.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('redactSecrets', () => {
|
||||
it('replaces every match and reports the patterns that fired', () => {
|
||||
const { text, found } = redactSecrets('key AKIAIOSFODNN7EXAMPLE and PASSWORD=supersecret123');
|
||||
expect(text).toContain('[REDACTED:aws-access-key]');
|
||||
expect(text).toContain('[REDACTED:env-password]');
|
||||
expect(text).not.toContain('AKIAIOSFODNN7EXAMPLE');
|
||||
expect(found.sort()).toEqual(['aws-access-key', 'env-password']);
|
||||
});
|
||||
|
||||
it('leaves clean text untouched', () => {
|
||||
expect(redactSecrets('nothing to hide')).toEqual({ text: 'nothing to hide', found: [] });
|
||||
});
|
||||
});
|
||||
@@ -90,59 +90,4 @@ describe('Self-Awareness', () => {
|
||||
expect(result).toContain('acquire_capability');
|
||||
expect(result).toContain('uncertain');
|
||||
});
|
||||
|
||||
it('includes improvement signals when awareness has actionable data', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [{
|
||||
id: 1,
|
||||
toolName: 'pdf_reader',
|
||||
occurrences: 3,
|
||||
suggestion: 'User has needed "pdf_reader" 3 times. Consider using acquire_capability to find it.',
|
||||
}],
|
||||
corrections: [{
|
||||
id: 2,
|
||||
patternKey: 'tone:too_formal',
|
||||
detail: 'Keep it casual',
|
||||
occurrences: 4,
|
||||
guidance: 'Keep it casual (corrected 4 times)',
|
||||
}],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 2,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).toContain('Learning from Past Sessions');
|
||||
expect(result).toContain('pdf_reader');
|
||||
expect(result).toContain('Keep it casual');
|
||||
expect(result).toContain('Recurring capability gaps');
|
||||
expect(result).toContain('User corrections to apply');
|
||||
});
|
||||
|
||||
it('omits improvement section when awareness has zero actionable signals', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [],
|
||||
corrections: [],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 0,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).not.toContain('Learning from Past Sessions');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
autoExtractAndCreateSkill,
|
||||
skillFilename,
|
||||
@@ -35,24 +34,18 @@ function repeatedSession(): AutoExtractMessage[] {
|
||||
|
||||
describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-autoextract-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates a personal-scope SKILL.md from a repeated workflow', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(result.filePath).not.toBeNull();
|
||||
@@ -67,19 +60,6 @@ describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
expect(body).toContain('## Tools Used');
|
||||
});
|
||||
|
||||
it('records a workflow_pattern signal with auto-extract pattern_key', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
expect(result.filePath).not.toBeNull();
|
||||
|
||||
const recorded = signals.getByCategory('workflow_pattern');
|
||||
const autoExtract = recorded.find(s => s.pattern_key.startsWith('auto-extract:'));
|
||||
expect(autoExtract).toBeDefined();
|
||||
expect(autoExtract!.detail).toContain('Auto-extracted');
|
||||
});
|
||||
|
||||
it('returns no_pattern for short or random sessions', async () => {
|
||||
const messages: AutoExtractMessage[] = [
|
||||
{ role: 'user', content: 'Hi' },
|
||||
|
||||
@@ -5,7 +5,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { retireStaleSkills } from '../src/skill-retirement.js';
|
||||
import {
|
||||
recordSkillUsage,
|
||||
@@ -74,8 +73,6 @@ describe('skill-usage — gap F', () => {
|
||||
describe('retireStaleSkills — gap F', () => {
|
||||
let waggleHome: string;
|
||||
let skillsDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
const NOW = new Date('2026-04-15T00:00:00Z');
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
@@ -83,12 +80,9 @@ describe('retireStaleSkills — gap F', () => {
|
||||
beforeEach(() => {
|
||||
waggleHome = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-retire-'));
|
||||
skillsDir = path.join(waggleHome, 'skills');
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(waggleHome, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
@@ -110,7 +104,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays: 90,
|
||||
now: () => NOW,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(report.scanned).toBe(2);
|
||||
@@ -123,10 +116,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
// Usage entry for the retired skill is cleared
|
||||
expect(loadSkillUsage(waggleHome).stale).toBeUndefined();
|
||||
expect(loadSkillUsage(waggleHome).fresh).toBeDefined();
|
||||
|
||||
// Signal recorded
|
||||
const recorded = signals.getByCategory('workflow_pattern').find(s => s.pattern_key === 'retire:stale');
|
||||
expect(recorded).toBeDefined();
|
||||
});
|
||||
|
||||
it('falls back to file mtime when no usage entry exists', () => {
|
||||
|
||||
Reference in New Issue
Block a user