refactor: remove self-learning and the optimizer package (P1.4)

Self-learning is excluded from the community edition (ADR-0004); the
project owner reworks it separately. Cost tracking stays.

- agent: delete behaviour evolution (evolve-schema, iterative GEPA,
  compose-evolution, evolution gates, orchestrator and deploy),
  improvement and correction detection, AgentLearning, the eval-dataset
  builder, optimization-capture and the agent_insights tool. Secret
  detection moves from eval-dataset to secret-patterns. The LLM wiring
  keeps only the Haiku adapter and judge call used by skill audit and
  hygiene, so @ax-llm/ax is now declared by agent instead of server.
- server: delete /api/evolution/*, the evolution and optimizer services,
  GEPA expansion of the first chat message, the prompt_optimization and
  monthly_assessment jobs, the idle-session reviewer and its
  session-reviewer persona, prompt injection of past corrections and
  thumbs-down recorded as correction signals. Existing job rows of the
  removed types are ignored, not deleted.
- core: delete OptimizationLogStore; stop re-exporting the improvement
  signal and evolution run stores.
- web: delete the Evolution screen and the two job types in the
  automation editor.
- delete packages/optimizer.

Also logs UF-027 (self-learning loops read data nothing collects) and
UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
shumeiko.it
2026-10-20 00:54:58 +02:00
parent b734c1d3d2
commit 2e1999a3f6
112 changed files with 336 additions and 16165 deletions

View File

@@ -1,373 +0,0 @@
import { describe, it, expect, vi } from 'vitest';
import {
ComposeEvolution,
defaultFeedbackFilter,
filterJudgeFeedback,
stripStructuralLines,
schemaExecutorFromInstructionRunner,
} from '../src/compose-evolution.js';
import type {
Schema,
SchemaExecuteFn,
} from '../src/evolve-schema.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(name => ({
name, type: 'string',
description: `description of ${name}`,
required: true,
constraints: [],
})),
version: 1,
};
}
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `q${i}`,
expected_output: `a${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeJudgeScore(overall: number, feedback: string): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
function makeFakeRunner(): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => f.name).join(','),
parsed: true,
});
}
// ── defaultFeedbackFilter ──────────────────────────────────────
describe('defaultFeedbackFilter', () => {
it('classifies "missing reasoning field" as structural', () => {
expect(defaultFeedbackFilter('Missing reasoning field.')).toBe('structural');
});
it('classifies "wrong field type" as structural', () => {
expect(defaultFeedbackFilter('Wrong field type for answer.')).toBe('structural');
});
it('classifies "schema mismatch" as structural', () => {
expect(defaultFeedbackFilter('Schema mismatch — expected 3 fields.')).toBe('structural');
});
it('classifies "should reorder fields" as structural', () => {
expect(defaultFeedbackFilter('Should reorder fields — put reasoning first.')).toBe('structural');
});
it('classifies value-level complaints as value', () => {
expect(defaultFeedbackFilter('The answer is too terse')).toBe('value');
expect(defaultFeedbackFilter('Response is too verbose')).toBe('value');
expect(defaultFeedbackFilter('Incorrect calculation')).toBe('value');
expect(defaultFeedbackFilter('Wrong tone — should be more formal')).toBe('value');
});
it('classifies empty feedback as value (safe default)', () => {
expect(defaultFeedbackFilter('')).toBe('value');
});
});
// ── stripStructuralLines ──────────────────────────────────────
describe('stripStructuralLines', () => {
it('returns empty for empty input', () => {
expect(stripStructuralLines('')).toBe('');
});
it('drops only structural lines from multi-line feedback', () => {
const input = [
'Missing reasoning field.',
'The answer is too terse.',
'Wrong field type for confidence.',
'Needs more detail.',
].join('\n');
const out = stripStructuralLines(input);
expect(out).not.toContain('Missing reasoning field');
expect(out).not.toContain('Wrong field type');
expect(out).toContain('too terse');
expect(out).toContain('Needs more detail');
});
it('keeps all lines when none are structural', () => {
const input = 'Too terse.\nNeeds more examples.';
expect(stripStructuralLines(input)).toBe(input);
});
it('returns empty when all lines are structural', () => {
const input = 'Missing reasoning field.\nSchema mismatch.';
expect(stripStructuralLines(input)).toBe('');
});
it('accepts a custom filter', () => {
const custom = (line: string) => line.includes('DROP') ? 'structural' : 'value';
const input = 'keep me\nDROP me\nkeep too';
expect(stripStructuralLines(input, custom)).toBe('keep me\nkeep too');
});
});
// ── filterJudgeFeedback ───────────────────────────────────────
describe('filterJudgeFeedback', () => {
it('strips structural feedback but preserves numeric scores', async () => {
const rawJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.7, 'Missing reasoning field.\nResponse is too terse.');
},
};
const filtered = filterJudgeFeedback(rawJudge);
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
expect(score.overall).toBe(0.7);
expect(score.correctness).toBe(0.7);
expect(score.feedback).not.toContain('Missing reasoning field');
expect(score.feedback).toContain('too terse');
});
it('returns empty feedback when all lines are structural', async () => {
const rawJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'Missing reasoning field.\nSchema mismatch.');
},
};
const filtered = filterJudgeFeedback(rawJudge);
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
expect(score.feedback).toBe('');
});
});
// ── schemaExecutorFromInstructionRunner ───────────────────────
describe('schemaExecutorFromInstructionRunner', () => {
it('builds a schema prefix and delegates to the runner', async () => {
let captured: { prompt: string; input: string } | null = null;
const runner = async (args: { prompt: string; input: string }) => {
captured = args;
return '{"answer":"42"}';
};
const schemaRunner = schemaExecutorFromInstructionRunner(runner);
const result = await schemaRunner({
schema: makeSchema(['reasoning', 'answer']),
input: 'What is 6*7?',
});
expect(result.actual).toBe('{"answer":"42"}');
expect(result.parsed).toBe(true);
expect(captured!.prompt).toContain('reasoning');
expect(captured!.prompt).toContain('answer');
expect(captured!.input).toBe('What is 6*7?');
});
it('reports parsed=false for non-JSON output', async () => {
const schemaRunner = schemaExecutorFromInstructionRunner(async () => 'just prose');
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
expect(result.parsed).toBe(false);
});
it('swallows runner errors and reports parsed=false', async () => {
const schemaRunner = schemaExecutorFromInstructionRunner(async () => {
throw new Error('boom');
});
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
expect(result.actual).toBe('');
expect(result.parsed).toBe(false);
});
});
// ── ComposeEvolution end-to-end ────────────────────────────────
describe('ComposeEvolution.run', () => {
it('runs schema stage then instruction stage', async () => {
const stages: string[] = [];
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeRunner(),
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'a baseline instruction prompt for testing purposes',
examples: makeExamples(5),
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
mutate: async ({ parent }) => `${parent.prompt} v2`,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
onProgress: (e) => stages.push(e.stage),
});
expect(stages).toContain('schema');
expect(stages).toContain('instructions');
expect(stages).toContain('done');
expect(result.schema.winner).toBeDefined();
expect(result.instructions.winner).toBeDefined();
expect(result.frozenSchema).toBe(result.schema.winner.schema);
});
it('feedback separation: GEPA cannot see structural feedback from the judge', async () => {
const capturedFeedbacks: string[] = [];
// Judge always emits structural + value feedback
const mixedJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'Missing reasoning field.\nResponse too terse.');
},
};
const mutateSpy = vi.fn(async ({ weaknessFeedback }: { parent: unknown; weaknessFeedback: string[]; strategy: string; targetKind: string; generation: number }) => {
capturedFeedbacks.push(...weaknessFeedback);
return 'mutated instruction prompt is a reasonable length';
});
await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeRunner(),
judge: mixedJudge,
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'a baseline instruction prompt for testing purposes',
examples: makeExamples(5),
judge: mixedJudge,
mutate: mutateSpy,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
});
// GEPA must never receive "Missing reasoning field" in weaknessFeedback.
for (const fb of capturedFeedbacks) {
expect(fb.toLowerCase()).not.toContain('missing reasoning field');
expect(fb.toLowerCase()).not.toContain('wrong field type');
}
// But it should see the value-level complaint
expect(capturedFeedbacks.join(' ')).toContain('too terse');
});
it('accepts a custom feedback filter', async () => {
let filterCalls = 0;
const customFilter = (feedback: string): 'structural' | 'value' => {
filterCalls++;
return feedback.includes('CUSTOM-DROP') ? 'structural' : 'value';
};
const judge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'CUSTOM-DROP this line\nkeep this line');
},
};
await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(3),
execute: makeFakeRunner(),
judge,
populationSize: 1, generations: 1,
evalSize: 2, anchorEvalSize: 2,
},
instructions: {
baseline: 'baseline prompt text is long enough',
examples: makeExamples(3),
judge,
mutate: async () => 'new prompt text that is long enough',
allowBareJudge: true,
populationSize: 1, generations: 1,
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
},
feedbackFilter: customFilter,
});
expect(filterCalls).toBeGreaterThan(0);
});
it('returns a stable shape when aborted before instruction stage', async () => {
const ctrl = new AbortController();
let progressCount = 0;
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(3),
execute: async () => {
// Abort during the schema stage
if (progressCount === 1) ctrl.abort();
progressCount++;
return { actual: '', parsed: false };
},
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
populationSize: 1, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'baseline prompt',
examples: makeExamples(3),
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
mutate: async () => 'should not run',
allowBareJudge: true,
populationSize: 1, generations: 1,
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
},
signal: ctrl.signal,
});
expect(result.schema).toBeDefined();
expect(result.instructions).toBeDefined();
expect(result.instructions.winner.prompt).toBe('baseline prompt');
});
it('combinedDelta is a number; frozenSchema matches ES winner', async () => {
const judge = {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
return makeJudgeScore(Math.min(1, args.actual.length / 50), 'ok');
},
};
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(8),
execute: makeFakeRunner(),
judge,
populationSize: 2, generations: 1,
evalSize: 4, anchorEvalSize: 4,
},
instructions: {
baseline: 'short',
examples: makeExamples(8),
judge,
mutate: async ({ parent }) => `${parent.prompt} more tokens here`,
allowBareJudge: true,
populationSize: 2, generations: 2,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 6,
},
});
expect(result.instructions.winner.score).not.toBeNull();
expect(result.frozenSchema).toBe(result.schema.winner.schema);
expect(typeof result.combinedDelta).toBe('number');
});
});

View File

@@ -1,204 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
detectCorrection,
detectCorrectionsInHistory,
} from '../src/correction-detector.js';
describe('correction-detector', () => {
// ── Basic detection ────────────────────────────────────────
describe('detectCorrection', () => {
it('returns null for normal messages', () => {
expect(detectCorrection('What is the weather today?')).toBeNull();
expect(detectCorrection('Please write a function that sorts an array')).toBeNull();
expect(detectCorrection('Thanks, that looks good')).toBeNull();
});
it('returns null for very short messages', () => {
expect(detectCorrection('ok')).toBeNull();
expect(detectCorrection('yes')).toBeNull();
});
it('detects strong correction: "no, not that"', () => {
const result = detectCorrection('No, not that. I wanted a bullet list, not paragraphs.');
expect(result).not.toBeNull();
expect(result!.confidence).toBeGreaterThanOrEqual(0.3);
});
it('detects strong correction: "I said / I told you"', () => {
const result = detectCorrection('I said to use markdown headers, not plain text.');
expect(result).not.toBeNull();
});
it('detects strong correction: "that\'s wrong"', () => {
const result = detectCorrection("That's wrong. The API endpoint should be POST, not GET.");
expect(result).not.toBeNull();
});
it('detects moderate correction: "actually, use X instead"', () => {
const result = detectCorrection('Actually, instead use TypeScript for this module.');
expect(result).not.toBeNull();
});
it('detects tone correction', () => {
const result = detectCorrection('No, that response was too formal. Keep it casual.');
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('tone');
});
it('detects format correction', () => {
const result = detectCorrection("No, don't use those headers. Use bullet formatting instead.");
expect(result).not.toBeNull();
});
it('detects length correction', () => {
const result = detectCorrection("That's not what I wanted. It's too long, make it shorter.");
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('length');
});
});
// ── Durability classification ──────────────────────────────
describe('durability classification', () => {
it('classifies "always" as durable', () => {
const result = detectCorrection('No, always use bullet points instead of numbered lists.');
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
expect(result!.durability).toBe('durable');
});
it('classifies "never" as durable', () => {
const result = detectCorrection("No, never use code blocks for simple text. That's wrong.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "from now on" as durable', () => {
const result = detectCorrection('From now on, please don\'t include timestamps in summaries.');
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "I prefer" as durable', () => {
const result = detectCorrection("No, I prefer shorter responses. That's not what I wanted.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "this time" as task_local', () => {
const result = detectCorrection("No, not that. This time just use a simple list.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(false);
expect(result!.durability).toBe('task_local');
});
it('classifies "for now" as task_local', () => {
const result = detectCorrection("That's wrong for now. Instead use the old API endpoint.");
expect(result).not.toBeNull();
expect(result!.durability).toBe('task_local');
});
it('defaults to task_local when no explicit signal', () => {
const result = detectCorrection('No, not that approach. Use the factory pattern.');
expect(result).not.toBeNull();
expect(result!.durability).toBe('task_local');
});
});
// ── Pattern key extraction ─────────────────────────────────
describe('pattern key extraction', () => {
it('extracts tone pattern key', () => {
const result = detectCorrection("No, that's too verbose. Stop doing that.");
expect(result).not.toBeNull();
// "too verbose" maps to tone category
// Note: might also match length depending on exact wording
});
it('extracts accuracy pattern key for "wrong"', () => {
const result = detectCorrection("That's incorrect. I said the opposite.");
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('accuracy');
});
it('falls back to general:correction for unrecognized patterns', () => {
const result = detectCorrection("No, not that. I told you to use something else entirely.");
expect(result).not.toBeNull();
expect(result!.patternKey).toBe('general:correction');
});
});
// ── Detail extraction ──────────────────────────────────────
describe('detail extraction', () => {
it('extracts first sentence as detail', () => {
const result = detectCorrection('No, not that. I wanted a completely different structure. Also fix the naming.');
expect(result).not.toBeNull();
expect(result!.detail).toBe('No, not that.');
});
it('truncates long messages', () => {
const longMessage = 'No, not that. ' + 'A'.repeat(200);
const result = detectCorrection(longMessage);
expect(result).not.toBeNull();
expect(result!.detail.length).toBeLessThanOrEqual(120);
});
});
// ── Confidence scoring ─────────────────────────────────────
describe('confidence', () => {
it('gives higher confidence to strong corrections', () => {
const strong = detectCorrection("No, that's wrong. I told you not to do that. Stop.");
const moderate = detectCorrection('Actually, let\'s try a different approach instead.');
expect(strong).not.toBeNull();
expect(moderate).not.toBeNull();
expect(strong!.confidence).toBeGreaterThan(moderate!.confidence);
});
it('caps confidence at 1.0', () => {
const result = detectCorrection(
"No, that's wrong. I said don't do that. Not what I wanted. Please stop. Instead use something else.",
);
expect(result).not.toBeNull();
expect(result!.confidence).toBeLessThanOrEqual(1.0);
});
});
// ── History analysis ───────────────────────────────────────
describe('detectCorrectionsInHistory', () => {
it('detects corrections in a message sequence', () => {
const messages = [
{ role: 'assistant', content: 'Here is a very detailed formal report...' },
{ role: 'user', content: "No, that's too formal. I said keep it casual." },
{ role: 'assistant', content: 'Got it. Here is a casual version...' },
{ role: 'user', content: 'Thanks, that looks better.' },
];
const corrections = detectCorrectionsInHistory(messages);
expect(corrections).toHaveLength(1);
expect(corrections[0].patternKey).toContain('tone');
});
it('returns empty array when no corrections found', () => {
const messages = [
{ role: 'user', content: 'Write a function to sort an array' },
{ role: 'assistant', content: 'Here you go...' },
{ role: 'user', content: 'Thanks!' },
];
expect(detectCorrectionsInHistory(messages)).toEqual([]);
});
it('handles multiple corrections in sequence', () => {
const messages = [
{ role: 'assistant', content: 'Draft version 1...' },
{ role: 'user', content: "No, not that. That's wrong. Use a different approach." },
{ role: 'assistant', content: 'Draft version 2...' },
{ role: 'user', content: "I said don't use bullet points. Instead use paragraphs." },
];
const corrections = detectCorrectionsInHistory(messages);
expect(corrections.length).toBeGreaterThanOrEqual(2);
});
});
});

View File

@@ -1,348 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, ExecutionTraceStore } from '@waggle/core';
import {
EvalDatasetBuilder,
detectSecrets,
SECRET_PATTERN_NAMES,
toJSONL as evalToJSONL,
fromJSONL as evalFromJSONL,
type EvalExample,
} from '../src/eval-dataset.js';
describe('detectSecrets', () => {
it('returns null for clean text', () => {
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
});
it('detects AWS access keys', () => {
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
});
it('detects GitHub PATs', () => {
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
});
it('detects OpenAI keys', () => {
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
});
it('detects Anthropic keys', () => {
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
});
it('detects Google API keys', () => {
// Google API keys = literal "AIza" + exactly 35 chars
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
});
it('detects Stripe keys', () => {
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
});
it('detects private key blocks', () => {
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
});
it('detects JWT tokens', () => {
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
expect(detectSecrets(jwt)).toBe('jwt');
});
it('detects basic-auth URLs', () => {
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
.toBe('basic-auth-url');
});
it('detects env-var password assignments', () => {
expect(detectSecrets('PASSWORD=supersecret123'))
.toBe('env-password');
});
it('detects postgres connection strings with creds', () => {
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
.toBe('pgsql-url');
});
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
});
});
describe('JSONL round-trip', () => {
const examples: EvalExample[] = [
{
input: 'write a haiku',
expected_output: 'Cherry blossoms fall / ...',
metadata: { source: 'trace', personaId: 'writer', outcome: 'success' },
},
{
input: 'explain closures',
expected_output: 'A closure captures variables from its enclosing scope.',
metadata: { source: 'trace', personaId: 'coder', outcome: 'verified' },
},
];
it('round-trips through JSONL', () => {
const jsonl = evalToJSONL(examples);
expect(jsonl.split('\n')).toHaveLength(2);
const parsed = evalFromJSONL(jsonl);
expect(parsed).toEqual(examples);
});
it('skips malformed lines silently', () => {
const jsonl = [
JSON.stringify(examples[0]),
'{ this is not valid json',
'',
JSON.stringify(examples[1]),
].join('\n');
const parsed = evalFromJSONL(jsonl);
expect(parsed).toHaveLength(2);
});
it('skips lines missing required fields', () => {
const jsonl = [
JSON.stringify({ input: 'x' }), // missing expected_output + metadata
JSON.stringify(examples[0]),
].join('\n');
const parsed = evalFromJSONL(jsonl);
expect(parsed).toHaveLength(1);
});
});
describe('EvalDatasetBuilder', () => {
let db: MindDB;
let store: ExecutionTraceStore;
let builder: EvalDatasetBuilder;
beforeEach(() => {
db = new MindDB(':memory:');
store = new ExecutionTraceStore(db);
builder = new EvalDatasetBuilder(store);
});
afterEach(() => {
db.close();
});
function seedTrace(input: string, output: string, outcome: 'success' | 'verified' | 'corrected' | 'abandoned' = 'success', personaId = 'coder'): number {
const id = store.start({ input, personaId });
store.finalize(id, { outcome, output });
return id;
}
// ── sourceFromTraces ──
describe('sourceFromTraces', () => {
it('pulls only traces matching the requested outcomes', () => {
seedTrace('a?', 'A.', 'success');
seedTrace('b?', 'B.', 'verified');
seedTrace('c?', 'C.', 'corrected');
seedTrace('d?', 'D.', 'abandoned');
const examples = builder.sourceFromTraces(['success', 'verified'], false);
expect(examples).toHaveLength(2);
expect(examples.map(e => e.input).sort()).toEqual(['a?', 'b?']);
});
it('includes corrections when flag set', () => {
seedTrace('a?', 'A.', 'success');
seedTrace('b?', 'B.', 'corrected');
const examples = builder.sourceFromTraces(['success'], true);
expect(examples).toHaveLength(2);
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
expect(corrected?.metadata.source).toBe('correction');
});
it('maps trace fields onto metadata', () => {
seedTrace('hello', 'world', 'success', 'writer');
const [example] = builder.sourceFromTraces(['success'], false);
expect(example.metadata.personaId).toBe('writer');
expect(example.metadata.traceId).toBeGreaterThan(0);
expect(example.metadata.source).toBe('trace');
});
});
// ── build (end-to-end) ──
describe('build', () => {
it('returns deterministic split with same seed', async () => {
for (let i = 0; i < 20; i++) {
seedTrace(`question ${i}?`, `answer ${i}.`, 'success');
}
const a = await builder.build({ seed: 42 });
const b = await builder.build({ seed: 42 });
expect(a.train.map(e => e.input)).toEqual(b.train.map(e => e.input));
expect(a.val.map(e => e.input)).toEqual(b.val.map(e => e.input));
expect(a.holdout.map(e => e.input)).toEqual(b.holdout.map(e => e.input));
});
it('different seeds produce different orderings', async () => {
for (let i = 0; i < 30; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const a = await builder.build({ seed: 1 });
const b = await builder.build({ seed: 999 });
const aIds = a.train.map(e => e.input).join('|');
const bIds = b.train.map(e => e.input).join('|');
expect(aIds).not.toBe(bIds);
});
it('splits 60/20/20 when enough examples', async () => {
for (let i = 0; i < 100; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const split = await builder.build({ seed: 1 });
expect(split.train.length).toBe(60);
expect(split.val.length).toBe(20);
expect(split.holdout.length).toBe(20);
expect(split.stats.total).toBe(100);
});
it('honors custom split ratios', async () => {
for (let i = 0; i < 50; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const split = await builder.build({ seed: 1, splitRatios: [0.8, 0.1, 0.1] });
expect(split.train.length).toBe(40);
expect(split.val.length).toBe(5);
expect(split.holdout.length).toBe(5);
});
it('rejects ratios that do not sum to 1', async () => {
await expect(builder.build({ splitRatios: [0.5, 0.3, 0.3] })).rejects.toThrow();
});
it('filters out examples containing secrets', async () => {
seedTrace('clean input', 'clean output', 'success');
seedTrace('here is a key: AKIAIOSFODNN7EXAMPLE', 'sure', 'success');
seedTrace('what is your token?', 'ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.sourced).toBe(3);
expect(split.stats.acceptedAfterSecretScan).toBe(1);
expect(split.rejected.filter(r => r.reason.startsWith('secret:'))).toHaveLength(2);
});
it('filters examples by min length', async () => {
seedTrace('?', 'ok', 'success'); // below both mins
seedTrace('a proper question', 'a proper answer', 'success');
const split = await builder.build({ seed: 1, minInputChars: 5, minOutputChars: 3 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'too-short-input')).toBe(true);
});
it('filters examples over max combined length', async () => {
seedTrace('this is an ok prompt', 'this is an ok answer', 'success');
seedTrace('x'.repeat(5000), 'y'.repeat(5000), 'success');
const split = await builder.build({ seed: 1, maxCombinedChars: 1000 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'too-long')).toBe(true);
});
it('filters low-signal garbage', async () => {
seedTrace('.......................', '!!!!!!!!!!!!!!!!', 'success');
seedTrace('real question?', 'real answer.', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'low-signal')).toBe(true);
});
it('dedupes by input hash', async () => {
seedTrace('same prompt', 'answer a', 'success');
seedTrace('same prompt', 'answer b', 'success');
seedTrace('different prompt', 'answer c', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.unique).toBe(2);
expect(split.rejected.some(r => r.reason === 'duplicate')).toBe(true);
});
it('applies optional LLM judge', async () => {
seedTrace('keep this example please', 'kept response', 'success');
seedTrace('drop this example please', 'dropped response', 'success');
const split = await builder.build({
seed: 1,
judge: async (ex) => ({
keep: !ex.input.includes('drop'),
reason: ex.input.includes('drop') ? 'contains drop' : undefined,
}),
});
expect(split.stats.acceptedAfterJudge).toBe(1);
expect(split.rejected.some(r => r.reason.startsWith('judge:'))).toBe(true);
});
it('keeps example when judge throws (fail-open)', async () => {
seedTrace('some prompt text', 'some answer text', 'success');
const split = await builder.build({
seed: 1,
judge: async () => {
throw new Error('llm down');
},
});
expect(split.stats.acceptedAfterJudge).toBe(1);
expect(split.stats.total).toBe(1);
});
it('accepts external harvest/correction examples', async () => {
const harvest: EvalExample = {
input: 'harvest prompt',
expected_output: 'harvest answer',
metadata: { source: 'harvest' },
};
const correction: EvalExample = {
input: 'correction prompt',
expected_output: 'correction answer',
metadata: { source: 'correction' },
};
const split = await builder.build({
seed: 1,
harvestExamples: [harvest],
correctionExamples: [correction],
});
const allInputs = [...split.train, ...split.val, ...split.holdout].map(e => e.input);
expect(allInputs).toContain('harvest prompt');
expect(allInputs).toContain('correction prompt');
});
it('maps correction trace output to correctionFeedback when available', async () => {
const id = store.start({ input: 'wrong approach', personaId: 'coder' });
store.finalize(id, { outcome: 'success', output: 'first try' });
store.markCorrected(id, 'Should use async/await here');
const examples = builder.sourceFromTraces(['success'], true);
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
expect(corrected?.expected_output).toBe('Should use async/await here');
});
it('returns empty splits when no traces match', async () => {
const split = await builder.build({ seed: 1 });
expect(split.stats.total).toBe(0);
expect(split.train).toEqual([]);
expect(split.val).toEqual([]);
expect(split.holdout).toEqual([]);
});
it('tiny datasets assign leftover to holdout', async () => {
for (let i = 0; i < 3; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
// Ratios 0.6/0.2/0.2 with n=3 → floor(1.8)=1, floor(0.6)=0, remainder=2 → holdout
const split = await builder.build({ seed: 1 });
expect(split.train.length + split.val.length + split.holdout.length).toBe(3);
});
});
});

View File

@@ -1,372 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import {
deployPersonaOverride,
rollbackPersonaOverride,
deployBehavioralSpecOverride,
rollbackBehavioralSpecOverride,
loadBehavioralSpecOverrides,
applyBehavioralSpecOverrides,
BEHAVIORAL_SPEC_SECTIONS,
type BehavioralSpecSection,
} from '../src/evolution-deploy.js';
import { getPersona, listPersonas, setPersonaDataDir } from '../src/personas.js';
import { loadCustomPersonas } from '../src/custom-personas.js';
// ── Test fixture: temp dir per-test ────────────────────────────
describe('evolution-deploy', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-deploy-'));
});
afterEach(() => {
vi.restoreAllMocks();
try {
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { /* Windows lock cleanup racy; ignore */ }
});
// ── deployPersonaOverride ──
describe('deployPersonaOverride', () => {
it('writes a JSON file named {id}.json in personas/', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'You are the evolved coder.',
});
expect(fs.existsSync(result.path)).toBe(true);
expect(result.path.endsWith(path.join('personas', 'coder.json'))).toBe(true);
expect(result.backupPath).toBeNull();
expect(result.deployedAt).toBeTruthy();
});
it('inherits built-in persona fields when targeting a known id', () => {
// 'coder' is a built-in persona
const builtin = getPersona('coder');
expect(builtin).not.toBeNull();
deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'EVOLVED coder prompt — tighter format.',
});
const [loaded] = loadCustomPersonas(tmpDir);
expect(loaded.id).toBe('coder');
expect(loaded.name).toBe(builtin!.name);
expect(loaded.icon).toBe(builtin!.icon);
expect(loaded.systemPrompt).toBe('EVOLVED coder prompt — tighter format.');
expect(loaded.tools).toEqual(builtin!.tools);
});
it('creates a backup on subsequent writes', () => {
const first = deployPersonaOverride(tmpDir, {
personaId: 'coder', systemPrompt: 'v1',
});
expect(first.backupPath).toBeNull();
const second = deployPersonaOverride(tmpDir, {
personaId: 'coder', systemPrompt: 'v2',
});
expect(second.backupPath).not.toBeNull();
expect(fs.existsSync(second.backupPath!)).toBe(true);
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).systemPrompt).toBe('v1');
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).systemPrompt).toBe('v2');
});
it('creates a minimal shell for unknown persona ids', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'made-up-persona',
systemPrompt: 'fresh prompt',
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.id).toBe('made-up-persona');
expect(loaded.name).toBe('made-up-persona');
expect(loaded.systemPrompt).toBe('fresh prompt');
expect(Array.isArray(loaded.tools)).toBe(true);
});
it('applies caller-supplied overrides (description, icon, tools)', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'made-up',
systemPrompt: 'x',
overrides: {
description: 'custom description',
icon: 'brain',
tools: ['tool_a', 'tool_b'],
},
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.description).toBe('custom description');
expect(loaded.icon).toBe('brain');
expect(loaded.tools).toEqual(['tool_a', 'tool_b']);
});
it('is picked up by listPersonas after setPersonaDataDir', () => {
deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'EVOLVED ROUTING',
});
setPersonaDataDir(tmpDir);
try {
const personas = listPersonas();
const evolved = personas.find(p => p.id === 'coder' && p.systemPrompt.includes('EVOLVED ROUTING'));
expect(evolved).toBeDefined();
} finally {
setPersonaDataDir(''); // reset so other tests aren't polluted
}
});
});
// ── rollbackPersonaOverride ──
describe('rollbackPersonaOverride', () => {
it('restores .bak when present', () => {
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
const second = deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v2' });
expect(second.backupPath).not.toBeNull();
const ok = rollbackPersonaOverride(tmpDir, 'coder');
expect(ok).toBe(true);
const [loaded] = loadCustomPersonas(tmpDir);
expect(loaded.systemPrompt).toBe('v1');
});
it('deletes the override entirely when no .bak exists', () => {
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
const ok = rollbackPersonaOverride(tmpDir, 'coder');
expect(ok).toBe(true);
expect(loadCustomPersonas(tmpDir)).toHaveLength(0);
});
it('returns false for non-existent personas', () => {
expect(rollbackPersonaOverride(tmpDir, 'ghost')).toBe(false);
});
});
// ── deployBehavioralSpecOverride ──
describe('deployBehavioralSpecOverride', () => {
it('writes a JSON file named {section}.json in behavioral-overrides/', () => {
const result = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop',
text: 'Evolved core loop instructions.',
});
expect(fs.existsSync(result.path)).toBe(true);
expect(result.path.endsWith(path.join('behavioral-overrides', 'coreLoop.json'))).toBe(true);
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.section).toBe('coreLoop');
expect(loaded.text).toBe('Evolved core loop instructions.');
expect(loaded.deployedAt).toBeTruthy();
});
it('stores the originating run uuid for audit', () => {
const result = deployBehavioralSpecOverride(tmpDir, {
section: 'qualityRules',
text: 'new quality rules',
runUuid: 'test-run-uuid-123',
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.runUuid).toBe('test-run-uuid-123');
});
it('creates backup on subsequent writes', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
expect(first.backupPath).toBeNull();
const second = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
});
expect(second.backupPath).not.toBeNull();
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
});
it('retries transient Windows rename locks before replacing an override', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync')
.mockImplementationOnce(() => {
throw Object.assign(new Error('locked'), { code: 'EPERM' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('busy'), { code: 'EBUSY' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('locked again'), { code: 'EPERM' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('still busy'), { code: 'EBUSY' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
const second = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
});
expect(rename).toHaveBeenCalledTimes(5);
expect(wait).toHaveBeenCalledTimes(4);
expect(JSON.parse(fs.readFileSync(second.path, 'utf-8')).text).toBe('v2');
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('preserves the current override when transient rename retries are exhausted', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
throw Object.assign(new Error('still locked'), { code: 'EACCES' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
expect(() => deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
})).toThrow(/still locked/);
expect(rename).toHaveBeenCalledTimes(10);
expect(wait).toHaveBeenCalledTimes(9);
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('does not retry non-transient rename failures', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
throw Object.assign(new Error('disk error'), { code: 'EIO' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
expect(() => deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
})).toThrow(/disk error/);
expect(rename).toHaveBeenCalledOnce();
expect(wait).not.toHaveBeenCalled();
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('rejects unknown sections', () => {
expect(() =>
deployBehavioralSpecOverride(tmpDir, {
section: 'notASection' as BehavioralSpecSection,
text: 'x',
}),
).toThrow(/Unknown behavioral-spec section/);
});
});
// ── rollbackBehavioralSpecOverride ──
describe('rollbackBehavioralSpecOverride', () => {
it('restores .bak when present', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v2' });
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
expect(ok).toBe(true);
const overrides = loadBehavioralSpecOverrides(tmpDir);
expect(overrides.coreLoop).toBe('v1');
});
it('removes the override when no backup exists', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
expect(ok).toBe(true);
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
});
});
// ── loadBehavioralSpecOverrides ──
describe('loadBehavioralSpecOverrides', () => {
it('returns empty object when no overrides exist', () => {
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
});
it('returns a section → text map for all written overrides', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'evolved core' });
deployBehavioralSpecOverride(tmpDir, { section: 'qualityRules', text: 'evolved quality' });
const overrides = loadBehavioralSpecOverrides(tmpDir);
expect(overrides.coreLoop).toBe('evolved core');
expect(overrides.qualityRules).toBe('evolved quality');
expect(overrides.behavioralRules).toBeUndefined();
});
it('ignores malformed override files', () => {
const dir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'coreLoop.json'), '{ not valid json');
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
});
it('ignores files with empty text', () => {
const dir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'coreLoop.json'), JSON.stringify({
section: 'coreLoop', text: '', deployedAt: '2026-04-14',
}));
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
});
});
// ── applyBehavioralSpecOverrides ──
describe('applyBehavioralSpecOverrides', () => {
const baseline: Record<BehavioralSpecSection, string> = {
coreLoop: 'base core',
qualityRules: 'base quality',
behavioralRules: 'base behavioral',
workPatterns: 'base patterns',
intelligenceDefaults: 'base intelligence',
};
it('returns baseline unchanged when no overrides', () => {
expect(applyBehavioralSpecOverrides(baseline, {})).toEqual(baseline);
});
it('overlays provided sections', () => {
const merged = applyBehavioralSpecOverrides(baseline, {
coreLoop: 'EVOLVED core',
qualityRules: 'EVOLVED quality',
});
expect(merged.coreLoop).toBe('EVOLVED core');
expect(merged.qualityRules).toBe('EVOLVED quality');
expect(merged.behavioralRules).toBe('base behavioral');
});
it('ignores empty/undefined overrides without clobbering baseline', () => {
const merged = applyBehavioralSpecOverrides(baseline, {
coreLoop: '',
qualityRules: undefined,
});
expect(merged.coreLoop).toBe('base core');
expect(merged.qualityRules).toBe('base quality');
});
});
// ── BEHAVIORAL_SPEC_SECTIONS constant ──
describe('BEHAVIORAL_SPEC_SECTIONS', () => {
it('contains the expected 5 sections in expected order', () => {
expect(BEHAVIORAL_SPEC_SECTIONS).toEqual([
'coreLoop', 'qualityRules', 'behavioralRules',
'workPatterns', 'intelligenceDefaults',
]);
});
});
});

View File

@@ -1,245 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
runGates, DEFAULT_SIZE_LIMITS,
checkNonEmpty, checkSize, checkGrowth,
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
} from '../src/evolution-gates.js';
describe('checkNonEmpty', () => {
it('fails on empty string', () => {
expect(checkNonEmpty('').verdict).toBe('fail');
});
it('fails on whitespace-only string', () => {
expect(checkNonEmpty(' \n\n ').verdict).toBe('fail');
});
it('passes on real content', () => {
expect(checkNonEmpty('hello').verdict).toBe('pass');
});
});
describe('checkSize', () => {
it('passes within the persona cap', () => {
const result = checkSize('x'.repeat(2000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('pass');
expect(result.detail?.current).toBe(2000);
expect(result.detail?.max).toBe(DEFAULT_SIZE_LIMITS.personaSystemPrompt);
});
it('fails over the persona cap', () => {
const result = checkSize('x'.repeat(4000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('fail');
expect(result.reason).toContain('persona-system-prompt');
});
it('uses tool-description cap (500)', () => {
expect(checkSize('x'.repeat(500), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
expect(checkSize('x'.repeat(501), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
});
it('uses skill-body cap (15k)', () => {
expect(checkSize('x'.repeat(15_000), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
expect(checkSize('x'.repeat(15_001), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
});
it('uses generic cap for unknown target', () => {
const result = checkSize('x'.repeat(5000), 'generic', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('pass');
});
it('accepts custom size limits', () => {
const limits = { ...DEFAULT_SIZE_LIMITS, generic: 100 };
const result = checkSize('x'.repeat(200), 'generic', limits);
expect(result.verdict).toBe('fail');
});
});
describe('checkGrowth', () => {
it('passes when candidate is smaller than baseline', () => {
const result = checkGrowth('abc', 'abcdefgh', 0.2);
expect(result.verdict).toBe('pass');
});
it('passes when growth is within budget', () => {
const result = checkGrowth('x'.repeat(110), 'x'.repeat(100), 0.2); // +10%
expect(result.verdict).toBe('pass');
});
it('fails when growth exceeds budget', () => {
const result = checkGrowth('x'.repeat(140), 'x'.repeat(100), 0.2); // +40%
expect(result.verdict).toBe('fail');
expect(result.detail?.ratio).toBeCloseTo(0.4, 5);
});
it('passes when baseline is empty (cannot compute ratio)', () => {
const result = checkGrowth('anything', '', 0.2);
expect(result.verdict).toBe('pass');
});
it('boundary: exactly 20% growth passes with default budget', () => {
// 100 → 120 is exactly 20%, not greater
const result = checkGrowth('x'.repeat(120), 'x'.repeat(100), 0.2);
expect(result.verdict).toBe('pass');
});
});
describe('checkBalancedFences', () => {
it('passes no fences', () => {
expect(checkBalancedFences('plain prose').verdict).toBe('pass');
});
it('passes two fences (one code block)', () => {
const text = 'here:\n```js\nconst x = 1;\n```\ndone';
expect(checkBalancedFences(text).verdict).toBe('pass');
});
it('fails one fence (unbalanced)', () => {
const text = 'here:\n```js\nconst x = 1;\nstill going';
const result = checkBalancedFences(text);
expect(result.verdict).toBe('fail');
expect(result.detail?.fences).toBe(1);
});
it('passes four fences (two blocks)', () => {
const text = '```a\nfoo\n```\n```b\nbar\n```';
expect(checkBalancedFences(text).verdict).toBe('pass');
});
});
describe('checkNoPlaceholders', () => {
it('passes clean text', () => {
expect(checkNoPlaceholders('A prompt with normal prose.').verdict).toBe('pass');
});
it('fails [TODO] bracket placeholder', () => {
expect(checkNoPlaceholders('Write about [TODO] here.').verdict).toBe('fail');
});
it('fails [PLACEHOLDER]', () => {
expect(checkNoPlaceholders('Fill in [PLACEHOLDER].').verdict).toBe('fail');
});
it('fails [YOUR NAME] style', () => {
expect(checkNoPlaceholders('Hi [YOUR NAME HERE]!').verdict).toBe('fail');
});
it('fails angle-brackets <placeholder>', () => {
expect(checkNoPlaceholders('Set <placeholder> before sending.').verdict).toBe('fail');
});
it('fails handlebars {{var}}', () => {
expect(checkNoPlaceholders('Hello {{name}}.').verdict).toBe('fail');
});
it('does not false-positive normal bracketed citations', () => {
expect(checkNoPlaceholders('See [1] and [Smith 2024] for details.').verdict).toBe('pass');
});
});
describe('checkNoObviousTodos', () => {
it('passes clean text', () => {
expect(checkNoObviousTodos('Write clear documentation.').verdict).toBe('pass');
});
it('fails lines starting with TODO:', () => {
expect(checkNoObviousTodos('Intro paragraph.\nTODO: finish this').verdict).toBe('fail');
});
it('fails lines starting with FIXME:', () => {
expect(checkNoObviousTodos('FIXME: tighten this up').verdict).toBe('fail');
});
it('does not fail mid-sentence "todo" mention', () => {
expect(checkNoObviousTodos('Build a todo list UI.').verdict).toBe('pass');
});
});
describe('checkRegression', () => {
it('passes when delta is within tolerance', () => {
const result = checkRegression(0.8, 0.79, -0.02); // -1pp, floor -2pp
expect(result.verdict).toBe('pass');
});
it('passes on improvement', () => {
const result = checkRegression(0.7, 0.85, -0.02);
expect(result.verdict).toBe('pass');
expect(result.detail?.delta).toBeCloseTo(0.15, 5);
});
it('fails when delta exceeds tolerance', () => {
const result = checkRegression(0.9, 0.7, -0.02); // -20pp
expect(result.verdict).toBe('fail');
});
it('boundary: equal to tolerance passes', () => {
const result = checkRegression(0.8, 0.78, -0.02); // exactly -2pp
expect(result.verdict).toBe('pass');
});
});
// ── runGates end-to-end ────────────────────────────────────────
describe('runGates', () => {
it('passes a clean candidate', () => {
const res = runGates({
candidate: 'A concise persona prompt for the researcher.',
baseline: 'A concise persona prompt for the researcher.',
}, { targetKind: 'persona-system-prompt' });
expect(res.verdict).toBe('pass');
expect(res.firstFailure).toBeNull();
});
it('fails an empty candidate on the first gate', () => {
const res = runGates({ candidate: '', baseline: 'whatever' });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('non-empty');
});
it('fails oversized candidate on size gate', () => {
const res = runGates({
candidate: 'x'.repeat(4000),
baseline: 'x'.repeat(3000),
}, { targetKind: 'persona-system-prompt' });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('size');
});
it('fails runaway growth', () => {
const res = runGates({
candidate: 'x'.repeat(200),
baseline: 'x'.repeat(100),
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('growth');
});
it('fails placeholder leakage', () => {
const res = runGates({
candidate: 'You are {{persona.name}} — answer questions.',
baseline: 'You are the researcher — answer questions.',
});
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('no-placeholders');
});
it('fails unbalanced markdown fences', () => {
// Pad baseline so the growth gate doesn't trip before the fence gate.
const padding = ' '.repeat(100);
const res = runGates({
candidate: `Instructions:\n\`\`\`js\nconst x = 1;\nmore prose${padding}`,
baseline: `Instructions: write code clearly.${padding}`,
});
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('balanced-fences');
});
it('applies the regression gate when scores are supplied', () => {
const res = runGates({
candidate: 'new prompt',
baseline: 'old prompt',
scores: { baseline: 0.85, candidate: 0.6 },
}, { maxRegression: -0.02 });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('regression');
});
it('skips the regression gate when scores are omitted', () => {
const res = runGates({ candidate: 'new', baseline: 'old' });
expect(res.results.find(r => r.gate === 'regression')).toBeUndefined();
});
it('returns results for every gate even when one fails', () => {
const res = runGates({
candidate: 'x'.repeat(200),
baseline: 'x'.repeat(100),
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
expect(res.results.length).toBeGreaterThan(1);
});
});

View File

@@ -1,11 +1,6 @@
import { describe, it, expect, vi } from 'vitest';
import {
buildJudgeLLMCall,
buildGEPAMutateFn,
buildSchemaExecuteFn,
makeRunningJudge,
buildReflectiveMutationPrompt,
buildSchemaFillPrompt,
retryWithBackoff,
wrapWithRetry,
isRetryableEvolutionError,
@@ -15,9 +10,6 @@ import {
type RetryOptions,
type RetryInfo,
} from '../src/evolution-llm-wiring.js';
import type { MutateArgs, GEPACandidate } from '../src/index.js';
import type { Schema } from '../src/evolve-schema.js';
import type { JudgeInput } from '../src/judge.js';
// ── Fixtures ──────────────────────────────────────────────────────
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
};
}
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
return {
id: 'g1-m0',
prompt: 'You are a tester. Respond with the exact expected word.',
generation: 1,
parent: 'g0-baseline',
strategy: 'expand-edge-cases',
score: null,
perExample: [],
...overrides,
};
}
function makeSchema(overrides: Partial<Schema> = {}): Schema {
return {
name: 'answer',
version: 1,
fields: [
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
],
...overrides,
};
}
// ── buildJudgeLLMCall ─────────────────────────────────────────────
describe('buildJudgeLLMCall', () => {
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
});
});
// ── buildGEPAMutateFn ─────────────────────────────────────────────
describe('buildGEPAMutateFn', () => {
const mutateArgs: MutateArgs = {
parent: makeCandidate(),
strategy: 'expand-edge-cases',
weaknessFeedback: [
'Fails to handle empty input',
'Sometimes forgets to include the final answer',
],
targetKind: 'persona-system-prompt',
generation: 2,
};
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe('EVOLVED SYSTEM PROMPT');
expect(calls).toHaveLength(1);
const prompt = calls[0];
expect(prompt).toContain(mutateArgs.parent.prompt);
expect(prompt).toContain(mutateArgs.strategy);
expect(prompt).toContain('Fails to handle empty input');
expect(prompt).toContain('persona-system-prompt');
});
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
const { llm } = makeMockLLM(() => ' \n ');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe(mutateArgs.parent.prompt);
});
it('falls back to the parent prompt when the LLM throws', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('boom'); },
};
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe(mutateArgs.parent.prompt);
});
it('strips markdown code fences from the LLM response', async () => {
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result.trim()).toBe('HIDDEN PROMPT');
});
});
describe('buildReflectiveMutationPrompt', () => {
it('includes the generation number and weakness bullets', () => {
const prompt = buildReflectiveMutationPrompt({
parent: 'PARENT',
strategy: 'tighten-format',
weaknessFeedback: ['a', 'b'],
targetKind: 'behavioral-spec-section',
generation: 3,
});
expect(prompt).toContain('generation 3');
expect(prompt).toContain('- a');
expect(prompt).toContain('- b');
expect(prompt).toContain('tighten-format');
expect(prompt).toContain('PARENT');
});
it('handles missing weakness feedback gracefully', () => {
const prompt = buildReflectiveMutationPrompt({
parent: 'PARENT',
strategy: 'add-examples',
weaknessFeedback: [],
targetKind: 'generic',
generation: 0,
});
expect(prompt).toContain('PARENT');
expect(prompt).toContain('(no specific weakness signals yet)');
});
});
// ── buildSchemaExecuteFn ──────────────────────────────────────────
describe('buildSchemaExecuteFn', () => {
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
expect(result.parsed).toBe(true);
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
expect(calls[0]).toContain('What is the answer?');
expect(calls[0]).toContain('"reasoning"');
expect(calls[0]).toContain('"answer"');
});
it('returns parsed=false when LLM returns non-JSON', async () => {
const { llm } = makeMockLLM(() => 'just some text');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(false);
expect(result.actual).toBe('just some text');
});
it('extracts JSON from markdown fences', async () => {
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(true);
expect(result.actual).toContain('"reasoning"');
});
it('returns parsed=false and empty actual when LLM throws', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('boom'); },
};
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(false);
expect(result.actual).toBe('');
});
});
describe('buildSchemaFillPrompt', () => {
it('serializes each field with name + type + description', () => {
const prompt = buildSchemaFillPrompt({
schema: makeSchema(),
input: 'What is 2+2?',
});
expect(prompt).toContain('"reasoning"');
expect(prompt).toContain('"answer"');
expect(prompt).toContain('step-by-step thinking');
expect(prompt).toContain('the final answer');
expect(prompt).toContain('What is 2+2?');
});
it('notes required constraints', () => {
const schema: Schema = {
name: 'x', version: 1,
fields: [
{
name: 'score',
type: 'number',
description: 'a rating',
required: true,
constraints: [{ kind: 'range', value: '0-10' }],
},
],
};
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
expect(prompt).toContain('score');
expect(prompt).toContain('number');
expect(prompt).toContain('range');
});
});
// ── makeRunningJudge ──────────────────────────────────────────────
describe('makeRunningJudge', () => {
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
const { llm, calls } = makeMockLLM((prompt) => {
// Simulate running the candidate prompt and returning a model response.
return `MODEL OUTPUT for: ${prompt}`;
});
const baseScore = vi.fn(async (args: JudgeInput) => ({
overall: 0.8, weighted: 0.8,
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
lengthPenalty: 1, feedback: 'passed through', parsed: true,
}));
const baseJudge = { score: baseScore };
const runningJudge = makeRunningJudge(baseJudge, llm);
const result = await runningJudge.score({
input: 'What is love?',
expected: 'baby don\'t hurt me',
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
});
// Model was asked to run the candidate prompt against the example input.
expect(calls).toHaveLength(1);
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
expect(calls[0]).toContain('What is love?');
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
expect(baseScore).toHaveBeenCalledTimes(1);
const passedArgs = baseScore.mock.calls[0][0];
expect(passedArgs.input).toBe('What is love?');
expect(passedArgs.expected).toBe('baby don\'t hurt me');
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
expect(result.overall).toBe(0.8);
});
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('rate limited'); },
};
const baseJudge = { score: vi.fn() };
const runningJudge = makeRunningJudge(baseJudge, llm);
const result = await runningJudge.score({
input: 'any', expected: 'any', actual: 'prompt',
});
expect(result.overall).toBe(0);
expect(result.parsed).toBe(false);
expect(result.feedback).toMatch(/execution failed/i);
expect(baseJudge.score).not.toHaveBeenCalled();
});
it('passes context through to the base judge', async () => {
const { llm } = makeMockLLM(() => 'EXECUTED');
const baseScore = vi.fn(async () => ({
overall: 0.5, weighted: 0.5,
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
lengthPenalty: 1, feedback: '', parsed: true,
}));
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
await runningJudge.score({
input: 'q', expected: 'a', actual: 'prompt',
context: 'persona:coder',
});
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
});
});
// ── isRetryableEvolutionError ─────────────────────────────────────
describe('isRetryableEvolutionError', () => {

View File

@@ -1,447 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
import {
EvolutionOrchestrator,
eligibleForEvolution,
summarizeRuns,
} from '../src/evolution-orchestrator.js';
import type { JudgeScore } from '../src/judge.js';
import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
import type { MutateFn } from '../src/iterative-optimizer.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(n => ({
name: n, type: 'string' as const,
description: `${n} field`,
required: true, constraints: [],
})),
version: 1,
};
}
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
// Executor that produces longer output for bigger schemas.
function makeExec(): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => f.name).join(','),
parsed: true,
});
}
// Judge that prefers candidates containing the "plus" token so the
// mutateAppend helper ("... plus") wins reliably. Creates a measurable
// delta between baseline and winner regardless of absolute lengths.
function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
return {
async score(args) {
const base = Math.min(1, args.actual.length / 100) * 0.6;
const bonus = args.actual.includes('plus') ? 0.3 : 0;
return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
},
};
}
// Mutate appends ~10 chars so growth ratio stays below the default +20%.
const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
const BASELINE =
'Return a concise answer that directly addresses the question. ' +
'Provide a brief explanation and the final answer. Be clear and accurate.';
function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
for (let i = 0; i < n; i++) {
const id = store.start({
personaId,
input: `test question ${i} with enough length to pass filters`,
workspaceId: 'ws-1',
taskShape: 'qa',
});
store.finalize(id, {
outcome: 'success',
output: `a full answer for question ${i} that is long enough for use`,
});
}
}
function baseComposeOptions() {
return {
schema: {
examples: [] as never[],
execute: makeExec(),
judge: makeJudge(),
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
examples: [] as never[],
judge: makeJudge(),
mutate: mutateAppend,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
};
}
// ── eligibleForEvolution ──
describe('eligibleForEvolution', () => {
it('keeps finalized traces with non-empty input and output', () => {
const eligible = eligibleForEvolution([
{
id: 1, session_id: 's', persona_id: null, workspace_id: null,
model: null, task_shape: null, outcome: 'success',
cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
payload: {
input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
tokens: { input: 0, output: 0 }, tags: [],
},
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(1);
});
it('drops pending traces', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'pending',
payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(0);
});
it('drops traces with empty input', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'success',
payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(0);
});
it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'corrected',
payload: {
input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
tokens: { input: 0, output: 0 }, tags: [],
correctionFeedback: 'use bullets',
},
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(1);
});
});
// ── summarizeRuns ──
describe('summarizeRuns', () => {
it('returns zero-aggregates on empty input', () => {
const s = summarizeRuns([]);
expect(s.total).toBe(0);
expect(s.byStatus.proposed).toBe(0);
expect(s.bestDelta).toBe(0);
});
it('counts by status and target kind, tracks best delta', () => {
const runs = [
{ status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
{ status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
{ status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
{ status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
] as unknown as EvolutionRun[];
const s = summarizeRuns(runs);
expect(s.total).toBe(4);
expect(s.byStatus.proposed).toBe(1);
expect(s.byStatus.accepted).toBe(1);
expect(s.byStatus.rejected).toBe(1);
expect(s.byStatus.deployed).toBe(1);
expect(s.byTargetKind['persona-system-prompt']).toBe(3);
expect(s.byTargetKind['tool-description']).toBe(1);
expect(s.bestDelta).toBeCloseTo(0.12, 5);
});
});
// ── EvolutionOrchestrator end-to-end ───────────────────────────
describe('EvolutionOrchestrator', () => {
let db: MindDB;
let traceStore: ExecutionTraceStore;
let runStore: EvolutionRunStore;
let orchestrator: EvolutionOrchestrator;
beforeEach(() => {
db = new MindDB(':memory:');
traceStore = new ExecutionTraceStore(db);
runStore = new EvolutionRunStore(db);
orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
});
afterEach(() => {
db.close();
});
// ── runOnce ──
describe('runOnce', () => {
it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
seedSuccessfulTraces(traceStore, 10);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
targetName: 'researcher',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
expect(result.outcome).toBe('proposed');
expect(result.run?.status).toBe('proposed');
expect(result.run?.delta_accuracy).toBeGreaterThan(0);
expect(result.compose).toBeDefined();
expect(result.gateResults).toBeDefined();
});
it('persists compose artifacts (schema JSON, gates, delta)', async () => {
seedSuccessfulTraces(traceStore, 8);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
targetName: 'researcher',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const run = result.run!;
expect(run.winner_schema_json).toBeTruthy();
const schema = JSON.parse(run.winner_schema_json!);
expect(schema.name).toBe('test');
expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
});
it('skips with skipped-delta when improvement is below threshold', async () => {
seedSuccessfulTraces(traceStore, 8);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: 'a reasonably detailed baseline prompt that already scores well',
schemaBaseline: makeSchema(['answer']),
compose: {
...baseComposeOptions(),
instructions: {
...baseComposeOptions().instructions,
// Mutate returns the same text → zero delta
mutate: async ({ parent }) => parent.prompt,
},
},
minDelta: 0.5, // artificially high
});
expect(result.outcome).toBe('skipped-delta');
expect(result.run).toBeUndefined();
});
it('auto-trigger: skips when too few traces', async () => {
seedSuccessfulTraces(traceStore, 2);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: 'baseline',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
autoTrigger: { minTraces: 50 },
});
expect(result.outcome).toBe('skipped-trigger');
expect(result.run).toBeUndefined();
});
it('auto-trigger: proceeds when threshold is met', async () => {
seedSuccessfulTraces(traceStore, 10);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
autoTrigger: { minTraces: 5 },
});
expect(result.outcome).toBe('proposed');
});
it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
seedSuccessfulTraces(traceStore, 8);
// Judge that prefers the mutated "y" text — ensures the GEPA winner
// is the oversized candidate, not the baseline.
const yPreferringJudge = {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
},
};
const result = await orchestrator.runOnce({
targetKind: 'tool-description',
baseline: 'x'.repeat(30),
schemaBaseline: makeSchema(['answer']),
compose: {
...baseComposeOptions(),
instructions: {
...baseComposeOptions().instructions,
judge: yPreferringJudge,
// Mutate produces candidate that blows past tool-description cap (500)
mutate: async () => 'y'.repeat(2000),
},
},
minDelta: 0, // allow any delta so we reach the gate stage
});
expect(result.outcome).toBe('skipped-gates');
expect(result.run?.status).toBe('rejected');
expect(result.run?.gate_verdict).toBe('fail');
expect(result.run?.user_note).toMatch(/gate failure/);
});
it('aborts cleanly when signal fires before start', async () => {
seedSuccessfulTraces(traceStore, 5);
const ctrl = new AbortController();
ctrl.abort();
const result = await orchestrator.runOnce({
targetKind: 'generic',
baseline: 'baseline',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
signal: ctrl.signal,
});
expect(result.outcome).toBe('aborted');
});
it('emits progress events', async () => {
seedSuccessfulTraces(traceStore, 5);
const phases: string[] = [];
await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
onProgress: (e) => phases.push(e.phase),
});
expect(phases).toContain('compose');
expect(phases).toContain('done');
});
});
// ── accept ──
describe('accept', () => {
it('invokes deploy callback and marks run deployed on success', async () => {
const deploy = vi.fn(async () => { /* success */ });
const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
seedSuccessfulTraces(traceStore, 8);
const run = await deployedOrch.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
expect(run.outcome).toBe('proposed');
const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
expect(accepted?.status).toBe('deployed');
expect(deploy).toHaveBeenCalledTimes(1);
});
it('marks run failed when deploy callback throws', async () => {
const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
seedSuccessfulTraces(traceStore, 8);
const run = await orch.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const result = await orch.accept(run.run!.run_uuid);
expect(result?.status).toBe('failed');
expect(result?.failure_reason).toContain('persona write failed');
});
it('leaves run as accepted when no deploy hook configured', async () => {
seedSuccessfulTraces(traceStore, 8);
const run = await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const accepted = await orchestrator.accept(run.run!.run_uuid);
expect(accepted?.status).toBe('accepted');
});
it('returns undefined for unknown uuid', async () => {
expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
});
});
// ── reject ──
describe('reject', () => {
it('marks proposed run as rejected', async () => {
seedSuccessfulTraces(traceStore, 8);
const run = await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
expect(rejected?.status).toBe('rejected');
expect(rejected?.user_note).toBe('regressed on custom cases');
});
});
// ── list / get ──
describe('list / get', () => {
it('returns runs in reverse-chronological order', async () => {
seedSuccessfulTraces(traceStore, 8);
await orchestrator.runOnce({
targetKind: 'generic', baseline: 'a',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
await orchestrator.runOnce({
targetKind: 'generic', baseline: 'b',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const list = orchestrator.list();
expect(list.length).toBeGreaterThanOrEqual(2);
});
it('get returns a specific run by uuid', async () => {
seedSuccessfulTraces(traceStore, 8);
const created = await orchestrator.runOnce({
targetKind: 'generic', baseline: 'short',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const fetched = orchestrator.get(created.run!.run_uuid);
expect(fetched?.id).toBe(created.run!.id);
});
});
});

View File

@@ -1,545 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
EvolveSchema,
addOutputField,
removeField,
editFieldDescription,
changeFieldType,
addConstraint,
removeConstraint,
reorderFields,
replaceOutputFields,
schemaComplexity,
aggregateSchemaScores,
paretoFrontSchema,
pickSchemaWinner,
generateStructureMutations,
generateOrderMutations,
generateRefinementMutations,
pickSample,
type Schema,
type SchemaField,
type SchemaCandidate,
type SchemaCandidateScore,
type SchemaExecuteFn,
} from '../src/evolve-schema.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeField(name: string, partial: Partial<SchemaField> = {}): SchemaField {
return {
name,
type: 'string',
description: `description for ${name}`,
required: true,
constraints: [],
...partial,
};
}
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(f => makeField(f)),
version: 1,
};
}
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `q${i}`,
expected_output: `a${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
function makeCandidate(id: string, schema: Schema, score: SchemaCandidateScore | null): SchemaCandidate {
return {
id,
schema,
generation: 0,
parent: null,
mutation: 'baseline',
mutationLabel: 'baseline',
score,
perExample: [],
};
}
// Fake executor: returns a deterministic output string that scores higher
// when schema has more fields (up to a cap) and scores lower when schema
// is bloated beyond 5 fields.
function makeFakeExecutor(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more'): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => `${f.name}=mock`).join('; '),
parsed: schema.fields.length > 0,
});
}
function makeJudge(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more') {
return {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
const fieldCount = (args.actual.match(/=/g) || []).length;
let overall: number;
if (bias === 'prefers-more') {
// More fields = better, up to 6 fields, then flat
overall = Math.min(1, fieldCount / 6);
} else if (bias === 'prefers-less') {
overall = Math.max(0, 1 - fieldCount / 10);
} else {
overall = 0.5;
}
return makeJudgeScore(overall);
},
};
}
// ── Pure mutation functions ────────────────────────────────────
describe('addOutputField', () => {
it('appends when no position given', () => {
const s = makeSchema(['a', 'b']);
const next = addOutputField(s, makeField('c'));
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
expect(next.version).toBe(2);
});
it('inserts at given position', () => {
const s = makeSchema(['a', 'c']);
const next = addOutputField(s, makeField('b'), 1);
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
});
it('does not mutate source', () => {
const s = makeSchema(['a']);
addOutputField(s, makeField('b'));
expect(s.fields.map(f => f.name)).toEqual(['a']);
});
});
describe('removeField', () => {
it('drops the named field', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = removeField(s, 'b');
expect(next.fields.map(f => f.name)).toEqual(['a', 'c']);
expect(next.version).toBe(2);
});
it('is a no-op for unknown field', () => {
const s = makeSchema(['a', 'b']);
const next = removeField(s, 'z');
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
});
});
describe('editFieldDescription', () => {
it('updates the description of the named field', () => {
const s = makeSchema(['a']);
const next = editFieldDescription(s, 'a', 'new');
expect(next.fields[0].description).toBe('new');
});
});
describe('changeFieldType', () => {
it('updates the type of the named field', () => {
const s = makeSchema(['a']);
const next = changeFieldType(s, 'a', 'number');
expect(next.fields[0].type).toBe('number');
});
});
describe('addConstraint / removeConstraint', () => {
it('adds a constraint', () => {
const s = makeSchema(['a']);
const next = addConstraint(s, 'a', { kind: 'maxLength', value: 10 });
expect(next.fields[0].constraints).toHaveLength(1);
});
it('removes the given constraint index', () => {
const s = addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 });
const next = removeConstraint(s, 'a', 0);
expect(next.fields[0].constraints).toHaveLength(0);
});
it('is a no-op for unknown field when adding', () => {
const s = makeSchema(['a']);
const next = addConstraint(s, 'z', { kind: 'maxLength', value: 10 });
expect(next.fields[0].constraints).toHaveLength(0);
});
});
describe('reorderFields', () => {
it('reorders by name', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = reorderFields(s, ['c', 'a', 'b']);
expect(next.fields.map(f => f.name)).toEqual(['c', 'a', 'b']);
});
it('appends fields omitted from newOrder', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = reorderFields(s, ['b']);
// 'b' first, then the rest in original order
expect(next.fields.map(f => f.name)).toEqual(['b', 'a', 'c']);
});
it('ignores unknown field names in newOrder', () => {
const s = makeSchema(['a', 'b']);
const next = reorderFields(s, ['unknown', 'a', 'b']);
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
});
});
describe('replaceOutputFields', () => {
it('replaces all fields', () => {
const s = makeSchema(['a', 'b']);
const next = replaceOutputFields(s, [makeField('x'), makeField('y')]);
expect(next.fields.map(f => f.name)).toEqual(['x', 'y']);
});
});
// ── Complexity + scoring ───────────────────────────────────────
describe('schemaComplexity', () => {
it('returns 0 for empty schema', () => {
expect(schemaComplexity(makeSchema([]))).toBe(0);
});
it('scales with field count, constraint count, and description length', () => {
const simple = schemaComplexity(makeSchema(['a']));
const twoField = schemaComplexity(makeSchema(['a', 'b']));
expect(twoField).toBeGreaterThan(simple);
const withConstraint = schemaComplexity(
addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 }),
);
expect(withConstraint).toBeGreaterThan(simple);
});
});
describe('aggregateSchemaScores', () => {
it('returns zero aggregate on empty results', () => {
const agg = aggregateSchemaScores([]);
expect(agg.n).toBe(0);
expect(agg.accuracy).toBe(0);
expect(agg.parseRate).toBe(0);
});
it('averages accuracy + computes parse rate', () => {
const results = [
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.8), parsed: true },
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.6), parsed: true },
{ input: 'c', expected: 'C', actual: 'X', score: makeJudgeScore(0.2), parsed: false },
];
const agg = aggregateSchemaScores(results);
expect(agg.n).toBe(3);
expect(agg.accuracy).toBeCloseTo((0.8 + 0.6 + 0.2) / 3, 5);
expect(agg.parseRate).toBeCloseTo(2 / 3, 5);
});
it('surfaces worst-3 example feedback', () => {
const results = [
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.9, 'best'), parsed: true },
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.1, 'worst'), parsed: true },
{ input: 'c', expected: 'C', actual: 'C', score: makeJudgeScore(0.5, 'middle'), parsed: true },
];
const agg = aggregateSchemaScores(results);
expect(agg.weaknessFeedback).toContain('worst');
expect(agg.weaknessFeedback).toContain('middle');
});
});
// ── Pareto ─────────────────────────────────────────────────────
describe('paretoFrontSchema', () => {
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
});
it('removes strictly dominated candidates', () => {
const dominated = makeCandidate('d', makeSchema(['a']), makeScore(0.5, 5));
const better = makeCandidate('b', makeSchema(['a']), makeScore(0.8, 3));
const front = paretoFrontSchema([dominated, better]);
expect(front.map(c => c.id)).toEqual(['b']);
});
it('keeps trade-off candidates (accurate vs simple)', () => {
const accurate = makeCandidate('acc', makeSchema(['a', 'b']), makeScore(0.9, 10));
const simple = makeCandidate('sim', makeSchema(['a']), makeScore(0.7, 2));
const front = paretoFrontSchema([accurate, simple]);
expect(front).toHaveLength(2);
});
it('ignores unscored candidates', () => {
const scored = makeCandidate('s', makeSchema(['a']), makeScore(0.5, 1));
const unscored = makeCandidate('u', makeSchema(['a']), null);
const front = paretoFrontSchema([scored, unscored]);
expect(front).toHaveLength(1);
});
});
describe('pickSchemaWinner', () => {
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
});
it('picks highest accuracy', () => {
const lo = makeCandidate('lo', makeSchema(['a']), makeScore(0.5, 1));
const hi = makeCandidate('hi', makeSchema(['a']), makeScore(0.9, 10));
expect(pickSchemaWinner([lo, hi]).id).toBe('hi');
});
it('ties on accuracy → prefers lower complexity', () => {
const complex = makeCandidate('cx', makeSchema(['a', 'b']), makeScore(0.8, 10));
const simple = makeCandidate('sm', makeSchema(['a']), makeScore(0.8, 3));
expect(pickSchemaWinner([complex, simple]).id).toBe('sm');
});
it('throws on empty', () => {
expect(() => pickSchemaWinner([])).toThrow();
});
});
// ── Mutation generators ────────────────────────────────────────
describe('generateStructureMutations', () => {
const rng = () => 0.5;
it('suggests adding reasoning if absent', () => {
const s = makeSchema(['answer']);
const muts = generateStructureMutations(s, 5, rng);
expect(muts.some(m => m.description.includes('reasoning'))).toBe(true);
});
it('does not re-suggest reasoning if already present', () => {
const s = makeSchema(['reasoning', 'answer']);
const muts = generateStructureMutations(s, 5, rng);
const hasReasoning = muts.some(m => m.description.includes('reasoning field'));
expect(hasReasoning).toBe(false);
});
it('suggests drop when schema is large', () => {
const s = makeSchema(['a', 'b', 'c', 'd', 'e']);
const muts = generateStructureMutations(s, 5, rng);
expect(muts.some(m => m.kind === 'remove_field')).toBe(true);
});
it('returns at most n mutations', () => {
const s = makeSchema(['answer']);
const muts = generateStructureMutations(s, 1, rng);
expect(muts.length).toBeLessThanOrEqual(1);
});
});
describe('generateOrderMutations', () => {
const rng = () => 0.5;
it('returns empty for 1-field schemas', () => {
expect(generateOrderMutations(makeSchema(['a']), 3, rng)).toEqual([]);
});
it('moves a reasoning field to the front', () => {
const s: Schema = {
name: 't', version: 1,
fields: [makeField('answer'), makeField('reasoning'), makeField('confidence')],
};
const muts = generateOrderMutations(s, 3, rng);
const applied = muts[0].apply(s);
expect(applied.fields[0].name).toBe('reasoning');
});
it('moves a confidence field to the end', () => {
const s: Schema = {
name: 't', version: 1,
fields: [makeField('confidence'), makeField('reasoning'), makeField('answer')],
};
const muts = generateOrderMutations(s, 3, rng);
const last = muts[0].apply(s);
// One of the generated mutations should put 'confidence' at the end
const someMovesConfidenceToEnd = muts.some(m => {
const applied = m.apply(s);
return applied.fields[applied.fields.length - 1].name === 'confidence';
});
expect(someMovesConfidenceToEnd).toBe(true);
});
});
describe('generateRefinementMutations', () => {
it('adds maxLength when feedback mentions verbosity', async () => {
const s = makeSchema(['reasoning', 'answer']);
const muts = await generateRefinementMutations(
s, ['too verbose, output wordy'], 5,
);
expect(muts.some(m => m.kind === 'add_constraint')).toBe(true);
});
it('adds minLength when feedback mentions incompleteness', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['response is too brief, missing detail'], 5,
);
expect(muts.some(m =>
m.description.includes('minLength'),
)).toBe(true);
});
it('edits field descriptions when feedback mentions format issues', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['wrong format, could not parse'], 5,
);
expect(muts.some(m => m.kind === 'edit_field_desc')).toBe(true);
});
it('falls back to a single description rewrite when no heuristic matches', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(s, ['something unrelated'], 5);
expect(muts.length).toBeGreaterThan(0);
expect(muts[0].kind).toBe('edit_field_desc');
});
it('uses LLM-provided editor when supplied', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['wrong format'], 5,
async ({ field }) => `LLM-REWRITE of ${field.name}`,
);
const edit = muts.find(m => m.kind === 'edit_field_desc');
const applied = edit!.apply(s);
expect(applied.fields[0].description).toBe('LLM-REWRITE of answer');
});
});
// ── pickSample ─────────────────────────────────────────────────
describe('pickSample', () => {
it('returns up to k items', () => {
expect(pickSample(makeExamples(5), 3, () => 0.5)).toHaveLength(3);
});
it('returns empty on k=0', () => {
expect(pickSample(makeExamples(5), 0, () => 0.5)).toEqual([]);
});
});
// ── EvolveSchema.run end-to-end ───────────────────────────────
describe('EvolveSchema.run', () => {
it('runs through all phases and returns a winner', async () => {
const baseline = makeSchema(['answer']);
const result = await new EvolveSchema().run({
baseline,
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 1,
evalSize: 5,
anchorEvalSize: 10,
});
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
expect(result.history.length).toBeGreaterThan(1);
});
it('winner accuracy >= baseline accuracy when judge prefers richer schemas', async () => {
const baseline = makeSchema(['answer']);
const result = await new EvolveSchema().run({
baseline,
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 2,
evalSize: 5,
anchorEvalSize: 10,
});
expect(result.winner.score!.accuracy).toBeGreaterThanOrEqual(
result.history[0].score!.accuracy,
);
});
it('emits progress events for each phase', async () => {
const phases: string[] = [];
await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 1,
evalSize: 3,
anchorEvalSize: 3,
onProgress: (e) => phases.push(e.phase),
});
expect(phases).toContain('start');
expect(phases).toContain('structure');
expect(phases).toContain('order');
expect(phases).toContain('refinement');
expect(phases).toContain('anchor');
expect(phases).toContain('done');
});
it('is deterministic with same seed', async () => {
const cfg = () => ({
baseline: makeSchema(['answer']),
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 1,
evalSize: 5,
anchorEvalSize: 8,
seed: 7,
});
const a = await new EvolveSchema().run(cfg());
const b = await new EvolveSchema().run(cfg());
expect(a.winner.schema.fields.map(f => f.name)).toEqual(
b.winner.schema.fields.map(f => f.name),
);
});
it('respects abort signal', async () => {
const ctrl = new AbortController();
ctrl.abort();
const result = await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 2,
evalSize: 3,
anchorEvalSize: 3,
signal: ctrl.signal,
});
expect(result.winner).toBeDefined();
});
it('records all candidates in history with generation numbers', async () => {
const result = await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 2,
mutationMix: { structure: 2, order: 0, refinement: 0 },
evalSize: 3,
anchorEvalSize: 3,
});
const gen0 = result.history.filter(c => c.generation === 0);
const gen1 = result.history.filter(c => c.generation === 1);
expect(gen0.length).toBe(1);
expect(gen1.length).toBeGreaterThan(0);
});
});

View File

@@ -1,226 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import {
recordCapabilityGap,
analyzeAndRecordCorrection,
recordWorkflowPattern,
buildAwarenessSummary,
formatAwarenessPrompt,
markSummarySurfaced,
} from '../src/improvement-detector.js';
describe('improvement-detector', () => {
let db: MindDB;
let store: ImprovementSignalStore;
beforeEach(() => {
db = new MindDB(':memory:');
store = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
});
// ── recordCapabilityGap ────────────────────────────────────
describe('recordCapabilityGap', () => {
it('records a capability gap signal', () => {
recordCapabilityGap(store, 'pdf_reader', 'user asked to read a PDF');
const gaps = store.getByCategory('capability_gap');
expect(gaps).toHaveLength(1);
expect(gaps[0].pattern_key).toBe('missing:pdf_reader');
expect(gaps[0].count).toBe(1);
});
it('increments count on repeated gap', () => {
recordCapabilityGap(store, 'web_search');
recordCapabilityGap(store, 'web_search');
const gaps = store.getByCategory('capability_gap');
expect(gaps).toHaveLength(1);
expect(gaps[0].count).toBe(2);
});
});
// ── analyzeAndRecordCorrection ─────────────────────────────
describe('analyzeAndRecordCorrection', () => {
it('returns null for non-correction messages', () => {
const result = analyzeAndRecordCorrection(store, 'Please write a sorting function');
expect(result).toBeNull();
expect(store.getByCategory('correction')).toHaveLength(0);
});
it('detects and records a correction', () => {
const result = analyzeAndRecordCorrection(
store,
"No, that's wrong. I told you to use markdown headers.",
);
expect(result).not.toBeNull();
expect(result!.patternKey).toBeDefined();
const corrections = store.getByCategory('correction');
expect(corrections).toHaveLength(1);
});
it('records both durable and task-local corrections', () => {
analyzeAndRecordCorrection(store, "No, always use shorter responses. I said keep it brief.");
analyzeAndRecordCorrection(store, "No, not that. Just use a list this time.");
const corrections = store.getByCategory('correction');
expect(corrections.length).toBeGreaterThanOrEqual(1);
});
});
// ── recordWorkflowPattern ──────────────────────────────────
describe('recordWorkflowPattern', () => {
it('records a workflow pattern', () => {
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
const patterns = store.getByCategory('workflow_pattern');
expect(patterns).toHaveLength(1);
expect(patterns[0].pattern_key).toBe('shape:research');
});
it('increments on repeated task shapes', () => {
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
recordWorkflowPattern(store, 'research', 'investigate market trends');
recordWorkflowPattern(store, 'research', 'review latest papers');
const patterns = store.getByCategory('workflow_pattern');
expect(patterns).toHaveLength(1);
expect(patterns[0].count).toBe(3);
});
});
// ── buildAwarenessSummary ──────────────────────────────────
describe('buildAwarenessSummary', () => {
it('returns empty summary when no actionable signals', () => {
const summary = buildAwarenessSummary(store);
expect(summary.totalActionable).toBe(0);
expect(summary.capabilityGaps).toEqual([]);
expect(summary.corrections).toEqual([]);
expect(summary.workflowPatterns).toEqual([]);
});
it('includes capability gaps above threshold', () => {
// Default threshold for capability_gap is 2
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
expect(summary.capabilityGaps).toHaveLength(1);
expect(summary.capabilityGaps[0].toolName).toBe('pdf_reader');
expect(summary.capabilityGaps[0].occurrences).toBe(2);
expect(summary.capabilityGaps[0].suggestion).toContain('pdf_reader');
});
it('includes corrections above threshold', () => {
// Default threshold for correction is 3
for (let i = 0; i < 3; i++) {
store.record('correction', 'tone:too_formal', 'Keep responses casual');
}
const summary = buildAwarenessSummary(store);
expect(summary.corrections).toHaveLength(1);
expect(summary.corrections[0].patternKey).toBe('tone:too_formal');
expect(summary.corrections[0].occurrences).toBe(3);
});
it('includes recent workflow patterns above threshold', () => {
// Default threshold for workflow_pattern is 3
for (let i = 0; i < 3; i++) {
recordWorkflowPattern(store, 'research', 'investigate topic');
}
const summary = buildAwarenessSummary(store);
expect(summary.workflowPatterns).toHaveLength(1);
expect(summary.workflowPatterns[0].patternKey).toBe('shape:research');
});
it('excludes stale workflow patterns', () => {
// Manually insert a workflow_pattern with old last_seen
const raw = db.getDatabase();
raw.prepare(`
INSERT INTO improvement_signals (category, pattern_key, count, last_seen)
VALUES ('workflow_pattern', 'shape:old_task', 5, datetime('now', '-30 days'))
`).run();
const summary = buildAwarenessSummary(store);
expect(summary.workflowPatterns).toHaveLength(0);
});
it('respects total cap of 3 actionable signals', () => {
// Create many signals above threshold
for (let i = 0; i < 5; i++) {
const key = `missing:tool_${i}`;
store.record('capability_gap', key);
store.record('capability_gap', key);
}
const summary = buildAwarenessSummary(store);
expect(summary.totalActionable).toBeLessThanOrEqual(3);
});
});
// ── formatAwarenessPrompt ──────────────────────────────────
describe('formatAwarenessPrompt', () => {
it('returns null for empty summary', () => {
const summary = buildAwarenessSummary(store);
expect(formatAwarenessPrompt(summary)).toBeNull();
});
it('formats capability gaps into prompt text', () => {
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).not.toBeNull();
expect(prompt).toContain('## Improvement Signals');
expect(prompt).toContain('pdf_reader');
expect(prompt).toContain('Missing capabilities');
});
it('formats corrections into prompt text', () => {
for (let i = 0; i < 3; i++) {
store.record('correction', 'tone:too_formal', 'Keep it casual');
}
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).toContain('Behavioral adjustments');
expect(prompt).toContain('casual');
});
it('formats workflow patterns into prompt text', () => {
for (let i = 0; i < 3; i++) {
recordWorkflowPattern(store, 'compare', 'compare options');
}
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).toContain('Recurring workflows');
expect(prompt).toContain('compare');
});
});
// ── markSummarySurfaced ────────────────────────────────────
describe('markSummarySurfaced', () => {
it('marks all signals in summary as surfaced', () => {
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
expect(summary.capabilityGaps).toHaveLength(1);
markSummarySurfaced(store, summary);
// Should not appear in next summary
const summary2 = buildAwarenessSummary(store);
expect(summary2.capabilityGaps).toHaveLength(0);
expect(summary2.totalActionable).toBe(0);
});
});
});

View File

@@ -1,153 +0,0 @@
import { describe, it, expect } from 'vitest';
import { processInteractionForImprovement } from '../src/improvement-wiring.js';
describe('improvement-wiring', () => {
const baseParams = {
workspaceId: 'ws-1',
sessionId: 'sess-1',
toolsUsed: [],
agentResponse: 'Here is the result.',
userMessage: '',
};
// ── Correction detection ─────────────────────────────────────
describe('correction detection', () => {
it('detects corrections in user messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: "No, that's wrong. I told you to use bullet points, not paragraphs.",
});
expect(result.wasCorrection).toBe(true);
expect(result.correctionDetail).toBeDefined();
expect(result.correctionDetail!.length).toBeGreaterThan(0);
});
it('returns wasCorrection false for normal messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you help me write a function to sort an array?',
});
expect(result.wasCorrection).toBe(false);
expect(result.correctionDetail).toBeUndefined();
});
it('returns wasCorrection false for very short messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'ok',
});
expect(result.wasCorrection).toBe(false);
});
it('detects "I said" corrections', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'I said to use markdown headers, not plain text.',
});
expect(result.wasCorrection).toBe(true);
});
it('detects "actually instead use" corrections', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Actually, instead use TypeScript for this module.',
});
expect(result.wasCorrection).toBe(true);
});
});
// ── Capability gap detection ─────────────────────────────────
describe('capability gap detection', () => {
it('identifies capability gaps from tool-not-found in response', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Read this PDF for me.',
agentResponse: 'Tool "pdf_reader" not found. I cannot read PDFs directly.',
});
expect(result.capabilityGap).toBe('pdf_reader');
});
it('identifies capability gaps from "I don\'t have a tool" in response', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you send this to Slack?',
agentResponse: "I don't have a tool for sending Slack messages.",
});
expect(result.capabilityGap).toBe('sending Slack messages');
});
it('identifies gaps from user requesting integration', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you send to Slack the summary?',
agentResponse: 'Let me try to do that.',
toolsUsed: [],
});
expect(result.capabilityGap).toBe('slack');
});
it('returns no gap for normal interactions', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'What is the weather today?',
agentResponse: 'The weather today is sunny and 72F.',
});
expect(result.capabilityGap).toBeUndefined();
});
});
// ── Workflow pattern detection ───────────────────────────────
describe('workflow pattern detection', () => {
it('detects multi-step workflow patterns with "then"', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'First research the competitors, then compare their features, then draft a summary report.',
});
expect(result.workflowPattern).toBeDefined();
});
it('does not flag single-step requests as workflow patterns', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Write a summary of this document.',
});
expect(result.workflowPattern).toBeUndefined();
});
it('returns undefined for simple questions', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'What is the status of the project?',
});
expect(result.workflowPattern).toBeUndefined();
});
});
// ── Combined signals ─────────────────────────────────────────
describe('combined signals', () => {
it('can detect correction and gap simultaneously', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: "No, that's wrong. I told you to use the Jira tool.",
agentResponse: 'Tool "jira_create" not found.',
});
expect(result.wasCorrection).toBe(true);
expect(result.capabilityGap).toBe('jira_create');
});
it('returns all-false for benign exchange', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Thanks, that looks great!',
agentResponse: "You're welcome! Let me know if you need anything else.",
});
expect(result.wasCorrection).toBe(false);
expect(result.capabilityGap).toBeUndefined();
expect(result.workflowPattern).toBeUndefined();
});
});
});

View File

@@ -1,655 +0,0 @@
import { describe, it, expect, vi } from 'vitest';
import {
IterativeGEPA,
paretoFront,
scoreCandidate,
aggregateScores,
pickWinner,
pickSample,
type Candidate,
type CandidateScore,
type IterativeGEPAOptions,
type MutateFn,
} from '../src/iterative-optimizer.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
/**
* H-09 G3 — unit tests mock the judge, so they opt into the running-judge
* safety-check bypass. Production code paths must still wrap with
* makeRunningJudge(); the check in IterativeGEPA enforces that.
*/
function runGEPA(options: Omit<IterativeGEPAOptions, 'allowBareJudge'>) {
const gepa = new IterativeGEPA();
return gepa.run({ ...options, allowBareJudge: true });
}
// ── Fixtures ───────────────────────────────────────────────────
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `question ${i}`,
expected_output: `answer ${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeScore(overall: number, extra: Partial<JudgeScore> = {}): JudgeScore {
return {
overall,
weighted: overall,
correctness: extra.correctness ?? overall,
procedureFollowing: extra.procedureFollowing ?? overall,
conciseness: extra.conciseness ?? overall,
lengthPenalty: extra.lengthPenalty ?? 1,
feedback: extra.feedback ?? `feedback for score ${overall}`,
parsed: true,
};
}
function makeCandidate(id: string, score: CandidateScore | null): Candidate {
return {
id,
prompt: `prompt-${id}`,
generation: 0,
parent: null,
strategy: 'baseline',
score,
perExample: [],
};
}
function makeCandidateScore(
overall: number,
dims: Partial<{ correctness: number; procedureFollowing: number; conciseness: number; lengthPenalty: number; n: number; weaknessFeedback: string[] }> = {},
): CandidateScore {
return {
overall,
correctness: dims.correctness ?? overall,
procedureFollowing: dims.procedureFollowing ?? overall,
conciseness: dims.conciseness ?? overall,
lengthPenalty: dims.lengthPenalty ?? 1,
n: dims.n ?? 10,
weaknessFeedback: dims.weaknessFeedback ?? [],
};
}
// Fake judge that computes a deterministic score based on candidate text length.
// Longer prompts → higher correctness, shorter → higher conciseness.
function makeFakeJudge(bias: 'long' | 'short' | 'uniform' = 'uniform') {
return {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
const len = args.actual.length;
const corr = bias === 'long' ? Math.min(1, len / 50) : bias === 'short' ? Math.max(0, 1 - len / 100) : 0.5;
const proc = 0.6;
const conc = bias === 'short' ? 0.9 : 0.5;
const overall = 0.5 * corr + 0.3 * proc + 0.2 * conc;
return {
overall, weighted: overall,
correctness: corr,
procedureFollowing: proc,
conciseness: conc,
lengthPenalty: 1,
feedback: `len=${len}`,
parsed: true,
};
},
};
}
// ── Pure helpers ───────────────────────────────────────────────
describe('aggregateScores', () => {
it('handles empty input', () => {
const agg = aggregateScores([]);
expect(agg.n).toBe(0);
expect(agg.overall).toBe(0);
expect(agg.weaknessFeedback).toEqual([]);
});
it('averages component dimensions', () => {
const agg = aggregateScores([
makeScore(0.8, { correctness: 0.9, procedureFollowing: 0.7, conciseness: 0.8 }),
makeScore(0.6, { correctness: 0.7, procedureFollowing: 0.5, conciseness: 0.6 }),
]);
expect(agg.n).toBe(2);
expect(agg.correctness).toBeCloseTo(0.8, 5);
expect(agg.procedureFollowing).toBeCloseTo(0.6, 5);
expect(agg.conciseness).toBeCloseTo(0.7, 5);
expect(agg.overall).toBeCloseTo(0.7, 5);
});
it('surfaces worst-3 feedback lines in weaknessFeedback', () => {
const agg = aggregateScores([
makeScore(0.9, { feedback: 'best' }),
makeScore(0.1, { feedback: 'worst' }),
makeScore(0.2, { feedback: 'second-worst' }),
makeScore(0.3, { feedback: 'third-worst' }),
makeScore(0.8, { feedback: 'good' }),
]);
// Worst 3 should contain 'worst', 'second-worst', 'third-worst' in some order
expect(agg.weaknessFeedback).toContain('worst');
expect(agg.weaknessFeedback).toContain('second-worst');
expect(agg.weaknessFeedback).toContain('third-worst');
expect(agg.weaknessFeedback).not.toContain('best');
});
it('drops empty feedback from weakness list', () => {
const agg = aggregateScores([
makeScore(0.1, { feedback: '' }),
makeScore(0.2, { feedback: 'real feedback' }),
]);
expect(agg.weaknessFeedback).toEqual(['real feedback']);
});
});
describe('paretoFront', () => {
it('keeps a single member when only one has a score', () => {
const a = makeCandidate('a', makeCandidateScore(0.8));
const b = makeCandidate('b', null);
expect(paretoFront([a, b])).toHaveLength(1);
});
it('removes strictly-dominated candidates', () => {
const weak = makeCandidate('weak', makeCandidateScore(0.5, {
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
}));
const strong = makeCandidate('strong', makeCandidateScore(0.8, {
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
}));
const front = paretoFront([weak, strong]);
expect(front).toHaveLength(1);
expect(front[0].id).toBe('strong');
});
it('keeps trade-off candidates on the front', () => {
const accurate = makeCandidate('accurate', makeCandidateScore(0.7, {
correctness: 0.9, procedureFollowing: 0.6, conciseness: 0.5,
}));
const concise = makeCandidate('concise', makeCandidateScore(0.7, {
correctness: 0.6, procedureFollowing: 0.6, conciseness: 0.9,
}));
const front = paretoFront([accurate, concise]);
expect(front).toHaveLength(2);
});
it('returns all candidates when they have identical scores', () => {
const a = makeCandidate('a', makeCandidateScore(0.5));
const b = makeCandidate('b', makeCandidateScore(0.5));
const front = paretoFront([a, b]);
expect(front).toHaveLength(2);
});
it('ignores candidates with null scores entirely', () => {
const scored = makeCandidate('scored', makeCandidateScore(0.5));
const unscored = makeCandidate('unscored', null);
const front = paretoFront([scored, unscored]);
expect(front).toHaveLength(1);
expect(front[0].id).toBe('scored');
});
});
describe('pickWinner', () => {
it('picks highest overall score', () => {
const a = makeCandidate('a', makeCandidateScore(0.6));
const b = makeCandidate('b', makeCandidateScore(0.9));
expect(pickWinner([a, b]).id).toBe('b');
});
it('breaks ties using length penalty (higher = more concise, preferred)', () => {
const verbose = makeCandidate('verbose', makeCandidateScore(0.7, { lengthPenalty: 0.6 }));
const concise = makeCandidate('concise', makeCandidateScore(0.7, { lengthPenalty: 0.9 }));
expect(pickWinner([verbose, concise]).id).toBe('concise');
});
it('breaks further ties by earliest generation', () => {
const early = makeCandidate('early', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
const late = makeCandidate('late', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
late.generation = 3;
expect(pickWinner([early, late]).id).toBe('early');
});
it('throws on empty input', () => {
expect(() => pickWinner([])).toThrow();
});
});
describe('pickSample', () => {
const rng = () => 0.5;
it('returns empty on k=0', () => {
expect(pickSample(makeExamples(10), 0, rng)).toEqual([]);
});
it('returns full set (shuffled) when k >= length', () => {
const sample = pickSample(makeExamples(5), 10, rng);
expect(sample).toHaveLength(5);
});
it('is deterministic with a fixed rng', () => {
const rngA = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
const rngB = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
const a = pickSample(makeExamples(20), 5, rngA).map(e => e.input);
const b = pickSample(makeExamples(20), 5, rngB).map(e => e.input);
expect(a).toEqual(b);
});
});
describe('scoreCandidate', () => {
it('scores candidate against examples and populates the score field', async () => {
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(3), makeFakeJudge('uniform'));
expect(score.n).toBe(3);
expect(cand.score).not.toBeNull();
expect(cand.perExample).toHaveLength(3);
});
it('returns zero-score aggregation when examples list is empty', async () => {
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, [], makeFakeJudge());
expect(score.n).toBe(0);
expect(score.overall).toBe(0);
});
it('continues iterating if one score call throws', async () => {
let callCount = 0;
const flakyJudge = {
async score(): Promise<JudgeScore> {
callCount++;
if (callCount === 2) throw new Error('transient');
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(3), flakyJudge);
expect(score.n).toBe(2); // 2 succeeded, 1 failed
});
it('stops early when abort signal fires', async () => {
const ctrl = new AbortController();
const judge = makeFakeJudge();
// Abort before any call
ctrl.abort();
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(10), judge, ctrl.signal);
expect(cand.perExample.length).toBeLessThan(10);
});
// ── Concurrency ────────────────────────────────────────────────
it('accepts an options object (new API) equivalently to AbortSignal (legacy)', async () => {
const judge = makeFakeJudge('uniform');
const candA = makeCandidate('a', null);
const candB = makeCandidate('b', null);
const ctrl = new AbortController();
ctrl.abort();
await scoreCandidate(candA, makeExamples(5), judge, ctrl.signal);
await scoreCandidate(candB, makeExamples(5), judge, { signal: ctrl.signal });
expect(candA.perExample.length).toBe(candB.perExample.length);
});
it('concurrency=1 produces identical aggregate to no option (sequential baseline)', async () => {
const examples = makeExamples(6);
const candSeq = makeCandidate('seq', null);
const candPar = makeCandidate('par', null);
const seqScore = await scoreCandidate(candSeq, examples, makeFakeJudge('uniform'));
const parScore = await scoreCandidate(candPar, examples, makeFakeJudge('uniform'), { concurrency: 1 });
expect(parScore.overall).toBeCloseTo(seqScore.overall);
expect(parScore.n).toBe(seqScore.n);
});
it('with concurrency > 1, runs scores in parallel (observable via in-flight counter)', async () => {
let inFlight = 0;
let peakInFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
// Await a microtask + a real delay so parallel workers can accumulate.
await new Promise(r => setTimeout(r, 5));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(8), judge, { concurrency: 4 });
expect(peakInFlight).toBe(4);
expect(cand.perExample).toHaveLength(8);
});
it('with concurrency=1, exactly 1 in-flight call at a time', async () => {
let inFlight = 0;
let peakInFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 3));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(5), judge, { concurrency: 1 });
expect(peakInFlight).toBe(1);
});
it('concurrency is capped at examples.length (does not start idle workers)', async () => {
let peakInFlight = 0;
let inFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 2));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(3), judge, { concurrency: 100 });
expect(peakInFlight).toBeLessThanOrEqual(3);
});
it('parallel mode still filters out thrown-error results without corrupting the batch', async () => {
let call = 0;
const judge = {
async score(): Promise<JudgeScore> {
call++;
if (call % 3 === 0) throw new Error('flaky');
return makeScore(0.7);
},
};
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(9), judge, { concurrency: 3 });
// 9 total, every 3rd throws → 6 succeed.
expect(score.n).toBe(6);
});
it('parallel mode respects abort signal by not dispatching further workers', async () => {
const ctrl = new AbortController();
ctrl.abort();
const judge = makeFakeJudge();
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(20), judge, {
signal: ctrl.signal,
concurrency: 4,
});
expect(cand.perExample.length).toBeLessThan(20);
});
});
// ── IterativeGEPA end-to-end with concurrency ────────────────────
describe('IterativeGEPA with concurrency', () => {
it('threads options.concurrency into every scoreCandidate call', async () => {
let peakInFlight = 0;
let inFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 2));
inFlight--;
return makeScore(0.5);
},
};
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt} :: ${strategy}`;
await runGEPA({
baseline: 'base',
examples: makeExamples(8),
judge,
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 4,
miniEvalSize: 4,
anchorEvalSize: 4,
concurrency: 3,
});
expect(peakInFlight).toBe(3);
});
});
// ── H-09 G3 · running-judge guard ──────────────────────────────
describe('IterativeGEPA.run · running-judge guard (H-09 G3)', () => {
it('throws when a bare judge is passed without allowBareJudge', async () => {
const bareJudge = {
async score(): Promise<JudgeScore> {
return {
overall: 0.5, weighted: 0.5, correctness: 0.5,
procedureFollowing: 0.5, conciseness: 0.5, lengthPenalty: 1,
feedback: '', parsed: true,
};
},
};
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} mutated`;
await expect(
new IterativeGEPA().run({
baseline: 'seed',
examples: makeExamples(3),
judge: bareJudge,
mutate,
}),
).rejects.toThrow(/not a running judge|makeRunningJudge/);
});
it('accepts a judge wrapped with makeRunningJudge', async () => {
const { makeRunningJudge } = await import('../src/evolution-llm-wiring.js');
const baseJudge = {
async score(): Promise<JudgeScore> {
return {
overall: 0.8, weighted: 0.8, correctness: 0.8,
procedureFollowing: 0.8, conciseness: 0.8, lengthPenalty: 1,
feedback: '', parsed: true,
};
},
};
const fakeLLM = { complete: async () => 'LLM output' };
const wrapped = makeRunningJudge(baseJudge, fakeLLM);
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} v2`;
// Should not throw — the wrapped judge carries the brand.
const result = await new IterativeGEPA().run({
baseline: 'seed',
examples: makeExamples(3),
judge: wrapped,
mutate,
populationSize: 1,
generations: 1,
microScreenSize: 1, miniEvalSize: 1, anchorEvalSize: 1,
});
expect(result.winner).toBeDefined();
});
});
// ── End-to-end run ─────────────────────────────────────────────
describe('IterativeGEPA.run', () => {
it('runs through all phases and produces a winner', async () => {
const baseline = 'short baseline prompt';
const examples = makeExamples(10);
const judge = makeFakeJudge('uniform');
// Mutate appends text — simulates generation of children
const mutate: MutateFn = async ({ parent, strategy }) => {
return `${parent.prompt} :: ${strategy}`;
};
const progress: string[] = [];
const result = await runGEPA({
baseline,
examples,
judge,
mutate,
populationSize: 3,
generations: 2,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 10,
onProgress: (e) => progress.push(`${e.phase}@g${e.generation}`),
});
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
expect(result.history.length).toBeGreaterThan(1);
expect(progress).toContain('start@g0');
expect(progress.some(p => p.startsWith('anchor'))).toBe(true);
expect(progress.some(p => p.startsWith('done'))).toBe(true);
});
it('winner dominates or matches baseline on overall score', async () => {
const baseline = 'x'; // very short
const examples = makeExamples(20);
// Long-biased judge: longer prompts score higher
const judge = makeFakeJudge('long');
// Mutate doubles length each time
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} ${parent.prompt}more`;
const result = await runGEPA({
baseline,
examples,
judge,
mutate,
populationSize: 3,
generations: 2,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 15,
});
// Winner should have evolved (not baseline prompt).
expect(result.winner.prompt.length).toBeGreaterThan(baseline.length);
expect(result.improved).toBe(true);
expect(result.delta).toBeGreaterThan(0);
});
it('passes weakness feedback from parent into mutate()', async () => {
const captured: Array<{ strategy: string; feedbacks: string[] }> = [];
const mutate: MutateFn = async ({ parent, strategy, weaknessFeedback }) => {
captured.push({ strategy, feedbacks: weaknessFeedback });
return `${parent.prompt} mutated`;
};
await runGEPA({
baseline: 'baseline',
examples: makeExamples(10),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 5,
});
// 2 mutations were spawned — each received the parent's feedback
expect(captured.length).toBe(2);
for (const c of captured) {
expect(Array.isArray(c.feedbacks)).toBe(true);
// Feedback array may be empty on first gen if baseline had no weaknesses;
// we just assert the prop is present and is an array.
}
});
it('recovers when mutate throws (falls back to parent prompt)', async () => {
let mutations = 0;
const mutate: MutateFn = async ({ parent }) => {
mutations++;
if (mutations === 1) throw new Error('boom');
return `${parent.prompt} ok`;
};
const result = await runGEPA({
baseline: 'baseline',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 3,
});
// Winner must be defined even though one mutation failed
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
});
it('respects abort signal and exits early without throwing', async () => {
const ctrl = new AbortController();
const mutate: MutateFn = vi.fn(async () => 'should never run');
ctrl.abort(); // abort before starting generations
const result = await runGEPA({
baseline: 'baseline',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 2,
microScreenSize: 2,
miniEvalSize: 2,
anchorEvalSize: 2,
signal: ctrl.signal,
});
// With abort before baseline scoring, no winner should evolve
expect(result.winner).toBeDefined();
expect(mutate).not.toHaveBeenCalled();
});
it('is deterministic with same seed', async () => {
const examples = makeExamples(20);
const judge = makeFakeJudge();
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt}::${strategy}`;
const run = () => runGEPA({
baseline: 'seed',
examples,
judge,
mutate,
populationSize: 3,
generations: 1,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 10,
seed: 42,
});
const r1 = await run();
const r2 = await run();
expect(r1.winner.prompt).toBe(r2.winner.prompt);
expect(r1.history.length).toBe(r2.history.length);
});
it('produces history containing baseline and all mutated children', async () => {
const result = await runGEPA({
baseline: 'base',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate: async ({ parent, strategy }) => `${parent.prompt}_${strategy}`,
populationSize: 3,
generations: 2,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 3,
});
expect(result.history[0].id).toBe('g0-baseline');
expect(result.history[0].strategy).toBe('baseline');
// 2 generations * 3 mutations + 1 baseline = 7 total
expect(result.history.length).toBe(7);
});
});

View File

@@ -1,280 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, OptimizationLogStore } from '@waggle/core';
import {
captureInteraction,
getRecentLogs,
getWorkspaceLogs,
isWithinBudget,
} from '../src/optimization-capture.js';
import type { AgentResponse } from '../src/agent-loop.js';
describe('optimization-capture', () => {
let db: MindDB;
let store: OptimizationLogStore;
beforeEach(() => {
db = new MindDB(':memory:');
store = new OptimizationLogStore(db);
});
afterEach(() => {
db.close();
});
function makeResponse(overrides?: Partial<AgentResponse>): AgentResponse {
return {
content: 'Hello, I can help with that.',
toolsUsed: ['search_memory', 'save_memory'],
usage: { inputTokens: 500, outputTokens: 200 },
...overrides,
};
}
// ── captureInteraction ─────────────────────────────────────
describe('captureInteraction', () => {
it('stores a log entry with correct fields', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'You are a helpful assistant.',
response: makeResponse(),
turnCount: 3,
wasCorrection: false,
});
expect(entry.id).toBeGreaterThan(0);
expect(entry.session_id).toBe('session-1');
expect(entry.workspace_id).toBe('ws-abc');
expect(entry.system_prompt).toBe('You are a helpful assistant.');
expect(JSON.parse(entry.tools_used)).toEqual(['search_memory', 'save_memory']);
expect(entry.turn_count).toBe(3);
expect(entry.was_correction).toBe(0);
expect(entry.input_tokens).toBe(500);
expect(entry.output_tokens).toBe(200);
expect(entry.timestamp).toBeTruthy();
});
it('records wasCorrection as 1 when true', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'test prompt',
response: makeResponse(),
turnCount: 1,
wasCorrection: true,
});
expect(entry.was_correction).toBe(1);
});
it('handles empty toolsUsed', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'test prompt',
response: makeResponse({ toolsUsed: [] }),
turnCount: 1,
wasCorrection: false,
});
expect(JSON.parse(entry.tools_used)).toEqual([]);
});
it('defaults token counts to 0 when missing', () => {
const response: AgentResponse = {
content: 'response',
toolsUsed: [],
usage: { inputTokens: 0, outputTokens: 0 },
};
const entry = captureInteraction(store, {
sessionId: 's1',
workspaceId: 'ws1',
systemPrompt: 'prompt',
response,
turnCount: 1,
wasCorrection: false,
});
expect(entry.input_tokens).toBe(0);
expect(entry.output_tokens).toBe(0);
});
});
// ── getRecentLogs ──────────────────────────────────────────
describe('getRecentLogs', () => {
it('retrieves stored entries ordered by most recent first', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p1',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p2',
response: makeResponse(), turnCount: 2, wasCorrection: true,
});
const logs = getRecentLogs(store, 10);
expect(logs).toHaveLength(2);
// Ordered by timestamp DESC, id DESC — s2 was inserted second so it comes first
expect(logs[0].session_id).toBe('s2');
expect(logs[1].session_id).toBe('s1');
});
it('respects the limit parameter', () => {
for (let i = 0; i < 5; i++) {
captureInteraction(store, {
sessionId: `s${i}`, workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
}
const logs = getRecentLogs(store, 3);
expect(logs).toHaveLength(3);
});
it('returns empty array when no entries exist', () => {
const logs = getRecentLogs(store);
expect(logs).toEqual([]);
});
});
// ── getWorkspaceLogs ───────────────────────────────────────
describe('getWorkspaceLogs', () => {
it('filters by workspace ID', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws-a', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws-b', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's3', workspaceId: 'ws-a', systemPrompt: 'p',
response: makeResponse(), turnCount: 2, wasCorrection: true,
});
const wsALogs = getWorkspaceLogs(store, 'ws-a');
expect(wsALogs).toHaveLength(2);
expect(wsALogs.every(l => l.workspace_id === 'ws-a')).toBe(true);
const wsBLogs = getWorkspaceLogs(store, 'ws-b');
expect(wsBLogs).toHaveLength(1);
});
});
// ── Entries have correct shape ─────────────────────────────
describe('entry shape', () => {
it('has all required fields with correct types', () => {
const entry = captureInteraction(store, {
sessionId: 'session-x',
workspaceId: 'workspace-y',
systemPrompt: 'You are helpful.',
response: makeResponse({ toolsUsed: ['bash', 'read_file'] }),
turnCount: 7,
wasCorrection: false,
});
// Check all fields exist and are of expected types
expect(typeof entry.id).toBe('number');
expect(typeof entry.session_id).toBe('string');
expect(typeof entry.workspace_id).toBe('string');
expect(typeof entry.system_prompt).toBe('string');
expect(typeof entry.tools_used).toBe('string');
expect(typeof entry.turn_count).toBe('number');
expect(typeof entry.was_correction).toBe('number');
expect(typeof entry.input_tokens).toBe('number');
expect(typeof entry.output_tokens).toBe('number');
expect(typeof entry.timestamp).toBe('string');
// tools_used is valid JSON array
const parsed = JSON.parse(entry.tools_used);
expect(Array.isArray(parsed)).toBe(true);
expect(parsed).toEqual(['bash', 'read_file']);
});
});
// ── Budget enforcement ─────────────────────────────────────
describe('isWithinBudget', () => {
it('returns true when no entries exist (under budget)', () => {
expect(isWithinBudget(store, 100)).toBe(true);
});
it('returns true when token cost is under budget', () => {
// Insert entry with small token count
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
turnCount: 1, wasCorrection: false,
});
// 100 input * $3/M + 50 output * $15/M = $0.0003 + $0.00075 = ~$0.00105 = ~0.1 cents
// Budget is 100 cents = $1, so well within budget
expect(isWithinBudget(store, 100)).toBe(true);
});
it('returns false when token cost exceeds budget', () => {
// Insert entry with large token count to exceed a tiny budget
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 1_000_000, outputTokens: 500_000 } }),
turnCount: 1, wasCorrection: false,
});
// 1M input * $3/M + 500K output * $15/M = $3 + $7.5 = $10.50 = 1050 cents
// Budget is 1 cent, so way over budget
expect(isWithinBudget(store, 1)).toBe(false);
});
it('respects custom cost-per-token rates', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 10_000, outputTokens: 5_000 } }),
turnCount: 1, wasCorrection: false,
});
// With very cheap rates: 10K * $0.0001/M + 5K * $0.0001/M ≈ ~0 cents
expect(isWithinBudget(store, 1, 0.0001 / 1_000_000, 0.0001 / 1_000_000)).toBe(true);
// With very expensive rates: 10K * $100/M + 5K * $100/M = $1 + $0.5 = 150 cents
expect(isWithinBudget(store, 1, 100 / 1_000_000, 100 / 1_000_000)).toBe(false);
});
});
// ── Store stats ────────────────────────────────────────────
describe('getStats (via store)', () => {
it('computes correct aggregate stats', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
turnCount: 3, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 200, outputTokens: 100 } }),
turnCount: 5, wasCorrection: true,
});
const stats = store.getStats();
expect(stats.total).toBe(2);
expect(stats.correctionRate).toBe(0.5); // 1/2
expect(stats.avgTurnCount).toBe(4); // (3+5)/2
expect(stats.totalInputTokens).toBe(300); // 100+200
expect(stats.totalOutputTokens).toBe(150); // 50+100
});
it('returns zeros when empty', () => {
const stats = store.getStats();
expect(stats.total).toBe(0);
expect(stats.correctionRate).toBe(0);
expect(stats.avgTurnCount).toBe(0);
expect(stats.totalInputTokens).toBe(0);
expect(stats.totalOutputTokens).toBe(0);
});
});
});

View File

@@ -2,10 +2,9 @@ import { describe, it, expect } from 'vitest';
import { PERSONAS, getPersona, listPersonas, composePersonaPrompt } from '../src/personas.js';
describe('Agent Personas', () => {
it('PERSONAS catalog has 23 entries', () => {
// 22 user-facing (8 universal + 14 specialists) + 1 internal review persona
// (session-reviewer) used only by the IdleSessionWatcher self-evolution loop.
expect(PERSONAS).toHaveLength(23);
it('PERSONAS catalog has 22 entries', () => {
// 8 universal + 14 specialists.
expect(PERSONAS).toHaveLength(22);
});
it('each persona has required fields', () => {
@@ -40,9 +39,9 @@ describe('Agent Personas', () => {
expect(getPersona('')).toBeNull();
});
it('listPersonas() returns all 23 personas', () => {
it('listPersonas() returns all 22 personas', () => {
const list = listPersonas();
expect(list).toHaveLength(23);
expect(list).toHaveLength(22);
expect(list).not.toBe(PERSONAS); // Returns a copy
});
@@ -208,19 +207,6 @@ describe('Soft tool model', () => {
expect(v!.disallowedTools).toContain('write_file');
expect(v!.isReadOnly).toBe(true);
});
it('session-reviewer is a trust-boundary persona: reads + create_skill only, no writes/exec/memory', () => {
const r = getPersona('session-reviewer');
expect(r, 'session-reviewer not found').toBeTruthy();
// create_skill is the proposal vehicle and must stay in the allowlist.
expect(r!.tools).toContain('create_skill');
// isReadOnly would strip create_skill, so it must be false.
expect(r!.isReadOnly).toBe(false);
// The critical tools ALWAYS_AVAILABLE_TOOLS re-adds must be explicitly denied.
for (const t of ['save_memory', 'delete_skill', 'install_capability', 'acquire_capability', 'spawn_agent', 'bash']) {
expect(r!.disallowedTools, `${t} must be disallowed`).toContain(t);
}
});
});
describe('New domain personas', () => {
@@ -239,8 +225,7 @@ describe('New domain personas', () => {
});
}
it('total persona count is 23', () => {
// 22 tiered + session-reviewer (internal self-evolution reviewer).
expect(PERSONAS.length).toBe(23);
it('total persona count is 22', () => {
expect(PERSONAS.length).toBe(22);
});
});

View File

@@ -3,13 +3,12 @@
*
* Tests the one-step promotion personal → workspace, the refusal of team and
* enterprise scopes in this edition, frontmatter rewrite with promoted_from
* history, file move, and skill_promotion improvement signal emission.
* history and file move.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import { createSkillTools, getSkillDirForScope } from '../src/skill-tools.js';
import { parseSkillFrontmatter } from '../src/skill-frontmatter.js';
@@ -22,8 +21,6 @@ function seedSkill(dir: string, name: string, scope: string | null, body = 'body
describe('promote_skill — gap E', () => {
let tmpDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
let reloadCount: number;
function createTools(overrides: Partial<Parameters<typeof createSkillTools>[0]> = {}) {
@@ -32,7 +29,6 @@ describe('promote_skill — gap E', () => {
onSkillsChanged: () => { reloadCount++; },
getWorkspaceId: () => 'ws-alpha',
getTeamId: () => 'team-bravo',
improvementSignals: signals,
...overrides,
});
}
@@ -45,18 +41,15 @@ describe('promote_skill — gap E', () => {
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-promote-test-'));
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
reloadCount = 0;
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
describe('happy path', () => {
it('promotes personal → workspace, moves file, appends history, records signal', async () => {
it('promotes personal → workspace, moves file, appends history', async () => {
const personalDir = getSkillDirForScope(tmpDir, 'personal', {})!;
seedSkill(personalDir, 'deploy-helper', 'personal');
const tools = createTools();
@@ -64,7 +57,6 @@ describe('promote_skill — gap E', () => {
const out = await run(tools, 'promote_skill', { skill_name: 'deploy-helper' });
expect(out).toContain('personal → workspace');
expect(out).toContain('Signal recorded');
// File moved
expect(fs.existsSync(path.join(personalDir, 'deploy-helper.md'))).toBe(false);
@@ -77,11 +69,6 @@ describe('promote_skill — gap E', () => {
expect(parsed.frontmatter.scope).toBe('workspace');
expect(parsed.frontmatter.promoted_from).toEqual(['personal']);
// Signal emitted
const recorded = signals.getByCategory('skill_promotion');
expect(recorded.length).toBe(1);
expect(recorded[0].pattern_key).toBe('promote:deploy-helper:personal->workspace');
// onSkillsChanged fired
expect(reloadCount).toBe(1);
});
@@ -125,7 +112,6 @@ describe('promote_skill — gap E', () => {
const out = await run(createTools(), 'promote_skill', { skill_name: 'shared-helper' });
expect(out).toContain('not available in this edition');
expect(fs.existsSync(path.join(wsDir, 'shared-helper.md'))).toBe(true);
expect(signals.getByCategory('skill_promotion')).toHaveLength(0);
});
it('refuses enterprise promotion', async () => {

View File

@@ -1,8 +1,7 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
loadSystemPrompt,
loadSystemPromptWithOverrides,
assertOverridesReachActiveSpec,
loadSkills,
} from '../src/prompt-loader.js';
import * as fs from 'node:fs';
@@ -68,7 +67,6 @@ describe('Prompt Loader', () => {
});
});
// ── H-08 G2 — override-aware loader ──────────────────────────
describe('loadSystemPromptWithOverrides', () => {
it('returns baseline when no overrides exist', () => {
const composed = loadSystemPromptWithOverrides(tmpDir);
@@ -86,55 +84,17 @@ describe('Prompt Loader', () => {
expect(composed.userPromptMd).toBe('hello from user');
});
it('applies behavioral-spec overrides into the active spec', () => {
it('ignores override files left on disk by earlier versions', () => {
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(overrideDir, { recursive: true });
fs.writeFileSync(
path.join(overrideDir, 'coreLoop.json'),
JSON.stringify({
section: 'coreLoop',
text: 'CUSTOM CORE LOOP CONTENT FOR TEST',
deployedAt: new Date().toISOString(),
}),
JSON.stringify({ section: 'coreLoop', text: 'STALE OVERRIDE TEXT', deployedAt: new Date().toISOString() }),
);
const composed = loadSystemPromptWithOverrides(tmpDir);
const flattened = Object.values(composed.behavioralSpec).join('\n');
expect(flattened).toContain('CUSTOM CORE LOOP CONTENT FOR TEST');
});
});
describe('assertOverridesReachActiveSpec', () => {
it('is quiet when no overrides are deployed', () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
const composed = loadSystemPromptWithOverrides(tmpDir);
assertOverridesReachActiveSpec(tmpDir, composed.behavioralSpec);
expect(warnSpy).not.toHaveBeenCalled();
warnSpy.mockRestore();
});
it('warns when disk has overrides but activeSpec does not contain them', () => {
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(overrideDir, { recursive: true });
fs.writeFileSync(
path.join(overrideDir, 'coreLoop.json'),
JSON.stringify({
section: 'coreLoop',
text: 'UNIQUE MARKER STRING FOR G2 TEST — LONG ENOUGH TO MATCH',
deployedAt: new Date().toISOString(),
}),
);
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
// Simulate using the BARE loader — activeSpec is the compile-time
// baseline with no overrides folded in.
const fakeActiveSpec = loadSystemPromptWithOverrides(path.join(tmpDir, 'nothing')).behavioralSpec;
assertOverridesReachActiveSpec(tmpDir, fakeActiveSpec);
expect(warnSpy).toHaveBeenCalled();
const message = warnSpy.mock.calls[0]?.[0] as string;
expect(message).toContain('coreLoop');
warnSpy.mockRestore();
expect(flattened).not.toContain('STALE OVERRIDE TEXT');
});
});
});

View File

@@ -2,13 +2,8 @@
* Skills 2.0 gap K — write-path conflict detection.
*
* When save_memory writes a new frame that contradicts an existing frame
* (detectContradiction), the save still succeeds but:
* - the return string carries a [flag: contradicts_existing …] marker
* - a correction-category improvement signal is recorded
*
* Previously detectContradiction existed but was never called on the write
* path — write-path conflicts silently overwrote reality, degrading the
* evolution-signal loop.
* (detectContradiction), the save still succeeds but the return string
* carries a [flag: contradicts_existing …] marker.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
@@ -19,7 +14,6 @@ import {
SessionStore,
KnowledgeGraph,
HybridSearch,
ImprovementSignalStore,
} from '@waggle/core';
import { createMindTools } from '../src/tools.js';
import type { ToolDefinition } from '../src/tools.js';
@@ -30,7 +24,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
let frames: FrameStore;
let sessions: SessionStore;
let tools: ToolDefinition[];
let improvementSignals: ImprovementSignalStore;
function getSaveMemory(): ToolDefinition {
const tool = tools.find(t => t.name === 'save_memory');
@@ -45,7 +38,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
sessions = new SessionStore(db);
const knowledge = new KnowledgeGraph(db);
const search = new HybridSearch(db, embedder);
improvementSignals = new ImprovementSignalStore(db);
tools = createMindTools({
db,
@@ -55,7 +47,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
sessions,
search,
knowledge,
improvementSignals,
// Intentionally NO cognify → exercises the raw-frame fallback path
// (no LLM-dependent entity extraction in unit tests).
});
@@ -94,17 +85,9 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
expect(second).toContain('Memory saved');
// But the return message is annotated
expect(second).toContain('[flag: contradicts_existing');
// And a correction signal is recorded
const top = improvementSignals.getByCategory('correction');
expect(top.length).toBeGreaterThan(0);
const writeConflict = top.find(s => s.pattern_key.startsWith('write-conflict:'));
expect(writeConflict).toBeDefined();
expect(writeConflict!.detail).toMatch(/contradicts existing frame/i);
expect(writeConflict!.category).toBe('correction');
});
it('non-contradicting saves do not flag or emit a signal', async () => {
it('non-contradicting saves are not flagged', async () => {
const save = getSaveMemory();
await save.execute({
content: 'Decision: we chose PostgreSQL for analytics; approved and confirmed.',
@@ -117,34 +100,5 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
source: 'user_stated',
});
expect(result).not.toContain('contradicts_existing');
const top = improvementSignals.getByCategory('correction');
expect(top.find(s => s.pattern_key.startsWith('write-conflict:'))).toBeUndefined();
});
it('works without improvementSignals dep (backwards-compatible)', async () => {
// Rebuild tools without signals dep
const db2 = new MindDB(':memory:');
const embedder = new MockEmbedder();
const frames2 = new FrameStore(db2);
const sessions2 = new SessionStore(db2);
const knowledge2 = new KnowledgeGraph(db2);
const search2 = new HybridSearch(db2, embedder);
const tools2 = createMindTools({
db: db2,
identity: new IdentityLayer(db2),
awareness: new AwarenessLayer(db2),
frames: frames2,
sessions: sessions2,
search: search2,
knowledge: knowledge2,
// No improvementSignals — should not throw
});
const save = tools2.find(t => t.name === 'save_memory')!;
await save.execute({ content: 'Decision: enable feature flag rollout for users; proceed.', target: 'personal' });
const result = await save.execute({ content: 'Decision: disable feature flag rollout for users; cancel.', target: 'personal' });
// Flag still appears (detectContradiction doesn't need the signal store)
expect(result).toContain('contradicts_existing');
db2.close();
});
});

View File

@@ -0,0 +1,77 @@
import { describe, it, expect } from 'vitest';
import { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from '../src/secret-patterns.js';
describe('detectSecrets', () => {
it('returns null for clean text', () => {
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
});
it('detects AWS access keys', () => {
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
});
it('detects GitHub PATs', () => {
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
});
it('detects OpenAI keys', () => {
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
});
it('detects Anthropic keys', () => {
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
});
it('detects Google API keys', () => {
// Google API keys = literal "AIza" + exactly 35 chars
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
});
it('detects Stripe keys', () => {
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
});
it('detects private key blocks', () => {
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
});
it('detects JWT tokens', () => {
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
expect(detectSecrets(jwt)).toBe('jwt');
});
it('detects basic-auth URLs', () => {
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
.toBe('basic-auth-url');
});
it('detects env-var password assignments', () => {
expect(detectSecrets('PASSWORD=supersecret123'))
.toBe('env-password');
});
it('detects postgres connection strings with creds', () => {
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
.toBe('pgsql-url');
});
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
});
});
describe('redactSecrets', () => {
it('replaces every match and reports the patterns that fired', () => {
const { text, found } = redactSecrets('key AKIAIOSFODNN7EXAMPLE and PASSWORD=supersecret123');
expect(text).toContain('[REDACTED:aws-access-key]');
expect(text).toContain('[REDACTED:env-password]');
expect(text).not.toContain('AKIAIOSFODNN7EXAMPLE');
expect(found.sort()).toEqual(['aws-access-key', 'env-password']);
});
it('leaves clean text untouched', () => {
expect(redactSecrets('nothing to hide')).toEqual({ text: 'nothing to hide', found: [] });
});
});

View File

@@ -90,59 +90,4 @@ describe('Self-Awareness', () => {
expect(result).toContain('acquire_capability');
expect(result).toContain('uncertain');
});
it('includes improvement signals when awareness has actionable data', () => {
const caps: AgentCapabilities = {
tools: [],
skills: [],
model: 'test',
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
mode: 'local',
version: '1.0.0',
awareness: {
capabilityGaps: [{
id: 1,
toolName: 'pdf_reader',
occurrences: 3,
suggestion: 'User has needed "pdf_reader" 3 times. Consider using acquire_capability to find it.',
}],
corrections: [{
id: 2,
patternKey: 'tone:too_formal',
detail: 'Keep it casual',
occurrences: 4,
guidance: 'Keep it casual (corrected 4 times)',
}],
workflowPatterns: [],
totalActionable: 2,
},
};
const result = buildSelfAwareness(caps);
expect(result).toContain('Learning from Past Sessions');
expect(result).toContain('pdf_reader');
expect(result).toContain('Keep it casual');
expect(result).toContain('Recurring capability gaps');
expect(result).toContain('User corrections to apply');
});
it('omits improvement section when awareness has zero actionable signals', () => {
const caps: AgentCapabilities = {
tools: [],
skills: [],
model: 'test',
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
mode: 'local',
version: '1.0.0',
awareness: {
capabilityGaps: [],
corrections: [],
workflowPatterns: [],
totalActionable: 0,
},
};
const result = buildSelfAwareness(caps);
expect(result).not.toContain('Learning from Past Sessions');
});
});

View File

@@ -9,7 +9,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import {
autoExtractAndCreateSkill,
skillFilename,
@@ -35,24 +34,18 @@ function repeatedSession(): AutoExtractMessage[] {
describe('autoExtractAndCreateSkill — gap A', () => {
let tmpDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-autoextract-'));
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('creates a personal-scope SKILL.md from a repeated workflow', async () => {
const result = await autoExtractAndCreateSkill(repeatedSession(), {
personalSkillsDir: tmpDir,
improvementSignals: signals,
});
expect(result.filePath).not.toBeNull();
@@ -67,19 +60,6 @@ describe('autoExtractAndCreateSkill — gap A', () => {
expect(body).toContain('## Tools Used');
});
it('records a workflow_pattern signal with auto-extract pattern_key', async () => {
const result = await autoExtractAndCreateSkill(repeatedSession(), {
personalSkillsDir: tmpDir,
improvementSignals: signals,
});
expect(result.filePath).not.toBeNull();
const recorded = signals.getByCategory('workflow_pattern');
const autoExtract = recorded.find(s => s.pattern_key.startsWith('auto-extract:'));
expect(autoExtract).toBeDefined();
expect(autoExtract!.detail).toContain('Auto-extracted');
});
it('returns no_pattern for short or random sessions', async () => {
const messages: AutoExtractMessage[] = [
{ role: 'user', content: 'Hi' },

View File

@@ -5,7 +5,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import { retireStaleSkills } from '../src/skill-retirement.js';
import {
recordSkillUsage,
@@ -74,8 +73,6 @@ describe('skill-usage — gap F', () => {
describe('retireStaleSkills — gap F', () => {
let waggleHome: string;
let skillsDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
const NOW = new Date('2026-04-15T00:00:00Z');
const DAY = 24 * 60 * 60 * 1000;
@@ -83,12 +80,9 @@ describe('retireStaleSkills — gap F', () => {
beforeEach(() => {
waggleHome = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-retire-'));
skillsDir = path.join(waggleHome, 'skills');
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(waggleHome, {
recursive: true,
force: true,
@@ -110,7 +104,6 @@ describe('retireStaleSkills — gap F', () => {
const report = retireStaleSkills(waggleHome, {
maxIdleDays: 90,
now: () => NOW,
improvementSignals: signals,
});
expect(report.scanned).toBe(2);
@@ -123,10 +116,6 @@ describe('retireStaleSkills — gap F', () => {
// Usage entry for the retired skill is cleared
expect(loadSkillUsage(waggleHome).stale).toBeUndefined();
expect(loadSkillUsage(waggleHome).fresh).toBeDefined();
// Signal recorded
const recorded = signals.getByCategory('workflow_pattern').find(s => s.pattern_key === 'retire:stale');
expect(recorded).toBeDefined();
});
it('falls back to file mtime when no usage entry exists', () => {