refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC, deployBehavioralSpecOverride } from '@waggle/agent';
|
||||
import { BEHAVIORAL_SPEC } from '@waggle/agent';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
@@ -21,7 +21,7 @@ describe('activeBehavioralSpec integration', () => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
it('loads the compiled baseline when no overrides exist on disk', async () => {
|
||||
it('serves the compiled behavioral spec baseline', async () => {
|
||||
const server = await buildLocalServer({ dataDir: tmpDir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec).toBeDefined();
|
||||
@@ -31,104 +31,4 @@ describe('activeBehavioralSpec integration', () => {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('applies a pre-existing override at boot', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec2-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
// Write an override BEFORE the server boots.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'EVOLVED core loop — test fixture',
|
||||
});
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('EVOLVED core loop — test fixture');
|
||||
// Unaffected sections pass through
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe(BEHAVIORAL_SPEC.qualityRules);
|
||||
// rules reflects the override
|
||||
expect(server.activeBehavioralSpec.rules).toContain('EVOLVED core loop — test fixture');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('re-derives activeBehavioralSpec on behavioral-spec:reloaded event', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec3-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Sanity: no override yet
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe(BEHAVIORAL_SPEC.coreLoop);
|
||||
|
||||
// Write an override and fire the reload event.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'HOT-RELOADED core loop',
|
||||
});
|
||||
server.eventBus.emit('behavioral-spec:reloaded', { section: 'coreLoop' });
|
||||
|
||||
// Give the sync listener a tick to update (it's synchronous actually,
|
||||
// but await-nothing keeps us defensive).
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('HOT-RELOADED core loop');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('accepted evolution run causes activeBehavioralSpec to pick up the override', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec4-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Propose a run directly on the store, then accept via HTTP
|
||||
const run = server.evolutionStore.create({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'qualityRules',
|
||||
baselineText: BEHAVIORAL_SPEC.qualityRules,
|
||||
winnerText: 'END-TO-END EVOLVED quality rules',
|
||||
deltaAccuracy: 0.1,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [],
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
headers: { 'content-type': 'application/json', authorization: 'Bearer test' },
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
// Event fires synchronously on emit (EventEmitter behavior); give
|
||||
// the listener a microtask just in case.
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe('END-TO-END EVOLVED quality rules');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,386 +0,0 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
describe('Evolution Routes', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-routes-'));
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
function seedRun(overrides: Partial<Parameters<typeof server.evolutionStore.create>[0]> = {}) {
|
||||
return server.evolutionStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'baseline prompt that is long enough',
|
||||
winnerText: 'evolved prompt that is long enough to pass the gate',
|
||||
deltaAccuracy: 0.08,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [{ gate: 'size', verdict: 'pass', reason: 'within limit' }],
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/evolution/runs ──
|
||||
|
||||
describe('GET /api/evolution/runs', () => {
|
||||
beforeAll(() => server.evolutionStore.clear());
|
||||
|
||||
it('returns an empty list when nothing has run', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toEqual([]);
|
||||
expect(body.count).toBe(0);
|
||||
});
|
||||
|
||||
it('returns seeded runs in reverse-chronological order', async () => {
|
||||
seedRun({ targetName: 'a' });
|
||||
seedRun({ targetName: 'b' });
|
||||
seedRun({ targetName: 'c' });
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.count).toBe(3);
|
||||
expect(body.runs[0].id).toBeGreaterThan(body.runs[body.runs.length - 1].id);
|
||||
});
|
||||
|
||||
it('filters by status', async () => {
|
||||
const a = seedRun({ targetName: 'x' });
|
||||
server.evolutionStore.reject(a.run_uuid, 'test reject');
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?status=rejected',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs.length).toBeGreaterThanOrEqual(1);
|
||||
expect(body.runs.every((r: { status: string }) => r.status === 'rejected')).toBe(true);
|
||||
});
|
||||
|
||||
it('respects the limit query param', async () => {
|
||||
for (let i = 0; i < 5; i++) seedRun({ targetName: `limit-${i}` });
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?limit=2',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/runs/:uuid ──
|
||||
|
||||
describe('GET /api/evolution/runs/:uuid', () => {
|
||||
it('returns a single run with parsed JSON blobs', async () => {
|
||||
const run = seedRun({
|
||||
winnerSchema: { name: 'test', fields: [{ name: 'answer', type: 'string' }] },
|
||||
artifacts: { seed: 42, generations: 2 },
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.run_uuid).toBe(run.run_uuid);
|
||||
expect(body.winnerSchema).toEqual({ name: 'test', fields: [{ name: 'answer', type: 'string' }] });
|
||||
expect(body.artifacts).toEqual({ seed: 42, generations: 2 });
|
||||
expect(body.gateReasons).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs/does-not-exist',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns run even when artifacts are null/malformed', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.winnerSchema).toBeNull();
|
||||
expect(body.artifacts).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/reject ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/reject', () => {
|
||||
it('moves proposed run to rejected with reason', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: { reason: 'not great' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('rejected');
|
||||
expect(body.user_note).toBe('not great');
|
||||
});
|
||||
|
||||
it('rejects 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/runs/ghost/reject',
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('rejects 409 when run is not proposed', async () => {
|
||||
const run = seedRun();
|
||||
server.evolutionStore.reject(run.run_uuid);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/accept ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/accept', () => {
|
||||
it('writes a persona override to disk and moves to deployed', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
winnerText: 'EVOLVED coder system prompt',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: { note: 'looks good' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('deployed');
|
||||
|
||||
const overridePath = path.join(tmpDir, 'personas', 'coder.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder system prompt');
|
||||
});
|
||||
|
||||
it('writes a behavioral-spec override to disk', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'coreLoop',
|
||||
winnerText: 'EVOLVED core loop text',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
const overridePath = path.join(tmpDir, 'behavioral-overrides', 'coreLoop.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.text).toBe('EVOLVED core loop text');
|
||||
expect(loaded.runUuid).toBe(run.run_uuid);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy throws (unsupported target_kind)', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'tool-description',
|
||||
targetName: 'some_tool',
|
||||
winnerText: 'new description',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('failed');
|
||||
expect(body.failure_reason).toMatch(/not yet implemented/);
|
||||
});
|
||||
|
||||
it('emits persona:reloaded event on persona deploy', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'writer',
|
||||
winnerText: 'EVOLVED writer',
|
||||
});
|
||||
|
||||
let emitted: unknown = null;
|
||||
const listener = (payload: unknown) => { emitted = payload; };
|
||||
server.eventBus.on('persona:reloaded', listener);
|
||||
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
|
||||
server.eventBus.off('persona:reloaded', listener);
|
||||
expect(emitted).toEqual({ personaId: 'writer' });
|
||||
});
|
||||
|
||||
it('returns 409 for already-accepted run', async () => {
|
||||
const run = seedRun();
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
const second = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(second.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/status ──
|
||||
|
||||
describe('GET /api/evolution/status', () => {
|
||||
it('returns aggregate counts per status', async () => {
|
||||
server.evolutionStore.clear();
|
||||
const a = seedRun();
|
||||
const b = seedRun();
|
||||
server.evolutionStore.reject(a.run_uuid);
|
||||
server.evolutionStore.accept(b.run_uuid);
|
||||
server.evolutionStore.markDeployed(b.run_uuid);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/status',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.counts.rejected).toBe(1);
|
||||
expect(body.counts.deployed).toBe(1);
|
||||
expect(body.pendingCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/targets ──
|
||||
|
||||
describe('GET /api/evolution/targets', () => {
|
||||
it('returns personas + behavioral-spec sections + a default schema', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(Array.isArray(body.personas)).toBe(true);
|
||||
expect(body.personas.length).toBeGreaterThan(0);
|
||||
expect(body.personas[0]).toHaveProperty('id');
|
||||
expect(body.personas[0]).toHaveProperty('name');
|
||||
// systemPrompt is intentionally omitted from /targets to match /api/personas privacy policy
|
||||
expect(body.personas[0]).not.toHaveProperty('systemPrompt');
|
||||
|
||||
expect(Array.isArray(body.sections)).toBe(true);
|
||||
expect(body.sections).toEqual(expect.arrayContaining([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules', 'workPatterns', 'intelligenceDefaults',
|
||||
]));
|
||||
|
||||
expect(body.defaultSchema).toMatchObject({
|
||||
name: expect.any(String),
|
||||
version: expect.any(Number),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
expect(body.defaultSchema.fields.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/baseline ──
|
||||
|
||||
describe('GET /api/evolution/baseline', () => {
|
||||
it('returns 400 when kind or name is missing', async () => {
|
||||
const a = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline',
|
||||
});
|
||||
expect(a.statusCode).toBe(400);
|
||||
|
||||
const b = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt',
|
||||
});
|
||||
expect(b.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('returns the persona systemPrompt for persona-system-prompt', async () => {
|
||||
// Pick whatever persona the registry lists first — avoids hardcoding
|
||||
// an id that might be renamed.
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
const { personas } = JSON.parse(listRes.body);
|
||||
const personaId = personas[0].id;
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/baseline?kind=persona-system-prompt&name=${encodeURIComponent(personaId)}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
expect(body.schemaBaseline).toMatchObject({
|
||||
name: expect.any(String),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
});
|
||||
|
||||
it('returns 404 for unknown persona', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt&name=ghost-persona',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns the active section text for behavioral-spec-section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=coreLoop',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
// coreLoop has identifiable content
|
||||
expect(body.baseline).toMatch(/HOW YOU THINK|Core Loop|Step 1/i);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=nopeSection',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 400 for unknown kind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=tool-description&name=x',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,397 +0,0 @@
|
||||
/**
|
||||
* Integration tests for POST /api/evolution/run.
|
||||
*
|
||||
* The endpoint needs an Anthropic key and an LLM. We stub the LLM by
|
||||
* installing a global factory override (`__waggleEvolutionLlmFactory`) so
|
||||
* every call goes through an in-memory responder — no network, no @ax-llm/ax.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, VaultStore } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { EvolutionLLM } from '@waggle/agent';
|
||||
|
||||
// ── Global LLM override hook ─────────────────────────────────────
|
||||
|
||||
type LLMFactory = (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
||||
|
||||
function installLLMFactory(factory: LLMFactory): void {
|
||||
(globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory = factory;
|
||||
}
|
||||
|
||||
function clearLLMFactory(): void {
|
||||
delete (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stub LLM that returns a canned answer based on a pattern match on the
|
||||
* prompt. Falls back to echoing the input so the judge still produces a
|
||||
* score. Every call increments a counter so tests can assert the LLM was
|
||||
* actually invoked.
|
||||
*/
|
||||
function makeStubLLM(responses: Array<{ match: RegExp; reply: string }>): {
|
||||
llm: EvolutionLLM;
|
||||
callCount: () => number;
|
||||
calls: string[];
|
||||
} {
|
||||
const calls: string[] = [];
|
||||
return {
|
||||
callCount: () => calls.length,
|
||||
calls,
|
||||
llm: {
|
||||
async complete(prompt: string): Promise<string> {
|
||||
calls.push(prompt);
|
||||
for (const r of responses) {
|
||||
if (r.match.test(prompt)) return r.reply;
|
||||
}
|
||||
// Default response that parses as a valid judge verdict.
|
||||
return '{"correctness":8,"procedure":8,"conciseness":7,"feedback":"ok"}';
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Test scaffolding ─────────────────────────────────────────────
|
||||
|
||||
describe('POST /api/evolution/run', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-run-'));
|
||||
|
||||
// Fresh mind database.
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
// Seed the vault with an Anthropic key before the server boots — the
|
||||
// server loads the vault from disk on startup.
|
||||
const vault = new VaultStore(tmpDir);
|
||||
vault.set('anthropic', 'sk-ant-stub-for-tests', { models: ['claude-haiku-4-5-20251001'] });
|
||||
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
clearLLMFactory();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(() => {
|
||||
server.evolutionStore.clear();
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
/** Seed a handful of finalized traces so the orchestrator's dataset miner has material. */
|
||||
function seedTraces(count = 8): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
const id = server.traceStore.start({
|
||||
input: `What is the capital of country ${i}?`,
|
||||
personaId: 'coder',
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
server.traceStore.finalize(id, {
|
||||
outcome: i % 3 === 0 ? 'verified' : 'success',
|
||||
output: `The capital is City${i}.`,
|
||||
tokens: { input: 40, output: 20 },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const baseBody = {
|
||||
targetKind: 'behavioral-spec-section' as const,
|
||||
targetName: 'coreLoop',
|
||||
baseline: 'You are a careful assistant. Follow instructions.',
|
||||
schemaBaseline: {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'answer', type: 'string', description: 'the answer', required: true, constraints: [] },
|
||||
],
|
||||
},
|
||||
// Keep populations tiny so the test stays fast even though we use a stub LLM.
|
||||
schema: { populationSize: 1, generations: 1, evalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
gepa: { populationSize: 1, generations: 1, miniEvalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
};
|
||||
|
||||
// ── 422 when no API key ────────────────────────────────────────
|
||||
|
||||
it('returns 422 when no Anthropic key is in the vault', async () => {
|
||||
// Temporarily hide the key for just this request.
|
||||
const saved = server.vault?.get('anthropic');
|
||||
server.vault?.delete('anthropic');
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(422);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toMatch(/Anthropic API key/i);
|
||||
} finally {
|
||||
if (saved) {
|
||||
server.vault?.set('anthropic', saved.value, saved.metadata);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ── 400 on validation errors ───────────────────────────────────
|
||||
|
||||
describe('validation', () => {
|
||||
it('rejects missing body', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects invalid targetKind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetKind: 'nope' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetKind/);
|
||||
});
|
||||
|
||||
it('rejects empty targetName', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetName: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetName/);
|
||||
});
|
||||
|
||||
it('rejects empty baseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, baseline: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/baseline/);
|
||||
});
|
||||
|
||||
it('rejects malformed schemaBaseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, schemaBaseline: { name: 'x' } },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/schemaBaseline/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 503 when LLM factory fails ─────────────────────────────────
|
||||
|
||||
it('returns 503 when the LLM factory returns null', async () => {
|
||||
installLLMFactory(() => null as unknown as EvolutionLLM);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(JSON.parse(res.body).error).toMatch(/ax-llm/);
|
||||
});
|
||||
|
||||
// ── Skip path: no traces ───────────────────────────────────────
|
||||
|
||||
it('returns outcome=skipped-trigger when the trace store has no eligible traces', async () => {
|
||||
const { llm } = makeStubLLM([]);
|
||||
installLLMFactory(() => llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.outcome).toBe('skipped-trigger');
|
||||
expect(body.reason).toMatch(/eligible traces/);
|
||||
});
|
||||
|
||||
// ── Happy path: runs + persists ────────────────────────────────
|
||||
|
||||
it('runs the full orchestrator when traces exist and returns a proposal summary', async () => {
|
||||
seedTraces(8);
|
||||
|
||||
const stub = makeStubLLM([
|
||||
// Mutation requests get a slightly different prompt text
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'You are an extremely careful assistant. Follow all instructions step by step.' },
|
||||
// Schema fill requests get valid JSON
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
// Running-judge execution requests get a short answer
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
// Any terminal outcome is acceptable — the point is that the orchestrator
|
||||
// ran end-to-end, talked to our stub LLM, and wrote something to the store.
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(body.outcome);
|
||||
expect(body.composeSummary).toBeTruthy();
|
||||
// Every run (including skipped-delta) returns a composeSummary; only
|
||||
// 'proposed' and 'skipped-gates' attach a persisted run.
|
||||
expect(stub.callCount()).toBeGreaterThan(0);
|
||||
|
||||
// If a run was persisted, check it shows up in the list endpoint.
|
||||
if (body.run) {
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
const listed = JSON.parse(listRes.body);
|
||||
expect(listed.count).toBeGreaterThan(0);
|
||||
const matching = listed.runs.find((r: { run_uuid: string }) => r.run_uuid === body.run.run_uuid);
|
||||
expect(matching).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
// ── SSE streaming path ─────────────────────────────────────────
|
||||
|
||||
describe('SSE streaming (Accept: text/event-stream)', () => {
|
||||
/** Tiny SSE parser — takes the raw event-stream body and emits parsed events. */
|
||||
function parseSseEvents(raw: string): Array<{ event: string; data: unknown }> {
|
||||
const out: Array<{ event: string; data: unknown }> = [];
|
||||
for (const block of raw.split('\n\n')) {
|
||||
if (!block.trim()) continue;
|
||||
let event = 'message';
|
||||
const dataLines: string[] = [];
|
||||
for (const line of block.split('\n')) {
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||
else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
|
||||
}
|
||||
if (dataLines.length > 0) {
|
||||
try {
|
||||
out.push({ event, data: JSON.parse(dataLines.join('\n')) });
|
||||
} catch {
|
||||
out.push({ event, data: dataLines.join('\n') });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
it('emits open → progress* → done sequence for a successful run', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'A careful assistant.' },
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/text\/event-stream/);
|
||||
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('done');
|
||||
|
||||
// Terminal event carries the same payload shape as the JSON path.
|
||||
const done = events.find(e => e.event === 'done');
|
||||
expect(done).toBeDefined();
|
||||
const donePayload = done!.data as { outcome: string; composeSummary: unknown };
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(donePayload.outcome);
|
||||
expect(donePayload.composeSummary).toBeTruthy();
|
||||
|
||||
// There should be at least one progress event reporting an orchestrator phase.
|
||||
const progresses = events.filter(e => e.event === 'progress');
|
||||
expect(progresses.length).toBeGreaterThan(0);
|
||||
const phases = progresses.map(p => (p.data as { phase: string }).phase);
|
||||
// Orchestrator emits at least 'dataset' before skipping or compose.
|
||||
expect(phases.some(p => ['dataset', 'compose', 'gates', 'persist', 'skipped', 'done'].includes(p))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
it('emits open → error when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
// Force a real error path like the JSON test does.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200); // SSE streams return 200 even on internal errors; the error is an SSE event
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('error');
|
||||
const errEvent = events.find(e => e.event === 'error');
|
||||
expect((errEvent!.data as { error: string }).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to JSON when Accept header is absent', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
// No accept header — should return JSON.
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/application\/json/);
|
||||
expect(() => JSON.parse(res.body)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Orchestrator error path ────────────────────────────────────
|
||||
|
||||
it('returns 500 when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
|
||||
// Judge swallows its own LLM errors with a zero score, so we need to
|
||||
// make the mutate + execute paths noisy enough that *something* escapes.
|
||||
// Easiest: swap out the trace store to blow up during buildExamples.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(JSON.parse(res.body).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -40,7 +40,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let includeViewerWorkspace: boolean;
|
||||
let literalGlobalIsViewer: boolean;
|
||||
let viewerOptimizationEnabled: boolean;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = new MindDB(':memory:');
|
||||
@@ -49,7 +48,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
failNext = false;
|
||||
includeViewerWorkspace = false;
|
||||
literalGlobalIsViewer = false;
|
||||
viewerOptimizationEnabled = 1 as unknown as boolean;
|
||||
scheduler = new LocalScheduler(
|
||||
cronStore,
|
||||
async (schedule) => {
|
||||
@@ -79,7 +77,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
};
|
||||
}
|
||||
if (id === 'global' && literalGlobalIsViewer) {
|
||||
@@ -96,7 +93,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
}]
|
||||
: []),
|
||||
],
|
||||
@@ -460,13 +456,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/cron/${id}/trigger` }),
|
||||
},
|
||||
{
|
||||
label: 'prompt optimization through automation run',
|
||||
jobType: 'prompt_optimization' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/automations/${id}/run` }),
|
||||
},
|
||||
{
|
||||
label: 'memory compaction through cron trigger',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
@@ -506,25 +495,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
expect(executed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('prompt optimization ignores a listed viewer workspace without optimization opt-in', async () => {
|
||||
includeViewerWorkspace = true;
|
||||
viewerOptimizationEnabled = false;
|
||||
const row = cronStore.create({
|
||||
name: 'Prompt optimization',
|
||||
cronExpr: '0 2 * * *',
|
||||
jobType: 'prompt_optimization',
|
||||
enabled: false,
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/cron/${row.id}/trigger`,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(executed).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: 'cron',
|
||||
@@ -625,12 +595,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
jobConfig: { action: 'morning_briefing' },
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'personal monthly assessment',
|
||||
jobType: 'monthly_assessment' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'non-matching spaced memory action',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
|
||||
@@ -377,7 +377,6 @@ describe('classifyExplicitTurnMutationPolicy', () => {
|
||||
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity',
|
||||
'get_awareness', 'read_other_workspace', 'save_memory', 'add_task',
|
||||
'correct_knowledge', 'read_file', 'web_search', 'mcp_external_read',
|
||||
'agent_insights',
|
||||
].map(name => ({ name }));
|
||||
const filtered = filterToolsByTurnMutationPolicy(
|
||||
tools,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Tests the two feedback endpoints:
|
||||
* POST /api/feedback — record user feedback
|
||||
* GET /api/feedback/stats — get improvement stats
|
||||
* GET /api/feedback/stats — get feedback stats
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
@@ -11,35 +11,27 @@ import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import Fastify from 'fastify';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { feedbackRoutes } from '../../src/local/routes/feedback.js';
|
||||
|
||||
/** Minimal multiMind mock that provides a real MindDB for the personal mind. */
|
||||
function createTestServer(db: MindDB) {
|
||||
const server = Fastify({ logger: false });
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
server.decorate('multiMind', {
|
||||
personal: db,
|
||||
});
|
||||
server.decorate('agentState', {
|
||||
orchestrator: {
|
||||
getImprovementSignals: () => signalStore,
|
||||
},
|
||||
});
|
||||
server.register(feedbackRoutes);
|
||||
return { server, signalStore };
|
||||
return { server };
|
||||
}
|
||||
|
||||
describe('Feedback Routes', () => {
|
||||
let db: MindDB;
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let signalStore: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
const result = createTestServer(db);
|
||||
server = result.server;
|
||||
signalStore = result.signalStore;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -80,26 +72,6 @@ describe('Feedback Routes', () => {
|
||||
expect(res.json()).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('records negative feedback as correction signal', async () => {
|
||||
await server.inject({
|
||||
method: 'POST',
|
||||
url: '/api/feedback',
|
||||
payload: {
|
||||
sessionId: 'sess-1',
|
||||
messageIndex: 2,
|
||||
rating: 'down',
|
||||
reason: 'wrong_tool',
|
||||
detail: 'Should have used web_search',
|
||||
},
|
||||
});
|
||||
|
||||
// Check that it was recorded in the improvement signals store
|
||||
const corrections = signalStore.getByCategory('correction');
|
||||
const feedbackSignal = corrections.find(c => c.pattern_key === 'feedback:wrong_tool');
|
||||
expect(feedbackSignal).toBeDefined();
|
||||
expect(feedbackSignal!.count).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects missing sessionId', async () => {
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
@@ -170,7 +142,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(0);
|
||||
expect(body.positiveRate).toBe(0);
|
||||
expect(body.topIssues).toEqual([]);
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
@@ -202,7 +173,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(3);
|
||||
expect(body.positiveRate).toBeCloseTo(0.67, 1);
|
||||
expect(body.topIssues).toContain('too_verbose');
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
/**
|
||||
* GEPA Prompt Optimization Cron Handler Tests
|
||||
*
|
||||
* Tests the prompt_optimization cron handler logic:
|
||||
* - Calls LLM when correction rate > 20% or avg turns > 15
|
||||
* - Stores generated variant in optimization_log
|
||||
* - Skips when budget is exceeded
|
||||
* - Skips when not enough data
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import { isWithinBudget, getRecentLogs } from '@waggle/agent';
|
||||
|
||||
describe('GEPA Prompt Optimization', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let optStore: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-gepa-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
optStore = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('Signal detection', () => {
|
||||
it('detects high correction rate as optimization signal', () => {
|
||||
// Insert logs with high correction rate (>20%)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: i < 5, // 50% correction rate
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeGreaterThan(0.2);
|
||||
expect(stats.total).toBe(10);
|
||||
});
|
||||
|
||||
it('detects high avg turn count as optimization signal', () => {
|
||||
// Insert logs with high turn count (>15)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 20, // >15 threshold
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.avgTurnCount).toBeGreaterThan(15);
|
||||
});
|
||||
|
||||
it('does not trigger when both metrics are within thresholds', () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeLessThanOrEqual(0.2);
|
||||
expect(stats.avgTurnCount).toBeLessThanOrEqual(15);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Budget check', () => {
|
||||
it('reports within budget when token costs are low', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
});
|
||||
}
|
||||
|
||||
// Budget is 100 cents ($1). 500 input + 250 output tokens is far under budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('reports over budget when token costs exceed limit', () => {
|
||||
// Insert a log with massive token counts to exceed budget
|
||||
optStore.insert({
|
||||
sessionId: 'sess-big',
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 10_000_000, // 10M input tokens at $3/M = $30
|
||||
outputTokens: 1_000_000, // 1M output tokens at $15/M = $15
|
||||
});
|
||||
|
||||
// Budget is 100 cents ($1). Total cost is ~$45 = 4500 cents. Way over budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Variant storage', () => {
|
||||
it('stores generated variant in optimization_log', () => {
|
||||
// Simulate what the cron handler does after generating a variant
|
||||
const variantText = 'Improved system prompt with better instructions for reducing corrections...'.repeat(5);
|
||||
|
||||
optStore.insert({
|
||||
sessionId: `gepa-variant-${Date.now()}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: variantText,
|
||||
toolsUsed: ['gepa_variant'],
|
||||
turnCount: 0,
|
||||
wasCorrection: false,
|
||||
inputTokens: 500,
|
||||
outputTokens: variantText.length,
|
||||
});
|
||||
|
||||
const recent = getRecentLogs(optStore, 10);
|
||||
const variant = recent.find(l => JSON.parse(l.tools_used).includes('gepa_variant'));
|
||||
expect(variant).toBeDefined();
|
||||
expect(variant!.system_prompt).toBe(variantText);
|
||||
expect(variant!.turn_count).toBe(0);
|
||||
expect(variant!.was_correction).toBe(0);
|
||||
});
|
||||
|
||||
it('retrieves recent logs for analysis', () => {
|
||||
for (let i = 0; i < 8; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: `Prompt v${i}`,
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: i + 1,
|
||||
wasCorrection: i % 3 === 0,
|
||||
inputTokens: 100 * (i + 1),
|
||||
outputTokens: 50 * (i + 1),
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(optStore, 5);
|
||||
expect(logs).toHaveLength(5);
|
||||
// Most recent first
|
||||
expect(logs[0].session_id).toBe('sess-7');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Minimum data requirement', () => {
|
||||
it('skips optimization when fewer than 5 logs', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 20,
|
||||
wasCorrection: true,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const recentLogs = getRecentLogs(optStore, 100);
|
||||
// The cron handler checks: if (recentLogs.length < 5) continue;
|
||||
expect(recentLogs.length).toBeLessThan(5);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,287 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import {
|
||||
IdleSessionWatcher,
|
||||
countTurns,
|
||||
readRecentTranscript,
|
||||
buildReviewInstruction,
|
||||
NOTHING_TO_DO,
|
||||
DEFAULT_CONFIG,
|
||||
type SelfEvolutionConfig,
|
||||
type ReviewTurnResult,
|
||||
} from '../../src/local/idle-watcher.js';
|
||||
import { getNotificationGate } from '../../src/local/notification-gate.js';
|
||||
|
||||
// A fixed synthetic "now" so idle math is deterministic.
|
||||
const NOW = 2_000_000_000_000;
|
||||
const MIN = 60_000;
|
||||
|
||||
describe('idle-watcher', () => {
|
||||
let dataDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-idle-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ── Fixtures ─────────────────────────────────────────────────────────
|
||||
function writeConfig(cfg: Partial<SelfEvolutionConfig> | string): void {
|
||||
const body = typeof cfg === 'string' ? cfg : JSON.stringify(cfg);
|
||||
fs.writeFileSync(path.join(dataDir, 'self-evolution.json'), body);
|
||||
}
|
||||
|
||||
/** Create a session .jsonl with `turns` message lines and a given mtime (ms). */
|
||||
function writeSession(workspaceId: string, sessionId: string, turns: number, mtimeMs: number): void {
|
||||
const dir = path.join(dataDir, 'workspaces', workspaceId, 'sessions');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const lines = [JSON.stringify({ type: 'meta', title: sessionId })];
|
||||
for (let i = 0; i < turns; i++) {
|
||||
lines.push(JSON.stringify({ role: i % 2 === 0 ? 'user' : 'assistant', content: `msg ${i}` }));
|
||||
}
|
||||
const filePath = path.join(dir, `${sessionId}.jsonl`);
|
||||
fs.writeFileSync(filePath, lines.join('\n') + '\n');
|
||||
fs.utimesSync(filePath, new Date(mtimeMs), new Date(mtimeMs));
|
||||
}
|
||||
|
||||
function makeWatcher(over?: {
|
||||
runReviewTurn?: (i: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
|
||||
}) {
|
||||
const reviewed: Array<{ sessionId: string; workspaceId: string }> = [];
|
||||
const emitCalls: Array<{ suppressed: boolean; dedupeKey?: string }> = [];
|
||||
const runReviewTurn =
|
||||
over?.runReviewTurn ??
|
||||
(async (i: { sessionId: string; workspaceId: string }) => {
|
||||
reviewed.push(i);
|
||||
return { content: NOTHING_TO_DO };
|
||||
});
|
||||
const runSpy = vi.fn(runReviewTurn);
|
||||
const watcher = new IdleSessionWatcher({
|
||||
dataDir,
|
||||
now: () => NOW,
|
||||
runReviewTurn: runSpy,
|
||||
// Mirror production emitNotification's gate so suppression is genuinely tested.
|
||||
emitNotification: (_event, options) => {
|
||||
let suppressed = false;
|
||||
if (options?.dedupeKey && options.materialHash) {
|
||||
suppressed = !getNotificationGate(dataDir).shouldNotify(options.dedupeKey, options.materialHash);
|
||||
}
|
||||
emitCalls.push({ suppressed, dedupeKey: options?.dedupeKey });
|
||||
return { suppressed };
|
||||
},
|
||||
log: { info: () => {}, warn: () => {} },
|
||||
});
|
||||
return { watcher, reviewed, emitCalls, runSpy };
|
||||
}
|
||||
|
||||
// ── Fire-condition matrix ────────────────────────────────────────────
|
||||
describe('fire condition', () => {
|
||||
beforeEach(() => writeConfig({ enabled: true })); // rest of fields default
|
||||
|
||||
it('reviews an idle session with enough turns', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-a', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
it('does NOT review a still-active session (mtime too recent)', async () => {
|
||||
writeSession('w1', 'sess-active', 10, NOW - 5 * MIN); // < 15 min idle
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT review a session below the turn threshold', async () => {
|
||||
writeSession('w1', 'sess-short', 3, NOW - 30 * MIN); // idle but only 3 turns
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Fired-set: no re-fire until mtime advances ───────────────────────
|
||||
it('does not re-review the same session until its file advances', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(await watcher.tick()).toBe(0); // same mtime — already fired
|
||||
expect(runSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// File advances (a new message lands) → new mtime key → re-fires once.
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Disabled / config handling ───────────────────────────────────────
|
||||
it('is a no-op when disabled (default)', async () => {
|
||||
writeConfig({ enabled: false });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is a no-op when the config file is absent (defaults ⇒ disabled)', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to defaults on a corrupt config (⇒ disabled)', async () => {
|
||||
writeConfig('{ this is not json');
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
expect(DEFAULT_CONFIG.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('applies partial config over defaults (enabled + custom idle threshold)', async () => {
|
||||
writeConfig({ enabled: true, idleMinutes: 60 }); // minTurns/cap default
|
||||
writeSession('w1', 'fresh', 6, NOW - 30 * MIN); // idle 30m < 60m ⇒ skipped
|
||||
writeSession('w1', 'old', 6, NOW - 90 * MIN); // idle 90m ⇒ reviewed
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'old', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Daily cap ────────────────────────────────────────────────────────
|
||||
it('enforces maxReviewsPerDay', async () => {
|
||||
writeConfig({ enabled: true, maxReviewsPerDay: 2 });
|
||||
writeSession('w1', 's1', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 's2', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 's3', 6, NOW - 22 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(2); // capped at 2 even though 3 are eligible
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
// A later tick in the same day stays capped (the 3rd never runs).
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Enumeration skips channel-/evolve- prefixes ──────────────────────
|
||||
it('skips channel-* and evolve-* sessions', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'channel-telegram-123', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'evolve-sess-a', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'sess-real', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-real', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Notification gating ──────────────────────────────────────────────
|
||||
it('does NOT notify on a NOTHING_TO_DO / empty / error result', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'ntd', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'empty', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 'err', 6, NOW - 22 * MIN);
|
||||
const results: Record<string, ReviewTurnResult> = {
|
||||
ntd: { content: NOTHING_TO_DO },
|
||||
empty: { content: ' ' },
|
||||
err: { content: 'partial', error: 'boom' },
|
||||
};
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async ({ sessionId }) => results[sessionId],
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('notifies once on a material finding and suppresses an identical re-finding', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async () => ({ content: 'Finding: the assistant promised a report and never produced it.' }),
|
||||
});
|
||||
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(1);
|
||||
expect(emitCalls[0].suppressed).toBe(false);
|
||||
expect(emitCalls[0].dedupeKey).toBe('self-evolution:sess-a');
|
||||
|
||||
// Session advances, same finding recurs → notification suppressed (anti-nag).
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(2);
|
||||
expect(emitCalls[1].suppressed).toBe(true);
|
||||
});
|
||||
|
||||
// ── Single-flight guard ──────────────────────────────────────────────
|
||||
it('tick is single-flight (a re-entrant tick returns 0)', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
let secondResult = -1;
|
||||
const { watcher } = makeWatcher({
|
||||
runReviewTurn: async () => {
|
||||
// Re-enter while the first tick is still in-flight.
|
||||
secondResult = await watcher.tick();
|
||||
return { content: NOTHING_TO_DO };
|
||||
},
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(secondResult).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────────────
|
||||
describe('idle-watcher pure helpers', () => {
|
||||
it('countTurns excludes the meta line', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
});
|
||||
|
||||
it('countTurns counts every non-empty line when there is no meta', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
expect(countTurns('')).toBe(0);
|
||||
});
|
||||
|
||||
it('readRecentTranscript formats messages and skips meta; null when unreadable', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-tr-'));
|
||||
try {
|
||||
const sessionsDir = path.join(dir, 'workspaces', 'w1', 'sessions');
|
||||
fs.mkdirSync(sessionsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(sessionsDir, 'sess-a.jsonl'),
|
||||
[
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'build me a report' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'on it' }),
|
||||
].join('\n'),
|
||||
);
|
||||
const t = readRecentTranscript(dir, 'w1', 'sess-a');
|
||||
expect(t).toContain('USER: build me a report');
|
||||
expect(t).toContain('ASSISTANT: on it');
|
||||
expect(t).not.toContain('meta');
|
||||
expect(readRecentTranscript(dir, 'w1', 'missing')).toBeNull();
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('buildReviewInstruction embeds the transcript and the NOTHING_TO_DO contract', () => {
|
||||
const msg = buildReviewInstruction('sess-a', 'USER: hi');
|
||||
expect(msg).toContain('sess-a');
|
||||
expect(msg).toContain('USER: hi');
|
||||
expect(msg).toContain(NOTHING_TO_DO);
|
||||
expect(msg).toContain('create_skill');
|
||||
});
|
||||
});
|
||||
@@ -8,8 +8,8 @@ import { buildLocalServer } from '../../src/local/index.js';
|
||||
/**
|
||||
* Integration test for the pre:memory-write lint handler registered in chat.ts.
|
||||
* Fires the real hook through the server's shared HookRegistry and asserts that
|
||||
* capability-failure symptoms are cancelled (and routed to the improvement-signal
|
||||
* path) while normal memories and dramatic-but-legitimate claims pass through.
|
||||
* capability-failure symptoms are cancelled while normal memories and
|
||||
* dramatic-but-legitimate claims pass through.
|
||||
*/
|
||||
describe('pre:memory-write capability-symptom lint (handler integration)', () => {
|
||||
let server: FastifyInstance;
|
||||
@@ -30,10 +30,8 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels a capability-failure symptom write and records an improvement signal', async () => {
|
||||
const { hookRegistry, orchestrator } = server.agentState;
|
||||
const store = orchestrator.getImprovementSignals();
|
||||
const before = store.getByCategory('capability_gap').length;
|
||||
it('cancels a capability-failure symptom write', async () => {
|
||||
const { hookRegistry } = server.agentState;
|
||||
|
||||
const result = await hookRegistry.fire('pre:memory-write', {
|
||||
toolName: 'save_memory',
|
||||
@@ -43,9 +41,6 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
expect(result.cancelled).toBe(true);
|
||||
expect(result.reason).toMatch(/capability/i);
|
||||
expect(result.reason).toMatch(/acquire_capability/);
|
||||
|
||||
const after = store.getByCategory('capability_gap');
|
||||
expect(after.length).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('allows a normal memory (tool preference) to pass', async () => {
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
/**
|
||||
* Monthly Self-Assessment Tests
|
||||
*
|
||||
* Verifies:
|
||||
* - Assessment shape and all required fields
|
||||
* - Cron registration (monthly_assessment job type is valid)
|
||||
* - Assessment saves to personal mind as I-frame
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
MindDB,
|
||||
CronStore,
|
||||
FrameStore,
|
||||
OptimizationLogStore,
|
||||
ImprovementSignalStore,
|
||||
InstallAuditStore,
|
||||
} from '@waggle/core';
|
||||
import { generateMonthlyAssessment, saveAssessmentToMind, type MonthlyAssessment } from '../../src/local/monthly-assessment.js';
|
||||
|
||||
describe('Monthly Self-Assessment', () => {
|
||||
let db: MindDB;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── Assessment shape ────────────────────────────────────────────────
|
||||
|
||||
describe('generateMonthlyAssessment', () => {
|
||||
it('returns a valid MonthlyAssessment shape', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment).toBeDefined();
|
||||
expect(assessment.period).toBe('2026-03');
|
||||
expect(typeof assessment.totalInteractions).toBe('number');
|
||||
expect(typeof assessment.correctionRate).toBe('number');
|
||||
expect(typeof assessment.improvementTrend).toBe('string');
|
||||
expect(Array.isArray(assessment.topStrengths)).toBe(true);
|
||||
expect(Array.isArray(assessment.topWeaknesses)).toBe(true);
|
||||
expect(Array.isArray(assessment.capabilityGapsDetected)).toBe(true);
|
||||
expect(typeof assessment.skillsInstalled).toBe('number');
|
||||
expect(typeof assessment.recommendation).toBe('string');
|
||||
expect(assessment.recommendation.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('defaults to previous month when no period override', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db);
|
||||
|
||||
// Period should be YYYY-MM format for the previous month
|
||||
expect(assessment.period).toMatch(/^\d{4}-\d{2}$/);
|
||||
});
|
||||
|
||||
it('computes correction rate from optimization logs', () => {
|
||||
// Insert some optimization log entries for March 2026
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
optStore.insert({
|
||||
sessionId: 's1',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['read_file'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
});
|
||||
optStore.insert({
|
||||
sessionId: 's2',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['write_file'],
|
||||
turnCount: 3,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
// Use current month since the entries use datetime('now')
|
||||
const now = new Date();
|
||||
const currentPeriod = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
||||
const assessment = generateMonthlyAssessment(config, db, currentPeriod);
|
||||
|
||||
expect(assessment.totalInteractions).toBe(2);
|
||||
expect(assessment.correctionRate).toBe(0.5);
|
||||
});
|
||||
|
||||
it('uses local calendar boundaries for UTC-backed monthly data', () => {
|
||||
const previousTimezone = process.env.TZ;
|
||||
process.env.TZ = 'Europe/Budapest';
|
||||
|
||||
try {
|
||||
const raw = db.getDatabase();
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
const insideInteraction = optStore.insert({
|
||||
sessionId: 'inside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
const outsideInteraction = optStore.insert({
|
||||
sessionId: 'outside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInteraction.id);
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInteraction.id);
|
||||
|
||||
raw.exec(`
|
||||
CREATE TABLE feedback_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
message_index INTEGER NOT NULL,
|
||||
rating TEXT NOT NULL,
|
||||
reason TEXT,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('inside-august', 'up', null, '2026-07-31 22:30:00');
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('outside-august', 'down', 'wrong_answer', '2026-08-31 22:30:00');
|
||||
|
||||
const auditStore = new InstallAuditStore(db);
|
||||
const recordInstall = (capabilityName: string) => auditStore.record({
|
||||
capabilityName,
|
||||
capabilityType: 'skill',
|
||||
source: 'starter-pack',
|
||||
riskLevel: 'low',
|
||||
trustSource: 'starter_pack',
|
||||
approvalClass: 'standard',
|
||||
action: 'installed',
|
||||
initiator: 'agent',
|
||||
});
|
||||
const insideInstallA = recordInstall('inside-skill-a');
|
||||
const insideInstallB = recordInstall('inside-skill-b');
|
||||
const outsideInstall = recordInstall('outside-skill');
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInstallA.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 23:30:00', insideInstallB.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInstall.id);
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-08');
|
||||
|
||||
expect(assessment.totalInteractions).toBe(1);
|
||||
expect(assessment.correctionRate).toBe(1);
|
||||
expect(assessment.topStrengths).toContain('Consistent positive user feedback');
|
||||
expect(assessment.topWeaknesses).not.toContain('wrong answer');
|
||||
expect(assessment.skillsInstalled).toBe(2);
|
||||
} finally {
|
||||
if (previousTimezone === undefined) delete process.env.TZ;
|
||||
else process.env.TZ = previousTimezone;
|
||||
}
|
||||
});
|
||||
|
||||
it('includes capability gaps from improvement signals', () => {
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
// Record the same gap twice (threshold for capability_gap is 2)
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search');
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search again');
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.capabilityGapsDetected).toContain('web_search');
|
||||
});
|
||||
|
||||
it('has non-empty top strengths even with zero data', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
// Should have at least one strength (e.g., "Stable operation")
|
||||
expect(assessment.topStrengths.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('limits top strengths and weaknesses to 5', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.topStrengths.length).toBeLessThanOrEqual(5);
|
||||
expect(assessment.topWeaknesses.length).toBeLessThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Save to mind ────────────────────────────────────────────────────
|
||||
|
||||
describe('saveAssessmentToMind', () => {
|
||||
it('saves assessment as an I-frame in the personal mind', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate', 'Consistent positive user feedback'],
|
||||
topWeaknesses: ['wrong answer'],
|
||||
capabilityGapsDetected: ['web_search'],
|
||||
skillsInstalled: 2,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
|
||||
// Verify the frame was created
|
||||
const frames = new FrameStore(db);
|
||||
const allFrames = frames.getRecent(10);
|
||||
expect(allFrames.length).toBe(1);
|
||||
|
||||
const frame = allFrames[0];
|
||||
expect(frame.frame_type).toBe('I');
|
||||
expect(frame.gop_id).toBe('assessment');
|
||||
expect(frame.importance).toBe('important');
|
||||
expect(frame.content).toContain('Monthly Agent Assessment');
|
||||
expect(frame.content).toContain('2026-03');
|
||||
expect(frame.content).toContain('15.0%');
|
||||
expect(frame.content).toContain('+5%');
|
||||
expect(frame.content).toContain('web_search');
|
||||
});
|
||||
|
||||
it('a zero-data month writes nothing — no template-noise frames', () => {
|
||||
saveAssessmentToMind(db, {
|
||||
period: '2026-03', totalInteractions: 0, correctionRate: 0,
|
||||
improvementTrend: '0%', topStrengths: ['Stable operation'], topWeaknesses: [],
|
||||
capabilityGapsDetected: [], skillsInstalled: 0,
|
||||
recommendation: 'No corrections recorded.',
|
||||
});
|
||||
expect(new FrameStore(db).getRecent(10)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('re-running the same period replaces the frame instead of duplicating it', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate'],
|
||||
topWeaknesses: [],
|
||||
capabilityGapsDetected: [],
|
||||
skillsInstalled: 0,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
saveAssessmentToMind(db, { ...assessment, totalInteractions: 120 });
|
||||
// A different period must NOT be replaced — one frame per month.
|
||||
saveAssessmentToMind(db, { ...assessment, period: '2026-04' });
|
||||
|
||||
const frames = new FrameStore(db);
|
||||
const assessments = frames
|
||||
.getRecent(10)
|
||||
.filter((f) => f.content.startsWith('# Monthly Agent Assessment'));
|
||||
expect(assessments).toHaveLength(2);
|
||||
const march = assessments.filter((f) => f.content.includes('2026-03'));
|
||||
expect(march).toHaveLength(1);
|
||||
expect(march[0].content).toContain('**Interactions**: 120');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Cron registration ───────────────────────────────────────────────
|
||||
|
||||
describe('cron registration', () => {
|
||||
it('monthly_assessment is a valid cron job type', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
|
||||
// This should not throw — monthly_assessment is in VALID_JOB_TYPES
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
expect(schedule.name).toBe('Monthly assessment');
|
||||
expect(schedule.cron_expr).toBe('0 6 1 * *');
|
||||
expect(schedule.job_type).toBe('monthly_assessment');
|
||||
});
|
||||
|
||||
it('cron expression fires on 1st of month at 6 AM', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
// next_run_at should be set and valid
|
||||
expect(schedule.next_run_at).toBeDefined();
|
||||
expect(schedule.next_run_at).not.toBeNull();
|
||||
|
||||
// The next run should be on the 1st of some month at 6 AM local time
|
||||
// cron-parser interprets expressions in local timezone
|
||||
const nextRun = new Date(schedule.next_run_at!);
|
||||
expect(nextRun.getDate()).toBe(1);
|
||||
expect(nextRun.getHours()).toBe(6);
|
||||
expect(nextRun.getMinutes()).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -20,7 +20,6 @@ const testState = vi.hoisted(() => {
|
||||
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
||||
return {
|
||||
previousPromptAssembler,
|
||||
optimizerExpand: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
};
|
||||
});
|
||||
@@ -41,12 +40,6 @@ vi.mock('../../src/local/model-availability.js', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../src/local/services/optimizer-service.js', () => ({
|
||||
getOptimizerService: async () => ({
|
||||
expandWithChoices: testState.optimizerExpand,
|
||||
}),
|
||||
}));
|
||||
|
||||
function parseSse(raw: string): Array<{ event: string; data: Record<string, unknown> }> {
|
||||
return raw.split(/\n\n/)
|
||||
.filter(Boolean)
|
||||
@@ -151,12 +144,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
resetRateLimiter(server);
|
||||
testState.optimizerExpand.mockReset().mockResolvedValue({
|
||||
expanded: null,
|
||||
clarifyingQuestions: null,
|
||||
intent: 'request',
|
||||
isVague: false,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -657,7 +644,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
expect(response.body).toContain(toolResultMarker);
|
||||
expect(response.body).toContain(responseMarker);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
@@ -740,7 +726,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
|
||||
@@ -14,10 +14,8 @@ import { afterEach, describe, it, expect } from 'vitest';
|
||||
import Fastify from 'fastify';
|
||||
import {
|
||||
deleteCustomPersona,
|
||||
deployPersonaOverride,
|
||||
isValidCustomPersonaId,
|
||||
listPersonas,
|
||||
rollbackPersonaOverride,
|
||||
saveCustomPersona,
|
||||
type AgentPersona,
|
||||
} from '@waggle/agent';
|
||||
@@ -118,14 +116,12 @@ describe('Personas Routes', () => {
|
||||
await server.close();
|
||||
});
|
||||
|
||||
it('contains all 23 personas', async () => {
|
||||
it('contains all 22 personas', async () => {
|
||||
const server = createTestServer();
|
||||
const res = await server.inject({ method: 'GET', url: '/api/personas' });
|
||||
const { personas } = res.json();
|
||||
|
||||
// 22 tiered personas + session-reviewer (internal self-evolution reviewer;
|
||||
// returned by the raw list but excluded from onboarding + PersonaSwitcher).
|
||||
expect(personas.length).toBe(23);
|
||||
expect(personas.length).toBe(22);
|
||||
const ids = personas.map((p: { id: string }) => p.id);
|
||||
expect(ids).toContain('researcher');
|
||||
expect(ids).toContain('writer');
|
||||
@@ -251,40 +247,5 @@ describe('Personas Routes', () => {
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
|
||||
it('deployPersonaOverride rejects traversal before writing an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
|
||||
expect(() => deployPersonaOverride(dataDir, {
|
||||
personaId: '../evolved',
|
||||
systemPrompt: 'Escaped override',
|
||||
})).toThrow('Invalid custom persona ID');
|
||||
expect(fs.existsSync(outsidePath)).toBe(false);
|
||||
});
|
||||
|
||||
it('deployPersonaOverride keeps its validated ID authoritative over overrides', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const result = deployPersonaOverride(dataDir, {
|
||||
personaId: 'safe-persona',
|
||||
systemPrompt: 'Validated prompt',
|
||||
overrides: { id: '../evolved', systemPrompt: 'Spoofed prompt' } as never,
|
||||
});
|
||||
|
||||
const deployed = JSON.parse(fs.readFileSync(result.path, 'utf-8')) as AgentPersona;
|
||||
expect(deployed.id).toBe('safe-persona');
|
||||
expect(deployed.systemPrompt).toBe('Validated prompt');
|
||||
expect(fs.existsSync(path.join(dataDir, 'evolved.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rollbackPersonaOverride rejects traversal before deleting an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
fs.writeFileSync(outsidePath, 'sentinel', 'utf-8');
|
||||
|
||||
expect(() => rollbackPersonaOverride(dataDir, '../evolved'))
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
/**
|
||||
* session-reviewer persona — trust-boundary tool policy.
|
||||
*
|
||||
* The reviewer runs headless; ALWAYS_AVAILABLE_TOOLS (persona-tool-filter) re-adds
|
||||
* write-side tools past any allowlist, so the persona's disallowedTools must strip
|
||||
* the dangerous ones while KEEPING create_skill (its one proposal write) and the
|
||||
* read tools. This test locks that against regression.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getPersona, type ToolDefinition } from '@waggle/agent';
|
||||
import { applyPersonaToolFilter } from '../../src/local/persona-tool-filter.js';
|
||||
|
||||
/** Minimal ToolDefinition stub — only `name` matters for the filter. */
|
||||
function tool(name: string): ToolDefinition {
|
||||
return { name, description: '', parameters: { type: 'object', properties: {} }, execute: async () => '' };
|
||||
}
|
||||
|
||||
// A representative superset spanning reads, writes, memory, exec, and skill tools.
|
||||
const POOL: ToolDefinition[] = [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'save_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'correct_knowledge', 'add_task',
|
||||
'list_skills', 'search_skills', 'read_skill', 'create_skill', 'delete_skill',
|
||||
'write_file', 'edit_file', 'generate_docx', 'bash',
|
||||
'git_commit', 'git_push',
|
||||
'spawn_agent', 'install_capability', 'acquire_capability',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
].map(tool);
|
||||
|
||||
describe('session-reviewer persona', () => {
|
||||
it('exists and is not read-only (so create_skill survives)', () => {
|
||||
const p = getPersona('session-reviewer');
|
||||
expect(p).toBeTruthy();
|
||||
expect(p!.isReadOnly).toBe(false);
|
||||
expect(p!.tools).toContain('create_skill');
|
||||
});
|
||||
|
||||
it('applyPersonaToolFilter keeps reads + create_skill, strips writes/exec/memory', () => {
|
||||
const p = getPersona('session-reviewer')!;
|
||||
const names = new Set(applyPersonaToolFilter(POOL, p).map(t => t.name));
|
||||
|
||||
// Kept — the reviewer's read surface + its one proposal write.
|
||||
for (const keep of ['read_file', 'search_content', 'search_memory', 'read_skill', 'list_skills', 'create_skill']) {
|
||||
expect(names.has(keep), `expected ${keep} to survive`).toBe(true);
|
||||
}
|
||||
|
||||
// Stripped — writes, exec, memory writes, and the ALWAYS_AVAILABLE re-adds.
|
||||
for (const drop of [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'bash', 'spawn_agent', 'write_file', 'edit_file', 'git_commit',
|
||||
'add_task', 'correct_knowledge', 'execute_step', 'compose_workflow',
|
||||
]) {
|
||||
expect(names.has(drop), `expected ${drop} to be stripped`).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -217,7 +217,7 @@ const SELECTOR_TOOL_NAMES = [
|
||||
'delete_schedule', 'trigger_schedule', 'perplexity_search', 'tavily_search', 'brave_search',
|
||||
'browser_navigate', 'browser_screenshot', 'browser_click', 'browser_fill', 'browser_evaluate',
|
||||
'browser_snapshot', 'lsp_diagnostics', 'lsp_definition', 'lsp_references', 'lsp_hover',
|
||||
'cli_discover', 'cli_execute', 'agent_insights', 'find_connector', 'list_connector_categories',
|
||||
'cli_discover', 'cli_execute', 'find_connector', 'list_connector_categories',
|
||||
'spawn_agent', 'list_agents', 'get_agent_result', 'compose_workflow', 'orchestrate_workflow',
|
||||
'list_harnesses', 'run_harness',
|
||||
] as const;
|
||||
|
||||
@@ -1,341 +0,0 @@
|
||||
/**
|
||||
* H-10 G1 tests — EvolutionService.
|
||||
*
|
||||
* Covers:
|
||||
* - minimum-dataset gate (skips when no target has enough new traces)
|
||||
* - api-key gate (skips when vault is empty)
|
||||
* - target selection (never-run first, then oldest lastRunAt, then most traces)
|
||||
* - baseline resolution (persona + behavioral-spec-section + overrides)
|
||||
* - start/stop/tickInFlight lifecycle
|
||||
* - env flag parsing
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC_SECTIONS } from '@waggle/agent';
|
||||
import {
|
||||
EvolutionService,
|
||||
isEvolutionAutoEnabled,
|
||||
type EvolutionServiceDeps,
|
||||
type EvolutionTargetId,
|
||||
type TickResult,
|
||||
} from '../../src/local/services/evolution-service.js';
|
||||
|
||||
// ── Shared test fixture ──────────────────────────────────────────
|
||||
|
||||
interface Fixture {
|
||||
tmpDir: string;
|
||||
mind: MindDB;
|
||||
traceStore: ExecutionTraceStore;
|
||||
runStore: EvolutionRunStore;
|
||||
apiKey: string | null;
|
||||
activeSpec: Record<string, string>;
|
||||
makeDeps: (overrides?: Partial<EvolutionServiceDeps>) => EvolutionServiceDeps;
|
||||
}
|
||||
|
||||
function setupFixture(): Fixture {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-svc-'));
|
||||
const mind = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
const traceStore = new ExecutionTraceStore(mind);
|
||||
const runStore = new EvolutionRunStore(mind);
|
||||
|
||||
const fx: Fixture = {
|
||||
tmpDir,
|
||||
mind,
|
||||
traceStore,
|
||||
runStore,
|
||||
apiKey: 'sk-test',
|
||||
activeSpec: {},
|
||||
makeDeps: (overrides = {}) => ({
|
||||
traceStore,
|
||||
runStore,
|
||||
getApiKey: () => fx.apiKey,
|
||||
getActiveBehavioralSpec: () => fx.activeSpec,
|
||||
...overrides,
|
||||
}),
|
||||
};
|
||||
return fx;
|
||||
}
|
||||
|
||||
function teardown(fx: Fixture): void {
|
||||
fx.mind.close();
|
||||
try { fs.rmSync(fx.tmpDir, { recursive: true, force: true }); } catch { /* EBUSY on win32 */ }
|
||||
}
|
||||
|
||||
/** Seed `count` successful finalized traces for a given personaId. */
|
||||
function seedTraces(
|
||||
fx: Fixture,
|
||||
opts: { personaId?: string; count: number; outcome?: 'success' | 'corrected' | 'pending' },
|
||||
): void {
|
||||
for (let i = 0; i < opts.count; i++) {
|
||||
const id = fx.traceStore.start({
|
||||
personaId: opts.personaId ?? null,
|
||||
input: `q${i}`,
|
||||
});
|
||||
const outcome = opts.outcome ?? 'success';
|
||||
if (outcome !== 'pending') {
|
||||
fx.traceStore.finalize(id, { outcome, output: `a${i}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────
|
||||
|
||||
describe('EvolutionService — H-10 G1', () => {
|
||||
let fx: Fixture;
|
||||
beforeEach(() => { fx = setupFixture(); });
|
||||
afterEach(() => { teardown(fx); });
|
||||
|
||||
describe('dataset gate + target selection', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
{ kind: 'persona-system-prompt', name: 'writer' },
|
||||
];
|
||||
|
||||
it('pickNextTarget returns null when no persona has enough traces', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('pickNextTarget picks the never-run target that clears the gate', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const chosen = svc.pickNextTarget();
|
||||
expect(chosen?.name).toBe('coder');
|
||||
expect(chosen?.newTraces).toBe(25);
|
||||
expect(chosen?.lastRunAt).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers never-run targets over those with an older lastRunAt', () => {
|
||||
// Both qualify by trace count.
|
||||
seedTraces(fx, { personaId: 'coder', count: 30 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 30 });
|
||||
// coder has a historical run; writer has none.
|
||||
fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()?.name).toBe('writer');
|
||||
});
|
||||
|
||||
it('only counts outcomes in eligibleOutcomes', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 30, outcome: 'pending' });
|
||||
const svc = new EvolutionService(fx.makeDeps(), {
|
||||
targets,
|
||||
minTracesPerTarget: 20,
|
||||
eligibleOutcomes: ['success'],
|
||||
});
|
||||
// All pending — 0 eligible even though 30 exist.
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('counts only traces created since the last run when computing newTraces', () => {
|
||||
// Seed historical traces first.
|
||||
seedTraces(fx, { personaId: 'coder', count: 100 });
|
||||
|
||||
// Record a run with a created_at that's strictly AFTER the trace rows.
|
||||
// SQLite datetime('now') has second-level precision, so we need to bump
|
||||
// the run's timestamp forward by an hour to avoid a same-second
|
||||
// collision that would make "since >= lastRunAt" include the seeded
|
||||
// traces. Using raw SQL is the least invasive way to simulate time
|
||||
// advancing between seed and run creation.
|
||||
const run = fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
const futureTs = new Date(Date.now() + 60 * 60 * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z$/, '');
|
||||
fx.mind.getDatabase()
|
||||
.prepare('UPDATE evolution_runs SET created_at = ? WHERE run_uuid = ?')
|
||||
.run(futureTs, run.run_uuid);
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const candidates = svc.enumerateCandidates();
|
||||
const coder = candidates.find(c => c.name === 'coder');
|
||||
expect(coder?.lastRunAt).toBe(futureTs);
|
||||
expect(coder?.newTraces).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('baseline resolution', () => {
|
||||
it('resolves a real persona baseline from the registry', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for an unknown persona', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'nope' })).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers the active behavioral spec when an override exists', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
fx.activeSpec[section] = 'OVERRIDDEN TEXT';
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section }))
|
||||
.toBe('OVERRIDDEN TEXT');
|
||||
});
|
||||
|
||||
it('falls back to compile-time BEHAVIORAL_SPEC when no override', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for tool-description / skill-body / generic (not auto-evolvable)', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'tool-description', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'skill-body', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'generic', name: 'anything' })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('tick outcomes', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
];
|
||||
|
||||
it('skips when no API key is configured', async () => {
|
||||
fx.apiKey = null;
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 1 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/api key/i);
|
||||
});
|
||||
|
||||
it('skips when no target clears the gate', async () => {
|
||||
// Zero traces for any target.
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/dataset gate/);
|
||||
});
|
||||
|
||||
it('invokes the runner for the picked target when gate passes', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn<Parameters<NonNullable<EvolutionServiceDeps['runner']>>, Promise<TickResult>>()
|
||||
.mockResolvedValue({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: 'uuid-stub',
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
const [calledTarget, calledBaseline, calledKey] = runner.mock.calls[0];
|
||||
// The service passes the full candidate (incl. lastRunAt + newTraces),
|
||||
// not just the bare target id.
|
||||
expect(calledTarget).toMatchObject({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(calledBaseline).toBeTruthy();
|
||||
expect(calledKey).toBe('sk-test');
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.runUuid).toBe('uuid-stub');
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces runner errors through the tick return, not a throw', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn().mockRejectedValue(new Error('boom'));
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/tick failed: boom/);
|
||||
});
|
||||
|
||||
it('reentrancy guard: concurrent tick returns "already in progress"', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
let resolve: ((v: TickResult) => void) | null = null;
|
||||
const runner = vi.fn(() => new Promise<TickResult>(r => { resolve = r; }));
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const first = svc.tick();
|
||||
const second = await svc.tick(); // before first resolves
|
||||
|
||||
expect(second.skipped).toBe(true);
|
||||
if (second.skipped) expect(second.reason).toMatch(/already in progress/);
|
||||
|
||||
// Unblock and let the first finish cleanly.
|
||||
resolve!({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: null,
|
||||
});
|
||||
await first;
|
||||
});
|
||||
});
|
||||
|
||||
describe('lifecycle', () => {
|
||||
it('start/stop toggles isRunning', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
svc.start();
|
||||
expect(svc.isRunning()).toBe(true);
|
||||
svc.stop();
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('start is idempotent — double-start leaves a single timer', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
svc.start();
|
||||
const firstTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
svc.start();
|
||||
const secondTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
expect(firstTimer).toBe(secondTimer);
|
||||
svc.stop();
|
||||
});
|
||||
|
||||
it('clamps tickIntervalMs to at least 60s', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 1 });
|
||||
expect(svc.config.tickIntervalMs).toBe(60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isEvolutionAutoEnabled', () => {
|
||||
it('returns false when the flag is missing or unrelated', () => {
|
||||
expect(isEvolutionAutoEnabled({})).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'false' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '0' })).toBe(false);
|
||||
});
|
||||
it('returns true for "1" / "true" / "yes"', () => {
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '1' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'true' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'TRUE' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'yes' })).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user