refactor: remove self-learning and the optimizer package (P1.4)

Self-learning is excluded from the community edition (ADR-0004); the
project owner reworks it separately. Cost tracking stays.

- agent: delete behaviour evolution (evolve-schema, iterative GEPA,
  compose-evolution, evolution gates, orchestrator and deploy),
  improvement and correction detection, AgentLearning, the eval-dataset
  builder, optimization-capture and the agent_insights tool. Secret
  detection moves from eval-dataset to secret-patterns. The LLM wiring
  keeps only the Haiku adapter and judge call used by skill audit and
  hygiene, so @ax-llm/ax is now declared by agent instead of server.
- server: delete /api/evolution/*, the evolution and optimizer services,
  GEPA expansion of the first chat message, the prompt_optimization and
  monthly_assessment jobs, the idle-session reviewer and its
  session-reviewer persona, prompt injection of past corrections and
  thumbs-down recorded as correction signals. Existing job rows of the
  removed types are ignored, not deleted.
- core: delete OptimizationLogStore; stop re-exporting the improvement
  signal and evolution run stores.
- web: delete the Evolution screen and the two job types in the
  automation editor.
- delete packages/optimizer.

Also logs UF-027 (self-learning loops read data nothing collects) and
UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
shumeiko.it
2026-10-20 00:54:58 +02:00
parent b734c1d3d2
commit 2e1999a3f6
112 changed files with 336 additions and 16165 deletions

View File

@@ -40,7 +40,6 @@ describe('Automations alias routes (Phase 3)', () => {
let server: ReturnType<typeof Fastify>;
let includeViewerWorkspace: boolean;
let literalGlobalIsViewer: boolean;
let viewerOptimizationEnabled: boolean;
beforeEach(async () => {
db = new MindDB(':memory:');
@@ -49,7 +48,6 @@ describe('Automations alias routes (Phase 3)', () => {
failNext = false;
includeViewerWorkspace = false;
literalGlobalIsViewer = false;
viewerOptimizationEnabled = 1 as unknown as boolean;
scheduler = new LocalScheduler(
cronStore,
async (schedule) => {
@@ -79,7 +77,6 @@ describe('Automations alias routes (Phase 3)', () => {
name: 'Viewer WS',
teamId: 'team-1',
teamRole: 'viewer',
optimizationEnabled: viewerOptimizationEnabled,
};
}
if (id === 'global' && literalGlobalIsViewer) {
@@ -96,7 +93,6 @@ describe('Automations alias routes (Phase 3)', () => {
name: 'Viewer WS',
teamId: 'team-1',
teamRole: 'viewer',
optimizationEnabled: viewerOptimizationEnabled,
}]
: []),
],
@@ -460,13 +456,6 @@ describe('Automations alias routes (Phase 3)', () => {
workspaceId: undefined,
request: (id: number) => ({ method: 'POST' as const, url: `/api/cron/${id}/trigger` }),
},
{
label: 'prompt optimization through automation run',
jobType: 'prompt_optimization' as const,
jobConfig: {},
workspaceId: undefined,
request: (id: number) => ({ method: 'POST' as const, url: `/api/automations/${id}/run` }),
},
{
label: 'memory compaction through cron trigger',
jobType: 'memory_consolidation' as const,
@@ -506,25 +495,6 @@ describe('Automations alias routes (Phase 3)', () => {
expect(executed).toHaveLength(0);
});
it('prompt optimization ignores a listed viewer workspace without optimization opt-in', async () => {
includeViewerWorkspace = true;
viewerOptimizationEnabled = false;
const row = cronStore.create({
name: 'Prompt optimization',
cronExpr: '0 2 * * *',
jobType: 'prompt_optimization',
enabled: false,
});
const res = await server.inject({
method: 'POST',
url: `/api/cron/${row.id}/trigger`,
});
expect(res.statusCode).toBe(200);
expect(executed).toHaveLength(1);
});
it.each([
{
label: 'cron',
@@ -625,12 +595,6 @@ describe('Automations alias routes (Phase 3)', () => {
jobConfig: { action: 'morning_briefing' },
workspaceId: undefined,
},
{
label: 'personal monthly assessment',
jobType: 'monthly_assessment' as const,
jobConfig: {},
workspaceId: undefined,
},
{
label: 'non-matching spaced memory action',
jobType: 'memory_consolidation' as const,

View File

@@ -377,7 +377,6 @@ describe('classifyExplicitTurnMutationPolicy', () => {
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity',
'get_awareness', 'read_other_workspace', 'save_memory', 'add_task',
'correct_knowledge', 'read_file', 'web_search', 'mcp_external_read',
'agent_insights',
].map(name => ({ name }));
const filtered = filterToolsByTurnMutationPolicy(
tools,

View File

@@ -3,7 +3,7 @@
*
* Tests the two feedback endpoints:
* POST /api/feedback — record user feedback
* GET /api/feedback/stats — get improvement stats
* GET /api/feedback/stats — get feedback stats
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
@@ -11,35 +11,27 @@ import path from 'node:path';
import fs from 'node:fs';
import os from 'node:os';
import Fastify from 'fastify';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import { MindDB } from '@waggle/core';
import { feedbackRoutes } from '../../src/local/routes/feedback.js';
/** Minimal multiMind mock that provides a real MindDB for the personal mind. */
function createTestServer(db: MindDB) {
const server = Fastify({ logger: false });
const signalStore = new ImprovementSignalStore(db);
server.decorate('multiMind', {
personal: db,
});
server.decorate('agentState', {
orchestrator: {
getImprovementSignals: () => signalStore,
},
});
server.register(feedbackRoutes);
return { server, signalStore };
return { server };
}
describe('Feedback Routes', () => {
let db: MindDB;
let server: ReturnType<typeof Fastify>;
let signalStore: ImprovementSignalStore;
beforeEach(() => {
db = new MindDB(':memory:');
const result = createTestServer(db);
server = result.server;
signalStore = result.signalStore;
});
afterEach(async () => {
@@ -80,26 +72,6 @@ describe('Feedback Routes', () => {
expect(res.json()).toEqual({ ok: true });
});
it('records negative feedback as correction signal', async () => {
await server.inject({
method: 'POST',
url: '/api/feedback',
payload: {
sessionId: 'sess-1',
messageIndex: 2,
rating: 'down',
reason: 'wrong_tool',
detail: 'Should have used web_search',
},
});
// Check that it was recorded in the improvement signals store
const corrections = signalStore.getByCategory('correction');
const feedbackSignal = corrections.find(c => c.pattern_key === 'feedback:wrong_tool');
expect(feedbackSignal).toBeDefined();
expect(feedbackSignal!.count).toBe(1);
});
it('rejects missing sessionId', async () => {
const res = await server.inject({
method: 'POST',
@@ -170,7 +142,6 @@ describe('Feedback Routes', () => {
expect(body.totalFeedback).toBe(0);
expect(body.positiveRate).toBe(0);
expect(body.topIssues).toEqual([]);
expect(typeof body.correctionsThisWeek).toBe('number');
expect(typeof body.improvementTrend).toBe('string');
});
@@ -202,7 +173,6 @@ describe('Feedback Routes', () => {
expect(body.totalFeedback).toBe(3);
expect(body.positiveRate).toBeCloseTo(0.67, 1);
expect(body.topIssues).toContain('too_verbose');
expect(typeof body.correctionsThisWeek).toBe('number');
expect(typeof body.improvementTrend).toBe('string');
});

View File

@@ -1,196 +0,0 @@
/**
* GEPA Prompt Optimization Cron Handler Tests
*
* Tests the prompt_optimization cron handler logic:
* - Calls LLM when correction rate > 20% or avg turns > 15
* - Stores generated variant in optimization_log
* - Skips when budget is exceeded
* - Skips when not enough data
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, OptimizationLogStore } from '@waggle/core';
import { isWithinBudget, getRecentLogs } from '@waggle/agent';
describe('GEPA Prompt Optimization', () => {
let tmpDir: string;
let db: MindDB;
let optStore: OptimizationLogStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-gepa-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
optStore = new OptimizationLogStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
describe('Signal detection', () => {
it('detects high correction rate as optimization signal', () => {
// Insert logs with high correction rate (>20%)
for (let i = 0; i < 10; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: 'Test system prompt for optimization',
toolsUsed: ['search_memory', 'save_memory'],
turnCount: 5,
wasCorrection: i < 5, // 50% correction rate
inputTokens: 100,
outputTokens: 200,
});
}
const stats = optStore.getStats();
expect(stats.correctionRate).toBeGreaterThan(0.2);
expect(stats.total).toBe(10);
});
it('detects high avg turn count as optimization signal', () => {
// Insert logs with high turn count (>15)
for (let i = 0; i < 10; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: 'Test system prompt for optimization',
toolsUsed: ['search_memory'],
turnCount: 20, // >15 threshold
wasCorrection: false,
inputTokens: 100,
outputTokens: 200,
});
}
const stats = optStore.getStats();
expect(stats.avgTurnCount).toBeGreaterThan(15);
});
it('does not trigger when both metrics are within thresholds', () => {
for (let i = 0; i < 10; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: 'Test system prompt',
toolsUsed: ['search_memory'],
turnCount: 5,
wasCorrection: false,
inputTokens: 100,
outputTokens: 200,
});
}
const stats = optStore.getStats();
expect(stats.correctionRate).toBeLessThanOrEqual(0.2);
expect(stats.avgTurnCount).toBeLessThanOrEqual(15);
});
});
describe('Budget check', () => {
it('reports within budget when token costs are low', () => {
for (let i = 0; i < 5; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: 'Test prompt',
toolsUsed: [],
turnCount: 3,
wasCorrection: false,
inputTokens: 100,
outputTokens: 50,
});
}
// Budget is 100 cents ($1). 500 input + 250 output tokens is far under budget.
expect(isWithinBudget(optStore, 100)).toBe(true);
});
it('reports over budget when token costs exceed limit', () => {
// Insert a log with massive token counts to exceed budget
optStore.insert({
sessionId: 'sess-big',
workspaceId: 'ws-test',
systemPrompt: 'Test prompt',
toolsUsed: [],
turnCount: 3,
wasCorrection: false,
inputTokens: 10_000_000, // 10M input tokens at $3/M = $30
outputTokens: 1_000_000, // 1M output tokens at $15/M = $15
});
// Budget is 100 cents ($1). Total cost is ~$45 = 4500 cents. Way over budget.
expect(isWithinBudget(optStore, 100)).toBe(false);
});
});
describe('Variant storage', () => {
it('stores generated variant in optimization_log', () => {
// Simulate what the cron handler does after generating a variant
const variantText = 'Improved system prompt with better instructions for reducing corrections...'.repeat(5);
optStore.insert({
sessionId: `gepa-variant-${Date.now()}`,
workspaceId: 'ws-test',
systemPrompt: variantText,
toolsUsed: ['gepa_variant'],
turnCount: 0,
wasCorrection: false,
inputTokens: 500,
outputTokens: variantText.length,
});
const recent = getRecentLogs(optStore, 10);
const variant = recent.find(l => JSON.parse(l.tools_used).includes('gepa_variant'));
expect(variant).toBeDefined();
expect(variant!.system_prompt).toBe(variantText);
expect(variant!.turn_count).toBe(0);
expect(variant!.was_correction).toBe(0);
});
it('retrieves recent logs for analysis', () => {
for (let i = 0; i < 8; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: `Prompt v${i}`,
toolsUsed: ['search_memory'],
turnCount: i + 1,
wasCorrection: i % 3 === 0,
inputTokens: 100 * (i + 1),
outputTokens: 50 * (i + 1),
});
}
const logs = getRecentLogs(optStore, 5);
expect(logs).toHaveLength(5);
// Most recent first
expect(logs[0].session_id).toBe('sess-7');
});
});
describe('Minimum data requirement', () => {
it('skips optimization when fewer than 5 logs', () => {
for (let i = 0; i < 3; i++) {
optStore.insert({
sessionId: `sess-${i}`,
workspaceId: 'ws-test',
systemPrompt: 'Test prompt',
toolsUsed: [],
turnCount: 20,
wasCorrection: true,
inputTokens: 100,
outputTokens: 200,
});
}
const recentLogs = getRecentLogs(optStore, 100);
// The cron handler checks: if (recentLogs.length < 5) continue;
expect(recentLogs.length).toBeLessThan(5);
});
});
});

View File

@@ -1,287 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import {
IdleSessionWatcher,
countTurns,
readRecentTranscript,
buildReviewInstruction,
NOTHING_TO_DO,
DEFAULT_CONFIG,
type SelfEvolutionConfig,
type ReviewTurnResult,
} from '../../src/local/idle-watcher.js';
import { getNotificationGate } from '../../src/local/notification-gate.js';
// A fixed synthetic "now" so idle math is deterministic.
const NOW = 2_000_000_000_000;
const MIN = 60_000;
describe('idle-watcher', () => {
let dataDir: string;
beforeEach(() => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-idle-'));
});
afterEach(() => {
fs.rmSync(dataDir, { recursive: true, force: true });
});
// ── Fixtures ─────────────────────────────────────────────────────────
function writeConfig(cfg: Partial<SelfEvolutionConfig> | string): void {
const body = typeof cfg === 'string' ? cfg : JSON.stringify(cfg);
fs.writeFileSync(path.join(dataDir, 'self-evolution.json'), body);
}
/** Create a session .jsonl with `turns` message lines and a given mtime (ms). */
function writeSession(workspaceId: string, sessionId: string, turns: number, mtimeMs: number): void {
const dir = path.join(dataDir, 'workspaces', workspaceId, 'sessions');
fs.mkdirSync(dir, { recursive: true });
const lines = [JSON.stringify({ type: 'meta', title: sessionId })];
for (let i = 0; i < turns; i++) {
lines.push(JSON.stringify({ role: i % 2 === 0 ? 'user' : 'assistant', content: `msg ${i}` }));
}
const filePath = path.join(dir, `${sessionId}.jsonl`);
fs.writeFileSync(filePath, lines.join('\n') + '\n');
fs.utimesSync(filePath, new Date(mtimeMs), new Date(mtimeMs));
}
function makeWatcher(over?: {
runReviewTurn?: (i: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
}) {
const reviewed: Array<{ sessionId: string; workspaceId: string }> = [];
const emitCalls: Array<{ suppressed: boolean; dedupeKey?: string }> = [];
const runReviewTurn =
over?.runReviewTurn ??
(async (i: { sessionId: string; workspaceId: string }) => {
reviewed.push(i);
return { content: NOTHING_TO_DO };
});
const runSpy = vi.fn(runReviewTurn);
const watcher = new IdleSessionWatcher({
dataDir,
now: () => NOW,
runReviewTurn: runSpy,
// Mirror production emitNotification's gate so suppression is genuinely tested.
emitNotification: (_event, options) => {
let suppressed = false;
if (options?.dedupeKey && options.materialHash) {
suppressed = !getNotificationGate(dataDir).shouldNotify(options.dedupeKey, options.materialHash);
}
emitCalls.push({ suppressed, dedupeKey: options?.dedupeKey });
return { suppressed };
},
log: { info: () => {}, warn: () => {} },
});
return { watcher, reviewed, emitCalls, runSpy };
}
// ── Fire-condition matrix ────────────────────────────────────────────
describe('fire condition', () => {
beforeEach(() => writeConfig({ enabled: true })); // rest of fields default
it('reviews an idle session with enough turns', async () => {
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, reviewed } = makeWatcher();
const n = await watcher.tick();
expect(n).toBe(1);
expect(reviewed).toEqual([{ sessionId: 'sess-a', workspaceId: 'w1' }]);
});
it('does NOT review a still-active session (mtime too recent)', async () => {
writeSession('w1', 'sess-active', 10, NOW - 5 * MIN); // < 15 min idle
const { watcher, reviewed } = makeWatcher();
expect(await watcher.tick()).toBe(0);
expect(reviewed).toEqual([]);
});
it('does NOT review a session below the turn threshold', async () => {
writeSession('w1', 'sess-short', 3, NOW - 30 * MIN); // idle but only 3 turns
const { watcher, reviewed } = makeWatcher();
expect(await watcher.tick()).toBe(0);
expect(reviewed).toEqual([]);
});
});
// ── Fired-set: no re-fire until mtime advances ───────────────────────
it('does not re-review the same session until its file advances', async () => {
writeConfig({ enabled: true });
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, runSpy } = makeWatcher();
expect(await watcher.tick()).toBe(1);
expect(await watcher.tick()).toBe(0); // same mtime — already fired
expect(runSpy).toHaveBeenCalledTimes(1);
// File advances (a new message lands) → new mtime key → re-fires once.
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
expect(await watcher.tick()).toBe(1);
expect(runSpy).toHaveBeenCalledTimes(2);
});
// ── Disabled / config handling ───────────────────────────────────────
it('is a no-op when disabled (default)', async () => {
writeConfig({ enabled: false });
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, runSpy } = makeWatcher();
expect(await watcher.tick()).toBe(0);
expect(runSpy).not.toHaveBeenCalled();
});
it('is a no-op when the config file is absent (defaults ⇒ disabled)', async () => {
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, runSpy } = makeWatcher();
expect(await watcher.tick()).toBe(0);
expect(runSpy).not.toHaveBeenCalled();
});
it('falls back to defaults on a corrupt config (⇒ disabled)', async () => {
writeConfig('{ this is not json');
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, runSpy } = makeWatcher();
expect(await watcher.tick()).toBe(0);
expect(runSpy).not.toHaveBeenCalled();
expect(DEFAULT_CONFIG.enabled).toBe(false);
});
it('applies partial config over defaults (enabled + custom idle threshold)', async () => {
writeConfig({ enabled: true, idleMinutes: 60 }); // minTurns/cap default
writeSession('w1', 'fresh', 6, NOW - 30 * MIN); // idle 30m < 60m ⇒ skipped
writeSession('w1', 'old', 6, NOW - 90 * MIN); // idle 90m ⇒ reviewed
const { watcher, reviewed } = makeWatcher();
expect(await watcher.tick()).toBe(1);
expect(reviewed).toEqual([{ sessionId: 'old', workspaceId: 'w1' }]);
});
// ── Daily cap ────────────────────────────────────────────────────────
it('enforces maxReviewsPerDay', async () => {
writeConfig({ enabled: true, maxReviewsPerDay: 2 });
writeSession('w1', 's1', 6, NOW - 20 * MIN);
writeSession('w1', 's2', 6, NOW - 21 * MIN);
writeSession('w1', 's3', 6, NOW - 22 * MIN);
const { watcher, runSpy } = makeWatcher();
const n = await watcher.tick();
expect(n).toBe(2); // capped at 2 even though 3 are eligible
expect(runSpy).toHaveBeenCalledTimes(2);
// A later tick in the same day stays capped (the 3rd never runs).
expect(await watcher.tick()).toBe(0);
expect(runSpy).toHaveBeenCalledTimes(2);
});
// ── Enumeration skips channel-/evolve- prefixes ──────────────────────
it('skips channel-* and evolve-* sessions', async () => {
writeConfig({ enabled: true });
writeSession('w1', 'channel-telegram-123', 6, NOW - 20 * MIN);
writeSession('w1', 'evolve-sess-a', 6, NOW - 20 * MIN);
writeSession('w1', 'sess-real', 6, NOW - 20 * MIN);
const { watcher, reviewed } = makeWatcher();
expect(await watcher.tick()).toBe(1);
expect(reviewed).toEqual([{ sessionId: 'sess-real', workspaceId: 'w1' }]);
});
// ── Notification gating ──────────────────────────────────────────────
it('does NOT notify on a NOTHING_TO_DO / empty / error result', async () => {
writeConfig({ enabled: true });
writeSession('w1', 'ntd', 6, NOW - 20 * MIN);
writeSession('w1', 'empty', 6, NOW - 21 * MIN);
writeSession('w1', 'err', 6, NOW - 22 * MIN);
const results: Record<string, ReviewTurnResult> = {
ntd: { content: NOTHING_TO_DO },
empty: { content: ' ' },
err: { content: 'partial', error: 'boom' },
};
const { watcher, emitCalls } = makeWatcher({
runReviewTurn: async ({ sessionId }) => results[sessionId],
});
await watcher.tick();
expect(emitCalls).toHaveLength(0);
});
it('notifies once on a material finding and suppresses an identical re-finding', async () => {
writeConfig({ enabled: true });
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
const { watcher, emitCalls } = makeWatcher({
runReviewTurn: async () => ({ content: 'Finding: the assistant promised a report and never produced it.' }),
});
await watcher.tick();
expect(emitCalls).toHaveLength(1);
expect(emitCalls[0].suppressed).toBe(false);
expect(emitCalls[0].dedupeKey).toBe('self-evolution:sess-a');
// Session advances, same finding recurs → notification suppressed (anti-nag).
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
await watcher.tick();
expect(emitCalls).toHaveLength(2);
expect(emitCalls[1].suppressed).toBe(true);
});
// ── Single-flight guard ──────────────────────────────────────────────
it('tick is single-flight (a re-entrant tick returns 0)', async () => {
writeConfig({ enabled: true });
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
let secondResult = -1;
const { watcher } = makeWatcher({
runReviewTurn: async () => {
// Re-enter while the first tick is still in-flight.
secondResult = await watcher.tick();
return { content: NOTHING_TO_DO };
},
});
await watcher.tick();
expect(secondResult).toBe(0);
});
});
// ── Pure helpers ───────────────────────────────────────────────────────
describe('idle-watcher pure helpers', () => {
it('countTurns excludes the meta line', () => {
const raw = [
JSON.stringify({ type: 'meta', title: 't' }),
JSON.stringify({ role: 'user', content: 'hi' }),
JSON.stringify({ role: 'assistant', content: 'yo' }),
].join('\n');
expect(countTurns(raw)).toBe(2);
});
it('countTurns counts every non-empty line when there is no meta', () => {
const raw = [
JSON.stringify({ role: 'user', content: 'hi' }),
JSON.stringify({ role: 'assistant', content: 'yo' }),
].join('\n');
expect(countTurns(raw)).toBe(2);
expect(countTurns('')).toBe(0);
});
it('readRecentTranscript formats messages and skips meta; null when unreadable', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-tr-'));
try {
const sessionsDir = path.join(dir, 'workspaces', 'w1', 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
fs.writeFileSync(
path.join(sessionsDir, 'sess-a.jsonl'),
[
JSON.stringify({ type: 'meta', title: 't' }),
JSON.stringify({ role: 'user', content: 'build me a report' }),
JSON.stringify({ role: 'assistant', content: 'on it' }),
].join('\n'),
);
const t = readRecentTranscript(dir, 'w1', 'sess-a');
expect(t).toContain('USER: build me a report');
expect(t).toContain('ASSISTANT: on it');
expect(t).not.toContain('meta');
expect(readRecentTranscript(dir, 'w1', 'missing')).toBeNull();
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
it('buildReviewInstruction embeds the transcript and the NOTHING_TO_DO contract', () => {
const msg = buildReviewInstruction('sess-a', 'USER: hi');
expect(msg).toContain('sess-a');
expect(msg).toContain('USER: hi');
expect(msg).toContain(NOTHING_TO_DO);
expect(msg).toContain('create_skill');
});
});

View File

@@ -8,8 +8,8 @@ import { buildLocalServer } from '../../src/local/index.js';
/**
* Integration test for the pre:memory-write lint handler registered in chat.ts.
* Fires the real hook through the server's shared HookRegistry and asserts that
* capability-failure symptoms are cancelled (and routed to the improvement-signal
* path) while normal memories and dramatic-but-legitimate claims pass through.
* capability-failure symptoms are cancelled while normal memories and
* dramatic-but-legitimate claims pass through.
*/
describe('pre:memory-write capability-symptom lint (handler integration)', () => {
let server: FastifyInstance;
@@ -30,10 +30,8 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
}
});
it('cancels a capability-failure symptom write and records an improvement signal', async () => {
const { hookRegistry, orchestrator } = server.agentState;
const store = orchestrator.getImprovementSignals();
const before = store.getByCategory('capability_gap').length;
it('cancels a capability-failure symptom write', async () => {
const { hookRegistry } = server.agentState;
const result = await hookRegistry.fire('pre:memory-write', {
toolName: 'save_memory',
@@ -43,9 +41,6 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
expect(result.cancelled).toBe(true);
expect(result.reason).toMatch(/capability/i);
expect(result.reason).toMatch(/acquire_capability/);
const after = store.getByCategory('capability_gap');
expect(after.length).toBe(before + 1);
});
it('allows a normal memory (tool preference) to pass', async () => {

View File

@@ -1,315 +0,0 @@
/**
* Monthly Self-Assessment Tests
*
* Verifies:
* - Assessment shape and all required fields
* - Cron registration (monthly_assessment job type is valid)
* - Assessment saves to personal mind as I-frame
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
MindDB,
CronStore,
FrameStore,
OptimizationLogStore,
ImprovementSignalStore,
InstallAuditStore,
} from '@waggle/core';
import { generateMonthlyAssessment, saveAssessmentToMind, type MonthlyAssessment } from '../../src/local/monthly-assessment.js';
describe('Monthly Self-Assessment', () => {
let db: MindDB;
beforeEach(() => {
db = new MindDB(':memory:');
});
afterEach(() => {
db.close();
});
// ── Assessment shape ────────────────────────────────────────────────
describe('generateMonthlyAssessment', () => {
it('returns a valid MonthlyAssessment shape', () => {
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db, '2026-03');
expect(assessment).toBeDefined();
expect(assessment.period).toBe('2026-03');
expect(typeof assessment.totalInteractions).toBe('number');
expect(typeof assessment.correctionRate).toBe('number');
expect(typeof assessment.improvementTrend).toBe('string');
expect(Array.isArray(assessment.topStrengths)).toBe(true);
expect(Array.isArray(assessment.topWeaknesses)).toBe(true);
expect(Array.isArray(assessment.capabilityGapsDetected)).toBe(true);
expect(typeof assessment.skillsInstalled).toBe('number');
expect(typeof assessment.recommendation).toBe('string');
expect(assessment.recommendation.length).toBeGreaterThan(0);
});
it('defaults to previous month when no period override', () => {
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db);
// Period should be YYYY-MM format for the previous month
expect(assessment.period).toMatch(/^\d{4}-\d{2}$/);
});
it('computes correction rate from optimization logs', () => {
// Insert some optimization log entries for March 2026
const optStore = new OptimizationLogStore(db);
optStore.insert({
sessionId: 's1',
workspaceId: 'w1',
systemPrompt: 'test',
toolsUsed: ['read_file'],
turnCount: 5,
wasCorrection: false,
});
optStore.insert({
sessionId: 's2',
workspaceId: 'w1',
systemPrompt: 'test',
toolsUsed: ['write_file'],
turnCount: 3,
wasCorrection: true,
});
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
// Use current month since the entries use datetime('now')
const now = new Date();
const currentPeriod = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
const assessment = generateMonthlyAssessment(config, db, currentPeriod);
expect(assessment.totalInteractions).toBe(2);
expect(assessment.correctionRate).toBe(0.5);
});
it('uses local calendar boundaries for UTC-backed monthly data', () => {
const previousTimezone = process.env.TZ;
process.env.TZ = 'Europe/Budapest';
try {
const raw = db.getDatabase();
const optStore = new OptimizationLogStore(db);
const insideInteraction = optStore.insert({
sessionId: 'inside-august',
workspaceId: 'w1',
systemPrompt: 'test',
toolsUsed: [],
turnCount: 1,
wasCorrection: true,
});
const outsideInteraction = optStore.insert({
sessionId: 'outside-august',
workspaceId: 'w1',
systemPrompt: 'test',
toolsUsed: [],
turnCount: 1,
wasCorrection: false,
});
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
.run('2026-07-31 22:30:00', insideInteraction.id);
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
.run('2026-08-31 22:30:00', outsideInteraction.id);
raw.exec(`
CREATE TABLE feedback_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL,
message_index INTEGER NOT NULL,
rating TEXT NOT NULL,
reason TEXT,
detail TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL
)
`);
raw.prepare(`
INSERT INTO feedback_entries
(session_id, message_index, rating, reason, created_at)
VALUES (?, 0, ?, ?, ?)
`).run('inside-august', 'up', null, '2026-07-31 22:30:00');
raw.prepare(`
INSERT INTO feedback_entries
(session_id, message_index, rating, reason, created_at)
VALUES (?, 0, ?, ?, ?)
`).run('outside-august', 'down', 'wrong_answer', '2026-08-31 22:30:00');
const auditStore = new InstallAuditStore(db);
const recordInstall = (capabilityName: string) => auditStore.record({
capabilityName,
capabilityType: 'skill',
source: 'starter-pack',
riskLevel: 'low',
trustSource: 'starter_pack',
approvalClass: 'standard',
action: 'installed',
initiator: 'agent',
});
const insideInstallA = recordInstall('inside-skill-a');
const insideInstallB = recordInstall('inside-skill-b');
const outsideInstall = recordInstall('outside-skill');
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
.run('2026-07-31 22:30:00', insideInstallA.id);
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
.run('2026-07-31 23:30:00', insideInstallB.id);
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
.run('2026-08-31 22:30:00', outsideInstall.id);
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db, '2026-08');
expect(assessment.totalInteractions).toBe(1);
expect(assessment.correctionRate).toBe(1);
expect(assessment.topStrengths).toContain('Consistent positive user feedback');
expect(assessment.topWeaknesses).not.toContain('wrong answer');
expect(assessment.skillsInstalled).toBe(2);
} finally {
if (previousTimezone === undefined) delete process.env.TZ;
else process.env.TZ = previousTimezone;
}
});
it('includes capability gaps from improvement signals', () => {
const signalStore = new ImprovementSignalStore(db);
// Record the same gap twice (threshold for capability_gap is 2)
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search');
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search again');
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db, '2026-03');
expect(assessment.capabilityGapsDetected).toContain('web_search');
});
it('has non-empty top strengths even with zero data', () => {
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db, '2026-03');
// Should have at least one strength (e.g., "Stable operation")
expect(assessment.topStrengths.length).toBeGreaterThan(0);
});
it('limits top strengths and weaknesses to 5', () => {
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
const assessment = generateMonthlyAssessment(config, db, '2026-03');
expect(assessment.topStrengths.length).toBeLessThanOrEqual(5);
expect(assessment.topWeaknesses.length).toBeLessThanOrEqual(5);
});
});
// ── Save to mind ────────────────────────────────────────────────────
describe('saveAssessmentToMind', () => {
it('saves assessment as an I-frame in the personal mind', () => {
const assessment: MonthlyAssessment = {
period: '2026-03',
totalInteractions: 100,
correctionRate: 0.15,
improvementTrend: '+5%',
topStrengths: ['Low correction rate', 'Consistent positive user feedback'],
topWeaknesses: ['wrong answer'],
capabilityGapsDetected: ['web_search'],
skillsInstalled: 2,
recommendation: 'Agent is performing well overall.',
};
saveAssessmentToMind(db, assessment);
// Verify the frame was created
const frames = new FrameStore(db);
const allFrames = frames.getRecent(10);
expect(allFrames.length).toBe(1);
const frame = allFrames[0];
expect(frame.frame_type).toBe('I');
expect(frame.gop_id).toBe('assessment');
expect(frame.importance).toBe('important');
expect(frame.content).toContain('Monthly Agent Assessment');
expect(frame.content).toContain('2026-03');
expect(frame.content).toContain('15.0%');
expect(frame.content).toContain('+5%');
expect(frame.content).toContain('web_search');
});
it('a zero-data month writes nothing — no template-noise frames', () => {
saveAssessmentToMind(db, {
period: '2026-03', totalInteractions: 0, correctionRate: 0,
improvementTrend: '0%', topStrengths: ['Stable operation'], topWeaknesses: [],
capabilityGapsDetected: [], skillsInstalled: 0,
recommendation: 'No corrections recorded.',
});
expect(new FrameStore(db).getRecent(10)).toHaveLength(0);
});
it('re-running the same period replaces the frame instead of duplicating it', () => {
const assessment: MonthlyAssessment = {
period: '2026-03',
totalInteractions: 100,
correctionRate: 0.15,
improvementTrend: '+5%',
topStrengths: ['Low correction rate'],
topWeaknesses: [],
capabilityGapsDetected: [],
skillsInstalled: 0,
recommendation: 'Agent is performing well overall.',
};
saveAssessmentToMind(db, assessment);
saveAssessmentToMind(db, { ...assessment, totalInteractions: 120 });
// A different period must NOT be replaced — one frame per month.
saveAssessmentToMind(db, { ...assessment, period: '2026-04' });
const frames = new FrameStore(db);
const assessments = frames
.getRecent(10)
.filter((f) => f.content.startsWith('# Monthly Agent Assessment'));
expect(assessments).toHaveLength(2);
const march = assessments.filter((f) => f.content.includes('2026-03'));
expect(march).toHaveLength(1);
expect(march[0].content).toContain('**Interactions**: 120');
});
});
// ── Cron registration ───────────────────────────────────────────────
describe('cron registration', () => {
it('monthly_assessment is a valid cron job type', () => {
const cronStore = new CronStore(db);
// This should not throw — monthly_assessment is in VALID_JOB_TYPES
const schedule = cronStore.create({
name: 'Monthly assessment',
cronExpr: '0 6 1 * *',
jobType: 'monthly_assessment',
});
expect(schedule.name).toBe('Monthly assessment');
expect(schedule.cron_expr).toBe('0 6 1 * *');
expect(schedule.job_type).toBe('monthly_assessment');
});
it('cron expression fires on 1st of month at 6 AM', () => {
const cronStore = new CronStore(db);
const schedule = cronStore.create({
name: 'Monthly assessment',
cronExpr: '0 6 1 * *',
jobType: 'monthly_assessment',
});
// next_run_at should be set and valid
expect(schedule.next_run_at).toBeDefined();
expect(schedule.next_run_at).not.toBeNull();
// The next run should be on the 1st of some month at 6 AM local time
// cron-parser interprets expressions in local timezone
const nextRun = new Date(schedule.next_run_at!);
expect(nextRun.getDate()).toBe(1);
expect(nextRun.getHours()).toBe(6);
expect(nextRun.getMinutes()).toBe(0);
});
});
});

View File

@@ -20,7 +20,6 @@ const testState = vi.hoisted(() => {
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
return {
previousPromptAssembler,
optimizerExpand: vi.fn(),
runAgentLoop: vi.fn(),
};
});
@@ -41,12 +40,6 @@ vi.mock('../../src/local/model-availability.js', async (importOriginal) => {
};
});
vi.mock('../../src/local/services/optimizer-service.js', () => ({
getOptimizerService: async () => ({
expandWithChoices: testState.optimizerExpand,
}),
}));
function parseSse(raw: string): Array<{ event: string; data: Record<string, unknown> }> {
return raw.split(/\n\n/)
.filter(Boolean)
@@ -151,12 +144,6 @@ describe('persona acceptance prompt budget', () => {
beforeEach(() => {
resetRateLimiter(server);
testState.optimizerExpand.mockReset().mockResolvedValue({
expanded: null,
clarifyingQuestions: null,
intent: 'request',
isVague: false,
});
});
afterAll(async () => {
@@ -657,7 +644,6 @@ describe('persona acceptance prompt budget', () => {
expect(response.body).toContain(toolResultMarker);
expect(response.body).toContain(responseMarker);
expect(capturedConfig?.traceRecording).toBeUndefined();
expect(testState.optimizerExpand).not.toHaveBeenCalled();
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
@@ -740,7 +726,6 @@ describe('persona acceptance prompt budget', () => {
expect(response.statusCode).toBe(200);
expect(capturedConfig?.traceRecording).toBeUndefined();
expect(testState.optimizerExpand).not.toHaveBeenCalled();
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(

View File

@@ -14,10 +14,8 @@ import { afterEach, describe, it, expect } from 'vitest';
import Fastify from 'fastify';
import {
deleteCustomPersona,
deployPersonaOverride,
isValidCustomPersonaId,
listPersonas,
rollbackPersonaOverride,
saveCustomPersona,
type AgentPersona,
} from '@waggle/agent';
@@ -118,14 +116,12 @@ describe('Personas Routes', () => {
await server.close();
});
it('contains all 23 personas', async () => {
it('contains all 22 personas', async () => {
const server = createTestServer();
const res = await server.inject({ method: 'GET', url: '/api/personas' });
const { personas } = res.json();
// 22 tiered personas + session-reviewer (internal self-evolution reviewer;
// returned by the raw list but excluded from onboarding + PersonaSwitcher).
expect(personas.length).toBe(23);
expect(personas.length).toBe(22);
const ids = personas.map((p: { id: string }) => p.id);
expect(ids).toContain('researcher');
expect(ids).toContain('writer');
@@ -251,40 +247,5 @@ describe('Personas Routes', () => {
.toThrow('Invalid custom persona ID');
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
});
it('deployPersonaOverride rejects traversal before writing an override', () => {
const dataDir = makeTempDir();
const outsidePath = path.join(dataDir, 'evolved.json');
expect(() => deployPersonaOverride(dataDir, {
personaId: '../evolved',
systemPrompt: 'Escaped override',
})).toThrow('Invalid custom persona ID');
expect(fs.existsSync(outsidePath)).toBe(false);
});
it('deployPersonaOverride keeps its validated ID authoritative over overrides', () => {
const dataDir = makeTempDir();
const result = deployPersonaOverride(dataDir, {
personaId: 'safe-persona',
systemPrompt: 'Validated prompt',
overrides: { id: '../evolved', systemPrompt: 'Spoofed prompt' } as never,
});
const deployed = JSON.parse(fs.readFileSync(result.path, 'utf-8')) as AgentPersona;
expect(deployed.id).toBe('safe-persona');
expect(deployed.systemPrompt).toBe('Validated prompt');
expect(fs.existsSync(path.join(dataDir, 'evolved.json'))).toBe(false);
});
it('rollbackPersonaOverride rejects traversal before deleting an override', () => {
const dataDir = makeTempDir();
const outsidePath = path.join(dataDir, 'evolved.json');
fs.writeFileSync(outsidePath, 'sentinel', 'utf-8');
expect(() => rollbackPersonaOverride(dataDir, '../evolved'))
.toThrow('Invalid custom persona ID');
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
});
});
});

View File

@@ -1,57 +0,0 @@
/**
* session-reviewer persona — trust-boundary tool policy.
*
* The reviewer runs headless; ALWAYS_AVAILABLE_TOOLS (persona-tool-filter) re-adds
* write-side tools past any allowlist, so the persona's disallowedTools must strip
* the dangerous ones while KEEPING create_skill (its one proposal write) and the
* read tools. This test locks that against regression.
*/
import { describe, it, expect } from 'vitest';
import { getPersona, type ToolDefinition } from '@waggle/agent';
import { applyPersonaToolFilter } from '../../src/local/persona-tool-filter.js';
/** Minimal ToolDefinition stub — only `name` matters for the filter. */
function tool(name: string): ToolDefinition {
return { name, description: '', parameters: { type: 'object', properties: {} }, execute: async () => '' };
}
// A representative superset spanning reads, writes, memory, exec, and skill tools.
const POOL: ToolDefinition[] = [
'read_file', 'search_files', 'search_content',
'search_memory', 'save_memory', 'query_knowledge', 'get_identity', 'get_awareness',
'correct_knowledge', 'add_task',
'list_skills', 'search_skills', 'read_skill', 'create_skill', 'delete_skill',
'write_file', 'edit_file', 'generate_docx', 'bash',
'git_commit', 'git_push',
'spawn_agent', 'install_capability', 'acquire_capability',
'execute_step', 'compose_workflow', 'orchestrate_workflow',
].map(tool);
describe('session-reviewer persona', () => {
it('exists and is not read-only (so create_skill survives)', () => {
const p = getPersona('session-reviewer');
expect(p).toBeTruthy();
expect(p!.isReadOnly).toBe(false);
expect(p!.tools).toContain('create_skill');
});
it('applyPersonaToolFilter keeps reads + create_skill, strips writes/exec/memory', () => {
const p = getPersona('session-reviewer')!;
const names = new Set(applyPersonaToolFilter(POOL, p).map(t => t.name));
// Kept — the reviewer's read surface + its one proposal write.
for (const keep of ['read_file', 'search_content', 'search_memory', 'read_skill', 'list_skills', 'create_skill']) {
expect(names.has(keep), `expected ${keep} to survive`).toBe(true);
}
// Stripped — writes, exec, memory writes, and the ALWAYS_AVAILABLE re-adds.
for (const drop of [
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
'bash', 'spawn_agent', 'write_file', 'edit_file', 'git_commit',
'add_task', 'correct_knowledge', 'execute_step', 'compose_workflow',
]) {
expect(names.has(drop), `expected ${drop} to be stripped`).toBe(false);
}
});
});