refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -18,7 +18,6 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.78.0",
|
||||
"@ax-llm/ax": "^19.0.12",
|
||||
"@clerk/fastify": "^3.1.3",
|
||||
"@fastify/cors": "^10.0.0",
|
||||
"@fastify/static": "^10.1.2",
|
||||
|
||||
@@ -28,7 +28,7 @@ export interface ChatTurnRequest {
|
||||
/** Persisted session id — one per IM conversation. */
|
||||
session: string;
|
||||
timeoutMs?: number;
|
||||
/** Per-turn persona override (e.g. 'session-reviewer' for the idle watcher). */
|
||||
/** Per-turn persona override. */
|
||||
persona?: string;
|
||||
/**
|
||||
* Headless turn: hold gated proposable tools for human approval instead of
|
||||
|
||||
@@ -38,11 +38,11 @@ export function isProposableTool(tool: string): boolean {
|
||||
if (tool === 'send_email' || tool === 'write_file' || tool === 'edit_file' || tool === 'generate_docx') {
|
||||
return true;
|
||||
}
|
||||
// create_skill is the self-evolution proposal vehicle (session-reviewer). Held
|
||||
// here so a headless review turn can never write a skill to disk without human
|
||||
// approval; on approve, executeHeldAction runs the real create_skill tool from
|
||||
// the workspace pool, which persists through the sanctioned, backup-protected
|
||||
// writeSkill path (skill-tools.ts → skill-write-service.ts).
|
||||
// create_skill is held here so a headless turn can never write a skill to
|
||||
// disk without human approval; on approve, executeHeldAction runs the real
|
||||
// create_skill tool from the workspace pool, which persists through the
|
||||
// sanctioned, backup-protected writeSkill path (skill-tools.ts →
|
||||
// skill-write-service.ts).
|
||||
if (tool === 'create_skill') return true;
|
||||
// Connector WRITE actions (connector_<id>_<action> where action mutates).
|
||||
if (tool.startsWith('connector_') && /_(create|update|delete|send|post|transition|remove|add|set|put)(_|$)/.test(tool)) {
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
/**
|
||||
* IdleSessionWatcher — the runtime half of idle-triggered self-evolution
|
||||
* (CowAgent steal #3, v1 "review-before-apply only").
|
||||
*
|
||||
* A 60s daemon (mirrors LocalScheduler's start/stop/tick shape) that scans a
|
||||
* workspace's chat sessions for ones that have gone idle after a real
|
||||
* conversation, and — once per idle session — spawns a RESTRICTED reviewer
|
||||
* (persona `session-reviewer`) through a loopback chat turn. The reviewer reads
|
||||
* the transcript and either proposes a skill patch (held for human approval,
|
||||
* never written to disk autonomously) or replies NOTHING_TO_DO.
|
||||
*
|
||||
* Trust + cost guardrails:
|
||||
* - default OFF (self-evolution.json `enabled:false`) — founder opt-in.
|
||||
* - a session fires at most once per mtime (RAM fired-set keyed sessionId:mtime);
|
||||
* it only re-fires after the file advances (a new message lands).
|
||||
* - a per-day cap bounds spend.
|
||||
* - `channel-*` and `evolve-*` sessions are skipped (IM threads + the reviewer's
|
||||
* own loopback sessions must not be reviewed → no feedback loop).
|
||||
*
|
||||
* Enumeration is PURE (fs.statSync + raw line count). It deliberately does NOT
|
||||
* use readSessionMeta (session-utils.ts) — that lazily backfills a title/summary
|
||||
* onto disk, a write side effect a read-only scan must not trigger.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from './logger.js';
|
||||
import { materialFingerprint } from './notification-gate.js';
|
||||
import type { EmitNotificationOptions, EmitNotificationResult, NotificationEvent } from './routes/notifications.js';
|
||||
|
||||
const log = createLogger('idle-watcher');
|
||||
|
||||
/** Sentinel the reviewer emits when it finds nothing material — never notified. */
|
||||
export const NOTHING_TO_DO = 'NOTHING_TO_DO';
|
||||
|
||||
/** Config file under dataDir. Absent/corrupt ⇒ DEFAULT_CONFIG. */
|
||||
const CONFIG_FILE = 'self-evolution.json';
|
||||
|
||||
export interface SelfEvolutionConfig {
|
||||
/** Master switch — default OFF (founder opt-in). */
|
||||
enabled: boolean;
|
||||
/** Minutes a session must be idle before it is eligible for review. */
|
||||
idleMinutes: number;
|
||||
/** Minimum turns (message lines) — skip trivial/empty sessions. */
|
||||
minTurns: number;
|
||||
/** Hard ceiling on reviews launched per calendar day (spend guard). */
|
||||
maxReviewsPerDay: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: SelfEvolutionConfig = {
|
||||
enabled: false,
|
||||
idleMinutes: 15,
|
||||
minTurns: 6,
|
||||
maxReviewsPerDay: 5,
|
||||
};
|
||||
|
||||
/** Result of one review turn (from the injected runReviewTurn). */
|
||||
export interface ReviewTurnResult {
|
||||
content: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Everything the watcher needs, injected for testability. */
|
||||
export interface IdleSessionWatcherDeps {
|
||||
dataDir: string;
|
||||
/** Run one restricted review turn for a session; returns the reviewer's reply. */
|
||||
runReviewTurn: (input: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
|
||||
/** Bound emitNotification (server-scoped) — supports the anti-nag gate. */
|
||||
emitNotification: (
|
||||
event: Omit<NotificationEvent, 'type' | 'timestamp' | 'id' | 'read'>,
|
||||
options?: EmitNotificationOptions,
|
||||
) => EmitNotificationResult;
|
||||
log?: { info: (msg: string) => void; warn: (msg: string) => void };
|
||||
/** Clock injection point for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Newest N fired keys retained — bounds RAM for a long-running process. */
|
||||
const FIRED_SET_CAP = 500;
|
||||
|
||||
export class IdleSessionWatcher {
|
||||
private readonly deps: IdleSessionWatcherDeps;
|
||||
private readonly now: () => number;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private ticking = false;
|
||||
/** `${sessionId}:${mtimeMs}` — a session re-fires only after its file advances. */
|
||||
private readonly firedKeys = new Set<string>();
|
||||
/** Day-keyed review counter (RAM — acceptable v1; a restart resets the cap). */
|
||||
private dayCount = { day: '', count: 0 };
|
||||
|
||||
constructor(deps: IdleSessionWatcherDeps) {
|
||||
this.deps = deps;
|
||||
this.now = deps.now ?? Date.now;
|
||||
}
|
||||
|
||||
/** Start the tick loop. Default interval is 60 seconds. */
|
||||
start(intervalMs = 60_000): void {
|
||||
if (this.timer) return;
|
||||
this.timer = setInterval(() => {
|
||||
this.tick().catch(() => { /* tick swallows its own errors */ });
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/** Stop the tick loop. */
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.timer !== null;
|
||||
}
|
||||
|
||||
/** Read + validate config. Absent/corrupt/partial ⇒ merged onto DEFAULT_CONFIG. */
|
||||
private loadConfig(): SelfEvolutionConfig {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(path.join(this.deps.dataDir, CONFIG_FILE), 'utf-8')) as unknown;
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { ...DEFAULT_CONFIG };
|
||||
const r = raw as Partial<SelfEvolutionConfig>;
|
||||
return {
|
||||
enabled: typeof r.enabled === 'boolean' ? r.enabled : DEFAULT_CONFIG.enabled,
|
||||
idleMinutes: typeof r.idleMinutes === 'number' && r.idleMinutes > 0 ? r.idleMinutes : DEFAULT_CONFIG.idleMinutes,
|
||||
minTurns: typeof r.minTurns === 'number' && r.minTurns > 0 ? r.minTurns : DEFAULT_CONFIG.minTurns,
|
||||
maxReviewsPerDay: typeof r.maxReviewsPerDay === 'number' && r.maxReviewsPerDay >= 0 ? r.maxReviewsPerDay : DEFAULT_CONFIG.maxReviewsPerDay,
|
||||
};
|
||||
} catch {
|
||||
return { ...DEFAULT_CONFIG };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate eligible sessions across all workspaces. PURE — statSync + raw
|
||||
* line count, no readSessionMeta (that writes). Skips channel-/evolve- prefixes
|
||||
* and sub-threshold sessions.
|
||||
*/
|
||||
private findIdleSessions(cfg: SelfEvolutionConfig): Array<{ sessionId: string; workspaceId: string; mtimeMs: number }> {
|
||||
const workspacesDir = path.join(this.deps.dataDir, 'workspaces');
|
||||
let workspaceIds: string[];
|
||||
try {
|
||||
workspaceIds = fs.readdirSync(workspacesDir, { withFileTypes: true }).filter(d => d.isDirectory()).map(d => d.name);
|
||||
} catch {
|
||||
return []; // no workspaces dir yet
|
||||
}
|
||||
|
||||
const idleMs = cfg.idleMinutes * 60_000;
|
||||
const nowMs = this.now();
|
||||
const out: Array<{ sessionId: string; workspaceId: string; mtimeMs: number }> = [];
|
||||
|
||||
for (const workspaceId of workspaceIds) {
|
||||
const sessionsDir = path.join(workspacesDir, workspaceId, 'sessions');
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(sessionsDir).filter(f => f.endsWith('.jsonl'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const file of files) {
|
||||
const sessionId = file.slice(0, -'.jsonl'.length);
|
||||
// The reviewer's own loopback sessions (evolve-*) and IM threads
|
||||
// (channel-*) are never reviewed — reviewing evolve-* would loop.
|
||||
if (sessionId.startsWith('channel-') || sessionId.startsWith('evolve-')) continue;
|
||||
|
||||
const filePath = path.join(sessionsDir, file);
|
||||
let mtimeMs: number;
|
||||
let turnCount: number;
|
||||
try {
|
||||
mtimeMs = fs.statSync(filePath).mtimeMs;
|
||||
turnCount = countTurns(fs.readFileSync(filePath, 'utf-8'));
|
||||
} catch {
|
||||
continue; // unreadable — skip
|
||||
}
|
||||
|
||||
if (turnCount < cfg.minTurns) continue;
|
||||
if (nowMs - mtimeMs < idleMs) continue; // still active
|
||||
out.push({ sessionId, workspaceId, mtimeMs });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private incrementDayCounter(): void {
|
||||
const today = new Date(this.now()).toISOString().slice(0, 10);
|
||||
if (this.dayCount.day !== today) this.dayCount = { day: today, count: 0 };
|
||||
this.dayCount.count += 1;
|
||||
}
|
||||
|
||||
private dayCountRemaining(cfg: SelfEvolutionConfig): number {
|
||||
const today = new Date(this.now()).toISOString().slice(0, 10);
|
||||
const count = this.dayCount.day === today ? this.dayCount.count : 0;
|
||||
return cfg.maxReviewsPerDay - count;
|
||||
}
|
||||
|
||||
private markFired(key: string): void {
|
||||
this.firedKeys.add(key);
|
||||
while (this.firedKeys.size > FIRED_SET_CAP) {
|
||||
const oldest = this.firedKeys.values().next().value as string | undefined;
|
||||
if (oldest === undefined) break;
|
||||
this.firedKeys.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One tick: find idle sessions, review the ones not yet fired (within the day
|
||||
* cap), notify only on a material, non-empty finding. Single-flight guarded.
|
||||
*/
|
||||
async tick(): Promise<number> {
|
||||
if (this.ticking) return 0;
|
||||
this.ticking = true;
|
||||
let reviewed = 0;
|
||||
try {
|
||||
const cfg = this.loadConfig();
|
||||
if (!cfg.enabled) return 0; // opt-in — no-op when disabled
|
||||
|
||||
const candidates = this.findIdleSessions(cfg);
|
||||
for (const { sessionId, workspaceId, mtimeMs } of candidates) {
|
||||
const key = `${sessionId}:${mtimeMs}`;
|
||||
if (this.firedKeys.has(key)) continue; // already reviewed at this mtime
|
||||
if (this.dayCountRemaining(cfg) <= 0) {
|
||||
(this.deps.log ?? log).info(`[idle-watcher] daily review cap (${cfg.maxReviewsPerDay}) reached — deferring`);
|
||||
break;
|
||||
}
|
||||
|
||||
// Count the review against the cap and mark fired BEFORE the turn so a
|
||||
// NOTHING_TO_DO result still consumes its slot and won't immediately re-fire.
|
||||
this.incrementDayCounter();
|
||||
this.markFired(key);
|
||||
reviewed += 1;
|
||||
|
||||
let result: ReviewTurnResult;
|
||||
try {
|
||||
result = await this.deps.runReviewTurn({ sessionId, workspaceId });
|
||||
} catch (err) {
|
||||
(this.deps.log ?? log).warn(`[idle-watcher] review turn threw for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = (result.content ?? '').trim();
|
||||
if (result.error || content === '' || content === NOTHING_TO_DO) {
|
||||
continue; // nothing material — stay silent
|
||||
}
|
||||
|
||||
// Material finding — notify, anti-nag gated so an identical finding for
|
||||
// the same session on a later tick is suppressed.
|
||||
this.deps.emitNotification(
|
||||
{
|
||||
title: 'Self-evolution: a review found something',
|
||||
body: content.length > 200 ? `${content.slice(0, 197)}...` : content,
|
||||
category: 'agent',
|
||||
actionUrl: '/approvals',
|
||||
},
|
||||
{ dedupeKey: `self-evolution:${sessionId}`, materialHash: materialFingerprint(content) },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
this.ticking = false;
|
||||
}
|
||||
return reviewed;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a bounded, compact transcript of a session's most recent messages.
|
||||
* PURE (readFileSync only). Returns null if unreadable or too short.
|
||||
*
|
||||
* The reviewer runs in a SEPARATE `evolve-*` session and read_file is jailed to
|
||||
* the workspace `files/` dir (resolveSafe), so it cannot open the session JSONL
|
||||
* itself. We therefore embed the transcript in the review message. (Deviation
|
||||
* from the spec's "it has read tools + workspace binding" — the sandbox blocks a
|
||||
* cross-directory read, so providing the transcript is the reliable path.)
|
||||
*/
|
||||
export function readRecentTranscript(
|
||||
dataDir: string,
|
||||
workspaceId: string,
|
||||
sessionId: string,
|
||||
maxMessages = 40,
|
||||
perMessageChars = 1500,
|
||||
): string | null {
|
||||
const filePath = path.join(dataDir, 'workspaces', workspaceId, 'sessions', `${sessionId}.jsonl`);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(filePath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const lines = raw.split('\n').filter(l => l.trim());
|
||||
const messages: string[] = [];
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = JSON.parse(line) as { type?: string; role?: string; content?: string };
|
||||
if (parsed.type === 'meta') continue;
|
||||
if (!parsed.role || !parsed.content) continue;
|
||||
const role = parsed.role.toUpperCase();
|
||||
const content = parsed.content.length > perMessageChars
|
||||
? `${parsed.content.slice(0, perMessageChars)}…`
|
||||
: parsed.content;
|
||||
messages.push(`${role}: ${content}`);
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
if (messages.length === 0) return null;
|
||||
return messages.slice(-maxMessages).join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* The review instruction sent to the `session-reviewer` persona. Embeds the
|
||||
* transcript and states the two things to look for, the default-silent contract,
|
||||
* and the never-invent rule.
|
||||
*/
|
||||
export function buildReviewInstruction(sessionId: string, transcript: string): string {
|
||||
return [
|
||||
`You are reviewing the finished session "${sessionId}". Its recent conversation is below, between the markers.`,
|
||||
'',
|
||||
'--- BEGIN SESSION TRANSCRIPT ---',
|
||||
transcript,
|
||||
'--- END SESSION TRANSCRIPT ---',
|
||||
'',
|
||||
'Examine ONLY the transcript above and look for exactly two things:',
|
||||
'1. Promised-but-undelivered deliverables — the assistant said it would do or produce something and never did.',
|
||||
'2. Recurring capability failures fixable by a skill patch — the same tool/skill/workflow failed more than once.',
|
||||
'',
|
||||
'If — and only if — you find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence: state the finding with its evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill (it will be held for the human to approve — it is NOT written to disk now).',
|
||||
`Otherwise reply with exactly: ${NOTHING_TO_DO}`,
|
||||
'Never invent a finding. If you cannot cite the transcript, reply NOTHING_TO_DO.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn count for a session JSONL: non-empty lines, minus the leading meta line
|
||||
* if present. Pure — parses only the first line to detect meta, never writes.
|
||||
*/
|
||||
export function countTurns(raw: string): number {
|
||||
const lines = raw.split('\n').filter(l => l.trim());
|
||||
if (lines.length === 0) return 0;
|
||||
try {
|
||||
const first = JSON.parse(lines[0]) as { type?: string };
|
||||
if (first?.type === 'meta') return lines.length - 1;
|
||||
} catch {
|
||||
// First line isn't JSON meta — count every non-empty line.
|
||||
}
|
||||
return lines.length;
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import websocket from '@fastify/websocket';
|
||||
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, OptimizationLogStore, ImprovementSignalStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, EvolutionRunStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
|
||||
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
|
||||
import { corsOriginAllowed } from './cors-config.js';
|
||||
import { getBoundTeamServer } from './team-server-binding.js';
|
||||
import { fetchTeamServer } from './team-server-egress.js';
|
||||
@@ -46,18 +46,14 @@ import {
|
||||
createBrowserTools,
|
||||
createLspTools,
|
||||
createCliTools,
|
||||
createInsightsTools,
|
||||
createConnectorSearchTools,
|
||||
createCrossWorkspaceTools,
|
||||
McpRuntime,
|
||||
McpToolRetriever,
|
||||
isWithinBudget,
|
||||
getRecentLogs,
|
||||
setPersonaDataDir,
|
||||
deliverCronResult,
|
||||
createDefaultDeliveryPreferences,
|
||||
buildActiveBehavioralSpec,
|
||||
loadBehavioralSpecOverrides,
|
||||
TraceRecorder,
|
||||
HarnessTraceBridge,
|
||||
safeFetch,
|
||||
@@ -108,7 +104,6 @@ import { homeRoutes } from './routes/home.js';
|
||||
import { onboardingRoutes } from './routes/onboarding.js';
|
||||
import { cronRoutes } from './routes/cron.js';
|
||||
import { notificationRoutes, emitNotification, emitSubagentStatus } from './routes/notifications.js';
|
||||
import { materialFingerprint } from './notification-gate.js';
|
||||
import { marketplaceDevRoutes } from './routes/marketplace-dev.js';
|
||||
import { marketplaceRoutes } from './routes/marketplace.js';
|
||||
import {
|
||||
@@ -129,7 +124,6 @@ import { personaRoutes } from './routes/personas.js';
|
||||
import { feedbackRoutes } from './routes/feedback.js';
|
||||
import { workflowRoutes } from './routes/workflows.js';
|
||||
import { workspaceTemplateRoutes } from './routes/workspace-templates.js';
|
||||
import { evolutionRoutes } from './routes/evolution.js';
|
||||
import { exportRoutes } from './routes/export.js';
|
||||
import { dataEraseRoutes } from './routes/data-erase.js';
|
||||
import { costRoutes } from './routes/cost.js';
|
||||
@@ -154,7 +148,6 @@ import { runChannelChatTurn } from './channels/chat-client.js';
|
||||
import { channelRoutes } from './channels/routes.js';
|
||||
import { isChannelPlatform } from './channels/types.js';
|
||||
import { WorkspaceTurnCoordinator } from './workspace-turn-coordinator.js';
|
||||
import { IdleSessionWatcher, readRecentTranscript, buildReviewInstruction, NOTHING_TO_DO } from './idle-watcher.js';
|
||||
import { DreamJournal } from './dream-journal.js';
|
||||
import { dreamRoutes } from './routes/dreams.js';
|
||||
import { oauthRoutes } from './routes/oauth.js';
|
||||
@@ -183,7 +176,6 @@ import {
|
||||
securityMiddleware,
|
||||
} from './security-middleware.js';
|
||||
import { LocalScheduler, makeRecordExecutionCallback } from './cron.js';
|
||||
import { EvolutionService, isEvolutionAutoEnabled } from './services/evolution-service.js';
|
||||
import {
|
||||
generateMorningBriefing,
|
||||
checkStaleWorkspaces,
|
||||
@@ -192,7 +184,6 @@ import {
|
||||
type ProactiveContext,
|
||||
type ProactiveMessage,
|
||||
} from './proactive-handlers.js';
|
||||
import { generateMonthlyAssessment, saveAssessmentToMind } from './monthly-assessment.js';
|
||||
import { MAX_WORKSPACE_SESSIONS, WorkspaceSessionManager } from './workspace-sessions.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { LocalJobStore } from './job-store.js';
|
||||
@@ -392,21 +383,12 @@ declare module 'fastify' {
|
||||
auditStore: import('@waggle/core').InstallAuditStore;
|
||||
cronStore: import('@waggle/core').CronStore;
|
||||
traceStore: import('@waggle/core').ExecutionTraceStore;
|
||||
evolutionStore: import('@waggle/core').EvolutionRunStore;
|
||||
/**
|
||||
* Active behavioral spec — baseline `BEHAVIORAL_SPEC` with any
|
||||
* accepted-and-deployed evolution overrides applied. Re-derived on
|
||||
* the 'behavioral-spec:reloaded' event.
|
||||
*/
|
||||
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
|
||||
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
|
||||
vault: import('@waggle/core').VaultStore;
|
||||
embeddingProvider: import('@waggle/core').EmbeddingProviderInstance;
|
||||
skillHashStore: import('@waggle/core').SkillHashStore;
|
||||
scheduler: import('./cron.js').LocalScheduler;
|
||||
/** Idle-session self-evolution watcher. Inert unless self-evolution.json enabled. */
|
||||
idleWatcher: import('./idle-watcher.js').IdleSessionWatcher;
|
||||
/** H-10 G1: autonomous evolution daemon. Inert unless WAGGLE_EVOLUTION_AUTO_ENABLED=1. */
|
||||
evolutionService: import('./services/evolution-service.js').EvolutionService;
|
||||
marketplace: import('@waggle/marketplace').MarketplaceDB | null;
|
||||
/** Shared per-workspace mind cache. Routes use this to reach an arbitrary
|
||||
* workspace's mind (e.g. FileIndexer for L-20 auto-indexing). */
|
||||
@@ -418,7 +400,7 @@ declare module 'fastify' {
|
||||
sessionManager: import('./workspace-sessions.js').WorkspaceSessionManager;
|
||||
/**
|
||||
* Bridges workflow-harness phase events into the execution trace store
|
||||
* so harnessed runs become training signal for the self-evolution loop.
|
||||
* so harnessed runs show up in cost tracking and run history.
|
||||
* Started at boot, stopped on close.
|
||||
*/
|
||||
harnessTraceBridge: HarnessTraceBridge;
|
||||
@@ -533,14 +515,14 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
|
||||
seedDefaultCrons(cronStore);
|
||||
|
||||
// Execution trace store — persistent log of agent runs (self-evolution substrate)
|
||||
// Execution trace store — persistent log of agent runs (cost tracking, run history)
|
||||
const traceStore = new ExecutionTraceStore(multiMind.personal);
|
||||
server.decorate('traceStore', traceStore);
|
||||
|
||||
// Workflow-harness → trace bridge. Any harness phase that completes or
|
||||
// aborts anywhere in the app emits on the shared `harnessEvents` emitter;
|
||||
// this bridge translates those into rows in `execution_traces` so harness
|
||||
// runs become training data for the self-evolution loop.
|
||||
// runs show up in cost tracking.
|
||||
const harnessTraceBridge = new HarnessTraceBridge({
|
||||
recorder: new TraceRecorder(traceStore),
|
||||
});
|
||||
@@ -550,22 +532,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
harnessTraceBridge.stop();
|
||||
});
|
||||
|
||||
// Evolution run store — audit trail of proposed/accepted/rejected self-evolution runs
|
||||
const evolutionStore = new EvolutionRunStore(multiMind.personal);
|
||||
server.decorate('evolutionStore', evolutionStore);
|
||||
|
||||
// Behavioral spec with deployed overrides applied (self-evolution deploy target).
|
||||
// Re-derived on the 'behavioral-spec:reloaded' event emitted by evolution routes.
|
||||
const initialOverrides = loadBehavioralSpecOverrides(fullConfig.dataDir);
|
||||
let activeBehavioralSpec = buildActiveBehavioralSpec(initialOverrides);
|
||||
server.decorate('activeBehavioralSpec', activeBehavioralSpec);
|
||||
eventBus.on('behavioral-spec:reloaded', () => {
|
||||
const latest = loadBehavioralSpecOverrides(fullConfig.dataDir);
|
||||
activeBehavioralSpec = buildActiveBehavioralSpec(latest);
|
||||
// Re-decorate so server.activeBehavioralSpec returns the fresh value.
|
||||
// Fastify's decorate() is idempotent for reassignment via the instance.
|
||||
(server as unknown as { activeBehavioralSpec: typeof activeBehavioralSpec }).activeBehavioralSpec = activeBehavioralSpec;
|
||||
});
|
||||
server.decorate('activeBehavioralSpec', buildActiveBehavioralSpec());
|
||||
|
||||
// Vault — encrypted secret storage
|
||||
const vault = new VaultStore(fullConfig.dataDir);
|
||||
@@ -819,86 +786,13 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Dynamic connector tools (initial — regenerated per workspace in buildToolsForWorkspace)
|
||||
const defaultConnectorTools = connectorRegistry.generateTools();
|
||||
|
||||
// Insights tools — self-analytics so the agent can reflect on its own performance.
|
||||
// Aggregates optimization logs + improvement signals across the personal mind and
|
||||
// every workspace so the agent sees a complete picture regardless of where it runs.
|
||||
const insightsTools = createInsightsTools({
|
||||
getOptimizationLogs: (limit: number) => {
|
||||
const rows: Array<{
|
||||
tools_used: string;
|
||||
was_correction: number;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
timestamp: string;
|
||||
}> = [];
|
||||
try {
|
||||
rows.push(...new OptimizationLogStore(multiMind.personal).getRecent(limit));
|
||||
} catch { /* personal store unavailable — keep going */ }
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
try {
|
||||
rows.push(...new OptimizationLogStore(wsDb).getRecent(limit));
|
||||
} catch { /* workspace store unavailable — skip */ }
|
||||
}
|
||||
rows.sort((a, b) => (b.timestamp ?? '').localeCompare(a.timestamp ?? ''));
|
||||
return rows.slice(0, limit).map(row => {
|
||||
let toolNames = '';
|
||||
try {
|
||||
const parsed = JSON.parse(row.tools_used ?? '[]');
|
||||
if (Array.isArray(parsed)) toolNames = parsed.join(',');
|
||||
} catch { /* malformed — leave empty */ }
|
||||
return {
|
||||
model: undefined,
|
||||
tool_names: toolNames,
|
||||
total_tokens: (row.input_tokens ?? 0) + (row.output_tokens ?? 0),
|
||||
estimated_cost: 0,
|
||||
was_correction: row.was_correction,
|
||||
created_at: row.timestamp,
|
||||
};
|
||||
});
|
||||
},
|
||||
getImprovementSignals: () => {
|
||||
const collectFrom = (db: MindDB) => {
|
||||
const store = new ImprovementSignalStore(db);
|
||||
return [
|
||||
...store.getByCategory('capability_gap'),
|
||||
...store.getByCategory('correction'),
|
||||
...store.getByCategory('workflow_pattern'),
|
||||
];
|
||||
};
|
||||
const all: Array<{
|
||||
category: string;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
}> = [];
|
||||
try { all.push(...collectFrom(multiMind.personal)); } catch { /* skip */ }
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
try { all.push(...collectFrom(wsDb)); } catch { /* skip */ }
|
||||
}
|
||||
return all.map(s => ({
|
||||
category: s.category,
|
||||
pattern_key: s.pattern_key,
|
||||
detail: s.detail,
|
||||
count: s.count,
|
||||
first_seen: s.first_seen,
|
||||
last_seen: s.last_seen,
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
// Connector search tools — give the agent natural-language discovery over
|
||||
// the 148-entry MCP catalog (find_connector, list_connector_categories).
|
||||
// Static data, zero I/O, no deps — cheap to always-include.
|
||||
const connectorSearchTools = createConnectorSearchTools();
|
||||
|
||||
// Collect all non-subagent tools first (sub-agent tools need the full list)
|
||||
const baseTools = [...mindTools, ...systemTools, ...planTools, ...gitTools, ...documentTools, ...skillTools, ...cronTools, ...searchTools, ...browserTools, ...lspTools, ...cliTools, ...insightsTools, ...connectorSearchTools, ...defaultConnectorTools];
|
||||
const baseTools = [...mindTools, ...systemTools, ...planTools, ...gitTools, ...documentTools, ...skillTools, ...cronTools, ...searchTools, ...browserTools, ...lspTools, ...cliTools, ...connectorSearchTools, ...defaultConnectorTools];
|
||||
|
||||
// PR4: expose the full native-tool name union to the agent-search route so
|
||||
// its "you already have a tool for this" lane isn't blind to search/browser/
|
||||
@@ -945,7 +839,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
const targetDb = wsDb ?? multiMind.personal;
|
||||
const mindLabel = wsDb ? `workspace:${wsId}` : 'personal';
|
||||
|
||||
// Structured content: tag provenance so search/evolution can filter
|
||||
// Structured content: tag provenance so search can filter
|
||||
// for sub-agent-derived frames and so humans can see where it came from.
|
||||
const content = `[Sub-agent: ${result.role}] ${result.agentName}\n\n${result.response}\n\n— tools: ${result.toolsUsed.join(', ') || 'none'} · ${(result.duration / 1000).toFixed(1)}s · ${result.usage.inputTokens + result.usage.outputTokens} tokens`;
|
||||
|
||||
@@ -1874,7 +1768,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// W4.3d: extraction of the benchmark-proven recall lanes
|
||||
// (facts/events/profiles) over frames written since the last run.
|
||||
// Routes through the built-in proxy ('fast' tier = haiku) — the
|
||||
// same internal-LLM pattern as prompt_optimization above.
|
||||
// same internal-LLM pattern as the other cron LLM calls.
|
||||
try {
|
||||
const llmCall = async (prompt: string, _model: 'fast' | 'accurate'): Promise<string> => {
|
||||
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
@@ -2041,143 +1935,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'prompt_optimization': {
|
||||
// Background GEPA/Ax prompt optimization — runs daily at 2 AM.
|
||||
// Iterates workspaces with optimizationEnabled, checks budget, reads logs,
|
||||
// and invokes PromptOptimizer for variant generation.
|
||||
try {
|
||||
const workspaces = wsManager.list().filter(ws => ws.optimizationEnabled);
|
||||
for (const ws of workspaces) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
|
||||
const optStore = new OptimizationLogStore(wsDb);
|
||||
const budget = ws.optimizationBudget ?? 100; // cents, default $1/day
|
||||
|
||||
// Check budget before spending any tokens
|
||||
if (!isWithinBudget(optStore, budget)) {
|
||||
log.info(`[cron] Prompt optimization: workspace "${ws.name}" over budget, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Read recent logs for analysis
|
||||
const recentLogs = getRecentLogs(optStore, 100);
|
||||
if (recentLogs.length < 5) {
|
||||
// Not enough data to optimize yet
|
||||
continue;
|
||||
}
|
||||
|
||||
// Analyze patterns: high correction rate, common tools, avg turn count
|
||||
const stats = optStore.getStats();
|
||||
const correctionRate = stats.correctionRate;
|
||||
const avgTurns = stats.avgTurnCount;
|
||||
|
||||
// Only trigger optimization if correction rate is above threshold
|
||||
// or turn count is significantly above average (indicating inefficiency)
|
||||
if (correctionRate > 0.2 || avgTurns > 15) {
|
||||
const rate = (correctionRate * 100).toFixed(1);
|
||||
log.info(`[cron] Prompt optimization: workspace "${ws.name}" — correction rate ${rate}%, avg turns ${avgTurns.toFixed(1)}`);
|
||||
|
||||
// Re-check budget before the LLM call (another workspace may have consumed tokens)
|
||||
if (!isWithinBudget(optStore, budget)) {
|
||||
log.info(` GEPA optimization skipped — daily budget exceeded`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get the most recent system prompt from the workspace's logs
|
||||
const currentSystemPrompt = recentLogs[0]?.system_prompt ?? '';
|
||||
if (!currentSystemPrompt) {
|
||||
log.info(`[cron] Prompt optimization: no system prompt in logs, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate variant via the built-in Anthropic proxy (localhost)
|
||||
try {
|
||||
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
const variantResponse = await fetch(proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'claude-sonnet-4-6',
|
||||
max_tokens: 4096,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You are a prompt optimization specialist. Your task is to improve an AI agent's system prompt based on performance signals.
|
||||
|
||||
The agent's correction rate is ${rate}% (target: under 20%) and average turn count is ${avgTurns.toFixed(1)} (target: under 15).
|
||||
|
||||
A high correction rate means the agent frequently misunderstands instructions or produces wrong outputs.
|
||||
A high turn count means the agent takes too many steps to complete tasks.
|
||||
|
||||
Analyze the current system prompt and generate an improved variant that:
|
||||
1. Adds clearer instructions for areas where corrections are common
|
||||
2. Reduces unnecessary verbosity that inflates turn count
|
||||
3. Preserves the core identity and capabilities
|
||||
4. Is production-ready (not a draft or explanation)
|
||||
|
||||
Return ONLY the improved system prompt text. No commentary, no markdown fences, no explanation.`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `Here is the current system prompt to improve:\n\n${currentSystemPrompt.slice(0, 12000)}`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
|
||||
if (!variantResponse.ok) {
|
||||
log.warn(`[cron] GEPA variant generation failed: HTTP ${variantResponse.status}`);
|
||||
} else {
|
||||
const variantBody = await variantResponse.json() as unknown;
|
||||
const variantText = parseOpenAiTextCompletion(variantBody).content;
|
||||
|
||||
if (variantText.length > 100) {
|
||||
// Store the variant in the optimization_log with a marker
|
||||
optStore.insert({
|
||||
sessionId: `gepa-variant-${Date.now()}`,
|
||||
workspaceId: ws.id,
|
||||
systemPrompt: variantText,
|
||||
toolsUsed: ['gepa_variant'],
|
||||
turnCount: 0,
|
||||
wasCorrection: false,
|
||||
inputTokens: currentSystemPrompt.length,
|
||||
outputTokens: variantText.length,
|
||||
});
|
||||
log.info(` GEPA generated prompt variant (correction_rate=${rate}%)`);
|
||||
} else {
|
||||
log.warn(`[cron] GEPA variant too short (${variantText.length} chars), discarding`);
|
||||
}
|
||||
}
|
||||
} catch (variantErr) {
|
||||
log.warn(`[cron] GEPA variant generation error: ${(variantErr as Error).message}`);
|
||||
}
|
||||
|
||||
// Anti-nag: one signal per (workspace, month, correction-rate bucket).
|
||||
// Re-runs on the same month with an unchanged rate bucket are suppressed.
|
||||
emitNotification(server, {
|
||||
title: 'Prompt optimization signal',
|
||||
body: `Workspace "${ws.name}" has ${(correctionRate * 100).toFixed(0)}% correction rate — optimization candidate`,
|
||||
category: 'agent',
|
||||
}, {
|
||||
dedupeKey: `prompt_optimization:${ws.id}`,
|
||||
materialHash: materialFingerprint({
|
||||
wsId: ws.id,
|
||||
period: new Date().toISOString().slice(0, 7),
|
||||
rateBucket: Math.round(correctionRate * 10),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// Prune old logs (>30 days) to keep the .mind file lean
|
||||
optStore.pruneOlderThan(30);
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Prompt optimization failed: ${(err as Error).message}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'agent_task': {
|
||||
// F9: Execute an agent task prompt in a workspace context
|
||||
const taskConfig = JSON.parse(schedule.job_config || '{}');
|
||||
@@ -2363,31 +2120,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'monthly_assessment': {
|
||||
try {
|
||||
const assessment = generateMonthlyAssessment(fullConfig, multiMind.personal);
|
||||
saveAssessmentToMind(multiMind.personal, assessment);
|
||||
log.info(`[cron] Monthly assessment for ${assessment.period}: ${assessment.totalInteractions} interactions, ${(assessment.correctionRate * 100).toFixed(1)}% correction rate`);
|
||||
emitNotification(server, {
|
||||
title: 'Monthly agent assessment ready',
|
||||
body: `${assessment.period} report: ${assessment.totalInteractions} interactions, ${(assessment.correctionRate * 100).toFixed(1)}% correction rate — check your workspace home`,
|
||||
category: 'agent',
|
||||
actionUrl: '/',
|
||||
}, {
|
||||
// Anti-nag: one per period + counters. Same month with unchanged
|
||||
// counters is suppressed; a new month or changed numbers re-fires.
|
||||
dedupeKey: 'monthly_assessment',
|
||||
materialHash: materialFingerprint({
|
||||
period: assessment.period,
|
||||
totalInteractions: assessment.totalInteractions,
|
||||
correctionRateBucket: Math.round(assessment.correctionRate * 100),
|
||||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Monthly assessment failed: ${(err as Error).message}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'loop': {
|
||||
// Loop v0 (L1, report-only): a stateful, memory-powered scheduled
|
||||
// automation. Resolve the target mind (a specific workspace, else the
|
||||
@@ -2509,66 +2241,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
scheduler.start();
|
||||
server.decorate('scheduler', scheduler);
|
||||
|
||||
// Self-evolution: idle-session watcher (CowAgent steal #3, v1). Default OFF via
|
||||
// <dataDir>/self-evolution.json; when enabled, it reviews idle sessions through
|
||||
// a restricted `session-reviewer` loopback turn whose only write (create_skill)
|
||||
// is HELD for human approval. Skipped under test to keep unit runs hermetic
|
||||
// (mirrors the ChannelManager VITEST guard below).
|
||||
const idleWatcher = new IdleSessionWatcher({
|
||||
dataDir: fullConfig.dataDir,
|
||||
emitNotification: (event, options) => emitNotification(server, event, options),
|
||||
log: { info: (msg: string) => log.info(msg), warn: (msg: string) => log.warn(msg) },
|
||||
runReviewTurn: async ({ sessionId, workspaceId }) => {
|
||||
const transcript = readRecentTranscript(fullConfig.dataDir, workspaceId, sessionId);
|
||||
if (!transcript) return { content: NOTHING_TO_DO };
|
||||
const res = await runChannelChatTurn({
|
||||
port: server.localConfig.port,
|
||||
sessionToken: server.agentState.wsSessionToken,
|
||||
message: buildReviewInstruction(sessionId, transcript),
|
||||
workspace: workspaceId,
|
||||
session: `evolve-${sessionId}`,
|
||||
persona: 'session-reviewer',
|
||||
proposeHeld: true,
|
||||
origin: 'automation',
|
||||
});
|
||||
return { content: res.content, ...(res.error ? { error: res.error } : {}) };
|
||||
},
|
||||
});
|
||||
server.decorate('idleWatcher', idleWatcher);
|
||||
if (!process.env.VITEST && process.env.NODE_ENV !== 'test') {
|
||||
idleWatcher.start();
|
||||
}
|
||||
server.addHook('onClose', async () => {
|
||||
idleWatcher.stop();
|
||||
});
|
||||
|
||||
// H-10 G1: Evolution service — autonomous self-evolution daemon.
|
||||
// Disabled by default; opt-in via WAGGLE_EVOLUTION_AUTO_ENABLED=1 so
|
||||
// existing deployments don't start burning Anthropic credits silently.
|
||||
// Even when enabled, the service only produces `proposed` runs — accept
|
||||
// still happens manually via the Memory → Evolution UI.
|
||||
const evolutionService = new EvolutionService(
|
||||
{
|
||||
traceStore,
|
||||
runStore: evolutionStore,
|
||||
getApiKey: () => server.vault?.get('anthropic')?.value ?? null,
|
||||
getActiveBehavioralSpec: () => (server as unknown as { activeBehavioralSpec: Record<string, unknown> }).activeBehavioralSpec,
|
||||
log: (level, msg) => {
|
||||
if (level === 'warn') log.warn(`[evolution-service] ${msg}`);
|
||||
else log.info(`[evolution-service] ${msg}`);
|
||||
},
|
||||
},
|
||||
{
|
||||
tickIntervalMs: parseInt(process.env.WAGGLE_EVOLUTION_TICK_INTERVAL_MS ?? '', 10) || undefined,
|
||||
minTracesPerTarget: parseInt(process.env.WAGGLE_EVOLUTION_MIN_TRACES ?? '', 10) || undefined,
|
||||
},
|
||||
);
|
||||
server.decorate('evolutionService', evolutionService);
|
||||
if (isEvolutionAutoEnabled()) {
|
||||
evolutionService.start();
|
||||
log.info('[evolution-service] autonomous evolution enabled (WAGGLE_EVOLUTION_AUTO_ENABLED=1)');
|
||||
}
|
||||
|
||||
// F2: Daily audit event cleanup (retain 90 days by default)
|
||||
const auditRetentionDays = parseInt(process.env.WAGGLE_AUDIT_RETENTION_DAYS ?? '90', 10);
|
||||
const auditCleanupTimer = setInterval(() => {
|
||||
@@ -2807,7 +2479,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
await server.register(personaRoutes);
|
||||
await server.register(feedbackRoutes);
|
||||
await server.register(workspaceTemplateRoutes);
|
||||
await server.register(evolutionRoutes);
|
||||
await server.register(exportRoutes);
|
||||
await server.register(dataEraseRoutes);
|
||||
await server.register(costRoutes);
|
||||
@@ -3288,7 +2959,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
agentRunRegistry.close();
|
||||
stopMarketplaceBackgroundSync();
|
||||
scheduler.stop();
|
||||
evolutionService.stop();
|
||||
await offlineManager.stop();
|
||||
clearInterval(auditCleanupTimer);
|
||||
|
||||
|
||||
@@ -1,346 +0,0 @@
|
||||
/**
|
||||
* Monthly Self-Assessment — generates a structured report of agent performance.
|
||||
*
|
||||
* Reads optimization logs, feedback stats, and capability gap signals from
|
||||
* the personal mind to produce a monthly assessment. Designed to be called
|
||||
* by the cron scheduler on the 1st of each month.
|
||||
*
|
||||
* The assessment is saved as a memory frame (I-frame) in the personal mind
|
||||
* so it becomes part of the agent's long-term self-awareness.
|
||||
*/
|
||||
|
||||
import { MindDB, OptimizationLogStore, FrameStore, SessionStore, ImprovementSignalStore } from '@waggle/core';
|
||||
import type { LocalConfig } from './index.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface MonthlyAssessment {
|
||||
period: string; // "2026-03"
|
||||
totalInteractions: number;
|
||||
correctionRate: number;
|
||||
improvementTrend: string;
|
||||
topStrengths: string[];
|
||||
topWeaknesses: string[];
|
||||
capabilityGapsDetected: string[];
|
||||
skillsInstalled: number;
|
||||
recommendation: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a user-visible local calendar month to UTC SQLite bounds.
|
||||
* Stored timestamps use SQLite datetime('now') (UTC), while monthly reports
|
||||
* follow the same local calendar as the cron schedule.
|
||||
*/
|
||||
function getUtcMonthBounds(yearMonth: string): { startDate: string; endDate: string } {
|
||||
const [year, month] = yearMonth.split('-').map(Number);
|
||||
const toSqliteUtc = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');
|
||||
|
||||
return {
|
||||
startDate: toSqliteUtc(new Date(year, month - 1, 1)),
|
||||
endDate: toSqliteUtc(new Date(year, month, 1)),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compute a correction rate for a given month from optimization logs.
|
||||
*/
|
||||
function computeMonthCorrectionRate(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): { total: number; correctionRate: number } {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate
|
||||
FROM optimization_log
|
||||
WHERE timestamp >= ? AND timestamp < ?
|
||||
`).get(startDate, endDate) as { total: number; correction_rate: number } | undefined;
|
||||
|
||||
return {
|
||||
total: row?.total ?? 0,
|
||||
correctionRate: row?.correction_rate ?? 0,
|
||||
};
|
||||
} catch {
|
||||
return { total: 0, correctionRate: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a correction rate for the prior month for trend comparison.
|
||||
*/
|
||||
function computePriorMonthCorrectionRate(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): number {
|
||||
const [year, month] = yearMonth.split('-').map(Number);
|
||||
const priorMonth = month === 1 ? 12 : month - 1;
|
||||
const priorYear = month === 1 ? year - 1 : year;
|
||||
const priorYearMonth = `${priorYear}-${String(priorMonth).padStart(2, '0')}`;
|
||||
const { correctionRate } = computeMonthCorrectionRate(db, priorYearMonth);
|
||||
return correctionRate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get top feedback reasons from the feedback_entries table for the month.
|
||||
*/
|
||||
function getTopFeedbackReasons(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): { positiveReasons: string[]; negativeReasons: string[] } {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
const negativeReasons: string[] = [];
|
||||
const positiveReasons: string[] = [];
|
||||
|
||||
try {
|
||||
const negRows = db.prepare(`
|
||||
SELECT reason, COUNT(*) as count
|
||||
FROM feedback_entries
|
||||
WHERE rating = 'down' AND reason IS NOT NULL
|
||||
AND created_at >= ? AND created_at < ?
|
||||
GROUP BY reason
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
`).all(startDate, endDate) as Array<{ reason: string; count: number }>;
|
||||
negativeReasons.push(...negRows.map(r => r.reason));
|
||||
} catch {
|
||||
// feedback_entries table might not exist yet
|
||||
}
|
||||
|
||||
// Strengths: areas where positive feedback was given (no reason column for up, so we derive from low correction areas)
|
||||
try {
|
||||
const posCount = db.prepare(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM feedback_entries
|
||||
WHERE rating = 'up'
|
||||
AND created_at >= ? AND created_at < ?
|
||||
`).get(startDate, endDate) as { count: number } | undefined;
|
||||
if (posCount && posCount.count > 0) {
|
||||
positiveReasons.push('Consistent positive user feedback');
|
||||
}
|
||||
} catch {
|
||||
// table might not exist
|
||||
}
|
||||
|
||||
return { positiveReasons, negativeReasons };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get capability gaps detected from the improvement signals store.
|
||||
*/
|
||||
function getCapabilityGaps(signalStore: ImprovementSignalStore): string[] {
|
||||
try {
|
||||
const gaps = signalStore.getByCategory('capability_gap');
|
||||
return gaps
|
||||
.filter(g => g.count >= 2) // Only surface gaps that occurred multiple times
|
||||
.sort((a, b) => b.count - a.count)
|
||||
.slice(0, 5)
|
||||
.map(g => g.pattern_key.replace('missing:', ''));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count skills installed this month from the install audit trail.
|
||||
*/
|
||||
function countSkillsInstalled(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): number {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM install_audit
|
||||
WHERE action = 'installed'
|
||||
AND timestamp >= ? AND timestamp < ?
|
||||
`).get(startDate, endDate) as { count: number } | undefined;
|
||||
return row?.count ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a text recommendation based on the assessment data.
|
||||
*/
|
||||
function generateRecommendation(
|
||||
correctionRate: number,
|
||||
trend: string,
|
||||
gaps: string[],
|
||||
weaknesses: string[],
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (correctionRate > 0.3) {
|
||||
parts.push('High correction rate detected. Consider reviewing recurring feedback patterns and adjusting behavior accordingly.');
|
||||
} else if (correctionRate > 0.15) {
|
||||
parts.push('Moderate correction rate. Room for improvement in accuracy and user alignment.');
|
||||
} else if (correctionRate > 0) {
|
||||
parts.push('Low correction rate. Agent is performing well overall.');
|
||||
} else {
|
||||
parts.push('No corrections recorded. Consider encouraging more user feedback to track performance.');
|
||||
}
|
||||
|
||||
if (gaps.length > 0) {
|
||||
parts.push(`Capability gaps detected: ${gaps.slice(0, 3).join(', ')}. Consider installing relevant skills.`);
|
||||
}
|
||||
|
||||
if (weaknesses.length > 0) {
|
||||
parts.push(`Most common issues: ${weaknesses.slice(0, 3).map(w => w.replace(/_/g, ' ')).join(', ')}.`);
|
||||
}
|
||||
|
||||
if (trend.startsWith('+')) {
|
||||
parts.push('Positive trend — keep up the good work.');
|
||||
} else if (trend.startsWith('-')) {
|
||||
parts.push('Negative trend — attention needed on quality.');
|
||||
}
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Generate a monthly self-assessment from the personal mind data.
|
||||
*
|
||||
* @param config - Local server configuration (provides dataDir)
|
||||
* @param personalMind - The personal MindDB instance
|
||||
* @param periodOverride - Optional YYYY-MM override (defaults to previous month)
|
||||
*/
|
||||
export function generateMonthlyAssessment(
|
||||
config: LocalConfig,
|
||||
personalMind: MindDB,
|
||||
periodOverride?: string,
|
||||
): MonthlyAssessment {
|
||||
const db = personalMind.getDatabase();
|
||||
|
||||
// Default to previous month
|
||||
const now = new Date();
|
||||
const prevMonth = now.getMonth() === 0 ? 12 : now.getMonth(); // getMonth() is 0-based
|
||||
const prevYear = now.getMonth() === 0 ? now.getFullYear() - 1 : now.getFullYear();
|
||||
const period = periodOverride ?? `${prevYear}-${String(prevMonth).padStart(2, '0')}`;
|
||||
|
||||
// Correction stats for this month
|
||||
const { total: totalInteractions, correctionRate } = computeMonthCorrectionRate(db, period);
|
||||
|
||||
// Trend: compare to prior month
|
||||
const priorRate = computePriorMonthCorrectionRate(db, period);
|
||||
let improvementTrend = '0%';
|
||||
if (totalInteractions > 0 && priorRate > 0) {
|
||||
const diff = Math.round((priorRate - correctionRate) * 100); // positive = improvement (fewer corrections)
|
||||
improvementTrend = diff >= 0 ? `+${diff}%` : `${diff}%`;
|
||||
}
|
||||
|
||||
// Feedback analysis
|
||||
const { positiveReasons, negativeReasons } = getTopFeedbackReasons(db, period);
|
||||
|
||||
// Capability gaps
|
||||
const signalStore = new ImprovementSignalStore(personalMind);
|
||||
const capabilityGaps = getCapabilityGaps(signalStore);
|
||||
|
||||
// Skills installed
|
||||
const skillsInstalled = countSkillsInstalled(db, period);
|
||||
|
||||
// Derive strengths
|
||||
const topStrengths: string[] = [...positiveReasons];
|
||||
if (correctionRate < 0.15) topStrengths.push('Low correction rate');
|
||||
if (skillsInstalled > 0) topStrengths.push(`${skillsInstalled} new skills adopted`);
|
||||
if (topStrengths.length === 0) topStrengths.push('Stable operation');
|
||||
|
||||
// Derive weaknesses
|
||||
const topWeaknesses: string[] = negativeReasons.map(r => r.replace(/_/g, ' '));
|
||||
if (correctionRate > 0.25) topWeaknesses.push('High correction rate');
|
||||
if (capabilityGaps.length > 0) topWeaknesses.push('Missing capabilities requested');
|
||||
|
||||
// Generate recommendation
|
||||
const recommendation = generateRecommendation(correctionRate, improvementTrend, capabilityGaps, negativeReasons);
|
||||
|
||||
return {
|
||||
period,
|
||||
totalInteractions,
|
||||
correctionRate: +correctionRate.toFixed(3),
|
||||
improvementTrend,
|
||||
topStrengths: topStrengths.slice(0, 5),
|
||||
topWeaknesses: topWeaknesses.slice(0, 5),
|
||||
capabilityGapsDetected: capabilityGaps,
|
||||
skillsInstalled,
|
||||
recommendation,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure an "assessment" session exists in the personal mind.
|
||||
* FrameStore requires a valid gop_id in the sessions table (FK constraint).
|
||||
* We create one permanent session for all assessment frames.
|
||||
*/
|
||||
function ensureAssessmentSession(personalMind: MindDB): string {
|
||||
const sessions = new SessionStore(personalMind);
|
||||
const existing = sessions.getByGopId('assessment');
|
||||
if (existing) return existing.gop_id;
|
||||
|
||||
// Insert a permanent session with gop_id = 'assessment'
|
||||
const raw = personalMind.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO sessions (gop_id, project_id, status, started_at)
|
||||
VALUES ('assessment', NULL, 'active', datetime('now'))
|
||||
`).run();
|
||||
return 'assessment';
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a monthly assessment as an I-frame in the personal mind.
|
||||
* Uses a dedicated GOP (Group of Pictures) namespace: "assessment".
|
||||
*/
|
||||
export function saveAssessmentToMind(personalMind: MindDB, assessment: MonthlyAssessment): void {
|
||||
// A zero-data month has nothing to report — writing "Interactions: 0 /
|
||||
// Correction Rate: 0.0%" frames graded the agent on no data and polluted
|
||||
// the user's memory list with template noise (judge-verified).
|
||||
if (assessment.totalInteractions === 0 && assessment.skillsInstalled === 0) {
|
||||
return;
|
||||
}
|
||||
ensureAssessmentSession(personalMind);
|
||||
const frames = new FrameStore(personalMind);
|
||||
|
||||
// Replace-on-update: one assessment frame per period. A re-fired overdue
|
||||
// monthly job (every sidecar boot re-runs due schedules) must update the
|
||||
// month's report, not accumulate duplicates.
|
||||
frames.deleteByContentPrefix(`# Monthly Agent Assessment — ${assessment.period}`);
|
||||
|
||||
const content = [
|
||||
`# Monthly Agent Assessment — ${assessment.period}`,
|
||||
'',
|
||||
`**Interactions**: ${assessment.totalInteractions}`,
|
||||
`**Correction Rate**: ${(assessment.correctionRate * 100).toFixed(1)}%`,
|
||||
`**Improvement Trend**: ${assessment.improvementTrend}`,
|
||||
'',
|
||||
`## Strengths`,
|
||||
...assessment.topStrengths.map(s => `- ${s}`),
|
||||
'',
|
||||
`## Weaknesses`,
|
||||
...assessment.topWeaknesses.map(w => `- ${w}`),
|
||||
'',
|
||||
`## Capability Gaps`,
|
||||
...(assessment.capabilityGapsDetected.length > 0
|
||||
? assessment.capabilityGapsDetected.map(g => `- ${g}`)
|
||||
: ['- None detected']),
|
||||
'',
|
||||
`## Recommendation`,
|
||||
assessment.recommendation,
|
||||
'',
|
||||
`---`,
|
||||
`Skills installed this month: ${assessment.skillsInstalled}`,
|
||||
`Generated: ${new Date().toISOString()}`,
|
||||
].join('\n');
|
||||
|
||||
// 'system' source: this is an agent-generated report, not something the
|
||||
// user said — stamping it user_stated was a judge-verified provenance lie.
|
||||
frames.createIFrame('assessment', content, 'important', 'system');
|
||||
}
|
||||
@@ -85,20 +85,6 @@ function countMemoryFrames(ctx: ProactiveContext, workspaceId: string): number {
|
||||
}
|
||||
}
|
||||
|
||||
/** Count total tool uses logged in improvement_signals (proxy for usage patterns). */
|
||||
function countToolSignals(ctx: ProactiveContext): number {
|
||||
// Use personal mind to check for improvement signals — they track tool patterns
|
||||
try {
|
||||
const personalMindPath = path.join(ctx.dataDir, 'personal.mind');
|
||||
if (!fs.existsSync(personalMindPath)) return 0;
|
||||
// We don't re-open the personal mind here; just check if improvement_signals table exists
|
||||
// and count entries. This is a lightweight heuristic.
|
||||
return 0; // Will be enriched when improvement signals are populated
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if any capability packs are installed. */
|
||||
function hasInstalledCapabilities(dataDir: string): boolean {
|
||||
const skillsDir = path.join(dataDir, 'skills');
|
||||
|
||||
@@ -990,7 +990,6 @@ const PERSISTED_MEMORY_ACCESS_TOOL_NAMES = new Set([
|
||||
'promote_skill',
|
||||
'auto_extract_skills',
|
||||
'retire_skills',
|
||||
'agent_insights',
|
||||
'compose_workflow',
|
||||
'list_agents',
|
||||
'get_agent_result',
|
||||
|
||||
@@ -6,7 +6,7 @@ import { performance } from 'node:perf_hooks';
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import { createLogger } from '../logger.js';
|
||||
const log = createLogger('chat');
|
||||
import { runAgentLoop, needsConfirmation, needsConfirmationWithAutonomy, isCriticalNeverAutopass, classifyGatedToolRisk, CapabilityRouter, analyzeAndRecordCorrection, recordCapabilityGap, lintMemoryWrite, assessTrust, formatTrustSummary, scanForInjection, AGENT_LOOP_REROUTE_PREFIX, extractEntities, IterationBudget, routeMessage, compressConversation, createDefaultCompressionConfig, needsCompression, computeInputTokenBudget, getModelContextWindow, CredentialPool, loadCredentialPool, extractStatusCode, filterAvailableTools, shouldSuggestCapture, planSkillDistillation, selectAgentRunBudget, TraceRecorder, generateTurnId, logTurnEvent, checkGrounding, READONLY_TOOLS, type ToolDefinition, type TraceHandle } from '@waggle/agent';
|
||||
import { runAgentLoop, needsConfirmation, needsConfirmationWithAutonomy, isCriticalNeverAutopass, classifyGatedToolRisk, CapabilityRouter, lintMemoryWrite, assessTrust, formatTrustSummary, scanForInjection, AGENT_LOOP_REROUTE_PREFIX, extractEntities, IterationBudget, routeMessage, compressConversation, createDefaultCompressionConfig, needsCompression, computeInputTokenBudget, getModelContextWindow, CredentialPool, loadCredentialPool, extractStatusCode, filterAvailableTools, shouldSuggestCapture, planSkillDistillation, selectAgentRunBudget, TraceRecorder, generateTurnId, logTurnEvent, checkGrounding, READONLY_TOOLS, type ToolDefinition, type TraceHandle } from '@waggle/agent';
|
||||
import type { AgentLoopConfig, AgentResponse, Orchestrator, AutonomyLevel, HookRegistry } from '@waggle/agent';
|
||||
import type { WorkspaceSession } from '../workspace-sessions.js';
|
||||
import { buildWorkspaceNowBlock, formatWorkspaceNowPrompt } from './workspace-context.js';
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
stripCapabilityRequestMarker,
|
||||
} from './capability-proposals.js';
|
||||
import { resolveGrantRiskLevel } from '../approval-grants.js';
|
||||
import { getOptimizerService } from '../services/optimizer-service.js';
|
||||
import { validateOrigin } from '../cors-config.js';
|
||||
import { listPersonas, BEHAVIORAL_SPEC, isEnabled, detectTaskShape, isClosedWorldRewriteRequest, type AssembledPrompt } from '@waggle/agent';
|
||||
|
||||
@@ -33,10 +32,7 @@ const NON_RETAINED_TURN_CONTENT = '[Not retained: memory disabled for this turn]
|
||||
*
|
||||
* Previously `getPersona(id)` was used here, which only consulted the static
|
||||
* built-in PERSONAS array — `listPersonas()` adds the custom ones from disk
|
||||
* via `loadCustomPersonas()`. The deploy comment in evolution-deploy.ts
|
||||
* explicitly says "loader picks it up on next listPersonas() call", but
|
||||
* the chat consumer was reading the wrong function. See FR #3 in
|
||||
* docs/GEPA-SCOPE-AUDIT-2026-04-30.md.
|
||||
* via `loadCustomPersonas()`, so custom personas resolve too.
|
||||
*/
|
||||
function resolvePersona(id: string) {
|
||||
return listPersonas().find(p => p.id === id) ?? null;
|
||||
@@ -666,23 +662,13 @@ export const chatRoutes: FastifyPluginAsync = async (server) => {
|
||||
// that it is broken — a symptom memory rots the moment the tool is repaired.
|
||||
const lint = lintMemoryWrite(content, ctx.memoryType);
|
||||
if (lint.verdict === 'capability_symptom') {
|
||||
// Route the observation to the improvement-signal path so a recurring gap
|
||||
// still surfaces (via acquire_capability), rather than into durable memory.
|
||||
// The pre:memory-write ctx carries no workspaceId, so we record on the base
|
||||
// orchestrator's (personal) signal store — the least-invasive reachable
|
||||
// ImprovementSignalStore. Recording is non-blocking.
|
||||
try {
|
||||
recordCapabilityGap(orchestrator.getImprovementSignals(), lint.capability ?? 'capability', lint.reason);
|
||||
} catch {
|
||||
// Non-blocking — a signal-store failure must not swallow the cancel.
|
||||
}
|
||||
log.info('[memory-validation] Capability-failure symptom blocked from memory:', content.slice(0, 100));
|
||||
return {
|
||||
cancel: true,
|
||||
reason:
|
||||
`This looks like a capability failure symptom ("${lint.capability ?? 'a capability'}"), not a durable fact. ` +
|
||||
`Don't memorize that a tool/connector is broken — it will rot when the capability is fixed. ` +
|
||||
`Instead, fix or flag the capability (e.g. use acquire_capability). The gap has been recorded as an improvement signal.`,
|
||||
`Instead, fix or flag the capability (e.g. use acquire_capability).`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1022,8 +1008,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
`;
|
||||
}
|
||||
|
||||
// Behavioral rules from the ACTIVE spec — baseline with any deployed
|
||||
// self-evolution overrides applied. Falls back to the compiled
|
||||
// Behavioral rules from the active spec. Falls back to the compiled
|
||||
// BEHAVIORAL_SPEC when the server hasn't decorated activeBehavioralSpec
|
||||
// (legacy test harness).
|
||||
const activeSpec = server.activeBehavioralSpec ?? BEHAVIORAL_SPEC;
|
||||
@@ -1067,22 +1052,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// W3.3: Inject actionable correction signals — user corrections from prior sessions
|
||||
if (includePersistedMemory) {
|
||||
try {
|
||||
const signalStore = orch.getImprovementSignals();
|
||||
if (signalStore) {
|
||||
const actionable = signalStore.getActionable();
|
||||
if (actionable.length > 0) {
|
||||
prompt += '\n\n# User Corrections (from prior sessions — follow these)\n';
|
||||
for (const signal of actionable) {
|
||||
prompt += `- ${signal.detail} (observed ${signal.count}x)\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
// W1.3/W7.3: add persona only on the legacy path; assembler-owned persona
|
||||
// content stays singular. DOCX guidance always applies; persisted workspace
|
||||
// tone is withheld when the user disables memory reads for this turn.
|
||||
@@ -1130,22 +1099,19 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
*/
|
||||
retry?: boolean;
|
||||
/**
|
||||
* Self-evolution: set by the IdleSessionWatcher's loopback review turn.
|
||||
* A review turn runs headless — no interactive client watches the SSE
|
||||
* Set by headless loopback turns. No interactive client watches the SSE
|
||||
* stream, so a live approval prompt would auto-deny after the timeout and
|
||||
* the proposal would be lost. When true, a gated proposable tool (e.g.
|
||||
* create_skill) is HELD for durable human approval in Approvals instead of
|
||||
* a live SSE prompt, and any other gated tool is denied. The reviewer can
|
||||
* therefore never write to disk without approval.
|
||||
* a live SSE prompt, and any other gated tool is denied.
|
||||
*/
|
||||
proposeHeld?: boolean;
|
||||
/**
|
||||
* Steal #13: automation-origin memory write-back gate. Set ONLY by
|
||||
* headless/automated callers (idle-watcher review turns, scheduled
|
||||
* loops) — an automated turn re-analyzes existing transcripts, so its
|
||||
* post-response write-back (auto-save, skill distillation, KG
|
||||
* extraction, correction detection) would pollute memory with
|
||||
* re-detected "decisions" and false correction signals. IM channel
|
||||
* headless/automated callers (scheduled loops) — an automated turn
|
||||
* re-analyzes existing transcripts, so its post-response write-back
|
||||
* (auto-save, skill distillation, KG extraction) would pollute memory
|
||||
* with re-detected "decisions". IM channel
|
||||
* adapters must NOT set this: inbound IM messages are real user turns.
|
||||
*/
|
||||
origin?: 'automation' | 'router';
|
||||
@@ -1210,10 +1176,9 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
const requestedSessionId = session ?? sessionIdAlias;
|
||||
|
||||
// #13: automated turns skip the post-response memory write-back seams
|
||||
// below. `proposeHeld` is belt-and-braces — the shipped idle-watcher
|
||||
// already sets it, so its review turns are gated even without `origin`.
|
||||
// Compliance execution traces stay ungated. Learned memory, improvement
|
||||
// signals, and skill capture are gated below by the resolved turn policy.
|
||||
// below. `proposeHeld` is belt-and-braces, so held-action turns are gated
|
||||
// even without `origin`. Execution traces stay ungated. Learned memory and
|
||||
// skill capture are gated below by the resolved turn policy.
|
||||
const isAutomatedTurn = origin === 'automation' || !!proposeHeldTurn;
|
||||
|
||||
// Phase B.5: resolve the effective autonomy level for this request.
|
||||
@@ -1467,8 +1432,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
|
||||
// H-07 G4: hoist trace recorder/handle so the outer catch can finalize
|
||||
// aborted/errored traces with outcome='abandoned'. Without this, a failed
|
||||
// turn leaves its row in the 'pending' state and EvalDatasetBuilder skips
|
||||
// it, starving the evolution loop of negative examples.
|
||||
// turn leaves its row in the 'pending' state and its cost goes unrecorded.
|
||||
let traceRecorder: TraceRecorder | null = null;
|
||||
let traceHandle: TraceHandle | null = null;
|
||||
let traceFinalized = false;
|
||||
@@ -1967,57 +1931,17 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// Count prior user messages FIRST — needed by both GEPA and ambiguity guards.
|
||||
// Count prior user messages FIRST — needed by the ambiguity guard.
|
||||
// Q11:A — Mid-conversation follow-ups like "yes", "run it", "LGTM" are valid
|
||||
// replies and should NOT be expanded or flagged as vague.
|
||||
const priorUserMessages = history.filter((m: { role: string }) => m.role === 'user').length;
|
||||
const isFirstUserMessage = priorUserMessages <= 1; // history already includes current message
|
||||
|
||||
// ── GEPA Optimizer: classify + expand vague prompts ──────────
|
||||
// Uses @ax-llm/ax with cheapest model (Haiku) to optimize user input
|
||||
// before the main LLM call. Non-blocking — falls back gracefully.
|
||||
// CRITICAL: Only run on the FIRST user message in a session. Mid-conversation
|
||||
// replies ("yes", "the first three", "thats fine") must NOT be expanded —
|
||||
// GEPA has no conversation context and will misinterpret them as standalone
|
||||
// vague requests, generating phantom instructions the user never intended.
|
||||
let gepaExpanded: string | null = null;
|
||||
if (!hasCustomRunner
|
||||
&& isFirstUserMessage
|
||||
&& !closedWorldRewrite
|
||||
&& !toolFreeAdvisory
|
||||
&& allowDerivedPersistence
|
||||
&& turnMutationPolicy.contextScope === 'default') {
|
||||
try {
|
||||
const optimizer = await getOptimizerService(server);
|
||||
if (optimizer) {
|
||||
const result = await optimizer.expandWithChoices(agentMessage);
|
||||
throwIfTurnAborted();
|
||||
if (result.isVague && result.expanded) {
|
||||
gepaExpanded = result.expanded;
|
||||
sendEvent('step', { content: `GEPA: Expanded prompt for better results` });
|
||||
// Send expansion choices to frontend for ask-first mode
|
||||
if (result.clarifyingQuestions && result.clarifyingQuestions.length > 0) {
|
||||
sendEvent('gepa_choices', {
|
||||
original: agentMessage,
|
||||
expanded: result.expanded,
|
||||
clarifyingQuestions: result.clarifyingQuestions,
|
||||
intent: result.intent,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
throwIfTurnAborted();
|
||||
// Non-blocking — optimizer failure doesn't affect chat
|
||||
}
|
||||
}
|
||||
|
||||
// Build system prompt (with workspace path awareness + recalled memories)
|
||||
// GAP-006: Prepend ambiguity guard when user message is too brief/vague
|
||||
const shouldCheckAmbiguity = isFirstUserMessage
|
||||
&& !gepaExpanded
|
||||
&& !closedWorldRewrite
|
||||
&& turnMutationPolicy.contextScope === 'default'; // Skip when expansion or an explicit evidence boundary already resolves intent
|
||||
&& turnMutationPolicy.contextScope === 'default'; // Skip when an explicit evidence boundary already resolves intent
|
||||
const ambiguityPrefix = (!hasCustomRunner && shouldCheckAmbiguity && isAmbiguousMessage(agentMessage)) ? AMBIGUITY_PROMPT : '';
|
||||
|
||||
// M2-7: Track session start on first user message
|
||||
@@ -2141,8 +2065,8 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// timeout. Convert a gated proposable tool into a DURABLE held action
|
||||
// that ApprovalsApp shows; deny any other gated tool. This runs before the
|
||||
// grant-store shortcut on purpose \u2014 a saved "Always allow" grant must
|
||||
// NOT let a headless reviewer write a skill to disk. The trust boundary:
|
||||
// the reviewer can never persist a skill without explicit human approval.
|
||||
// NOT let a headless turn write a skill to disk. The trust boundary:
|
||||
// a headless turn can never persist a skill without explicit human approval.
|
||||
if (proposeHeldTurn) {
|
||||
if (!allowDerivedPersistence) {
|
||||
sendEvent('step', { content: 'Tool proposal denied because memory is disabled for this turn.' });
|
||||
@@ -2150,7 +2074,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
const heldSource = sessionId.startsWith('channel-')
|
||||
? `channel:${sessionId}`
|
||||
: `session-reviewer:${sessionId}`;
|
||||
: `session:${sessionId}`;
|
||||
const decision = decideReviewTurnTool(server, {
|
||||
workspaceId: effectiveWorkspace || null,
|
||||
source: heldSource,
|
||||
@@ -2573,17 +2497,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
windowedMessages = applyContextWindow(history);
|
||||
}
|
||||
|
||||
// GEPA: if the prompt was expanded, replace the last user message
|
||||
// so the LLM sees the optimized version (original stays in disk history)
|
||||
if (gepaExpanded) {
|
||||
windowedMessages = windowedMessages.map((m, i, arr) => {
|
||||
if (i === arr.length - 1 && m.role === 'user') {
|
||||
return { ...m, content: `${gepaExpanded}\n\n(Original: "${m.content}")` };
|
||||
}
|
||||
return m;
|
||||
});
|
||||
}
|
||||
|
||||
// Governance policies for team workspaces — direct call (no HTTP loopback)
|
||||
let governancePolicies: { blockedTools?: string[]; allowedSources?: string[] } | undefined;
|
||||
if (wsConfig?.teamId && effectiveWorkspace) {
|
||||
@@ -3046,7 +2959,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
const poolKey = credPool?.getKey();
|
||||
const effectiveApiKey = poolKey ?? server.agentState.litellmApiKey;
|
||||
|
||||
// ── Start execution trace (self-evolution substrate) ──
|
||||
// ── Start execution trace (cost tracking, run history) ──
|
||||
// Lazy-created per request so unit tests with no traceStore decorator
|
||||
// (legacy suites) still pass. Assigned to the hoisted outer-scope
|
||||
// variables so the outer catch can finalize with outcome='abandoned'
|
||||
@@ -3340,9 +3253,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
);
|
||||
}
|
||||
|
||||
// ── Finalize the execution trace (self-evolution substrate) ──
|
||||
// Default outcome is 'success'; correction-detector may downgrade to
|
||||
// 'corrected' on the next turn via traceStore.markCorrected().
|
||||
// ── Finalize the execution trace (cost tracking, run history) ──
|
||||
if (traceRecorder && traceHandle) {
|
||||
try {
|
||||
traceRecorder.finalize(traceHandle, {
|
||||
@@ -3359,9 +3270,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
} catch { /* tracing is best-effort — don't fail the response */ }
|
||||
}
|
||||
|
||||
// M8: commit deferred signal markings now that model call succeeded
|
||||
if (!hasCustomRunner && allowDerivedPersistence) sessionOrch.commitSurfacedSignals();
|
||||
|
||||
// ── Post-response memory write-back ──────────────────────
|
||||
// If the agent didn't save memory itself, check if the exchange
|
||||
// contains save-worthy content and auto-save it.
|
||||
@@ -3403,24 +3311,12 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// behavioral-spec prose — detects a successful ≥5-tool turn and
|
||||
// surfaces the distillation directive, so the agent reliably authors
|
||||
// a reusable skill via its own create_skill tool. R2-gated end to
|
||||
// end: a refusal / self-incapacity turn yields no plan. The signal
|
||||
// is recorded idempotently (skill_promotion) so recurring workflows
|
||||
// bubble up through the existing actionable-signal substrate.
|
||||
// end: a refusal / self-incapacity turn yields no plan.
|
||||
// Skipped whenever learned/derived persistence is disabled.
|
||||
if (!hasCustomRunner && allowDerivedPersistence) {
|
||||
const distillPlan = planSkillDistillation(result.toolsUsed ?? [], result.content ?? '');
|
||||
if (distillPlan) {
|
||||
sendEvent('step', { content: distillPlan.directive });
|
||||
try {
|
||||
sessionOrch.getImprovementSignals().record(
|
||||
'skill_promotion',
|
||||
distillPlan.patternKey,
|
||||
distillPlan.directive,
|
||||
{ sessionId, toolCalls: (result.toolsUsed ?? []).length },
|
||||
);
|
||||
} catch {
|
||||
// Signal recording is best-effort — never fail the response.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3459,29 +3355,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// ── Correction detection ──────────────────────────────────
|
||||
// Analyze user message for corrections and record improvement signals.
|
||||
// Non-blocking — detection failure shouldn't affect the response.
|
||||
// Skipped whenever learned/derived persistence is disabled; a review
|
||||
// instruction can quote old corrections that must not re-fire.
|
||||
if (!hasCustomRunner && allowDerivedPersistence) {
|
||||
try {
|
||||
const signalStore = sessionOrch.getImprovementSignals();
|
||||
analyzeAndRecordCorrection(signalStore, message);
|
||||
|
||||
// Record capability gaps from tool-not-found events
|
||||
const toolNotFoundPattern = /Tool "(.+?)" not found/;
|
||||
if (result.content) {
|
||||
const match = result.content.match(toolNotFoundPattern);
|
||||
if (match) {
|
||||
recordCapabilityGap(signalStore, match[1]);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-blocking
|
||||
}
|
||||
}
|
||||
|
||||
// ── Auto skill capture — detect repeatable workflow patterns ──
|
||||
if (!hasCustomRunner
|
||||
&& allowDerivedPersistence
|
||||
@@ -3698,9 +3571,8 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
);
|
||||
}
|
||||
}
|
||||
// H-07 G4: finalize aborted trace so the evolution dataset builder can
|
||||
// mine it as a negative example. Without this the row stays 'pending'
|
||||
// and GEPA never sees it — starving the loop of counterexamples.
|
||||
// H-07 G4: finalize the aborted trace so its cost is accounted for and
|
||||
// the row does not stay 'pending'.
|
||||
if (traceRecorder && traceHandle && !traceFinalized) {
|
||||
try {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
@@ -3822,7 +3694,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// H-07 G4: defensive trace finalization. Catches SSE-disconnect and any
|
||||
// exotic exit path where the outer catch didn't run. Outcome stays
|
||||
// 'abandoned' because we don't know if the agent produced a usable
|
||||
// output — the correction-detector can upgrade it later if appropriate.
|
||||
// output.
|
||||
if (traceRecorder && traceHandle && !traceFinalized) {
|
||||
try {
|
||||
traceRecorder.finalize(traceHandle, {
|
||||
|
||||
@@ -1,661 +0,0 @@
|
||||
/**
|
||||
* Evolution Routes — Phase 7 of the self-evolution loop.
|
||||
*
|
||||
* HTTP surface for the EvolutionOrchestrator + EvolutionRunStore. The
|
||||
* server owns the deploy callback (file write + cache invalidation);
|
||||
* callers only manipulate run state.
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /api/evolution/runs — list proposals (with filters)
|
||||
* GET /api/evolution/runs/:uuid — single run detail
|
||||
* POST /api/evolution/runs/:uuid/accept — deploy + mark deployed
|
||||
* POST /api/evolution/runs/:uuid/reject — mark rejected
|
||||
* GET /api/evolution/status — aggregate counts for dashboard
|
||||
*/
|
||||
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import type { EvolutionRun, EvolutionRunStatus } from '@waggle/core';
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
LLMJudge,
|
||||
deployPersonaOverride,
|
||||
deployBehavioralSpecOverride,
|
||||
createAnthropicEvolutionLLM,
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
listPersonas,
|
||||
getPersona,
|
||||
BEHAVIORAL_SPEC,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
type EvolutionLLM,
|
||||
type Schema,
|
||||
type SchemaBaselineInput,
|
||||
type EvolutionTarget,
|
||||
type GateOptions,
|
||||
} from '@waggle/agent';
|
||||
import { createLogger } from '../logger.js';
|
||||
|
||||
const log = createLogger('evolution');
|
||||
|
||||
// ── Deploy dispatcher ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Given an accepted run, write the evolved text to the right place on
|
||||
* disk. Returns a structured result for audit. Throws when the run's
|
||||
* target kind isn't one the deployer knows how to handle — the
|
||||
* orchestrator converts that throw into `failed` status.
|
||||
*/
|
||||
function deployFromRun(dataDir: string, run: EvolutionRun): { path: string } {
|
||||
switch (run.target_kind) {
|
||||
case 'persona-system-prompt': {
|
||||
if (!run.target_name) {
|
||||
throw new Error('persona-system-prompt deploy needs target_name (persona id)');
|
||||
}
|
||||
const result = deployPersonaOverride(dataDir, {
|
||||
personaId: run.target_name,
|
||||
systemPrompt: run.winner_text,
|
||||
});
|
||||
return { path: result.path };
|
||||
}
|
||||
case 'behavioral-spec-section': {
|
||||
if (!run.target_name) {
|
||||
throw new Error('behavioral-spec-section deploy needs target_name (section id)');
|
||||
}
|
||||
const result = deployBehavioralSpecOverride(dataDir, {
|
||||
section: run.target_name as BehavioralSpecSection,
|
||||
text: run.winner_text,
|
||||
runUuid: run.run_uuid,
|
||||
});
|
||||
return { path: result.path };
|
||||
}
|
||||
case 'tool-description':
|
||||
case 'skill-body':
|
||||
case 'generic':
|
||||
default:
|
||||
throw new Error(
|
||||
`Deploy for target_kind "${run.target_kind}" is not yet implemented`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Plugin ───────────────────────────────────────────────────────
|
||||
|
||||
export const evolutionRoutes: FastifyPluginAsync = async (server) => {
|
||||
/**
|
||||
* GET /api/evolution/runs — list runs (newest first).
|
||||
*
|
||||
* Query:
|
||||
* ?status=proposed|accepted|rejected|deployed|failed (repeat for multiple)
|
||||
* ?targetKind=persona-system-prompt|...
|
||||
* ?targetName=coder
|
||||
* ?since=2026-04-01T00:00:00Z
|
||||
* ?limit=50
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: {
|
||||
status?: string | string[];
|
||||
targetKind?: string;
|
||||
targetName?: string;
|
||||
since?: string;
|
||||
limit?: string;
|
||||
};
|
||||
}>('/api/evolution/runs', async (request) => {
|
||||
const q = request.query;
|
||||
const statusFilter = Array.isArray(q.status)
|
||||
? q.status as EvolutionRunStatus[]
|
||||
: q.status
|
||||
? [q.status as EvolutionRunStatus]
|
||||
: undefined;
|
||||
|
||||
const rows = server.evolutionStore.list({
|
||||
status: statusFilter,
|
||||
targetKind: q.targetKind as EvolutionRun['target_kind'] | undefined,
|
||||
targetName: q.targetName,
|
||||
since: q.since,
|
||||
limit: q.limit ? Math.max(1, Math.min(500, parseInt(q.limit, 10) || 50)) : 50,
|
||||
});
|
||||
|
||||
return { runs: rows, count: rows.length };
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/runs/:uuid — single run with parsed JSON blobs.
|
||||
*/
|
||||
server.get<{ Params: { uuid: string } }>(
|
||||
'/api/evolution/runs/:uuid',
|
||||
async (request, reply) => {
|
||||
const run = server.evolutionStore.getByUuid(request.params.uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
|
||||
let winnerSchema: unknown = null;
|
||||
if (run.winner_schema_json) {
|
||||
try { winnerSchema = JSON.parse(run.winner_schema_json); } catch { /* keep null */ }
|
||||
}
|
||||
let artifacts: unknown = null;
|
||||
if (run.artifacts_json) {
|
||||
try { artifacts = JSON.parse(run.artifacts_json); } catch { /* keep null */ }
|
||||
}
|
||||
let gateReasons: unknown = [];
|
||||
try { gateReasons = JSON.parse(run.gate_reasons_json); } catch { /* keep [] */ }
|
||||
|
||||
return { ...run, winnerSchema, artifacts, gateReasons };
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* POST /api/evolution/runs/:uuid/accept
|
||||
*
|
||||
* Body: { note?: string }
|
||||
*
|
||||
* Marks the run accepted, invokes the deploy dispatcher, then moves
|
||||
* to deployed (on success) or failed (on throw). Emits a
|
||||
* 'persona:reloaded' or 'behavioral-spec:reloaded' event on the
|
||||
* server event bus so other subsystems (chat route's
|
||||
* systemPromptCache) can invalidate their caches.
|
||||
*/
|
||||
server.post<{
|
||||
Params: { uuid: string };
|
||||
Body: { note?: string };
|
||||
}>('/api/evolution/runs/:uuid/accept', async (request, reply) => {
|
||||
const { uuid } = request.params;
|
||||
const note = (request.body ?? {}).note;
|
||||
|
||||
const run = server.evolutionStore.getByUuid(uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
if (run.status !== 'proposed') {
|
||||
return reply.status(409).send({
|
||||
error: `Run is in status "${run.status}" — only proposed runs can be accepted`,
|
||||
});
|
||||
}
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: server.traceStore,
|
||||
runStore: server.evolutionStore,
|
||||
deploy: async (acceptedRun) => {
|
||||
const result = deployFromRun(server.localConfig.dataDir, acceptedRun);
|
||||
log.info(
|
||||
`Evolution deployed: ${acceptedRun.target_kind}/${acceptedRun.target_name ?? '?'} → ${result.path}`,
|
||||
);
|
||||
// Signal subsystems that the persona / spec source changed so
|
||||
// they can drop any cached derivations.
|
||||
if (acceptedRun.target_kind === 'persona-system-prompt') {
|
||||
server.eventBus.emit('persona:reloaded', { personaId: acceptedRun.target_name });
|
||||
} else if (acceptedRun.target_kind === 'behavioral-spec-section') {
|
||||
server.eventBus.emit('behavioral-spec:reloaded', { section: acceptedRun.target_name });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const updated = await orchestrator.accept(uuid, note);
|
||||
if (!updated) return reply.status(500).send({ error: 'Accept returned no record' });
|
||||
|
||||
return reply.status(200).send(updated);
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/evolution/runs/:uuid/reject
|
||||
*
|
||||
* Body: { reason?: string }
|
||||
*/
|
||||
server.post<{
|
||||
Params: { uuid: string };
|
||||
Body: { reason?: string };
|
||||
}>('/api/evolution/runs/:uuid/reject', async (request, reply) => {
|
||||
const { uuid } = request.params;
|
||||
const reason = (request.body ?? {}).reason;
|
||||
|
||||
const run = server.evolutionStore.getByUuid(uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
if (run.status !== 'proposed') {
|
||||
return reply.status(409).send({
|
||||
error: `Run is in status "${run.status}" — only proposed runs can be rejected`,
|
||||
});
|
||||
}
|
||||
|
||||
const updated = server.evolutionStore.reject(uuid, reason);
|
||||
return reply.status(200).send(updated);
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/targets — enumerate what the user can evolve.
|
||||
*
|
||||
* Returns the available persona ids + the fixed behavioral-spec section
|
||||
* list so the UI can populate its Run-form dropdowns without hardcoding.
|
||||
*/
|
||||
server.get('/api/evolution/targets', async () => {
|
||||
const personas = listPersonas().map(p => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
description: p.description,
|
||||
icon: p.icon,
|
||||
}));
|
||||
return {
|
||||
personas,
|
||||
sections: BEHAVIORAL_SPEC_SECTIONS,
|
||||
defaultSchema: defaultSchemaBaseline('generic'),
|
||||
};
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/baseline?kind=X&name=Y — fetch the current baseline.
|
||||
*
|
||||
* For `persona-system-prompt`, returns the persona's live systemPrompt.
|
||||
* For `behavioral-spec-section`, returns the active section text from
|
||||
* `server.activeBehavioralSpec` (which already accounts for any
|
||||
* deployed overrides). Also returns a sensible default schemaBaseline.
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: { kind?: string; name?: string };
|
||||
}>('/api/evolution/baseline', async (request, reply) => {
|
||||
const { kind, name } = request.query;
|
||||
if (!kind || !name) {
|
||||
return reply.status(400).send({ error: 'kind and name query params are required' });
|
||||
}
|
||||
|
||||
if (kind === 'persona-system-prompt') {
|
||||
const persona = getPersona(name);
|
||||
if (!persona) {
|
||||
return reply.status(404).send({ error: `Unknown persona: ${name}` });
|
||||
}
|
||||
return {
|
||||
baseline: persona.systemPrompt,
|
||||
schemaBaseline: defaultSchemaBaseline('persona-system-prompt'),
|
||||
};
|
||||
}
|
||||
|
||||
if (kind === 'behavioral-spec-section') {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(name as BehavioralSpecSection)) {
|
||||
return reply.status(404).send({ error: `Unknown section: ${name}` });
|
||||
}
|
||||
// Prefer the live active spec (accounts for any deployed overrides)
|
||||
// over BEHAVIORAL_SPEC's compile-time text.
|
||||
const section = name as BehavioralSpecSection;
|
||||
const activeSpec = server.activeBehavioralSpec as unknown as Record<string, string> | undefined;
|
||||
const activeText =
|
||||
activeSpec?.[section]
|
||||
?? (BEHAVIORAL_SPEC as unknown as Record<string, string>)[section];
|
||||
return {
|
||||
baseline: activeText ?? '',
|
||||
schemaBaseline: defaultSchemaBaseline('behavioral-spec-section'),
|
||||
};
|
||||
}
|
||||
|
||||
return reply.status(400).send({
|
||||
error: 'kind must be "persona-system-prompt" or "behavioral-spec-section"',
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/evolution/run
|
||||
*
|
||||
* Trigger a real evolution run synchronously. Uses the Anthropic key from
|
||||
* the vault to build Haiku-backed judge / mutate / execute functions and
|
||||
* invokes `EvolutionOrchestrator.runOnce`. The resulting proposal (if
|
||||
* any) is persisted to the evolution store and surfaced in the response
|
||||
* so the caller can immediately show it to the user.
|
||||
*
|
||||
* Body:
|
||||
* {
|
||||
* targetKind: 'persona-system-prompt' | 'behavioral-spec-section' | ...
|
||||
* targetName: string
|
||||
* baseline: string // current instruction text
|
||||
* schemaBaseline: Schema // current structural baseline (DSPy signature)
|
||||
* minDelta?: number // default 0.02
|
||||
* gepa?: { populationSize?, generations?, miniEvalSize?, anchorEvalSize?, seed? }
|
||||
* schema?: { populationSize?, generations?, evalSize?, anchorEvalSize?, seed? }
|
||||
* gateOptions?: GateOptions
|
||||
* }
|
||||
*
|
||||
* Status codes:
|
||||
* 200 — orchestrator ran (see body.outcome for proposed / skipped-*)
|
||||
* 400 — validation error (missing/invalid body fields)
|
||||
* 422 — no Anthropic API key configured in the vault
|
||||
* 503 — @ax-llm/ax unavailable or LLM init failed
|
||||
*/
|
||||
server.post<{
|
||||
Body: {
|
||||
targetKind?: EvolutionTarget;
|
||||
targetName?: string;
|
||||
baseline?: string;
|
||||
schemaBaseline?: Schema;
|
||||
minDelta?: number;
|
||||
gepa?: {
|
||||
populationSize?: number;
|
||||
generations?: number;
|
||||
miniEvalSize?: number;
|
||||
anchorEvalSize?: number;
|
||||
seed?: number;
|
||||
/**
|
||||
* Max concurrent judge.score() calls during eval stages. Default
|
||||
* 4 — a safe tradeoff between wall time and API politeness. Bump
|
||||
* higher for dedicated endpoints, drop to 1 for strict ordering.
|
||||
*/
|
||||
concurrency?: number;
|
||||
};
|
||||
schema?: {
|
||||
populationSize?: number;
|
||||
generations?: number;
|
||||
evalSize?: number;
|
||||
anchorEvalSize?: number;
|
||||
seed?: number;
|
||||
};
|
||||
gateOptions?: GateOptions;
|
||||
};
|
||||
}>('/api/evolution/run', async (request, reply) => {
|
||||
const body = request.body ?? {};
|
||||
const validation = validateRunBody(body);
|
||||
if (!validation.ok) {
|
||||
return reply.status(400).send({ error: validation.error });
|
||||
}
|
||||
const { targetKind, targetName, baseline, schemaBaseline } = validation;
|
||||
|
||||
// Resolve the Anthropic key from the vault. Evolution is opt-in — if
|
||||
// the user has not added their key we stop here with an actionable
|
||||
// message rather than burning CPU on stub LLMs.
|
||||
const apiKey = server.vault?.get('anthropic')?.value;
|
||||
if (!apiKey) {
|
||||
return reply.status(422).send({
|
||||
error: 'No Anthropic API key configured. Add one in Settings → Vault.',
|
||||
});
|
||||
}
|
||||
|
||||
const llm = await buildEvolutionLLM(apiKey, request.log);
|
||||
if (!llm) {
|
||||
return reply.status(503).send({
|
||||
error: '@ax-llm/ax is not available — cannot initialize the evolution LLM.',
|
||||
});
|
||||
}
|
||||
|
||||
// Compose adapters — shared judge for both stages, wrapped with a
|
||||
// running judge for GEPA so prompts are executed against real LLM
|
||||
// output before scoring. The ES stage has its own executor and uses
|
||||
// the base judge directly.
|
||||
const baseJudge = new LLMJudge(buildJudgeLLMCall(llm));
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
const schemaExecute = buildSchemaExecuteFn(llm);
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: server.traceStore,
|
||||
runStore: server.evolutionStore,
|
||||
// No deploy callback — accept endpoint handles that separately.
|
||||
});
|
||||
|
||||
const runOptions = {
|
||||
targetKind,
|
||||
targetName,
|
||||
baseline,
|
||||
schemaBaseline: schemaBaseline as SchemaBaselineInput['baseline'],
|
||||
minDelta: body.minDelta,
|
||||
gateOptions: body.gateOptions,
|
||||
compose: {
|
||||
schema: {
|
||||
execute: schemaExecute,
|
||||
judge: baseJudge,
|
||||
// Orchestrator mines examples from the trace store when empty;
|
||||
// the field is present for type-completeness only.
|
||||
examples: [],
|
||||
...(body.schema ?? {}),
|
||||
},
|
||||
instructions: {
|
||||
judge: runningJudge,
|
||||
mutate,
|
||||
targetKind,
|
||||
examples: [],
|
||||
// Parallelize the per-candidate mini-eval by default.
|
||||
// Historical sequential runs took 54 min for 10×3×2; a
|
||||
// concurrency of 4 gets that down to roughly a quarter of
|
||||
// the wall time without saturating the Anthropic API.
|
||||
concurrency: 4,
|
||||
...(body.gepa ?? {}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const buildResultPayload = (result: Awaited<ReturnType<typeof orchestrator.runOnce>>) => ({
|
||||
outcome: result.outcome,
|
||||
reason: result.reason,
|
||||
run: result.run,
|
||||
gateResults: result.gateResults,
|
||||
// Compose result can be large; only include the deltas + winner id.
|
||||
composeSummary: result.compose
|
||||
? {
|
||||
combinedDelta: result.compose.combinedDelta,
|
||||
fullyImproved: result.compose.fullyImproved,
|
||||
schemaImproved: result.compose.schema.improved,
|
||||
schemaDelta: result.compose.schema.deltaAccuracy,
|
||||
instructionImproved: result.compose.instructions.improved,
|
||||
instructionDelta: result.compose.instructions.delta,
|
||||
winnerId: result.compose.instructions.winner.id,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
|
||||
// Opt-in SSE: when the client sends Accept: text/event-stream, stream
|
||||
// per-phase progress events. Without the header the route returns a
|
||||
// single JSON body as before (backwards compat for tests + cron).
|
||||
const wantsSse = (request.headers.accept ?? '').toLowerCase().includes('text/event-stream');
|
||||
|
||||
if (wantsSse) {
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
// Disable reverse-proxy buffering (nginx, etc) — streamed data must
|
||||
// reach the client immediately, not in chunks.
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
reply.raw.flushHeaders?.();
|
||||
|
||||
const writeSse = (event: string, data: unknown): void => {
|
||||
// Guard against writes after the socket has already closed.
|
||||
if (reply.raw.writableEnded || reply.raw.destroyed) return;
|
||||
reply.raw.write(`event: ${event}\n`);
|
||||
reply.raw.write(`data: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
|
||||
// If the client disconnects, stop emitting. The orchestrator run
|
||||
// itself continues to completion — cancelling mid-flight LLM work
|
||||
// would waste spend already incurred.
|
||||
let clientClosed = false;
|
||||
request.raw.on('close', () => { clientClosed = true; });
|
||||
|
||||
// Emit an initial event so clients know the stream is live.
|
||||
writeSse('open', { targetKind, targetName });
|
||||
|
||||
try {
|
||||
const result = await orchestrator.runOnce({
|
||||
...runOptions,
|
||||
onProgress: (progress) => {
|
||||
if (!clientClosed) writeSse('progress', progress);
|
||||
},
|
||||
});
|
||||
|
||||
log.info(
|
||||
`Evolution run (SSE): ${targetKind}/${targetName ?? '?'} → ${result.outcome}` +
|
||||
(result.run ? ` (run ${result.run.run_uuid})` : ''),
|
||||
);
|
||||
writeSse('done', buildResultPayload(result));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`Evolution run failed (SSE): ${msg}`);
|
||||
writeSse('error', { error: `Evolution run failed: ${msg}` });
|
||||
} finally {
|
||||
if (!reply.raw.writableEnded) reply.raw.end();
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
// ── JSON path (legacy / non-streaming callers) ──────────────
|
||||
try {
|
||||
const result = await orchestrator.runOnce(runOptions);
|
||||
|
||||
log.info(
|
||||
`Evolution run: ${targetKind}/${targetName ?? '?'} → ${result.outcome}` +
|
||||
(result.run ? ` (run ${result.run.run_uuid})` : ''),
|
||||
);
|
||||
|
||||
return reply.status(200).send(buildResultPayload(result));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`Evolution run failed: ${msg}`);
|
||||
return reply.status(500).send({ error: `Evolution run failed: ${msg}` });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/status — aggregate counts for the dashboard.
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: {
|
||||
targetKind?: string;
|
||||
targetName?: string;
|
||||
since?: string;
|
||||
};
|
||||
}>('/api/evolution/status', async (request) => {
|
||||
const q = request.query;
|
||||
const counts = server.evolutionStore.statusCounts({
|
||||
targetKind: q.targetKind as EvolutionRun['target_kind'] | undefined,
|
||||
targetName: q.targetName,
|
||||
since: q.since,
|
||||
});
|
||||
const pendingCount = counts.proposed;
|
||||
return { counts, pendingCount };
|
||||
});
|
||||
};
|
||||
|
||||
// ── /run helpers ────────────────────────────────────────────────
|
||||
|
||||
const VALID_TARGET_KINDS: EvolutionTarget[] = [
|
||||
'persona-system-prompt',
|
||||
'behavioral-spec-section',
|
||||
'tool-description',
|
||||
'skill-body',
|
||||
'generic',
|
||||
];
|
||||
|
||||
type ValidationOk = {
|
||||
ok: true;
|
||||
targetKind: EvolutionTarget;
|
||||
targetName: string;
|
||||
baseline: string;
|
||||
schemaBaseline: Schema;
|
||||
};
|
||||
type ValidationErr = { ok: false; error: string };
|
||||
|
||||
/**
|
||||
* Validate the POST /api/evolution/run body. Returns a discriminated union
|
||||
* so the handler can early-return with a 400 on any shape error.
|
||||
*/
|
||||
function validateRunBody(body: unknown): ValidationOk | ValidationErr {
|
||||
if (!body || typeof body !== 'object') {
|
||||
return { ok: false, error: 'Body must be a JSON object.' };
|
||||
}
|
||||
const b = body as Record<string, unknown>;
|
||||
|
||||
if (typeof b.targetKind !== 'string' || !VALID_TARGET_KINDS.includes(b.targetKind as EvolutionTarget)) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `targetKind must be one of: ${VALID_TARGET_KINDS.join(', ')}`,
|
||||
};
|
||||
}
|
||||
if (typeof b.targetName !== 'string' || b.targetName.trim().length === 0) {
|
||||
return { ok: false, error: 'targetName must be a non-empty string.' };
|
||||
}
|
||||
if (typeof b.baseline !== 'string' || b.baseline.trim().length === 0) {
|
||||
return { ok: false, error: 'baseline must be a non-empty string.' };
|
||||
}
|
||||
const sb = b.schemaBaseline;
|
||||
if (!sb || typeof sb !== 'object') {
|
||||
return { ok: false, error: 'schemaBaseline must be an object with {name, fields, version}.' };
|
||||
}
|
||||
const schemaObj = sb as Record<string, unknown>;
|
||||
if (typeof schemaObj.name !== 'string' || !Array.isArray(schemaObj.fields)) {
|
||||
return { ok: false, error: 'schemaBaseline requires string "name" and array "fields".' };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
targetKind: b.targetKind as EvolutionTarget,
|
||||
targetName: b.targetName,
|
||||
baseline: b.baseline,
|
||||
schemaBaseline: sb as Schema,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sensible default schema for a given target kind. Returning this from
|
||||
* the baseline endpoint lets the UI submit without forcing the user to
|
||||
* hand-write a DSPy signature up front.
|
||||
*/
|
||||
function defaultSchemaBaseline(kind: EvolutionTarget): Schema {
|
||||
return {
|
||||
name: kind.replace(/-/g, '_') + '_baseline',
|
||||
version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'short step-by-step justification before the answer',
|
||||
required: false,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'the assistant\u0027s direct response',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal logger shape — structured enough for Fastify's logger, noop-safe
|
||||
* when omitted. Kept narrow so we don't pull in Fastify types here.
|
||||
*/
|
||||
interface RunLogger {
|
||||
warn(payload: unknown, msg?: string): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook point for tests: override the LLM factory by setting
|
||||
* `(globalThis as any).__waggleEvolutionLlmFactory` to an `(apiKey) => EvolutionLLM`.
|
||||
* Production code flows through `createAnthropicEvolutionLLM` with the
|
||||
* default retry policy (5s → 15s → 45s → 135s → 150s, 6 attempts). When a
|
||||
* logger is supplied each backoff is surfaced as a structured warn event
|
||||
* so rate-limit backoffs show up in ops logs instead of looking like a
|
||||
* silent hang.
|
||||
*/
|
||||
async function buildEvolutionLLM(
|
||||
apiKey: string,
|
||||
logger?: RunLogger,
|
||||
): Promise<EvolutionLLM | null> {
|
||||
const override = (globalThis as unknown as {
|
||||
__waggleEvolutionLlmFactory?: (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
||||
}).__waggleEvolutionLlmFactory;
|
||||
if (override) {
|
||||
return await override(apiKey);
|
||||
}
|
||||
return createAnthropicEvolutionLLM(apiKey, {
|
||||
retry: logger
|
||||
? {
|
||||
onRetry: ({ attempt, delayMs, error }) => {
|
||||
const e = error as { status?: number; message?: string };
|
||||
logger.warn(
|
||||
{
|
||||
component: 'evolution.run',
|
||||
attempt,
|
||||
delayMs,
|
||||
status: e?.status,
|
||||
error: e?.message ?? String(error),
|
||||
},
|
||||
`evolution LLM rate-limited — backing off ${delayMs}ms (attempt ${attempt})`,
|
||||
);
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
/**
|
||||
* Feedback REST API Routes — records user feedback on agent responses
|
||||
* and surfaces improvement stats.
|
||||
* and surfaces feedback stats.
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /api/feedback — record user feedback (thumbs up/down + reason)
|
||||
* GET /api/feedback/stats — return improvement stats and trends
|
||||
* GET /api/feedback/stats — return feedback stats and trends
|
||||
*
|
||||
* Feedback is stored in the .mind database in a dedicated feedback_entries table.
|
||||
* Negative feedback is also cross-recorded as correction signals in the
|
||||
* improvement_signals table so the self-improvement loop can track recurring issues.
|
||||
*/
|
||||
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
@@ -86,13 +84,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(sessionId, messageIndex, rating, reason ?? null, detail ?? '');
|
||||
|
||||
// Also record negative feedback as a correction signal in the improvement store
|
||||
// so the self-improvement loop can track recurring issues
|
||||
if (rating === 'down' && reason) {
|
||||
const signalStore = fastify.agentState.orchestrator.getImprovementSignals();
|
||||
signalStore.record('correction', `feedback:${reason}`, detail ?? `User thumbs-down: ${reason}`);
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Unknown error';
|
||||
@@ -100,7 +91,7 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/feedback/stats — return improvement stats
|
||||
// GET /api/feedback/stats — return feedback stats
|
||||
fastify.get('/api/feedback/stats', async (_request, reply) => {
|
||||
try {
|
||||
const db = fastify.multiMind.personal.getDatabase();
|
||||
@@ -130,18 +121,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
`).all() as Array<{ reason: string; count: number }>;
|
||||
const topIssues = topIssuesRows.map(r => r.reason);
|
||||
|
||||
// Corrections this week (from improvement_signals table)
|
||||
let correctionsThisWeek = 0;
|
||||
try {
|
||||
const corrRow = db.prepare(`
|
||||
SELECT COUNT(*) as count FROM improvement_signals
|
||||
WHERE category = 'correction'
|
||||
AND last_seen >= datetime('now', '-7 days')
|
||||
`).get() as { count: number } | undefined;
|
||||
correctionsThisWeek = corrRow?.count ?? 0;
|
||||
} catch {
|
||||
// improvement_signals table might not exist yet
|
||||
}
|
||||
|
||||
// Improvement trend: compare last 7 days positive rate to prior 7 days
|
||||
let improvementTrend = '0%';
|
||||
@@ -173,7 +152,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
totalFeedback,
|
||||
positiveRate,
|
||||
topIssues,
|
||||
correctionsThisWeek,
|
||||
improvementTrend,
|
||||
};
|
||||
} catch (err) {
|
||||
|
||||
@@ -114,9 +114,10 @@ export interface CronScheduleLike {
|
||||
*/
|
||||
const SYSTEM_JOB_TYPES = new Set([
|
||||
'memory_consolidation',
|
||||
'workspace_health',
|
||||
// Self-learning job types seeded by earlier versions; they no longer run.
|
||||
'monthly_assessment',
|
||||
'prompt_optimization',
|
||||
'workspace_health',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -448,12 +448,6 @@ function cronScheduleWorkspaceIds(
|
||||
if (ALL_WORKSPACE_CRON_JOB_TYPES.has(jobType)) {
|
||||
return [...explicitOwnerIds, ...allWorkspaceIds(fastify)];
|
||||
}
|
||||
if (jobType === 'prompt_optimization') {
|
||||
const optimizedWorkspaceIds = fastify.workspaceManager?.list()
|
||||
.filter((workspace) => Boolean(workspace.optimizationEnabled))
|
||||
.map((workspace) => workspace.id) ?? [];
|
||||
return [...explicitOwnerIds, ...optimizedWorkspaceIds];
|
||||
}
|
||||
if (jobType === 'memory_consolidation') {
|
||||
const rawAction = parsedCronJobConfig(jobConfig, serializedJobConfig)?.action;
|
||||
const action = typeof rawAction === 'string' ? rawAction : undefined;
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
/**
|
||||
* H-10 G1 — Evolution Service.
|
||||
*
|
||||
* Autonomy layer for the self-evolution loop. Runs a quiet background daemon
|
||||
* that, on each tick:
|
||||
*
|
||||
* 1. Picks one eligible evolve target (persona or behavioral-spec section)
|
||||
* whose trace count since its last run passes a minimum-dataset gate.
|
||||
* 2. Invokes EvolutionOrchestrator.runOnce with Anthropic-backed judge /
|
||||
* mutate / execute adapters — same plumbing as POST /api/evolution/run.
|
||||
* 3. Produces a `proposed` EvolutionRun. Accept/reject stays manual via the
|
||||
* existing Memory → Evolution UI; the service never auto-deploys.
|
||||
*
|
||||
* Why a standalone setInterval rather than CronStore integration: CronStore's
|
||||
* job-type enum is a breaking-change surface and the existing cron executor
|
||||
* dispatches by enum. A standalone daemon keeps H-10 surgical, testable, and
|
||||
* independent — the Memory → Evolution "Run now" button shipped in Phase 8.5
|
||||
* still covers manual triggering through the HTTP route.
|
||||
*
|
||||
* Disabled by default. Opt-in via WAGGLE_EVOLUTION_AUTO_ENABLED=1.
|
||||
*/
|
||||
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
LLMJudge,
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
createAnthropicEvolutionLLM,
|
||||
listPersonas,
|
||||
getPersona,
|
||||
BEHAVIORAL_SPEC,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
type EvolutionLLM,
|
||||
type Schema,
|
||||
type EvolutionTarget,
|
||||
} from '@waggle/agent';
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
EvolutionRunStore,
|
||||
TraceOutcome,
|
||||
} from '@waggle/core';
|
||||
|
||||
// ── Public types ─────────────────────────────────────────────────
|
||||
|
||||
export interface EvolutionTargetId {
|
||||
kind: EvolutionTarget;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface EvolutionCandidate extends EvolutionTargetId {
|
||||
/** ISO timestamp of the most recent run for this target, or null if never run. */
|
||||
lastRunAt: string | null;
|
||||
/** Finalized traces recorded since `lastRunAt` (or total when never run). */
|
||||
newTraces: number;
|
||||
}
|
||||
|
||||
/** Result of a single tick — a structured report the caller can log. */
|
||||
export type TickResult =
|
||||
| { skipped: true; reason: string }
|
||||
| {
|
||||
skipped: false;
|
||||
targetKind: EvolutionTarget;
|
||||
targetName: string;
|
||||
outcome: string;
|
||||
runUuid: string | null;
|
||||
};
|
||||
|
||||
export interface ActiveBehavioralSpecLike {
|
||||
[section: string]: unknown;
|
||||
}
|
||||
|
||||
export interface EvolutionServiceDeps {
|
||||
traceStore: ExecutionTraceStore;
|
||||
runStore: EvolutionRunStore;
|
||||
/** Returns the current Anthropic API key or null when none configured. */
|
||||
getApiKey: () => string | null;
|
||||
/** Active behavioral spec (with deployed overrides applied). */
|
||||
getActiveBehavioralSpec: () => ActiveBehavioralSpecLike;
|
||||
/** Test hook — defaults to createAnthropicEvolutionLLM. */
|
||||
llmFactory?: (apiKey: string) => Promise<EvolutionLLM | null>;
|
||||
/**
|
||||
* Test hook — by default we build and invoke an EvolutionOrchestrator.
|
||||
* Overriding this lets unit tests inject a deterministic runner without
|
||||
* spinning up the real LLM plumbing.
|
||||
*/
|
||||
runner?: (target: EvolutionTargetId, baseline: string, apiKey: string) => Promise<TickResult>;
|
||||
/** Structured log sink — defaults to no-op. */
|
||||
log?: (level: 'info' | 'warn', msg: string) => void;
|
||||
}
|
||||
|
||||
export interface EvolutionServiceConfig {
|
||||
/** Milliseconds between tick attempts. Default 6h. Clamped to ≥ 60s. */
|
||||
tickIntervalMs?: number;
|
||||
/** Minimum finalized traces since a target's last run for it to be eligible. */
|
||||
minTracesPerTarget?: number;
|
||||
/** Explicit targets list. Defaults to all personas + all spec sections. */
|
||||
targets?: EvolutionTargetId[];
|
||||
/** Trace outcomes that count toward the dataset gate. */
|
||||
eligibleOutcomes?: TraceOutcome[];
|
||||
}
|
||||
|
||||
// ── Service ──────────────────────────────────────────────────────
|
||||
|
||||
const MIN_TICK_INTERVAL_MS = 60_000;
|
||||
const DEFAULT_TICK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6h
|
||||
const DEFAULT_MIN_TRACES = 20;
|
||||
const DEFAULT_ELIGIBLE_OUTCOMES: TraceOutcome[] = ['success', 'corrected', 'verified'];
|
||||
|
||||
export class EvolutionService {
|
||||
private readonly deps: EvolutionServiceDeps;
|
||||
private readonly cfg: Required<EvolutionServiceConfig>;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private tickInFlight = false;
|
||||
|
||||
constructor(deps: EvolutionServiceDeps, cfg: EvolutionServiceConfig = {}) {
|
||||
this.deps = deps;
|
||||
this.cfg = {
|
||||
tickIntervalMs: Math.max(MIN_TICK_INTERVAL_MS, cfg.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS),
|
||||
minTracesPerTarget: Math.max(1, cfg.minTracesPerTarget ?? DEFAULT_MIN_TRACES),
|
||||
targets: cfg.targets ?? EvolutionService.defaultTargets(),
|
||||
eligibleOutcomes: cfg.eligibleOutcomes ?? DEFAULT_ELIGIBLE_OUTCOMES,
|
||||
};
|
||||
}
|
||||
|
||||
/** Default targets = every registered persona + every behavioral-spec section. */
|
||||
static defaultTargets(): EvolutionTargetId[] {
|
||||
const personas: EvolutionTargetId[] = listPersonas().map(p => ({
|
||||
kind: 'persona-system-prompt',
|
||||
name: p.id,
|
||||
}));
|
||||
const sections: EvolutionTargetId[] = BEHAVIORAL_SPEC_SECTIONS.map(s => ({
|
||||
kind: 'behavioral-spec-section',
|
||||
name: s,
|
||||
}));
|
||||
return [...personas, ...sections];
|
||||
}
|
||||
|
||||
/** Kick off the daemon. Fires one tick immediately, then on interval. */
|
||||
start(): void {
|
||||
if (this.timer) return;
|
||||
// Run an initial tick on the next event-loop turn so callers can await
|
||||
// startup side-effects deterministically.
|
||||
setImmediate(() => { void this.tick(); });
|
||||
this.timer = setInterval(() => { void this.tick(); }, this.cfg.tickIntervalMs);
|
||||
// Don't pin the process alive just for the evolution timer.
|
||||
if (typeof this.timer.unref === 'function') this.timer.unref();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.timer !== null;
|
||||
}
|
||||
|
||||
get config(): Readonly<Required<EvolutionServiceConfig>> {
|
||||
return this.cfg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one tick. Returns a structured report. Never throws — errors are
|
||||
* caught, logged, and surfaced through the return value.
|
||||
*/
|
||||
async tick(): Promise<TickResult> {
|
||||
if (this.tickInFlight) {
|
||||
return { skipped: true, reason: 'tick already in progress' };
|
||||
}
|
||||
this.tickInFlight = true;
|
||||
try {
|
||||
const apiKey = this.deps.getApiKey();
|
||||
if (!apiKey) {
|
||||
return { skipped: true, reason: 'no Anthropic API key in vault' };
|
||||
}
|
||||
|
||||
const target = this.pickNextTarget();
|
||||
if (!target) {
|
||||
return { skipped: true, reason: 'no target meets the dataset gate' };
|
||||
}
|
||||
|
||||
const baseline = this.resolveBaseline(target);
|
||||
if (!baseline) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: `no baseline for ${target.kind}/${target.name}`,
|
||||
};
|
||||
}
|
||||
|
||||
const runner = this.deps.runner ?? this.defaultRunner.bind(this);
|
||||
const result = await runner(target, baseline, apiKey);
|
||||
|
||||
if (!result.skipped) {
|
||||
this.deps.log?.(
|
||||
'info',
|
||||
`evolution tick: ${result.targetKind}/${result.targetName} → ${result.outcome}` +
|
||||
(result.runUuid ? ` (run ${result.runUuid})` : ''),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
this.deps.log?.('warn', `evolution tick failed: ${msg}`);
|
||||
return { skipped: true, reason: `tick failed: ${msg}` };
|
||||
} finally {
|
||||
this.tickInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate every configured target with its current dataset-gate status.
|
||||
* Exposed for inspection + the "status" UI; `pickNextTarget` consumes the
|
||||
* same data to choose a winner.
|
||||
*/
|
||||
enumerateCandidates(): EvolutionCandidate[] {
|
||||
return this.cfg.targets.map(t => {
|
||||
const lastRun = this.deps.runStore.list({
|
||||
targetKind: t.kind,
|
||||
targetName: t.name,
|
||||
limit: 1,
|
||||
})[0];
|
||||
const lastRunAt = lastRun?.created_at ?? null;
|
||||
|
||||
const traceFilter: {
|
||||
personaId?: string;
|
||||
since?: string;
|
||||
outcome: TraceOutcome[];
|
||||
} = { outcome: this.cfg.eligibleOutcomes };
|
||||
if (t.kind === 'persona-system-prompt') {
|
||||
traceFilter.personaId = t.name;
|
||||
}
|
||||
if (lastRunAt) traceFilter.since = lastRunAt;
|
||||
|
||||
const newTraces = this.deps.traceStore.count(traceFilter);
|
||||
return { kind: t.kind, name: t.name, lastRunAt, newTraces };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the best eligible target. "Eligible" = newTraces ≥ minTracesPerTarget.
|
||||
* "Best" = never-run first, then oldest lastRunAt, then most newTraces as a
|
||||
* tiebreaker. Returns null when no target clears the gate.
|
||||
*/
|
||||
pickNextTarget(): EvolutionCandidate | null {
|
||||
const eligible = this.enumerateCandidates().filter(
|
||||
c => c.newTraces >= this.cfg.minTracesPerTarget,
|
||||
);
|
||||
if (eligible.length === 0) return null;
|
||||
|
||||
eligible.sort((a, b) => {
|
||||
// Never-run first
|
||||
if (a.lastRunAt === null && b.lastRunAt !== null) return -1;
|
||||
if (a.lastRunAt !== null && b.lastRunAt === null) return 1;
|
||||
// Oldest last run first
|
||||
if (a.lastRunAt !== b.lastRunAt && a.lastRunAt && b.lastRunAt) {
|
||||
return a.lastRunAt.localeCompare(b.lastRunAt);
|
||||
}
|
||||
// Most new traces as tiebreaker
|
||||
return b.newTraces - a.newTraces;
|
||||
});
|
||||
return eligible[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the current baseline text for a target:
|
||||
* - persona-system-prompt: persona.systemPrompt from the live registry.
|
||||
* - behavioral-spec-section: the active spec (with overrides) with a
|
||||
* fallback to compile-time BEHAVIORAL_SPEC.
|
||||
*/
|
||||
resolveBaseline(target: EvolutionTargetId): string | null {
|
||||
if (target.kind === 'persona-system-prompt') {
|
||||
return getPersona(target.name)?.systemPrompt ?? null;
|
||||
}
|
||||
if (target.kind === 'behavioral-spec-section') {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(target.name as BehavioralSpecSection)) {
|
||||
return null;
|
||||
}
|
||||
const active = this.deps.getActiveBehavioralSpec() as Record<string, unknown>;
|
||||
const fallback = BEHAVIORAL_SPEC as unknown as Record<string, unknown>;
|
||||
const value = active?.[target.name] ?? fallback[target.name];
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
// tool-description / skill-body / generic aren't auto-evolvable yet —
|
||||
// they would need a baseline source we don't have on disk.
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default runner — builds the Anthropic LLM + adapter stack and invokes
|
||||
* EvolutionOrchestrator.runOnce. Kept as an instance method so tests can
|
||||
* bypass it entirely via `deps.runner`.
|
||||
*/
|
||||
private async defaultRunner(
|
||||
target: EvolutionTargetId,
|
||||
baseline: string,
|
||||
apiKey: string,
|
||||
): Promise<TickResult> {
|
||||
const factory = this.deps.llmFactory ?? ((key: string) => createAnthropicEvolutionLLM(key));
|
||||
const llm = await factory(apiKey);
|
||||
if (!llm) {
|
||||
return { skipped: true, reason: '@ax-llm/ax unavailable — evolution LLM not initialized' };
|
||||
}
|
||||
|
||||
const baseJudge = new LLMJudge(buildJudgeLLMCall(llm));
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
const schemaExecute = buildSchemaExecuteFn(llm);
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: this.deps.traceStore,
|
||||
runStore: this.deps.runStore,
|
||||
// No deploy — auto-accept is an explicit non-goal. User reviews proposals.
|
||||
});
|
||||
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: target.kind,
|
||||
targetName: target.name,
|
||||
baseline,
|
||||
schemaBaseline: defaultSchemaBaseline(target.kind),
|
||||
compose: {
|
||||
schema: {
|
||||
execute: schemaExecute,
|
||||
judge: baseJudge,
|
||||
examples: [],
|
||||
},
|
||||
instructions: {
|
||||
judge: runningJudge,
|
||||
mutate,
|
||||
targetKind: target.kind,
|
||||
examples: [],
|
||||
concurrency: 4,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
skipped: false,
|
||||
targetKind: target.kind,
|
||||
targetName: target.name,
|
||||
outcome: result.outcome,
|
||||
runUuid: result.run?.run_uuid ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Sensible default Schema when a target has no structural baseline on disk.
|
||||
* Mirrors the default in packages/server/src/local/routes/evolution.ts so the
|
||||
* service produces runs with the same shape as the manual route.
|
||||
*/
|
||||
function defaultSchemaBaseline(kind: EvolutionTarget): Schema {
|
||||
return {
|
||||
name: kind.replace(/-/g, '_') + '_baseline',
|
||||
version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'short step-by-step justification before the answer',
|
||||
required: false,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'the assistant\u0027s direct response',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the env flag that gates automatic evolution. Returns true only when
|
||||
* the value is an explicit opt-in ("1" / "true"). Anything else — unset,
|
||||
* empty, "false", "0" — leaves the daemon dormant.
|
||||
*/
|
||||
export function isEvolutionAutoEnabled(
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): boolean {
|
||||
const v = (env.WAGGLE_EVOLUTION_AUTO_ENABLED ?? '').toLowerCase().trim();
|
||||
return v === '1' || v === 'true' || v === 'yes';
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
/**
|
||||
* Optimizer Service — wraps @waggle/optimizer with vault key resolution,
|
||||
* caching, and graceful fallback.
|
||||
*
|
||||
* Uses @ax-llm/ax directly (no LiteLLM dependency) with the Anthropic
|
||||
* key from the vault.
|
||||
*
|
||||
* Usage in chat loop:
|
||||
* const optimizer = getOptimizerService(server);
|
||||
* if (optimizer) {
|
||||
* const intent = await optimizer.classify(message);
|
||||
* const expanded = await optimizer.expandIfVague(message, intent);
|
||||
* }
|
||||
*/
|
||||
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
let _optimizerInstance: OptimizerService | null = null;
|
||||
let _lastKeyHash: string | null = null;
|
||||
|
||||
export interface GEPAExpansionResult {
|
||||
expanded: string | null;
|
||||
clarifyingQuestions: string[] | null;
|
||||
intent: string;
|
||||
isVague: boolean;
|
||||
}
|
||||
|
||||
interface OptimizerService {
|
||||
classify(text: string): Promise<string>;
|
||||
expandIfVague(text: string, intent: string): Promise<string | null>;
|
||||
expandWithChoices(text: string): Promise<GEPAExpansionResult>;
|
||||
summarize(text: string): Promise<string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create the optimizer service. Returns null if no API key available.
|
||||
* Caches the instance and recreates if the vault key changes.
|
||||
*/
|
||||
export async function getOptimizerService(server: FastifyInstance): Promise<OptimizerService | null> {
|
||||
// Get Anthropic key from vault
|
||||
const apiKey = server.vault?.get('anthropic')?.value;
|
||||
if (!apiKey) return null;
|
||||
|
||||
// Check if key changed (user updated vault)
|
||||
const keyHash = apiKey.slice(-8);
|
||||
if (_optimizerInstance && _lastKeyHash === keyHash) {
|
||||
return _optimizerInstance;
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamic import to avoid loading @ax-llm/ax at startup if not needed
|
||||
const { AxAI, AxAIAnthropicModel } = await import('@ax-llm/ax');
|
||||
const { PromptOptimizer } = await import('@waggle/optimizer');
|
||||
|
||||
const ai = new AxAI({
|
||||
name: 'anthropic',
|
||||
apiKey,
|
||||
config: { model: AxAIAnthropicModel.Claude45Haiku }, // Use cheapest model for optimization
|
||||
});
|
||||
|
||||
const optimizer = new PromptOptimizer({ ai });
|
||||
|
||||
// Simple cache to avoid re-classifying the same message
|
||||
const classifyCache = new Map<string, string>();
|
||||
const expandCache = new Map<string, string>();
|
||||
|
||||
_optimizerInstance = {
|
||||
async classify(text: string): Promise<string> {
|
||||
const cacheKey = text.slice(0, 100);
|
||||
const cached = classifyCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const result = await optimizer.classify(text);
|
||||
classifyCache.set(cacheKey, result);
|
||||
// Cap cache size
|
||||
if (classifyCache.size > 200) {
|
||||
const firstKey = classifyCache.keys().next().value;
|
||||
if (firstKey) classifyCache.delete(firstKey);
|
||||
}
|
||||
return result;
|
||||
} catch {
|
||||
return 'request'; // Default fallback
|
||||
}
|
||||
},
|
||||
|
||||
async expandIfVague(text: string, intent: string): Promise<string | null> {
|
||||
// Only expand vague requests, not questions/greetings/commands
|
||||
if (intent === 'greeting' || intent === 'question') return null;
|
||||
|
||||
// Don't expand if message is already detailed (>100 chars)
|
||||
if (text.length > 100) return null;
|
||||
|
||||
// Don't expand commands (starting with /)
|
||||
if (text.startsWith('/')) return null;
|
||||
|
||||
const cacheKey = text.slice(0, 100);
|
||||
const cached = expandCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const expanded = await optimizer.expandPrompt(text);
|
||||
// Only use expansion if it's meaningfully different
|
||||
if (expanded && expanded.length > text.length * 1.5) {
|
||||
expandCache.set(cacheKey, expanded);
|
||||
if (expandCache.size > 100) {
|
||||
const firstKey = expandCache.keys().next().value;
|
||||
if (firstKey) expandCache.delete(firstKey);
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null; // Graceful fallback — use original message
|
||||
}
|
||||
},
|
||||
|
||||
async expandWithChoices(text: string): Promise<GEPAExpansionResult> {
|
||||
const intent = await this.classify(text);
|
||||
const isVague = intent !== 'greeting' && intent !== 'question' && text.length <= 100 && !text.startsWith('/');
|
||||
|
||||
if (!isVague) {
|
||||
return { expanded: null, clarifyingQuestions: null, intent, isVague: false };
|
||||
}
|
||||
|
||||
const expanded = await this.expandIfVague(text, intent);
|
||||
|
||||
// Generate clarifying questions using the optimizer's summarizer
|
||||
// (repurposed — it's a cheap Haiku call that can handle short prompts)
|
||||
let clarifyingQuestions: string[] | null = null;
|
||||
try {
|
||||
const questionsText = await optimizer.summarize(
|
||||
`Generate 3 short clarifying questions for this vague request: "${text}". One question per line, no numbering.`,
|
||||
);
|
||||
clarifyingQuestions = questionsText
|
||||
.split('\n')
|
||||
.map((q: string) => q.trim())
|
||||
.filter((q: string) => q.length > 5 && q.includes('?'))
|
||||
.slice(0, 3);
|
||||
if (clarifyingQuestions.length === 0) clarifyingQuestions = null;
|
||||
} catch {
|
||||
clarifyingQuestions = null;
|
||||
}
|
||||
|
||||
return { expanded, clarifyingQuestions, intent, isVague: true };
|
||||
},
|
||||
|
||||
async summarize(text: string): Promise<string> {
|
||||
try {
|
||||
return await optimizer.summarize(text);
|
||||
} catch {
|
||||
return text; // Fallback to original
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
_lastKeyHash = keyHash;
|
||||
return _optimizerInstance;
|
||||
} catch (err) {
|
||||
// @ax-llm/ax or @waggle/optimizer not available — graceful degradation
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,6 @@ export function seedDefaultCrons(cronStore: CronStore): void {
|
||||
ensure('Stale workspace check', '0 9 * * 1', 'proactive', { action: 'stale_workspace_check' });
|
||||
ensure('Task reminder', '30 8 * * *', 'proactive', { action: 'task_reminder' });
|
||||
ensure('Capability suggestion', '0 10 * * 3', 'proactive', { action: 'capability_suggestion' });
|
||||
ensure('Prompt optimization', '0 2 * * *', 'prompt_optimization', undefined, false);
|
||||
ensure('Monthly assessment', '0 6 1 * *', 'monthly_assessment');
|
||||
ensure('Index reconciliation', '0 4 * * 0', 'memory_consolidation', { action: 'index_reconcile' });
|
||||
|
||||
// Phase 7: Memory Harvest sync — scan local sources (Claude Code, Cursor) for new data
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC, deployBehavioralSpecOverride } from '@waggle/agent';
|
||||
import { BEHAVIORAL_SPEC } from '@waggle/agent';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
@@ -21,7 +21,7 @@ describe('activeBehavioralSpec integration', () => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
it('loads the compiled baseline when no overrides exist on disk', async () => {
|
||||
it('serves the compiled behavioral spec baseline', async () => {
|
||||
const server = await buildLocalServer({ dataDir: tmpDir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec).toBeDefined();
|
||||
@@ -31,104 +31,4 @@ describe('activeBehavioralSpec integration', () => {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('applies a pre-existing override at boot', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec2-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
// Write an override BEFORE the server boots.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'EVOLVED core loop — test fixture',
|
||||
});
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('EVOLVED core loop — test fixture');
|
||||
// Unaffected sections pass through
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe(BEHAVIORAL_SPEC.qualityRules);
|
||||
// rules reflects the override
|
||||
expect(server.activeBehavioralSpec.rules).toContain('EVOLVED core loop — test fixture');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('re-derives activeBehavioralSpec on behavioral-spec:reloaded event', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec3-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Sanity: no override yet
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe(BEHAVIORAL_SPEC.coreLoop);
|
||||
|
||||
// Write an override and fire the reload event.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'HOT-RELOADED core loop',
|
||||
});
|
||||
server.eventBus.emit('behavioral-spec:reloaded', { section: 'coreLoop' });
|
||||
|
||||
// Give the sync listener a tick to update (it's synchronous actually,
|
||||
// but await-nothing keeps us defensive).
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('HOT-RELOADED core loop');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('accepted evolution run causes activeBehavioralSpec to pick up the override', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec4-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Propose a run directly on the store, then accept via HTTP
|
||||
const run = server.evolutionStore.create({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'qualityRules',
|
||||
baselineText: BEHAVIORAL_SPEC.qualityRules,
|
||||
winnerText: 'END-TO-END EVOLVED quality rules',
|
||||
deltaAccuracy: 0.1,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [],
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
headers: { 'content-type': 'application/json', authorization: 'Bearer test' },
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
// Event fires synchronously on emit (EventEmitter behavior); give
|
||||
// the listener a microtask just in case.
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe('END-TO-END EVOLVED quality rules');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,386 +0,0 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
describe('Evolution Routes', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-routes-'));
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
function seedRun(overrides: Partial<Parameters<typeof server.evolutionStore.create>[0]> = {}) {
|
||||
return server.evolutionStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'baseline prompt that is long enough',
|
||||
winnerText: 'evolved prompt that is long enough to pass the gate',
|
||||
deltaAccuracy: 0.08,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [{ gate: 'size', verdict: 'pass', reason: 'within limit' }],
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/evolution/runs ──
|
||||
|
||||
describe('GET /api/evolution/runs', () => {
|
||||
beforeAll(() => server.evolutionStore.clear());
|
||||
|
||||
it('returns an empty list when nothing has run', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toEqual([]);
|
||||
expect(body.count).toBe(0);
|
||||
});
|
||||
|
||||
it('returns seeded runs in reverse-chronological order', async () => {
|
||||
seedRun({ targetName: 'a' });
|
||||
seedRun({ targetName: 'b' });
|
||||
seedRun({ targetName: 'c' });
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.count).toBe(3);
|
||||
expect(body.runs[0].id).toBeGreaterThan(body.runs[body.runs.length - 1].id);
|
||||
});
|
||||
|
||||
it('filters by status', async () => {
|
||||
const a = seedRun({ targetName: 'x' });
|
||||
server.evolutionStore.reject(a.run_uuid, 'test reject');
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?status=rejected',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs.length).toBeGreaterThanOrEqual(1);
|
||||
expect(body.runs.every((r: { status: string }) => r.status === 'rejected')).toBe(true);
|
||||
});
|
||||
|
||||
it('respects the limit query param', async () => {
|
||||
for (let i = 0; i < 5; i++) seedRun({ targetName: `limit-${i}` });
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?limit=2',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/runs/:uuid ──
|
||||
|
||||
describe('GET /api/evolution/runs/:uuid', () => {
|
||||
it('returns a single run with parsed JSON blobs', async () => {
|
||||
const run = seedRun({
|
||||
winnerSchema: { name: 'test', fields: [{ name: 'answer', type: 'string' }] },
|
||||
artifacts: { seed: 42, generations: 2 },
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.run_uuid).toBe(run.run_uuid);
|
||||
expect(body.winnerSchema).toEqual({ name: 'test', fields: [{ name: 'answer', type: 'string' }] });
|
||||
expect(body.artifacts).toEqual({ seed: 42, generations: 2 });
|
||||
expect(body.gateReasons).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs/does-not-exist',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns run even when artifacts are null/malformed', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.winnerSchema).toBeNull();
|
||||
expect(body.artifacts).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/reject ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/reject', () => {
|
||||
it('moves proposed run to rejected with reason', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: { reason: 'not great' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('rejected');
|
||||
expect(body.user_note).toBe('not great');
|
||||
});
|
||||
|
||||
it('rejects 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/runs/ghost/reject',
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('rejects 409 when run is not proposed', async () => {
|
||||
const run = seedRun();
|
||||
server.evolutionStore.reject(run.run_uuid);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/accept ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/accept', () => {
|
||||
it('writes a persona override to disk and moves to deployed', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
winnerText: 'EVOLVED coder system prompt',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: { note: 'looks good' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('deployed');
|
||||
|
||||
const overridePath = path.join(tmpDir, 'personas', 'coder.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder system prompt');
|
||||
});
|
||||
|
||||
it('writes a behavioral-spec override to disk', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'coreLoop',
|
||||
winnerText: 'EVOLVED core loop text',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
const overridePath = path.join(tmpDir, 'behavioral-overrides', 'coreLoop.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.text).toBe('EVOLVED core loop text');
|
||||
expect(loaded.runUuid).toBe(run.run_uuid);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy throws (unsupported target_kind)', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'tool-description',
|
||||
targetName: 'some_tool',
|
||||
winnerText: 'new description',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('failed');
|
||||
expect(body.failure_reason).toMatch(/not yet implemented/);
|
||||
});
|
||||
|
||||
it('emits persona:reloaded event on persona deploy', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'writer',
|
||||
winnerText: 'EVOLVED writer',
|
||||
});
|
||||
|
||||
let emitted: unknown = null;
|
||||
const listener = (payload: unknown) => { emitted = payload; };
|
||||
server.eventBus.on('persona:reloaded', listener);
|
||||
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
|
||||
server.eventBus.off('persona:reloaded', listener);
|
||||
expect(emitted).toEqual({ personaId: 'writer' });
|
||||
});
|
||||
|
||||
it('returns 409 for already-accepted run', async () => {
|
||||
const run = seedRun();
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
const second = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(second.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/status ──
|
||||
|
||||
describe('GET /api/evolution/status', () => {
|
||||
it('returns aggregate counts per status', async () => {
|
||||
server.evolutionStore.clear();
|
||||
const a = seedRun();
|
||||
const b = seedRun();
|
||||
server.evolutionStore.reject(a.run_uuid);
|
||||
server.evolutionStore.accept(b.run_uuid);
|
||||
server.evolutionStore.markDeployed(b.run_uuid);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/status',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.counts.rejected).toBe(1);
|
||||
expect(body.counts.deployed).toBe(1);
|
||||
expect(body.pendingCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/targets ──
|
||||
|
||||
describe('GET /api/evolution/targets', () => {
|
||||
it('returns personas + behavioral-spec sections + a default schema', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(Array.isArray(body.personas)).toBe(true);
|
||||
expect(body.personas.length).toBeGreaterThan(0);
|
||||
expect(body.personas[0]).toHaveProperty('id');
|
||||
expect(body.personas[0]).toHaveProperty('name');
|
||||
// systemPrompt is intentionally omitted from /targets to match /api/personas privacy policy
|
||||
expect(body.personas[0]).not.toHaveProperty('systemPrompt');
|
||||
|
||||
expect(Array.isArray(body.sections)).toBe(true);
|
||||
expect(body.sections).toEqual(expect.arrayContaining([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules', 'workPatterns', 'intelligenceDefaults',
|
||||
]));
|
||||
|
||||
expect(body.defaultSchema).toMatchObject({
|
||||
name: expect.any(String),
|
||||
version: expect.any(Number),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
expect(body.defaultSchema.fields.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/baseline ──
|
||||
|
||||
describe('GET /api/evolution/baseline', () => {
|
||||
it('returns 400 when kind or name is missing', async () => {
|
||||
const a = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline',
|
||||
});
|
||||
expect(a.statusCode).toBe(400);
|
||||
|
||||
const b = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt',
|
||||
});
|
||||
expect(b.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('returns the persona systemPrompt for persona-system-prompt', async () => {
|
||||
// Pick whatever persona the registry lists first — avoids hardcoding
|
||||
// an id that might be renamed.
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
const { personas } = JSON.parse(listRes.body);
|
||||
const personaId = personas[0].id;
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/baseline?kind=persona-system-prompt&name=${encodeURIComponent(personaId)}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
expect(body.schemaBaseline).toMatchObject({
|
||||
name: expect.any(String),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
});
|
||||
|
||||
it('returns 404 for unknown persona', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt&name=ghost-persona',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns the active section text for behavioral-spec-section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=coreLoop',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
// coreLoop has identifiable content
|
||||
expect(body.baseline).toMatch(/HOW YOU THINK|Core Loop|Step 1/i);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=nopeSection',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 400 for unknown kind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=tool-description&name=x',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,397 +0,0 @@
|
||||
/**
|
||||
* Integration tests for POST /api/evolution/run.
|
||||
*
|
||||
* The endpoint needs an Anthropic key and an LLM. We stub the LLM by
|
||||
* installing a global factory override (`__waggleEvolutionLlmFactory`) so
|
||||
* every call goes through an in-memory responder — no network, no @ax-llm/ax.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, VaultStore } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { EvolutionLLM } from '@waggle/agent';
|
||||
|
||||
// ── Global LLM override hook ─────────────────────────────────────
|
||||
|
||||
type LLMFactory = (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
||||
|
||||
function installLLMFactory(factory: LLMFactory): void {
|
||||
(globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory = factory;
|
||||
}
|
||||
|
||||
function clearLLMFactory(): void {
|
||||
delete (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stub LLM that returns a canned answer based on a pattern match on the
|
||||
* prompt. Falls back to echoing the input so the judge still produces a
|
||||
* score. Every call increments a counter so tests can assert the LLM was
|
||||
* actually invoked.
|
||||
*/
|
||||
function makeStubLLM(responses: Array<{ match: RegExp; reply: string }>): {
|
||||
llm: EvolutionLLM;
|
||||
callCount: () => number;
|
||||
calls: string[];
|
||||
} {
|
||||
const calls: string[] = [];
|
||||
return {
|
||||
callCount: () => calls.length,
|
||||
calls,
|
||||
llm: {
|
||||
async complete(prompt: string): Promise<string> {
|
||||
calls.push(prompt);
|
||||
for (const r of responses) {
|
||||
if (r.match.test(prompt)) return r.reply;
|
||||
}
|
||||
// Default response that parses as a valid judge verdict.
|
||||
return '{"correctness":8,"procedure":8,"conciseness":7,"feedback":"ok"}';
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Test scaffolding ─────────────────────────────────────────────
|
||||
|
||||
describe('POST /api/evolution/run', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-run-'));
|
||||
|
||||
// Fresh mind database.
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
// Seed the vault with an Anthropic key before the server boots — the
|
||||
// server loads the vault from disk on startup.
|
||||
const vault = new VaultStore(tmpDir);
|
||||
vault.set('anthropic', 'sk-ant-stub-for-tests', { models: ['claude-haiku-4-5-20251001'] });
|
||||
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
clearLLMFactory();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(() => {
|
||||
server.evolutionStore.clear();
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
/** Seed a handful of finalized traces so the orchestrator's dataset miner has material. */
|
||||
function seedTraces(count = 8): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
const id = server.traceStore.start({
|
||||
input: `What is the capital of country ${i}?`,
|
||||
personaId: 'coder',
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
server.traceStore.finalize(id, {
|
||||
outcome: i % 3 === 0 ? 'verified' : 'success',
|
||||
output: `The capital is City${i}.`,
|
||||
tokens: { input: 40, output: 20 },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const baseBody = {
|
||||
targetKind: 'behavioral-spec-section' as const,
|
||||
targetName: 'coreLoop',
|
||||
baseline: 'You are a careful assistant. Follow instructions.',
|
||||
schemaBaseline: {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'answer', type: 'string', description: 'the answer', required: true, constraints: [] },
|
||||
],
|
||||
},
|
||||
// Keep populations tiny so the test stays fast even though we use a stub LLM.
|
||||
schema: { populationSize: 1, generations: 1, evalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
gepa: { populationSize: 1, generations: 1, miniEvalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
};
|
||||
|
||||
// ── 422 when no API key ────────────────────────────────────────
|
||||
|
||||
it('returns 422 when no Anthropic key is in the vault', async () => {
|
||||
// Temporarily hide the key for just this request.
|
||||
const saved = server.vault?.get('anthropic');
|
||||
server.vault?.delete('anthropic');
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(422);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toMatch(/Anthropic API key/i);
|
||||
} finally {
|
||||
if (saved) {
|
||||
server.vault?.set('anthropic', saved.value, saved.metadata);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ── 400 on validation errors ───────────────────────────────────
|
||||
|
||||
describe('validation', () => {
|
||||
it('rejects missing body', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects invalid targetKind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetKind: 'nope' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetKind/);
|
||||
});
|
||||
|
||||
it('rejects empty targetName', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetName: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetName/);
|
||||
});
|
||||
|
||||
it('rejects empty baseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, baseline: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/baseline/);
|
||||
});
|
||||
|
||||
it('rejects malformed schemaBaseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, schemaBaseline: { name: 'x' } },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/schemaBaseline/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 503 when LLM factory fails ─────────────────────────────────
|
||||
|
||||
it('returns 503 when the LLM factory returns null', async () => {
|
||||
installLLMFactory(() => null as unknown as EvolutionLLM);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(JSON.parse(res.body).error).toMatch(/ax-llm/);
|
||||
});
|
||||
|
||||
// ── Skip path: no traces ───────────────────────────────────────
|
||||
|
||||
it('returns outcome=skipped-trigger when the trace store has no eligible traces', async () => {
|
||||
const { llm } = makeStubLLM([]);
|
||||
installLLMFactory(() => llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.outcome).toBe('skipped-trigger');
|
||||
expect(body.reason).toMatch(/eligible traces/);
|
||||
});
|
||||
|
||||
// ── Happy path: runs + persists ────────────────────────────────
|
||||
|
||||
it('runs the full orchestrator when traces exist and returns a proposal summary', async () => {
|
||||
seedTraces(8);
|
||||
|
||||
const stub = makeStubLLM([
|
||||
// Mutation requests get a slightly different prompt text
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'You are an extremely careful assistant. Follow all instructions step by step.' },
|
||||
// Schema fill requests get valid JSON
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
// Running-judge execution requests get a short answer
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
// Any terminal outcome is acceptable — the point is that the orchestrator
|
||||
// ran end-to-end, talked to our stub LLM, and wrote something to the store.
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(body.outcome);
|
||||
expect(body.composeSummary).toBeTruthy();
|
||||
// Every run (including skipped-delta) returns a composeSummary; only
|
||||
// 'proposed' and 'skipped-gates' attach a persisted run.
|
||||
expect(stub.callCount()).toBeGreaterThan(0);
|
||||
|
||||
// If a run was persisted, check it shows up in the list endpoint.
|
||||
if (body.run) {
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
const listed = JSON.parse(listRes.body);
|
||||
expect(listed.count).toBeGreaterThan(0);
|
||||
const matching = listed.runs.find((r: { run_uuid: string }) => r.run_uuid === body.run.run_uuid);
|
||||
expect(matching).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
// ── SSE streaming path ─────────────────────────────────────────
|
||||
|
||||
describe('SSE streaming (Accept: text/event-stream)', () => {
|
||||
/** Tiny SSE parser — takes the raw event-stream body and emits parsed events. */
|
||||
function parseSseEvents(raw: string): Array<{ event: string; data: unknown }> {
|
||||
const out: Array<{ event: string; data: unknown }> = [];
|
||||
for (const block of raw.split('\n\n')) {
|
||||
if (!block.trim()) continue;
|
||||
let event = 'message';
|
||||
const dataLines: string[] = [];
|
||||
for (const line of block.split('\n')) {
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||
else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
|
||||
}
|
||||
if (dataLines.length > 0) {
|
||||
try {
|
||||
out.push({ event, data: JSON.parse(dataLines.join('\n')) });
|
||||
} catch {
|
||||
out.push({ event, data: dataLines.join('\n') });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
it('emits open → progress* → done sequence for a successful run', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'A careful assistant.' },
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/text\/event-stream/);
|
||||
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('done');
|
||||
|
||||
// Terminal event carries the same payload shape as the JSON path.
|
||||
const done = events.find(e => e.event === 'done');
|
||||
expect(done).toBeDefined();
|
||||
const donePayload = done!.data as { outcome: string; composeSummary: unknown };
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(donePayload.outcome);
|
||||
expect(donePayload.composeSummary).toBeTruthy();
|
||||
|
||||
// There should be at least one progress event reporting an orchestrator phase.
|
||||
const progresses = events.filter(e => e.event === 'progress');
|
||||
expect(progresses.length).toBeGreaterThan(0);
|
||||
const phases = progresses.map(p => (p.data as { phase: string }).phase);
|
||||
// Orchestrator emits at least 'dataset' before skipping or compose.
|
||||
expect(phases.some(p => ['dataset', 'compose', 'gates', 'persist', 'skipped', 'done'].includes(p))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
it('emits open → error when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
// Force a real error path like the JSON test does.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200); // SSE streams return 200 even on internal errors; the error is an SSE event
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('error');
|
||||
const errEvent = events.find(e => e.event === 'error');
|
||||
expect((errEvent!.data as { error: string }).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to JSON when Accept header is absent', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
// No accept header — should return JSON.
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/application\/json/);
|
||||
expect(() => JSON.parse(res.body)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Orchestrator error path ────────────────────────────────────
|
||||
|
||||
it('returns 500 when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
|
||||
// Judge swallows its own LLM errors with a zero score, so we need to
|
||||
// make the mutate + execute paths noisy enough that *something* escapes.
|
||||
// Easiest: swap out the trace store to blow up during buildExamples.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(JSON.parse(res.body).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -40,7 +40,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let includeViewerWorkspace: boolean;
|
||||
let literalGlobalIsViewer: boolean;
|
||||
let viewerOptimizationEnabled: boolean;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = new MindDB(':memory:');
|
||||
@@ -49,7 +48,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
failNext = false;
|
||||
includeViewerWorkspace = false;
|
||||
literalGlobalIsViewer = false;
|
||||
viewerOptimizationEnabled = 1 as unknown as boolean;
|
||||
scheduler = new LocalScheduler(
|
||||
cronStore,
|
||||
async (schedule) => {
|
||||
@@ -79,7 +77,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
};
|
||||
}
|
||||
if (id === 'global' && literalGlobalIsViewer) {
|
||||
@@ -96,7 +93,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
}]
|
||||
: []),
|
||||
],
|
||||
@@ -460,13 +456,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/cron/${id}/trigger` }),
|
||||
},
|
||||
{
|
||||
label: 'prompt optimization through automation run',
|
||||
jobType: 'prompt_optimization' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/automations/${id}/run` }),
|
||||
},
|
||||
{
|
||||
label: 'memory compaction through cron trigger',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
@@ -506,25 +495,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
expect(executed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('prompt optimization ignores a listed viewer workspace without optimization opt-in', async () => {
|
||||
includeViewerWorkspace = true;
|
||||
viewerOptimizationEnabled = false;
|
||||
const row = cronStore.create({
|
||||
name: 'Prompt optimization',
|
||||
cronExpr: '0 2 * * *',
|
||||
jobType: 'prompt_optimization',
|
||||
enabled: false,
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/cron/${row.id}/trigger`,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(executed).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: 'cron',
|
||||
@@ -625,12 +595,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
jobConfig: { action: 'morning_briefing' },
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'personal monthly assessment',
|
||||
jobType: 'monthly_assessment' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'non-matching spaced memory action',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
|
||||
@@ -377,7 +377,6 @@ describe('classifyExplicitTurnMutationPolicy', () => {
|
||||
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity',
|
||||
'get_awareness', 'read_other_workspace', 'save_memory', 'add_task',
|
||||
'correct_knowledge', 'read_file', 'web_search', 'mcp_external_read',
|
||||
'agent_insights',
|
||||
].map(name => ({ name }));
|
||||
const filtered = filterToolsByTurnMutationPolicy(
|
||||
tools,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Tests the two feedback endpoints:
|
||||
* POST /api/feedback — record user feedback
|
||||
* GET /api/feedback/stats — get improvement stats
|
||||
* GET /api/feedback/stats — get feedback stats
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
@@ -11,35 +11,27 @@ import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import Fastify from 'fastify';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { feedbackRoutes } from '../../src/local/routes/feedback.js';
|
||||
|
||||
/** Minimal multiMind mock that provides a real MindDB for the personal mind. */
|
||||
function createTestServer(db: MindDB) {
|
||||
const server = Fastify({ logger: false });
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
server.decorate('multiMind', {
|
||||
personal: db,
|
||||
});
|
||||
server.decorate('agentState', {
|
||||
orchestrator: {
|
||||
getImprovementSignals: () => signalStore,
|
||||
},
|
||||
});
|
||||
server.register(feedbackRoutes);
|
||||
return { server, signalStore };
|
||||
return { server };
|
||||
}
|
||||
|
||||
describe('Feedback Routes', () => {
|
||||
let db: MindDB;
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let signalStore: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
const result = createTestServer(db);
|
||||
server = result.server;
|
||||
signalStore = result.signalStore;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -80,26 +72,6 @@ describe('Feedback Routes', () => {
|
||||
expect(res.json()).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('records negative feedback as correction signal', async () => {
|
||||
await server.inject({
|
||||
method: 'POST',
|
||||
url: '/api/feedback',
|
||||
payload: {
|
||||
sessionId: 'sess-1',
|
||||
messageIndex: 2,
|
||||
rating: 'down',
|
||||
reason: 'wrong_tool',
|
||||
detail: 'Should have used web_search',
|
||||
},
|
||||
});
|
||||
|
||||
// Check that it was recorded in the improvement signals store
|
||||
const corrections = signalStore.getByCategory('correction');
|
||||
const feedbackSignal = corrections.find(c => c.pattern_key === 'feedback:wrong_tool');
|
||||
expect(feedbackSignal).toBeDefined();
|
||||
expect(feedbackSignal!.count).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects missing sessionId', async () => {
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
@@ -170,7 +142,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(0);
|
||||
expect(body.positiveRate).toBe(0);
|
||||
expect(body.topIssues).toEqual([]);
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
@@ -202,7 +173,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(3);
|
||||
expect(body.positiveRate).toBeCloseTo(0.67, 1);
|
||||
expect(body.topIssues).toContain('too_verbose');
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
/**
|
||||
* GEPA Prompt Optimization Cron Handler Tests
|
||||
*
|
||||
* Tests the prompt_optimization cron handler logic:
|
||||
* - Calls LLM when correction rate > 20% or avg turns > 15
|
||||
* - Stores generated variant in optimization_log
|
||||
* - Skips when budget is exceeded
|
||||
* - Skips when not enough data
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import { isWithinBudget, getRecentLogs } from '@waggle/agent';
|
||||
|
||||
describe('GEPA Prompt Optimization', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let optStore: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-gepa-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
optStore = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('Signal detection', () => {
|
||||
it('detects high correction rate as optimization signal', () => {
|
||||
// Insert logs with high correction rate (>20%)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: i < 5, // 50% correction rate
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeGreaterThan(0.2);
|
||||
expect(stats.total).toBe(10);
|
||||
});
|
||||
|
||||
it('detects high avg turn count as optimization signal', () => {
|
||||
// Insert logs with high turn count (>15)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 20, // >15 threshold
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.avgTurnCount).toBeGreaterThan(15);
|
||||
});
|
||||
|
||||
it('does not trigger when both metrics are within thresholds', () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeLessThanOrEqual(0.2);
|
||||
expect(stats.avgTurnCount).toBeLessThanOrEqual(15);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Budget check', () => {
|
||||
it('reports within budget when token costs are low', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
});
|
||||
}
|
||||
|
||||
// Budget is 100 cents ($1). 500 input + 250 output tokens is far under budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('reports over budget when token costs exceed limit', () => {
|
||||
// Insert a log with massive token counts to exceed budget
|
||||
optStore.insert({
|
||||
sessionId: 'sess-big',
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 10_000_000, // 10M input tokens at $3/M = $30
|
||||
outputTokens: 1_000_000, // 1M output tokens at $15/M = $15
|
||||
});
|
||||
|
||||
// Budget is 100 cents ($1). Total cost is ~$45 = 4500 cents. Way over budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Variant storage', () => {
|
||||
it('stores generated variant in optimization_log', () => {
|
||||
// Simulate what the cron handler does after generating a variant
|
||||
const variantText = 'Improved system prompt with better instructions for reducing corrections...'.repeat(5);
|
||||
|
||||
optStore.insert({
|
||||
sessionId: `gepa-variant-${Date.now()}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: variantText,
|
||||
toolsUsed: ['gepa_variant'],
|
||||
turnCount: 0,
|
||||
wasCorrection: false,
|
||||
inputTokens: 500,
|
||||
outputTokens: variantText.length,
|
||||
});
|
||||
|
||||
const recent = getRecentLogs(optStore, 10);
|
||||
const variant = recent.find(l => JSON.parse(l.tools_used).includes('gepa_variant'));
|
||||
expect(variant).toBeDefined();
|
||||
expect(variant!.system_prompt).toBe(variantText);
|
||||
expect(variant!.turn_count).toBe(0);
|
||||
expect(variant!.was_correction).toBe(0);
|
||||
});
|
||||
|
||||
it('retrieves recent logs for analysis', () => {
|
||||
for (let i = 0; i < 8; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: `Prompt v${i}`,
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: i + 1,
|
||||
wasCorrection: i % 3 === 0,
|
||||
inputTokens: 100 * (i + 1),
|
||||
outputTokens: 50 * (i + 1),
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(optStore, 5);
|
||||
expect(logs).toHaveLength(5);
|
||||
// Most recent first
|
||||
expect(logs[0].session_id).toBe('sess-7');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Minimum data requirement', () => {
|
||||
it('skips optimization when fewer than 5 logs', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 20,
|
||||
wasCorrection: true,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const recentLogs = getRecentLogs(optStore, 100);
|
||||
// The cron handler checks: if (recentLogs.length < 5) continue;
|
||||
expect(recentLogs.length).toBeLessThan(5);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,287 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import {
|
||||
IdleSessionWatcher,
|
||||
countTurns,
|
||||
readRecentTranscript,
|
||||
buildReviewInstruction,
|
||||
NOTHING_TO_DO,
|
||||
DEFAULT_CONFIG,
|
||||
type SelfEvolutionConfig,
|
||||
type ReviewTurnResult,
|
||||
} from '../../src/local/idle-watcher.js';
|
||||
import { getNotificationGate } from '../../src/local/notification-gate.js';
|
||||
|
||||
// A fixed synthetic "now" so idle math is deterministic.
|
||||
const NOW = 2_000_000_000_000;
|
||||
const MIN = 60_000;
|
||||
|
||||
describe('idle-watcher', () => {
|
||||
let dataDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-idle-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ── Fixtures ─────────────────────────────────────────────────────────
|
||||
function writeConfig(cfg: Partial<SelfEvolutionConfig> | string): void {
|
||||
const body = typeof cfg === 'string' ? cfg : JSON.stringify(cfg);
|
||||
fs.writeFileSync(path.join(dataDir, 'self-evolution.json'), body);
|
||||
}
|
||||
|
||||
/** Create a session .jsonl with `turns` message lines and a given mtime (ms). */
|
||||
function writeSession(workspaceId: string, sessionId: string, turns: number, mtimeMs: number): void {
|
||||
const dir = path.join(dataDir, 'workspaces', workspaceId, 'sessions');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const lines = [JSON.stringify({ type: 'meta', title: sessionId })];
|
||||
for (let i = 0; i < turns; i++) {
|
||||
lines.push(JSON.stringify({ role: i % 2 === 0 ? 'user' : 'assistant', content: `msg ${i}` }));
|
||||
}
|
||||
const filePath = path.join(dir, `${sessionId}.jsonl`);
|
||||
fs.writeFileSync(filePath, lines.join('\n') + '\n');
|
||||
fs.utimesSync(filePath, new Date(mtimeMs), new Date(mtimeMs));
|
||||
}
|
||||
|
||||
function makeWatcher(over?: {
|
||||
runReviewTurn?: (i: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
|
||||
}) {
|
||||
const reviewed: Array<{ sessionId: string; workspaceId: string }> = [];
|
||||
const emitCalls: Array<{ suppressed: boolean; dedupeKey?: string }> = [];
|
||||
const runReviewTurn =
|
||||
over?.runReviewTurn ??
|
||||
(async (i: { sessionId: string; workspaceId: string }) => {
|
||||
reviewed.push(i);
|
||||
return { content: NOTHING_TO_DO };
|
||||
});
|
||||
const runSpy = vi.fn(runReviewTurn);
|
||||
const watcher = new IdleSessionWatcher({
|
||||
dataDir,
|
||||
now: () => NOW,
|
||||
runReviewTurn: runSpy,
|
||||
// Mirror production emitNotification's gate so suppression is genuinely tested.
|
||||
emitNotification: (_event, options) => {
|
||||
let suppressed = false;
|
||||
if (options?.dedupeKey && options.materialHash) {
|
||||
suppressed = !getNotificationGate(dataDir).shouldNotify(options.dedupeKey, options.materialHash);
|
||||
}
|
||||
emitCalls.push({ suppressed, dedupeKey: options?.dedupeKey });
|
||||
return { suppressed };
|
||||
},
|
||||
log: { info: () => {}, warn: () => {} },
|
||||
});
|
||||
return { watcher, reviewed, emitCalls, runSpy };
|
||||
}
|
||||
|
||||
// ── Fire-condition matrix ────────────────────────────────────────────
|
||||
describe('fire condition', () => {
|
||||
beforeEach(() => writeConfig({ enabled: true })); // rest of fields default
|
||||
|
||||
it('reviews an idle session with enough turns', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-a', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
it('does NOT review a still-active session (mtime too recent)', async () => {
|
||||
writeSession('w1', 'sess-active', 10, NOW - 5 * MIN); // < 15 min idle
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT review a session below the turn threshold', async () => {
|
||||
writeSession('w1', 'sess-short', 3, NOW - 30 * MIN); // idle but only 3 turns
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Fired-set: no re-fire until mtime advances ───────────────────────
|
||||
it('does not re-review the same session until its file advances', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(await watcher.tick()).toBe(0); // same mtime — already fired
|
||||
expect(runSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// File advances (a new message lands) → new mtime key → re-fires once.
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Disabled / config handling ───────────────────────────────────────
|
||||
it('is a no-op when disabled (default)', async () => {
|
||||
writeConfig({ enabled: false });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is a no-op when the config file is absent (defaults ⇒ disabled)', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to defaults on a corrupt config (⇒ disabled)', async () => {
|
||||
writeConfig('{ this is not json');
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
expect(DEFAULT_CONFIG.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('applies partial config over defaults (enabled + custom idle threshold)', async () => {
|
||||
writeConfig({ enabled: true, idleMinutes: 60 }); // minTurns/cap default
|
||||
writeSession('w1', 'fresh', 6, NOW - 30 * MIN); // idle 30m < 60m ⇒ skipped
|
||||
writeSession('w1', 'old', 6, NOW - 90 * MIN); // idle 90m ⇒ reviewed
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'old', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Daily cap ────────────────────────────────────────────────────────
|
||||
it('enforces maxReviewsPerDay', async () => {
|
||||
writeConfig({ enabled: true, maxReviewsPerDay: 2 });
|
||||
writeSession('w1', 's1', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 's2', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 's3', 6, NOW - 22 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(2); // capped at 2 even though 3 are eligible
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
// A later tick in the same day stays capped (the 3rd never runs).
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Enumeration skips channel-/evolve- prefixes ──────────────────────
|
||||
it('skips channel-* and evolve-* sessions', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'channel-telegram-123', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'evolve-sess-a', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'sess-real', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-real', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Notification gating ──────────────────────────────────────────────
|
||||
it('does NOT notify on a NOTHING_TO_DO / empty / error result', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'ntd', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'empty', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 'err', 6, NOW - 22 * MIN);
|
||||
const results: Record<string, ReviewTurnResult> = {
|
||||
ntd: { content: NOTHING_TO_DO },
|
||||
empty: { content: ' ' },
|
||||
err: { content: 'partial', error: 'boom' },
|
||||
};
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async ({ sessionId }) => results[sessionId],
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('notifies once on a material finding and suppresses an identical re-finding', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async () => ({ content: 'Finding: the assistant promised a report and never produced it.' }),
|
||||
});
|
||||
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(1);
|
||||
expect(emitCalls[0].suppressed).toBe(false);
|
||||
expect(emitCalls[0].dedupeKey).toBe('self-evolution:sess-a');
|
||||
|
||||
// Session advances, same finding recurs → notification suppressed (anti-nag).
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(2);
|
||||
expect(emitCalls[1].suppressed).toBe(true);
|
||||
});
|
||||
|
||||
// ── Single-flight guard ──────────────────────────────────────────────
|
||||
it('tick is single-flight (a re-entrant tick returns 0)', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
let secondResult = -1;
|
||||
const { watcher } = makeWatcher({
|
||||
runReviewTurn: async () => {
|
||||
// Re-enter while the first tick is still in-flight.
|
||||
secondResult = await watcher.tick();
|
||||
return { content: NOTHING_TO_DO };
|
||||
},
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(secondResult).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────────────
|
||||
describe('idle-watcher pure helpers', () => {
|
||||
it('countTurns excludes the meta line', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
});
|
||||
|
||||
it('countTurns counts every non-empty line when there is no meta', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
expect(countTurns('')).toBe(0);
|
||||
});
|
||||
|
||||
it('readRecentTranscript formats messages and skips meta; null when unreadable', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-tr-'));
|
||||
try {
|
||||
const sessionsDir = path.join(dir, 'workspaces', 'w1', 'sessions');
|
||||
fs.mkdirSync(sessionsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(sessionsDir, 'sess-a.jsonl'),
|
||||
[
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'build me a report' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'on it' }),
|
||||
].join('\n'),
|
||||
);
|
||||
const t = readRecentTranscript(dir, 'w1', 'sess-a');
|
||||
expect(t).toContain('USER: build me a report');
|
||||
expect(t).toContain('ASSISTANT: on it');
|
||||
expect(t).not.toContain('meta');
|
||||
expect(readRecentTranscript(dir, 'w1', 'missing')).toBeNull();
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('buildReviewInstruction embeds the transcript and the NOTHING_TO_DO contract', () => {
|
||||
const msg = buildReviewInstruction('sess-a', 'USER: hi');
|
||||
expect(msg).toContain('sess-a');
|
||||
expect(msg).toContain('USER: hi');
|
||||
expect(msg).toContain(NOTHING_TO_DO);
|
||||
expect(msg).toContain('create_skill');
|
||||
});
|
||||
});
|
||||
@@ -8,8 +8,8 @@ import { buildLocalServer } from '../../src/local/index.js';
|
||||
/**
|
||||
* Integration test for the pre:memory-write lint handler registered in chat.ts.
|
||||
* Fires the real hook through the server's shared HookRegistry and asserts that
|
||||
* capability-failure symptoms are cancelled (and routed to the improvement-signal
|
||||
* path) while normal memories and dramatic-but-legitimate claims pass through.
|
||||
* capability-failure symptoms are cancelled while normal memories and
|
||||
* dramatic-but-legitimate claims pass through.
|
||||
*/
|
||||
describe('pre:memory-write capability-symptom lint (handler integration)', () => {
|
||||
let server: FastifyInstance;
|
||||
@@ -30,10 +30,8 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels a capability-failure symptom write and records an improvement signal', async () => {
|
||||
const { hookRegistry, orchestrator } = server.agentState;
|
||||
const store = orchestrator.getImprovementSignals();
|
||||
const before = store.getByCategory('capability_gap').length;
|
||||
it('cancels a capability-failure symptom write', async () => {
|
||||
const { hookRegistry } = server.agentState;
|
||||
|
||||
const result = await hookRegistry.fire('pre:memory-write', {
|
||||
toolName: 'save_memory',
|
||||
@@ -43,9 +41,6 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
expect(result.cancelled).toBe(true);
|
||||
expect(result.reason).toMatch(/capability/i);
|
||||
expect(result.reason).toMatch(/acquire_capability/);
|
||||
|
||||
const after = store.getByCategory('capability_gap');
|
||||
expect(after.length).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('allows a normal memory (tool preference) to pass', async () => {
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
/**
|
||||
* Monthly Self-Assessment Tests
|
||||
*
|
||||
* Verifies:
|
||||
* - Assessment shape and all required fields
|
||||
* - Cron registration (monthly_assessment job type is valid)
|
||||
* - Assessment saves to personal mind as I-frame
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
MindDB,
|
||||
CronStore,
|
||||
FrameStore,
|
||||
OptimizationLogStore,
|
||||
ImprovementSignalStore,
|
||||
InstallAuditStore,
|
||||
} from '@waggle/core';
|
||||
import { generateMonthlyAssessment, saveAssessmentToMind, type MonthlyAssessment } from '../../src/local/monthly-assessment.js';
|
||||
|
||||
describe('Monthly Self-Assessment', () => {
|
||||
let db: MindDB;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── Assessment shape ────────────────────────────────────────────────
|
||||
|
||||
describe('generateMonthlyAssessment', () => {
|
||||
it('returns a valid MonthlyAssessment shape', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment).toBeDefined();
|
||||
expect(assessment.period).toBe('2026-03');
|
||||
expect(typeof assessment.totalInteractions).toBe('number');
|
||||
expect(typeof assessment.correctionRate).toBe('number');
|
||||
expect(typeof assessment.improvementTrend).toBe('string');
|
||||
expect(Array.isArray(assessment.topStrengths)).toBe(true);
|
||||
expect(Array.isArray(assessment.topWeaknesses)).toBe(true);
|
||||
expect(Array.isArray(assessment.capabilityGapsDetected)).toBe(true);
|
||||
expect(typeof assessment.skillsInstalled).toBe('number');
|
||||
expect(typeof assessment.recommendation).toBe('string');
|
||||
expect(assessment.recommendation.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('defaults to previous month when no period override', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db);
|
||||
|
||||
// Period should be YYYY-MM format for the previous month
|
||||
expect(assessment.period).toMatch(/^\d{4}-\d{2}$/);
|
||||
});
|
||||
|
||||
it('computes correction rate from optimization logs', () => {
|
||||
// Insert some optimization log entries for March 2026
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
optStore.insert({
|
||||
sessionId: 's1',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['read_file'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
});
|
||||
optStore.insert({
|
||||
sessionId: 's2',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['write_file'],
|
||||
turnCount: 3,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
// Use current month since the entries use datetime('now')
|
||||
const now = new Date();
|
||||
const currentPeriod = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
||||
const assessment = generateMonthlyAssessment(config, db, currentPeriod);
|
||||
|
||||
expect(assessment.totalInteractions).toBe(2);
|
||||
expect(assessment.correctionRate).toBe(0.5);
|
||||
});
|
||||
|
||||
it('uses local calendar boundaries for UTC-backed monthly data', () => {
|
||||
const previousTimezone = process.env.TZ;
|
||||
process.env.TZ = 'Europe/Budapest';
|
||||
|
||||
try {
|
||||
const raw = db.getDatabase();
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
const insideInteraction = optStore.insert({
|
||||
sessionId: 'inside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
const outsideInteraction = optStore.insert({
|
||||
sessionId: 'outside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInteraction.id);
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInteraction.id);
|
||||
|
||||
raw.exec(`
|
||||
CREATE TABLE feedback_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
message_index INTEGER NOT NULL,
|
||||
rating TEXT NOT NULL,
|
||||
reason TEXT,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('inside-august', 'up', null, '2026-07-31 22:30:00');
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('outside-august', 'down', 'wrong_answer', '2026-08-31 22:30:00');
|
||||
|
||||
const auditStore = new InstallAuditStore(db);
|
||||
const recordInstall = (capabilityName: string) => auditStore.record({
|
||||
capabilityName,
|
||||
capabilityType: 'skill',
|
||||
source: 'starter-pack',
|
||||
riskLevel: 'low',
|
||||
trustSource: 'starter_pack',
|
||||
approvalClass: 'standard',
|
||||
action: 'installed',
|
||||
initiator: 'agent',
|
||||
});
|
||||
const insideInstallA = recordInstall('inside-skill-a');
|
||||
const insideInstallB = recordInstall('inside-skill-b');
|
||||
const outsideInstall = recordInstall('outside-skill');
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInstallA.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 23:30:00', insideInstallB.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInstall.id);
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-08');
|
||||
|
||||
expect(assessment.totalInteractions).toBe(1);
|
||||
expect(assessment.correctionRate).toBe(1);
|
||||
expect(assessment.topStrengths).toContain('Consistent positive user feedback');
|
||||
expect(assessment.topWeaknesses).not.toContain('wrong answer');
|
||||
expect(assessment.skillsInstalled).toBe(2);
|
||||
} finally {
|
||||
if (previousTimezone === undefined) delete process.env.TZ;
|
||||
else process.env.TZ = previousTimezone;
|
||||
}
|
||||
});
|
||||
|
||||
it('includes capability gaps from improvement signals', () => {
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
// Record the same gap twice (threshold for capability_gap is 2)
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search');
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search again');
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.capabilityGapsDetected).toContain('web_search');
|
||||
});
|
||||
|
||||
it('has non-empty top strengths even with zero data', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
// Should have at least one strength (e.g., "Stable operation")
|
||||
expect(assessment.topStrengths.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('limits top strengths and weaknesses to 5', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.topStrengths.length).toBeLessThanOrEqual(5);
|
||||
expect(assessment.topWeaknesses.length).toBeLessThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Save to mind ────────────────────────────────────────────────────
|
||||
|
||||
describe('saveAssessmentToMind', () => {
|
||||
it('saves assessment as an I-frame in the personal mind', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate', 'Consistent positive user feedback'],
|
||||
topWeaknesses: ['wrong answer'],
|
||||
capabilityGapsDetected: ['web_search'],
|
||||
skillsInstalled: 2,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
|
||||
// Verify the frame was created
|
||||
const frames = new FrameStore(db);
|
||||
const allFrames = frames.getRecent(10);
|
||||
expect(allFrames.length).toBe(1);
|
||||
|
||||
const frame = allFrames[0];
|
||||
expect(frame.frame_type).toBe('I');
|
||||
expect(frame.gop_id).toBe('assessment');
|
||||
expect(frame.importance).toBe('important');
|
||||
expect(frame.content).toContain('Monthly Agent Assessment');
|
||||
expect(frame.content).toContain('2026-03');
|
||||
expect(frame.content).toContain('15.0%');
|
||||
expect(frame.content).toContain('+5%');
|
||||
expect(frame.content).toContain('web_search');
|
||||
});
|
||||
|
||||
it('a zero-data month writes nothing — no template-noise frames', () => {
|
||||
saveAssessmentToMind(db, {
|
||||
period: '2026-03', totalInteractions: 0, correctionRate: 0,
|
||||
improvementTrend: '0%', topStrengths: ['Stable operation'], topWeaknesses: [],
|
||||
capabilityGapsDetected: [], skillsInstalled: 0,
|
||||
recommendation: 'No corrections recorded.',
|
||||
});
|
||||
expect(new FrameStore(db).getRecent(10)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('re-running the same period replaces the frame instead of duplicating it', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate'],
|
||||
topWeaknesses: [],
|
||||
capabilityGapsDetected: [],
|
||||
skillsInstalled: 0,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
saveAssessmentToMind(db, { ...assessment, totalInteractions: 120 });
|
||||
// A different period must NOT be replaced — one frame per month.
|
||||
saveAssessmentToMind(db, { ...assessment, period: '2026-04' });
|
||||
|
||||
const frames = new FrameStore(db);
|
||||
const assessments = frames
|
||||
.getRecent(10)
|
||||
.filter((f) => f.content.startsWith('# Monthly Agent Assessment'));
|
||||
expect(assessments).toHaveLength(2);
|
||||
const march = assessments.filter((f) => f.content.includes('2026-03'));
|
||||
expect(march).toHaveLength(1);
|
||||
expect(march[0].content).toContain('**Interactions**: 120');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Cron registration ───────────────────────────────────────────────
|
||||
|
||||
describe('cron registration', () => {
|
||||
it('monthly_assessment is a valid cron job type', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
|
||||
// This should not throw — monthly_assessment is in VALID_JOB_TYPES
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
expect(schedule.name).toBe('Monthly assessment');
|
||||
expect(schedule.cron_expr).toBe('0 6 1 * *');
|
||||
expect(schedule.job_type).toBe('monthly_assessment');
|
||||
});
|
||||
|
||||
it('cron expression fires on 1st of month at 6 AM', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
// next_run_at should be set and valid
|
||||
expect(schedule.next_run_at).toBeDefined();
|
||||
expect(schedule.next_run_at).not.toBeNull();
|
||||
|
||||
// The next run should be on the 1st of some month at 6 AM local time
|
||||
// cron-parser interprets expressions in local timezone
|
||||
const nextRun = new Date(schedule.next_run_at!);
|
||||
expect(nextRun.getDate()).toBe(1);
|
||||
expect(nextRun.getHours()).toBe(6);
|
||||
expect(nextRun.getMinutes()).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -20,7 +20,6 @@ const testState = vi.hoisted(() => {
|
||||
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
||||
return {
|
||||
previousPromptAssembler,
|
||||
optimizerExpand: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
};
|
||||
});
|
||||
@@ -41,12 +40,6 @@ vi.mock('../../src/local/model-availability.js', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../src/local/services/optimizer-service.js', () => ({
|
||||
getOptimizerService: async () => ({
|
||||
expandWithChoices: testState.optimizerExpand,
|
||||
}),
|
||||
}));
|
||||
|
||||
function parseSse(raw: string): Array<{ event: string; data: Record<string, unknown> }> {
|
||||
return raw.split(/\n\n/)
|
||||
.filter(Boolean)
|
||||
@@ -151,12 +144,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
resetRateLimiter(server);
|
||||
testState.optimizerExpand.mockReset().mockResolvedValue({
|
||||
expanded: null,
|
||||
clarifyingQuestions: null,
|
||||
intent: 'request',
|
||||
isVague: false,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -657,7 +644,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
expect(response.body).toContain(toolResultMarker);
|
||||
expect(response.body).toContain(responseMarker);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
@@ -740,7 +726,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
|
||||
@@ -14,10 +14,8 @@ import { afterEach, describe, it, expect } from 'vitest';
|
||||
import Fastify from 'fastify';
|
||||
import {
|
||||
deleteCustomPersona,
|
||||
deployPersonaOverride,
|
||||
isValidCustomPersonaId,
|
||||
listPersonas,
|
||||
rollbackPersonaOverride,
|
||||
saveCustomPersona,
|
||||
type AgentPersona,
|
||||
} from '@waggle/agent';
|
||||
@@ -118,14 +116,12 @@ describe('Personas Routes', () => {
|
||||
await server.close();
|
||||
});
|
||||
|
||||
it('contains all 23 personas', async () => {
|
||||
it('contains all 22 personas', async () => {
|
||||
const server = createTestServer();
|
||||
const res = await server.inject({ method: 'GET', url: '/api/personas' });
|
||||
const { personas } = res.json();
|
||||
|
||||
// 22 tiered personas + session-reviewer (internal self-evolution reviewer;
|
||||
// returned by the raw list but excluded from onboarding + PersonaSwitcher).
|
||||
expect(personas.length).toBe(23);
|
||||
expect(personas.length).toBe(22);
|
||||
const ids = personas.map((p: { id: string }) => p.id);
|
||||
expect(ids).toContain('researcher');
|
||||
expect(ids).toContain('writer');
|
||||
@@ -251,40 +247,5 @@ describe('Personas Routes', () => {
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
|
||||
it('deployPersonaOverride rejects traversal before writing an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
|
||||
expect(() => deployPersonaOverride(dataDir, {
|
||||
personaId: '../evolved',
|
||||
systemPrompt: 'Escaped override',
|
||||
})).toThrow('Invalid custom persona ID');
|
||||
expect(fs.existsSync(outsidePath)).toBe(false);
|
||||
});
|
||||
|
||||
it('deployPersonaOverride keeps its validated ID authoritative over overrides', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const result = deployPersonaOverride(dataDir, {
|
||||
personaId: 'safe-persona',
|
||||
systemPrompt: 'Validated prompt',
|
||||
overrides: { id: '../evolved', systemPrompt: 'Spoofed prompt' } as never,
|
||||
});
|
||||
|
||||
const deployed = JSON.parse(fs.readFileSync(result.path, 'utf-8')) as AgentPersona;
|
||||
expect(deployed.id).toBe('safe-persona');
|
||||
expect(deployed.systemPrompt).toBe('Validated prompt');
|
||||
expect(fs.existsSync(path.join(dataDir, 'evolved.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rollbackPersonaOverride rejects traversal before deleting an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
fs.writeFileSync(outsidePath, 'sentinel', 'utf-8');
|
||||
|
||||
expect(() => rollbackPersonaOverride(dataDir, '../evolved'))
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
/**
|
||||
* session-reviewer persona — trust-boundary tool policy.
|
||||
*
|
||||
* The reviewer runs headless; ALWAYS_AVAILABLE_TOOLS (persona-tool-filter) re-adds
|
||||
* write-side tools past any allowlist, so the persona's disallowedTools must strip
|
||||
* the dangerous ones while KEEPING create_skill (its one proposal write) and the
|
||||
* read tools. This test locks that against regression.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getPersona, type ToolDefinition } from '@waggle/agent';
|
||||
import { applyPersonaToolFilter } from '../../src/local/persona-tool-filter.js';
|
||||
|
||||
/** Minimal ToolDefinition stub — only `name` matters for the filter. */
|
||||
function tool(name: string): ToolDefinition {
|
||||
return { name, description: '', parameters: { type: 'object', properties: {} }, execute: async () => '' };
|
||||
}
|
||||
|
||||
// A representative superset spanning reads, writes, memory, exec, and skill tools.
|
||||
const POOL: ToolDefinition[] = [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'save_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'correct_knowledge', 'add_task',
|
||||
'list_skills', 'search_skills', 'read_skill', 'create_skill', 'delete_skill',
|
||||
'write_file', 'edit_file', 'generate_docx', 'bash',
|
||||
'git_commit', 'git_push',
|
||||
'spawn_agent', 'install_capability', 'acquire_capability',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
].map(tool);
|
||||
|
||||
describe('session-reviewer persona', () => {
|
||||
it('exists and is not read-only (so create_skill survives)', () => {
|
||||
const p = getPersona('session-reviewer');
|
||||
expect(p).toBeTruthy();
|
||||
expect(p!.isReadOnly).toBe(false);
|
||||
expect(p!.tools).toContain('create_skill');
|
||||
});
|
||||
|
||||
it('applyPersonaToolFilter keeps reads + create_skill, strips writes/exec/memory', () => {
|
||||
const p = getPersona('session-reviewer')!;
|
||||
const names = new Set(applyPersonaToolFilter(POOL, p).map(t => t.name));
|
||||
|
||||
// Kept — the reviewer's read surface + its one proposal write.
|
||||
for (const keep of ['read_file', 'search_content', 'search_memory', 'read_skill', 'list_skills', 'create_skill']) {
|
||||
expect(names.has(keep), `expected ${keep} to survive`).toBe(true);
|
||||
}
|
||||
|
||||
// Stripped — writes, exec, memory writes, and the ALWAYS_AVAILABLE re-adds.
|
||||
for (const drop of [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'bash', 'spawn_agent', 'write_file', 'edit_file', 'git_commit',
|
||||
'add_task', 'correct_knowledge', 'execute_step', 'compose_workflow',
|
||||
]) {
|
||||
expect(names.has(drop), `expected ${drop} to be stripped`).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -217,7 +217,7 @@ const SELECTOR_TOOL_NAMES = [
|
||||
'delete_schedule', 'trigger_schedule', 'perplexity_search', 'tavily_search', 'brave_search',
|
||||
'browser_navigate', 'browser_screenshot', 'browser_click', 'browser_fill', 'browser_evaluate',
|
||||
'browser_snapshot', 'lsp_diagnostics', 'lsp_definition', 'lsp_references', 'lsp_hover',
|
||||
'cli_discover', 'cli_execute', 'agent_insights', 'find_connector', 'list_connector_categories',
|
||||
'cli_discover', 'cli_execute', 'find_connector', 'list_connector_categories',
|
||||
'spawn_agent', 'list_agents', 'get_agent_result', 'compose_workflow', 'orchestrate_workflow',
|
||||
'list_harnesses', 'run_harness',
|
||||
] as const;
|
||||
|
||||
@@ -1,341 +0,0 @@
|
||||
/**
|
||||
* H-10 G1 tests — EvolutionService.
|
||||
*
|
||||
* Covers:
|
||||
* - minimum-dataset gate (skips when no target has enough new traces)
|
||||
* - api-key gate (skips when vault is empty)
|
||||
* - target selection (never-run first, then oldest lastRunAt, then most traces)
|
||||
* - baseline resolution (persona + behavioral-spec-section + overrides)
|
||||
* - start/stop/tickInFlight lifecycle
|
||||
* - env flag parsing
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC_SECTIONS } from '@waggle/agent';
|
||||
import {
|
||||
EvolutionService,
|
||||
isEvolutionAutoEnabled,
|
||||
type EvolutionServiceDeps,
|
||||
type EvolutionTargetId,
|
||||
type TickResult,
|
||||
} from '../../src/local/services/evolution-service.js';
|
||||
|
||||
// ── Shared test fixture ──────────────────────────────────────────
|
||||
|
||||
interface Fixture {
|
||||
tmpDir: string;
|
||||
mind: MindDB;
|
||||
traceStore: ExecutionTraceStore;
|
||||
runStore: EvolutionRunStore;
|
||||
apiKey: string | null;
|
||||
activeSpec: Record<string, string>;
|
||||
makeDeps: (overrides?: Partial<EvolutionServiceDeps>) => EvolutionServiceDeps;
|
||||
}
|
||||
|
||||
function setupFixture(): Fixture {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-svc-'));
|
||||
const mind = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
const traceStore = new ExecutionTraceStore(mind);
|
||||
const runStore = new EvolutionRunStore(mind);
|
||||
|
||||
const fx: Fixture = {
|
||||
tmpDir,
|
||||
mind,
|
||||
traceStore,
|
||||
runStore,
|
||||
apiKey: 'sk-test',
|
||||
activeSpec: {},
|
||||
makeDeps: (overrides = {}) => ({
|
||||
traceStore,
|
||||
runStore,
|
||||
getApiKey: () => fx.apiKey,
|
||||
getActiveBehavioralSpec: () => fx.activeSpec,
|
||||
...overrides,
|
||||
}),
|
||||
};
|
||||
return fx;
|
||||
}
|
||||
|
||||
function teardown(fx: Fixture): void {
|
||||
fx.mind.close();
|
||||
try { fs.rmSync(fx.tmpDir, { recursive: true, force: true }); } catch { /* EBUSY on win32 */ }
|
||||
}
|
||||
|
||||
/** Seed `count` successful finalized traces for a given personaId. */
|
||||
function seedTraces(
|
||||
fx: Fixture,
|
||||
opts: { personaId?: string; count: number; outcome?: 'success' | 'corrected' | 'pending' },
|
||||
): void {
|
||||
for (let i = 0; i < opts.count; i++) {
|
||||
const id = fx.traceStore.start({
|
||||
personaId: opts.personaId ?? null,
|
||||
input: `q${i}`,
|
||||
});
|
||||
const outcome = opts.outcome ?? 'success';
|
||||
if (outcome !== 'pending') {
|
||||
fx.traceStore.finalize(id, { outcome, output: `a${i}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────
|
||||
|
||||
describe('EvolutionService — H-10 G1', () => {
|
||||
let fx: Fixture;
|
||||
beforeEach(() => { fx = setupFixture(); });
|
||||
afterEach(() => { teardown(fx); });
|
||||
|
||||
describe('dataset gate + target selection', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
{ kind: 'persona-system-prompt', name: 'writer' },
|
||||
];
|
||||
|
||||
it('pickNextTarget returns null when no persona has enough traces', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('pickNextTarget picks the never-run target that clears the gate', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const chosen = svc.pickNextTarget();
|
||||
expect(chosen?.name).toBe('coder');
|
||||
expect(chosen?.newTraces).toBe(25);
|
||||
expect(chosen?.lastRunAt).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers never-run targets over those with an older lastRunAt', () => {
|
||||
// Both qualify by trace count.
|
||||
seedTraces(fx, { personaId: 'coder', count: 30 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 30 });
|
||||
// coder has a historical run; writer has none.
|
||||
fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()?.name).toBe('writer');
|
||||
});
|
||||
|
||||
it('only counts outcomes in eligibleOutcomes', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 30, outcome: 'pending' });
|
||||
const svc = new EvolutionService(fx.makeDeps(), {
|
||||
targets,
|
||||
minTracesPerTarget: 20,
|
||||
eligibleOutcomes: ['success'],
|
||||
});
|
||||
// All pending — 0 eligible even though 30 exist.
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('counts only traces created since the last run when computing newTraces', () => {
|
||||
// Seed historical traces first.
|
||||
seedTraces(fx, { personaId: 'coder', count: 100 });
|
||||
|
||||
// Record a run with a created_at that's strictly AFTER the trace rows.
|
||||
// SQLite datetime('now') has second-level precision, so we need to bump
|
||||
// the run's timestamp forward by an hour to avoid a same-second
|
||||
// collision that would make "since >= lastRunAt" include the seeded
|
||||
// traces. Using raw SQL is the least invasive way to simulate time
|
||||
// advancing between seed and run creation.
|
||||
const run = fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
const futureTs = new Date(Date.now() + 60 * 60 * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z$/, '');
|
||||
fx.mind.getDatabase()
|
||||
.prepare('UPDATE evolution_runs SET created_at = ? WHERE run_uuid = ?')
|
||||
.run(futureTs, run.run_uuid);
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const candidates = svc.enumerateCandidates();
|
||||
const coder = candidates.find(c => c.name === 'coder');
|
||||
expect(coder?.lastRunAt).toBe(futureTs);
|
||||
expect(coder?.newTraces).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('baseline resolution', () => {
|
||||
it('resolves a real persona baseline from the registry', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for an unknown persona', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'nope' })).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers the active behavioral spec when an override exists', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
fx.activeSpec[section] = 'OVERRIDDEN TEXT';
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section }))
|
||||
.toBe('OVERRIDDEN TEXT');
|
||||
});
|
||||
|
||||
it('falls back to compile-time BEHAVIORAL_SPEC when no override', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for tool-description / skill-body / generic (not auto-evolvable)', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'tool-description', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'skill-body', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'generic', name: 'anything' })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('tick outcomes', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
];
|
||||
|
||||
it('skips when no API key is configured', async () => {
|
||||
fx.apiKey = null;
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 1 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/api key/i);
|
||||
});
|
||||
|
||||
it('skips when no target clears the gate', async () => {
|
||||
// Zero traces for any target.
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/dataset gate/);
|
||||
});
|
||||
|
||||
it('invokes the runner for the picked target when gate passes', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn<Parameters<NonNullable<EvolutionServiceDeps['runner']>>, Promise<TickResult>>()
|
||||
.mockResolvedValue({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: 'uuid-stub',
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
const [calledTarget, calledBaseline, calledKey] = runner.mock.calls[0];
|
||||
// The service passes the full candidate (incl. lastRunAt + newTraces),
|
||||
// not just the bare target id.
|
||||
expect(calledTarget).toMatchObject({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(calledBaseline).toBeTruthy();
|
||||
expect(calledKey).toBe('sk-test');
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.runUuid).toBe('uuid-stub');
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces runner errors through the tick return, not a throw', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn().mockRejectedValue(new Error('boom'));
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/tick failed: boom/);
|
||||
});
|
||||
|
||||
it('reentrancy guard: concurrent tick returns "already in progress"', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
let resolve: ((v: TickResult) => void) | null = null;
|
||||
const runner = vi.fn(() => new Promise<TickResult>(r => { resolve = r; }));
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const first = svc.tick();
|
||||
const second = await svc.tick(); // before first resolves
|
||||
|
||||
expect(second.skipped).toBe(true);
|
||||
if (second.skipped) expect(second.reason).toMatch(/already in progress/);
|
||||
|
||||
// Unblock and let the first finish cleanly.
|
||||
resolve!({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: null,
|
||||
});
|
||||
await first;
|
||||
});
|
||||
});
|
||||
|
||||
describe('lifecycle', () => {
|
||||
it('start/stop toggles isRunning', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
svc.start();
|
||||
expect(svc.isRunning()).toBe(true);
|
||||
svc.stop();
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('start is idempotent — double-start leaves a single timer', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
svc.start();
|
||||
const firstTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
svc.start();
|
||||
const secondTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
expect(firstTimer).toBe(secondTimer);
|
||||
svc.stop();
|
||||
});
|
||||
|
||||
it('clamps tickIntervalMs to at least 60s', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 1 });
|
||||
expect(svc.config.tickIntervalMs).toBe(60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isEvolutionAutoEnabled', () => {
|
||||
it('returns false when the flag is missing or unrelated', () => {
|
||||
expect(isEvolutionAutoEnabled({})).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'false' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '0' })).toBe(false);
|
||||
});
|
||||
it('returns true for "1" / "true" / "yes"', () => {
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '1' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'true' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'TRUE' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'yes' })).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user