refactor: remove self-learning and the optimizer package (P1.4)

Self-learning is excluded from the community edition (ADR-0004); the
project owner reworks it separately. Cost tracking stays.

- agent: delete behaviour evolution (evolve-schema, iterative GEPA,
  compose-evolution, evolution gates, orchestrator and deploy),
  improvement and correction detection, AgentLearning, the eval-dataset
  builder, optimization-capture and the agent_insights tool. Secret
  detection moves from eval-dataset to secret-patterns. The LLM wiring
  keeps only the Haiku adapter and judge call used by skill audit and
  hygiene, so @ax-llm/ax is now declared by agent instead of server.
- server: delete /api/evolution/*, the evolution and optimizer services,
  GEPA expansion of the first chat message, the prompt_optimization and
  monthly_assessment jobs, the idle-session reviewer and its
  session-reviewer persona, prompt injection of past corrections and
  thumbs-down recorded as correction signals. Existing job rows of the
  removed types are ignored, not deleted.
- core: delete OptimizationLogStore; stop re-exporting the improvement
  signal and evolution run stores.
- web: delete the Evolution screen and the two job types in the
  automation editor.
- delete packages/optimizer.

Also logs UF-027 (self-learning loops read data nothing collects) and
UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
shumeiko.it
2026-10-20 00:54:58 +02:00
parent b734c1d3d2
commit 2e1999a3f6
112 changed files with 336 additions and 16165 deletions

View File

@@ -24,6 +24,7 @@
"test": "vitest run"
},
"dependencies": {
"@ax-llm/ax": "^19.0.12",
"@waggle/core": "*",
"@waggle/marketplace": "*",
"@waggle/shared": "*",

View File

@@ -1,194 +0,0 @@
/**
* Agent Learning — persistent behavioral adjustments from performance signals.
*
* Tracks what works and what doesn't across sessions, adjusts agent behavior
* by building a learned context section injected into the system prompt.
*
* Three learning channels:
* 1. Correction patterns — user corrections become persistent rules
* 2. Success patterns — approaches that got positive feedback
* 3. Persona effectiveness — per-persona quality signals
*
* Uses the improvement_signals table for persistence.
*/
import type { ImprovementSignalStore } from '@waggle/core';
export interface LearnedBehavior {
rule: string;
source: 'correction' | 'success' | 'observation';
confidence: number;
occurrences: number;
lastSeen: string;
}
export interface PersonaEffectiveness {
personaId: string;
tasksCompleted: number;
correctionsReceived: number;
positiveSignals: number;
effectivenessScore: number; // 0-100
}
export interface LearningSnapshot {
learnedBehaviors: LearnedBehavior[];
personaStats: PersonaEffectiveness[];
totalCorrections: number;
totalSuccesses: number;
adaptationLevel: 'new' | 'learning' | 'adapted' | 'expert';
}
export class AgentLearning {
private store: ImprovementSignalStore;
constructor(store: ImprovementSignalStore) {
this.store = store;
}
/** Record a successful interaction pattern. */
recordSuccess(pattern: string, personaId?: string): void {
this.store.record('workflow_pattern', `success:${pattern}`, `Approach worked well`, {
type: 'success',
personaId,
recordedAt: new Date().toISOString(),
});
}
/** Record that the user gave positive feedback. */
recordPositiveFeedback(context: string, personaId?: string): void {
this.store.record('correction', `positive:${context.slice(0, 50)}`, context, {
type: 'positive',
personaId,
recordedAt: new Date().toISOString(),
});
}
/** Record a persona's task completion. */
recordPersonaTask(personaId: string, corrected: boolean): void {
const key = corrected ? `persona:${personaId}:corrected` : `persona:${personaId}:completed`;
this.store.record('workflow_pattern', key, personaId, {
type: 'persona_task',
personaId,
corrected,
});
}
/** Build a learning snapshot from accumulated signals. */
getSnapshot(): LearningSnapshot {
const all = this.store.getActionable();
const learnedBehaviors: LearnedBehavior[] = [];
const personaMap = new Map<string, PersonaEffectiveness>();
let totalCorrections = 0;
let totalSuccesses = 0;
for (const signal of all) {
// Corrections become learned behavioral rules
if (signal.category === 'correction' && !signal.pattern_key.startsWith('positive:')) {
totalCorrections += signal.count;
if (signal.count >= 2) { // Only learn from repeated corrections
learnedBehaviors.push({
rule: signal.detail || signal.pattern_key,
source: 'correction',
confidence: Math.min(1.0, 0.5 + signal.count * 0.1),
occurrences: signal.count,
lastSeen: signal.last_seen,
});
}
}
// Positive feedback
if (signal.pattern_key.startsWith('positive:')) {
totalSuccesses += signal.count;
learnedBehaviors.push({
rule: signal.detail || 'Positive approach',
source: 'success',
confidence: Math.min(1.0, 0.6 + signal.count * 0.1),
occurrences: signal.count,
lastSeen: signal.last_seen,
});
}
// Success patterns
if (signal.pattern_key.startsWith('success:')) {
totalSuccesses += signal.count;
}
// Persona stats
if (signal.pattern_key.startsWith('persona:')) {
const parts = signal.pattern_key.split(':');
const personaId = parts[1];
const isCorrected = parts[2] === 'corrected';
let stats = personaMap.get(personaId);
if (!stats) {
stats = { personaId, tasksCompleted: 0, correctionsReceived: 0, positiveSignals: 0, effectivenessScore: 50 };
personaMap.set(personaId, stats);
}
if (isCorrected) {
stats.correctionsReceived += signal.count;
} else {
stats.tasksCompleted += signal.count;
}
}
}
// Calculate persona effectiveness
for (const stats of personaMap.values()) {
const total = stats.tasksCompleted + stats.correctionsReceived;
stats.effectivenessScore = total > 0
? Math.round((stats.tasksCompleted / total) * 100)
: 50;
}
// Determine adaptation level
const totalSignals = totalCorrections + totalSuccesses;
const adaptationLevel = totalSignals < 5 ? 'new'
: totalSignals < 20 ? 'learning'
: totalSignals < 50 ? 'adapted'
: 'expert';
return {
learnedBehaviors: learnedBehaviors.slice(0, 10), // Cap at 10 rules
personaStats: Array.from(personaMap.values()),
totalCorrections,
totalSuccesses,
adaptationLevel,
};
}
/**
* Format learned behaviors as a system prompt section.
* Only included when there are meaningful learned rules.
*/
formatLearningPrompt(): string | null {
const snapshot = this.getSnapshot();
if (snapshot.learnedBehaviors.length === 0) return null;
const lines: string[] = [
'## Learned Behaviors',
`*Adaptation level: ${snapshot.adaptationLevel} (${snapshot.totalCorrections} corrections, ${snapshot.totalSuccesses} successes)*`,
'',
];
const corrections = snapshot.learnedBehaviors.filter(b => b.source === 'correction');
if (corrections.length > 0) {
lines.push('**Avoid these patterns** (user has corrected before):');
for (const b of corrections) {
lines.push(`- ${b.rule} (corrected ${b.occurrences}x, confidence: ${(b.confidence * 100).toFixed(0)}%)`);
}
lines.push('');
}
const successes = snapshot.learnedBehaviors.filter(b => b.source === 'success');
if (successes.length > 0) {
lines.push('**Keep doing** (positive feedback received):');
for (const b of successes) {
lines.push(`- ${b.rule}`);
}
lines.push('');
}
return lines.join('\n');
}
}

View File

@@ -384,8 +384,7 @@ When approaching any task:
};
/**
* Section name literal type — matches the evolution-deploy module.
* Kept minimal here so behavioral-spec.ts stays free of cross-imports.
* Section name literal type for `buildActiveBehavioralSpec` overrides.
*/
export type BehavioralSpecSectionName =
| 'coreLoop'
@@ -402,8 +401,7 @@ export type BehavioralSpecSectionName =
* the compiled baseline unchanged.
*
* Typical usage at server boot:
* const overrides = loadBehavioralSpecOverrides(dataDir);
* const spec = buildActiveBehavioralSpec(overrides);
* const spec = buildActiveBehavioralSpec();
* // ...pass spec.rules into the system prompt...
*/
export function buildActiveBehavioralSpec(

View File

@@ -1,281 +0,0 @@
/**
* Compose Evolution — Phase 3.2 of the self-evolution loop.
*
* Two-stage pipeline:
*
* Stage 1 — EvolveSchema
* Evolve the output STRUCTURE (fields, order, types, constraints).
* Returns a winner schema.
*
* Stage 2 — IterativeGEPA
* Freeze the winner schema. Evolve the INSTRUCTION prompt that fills
* that schema. Only instruction text changes here.
*
* Critical design detail (from Mikhail Pavlukhin's paper): **feedback
* separation**. If the judge complains "missing reasoning field" during
* Stage 2, and we let GEPA see that feedback, GEPA will mutate the
* instruction to try to remove reasoning — undoing the schema Stage 1
* just evolved. So Stage 2 receives a *filtered* judge that strips
* structural complaints and keeps only value-level signals (correctness,
* conciseness, format-of-values, tone).
*
* The composition is stateless: callers provide the baseline schema and
* instruction separately; the result returns both winners + aggregate
* accuracy improvement.
*/
import {
EvolveSchema,
type EvolveSchemaOptions,
type EvolveSchemaResult,
type Schema,
type SchemaExecuteFn,
} from './evolve-schema.js';
import {
IterativeGEPA,
type IterativeGEPAOptions,
type GEPARunResult,
} from './iterative-optimizer.js';
import type { LLMJudge, JudgeScore } from './judge.js';
import type { EvalExample } from './eval-dataset.js';
import { RUNNING_JUDGE_BRAND, isRunningJudge } from './evolution-llm-wiring.js';
// ── Types ───────────────────────────────────────────────────────
export interface ComposeEvolutionOptions {
/** EvolveSchema stage configuration */
schema: Omit<EvolveSchemaOptions, 'onProgress' | 'signal'>;
/** IterativeGEPA stage configuration */
instructions: Omit<IterativeGEPAOptions, 'onProgress' | 'signal'>;
/**
* Classifier — returns 'structural' to drop a feedback line from GEPA,
* or 'value' to keep it. Default: `defaultFeedbackFilter`.
*/
feedbackFilter?: FeedbackFilter;
/** Optional progress reporter across both stages */
onProgress?: (event: ComposeProgress) => void;
/** Optional abort signal (passed through to both stages) */
signal?: AbortSignal;
}
export interface ComposeEvolutionResult {
schema: EvolveSchemaResult;
instructions: GEPARunResult;
/**
* Accuracy delta after both stages compared to the raw baseline schema +
* baseline instructions. Positive = improvement.
*/
combinedDelta: number;
/** True if both stages improved over their individual baselines. */
fullyImproved: boolean;
/** Frozen schema used as input to the GEPA stage — convenience pointer. */
frozenSchema: Schema;
}
export interface ComposeProgress {
stage: 'schema' | 'instructions' | 'done';
/** Underlying stage progress event, if available */
detail?: unknown;
message?: string;
}
export type FeedbackFilter = (feedback: string) => 'structural' | 'value';
// ── Feedback filtering ─────────────────────────────────────────
/**
* Default heuristic: classifies feedback lines as 'structural' when they
* mention schema-level concepts (field, schema, missing/extra property,
* wrong type, reorder, etc). Everything else is 'value'.
*
* Tight enough that ambiguous feedback still flows to GEPA — only obvious
* structural complaints are dropped.
*/
export function defaultFeedbackFilter(feedback: string): 'structural' | 'value' {
if (!feedback) return 'value';
const lower = feedback.toLowerCase();
const structuralPatterns: RegExp[] = [
/\bmissing\s+(?:the\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
/\badd(?:ed|ing)?\s+(?:a|the)?\s*(?:new\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
/\bextra\s+field\b/,
/\bunexpected\s+field\b/,
/\bwrong\s+(?:field\s+)?(?:type|order)\b/,
/\bfield\s+type\s+(?:mismatch|wrong)/,
/\b(?:should\s+)?(?:re)?order\s+(?:the\s+)?fields\b/,
/\bschema\s+(?:mismatch|violation|error)\b/,
/\binvalid\s+json\s+shape\b/,
/\bmissing\s+required\s+property\b/,
/\bproperty\s+missing\b/,
/\bshape\s+is\s+wrong\b/,
];
for (const pattern of structuralPatterns) {
if (pattern.test(lower)) return 'structural';
}
return 'value';
}
// ── Judge wrapping ─────────────────────────────────────────────
/**
* Wrap an LLMJudge-like object so the feedback it emits has structural
* lines stripped according to `filter`. Only `feedback` is filtered —
* the numerical scores (correctness, procedure, conciseness, overall)
* are preserved as-is so GEPA sees the true accuracy signal.
*
* The multi-line feedback from the judge is split on newlines and each
* line classified individually.
*/
export function filterJudgeFeedback(
judge: Pick<LLMJudge, 'score'>,
filter: FeedbackFilter = defaultFeedbackFilter,
): Pick<LLMJudge, 'score'> {
const wrapped: Pick<LLMJudge, 'score'> & Partial<Record<symbol, unknown>> = {
async score(args) {
const raw = await judge.score(args);
const filteredFeedback = stripStructuralLines(raw.feedback, filter);
const result: JudgeScore = { ...raw, feedback: filteredFeedback };
return result;
},
};
// H-09 G3: preserve the running-judge brand so IterativeGEPA's check
// still passes after feedback filtering. Without this, wrapping a
// running judge with filterJudgeFeedback would silently strip the
// brand and GEPA would reject it.
if (isRunningJudge(judge)) {
wrapped[RUNNING_JUDGE_BRAND] = true;
}
return wrapped;
}
/** Public helper — strips structural lines from a feedback string. */
export function stripStructuralLines(
feedback: string,
filter: FeedbackFilter = defaultFeedbackFilter,
): string {
if (!feedback) return feedback;
return feedback
.split(/\r?\n/)
.filter(line => filter(line) === 'value')
.join('\n')
.trim();
}
// ── Orchestrator ───────────────────────────────────────────────
export class ComposeEvolution {
async run(options: ComposeEvolutionOptions): Promise<ComposeEvolutionResult> {
const signal = options.signal;
const filter = options.feedbackFilter ?? defaultFeedbackFilter;
// Stage 1 — evolve the schema.
options.onProgress?.({ stage: 'schema', message: 'starting schema evolution' });
const schemaResult = await new EvolveSchema().run({
...options.schema,
signal,
onProgress: (e) => options.onProgress?.({ stage: 'schema', detail: e }),
});
if (signal?.aborted) {
return assembleAbortedResult(schemaResult, options.instructions.baseline);
}
const frozenSchema = schemaResult.winner.schema;
// Stage 2 — evolve the instructions, with a filtered judge so GEPA
// never sees structural feedback.
options.onProgress?.({ stage: 'instructions', message: 'starting instruction evolution' });
const filteredJudge = filterJudgeFeedback(options.instructions.judge, filter);
const gepaResult = await new IterativeGEPA().run({
...options.instructions,
judge: filteredJudge,
signal,
onProgress: (e) => options.onProgress?.({ stage: 'instructions', detail: e }),
});
const combinedDelta =
(gepaResult.winner.score?.overall ?? 0) -
((schemaResult.history[0]?.score?.accuracy ?? 0));
const fullyImproved = schemaResult.improved && gepaResult.improved;
options.onProgress?.({ stage: 'done' });
return {
schema: schemaResult,
instructions: gepaResult,
combinedDelta,
fullyImproved,
frozenSchema,
};
}
}
// ── Helpers ─────────────────────────────────────────────────────
/**
* Build a convenience `SchemaExecuteFn` from a user-supplied runner that
* only cares about the instruction prompt (not the schema itself). Used
* by callers that want to compose — they supply one "execute" function
* for the instructional stage and this helper wraps it for Stage 1.
*
* The wrapped executor serializes the schema into a short prefix the
* model can follow (`Return JSON with fields: <field1>, <field2>, ...`)
* before delegating to the caller's function.
*/
export function schemaExecutorFromInstructionRunner(
runInstructions: (args: { prompt: string; input: string }) => Promise<string>,
): SchemaExecuteFn {
return async ({ schema, input }) => {
const fieldList = schema.fields.map(f => `"${f.name}" (${f.type})`).join(', ');
const prompt = `Return a JSON object with these fields: ${fieldList}.`;
try {
const actual = await runInstructions({ prompt, input });
return { actual, parsed: actualLooksLikeJson(actual) };
} catch {
return { actual: '', parsed: false };
}
};
}
function actualLooksLikeJson(s: string): boolean {
const trimmed = s.trim();
if (!trimmed) return false;
return (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
(trimmed.startsWith('[') && trimmed.endsWith(']'));
}
function assembleAbortedResult(
schemaResult: EvolveSchemaResult,
instructionBaseline: string,
): ComposeEvolutionResult {
// When aborted before Stage 2, produce a minimal GEPARunResult pointing
// at the baseline instruction so callers have a stable shape.
const baselineInstruction = {
id: 'aborted',
prompt: instructionBaseline,
generation: 0,
parent: null,
strategy: 'aborted',
score: null,
perExample: [],
} as GEPARunResult['winner'];
return {
schema: schemaResult,
instructions: {
winner: baselineInstruction,
paretoFront: [baselineInstruction],
history: [baselineInstruction],
improved: false,
delta: 0,
},
combinedDelta: 0,
fullyImproved: false,
frozenSchema: schemaResult.winner.schema,
};
}
/** Re-export types so callers can `import { type Schema } from '...'`. */
export type { EvolveSchemaResult, GEPARunResult };

View File

@@ -1,199 +0,0 @@
/**
* Correction Detector — identifies when a user is correcting the agent
* and classifies corrections as durable (behavior-changing) vs task-local.
*
* Per Slice 7 correction #2:
* - Durable corrections: repeated patterns that should influence future behavior → improvement signals
* - Task-local corrections: one-off adjustments within a session → NOT signals
* - One-off disagreements: not corrections at all → ignored
*/
export type CorrectionDurability = 'durable' | 'task_local' | 'not_correction';
export interface DetectedCorrection {
isDurable: boolean;
durability: CorrectionDurability;
patternKey: string;
detail: string;
confidence: number; // 0-1
}
// ── Correction signal patterns ──────────────────────────────
/** Strong correction signals — high confidence the user is correcting agent behavior */
const STRONG_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
{ pattern: /\bno[,.]?\s*(?:not that|don'?t|do not|stop|never)\b/i, weight: 3 },
{ pattern: /\bI (?:said|told you|asked)\b/i, weight: 3 },
{ pattern: /\bthat'?s (?:wrong|incorrect|not (?:what|right))\b/i, weight: 3 },
{ pattern: /\bplease (?:don'?t|do not|stop|never)\b/i, weight: 2 },
{ pattern: /\binstead[,.]?\s*(?:use|do|try|go with)\b/i, weight: 2 },
{ pattern: /\bwrong (?:approach|way|format|style|tone)\b/i, weight: 2 },
{ pattern: /\bnot what I (?:wanted|meant|asked)\b/i, weight: 3 },
];
/** Moderate correction signals — may be correction or just refinement */
const MODERATE_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
{ pattern: /^no[,.:!]\s/i, weight: 2 }, // "No, ..." at start of message — strong disagreement
{ pattern: /\bactually[,.]?\s/i, weight: 1 },
{ pattern: /\brather[,.]?\s/i, weight: 1 },
{ pattern: /\blet'?s (?:not|try|go with|use)\b/i, weight: 1 },
{ pattern: /\bprefer\b/i, weight: 1 },
{ pattern: /\bshould (?:be|have been|use)\b/i, weight: 1 },
{ pattern: /\bchange (?:it|this|that) to\b/i, weight: 1 },
{ pattern: /\btoo (?:verbose|long|short|formal|casual|technical|simple)\b/i, weight: 1 },
{ pattern: /\bstop (?:doing|using|adding)\b/i, weight: 1 },
{ pattern: /\bkeep (?:it|things) (?:simple|short|brief|casual|formal)\b/i, weight: 1 },
];
// ── Durability classification patterns ──────────────────────
/** Durable signals — the correction applies beyond this specific task */
const DURABLE_SIGNALS: RegExp[] = [
/\balways\b/i,
/\bnever\b/i,
/\bfrom now on\b/i,
/\bin (?:the )?future\b/i,
/\bwhenever\b/i,
/\bevery time\b/i,
/\bin general\b/i,
/\bI (?:always |usually )?prefer\b/i,
/\bmy (?:preference|style|approach)\b/i,
/\bdon'?t (?:ever|again)\b/i,
/\bremember (?:to|that)\b/i,
/\bkeep (?:it|things|this)\b/i,
];
/** Task-local signals — the correction is specific to this task */
const TASK_LOCAL_SIGNALS: RegExp[] = [
/\bthis (?:time|one|specific|particular)\b/i,
/\bfor (?:this|now)\b/i,
/\bjust (?:here|this|now)\b/i,
/\bin this (?:case|instance|response)\b/i,
/\bright now\b/i,
/\bhere\b/i,
];
// ── Pattern key extraction ──────────────────────────────────
/** Categories of behavioral corrections we can extract pattern keys for */
const PATTERN_KEY_EXTRACTORS: Array<{ category: string; pattern: RegExp }> = [
{ category: 'tone', pattern: /\btoo (?:formal|casual|verbose|terse|technical|simple)\b/i },
{ category: 'format', pattern: /\b(?:format|formatting|headers?|bullet|numbering|markdown)\b/i },
{ category: 'length', pattern: /\btoo (?:long|short|brief|detailed)\b/i },
{ category: 'approach', pattern: /\bwrong (?:approach|way|method|strategy)\b/i },
{ category: 'scope', pattern: /\btoo (?:much|many|broad|narrow|specific|general)\b/i },
{ category: 'accuracy', pattern: /\b(?:wrong|incorrect|inaccurate|mistake|error)\b/i },
{ category: 'style', pattern: /\b(?:style|voice|writing|wording|phrasing)\b/i },
];
/**
* Detect whether a user message contains a correction and classify it.
*
* Returns null if no correction detected.
* Returns DetectedCorrection with durability classification if correction found.
*/
export function detectCorrection(
userMessage: string,
previousAssistantMessage?: string,
): DetectedCorrection | null {
if (!userMessage || userMessage.length < 5) return null;
// Score correction strength
let correctionScore = 0;
for (const { pattern, weight } of STRONG_CORRECTION_PATTERNS) {
if (pattern.test(userMessage)) correctionScore += weight;
}
for (const { pattern, weight } of MODERATE_CORRECTION_PATTERNS) {
if (pattern.test(userMessage)) correctionScore += weight;
}
// Need minimum score to count as correction
if (correctionScore < 2) return null;
// Classify durability
const durability = classifyDurability(userMessage);
// Extract pattern key
const patternKey = extractPatternKey(userMessage);
// Compute confidence (0-1)
const confidence = Math.min(correctionScore / 6, 1);
// Build detail: first sentence or first 120 chars
const detail = extractDetail(userMessage);
return {
isDurable: durability === 'durable',
durability,
patternKey,
detail,
confidence,
};
}
function classifyDurability(message: string): CorrectionDurability {
let durableScore = 0;
let taskLocalScore = 0;
for (const pattern of DURABLE_SIGNALS) {
if (pattern.test(message)) durableScore++;
}
for (const pattern of TASK_LOCAL_SIGNALS) {
if (pattern.test(message)) taskLocalScore++;
}
// Explicit durable signals win
if (durableScore > 0 && durableScore >= taskLocalScore) return 'durable';
// Explicit task-local signals
if (taskLocalScore > 0) return 'task_local';
// No explicit signals — default to task_local (conservative; only promote to durable
// when the same pattern_key recurs across sessions via ImprovementSignalStore)
return 'task_local';
}
function extractPatternKey(message: string): string {
for (const { category, pattern } of PATTERN_KEY_EXTRACTORS) {
const match = message.match(pattern);
if (match) {
const qualifier = match[0].toLowerCase().replace(/\s+/g, '_');
return `${category}:${qualifier}`;
}
}
// Fallback: generic correction key
return 'general:correction';
}
function extractDetail(message: string): string {
// Take first sentence or first 120 chars
const sentenceMatch = message.match(/^(.+?[.!?])\s/);
if (sentenceMatch && sentenceMatch[1].length <= 120) {
return sentenceMatch[1];
}
return message.length > 120 ? message.slice(0, 117) + '...' : message;
}
/**
* Analyze a sequence of message pairs to detect corrections.
* Useful for batch analysis of session history.
*/
export function detectCorrectionsInHistory(
messages: Array<{ role: string; content: string }>,
): DetectedCorrection[] {
const corrections: DetectedCorrection[] = [];
for (let i = 1; i < messages.length; i++) {
const msg = messages[i];
if (msg.role !== 'user') continue;
const prevAssistant = i > 0 ? messages[i - 1] : undefined;
const correction = detectCorrection(
msg.content,
prevAssistant?.role === 'assistant' ? prevAssistant.content : undefined,
);
if (correction) {
corrections.push(correction);
}
}
return corrections;
}

View File

@@ -1,441 +0,0 @@
/**
* Eval Dataset Builder — turns execution traces and harvest frames into
* train/val/holdout splits for the self-evolution loop.
*
* Sources mined:
* - ExecutionTraceStore (outcome = success | verified → positive examples;
* outcome = corrected → negative example with correctionFeedback as ground truth)
* - Optional harvest frames (DistilledKnowledge) for Q&A-style augmentation
* - Optional corrections from ImprovementSignalStore
*
* Filter pipeline (in order):
* 1. Secret scanning — reject any example containing credentials / tokens
* 2. Keyword heuristic — min length, non-trivial content, no duplicate inputs
* 3. Optional LLM-as-judge relevance filter (pass an llmCall callback)
*
* Split is deterministic given a seed — same traces + same seed = same split.
* Default split ratio is 60/20/20 train/val/holdout.
*/
import type {
ExecutionTraceStore,
ParsedExecutionTrace,
TraceOutcome,
TraceQueryFilter,
} from '@waggle/core';
// ── Types ───────────────────────────────────────────────────────
export interface EvalExample {
input: string;
expected_output: string;
metadata: EvalExampleMetadata;
}
export interface EvalExampleMetadata {
traceId?: number;
personaId?: string | null;
taskShape?: string | null;
model?: string | null;
outcome?: TraceOutcome;
tags?: string[];
/** Where this example came from */
source: 'trace' | 'harvest' | 'correction';
/** Optional opaque identifier from the source system */
sourceId?: string;
}
export interface DatasetSplit {
train: EvalExample[];
val: EvalExample[];
holdout: EvalExample[];
/** Examples that were filtered out — useful for debugging */
rejected: Array<{ reason: string; preview: string }>;
/** Stats for logging / UI */
stats: {
sourced: number;
acceptedAfterSecretScan: number;
acceptedAfterHeuristic: number;
acceptedAfterJudge: number;
unique: number;
total: number;
};
}
export interface BuildOptions {
/** Which trace outcomes count as positive examples (default: ['success', 'verified']). */
positiveOutcomes?: TraceOutcome[];
/** Whether to include correction traces as negative examples with feedback as expected_output (default true). */
includeCorrections?: boolean;
/** Filter passed straight through to ExecutionTraceStore.query */
traceFilter?: Omit<TraceQueryFilter, 'outcome' | 'limit'> & { limit?: number };
/** Optional external augmenters */
harvestExamples?: EvalExample[];
correctionExamples?: EvalExample[];
/** Minimum input characters to keep the example (default 10). */
minInputChars?: number;
/** Minimum expected_output characters to keep the example (default 5). */
minOutputChars?: number;
/** Maximum input + output characters combined; over this, the example is rejected (default 16384). */
maxCombinedChars?: number;
/** Optional LLM-as-judge relevance filter. Pass null or omit to skip. */
judge?: (example: EvalExample) => Promise<JudgeVerdict>;
/** Ratio triple summing to 1.0 (default [0.6, 0.2, 0.2]). */
splitRatios?: [number, number, number];
/** Deterministic seed (default 1). */
seed?: number;
}
export interface JudgeVerdict {
keep: boolean;
reason?: string;
}
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
];
/** Returns the first secret pattern the text matches, or null if clean. */
export function detectSecrets(text: string): string | null {
for (const { name, re } of SECRET_PATTERNS) {
if (re.test(text)) return name;
}
return null;
}
/** Exported for tests / tools that want the full list. */
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
/**
* Redact every secret-pattern match in a string, replacing each with
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
* detectSecrets so there is a single source of truth for "what is a secret".
*/
export function redactSecrets(text: string): { text: string; found: string[] } {
let out = text;
const found: string[] = [];
for (const { name, re } of SECRET_PATTERNS) {
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
const replaced = out.replace(global, `[REDACTED:${name}]`);
if (replaced !== out) {
found.push(name);
out = replaced;
}
}
return { text: out, found };
}
// ── Deterministic PRNG (mulberry32) ─────────────────────────────
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
/** Fisher-Yates shuffle in-place using provided rng. */
function shuffle<T>(arr: T[], rng: () => number): T[] {
for (let i = arr.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr;
}
// ── Builder ─────────────────────────────────────────────────────
export class EvalDatasetBuilder {
private store: ExecutionTraceStore;
constructor(store: ExecutionTraceStore) {
this.store = store;
}
/** Mine traces into a typed example list (before filtering). */
sourceFromTraces(
positiveOutcomes: TraceOutcome[],
includeCorrections: boolean,
filter: BuildOptions['traceFilter'] = {},
): EvalExample[] {
const outcomes: TraceOutcome[] = [...positiveOutcomes];
if (includeCorrections) outcomes.push('corrected');
const rows = this.store.queryParsed({
...filter,
outcome: outcomes,
limit: filter.limit ?? 10_000,
});
return rows.map(row => traceToExample(row, includeCorrections));
}
/**
* Build the full dataset split.
*
* Filter order:
* 1. Source (traces + augmenters)
* 2. Secret scan
* 3. Heuristic (length, non-trivial)
* 4. Optional judge
* 5. Dedup on input hash
* 6. Shuffle + split
*/
async build(options: BuildOptions = {}): Promise<DatasetSplit> {
const positiveOutcomes = options.positiveOutcomes ?? ['success', 'verified'];
const includeCorrections = options.includeCorrections ?? true;
const minInputChars = options.minInputChars ?? 10;
const minOutputChars = options.minOutputChars ?? 5;
const maxCombinedChars = options.maxCombinedChars ?? 16_384;
const splitRatios = options.splitRatios ?? [0.6, 0.2, 0.2];
const seed = options.seed ?? 1;
validateRatios(splitRatios);
const rejected: DatasetSplit['rejected'] = [];
// 1. Source
const traceExamples = this.sourceFromTraces(
positiveOutcomes,
includeCorrections,
options.traceFilter,
);
const sourced: EvalExample[] = [
...traceExamples,
...(options.harvestExamples ?? []),
...(options.correctionExamples ?? []),
];
// 2. Secret scan
const afterSecretScan: EvalExample[] = [];
for (const ex of sourced) {
const hitInput = detectSecrets(ex.input);
const hitOutput = detectSecrets(ex.expected_output);
if (hitInput || hitOutput) {
rejected.push({
reason: `secret:${hitInput ?? hitOutput}`,
preview: ex.input.slice(0, 80),
});
continue;
}
afterSecretScan.push(ex);
}
// 3. Heuristic
const afterHeuristic: EvalExample[] = [];
for (const ex of afterSecretScan) {
const input = ex.input.trim();
const output = ex.expected_output.trim();
if (input.length < minInputChars) {
rejected.push({ reason: 'too-short-input', preview: input.slice(0, 80) });
continue;
}
if (output.length < minOutputChars) {
rejected.push({ reason: 'too-short-output', preview: input.slice(0, 80) });
continue;
}
if (input.length + output.length > maxCombinedChars) {
rejected.push({ reason: 'too-long', preview: input.slice(0, 80) });
continue;
}
if (isLowSignal(input) || isLowSignal(output)) {
rejected.push({ reason: 'low-signal', preview: input.slice(0, 80) });
continue;
}
afterHeuristic.push({ ...ex, input, expected_output: output });
}
// 4. Optional judge
let afterJudge = afterHeuristic;
if (options.judge) {
afterJudge = [];
for (const ex of afterHeuristic) {
try {
const verdict = await options.judge(ex);
if (verdict.keep) {
afterJudge.push(ex);
} else {
rejected.push({
reason: `judge:${verdict.reason ?? 'rejected'}`,
preview: ex.input.slice(0, 80),
});
}
} catch (err) {
// On judge failure, keep the example — err on the side of more data.
afterJudge.push(ex);
}
}
}
// 5. Dedup on input hash
const seen = new Set<string>();
const unique: EvalExample[] = [];
for (const ex of afterJudge) {
const key = hashKey(ex.input);
if (seen.has(key)) {
rejected.push({ reason: 'duplicate', preview: ex.input.slice(0, 80) });
continue;
}
seen.add(key);
unique.push(ex);
}
// 6. Shuffle + split
const rng = makeRng(seed);
const shuffled = shuffle([...unique], rng);
const { train, val, holdout } = splitExamples(shuffled, splitRatios);
return {
train,
val,
holdout,
rejected,
stats: {
sourced: sourced.length,
acceptedAfterSecretScan: afterSecretScan.length,
acceptedAfterHeuristic: afterHeuristic.length,
acceptedAfterJudge: afterJudge.length,
unique: unique.length,
total: train.length + val.length + holdout.length,
},
};
}
}
// ── JSONL IO ────────────────────────────────────────────────────
/** Serialize examples as JSONL (one JSON object per line). */
export function toJSONL(examples: EvalExample[]): string {
return examples.map(ex => JSON.stringify(ex)).join('\n');
}
/** Parse JSONL into examples. Silently skips unparseable lines. */
export function fromJSONL(jsonl: string): EvalExample[] {
const out: EvalExample[] = [];
for (const line of jsonl.split(/\r?\n/)) {
if (!line.trim()) continue;
try {
const parsed = JSON.parse(line);
if (
parsed &&
typeof parsed.input === 'string' &&
typeof parsed.expected_output === 'string' &&
parsed.metadata
) {
out.push(parsed as EvalExample);
}
} catch {
// skip malformed line
}
}
return out;
}
// ── Helpers ─────────────────────────────────────────────────────
function traceToExample(
trace: ParsedExecutionTrace,
includeCorrections: boolean,
): EvalExample {
const isCorrected = trace.outcome === 'corrected';
const expected = isCorrected && includeCorrections
? (trace.payload.correctionFeedback ?? trace.payload.output)
: trace.payload.output;
return {
input: trace.payload.input,
expected_output: expected,
metadata: {
traceId: trace.id,
personaId: trace.persona_id,
taskShape: trace.task_shape,
model: trace.model,
outcome: trace.outcome,
tags: trace.payload.tags ?? [],
source: isCorrected ? 'correction' : 'trace',
},
};
}
function validateRatios(ratios: [number, number, number]): void {
const sum = ratios[0] + ratios[1] + ratios[2];
if (Math.abs(sum - 1.0) > 1e-6) {
throw new Error(`Split ratios must sum to 1.0, got ${sum}`);
}
if (ratios.some(r => r < 0)) {
throw new Error('Split ratios must be non-negative');
}
}
function splitExamples(
examples: EvalExample[],
ratios: [number, number, number],
): { train: EvalExample[]; val: EvalExample[]; holdout: EvalExample[] } {
const n = examples.length;
// Floor-then-assign-remainders so every example ends up in exactly one split.
const trainN = Math.floor(n * ratios[0]);
const valN = Math.floor(n * ratios[1]);
const holdoutN = n - trainN - valN;
return {
train: examples.slice(0, trainN),
val: examples.slice(trainN, trainN + valN),
holdout: examples.slice(trainN + valN, trainN + valN + holdoutN),
};
}
/**
* Cheap signal check — reject examples that are mostly whitespace, repeated
* punctuation, or otherwise meaningless. Not a language detector.
*/
function isLowSignal(text: string): boolean {
if (!text) return true;
const stripped = text.replace(/\s+/g, '');
if (stripped.length < 3) return true;
// 80%+ of the same character → reject
const counts = new Map<string, number>();
for (const ch of stripped) {
counts.set(ch, (counts.get(ch) ?? 0) + 1);
}
const maxCount = Math.max(...counts.values());
if (maxCount / stripped.length > 0.8) return true;
// No alphanumeric at all → reject
if (!/[A-Za-z0-9]/.test(stripped)) return true;
return false;
}
/**
* FNV-1a 32-bit hash — stable across processes, no crypto import needed.
* Used for input-level dedup.
*/
function hashKey(text: string): string {
let h = 0x811c9dc5;
for (let i = 0; i < text.length; i++) {
h ^= text.charCodeAt(i);
h = Math.imul(h, 0x01000193);
}
return (h >>> 0).toString(16);
}

View File

@@ -1,299 +0,0 @@
/**
* Evolution Deploy Helpers — Phase 6.1 of the self-evolution loop.
*
* When the user accepts an EvolutionRun, the orchestrator calls the
* caller-supplied `deploy` callback. This module provides the built-in
* building blocks for that callback:
*
* deployPersonaOverride — writes a persona JSON file that
* shadows the built-in persona at
* runtime via loadCustomPersonas.
* deployBehavioralSpecOverride — writes a JSON file in
* {dataDir}/behavioral-overrides/
* that can be merged into
* BEHAVIORAL_SPEC sections at load.
* loadBehavioralSpecOverrides — reads those overrides from disk.
*
* Every writer:
* - Creates the target directory if missing.
* - Writes atomically (write to .tmp, rename) so a crash mid-write
* never corrupts the config.
* - Backs up the previous version to {file}.bak before overwriting.
* - Returns a structured DeployResult with paths + timestamps.
*
* These helpers are pure filesystem operations — they don't talk to
* any runtime singletons, so the server wiring is trivial:
* orchestrator.accept(uuid) → onDeploy(run)
* → deployPersonaOverride(dataDir, ...)
* → reloadPersonas()
*/
import fs from 'node:fs';
import path from 'node:path';
import type { AgentPersona } from './personas.js';
import { getPersona } from './personas.js';
import { assertValidCustomPersonaId } from './custom-personas.js';
// ── Public result shape ────────────────────────────────────────
export interface DeployResult {
/** Absolute path that was written */
path: string;
/** Path to the backup of the previous version (null if first write) */
backupPath: string | null;
/** When the deploy happened */
deployedAt: string;
}
// ── Persona override ───────────────────────────────────────────
export interface DeployPersonaInput {
/** Persona id — matches a built-in persona (the deploy SHADOWS it). */
personaId: string;
/** The evolved system prompt text */
systemPrompt: string;
/** Optional additional field overrides */
overrides?: Partial<Omit<AgentPersona, 'id' | 'systemPrompt'>>;
}
/**
* Write an evolved persona system prompt to disk so the custom-personas
* loader picks it up on next `listPersonas()` call.
*
* When a built-in persona with this id exists, the override inherits
* all its properties (name, tools, tagline, etc) except systemPrompt
* (replaced with the evolved one). When no built-in exists, a minimal
* persona shell is built from the input.
*/
export function deployPersonaOverride(
dataDir: string,
input: DeployPersonaInput,
): DeployResult {
assertValidCustomPersonaId(input.personaId);
const personasDir = path.join(dataDir, 'personas');
if (!fs.existsSync(personasDir)) {
fs.mkdirSync(personasDir, { recursive: true });
}
const filePath = path.join(personasDir, `${input.personaId}.json`);
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
if (backupPath) {
fs.copyFileSync(filePath, backupPath);
}
const builtin = getPersona(input.personaId);
const persona: AgentPersona = builtin
? {
...builtin,
...input.overrides,
id: input.personaId,
systemPrompt: input.systemPrompt,
}
: {
name: input.personaId,
description: input.overrides?.description ?? `${input.personaId} (evolved)`,
icon: input.overrides?.icon ?? 'sparkles',
modelPreference: input.overrides?.modelPreference ?? 'claude-sonnet-4-6',
tools: input.overrides?.tools ?? [],
workspaceAffinity: input.overrides?.workspaceAffinity ?? [],
suggestedCommands: input.overrides?.suggestedCommands ?? [],
defaultWorkflow: input.overrides?.defaultWorkflow ?? null,
...input.overrides,
id: input.personaId,
systemPrompt: input.systemPrompt,
};
writeAtomic(filePath, JSON.stringify(persona, null, 2));
return {
path: filePath,
backupPath,
deployedAt: new Date().toISOString(),
};
}
/**
* Roll back a previously-deployed persona override to the .bak version
* (or remove the override entirely if there was no previous version).
*/
export function rollbackPersonaOverride(
dataDir: string,
personaId: string,
): boolean {
assertValidCustomPersonaId(personaId);
const filePath = path.join(dataDir, 'personas', `${personaId}.json`);
const backupPath = `${filePath}.bak`;
if (fs.existsSync(backupPath)) {
fs.renameSync(backupPath, filePath);
return true;
}
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
return true;
}
return false;
}
// ── Behavioral-spec override ──────────────────────────────────
/**
* BEHAVIORAL_SPEC is compiled into the bundle, so we cannot mutate it
* on disk without rebuilding. Instead, overrides live as JSON files in
* {dataDir}/behavioral-overrides/ and the runtime merges them at load.
*
* One file per section, filename = section name (coreLoop, qualityRules,
* behavioralRules, workPatterns, intelligenceDefaults).
*/
export type BehavioralSpecSection =
| 'coreLoop'
| 'qualityRules'
| 'behavioralRules'
| 'workPatterns'
| 'intelligenceDefaults';
export const BEHAVIORAL_SPEC_SECTIONS: BehavioralSpecSection[] = [
'coreLoop',
'qualityRules',
'behavioralRules',
'workPatterns',
'intelligenceDefaults',
];
export interface DeployBehavioralSpecInput {
/** Which section to override */
section: BehavioralSpecSection;
/** The evolved text for that section */
text: string;
/** Optional evolution-run uuid for audit */
runUuid?: string;
}
export interface BehavioralSpecOverride {
section: BehavioralSpecSection;
text: string;
runUuid?: string;
deployedAt: string;
}
/**
* Write a behavioral-spec section override to disk. The runtime loader
* (loadBehavioralSpecOverrides) reads these on startup and merges them
* into the compiled BEHAVIORAL_SPEC at runtime.
*/
export function deployBehavioralSpecOverride(
dataDir: string,
input: DeployBehavioralSpecInput,
): DeployResult {
if (!BEHAVIORAL_SPEC_SECTIONS.includes(input.section)) {
throw new Error(`Unknown behavioral-spec section: ${input.section}`);
}
const dir = path.join(dataDir, 'behavioral-overrides');
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
const filePath = path.join(dir, `${input.section}.json`);
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
if (backupPath) {
fs.copyFileSync(filePath, backupPath);
}
const payload: BehavioralSpecOverride = {
section: input.section,
text: input.text,
runUuid: input.runUuid,
deployedAt: new Date().toISOString(),
};
writeAtomic(filePath, JSON.stringify(payload, null, 2));
return {
path: filePath,
backupPath,
deployedAt: payload.deployedAt,
};
}
/** Roll back one section (restores .bak, or removes if no previous override). */
export function rollbackBehavioralSpecOverride(
dataDir: string,
section: BehavioralSpecSection,
): boolean {
const filePath = path.join(dataDir, 'behavioral-overrides', `${section}.json`);
const backupPath = `${filePath}.bak`;
if (fs.existsSync(backupPath)) {
fs.renameSync(backupPath, filePath);
return true;
}
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
return true;
}
return false;
}
/** Read all behavioral-spec overrides from disk as a map section → text. */
export function loadBehavioralSpecOverrides(
dataDir: string,
): Partial<Record<BehavioralSpecSection, string>> {
const dir = path.join(dataDir, 'behavioral-overrides');
if (!fs.existsSync(dir)) return {};
const out: Partial<Record<BehavioralSpecSection, string>> = {};
try {
for (const section of BEHAVIORAL_SPEC_SECTIONS) {
const filePath = path.join(dir, `${section}.json`);
if (!fs.existsSync(filePath)) continue;
try {
const raw = fs.readFileSync(filePath, 'utf-8');
const parsed = JSON.parse(raw) as Partial<BehavioralSpecOverride>;
if (parsed && typeof parsed.text === 'string' && parsed.text.length > 0) {
out[section] = parsed.text;
}
} catch { /* skip malformed */ }
}
} catch { /* dir unreadable */ }
return out;
}
/** Merge overrides into a baseline spec object, preferring the override text. */
export function applyBehavioralSpecOverrides(
baseline: Record<BehavioralSpecSection, string>,
overrides: Partial<Record<BehavioralSpecSection, string>>,
): Record<BehavioralSpecSection, string> {
return {
...baseline,
...Object.fromEntries(
(Object.entries(overrides) as Array<[BehavioralSpecSection, string]>)
.filter(([, v]) => typeof v === 'string' && v.length > 0),
),
} as Record<BehavioralSpecSection, string>;
}
// ── Internal ───────────────────────────────────────────────────
/** Atomic write — write to .tmp then rename into place. */
function writeAtomic(filePath: string, contents: string): void {
const tmpPath = `${filePath}.tmp`;
fs.writeFileSync(tmpPath, contents, 'utf-8');
const waitBuffer = new Int32Array(new SharedArrayBuffer(4));
try {
for (let attempt = 1; attempt <= 10; attempt++) {
try {
fs.renameSync(tmpPath, filePath);
return;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
const transient = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
if (!transient || attempt === 10) throw error;
// Windows antivirus and indexers can briefly hold an exclusive handle.
Atomics.wait(waitBuffer, 0, 0, 25 * attempt);
}
}
} finally {
try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ }
}
}

View File

@@ -1,316 +0,0 @@
/**
* Evolution Gates — Phase 2.2 of the self-evolution loop.
*
* Gatekeeping for prompts/skills produced by the iterative GEPA optimizer
* before they reach production. A candidate must pass ALL gates; the first
* failure short-circuits and returns a `fail` verdict with the reason.
*
* Four gate categories:
*
* 1. Size gates — hard character caps per target type (persona prompt
* ≤ 3 000 chars, tool description ≤ 500, skill ≤ 15 KB,
* behavioral-spec section ≤ 4 000).
* 2. Growth gates — percentage cap over baseline (default +20%) prevents
* runaway prompt bloat across generations.
* 3. Structural — non-empty, no unresolved placeholders, balanced
* markdown fences, no obvious template accidents
* (e.g. "[PLACEHOLDER]", "TODO:" left in).
* 4. Regression — score delta on a held-out eval set must not drop
* below the allowed tolerance (default -0.02, i.e. a
* 2% accuracy regression is the worst allowed).
*
* Each gate returns structured reasons so the orchestrator (phase 4)
* can log WHY a candidate was rejected and surface it in the UI.
*/
import type { EvolutionTarget } from './iterative-optimizer.js';
// ── Public types ───────────────────────────────────────────────
export type GateVerdict = 'pass' | 'fail';
export interface GateResult {
gate: string;
verdict: GateVerdict;
reason: string;
/** Optional measurable detail: e.g. `{ current: 3520, max: 3000 }` */
detail?: Record<string, number | string>;
}
export interface GateCheckResult {
verdict: GateVerdict;
/** First failing gate, or null if all passed */
firstFailure: GateResult | null;
/** All gate results in order */
results: GateResult[];
}
export interface SizeLimits {
personaSystemPrompt: number;
toolDescription: number;
skillBody: number;
behavioralSpecSection: number;
generic: number;
}
export const DEFAULT_SIZE_LIMITS: SizeLimits = {
personaSystemPrompt: 3_000,
toolDescription: 500,
skillBody: 15_000,
behavioralSpecSection: 4_000,
generic: 8_000,
};
export interface GateOptions {
/** What is being evolved — drives which size cap applies. Default 'generic'. */
targetKind?: EvolutionTarget;
/** Max allowed % growth over baseline. Default 0.2 (i.e. +20%). */
maxGrowthRatio?: number;
/** Largest allowed accuracy regression vs baseline. Default -0.02 (i.e. -2%). */
maxRegression?: number;
/** Override the default size limits */
sizeLimits?: Partial<SizeLimits>;
/** Allow empty candidates (rarely useful — default false) */
allowEmpty?: boolean;
}
export interface CheckInput {
/** The new candidate text */
candidate: string;
/** The baseline (pre-evolution) text — used for growth + regression checks */
baseline: string;
/** Optional scores to enable the regression gate */
scores?: {
baseline: number;
candidate: number;
};
}
// ── Public API ─────────────────────────────────────────────────
/**
* Run all gates in order. Returns a structured verdict. The overall
* verdict is `pass` iff every gate passed; `fail` with `firstFailure`
* populated otherwise.
*
* Callers can choose to short-circuit on the first failure or to
* collect all of them — `runGates` returns every gate result either way.
*/
export function runGates(input: CheckInput, options: GateOptions = {}): GateCheckResult {
const targetKind = options.targetKind ?? 'generic';
const sizeLimits = { ...DEFAULT_SIZE_LIMITS, ...options.sizeLimits };
const maxGrowthRatio = options.maxGrowthRatio ?? 0.2;
const maxRegression = options.maxRegression ?? -0.02;
const allowEmpty = options.allowEmpty ?? false;
const results: GateResult[] = [];
// 1. Non-empty
if (!allowEmpty) {
results.push(checkNonEmpty(input.candidate));
}
// 2. Size
results.push(checkSize(input.candidate, targetKind, sizeLimits));
// 3. Growth
results.push(checkGrowth(input.candidate, input.baseline, maxGrowthRatio));
// 4. Structural integrity
results.push(checkBalancedFences(input.candidate));
results.push(checkNoPlaceholders(input.candidate));
results.push(checkNoObviousTodos(input.candidate));
// 5. Regression (optional — skipped if scores not provided)
if (input.scores) {
results.push(checkRegression(input.scores.baseline, input.scores.candidate, maxRegression));
}
const firstFailure = results.find(r => r.verdict === 'fail') ?? null;
return {
verdict: firstFailure ? 'fail' : 'pass',
firstFailure,
results,
};
}
// ── Individual gates (exported for unit tests + fine-grained use) ──
export function checkNonEmpty(candidate: string): GateResult {
const trimmed = candidate.trim();
if (trimmed.length === 0) {
return {
gate: 'non-empty',
verdict: 'fail',
reason: 'Candidate is empty or whitespace only',
};
}
return { gate: 'non-empty', verdict: 'pass', reason: 'non-empty' };
}
export function checkSize(
candidate: string,
targetKind: EvolutionTarget,
limits: SizeLimits,
): GateResult {
const max = resolveSizeLimit(targetKind, limits);
const len = candidate.length;
if (len > max) {
return {
gate: 'size',
verdict: 'fail',
reason: `Candidate is ${len} chars but ${targetKind} cap is ${max}`,
detail: { current: len, max },
};
}
return {
gate: 'size',
verdict: 'pass',
reason: `Within ${targetKind} limit (${len}/${max})`,
detail: { current: len, max },
};
}
export function checkGrowth(
candidate: string,
baseline: string,
maxGrowthRatio: number,
): GateResult {
// A candidate shorter than baseline always passes the growth gate.
if (candidate.length <= baseline.length) {
return {
gate: 'growth',
verdict: 'pass',
reason: 'Candidate is not larger than baseline',
detail: { baseline: baseline.length, candidate: candidate.length },
};
}
// Empty baseline means any candidate can't be measured as a ratio —
// defer to the size gate instead.
if (baseline.length === 0) {
return {
gate: 'growth',
verdict: 'pass',
reason: 'No baseline to compare against',
};
}
const ratio = (candidate.length - baseline.length) / baseline.length;
if (ratio > maxGrowthRatio) {
return {
gate: 'growth',
verdict: 'fail',
reason: `Candidate grew ${(ratio * 100).toFixed(1)}% over baseline (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
};
}
return {
gate: 'growth',
verdict: 'pass',
reason: `Grew ${(ratio * 100).toFixed(1)}% (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
};
}
export function checkBalancedFences(candidate: string): GateResult {
const fenceMatches = candidate.match(/```/g);
const count = fenceMatches?.length ?? 0;
if (count % 2 !== 0) {
return {
gate: 'balanced-fences',
verdict: 'fail',
reason: `Odd number of markdown code fences (${count}) — unbalanced`,
detail: { fences: count },
};
}
return {
gate: 'balanced-fences',
verdict: 'pass',
reason: `${count} fence(s), balanced`,
detail: { fences: count },
};
}
/**
* Reject candidates that still contain placeholder text like `[TODO]`,
* `<placeholder>`, or `{{var}}` — these are almost always unfinished
* generations.
*/
export function checkNoPlaceholders(candidate: string): GateResult {
const patterns: Array<{ name: string; re: RegExp }> = [
{ name: 'bracket-placeholder', re: /\[(?:PLACEHOLDER|TODO|FIXME|INSERT|XXX|YOUR[_ ][A-Z ]+)\]/i },
{ name: 'angle-placeholder', re: /<(?:placeholder|todo|your[-_ ][a-z ]+)>/i },
{ name: 'handlebars-placeholder', re: /\{\{[a-zA-Z_][a-zA-Z0-9_.]*\}\}/ },
];
for (const { name, re } of patterns) {
const m = candidate.match(re);
if (m) {
return {
gate: 'no-placeholders',
verdict: 'fail',
reason: `Found unresolved placeholder (${name}): "${m[0]}"`,
};
}
}
return { gate: 'no-placeholders', verdict: 'pass', reason: 'no unresolved placeholders' };
}
/**
* Reject candidates with "TODO:" or "FIXME:" headers — a prompt shouldn't
* ship with unfinished author notes.
*/
export function checkNoObviousTodos(candidate: string): GateResult {
// Only match lines that START with the marker (as a label), not prose
// that happens to mention "todo" (e.g. a task-list prompt).
const re = /(?:^|\n)\s*(?:TODO|FIXME|XXX)\s*:/i;
const m = candidate.match(re);
if (m) {
return {
gate: 'no-todos',
verdict: 'fail',
reason: `Found leftover author marker: "${m[0].trim()}"`,
};
}
return { gate: 'no-todos', verdict: 'pass', reason: 'no leftover author markers' };
}
export function checkRegression(
baselineScore: number,
candidateScore: number,
maxRegression: number,
): GateResult {
const delta = candidateScore - baselineScore;
// Small epsilon avoids float-precision false negatives when delta is
// numerically equal to the tolerance (e.g. 0.78 - 0.8 === -0.02000002).
const epsilon = 1e-9;
if (delta < maxRegression - epsilon) {
return {
gate: 'regression',
verdict: 'fail',
reason: `Candidate score regressed by ${(delta * 100).toFixed(2)}pp (allowed ${(maxRegression * 100).toFixed(2)}pp)`,
detail: { baseline: baselineScore, candidate: candidateScore, delta },
};
}
return {
gate: 'regression',
verdict: 'pass',
reason: `Score delta ${(delta * 100).toFixed(2)}pp (allowed floor ${(maxRegression * 100).toFixed(2)}pp)`,
detail: { baseline: baselineScore, candidate: candidateScore, delta },
};
}
// ── Helpers ─────────────────────────────────────────────────────
function resolveSizeLimit(target: EvolutionTarget, limits: SizeLimits): number {
switch (target) {
case 'persona-system-prompt': return limits.personaSystemPrompt;
case 'tool-description': return limits.toolDescription;
case 'skill-body': return limits.skillBody;
case 'behavioral-spec-section': return limits.behavioralSpecSection;
case 'generic':
default:
return limits.generic;
}
}

View File

@@ -1,43 +1,29 @@
/**
* Evolution LLM Wiring — binds real LLMs into the self-evolution loop.
* LLM wiring for the skill audit and skill hygiene routes.
*
* The evolution primitives (judge, iterative-optimizer, evolve-schema) are
* deliberately model-agnostic: they accept plain callables. This module
* supplies production adapters that turn a Haiku-class LLM into those
* callables, so the `/api/evolution/run` endpoint can perform a real run.
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
* This module supplies the production adapter that turns a Haiku-class LLM
* into that callable, plus a retry wrapper for transient provider errors.
*
* Public surface:
*
* createAnthropicEvolutionLLM(apiKey, model?)
* Builds the default Haiku-backed LLM via @ax-llm/ax — same pattern as
* `optimizer-service.ts`. Dynamic import to avoid startup cost when
* evolution is not in use.
* createAnthropicEvolutionLLM(apiKey, options?)
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
* avoid startup cost when no audit runs.
*
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
* buildGEPAMutateFn(llm) → MutateFn (for IterativeGEPA)
* buildSchemaExecuteFn(llm) → SchemaExecuteFn (for EvolveSchema)
* makeRunningJudge(base, llm) → Pick<LLMJudge,'score'>
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
*
* The `makeRunningJudge` wrapper is what enables GEPA to evaluate real
* prompt effectiveness: on each score() call it first executes the
* candidate prompt against the example input via the LLM, then delegates
* to the wrapped judge with the model's actual response. Without this,
* GEPA compares prompt text to expected output directly — less useful.
*
* All adapters are pure wrappers over `EvolutionLLM.complete()`, which is a
* minimal `(prompt: string) => Promise<string>` contract. Tests supply
* in-memory mocks of that contract; production code goes through AxAI.
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
* mocks of that contract; production code goes through AxAI.
*/
import type { JudgeLLMCall, JudgeInput, JudgeScore } from './judge.js';
import type { LLMJudge } from './judge.js';
import type { MutateArgs, MutateFn, EvolutionTarget } from './iterative-optimizer.js';
import type { SchemaExecuteFn, Schema, SchemaField } from './evolve-schema.js';
import type { JudgeLLMCall } from './judge.js';
// ── Minimal LLM contract ────────────────────────────────────────
/**
* A minimal single-turn completion contract. All evolution adapters depend
* A minimal single-turn completion contract. All adapters depend
* on this interface rather than on @ax-llm/ax directly so they stay easy to
* mock in unit tests.
*/
@@ -206,10 +192,10 @@ export interface CreateAnthropicEvolutionLLMOptions {
/**
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
* @ax-llm/ax so evolution is a zero-cost dependency until used.
* @ax-llm/ax so it is a zero-cost dependency until used.
*
* Returns null when @ax-llm/ax is unavailable — callers should treat null
* as "evolution disabled" and surface a clear error to the user.
* as "LLM unavailable" and surface a clear error to the user.
*/
export async function createAnthropicEvolutionLLM(
apiKey: string,
@@ -240,8 +226,8 @@ export async function createAnthropicEvolutionLLM(
const response = await ai.chat({
chatPrompt: [{ role: 'user', content: prompt }],
});
// chat() may return a ReadableStream in streaming mode — evolution
// does not stream, so we only handle the non-stream case.
// chat() may return a ReadableStream in streaming mode — audits
// do not stream, so we only handle the non-stream case.
const typed = response as {
results?: readonly { content?: string }[];
};
@@ -249,7 +235,7 @@ export async function createAnthropicEvolutionLLM(
return typeof first === 'string' ? first : '';
},
};
// Apply retry policy — defaults are sized for background evolution runs
// Apply retry policy — defaults are sized for background runs
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
// by passing `retry: { maxAttempts: 1 }`.
return wrapWithRetry(inner, options.retry ?? {});
@@ -268,217 +254,3 @@ export async function createAnthropicEvolutionLLM(
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
return (prompt: string) => llm.complete(prompt);
}
// ── Reflective mutation (GEPA) ──────────────────────────────────
export interface BuildReflectiveMutationPromptArgs {
parent: string;
strategy: string;
weaknessFeedback: string[];
targetKind: EvolutionTarget;
generation: number;
}
/**
* Build a reflective mutation prompt for GEPA. Returns the *mutated child*
* text only — no commentary, no wrapping — per the instruction in the
* prompt body. The LLM is told what to fix, using the weakness signal from
* the parent's worst-scoring eval examples.
*/
export function buildReflectiveMutationPrompt(
args: BuildReflectiveMutationPromptArgs,
): string {
const weaknessBullets = args.weaknessFeedback.length > 0
? args.weaknessFeedback.map(fb => `- ${fb}`).join('\n')
: '(no specific weakness signals yet)';
return `You are evolving an AI prompt. This is generation ${args.generation}.
TARGET KIND: ${args.targetKind}
STRATEGY: ${args.strategy}
PARENT PROMPT (do not return this unchanged — produce a *better* variant):
---
${args.parent}
---
WEAKNESS SIGNALS from the parent's lowest-scoring eval examples:
${weaknessBullets}
Apply the strategy above to address the weakness signals. Keep the same
role and overall structure — do not change the prompt's purpose. Produce
a concrete variant that should score higher on the next eval.
Return ONLY the new prompt text. No preamble, no explanation, no markdown
fences. Plain text only.`;
}
/**
* Build a GEPA MutateFn that uses an LLM to produce reflective mutations.
* Graceful fallback: if the LLM throws or returns empty, returns the
* parent prompt unchanged (GEPA will still evaluate it as part of the
* Pareto population — no progress that generation, but no crash).
*/
export function buildGEPAMutateFn(llm: EvolutionLLM): MutateFn {
return async (args: MutateArgs): Promise<string> => {
const prompt = buildReflectiveMutationPrompt({
parent: args.parent.prompt,
strategy: args.strategy,
weaknessFeedback: args.weaknessFeedback,
targetKind: args.targetKind,
generation: args.generation,
});
let raw = '';
try {
raw = await llm.complete(prompt);
} catch {
return args.parent.prompt;
}
const cleaned = stripFences(raw).trim();
if (cleaned.length === 0) return args.parent.prompt;
return cleaned;
};
}
// ── Schema execute (ES) ─────────────────────────────────────────
/**
* Build a schema-filling prompt that asks the LLM to return JSON matching
* the given schema for the given user input. Every field is documented by
* name + type + description + required flag + constraints.
*/
export function buildSchemaFillPrompt(args: { schema: Schema; input: string }): string {
const fieldLines = args.schema.fields.map(f => formatField(f)).join('\n');
return `Fill the JSON schema below using the user input. Respond with
valid JSON — no markdown, no prose, no trailing commas.
SCHEMA (name: ${args.schema.name}, version: ${args.schema.version}):
${fieldLines}
USER INPUT:
${args.input}
Return ONLY the JSON object.`;
}
function formatField(field: SchemaField): string {
const req = field.required ? 'required' : 'optional';
const constraints = field.constraints.length > 0
? ' constraints: ' + field.constraints.map(c => `${c.kind}=${stringifyConstraintValue(c.value)}`).join(', ')
: '';
return `- "${field.name}" (${field.type}, ${req}): ${field.description}${constraints}`;
}
function stringifyConstraintValue(value: string | number | string[]): string {
if (Array.isArray(value)) return `[${value.join(',')}]`;
return String(value);
}
/**
* Build a SchemaExecuteFn that asks the LLM to fill a schema. Returns
* `parsed: true` only when the response parses as JSON. On any error
* (network, timeout, bad model), returns `{actual: '', parsed: false}`.
*/
export function buildSchemaExecuteFn(llm: EvolutionLLM): SchemaExecuteFn {
return async ({ schema, input }) => {
let raw = '';
try {
raw = await llm.complete(buildSchemaFillPrompt({ schema, input }));
} catch {
return { actual: '', parsed: false };
}
const cleaned = stripFences(raw);
return { actual: cleaned, parsed: looksLikeJSON(cleaned) };
};
}
// ── Running judge (executes candidate prompt first, then scores) ─
/**
* Wrap a base judge so that each score() call first runs the candidate
* prompt through the LLM on the example's input. The base judge then
* receives the LLM's output as `actual`, giving a real behavioral signal
* instead of a prompt-text-to-expected-output comparison.
*
* When the LLM call fails, returns a zero score with clear feedback so the
* Pareto population treats the candidate as a loser without crashing the
* whole generation.
*/
/**
* Phantom property that marks a judge as running-capable (i.e. it executes
* the candidate prompt against a real LLM before scoring). IterativeGEPA
* checks for this to prevent regressions where a bare prompt-text-similarity
* judge is passed by mistake.
*
* Symbol-keyed to avoid accidental conflicts with user-defined fields.
*/
export const RUNNING_JUDGE_BRAND = Symbol.for('waggle.runningJudge');
export interface RunningJudge extends Pick<LLMJudge, 'score'> {
readonly [RUNNING_JUDGE_BRAND]: true;
}
export function isRunningJudge(j: Pick<LLMJudge, 'score'> | RunningJudge): j is RunningJudge {
return (j as Record<symbol, unknown>)[RUNNING_JUDGE_BRAND] === true;
}
export function makeRunningJudge(
baseJudge: Pick<LLMJudge, 'score'>,
llm: EvolutionLLM,
): RunningJudge {
return {
[RUNNING_JUDGE_BRAND]: true as const,
async score(args: JudgeInput): Promise<JudgeScore> {
// `args.actual` is the candidate prompt supplied by GEPA. Treat it as
// a system prompt, run it against the example input via the LLM, and
// use the LLM's reply as the new `actual` for the downstream judge.
const runPrompt = buildRunPrompt(args.actual, args.input);
let modelOutput = '';
try {
modelOutput = await llm.complete(runPrompt);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return {
overall: 0, weighted: 0,
correctness: 0, procedureFollowing: 0, conciseness: 0,
lengthPenalty: 1,
feedback: `Candidate execution failed: ${msg}`,
parsed: false,
};
}
return baseJudge.score({ ...args, actual: modelOutput });
},
};
}
function buildRunPrompt(candidatePrompt: string, userInput: string): string {
return `${candidatePrompt}
USER INPUT:
${userInput}
Respond now.`;
}
// ── Shared helpers ──────────────────────────────────────────────
/** Strip ``` and ```json fences if present. Never throws. */
function stripFences(raw: string): string {
if (!raw) return '';
return raw
.replace(/```json\s*/gi, '')
.replace(/```\s*/g, '')
.trim();
}
function looksLikeJSON(s: string): boolean {
const trimmed = s.trim();
if (!trimmed) return false;
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return false;
try {
JSON.parse(trimmed);
return true;
} catch {
return false;
}
}

View File

@@ -1,371 +0,0 @@
/**
* Evolution Orchestrator — Phase 4 of the self-evolution loop.
*
* Wires the primitives into a closed loop:
*
* traces → eval dataset → ComposeEvolution → gates → run store
* ↓
* user accepts/rejects
* ↓
* deploy callback fires
* ↓
* runStore.markDeployed()
*
* The orchestrator itself does NOT hot-reload personas or rewrite the
* behavioral spec — it calls a pluggable `deploy` callback supplied by
* the caller. This keeps the orchestrator pure (easy to unit-test) and
* lets the server/desktop app wire up the real side-effects (file write,
* persona reload, event emission) separately.
*
* Persistence is in @waggle/core's `EvolutionRunStore`. Every run — even
* rejected or failed — is preserved for audit.
*/
import type {
ExecutionTraceStore,
EvolutionRun,
EvolutionRunStore,
EvolutionRunStatus,
EvolutionRunTarget,
ParsedExecutionTrace,
TraceOutcome,
} from '@waggle/core';
import { ComposeEvolution, type ComposeEvolutionOptions, type ComposeEvolutionResult } from './compose-evolution.js';
import { runGates, type GateResult, type GateOptions } from './evolution-gates.js';
import { EvalDatasetBuilder, type EvalExample } from './eval-dataset.js';
// ── Dependency injection types ─────────────────────────────────
export interface EvolutionOrchestratorDeps {
/** Source of traces (from @waggle/core) */
traceStore: ExecutionTraceStore;
/** Persistent run history (from @waggle/core) */
runStore: EvolutionRunStore;
/**
* Function invoked when a run moves from accepted → (deployed | failed).
* Returning normally marks the run deployed; throwing marks it failed
* with the error message as reason.
*
* This is where the server writes to the persona file, reloads the
* orchestrator cache, updates behavioral-spec.ts, etc.
*/
deploy?: (run: EvolutionRun) => Promise<void>;
}
export interface EvolutionAutoTriggerConfig {
/** Minimum traces (with the given outcome filter) before a run is triggered */
minTraces: number;
/** Restrict trigger to a specific persona / workspace / taskShape */
traceFilter?: {
personaId?: string;
workspaceId?: string;
taskShape?: string;
};
}
export interface EvolutionOrchestratorOptions {
/** What is being evolved (drives gate policy + persistence tag) */
targetKind: EvolutionRunTarget;
/** Stable identifier — e.g. persona id, spec-section name */
targetName?: string;
/** The current baseline text to evolve against */
baseline: string;
/** Compose options — already wired with examples, judge, execute, mutate */
compose: Omit<ComposeEvolutionOptions, 'schema' | 'instructions'> & {
schema: Omit<ComposeEvolutionOptions['schema'], 'baseline'>;
instructions: Omit<ComposeEvolutionOptions['instructions'], 'baseline'>;
};
/** Optional gate overrides (size caps, growth ratio, etc) */
gateOptions?: GateOptions;
/** Improvement threshold required to create a proposal (default 0.02) */
minDelta?: number;
/** Auto-trigger configuration — when omitted, runOnce() is manual-only */
autoTrigger?: EvolutionAutoTriggerConfig;
/** Abort the run early */
signal?: AbortSignal;
/** Optional progress pass-through */
onProgress?: (event: EvolutionProgress) => void;
}
export interface EvolutionProgress {
phase: 'trigger-check' | 'dataset' | 'compose' | 'gates' | 'persist' | 'skipped' | 'done';
message?: string;
detail?: unknown;
}
// ── Schema baseline input ──────────────────────────────────────
/** Optional structural baseline when the target has one (e.g. persona's DSPy signature). */
export interface SchemaBaselineInput {
baseline: ComposeEvolutionOptions['schema']['baseline'];
}
// ── Run result ─────────────────────────────────────────────────
export type OrchestratorOutcome =
| 'proposed' // Run created, awaiting accept/reject
| 'skipped-trigger'
| 'skipped-gates' // Completed but gates failed
| 'skipped-delta' // Improvement below minDelta
| 'aborted';
export interface OrchestratorRunResult {
outcome: OrchestratorOutcome;
/** Populated for 'proposed' and 'skipped-gates' */
run?: EvolutionRun;
/** Raw ComposeEvolution result when the compose stage ran */
compose?: ComposeEvolutionResult;
/** Gate results for audit */
gateResults?: GateResult[];
/** Reason if outcome is skipped/aborted */
reason?: string;
}
// ── Orchestrator ──────────────────────────────────────────────
export class EvolutionOrchestrator {
private deps: EvolutionOrchestratorDeps;
constructor(deps: EvolutionOrchestratorDeps) {
this.deps = deps;
}
/**
* Run the full orchestration pipeline once. Returns structured outcome.
* Does NOT deploy — that happens when the caller calls `accept(uuid)`.
*/
async runOnce(options: EvolutionOrchestratorOptions & {
schemaBaseline: SchemaBaselineInput['baseline'];
}): Promise<OrchestratorRunResult> {
const emit = (phase: EvolutionProgress['phase'], message?: string, detail?: unknown) => {
options.onProgress?.({ phase, message, detail });
};
if (options.signal?.aborted) {
return { outcome: 'aborted', reason: 'aborted before start' };
}
// Trigger check — skip if auto-trigger thresholds aren't met.
if (options.autoTrigger) {
emit('trigger-check');
const ok = this.meetsAutoTrigger(options.autoTrigger);
if (!ok) {
emit('skipped');
return {
outcome: 'skipped-trigger',
reason: `fewer than ${options.autoTrigger.minTraces} eligible traces`,
};
}
}
// Derive the eval dataset from finalized traces of the requested target.
emit('dataset');
const examples = this.buildExamplesFromTraces(options);
if (examples.length === 0) {
emit('skipped', 'no eval examples');
return { outcome: 'skipped-trigger', reason: 'no eligible traces to form dataset' };
}
// Run the compose pipeline.
emit('compose');
const composeResult = await new ComposeEvolution().run({
schema: { ...options.compose.schema, baseline: options.schemaBaseline, examples },
instructions: { ...options.compose.instructions, baseline: options.baseline, examples },
feedbackFilter: options.compose.feedbackFilter,
signal: options.signal,
onProgress: (e) => emit('compose', undefined, e),
});
if (options.signal?.aborted) {
return { outcome: 'aborted', reason: 'aborted during compose', compose: composeResult };
}
const winnerText = composeResult.instructions.winner.prompt;
// Apples-to-apples: compare the GEPA baseline candidate (history[0])
// to the GEPA winner. Both are scored by the same judge on the same
// eval stage, so the delta is a clean signal.
const baselineOverall = composeResult.instructions.history[0]?.score?.overall ?? 0;
const winnerOverall = composeResult.instructions.winner.score?.overall ?? 0;
const delta = winnerOverall - baselineOverall;
const minDelta = options.minDelta ?? 0.02;
// Gate check.
emit('gates');
const gateResult = runGates(
{
candidate: winnerText,
baseline: options.baseline,
scores: { baseline: baselineOverall, candidate: winnerOverall },
},
{
targetKind: options.targetKind,
...options.gateOptions,
},
);
if (delta < minDelta) {
emit('skipped', `delta ${(delta * 100).toFixed(2)}pp below minimum ${(minDelta * 100).toFixed(2)}pp`);
emit('done');
return {
outcome: 'skipped-delta',
reason: `winner improved only ${(delta * 100).toFixed(2)}pp (min ${(minDelta * 100).toFixed(2)}pp required)`,
compose: composeResult,
gateResults: gateResult.results,
};
}
// Persist — create the proposed run even if gates failed, so there's audit.
emit('persist');
const run = this.deps.runStore.create({
targetKind: options.targetKind,
targetName: options.targetName ?? null,
baselineText: options.baseline,
winnerText,
winnerSchema: composeResult.frozenSchema,
deltaAccuracy: delta,
gateVerdict: gateResult.verdict,
gateReasons: gateResult.results.map(r => ({
gate: r.gate, verdict: r.verdict, reason: r.reason,
})),
artifacts: {
runSeed: options.compose.instructions.seed ?? null,
generations: options.compose.instructions.generations ?? null,
paretoFrontSize: composeResult.schema.paretoFront.length,
exampleCount: examples.length,
},
});
if (gateResult.verdict === 'fail') {
// Immediately reject runs that fail gates so the user doesn't see
// dangerous candidates in the "review" queue. Run history is kept.
const rejected = this.deps.runStore.reject(
run.run_uuid,
`gate failure: ${gateResult.firstFailure?.reason ?? 'unknown'}`,
);
emit('done');
return {
outcome: 'skipped-gates',
reason: gateResult.firstFailure?.reason ?? 'gate failure',
run: rejected ?? run,
compose: composeResult,
gateResults: gateResult.results,
};
}
emit('done');
return {
outcome: 'proposed',
run,
compose: composeResult,
gateResults: gateResult.results,
};
}
/** Accept a proposed run and invoke the deploy callback. */
async accept(runUuid: string, userNote?: string): Promise<EvolutionRun | undefined> {
const accepted = this.deps.runStore.accept(runUuid, userNote);
if (!accepted || accepted.status !== 'accepted') return accepted;
if (!this.deps.deploy) {
// No deploy hook configured — leave as 'accepted' and let the caller
// mark deployed manually.
return accepted;
}
try {
await this.deps.deploy(accepted);
return this.deps.runStore.markDeployed(runUuid);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
return this.deps.runStore.markFailed(runUuid, reason);
}
}
/** Reject a proposed run with optional reason. */
reject(runUuid: string, reason?: string): EvolutionRun | undefined {
return this.deps.runStore.reject(runUuid, reason);
}
/** List runs (thin pass-through, handy for the UI). */
list(filter?: Parameters<EvolutionRunStore['list']>[0]): EvolutionRun[] {
return this.deps.runStore.list(filter);
}
/** Get a single run by uuid. */
get(runUuid: string): EvolutionRun | undefined {
return this.deps.runStore.getByUuid(runUuid);
}
// ── Internals ───────────────────────────────────────────────
private meetsAutoTrigger(trigger: EvolutionAutoTriggerConfig): boolean {
const outcomes: TraceOutcome[] = ['success', 'verified', 'corrected'];
const traces = this.deps.traceStore.query({
...(trigger.traceFilter ?? {}),
outcome: outcomes,
limit: Math.max(trigger.minTraces * 2, 50),
});
return traces.length >= trigger.minTraces;
}
private buildExamplesFromTraces(
options: EvolutionOrchestratorOptions,
): EvalExample[] {
// If the caller already supplied examples via compose.schema/instructions,
// we respect that. Otherwise we mine them from the trace store.
const existing = options.compose.schema.examples ?? options.compose.instructions.examples;
if (existing && existing.length > 0) return existing;
const builder = new EvalDatasetBuilder(this.deps.traceStore);
const traceFilter = options.autoTrigger?.traceFilter ?? {};
const traces = builder.sourceFromTraces(['success', 'verified'], true, {
...traceFilter,
limit: 500,
});
return traces;
}
}
// ── Convenience helpers ────────────────────────────────────────
/**
* Filter a list of parsed traces down to those likely useful for evolution.
* Keeps finalized (non-pending) traces with non-empty I/O. Used by callers
* that want to manually pre-filter before handing to the orchestrator.
*/
export function eligibleForEvolution(traces: ParsedExecutionTrace[]): ParsedExecutionTrace[] {
return traces.filter(t =>
t.outcome !== 'pending' &&
t.payload.input.trim().length > 0 &&
(t.payload.output.trim().length > 0 || t.payload.correctionFeedback),
);
}
/**
* Helper that summarizes runs by status/target for dashboard display.
*/
export function summarizeRuns(runs: EvolutionRun[]): {
total: number;
byStatus: Record<EvolutionRunStatus, number>;
byTargetKind: Record<string, number>;
bestDelta: number;
} {
const byStatus: Record<EvolutionRunStatus, number> = {
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
};
const byTargetKind: Record<string, number> = {};
let bestDelta = 0;
for (const r of runs) {
byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
byTargetKind[r.target_kind] = (byTargetKind[r.target_kind] ?? 0) + 1;
if (r.delta_accuracy > bestDelta) bestDelta = r.delta_accuracy;
}
return {
total: runs.length,
byStatus,
byTargetKind,
bestDelta,
};
}

View File

@@ -1,852 +0,0 @@
/**
* EvolveSchema — Phase 3.1 of the self-evolution loop.
*
* Port of Mikhail Pavlukhin's EvolveSchema algorithm (Egzakta research).
* Evolves the STRUCTURE of a typed output schema (DSPy-style signatures):
* field names, types, descriptions, ordering, and constraints. This is
* orthogonal to the GEPA optimizer which evolves INSTRUCTIONS against a
* fixed schema.
*
* Three-phase per-generation loop:
*
* Phase A — Structure Discovery
* Spawn variants that add, replace, or drop output fields. This is
* the single biggest-impact mutation class (Mikhail's paper shows
* ~74% of HotPotQA gains come from one structural mutation).
*
* Phase B — Field-Order Probes
* Permute field order. LLMs condition on earlier fields when filling
* later ones, so putting a "reasoning" field BEFORE "answer" routinely
* outperforms the reverse.
*
* Phase C — Failure-Driven Refinement
* Use the per-example judge feedback from the previous generation to
* target specific fields — edit their descriptions, tighten / loosen
* their constraints, or change their types where signals suggest.
*
* 8 typed mutations are available:
* add_output_field, remove_field, edit_field_desc, change_field_type,
* add_constraint, remove_constraint, reorder_fields, replace_output_fields
*
* Pareto selection is 2-dimensional: (accuracy, -complexity) — an
* accurate-but-complex schema is not automatically preferred over a
* slightly-less-accurate simpler one.
*
* Callers provide:
* - `execute(schema, input) → actualOutput` the LLM runner
* - `judge.score({input, expected, actual})` the LLMJudge from Phase 1.3
* - `mutate` (optional) a custom LLM-driven mutator
* for edit_field_desc; falls
* back to a deterministic
* rewriter when omitted.
*/
import type { EvalExample } from './eval-dataset.js';
import type { LLMJudge, JudgeScore } from './judge.js';
// ── Schema types ───────────────────────────────────────────────
export type FieldType = 'string' | 'number' | 'boolean' | 'array' | 'object' | 'enum';
export interface FieldConstraint {
kind: 'minLength' | 'maxLength' | 'pattern' | 'enum' | 'range' | 'custom';
value: string | number | string[];
}
export interface SchemaField {
name: string;
type: FieldType;
description: string;
required: boolean;
constraints: FieldConstraint[];
}
export interface Schema {
name: string;
fields: SchemaField[];
version: number;
}
// ── Mutation types ─────────────────────────────────────────────
export type MutationKind =
| 'add_output_field'
| 'remove_field'
| 'edit_field_desc'
| 'change_field_type'
| 'add_constraint'
| 'remove_constraint'
| 'reorder_fields'
| 'replace_output_fields';
export interface Mutation {
kind: MutationKind;
/** What changed, for logging + feedback */
description: string;
/** Functional transformer (pure) */
apply: (schema: Schema) => Schema;
}
// ── Candidate + result types ──────────────────────────────────
export interface SchemaCandidate {
id: string;
schema: Schema;
generation: number;
parent: string | null;
mutation: MutationKind | 'baseline';
mutationLabel: string;
score: SchemaCandidateScore | null;
perExample: SchemaExampleResult[];
}
export interface SchemaExampleResult {
input: string;
expected: string;
actual: string;
score: JudgeScore;
/** Diagnostic: did the output parse against the schema? */
parsed: boolean;
}
export interface SchemaCandidateScore {
/** Mean overall judge score across the eval set (0..1) */
accuracy: number;
/** Schema "complexity" — a scalar that grows with fields, descriptions, constraints */
complexity: number;
/** Share of examples that produced parseable output against this schema (0..1) */
parseRate: number;
/** Weakness feedback taken from the worst-scoring 3 examples */
weaknessFeedback: string[];
/** Number of examples scored */
n: number;
}
export type SchemaExecuteFn = (args: {
schema: Schema;
input: string;
}) => Promise<{ actual: string; parsed: boolean }>;
export interface EvolveSchemaOptions {
/** Starting schema */
baseline: Schema;
/** Eval dataset */
examples: EvalExample[];
/** LLM runner that takes a schema + input and returns an attempted output */
execute: SchemaExecuteFn;
/** Judge for scoring individual attempts */
judge: Pick<LLMJudge, 'score'>;
/** Candidates per generation (default 5) */
populationSize?: number;
/** Generations (default 3) */
generations?: number;
/** Mix of phase mutations per generation */
mutationMix?: { structure: number; order: number; refinement: number };
/** Eval sample size per generation (default 32) */
evalSize?: number;
/** Anchor eval size at the end (default 100) */
anchorEvalSize?: number;
/** Seed for sampling (default 1) */
seed?: number;
/** Optional LLM helper for description edits — deterministic fallback used if omitted */
editFieldDescription?: (args: {
field: SchemaField;
feedback: string[];
}) => Promise<string>;
/** Optional progress emitter */
onProgress?: (event: EvolveSchemaProgress) => void;
/** Optional abort signal */
signal?: AbortSignal;
}
export interface EvolveSchemaProgress {
phase: 'start' | 'structure' | 'order' | 'refinement' | 'anchor' | 'done';
generation: number;
populationSize: number;
bestAccuracy: number;
bestComplexity: number;
message?: string;
}
export interface EvolveSchemaResult {
winner: SchemaCandidate;
/** Full Pareto-non-dominated set from the anchor stage */
paretoFront: SchemaCandidate[];
/** All candidates across all generations */
history: SchemaCandidate[];
/** Accuracy delta: winner.accuracy - baseline.accuracy */
deltaAccuracy: number;
/** True if the winner's accuracy improved over baseline */
improved: boolean;
}
// ── Pure mutation functions ───────────────────────────────────
export function addOutputField(
schema: Schema,
field: SchemaField,
position?: number,
): Schema {
const pos = position ?? schema.fields.length;
const fields = [...schema.fields];
fields.splice(pos, 0, { ...field, constraints: [...field.constraints] });
return { ...schema, fields, version: schema.version + 1 };
}
export function removeField(schema: Schema, fieldName: string): Schema {
return {
...schema,
fields: schema.fields.filter(f => f.name !== fieldName),
version: schema.version + 1,
};
}
export function editFieldDescription(
schema: Schema,
fieldName: string,
newDescription: string,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, description: newDescription } : f,
),
version: schema.version + 1,
};
}
export function changeFieldType(
schema: Schema,
fieldName: string,
newType: FieldType,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, type: newType } : f,
),
version: schema.version + 1,
};
}
export function addConstraint(
schema: Schema,
fieldName: string,
constraint: FieldConstraint,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, constraints: [...f.constraints, constraint] } : f,
),
version: schema.version + 1,
};
}
export function removeConstraint(
schema: Schema,
fieldName: string,
constraintIndex: number,
): Schema {
return {
...schema,
fields: schema.fields.map(f => {
if (f.name !== fieldName) return f;
const constraints = f.constraints.filter((_, i) => i !== constraintIndex);
return { ...f, constraints };
}),
version: schema.version + 1,
};
}
export function reorderFields(schema: Schema, newOrder: string[]): Schema {
const byName = new Map(schema.fields.map(f => [f.name, f]));
const reordered: SchemaField[] = [];
for (const name of newOrder) {
const field = byName.get(name);
if (field) reordered.push(field);
}
// Append any fields that were omitted from newOrder (keeps their position stable).
for (const field of schema.fields) {
if (!newOrder.includes(field.name)) reordered.push(field);
}
return { ...schema, fields: reordered, version: schema.version + 1 };
}
export function replaceOutputFields(schema: Schema, newFields: SchemaField[]): Schema {
return {
...schema,
fields: newFields.map(f => ({ ...f, constraints: [...f.constraints] })),
version: schema.version + 1,
};
}
// ── Complexity + scoring helpers ───────────────────────────────
/**
* Scalar complexity measure: 1 per field + 0.3 per constraint +
* 0.01 per character of description. Calibrated so a 3-field schema
* with short descriptions ≈ 3.6, a 10-field schema with long
* descriptions ≈ 15+.
*/
export function schemaComplexity(schema: Schema): number {
let c = 0;
for (const field of schema.fields) {
c += 1;
c += field.constraints.length * 0.3;
c += (field.description?.length ?? 0) * 0.01;
}
return c;
}
export function aggregateSchemaScores(results: SchemaExampleResult[]): SchemaCandidateScore {
if (results.length === 0) {
return {
accuracy: 0, complexity: 0, parseRate: 0, weaknessFeedback: [], n: 0,
};
}
const n = results.length;
const accuracy = results.reduce((s, r) => s + r.score.overall, 0) / n;
const parseRate = results.filter(r => r.parsed).length / n;
const worst = [...results].sort((a, b) => a.score.overall - b.score.overall).slice(0, 3);
const weaknessFeedback = worst
.map(r => r.score.feedback)
.filter(fb => fb && fb.length > 0);
return { accuracy, complexity: 0, parseRate, weaknessFeedback, n };
}
// ── Pareto (2D) ───────────────────────────────────────────────
/**
* Pareto front on (accuracy↑, complexity↓). A candidate dominates
* another iff it is at least as accurate AND no more complex, and
* strictly better on at least one dimension.
*/
export function paretoFrontSchema(candidates: SchemaCandidate[]): SchemaCandidate[] {
const scored = candidates.filter(c => c.score !== null);
const front: SchemaCandidate[] = [];
for (const cand of scored) {
let dominated = false;
for (const other of scored) {
if (other === cand) continue;
if (dominatesSchema(other.score!, cand.score!)) {
dominated = true;
break;
}
}
if (!dominated) front.push(cand);
}
return front.length > 0 ? front : scored;
}
function dominatesSchema(a: SchemaCandidateScore, b: SchemaCandidateScore): boolean {
const geAll = a.accuracy >= b.accuracy && a.complexity <= b.complexity;
const gtAny = a.accuracy > b.accuracy || a.complexity < b.complexity;
return geAll && gtAny;
}
// ── Candidate scoring ─────────────────────────────────────────
export async function scoreSchemaCandidate(
candidate: SchemaCandidate,
examples: EvalExample[],
execute: SchemaExecuteFn,
judge: Pick<LLMJudge, 'score'>,
signal?: AbortSignal,
): Promise<SchemaCandidateScore> {
const results: SchemaExampleResult[] = [];
for (const ex of examples) {
if (signal?.aborted) break;
try {
const { actual, parsed } = await execute({ schema: candidate.schema, input: ex.input });
const score = await judge.score({
input: ex.input,
expected: ex.expected_output,
actual,
});
results.push({ input: ex.input, expected: ex.expected_output, actual, score, parsed });
} catch {
// Keep iterating — one broken example shouldn't poison the whole batch.
}
}
const agg = aggregateSchemaScores(results);
agg.complexity = schemaComplexity(candidate.schema);
candidate.perExample = results;
candidate.score = agg;
return agg;
}
// ── Mutation generators per phase ─────────────────────────────
/**
* Structure-Discovery mutations: add a new field, replace fields wholesale,
* or drop one if the schema is too large. Returns up to `n` mutations.
*/
export function generateStructureMutations(
parent: Schema,
n: number,
rng: () => number,
): Mutation[] {
const out: Mutation[] = [];
// Add a reasoning field if absent.
if (!parent.fields.some(f => /reason|thought|scratch/i.test(f.name))) {
out.push({
kind: 'add_output_field',
description: 'add reasoning field before answer',
apply: (s) =>
addOutputField(s, {
name: 'reasoning',
type: 'string',
description: 'Concise chain-of-thought explaining how the answer was derived.',
required: true,
constraints: [],
}, 0),
});
}
// Add a confidence field if absent.
if (!parent.fields.some(f => /confidence|certainty|score/i.test(f.name))) {
out.push({
kind: 'add_output_field',
description: 'add confidence field at end',
apply: (s) =>
addOutputField(s, {
name: 'confidence',
type: 'number',
description: 'Self-estimated confidence in the answer (0.0 - 1.0).',
required: false,
constraints: [{ kind: 'range', value: '0..1' }],
}),
});
}
// Drop the lowest-priority field if > 4 fields.
if (parent.fields.length > 4) {
const candidate = parent.fields[parent.fields.length - 1];
out.push({
kind: 'remove_field',
description: `drop trailing field "${candidate.name}"`,
apply: (s) => removeField(s, candidate.name),
});
}
// Replace all output fields with a minimal skeleton (reasoning + answer).
if (parent.fields.length >= 2) {
out.push({
kind: 'replace_output_fields',
description: 'collapse to {reasoning, answer}',
apply: (s) =>
replaceOutputFields(s, [
{
name: 'reasoning',
type: 'string',
description: 'Brief chain-of-thought.',
required: true,
constraints: [],
},
{
name: 'answer',
type: 'string',
description: 'Final answer.',
required: true,
constraints: [],
},
]),
});
}
return pickN(out, n, rng);
}
/**
* Order-Probe mutations: generate permutations of field order. Limits
* output to `n` by random sampling from the set of useful permutations.
*/
export function generateOrderMutations(
parent: Schema,
n: number,
rng: () => number,
): Mutation[] {
const names = parent.fields.map(f => f.name);
if (names.length < 2) return [];
const out: Mutation[] = [];
// Heuristic 1: move any reasoning/thought field to the front.
const reasoningIdx = parent.fields.findIndex(f => /reason|thought|scratch/i.test(f.name));
if (reasoningIdx > 0) {
const newOrder = [names[reasoningIdx], ...names.filter((_, i) => i !== reasoningIdx)];
out.push({
kind: 'reorder_fields',
description: `move "${names[reasoningIdx]}" to front`,
apply: (s) => reorderFields(s, newOrder),
});
}
// Heuristic 2: move any confidence/score field to the end.
const confIdx = parent.fields.findIndex(f => /confidence|certainty/i.test(f.name));
if (confIdx >= 0 && confIdx !== parent.fields.length - 1) {
const newOrder = [...names.filter((_, i) => i !== confIdx), names[confIdx]];
out.push({
kind: 'reorder_fields',
description: `move "${names[confIdx]}" to end`,
apply: (s) => reorderFields(s, newOrder),
});
}
// Fill remaining with random permutations.
while (out.length < n) {
const shuffled = shuffle([...names], rng);
if (!arraysEqual(shuffled, names)) {
out.push({
kind: 'reorder_fields',
description: `reorder to [${shuffled.join(', ')}]`,
apply: (s) => reorderFields(s, shuffled),
});
}
if (out.length > n * 3) break; // safety — very short schemas
}
return pickN(out, n, rng);
}
/**
* Refinement mutations driven by worst-scoring example feedback.
* Edits descriptions, tightens constraints, or switches types based on
* heuristic patterns in the feedback.
*/
export async function generateRefinementMutations(
parent: Schema,
weakness: string[],
n: number,
editFieldDescription?: (args: {
field: SchemaField;
feedback: string[];
}) => Promise<string>,
): Promise<Mutation[]> {
const out: Mutation[] = [];
const joined = weakness.join(' ').toLowerCase();
// Heuristic: if feedback mentions "too verbose" / "long" — add a maxLength to any string field.
if (/verbose|too long|wordy/i.test(joined)) {
const stringFields = parent.fields.filter(f => f.type === 'string');
for (const f of stringFields.slice(0, 2)) {
out.push({
kind: 'add_constraint',
description: `tighten "${f.name}" with maxLength: 200`,
apply: (s) => addConstraint(s, f.name, { kind: 'maxLength', value: 200 }),
});
}
}
// Heuristic: if feedback mentions "too brief" / "incomplete" — add minLength.
if (/brief|incomplete|missing|short/i.test(joined)) {
const stringFields = parent.fields.filter(f => f.type === 'string');
for (const f of stringFields.slice(0, 2)) {
out.push({
kind: 'add_constraint',
description: `require "${f.name}" minLength: 20`,
apply: (s) => addConstraint(s, f.name, { kind: 'minLength', value: 20 }),
});
}
}
// Heuristic: if feedback mentions "wrong format" / "parse" — clarify description.
if (/format|parse|wrong type/i.test(joined)) {
for (const field of parent.fields.slice(0, 2)) {
const newDesc = editFieldDescription
? await editFieldDescription({ field, feedback: weakness }).catch(() => field.description)
: deterministicClarifyDescription(field);
if (newDesc && newDesc !== field.description) {
out.push({
kind: 'edit_field_desc',
description: `clarify "${field.name}" description`,
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
});
}
}
}
// Fallback: always consider one description rewrite on the first field.
if (out.length === 0 && parent.fields.length > 0) {
const field = parent.fields[0];
const newDesc = deterministicClarifyDescription(field);
if (newDesc !== field.description) {
out.push({
kind: 'edit_field_desc',
description: `clarify "${field.name}" description`,
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
});
}
}
return out.slice(0, n);
}
// ── EvolveSchema engine ──────────────────────────────────────
export class EvolveSchema {
async run(options: EvolveSchemaOptions): Promise<EvolveSchemaResult> {
const config = normalizeOptions(options);
const rng = makeRng(config.seed);
const baseline: SchemaCandidate = {
id: 'g0-baseline',
schema: cloneSchema(config.baseline),
generation: 0,
parent: null,
mutation: 'baseline',
mutationLabel: 'baseline',
score: null,
perExample: [],
};
emitProgress(config, 'start', 0, 1, 0, 0, 'scoring baseline');
const evalSample = pickSample(config.examples, config.evalSize, rng);
await scoreSchemaCandidate(baseline, evalSample, config.execute, config.judge, config.signal);
const history: SchemaCandidate[] = [baseline];
let survivors: SchemaCandidate[] = [baseline];
for (let gen = 1; gen <= config.generations; gen++) {
if (config.signal?.aborted) break;
const topParent = pickTopByAccuracy(survivors);
if (!topParent) break;
const children: SchemaCandidate[] = [];
// Phase A — Structure Discovery
emitProgress(
config, 'structure', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const structureMutations = generateStructureMutations(
topParent.schema, config.mutationMix.structure, rng,
);
children.push(...applyMutations(structureMutations, topParent, gen));
// Phase B — Field-Order Probes
emitProgress(
config, 'order', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const orderMutations = generateOrderMutations(
topParent.schema, config.mutationMix.order, rng,
);
children.push(...applyMutations(orderMutations, topParent, gen));
// Phase C — Failure-Driven Refinement
emitProgress(
config, 'refinement', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const refinementMutations = await generateRefinementMutations(
topParent.schema,
topParent.score?.weaknessFeedback ?? [],
config.mutationMix.refinement,
config.editFieldDescription,
);
children.push(...applyMutations(refinementMutations, topParent, gen));
// Score all children on the per-generation sample.
const genSample = pickSample(config.examples, config.evalSize, rng);
for (const child of children) {
if (config.signal?.aborted) break;
await scoreSchemaCandidate(child, genSample, config.execute, config.judge, config.signal);
history.push(child);
}
// Pareto-select survivors: parent + children.
survivors = paretoFrontSchema([topParent, ...children]);
if (survivors.length > config.populationSize) {
// Trim by picking the best-accuracy members.
survivors = survivors
.slice()
.sort((a, b) => (b.score?.accuracy ?? 0) - (a.score?.accuracy ?? 0))
.slice(0, config.populationSize);
}
}
// Anchor stage: larger eval on final survivors.
emitProgress(
config, 'anchor', config.generations, survivors.length,
bestAccuracy(survivors), minComplexity(survivors),
);
const anchorSample = pickSample(config.examples, config.anchorEvalSize, rng);
for (const cand of survivors) {
if (config.signal?.aborted) break;
await scoreSchemaCandidate(cand, anchorSample, config.execute, config.judge, config.signal);
}
const paretoFront = paretoFrontSchema(survivors);
const winner = pickSchemaWinner(paretoFront);
const deltaAccuracy = (winner.score?.accuracy ?? 0) - (baseline.score?.accuracy ?? 0);
emitProgress(
config, 'done', config.generations, paretoFront.length,
winner.score?.accuracy ?? 0, winner.score?.complexity ?? 0,
);
return {
winner,
paretoFront,
history,
deltaAccuracy,
improved: deltaAccuracy > 0,
};
}
}
// ── Helpers ────────────────────────────────────────────────────
function applyMutations(
mutations: Mutation[],
parent: SchemaCandidate,
generation: number,
): SchemaCandidate[] {
return mutations.map((m, i) => ({
id: `g${generation}-${m.kind}-${i}`,
schema: m.apply(parent.schema),
generation,
parent: parent.id,
mutation: m.kind,
mutationLabel: m.description,
score: null,
perExample: [],
}));
}
function pickTopByAccuracy(candidates: SchemaCandidate[]): SchemaCandidate | undefined {
if (candidates.length === 0) return undefined;
return candidates.reduce((a, b) =>
(a.score?.accuracy ?? 0) >= (b.score?.accuracy ?? 0) ? a : b,
);
}
export function pickSchemaWinner(candidates: SchemaCandidate[]): SchemaCandidate {
if (candidates.length === 0) {
throw new Error('pickSchemaWinner called on empty candidate list');
}
return candidates.reduce((best, cand) => {
const bA = best.score?.accuracy ?? -1;
const cA = cand.score?.accuracy ?? -1;
if (cA > bA) return cand;
if (cA < bA) return best;
// Tie on accuracy → prefer lower complexity.
const bC = best.score?.complexity ?? Infinity;
const cC = cand.score?.complexity ?? Infinity;
return cC < bC ? cand : best;
});
}
function bestAccuracy(cs: SchemaCandidate[]): number {
if (cs.length === 0) return 0;
return Math.max(...cs.map(c => c.score?.accuracy ?? 0));
}
function minComplexity(cs: SchemaCandidate[]): number {
const scored = cs.filter(c => c.score !== null);
if (scored.length === 0) return 0;
return Math.min(...scored.map(c => c.score!.complexity));
}
function cloneSchema(s: Schema): Schema {
return {
...s,
fields: s.fields.map(f => ({ ...f, constraints: [...f.constraints] })),
};
}
function deterministicClarifyDescription(field: SchemaField): string {
const base = field.description.trim();
if (!base) return `Return the value for "${field.name}" as ${field.type}.`;
if (/\b(must|should|return)\b/i.test(base)) return base;
return `${base} Must be a ${field.type}.`;
}
// Disambiguation re-export — the generator calls this internally so mutations
// can reference it without colliding with the public `editFieldDescription`
// option callback in `EvolveSchemaOptions`.
const editFieldDescription_pure = editFieldDescription;
function pickN<T>(arr: T[], n: number, rng: () => number): T[] {
if (arr.length <= n) return arr.slice();
const copy = arr.slice();
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, n);
}
function shuffle<T>(arr: T[], rng: () => number): T[] {
for (let i = arr.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr;
}
function arraysEqual<T>(a: T[], b: T[]): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
return true;
}
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
if (k <= 0 || examples.length === 0) return [];
const copy = [...examples];
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, Math.min(k, copy.length));
}
function emitProgress(
config: { onProgress?: (e: EvolveSchemaProgress) => void },
phase: EvolveSchemaProgress['phase'],
generation: number,
populationSize: number,
bestAcc: number,
bestCmp: number,
message?: string,
): void {
if (config.onProgress) {
config.onProgress({
phase, generation, populationSize,
bestAccuracy: bestAcc,
bestComplexity: bestCmp,
message,
});
}
}
function normalizeOptions(opts: EvolveSchemaOptions) {
return {
baseline: opts.baseline,
examples: opts.examples,
execute: opts.execute,
judge: opts.judge,
populationSize: opts.populationSize ?? 5,
generations: opts.generations ?? 3,
mutationMix: opts.mutationMix ?? { structure: 3, order: 1, refinement: 1 },
evalSize: opts.evalSize ?? 32,
anchorEvalSize: opts.anchorEvalSize ?? 100,
seed: opts.seed ?? 1,
editFieldDescription: opts.editFieldDescription,
onProgress: opts.onProgress,
signal: opts.signal,
};
}
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}

View File

@@ -1,19 +1,17 @@
/**
* HarnessTraceBridge — translates workflow-harness phase events into
* execution traces so harnessed agent runs feed the self-evolution loop.
* execution traces so harnessed agent runs show up in cost tracking and
* run history.
*
* Without this bridge the workflow harness is an island: phases run,
* gates pass or fail, but nothing flows into the `execution_traces`
* table that the evaluation dataset builder mines. This module closes
* that loop by subscribing to the shared `harnessEvents` emitter and
* writing one trace per phase outcome:
* table. This module subscribes to the shared `harnessEvents` emitter and
* writes one trace per phase outcome:
*
* harness:phase:complete → trace outcome 'verified'
* harness:phase:fail (aborted === true) → trace outcome 'abandoned'
* harness:phase:fail (will retry) → skipped — only final
* outcomes become training
* signal, mid-retry noise
* stays out of the dataset
* outcomes are recorded
*
* Each trace records:
* - input: the phase's instruction text

View File

@@ -1,249 +0,0 @@
/**
* Improvement Detector — produces structured, runtime-consumable awareness signals.
*
* Per correction #1: Returns actionable signals + surfaced suggestions as structured data,
* not just prompt text changes.
*
* Per correction #5: Awareness is operational — route better, suggest better, adapt better,
* avoid repeated mistakes.
*
* Per correction #3: Workflow suggestions require recency + pattern similarity,
* not just raw shape count.
*/
import type { ImprovementSignalStore, ActionableSignal, SignalCategory } from '@waggle/core';
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
// ── Structured output types ──────────────────────────────────
export interface AwarenessSummary {
/** Capability gaps the agent has encountered repeatedly */
capabilityGaps: CapabilityGapSignal[];
/** Behavioral corrections the user has made repeatedly */
corrections: CorrectionSignal[];
/** Workflow patterns that recur and could benefit from templates */
workflowPatterns: WorkflowPatternSignal[];
/** Total actionable signal count (for deciding whether to inject into prompt) */
totalActionable: number;
}
export interface CapabilityGapSignal {
id: number;
toolName: string;
occurrences: number;
suggestion: string;
}
export interface CorrectionSignal {
id: number;
patternKey: string;
detail: string;
occurrences: number;
guidance: string;
}
export interface WorkflowPatternSignal {
id: number;
patternKey: string;
occurrences: number;
suggestion: string;
}
// ── Capability gap recording ─────────────────────────────────
/**
* Record a capability gap when a tool is not found or a skill is missing.
* Called from agent-loop when tool lookup fails.
*/
export function recordCapabilityGap(
store: ImprovementSignalStore,
toolName: string,
context?: string,
): void {
store.record('capability_gap', `missing:${toolName}`, context, {
tool: toolName,
lastContext: context,
});
}
// ── Correction recording ─────────────────────────────────────
/**
* Analyze a user message for corrections and record if detected.
* Only task-local and durable corrections are recorded (both go into the store;
* the store's threshold mechanism handles promotion to actionable).
*
* Returns the detected correction (if any) for caller use.
*/
export function analyzeAndRecordCorrection(
store: ImprovementSignalStore,
userMessage: string,
previousAssistantMessage?: string,
): DetectedCorrection | null {
const correction = detectCorrection(userMessage, previousAssistantMessage);
if (!correction) return null;
// Record in store — both durable and task-local go in.
// Durable corrections get recorded; task-local ones also get recorded but
// only become actionable if the same pattern_key recurs (count >= threshold).
store.record('correction', correction.patternKey, correction.detail, {
isDurable: correction.isDurable,
confidence: correction.confidence,
});
return correction;
}
// ── Workflow pattern recording ────────────────────────────────
const RECENCY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
/**
* Record a workflow pattern occurrence.
* Per correction #3: requires recency context, not just raw shape count.
*/
export function recordWorkflowPattern(
store: ImprovementSignalStore,
taskShape: string,
taskDescription: string,
): void {
store.record('workflow_pattern', `shape:${taskShape}`, taskDescription, {
lastTask: taskDescription.slice(0, 100),
recordedAt: new Date().toISOString(),
});
}
// ── Structured awareness builder ─────────────────────────────
/**
* Build a structured awareness summary from the signal store.
* Returns runtime-consumable data that can be:
* 1. Injected into system prompt as guidance
* 2. Used by the agent loop for routing decisions
* 3. Displayed in UI as improvement suggestions
*
* Per correction #6: capped at 3 actionable signals total, non-repeating.
*/
export function buildAwarenessSummary(store: ImprovementSignalStore): AwarenessSummary {
const actionable = store.getActionable();
const capabilityGaps: CapabilityGapSignal[] = [];
const corrections: CorrectionSignal[] = [];
const workflowPatterns: WorkflowPatternSignal[] = [];
for (const signal of actionable) {
switch (signal.category) {
case 'capability_gap':
capabilityGaps.push(formatCapabilityGap(signal));
break;
case 'correction':
corrections.push(formatCorrection(signal));
break;
case 'workflow_pattern':
// Per correction #3: only surface if recent (within 7 days)
if (isRecent(signal.last_seen)) {
workflowPatterns.push(formatWorkflowPattern(signal));
}
break;
}
}
return {
capabilityGaps,
corrections,
workflowPatterns,
totalActionable: capabilityGaps.length + corrections.length + workflowPatterns.length,
};
}
/**
* Format awareness summary as a prompt section for system prompt injection.
* Only included when there are actionable signals.
*/
export function formatAwarenessPrompt(summary: AwarenessSummary): string | null {
if (summary.totalActionable === 0) return null;
const lines: string[] = ['## Improvement Signals'];
if (summary.capabilityGaps.length > 0) {
lines.push('');
lines.push('**Missing capabilities** (user has needed these before):');
for (const gap of summary.capabilityGaps) {
lines.push(`- ${gap.suggestion}`);
}
}
if (summary.corrections.length > 0) {
lines.push('');
lines.push('**Behavioral adjustments** (user has corrected these patterns):');
for (const correction of summary.corrections) {
lines.push(`- ${correction.guidance}`);
}
}
if (summary.workflowPatterns.length > 0) {
lines.push('');
lines.push('**Recurring workflows** (consider suggesting a template):');
for (const pattern of summary.workflowPatterns) {
lines.push(`- ${pattern.suggestion}`);
}
}
return lines.join('\n');
}
/**
* Mark all signals in a summary as surfaced.
* Call this after the signals have been injected into a prompt or shown to the user.
*/
export function markSummarySurfaced(
store: ImprovementSignalStore,
summary: AwarenessSummary,
): void {
for (const gap of summary.capabilityGaps) store.markSurfaced(gap.id);
for (const correction of summary.corrections) store.markSurfaced(correction.id);
for (const pattern of summary.workflowPatterns) store.markSurfaced(pattern.id);
}
// ── Helpers ──────────────────────────────────────────────────
function formatCapabilityGap(signal: ActionableSignal): CapabilityGapSignal {
const toolName = signal.parsedMetadata.tool as string || signal.pattern_key.replace('missing:', '');
return {
id: signal.id,
toolName,
occurrences: signal.count,
suggestion: `User has needed "${toolName}" ${signal.count} times. Consider using acquire_capability to find it.`,
};
}
function formatCorrection(signal: ActionableSignal): CorrectionSignal {
return {
id: signal.id,
patternKey: signal.pattern_key,
detail: signal.detail,
occurrences: signal.count,
guidance: signal.detail
? `${signal.detail} (corrected ${signal.count} times)`
: `Correction pattern "${signal.pattern_key}" observed ${signal.count} times`,
};
}
function formatWorkflowPattern(signal: ActionableSignal): WorkflowPatternSignal {
const shape = signal.pattern_key.replace('shape:', '');
return {
id: signal.id,
patternKey: signal.pattern_key,
occurrences: signal.count,
suggestion: `"${shape}" tasks recur frequently (${signal.count} times). A workflow template could streamline this.`,
};
}
function isRecent(dateStr: string): boolean {
try {
const date = new Date(dateStr);
return Date.now() - date.getTime() < RECENCY_WINDOW_MS;
} catch {
return false;
}
}

View File

@@ -1,181 +0,0 @@
/**
* @internal Not exported from index.ts — future feature, tested but not wired.
*
* Improvement Wiring — processes each interaction for improvement signals.
*
* Bridges the gap between the chat route (which has raw messages) and the
* existing correction-detector / improvement-detector / capability-acquisition
* modules. Call processInteractionForImprovement after every agent response
* to detect corrections, capability gaps, and recurring workflow patterns.
*/
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
import { detectTaskShape, type TaskShape } from './task-shape.js';
// ── Types ──────────────────────────────────────────────────────────────
export interface ImprovementWiringParams {
userMessage: string;
agentResponse: string;
toolsUsed: string[];
workspaceId: string;
sessionId: string;
}
export interface ImprovementWiringResult {
wasCorrection: boolean;
correctionDetail?: string;
capabilityGap?: string;
workflowPattern?: string;
}
// ── Well-known tool names (built-in tools the agent can use) ───────────
const KNOWN_TOOL_NAMES = new Set([
'web_search', 'web_fetch', 'search_memory', 'save_memory',
'get_identity', 'get_awareness', 'query_knowledge', 'add_task',
'correct_knowledge', 'bash', 'read_file', 'write_file', 'edit_file',
'search_files', 'search_content', 'git_status', 'git_diff', 'git_log',
'git_commit', 'generate_docx', 'create_plan', 'add_plan_step',
'execute_step', 'show_plan', 'list_skills', 'create_skill',
'delete_skill', 'read_skill', 'search_skills', 'suggest_skill',
'acquire_capability', 'install_capability', 'compose_workflow',
'orchestrate_workflow', 'spawn_agent', 'list_agents', 'get_agent_result',
]);
// ── Capability gap detection patterns ──────────────────────────────────
/**
* Patterns in user messages that suggest the user wants a tool/capability
* the agent doesn't have. These are request-like phrases paired with
* tool/domain keywords.
*/
const GAP_REQUEST_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
{ pattern: /\bcan you (?:send|post|publish|push) (?:to|on|via) (\w+)/i, domain: 'integration' },
{ pattern: /\bconnect (?:to|with) (\w+)/i, domain: 'connector' },
{ pattern: /\buse (\w+) (?:api|tool|service)/i, domain: 'integration' },
{ pattern: /\b(?:read|open|parse|convert) (?:this |the |a )?(\w+) file/i, domain: 'file_format' },
{ pattern: /\brun (?:this |the |a )?(\w+) (?:test|check|scan|lint)/i, domain: 'tooling' },
];
/**
* Detect if the agent response itself indicates a tool was missing.
* The agent often says things like "I don't have a tool for X" or
* "Tool 'X' not found".
*/
const TOOL_NOT_FOUND_PATTERNS: RegExp[] = [
/Tool "(.+?)" not found/,
/I don't have (?:a |the )?(?:tool|capability|ability) (?:for|to) (.+?)(?:\.|$)/i,
/no (?:tool|capability) available for (.+?)(?:\.|$)/i,
];
// ── Workflow pattern detection ─────────────────────────────────────────
/**
* Multi-step request indicators — the user is asking for a sequence of actions.
*/
const MULTI_STEP_PATTERNS: RegExp[] = [
/\bthen\b.*\bthen\b/i, // "do X, then Y, then Z"
/\bfirst\b.*\bthen\b/i, // "first do X, then Y"
/\bstep\s*\d+/i, // "step 1, step 2"
/\b(?:and then|after that|next|finally)\b/i, // sequence words
];
// ── Main function ──────────────────────────────────────────────────────
/**
* Process an interaction for improvement signals.
*
* This is the main entry point called after every agent response.
* It analyzes the user message and agent response to detect:
* 1. Corrections (using the existing correction-detector)
* 2. Capability gaps (tools requested but not available)
* 3. Workflow patterns (recurring multi-step sequences)
*
* Returns structured signals for storage and display.
*/
export function processInteractionForImprovement(
params: ImprovementWiringParams,
): ImprovementWiringResult {
const { userMessage, agentResponse, toolsUsed } = params;
const result: ImprovementWiringResult = {
wasCorrection: false,
};
// 1. Run correction detector on userMessage
const correction = detectCorrection(userMessage);
if (correction) {
result.wasCorrection = true;
result.correctionDetail = correction.detail;
}
// 2. Check for capability gaps
const gap = detectCapabilityGap(userMessage, agentResponse, toolsUsed);
if (gap) {
result.capabilityGap = gap;
}
// 3. Check for workflow patterns (repeated multi-step sequences)
const pattern = detectWorkflowPattern(userMessage);
if (pattern) {
result.workflowPattern = pattern;
}
return result;
}
// ── Internal helpers ───────────────────────────────────────────────────
/**
* Detect capability gaps from the interaction.
* Checks:
* - Agent response mentions a tool not found
* - User message requests a tool/integration not in the known set
*/
function detectCapabilityGap(
userMessage: string,
agentResponse: string,
toolsUsed: string[],
): string | undefined {
// Check agent response for explicit tool-not-found messages
for (const pattern of TOOL_NOT_FOUND_PATTERNS) {
const match = agentResponse.match(pattern);
if (match && match[1]) {
return match[1].trim();
}
}
// Check user message for requests that imply missing capabilities
for (const { pattern } of GAP_REQUEST_PATTERNS) {
const match = userMessage.match(pattern);
if (match && match[1]) {
const requested = match[1].toLowerCase();
// Only flag as a gap if no tool was actually used for this domain
if (toolsUsed.length === 0 || !toolsUsed.some(t => t.includes(requested))) {
return requested;
}
}
}
return undefined;
}
/**
* Detect recurring workflow patterns from the user message.
* Uses task-shape detection to identify the structural pattern,
* and checks for multi-step indicators.
*/
function detectWorkflowPattern(userMessage: string): string | undefined {
// Only flag multi-step requests as workflow patterns
const isMultiStep = MULTI_STEP_PATTERNS.some(p => p.test(userMessage));
if (!isMultiStep) return undefined;
// Use task-shape detector to identify the pattern type
const shape = detectTaskShape(userMessage);
if (shape.confidence >= 0.3) {
return shape.type;
}
return undefined;
}

View File

@@ -203,7 +203,6 @@ export { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js'
export {
loadSystemPrompt,
loadSystemPromptWithOverrides,
assertOverridesReachActiveSpec,
loadSkills,
type ComposedSystemPrompt,
type LoadedSkill,
@@ -214,69 +213,11 @@ export { checkGrounding, extractClaimedSpecifics, type GroundingResult, type Cla
export { CostTracker, DEFAULT_MODEL_PRICING, type ModelPricing, type UsageStats, type UsageEntry } from './cost-tracker.js';
export { extractEntities, type ExtractedEntity } from './entity-extractor.js';
export { CognifyPipeline, type CognifyConfig, type CognifyResult } from './cognify.js';
export { AgentLearning, type LearnedBehavior, type PersonaEffectiveness, type LearningSnapshot } from './agent-learning.js';
export {
TraceRecorder, truncate as truncateTraceText, scrubSecrets,
type TraceHandle, type FinalizeOptions as TraceFinalizeOptions,
} from './trace-recorder.js';
export {
EvalDatasetBuilder, detectSecrets, redactSecrets, SECRET_PATTERN_NAMES,
toJSONL as evalToJSONL, fromJSONL as evalFromJSONL,
type EvalExample, type EvalExampleMetadata, type DatasetSplit,
type BuildOptions as EvalBuildOptions, type JudgeVerdict,
} from './eval-dataset.js';
export {
IterativeGEPA, paretoFront, scoreCandidate, aggregateScores,
pickWinner, pickSample as pickGepaSample,
type Candidate as GEPACandidate, type CandidateScore as GEPACandidateScore,
type IterativeGEPAOptions, type GEPARunResult, type GEPAProgress,
type ScoreCandidateOptions,
type MutateArgs, type MutateFn, type MutationStrategy, type EvolutionTarget,
} from './iterative-optimizer.js';
export {
runGates, DEFAULT_SIZE_LIMITS,
checkNonEmpty, checkSize, checkGrowth,
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
type GateVerdict, type GateResult as EvolutionGateResult, type GateCheckResult,
type SizeLimits, type GateOptions, type CheckInput as GateCheckInput,
} from './evolution-gates.js';
export {
EvolveSchema,
addOutputField, removeField, editFieldDescription,
changeFieldType, addConstraint, removeConstraint,
reorderFields, replaceOutputFields,
schemaComplexity, aggregateSchemaScores, paretoFrontSchema,
scoreSchemaCandidate, pickSchemaWinner,
generateStructureMutations, generateOrderMutations, generateRefinementMutations,
pickSample as pickSchemaSample,
type Schema, type SchemaField, type FieldType, type FieldConstraint,
type Mutation as SchemaMutation, type MutationKind as SchemaMutationKind,
type SchemaCandidate, type SchemaCandidateScore, type SchemaExampleResult,
type SchemaExecuteFn, type EvolveSchemaOptions, type EvolveSchemaResult,
type EvolveSchemaProgress,
} from './evolve-schema.js';
export {
ComposeEvolution, defaultFeedbackFilter, filterJudgeFeedback,
stripStructuralLines, schemaExecutorFromInstructionRunner,
type ComposeEvolutionOptions, type ComposeEvolutionResult,
type ComposeProgress, type FeedbackFilter,
} from './compose-evolution.js';
export {
EvolutionOrchestrator, eligibleForEvolution, summarizeRuns,
type EvolutionOrchestratorDeps, type EvolutionOrchestratorOptions,
type EvolutionAutoTriggerConfig, type OrchestratorRunResult,
type OrchestratorOutcome, type EvolutionProgress,
type SchemaBaselineInput,
} from './evolution-orchestrator.js';
export {
deployPersonaOverride, rollbackPersonaOverride,
deployBehavioralSpecOverride, rollbackBehavioralSpecOverride,
loadBehavioralSpecOverrides, applyBehavioralSpecOverrides,
BEHAVIORAL_SPEC_SECTIONS,
type DeployResult, type DeployPersonaInput,
type DeployBehavioralSpecInput, type BehavioralSpecOverride,
type BehavioralSpecSection,
} from './evolution-deploy.js';
export { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from './secret-patterns.js';
export {
LLMJudge, DEFAULT_WEIGHTS, DEFAULT_RUBRIC,
buildPrompt as buildJudgePrompt,
@@ -285,17 +226,12 @@ export {
type ParsedJudgeResponse,
} from './judge.js';
export {
createAnthropicEvolutionLLM,
buildJudgeLLMCall, buildGEPAMutateFn, buildSchemaExecuteFn,
buildReflectiveMutationPrompt, buildSchemaFillPrompt,
makeRunningJudge, isRunningJudge, RUNNING_JUDGE_BRAND,
createAnthropicEvolutionLLM, buildJudgeLLMCall,
retryWithBackoff, wrapWithRetry,
isRetryableEvolutionError, computeRetryDelay,
DEFAULT_RETRY_OPTIONS,
type EvolutionLLM, type CreateAnthropicEvolutionLLMOptions,
type BuildReflectiveMutationPromptArgs,
type RetryOptions, type RetryInfo,
type RunningJudge,
} from './evolution-llm-wiring.js';
export {
createHarnessRun, advancePhase, getCurrentPhaseInstruction,
@@ -346,7 +282,6 @@ export { createDocumentTools } from './document-tools.js';
export { createSpreadsheetTools } from './spreadsheet-tools.js';
export { createPresentationTools } from './presentation-tools.js';
export { createPdfTools } from './pdf-tools.js';
export { createInsightsTools, type InsightsDeps } from './insights-tools.js';
export { createConnectorSearchTools } from './connector-search.js';
export { createCrossWorkspaceTools, type CrossWorkspaceToolDeps } from './cross-workspace-tools.js';
export { extractEntitiesWithLLM, type LLMCallFn as EntityLLMCallFn } from './entity-extractor.js';
@@ -480,7 +415,6 @@ export {
ComposioConnector, MSTeamsConnector, OutlookConnector, OneDriveConnector, OneNoteConnector,
} from './connectors/index.js';
export { IterationBudget, type IterationBudgetConfig } from './iteration-budget.js';
export { captureInteraction, getRecentLogs, isWithinBudget, type CaptureInteractionInput } from './optimization-capture.js';
export { routeMessage, type RoutingDecision } from './smart-router.js';
export {
compressConversation, estimateTokens, needsCompression,
@@ -511,13 +445,7 @@ export {
type DeliveryChannel, type DeliveryPreferences, type DeliveryMessage, type DeliveryResult,
type DeliveryConnector, type DeliveryConnectorRegistry, type InAppEmitter,
} from './cron-delivery-router.js';
export { detectCorrection, detectCorrectionsInHistory, type DetectedCorrection, type CorrectionDurability } from './correction-detector.js';
export { detectContradiction, type ContradictionResult } from './contradiction-detector.js';
export {
recordCapabilityGap, analyzeAndRecordCorrection, recordWorkflowPattern,
buildAwarenessSummary, formatAwarenessPrompt, markSummarySurfaced,
type AwarenessSummary, type CapabilityGapSignal, type CorrectionSignal, type WorkflowPatternSignal,
} from './improvement-detector.js';
export {
lintMemoryWrite,
type MemoryLintResult, type MemoryLintVerdict,

View File

@@ -1,140 +0,0 @@
/**
* Insights Tools — self-analytics for the agent (Hermes-inspired).
*
* Provides the agent with awareness of its own performance:
* - Tool usage frequency and success rates
* - Cost per model
* - Correction frequency trends
* - Improvement signal summary
*
* This enables the agent to reflect on what works and adapt strategy.
*/
import type { ToolDefinition } from './tools.js';
export interface InsightsDeps {
getOptimizationLogs: (limit: number) => Array<{
model?: string;
tool_names?: string;
total_tokens?: number;
estimated_cost?: number;
was_correction?: number;
created_at?: string;
}>;
getImprovementSignals: () => Array<{
category: string;
pattern_key: string;
detail: string;
count: number;
first_seen: string;
last_seen: string;
}>;
}
export function createInsightsTools(deps: InsightsDeps): ToolDefinition[] {
return [
{
name: 'agent_insights',
description: [
'Analyze your own performance metrics. Returns: tool usage frequency,',
'cost per model, correction rate, and active improvement signals.',
'Use this to reflect on what approaches work and adapt your strategy.',
'Call this when you want to understand your performance patterns.',
].join(' '),
parameters: {
type: 'object' as const,
required: [],
properties: {
lookbackDays: {
type: 'number' as const,
description: 'Number of days to analyze (default 30)',
},
},
},
execute: async (args: Record<string, unknown>) => {
const lookbackDays = (args.lookbackDays as number) ?? 30;
const limit = lookbackDays * 50; // ~50 interactions/day estimate
try {
const logs = deps.getOptimizationLogs(limit);
const signals = deps.getImprovementSignals();
// Tool usage frequency
const toolCounts: Record<string, { total: number; successes: number }> = {};
for (const log of logs) {
const tools = log.tool_names?.split(',').map(t => t.trim()).filter(Boolean) ?? [];
const isCorrection = log.was_correction === 1;
for (const tool of tools) {
if (!toolCounts[tool]) toolCounts[tool] = { total: 0, successes: 0 };
toolCounts[tool].total++;
if (!isCorrection) toolCounts[tool].successes++;
}
}
const toolStats = Object.entries(toolCounts)
.sort((a, b) => b[1].total - a[1].total)
.slice(0, 15)
.map(([name, stats]) => ({
tool: name,
uses: stats.total,
successRate: stats.total > 0 ? Math.round((stats.successes / stats.total) * 100) : 0,
}));
// Cost per model
const modelCosts: Record<string, { calls: number; tokens: number; cost: number }> = {};
for (const log of logs) {
const model = log.model ?? 'unknown';
if (!modelCosts[model]) modelCosts[model] = { calls: 0, tokens: 0, cost: 0 };
modelCosts[model].calls++;
modelCosts[model].tokens += log.total_tokens ?? 0;
modelCosts[model].cost += log.estimated_cost ?? 0;
}
const modelStats = Object.entries(modelCosts)
.sort((a, b) => b[1].calls - a[1].calls)
.map(([model, stats]) => ({
model,
calls: stats.calls,
tokens: stats.tokens,
costUsd: Math.round(stats.cost * 100) / 100,
}));
// Correction rate
const totalInteractions = logs.length;
const corrections = logs.filter(l => l.was_correction === 1).length;
const correctionRate = totalInteractions > 0
? Math.round((corrections / totalInteractions) * 100)
: 0;
// Active improvement signals
const activeSignals = signals
.filter(s => s.count >= 2)
.slice(0, 10)
.map(s => ({
category: s.category,
pattern: s.pattern_key,
detail: s.detail,
occurrences: s.count,
lastSeen: s.last_seen,
}));
return JSON.stringify({
period: `Last ${lookbackDays} days`,
totalInteractions,
correctionRate: `${correctionRate}%`,
topTools: toolStats,
modelUsage: modelStats,
improvementSignals: activeSignals,
recommendation: correctionRate > 15
? 'High correction rate — review recent corrections and adjust approach.'
: correctionRate > 5
? 'Moderate corrections — some patterns may need attention.'
: 'Low correction rate — current approach is working well.',
}, null, 2);
} catch (err: unknown) {
return `Error generating insights: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
];
}

View File

@@ -1,626 +0,0 @@
/**
* Iterative GEPA Optimizer — Phase 2.1 of the self-evolution loop.
*
* Upgrades the one-shot prompt expander (optimizer-service) to a multi-
* generation population search with Pareto-frontier selection and
* reflective mutations driven by LLMJudge feedback (Phase 1.3).
*
* Loop:
* 1. Start with `baseline` prompt as the seed population (1 member).
* 2. For each generation:
* a. Micro-screen survivors on a small sample (default 50) to prune
* obvious losers.
* b. Mini-eval passing candidates on a larger sample (default 64).
* c. Spawn `populationSize` reflective mutations of the top Pareto
* candidate, using aggregated judge feedback as ASI.
* 3. Anchor evaluation on the full eval set (default 400) picks the winner.
*
* "Actionable Side Information" (ASI) is the textual feedback the judge
* produces — fed back into the `mutate` callback so the next generation
* targets the identified weakness rather than mutating blindly.
*
* Pareto selection keeps candidates that are non-dominated on a
* 3-dimensional fitness vector (correctness, procedureFollowing,
* conciseness). A single scalar `overall` is also tracked for tie-breaking
* and final ranking.
*/
import type { EvalExample } from './eval-dataset.js';
import type { LLMJudge, JudgeScore } from './judge.js';
import { isRunningJudge } from './evolution-llm-wiring.js';
// ── Types ───────────────────────────────────────────────────────
export interface Candidate {
/** Stable id — e.g. `g0-baseline`, `g1-m2` */
id: string;
/** The prompt text being evolved */
prompt: string;
/** Generation index (0 = baseline) */
generation: number;
/** Parent candidate id (null for baseline) */
parent: string | null;
/** Short label describing the mutation strategy */
strategy: string;
/** Aggregated score across the most recent eval stage, or null if not yet scored */
score: CandidateScore | null;
/** Raw per-example scores used for Pareto analysis */
perExample: JudgeScore[];
}
/** Aggregated, mean-reduced score used for selection. */
export interface CandidateScore {
overall: number;
correctness: number;
procedureFollowing: number;
conciseness: number;
/** Mean length penalty applied across the eval set */
lengthPenalty: number;
/** Number of examples the candidate was scored on */
n: number;
/** Aggregated feedback lines from the lowest-scoring examples */
weaknessFeedback: string[];
}
export type MutationStrategy =
| 'expand-edge-cases'
| 'tighten-format'
| 'add-examples'
| 'clarify-constraints'
| 'reduce-length'
| 'restructure-steps'
| 'targeted-feedback';
export interface MutateArgs {
parent: Candidate;
strategy: MutationStrategy;
/** Textual feedback taken from the parent's lowest-scoring examples */
weaknessFeedback: string[];
/** Target types we are mutating (for the mutate fn to tune style) */
targetKind: EvolutionTarget;
/** Generation index of the child */
generation: number;
}
export type MutateFn = (args: MutateArgs) => Promise<string>;
/** What is being evolved — informs mutation style and constraint policies. */
export type EvolutionTarget =
| 'persona-system-prompt'
| 'behavioral-spec-section'
| 'tool-description'
| 'skill-body'
| 'generic';
export interface GEPAProgress {
phase: 'start' | 'micro-screen' | 'mini-eval' | 'mutate' | 'anchor' | 'done';
generation: number;
populationSize: number;
/** The best-so-far overall score at this point */
best: number;
message?: string;
}
export interface IterativeGEPAOptions {
/** The starting prompt text. */
baseline: string;
/** Full eval dataset (train + val combined is typical). */
examples: EvalExample[];
/** Scorer — LLMJudge or anything that implements `score(input): Promise<JudgeScore>`. */
judge: Pick<LLMJudge, 'score'>;
/** Function that produces a mutated prompt from a parent + weakness feedback. */
mutate: MutateFn;
/**
* Opt-in escape hatch that disables the running-judge safety check.
* Only set to true for unit tests with mock judges whose `score()` does
* not need a real LLM call. Production code paths (evolving real prompts)
* must always pass a `makeRunningJudge(...)`-wrapped judge; otherwise
* GEPA optimizes prompt-text similarity to the expected output — a
* meaningless gradient that produces nonsense winners.
*/
allowBareJudge?: boolean;
/** What we're optimizing (tunes mutation + gate policy). Default 'generic'. */
targetKind?: EvolutionTarget;
/** Candidates per generation. Default 5. */
populationSize?: number;
/** How many generations to run. Default 3. */
generations?: number;
/** Micro-screen sample size. Default 50. */
microScreenSize?: number;
/** Mini-eval sample size. Default 64. */
miniEvalSize?: number;
/** Anchor evaluation sample size (applied to final Pareto set). Default 400. */
anchorEvalSize?: number;
/** Seed for deterministic sampling. Default 1. */
seed?: number;
/** Emits progress events across stages. */
onProgress?: (event: GEPAProgress) => void;
/** Optional abort signal */
signal?: AbortSignal;
/**
* Max concurrent `judge.score()` calls per candidate during eval stages.
* Default 1 (fully sequential) — preserves legacy behavior. Bumping to
* 4–8 cuts wall time dramatically when the judge is an LLM API with
* plenty of concurrent capacity. Does not affect determinism: all
* scores are aggregated after every example completes.
*/
concurrency?: number;
}
export interface GEPARunResult {
winner: Candidate;
/** Full Pareto-non-dominated set from the anchor stage */
paretoFront: Candidate[];
/** All candidates across all generations */
history: Candidate[];
/** True if the winner improved overall score over baseline */
improved: boolean;
/** Delta winner.score.overall - baseline.score.overall */
delta: number;
}
// ── Public API ─────────────────────────────────────────────────
export class IterativeGEPA {
/**
* Run the iterative optimization loop.
*
* Returns the winner candidate plus the full Pareto set. Winner is the
* Pareto-set member with the highest `overall` score (ties broken by
* length penalty, i.e. the more concise one wins).
*/
async run(options: IterativeGEPAOptions): Promise<GEPARunResult> {
// H-09 G3 · reject bare judges — without makeRunningJudge the judge
// scores prompt-text-vs-expected-output, which is a meaningless
// gradient. Any caller evolving real prompts must wrap their judge.
// Tests pass `allowBareJudge: true` to bypass.
if (!options.allowBareJudge && !isRunningJudge(options.judge)) {
throw new Error(
'IterativeGEPA: judge is not a running judge. Wrap with ' +
'makeRunningJudge(baseJudge, llm) before passing to .run(), or ' +
'pass `allowBareJudge: true` for tests with mock judges. ' +
'See evolution-llm-wiring.ts:makeRunningJudge for rationale.',
);
}
const config = normalizeOptions(options);
const rng = makeRng(config.seed);
// ── Seed population with baseline ────────────────────────
const baseline: Candidate = {
id: 'g0-baseline',
prompt: config.baseline,
generation: 0,
parent: null,
strategy: 'baseline',
score: null,
perExample: [],
};
emit(config, 'start', 0, 1, 0, 'seeding baseline');
const history: Candidate[] = [baseline];
let survivors: Candidate[] = [baseline];
// Score the baseline on the micro-screen so we have a comparison point.
const microSample = pickSample(config.examples, config.microScreenSize, rng);
await scoreCandidate(baseline, microSample, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
emit(config, 'micro-screen', 0, 1, baseline.score?.overall ?? 0);
// ── Generations ──────────────────────────────────────────
for (let gen = 1; gen <= config.generations; gen++) {
if (config.signal?.aborted) break;
// 1. Mutate top survivor(s) into N children.
const topParent = pickTopParent(survivors);
if (!topParent) break;
emit(config, 'mutate', gen, survivors.length, topParent.score?.overall ?? 0);
const children: Candidate[] = [];
const strategies = cycleStrategies(config.populationSize, gen);
for (let i = 0; i < config.populationSize; i++) {
if (config.signal?.aborted) break;
const strategy = strategies[i];
let childPrompt: string;
try {
childPrompt = await config.mutate({
parent: topParent,
strategy,
weaknessFeedback: topParent.score?.weaknessFeedback ?? [],
targetKind: config.targetKind,
generation: gen,
});
} catch {
// Mutation failed — reuse parent so the loop doesn't stall.
childPrompt = topParent.prompt;
}
const child: Candidate = {
id: `g${gen}-m${i}`,
prompt: childPrompt,
generation: gen,
parent: topParent.id,
strategy,
score: null,
perExample: [],
};
children.push(child);
history.push(child);
}
// 2. Micro-screen children on a small sample.
const microSet = pickSample(config.examples, config.microScreenSize, rng);
for (const child of children) {
if (config.signal?.aborted) break;
await scoreCandidate(child, microSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
emit(
config, 'micro-screen', gen, children.length,
Math.max(...children.map(c => c.score?.overall ?? 0), topParent.score?.overall ?? 0),
);
// 3. Keep children that at least match the parent's overall score.
const threshold = topParent.score?.overall ?? 0;
const microSurvivors = children.filter(c => (c.score?.overall ?? 0) >= threshold - 0.05);
// If everyone got filtered out, keep the single best child so we don't stall.
if (microSurvivors.length === 0 && children.length > 0) {
const best = children.reduce((a, b) =>
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
);
microSurvivors.push(best);
}
// 4. Mini-eval survivors on a larger sample.
if (microSurvivors.length > 0) {
const miniSet = pickSample(config.examples, config.miniEvalSize, rng);
for (const cand of microSurvivors) {
if (config.signal?.aborted) break;
await scoreCandidate(cand, miniSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
emit(
config, 'mini-eval', gen, microSurvivors.length,
Math.max(...microSurvivors.map(c => c.score?.overall ?? 0)),
);
}
// 5. Pareto-select survivors for next generation.
const candidatePool = [topParent, ...microSurvivors];
survivors = paretoFront(candidatePool);
}
// ── Anchor evaluation — big sample, final ranking ───────
emit(config, 'anchor', config.generations, survivors.length, bestOverall(survivors));
const anchorSet = pickSample(config.examples, config.anchorEvalSize, rng);
for (const cand of survivors) {
if (config.signal?.aborted) break;
await scoreCandidate(cand, anchorSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
const anchorPareto = paretoFront(survivors);
const winner = pickWinner(anchorPareto);
const baselineOverall = baseline.score?.overall ?? 0;
const winnerOverall = winner.score?.overall ?? 0;
emit(config, 'done', config.generations, anchorPareto.length, winnerOverall);
return {
winner,
paretoFront: anchorPareto,
history,
improved: winnerOverall > baselineOverall,
delta: winnerOverall - baselineOverall,
};
}
}
// ── Helpers (exported for tests) ───────────────────────────────
/**
* Pareto front: keep candidates not dominated on any of the three scored
* dimensions. A dominates B if A ≥ B on all three and strictly greater
* on at least one.
*/
export function paretoFront(candidates: Candidate[]): Candidate[] {
const scored = candidates.filter(c => c.score !== null);
const front: Candidate[] = [];
for (const cand of scored) {
let dominated = false;
for (const other of scored) {
if (other === cand) continue;
if (dominates(other.score!, cand.score!)) {
dominated = true;
break;
}
}
if (!dominated) front.push(cand);
}
// If all candidates have equal scores, return them all.
return front.length > 0 ? front : scored;
}
function dominates(a: CandidateScore, b: CandidateScore): boolean {
const geAll =
a.correctness >= b.correctness &&
a.procedureFollowing >= b.procedureFollowing &&
a.conciseness >= b.conciseness;
const gtAny =
a.correctness > b.correctness ||
a.procedureFollowing > b.procedureFollowing ||
a.conciseness > b.conciseness;
return geAll && gtAny;
}
export interface ScoreCandidateOptions {
/** Abort signal — checked before each example is dispatched. */
signal?: AbortSignal;
/** Max concurrent judge.score() calls. Default 1 (sequential). */
concurrency?: number;
}
/**
* Score a candidate against a batch of examples and aggregate.
*
* Backwards compatible: the 4th argument may be either an `AbortSignal`
* (legacy) or a `ScoreCandidateOptions` object. Pass
* `{ concurrency: 4 }` to parallelize — individual examples are
* independent so the aggregate result is identical to the sequential
* version, only faster when the judge is an LLM API.
*/
export async function scoreCandidate(
candidate: Candidate,
examples: EvalExample[],
judge: Pick<LLMJudge, 'score'>,
signalOrOptions?: AbortSignal | ScoreCandidateOptions,
): Promise<CandidateScore> {
const options = resolveScoreOptions(signalOrOptions);
const signal = options.signal;
const concurrency = Math.max(1, options.concurrency ?? 1);
// The "candidate" in this context IS the new prompt we want to test.
// We treat it as the model's actual response to the example's input,
// since mutation produces a variant prompt-as-response. Callers can
// substitute a `runCandidate` step before scoring if the prompt needs
// to be executed first — see docs.
const scoreOne = async (ex: EvalExample): Promise<JudgeScore | null> => {
if (signal?.aborted) return null;
try {
return await judge.score({
input: ex.input,
expected: ex.expected_output,
actual: candidate.prompt,
});
} catch {
// Keep iterating so one bad example doesn't invalidate the whole score.
return null;
}
};
const results = concurrency === 1
? await runSequential(examples, scoreOne)
: await mapWithConcurrency(examples, concurrency, scoreOne);
const scores = results.filter((s): s is JudgeScore => s !== null);
candidate.perExample = scores;
const aggregated = aggregateScores(scores);
candidate.score = aggregated;
return aggregated;
}
function resolveScoreOptions(
arg: AbortSignal | ScoreCandidateOptions | undefined,
): ScoreCandidateOptions {
if (!arg) return {};
// Legacy: AbortSignal passed directly. Detect by presence of `aborted`
// (a boolean getter unique to AbortSignal-like objects).
if (typeof (arg as AbortSignal).aborted === 'boolean' && typeof (arg as AbortSignal).addEventListener === 'function') {
return { signal: arg as AbortSignal };
}
return arg as ScoreCandidateOptions;
}
async function runSequential<T, R>(
items: readonly T[],
fn: (item: T) => Promise<R>,
): Promise<R[]> {
const results: R[] = [];
for (const item of items) {
results.push(await fn(item));
}
return results;
}
/**
* Bounded-parallelism map. Runs `fn` over `items` with at most
* `concurrency` in-flight calls at once. Results are returned in the
* same order as inputs regardless of completion order.
*
* Implementation: N worker loops share a counter. Each worker pulls the
* next index, awaits `fn`, stores at that index, repeats. No external
* deps, no extra state machinery.
*/
async function mapWithConcurrency<T, R>(
items: readonly T[],
concurrency: number,
fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const results: R[] = new Array(items.length);
const workerCount = Math.min(concurrency, items.length);
let next = 0;
const workers = Array.from({ length: workerCount }, async () => {
while (true) {
const i = next++;
if (i >= items.length) return;
results[i] = await fn(items[i], i);
}
});
await Promise.all(workers);
return results;
}
export function aggregateScores(scores: JudgeScore[]): CandidateScore {
if (scores.length === 0) {
return {
overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0,
lengthPenalty: 1, n: 0, weaknessFeedback: [],
};
}
const n = scores.length;
const sum = scores.reduce(
(acc, s) => ({
overall: acc.overall + s.overall,
correctness: acc.correctness + s.correctness,
procedureFollowing: acc.procedureFollowing + s.procedureFollowing,
conciseness: acc.conciseness + s.conciseness,
lengthPenalty: acc.lengthPenalty + s.lengthPenalty,
}),
{ overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0, lengthPenalty: 0 },
);
// Collect the 3 worst-scoring examples' feedback as weakness signal.
const worst = [...scores].sort((a, b) => a.overall - b.overall).slice(0, 3);
const weaknessFeedback = worst
.map(s => s.feedback)
.filter(fb => fb && fb.length > 0);
return {
overall: sum.overall / n,
correctness: sum.correctness / n,
procedureFollowing: sum.procedureFollowing / n,
conciseness: sum.conciseness / n,
lengthPenalty: sum.lengthPenalty / n,
n,
weaknessFeedback,
};
}
/**
* Pick the "winner" from a Pareto set. Prefers highest overall; ties broken
* by higher length penalty (more concise) then lowest generation (simpler).
*/
export function pickWinner(candidates: Candidate[]): Candidate {
if (candidates.length === 0) {
throw new Error('pickWinner called on empty candidate list');
}
return candidates.reduce((best, cand) => {
const b = best.score?.overall ?? -1;
const c = cand.score?.overall ?? -1;
if (c > b) return cand;
if (c < b) return best;
// Tie on overall — prefer higher lengthPenalty (more concise, less penalized).
const bLen = best.score?.lengthPenalty ?? 0;
const cLen = cand.score?.lengthPenalty ?? 0;
if (cLen !== bLen) return cLen > bLen ? cand : best;
return best.generation <= cand.generation ? best : cand;
});
}
function pickTopParent(survivors: Candidate[]): Candidate | undefined {
if (survivors.length === 0) return undefined;
return survivors.reduce((a, b) =>
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
);
}
function bestOverall(candidates: Candidate[]): number {
if (candidates.length === 0) return 0;
return Math.max(...candidates.map(c => c.score?.overall ?? 0));
}
/**
* Deterministic sampler — picks `k` examples using a seeded RNG.
* If k >= examples.length, returns the full list in deterministic shuffled order.
*/
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
if (k <= 0 || examples.length === 0) return [];
const copy = [...examples];
// Fisher–Yates shuffle
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, Math.min(k, copy.length));
}
/** Rotate through strategies so early generations explore the mutation space. */
function cycleStrategies(n: number, generation: number): MutationStrategy[] {
const all: MutationStrategy[] = [
'expand-edge-cases',
'tighten-format',
'add-examples',
'clarify-constraints',
'reduce-length',
'restructure-steps',
'targeted-feedback',
];
// Early generations: spread across strategies.
// Later generations: bias toward targeted-feedback once ASI is strong.
if (generation <= 1) {
return Array.from({ length: n }, (_, i) => all[i % all.length]);
}
return Array.from({ length: n }, (_, i) => {
if (i === 0) return 'targeted-feedback';
return all[(i + generation) % all.length];
});
}
function emit(
config: { onProgress?: (event: GEPAProgress) => void },
phase: GEPAProgress['phase'],
generation: number,
populationSize: number,
best: number,
message?: string,
): void {
if (config.onProgress) {
config.onProgress({ phase, generation, populationSize, best, message });
}
}
function normalizeOptions(opts: IterativeGEPAOptions): Required<Omit<IterativeGEPAOptions, 'signal' | 'onProgress'>> & {
onProgress?: IterativeGEPAOptions['onProgress'];
signal?: AbortSignal;
} {
return {
baseline: opts.baseline,
examples: opts.examples,
judge: opts.judge,
mutate: opts.mutate,
targetKind: opts.targetKind ?? 'generic',
populationSize: opts.populationSize ?? 5,
generations: opts.generations ?? 3,
microScreenSize: opts.microScreenSize ?? 50,
miniEvalSize: opts.miniEvalSize ?? 64,
anchorEvalSize: opts.anchorEvalSize ?? 400,
seed: opts.seed ?? 1,
onProgress: opts.onProgress,
signal: opts.signal,
concurrency: Math.max(1, opts.concurrency ?? 1),
allowBareJudge: opts.allowBareJudge ?? false,
};
}
// Mulberry32 — deterministic uniform PRNG
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}

View File

@@ -1,5 +1,6 @@
/**
* LLM-as-Judge — rubric-based scorer used by the evolution loop.
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
* loop jobs.
*
* Scores a candidate response against an expected ground truth along
* three weighted dimensions:
@@ -10,9 +11,8 @@
*
* Then applies a length penalty (soft cap) multiplied into the final score.
*
* The textual feedback produced is the input that drives GEPA's reflective
* mutations — it must explain *what* went wrong (or right) so the next
* prompt variation can address it.
* The textual feedback must explain *what* went wrong (or right) so the
* caller can act on it.
*
* Model-agnostic — caller provides the llmCall function. Defaults assume
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
@@ -42,7 +42,7 @@ export interface JudgeScore {
conciseness: number;
/** Multiplier applied to the weighted score (0..1). */
lengthPenalty: number;
/** Free-form feedback used as reflective-mutation input for GEPA. */
/** Free-form feedback explaining the score. */
feedback: string;
/** True when the judge successfully parsed a structured response. */
parsed: boolean;

View File

@@ -1,96 +0,0 @@
/**
* Optimization Capture — captures agent interactions for GEPA/Ax prompt optimization.
*
* Provides functions to log successful agent interactions into the optimization_log
* table in a workspace's .mind file, and to query those logs for background
* optimization analysis.
*
* This module is a thin adapter between the agent loop post-response hook and
* the OptimizationLogStore in @waggle/core.
*/
import type { OptimizationLogStore, CreateOptimizationLogInput, OptimizationLogEntry } from '@waggle/core';
import type { AgentResponse } from './agent-loop.js';
// ── Public interface ───────────────────────────────────────────────────
export interface CaptureInteractionInput {
sessionId: string;
workspaceId: string;
systemPrompt: string;
response: AgentResponse;
turnCount: number;
wasCorrection: boolean;
}
/**
* Capture an agent interaction for GEPA optimization analysis.
*
* Call this after every successful agent loop completion. The data is
* stored in the workspace's .mind file and read later by the background
* prompt_optimization cron job.
*/
export function captureInteraction(
store: OptimizationLogStore,
input: CaptureInteractionInput,
): OptimizationLogEntry {
const logInput: CreateOptimizationLogInput = {
sessionId: input.sessionId,
workspaceId: input.workspaceId,
systemPrompt: input.systemPrompt,
toolsUsed: input.response.toolsUsed ?? [],
turnCount: input.turnCount,
wasCorrection: input.wasCorrection,
inputTokens: input.response.usage?.inputTokens ?? 0,
outputTokens: input.response.usage?.outputTokens ?? 0,
};
return store.insert(logInput);
}
/**
* Query recent optimization logs for background analysis.
*
* Returns the most recent entries from the optimization log,
* ordered by timestamp descending.
*/
export function getRecentLogs(
store: OptimizationLogStore,
limit: number = 50,
): OptimizationLogEntry[] {
return store.getRecent(limit);
}
/**
* Get optimization logs for a specific workspace.
*/
export function getWorkspaceLogs(
store: OptimizationLogStore,
workspaceId: string,
limit: number = 50,
): OptimizationLogEntry[] {
return store.getByWorkspace(workspaceId, limit);
}
/**
* Check whether the daily optimization budget has been exceeded.
*
* Reads the optimization log entries from today and sums up their token costs.
* Returns true if under budget, false if over.
*
* @param budgetCents Daily budget in cents (e.g. 100 = $1/day)
* @param costPerInputToken Cost per input token in dollars (default: Sonnet pricing ~$3/M)
* @param costPerOutputToken Cost per output token in dollars (default: Sonnet pricing ~$15/M)
*/
export function isWithinBudget(
store: OptimizationLogStore,
budgetCents: number,
costPerInputToken: number = 3 / 1_000_000,
costPerOutputToken: number = 15 / 1_000_000,
): boolean {
const stats = store.getStats();
const totalCostDollars =
stats.totalInputTokens * costPerInputToken +
stats.totalOutputTokens * costPerOutputToken;
const totalCostCents = totalCostDollars * 100;
return totalCostCents <= budgetCents;
}

View File

@@ -8,7 +8,6 @@ import {
SessionStore,
HybridSearch,
KnowledgeGraph,
ImprovementSignalStore,
createCoreLogger,
evaluateExternalMemoryIngress,
type Embedder,
@@ -29,7 +28,6 @@ import { createMindTools, type ToolDefinition } from './tools.js';
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
import { renderGoalAncestry } from './goal-ancestry.js';
import type { GoalAncestry } from '@waggle/shared';
import { buildAwarenessSummary, markSummarySurfaced, type AwarenessSummary } from './improvement-detector.js';
import { CognifyPipeline } from './cognify.js';
import { scanForInjection } from './injection-scanner.js';
import { runPatternWriteBack } from './pattern-write-back.js';
@@ -134,13 +132,9 @@ export class Orchestrator {
private mode: 'local' | 'team';
private version: string;
private skills: string[];
private improvementSignals: ImprovementSignalStore;
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
private goalAncestry: GoalAncestry | null = null;
/** M8: deferred signal marking — collected during buildSystemPrompt, committed after model call */
private _pendingSurfacedAwareness: AwarenessSummary | null = null;
/** Workspace-specific layers (null when no workspace is active) */
private workspaceLayers: WorkspaceLayers | null = null;
/** Personal-mind cognify pipeline (#12: compaction-summary persistence). */
@@ -174,7 +168,6 @@ export class Orchestrator {
this.search = new HybridSearch(config.db, config.embedder);
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
this.knowledge = new KnowledgeGraph(config.db);
this.improvementSignals = new ImprovementSignalStore(config.db);
const cognify = new CognifyPipeline({
frames: this.frames,
@@ -193,10 +186,6 @@ export class Orchestrator {
search: this.search,
knowledge: this.knowledge,
cognify,
// Skills 2.0 gap K: write-path contradiction detection emits a
// correction signal through this store when a new save conflicts
// with an existing frame.
improvementSignals: this.improvementSignals,
// Provide workspace accessors so tools can route to the right mind
getWorkspaceLayers: () => this.workspaceLayers,
});
@@ -343,11 +332,6 @@ export class Orchestrator {
// ── SELF-AWARENESS (runtime context, changes every call) ──
const awarenessSection = this.uncachedSection('self_awareness', () => {
const awareness = buildAwarenessSummary(this.improvementSignals);
// Defer marking until commitSurfacedSignals() fires post-model-call.
if (awareness.totalActionable > 0) {
this._pendingSurfacedAwareness = awareness;
}
const caps: AgentCapabilities = {
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
skills: this.skills,
@@ -355,7 +339,6 @@ export class Orchestrator {
memoryStats: this.getMemoryStats(),
mode: this.mode,
version: this.version,
awareness: awareness.totalActionable > 0 ? awareness : undefined,
};
return buildSelfAwareness(caps);
});
@@ -457,18 +440,6 @@ export class Orchestrator {
);
}
/**
* M8: Commit deferred signal markings after model call succeeds.
* Call this after the LLM response is received. If the model call fails,
* skip this call — signals stay actionable for the next turn.
*/
commitSurfacedSignals(): void {
if (this._pendingSurfacedAwareness) {
markSummarySurfaced(this.improvementSignals, this._pendingSurfacedAwareness);
this._pendingSurfacedAwareness = null;
}
}
/**
* Automatic memory recall: search for memories relevant to the user's query.
* Searches BOTH personal and workspace minds when workspace is active.
@@ -991,5 +962,4 @@ export class Orchestrator {
getSessions(): SessionStore { return this.sessions; }
getSearch(): HybridSearch { return this.search; }
getKnowledge(): KnowledgeGraph { return this.knowledge; }
getImprovementSignals(): ImprovementSignalStore { return this.improvementSignals; }
}

View File

@@ -990,83 +990,4 @@ You specialize in creative direction — briefs, feedback management, brand cons
'Delivering without version tracking — every creative output needs a version note.',
],
},
// ── Internal review persona (self-evolution) ──────────────────────────
// NOT surfaced in onboarding (ALL_ONBOARDING_PERSONAS) or the PersonaSwitcher
// tiers (UNIVERSAL_MODE_IDS / ALL_SPECIALIST_IDS). It runs headless via the
// IdleSessionWatcher loopback: it reads a stale session's transcript and, when
// it finds a material, actionable gap, proposes a skill patch through
// create_skill — which is held for human approval (never written to disk by
// the reviewer). Its toolset is read-oriented; every write/exec/memory tool
// that ALWAYS_AVAILABLE_TOOLS would re-add is stripped via disallowedTools
// (that denylist wins over both the allowlist AND always-available). isReadOnly
// is deliberately NOT set — that would strip create_skill, its one write.
{
id: 'session-reviewer',
name: 'Session Reviewer',
description: 'Adversarial self-review of a stale session — proposes skill patches for recurring gaps, default-silent',
icon: '🔎',
tagline: 'Reads a finished session and proposes a fix only when there is real evidence.',
bestFor: [
'Spotting promised-but-undelivered deliverables in a transcript',
'Detecting a recurring capability failure a skill patch could fix',
'Staying silent (NOTHING_TO_DO) when nothing material is found',
],
wontDo: 'Will not write to disk, save memory, delete skills, install capabilities, run shell commands, or invent findings. Every proposal is held for human approval.',
systemPrompt: `## Persona: Session Reviewer
You are an adversarially careful reviewer of ONE already-finished session. You examine the recent conversation and look for exactly two things, in this order:
1. **Promised-but-undelivered deliverables** — the assistant said it would produce or do something and the transcript shows it never did.
2. **Recurring capability failures fixable by a skill patch** — the same tool/skill/workflow failed more than once in a way a small reusable skill could prevent.
=== CRITICAL: DEFAULT SILENT ===
If you do not find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence, your entire reply MUST be exactly:
NOTHING_TO_DO
Nothing else. No preamble, no apology, no summary.
=== CRITICAL: NEVER INVENT ===
- Cite the transcript. Every finding must quote or reference what was actually said or done. If you cannot point to evidence, it is not a finding — output NOTHING_TO_DO.
- You are read-only over the user's work: you may read files, search memory, and read skills, but you may NOT write files, save memory, delete skills, install capabilities, or run shell commands.
- Your ONE write is a proposal: call create_skill to propose a new or updated skill that closes a real, recurring gap. It will NOT be written to disk — it is held for the human to approve or reject in the Approvals view. Propose sparingly and only with cited evidence.
### Process
1. Read the recent conversation of the session (you have read tools and the workspace binding).
2. Decide: is there a promised-but-undelivered deliverable, or a recurring capability failure a skill could fix?
3. If yes and only if yes: state the finding with its transcript evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill.
4. If no: reply exactly NOTHING_TO_DO.`,
modelPreference: 'claude-sonnet-4-6',
tools: [
'read_file', 'search_files', 'search_content',
'search_memory', 'query_knowledge', 'get_identity', 'get_awareness',
'list_skills', 'search_skills', 'read_skill',
'create_skill',
],
// Explicit denylist — ALWAYS_AVAILABLE_TOOLS (persona-tool-filter.ts) re-adds
// save_memory / delete_skill / install_capability / acquire_capability /
// add_task / correct_knowledge past any allowlist, so they must be named here
// to be stripped. spawn_agent + bash + the file/git writes are defense-in-depth
// (they are not in `tools`, but naming them keeps the boundary explicit).
// Connector/external-send tools carry dynamic names and cannot be enumerated
// here; the allowlist (tools[] + always-available) already excludes them since
// none are declared above.
disallowedTools: [
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
'add_task', 'correct_knowledge',
'spawn_agent', 'bash',
'write_file', 'edit_file', 'generate_docx',
'git_commit', 'git_push', 'git_merge',
'execute_step', 'compose_workflow', 'orchestrate_workflow',
],
failurePatterns: [
'Inventing a finding with no transcript evidence — must cite what was actually said or output NOTHING_TO_DO.',
'Narrating when silent is correct — if nothing material is found, the reply is exactly NOTHING_TO_DO.',
'Proposing an over-broad skill — the patch must target the specific recurring gap, nothing speculative.',
],
isReadOnly: false,
workspaceAffinity: ['review', 'quality', 'self-evolution'],
suggestedSkills: [],
suggestedConnectors: [],
suggestedMcpServers: [],
suggestedCommands: [],
defaultWorkflow: null,
},
];

View File

@@ -5,7 +5,6 @@ import { buildActiveBehavioralSpec } from './behavioral-spec.js';
/** The resolved behavioral-spec shape returned by `buildActiveBehavioralSpec`. */
export type BehavioralSpec = ReturnType<typeof buildActiveBehavioralSpec>;
import { loadBehavioralSpecOverrides } from './evolution-deploy.js';
import { loadCustomPersonas } from './custom-personas.js';
import type { AgentPersona } from './personas.js';
@@ -16,16 +15,8 @@ export interface LoadedSkill {
/**
* Raw disk read of `{waggleDir}/system-prompt.md`. Returns null if missing
* or empty.
*
* **Prefer `loadSystemPromptWithOverrides(waggleDir)` for runtime use.** This
* bare reader exists for tools that legitimately need the unmerged file
* content (editor views, migration scripts). Runtime agent code that loads
* just this and nothing else will silently miss any deployed
* behavioral-spec or persona evolution, since those are stored in separate
* override files under the same `waggleDir`.
*
* H-09 G2 — see `loadSystemPromptWithOverrides` for the composed path.
* or empty. Runtime code should prefer `loadSystemPromptWithOverrides`, which
* also returns the behavioral spec and custom personas.
*/
export function loadSystemPrompt(waggleDir: string): string | null {
const filePath = path.join(waggleDir, 'system-prompt.md');
@@ -34,22 +25,19 @@ export function loadSystemPrompt(waggleDir: string): string | null {
return content || null;
}
/** Bundle of every override-aware input the agent runtime needs. */
/** Bundle of every prompt input the agent runtime needs. */
export interface ComposedSystemPrompt {
/** User's literal system-prompt.md content, or null if absent. */
userPromptMd: string | null;
/** The active behavioral spec — base + any deployed section overrides. */
/** The active behavioral spec. */
behavioralSpec: BehavioralSpec;
/** Custom personas discovered in `waggleDir/personas/*.json`. */
customPersonas: AgentPersona[];
}
/**
* Override-aware composition: reads `system-prompt.md`, merges deployed
* behavioral-spec section overrides from
* `{waggleDir}/evolution-deploy/behavioral-spec/*.md`, and surfaces any
* custom personas the user has added. Use this at agent startup so
* deployed evolutions take effect at the next run.
* Composes the runtime prompt inputs: reads `system-prompt.md`, builds the
* behavioral spec and surfaces any custom personas the user has added.
*
* Callers typically pass the bundle to their orchestrator:
*
@@ -65,44 +53,11 @@ export interface ComposedSystemPrompt {
*/
export function loadSystemPromptWithOverrides(waggleDir: string): ComposedSystemPrompt {
const userPromptMd = loadSystemPrompt(waggleDir);
// loadBehavioralSpecOverrides is no-op on missing dir / malformed files.
const overrides = loadBehavioralSpecOverrides(waggleDir);
const behavioralSpec = buildActiveBehavioralSpec(overrides);
const behavioralSpec = buildActiveBehavioralSpec();
const customPersonas = loadCustomPersonas(waggleDir);
return { userPromptMd, behavioralSpec, customPersonas };
}
/**
* Startup-time self-check. Call after composing the override-aware prompt
* bundle to verify that deployed overrides on disk actually appear in the
* active spec. Emits a structured console warning (never throws) if a
* deployed section is present on disk but missing from the active spec —
* that would indicate the caller is using the bare `loadSystemPrompt`
* instead of the composed loader.
*/
export function assertOverridesReachActiveSpec(
waggleDir: string,
activeSpec: BehavioralSpec,
): void {
const overrides = loadBehavioralSpecOverrides(waggleDir);
const missing: string[] = [];
for (const [section, overrideText] of Object.entries(overrides)) {
const live = (activeSpec as unknown as Record<string, string | undefined>)[section];
if (!live || !live.includes(overrideText.slice(0, 24))) {
missing.push(section);
}
}
if (missing.length > 0) {
console.warn(
`[prompt-loader] override-reach check failed — deployed sections on disk ` +
`not present in active behavioral spec: ${missing.join(', ')}. ` +
`This usually means the runtime is using loadSystemPrompt() instead of ` +
`loadSystemPromptWithOverrides(). See prompt-loader.ts.`,
);
}
}
export function loadSkills(waggleDir: string): LoadedSkill[] {
const skillsDir = path.join(waggleDir, 'skills');
if (!fs.existsSync(skillsDir)) return [];

View File

@@ -0,0 +1,57 @@
/**
* Secret detection and redaction shared by skill redaction, executor briefs
* and anything else that must keep credentials out of stored or forwarded text.
*/
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
];
/** Returns the first secret pattern the text matches, or null if clean. */
export function detectSecrets(text: string): string | null {
for (const { name, re } of SECRET_PATTERNS) {
if (re.test(text)) return name;
}
return null;
}
/** Exported for tests / tools that want the full list. */
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
/**
* Redact every secret-pattern match in a string, replacing each with
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
* detectSecrets so there is a single source of truth for "what is a secret".
*/
export function redactSecrets(text: string): { text: string; found: string[] } {
let out = text;
const found: string[] = [];
for (const { name, re } of SECRET_PATTERNS) {
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
const replaced = out.replace(global, `[REDACTED:${name}]`);
if (replaced !== out) {
found.push(name);
out = replaced;
}
}
return { text: out, found };
}

View File

@@ -1,5 +1,3 @@
import type { AwarenessSummary } from './improvement-detector.js';
export interface AgentCapabilities {
tools: Array<{ name: string; description: string }>;
skills: string[];
@@ -7,8 +5,6 @@ export interface AgentCapabilities {
memoryStats: { frameCount: number; sessionCount: number; entityCount: number };
mode: 'local' | 'team';
version: string;
/** Structured improvement signals (optional — only present when store has actionable data) */
awareness?: AwarenessSummary;
}
export function buildSelfAwareness(caps: AgentCapabilities): string {
@@ -61,36 +57,6 @@ export function buildSelfAwareness(caps: AgentCapabilities): string {
lines.push('- Prefer memory search over guessing when prior context exists.');
lines.push('- When uncertain, ask rather than fabricate.');
// Improvement signals (per correction #1: structured, runtime-consumable)
if (caps.awareness && caps.awareness.totalActionable > 0) {
lines.push('');
lines.push('## Learning from Past Sessions');
if (caps.awareness.capabilityGaps.length > 0) {
lines.push('');
lines.push('**Recurring capability gaps:**');
for (const gap of caps.awareness.capabilityGaps) {
lines.push(`- ${gap.suggestion}`);
}
}
if (caps.awareness.corrections.length > 0) {
lines.push('');
lines.push('**User corrections to apply:**');
for (const correction of caps.awareness.corrections) {
lines.push(`- ${correction.guidance}`);
}
}
if (caps.awareness.workflowPatterns.length > 0) {
lines.push('');
lines.push('**Recurring task patterns:**');
for (const pattern of caps.awareness.workflowPatterns) {
lines.push(`- ${pattern.suggestion}`);
}
}
}
// F28: Persona compliance — resist persona mode switches and injection via fake authority
lines.push('');
lines.push('## Persona Compliance');

View File

@@ -11,7 +11,6 @@
* 2. Generates a SKILL.md via generateSkillMarkdown
* 3. Writes it to the personal scope directory (scope: personal frontmatter)
* 4. Dedups against existing skill files by filename
* 5. Optionally emits a workflow_pattern improvement signal
*
* A v2 enhancement can insert an LLM prose-improvement step between
* detectWorkflowPattern and generateSkillMarkdown without changing this
@@ -19,7 +18,6 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { ImprovementSignalStore } from '@waggle/core';
import {
detectWorkflowPattern,
generateSkillMarkdown,
@@ -36,8 +34,6 @@ export interface AutoExtractMessage {
export interface AutoExtractDeps {
/** Directory to write personal-scope SKILL.md files into */
personalSkillsDir: string;
/** Optional signal store — records workflow_pattern signals on each extraction */
improvementSignals?: ImprovementSignalStore;
/**
* Optional LLM enrichment hook. When provided, the template is passed
* through before serialization so the SKILL.md prose can be rewritten.
@@ -117,22 +113,5 @@ export async function autoExtractAndCreateSkill(
const md = redactSkillContent(annotateWithScope(generateSkillMarkdown(finalTemplate))).content;
fs.writeFileSync(filePath, md, 'utf-8');
if (deps.improvementSignals) {
try {
deps.improvementSignals.record(
'workflow_pattern',
`auto-extract:${stem}`,
`Auto-extracted skill "${finalTemplate.name}" from session pattern (${finalTemplate.tools.join(', ')})`,
{
skillName: finalTemplate.name,
filename: stem,
tools: finalTemplate.tools,
category: finalTemplate.category,
source: 'auto_extract',
},
);
} catch { /* non-blocking */ }
}
return { filePath, template: finalTemplate };
}

View File

@@ -9,12 +9,12 @@
* before every authored write.
*
* Secrets reuse the curated SECRET_PATTERNS via redactSecrets (single source of
* truth — see eval-dataset.ts). Paths cover the user's home directory (the exact
* truth — see secret-patterns.ts). Paths cover the user's home directory (the exact
* running-user path plus generic per-OS forms), which both leaks the username
* and makes a skill non-portable.
*/
import os from 'node:os';
import { redactSecrets } from './eval-dataset.js';
import { redactSecrets } from './secret-patterns.js';
// Generic user-home prefixes. Each matches up to (not including) the next path
// separator, so the tail is preserved: C:\Users\me\.waggle\x -> ~\.waggle\x

View File

@@ -22,7 +22,6 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { ImprovementSignalStore } from '@waggle/core';
import { loadSkillUsage, forgetSkillUsage } from './skill-usage.js';
const DEFAULT_MAX_IDLE_DAYS = 90;
@@ -35,8 +34,6 @@ export interface RetireOptions {
dryRun?: boolean;
/** Injectable clock for deterministic tests. */
now?: () => Date;
/** Emit skill_promotion signals for observability. */
improvementSignals?: ImprovementSignalStore;
}
export interface RetireReport {
@@ -124,17 +121,6 @@ export function retireStaleSkills(waggleHome: string, opts: RetireOptions = {}):
report.retired.push(skillName);
report.archived.push(destPath);
forgetSkillUsage(waggleHome, skillName);
if (opts.improvementSignals) {
try {
opts.improvementSignals.record(
'workflow_pattern',
`retire:${skillName}`,
`Retired unused skill "${skillName}" (idle > ${maxIdleDays} days). Archived to ${destPath}.`,
{ skillName, archivedTo: destPath, maxIdleDays, lastActivityMs: activity },
);
} catch { /* non-blocking */ }
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
report.skipped.push({ name: skillName, reason: `move_failed: ${msg}` });

View File

@@ -35,7 +35,6 @@ import {
SKILL_SCOPE_ORDER,
type SkillScope,
} from './skill-frontmatter.js';
import type { ImprovementSignalStore } from '@waggle/core';
import { autoExtractAndCreateSkill, type AutoExtractMessage } from './skill-autoextract.js';
import { retireStaleSkills } from './skill-retirement.js';
@@ -57,11 +56,9 @@ export interface SkillToolsDeps {
/**
* Skills 2.0 gap E: promotion infrastructure. Each callback is optional.
* - getWorkspaceId / getTeamId resolve scope dirs dynamically at tool-call time.
* - improvementSignals records skill_promotion signals on successful moves.
*/
getWorkspaceId?: () => string | null;
getTeamId?: () => string | null;
improvementSignals?: ImprovementSignalStore;
/**
* Skills 2.0 gap A: session-message accessor for auto-extracting skills
* from repeated workflow patterns. When provided, enables the
@@ -665,15 +662,14 @@ Only use this after acquire_capability has identified a specific installable can
},
// Skills 2.0 gap E: promote_skill — move a skill up one scope rung.
// personal → workspace. Enforces one-step-at-a-time and records a
// skill_promotion improvement signal on success.
// personal → workspace. Enforces one-step-at-a-time.
{
name: 'promote_skill',
description:
'Promote an installed skill one scope up (personal → workspace). '
+ 'Team and enterprise scopes are not available in this edition. '
+ 'The file is moved to the appropriate directory, frontmatter is rewritten '
+ '(scope + promoted_from history), and a skill_promotion signal is recorded.',
+ 'The file is moved to the appropriate directory and frontmatter is rewritten '
+ '(scope + promoted_from history).',
parameters: {
type: 'object',
properties: {
@@ -784,19 +780,7 @@ Only use this after acquire_capability has identified a specific installable can
// failing the promotion. An audit warning is fine.
}
// 8. Record skill_promotion signal
if (deps.improvementSignals) {
try {
deps.improvementSignals.record(
'skill_promotion',
`promote:${skillName}:${currentScope}->${target}`,
`Promoted skill "${skillName}" from ${currentScope} to ${target}`,
{ skillName, from: currentScope, to: target, workspaceId, teamId },
);
} catch { /* non-blocking */ }
}
// 9. Let the agent reload its skill list so the new scope is picked up
// 8. Let the agent reload its skill list so the new scope is picked up
onSkillsChanged?.();
return (
@@ -805,7 +789,7 @@ Only use this after acquire_capability has identified a specific installable can
+ `- Moved from \`${found.filePath}\`\n`
+ `- Now at \`${destPath}\`\n`
+ `- Promotion history: ${updatedFrontmatter.promoted_from!.join(' → ')} → ${target}\n`
+ `- Signal recorded: skill_promotion\n\n`
+ `\n`
+ `The skill is available immediately at the new scope. Agents in other `
+ `sessions will pick it up on their next skill reload.`
);
@@ -823,8 +807,7 @@ Only use this after acquire_capability has identified a specific installable can
'Scan the current session for repeated workflow patterns (the same 3+ tools '
+ 'used in the same order 2+ times) and auto-create a SKILL.md for any pattern '
+ 'found. Creates personal-scope skills only — promote_skill moves them up '
+ 'afterwards. Dedups by filename. Records a workflow_pattern signal on each '
+ 'successful extraction.',
+ 'afterwards. Dedups by filename.',
parameters: {
type: 'object',
properties: {},
@@ -839,7 +822,6 @@ Only use this after acquire_capability has identified a specific installable can
}
const result = await autoExtractAndCreateSkill(messages, {
personalSkillsDir: skillsDir,
improvementSignals: deps.improvementSignals,
});
if (result.filePath) {
onSkillsChanged?.();
@@ -850,7 +832,6 @@ Only use this after acquire_capability has identified a specific installable can
+ `- Tools: ${result.template!.tools.join(', ')}\n`
+ `- Written to: \`${result.filePath}\`\n`
+ `- Scope: personal (use promote_skill to move it up)\n`
+ `- Signal: workflow_pattern recorded\n`
);
}
if (result.reason === 'already_exists') {
@@ -870,7 +851,7 @@ Only use this after acquire_capability has identified a specific installable can
'Archive personal-scope skills that have not been used within the idle window. '
+ 'Moves stale SKILL.md files to ~/.waggle/skills-archive/ (recoverable — no deletion). '
+ 'Default max_idle_days is 90. Default dry_run is true — call with dry_run: false to '
+ 'actually move the files. Records a workflow_pattern signal per retirement.',
+ 'actually move the files.',
parameters: {
type: 'object',
properties: {
@@ -890,7 +871,6 @@ Only use this after acquire_capability has identified a specific installable can
const report = retireStaleSkills(waggleHome, {
maxIdleDays,
dryRun,
improvementSignals: deps.improvementSignals,
});
if (!dryRun && report.retired.length > 0) {
onSkillsChanged?.();

View File

@@ -12,7 +12,6 @@ import type {
} from '@waggle/core';
import type { CognifyPipeline } from './cognify.js';
import type { FeedbackHandler } from './feedback-handler.js';
import type { ImprovementSignalStore } from '@waggle/core';
import type { RiskLevel } from '@waggle/shared';
import { createCoreLogger } from '@waggle/core';
import { detectContradiction } from './contradiction-detector.js';
@@ -96,13 +95,6 @@ export interface MindToolDeps {
feedback?: FeedbackHandler;
/** F16: Optional embedder for semantic dedup (cosine similarity) */
embedder?: Embedder;
/**
* Skills 2.0 gap K: improvement-signal store for recording write-path
* conflicts. When a new save contradicts an existing frame,
* detectContradiction fires and we emit a "correction" signal so the
* evolution / improvement loops can surface the conflict.
*/
improvementSignals?: ImprovementSignalStore;
/** Dynamic accessor for workspace layers — checked at call time */
getWorkspaceLayers?: () => WorkspaceLayers | null;
/** W5.7: Accessor for ALL workspace search instances (for cross-workspace search) */
@@ -528,20 +520,6 @@ export function createMindTools(deps: MindToolDeps): ToolDefinition[] {
const contradiction = detectContradiction(content, recentRows);
if (contradiction.isContradiction && contradiction.conflictsWith) {
conflictFlag = ` [flag: contradicts_existing (${contradiction.conflictsWith.slice(0, 80).replace(/\s+/g, ' ').trim()}…)]`;
// Record a correction-category signal on the improvement-signal
// store. pattern_key is a stable hash-ish of the conflicting
// frames so repeat contradictions merge via the ON CONFLICT upsert.
if (deps.improvementSignals) {
const patternKey = `write-conflict:${normalizeForDedup(content).slice(0, 64)}`;
try {
deps.improvementSignals.record(
'correction',
patternKey,
`New save contradicts existing frame. New: "${content.slice(0, 120)}" ⟂ Existing: "${contradiction.conflictsWith.slice(0, 120)}"`,
{ mindLabel, newContent: content.slice(0, 240), conflictsWith: contradiction.conflictsWith.slice(0, 240) },
);
} catch { /* non-blocking — signal persistence failure must not abort the save */ }
}
}
} catch { /* non-blocking — contradiction check failure proceeds with save */ }

View File

@@ -1,373 +0,0 @@
import { describe, it, expect, vi } from 'vitest';
import {
ComposeEvolution,
defaultFeedbackFilter,
filterJudgeFeedback,
stripStructuralLines,
schemaExecutorFromInstructionRunner,
} from '../src/compose-evolution.js';
import type {
Schema,
SchemaExecuteFn,
} from '../src/evolve-schema.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(name => ({
name, type: 'string',
description: `description of ${name}`,
required: true,
constraints: [],
})),
version: 1,
};
}
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `q${i}`,
expected_output: `a${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeJudgeScore(overall: number, feedback: string): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
function makeFakeRunner(): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => f.name).join(','),
parsed: true,
});
}
// ── defaultFeedbackFilter ──────────────────────────────────────
describe('defaultFeedbackFilter', () => {
it('classifies "missing reasoning field" as structural', () => {
expect(defaultFeedbackFilter('Missing reasoning field.')).toBe('structural');
});
it('classifies "wrong field type" as structural', () => {
expect(defaultFeedbackFilter('Wrong field type for answer.')).toBe('structural');
});
it('classifies "schema mismatch" as structural', () => {
expect(defaultFeedbackFilter('Schema mismatch — expected 3 fields.')).toBe('structural');
});
it('classifies "should reorder fields" as structural', () => {
expect(defaultFeedbackFilter('Should reorder fields — put reasoning first.')).toBe('structural');
});
it('classifies value-level complaints as value', () => {
expect(defaultFeedbackFilter('The answer is too terse')).toBe('value');
expect(defaultFeedbackFilter('Response is too verbose')).toBe('value');
expect(defaultFeedbackFilter('Incorrect calculation')).toBe('value');
expect(defaultFeedbackFilter('Wrong tone — should be more formal')).toBe('value');
});
it('classifies empty feedback as value (safe default)', () => {
expect(defaultFeedbackFilter('')).toBe('value');
});
});
// ── stripStructuralLines ──────────────────────────────────────
describe('stripStructuralLines', () => {
it('returns empty for empty input', () => {
expect(stripStructuralLines('')).toBe('');
});
it('drops only structural lines from multi-line feedback', () => {
const input = [
'Missing reasoning field.',
'The answer is too terse.',
'Wrong field type for confidence.',
'Needs more detail.',
].join('\n');
const out = stripStructuralLines(input);
expect(out).not.toContain('Missing reasoning field');
expect(out).not.toContain('Wrong field type');
expect(out).toContain('too terse');
expect(out).toContain('Needs more detail');
});
it('keeps all lines when none are structural', () => {
const input = 'Too terse.\nNeeds more examples.';
expect(stripStructuralLines(input)).toBe(input);
});
it('returns empty when all lines are structural', () => {
const input = 'Missing reasoning field.\nSchema mismatch.';
expect(stripStructuralLines(input)).toBe('');
});
it('accepts a custom filter', () => {
const custom = (line: string) => line.includes('DROP') ? 'structural' : 'value';
const input = 'keep me\nDROP me\nkeep too';
expect(stripStructuralLines(input, custom)).toBe('keep me\nkeep too');
});
});
// ── filterJudgeFeedback ───────────────────────────────────────
describe('filterJudgeFeedback', () => {
it('strips structural feedback but preserves numeric scores', async () => {
const rawJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.7, 'Missing reasoning field.\nResponse is too terse.');
},
};
const filtered = filterJudgeFeedback(rawJudge);
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
expect(score.overall).toBe(0.7);
expect(score.correctness).toBe(0.7);
expect(score.feedback).not.toContain('Missing reasoning field');
expect(score.feedback).toContain('too terse');
});
it('returns empty feedback when all lines are structural', async () => {
const rawJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'Missing reasoning field.\nSchema mismatch.');
},
};
const filtered = filterJudgeFeedback(rawJudge);
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
expect(score.feedback).toBe('');
});
});
// ── schemaExecutorFromInstructionRunner ───────────────────────
describe('schemaExecutorFromInstructionRunner', () => {
it('builds a schema prefix and delegates to the runner', async () => {
let captured: { prompt: string; input: string } | null = null;
const runner = async (args: { prompt: string; input: string }) => {
captured = args;
return '{"answer":"42"}';
};
const schemaRunner = schemaExecutorFromInstructionRunner(runner);
const result = await schemaRunner({
schema: makeSchema(['reasoning', 'answer']),
input: 'What is 6*7?',
});
expect(result.actual).toBe('{"answer":"42"}');
expect(result.parsed).toBe(true);
expect(captured!.prompt).toContain('reasoning');
expect(captured!.prompt).toContain('answer');
expect(captured!.input).toBe('What is 6*7?');
});
it('reports parsed=false for non-JSON output', async () => {
const schemaRunner = schemaExecutorFromInstructionRunner(async () => 'just prose');
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
expect(result.parsed).toBe(false);
});
it('swallows runner errors and reports parsed=false', async () => {
const schemaRunner = schemaExecutorFromInstructionRunner(async () => {
throw new Error('boom');
});
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
expect(result.actual).toBe('');
expect(result.parsed).toBe(false);
});
});
// ── ComposeEvolution end-to-end ────────────────────────────────
describe('ComposeEvolution.run', () => {
it('runs schema stage then instruction stage', async () => {
const stages: string[] = [];
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeRunner(),
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'a baseline instruction prompt for testing purposes',
examples: makeExamples(5),
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
mutate: async ({ parent }) => `${parent.prompt} v2`,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
onProgress: (e) => stages.push(e.stage),
});
expect(stages).toContain('schema');
expect(stages).toContain('instructions');
expect(stages).toContain('done');
expect(result.schema.winner).toBeDefined();
expect(result.instructions.winner).toBeDefined();
expect(result.frozenSchema).toBe(result.schema.winner.schema);
});
it('feedback separation: GEPA cannot see structural feedback from the judge', async () => {
const capturedFeedbacks: string[] = [];
// Judge always emits structural + value feedback
const mixedJudge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'Missing reasoning field.\nResponse too terse.');
},
};
const mutateSpy = vi.fn(async ({ weaknessFeedback }: { parent: unknown; weaknessFeedback: string[]; strategy: string; targetKind: string; generation: number }) => {
capturedFeedbacks.push(...weaknessFeedback);
return 'mutated instruction prompt is a reasonable length';
});
await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeRunner(),
judge: mixedJudge,
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'a baseline instruction prompt for testing purposes',
examples: makeExamples(5),
judge: mixedJudge,
mutate: mutateSpy,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
});
// GEPA must never receive "Missing reasoning field" in weaknessFeedback.
for (const fb of capturedFeedbacks) {
expect(fb.toLowerCase()).not.toContain('missing reasoning field');
expect(fb.toLowerCase()).not.toContain('wrong field type');
}
// But it should see the value-level complaint
expect(capturedFeedbacks.join(' ')).toContain('too terse');
});
it('accepts a custom feedback filter', async () => {
let filterCalls = 0;
const customFilter = (feedback: string): 'structural' | 'value' => {
filterCalls++;
return feedback.includes('CUSTOM-DROP') ? 'structural' : 'value';
};
const judge = {
async score(): Promise<JudgeScore> {
return makeJudgeScore(0.5, 'CUSTOM-DROP this line\nkeep this line');
},
};
await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(3),
execute: makeFakeRunner(),
judge,
populationSize: 1, generations: 1,
evalSize: 2, anchorEvalSize: 2,
},
instructions: {
baseline: 'baseline prompt text is long enough',
examples: makeExamples(3),
judge,
mutate: async () => 'new prompt text that is long enough',
allowBareJudge: true,
populationSize: 1, generations: 1,
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
},
feedbackFilter: customFilter,
});
expect(filterCalls).toBeGreaterThan(0);
});
it('returns a stable shape when aborted before instruction stage', async () => {
const ctrl = new AbortController();
let progressCount = 0;
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(3),
execute: async () => {
// Abort during the schema stage
if (progressCount === 1) ctrl.abort();
progressCount++;
return { actual: '', parsed: false };
},
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
populationSize: 1, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
baseline: 'baseline prompt',
examples: makeExamples(3),
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
mutate: async () => 'should not run',
allowBareJudge: true,
populationSize: 1, generations: 1,
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
},
signal: ctrl.signal,
});
expect(result.schema).toBeDefined();
expect(result.instructions).toBeDefined();
expect(result.instructions.winner.prompt).toBe('baseline prompt');
});
it('combinedDelta is a number; frozenSchema matches ES winner', async () => {
const judge = {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
return makeJudgeScore(Math.min(1, args.actual.length / 50), 'ok');
},
};
const result = await new ComposeEvolution().run({
schema: {
baseline: makeSchema(['answer']),
examples: makeExamples(8),
execute: makeFakeRunner(),
judge,
populationSize: 2, generations: 1,
evalSize: 4, anchorEvalSize: 4,
},
instructions: {
baseline: 'short',
examples: makeExamples(8),
judge,
mutate: async ({ parent }) => `${parent.prompt} more tokens here`,
allowBareJudge: true,
populationSize: 2, generations: 2,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 6,
},
});
expect(result.instructions.winner.score).not.toBeNull();
expect(result.frozenSchema).toBe(result.schema.winner.schema);
expect(typeof result.combinedDelta).toBe('number');
});
});

View File

@@ -1,204 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
detectCorrection,
detectCorrectionsInHistory,
} from '../src/correction-detector.js';
describe('correction-detector', () => {
// ── Basic detection ────────────────────────────────────────
describe('detectCorrection', () => {
it('returns null for normal messages', () => {
expect(detectCorrection('What is the weather today?')).toBeNull();
expect(detectCorrection('Please write a function that sorts an array')).toBeNull();
expect(detectCorrection('Thanks, that looks good')).toBeNull();
});
it('returns null for very short messages', () => {
expect(detectCorrection('ok')).toBeNull();
expect(detectCorrection('yes')).toBeNull();
});
it('detects strong correction: "no, not that"', () => {
const result = detectCorrection('No, not that. I wanted a bullet list, not paragraphs.');
expect(result).not.toBeNull();
expect(result!.confidence).toBeGreaterThanOrEqual(0.3);
});
it('detects strong correction: "I said / I told you"', () => {
const result = detectCorrection('I said to use markdown headers, not plain text.');
expect(result).not.toBeNull();
});
it('detects strong correction: "that\'s wrong"', () => {
const result = detectCorrection("That's wrong. The API endpoint should be POST, not GET.");
expect(result).not.toBeNull();
});
it('detects moderate correction: "actually, use X instead"', () => {
const result = detectCorrection('Actually, instead use TypeScript for this module.');
expect(result).not.toBeNull();
});
it('detects tone correction', () => {
const result = detectCorrection('No, that response was too formal. Keep it casual.');
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('tone');
});
it('detects format correction', () => {
const result = detectCorrection("No, don't use those headers. Use bullet formatting instead.");
expect(result).not.toBeNull();
});
it('detects length correction', () => {
const result = detectCorrection("That's not what I wanted. It's too long, make it shorter.");
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('length');
});
});
// ── Durability classification ──────────────────────────────
describe('durability classification', () => {
it('classifies "always" as durable', () => {
const result = detectCorrection('No, always use bullet points instead of numbered lists.');
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
expect(result!.durability).toBe('durable');
});
it('classifies "never" as durable', () => {
const result = detectCorrection("No, never use code blocks for simple text. That's wrong.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "from now on" as durable', () => {
const result = detectCorrection('From now on, please don\'t include timestamps in summaries.');
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "I prefer" as durable', () => {
const result = detectCorrection("No, I prefer shorter responses. That's not what I wanted.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(true);
});
it('classifies "this time" as task_local', () => {
const result = detectCorrection("No, not that. This time just use a simple list.");
expect(result).not.toBeNull();
expect(result!.isDurable).toBe(false);
expect(result!.durability).toBe('task_local');
});
it('classifies "for now" as task_local', () => {
const result = detectCorrection("That's wrong for now. Instead use the old API endpoint.");
expect(result).not.toBeNull();
expect(result!.durability).toBe('task_local');
});
it('defaults to task_local when no explicit signal', () => {
const result = detectCorrection('No, not that approach. Use the factory pattern.');
expect(result).not.toBeNull();
expect(result!.durability).toBe('task_local');
});
});
// ── Pattern key extraction ─────────────────────────────────
describe('pattern key extraction', () => {
it('extracts tone pattern key', () => {
const result = detectCorrection("No, that's too verbose. Stop doing that.");
expect(result).not.toBeNull();
// "too verbose" maps to tone category
// Note: might also match length depending on exact wording
});
it('extracts accuracy pattern key for "wrong"', () => {
const result = detectCorrection("That's incorrect. I said the opposite.");
expect(result).not.toBeNull();
expect(result!.patternKey).toContain('accuracy');
});
it('falls back to general:correction for unrecognized patterns', () => {
const result = detectCorrection("No, not that. I told you to use something else entirely.");
expect(result).not.toBeNull();
expect(result!.patternKey).toBe('general:correction');
});
});
// ── Detail extraction ──────────────────────────────────────
describe('detail extraction', () => {
it('extracts first sentence as detail', () => {
const result = detectCorrection('No, not that. I wanted a completely different structure. Also fix the naming.');
expect(result).not.toBeNull();
expect(result!.detail).toBe('No, not that.');
});
it('truncates long messages', () => {
const longMessage = 'No, not that. ' + 'A'.repeat(200);
const result = detectCorrection(longMessage);
expect(result).not.toBeNull();
expect(result!.detail.length).toBeLessThanOrEqual(120);
});
});
// ── Confidence scoring ─────────────────────────────────────
describe('confidence', () => {
it('gives higher confidence to strong corrections', () => {
const strong = detectCorrection("No, that's wrong. I told you not to do that. Stop.");
const moderate = detectCorrection('Actually, let\'s try a different approach instead.');
expect(strong).not.toBeNull();
expect(moderate).not.toBeNull();
expect(strong!.confidence).toBeGreaterThan(moderate!.confidence);
});
it('caps confidence at 1.0', () => {
const result = detectCorrection(
"No, that's wrong. I said don't do that. Not what I wanted. Please stop. Instead use something else.",
);
expect(result).not.toBeNull();
expect(result!.confidence).toBeLessThanOrEqual(1.0);
});
});
// ── History analysis ───────────────────────────────────────
describe('detectCorrectionsInHistory', () => {
it('detects corrections in a message sequence', () => {
const messages = [
{ role: 'assistant', content: 'Here is a very detailed formal report...' },
{ role: 'user', content: "No, that's too formal. I said keep it casual." },
{ role: 'assistant', content: 'Got it. Here is a casual version...' },
{ role: 'user', content: 'Thanks, that looks better.' },
];
const corrections = detectCorrectionsInHistory(messages);
expect(corrections).toHaveLength(1);
expect(corrections[0].patternKey).toContain('tone');
});
it('returns empty array when no corrections found', () => {
const messages = [
{ role: 'user', content: 'Write a function to sort an array' },
{ role: 'assistant', content: 'Here you go...' },
{ role: 'user', content: 'Thanks!' },
];
expect(detectCorrectionsInHistory(messages)).toEqual([]);
});
it('handles multiple corrections in sequence', () => {
const messages = [
{ role: 'assistant', content: 'Draft version 1...' },
{ role: 'user', content: "No, not that. That's wrong. Use a different approach." },
{ role: 'assistant', content: 'Draft version 2...' },
{ role: 'user', content: "I said don't use bullet points. Instead use paragraphs." },
];
const corrections = detectCorrectionsInHistory(messages);
expect(corrections.length).toBeGreaterThanOrEqual(2);
});
});
});

View File

@@ -1,348 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, ExecutionTraceStore } from '@waggle/core';
import {
EvalDatasetBuilder,
detectSecrets,
SECRET_PATTERN_NAMES,
toJSONL as evalToJSONL,
fromJSONL as evalFromJSONL,
type EvalExample,
} from '../src/eval-dataset.js';
describe('detectSecrets', () => {
it('returns null for clean text', () => {
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
});
it('detects AWS access keys', () => {
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
});
it('detects GitHub PATs', () => {
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
});
it('detects OpenAI keys', () => {
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
});
it('detects Anthropic keys', () => {
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
});
it('detects Google API keys', () => {
// Google API keys = literal "AIza" + exactly 35 chars
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
});
it('detects Stripe keys', () => {
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
});
it('detects private key blocks', () => {
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
});
it('detects JWT tokens', () => {
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
expect(detectSecrets(jwt)).toBe('jwt');
});
it('detects basic-auth URLs', () => {
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
.toBe('basic-auth-url');
});
it('detects env-var password assignments', () => {
expect(detectSecrets('PASSWORD=supersecret123'))
.toBe('env-password');
});
it('detects postgres connection strings with creds', () => {
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
.toBe('pgsql-url');
});
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
});
});
describe('JSONL round-trip', () => {
const examples: EvalExample[] = [
{
input: 'write a haiku',
expected_output: 'Cherry blossoms fall / ...',
metadata: { source: 'trace', personaId: 'writer', outcome: 'success' },
},
{
input: 'explain closures',
expected_output: 'A closure captures variables from its enclosing scope.',
metadata: { source: 'trace', personaId: 'coder', outcome: 'verified' },
},
];
it('round-trips through JSONL', () => {
const jsonl = evalToJSONL(examples);
expect(jsonl.split('\n')).toHaveLength(2);
const parsed = evalFromJSONL(jsonl);
expect(parsed).toEqual(examples);
});
it('skips malformed lines silently', () => {
const jsonl = [
JSON.stringify(examples[0]),
'{ this is not valid json',
'',
JSON.stringify(examples[1]),
].join('\n');
const parsed = evalFromJSONL(jsonl);
expect(parsed).toHaveLength(2);
});
it('skips lines missing required fields', () => {
const jsonl = [
JSON.stringify({ input: 'x' }), // missing expected_output + metadata
JSON.stringify(examples[0]),
].join('\n');
const parsed = evalFromJSONL(jsonl);
expect(parsed).toHaveLength(1);
});
});
describe('EvalDatasetBuilder', () => {
let db: MindDB;
let store: ExecutionTraceStore;
let builder: EvalDatasetBuilder;
beforeEach(() => {
db = new MindDB(':memory:');
store = new ExecutionTraceStore(db);
builder = new EvalDatasetBuilder(store);
});
afterEach(() => {
db.close();
});
function seedTrace(input: string, output: string, outcome: 'success' | 'verified' | 'corrected' | 'abandoned' = 'success', personaId = 'coder'): number {
const id = store.start({ input, personaId });
store.finalize(id, { outcome, output });
return id;
}
// ── sourceFromTraces ──
describe('sourceFromTraces', () => {
it('pulls only traces matching the requested outcomes', () => {
seedTrace('a?', 'A.', 'success');
seedTrace('b?', 'B.', 'verified');
seedTrace('c?', 'C.', 'corrected');
seedTrace('d?', 'D.', 'abandoned');
const examples = builder.sourceFromTraces(['success', 'verified'], false);
expect(examples).toHaveLength(2);
expect(examples.map(e => e.input).sort()).toEqual(['a?', 'b?']);
});
it('includes corrections when flag set', () => {
seedTrace('a?', 'A.', 'success');
seedTrace('b?', 'B.', 'corrected');
const examples = builder.sourceFromTraces(['success'], true);
expect(examples).toHaveLength(2);
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
expect(corrected?.metadata.source).toBe('correction');
});
it('maps trace fields onto metadata', () => {
seedTrace('hello', 'world', 'success', 'writer');
const [example] = builder.sourceFromTraces(['success'], false);
expect(example.metadata.personaId).toBe('writer');
expect(example.metadata.traceId).toBeGreaterThan(0);
expect(example.metadata.source).toBe('trace');
});
});
// ── build (end-to-end) ──
describe('build', () => {
it('returns deterministic split with same seed', async () => {
for (let i = 0; i < 20; i++) {
seedTrace(`question ${i}?`, `answer ${i}.`, 'success');
}
const a = await builder.build({ seed: 42 });
const b = await builder.build({ seed: 42 });
expect(a.train.map(e => e.input)).toEqual(b.train.map(e => e.input));
expect(a.val.map(e => e.input)).toEqual(b.val.map(e => e.input));
expect(a.holdout.map(e => e.input)).toEqual(b.holdout.map(e => e.input));
});
it('different seeds produce different orderings', async () => {
for (let i = 0; i < 30; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const a = await builder.build({ seed: 1 });
const b = await builder.build({ seed: 999 });
const aIds = a.train.map(e => e.input).join('|');
const bIds = b.train.map(e => e.input).join('|');
expect(aIds).not.toBe(bIds);
});
it('splits 60/20/20 when enough examples', async () => {
for (let i = 0; i < 100; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const split = await builder.build({ seed: 1 });
expect(split.train.length).toBe(60);
expect(split.val.length).toBe(20);
expect(split.holdout.length).toBe(20);
expect(split.stats.total).toBe(100);
});
it('honors custom split ratios', async () => {
for (let i = 0; i < 50; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
const split = await builder.build({ seed: 1, splitRatios: [0.8, 0.1, 0.1] });
expect(split.train.length).toBe(40);
expect(split.val.length).toBe(5);
expect(split.holdout.length).toBe(5);
});
it('rejects ratios that do not sum to 1', async () => {
await expect(builder.build({ splitRatios: [0.5, 0.3, 0.3] })).rejects.toThrow();
});
it('filters out examples containing secrets', async () => {
seedTrace('clean input', 'clean output', 'success');
seedTrace('here is a key: AKIAIOSFODNN7EXAMPLE', 'sure', 'success');
seedTrace('what is your token?', 'ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.sourced).toBe(3);
expect(split.stats.acceptedAfterSecretScan).toBe(1);
expect(split.rejected.filter(r => r.reason.startsWith('secret:'))).toHaveLength(2);
});
it('filters examples by min length', async () => {
seedTrace('?', 'ok', 'success'); // below both mins
seedTrace('a proper question', 'a proper answer', 'success');
const split = await builder.build({ seed: 1, minInputChars: 5, minOutputChars: 3 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'too-short-input')).toBe(true);
});
it('filters examples over max combined length', async () => {
seedTrace('this is an ok prompt', 'this is an ok answer', 'success');
seedTrace('x'.repeat(5000), 'y'.repeat(5000), 'success');
const split = await builder.build({ seed: 1, maxCombinedChars: 1000 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'too-long')).toBe(true);
});
it('filters low-signal garbage', async () => {
seedTrace('.......................', '!!!!!!!!!!!!!!!!', 'success');
seedTrace('real question?', 'real answer.', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.total).toBe(1);
expect(split.rejected.some(r => r.reason === 'low-signal')).toBe(true);
});
it('dedupes by input hash', async () => {
seedTrace('same prompt', 'answer a', 'success');
seedTrace('same prompt', 'answer b', 'success');
seedTrace('different prompt', 'answer c', 'success');
const split = await builder.build({ seed: 1 });
expect(split.stats.unique).toBe(2);
expect(split.rejected.some(r => r.reason === 'duplicate')).toBe(true);
});
it('applies optional LLM judge', async () => {
seedTrace('keep this example please', 'kept response', 'success');
seedTrace('drop this example please', 'dropped response', 'success');
const split = await builder.build({
seed: 1,
judge: async (ex) => ({
keep: !ex.input.includes('drop'),
reason: ex.input.includes('drop') ? 'contains drop' : undefined,
}),
});
expect(split.stats.acceptedAfterJudge).toBe(1);
expect(split.rejected.some(r => r.reason.startsWith('judge:'))).toBe(true);
});
it('keeps example when judge throws (fail-open)', async () => {
seedTrace('some prompt text', 'some answer text', 'success');
const split = await builder.build({
seed: 1,
judge: async () => {
throw new Error('llm down');
},
});
expect(split.stats.acceptedAfterJudge).toBe(1);
expect(split.stats.total).toBe(1);
});
it('accepts external harvest/correction examples', async () => {
const harvest: EvalExample = {
input: 'harvest prompt',
expected_output: 'harvest answer',
metadata: { source: 'harvest' },
};
const correction: EvalExample = {
input: 'correction prompt',
expected_output: 'correction answer',
metadata: { source: 'correction' },
};
const split = await builder.build({
seed: 1,
harvestExamples: [harvest],
correctionExamples: [correction],
});
const allInputs = [...split.train, ...split.val, ...split.holdout].map(e => e.input);
expect(allInputs).toContain('harvest prompt');
expect(allInputs).toContain('correction prompt');
});
it('maps correction trace output to correctionFeedback when available', async () => {
const id = store.start({ input: 'wrong approach', personaId: 'coder' });
store.finalize(id, { outcome: 'success', output: 'first try' });
store.markCorrected(id, 'Should use async/await here');
const examples = builder.sourceFromTraces(['success'], true);
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
expect(corrected?.expected_output).toBe('Should use async/await here');
});
it('returns empty splits when no traces match', async () => {
const split = await builder.build({ seed: 1 });
expect(split.stats.total).toBe(0);
expect(split.train).toEqual([]);
expect(split.val).toEqual([]);
expect(split.holdout).toEqual([]);
});
it('tiny datasets assign leftover to holdout', async () => {
for (let i = 0; i < 3; i++) {
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
}
// Ratios 0.6/0.2/0.2 with n=3 → floor(1.8)=1, floor(0.6)=0, remainder=2 → holdout
const split = await builder.build({ seed: 1 });
expect(split.train.length + split.val.length + split.holdout.length).toBe(3);
});
});
});

View File

@@ -1,372 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import {
deployPersonaOverride,
rollbackPersonaOverride,
deployBehavioralSpecOverride,
rollbackBehavioralSpecOverride,
loadBehavioralSpecOverrides,
applyBehavioralSpecOverrides,
BEHAVIORAL_SPEC_SECTIONS,
type BehavioralSpecSection,
} from '../src/evolution-deploy.js';
import { getPersona, listPersonas, setPersonaDataDir } from '../src/personas.js';
import { loadCustomPersonas } from '../src/custom-personas.js';
// ── Test fixture: temp dir per-test ────────────────────────────
describe('evolution-deploy', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-deploy-'));
});
afterEach(() => {
vi.restoreAllMocks();
try {
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { /* Windows lock cleanup racy; ignore */ }
});
// ── deployPersonaOverride ──
describe('deployPersonaOverride', () => {
it('writes a JSON file named {id}.json in personas/', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'You are the evolved coder.',
});
expect(fs.existsSync(result.path)).toBe(true);
expect(result.path.endsWith(path.join('personas', 'coder.json'))).toBe(true);
expect(result.backupPath).toBeNull();
expect(result.deployedAt).toBeTruthy();
});
it('inherits built-in persona fields when targeting a known id', () => {
// 'coder' is a built-in persona
const builtin = getPersona('coder');
expect(builtin).not.toBeNull();
deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'EVOLVED coder prompt — tighter format.',
});
const [loaded] = loadCustomPersonas(tmpDir);
expect(loaded.id).toBe('coder');
expect(loaded.name).toBe(builtin!.name);
expect(loaded.icon).toBe(builtin!.icon);
expect(loaded.systemPrompt).toBe('EVOLVED coder prompt — tighter format.');
expect(loaded.tools).toEqual(builtin!.tools);
});
it('creates a backup on subsequent writes', () => {
const first = deployPersonaOverride(tmpDir, {
personaId: 'coder', systemPrompt: 'v1',
});
expect(first.backupPath).toBeNull();
const second = deployPersonaOverride(tmpDir, {
personaId: 'coder', systemPrompt: 'v2',
});
expect(second.backupPath).not.toBeNull();
expect(fs.existsSync(second.backupPath!)).toBe(true);
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).systemPrompt).toBe('v1');
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).systemPrompt).toBe('v2');
});
it('creates a minimal shell for unknown persona ids', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'made-up-persona',
systemPrompt: 'fresh prompt',
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.id).toBe('made-up-persona');
expect(loaded.name).toBe('made-up-persona');
expect(loaded.systemPrompt).toBe('fresh prompt');
expect(Array.isArray(loaded.tools)).toBe(true);
});
it('applies caller-supplied overrides (description, icon, tools)', () => {
const result = deployPersonaOverride(tmpDir, {
personaId: 'made-up',
systemPrompt: 'x',
overrides: {
description: 'custom description',
icon: 'brain',
tools: ['tool_a', 'tool_b'],
},
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.description).toBe('custom description');
expect(loaded.icon).toBe('brain');
expect(loaded.tools).toEqual(['tool_a', 'tool_b']);
});
it('is picked up by listPersonas after setPersonaDataDir', () => {
deployPersonaOverride(tmpDir, {
personaId: 'coder',
systemPrompt: 'EVOLVED ROUTING',
});
setPersonaDataDir(tmpDir);
try {
const personas = listPersonas();
const evolved = personas.find(p => p.id === 'coder' && p.systemPrompt.includes('EVOLVED ROUTING'));
expect(evolved).toBeDefined();
} finally {
setPersonaDataDir(''); // reset so other tests aren't polluted
}
});
});
// ── rollbackPersonaOverride ──
describe('rollbackPersonaOverride', () => {
it('restores .bak when present', () => {
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
const second = deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v2' });
expect(second.backupPath).not.toBeNull();
const ok = rollbackPersonaOverride(tmpDir, 'coder');
expect(ok).toBe(true);
const [loaded] = loadCustomPersonas(tmpDir);
expect(loaded.systemPrompt).toBe('v1');
});
it('deletes the override entirely when no .bak exists', () => {
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
const ok = rollbackPersonaOverride(tmpDir, 'coder');
expect(ok).toBe(true);
expect(loadCustomPersonas(tmpDir)).toHaveLength(0);
});
it('returns false for non-existent personas', () => {
expect(rollbackPersonaOverride(tmpDir, 'ghost')).toBe(false);
});
});
// ── deployBehavioralSpecOverride ──
describe('deployBehavioralSpecOverride', () => {
it('writes a JSON file named {section}.json in behavioral-overrides/', () => {
const result = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop',
text: 'Evolved core loop instructions.',
});
expect(fs.existsSync(result.path)).toBe(true);
expect(result.path.endsWith(path.join('behavioral-overrides', 'coreLoop.json'))).toBe(true);
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.section).toBe('coreLoop');
expect(loaded.text).toBe('Evolved core loop instructions.');
expect(loaded.deployedAt).toBeTruthy();
});
it('stores the originating run uuid for audit', () => {
const result = deployBehavioralSpecOverride(tmpDir, {
section: 'qualityRules',
text: 'new quality rules',
runUuid: 'test-run-uuid-123',
});
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
expect(loaded.runUuid).toBe('test-run-uuid-123');
});
it('creates backup on subsequent writes', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
expect(first.backupPath).toBeNull();
const second = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
});
expect(second.backupPath).not.toBeNull();
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
});
it('retries transient Windows rename locks before replacing an override', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync')
.mockImplementationOnce(() => {
throw Object.assign(new Error('locked'), { code: 'EPERM' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('busy'), { code: 'EBUSY' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('locked again'), { code: 'EPERM' });
})
.mockImplementationOnce(() => {
throw Object.assign(new Error('still busy'), { code: 'EBUSY' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
const second = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
});
expect(rename).toHaveBeenCalledTimes(5);
expect(wait).toHaveBeenCalledTimes(4);
expect(JSON.parse(fs.readFileSync(second.path, 'utf-8')).text).toBe('v2');
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('preserves the current override when transient rename retries are exhausted', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
throw Object.assign(new Error('still locked'), { code: 'EACCES' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
expect(() => deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
})).toThrow(/still locked/);
expect(rename).toHaveBeenCalledTimes(10);
expect(wait).toHaveBeenCalledTimes(9);
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('does not retry non-transient rename failures', () => {
const first = deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v1',
});
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
throw Object.assign(new Error('disk error'), { code: 'EIO' });
});
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
expect(() => deployBehavioralSpecOverride(tmpDir, {
section: 'coreLoop', text: 'v2',
})).toThrow(/disk error/);
expect(rename).toHaveBeenCalledOnce();
expect(wait).not.toHaveBeenCalled();
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
});
it('rejects unknown sections', () => {
expect(() =>
deployBehavioralSpecOverride(tmpDir, {
section: 'notASection' as BehavioralSpecSection,
text: 'x',
}),
).toThrow(/Unknown behavioral-spec section/);
});
});
// ── rollbackBehavioralSpecOverride ──
describe('rollbackBehavioralSpecOverride', () => {
it('restores .bak when present', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v2' });
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
expect(ok).toBe(true);
const overrides = loadBehavioralSpecOverrides(tmpDir);
expect(overrides.coreLoop).toBe('v1');
});
it('removes the override when no backup exists', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
expect(ok).toBe(true);
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
});
});
// ── loadBehavioralSpecOverrides ──
describe('loadBehavioralSpecOverrides', () => {
it('returns empty object when no overrides exist', () => {
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
});
it('returns a section → text map for all written overrides', () => {
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'evolved core' });
deployBehavioralSpecOverride(tmpDir, { section: 'qualityRules', text: 'evolved quality' });
const overrides = loadBehavioralSpecOverrides(tmpDir);
expect(overrides.coreLoop).toBe('evolved core');
expect(overrides.qualityRules).toBe('evolved quality');
expect(overrides.behavioralRules).toBeUndefined();
});
it('ignores malformed override files', () => {
const dir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'coreLoop.json'), '{ not valid json');
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
});
it('ignores files with empty text', () => {
const dir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'coreLoop.json'), JSON.stringify({
section: 'coreLoop', text: '', deployedAt: '2026-04-14',
}));
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
});
});
// ── applyBehavioralSpecOverrides ──
describe('applyBehavioralSpecOverrides', () => {
const baseline: Record<BehavioralSpecSection, string> = {
coreLoop: 'base core',
qualityRules: 'base quality',
behavioralRules: 'base behavioral',
workPatterns: 'base patterns',
intelligenceDefaults: 'base intelligence',
};
it('returns baseline unchanged when no overrides', () => {
expect(applyBehavioralSpecOverrides(baseline, {})).toEqual(baseline);
});
it('overlays provided sections', () => {
const merged = applyBehavioralSpecOverrides(baseline, {
coreLoop: 'EVOLVED core',
qualityRules: 'EVOLVED quality',
});
expect(merged.coreLoop).toBe('EVOLVED core');
expect(merged.qualityRules).toBe('EVOLVED quality');
expect(merged.behavioralRules).toBe('base behavioral');
});
it('ignores empty/undefined overrides without clobbering baseline', () => {
const merged = applyBehavioralSpecOverrides(baseline, {
coreLoop: '',
qualityRules: undefined,
});
expect(merged.coreLoop).toBe('base core');
expect(merged.qualityRules).toBe('base quality');
});
});
// ── BEHAVIORAL_SPEC_SECTIONS constant ──
describe('BEHAVIORAL_SPEC_SECTIONS', () => {
it('contains the expected 5 sections in expected order', () => {
expect(BEHAVIORAL_SPEC_SECTIONS).toEqual([
'coreLoop', 'qualityRules', 'behavioralRules',
'workPatterns', 'intelligenceDefaults',
]);
});
});
});

View File

@@ -1,245 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
runGates, DEFAULT_SIZE_LIMITS,
checkNonEmpty, checkSize, checkGrowth,
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
} from '../src/evolution-gates.js';
describe('checkNonEmpty', () => {
it('fails on empty string', () => {
expect(checkNonEmpty('').verdict).toBe('fail');
});
it('fails on whitespace-only string', () => {
expect(checkNonEmpty(' \n\n ').verdict).toBe('fail');
});
it('passes on real content', () => {
expect(checkNonEmpty('hello').verdict).toBe('pass');
});
});
describe('checkSize', () => {
it('passes within the persona cap', () => {
const result = checkSize('x'.repeat(2000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('pass');
expect(result.detail?.current).toBe(2000);
expect(result.detail?.max).toBe(DEFAULT_SIZE_LIMITS.personaSystemPrompt);
});
it('fails over the persona cap', () => {
const result = checkSize('x'.repeat(4000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('fail');
expect(result.reason).toContain('persona-system-prompt');
});
it('uses tool-description cap (500)', () => {
expect(checkSize('x'.repeat(500), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
expect(checkSize('x'.repeat(501), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
});
it('uses skill-body cap (15k)', () => {
expect(checkSize('x'.repeat(15_000), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
expect(checkSize('x'.repeat(15_001), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
});
it('uses generic cap for unknown target', () => {
const result = checkSize('x'.repeat(5000), 'generic', DEFAULT_SIZE_LIMITS);
expect(result.verdict).toBe('pass');
});
it('accepts custom size limits', () => {
const limits = { ...DEFAULT_SIZE_LIMITS, generic: 100 };
const result = checkSize('x'.repeat(200), 'generic', limits);
expect(result.verdict).toBe('fail');
});
});
describe('checkGrowth', () => {
it('passes when candidate is smaller than baseline', () => {
const result = checkGrowth('abc', 'abcdefgh', 0.2);
expect(result.verdict).toBe('pass');
});
it('passes when growth is within budget', () => {
const result = checkGrowth('x'.repeat(110), 'x'.repeat(100), 0.2); // +10%
expect(result.verdict).toBe('pass');
});
it('fails when growth exceeds budget', () => {
const result = checkGrowth('x'.repeat(140), 'x'.repeat(100), 0.2); // +40%
expect(result.verdict).toBe('fail');
expect(result.detail?.ratio).toBeCloseTo(0.4, 5);
});
it('passes when baseline is empty (cannot compute ratio)', () => {
const result = checkGrowth('anything', '', 0.2);
expect(result.verdict).toBe('pass');
});
it('boundary: exactly 20% growth passes with default budget', () => {
// 100 → 120 is exactly 20%, not greater
const result = checkGrowth('x'.repeat(120), 'x'.repeat(100), 0.2);
expect(result.verdict).toBe('pass');
});
});
describe('checkBalancedFences', () => {
it('passes no fences', () => {
expect(checkBalancedFences('plain prose').verdict).toBe('pass');
});
it('passes two fences (one code block)', () => {
const text = 'here:\n```js\nconst x = 1;\n```\ndone';
expect(checkBalancedFences(text).verdict).toBe('pass');
});
it('fails one fence (unbalanced)', () => {
const text = 'here:\n```js\nconst x = 1;\nstill going';
const result = checkBalancedFences(text);
expect(result.verdict).toBe('fail');
expect(result.detail?.fences).toBe(1);
});
it('passes four fences (two blocks)', () => {
const text = '```a\nfoo\n```\n```b\nbar\n```';
expect(checkBalancedFences(text).verdict).toBe('pass');
});
});
describe('checkNoPlaceholders', () => {
it('passes clean text', () => {
expect(checkNoPlaceholders('A prompt with normal prose.').verdict).toBe('pass');
});
it('fails [TODO] bracket placeholder', () => {
expect(checkNoPlaceholders('Write about [TODO] here.').verdict).toBe('fail');
});
it('fails [PLACEHOLDER]', () => {
expect(checkNoPlaceholders('Fill in [PLACEHOLDER].').verdict).toBe('fail');
});
it('fails [YOUR NAME] style', () => {
expect(checkNoPlaceholders('Hi [YOUR NAME HERE]!').verdict).toBe('fail');
});
it('fails angle-brackets <placeholder>', () => {
expect(checkNoPlaceholders('Set <placeholder> before sending.').verdict).toBe('fail');
});
it('fails handlebars {{var}}', () => {
expect(checkNoPlaceholders('Hello {{name}}.').verdict).toBe('fail');
});
it('does not false-positive normal bracketed citations', () => {
expect(checkNoPlaceholders('See [1] and [Smith 2024] for details.').verdict).toBe('pass');
});
});
describe('checkNoObviousTodos', () => {
it('passes clean text', () => {
expect(checkNoObviousTodos('Write clear documentation.').verdict).toBe('pass');
});
it('fails lines starting with TODO:', () => {
expect(checkNoObviousTodos('Intro paragraph.\nTODO: finish this').verdict).toBe('fail');
});
it('fails lines starting with FIXME:', () => {
expect(checkNoObviousTodos('FIXME: tighten this up').verdict).toBe('fail');
});
it('does not fail mid-sentence "todo" mention', () => {
expect(checkNoObviousTodos('Build a todo list UI.').verdict).toBe('pass');
});
});
describe('checkRegression', () => {
it('passes when delta is within tolerance', () => {
const result = checkRegression(0.8, 0.79, -0.02); // -1pp, floor -2pp
expect(result.verdict).toBe('pass');
});
it('passes on improvement', () => {
const result = checkRegression(0.7, 0.85, -0.02);
expect(result.verdict).toBe('pass');
expect(result.detail?.delta).toBeCloseTo(0.15, 5);
});
it('fails when delta exceeds tolerance', () => {
const result = checkRegression(0.9, 0.7, -0.02); // -20pp
expect(result.verdict).toBe('fail');
});
it('boundary: equal to tolerance passes', () => {
const result = checkRegression(0.8, 0.78, -0.02); // exactly -2pp
expect(result.verdict).toBe('pass');
});
});
// ── runGates end-to-end ────────────────────────────────────────
describe('runGates', () => {
it('passes a clean candidate', () => {
const res = runGates({
candidate: 'A concise persona prompt for the researcher.',
baseline: 'A concise persona prompt for the researcher.',
}, { targetKind: 'persona-system-prompt' });
expect(res.verdict).toBe('pass');
expect(res.firstFailure).toBeNull();
});
it('fails an empty candidate on the first gate', () => {
const res = runGates({ candidate: '', baseline: 'whatever' });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('non-empty');
});
it('fails oversized candidate on size gate', () => {
const res = runGates({
candidate: 'x'.repeat(4000),
baseline: 'x'.repeat(3000),
}, { targetKind: 'persona-system-prompt' });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('size');
});
it('fails runaway growth', () => {
const res = runGates({
candidate: 'x'.repeat(200),
baseline: 'x'.repeat(100),
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('growth');
});
it('fails placeholder leakage', () => {
const res = runGates({
candidate: 'You are {{persona.name}} — answer questions.',
baseline: 'You are the researcher — answer questions.',
});
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('no-placeholders');
});
it('fails unbalanced markdown fences', () => {
// Pad baseline so the growth gate doesn't trip before the fence gate.
const padding = ' '.repeat(100);
const res = runGates({
candidate: `Instructions:\n\`\`\`js\nconst x = 1;\nmore prose${padding}`,
baseline: `Instructions: write code clearly.${padding}`,
});
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('balanced-fences');
});
it('applies the regression gate when scores are supplied', () => {
const res = runGates({
candidate: 'new prompt',
baseline: 'old prompt',
scores: { baseline: 0.85, candidate: 0.6 },
}, { maxRegression: -0.02 });
expect(res.verdict).toBe('fail');
expect(res.firstFailure?.gate).toBe('regression');
});
it('skips the regression gate when scores are omitted', () => {
const res = runGates({ candidate: 'new', baseline: 'old' });
expect(res.results.find(r => r.gate === 'regression')).toBeUndefined();
});
it('returns results for every gate even when one fails', () => {
const res = runGates({
candidate: 'x'.repeat(200),
baseline: 'x'.repeat(100),
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
expect(res.results.length).toBeGreaterThan(1);
});
});

View File

@@ -1,11 +1,6 @@
import { describe, it, expect, vi } from 'vitest';
import {
buildJudgeLLMCall,
buildGEPAMutateFn,
buildSchemaExecuteFn,
makeRunningJudge,
buildReflectiveMutationPrompt,
buildSchemaFillPrompt,
retryWithBackoff,
wrapWithRetry,
isRetryableEvolutionError,
@@ -15,9 +10,6 @@ import {
type RetryOptions,
type RetryInfo,
} from '../src/evolution-llm-wiring.js';
import type { MutateArgs, GEPACandidate } from '../src/index.js';
import type { Schema } from '../src/evolve-schema.js';
import type { JudgeInput } from '../src/judge.js';
// ── Fixtures ──────────────────────────────────────────────────────
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
};
}
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
return {
id: 'g1-m0',
prompt: 'You are a tester. Respond with the exact expected word.',
generation: 1,
parent: 'g0-baseline',
strategy: 'expand-edge-cases',
score: null,
perExample: [],
...overrides,
};
}
function makeSchema(overrides: Partial<Schema> = {}): Schema {
return {
name: 'answer',
version: 1,
fields: [
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
],
...overrides,
};
}
// ── buildJudgeLLMCall ─────────────────────────────────────────────
describe('buildJudgeLLMCall', () => {
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
});
});
// ── buildGEPAMutateFn ─────────────────────────────────────────────
describe('buildGEPAMutateFn', () => {
const mutateArgs: MutateArgs = {
parent: makeCandidate(),
strategy: 'expand-edge-cases',
weaknessFeedback: [
'Fails to handle empty input',
'Sometimes forgets to include the final answer',
],
targetKind: 'persona-system-prompt',
generation: 2,
};
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe('EVOLVED SYSTEM PROMPT');
expect(calls).toHaveLength(1);
const prompt = calls[0];
expect(prompt).toContain(mutateArgs.parent.prompt);
expect(prompt).toContain(mutateArgs.strategy);
expect(prompt).toContain('Fails to handle empty input');
expect(prompt).toContain('persona-system-prompt');
});
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
const { llm } = makeMockLLM(() => ' \n ');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe(mutateArgs.parent.prompt);
});
it('falls back to the parent prompt when the LLM throws', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('boom'); },
};
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result).toBe(mutateArgs.parent.prompt);
});
it('strips markdown code fences from the LLM response', async () => {
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
const mutate = buildGEPAMutateFn(llm);
const result = await mutate(mutateArgs);
expect(result.trim()).toBe('HIDDEN PROMPT');
});
});
describe('buildReflectiveMutationPrompt', () => {
it('includes the generation number and weakness bullets', () => {
const prompt = buildReflectiveMutationPrompt({
parent: 'PARENT',
strategy: 'tighten-format',
weaknessFeedback: ['a', 'b'],
targetKind: 'behavioral-spec-section',
generation: 3,
});
expect(prompt).toContain('generation 3');
expect(prompt).toContain('- a');
expect(prompt).toContain('- b');
expect(prompt).toContain('tighten-format');
expect(prompt).toContain('PARENT');
});
it('handles missing weakness feedback gracefully', () => {
const prompt = buildReflectiveMutationPrompt({
parent: 'PARENT',
strategy: 'add-examples',
weaknessFeedback: [],
targetKind: 'generic',
generation: 0,
});
expect(prompt).toContain('PARENT');
expect(prompt).toContain('(no specific weakness signals yet)');
});
});
// ── buildSchemaExecuteFn ──────────────────────────────────────────
describe('buildSchemaExecuteFn', () => {
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
expect(result.parsed).toBe(true);
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
expect(calls[0]).toContain('What is the answer?');
expect(calls[0]).toContain('"reasoning"');
expect(calls[0]).toContain('"answer"');
});
it('returns parsed=false when LLM returns non-JSON', async () => {
const { llm } = makeMockLLM(() => 'just some text');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(false);
expect(result.actual).toBe('just some text');
});
it('extracts JSON from markdown fences', async () => {
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(true);
expect(result.actual).toContain('"reasoning"');
});
it('returns parsed=false and empty actual when LLM throws', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('boom'); },
};
const execute = buildSchemaExecuteFn(llm);
const result = await execute({ schema: makeSchema(), input: 'hi' });
expect(result.parsed).toBe(false);
expect(result.actual).toBe('');
});
});
describe('buildSchemaFillPrompt', () => {
it('serializes each field with name + type + description', () => {
const prompt = buildSchemaFillPrompt({
schema: makeSchema(),
input: 'What is 2+2?',
});
expect(prompt).toContain('"reasoning"');
expect(prompt).toContain('"answer"');
expect(prompt).toContain('step-by-step thinking');
expect(prompt).toContain('the final answer');
expect(prompt).toContain('What is 2+2?');
});
it('notes required constraints', () => {
const schema: Schema = {
name: 'x', version: 1,
fields: [
{
name: 'score',
type: 'number',
description: 'a rating',
required: true,
constraints: [{ kind: 'range', value: '0-10' }],
},
],
};
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
expect(prompt).toContain('score');
expect(prompt).toContain('number');
expect(prompt).toContain('range');
});
});
// ── makeRunningJudge ──────────────────────────────────────────────
describe('makeRunningJudge', () => {
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
const { llm, calls } = makeMockLLM((prompt) => {
// Simulate running the candidate prompt and returning a model response.
return `MODEL OUTPUT for: ${prompt}`;
});
const baseScore = vi.fn(async (args: JudgeInput) => ({
overall: 0.8, weighted: 0.8,
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
lengthPenalty: 1, feedback: 'passed through', parsed: true,
}));
const baseJudge = { score: baseScore };
const runningJudge = makeRunningJudge(baseJudge, llm);
const result = await runningJudge.score({
input: 'What is love?',
expected: 'baby don\'t hurt me',
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
});
// Model was asked to run the candidate prompt against the example input.
expect(calls).toHaveLength(1);
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
expect(calls[0]).toContain('What is love?');
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
expect(baseScore).toHaveBeenCalledTimes(1);
const passedArgs = baseScore.mock.calls[0][0];
expect(passedArgs.input).toBe('What is love?');
expect(passedArgs.expected).toBe('baby don\'t hurt me');
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
expect(result.overall).toBe(0.8);
});
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
const llm: EvolutionLLM = {
async complete() { throw new Error('rate limited'); },
};
const baseJudge = { score: vi.fn() };
const runningJudge = makeRunningJudge(baseJudge, llm);
const result = await runningJudge.score({
input: 'any', expected: 'any', actual: 'prompt',
});
expect(result.overall).toBe(0);
expect(result.parsed).toBe(false);
expect(result.feedback).toMatch(/execution failed/i);
expect(baseJudge.score).not.toHaveBeenCalled();
});
it('passes context through to the base judge', async () => {
const { llm } = makeMockLLM(() => 'EXECUTED');
const baseScore = vi.fn(async () => ({
overall: 0.5, weighted: 0.5,
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
lengthPenalty: 1, feedback: '', parsed: true,
}));
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
await runningJudge.score({
input: 'q', expected: 'a', actual: 'prompt',
context: 'persona:coder',
});
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
});
});
// ── isRetryableEvolutionError ─────────────────────────────────────
describe('isRetryableEvolutionError', () => {

View File

@@ -1,447 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
import {
EvolutionOrchestrator,
eligibleForEvolution,
summarizeRuns,
} from '../src/evolution-orchestrator.js';
import type { JudgeScore } from '../src/judge.js';
import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
import type { MutateFn } from '../src/iterative-optimizer.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(n => ({
name: n, type: 'string' as const,
description: `${n} field`,
required: true, constraints: [],
})),
version: 1,
};
}
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
// Executor that produces longer output for bigger schemas.
function makeExec(): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => f.name).join(','),
parsed: true,
});
}
// Judge that prefers candidates containing the "plus" token so the
// mutateAppend helper ("... plus") wins reliably. Creates a measurable
// delta between baseline and winner regardless of absolute lengths.
function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
return {
async score(args) {
const base = Math.min(1, args.actual.length / 100) * 0.6;
const bonus = args.actual.includes('plus') ? 0.3 : 0;
return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
},
};
}
// Mutate appends ~10 chars so growth ratio stays below the default +20%.
const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
const BASELINE =
'Return a concise answer that directly addresses the question. ' +
'Provide a brief explanation and the final answer. Be clear and accurate.';
function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
for (let i = 0; i < n; i++) {
const id = store.start({
personaId,
input: `test question ${i} with enough length to pass filters`,
workspaceId: 'ws-1',
taskShape: 'qa',
});
store.finalize(id, {
outcome: 'success',
output: `a full answer for question ${i} that is long enough for use`,
});
}
}
function baseComposeOptions() {
return {
schema: {
examples: [] as never[],
execute: makeExec(),
judge: makeJudge(),
populationSize: 2, generations: 1,
evalSize: 3, anchorEvalSize: 3,
},
instructions: {
examples: [] as never[],
judge: makeJudge(),
mutate: mutateAppend,
allowBareJudge: true,
populationSize: 2, generations: 1,
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
},
};
}
// ── eligibleForEvolution ──
describe('eligibleForEvolution', () => {
it('keeps finalized traces with non-empty input and output', () => {
const eligible = eligibleForEvolution([
{
id: 1, session_id: 's', persona_id: null, workspace_id: null,
model: null, task_shape: null, outcome: 'success',
cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
payload: {
input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
tokens: { input: 0, output: 0 }, tags: [],
},
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(1);
});
it('drops pending traces', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'pending',
payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(0);
});
it('drops traces with empty input', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'success',
payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(0);
});
it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
const eligible = eligibleForEvolution([
{
id: 1, outcome: 'corrected',
payload: {
input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
tokens: { input: 0, output: 0 }, tags: [],
correctionFeedback: 'use bullets',
},
} as unknown as ParsedExecutionTrace,
]);
expect(eligible).toHaveLength(1);
});
});
// ── summarizeRuns ──
describe('summarizeRuns', () => {
it('returns zero-aggregates on empty input', () => {
const s = summarizeRuns([]);
expect(s.total).toBe(0);
expect(s.byStatus.proposed).toBe(0);
expect(s.bestDelta).toBe(0);
});
it('counts by status and target kind, tracks best delta', () => {
const runs = [
{ status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
{ status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
{ status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
{ status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
] as unknown as EvolutionRun[];
const s = summarizeRuns(runs);
expect(s.total).toBe(4);
expect(s.byStatus.proposed).toBe(1);
expect(s.byStatus.accepted).toBe(1);
expect(s.byStatus.rejected).toBe(1);
expect(s.byStatus.deployed).toBe(1);
expect(s.byTargetKind['persona-system-prompt']).toBe(3);
expect(s.byTargetKind['tool-description']).toBe(1);
expect(s.bestDelta).toBeCloseTo(0.12, 5);
});
});
// ── EvolutionOrchestrator end-to-end ───────────────────────────
describe('EvolutionOrchestrator', () => {
let db: MindDB;
let traceStore: ExecutionTraceStore;
let runStore: EvolutionRunStore;
let orchestrator: EvolutionOrchestrator;
beforeEach(() => {
db = new MindDB(':memory:');
traceStore = new ExecutionTraceStore(db);
runStore = new EvolutionRunStore(db);
orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
});
afterEach(() => {
db.close();
});
// ── runOnce ──
describe('runOnce', () => {
it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
seedSuccessfulTraces(traceStore, 10);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
targetName: 'researcher',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
expect(result.outcome).toBe('proposed');
expect(result.run?.status).toBe('proposed');
expect(result.run?.delta_accuracy).toBeGreaterThan(0);
expect(result.compose).toBeDefined();
expect(result.gateResults).toBeDefined();
});
it('persists compose artifacts (schema JSON, gates, delta)', async () => {
seedSuccessfulTraces(traceStore, 8);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
targetName: 'researcher',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const run = result.run!;
expect(run.winner_schema_json).toBeTruthy();
const schema = JSON.parse(run.winner_schema_json!);
expect(schema.name).toBe('test');
expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
});
it('skips with skipped-delta when improvement is below threshold', async () => {
seedSuccessfulTraces(traceStore, 8);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: 'a reasonably detailed baseline prompt that already scores well',
schemaBaseline: makeSchema(['answer']),
compose: {
...baseComposeOptions(),
instructions: {
...baseComposeOptions().instructions,
// Mutate returns the same text → zero delta
mutate: async ({ parent }) => parent.prompt,
},
},
minDelta: 0.5, // artificially high
});
expect(result.outcome).toBe('skipped-delta');
expect(result.run).toBeUndefined();
});
it('auto-trigger: skips when too few traces', async () => {
seedSuccessfulTraces(traceStore, 2);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: 'baseline',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
autoTrigger: { minTraces: 50 },
});
expect(result.outcome).toBe('skipped-trigger');
expect(result.run).toBeUndefined();
});
it('auto-trigger: proceeds when threshold is met', async () => {
seedSuccessfulTraces(traceStore, 10);
const result = await orchestrator.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
autoTrigger: { minTraces: 5 },
});
expect(result.outcome).toBe('proposed');
});
it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
seedSuccessfulTraces(traceStore, 8);
// Judge that prefers the mutated "y" text — ensures the GEPA winner
// is the oversized candidate, not the baseline.
const yPreferringJudge = {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
},
};
const result = await orchestrator.runOnce({
targetKind: 'tool-description',
baseline: 'x'.repeat(30),
schemaBaseline: makeSchema(['answer']),
compose: {
...baseComposeOptions(),
instructions: {
...baseComposeOptions().instructions,
judge: yPreferringJudge,
// Mutate produces candidate that blows past tool-description cap (500)
mutate: async () => 'y'.repeat(2000),
},
},
minDelta: 0, // allow any delta so we reach the gate stage
});
expect(result.outcome).toBe('skipped-gates');
expect(result.run?.status).toBe('rejected');
expect(result.run?.gate_verdict).toBe('fail');
expect(result.run?.user_note).toMatch(/gate failure/);
});
it('aborts cleanly when signal fires before start', async () => {
seedSuccessfulTraces(traceStore, 5);
const ctrl = new AbortController();
ctrl.abort();
const result = await orchestrator.runOnce({
targetKind: 'generic',
baseline: 'baseline',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
signal: ctrl.signal,
});
expect(result.outcome).toBe('aborted');
});
it('emits progress events', async () => {
seedSuccessfulTraces(traceStore, 5);
const phases: string[] = [];
await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
onProgress: (e) => phases.push(e.phase),
});
expect(phases).toContain('compose');
expect(phases).toContain('done');
});
});
// ── accept ──
describe('accept', () => {
it('invokes deploy callback and marks run deployed on success', async () => {
const deploy = vi.fn(async () => { /* success */ });
const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
seedSuccessfulTraces(traceStore, 8);
const run = await deployedOrch.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
expect(run.outcome).toBe('proposed');
const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
expect(accepted?.status).toBe('deployed');
expect(deploy).toHaveBeenCalledTimes(1);
});
it('marks run failed when deploy callback throws', async () => {
const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
seedSuccessfulTraces(traceStore, 8);
const run = await orch.runOnce({
targetKind: 'persona-system-prompt',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const result = await orch.accept(run.run!.run_uuid);
expect(result?.status).toBe('failed');
expect(result?.failure_reason).toContain('persona write failed');
});
it('leaves run as accepted when no deploy hook configured', async () => {
seedSuccessfulTraces(traceStore, 8);
const run = await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const accepted = await orchestrator.accept(run.run!.run_uuid);
expect(accepted?.status).toBe('accepted');
});
it('returns undefined for unknown uuid', async () => {
expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
});
});
// ── reject ──
describe('reject', () => {
it('marks proposed run as rejected', async () => {
seedSuccessfulTraces(traceStore, 8);
const run = await orchestrator.runOnce({
targetKind: 'generic',
baseline: BASELINE,
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
expect(rejected?.status).toBe('rejected');
expect(rejected?.user_note).toBe('regressed on custom cases');
});
});
// ── list / get ──
describe('list / get', () => {
it('returns runs in reverse-chronological order', async () => {
seedSuccessfulTraces(traceStore, 8);
await orchestrator.runOnce({
targetKind: 'generic', baseline: 'a',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
await orchestrator.runOnce({
targetKind: 'generic', baseline: 'b',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const list = orchestrator.list();
expect(list.length).toBeGreaterThanOrEqual(2);
});
it('get returns a specific run by uuid', async () => {
seedSuccessfulTraces(traceStore, 8);
const created = await orchestrator.runOnce({
targetKind: 'generic', baseline: 'short',
schemaBaseline: makeSchema(['answer']),
compose: baseComposeOptions(),
});
const fetched = orchestrator.get(created.run!.run_uuid);
expect(fetched?.id).toBe(created.run!.id);
});
});
});

View File

@@ -1,545 +0,0 @@
import { describe, it, expect } from 'vitest';
import {
EvolveSchema,
addOutputField,
removeField,
editFieldDescription,
changeFieldType,
addConstraint,
removeConstraint,
reorderFields,
replaceOutputFields,
schemaComplexity,
aggregateSchemaScores,
paretoFrontSchema,
pickSchemaWinner,
generateStructureMutations,
generateOrderMutations,
generateRefinementMutations,
pickSample,
type Schema,
type SchemaField,
type SchemaCandidate,
type SchemaCandidateScore,
type SchemaExecuteFn,
} from '../src/evolve-schema.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
// ── Fixtures ───────────────────────────────────────────────────
function makeField(name: string, partial: Partial<SchemaField> = {}): SchemaField {
return {
name,
type: 'string',
description: `description for ${name}`,
required: true,
constraints: [],
...partial,
};
}
function makeSchema(fields: string[]): Schema {
return {
name: 'test',
fields: fields.map(f => makeField(f)),
version: 1,
};
}
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `q${i}`,
expected_output: `a${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
return {
overall, weighted: overall,
correctness: overall, procedureFollowing: overall, conciseness: overall,
lengthPenalty: 1, feedback, parsed: true,
};
}
function makeCandidate(id: string, schema: Schema, score: SchemaCandidateScore | null): SchemaCandidate {
return {
id,
schema,
generation: 0,
parent: null,
mutation: 'baseline',
mutationLabel: 'baseline',
score,
perExample: [],
};
}
// Fake executor: returns a deterministic output string that scores higher
// when schema has more fields (up to a cap) and scores lower when schema
// is bloated beyond 5 fields.
function makeFakeExecutor(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more'): SchemaExecuteFn {
return async ({ schema }) => ({
actual: schema.fields.map(f => `${f.name}=mock`).join('; '),
parsed: schema.fields.length > 0,
});
}
function makeJudge(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more') {
return {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
const fieldCount = (args.actual.match(/=/g) || []).length;
let overall: number;
if (bias === 'prefers-more') {
// More fields = better, up to 6 fields, then flat
overall = Math.min(1, fieldCount / 6);
} else if (bias === 'prefers-less') {
overall = Math.max(0, 1 - fieldCount / 10);
} else {
overall = 0.5;
}
return makeJudgeScore(overall);
},
};
}
// ── Pure mutation functions ────────────────────────────────────
describe('addOutputField', () => {
it('appends when no position given', () => {
const s = makeSchema(['a', 'b']);
const next = addOutputField(s, makeField('c'));
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
expect(next.version).toBe(2);
});
it('inserts at given position', () => {
const s = makeSchema(['a', 'c']);
const next = addOutputField(s, makeField('b'), 1);
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
});
it('does not mutate source', () => {
const s = makeSchema(['a']);
addOutputField(s, makeField('b'));
expect(s.fields.map(f => f.name)).toEqual(['a']);
});
});
describe('removeField', () => {
it('drops the named field', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = removeField(s, 'b');
expect(next.fields.map(f => f.name)).toEqual(['a', 'c']);
expect(next.version).toBe(2);
});
it('is a no-op for unknown field', () => {
const s = makeSchema(['a', 'b']);
const next = removeField(s, 'z');
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
});
});
describe('editFieldDescription', () => {
it('updates the description of the named field', () => {
const s = makeSchema(['a']);
const next = editFieldDescription(s, 'a', 'new');
expect(next.fields[0].description).toBe('new');
});
});
describe('changeFieldType', () => {
it('updates the type of the named field', () => {
const s = makeSchema(['a']);
const next = changeFieldType(s, 'a', 'number');
expect(next.fields[0].type).toBe('number');
});
});
describe('addConstraint / removeConstraint', () => {
it('adds a constraint', () => {
const s = makeSchema(['a']);
const next = addConstraint(s, 'a', { kind: 'maxLength', value: 10 });
expect(next.fields[0].constraints).toHaveLength(1);
});
it('removes the given constraint index', () => {
const s = addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 });
const next = removeConstraint(s, 'a', 0);
expect(next.fields[0].constraints).toHaveLength(0);
});
it('is a no-op for unknown field when adding', () => {
const s = makeSchema(['a']);
const next = addConstraint(s, 'z', { kind: 'maxLength', value: 10 });
expect(next.fields[0].constraints).toHaveLength(0);
});
});
describe('reorderFields', () => {
it('reorders by name', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = reorderFields(s, ['c', 'a', 'b']);
expect(next.fields.map(f => f.name)).toEqual(['c', 'a', 'b']);
});
it('appends fields omitted from newOrder', () => {
const s = makeSchema(['a', 'b', 'c']);
const next = reorderFields(s, ['b']);
// 'b' first, then the rest in original order
expect(next.fields.map(f => f.name)).toEqual(['b', 'a', 'c']);
});
it('ignores unknown field names in newOrder', () => {
const s = makeSchema(['a', 'b']);
const next = reorderFields(s, ['unknown', 'a', 'b']);
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
});
});
describe('replaceOutputFields', () => {
it('replaces all fields', () => {
const s = makeSchema(['a', 'b']);
const next = replaceOutputFields(s, [makeField('x'), makeField('y')]);
expect(next.fields.map(f => f.name)).toEqual(['x', 'y']);
});
});
// ── Complexity + scoring ───────────────────────────────────────
describe('schemaComplexity', () => {
it('returns 0 for empty schema', () => {
expect(schemaComplexity(makeSchema([]))).toBe(0);
});
it('scales with field count, constraint count, and description length', () => {
const simple = schemaComplexity(makeSchema(['a']));
const twoField = schemaComplexity(makeSchema(['a', 'b']));
expect(twoField).toBeGreaterThan(simple);
const withConstraint = schemaComplexity(
addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 }),
);
expect(withConstraint).toBeGreaterThan(simple);
});
});
describe('aggregateSchemaScores', () => {
it('returns zero aggregate on empty results', () => {
const agg = aggregateSchemaScores([]);
expect(agg.n).toBe(0);
expect(agg.accuracy).toBe(0);
expect(agg.parseRate).toBe(0);
});
it('averages accuracy + computes parse rate', () => {
const results = [
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.8), parsed: true },
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.6), parsed: true },
{ input: 'c', expected: 'C', actual: 'X', score: makeJudgeScore(0.2), parsed: false },
];
const agg = aggregateSchemaScores(results);
expect(agg.n).toBe(3);
expect(agg.accuracy).toBeCloseTo((0.8 + 0.6 + 0.2) / 3, 5);
expect(agg.parseRate).toBeCloseTo(2 / 3, 5);
});
it('surfaces worst-3 example feedback', () => {
const results = [
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.9, 'best'), parsed: true },
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.1, 'worst'), parsed: true },
{ input: 'c', expected: 'C', actual: 'C', score: makeJudgeScore(0.5, 'middle'), parsed: true },
];
const agg = aggregateSchemaScores(results);
expect(agg.weaknessFeedback).toContain('worst');
expect(agg.weaknessFeedback).toContain('middle');
});
});
// ── Pareto ─────────────────────────────────────────────────────
describe('paretoFrontSchema', () => {
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
});
it('removes strictly dominated candidates', () => {
const dominated = makeCandidate('d', makeSchema(['a']), makeScore(0.5, 5));
const better = makeCandidate('b', makeSchema(['a']), makeScore(0.8, 3));
const front = paretoFrontSchema([dominated, better]);
expect(front.map(c => c.id)).toEqual(['b']);
});
it('keeps trade-off candidates (accurate vs simple)', () => {
const accurate = makeCandidate('acc', makeSchema(['a', 'b']), makeScore(0.9, 10));
const simple = makeCandidate('sim', makeSchema(['a']), makeScore(0.7, 2));
const front = paretoFrontSchema([accurate, simple]);
expect(front).toHaveLength(2);
});
it('ignores unscored candidates', () => {
const scored = makeCandidate('s', makeSchema(['a']), makeScore(0.5, 1));
const unscored = makeCandidate('u', makeSchema(['a']), null);
const front = paretoFrontSchema([scored, unscored]);
expect(front).toHaveLength(1);
});
});
describe('pickSchemaWinner', () => {
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
});
it('picks highest accuracy', () => {
const lo = makeCandidate('lo', makeSchema(['a']), makeScore(0.5, 1));
const hi = makeCandidate('hi', makeSchema(['a']), makeScore(0.9, 10));
expect(pickSchemaWinner([lo, hi]).id).toBe('hi');
});
it('ties on accuracy → prefers lower complexity', () => {
const complex = makeCandidate('cx', makeSchema(['a', 'b']), makeScore(0.8, 10));
const simple = makeCandidate('sm', makeSchema(['a']), makeScore(0.8, 3));
expect(pickSchemaWinner([complex, simple]).id).toBe('sm');
});
it('throws on empty', () => {
expect(() => pickSchemaWinner([])).toThrow();
});
});
// ── Mutation generators ────────────────────────────────────────
describe('generateStructureMutations', () => {
const rng = () => 0.5;
it('suggests adding reasoning if absent', () => {
const s = makeSchema(['answer']);
const muts = generateStructureMutations(s, 5, rng);
expect(muts.some(m => m.description.includes('reasoning'))).toBe(true);
});
it('does not re-suggest reasoning if already present', () => {
const s = makeSchema(['reasoning', 'answer']);
const muts = generateStructureMutations(s, 5, rng);
const hasReasoning = muts.some(m => m.description.includes('reasoning field'));
expect(hasReasoning).toBe(false);
});
it('suggests drop when schema is large', () => {
const s = makeSchema(['a', 'b', 'c', 'd', 'e']);
const muts = generateStructureMutations(s, 5, rng);
expect(muts.some(m => m.kind === 'remove_field')).toBe(true);
});
it('returns at most n mutations', () => {
const s = makeSchema(['answer']);
const muts = generateStructureMutations(s, 1, rng);
expect(muts.length).toBeLessThanOrEqual(1);
});
});
describe('generateOrderMutations', () => {
const rng = () => 0.5;
it('returns empty for 1-field schemas', () => {
expect(generateOrderMutations(makeSchema(['a']), 3, rng)).toEqual([]);
});
it('moves a reasoning field to the front', () => {
const s: Schema = {
name: 't', version: 1,
fields: [makeField('answer'), makeField('reasoning'), makeField('confidence')],
};
const muts = generateOrderMutations(s, 3, rng);
const applied = muts[0].apply(s);
expect(applied.fields[0].name).toBe('reasoning');
});
it('moves a confidence field to the end', () => {
const s: Schema = {
name: 't', version: 1,
fields: [makeField('confidence'), makeField('reasoning'), makeField('answer')],
};
const muts = generateOrderMutations(s, 3, rng);
const last = muts[0].apply(s);
// One of the generated mutations should put 'confidence' at the end
const someMovesConfidenceToEnd = muts.some(m => {
const applied = m.apply(s);
return applied.fields[applied.fields.length - 1].name === 'confidence';
});
expect(someMovesConfidenceToEnd).toBe(true);
});
});
describe('generateRefinementMutations', () => {
it('adds maxLength when feedback mentions verbosity', async () => {
const s = makeSchema(['reasoning', 'answer']);
const muts = await generateRefinementMutations(
s, ['too verbose, output wordy'], 5,
);
expect(muts.some(m => m.kind === 'add_constraint')).toBe(true);
});
it('adds minLength when feedback mentions incompleteness', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['response is too brief, missing detail'], 5,
);
expect(muts.some(m =>
m.description.includes('minLength'),
)).toBe(true);
});
it('edits field descriptions when feedback mentions format issues', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['wrong format, could not parse'], 5,
);
expect(muts.some(m => m.kind === 'edit_field_desc')).toBe(true);
});
it('falls back to a single description rewrite when no heuristic matches', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(s, ['something unrelated'], 5);
expect(muts.length).toBeGreaterThan(0);
expect(muts[0].kind).toBe('edit_field_desc');
});
it('uses LLM-provided editor when supplied', async () => {
const s = makeSchema(['answer']);
const muts = await generateRefinementMutations(
s, ['wrong format'], 5,
async ({ field }) => `LLM-REWRITE of ${field.name}`,
);
const edit = muts.find(m => m.kind === 'edit_field_desc');
const applied = edit!.apply(s);
expect(applied.fields[0].description).toBe('LLM-REWRITE of answer');
});
});
// ── pickSample ─────────────────────────────────────────────────
describe('pickSample', () => {
it('returns up to k items', () => {
expect(pickSample(makeExamples(5), 3, () => 0.5)).toHaveLength(3);
});
it('returns empty on k=0', () => {
expect(pickSample(makeExamples(5), 0, () => 0.5)).toEqual([]);
});
});
// ── EvolveSchema.run end-to-end ───────────────────────────────
describe('EvolveSchema.run', () => {
it('runs through all phases and returns a winner', async () => {
const baseline = makeSchema(['answer']);
const result = await new EvolveSchema().run({
baseline,
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 1,
evalSize: 5,
anchorEvalSize: 10,
});
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
expect(result.history.length).toBeGreaterThan(1);
});
it('winner accuracy >= baseline accuracy when judge prefers richer schemas', async () => {
const baseline = makeSchema(['answer']);
const result = await new EvolveSchema().run({
baseline,
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 2,
evalSize: 5,
anchorEvalSize: 10,
});
expect(result.winner.score!.accuracy).toBeGreaterThanOrEqual(
result.history[0].score!.accuracy,
);
});
it('emits progress events for each phase', async () => {
const phases: string[] = [];
await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 1,
evalSize: 3,
anchorEvalSize: 3,
onProgress: (e) => phases.push(e.phase),
});
expect(phases).toContain('start');
expect(phases).toContain('structure');
expect(phases).toContain('order');
expect(phases).toContain('refinement');
expect(phases).toContain('anchor');
expect(phases).toContain('done');
});
it('is deterministic with same seed', async () => {
const cfg = () => ({
baseline: makeSchema(['answer']),
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge('prefers-more'),
populationSize: 3,
generations: 1,
evalSize: 5,
anchorEvalSize: 8,
seed: 7,
});
const a = await new EvolveSchema().run(cfg());
const b = await new EvolveSchema().run(cfg());
expect(a.winner.schema.fields.map(f => f.name)).toEqual(
b.winner.schema.fields.map(f => f.name),
);
});
it('respects abort signal', async () => {
const ctrl = new AbortController();
ctrl.abort();
const result = await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(10),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 2,
evalSize: 3,
anchorEvalSize: 3,
signal: ctrl.signal,
});
expect(result.winner).toBeDefined();
});
it('records all candidates in history with generation numbers', async () => {
const result = await new EvolveSchema().run({
baseline: makeSchema(['answer']),
examples: makeExamples(5),
execute: makeFakeExecutor(),
judge: makeJudge(),
populationSize: 2,
generations: 2,
mutationMix: { structure: 2, order: 0, refinement: 0 },
evalSize: 3,
anchorEvalSize: 3,
});
const gen0 = result.history.filter(c => c.generation === 0);
const gen1 = result.history.filter(c => c.generation === 1);
expect(gen0.length).toBe(1);
expect(gen1.length).toBeGreaterThan(0);
});
});

View File

@@ -1,226 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import {
recordCapabilityGap,
analyzeAndRecordCorrection,
recordWorkflowPattern,
buildAwarenessSummary,
formatAwarenessPrompt,
markSummarySurfaced,
} from '../src/improvement-detector.js';
describe('improvement-detector', () => {
let db: MindDB;
let store: ImprovementSignalStore;
beforeEach(() => {
db = new MindDB(':memory:');
store = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
});
// ── recordCapabilityGap ────────────────────────────────────
describe('recordCapabilityGap', () => {
it('records a capability gap signal', () => {
recordCapabilityGap(store, 'pdf_reader', 'user asked to read a PDF');
const gaps = store.getByCategory('capability_gap');
expect(gaps).toHaveLength(1);
expect(gaps[0].pattern_key).toBe('missing:pdf_reader');
expect(gaps[0].count).toBe(1);
});
it('increments count on repeated gap', () => {
recordCapabilityGap(store, 'web_search');
recordCapabilityGap(store, 'web_search');
const gaps = store.getByCategory('capability_gap');
expect(gaps).toHaveLength(1);
expect(gaps[0].count).toBe(2);
});
});
// ── analyzeAndRecordCorrection ─────────────────────────────
describe('analyzeAndRecordCorrection', () => {
it('returns null for non-correction messages', () => {
const result = analyzeAndRecordCorrection(store, 'Please write a sorting function');
expect(result).toBeNull();
expect(store.getByCategory('correction')).toHaveLength(0);
});
it('detects and records a correction', () => {
const result = analyzeAndRecordCorrection(
store,
"No, that's wrong. I told you to use markdown headers.",
);
expect(result).not.toBeNull();
expect(result!.patternKey).toBeDefined();
const corrections = store.getByCategory('correction');
expect(corrections).toHaveLength(1);
});
it('records both durable and task-local corrections', () => {
analyzeAndRecordCorrection(store, "No, always use shorter responses. I said keep it brief.");
analyzeAndRecordCorrection(store, "No, not that. Just use a list this time.");
const corrections = store.getByCategory('correction');
expect(corrections.length).toBeGreaterThanOrEqual(1);
});
});
// ── recordWorkflowPattern ──────────────────────────────────
describe('recordWorkflowPattern', () => {
it('records a workflow pattern', () => {
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
const patterns = store.getByCategory('workflow_pattern');
expect(patterns).toHaveLength(1);
expect(patterns[0].pattern_key).toBe('shape:research');
});
it('increments on repeated task shapes', () => {
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
recordWorkflowPattern(store, 'research', 'investigate market trends');
recordWorkflowPattern(store, 'research', 'review latest papers');
const patterns = store.getByCategory('workflow_pattern');
expect(patterns).toHaveLength(1);
expect(patterns[0].count).toBe(3);
});
});
// ── buildAwarenessSummary ──────────────────────────────────
describe('buildAwarenessSummary', () => {
it('returns empty summary when no actionable signals', () => {
const summary = buildAwarenessSummary(store);
expect(summary.totalActionable).toBe(0);
expect(summary.capabilityGaps).toEqual([]);
expect(summary.corrections).toEqual([]);
expect(summary.workflowPatterns).toEqual([]);
});
it('includes capability gaps above threshold', () => {
// Default threshold for capability_gap is 2
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
expect(summary.capabilityGaps).toHaveLength(1);
expect(summary.capabilityGaps[0].toolName).toBe('pdf_reader');
expect(summary.capabilityGaps[0].occurrences).toBe(2);
expect(summary.capabilityGaps[0].suggestion).toContain('pdf_reader');
});
it('includes corrections above threshold', () => {
// Default threshold for correction is 3
for (let i = 0; i < 3; i++) {
store.record('correction', 'tone:too_formal', 'Keep responses casual');
}
const summary = buildAwarenessSummary(store);
expect(summary.corrections).toHaveLength(1);
expect(summary.corrections[0].patternKey).toBe('tone:too_formal');
expect(summary.corrections[0].occurrences).toBe(3);
});
it('includes recent workflow patterns above threshold', () => {
// Default threshold for workflow_pattern is 3
for (let i = 0; i < 3; i++) {
recordWorkflowPattern(store, 'research', 'investigate topic');
}
const summary = buildAwarenessSummary(store);
expect(summary.workflowPatterns).toHaveLength(1);
expect(summary.workflowPatterns[0].patternKey).toBe('shape:research');
});
it('excludes stale workflow patterns', () => {
// Manually insert a workflow_pattern with old last_seen
const raw = db.getDatabase();
raw.prepare(`
INSERT INTO improvement_signals (category, pattern_key, count, last_seen)
VALUES ('workflow_pattern', 'shape:old_task', 5, datetime('now', '-30 days'))
`).run();
const summary = buildAwarenessSummary(store);
expect(summary.workflowPatterns).toHaveLength(0);
});
it('respects total cap of 3 actionable signals', () => {
// Create many signals above threshold
for (let i = 0; i < 5; i++) {
const key = `missing:tool_${i}`;
store.record('capability_gap', key);
store.record('capability_gap', key);
}
const summary = buildAwarenessSummary(store);
expect(summary.totalActionable).toBeLessThanOrEqual(3);
});
});
// ── formatAwarenessPrompt ──────────────────────────────────
describe('formatAwarenessPrompt', () => {
it('returns null for empty summary', () => {
const summary = buildAwarenessSummary(store);
expect(formatAwarenessPrompt(summary)).toBeNull();
});
it('formats capability gaps into prompt text', () => {
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).not.toBeNull();
expect(prompt).toContain('## Improvement Signals');
expect(prompt).toContain('pdf_reader');
expect(prompt).toContain('Missing capabilities');
});
it('formats corrections into prompt text', () => {
for (let i = 0; i < 3; i++) {
store.record('correction', 'tone:too_formal', 'Keep it casual');
}
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).toContain('Behavioral adjustments');
expect(prompt).toContain('casual');
});
it('formats workflow patterns into prompt text', () => {
for (let i = 0; i < 3; i++) {
recordWorkflowPattern(store, 'compare', 'compare options');
}
const summary = buildAwarenessSummary(store);
const prompt = formatAwarenessPrompt(summary);
expect(prompt).toContain('Recurring workflows');
expect(prompt).toContain('compare');
});
});
// ── markSummarySurfaced ────────────────────────────────────
describe('markSummarySurfaced', () => {
it('marks all signals in summary as surfaced', () => {
recordCapabilityGap(store, 'pdf_reader');
recordCapabilityGap(store, 'pdf_reader');
const summary = buildAwarenessSummary(store);
expect(summary.capabilityGaps).toHaveLength(1);
markSummarySurfaced(store, summary);
// Should not appear in next summary
const summary2 = buildAwarenessSummary(store);
expect(summary2.capabilityGaps).toHaveLength(0);
expect(summary2.totalActionable).toBe(0);
});
});
});

View File

@@ -1,153 +0,0 @@
import { describe, it, expect } from 'vitest';
import { processInteractionForImprovement } from '../src/improvement-wiring.js';
describe('improvement-wiring', () => {
const baseParams = {
workspaceId: 'ws-1',
sessionId: 'sess-1',
toolsUsed: [],
agentResponse: 'Here is the result.',
userMessage: '',
};
// ── Correction detection ─────────────────────────────────────
describe('correction detection', () => {
it('detects corrections in user messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: "No, that's wrong. I told you to use bullet points, not paragraphs.",
});
expect(result.wasCorrection).toBe(true);
expect(result.correctionDetail).toBeDefined();
expect(result.correctionDetail!.length).toBeGreaterThan(0);
});
it('returns wasCorrection false for normal messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you help me write a function to sort an array?',
});
expect(result.wasCorrection).toBe(false);
expect(result.correctionDetail).toBeUndefined();
});
it('returns wasCorrection false for very short messages', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'ok',
});
expect(result.wasCorrection).toBe(false);
});
it('detects "I said" corrections', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'I said to use markdown headers, not plain text.',
});
expect(result.wasCorrection).toBe(true);
});
it('detects "actually instead use" corrections', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Actually, instead use TypeScript for this module.',
});
expect(result.wasCorrection).toBe(true);
});
});
// ── Capability gap detection ─────────────────────────────────
describe('capability gap detection', () => {
it('identifies capability gaps from tool-not-found in response', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Read this PDF for me.',
agentResponse: 'Tool "pdf_reader" not found. I cannot read PDFs directly.',
});
expect(result.capabilityGap).toBe('pdf_reader');
});
it('identifies capability gaps from "I don\'t have a tool" in response', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you send this to Slack?',
agentResponse: "I don't have a tool for sending Slack messages.",
});
expect(result.capabilityGap).toBe('sending Slack messages');
});
it('identifies gaps from user requesting integration', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Can you send to Slack the summary?',
agentResponse: 'Let me try to do that.',
toolsUsed: [],
});
expect(result.capabilityGap).toBe('slack');
});
it('returns no gap for normal interactions', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'What is the weather today?',
agentResponse: 'The weather today is sunny and 72F.',
});
expect(result.capabilityGap).toBeUndefined();
});
});
// ── Workflow pattern detection ───────────────────────────────
describe('workflow pattern detection', () => {
it('detects multi-step workflow patterns with "then"', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'First research the competitors, then compare their features, then draft a summary report.',
});
expect(result.workflowPattern).toBeDefined();
});
it('does not flag single-step requests as workflow patterns', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Write a summary of this document.',
});
expect(result.workflowPattern).toBeUndefined();
});
it('returns undefined for simple questions', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'What is the status of the project?',
});
expect(result.workflowPattern).toBeUndefined();
});
});
// ── Combined signals ─────────────────────────────────────────
describe('combined signals', () => {
it('can detect correction and gap simultaneously', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: "No, that's wrong. I told you to use the Jira tool.",
agentResponse: 'Tool "jira_create" not found.',
});
expect(result.wasCorrection).toBe(true);
expect(result.capabilityGap).toBe('jira_create');
});
it('returns all-false for benign exchange', () => {
const result = processInteractionForImprovement({
...baseParams,
userMessage: 'Thanks, that looks great!',
agentResponse: "You're welcome! Let me know if you need anything else.",
});
expect(result.wasCorrection).toBe(false);
expect(result.capabilityGap).toBeUndefined();
expect(result.workflowPattern).toBeUndefined();
});
});
});

View File

@@ -1,655 +0,0 @@
import { describe, it, expect, vi } from 'vitest';
import {
IterativeGEPA,
paretoFront,
scoreCandidate,
aggregateScores,
pickWinner,
pickSample,
type Candidate,
type CandidateScore,
type IterativeGEPAOptions,
type MutateFn,
} from '../src/iterative-optimizer.js';
import type { EvalExample } from '../src/eval-dataset.js';
import type { JudgeScore } from '../src/judge.js';
/**
* H-09 G3 — unit tests mock the judge, so they opt into the running-judge
* safety-check bypass. Production code paths must still wrap with
* makeRunningJudge(); the check in IterativeGEPA enforces that.
*/
function runGEPA(options: Omit<IterativeGEPAOptions, 'allowBareJudge'>) {
const gepa = new IterativeGEPA();
return gepa.run({ ...options, allowBareJudge: true });
}
// ── Fixtures ───────────────────────────────────────────────────
function makeExamples(n: number): EvalExample[] {
return Array.from({ length: n }, (_, i) => ({
input: `question ${i}`,
expected_output: `answer ${i}`,
metadata: { source: 'trace' as const },
}));
}
function makeScore(overall: number, extra: Partial<JudgeScore> = {}): JudgeScore {
return {
overall,
weighted: overall,
correctness: extra.correctness ?? overall,
procedureFollowing: extra.procedureFollowing ?? overall,
conciseness: extra.conciseness ?? overall,
lengthPenalty: extra.lengthPenalty ?? 1,
feedback: extra.feedback ?? `feedback for score ${overall}`,
parsed: true,
};
}
function makeCandidate(id: string, score: CandidateScore | null): Candidate {
return {
id,
prompt: `prompt-${id}`,
generation: 0,
parent: null,
strategy: 'baseline',
score,
perExample: [],
};
}
function makeCandidateScore(
overall: number,
dims: Partial<{ correctness: number; procedureFollowing: number; conciseness: number; lengthPenalty: number; n: number; weaknessFeedback: string[] }> = {},
): CandidateScore {
return {
overall,
correctness: dims.correctness ?? overall,
procedureFollowing: dims.procedureFollowing ?? overall,
conciseness: dims.conciseness ?? overall,
lengthPenalty: dims.lengthPenalty ?? 1,
n: dims.n ?? 10,
weaknessFeedback: dims.weaknessFeedback ?? [],
};
}
// Fake judge that computes a deterministic score based on candidate text length.
// Longer prompts → higher correctness, shorter → higher conciseness.
function makeFakeJudge(bias: 'long' | 'short' | 'uniform' = 'uniform') {
return {
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
const len = args.actual.length;
const corr = bias === 'long' ? Math.min(1, len / 50) : bias === 'short' ? Math.max(0, 1 - len / 100) : 0.5;
const proc = 0.6;
const conc = bias === 'short' ? 0.9 : 0.5;
const overall = 0.5 * corr + 0.3 * proc + 0.2 * conc;
return {
overall, weighted: overall,
correctness: corr,
procedureFollowing: proc,
conciseness: conc,
lengthPenalty: 1,
feedback: `len=${len}`,
parsed: true,
};
},
};
}
// ── Pure helpers ───────────────────────────────────────────────
describe('aggregateScores', () => {
it('handles empty input', () => {
const agg = aggregateScores([]);
expect(agg.n).toBe(0);
expect(agg.overall).toBe(0);
expect(agg.weaknessFeedback).toEqual([]);
});
it('averages component dimensions', () => {
const agg = aggregateScores([
makeScore(0.8, { correctness: 0.9, procedureFollowing: 0.7, conciseness: 0.8 }),
makeScore(0.6, { correctness: 0.7, procedureFollowing: 0.5, conciseness: 0.6 }),
]);
expect(agg.n).toBe(2);
expect(agg.correctness).toBeCloseTo(0.8, 5);
expect(agg.procedureFollowing).toBeCloseTo(0.6, 5);
expect(agg.conciseness).toBeCloseTo(0.7, 5);
expect(agg.overall).toBeCloseTo(0.7, 5);
});
it('surfaces worst-3 feedback lines in weaknessFeedback', () => {
const agg = aggregateScores([
makeScore(0.9, { feedback: 'best' }),
makeScore(0.1, { feedback: 'worst' }),
makeScore(0.2, { feedback: 'second-worst' }),
makeScore(0.3, { feedback: 'third-worst' }),
makeScore(0.8, { feedback: 'good' }),
]);
// Worst 3 should contain 'worst', 'second-worst', 'third-worst' in some order
expect(agg.weaknessFeedback).toContain('worst');
expect(agg.weaknessFeedback).toContain('second-worst');
expect(agg.weaknessFeedback).toContain('third-worst');
expect(agg.weaknessFeedback).not.toContain('best');
});
it('drops empty feedback from weakness list', () => {
const agg = aggregateScores([
makeScore(0.1, { feedback: '' }),
makeScore(0.2, { feedback: 'real feedback' }),
]);
expect(agg.weaknessFeedback).toEqual(['real feedback']);
});
});
describe('paretoFront', () => {
it('keeps a single member when only one has a score', () => {
const a = makeCandidate('a', makeCandidateScore(0.8));
const b = makeCandidate('b', null);
expect(paretoFront([a, b])).toHaveLength(1);
});
it('removes strictly-dominated candidates', () => {
const weak = makeCandidate('weak', makeCandidateScore(0.5, {
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
}));
const strong = makeCandidate('strong', makeCandidateScore(0.8, {
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
}));
const front = paretoFront([weak, strong]);
expect(front).toHaveLength(1);
expect(front[0].id).toBe('strong');
});
it('keeps trade-off candidates on the front', () => {
const accurate = makeCandidate('accurate', makeCandidateScore(0.7, {
correctness: 0.9, procedureFollowing: 0.6, conciseness: 0.5,
}));
const concise = makeCandidate('concise', makeCandidateScore(0.7, {
correctness: 0.6, procedureFollowing: 0.6, conciseness: 0.9,
}));
const front = paretoFront([accurate, concise]);
expect(front).toHaveLength(2);
});
it('returns all candidates when they have identical scores', () => {
const a = makeCandidate('a', makeCandidateScore(0.5));
const b = makeCandidate('b', makeCandidateScore(0.5));
const front = paretoFront([a, b]);
expect(front).toHaveLength(2);
});
it('ignores candidates with null scores entirely', () => {
const scored = makeCandidate('scored', makeCandidateScore(0.5));
const unscored = makeCandidate('unscored', null);
const front = paretoFront([scored, unscored]);
expect(front).toHaveLength(1);
expect(front[0].id).toBe('scored');
});
});
describe('pickWinner', () => {
it('picks highest overall score', () => {
const a = makeCandidate('a', makeCandidateScore(0.6));
const b = makeCandidate('b', makeCandidateScore(0.9));
expect(pickWinner([a, b]).id).toBe('b');
});
it('breaks ties using length penalty (higher = more concise, preferred)', () => {
const verbose = makeCandidate('verbose', makeCandidateScore(0.7, { lengthPenalty: 0.6 }));
const concise = makeCandidate('concise', makeCandidateScore(0.7, { lengthPenalty: 0.9 }));
expect(pickWinner([verbose, concise]).id).toBe('concise');
});
it('breaks further ties by earliest generation', () => {
const early = makeCandidate('early', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
const late = makeCandidate('late', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
late.generation = 3;
expect(pickWinner([early, late]).id).toBe('early');
});
it('throws on empty input', () => {
expect(() => pickWinner([])).toThrow();
});
});
describe('pickSample', () => {
const rng = () => 0.5;
it('returns empty on k=0', () => {
expect(pickSample(makeExamples(10), 0, rng)).toEqual([]);
});
it('returns full set (shuffled) when k >= length', () => {
const sample = pickSample(makeExamples(5), 10, rng);
expect(sample).toHaveLength(5);
});
it('is deterministic with a fixed rng', () => {
const rngA = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
const rngB = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
const a = pickSample(makeExamples(20), 5, rngA).map(e => e.input);
const b = pickSample(makeExamples(20), 5, rngB).map(e => e.input);
expect(a).toEqual(b);
});
});
describe('scoreCandidate', () => {
it('scores candidate against examples and populates the score field', async () => {
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(3), makeFakeJudge('uniform'));
expect(score.n).toBe(3);
expect(cand.score).not.toBeNull();
expect(cand.perExample).toHaveLength(3);
});
it('returns zero-score aggregation when examples list is empty', async () => {
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, [], makeFakeJudge());
expect(score.n).toBe(0);
expect(score.overall).toBe(0);
});
it('continues iterating if one score call throws', async () => {
let callCount = 0;
const flakyJudge = {
async score(): Promise<JudgeScore> {
callCount++;
if (callCount === 2) throw new Error('transient');
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(3), flakyJudge);
expect(score.n).toBe(2); // 2 succeeded, 1 failed
});
it('stops early when abort signal fires', async () => {
const ctrl = new AbortController();
const judge = makeFakeJudge();
// Abort before any call
ctrl.abort();
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(10), judge, ctrl.signal);
expect(cand.perExample.length).toBeLessThan(10);
});
// ── Concurrency ────────────────────────────────────────────────
it('accepts an options object (new API) equivalently to AbortSignal (legacy)', async () => {
const judge = makeFakeJudge('uniform');
const candA = makeCandidate('a', null);
const candB = makeCandidate('b', null);
const ctrl = new AbortController();
ctrl.abort();
await scoreCandidate(candA, makeExamples(5), judge, ctrl.signal);
await scoreCandidate(candB, makeExamples(5), judge, { signal: ctrl.signal });
expect(candA.perExample.length).toBe(candB.perExample.length);
});
it('concurrency=1 produces identical aggregate to no option (sequential baseline)', async () => {
const examples = makeExamples(6);
const candSeq = makeCandidate('seq', null);
const candPar = makeCandidate('par', null);
const seqScore = await scoreCandidate(candSeq, examples, makeFakeJudge('uniform'));
const parScore = await scoreCandidate(candPar, examples, makeFakeJudge('uniform'), { concurrency: 1 });
expect(parScore.overall).toBeCloseTo(seqScore.overall);
expect(parScore.n).toBe(seqScore.n);
});
it('with concurrency > 1, runs scores in parallel (observable via in-flight counter)', async () => {
let inFlight = 0;
let peakInFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
// Await a microtask + a real delay so parallel workers can accumulate.
await new Promise(r => setTimeout(r, 5));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(8), judge, { concurrency: 4 });
expect(peakInFlight).toBe(4);
expect(cand.perExample).toHaveLength(8);
});
it('with concurrency=1, exactly 1 in-flight call at a time', async () => {
let inFlight = 0;
let peakInFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 3));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(5), judge, { concurrency: 1 });
expect(peakInFlight).toBe(1);
});
it('concurrency is capped at examples.length (does not start idle workers)', async () => {
let peakInFlight = 0;
let inFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 2));
inFlight--;
return makeScore(0.5);
},
};
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(3), judge, { concurrency: 100 });
expect(peakInFlight).toBeLessThanOrEqual(3);
});
it('parallel mode still filters out thrown-error results without corrupting the batch', async () => {
let call = 0;
const judge = {
async score(): Promise<JudgeScore> {
call++;
if (call % 3 === 0) throw new Error('flaky');
return makeScore(0.7);
},
};
const cand = makeCandidate('c', null);
const score = await scoreCandidate(cand, makeExamples(9), judge, { concurrency: 3 });
// 9 total, every 3rd throws → 6 succeed.
expect(score.n).toBe(6);
});
it('parallel mode respects abort signal by not dispatching further workers', async () => {
const ctrl = new AbortController();
ctrl.abort();
const judge = makeFakeJudge();
const cand = makeCandidate('c', null);
await scoreCandidate(cand, makeExamples(20), judge, {
signal: ctrl.signal,
concurrency: 4,
});
expect(cand.perExample.length).toBeLessThan(20);
});
});
// ── IterativeGEPA end-to-end with concurrency ────────────────────
describe('IterativeGEPA with concurrency', () => {
it('threads options.concurrency into every scoreCandidate call', async () => {
let peakInFlight = 0;
let inFlight = 0;
const judge = {
async score(): Promise<JudgeScore> {
inFlight++;
peakInFlight = Math.max(peakInFlight, inFlight);
await new Promise(r => setTimeout(r, 2));
inFlight--;
return makeScore(0.5);
},
};
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt} :: ${strategy}`;
await runGEPA({
baseline: 'base',
examples: makeExamples(8),
judge,
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 4,
miniEvalSize: 4,
anchorEvalSize: 4,
concurrency: 3,
});
expect(peakInFlight).toBe(3);
});
});
// ── H-09 G3 · running-judge guard ──────────────────────────────
describe('IterativeGEPA.run · running-judge guard (H-09 G3)', () => {
it('throws when a bare judge is passed without allowBareJudge', async () => {
const bareJudge = {
async score(): Promise<JudgeScore> {
return {
overall: 0.5, weighted: 0.5, correctness: 0.5,
procedureFollowing: 0.5, conciseness: 0.5, lengthPenalty: 1,
feedback: '', parsed: true,
};
},
};
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} mutated`;
await expect(
new IterativeGEPA().run({
baseline: 'seed',
examples: makeExamples(3),
judge: bareJudge,
mutate,
}),
).rejects.toThrow(/not a running judge|makeRunningJudge/);
});
it('accepts a judge wrapped with makeRunningJudge', async () => {
const { makeRunningJudge } = await import('../src/evolution-llm-wiring.js');
const baseJudge = {
async score(): Promise<JudgeScore> {
return {
overall: 0.8, weighted: 0.8, correctness: 0.8,
procedureFollowing: 0.8, conciseness: 0.8, lengthPenalty: 1,
feedback: '', parsed: true,
};
},
};
const fakeLLM = { complete: async () => 'LLM output' };
const wrapped = makeRunningJudge(baseJudge, fakeLLM);
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} v2`;
// Should not throw — the wrapped judge carries the brand.
const result = await new IterativeGEPA().run({
baseline: 'seed',
examples: makeExamples(3),
judge: wrapped,
mutate,
populationSize: 1,
generations: 1,
microScreenSize: 1, miniEvalSize: 1, anchorEvalSize: 1,
});
expect(result.winner).toBeDefined();
});
});
// ── End-to-end run ─────────────────────────────────────────────
describe('IterativeGEPA.run', () => {
it('runs through all phases and produces a winner', async () => {
const baseline = 'short baseline prompt';
const examples = makeExamples(10);
const judge = makeFakeJudge('uniform');
// Mutate appends text — simulates generation of children
const mutate: MutateFn = async ({ parent, strategy }) => {
return `${parent.prompt} :: ${strategy}`;
};
const progress: string[] = [];
const result = await runGEPA({
baseline,
examples,
judge,
mutate,
populationSize: 3,
generations: 2,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 10,
onProgress: (e) => progress.push(`${e.phase}@g${e.generation}`),
});
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
expect(result.history.length).toBeGreaterThan(1);
expect(progress).toContain('start@g0');
expect(progress.some(p => p.startsWith('anchor'))).toBe(true);
expect(progress.some(p => p.startsWith('done'))).toBe(true);
});
it('winner dominates or matches baseline on overall score', async () => {
const baseline = 'x'; // very short
const examples = makeExamples(20);
// Long-biased judge: longer prompts score higher
const judge = makeFakeJudge('long');
// Mutate doubles length each time
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} ${parent.prompt}more`;
const result = await runGEPA({
baseline,
examples,
judge,
mutate,
populationSize: 3,
generations: 2,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 15,
});
// Winner should have evolved (not baseline prompt).
expect(result.winner.prompt.length).toBeGreaterThan(baseline.length);
expect(result.improved).toBe(true);
expect(result.delta).toBeGreaterThan(0);
});
it('passes weakness feedback from parent into mutate()', async () => {
const captured: Array<{ strategy: string; feedbacks: string[] }> = [];
const mutate: MutateFn = async ({ parent, strategy, weaknessFeedback }) => {
captured.push({ strategy, feedbacks: weaknessFeedback });
return `${parent.prompt} mutated`;
};
await runGEPA({
baseline: 'baseline',
examples: makeExamples(10),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 5,
});
// 2 mutations were spawned — each received the parent's feedback
expect(captured.length).toBe(2);
for (const c of captured) {
expect(Array.isArray(c.feedbacks)).toBe(true);
// Feedback array may be empty on first gen if baseline had no weaknesses;
// we just assert the prop is present and is an array.
}
});
it('recovers when mutate throws (falls back to parent prompt)', async () => {
let mutations = 0;
const mutate: MutateFn = async ({ parent }) => {
mutations++;
if (mutations === 1) throw new Error('boom');
return `${parent.prompt} ok`;
};
const result = await runGEPA({
baseline: 'baseline',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 1,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 3,
});
// Winner must be defined even though one mutation failed
expect(result.winner).toBeDefined();
expect(result.winner.score).not.toBeNull();
});
it('respects abort signal and exits early without throwing', async () => {
const ctrl = new AbortController();
const mutate: MutateFn = vi.fn(async () => 'should never run');
ctrl.abort(); // abort before starting generations
const result = await runGEPA({
baseline: 'baseline',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate,
populationSize: 2,
generations: 2,
microScreenSize: 2,
miniEvalSize: 2,
anchorEvalSize: 2,
signal: ctrl.signal,
});
// With abort before baseline scoring, no winner should evolve
expect(result.winner).toBeDefined();
expect(mutate).not.toHaveBeenCalled();
});
it('is deterministic with same seed', async () => {
const examples = makeExamples(20);
const judge = makeFakeJudge();
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt}::${strategy}`;
const run = () => runGEPA({
baseline: 'seed',
examples,
judge,
mutate,
populationSize: 3,
generations: 1,
microScreenSize: 5,
miniEvalSize: 5,
anchorEvalSize: 10,
seed: 42,
});
const r1 = await run();
const r2 = await run();
expect(r1.winner.prompt).toBe(r2.winner.prompt);
expect(r1.history.length).toBe(r2.history.length);
});
it('produces history containing baseline and all mutated children', async () => {
const result = await runGEPA({
baseline: 'base',
examples: makeExamples(5),
judge: makeFakeJudge(),
mutate: async ({ parent, strategy }) => `${parent.prompt}_${strategy}`,
populationSize: 3,
generations: 2,
microScreenSize: 3,
miniEvalSize: 3,
anchorEvalSize: 3,
});
expect(result.history[0].id).toBe('g0-baseline');
expect(result.history[0].strategy).toBe('baseline');
// 2 generations * 3 mutations + 1 baseline = 7 total
expect(result.history.length).toBe(7);
});
});

View File

@@ -1,280 +0,0 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB, OptimizationLogStore } from '@waggle/core';
import {
captureInteraction,
getRecentLogs,
getWorkspaceLogs,
isWithinBudget,
} from '../src/optimization-capture.js';
import type { AgentResponse } from '../src/agent-loop.js';
describe('optimization-capture', () => {
let db: MindDB;
let store: OptimizationLogStore;
beforeEach(() => {
db = new MindDB(':memory:');
store = new OptimizationLogStore(db);
});
afterEach(() => {
db.close();
});
function makeResponse(overrides?: Partial<AgentResponse>): AgentResponse {
return {
content: 'Hello, I can help with that.',
toolsUsed: ['search_memory', 'save_memory'],
usage: { inputTokens: 500, outputTokens: 200 },
...overrides,
};
}
// ── captureInteraction ─────────────────────────────────────
describe('captureInteraction', () => {
it('stores a log entry with correct fields', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'You are a helpful assistant.',
response: makeResponse(),
turnCount: 3,
wasCorrection: false,
});
expect(entry.id).toBeGreaterThan(0);
expect(entry.session_id).toBe('session-1');
expect(entry.workspace_id).toBe('ws-abc');
expect(entry.system_prompt).toBe('You are a helpful assistant.');
expect(JSON.parse(entry.tools_used)).toEqual(['search_memory', 'save_memory']);
expect(entry.turn_count).toBe(3);
expect(entry.was_correction).toBe(0);
expect(entry.input_tokens).toBe(500);
expect(entry.output_tokens).toBe(200);
expect(entry.timestamp).toBeTruthy();
});
it('records wasCorrection as 1 when true', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'test prompt',
response: makeResponse(),
turnCount: 1,
wasCorrection: true,
});
expect(entry.was_correction).toBe(1);
});
it('handles empty toolsUsed', () => {
const entry = captureInteraction(store, {
sessionId: 'session-1',
workspaceId: 'ws-abc',
systemPrompt: 'test prompt',
response: makeResponse({ toolsUsed: [] }),
turnCount: 1,
wasCorrection: false,
});
expect(JSON.parse(entry.tools_used)).toEqual([]);
});
it('defaults token counts to 0 when missing', () => {
const response: AgentResponse = {
content: 'response',
toolsUsed: [],
usage: { inputTokens: 0, outputTokens: 0 },
};
const entry = captureInteraction(store, {
sessionId: 's1',
workspaceId: 'ws1',
systemPrompt: 'prompt',
response,
turnCount: 1,
wasCorrection: false,
});
expect(entry.input_tokens).toBe(0);
expect(entry.output_tokens).toBe(0);
});
});
// ── getRecentLogs ──────────────────────────────────────────
describe('getRecentLogs', () => {
it('retrieves stored entries ordered by most recent first', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p1',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p2',
response: makeResponse(), turnCount: 2, wasCorrection: true,
});
const logs = getRecentLogs(store, 10);
expect(logs).toHaveLength(2);
// Ordered by timestamp DESC, id DESC — s2 was inserted second so it comes first
expect(logs[0].session_id).toBe('s2');
expect(logs[1].session_id).toBe('s1');
});
it('respects the limit parameter', () => {
for (let i = 0; i < 5; i++) {
captureInteraction(store, {
sessionId: `s${i}`, workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
}
const logs = getRecentLogs(store, 3);
expect(logs).toHaveLength(3);
});
it('returns empty array when no entries exist', () => {
const logs = getRecentLogs(store);
expect(logs).toEqual([]);
});
});
// ── getWorkspaceLogs ───────────────────────────────────────
describe('getWorkspaceLogs', () => {
it('filters by workspace ID', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws-a', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws-b', systemPrompt: 'p',
response: makeResponse(), turnCount: 1, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's3', workspaceId: 'ws-a', systemPrompt: 'p',
response: makeResponse(), turnCount: 2, wasCorrection: true,
});
const wsALogs = getWorkspaceLogs(store, 'ws-a');
expect(wsALogs).toHaveLength(2);
expect(wsALogs.every(l => l.workspace_id === 'ws-a')).toBe(true);
const wsBLogs = getWorkspaceLogs(store, 'ws-b');
expect(wsBLogs).toHaveLength(1);
});
});
// ── Entries have correct shape ─────────────────────────────
describe('entry shape', () => {
it('has all required fields with correct types', () => {
const entry = captureInteraction(store, {
sessionId: 'session-x',
workspaceId: 'workspace-y',
systemPrompt: 'You are helpful.',
response: makeResponse({ toolsUsed: ['bash', 'read_file'] }),
turnCount: 7,
wasCorrection: false,
});
// Check all fields exist and are of expected types
expect(typeof entry.id).toBe('number');
expect(typeof entry.session_id).toBe('string');
expect(typeof entry.workspace_id).toBe('string');
expect(typeof entry.system_prompt).toBe('string');
expect(typeof entry.tools_used).toBe('string');
expect(typeof entry.turn_count).toBe('number');
expect(typeof entry.was_correction).toBe('number');
expect(typeof entry.input_tokens).toBe('number');
expect(typeof entry.output_tokens).toBe('number');
expect(typeof entry.timestamp).toBe('string');
// tools_used is valid JSON array
const parsed = JSON.parse(entry.tools_used);
expect(Array.isArray(parsed)).toBe(true);
expect(parsed).toEqual(['bash', 'read_file']);
});
});
// ── Budget enforcement ─────────────────────────────────────
describe('isWithinBudget', () => {
it('returns true when no entries exist (under budget)', () => {
expect(isWithinBudget(store, 100)).toBe(true);
});
it('returns true when token cost is under budget', () => {
// Insert entry with small token count
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
turnCount: 1, wasCorrection: false,
});
// 100 input * $3/M + 50 output * $15/M = $0.0003 + $0.00075 = ~$0.00105 = ~0.1 cents
// Budget is 100 cents = $1, so well within budget
expect(isWithinBudget(store, 100)).toBe(true);
});
it('returns false when token cost exceeds budget', () => {
// Insert entry with large token count to exceed a tiny budget
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 1_000_000, outputTokens: 500_000 } }),
turnCount: 1, wasCorrection: false,
});
// 1M input * $3/M + 500K output * $15/M = $3 + $7.5 = $10.50 = 1050 cents
// Budget is 1 cent, so way over budget
expect(isWithinBudget(store, 1)).toBe(false);
});
it('respects custom cost-per-token rates', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 10_000, outputTokens: 5_000 } }),
turnCount: 1, wasCorrection: false,
});
// With very cheap rates: 10K * $0.0001/M + 5K * $0.0001/M ≈ ~0 cents
expect(isWithinBudget(store, 1, 0.0001 / 1_000_000, 0.0001 / 1_000_000)).toBe(true);
// With very expensive rates: 10K * $100/M + 5K * $100/M = $1 + $0.5 = 150 cents
expect(isWithinBudget(store, 1, 100 / 1_000_000, 100 / 1_000_000)).toBe(false);
});
});
// ── Store stats ────────────────────────────────────────────
describe('getStats (via store)', () => {
it('computes correct aggregate stats', () => {
captureInteraction(store, {
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
turnCount: 3, wasCorrection: false,
});
captureInteraction(store, {
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p',
response: makeResponse({ usage: { inputTokens: 200, outputTokens: 100 } }),
turnCount: 5, wasCorrection: true,
});
const stats = store.getStats();
expect(stats.total).toBe(2);
expect(stats.correctionRate).toBe(0.5); // 1/2
expect(stats.avgTurnCount).toBe(4); // (3+5)/2
expect(stats.totalInputTokens).toBe(300); // 100+200
expect(stats.totalOutputTokens).toBe(150); // 50+100
});
it('returns zeros when empty', () => {
const stats = store.getStats();
expect(stats.total).toBe(0);
expect(stats.correctionRate).toBe(0);
expect(stats.avgTurnCount).toBe(0);
expect(stats.totalInputTokens).toBe(0);
expect(stats.totalOutputTokens).toBe(0);
});
});
});

View File

@@ -2,10 +2,9 @@ import { describe, it, expect } from 'vitest';
import { PERSONAS, getPersona, listPersonas, composePersonaPrompt } from '../src/personas.js';
describe('Agent Personas', () => {
it('PERSONAS catalog has 23 entries', () => {
// 22 user-facing (8 universal + 14 specialists) + 1 internal review persona
// (session-reviewer) used only by the IdleSessionWatcher self-evolution loop.
expect(PERSONAS).toHaveLength(23);
it('PERSONAS catalog has 22 entries', () => {
// 8 universal + 14 specialists.
expect(PERSONAS).toHaveLength(22);
});
it('each persona has required fields', () => {
@@ -40,9 +39,9 @@ describe('Agent Personas', () => {
expect(getPersona('')).toBeNull();
});
it('listPersonas() returns all 23 personas', () => {
it('listPersonas() returns all 22 personas', () => {
const list = listPersonas();
expect(list).toHaveLength(23);
expect(list).toHaveLength(22);
expect(list).not.toBe(PERSONAS); // Returns a copy
});
@@ -208,19 +207,6 @@ describe('Soft tool model', () => {
expect(v!.disallowedTools).toContain('write_file');
expect(v!.isReadOnly).toBe(true);
});
it('session-reviewer is a trust-boundary persona: reads + create_skill only, no writes/exec/memory', () => {
const r = getPersona('session-reviewer');
expect(r, 'session-reviewer not found').toBeTruthy();
// create_skill is the proposal vehicle and must stay in the allowlist.
expect(r!.tools).toContain('create_skill');
// isReadOnly would strip create_skill, so it must be false.
expect(r!.isReadOnly).toBe(false);
// The critical tools ALWAYS_AVAILABLE_TOOLS re-adds must be explicitly denied.
for (const t of ['save_memory', 'delete_skill', 'install_capability', 'acquire_capability', 'spawn_agent', 'bash']) {
expect(r!.disallowedTools, `${t} must be disallowed`).toContain(t);
}
});
});
describe('New domain personas', () => {
@@ -239,8 +225,7 @@ describe('New domain personas', () => {
});
}
it('total persona count is 23', () => {
// 22 tiered + session-reviewer (internal self-evolution reviewer).
expect(PERSONAS.length).toBe(23);
it('total persona count is 22', () => {
expect(PERSONAS.length).toBe(22);
});
});

View File

@@ -3,13 +3,12 @@
*
* Tests the one-step promotion personal → workspace, the refusal of team and
* enterprise scopes in this edition, frontmatter rewrite with promoted_from
* history, file move, and skill_promotion improvement signal emission.
* history and file move.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import { createSkillTools, getSkillDirForScope } from '../src/skill-tools.js';
import { parseSkillFrontmatter } from '../src/skill-frontmatter.js';
@@ -22,8 +21,6 @@ function seedSkill(dir: string, name: string, scope: string | null, body = 'body
describe('promote_skill — gap E', () => {
let tmpDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
let reloadCount: number;
function createTools(overrides: Partial<Parameters<typeof createSkillTools>[0]> = {}) {
@@ -32,7 +29,6 @@ describe('promote_skill — gap E', () => {
onSkillsChanged: () => { reloadCount++; },
getWorkspaceId: () => 'ws-alpha',
getTeamId: () => 'team-bravo',
improvementSignals: signals,
...overrides,
});
}
@@ -45,18 +41,15 @@ describe('promote_skill — gap E', () => {
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-promote-test-'));
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
reloadCount = 0;
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
describe('happy path', () => {
it('promotes personal → workspace, moves file, appends history, records signal', async () => {
it('promotes personal → workspace, moves file, appends history', async () => {
const personalDir = getSkillDirForScope(tmpDir, 'personal', {})!;
seedSkill(personalDir, 'deploy-helper', 'personal');
const tools = createTools();
@@ -64,7 +57,6 @@ describe('promote_skill — gap E', () => {
const out = await run(tools, 'promote_skill', { skill_name: 'deploy-helper' });
expect(out).toContain('personal → workspace');
expect(out).toContain('Signal recorded');
// File moved
expect(fs.existsSync(path.join(personalDir, 'deploy-helper.md'))).toBe(false);
@@ -77,11 +69,6 @@ describe('promote_skill — gap E', () => {
expect(parsed.frontmatter.scope).toBe('workspace');
expect(parsed.frontmatter.promoted_from).toEqual(['personal']);
// Signal emitted
const recorded = signals.getByCategory('skill_promotion');
expect(recorded.length).toBe(1);
expect(recorded[0].pattern_key).toBe('promote:deploy-helper:personal->workspace');
// onSkillsChanged fired
expect(reloadCount).toBe(1);
});
@@ -125,7 +112,6 @@ describe('promote_skill — gap E', () => {
const out = await run(createTools(), 'promote_skill', { skill_name: 'shared-helper' });
expect(out).toContain('not available in this edition');
expect(fs.existsSync(path.join(wsDir, 'shared-helper.md'))).toBe(true);
expect(signals.getByCategory('skill_promotion')).toHaveLength(0);
});
it('refuses enterprise promotion', async () => {

View File

@@ -1,8 +1,7 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
loadSystemPrompt,
loadSystemPromptWithOverrides,
assertOverridesReachActiveSpec,
loadSkills,
} from '../src/prompt-loader.js';
import * as fs from 'node:fs';
@@ -68,7 +67,6 @@ describe('Prompt Loader', () => {
});
});
// ── H-08 G2 — override-aware loader ──────────────────────────
describe('loadSystemPromptWithOverrides', () => {
it('returns baseline when no overrides exist', () => {
const composed = loadSystemPromptWithOverrides(tmpDir);
@@ -86,55 +84,17 @@ describe('Prompt Loader', () => {
expect(composed.userPromptMd).toBe('hello from user');
});
it('applies behavioral-spec overrides into the active spec', () => {
it('ignores override files left on disk by earlier versions', () => {
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(overrideDir, { recursive: true });
fs.writeFileSync(
path.join(overrideDir, 'coreLoop.json'),
JSON.stringify({
section: 'coreLoop',
text: 'CUSTOM CORE LOOP CONTENT FOR TEST',
deployedAt: new Date().toISOString(),
}),
JSON.stringify({ section: 'coreLoop', text: 'STALE OVERRIDE TEXT', deployedAt: new Date().toISOString() }),
);
const composed = loadSystemPromptWithOverrides(tmpDir);
const flattened = Object.values(composed.behavioralSpec).join('\n');
expect(flattened).toContain('CUSTOM CORE LOOP CONTENT FOR TEST');
});
});
describe('assertOverridesReachActiveSpec', () => {
it('is quiet when no overrides are deployed', () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
const composed = loadSystemPromptWithOverrides(tmpDir);
assertOverridesReachActiveSpec(tmpDir, composed.behavioralSpec);
expect(warnSpy).not.toHaveBeenCalled();
warnSpy.mockRestore();
});
it('warns when disk has overrides but activeSpec does not contain them', () => {
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
fs.mkdirSync(overrideDir, { recursive: true });
fs.writeFileSync(
path.join(overrideDir, 'coreLoop.json'),
JSON.stringify({
section: 'coreLoop',
text: 'UNIQUE MARKER STRING FOR G2 TEST — LONG ENOUGH TO MATCH',
deployedAt: new Date().toISOString(),
}),
);
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
// Simulate using the BARE loader — activeSpec is the compile-time
// baseline with no overrides folded in.
const fakeActiveSpec = loadSystemPromptWithOverrides(path.join(tmpDir, 'nothing')).behavioralSpec;
assertOverridesReachActiveSpec(tmpDir, fakeActiveSpec);
expect(warnSpy).toHaveBeenCalled();
const message = warnSpy.mock.calls[0]?.[0] as string;
expect(message).toContain('coreLoop');
warnSpy.mockRestore();
expect(flattened).not.toContain('STALE OVERRIDE TEXT');
});
});
});

View File

@@ -2,13 +2,8 @@
* Skills 2.0 gap K — write-path conflict detection.
*
* When save_memory writes a new frame that contradicts an existing frame
* (detectContradiction), the save still succeeds but:
* - the return string carries a [flag: contradicts_existing …] marker
* - a correction-category improvement signal is recorded
*
* Previously detectContradiction existed but was never called on the write
* path — write-path conflicts silently overwrote reality, degrading the
* evolution-signal loop.
* (detectContradiction), the save still succeeds but the return string
* carries a [flag: contradicts_existing …] marker.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
@@ -19,7 +14,6 @@ import {
SessionStore,
KnowledgeGraph,
HybridSearch,
ImprovementSignalStore,
} from '@waggle/core';
import { createMindTools } from '../src/tools.js';
import type { ToolDefinition } from '../src/tools.js';
@@ -30,7 +24,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
let frames: FrameStore;
let sessions: SessionStore;
let tools: ToolDefinition[];
let improvementSignals: ImprovementSignalStore;
function getSaveMemory(): ToolDefinition {
const tool = tools.find(t => t.name === 'save_memory');
@@ -45,7 +38,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
sessions = new SessionStore(db);
const knowledge = new KnowledgeGraph(db);
const search = new HybridSearch(db, embedder);
improvementSignals = new ImprovementSignalStore(db);
tools = createMindTools({
db,
@@ -55,7 +47,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
sessions,
search,
knowledge,
improvementSignals,
// Intentionally NO cognify → exercises the raw-frame fallback path
// (no LLM-dependent entity extraction in unit tests).
});
@@ -94,17 +85,9 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
expect(second).toContain('Memory saved');
// But the return message is annotated
expect(second).toContain('[flag: contradicts_existing');
// And a correction signal is recorded
const top = improvementSignals.getByCategory('correction');
expect(top.length).toBeGreaterThan(0);
const writeConflict = top.find(s => s.pattern_key.startsWith('write-conflict:'));
expect(writeConflict).toBeDefined();
expect(writeConflict!.detail).toMatch(/contradicts existing frame/i);
expect(writeConflict!.category).toBe('correction');
});
it('non-contradicting saves do not flag or emit a signal', async () => {
it('non-contradicting saves are not flagged', async () => {
const save = getSaveMemory();
await save.execute({
content: 'Decision: we chose PostgreSQL for analytics; approved and confirmed.',
@@ -117,34 +100,5 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
source: 'user_stated',
});
expect(result).not.toContain('contradicts_existing');
const top = improvementSignals.getByCategory('correction');
expect(top.find(s => s.pattern_key.startsWith('write-conflict:'))).toBeUndefined();
});
it('works without improvementSignals dep (backwards-compatible)', async () => {
// Rebuild tools without signals dep
const db2 = new MindDB(':memory:');
const embedder = new MockEmbedder();
const frames2 = new FrameStore(db2);
const sessions2 = new SessionStore(db2);
const knowledge2 = new KnowledgeGraph(db2);
const search2 = new HybridSearch(db2, embedder);
const tools2 = createMindTools({
db: db2,
identity: new IdentityLayer(db2),
awareness: new AwarenessLayer(db2),
frames: frames2,
sessions: sessions2,
search: search2,
knowledge: knowledge2,
// No improvementSignals — should not throw
});
const save = tools2.find(t => t.name === 'save_memory')!;
await save.execute({ content: 'Decision: enable feature flag rollout for users; proceed.', target: 'personal' });
const result = await save.execute({ content: 'Decision: disable feature flag rollout for users; cancel.', target: 'personal' });
// Flag still appears (detectContradiction doesn't need the signal store)
expect(result).toContain('contradicts_existing');
db2.close();
});
});

View File

@@ -0,0 +1,77 @@
import { describe, it, expect } from 'vitest';
import { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from '../src/secret-patterns.js';
describe('detectSecrets', () => {
it('returns null for clean text', () => {
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
});
it('detects AWS access keys', () => {
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
});
it('detects GitHub PATs', () => {
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
});
it('detects OpenAI keys', () => {
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
});
it('detects Anthropic keys', () => {
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
});
it('detects Google API keys', () => {
// Google API keys = literal "AIza" + exactly 35 chars
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
});
it('detects Stripe keys', () => {
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
});
it('detects private key blocks', () => {
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
});
it('detects JWT tokens', () => {
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
expect(detectSecrets(jwt)).toBe('jwt');
});
it('detects basic-auth URLs', () => {
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
.toBe('basic-auth-url');
});
it('detects env-var password assignments', () => {
expect(detectSecrets('PASSWORD=supersecret123'))
.toBe('env-password');
});
it('detects postgres connection strings with creds', () => {
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
.toBe('pgsql-url');
});
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
});
});
describe('redactSecrets', () => {
it('replaces every match and reports the patterns that fired', () => {
const { text, found } = redactSecrets('key AKIAIOSFODNN7EXAMPLE and PASSWORD=supersecret123');
expect(text).toContain('[REDACTED:aws-access-key]');
expect(text).toContain('[REDACTED:env-password]');
expect(text).not.toContain('AKIAIOSFODNN7EXAMPLE');
expect(found.sort()).toEqual(['aws-access-key', 'env-password']);
});
it('leaves clean text untouched', () => {
expect(redactSecrets('nothing to hide')).toEqual({ text: 'nothing to hide', found: [] });
});
});

View File

@@ -90,59 +90,4 @@ describe('Self-Awareness', () => {
expect(result).toContain('acquire_capability');
expect(result).toContain('uncertain');
});
it('includes improvement signals when awareness has actionable data', () => {
const caps: AgentCapabilities = {
tools: [],
skills: [],
model: 'test',
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
mode: 'local',
version: '1.0.0',
awareness: {
capabilityGaps: [{
id: 1,
toolName: 'pdf_reader',
occurrences: 3,
suggestion: 'User has needed "pdf_reader" 3 times. Consider using acquire_capability to find it.',
}],
corrections: [{
id: 2,
patternKey: 'tone:too_formal',
detail: 'Keep it casual',
occurrences: 4,
guidance: 'Keep it casual (corrected 4 times)',
}],
workflowPatterns: [],
totalActionable: 2,
},
};
const result = buildSelfAwareness(caps);
expect(result).toContain('Learning from Past Sessions');
expect(result).toContain('pdf_reader');
expect(result).toContain('Keep it casual');
expect(result).toContain('Recurring capability gaps');
expect(result).toContain('User corrections to apply');
});
it('omits improvement section when awareness has zero actionable signals', () => {
const caps: AgentCapabilities = {
tools: [],
skills: [],
model: 'test',
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
mode: 'local',
version: '1.0.0',
awareness: {
capabilityGaps: [],
corrections: [],
workflowPatterns: [],
totalActionable: 0,
},
};
const result = buildSelfAwareness(caps);
expect(result).not.toContain('Learning from Past Sessions');
});
});

View File

@@ -9,7 +9,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import {
autoExtractAndCreateSkill,
skillFilename,
@@ -35,24 +34,18 @@ function repeatedSession(): AutoExtractMessage[] {
describe('autoExtractAndCreateSkill — gap A', () => {
let tmpDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-autoextract-'));
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('creates a personal-scope SKILL.md from a repeated workflow', async () => {
const result = await autoExtractAndCreateSkill(repeatedSession(), {
personalSkillsDir: tmpDir,
improvementSignals: signals,
});
expect(result.filePath).not.toBeNull();
@@ -67,19 +60,6 @@ describe('autoExtractAndCreateSkill — gap A', () => {
expect(body).toContain('## Tools Used');
});
it('records a workflow_pattern signal with auto-extract pattern_key', async () => {
const result = await autoExtractAndCreateSkill(repeatedSession(), {
personalSkillsDir: tmpDir,
improvementSignals: signals,
});
expect(result.filePath).not.toBeNull();
const recorded = signals.getByCategory('workflow_pattern');
const autoExtract = recorded.find(s => s.pattern_key.startsWith('auto-extract:'));
expect(autoExtract).toBeDefined();
expect(autoExtract!.detail).toContain('Auto-extracted');
});
it('returns no_pattern for short or random sessions', async () => {
const messages: AutoExtractMessage[] = [
{ role: 'user', content: 'Hi' },

View File

@@ -5,7 +5,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB, ImprovementSignalStore } from '@waggle/core';
import { retireStaleSkills } from '../src/skill-retirement.js';
import {
recordSkillUsage,
@@ -74,8 +73,6 @@ describe('skill-usage — gap F', () => {
describe('retireStaleSkills — gap F', () => {
let waggleHome: string;
let skillsDir: string;
let db: MindDB;
let signals: ImprovementSignalStore;
const NOW = new Date('2026-04-15T00:00:00Z');
const DAY = 24 * 60 * 60 * 1000;
@@ -83,12 +80,9 @@ describe('retireStaleSkills — gap F', () => {
beforeEach(() => {
waggleHome = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-retire-'));
skillsDir = path.join(waggleHome, 'skills');
db = new MindDB(':memory:');
signals = new ImprovementSignalStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(waggleHome, {
recursive: true,
force: true,
@@ -110,7 +104,6 @@ describe('retireStaleSkills — gap F', () => {
const report = retireStaleSkills(waggleHome, {
maxIdleDays: 90,
now: () => NOW,
improvementSignals: signals,
});
expect(report.scanned).toBe(2);
@@ -123,10 +116,6 @@ describe('retireStaleSkills — gap F', () => {
// Usage entry for the retired skill is cleared
expect(loadSkillUsage(waggleHome).stale).toBeUndefined();
expect(loadSkillUsage(waggleHome).fresh).toBeDefined();
// Signal recorded
const recorded = signals.getByCategory('workflow_pattern').find(s => s.pattern_key === 'retire:stale');
expect(recorded).toBeDefined();
});
it('falls back to file mtime when no usage entry exists', () => {