refactor: remove self-learning and the optimizer package (P1.4)

Self-learning is excluded from the community edition (ADR-0004); the
project owner reworks it separately. Cost tracking stays.

- agent: delete behaviour evolution (evolve-schema, iterative GEPA,
  compose-evolution, evolution gates, orchestrator and deploy),
  improvement and correction detection, AgentLearning, the eval-dataset
  builder, optimization-capture and the agent_insights tool. Secret
  detection moves from eval-dataset to secret-patterns. The LLM wiring
  keeps only the Haiku adapter and judge call used by skill audit and
  hygiene, so @ax-llm/ax is now declared by agent instead of server.
- server: delete /api/evolution/*, the evolution and optimizer services,
  GEPA expansion of the first chat message, the prompt_optimization and
  monthly_assessment jobs, the idle-session reviewer and its
  session-reviewer persona, prompt injection of past corrections and
  thumbs-down recorded as correction signals. Existing job rows of the
  removed types are ignored, not deleted.
- core: delete OptimizationLogStore; stop re-exporting the improvement
  signal and evolution run stores.
- web: delete the Evolution screen and the two job types in the
  automation editor.
- delete packages/optimizer.

Also logs UF-027 (self-learning loops read data nothing collects) and
UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
shumeiko.it
2026-10-20 00:54:58 +02:00
parent b734c1d3d2
commit 2e1999a3f6
112 changed files with 336 additions and 16165 deletions

View File

@@ -1,194 +0,0 @@
/**
* Agent Learning — persistent behavioral adjustments from performance signals.
*
* Tracks what works and what doesn't across sessions, adjusts agent behavior
* by building a learned context section injected into the system prompt.
*
* Three learning channels:
* 1. Correction patterns — user corrections become persistent rules
* 2. Success patterns — approaches that got positive feedback
* 3. Persona effectiveness — per-persona quality signals
*
* Uses the improvement_signals table for persistence.
*/
import type { ImprovementSignalStore } from '@waggle/core';
export interface LearnedBehavior {
rule: string;
source: 'correction' | 'success' | 'observation';
confidence: number;
occurrences: number;
lastSeen: string;
}
export interface PersonaEffectiveness {
personaId: string;
tasksCompleted: number;
correctionsReceived: number;
positiveSignals: number;
effectivenessScore: number; // 0-100
}
export interface LearningSnapshot {
learnedBehaviors: LearnedBehavior[];
personaStats: PersonaEffectiveness[];
totalCorrections: number;
totalSuccesses: number;
adaptationLevel: 'new' | 'learning' | 'adapted' | 'expert';
}
export class AgentLearning {
private store: ImprovementSignalStore;
constructor(store: ImprovementSignalStore) {
this.store = store;
}
/** Record a successful interaction pattern. */
recordSuccess(pattern: string, personaId?: string): void {
this.store.record('workflow_pattern', `success:${pattern}`, `Approach worked well`, {
type: 'success',
personaId,
recordedAt: new Date().toISOString(),
});
}
/** Record that the user gave positive feedback. */
recordPositiveFeedback(context: string, personaId?: string): void {
this.store.record('correction', `positive:${context.slice(0, 50)}`, context, {
type: 'positive',
personaId,
recordedAt: new Date().toISOString(),
});
}
/** Record a persona's task completion. */
recordPersonaTask(personaId: string, corrected: boolean): void {
const key = corrected ? `persona:${personaId}:corrected` : `persona:${personaId}:completed`;
this.store.record('workflow_pattern', key, personaId, {
type: 'persona_task',
personaId,
corrected,
});
}
/** Build a learning snapshot from accumulated signals. */
getSnapshot(): LearningSnapshot {
const all = this.store.getActionable();
const learnedBehaviors: LearnedBehavior[] = [];
const personaMap = new Map<string, PersonaEffectiveness>();
let totalCorrections = 0;
let totalSuccesses = 0;
for (const signal of all) {
// Corrections become learned behavioral rules
if (signal.category === 'correction' && !signal.pattern_key.startsWith('positive:')) {
totalCorrections += signal.count;
if (signal.count >= 2) { // Only learn from repeated corrections
learnedBehaviors.push({
rule: signal.detail || signal.pattern_key,
source: 'correction',
confidence: Math.min(1.0, 0.5 + signal.count * 0.1),
occurrences: signal.count,
lastSeen: signal.last_seen,
});
}
}
// Positive feedback
if (signal.pattern_key.startsWith('positive:')) {
totalSuccesses += signal.count;
learnedBehaviors.push({
rule: signal.detail || 'Positive approach',
source: 'success',
confidence: Math.min(1.0, 0.6 + signal.count * 0.1),
occurrences: signal.count,
lastSeen: signal.last_seen,
});
}
// Success patterns
if (signal.pattern_key.startsWith('success:')) {
totalSuccesses += signal.count;
}
// Persona stats
if (signal.pattern_key.startsWith('persona:')) {
const parts = signal.pattern_key.split(':');
const personaId = parts[1];
const isCorrected = parts[2] === 'corrected';
let stats = personaMap.get(personaId);
if (!stats) {
stats = { personaId, tasksCompleted: 0, correctionsReceived: 0, positiveSignals: 0, effectivenessScore: 50 };
personaMap.set(personaId, stats);
}
if (isCorrected) {
stats.correctionsReceived += signal.count;
} else {
stats.tasksCompleted += signal.count;
}
}
}
// Calculate persona effectiveness
for (const stats of personaMap.values()) {
const total = stats.tasksCompleted + stats.correctionsReceived;
stats.effectivenessScore = total > 0
? Math.round((stats.tasksCompleted / total) * 100)
: 50;
}
// Determine adaptation level
const totalSignals = totalCorrections + totalSuccesses;
const adaptationLevel = totalSignals < 5 ? 'new'
: totalSignals < 20 ? 'learning'
: totalSignals < 50 ? 'adapted'
: 'expert';
return {
learnedBehaviors: learnedBehaviors.slice(0, 10), // Cap at 10 rules
personaStats: Array.from(personaMap.values()),
totalCorrections,
totalSuccesses,
adaptationLevel,
};
}
/**
* Format learned behaviors as a system prompt section.
* Only included when there are meaningful learned rules.
*/
formatLearningPrompt(): string | null {
const snapshot = this.getSnapshot();
if (snapshot.learnedBehaviors.length === 0) return null;
const lines: string[] = [
'## Learned Behaviors',
`*Adaptation level: ${snapshot.adaptationLevel} (${snapshot.totalCorrections} corrections, ${snapshot.totalSuccesses} successes)*`,
'',
];
const corrections = snapshot.learnedBehaviors.filter(b => b.source === 'correction');
if (corrections.length > 0) {
lines.push('**Avoid these patterns** (user has corrected before):');
for (const b of corrections) {
lines.push(`- ${b.rule} (corrected ${b.occurrences}x, confidence: ${(b.confidence * 100).toFixed(0)}%)`);
}
lines.push('');
}
const successes = snapshot.learnedBehaviors.filter(b => b.source === 'success');
if (successes.length > 0) {
lines.push('**Keep doing** (positive feedback received):');
for (const b of successes) {
lines.push(`- ${b.rule}`);
}
lines.push('');
}
return lines.join('\n');
}
}

View File

@@ -384,8 +384,7 @@ When approaching any task:
};
/**
* Section name literal type — matches the evolution-deploy module.
* Kept minimal here so behavioral-spec.ts stays free of cross-imports.
* Section name literal type for `buildActiveBehavioralSpec` overrides.
*/
export type BehavioralSpecSectionName =
| 'coreLoop'
@@ -402,8 +401,7 @@ export type BehavioralSpecSectionName =
* the compiled baseline unchanged.
*
* Typical usage at server boot:
* const overrides = loadBehavioralSpecOverrides(dataDir);
* const spec = buildActiveBehavioralSpec(overrides);
* const spec = buildActiveBehavioralSpec();
* // ...pass spec.rules into the system prompt...
*/
export function buildActiveBehavioralSpec(

View File

@@ -1,281 +0,0 @@
/**
* Compose Evolution — Phase 3.2 of the self-evolution loop.
*
* Two-stage pipeline:
*
* Stage 1 — EvolveSchema
* Evolve the output STRUCTURE (fields, order, types, constraints).
* Returns a winner schema.
*
* Stage 2 — IterativeGEPA
* Freeze the winner schema. Evolve the INSTRUCTION prompt that fills
* that schema. Only instruction text changes here.
*
* Critical design detail (from Mikhail Pavlukhin's paper): **feedback
* separation**. If the judge complains "missing reasoning field" during
* Stage 2, and we let GEPA see that feedback, GEPA will mutate the
* instruction to try to remove reasoning — undoing the schema Stage 1
* just evolved. So Stage 2 receives a *filtered* judge that strips
* structural complaints and keeps only value-level signals (correctness,
* conciseness, format-of-values, tone).
*
* The composition is stateless: callers provide the baseline schema and
* instruction separately; the result returns both winners + aggregate
* accuracy improvement.
*/
import {
EvolveSchema,
type EvolveSchemaOptions,
type EvolveSchemaResult,
type Schema,
type SchemaExecuteFn,
} from './evolve-schema.js';
import {
IterativeGEPA,
type IterativeGEPAOptions,
type GEPARunResult,
} from './iterative-optimizer.js';
import type { LLMJudge, JudgeScore } from './judge.js';
import type { EvalExample } from './eval-dataset.js';
import { RUNNING_JUDGE_BRAND, isRunningJudge } from './evolution-llm-wiring.js';
// ── Types ───────────────────────────────────────────────────────
export interface ComposeEvolutionOptions {
/** EvolveSchema stage configuration */
schema: Omit<EvolveSchemaOptions, 'onProgress' | 'signal'>;
/** IterativeGEPA stage configuration */
instructions: Omit<IterativeGEPAOptions, 'onProgress' | 'signal'>;
/**
* Classifier — returns 'structural' to drop a feedback line from GEPA,
* or 'value' to keep it. Default: `defaultFeedbackFilter`.
*/
feedbackFilter?: FeedbackFilter;
/** Optional progress reporter across both stages */
onProgress?: (event: ComposeProgress) => void;
/** Optional abort signal (passed through to both stages) */
signal?: AbortSignal;
}
export interface ComposeEvolutionResult {
schema: EvolveSchemaResult;
instructions: GEPARunResult;
/**
* Accuracy delta after both stages compared to the raw baseline schema +
* baseline instructions. Positive = improvement.
*/
combinedDelta: number;
/** True if both stages improved over their individual baselines. */
fullyImproved: boolean;
/** Frozen schema used as input to the GEPA stage — convenience pointer. */
frozenSchema: Schema;
}
export interface ComposeProgress {
stage: 'schema' | 'instructions' | 'done';
/** Underlying stage progress event, if available */
detail?: unknown;
message?: string;
}
export type FeedbackFilter = (feedback: string) => 'structural' | 'value';
// ── Feedback filtering ─────────────────────────────────────────
/**
* Default heuristic: classifies feedback lines as 'structural' when they
* mention schema-level concepts (field, schema, missing/extra property,
* wrong type, reorder, etc). Everything else is 'value'.
*
* Tight enough that ambiguous feedback still flows to GEPA — only obvious
* structural complaints are dropped.
*/
export function defaultFeedbackFilter(feedback: string): 'structural' | 'value' {
if (!feedback) return 'value';
const lower = feedback.toLowerCase();
const structuralPatterns: RegExp[] = [
/\bmissing\s+(?:the\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
/\badd(?:ed|ing)?\s+(?:a|the)?\s*(?:new\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
/\bextra\s+field\b/,
/\bunexpected\s+field\b/,
/\bwrong\s+(?:field\s+)?(?:type|order)\b/,
/\bfield\s+type\s+(?:mismatch|wrong)/,
/\b(?:should\s+)?(?:re)?order\s+(?:the\s+)?fields\b/,
/\bschema\s+(?:mismatch|violation|error)\b/,
/\binvalid\s+json\s+shape\b/,
/\bmissing\s+required\s+property\b/,
/\bproperty\s+missing\b/,
/\bshape\s+is\s+wrong\b/,
];
for (const pattern of structuralPatterns) {
if (pattern.test(lower)) return 'structural';
}
return 'value';
}
// ── Judge wrapping ─────────────────────────────────────────────
/**
* Wrap an LLMJudge-like object so the feedback it emits has structural
* lines stripped according to `filter`. Only `feedback` is filtered —
* the numerical scores (correctness, procedure, conciseness, overall)
* are preserved as-is so GEPA sees the true accuracy signal.
*
* The multi-line feedback from the judge is split on newlines and each
* line classified individually.
*/
export function filterJudgeFeedback(
judge: Pick<LLMJudge, 'score'>,
filter: FeedbackFilter = defaultFeedbackFilter,
): Pick<LLMJudge, 'score'> {
const wrapped: Pick<LLMJudge, 'score'> & Partial<Record<symbol, unknown>> = {
async score(args) {
const raw = await judge.score(args);
const filteredFeedback = stripStructuralLines(raw.feedback, filter);
const result: JudgeScore = { ...raw, feedback: filteredFeedback };
return result;
},
};
// H-09 G3: preserve the running-judge brand so IterativeGEPA's check
// still passes after feedback filtering. Without this, wrapping a
// running judge with filterJudgeFeedback would silently strip the
// brand and GEPA would reject it.
if (isRunningJudge(judge)) {
wrapped[RUNNING_JUDGE_BRAND] = true;
}
return wrapped;
}
/** Public helper — strips structural lines from a feedback string. */
export function stripStructuralLines(
feedback: string,
filter: FeedbackFilter = defaultFeedbackFilter,
): string {
if (!feedback) return feedback;
return feedback
.split(/\r?\n/)
.filter(line => filter(line) === 'value')
.join('\n')
.trim();
}
// ── Orchestrator ───────────────────────────────────────────────
export class ComposeEvolution {
async run(options: ComposeEvolutionOptions): Promise<ComposeEvolutionResult> {
const signal = options.signal;
const filter = options.feedbackFilter ?? defaultFeedbackFilter;
// Stage 1 — evolve the schema.
options.onProgress?.({ stage: 'schema', message: 'starting schema evolution' });
const schemaResult = await new EvolveSchema().run({
...options.schema,
signal,
onProgress: (e) => options.onProgress?.({ stage: 'schema', detail: e }),
});
if (signal?.aborted) {
return assembleAbortedResult(schemaResult, options.instructions.baseline);
}
const frozenSchema = schemaResult.winner.schema;
// Stage 2 — evolve the instructions, with a filtered judge so GEPA
// never sees structural feedback.
options.onProgress?.({ stage: 'instructions', message: 'starting instruction evolution' });
const filteredJudge = filterJudgeFeedback(options.instructions.judge, filter);
const gepaResult = await new IterativeGEPA().run({
...options.instructions,
judge: filteredJudge,
signal,
onProgress: (e) => options.onProgress?.({ stage: 'instructions', detail: e }),
});
const combinedDelta =
(gepaResult.winner.score?.overall ?? 0) -
((schemaResult.history[0]?.score?.accuracy ?? 0));
const fullyImproved = schemaResult.improved && gepaResult.improved;
options.onProgress?.({ stage: 'done' });
return {
schema: schemaResult,
instructions: gepaResult,
combinedDelta,
fullyImproved,
frozenSchema,
};
}
}
// ── Helpers ─────────────────────────────────────────────────────
/**
* Build a convenience `SchemaExecuteFn` from a user-supplied runner that
* only cares about the instruction prompt (not the schema itself). Used
* by callers that want to compose — they supply one "execute" function
* for the instructional stage and this helper wraps it for Stage 1.
*
* The wrapped executor serializes the schema into a short prefix the
* model can follow (`Return JSON with fields: <field1>, <field2>, ...`)
* before delegating to the caller's function.
*/
export function schemaExecutorFromInstructionRunner(
runInstructions: (args: { prompt: string; input: string }) => Promise<string>,
): SchemaExecuteFn {
return async ({ schema, input }) => {
const fieldList = schema.fields.map(f => `"${f.name}" (${f.type})`).join(', ');
const prompt = `Return a JSON object with these fields: ${fieldList}.`;
try {
const actual = await runInstructions({ prompt, input });
return { actual, parsed: actualLooksLikeJson(actual) };
} catch {
return { actual: '', parsed: false };
}
};
}
function actualLooksLikeJson(s: string): boolean {
const trimmed = s.trim();
if (!trimmed) return false;
return (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
(trimmed.startsWith('[') && trimmed.endsWith(']'));
}
function assembleAbortedResult(
schemaResult: EvolveSchemaResult,
instructionBaseline: string,
): ComposeEvolutionResult {
// When aborted before Stage 2, produce a minimal GEPARunResult pointing
// at the baseline instruction so callers have a stable shape.
const baselineInstruction = {
id: 'aborted',
prompt: instructionBaseline,
generation: 0,
parent: null,
strategy: 'aborted',
score: null,
perExample: [],
} as GEPARunResult['winner'];
return {
schema: schemaResult,
instructions: {
winner: baselineInstruction,
paretoFront: [baselineInstruction],
history: [baselineInstruction],
improved: false,
delta: 0,
},
combinedDelta: 0,
fullyImproved: false,
frozenSchema: schemaResult.winner.schema,
};
}
/** Re-export types so callers can `import { type Schema } from '...'`. */
export type { EvolveSchemaResult, GEPARunResult };

View File

@@ -1,199 +0,0 @@
/**
* Correction Detector — identifies when a user is correcting the agent
* and classifies corrections as durable (behavior-changing) vs task-local.
*
* Per Slice 7 correction #2:
* - Durable corrections: repeated patterns that should influence future behavior → improvement signals
* - Task-local corrections: one-off adjustments within a session → NOT signals
* - One-off disagreements: not corrections at all → ignored
*/
export type CorrectionDurability = 'durable' | 'task_local' | 'not_correction';
export interface DetectedCorrection {
isDurable: boolean;
durability: CorrectionDurability;
patternKey: string;
detail: string;
confidence: number; // 0-1
}
// ── Correction signal patterns ──────────────────────────────
/** Strong correction signals — high confidence the user is correcting agent behavior */
const STRONG_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
{ pattern: /\bno[,.]?\s*(?:not that|don'?t|do not|stop|never)\b/i, weight: 3 },
{ pattern: /\bI (?:said|told you|asked)\b/i, weight: 3 },
{ pattern: /\bthat'?s (?:wrong|incorrect|not (?:what|right))\b/i, weight: 3 },
{ pattern: /\bplease (?:don'?t|do not|stop|never)\b/i, weight: 2 },
{ pattern: /\binstead[,.]?\s*(?:use|do|try|go with)\b/i, weight: 2 },
{ pattern: /\bwrong (?:approach|way|format|style|tone)\b/i, weight: 2 },
{ pattern: /\bnot what I (?:wanted|meant|asked)\b/i, weight: 3 },
];
/** Moderate correction signals — may be correction or just refinement */
const MODERATE_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
{ pattern: /^no[,.:!]\s/i, weight: 2 }, // "No, ..." at start of message — strong disagreement
{ pattern: /\bactually[,.]?\s/i, weight: 1 },
{ pattern: /\brather[,.]?\s/i, weight: 1 },
{ pattern: /\blet'?s (?:not|try|go with|use)\b/i, weight: 1 },
{ pattern: /\bprefer\b/i, weight: 1 },
{ pattern: /\bshould (?:be|have been|use)\b/i, weight: 1 },
{ pattern: /\bchange (?:it|this|that) to\b/i, weight: 1 },
{ pattern: /\btoo (?:verbose|long|short|formal|casual|technical|simple)\b/i, weight: 1 },
{ pattern: /\bstop (?:doing|using|adding)\b/i, weight: 1 },
{ pattern: /\bkeep (?:it|things) (?:simple|short|brief|casual|formal)\b/i, weight: 1 },
];
// ── Durability classification patterns ──────────────────────
/** Durable signals — the correction applies beyond this specific task */
const DURABLE_SIGNALS: RegExp[] = [
/\balways\b/i,
/\bnever\b/i,
/\bfrom now on\b/i,
/\bin (?:the )?future\b/i,
/\bwhenever\b/i,
/\bevery time\b/i,
/\bin general\b/i,
/\bI (?:always |usually )?prefer\b/i,
/\bmy (?:preference|style|approach)\b/i,
/\bdon'?t (?:ever|again)\b/i,
/\bremember (?:to|that)\b/i,
/\bkeep (?:it|things|this)\b/i,
];
/** Task-local signals — the correction is specific to this task */
const TASK_LOCAL_SIGNALS: RegExp[] = [
/\bthis (?:time|one|specific|particular)\b/i,
/\bfor (?:this|now)\b/i,
/\bjust (?:here|this|now)\b/i,
/\bin this (?:case|instance|response)\b/i,
/\bright now\b/i,
/\bhere\b/i,
];
// ── Pattern key extraction ──────────────────────────────────
/** Categories of behavioral corrections we can extract pattern keys for */
const PATTERN_KEY_EXTRACTORS: Array<{ category: string; pattern: RegExp }> = [
{ category: 'tone', pattern: /\btoo (?:formal|casual|verbose|terse|technical|simple)\b/i },
{ category: 'format', pattern: /\b(?:format|formatting|headers?|bullet|numbering|markdown)\b/i },
{ category: 'length', pattern: /\btoo (?:long|short|brief|detailed)\b/i },
{ category: 'approach', pattern: /\bwrong (?:approach|way|method|strategy)\b/i },
{ category: 'scope', pattern: /\btoo (?:much|many|broad|narrow|specific|general)\b/i },
{ category: 'accuracy', pattern: /\b(?:wrong|incorrect|inaccurate|mistake|error)\b/i },
{ category: 'style', pattern: /\b(?:style|voice|writing|wording|phrasing)\b/i },
];
/**
* Detect whether a user message contains a correction and classify it.
*
* Returns null if no correction detected.
* Returns DetectedCorrection with durability classification if correction found.
*/
export function detectCorrection(
userMessage: string,
previousAssistantMessage?: string,
): DetectedCorrection | null {
if (!userMessage || userMessage.length < 5) return null;
// Score correction strength
let correctionScore = 0;
for (const { pattern, weight } of STRONG_CORRECTION_PATTERNS) {
if (pattern.test(userMessage)) correctionScore += weight;
}
for (const { pattern, weight } of MODERATE_CORRECTION_PATTERNS) {
if (pattern.test(userMessage)) correctionScore += weight;
}
// Need minimum score to count as correction
if (correctionScore < 2) return null;
// Classify durability
const durability = classifyDurability(userMessage);
// Extract pattern key
const patternKey = extractPatternKey(userMessage);
// Compute confidence (0-1)
const confidence = Math.min(correctionScore / 6, 1);
// Build detail: first sentence or first 120 chars
const detail = extractDetail(userMessage);
return {
isDurable: durability === 'durable',
durability,
patternKey,
detail,
confidence,
};
}
function classifyDurability(message: string): CorrectionDurability {
let durableScore = 0;
let taskLocalScore = 0;
for (const pattern of DURABLE_SIGNALS) {
if (pattern.test(message)) durableScore++;
}
for (const pattern of TASK_LOCAL_SIGNALS) {
if (pattern.test(message)) taskLocalScore++;
}
// Explicit durable signals win
if (durableScore > 0 && durableScore >= taskLocalScore) return 'durable';
// Explicit task-local signals
if (taskLocalScore > 0) return 'task_local';
// No explicit signals — default to task_local (conservative; only promote to durable
// when the same pattern_key recurs across sessions via ImprovementSignalStore)
return 'task_local';
}
function extractPatternKey(message: string): string {
for (const { category, pattern } of PATTERN_KEY_EXTRACTORS) {
const match = message.match(pattern);
if (match) {
const qualifier = match[0].toLowerCase().replace(/\s+/g, '_');
return `${category}:${qualifier}`;
}
}
// Fallback: generic correction key
return 'general:correction';
}
function extractDetail(message: string): string {
// Take first sentence or first 120 chars
const sentenceMatch = message.match(/^(.+?[.!?])\s/);
if (sentenceMatch && sentenceMatch[1].length <= 120) {
return sentenceMatch[1];
}
return message.length > 120 ? message.slice(0, 117) + '...' : message;
}
/**
* Analyze a sequence of message pairs to detect corrections.
* Useful for batch analysis of session history.
*/
export function detectCorrectionsInHistory(
messages: Array<{ role: string; content: string }>,
): DetectedCorrection[] {
const corrections: DetectedCorrection[] = [];
for (let i = 1; i < messages.length; i++) {
const msg = messages[i];
if (msg.role !== 'user') continue;
const prevAssistant = i > 0 ? messages[i - 1] : undefined;
const correction = detectCorrection(
msg.content,
prevAssistant?.role === 'assistant' ? prevAssistant.content : undefined,
);
if (correction) {
corrections.push(correction);
}
}
return corrections;
}

View File

@@ -1,441 +0,0 @@
/**
* Eval Dataset Builder — turns execution traces and harvest frames into
* train/val/holdout splits for the self-evolution loop.
*
* Sources mined:
* - ExecutionTraceStore (outcome = success | verified → positive examples;
* outcome = corrected → negative example with correctionFeedback as ground truth)
* - Optional harvest frames (DistilledKnowledge) for Q&A-style augmentation
* - Optional corrections from ImprovementSignalStore
*
* Filter pipeline (in order):
* 1. Secret scanning — reject any example containing credentials / tokens
* 2. Keyword heuristic — min length, non-trivial content, no duplicate inputs
* 3. Optional LLM-as-judge relevance filter (pass an llmCall callback)
*
* Split is deterministic given a seed — same traces + same seed = same split.
* Default split ratio is 60/20/20 train/val/holdout.
*/
import type {
ExecutionTraceStore,
ParsedExecutionTrace,
TraceOutcome,
TraceQueryFilter,
} from '@waggle/core';
// ── Types ───────────────────────────────────────────────────────
export interface EvalExample {
input: string;
expected_output: string;
metadata: EvalExampleMetadata;
}
export interface EvalExampleMetadata {
traceId?: number;
personaId?: string | null;
taskShape?: string | null;
model?: string | null;
outcome?: TraceOutcome;
tags?: string[];
/** Where this example came from */
source: 'trace' | 'harvest' | 'correction';
/** Optional opaque identifier from the source system */
sourceId?: string;
}
export interface DatasetSplit {
train: EvalExample[];
val: EvalExample[];
holdout: EvalExample[];
/** Examples that were filtered out — useful for debugging */
rejected: Array<{ reason: string; preview: string }>;
/** Stats for logging / UI */
stats: {
sourced: number;
acceptedAfterSecretScan: number;
acceptedAfterHeuristic: number;
acceptedAfterJudge: number;
unique: number;
total: number;
};
}
export interface BuildOptions {
/** Which trace outcomes count as positive examples (default: ['success', 'verified']). */
positiveOutcomes?: TraceOutcome[];
/** Whether to include correction traces as negative examples with feedback as expected_output (default true). */
includeCorrections?: boolean;
/** Filter passed straight through to ExecutionTraceStore.query */
traceFilter?: Omit<TraceQueryFilter, 'outcome' | 'limit'> & { limit?: number };
/** Optional external augmenters */
harvestExamples?: EvalExample[];
correctionExamples?: EvalExample[];
/** Minimum input characters to keep the example (default 10). */
minInputChars?: number;
/** Minimum expected_output characters to keep the example (default 5). */
minOutputChars?: number;
/** Maximum input + output characters combined; over this, the example is rejected (default 16384). */
maxCombinedChars?: number;
/** Optional LLM-as-judge relevance filter. Pass null or omit to skip. */
judge?: (example: EvalExample) => Promise<JudgeVerdict>;
/** Ratio triple summing to 1.0 (default [0.6, 0.2, 0.2]). */
splitRatios?: [number, number, number];
/** Deterministic seed (default 1). */
seed?: number;
}
export interface JudgeVerdict {
keep: boolean;
reason?: string;
}
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
];
/** Returns the first secret pattern the text matches, or null if clean. */
export function detectSecrets(text: string): string | null {
for (const { name, re } of SECRET_PATTERNS) {
if (re.test(text)) return name;
}
return null;
}
/** Exported for tests / tools that want the full list. */
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
/**
* Redact every secret-pattern match in a string, replacing each with
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
* detectSecrets so there is a single source of truth for "what is a secret".
*/
export function redactSecrets(text: string): { text: string; found: string[] } {
let out = text;
const found: string[] = [];
for (const { name, re } of SECRET_PATTERNS) {
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
const replaced = out.replace(global, `[REDACTED:${name}]`);
if (replaced !== out) {
found.push(name);
out = replaced;
}
}
return { text: out, found };
}
// ── Deterministic PRNG (mulberry32) ─────────────────────────────
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
/** Fisher-Yates shuffle in-place using provided rng. */
function shuffle<T>(arr: T[], rng: () => number): T[] {
for (let i = arr.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr;
}
// ── Builder ─────────────────────────────────────────────────────
export class EvalDatasetBuilder {
private store: ExecutionTraceStore;
constructor(store: ExecutionTraceStore) {
this.store = store;
}
/** Mine traces into a typed example list (before filtering). */
sourceFromTraces(
positiveOutcomes: TraceOutcome[],
includeCorrections: boolean,
filter: BuildOptions['traceFilter'] = {},
): EvalExample[] {
const outcomes: TraceOutcome[] = [...positiveOutcomes];
if (includeCorrections) outcomes.push('corrected');
const rows = this.store.queryParsed({
...filter,
outcome: outcomes,
limit: filter.limit ?? 10_000,
});
return rows.map(row => traceToExample(row, includeCorrections));
}
/**
* Build the full dataset split.
*
* Filter order:
* 1. Source (traces + augmenters)
* 2. Secret scan
* 3. Heuristic (length, non-trivial)
* 4. Optional judge
* 5. Dedup on input hash
* 6. Shuffle + split
*/
async build(options: BuildOptions = {}): Promise<DatasetSplit> {
const positiveOutcomes = options.positiveOutcomes ?? ['success', 'verified'];
const includeCorrections = options.includeCorrections ?? true;
const minInputChars = options.minInputChars ?? 10;
const minOutputChars = options.minOutputChars ?? 5;
const maxCombinedChars = options.maxCombinedChars ?? 16_384;
const splitRatios = options.splitRatios ?? [0.6, 0.2, 0.2];
const seed = options.seed ?? 1;
validateRatios(splitRatios);
const rejected: DatasetSplit['rejected'] = [];
// 1. Source
const traceExamples = this.sourceFromTraces(
positiveOutcomes,
includeCorrections,
options.traceFilter,
);
const sourced: EvalExample[] = [
...traceExamples,
...(options.harvestExamples ?? []),
...(options.correctionExamples ?? []),
];
// 2. Secret scan
const afterSecretScan: EvalExample[] = [];
for (const ex of sourced) {
const hitInput = detectSecrets(ex.input);
const hitOutput = detectSecrets(ex.expected_output);
if (hitInput || hitOutput) {
rejected.push({
reason: `secret:${hitInput ?? hitOutput}`,
preview: ex.input.slice(0, 80),
});
continue;
}
afterSecretScan.push(ex);
}
// 3. Heuristic
const afterHeuristic: EvalExample[] = [];
for (const ex of afterSecretScan) {
const input = ex.input.trim();
const output = ex.expected_output.trim();
if (input.length < minInputChars) {
rejected.push({ reason: 'too-short-input', preview: input.slice(0, 80) });
continue;
}
if (output.length < minOutputChars) {
rejected.push({ reason: 'too-short-output', preview: input.slice(0, 80) });
continue;
}
if (input.length + output.length > maxCombinedChars) {
rejected.push({ reason: 'too-long', preview: input.slice(0, 80) });
continue;
}
if (isLowSignal(input) || isLowSignal(output)) {
rejected.push({ reason: 'low-signal', preview: input.slice(0, 80) });
continue;
}
afterHeuristic.push({ ...ex, input, expected_output: output });
}
// 4. Optional judge
let afterJudge = afterHeuristic;
if (options.judge) {
afterJudge = [];
for (const ex of afterHeuristic) {
try {
const verdict = await options.judge(ex);
if (verdict.keep) {
afterJudge.push(ex);
} else {
rejected.push({
reason: `judge:${verdict.reason ?? 'rejected'}`,
preview: ex.input.slice(0, 80),
});
}
} catch (err) {
// On judge failure, keep the example — err on the side of more data.
afterJudge.push(ex);
}
}
}
// 5. Dedup on input hash
const seen = new Set<string>();
const unique: EvalExample[] = [];
for (const ex of afterJudge) {
const key = hashKey(ex.input);
if (seen.has(key)) {
rejected.push({ reason: 'duplicate', preview: ex.input.slice(0, 80) });
continue;
}
seen.add(key);
unique.push(ex);
}
// 6. Shuffle + split
const rng = makeRng(seed);
const shuffled = shuffle([...unique], rng);
const { train, val, holdout } = splitExamples(shuffled, splitRatios);
return {
train,
val,
holdout,
rejected,
stats: {
sourced: sourced.length,
acceptedAfterSecretScan: afterSecretScan.length,
acceptedAfterHeuristic: afterHeuristic.length,
acceptedAfterJudge: afterJudge.length,
unique: unique.length,
total: train.length + val.length + holdout.length,
},
};
}
}
// ── JSONL IO ────────────────────────────────────────────────────
/** Serialize examples as JSONL (one JSON object per line). */
export function toJSONL(examples: EvalExample[]): string {
return examples.map(ex => JSON.stringify(ex)).join('\n');
}
/** Parse JSONL into examples. Silently skips unparseable lines. */
export function fromJSONL(jsonl: string): EvalExample[] {
const out: EvalExample[] = [];
for (const line of jsonl.split(/\r?\n/)) {
if (!line.trim()) continue;
try {
const parsed = JSON.parse(line);
if (
parsed &&
typeof parsed.input === 'string' &&
typeof parsed.expected_output === 'string' &&
parsed.metadata
) {
out.push(parsed as EvalExample);
}
} catch {
// skip malformed line
}
}
return out;
}
// ── Helpers ─────────────────────────────────────────────────────
function traceToExample(
trace: ParsedExecutionTrace,
includeCorrections: boolean,
): EvalExample {
const isCorrected = trace.outcome === 'corrected';
const expected = isCorrected && includeCorrections
? (trace.payload.correctionFeedback ?? trace.payload.output)
: trace.payload.output;
return {
input: trace.payload.input,
expected_output: expected,
metadata: {
traceId: trace.id,
personaId: trace.persona_id,
taskShape: trace.task_shape,
model: trace.model,
outcome: trace.outcome,
tags: trace.payload.tags ?? [],
source: isCorrected ? 'correction' : 'trace',
},
};
}
function validateRatios(ratios: [number, number, number]): void {
const sum = ratios[0] + ratios[1] + ratios[2];
if (Math.abs(sum - 1.0) > 1e-6) {
throw new Error(`Split ratios must sum to 1.0, got ${sum}`);
}
if (ratios.some(r => r < 0)) {
throw new Error('Split ratios must be non-negative');
}
}
function splitExamples(
examples: EvalExample[],
ratios: [number, number, number],
): { train: EvalExample[]; val: EvalExample[]; holdout: EvalExample[] } {
const n = examples.length;
// Floor-then-assign-remainders so every example ends up in exactly one split.
const trainN = Math.floor(n * ratios[0]);
const valN = Math.floor(n * ratios[1]);
const holdoutN = n - trainN - valN;
return {
train: examples.slice(0, trainN),
val: examples.slice(trainN, trainN + valN),
holdout: examples.slice(trainN + valN, trainN + valN + holdoutN),
};
}
/**
* Cheap signal check — reject examples that are mostly whitespace, repeated
* punctuation, or otherwise meaningless. Not a language detector.
*/
function isLowSignal(text: string): boolean {
if (!text) return true;
const stripped = text.replace(/\s+/g, '');
if (stripped.length < 3) return true;
// 80%+ of the same character → reject
const counts = new Map<string, number>();
for (const ch of stripped) {
counts.set(ch, (counts.get(ch) ?? 0) + 1);
}
const maxCount = Math.max(...counts.values());
if (maxCount / stripped.length > 0.8) return true;
// No alphanumeric at all → reject
if (!/[A-Za-z0-9]/.test(stripped)) return true;
return false;
}
/**
* FNV-1a 32-bit hash — stable across processes, no crypto import needed.
* Used for input-level dedup.
*/
function hashKey(text: string): string {
let h = 0x811c9dc5;
for (let i = 0; i < text.length; i++) {
h ^= text.charCodeAt(i);
h = Math.imul(h, 0x01000193);
}
return (h >>> 0).toString(16);
}

View File

@@ -1,299 +0,0 @@
/**
* Evolution Deploy Helpers — Phase 6.1 of the self-evolution loop.
*
* When the user accepts an EvolutionRun, the orchestrator calls the
* caller-supplied `deploy` callback. This module provides the built-in
* building blocks for that callback:
*
* deployPersonaOverride — writes a persona JSON file that
* shadows the built-in persona at
* runtime via loadCustomPersonas.
* deployBehavioralSpecOverride — writes a JSON file in
* {dataDir}/behavioral-overrides/
* that can be merged into
* BEHAVIORAL_SPEC sections at load.
* loadBehavioralSpecOverrides — reads those overrides from disk.
*
* Every writer:
* - Creates the target directory if missing.
* - Writes atomically (write to .tmp, rename) so a crash mid-write
* never corrupts the config.
* - Backs up the previous version to {file}.bak before overwriting.
* - Returns a structured DeployResult with paths + timestamps.
*
* These helpers are pure filesystem operations — they don't talk to
* any runtime singletons, so the server wiring is trivial:
* orchestrator.accept(uuid) → onDeploy(run)
* → deployPersonaOverride(dataDir, ...)
* → reloadPersonas()
*/
import fs from 'node:fs';
import path from 'node:path';
import type { AgentPersona } from './personas.js';
import { getPersona } from './personas.js';
import { assertValidCustomPersonaId } from './custom-personas.js';
// ── Public result shape ────────────────────────────────────────
export interface DeployResult {
/** Absolute path that was written */
path: string;
/** Path to the backup of the previous version (null if first write) */
backupPath: string | null;
/** When the deploy happened */
deployedAt: string;
}
// ── Persona override ───────────────────────────────────────────
export interface DeployPersonaInput {
/** Persona id — matches a built-in persona (the deploy SHADOWS it). */
personaId: string;
/** The evolved system prompt text */
systemPrompt: string;
/** Optional additional field overrides */
overrides?: Partial<Omit<AgentPersona, 'id' | 'systemPrompt'>>;
}
/**
* Write an evolved persona system prompt to disk so the custom-personas
* loader picks it up on next `listPersonas()` call.
*
* When a built-in persona with this id exists, the override inherits
* all its properties (name, tools, tagline, etc) except systemPrompt
* (replaced with the evolved one). When no built-in exists, a minimal
* persona shell is built from the input.
*/
export function deployPersonaOverride(
dataDir: string,
input: DeployPersonaInput,
): DeployResult {
assertValidCustomPersonaId(input.personaId);
const personasDir = path.join(dataDir, 'personas');
if (!fs.existsSync(personasDir)) {
fs.mkdirSync(personasDir, { recursive: true });
}
const filePath = path.join(personasDir, `${input.personaId}.json`);
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
if (backupPath) {
fs.copyFileSync(filePath, backupPath);
}
const builtin = getPersona(input.personaId);
const persona: AgentPersona = builtin
? {
...builtin,
...input.overrides,
id: input.personaId,
systemPrompt: input.systemPrompt,
}
: {
name: input.personaId,
description: input.overrides?.description ?? `${input.personaId} (evolved)`,
icon: input.overrides?.icon ?? 'sparkles',
modelPreference: input.overrides?.modelPreference ?? 'claude-sonnet-4-6',
tools: input.overrides?.tools ?? [],
workspaceAffinity: input.overrides?.workspaceAffinity ?? [],
suggestedCommands: input.overrides?.suggestedCommands ?? [],
defaultWorkflow: input.overrides?.defaultWorkflow ?? null,
...input.overrides,
id: input.personaId,
systemPrompt: input.systemPrompt,
};
writeAtomic(filePath, JSON.stringify(persona, null, 2));
return {
path: filePath,
backupPath,
deployedAt: new Date().toISOString(),
};
}
/**
* Roll back a previously-deployed persona override to the .bak version
* (or remove the override entirely if there was no previous version).
*/
export function rollbackPersonaOverride(
dataDir: string,
personaId: string,
): boolean {
assertValidCustomPersonaId(personaId);
const filePath = path.join(dataDir, 'personas', `${personaId}.json`);
const backupPath = `${filePath}.bak`;
if (fs.existsSync(backupPath)) {
fs.renameSync(backupPath, filePath);
return true;
}
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
return true;
}
return false;
}
// ── Behavioral-spec override ──────────────────────────────────
/**
* BEHAVIORAL_SPEC is compiled into the bundle, so we cannot mutate it
* on disk without rebuilding. Instead, overrides live as JSON files in
* {dataDir}/behavioral-overrides/ and the runtime merges them at load.
*
* One file per section, filename = section name (coreLoop, qualityRules,
* behavioralRules, workPatterns, intelligenceDefaults).
*/
export type BehavioralSpecSection =
| 'coreLoop'
| 'qualityRules'
| 'behavioralRules'
| 'workPatterns'
| 'intelligenceDefaults';
export const BEHAVIORAL_SPEC_SECTIONS: BehavioralSpecSection[] = [
'coreLoop',
'qualityRules',
'behavioralRules',
'workPatterns',
'intelligenceDefaults',
];
export interface DeployBehavioralSpecInput {
/** Which section to override */
section: BehavioralSpecSection;
/** The evolved text for that section */
text: string;
/** Optional evolution-run uuid for audit */
runUuid?: string;
}
export interface BehavioralSpecOverride {
section: BehavioralSpecSection;
text: string;
runUuid?: string;
deployedAt: string;
}
/**
* Write a behavioral-spec section override to disk. The runtime loader
* (loadBehavioralSpecOverrides) reads these on startup and merges them
* into the compiled BEHAVIORAL_SPEC at runtime.
*/
export function deployBehavioralSpecOverride(
dataDir: string,
input: DeployBehavioralSpecInput,
): DeployResult {
if (!BEHAVIORAL_SPEC_SECTIONS.includes(input.section)) {
throw new Error(`Unknown behavioral-spec section: ${input.section}`);
}
const dir = path.join(dataDir, 'behavioral-overrides');
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
const filePath = path.join(dir, `${input.section}.json`);
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
if (backupPath) {
fs.copyFileSync(filePath, backupPath);
}
const payload: BehavioralSpecOverride = {
section: input.section,
text: input.text,
runUuid: input.runUuid,
deployedAt: new Date().toISOString(),
};
writeAtomic(filePath, JSON.stringify(payload, null, 2));
return {
path: filePath,
backupPath,
deployedAt: payload.deployedAt,
};
}
/** Roll back one section (restores .bak, or removes if no previous override). */
export function rollbackBehavioralSpecOverride(
dataDir: string,
section: BehavioralSpecSection,
): boolean {
const filePath = path.join(dataDir, 'behavioral-overrides', `${section}.json`);
const backupPath = `${filePath}.bak`;
if (fs.existsSync(backupPath)) {
fs.renameSync(backupPath, filePath);
return true;
}
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
return true;
}
return false;
}
/** Read all behavioral-spec overrides from disk as a map section → text. */
export function loadBehavioralSpecOverrides(
dataDir: string,
): Partial<Record<BehavioralSpecSection, string>> {
const dir = path.join(dataDir, 'behavioral-overrides');
if (!fs.existsSync(dir)) return {};
const out: Partial<Record<BehavioralSpecSection, string>> = {};
try {
for (const section of BEHAVIORAL_SPEC_SECTIONS) {
const filePath = path.join(dir, `${section}.json`);
if (!fs.existsSync(filePath)) continue;
try {
const raw = fs.readFileSync(filePath, 'utf-8');
const parsed = JSON.parse(raw) as Partial<BehavioralSpecOverride>;
if (parsed && typeof parsed.text === 'string' && parsed.text.length > 0) {
out[section] = parsed.text;
}
} catch { /* skip malformed */ }
}
} catch { /* dir unreadable */ }
return out;
}
/** Merge overrides into a baseline spec object, preferring the override text. */
export function applyBehavioralSpecOverrides(
baseline: Record<BehavioralSpecSection, string>,
overrides: Partial<Record<BehavioralSpecSection, string>>,
): Record<BehavioralSpecSection, string> {
return {
...baseline,
...Object.fromEntries(
(Object.entries(overrides) as Array<[BehavioralSpecSection, string]>)
.filter(([, v]) => typeof v === 'string' && v.length > 0),
),
} as Record<BehavioralSpecSection, string>;
}
// ── Internal ───────────────────────────────────────────────────
/** Atomic write — write to .tmp then rename into place. */
function writeAtomic(filePath: string, contents: string): void {
const tmpPath = `${filePath}.tmp`;
fs.writeFileSync(tmpPath, contents, 'utf-8');
const waitBuffer = new Int32Array(new SharedArrayBuffer(4));
try {
for (let attempt = 1; attempt <= 10; attempt++) {
try {
fs.renameSync(tmpPath, filePath);
return;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
const transient = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
if (!transient || attempt === 10) throw error;
// Windows antivirus and indexers can briefly hold an exclusive handle.
Atomics.wait(waitBuffer, 0, 0, 25 * attempt);
}
}
} finally {
try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ }
}
}

View File

@@ -1,316 +0,0 @@
/**
* Evolution Gates — Phase 2.2 of the self-evolution loop.
*
* Gatekeeping for prompts/skills produced by the iterative GEPA optimizer
* before they reach production. A candidate must pass ALL gates; the first
* failure short-circuits and returns a `fail` verdict with the reason.
*
* Four gate categories:
*
* 1. Size gates — hard character caps per target type (persona prompt
* ≤ 3 000 chars, tool description ≤ 500, skill ≤ 15 KB,
* behavioral-spec section ≤ 4 000).
* 2. Growth gates — percentage cap over baseline (default +20%) prevents
* runaway prompt bloat across generations.
* 3. Structural — non-empty, no unresolved placeholders, balanced
* markdown fences, no obvious template accidents
* (e.g. "[PLACEHOLDER]", "TODO:" left in).
* 4. Regression — score delta on a held-out eval set must not drop
* below the allowed tolerance (default -0.02, i.e. a
* 2% accuracy regression is the worst allowed).
*
* Each gate returns structured reasons so the orchestrator (phase 4)
* can log WHY a candidate was rejected and surface it in the UI.
*/
import type { EvolutionTarget } from './iterative-optimizer.js';
// ── Public types ───────────────────────────────────────────────
export type GateVerdict = 'pass' | 'fail';
export interface GateResult {
gate: string;
verdict: GateVerdict;
reason: string;
/** Optional measurable detail: e.g. `{ current: 3520, max: 3000 }` */
detail?: Record<string, number | string>;
}
export interface GateCheckResult {
verdict: GateVerdict;
/** First failing gate, or null if all passed */
firstFailure: GateResult | null;
/** All gate results in order */
results: GateResult[];
}
export interface SizeLimits {
personaSystemPrompt: number;
toolDescription: number;
skillBody: number;
behavioralSpecSection: number;
generic: number;
}
export const DEFAULT_SIZE_LIMITS: SizeLimits = {
personaSystemPrompt: 3_000,
toolDescription: 500,
skillBody: 15_000,
behavioralSpecSection: 4_000,
generic: 8_000,
};
export interface GateOptions {
/** What is being evolved — drives which size cap applies. Default 'generic'. */
targetKind?: EvolutionTarget;
/** Max allowed % growth over baseline. Default 0.2 (i.e. +20%). */
maxGrowthRatio?: number;
/** Largest allowed accuracy regression vs baseline. Default -0.02 (i.e. -2%). */
maxRegression?: number;
/** Override the default size limits */
sizeLimits?: Partial<SizeLimits>;
/** Allow empty candidates (rarely useful — default false) */
allowEmpty?: boolean;
}
export interface CheckInput {
/** The new candidate text */
candidate: string;
/** The baseline (pre-evolution) text — used for growth + regression checks */
baseline: string;
/** Optional scores to enable the regression gate */
scores?: {
baseline: number;
candidate: number;
};
}
// ── Public API ─────────────────────────────────────────────────
/**
* Run all gates in order. Returns a structured verdict. The overall
* verdict is `pass` iff every gate passed; `fail` with `firstFailure`
* populated otherwise.
*
* Callers can choose to short-circuit on the first failure or to
* collect all of them — `runGates` returns every gate result either way.
*/
export function runGates(input: CheckInput, options: GateOptions = {}): GateCheckResult {
const targetKind = options.targetKind ?? 'generic';
const sizeLimits = { ...DEFAULT_SIZE_LIMITS, ...options.sizeLimits };
const maxGrowthRatio = options.maxGrowthRatio ?? 0.2;
const maxRegression = options.maxRegression ?? -0.02;
const allowEmpty = options.allowEmpty ?? false;
const results: GateResult[] = [];
// 1. Non-empty
if (!allowEmpty) {
results.push(checkNonEmpty(input.candidate));
}
// 2. Size
results.push(checkSize(input.candidate, targetKind, sizeLimits));
// 3. Growth
results.push(checkGrowth(input.candidate, input.baseline, maxGrowthRatio));
// 4. Structural integrity
results.push(checkBalancedFences(input.candidate));
results.push(checkNoPlaceholders(input.candidate));
results.push(checkNoObviousTodos(input.candidate));
// 5. Regression (optional — skipped if scores not provided)
if (input.scores) {
results.push(checkRegression(input.scores.baseline, input.scores.candidate, maxRegression));
}
const firstFailure = results.find(r => r.verdict === 'fail') ?? null;
return {
verdict: firstFailure ? 'fail' : 'pass',
firstFailure,
results,
};
}
// ── Individual gates (exported for unit tests + fine-grained use) ──
export function checkNonEmpty(candidate: string): GateResult {
const trimmed = candidate.trim();
if (trimmed.length === 0) {
return {
gate: 'non-empty',
verdict: 'fail',
reason: 'Candidate is empty or whitespace only',
};
}
return { gate: 'non-empty', verdict: 'pass', reason: 'non-empty' };
}
export function checkSize(
candidate: string,
targetKind: EvolutionTarget,
limits: SizeLimits,
): GateResult {
const max = resolveSizeLimit(targetKind, limits);
const len = candidate.length;
if (len > max) {
return {
gate: 'size',
verdict: 'fail',
reason: `Candidate is ${len} chars but ${targetKind} cap is ${max}`,
detail: { current: len, max },
};
}
return {
gate: 'size',
verdict: 'pass',
reason: `Within ${targetKind} limit (${len}/${max})`,
detail: { current: len, max },
};
}
export function checkGrowth(
candidate: string,
baseline: string,
maxGrowthRatio: number,
): GateResult {
// A candidate shorter than baseline always passes the growth gate.
if (candidate.length <= baseline.length) {
return {
gate: 'growth',
verdict: 'pass',
reason: 'Candidate is not larger than baseline',
detail: { baseline: baseline.length, candidate: candidate.length },
};
}
// Empty baseline means any candidate can't be measured as a ratio —
// defer to the size gate instead.
if (baseline.length === 0) {
return {
gate: 'growth',
verdict: 'pass',
reason: 'No baseline to compare against',
};
}
const ratio = (candidate.length - baseline.length) / baseline.length;
if (ratio > maxGrowthRatio) {
return {
gate: 'growth',
verdict: 'fail',
reason: `Candidate grew ${(ratio * 100).toFixed(1)}% over baseline (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
};
}
return {
gate: 'growth',
verdict: 'pass',
reason: `Grew ${(ratio * 100).toFixed(1)}% (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
};
}
export function checkBalancedFences(candidate: string): GateResult {
const fenceMatches = candidate.match(/```/g);
const count = fenceMatches?.length ?? 0;
if (count % 2 !== 0) {
return {
gate: 'balanced-fences',
verdict: 'fail',
reason: `Odd number of markdown code fences (${count}) — unbalanced`,
detail: { fences: count },
};
}
return {
gate: 'balanced-fences',
verdict: 'pass',
reason: `${count} fence(s), balanced`,
detail: { fences: count },
};
}
/**
* Reject candidates that still contain placeholder text like `[TODO]`,
* `<placeholder>`, or `{{var}}` — these are almost always unfinished
* generations.
*/
export function checkNoPlaceholders(candidate: string): GateResult {
const patterns: Array<{ name: string; re: RegExp }> = [
{ name: 'bracket-placeholder', re: /\[(?:PLACEHOLDER|TODO|FIXME|INSERT|XXX|YOUR[_ ][A-Z ]+)\]/i },
{ name: 'angle-placeholder', re: /<(?:placeholder|todo|your[-_ ][a-z ]+)>/i },
{ name: 'handlebars-placeholder', re: /\{\{[a-zA-Z_][a-zA-Z0-9_.]*\}\}/ },
];
for (const { name, re } of patterns) {
const m = candidate.match(re);
if (m) {
return {
gate: 'no-placeholders',
verdict: 'fail',
reason: `Found unresolved placeholder (${name}): "${m[0]}"`,
};
}
}
return { gate: 'no-placeholders', verdict: 'pass', reason: 'no unresolved placeholders' };
}
/**
* Reject candidates with "TODO:" or "FIXME:" headers — a prompt shouldn't
* ship with unfinished author notes.
*/
export function checkNoObviousTodos(candidate: string): GateResult {
// Only match lines that START with the marker (as a label), not prose
// that happens to mention "todo" (e.g. a task-list prompt).
const re = /(?:^|\n)\s*(?:TODO|FIXME|XXX)\s*:/i;
const m = candidate.match(re);
if (m) {
return {
gate: 'no-todos',
verdict: 'fail',
reason: `Found leftover author marker: "${m[0].trim()}"`,
};
}
return { gate: 'no-todos', verdict: 'pass', reason: 'no leftover author markers' };
}
export function checkRegression(
baselineScore: number,
candidateScore: number,
maxRegression: number,
): GateResult {
const delta = candidateScore - baselineScore;
// Small epsilon avoids float-precision false negatives when delta is
// numerically equal to the tolerance (e.g. 0.78 - 0.8 === -0.02000002).
const epsilon = 1e-9;
if (delta < maxRegression - epsilon) {
return {
gate: 'regression',
verdict: 'fail',
reason: `Candidate score regressed by ${(delta * 100).toFixed(2)}pp (allowed ${(maxRegression * 100).toFixed(2)}pp)`,
detail: { baseline: baselineScore, candidate: candidateScore, delta },
};
}
return {
gate: 'regression',
verdict: 'pass',
reason: `Score delta ${(delta * 100).toFixed(2)}pp (allowed floor ${(maxRegression * 100).toFixed(2)}pp)`,
detail: { baseline: baselineScore, candidate: candidateScore, delta },
};
}
// ── Helpers ─────────────────────────────────────────────────────
function resolveSizeLimit(target: EvolutionTarget, limits: SizeLimits): number {
switch (target) {
case 'persona-system-prompt': return limits.personaSystemPrompt;
case 'tool-description': return limits.toolDescription;
case 'skill-body': return limits.skillBody;
case 'behavioral-spec-section': return limits.behavioralSpecSection;
case 'generic':
default:
return limits.generic;
}
}

View File

@@ -1,43 +1,29 @@
/**
* Evolution LLM Wiring — binds real LLMs into the self-evolution loop.
* LLM wiring for the skill audit and skill hygiene routes.
*
* The evolution primitives (judge, iterative-optimizer, evolve-schema) are
* deliberately model-agnostic: they accept plain callables. This module
* supplies production adapters that turn a Haiku-class LLM into those
* callables, so the `/api/evolution/run` endpoint can perform a real run.
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
* This module supplies the production adapter that turns a Haiku-class LLM
* into that callable, plus a retry wrapper for transient provider errors.
*
* Public surface:
*
* createAnthropicEvolutionLLM(apiKey, model?)
* Builds the default Haiku-backed LLM via @ax-llm/ax — same pattern as
* `optimizer-service.ts`. Dynamic import to avoid startup cost when
* evolution is not in use.
* createAnthropicEvolutionLLM(apiKey, options?)
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
* avoid startup cost when no audit runs.
*
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
* buildGEPAMutateFn(llm) → MutateFn (for IterativeGEPA)
* buildSchemaExecuteFn(llm) → SchemaExecuteFn (for EvolveSchema)
* makeRunningJudge(base, llm) → Pick<LLMJudge,'score'>
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
*
* The `makeRunningJudge` wrapper is what enables GEPA to evaluate real
* prompt effectiveness: on each score() call it first executes the
* candidate prompt against the example input via the LLM, then delegates
* to the wrapped judge with the model's actual response. Without this,
* GEPA compares prompt text to expected output directly — less useful.
*
* All adapters are pure wrappers over `EvolutionLLM.complete()`, which is a
* minimal `(prompt: string) => Promise<string>` contract. Tests supply
* in-memory mocks of that contract; production code goes through AxAI.
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
* mocks of that contract; production code goes through AxAI.
*/
import type { JudgeLLMCall, JudgeInput, JudgeScore } from './judge.js';
import type { LLMJudge } from './judge.js';
import type { MutateArgs, MutateFn, EvolutionTarget } from './iterative-optimizer.js';
import type { SchemaExecuteFn, Schema, SchemaField } from './evolve-schema.js';
import type { JudgeLLMCall } from './judge.js';
// ── Minimal LLM contract ────────────────────────────────────────
/**
* A minimal single-turn completion contract. All evolution adapters depend
* A minimal single-turn completion contract. All adapters depend
* on this interface rather than on @ax-llm/ax directly so they stay easy to
* mock in unit tests.
*/
@@ -206,10 +192,10 @@ export interface CreateAnthropicEvolutionLLMOptions {
/**
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
* @ax-llm/ax so evolution is a zero-cost dependency until used.
* @ax-llm/ax so it is a zero-cost dependency until used.
*
* Returns null when @ax-llm/ax is unavailable — callers should treat null
* as "evolution disabled" and surface a clear error to the user.
* as "LLM unavailable" and surface a clear error to the user.
*/
export async function createAnthropicEvolutionLLM(
apiKey: string,
@@ -240,8 +226,8 @@ export async function createAnthropicEvolutionLLM(
const response = await ai.chat({
chatPrompt: [{ role: 'user', content: prompt }],
});
// chat() may return a ReadableStream in streaming mode — evolution
// does not stream, so we only handle the non-stream case.
// chat() may return a ReadableStream in streaming mode — audits
// do not stream, so we only handle the non-stream case.
const typed = response as {
results?: readonly { content?: string }[];
};
@@ -249,7 +235,7 @@ export async function createAnthropicEvolutionLLM(
return typeof first === 'string' ? first : '';
},
};
// Apply retry policy — defaults are sized for background evolution runs
// Apply retry policy — defaults are sized for background runs
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
// by passing `retry: { maxAttempts: 1 }`.
return wrapWithRetry(inner, options.retry ?? {});
@@ -268,217 +254,3 @@ export async function createAnthropicEvolutionLLM(
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
return (prompt: string) => llm.complete(prompt);
}
// ── Reflective mutation (GEPA) ──────────────────────────────────
export interface BuildReflectiveMutationPromptArgs {
parent: string;
strategy: string;
weaknessFeedback: string[];
targetKind: EvolutionTarget;
generation: number;
}
/**
* Build a reflective mutation prompt for GEPA. Returns the *mutated child*
* text only — no commentary, no wrapping — per the instruction in the
* prompt body. The LLM is told what to fix, using the weakness signal from
* the parent's worst-scoring eval examples.
*/
export function buildReflectiveMutationPrompt(
args: BuildReflectiveMutationPromptArgs,
): string {
const weaknessBullets = args.weaknessFeedback.length > 0
? args.weaknessFeedback.map(fb => `- ${fb}`).join('\n')
: '(no specific weakness signals yet)';
return `You are evolving an AI prompt. This is generation ${args.generation}.
TARGET KIND: ${args.targetKind}
STRATEGY: ${args.strategy}
PARENT PROMPT (do not return this unchanged — produce a *better* variant):
---
${args.parent}
---
WEAKNESS SIGNALS from the parent's lowest-scoring eval examples:
${weaknessBullets}
Apply the strategy above to address the weakness signals. Keep the same
role and overall structure — do not change the prompt's purpose. Produce
a concrete variant that should score higher on the next eval.
Return ONLY the new prompt text. No preamble, no explanation, no markdown
fences. Plain text only.`;
}
/**
* Build a GEPA MutateFn that uses an LLM to produce reflective mutations.
* Graceful fallback: if the LLM throws or returns empty, returns the
* parent prompt unchanged (GEPA will still evaluate it as part of the
* Pareto population — no progress that generation, but no crash).
*/
export function buildGEPAMutateFn(llm: EvolutionLLM): MutateFn {
return async (args: MutateArgs): Promise<string> => {
const prompt = buildReflectiveMutationPrompt({
parent: args.parent.prompt,
strategy: args.strategy,
weaknessFeedback: args.weaknessFeedback,
targetKind: args.targetKind,
generation: args.generation,
});
let raw = '';
try {
raw = await llm.complete(prompt);
} catch {
return args.parent.prompt;
}
const cleaned = stripFences(raw).trim();
if (cleaned.length === 0) return args.parent.prompt;
return cleaned;
};
}
// ── Schema execute (ES) ─────────────────────────────────────────
/**
* Build a schema-filling prompt that asks the LLM to return JSON matching
* the given schema for the given user input. Every field is documented by
* name + type + description + required flag + constraints.
*/
export function buildSchemaFillPrompt(args: { schema: Schema; input: string }): string {
const fieldLines = args.schema.fields.map(f => formatField(f)).join('\n');
return `Fill the JSON schema below using the user input. Respond with
valid JSON — no markdown, no prose, no trailing commas.
SCHEMA (name: ${args.schema.name}, version: ${args.schema.version}):
${fieldLines}
USER INPUT:
${args.input}
Return ONLY the JSON object.`;
}
function formatField(field: SchemaField): string {
const req = field.required ? 'required' : 'optional';
const constraints = field.constraints.length > 0
? ' constraints: ' + field.constraints.map(c => `${c.kind}=${stringifyConstraintValue(c.value)}`).join(', ')
: '';
return `- "${field.name}" (${field.type}, ${req}): ${field.description}${constraints}`;
}
function stringifyConstraintValue(value: string | number | string[]): string {
if (Array.isArray(value)) return `[${value.join(',')}]`;
return String(value);
}
/**
* Build a SchemaExecuteFn that asks the LLM to fill a schema. Returns
* `parsed: true` only when the response parses as JSON. On any error
* (network, timeout, bad model), returns `{actual: '', parsed: false}`.
*/
export function buildSchemaExecuteFn(llm: EvolutionLLM): SchemaExecuteFn {
return async ({ schema, input }) => {
let raw = '';
try {
raw = await llm.complete(buildSchemaFillPrompt({ schema, input }));
} catch {
return { actual: '', parsed: false };
}
const cleaned = stripFences(raw);
return { actual: cleaned, parsed: looksLikeJSON(cleaned) };
};
}
// ── Running judge (executes candidate prompt first, then scores) ─
/**
* Wrap a base judge so that each score() call first runs the candidate
* prompt through the LLM on the example's input. The base judge then
* receives the LLM's output as `actual`, giving a real behavioral signal
* instead of a prompt-text-to-expected-output comparison.
*
* When the LLM call fails, returns a zero score with clear feedback so the
* Pareto population treats the candidate as a loser without crashing the
* whole generation.
*/
/**
* Phantom property that marks a judge as running-capable (i.e. it executes
* the candidate prompt against a real LLM before scoring). IterativeGEPA
* checks for this to prevent regressions where a bare prompt-text-similarity
* judge is passed by mistake.
*
* Symbol-keyed to avoid accidental conflicts with user-defined fields.
*/
export const RUNNING_JUDGE_BRAND = Symbol.for('waggle.runningJudge');
export interface RunningJudge extends Pick<LLMJudge, 'score'> {
readonly [RUNNING_JUDGE_BRAND]: true;
}
export function isRunningJudge(j: Pick<LLMJudge, 'score'> | RunningJudge): j is RunningJudge {
return (j as Record<symbol, unknown>)[RUNNING_JUDGE_BRAND] === true;
}
export function makeRunningJudge(
baseJudge: Pick<LLMJudge, 'score'>,
llm: EvolutionLLM,
): RunningJudge {
return {
[RUNNING_JUDGE_BRAND]: true as const,
async score(args: JudgeInput): Promise<JudgeScore> {
// `args.actual` is the candidate prompt supplied by GEPA. Treat it as
// a system prompt, run it against the example input via the LLM, and
// use the LLM's reply as the new `actual` for the downstream judge.
const runPrompt = buildRunPrompt(args.actual, args.input);
let modelOutput = '';
try {
modelOutput = await llm.complete(runPrompt);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return {
overall: 0, weighted: 0,
correctness: 0, procedureFollowing: 0, conciseness: 0,
lengthPenalty: 1,
feedback: `Candidate execution failed: ${msg}`,
parsed: false,
};
}
return baseJudge.score({ ...args, actual: modelOutput });
},
};
}
function buildRunPrompt(candidatePrompt: string, userInput: string): string {
return `${candidatePrompt}
USER INPUT:
${userInput}
Respond now.`;
}
// ── Shared helpers ──────────────────────────────────────────────
/** Strip ``` and ```json fences if present. Never throws. */
function stripFences(raw: string): string {
if (!raw) return '';
return raw
.replace(/```json\s*/gi, '')
.replace(/```\s*/g, '')
.trim();
}
function looksLikeJSON(s: string): boolean {
const trimmed = s.trim();
if (!trimmed) return false;
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return false;
try {
JSON.parse(trimmed);
return true;
} catch {
return false;
}
}

View File

@@ -1,371 +0,0 @@
/**
* Evolution Orchestrator — Phase 4 of the self-evolution loop.
*
* Wires the primitives into a closed loop:
*
* traces → eval dataset → ComposeEvolution → gates → run store
* ↓
* user accepts/rejects
* ↓
* deploy callback fires
* ↓
* runStore.markDeployed()
*
* The orchestrator itself does NOT hot-reload personas or rewrite the
* behavioral spec — it calls a pluggable `deploy` callback supplied by
* the caller. This keeps the orchestrator pure (easy to unit-test) and
* lets the server/desktop app wire up the real side-effects (file write,
* persona reload, event emission) separately.
*
* Persistence is in @waggle/core's `EvolutionRunStore`. Every run — even
* rejected or failed — is preserved for audit.
*/
import type {
ExecutionTraceStore,
EvolutionRun,
EvolutionRunStore,
EvolutionRunStatus,
EvolutionRunTarget,
ParsedExecutionTrace,
TraceOutcome,
} from '@waggle/core';
import { ComposeEvolution, type ComposeEvolutionOptions, type ComposeEvolutionResult } from './compose-evolution.js';
import { runGates, type GateResult, type GateOptions } from './evolution-gates.js';
import { EvalDatasetBuilder, type EvalExample } from './eval-dataset.js';
// ── Dependency injection types ─────────────────────────────────
export interface EvolutionOrchestratorDeps {
/** Source of traces (from @waggle/core) */
traceStore: ExecutionTraceStore;
/** Persistent run history (from @waggle/core) */
runStore: EvolutionRunStore;
/**
* Function invoked when a run moves from accepted → (deployed | failed).
* Returning normally marks the run deployed; throwing marks it failed
* with the error message as reason.
*
* This is where the server writes to the persona file, reloads the
* orchestrator cache, updates behavioral-spec.ts, etc.
*/
deploy?: (run: EvolutionRun) => Promise<void>;
}
export interface EvolutionAutoTriggerConfig {
/** Minimum traces (with the given outcome filter) before a run is triggered */
minTraces: number;
/** Restrict trigger to a specific persona / workspace / taskShape */
traceFilter?: {
personaId?: string;
workspaceId?: string;
taskShape?: string;
};
}
export interface EvolutionOrchestratorOptions {
/** What is being evolved (drives gate policy + persistence tag) */
targetKind: EvolutionRunTarget;
/** Stable identifier — e.g. persona id, spec-section name */
targetName?: string;
/** The current baseline text to evolve against */
baseline: string;
/** Compose options — already wired with examples, judge, execute, mutate */
compose: Omit<ComposeEvolutionOptions, 'schema' | 'instructions'> & {
schema: Omit<ComposeEvolutionOptions['schema'], 'baseline'>;
instructions: Omit<ComposeEvolutionOptions['instructions'], 'baseline'>;
};
/** Optional gate overrides (size caps, growth ratio, etc) */
gateOptions?: GateOptions;
/** Improvement threshold required to create a proposal (default 0.02) */
minDelta?: number;
/** Auto-trigger configuration — when omitted, runOnce() is manual-only */
autoTrigger?: EvolutionAutoTriggerConfig;
/** Abort the run early */
signal?: AbortSignal;
/** Optional progress pass-through */
onProgress?: (event: EvolutionProgress) => void;
}
export interface EvolutionProgress {
phase: 'trigger-check' | 'dataset' | 'compose' | 'gates' | 'persist' | 'skipped' | 'done';
message?: string;
detail?: unknown;
}
// ── Schema baseline input ──────────────────────────────────────
/** Optional structural baseline when the target has one (e.g. persona's DSPy signature). */
export interface SchemaBaselineInput {
baseline: ComposeEvolutionOptions['schema']['baseline'];
}
// ── Run result ─────────────────────────────────────────────────
export type OrchestratorOutcome =
| 'proposed' // Run created, awaiting accept/reject
| 'skipped-trigger'
| 'skipped-gates' // Completed but gates failed
| 'skipped-delta' // Improvement below minDelta
| 'aborted';
export interface OrchestratorRunResult {
outcome: OrchestratorOutcome;
/** Populated for 'proposed' and 'skipped-gates' */
run?: EvolutionRun;
/** Raw ComposeEvolution result when the compose stage ran */
compose?: ComposeEvolutionResult;
/** Gate results for audit */
gateResults?: GateResult[];
/** Reason if outcome is skipped/aborted */
reason?: string;
}
// ── Orchestrator ──────────────────────────────────────────────
export class EvolutionOrchestrator {
private deps: EvolutionOrchestratorDeps;
constructor(deps: EvolutionOrchestratorDeps) {
this.deps = deps;
}
/**
* Run the full orchestration pipeline once. Returns structured outcome.
* Does NOT deploy — that happens when the caller calls `accept(uuid)`.
*/
async runOnce(options: EvolutionOrchestratorOptions & {
schemaBaseline: SchemaBaselineInput['baseline'];
}): Promise<OrchestratorRunResult> {
const emit = (phase: EvolutionProgress['phase'], message?: string, detail?: unknown) => {
options.onProgress?.({ phase, message, detail });
};
if (options.signal?.aborted) {
return { outcome: 'aborted', reason: 'aborted before start' };
}
// Trigger check — skip if auto-trigger thresholds aren't met.
if (options.autoTrigger) {
emit('trigger-check');
const ok = this.meetsAutoTrigger(options.autoTrigger);
if (!ok) {
emit('skipped');
return {
outcome: 'skipped-trigger',
reason: `fewer than ${options.autoTrigger.minTraces} eligible traces`,
};
}
}
// Derive the eval dataset from finalized traces of the requested target.
emit('dataset');
const examples = this.buildExamplesFromTraces(options);
if (examples.length === 0) {
emit('skipped', 'no eval examples');
return { outcome: 'skipped-trigger', reason: 'no eligible traces to form dataset' };
}
// Run the compose pipeline.
emit('compose');
const composeResult = await new ComposeEvolution().run({
schema: { ...options.compose.schema, baseline: options.schemaBaseline, examples },
instructions: { ...options.compose.instructions, baseline: options.baseline, examples },
feedbackFilter: options.compose.feedbackFilter,
signal: options.signal,
onProgress: (e) => emit('compose', undefined, e),
});
if (options.signal?.aborted) {
return { outcome: 'aborted', reason: 'aborted during compose', compose: composeResult };
}
const winnerText = composeResult.instructions.winner.prompt;
// Apples-to-apples: compare the GEPA baseline candidate (history[0])
// to the GEPA winner. Both are scored by the same judge on the same
// eval stage, so the delta is a clean signal.
const baselineOverall = composeResult.instructions.history[0]?.score?.overall ?? 0;
const winnerOverall = composeResult.instructions.winner.score?.overall ?? 0;
const delta = winnerOverall - baselineOverall;
const minDelta = options.minDelta ?? 0.02;
// Gate check.
emit('gates');
const gateResult = runGates(
{
candidate: winnerText,
baseline: options.baseline,
scores: { baseline: baselineOverall, candidate: winnerOverall },
},
{
targetKind: options.targetKind,
...options.gateOptions,
},
);
if (delta < minDelta) {
emit('skipped', `delta ${(delta * 100).toFixed(2)}pp below minimum ${(minDelta * 100).toFixed(2)}pp`);
emit('done');
return {
outcome: 'skipped-delta',
reason: `winner improved only ${(delta * 100).toFixed(2)}pp (min ${(minDelta * 100).toFixed(2)}pp required)`,
compose: composeResult,
gateResults: gateResult.results,
};
}
// Persist — create the proposed run even if gates failed, so there's audit.
emit('persist');
const run = this.deps.runStore.create({
targetKind: options.targetKind,
targetName: options.targetName ?? null,
baselineText: options.baseline,
winnerText,
winnerSchema: composeResult.frozenSchema,
deltaAccuracy: delta,
gateVerdict: gateResult.verdict,
gateReasons: gateResult.results.map(r => ({
gate: r.gate, verdict: r.verdict, reason: r.reason,
})),
artifacts: {
runSeed: options.compose.instructions.seed ?? null,
generations: options.compose.instructions.generations ?? null,
paretoFrontSize: composeResult.schema.paretoFront.length,
exampleCount: examples.length,
},
});
if (gateResult.verdict === 'fail') {
// Immediately reject runs that fail gates so the user doesn't see
// dangerous candidates in the "review" queue. Run history is kept.
const rejected = this.deps.runStore.reject(
run.run_uuid,
`gate failure: ${gateResult.firstFailure?.reason ?? 'unknown'}`,
);
emit('done');
return {
outcome: 'skipped-gates',
reason: gateResult.firstFailure?.reason ?? 'gate failure',
run: rejected ?? run,
compose: composeResult,
gateResults: gateResult.results,
};
}
emit('done');
return {
outcome: 'proposed',
run,
compose: composeResult,
gateResults: gateResult.results,
};
}
/** Accept a proposed run and invoke the deploy callback. */
async accept(runUuid: string, userNote?: string): Promise<EvolutionRun | undefined> {
const accepted = this.deps.runStore.accept(runUuid, userNote);
if (!accepted || accepted.status !== 'accepted') return accepted;
if (!this.deps.deploy) {
// No deploy hook configured — leave as 'accepted' and let the caller
// mark deployed manually.
return accepted;
}
try {
await this.deps.deploy(accepted);
return this.deps.runStore.markDeployed(runUuid);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
return this.deps.runStore.markFailed(runUuid, reason);
}
}
/** Reject a proposed run with optional reason. */
reject(runUuid: string, reason?: string): EvolutionRun | undefined {
return this.deps.runStore.reject(runUuid, reason);
}
/** List runs (thin pass-through, handy for the UI). */
list(filter?: Parameters<EvolutionRunStore['list']>[0]): EvolutionRun[] {
return this.deps.runStore.list(filter);
}
/** Get a single run by uuid. */
get(runUuid: string): EvolutionRun | undefined {
return this.deps.runStore.getByUuid(runUuid);
}
// ── Internals ───────────────────────────────────────────────
private meetsAutoTrigger(trigger: EvolutionAutoTriggerConfig): boolean {
const outcomes: TraceOutcome[] = ['success', 'verified', 'corrected'];
const traces = this.deps.traceStore.query({
...(trigger.traceFilter ?? {}),
outcome: outcomes,
limit: Math.max(trigger.minTraces * 2, 50),
});
return traces.length >= trigger.minTraces;
}
private buildExamplesFromTraces(
options: EvolutionOrchestratorOptions,
): EvalExample[] {
// If the caller already supplied examples via compose.schema/instructions,
// we respect that. Otherwise we mine them from the trace store.
const existing = options.compose.schema.examples ?? options.compose.instructions.examples;
if (existing && existing.length > 0) return existing;
const builder = new EvalDatasetBuilder(this.deps.traceStore);
const traceFilter = options.autoTrigger?.traceFilter ?? {};
const traces = builder.sourceFromTraces(['success', 'verified'], true, {
...traceFilter,
limit: 500,
});
return traces;
}
}
// ── Convenience helpers ────────────────────────────────────────
/**
* Filter a list of parsed traces down to those likely useful for evolution.
* Keeps finalized (non-pending) traces with non-empty I/O. Used by callers
* that want to manually pre-filter before handing to the orchestrator.
*/
export function eligibleForEvolution(traces: ParsedExecutionTrace[]): ParsedExecutionTrace[] {
return traces.filter(t =>
t.outcome !== 'pending' &&
t.payload.input.trim().length > 0 &&
(t.payload.output.trim().length > 0 || t.payload.correctionFeedback),
);
}
/**
* Helper that summarizes runs by status/target for dashboard display.
*/
export function summarizeRuns(runs: EvolutionRun[]): {
total: number;
byStatus: Record<EvolutionRunStatus, number>;
byTargetKind: Record<string, number>;
bestDelta: number;
} {
const byStatus: Record<EvolutionRunStatus, number> = {
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
};
const byTargetKind: Record<string, number> = {};
let bestDelta = 0;
for (const r of runs) {
byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
byTargetKind[r.target_kind] = (byTargetKind[r.target_kind] ?? 0) + 1;
if (r.delta_accuracy > bestDelta) bestDelta = r.delta_accuracy;
}
return {
total: runs.length,
byStatus,
byTargetKind,
bestDelta,
};
}

View File

@@ -1,852 +0,0 @@
/**
* EvolveSchema — Phase 3.1 of the self-evolution loop.
*
* Port of Mikhail Pavlukhin's EvolveSchema algorithm (Egzakta research).
* Evolves the STRUCTURE of a typed output schema (DSPy-style signatures):
* field names, types, descriptions, ordering, and constraints. This is
* orthogonal to the GEPA optimizer which evolves INSTRUCTIONS against a
* fixed schema.
*
* Three-phase per-generation loop:
*
* Phase A — Structure Discovery
* Spawn variants that add, replace, or drop output fields. This is
* the single biggest-impact mutation class (Mikhail's paper shows
* ~74% of HotPotQA gains come from one structural mutation).
*
* Phase B — Field-Order Probes
* Permute field order. LLMs condition on earlier fields when filling
* later ones, so putting a "reasoning" field BEFORE "answer" routinely
* outperforms the reverse.
*
* Phase C — Failure-Driven Refinement
* Use the per-example judge feedback from the previous generation to
* target specific fields — edit their descriptions, tighten / loosen
* their constraints, or change their types where signals suggest.
*
* 8 typed mutations are available:
* add_output_field, remove_field, edit_field_desc, change_field_type,
* add_constraint, remove_constraint, reorder_fields, replace_output_fields
*
* Pareto selection is 2-dimensional: (accuracy, -complexity) — an
* accurate-but-complex schema is not automatically preferred over a
* slightly-less-accurate simpler one.
*
* Callers provide:
* - `execute(schema, input) → actualOutput` the LLM runner
* - `judge.score({input, expected, actual})` the LLMJudge from Phase 1.3
* - `mutate` (optional) a custom LLM-driven mutator
* for edit_field_desc; falls
* back to a deterministic
* rewriter when omitted.
*/
import type { EvalExample } from './eval-dataset.js';
import type { LLMJudge, JudgeScore } from './judge.js';
// ── Schema types ───────────────────────────────────────────────
export type FieldType = 'string' | 'number' | 'boolean' | 'array' | 'object' | 'enum';
export interface FieldConstraint {
kind: 'minLength' | 'maxLength' | 'pattern' | 'enum' | 'range' | 'custom';
value: string | number | string[];
}
export interface SchemaField {
name: string;
type: FieldType;
description: string;
required: boolean;
constraints: FieldConstraint[];
}
export interface Schema {
name: string;
fields: SchemaField[];
version: number;
}
// ── Mutation types ─────────────────────────────────────────────
export type MutationKind =
| 'add_output_field'
| 'remove_field'
| 'edit_field_desc'
| 'change_field_type'
| 'add_constraint'
| 'remove_constraint'
| 'reorder_fields'
| 'replace_output_fields';
export interface Mutation {
kind: MutationKind;
/** What changed, for logging + feedback */
description: string;
/** Functional transformer (pure) */
apply: (schema: Schema) => Schema;
}
// ── Candidate + result types ──────────────────────────────────
export interface SchemaCandidate {
id: string;
schema: Schema;
generation: number;
parent: string | null;
mutation: MutationKind | 'baseline';
mutationLabel: string;
score: SchemaCandidateScore | null;
perExample: SchemaExampleResult[];
}
export interface SchemaExampleResult {
input: string;
expected: string;
actual: string;
score: JudgeScore;
/** Diagnostic: did the output parse against the schema? */
parsed: boolean;
}
export interface SchemaCandidateScore {
/** Mean overall judge score across the eval set (0..1) */
accuracy: number;
/** Schema "complexity" — a scalar that grows with fields, descriptions, constraints */
complexity: number;
/** Share of examples that produced parseable output against this schema (0..1) */
parseRate: number;
/** Weakness feedback taken from the worst-scoring 3 examples */
weaknessFeedback: string[];
/** Number of examples scored */
n: number;
}
export type SchemaExecuteFn = (args: {
schema: Schema;
input: string;
}) => Promise<{ actual: string; parsed: boolean }>;
export interface EvolveSchemaOptions {
/** Starting schema */
baseline: Schema;
/** Eval dataset */
examples: EvalExample[];
/** LLM runner that takes a schema + input and returns an attempted output */
execute: SchemaExecuteFn;
/** Judge for scoring individual attempts */
judge: Pick<LLMJudge, 'score'>;
/** Candidates per generation (default 5) */
populationSize?: number;
/** Generations (default 3) */
generations?: number;
/** Mix of phase mutations per generation */
mutationMix?: { structure: number; order: number; refinement: number };
/** Eval sample size per generation (default 32) */
evalSize?: number;
/** Anchor eval size at the end (default 100) */
anchorEvalSize?: number;
/** Seed for sampling (default 1) */
seed?: number;
/** Optional LLM helper for description edits — deterministic fallback used if omitted */
editFieldDescription?: (args: {
field: SchemaField;
feedback: string[];
}) => Promise<string>;
/** Optional progress emitter */
onProgress?: (event: EvolveSchemaProgress) => void;
/** Optional abort signal */
signal?: AbortSignal;
}
export interface EvolveSchemaProgress {
phase: 'start' | 'structure' | 'order' | 'refinement' | 'anchor' | 'done';
generation: number;
populationSize: number;
bestAccuracy: number;
bestComplexity: number;
message?: string;
}
export interface EvolveSchemaResult {
winner: SchemaCandidate;
/** Full Pareto-non-dominated set from the anchor stage */
paretoFront: SchemaCandidate[];
/** All candidates across all generations */
history: SchemaCandidate[];
/** Accuracy delta: winner.accuracy - baseline.accuracy */
deltaAccuracy: number;
/** True if the winner's accuracy improved over baseline */
improved: boolean;
}
// ── Pure mutation functions ───────────────────────────────────
export function addOutputField(
schema: Schema,
field: SchemaField,
position?: number,
): Schema {
const pos = position ?? schema.fields.length;
const fields = [...schema.fields];
fields.splice(pos, 0, { ...field, constraints: [...field.constraints] });
return { ...schema, fields, version: schema.version + 1 };
}
export function removeField(schema: Schema, fieldName: string): Schema {
return {
...schema,
fields: schema.fields.filter(f => f.name !== fieldName),
version: schema.version + 1,
};
}
export function editFieldDescription(
schema: Schema,
fieldName: string,
newDescription: string,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, description: newDescription } : f,
),
version: schema.version + 1,
};
}
export function changeFieldType(
schema: Schema,
fieldName: string,
newType: FieldType,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, type: newType } : f,
),
version: schema.version + 1,
};
}
export function addConstraint(
schema: Schema,
fieldName: string,
constraint: FieldConstraint,
): Schema {
return {
...schema,
fields: schema.fields.map(f =>
f.name === fieldName ? { ...f, constraints: [...f.constraints, constraint] } : f,
),
version: schema.version + 1,
};
}
export function removeConstraint(
schema: Schema,
fieldName: string,
constraintIndex: number,
): Schema {
return {
...schema,
fields: schema.fields.map(f => {
if (f.name !== fieldName) return f;
const constraints = f.constraints.filter((_, i) => i !== constraintIndex);
return { ...f, constraints };
}),
version: schema.version + 1,
};
}
export function reorderFields(schema: Schema, newOrder: string[]): Schema {
const byName = new Map(schema.fields.map(f => [f.name, f]));
const reordered: SchemaField[] = [];
for (const name of newOrder) {
const field = byName.get(name);
if (field) reordered.push(field);
}
// Append any fields that were omitted from newOrder (keeps their position stable).
for (const field of schema.fields) {
if (!newOrder.includes(field.name)) reordered.push(field);
}
return { ...schema, fields: reordered, version: schema.version + 1 };
}
export function replaceOutputFields(schema: Schema, newFields: SchemaField[]): Schema {
return {
...schema,
fields: newFields.map(f => ({ ...f, constraints: [...f.constraints] })),
version: schema.version + 1,
};
}
// ── Complexity + scoring helpers ───────────────────────────────
/**
* Scalar complexity measure: 1 per field + 0.3 per constraint +
* 0.01 per character of description. Calibrated so a 3-field schema
* with short descriptions ≈ 3.6, a 10-field schema with long
* descriptions ≈ 15+.
*/
export function schemaComplexity(schema: Schema): number {
let c = 0;
for (const field of schema.fields) {
c += 1;
c += field.constraints.length * 0.3;
c += (field.description?.length ?? 0) * 0.01;
}
return c;
}
export function aggregateSchemaScores(results: SchemaExampleResult[]): SchemaCandidateScore {
if (results.length === 0) {
return {
accuracy: 0, complexity: 0, parseRate: 0, weaknessFeedback: [], n: 0,
};
}
const n = results.length;
const accuracy = results.reduce((s, r) => s + r.score.overall, 0) / n;
const parseRate = results.filter(r => r.parsed).length / n;
const worst = [...results].sort((a, b) => a.score.overall - b.score.overall).slice(0, 3);
const weaknessFeedback = worst
.map(r => r.score.feedback)
.filter(fb => fb && fb.length > 0);
return { accuracy, complexity: 0, parseRate, weaknessFeedback, n };
}
// ── Pareto (2D) ───────────────────────────────────────────────
/**
* Pareto front on (accuracy↑, complexity↓). A candidate dominates
* another iff it is at least as accurate AND no more complex, and
* strictly better on at least one dimension.
*/
export function paretoFrontSchema(candidates: SchemaCandidate[]): SchemaCandidate[] {
const scored = candidates.filter(c => c.score !== null);
const front: SchemaCandidate[] = [];
for (const cand of scored) {
let dominated = false;
for (const other of scored) {
if (other === cand) continue;
if (dominatesSchema(other.score!, cand.score!)) {
dominated = true;
break;
}
}
if (!dominated) front.push(cand);
}
return front.length > 0 ? front : scored;
}
function dominatesSchema(a: SchemaCandidateScore, b: SchemaCandidateScore): boolean {
const geAll = a.accuracy >= b.accuracy && a.complexity <= b.complexity;
const gtAny = a.accuracy > b.accuracy || a.complexity < b.complexity;
return geAll && gtAny;
}
// ── Candidate scoring ─────────────────────────────────────────
export async function scoreSchemaCandidate(
candidate: SchemaCandidate,
examples: EvalExample[],
execute: SchemaExecuteFn,
judge: Pick<LLMJudge, 'score'>,
signal?: AbortSignal,
): Promise<SchemaCandidateScore> {
const results: SchemaExampleResult[] = [];
for (const ex of examples) {
if (signal?.aborted) break;
try {
const { actual, parsed } = await execute({ schema: candidate.schema, input: ex.input });
const score = await judge.score({
input: ex.input,
expected: ex.expected_output,
actual,
});
results.push({ input: ex.input, expected: ex.expected_output, actual, score, parsed });
} catch {
// Keep iterating — one broken example shouldn't poison the whole batch.
}
}
const agg = aggregateSchemaScores(results);
agg.complexity = schemaComplexity(candidate.schema);
candidate.perExample = results;
candidate.score = agg;
return agg;
}
// ── Mutation generators per phase ─────────────────────────────
/**
* Structure-Discovery mutations: add a new field, replace fields wholesale,
* or drop one if the schema is too large. Returns up to `n` mutations.
*/
export function generateStructureMutations(
parent: Schema,
n: number,
rng: () => number,
): Mutation[] {
const out: Mutation[] = [];
// Add a reasoning field if absent.
if (!parent.fields.some(f => /reason|thought|scratch/i.test(f.name))) {
out.push({
kind: 'add_output_field',
description: 'add reasoning field before answer',
apply: (s) =>
addOutputField(s, {
name: 'reasoning',
type: 'string',
description: 'Concise chain-of-thought explaining how the answer was derived.',
required: true,
constraints: [],
}, 0),
});
}
// Add a confidence field if absent.
if (!parent.fields.some(f => /confidence|certainty|score/i.test(f.name))) {
out.push({
kind: 'add_output_field',
description: 'add confidence field at end',
apply: (s) =>
addOutputField(s, {
name: 'confidence',
type: 'number',
description: 'Self-estimated confidence in the answer (0.0 - 1.0).',
required: false,
constraints: [{ kind: 'range', value: '0..1' }],
}),
});
}
// Drop the lowest-priority field if > 4 fields.
if (parent.fields.length > 4) {
const candidate = parent.fields[parent.fields.length - 1];
out.push({
kind: 'remove_field',
description: `drop trailing field "${candidate.name}"`,
apply: (s) => removeField(s, candidate.name),
});
}
// Replace all output fields with a minimal skeleton (reasoning + answer).
if (parent.fields.length >= 2) {
out.push({
kind: 'replace_output_fields',
description: 'collapse to {reasoning, answer}',
apply: (s) =>
replaceOutputFields(s, [
{
name: 'reasoning',
type: 'string',
description: 'Brief chain-of-thought.',
required: true,
constraints: [],
},
{
name: 'answer',
type: 'string',
description: 'Final answer.',
required: true,
constraints: [],
},
]),
});
}
return pickN(out, n, rng);
}
/**
* Order-Probe mutations: generate permutations of field order. Limits
* output to `n` by random sampling from the set of useful permutations.
*/
export function generateOrderMutations(
parent: Schema,
n: number,
rng: () => number,
): Mutation[] {
const names = parent.fields.map(f => f.name);
if (names.length < 2) return [];
const out: Mutation[] = [];
// Heuristic 1: move any reasoning/thought field to the front.
const reasoningIdx = parent.fields.findIndex(f => /reason|thought|scratch/i.test(f.name));
if (reasoningIdx > 0) {
const newOrder = [names[reasoningIdx], ...names.filter((_, i) => i !== reasoningIdx)];
out.push({
kind: 'reorder_fields',
description: `move "${names[reasoningIdx]}" to front`,
apply: (s) => reorderFields(s, newOrder),
});
}
// Heuristic 2: move any confidence/score field to the end.
const confIdx = parent.fields.findIndex(f => /confidence|certainty/i.test(f.name));
if (confIdx >= 0 && confIdx !== parent.fields.length - 1) {
const newOrder = [...names.filter((_, i) => i !== confIdx), names[confIdx]];
out.push({
kind: 'reorder_fields',
description: `move "${names[confIdx]}" to end`,
apply: (s) => reorderFields(s, newOrder),
});
}
// Fill remaining with random permutations.
while (out.length < n) {
const shuffled = shuffle([...names], rng);
if (!arraysEqual(shuffled, names)) {
out.push({
kind: 'reorder_fields',
description: `reorder to [${shuffled.join(', ')}]`,
apply: (s) => reorderFields(s, shuffled),
});
}
if (out.length > n * 3) break; // safety — very short schemas
}
return pickN(out, n, rng);
}
/**
* Refinement mutations driven by worst-scoring example feedback.
* Edits descriptions, tightens constraints, or switches types based on
* heuristic patterns in the feedback.
*/
export async function generateRefinementMutations(
parent: Schema,
weakness: string[],
n: number,
editFieldDescription?: (args: {
field: SchemaField;
feedback: string[];
}) => Promise<string>,
): Promise<Mutation[]> {
const out: Mutation[] = [];
const joined = weakness.join(' ').toLowerCase();
// Heuristic: if feedback mentions "too verbose" / "long" — add a maxLength to any string field.
if (/verbose|too long|wordy/i.test(joined)) {
const stringFields = parent.fields.filter(f => f.type === 'string');
for (const f of stringFields.slice(0, 2)) {
out.push({
kind: 'add_constraint',
description: `tighten "${f.name}" with maxLength: 200`,
apply: (s) => addConstraint(s, f.name, { kind: 'maxLength', value: 200 }),
});
}
}
// Heuristic: if feedback mentions "too brief" / "incomplete" — add minLength.
if (/brief|incomplete|missing|short/i.test(joined)) {
const stringFields = parent.fields.filter(f => f.type === 'string');
for (const f of stringFields.slice(0, 2)) {
out.push({
kind: 'add_constraint',
description: `require "${f.name}" minLength: 20`,
apply: (s) => addConstraint(s, f.name, { kind: 'minLength', value: 20 }),
});
}
}
// Heuristic: if feedback mentions "wrong format" / "parse" — clarify description.
if (/format|parse|wrong type/i.test(joined)) {
for (const field of parent.fields.slice(0, 2)) {
const newDesc = editFieldDescription
? await editFieldDescription({ field, feedback: weakness }).catch(() => field.description)
: deterministicClarifyDescription(field);
if (newDesc && newDesc !== field.description) {
out.push({
kind: 'edit_field_desc',
description: `clarify "${field.name}" description`,
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
});
}
}
}
// Fallback: always consider one description rewrite on the first field.
if (out.length === 0 && parent.fields.length > 0) {
const field = parent.fields[0];
const newDesc = deterministicClarifyDescription(field);
if (newDesc !== field.description) {
out.push({
kind: 'edit_field_desc',
description: `clarify "${field.name}" description`,
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
});
}
}
return out.slice(0, n);
}
// ── EvolveSchema engine ──────────────────────────────────────
export class EvolveSchema {
async run(options: EvolveSchemaOptions): Promise<EvolveSchemaResult> {
const config = normalizeOptions(options);
const rng = makeRng(config.seed);
const baseline: SchemaCandidate = {
id: 'g0-baseline',
schema: cloneSchema(config.baseline),
generation: 0,
parent: null,
mutation: 'baseline',
mutationLabel: 'baseline',
score: null,
perExample: [],
};
emitProgress(config, 'start', 0, 1, 0, 0, 'scoring baseline');
const evalSample = pickSample(config.examples, config.evalSize, rng);
await scoreSchemaCandidate(baseline, evalSample, config.execute, config.judge, config.signal);
const history: SchemaCandidate[] = [baseline];
let survivors: SchemaCandidate[] = [baseline];
for (let gen = 1; gen <= config.generations; gen++) {
if (config.signal?.aborted) break;
const topParent = pickTopByAccuracy(survivors);
if (!topParent) break;
const children: SchemaCandidate[] = [];
// Phase A — Structure Discovery
emitProgress(
config, 'structure', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const structureMutations = generateStructureMutations(
topParent.schema, config.mutationMix.structure, rng,
);
children.push(...applyMutations(structureMutations, topParent, gen));
// Phase B — Field-Order Probes
emitProgress(
config, 'order', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const orderMutations = generateOrderMutations(
topParent.schema, config.mutationMix.order, rng,
);
children.push(...applyMutations(orderMutations, topParent, gen));
// Phase C — Failure-Driven Refinement
emitProgress(
config, 'refinement', gen, survivors.length,
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
);
const refinementMutations = await generateRefinementMutations(
topParent.schema,
topParent.score?.weaknessFeedback ?? [],
config.mutationMix.refinement,
config.editFieldDescription,
);
children.push(...applyMutations(refinementMutations, topParent, gen));
// Score all children on the per-generation sample.
const genSample = pickSample(config.examples, config.evalSize, rng);
for (const child of children) {
if (config.signal?.aborted) break;
await scoreSchemaCandidate(child, genSample, config.execute, config.judge, config.signal);
history.push(child);
}
// Pareto-select survivors: parent + children.
survivors = paretoFrontSchema([topParent, ...children]);
if (survivors.length > config.populationSize) {
// Trim by picking the best-accuracy members.
survivors = survivors
.slice()
.sort((a, b) => (b.score?.accuracy ?? 0) - (a.score?.accuracy ?? 0))
.slice(0, config.populationSize);
}
}
// Anchor stage: larger eval on final survivors.
emitProgress(
config, 'anchor', config.generations, survivors.length,
bestAccuracy(survivors), minComplexity(survivors),
);
const anchorSample = pickSample(config.examples, config.anchorEvalSize, rng);
for (const cand of survivors) {
if (config.signal?.aborted) break;
await scoreSchemaCandidate(cand, anchorSample, config.execute, config.judge, config.signal);
}
const paretoFront = paretoFrontSchema(survivors);
const winner = pickSchemaWinner(paretoFront);
const deltaAccuracy = (winner.score?.accuracy ?? 0) - (baseline.score?.accuracy ?? 0);
emitProgress(
config, 'done', config.generations, paretoFront.length,
winner.score?.accuracy ?? 0, winner.score?.complexity ?? 0,
);
return {
winner,
paretoFront,
history,
deltaAccuracy,
improved: deltaAccuracy > 0,
};
}
}
// ── Helpers ────────────────────────────────────────────────────
function applyMutations(
mutations: Mutation[],
parent: SchemaCandidate,
generation: number,
): SchemaCandidate[] {
return mutations.map((m, i) => ({
id: `g${generation}-${m.kind}-${i}`,
schema: m.apply(parent.schema),
generation,
parent: parent.id,
mutation: m.kind,
mutationLabel: m.description,
score: null,
perExample: [],
}));
}
function pickTopByAccuracy(candidates: SchemaCandidate[]): SchemaCandidate | undefined {
if (candidates.length === 0) return undefined;
return candidates.reduce((a, b) =>
(a.score?.accuracy ?? 0) >= (b.score?.accuracy ?? 0) ? a : b,
);
}
export function pickSchemaWinner(candidates: SchemaCandidate[]): SchemaCandidate {
if (candidates.length === 0) {
throw new Error('pickSchemaWinner called on empty candidate list');
}
return candidates.reduce((best, cand) => {
const bA = best.score?.accuracy ?? -1;
const cA = cand.score?.accuracy ?? -1;
if (cA > bA) return cand;
if (cA < bA) return best;
// Tie on accuracy → prefer lower complexity.
const bC = best.score?.complexity ?? Infinity;
const cC = cand.score?.complexity ?? Infinity;
return cC < bC ? cand : best;
});
}
function bestAccuracy(cs: SchemaCandidate[]): number {
if (cs.length === 0) return 0;
return Math.max(...cs.map(c => c.score?.accuracy ?? 0));
}
function minComplexity(cs: SchemaCandidate[]): number {
const scored = cs.filter(c => c.score !== null);
if (scored.length === 0) return 0;
return Math.min(...scored.map(c => c.score!.complexity));
}
function cloneSchema(s: Schema): Schema {
return {
...s,
fields: s.fields.map(f => ({ ...f, constraints: [...f.constraints] })),
};
}
function deterministicClarifyDescription(field: SchemaField): string {
const base = field.description.trim();
if (!base) return `Return the value for "${field.name}" as ${field.type}.`;
if (/\b(must|should|return)\b/i.test(base)) return base;
return `${base} Must be a ${field.type}.`;
}
// Disambiguation re-export — the generator calls this internally so mutations
// can reference it without colliding with the public `editFieldDescription`
// option callback in `EvolveSchemaOptions`.
const editFieldDescription_pure = editFieldDescription;
function pickN<T>(arr: T[], n: number, rng: () => number): T[] {
if (arr.length <= n) return arr.slice();
const copy = arr.slice();
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, n);
}
function shuffle<T>(arr: T[], rng: () => number): T[] {
for (let i = arr.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr;
}
function arraysEqual<T>(a: T[], b: T[]): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
return true;
}
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
if (k <= 0 || examples.length === 0) return [];
const copy = [...examples];
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, Math.min(k, copy.length));
}
function emitProgress(
config: { onProgress?: (e: EvolveSchemaProgress) => void },
phase: EvolveSchemaProgress['phase'],
generation: number,
populationSize: number,
bestAcc: number,
bestCmp: number,
message?: string,
): void {
if (config.onProgress) {
config.onProgress({
phase, generation, populationSize,
bestAccuracy: bestAcc,
bestComplexity: bestCmp,
message,
});
}
}
function normalizeOptions(opts: EvolveSchemaOptions) {
return {
baseline: opts.baseline,
examples: opts.examples,
execute: opts.execute,
judge: opts.judge,
populationSize: opts.populationSize ?? 5,
generations: opts.generations ?? 3,
mutationMix: opts.mutationMix ?? { structure: 3, order: 1, refinement: 1 },
evalSize: opts.evalSize ?? 32,
anchorEvalSize: opts.anchorEvalSize ?? 100,
seed: opts.seed ?? 1,
editFieldDescription: opts.editFieldDescription,
onProgress: opts.onProgress,
signal: opts.signal,
};
}
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}

View File

@@ -1,19 +1,17 @@
/**
* HarnessTraceBridge — translates workflow-harness phase events into
* execution traces so harnessed agent runs feed the self-evolution loop.
* execution traces so harnessed agent runs show up in cost tracking and
* run history.
*
* Without this bridge the workflow harness is an island: phases run,
* gates pass or fail, but nothing flows into the `execution_traces`
* table that the evaluation dataset builder mines. This module closes
* that loop by subscribing to the shared `harnessEvents` emitter and
* writing one trace per phase outcome:
* table. This module subscribes to the shared `harnessEvents` emitter and
* writes one trace per phase outcome:
*
* harness:phase:complete → trace outcome 'verified'
* harness:phase:fail (aborted === true) → trace outcome 'abandoned'
* harness:phase:fail (will retry) → skipped — only final
* outcomes become training
* signal, mid-retry noise
* stays out of the dataset
* outcomes are recorded
*
* Each trace records:
* - input: the phase's instruction text

View File

@@ -1,249 +0,0 @@
/**
* Improvement Detector — produces structured, runtime-consumable awareness signals.
*
* Per correction #1: Returns actionable signals + surfaced suggestions as structured data,
* not just prompt text changes.
*
* Per correction #5: Awareness is operational — route better, suggest better, adapt better,
* avoid repeated mistakes.
*
* Per correction #3: Workflow suggestions require recency + pattern similarity,
* not just raw shape count.
*/
import type { ImprovementSignalStore, ActionableSignal, SignalCategory } from '@waggle/core';
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
// ── Structured output types ──────────────────────────────────
export interface AwarenessSummary {
/** Capability gaps the agent has encountered repeatedly */
capabilityGaps: CapabilityGapSignal[];
/** Behavioral corrections the user has made repeatedly */
corrections: CorrectionSignal[];
/** Workflow patterns that recur and could benefit from templates */
workflowPatterns: WorkflowPatternSignal[];
/** Total actionable signal count (for deciding whether to inject into prompt) */
totalActionable: number;
}
export interface CapabilityGapSignal {
id: number;
toolName: string;
occurrences: number;
suggestion: string;
}
export interface CorrectionSignal {
id: number;
patternKey: string;
detail: string;
occurrences: number;
guidance: string;
}
export interface WorkflowPatternSignal {
id: number;
patternKey: string;
occurrences: number;
suggestion: string;
}
// ── Capability gap recording ─────────────────────────────────
/**
* Record a capability gap when a tool is not found or a skill is missing.
* Called from agent-loop when tool lookup fails.
*/
export function recordCapabilityGap(
store: ImprovementSignalStore,
toolName: string,
context?: string,
): void {
store.record('capability_gap', `missing:${toolName}`, context, {
tool: toolName,
lastContext: context,
});
}
// ── Correction recording ─────────────────────────────────────
/**
* Analyze a user message for corrections and record if detected.
* Only task-local and durable corrections are recorded (both go into the store;
* the store's threshold mechanism handles promotion to actionable).
*
* Returns the detected correction (if any) for caller use.
*/
export function analyzeAndRecordCorrection(
store: ImprovementSignalStore,
userMessage: string,
previousAssistantMessage?: string,
): DetectedCorrection | null {
const correction = detectCorrection(userMessage, previousAssistantMessage);
if (!correction) return null;
// Record in store — both durable and task-local go in.
// Durable corrections get recorded; task-local ones also get recorded but
// only become actionable if the same pattern_key recurs (count >= threshold).
store.record('correction', correction.patternKey, correction.detail, {
isDurable: correction.isDurable,
confidence: correction.confidence,
});
return correction;
}
// ── Workflow pattern recording ────────────────────────────────
const RECENCY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
/**
* Record a workflow pattern occurrence.
* Per correction #3: requires recency context, not just raw shape count.
*/
export function recordWorkflowPattern(
store: ImprovementSignalStore,
taskShape: string,
taskDescription: string,
): void {
store.record('workflow_pattern', `shape:${taskShape}`, taskDescription, {
lastTask: taskDescription.slice(0, 100),
recordedAt: new Date().toISOString(),
});
}
// ── Structured awareness builder ─────────────────────────────
/**
* Build a structured awareness summary from the signal store.
* Returns runtime-consumable data that can be:
* 1. Injected into system prompt as guidance
* 2. Used by the agent loop for routing decisions
* 3. Displayed in UI as improvement suggestions
*
* Per correction #6: capped at 3 actionable signals total, non-repeating.
*/
export function buildAwarenessSummary(store: ImprovementSignalStore): AwarenessSummary {
const actionable = store.getActionable();
const capabilityGaps: CapabilityGapSignal[] = [];
const corrections: CorrectionSignal[] = [];
const workflowPatterns: WorkflowPatternSignal[] = [];
for (const signal of actionable) {
switch (signal.category) {
case 'capability_gap':
capabilityGaps.push(formatCapabilityGap(signal));
break;
case 'correction':
corrections.push(formatCorrection(signal));
break;
case 'workflow_pattern':
// Per correction #3: only surface if recent (within 7 days)
if (isRecent(signal.last_seen)) {
workflowPatterns.push(formatWorkflowPattern(signal));
}
break;
}
}
return {
capabilityGaps,
corrections,
workflowPatterns,
totalActionable: capabilityGaps.length + corrections.length + workflowPatterns.length,
};
}
/**
* Format awareness summary as a prompt section for system prompt injection.
* Only included when there are actionable signals.
*/
export function formatAwarenessPrompt(summary: AwarenessSummary): string | null {
if (summary.totalActionable === 0) return null;
const lines: string[] = ['## Improvement Signals'];
if (summary.capabilityGaps.length > 0) {
lines.push('');
lines.push('**Missing capabilities** (user has needed these before):');
for (const gap of summary.capabilityGaps) {
lines.push(`- ${gap.suggestion}`);
}
}
if (summary.corrections.length > 0) {
lines.push('');
lines.push('**Behavioral adjustments** (user has corrected these patterns):');
for (const correction of summary.corrections) {
lines.push(`- ${correction.guidance}`);
}
}
if (summary.workflowPatterns.length > 0) {
lines.push('');
lines.push('**Recurring workflows** (consider suggesting a template):');
for (const pattern of summary.workflowPatterns) {
lines.push(`- ${pattern.suggestion}`);
}
}
return lines.join('\n');
}
/**
* Mark all signals in a summary as surfaced.
* Call this after the signals have been injected into a prompt or shown to the user.
*/
export function markSummarySurfaced(
store: ImprovementSignalStore,
summary: AwarenessSummary,
): void {
for (const gap of summary.capabilityGaps) store.markSurfaced(gap.id);
for (const correction of summary.corrections) store.markSurfaced(correction.id);
for (const pattern of summary.workflowPatterns) store.markSurfaced(pattern.id);
}
// ── Helpers ──────────────────────────────────────────────────
function formatCapabilityGap(signal: ActionableSignal): CapabilityGapSignal {
const toolName = signal.parsedMetadata.tool as string || signal.pattern_key.replace('missing:', '');
return {
id: signal.id,
toolName,
occurrences: signal.count,
suggestion: `User has needed "${toolName}" ${signal.count} times. Consider using acquire_capability to find it.`,
};
}
function formatCorrection(signal: ActionableSignal): CorrectionSignal {
return {
id: signal.id,
patternKey: signal.pattern_key,
detail: signal.detail,
occurrences: signal.count,
guidance: signal.detail
? `${signal.detail} (corrected ${signal.count} times)`
: `Correction pattern "${signal.pattern_key}" observed ${signal.count} times`,
};
}
function formatWorkflowPattern(signal: ActionableSignal): WorkflowPatternSignal {
const shape = signal.pattern_key.replace('shape:', '');
return {
id: signal.id,
patternKey: signal.pattern_key,
occurrences: signal.count,
suggestion: `"${shape}" tasks recur frequently (${signal.count} times). A workflow template could streamline this.`,
};
}
function isRecent(dateStr: string): boolean {
try {
const date = new Date(dateStr);
return Date.now() - date.getTime() < RECENCY_WINDOW_MS;
} catch {
return false;
}
}

View File

@@ -1,181 +0,0 @@
/**
* @internal Not exported from index.ts — future feature, tested but not wired.
*
* Improvement Wiring — processes each interaction for improvement signals.
*
* Bridges the gap between the chat route (which has raw messages) and the
* existing correction-detector / improvement-detector / capability-acquisition
* modules. Call processInteractionForImprovement after every agent response
* to detect corrections, capability gaps, and recurring workflow patterns.
*/
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
import { detectTaskShape, type TaskShape } from './task-shape.js';
// ── Types ──────────────────────────────────────────────────────────────
export interface ImprovementWiringParams {
userMessage: string;
agentResponse: string;
toolsUsed: string[];
workspaceId: string;
sessionId: string;
}
export interface ImprovementWiringResult {
wasCorrection: boolean;
correctionDetail?: string;
capabilityGap?: string;
workflowPattern?: string;
}
// ── Well-known tool names (built-in tools the agent can use) ───────────
const KNOWN_TOOL_NAMES = new Set([
'web_search', 'web_fetch', 'search_memory', 'save_memory',
'get_identity', 'get_awareness', 'query_knowledge', 'add_task',
'correct_knowledge', 'bash', 'read_file', 'write_file', 'edit_file',
'search_files', 'search_content', 'git_status', 'git_diff', 'git_log',
'git_commit', 'generate_docx', 'create_plan', 'add_plan_step',
'execute_step', 'show_plan', 'list_skills', 'create_skill',
'delete_skill', 'read_skill', 'search_skills', 'suggest_skill',
'acquire_capability', 'install_capability', 'compose_workflow',
'orchestrate_workflow', 'spawn_agent', 'list_agents', 'get_agent_result',
]);
// ── Capability gap detection patterns ──────────────────────────────────
/**
* Patterns in user messages that suggest the user wants a tool/capability
* the agent doesn't have. These are request-like phrases paired with
* tool/domain keywords.
*/
const GAP_REQUEST_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
{ pattern: /\bcan you (?:send|post|publish|push) (?:to|on|via) (\w+)/i, domain: 'integration' },
{ pattern: /\bconnect (?:to|with) (\w+)/i, domain: 'connector' },
{ pattern: /\buse (\w+) (?:api|tool|service)/i, domain: 'integration' },
{ pattern: /\b(?:read|open|parse|convert) (?:this |the |a )?(\w+) file/i, domain: 'file_format' },
{ pattern: /\brun (?:this |the |a )?(\w+) (?:test|check|scan|lint)/i, domain: 'tooling' },
];
/**
* Detect if the agent response itself indicates a tool was missing.
* The agent often says things like "I don't have a tool for X" or
* "Tool 'X' not found".
*/
const TOOL_NOT_FOUND_PATTERNS: RegExp[] = [
/Tool "(.+?)" not found/,
/I don't have (?:a |the )?(?:tool|capability|ability) (?:for|to) (.+?)(?:\.|$)/i,
/no (?:tool|capability) available for (.+?)(?:\.|$)/i,
];
// ── Workflow pattern detection ─────────────────────────────────────────
/**
* Multi-step request indicators — the user is asking for a sequence of actions.
*/
const MULTI_STEP_PATTERNS: RegExp[] = [
/\bthen\b.*\bthen\b/i, // "do X, then Y, then Z"
/\bfirst\b.*\bthen\b/i, // "first do X, then Y"
/\bstep\s*\d+/i, // "step 1, step 2"
/\b(?:and then|after that|next|finally)\b/i, // sequence words
];
// ── Main function ──────────────────────────────────────────────────────
/**
* Process an interaction for improvement signals.
*
* This is the main entry point called after every agent response.
* It analyzes the user message and agent response to detect:
* 1. Corrections (using the existing correction-detector)
* 2. Capability gaps (tools requested but not available)
* 3. Workflow patterns (recurring multi-step sequences)
*
* Returns structured signals for storage and display.
*/
export function processInteractionForImprovement(
params: ImprovementWiringParams,
): ImprovementWiringResult {
const { userMessage, agentResponse, toolsUsed } = params;
const result: ImprovementWiringResult = {
wasCorrection: false,
};
// 1. Run correction detector on userMessage
const correction = detectCorrection(userMessage);
if (correction) {
result.wasCorrection = true;
result.correctionDetail = correction.detail;
}
// 2. Check for capability gaps
const gap = detectCapabilityGap(userMessage, agentResponse, toolsUsed);
if (gap) {
result.capabilityGap = gap;
}
// 3. Check for workflow patterns (repeated multi-step sequences)
const pattern = detectWorkflowPattern(userMessage);
if (pattern) {
result.workflowPattern = pattern;
}
return result;
}
// ── Internal helpers ───────────────────────────────────────────────────
/**
* Detect capability gaps from the interaction.
* Checks:
* - Agent response mentions a tool not found
* - User message requests a tool/integration not in the known set
*/
function detectCapabilityGap(
userMessage: string,
agentResponse: string,
toolsUsed: string[],
): string | undefined {
// Check agent response for explicit tool-not-found messages
for (const pattern of TOOL_NOT_FOUND_PATTERNS) {
const match = agentResponse.match(pattern);
if (match && match[1]) {
return match[1].trim();
}
}
// Check user message for requests that imply missing capabilities
for (const { pattern } of GAP_REQUEST_PATTERNS) {
const match = userMessage.match(pattern);
if (match && match[1]) {
const requested = match[1].toLowerCase();
// Only flag as a gap if no tool was actually used for this domain
if (toolsUsed.length === 0 || !toolsUsed.some(t => t.includes(requested))) {
return requested;
}
}
}
return undefined;
}
/**
* Detect recurring workflow patterns from the user message.
* Uses task-shape detection to identify the structural pattern,
* and checks for multi-step indicators.
*/
function detectWorkflowPattern(userMessage: string): string | undefined {
// Only flag multi-step requests as workflow patterns
const isMultiStep = MULTI_STEP_PATTERNS.some(p => p.test(userMessage));
if (!isMultiStep) return undefined;
// Use task-shape detector to identify the pattern type
const shape = detectTaskShape(userMessage);
if (shape.confidence >= 0.3) {
return shape.type;
}
return undefined;
}

View File

@@ -203,7 +203,6 @@ export { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js'
export {
loadSystemPrompt,
loadSystemPromptWithOverrides,
assertOverridesReachActiveSpec,
loadSkills,
type ComposedSystemPrompt,
type LoadedSkill,
@@ -214,69 +213,11 @@ export { checkGrounding, extractClaimedSpecifics, type GroundingResult, type Cla
export { CostTracker, DEFAULT_MODEL_PRICING, type ModelPricing, type UsageStats, type UsageEntry } from './cost-tracker.js';
export { extractEntities, type ExtractedEntity } from './entity-extractor.js';
export { CognifyPipeline, type CognifyConfig, type CognifyResult } from './cognify.js';
export { AgentLearning, type LearnedBehavior, type PersonaEffectiveness, type LearningSnapshot } from './agent-learning.js';
export {
TraceRecorder, truncate as truncateTraceText, scrubSecrets,
type TraceHandle, type FinalizeOptions as TraceFinalizeOptions,
} from './trace-recorder.js';
export {
EvalDatasetBuilder, detectSecrets, redactSecrets, SECRET_PATTERN_NAMES,
toJSONL as evalToJSONL, fromJSONL as evalFromJSONL,
type EvalExample, type EvalExampleMetadata, type DatasetSplit,
type BuildOptions as EvalBuildOptions, type JudgeVerdict,
} from './eval-dataset.js';
export {
IterativeGEPA, paretoFront, scoreCandidate, aggregateScores,
pickWinner, pickSample as pickGepaSample,
type Candidate as GEPACandidate, type CandidateScore as GEPACandidateScore,
type IterativeGEPAOptions, type GEPARunResult, type GEPAProgress,
type ScoreCandidateOptions,
type MutateArgs, type MutateFn, type MutationStrategy, type EvolutionTarget,
} from './iterative-optimizer.js';
export {
runGates, DEFAULT_SIZE_LIMITS,
checkNonEmpty, checkSize, checkGrowth,
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
type GateVerdict, type GateResult as EvolutionGateResult, type GateCheckResult,
type SizeLimits, type GateOptions, type CheckInput as GateCheckInput,
} from './evolution-gates.js';
export {
EvolveSchema,
addOutputField, removeField, editFieldDescription,
changeFieldType, addConstraint, removeConstraint,
reorderFields, replaceOutputFields,
schemaComplexity, aggregateSchemaScores, paretoFrontSchema,
scoreSchemaCandidate, pickSchemaWinner,
generateStructureMutations, generateOrderMutations, generateRefinementMutations,
pickSample as pickSchemaSample,
type Schema, type SchemaField, type FieldType, type FieldConstraint,
type Mutation as SchemaMutation, type MutationKind as SchemaMutationKind,
type SchemaCandidate, type SchemaCandidateScore, type SchemaExampleResult,
type SchemaExecuteFn, type EvolveSchemaOptions, type EvolveSchemaResult,
type EvolveSchemaProgress,
} from './evolve-schema.js';
export {
ComposeEvolution, defaultFeedbackFilter, filterJudgeFeedback,
stripStructuralLines, schemaExecutorFromInstructionRunner,
type ComposeEvolutionOptions, type ComposeEvolutionResult,
type ComposeProgress, type FeedbackFilter,
} from './compose-evolution.js';
export {
EvolutionOrchestrator, eligibleForEvolution, summarizeRuns,
type EvolutionOrchestratorDeps, type EvolutionOrchestratorOptions,
type EvolutionAutoTriggerConfig, type OrchestratorRunResult,
type OrchestratorOutcome, type EvolutionProgress,
type SchemaBaselineInput,
} from './evolution-orchestrator.js';
export {
deployPersonaOverride, rollbackPersonaOverride,
deployBehavioralSpecOverride, rollbackBehavioralSpecOverride,
loadBehavioralSpecOverrides, applyBehavioralSpecOverrides,
BEHAVIORAL_SPEC_SECTIONS,
type DeployResult, type DeployPersonaInput,
type DeployBehavioralSpecInput, type BehavioralSpecOverride,
type BehavioralSpecSection,
} from './evolution-deploy.js';
export { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from './secret-patterns.js';
export {
LLMJudge, DEFAULT_WEIGHTS, DEFAULT_RUBRIC,
buildPrompt as buildJudgePrompt,
@@ -285,17 +226,12 @@ export {
type ParsedJudgeResponse,
} from './judge.js';
export {
createAnthropicEvolutionLLM,
buildJudgeLLMCall, buildGEPAMutateFn, buildSchemaExecuteFn,
buildReflectiveMutationPrompt, buildSchemaFillPrompt,
makeRunningJudge, isRunningJudge, RUNNING_JUDGE_BRAND,
createAnthropicEvolutionLLM, buildJudgeLLMCall,
retryWithBackoff, wrapWithRetry,
isRetryableEvolutionError, computeRetryDelay,
DEFAULT_RETRY_OPTIONS,
type EvolutionLLM, type CreateAnthropicEvolutionLLMOptions,
type BuildReflectiveMutationPromptArgs,
type RetryOptions, type RetryInfo,
type RunningJudge,
} from './evolution-llm-wiring.js';
export {
createHarnessRun, advancePhase, getCurrentPhaseInstruction,
@@ -346,7 +282,6 @@ export { createDocumentTools } from './document-tools.js';
export { createSpreadsheetTools } from './spreadsheet-tools.js';
export { createPresentationTools } from './presentation-tools.js';
export { createPdfTools } from './pdf-tools.js';
export { createInsightsTools, type InsightsDeps } from './insights-tools.js';
export { createConnectorSearchTools } from './connector-search.js';
export { createCrossWorkspaceTools, type CrossWorkspaceToolDeps } from './cross-workspace-tools.js';
export { extractEntitiesWithLLM, type LLMCallFn as EntityLLMCallFn } from './entity-extractor.js';
@@ -480,7 +415,6 @@ export {
ComposioConnector, MSTeamsConnector, OutlookConnector, OneDriveConnector, OneNoteConnector,
} from './connectors/index.js';
export { IterationBudget, type IterationBudgetConfig } from './iteration-budget.js';
export { captureInteraction, getRecentLogs, isWithinBudget, type CaptureInteractionInput } from './optimization-capture.js';
export { routeMessage, type RoutingDecision } from './smart-router.js';
export {
compressConversation, estimateTokens, needsCompression,
@@ -511,13 +445,7 @@ export {
type DeliveryChannel, type DeliveryPreferences, type DeliveryMessage, type DeliveryResult,
type DeliveryConnector, type DeliveryConnectorRegistry, type InAppEmitter,
} from './cron-delivery-router.js';
export { detectCorrection, detectCorrectionsInHistory, type DetectedCorrection, type CorrectionDurability } from './correction-detector.js';
export { detectContradiction, type ContradictionResult } from './contradiction-detector.js';
export {
recordCapabilityGap, analyzeAndRecordCorrection, recordWorkflowPattern,
buildAwarenessSummary, formatAwarenessPrompt, markSummarySurfaced,
type AwarenessSummary, type CapabilityGapSignal, type CorrectionSignal, type WorkflowPatternSignal,
} from './improvement-detector.js';
export {
lintMemoryWrite,
type MemoryLintResult, type MemoryLintVerdict,

View File

@@ -1,140 +0,0 @@
/**
* Insights Tools — self-analytics for the agent (Hermes-inspired).
*
* Provides the agent with awareness of its own performance:
* - Tool usage frequency and success rates
* - Cost per model
* - Correction frequency trends
* - Improvement signal summary
*
* This enables the agent to reflect on what works and adapt strategy.
*/
import type { ToolDefinition } from './tools.js';
export interface InsightsDeps {
getOptimizationLogs: (limit: number) => Array<{
model?: string;
tool_names?: string;
total_tokens?: number;
estimated_cost?: number;
was_correction?: number;
created_at?: string;
}>;
getImprovementSignals: () => Array<{
category: string;
pattern_key: string;
detail: string;
count: number;
first_seen: string;
last_seen: string;
}>;
}
export function createInsightsTools(deps: InsightsDeps): ToolDefinition[] {
return [
{
name: 'agent_insights',
description: [
'Analyze your own performance metrics. Returns: tool usage frequency,',
'cost per model, correction rate, and active improvement signals.',
'Use this to reflect on what approaches work and adapt your strategy.',
'Call this when you want to understand your performance patterns.',
].join(' '),
parameters: {
type: 'object' as const,
required: [],
properties: {
lookbackDays: {
type: 'number' as const,
description: 'Number of days to analyze (default 30)',
},
},
},
execute: async (args: Record<string, unknown>) => {
const lookbackDays = (args.lookbackDays as number) ?? 30;
const limit = lookbackDays * 50; // ~50 interactions/day estimate
try {
const logs = deps.getOptimizationLogs(limit);
const signals = deps.getImprovementSignals();
// Tool usage frequency
const toolCounts: Record<string, { total: number; successes: number }> = {};
for (const log of logs) {
const tools = log.tool_names?.split(',').map(t => t.trim()).filter(Boolean) ?? [];
const isCorrection = log.was_correction === 1;
for (const tool of tools) {
if (!toolCounts[tool]) toolCounts[tool] = { total: 0, successes: 0 };
toolCounts[tool].total++;
if (!isCorrection) toolCounts[tool].successes++;
}
}
const toolStats = Object.entries(toolCounts)
.sort((a, b) => b[1].total - a[1].total)
.slice(0, 15)
.map(([name, stats]) => ({
tool: name,
uses: stats.total,
successRate: stats.total > 0 ? Math.round((stats.successes / stats.total) * 100) : 0,
}));
// Cost per model
const modelCosts: Record<string, { calls: number; tokens: number; cost: number }> = {};
for (const log of logs) {
const model = log.model ?? 'unknown';
if (!modelCosts[model]) modelCosts[model] = { calls: 0, tokens: 0, cost: 0 };
modelCosts[model].calls++;
modelCosts[model].tokens += log.total_tokens ?? 0;
modelCosts[model].cost += log.estimated_cost ?? 0;
}
const modelStats = Object.entries(modelCosts)
.sort((a, b) => b[1].calls - a[1].calls)
.map(([model, stats]) => ({
model,
calls: stats.calls,
tokens: stats.tokens,
costUsd: Math.round(stats.cost * 100) / 100,
}));
// Correction rate
const totalInteractions = logs.length;
const corrections = logs.filter(l => l.was_correction === 1).length;
const correctionRate = totalInteractions > 0
? Math.round((corrections / totalInteractions) * 100)
: 0;
// Active improvement signals
const activeSignals = signals
.filter(s => s.count >= 2)
.slice(0, 10)
.map(s => ({
category: s.category,
pattern: s.pattern_key,
detail: s.detail,
occurrences: s.count,
lastSeen: s.last_seen,
}));
return JSON.stringify({
period: `Last ${lookbackDays} days`,
totalInteractions,
correctionRate: `${correctionRate}%`,
topTools: toolStats,
modelUsage: modelStats,
improvementSignals: activeSignals,
recommendation: correctionRate > 15
? 'High correction rate — review recent corrections and adjust approach.'
: correctionRate > 5
? 'Moderate corrections — some patterns may need attention.'
: 'Low correction rate — current approach is working well.',
}, null, 2);
} catch (err: unknown) {
return `Error generating insights: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
];
}

View File

@@ -1,626 +0,0 @@
/**
* Iterative GEPA Optimizer — Phase 2.1 of the self-evolution loop.
*
* Upgrades the one-shot prompt expander (optimizer-service) to a multi-
* generation population search with Pareto-frontier selection and
* reflective mutations driven by LLMJudge feedback (Phase 1.3).
*
* Loop:
* 1. Start with `baseline` prompt as the seed population (1 member).
* 2. For each generation:
* a. Micro-screen survivors on a small sample (default 50) to prune
* obvious losers.
* b. Mini-eval passing candidates on a larger sample (default 64).
* c. Spawn `populationSize` reflective mutations of the top Pareto
* candidate, using aggregated judge feedback as ASI.
* 3. Anchor evaluation on the full eval set (default 400) picks the winner.
*
* "Actionable Side Information" (ASI) is the textual feedback the judge
* produces — fed back into the `mutate` callback so the next generation
* targets the identified weakness rather than mutating blindly.
*
* Pareto selection keeps candidates that are non-dominated on a
* 3-dimensional fitness vector (correctness, procedureFollowing,
* conciseness). A single scalar `overall` is also tracked for tie-breaking
* and final ranking.
*/
import type { EvalExample } from './eval-dataset.js';
import type { LLMJudge, JudgeScore } from './judge.js';
import { isRunningJudge } from './evolution-llm-wiring.js';
// ── Types ───────────────────────────────────────────────────────
export interface Candidate {
/** Stable id — e.g. `g0-baseline`, `g1-m2` */
id: string;
/** The prompt text being evolved */
prompt: string;
/** Generation index (0 = baseline) */
generation: number;
/** Parent candidate id (null for baseline) */
parent: string | null;
/** Short label describing the mutation strategy */
strategy: string;
/** Aggregated score across the most recent eval stage, or null if not yet scored */
score: CandidateScore | null;
/** Raw per-example scores used for Pareto analysis */
perExample: JudgeScore[];
}
/** Aggregated, mean-reduced score used for selection. */
export interface CandidateScore {
overall: number;
correctness: number;
procedureFollowing: number;
conciseness: number;
/** Mean length penalty applied across the eval set */
lengthPenalty: number;
/** Number of examples the candidate was scored on */
n: number;
/** Aggregated feedback lines from the lowest-scoring examples */
weaknessFeedback: string[];
}
export type MutationStrategy =
| 'expand-edge-cases'
| 'tighten-format'
| 'add-examples'
| 'clarify-constraints'
| 'reduce-length'
| 'restructure-steps'
| 'targeted-feedback';
export interface MutateArgs {
parent: Candidate;
strategy: MutationStrategy;
/** Textual feedback taken from the parent's lowest-scoring examples */
weaknessFeedback: string[];
/** Target types we are mutating (for the mutate fn to tune style) */
targetKind: EvolutionTarget;
/** Generation index of the child */
generation: number;
}
export type MutateFn = (args: MutateArgs) => Promise<string>;
/** What is being evolved — informs mutation style and constraint policies. */
export type EvolutionTarget =
| 'persona-system-prompt'
| 'behavioral-spec-section'
| 'tool-description'
| 'skill-body'
| 'generic';
export interface GEPAProgress {
phase: 'start' | 'micro-screen' | 'mini-eval' | 'mutate' | 'anchor' | 'done';
generation: number;
populationSize: number;
/** The best-so-far overall score at this point */
best: number;
message?: string;
}
export interface IterativeGEPAOptions {
/** The starting prompt text. */
baseline: string;
/** Full eval dataset (train + val combined is typical). */
examples: EvalExample[];
/** Scorer — LLMJudge or anything that implements `score(input): Promise<JudgeScore>`. */
judge: Pick<LLMJudge, 'score'>;
/** Function that produces a mutated prompt from a parent + weakness feedback. */
mutate: MutateFn;
/**
* Opt-in escape hatch that disables the running-judge safety check.
* Only set to true for unit tests with mock judges whose `score()` does
* not need a real LLM call. Production code paths (evolving real prompts)
* must always pass a `makeRunningJudge(...)`-wrapped judge; otherwise
* GEPA optimizes prompt-text similarity to the expected output — a
* meaningless gradient that produces nonsense winners.
*/
allowBareJudge?: boolean;
/** What we're optimizing (tunes mutation + gate policy). Default 'generic'. */
targetKind?: EvolutionTarget;
/** Candidates per generation. Default 5. */
populationSize?: number;
/** How many generations to run. Default 3. */
generations?: number;
/** Micro-screen sample size. Default 50. */
microScreenSize?: number;
/** Mini-eval sample size. Default 64. */
miniEvalSize?: number;
/** Anchor evaluation sample size (applied to final Pareto set). Default 400. */
anchorEvalSize?: number;
/** Seed for deterministic sampling. Default 1. */
seed?: number;
/** Emits progress events across stages. */
onProgress?: (event: GEPAProgress) => void;
/** Optional abort signal */
signal?: AbortSignal;
/**
* Max concurrent `judge.score()` calls per candidate during eval stages.
* Default 1 (fully sequential) — preserves legacy behavior. Bumping to
* 4–8 cuts wall time dramatically when the judge is an LLM API with
* plenty of concurrent capacity. Does not affect determinism: all
* scores are aggregated after every example completes.
*/
concurrency?: number;
}
export interface GEPARunResult {
winner: Candidate;
/** Full Pareto-non-dominated set from the anchor stage */
paretoFront: Candidate[];
/** All candidates across all generations */
history: Candidate[];
/** True if the winner improved overall score over baseline */
improved: boolean;
/** Delta winner.score.overall - baseline.score.overall */
delta: number;
}
// ── Public API ─────────────────────────────────────────────────
export class IterativeGEPA {
/**
* Run the iterative optimization loop.
*
* Returns the winner candidate plus the full Pareto set. Winner is the
* Pareto-set member with the highest `overall` score (ties broken by
* length penalty, i.e. the more concise one wins).
*/
async run(options: IterativeGEPAOptions): Promise<GEPARunResult> {
// H-09 G3 · reject bare judges — without makeRunningJudge the judge
// scores prompt-text-vs-expected-output, which is a meaningless
// gradient. Any caller evolving real prompts must wrap their judge.
// Tests pass `allowBareJudge: true` to bypass.
if (!options.allowBareJudge && !isRunningJudge(options.judge)) {
throw new Error(
'IterativeGEPA: judge is not a running judge. Wrap with ' +
'makeRunningJudge(baseJudge, llm) before passing to .run(), or ' +
'pass `allowBareJudge: true` for tests with mock judges. ' +
'See evolution-llm-wiring.ts:makeRunningJudge for rationale.',
);
}
const config = normalizeOptions(options);
const rng = makeRng(config.seed);
// ── Seed population with baseline ────────────────────────
const baseline: Candidate = {
id: 'g0-baseline',
prompt: config.baseline,
generation: 0,
parent: null,
strategy: 'baseline',
score: null,
perExample: [],
};
emit(config, 'start', 0, 1, 0, 'seeding baseline');
const history: Candidate[] = [baseline];
let survivors: Candidate[] = [baseline];
// Score the baseline on the micro-screen so we have a comparison point.
const microSample = pickSample(config.examples, config.microScreenSize, rng);
await scoreCandidate(baseline, microSample, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
emit(config, 'micro-screen', 0, 1, baseline.score?.overall ?? 0);
// ── Generations ──────────────────────────────────────────
for (let gen = 1; gen <= config.generations; gen++) {
if (config.signal?.aborted) break;
// 1. Mutate top survivor(s) into N children.
const topParent = pickTopParent(survivors);
if (!topParent) break;
emit(config, 'mutate', gen, survivors.length, topParent.score?.overall ?? 0);
const children: Candidate[] = [];
const strategies = cycleStrategies(config.populationSize, gen);
for (let i = 0; i < config.populationSize; i++) {
if (config.signal?.aborted) break;
const strategy = strategies[i];
let childPrompt: string;
try {
childPrompt = await config.mutate({
parent: topParent,
strategy,
weaknessFeedback: topParent.score?.weaknessFeedback ?? [],
targetKind: config.targetKind,
generation: gen,
});
} catch {
// Mutation failed — reuse parent so the loop doesn't stall.
childPrompt = topParent.prompt;
}
const child: Candidate = {
id: `g${gen}-m${i}`,
prompt: childPrompt,
generation: gen,
parent: topParent.id,
strategy,
score: null,
perExample: [],
};
children.push(child);
history.push(child);
}
// 2. Micro-screen children on a small sample.
const microSet = pickSample(config.examples, config.microScreenSize, rng);
for (const child of children) {
if (config.signal?.aborted) break;
await scoreCandidate(child, microSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
emit(
config, 'micro-screen', gen, children.length,
Math.max(...children.map(c => c.score?.overall ?? 0), topParent.score?.overall ?? 0),
);
// 3. Keep children that at least match the parent's overall score.
const threshold = topParent.score?.overall ?? 0;
const microSurvivors = children.filter(c => (c.score?.overall ?? 0) >= threshold - 0.05);
// If everyone got filtered out, keep the single best child so we don't stall.
if (microSurvivors.length === 0 && children.length > 0) {
const best = children.reduce((a, b) =>
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
);
microSurvivors.push(best);
}
// 4. Mini-eval survivors on a larger sample.
if (microSurvivors.length > 0) {
const miniSet = pickSample(config.examples, config.miniEvalSize, rng);
for (const cand of microSurvivors) {
if (config.signal?.aborted) break;
await scoreCandidate(cand, miniSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
emit(
config, 'mini-eval', gen, microSurvivors.length,
Math.max(...microSurvivors.map(c => c.score?.overall ?? 0)),
);
}
// 5. Pareto-select survivors for next generation.
const candidatePool = [topParent, ...microSurvivors];
survivors = paretoFront(candidatePool);
}
// ── Anchor evaluation — big sample, final ranking ───────
emit(config, 'anchor', config.generations, survivors.length, bestOverall(survivors));
const anchorSet = pickSample(config.examples, config.anchorEvalSize, rng);
for (const cand of survivors) {
if (config.signal?.aborted) break;
await scoreCandidate(cand, anchorSet, config.judge, {
signal: config.signal,
concurrency: config.concurrency,
});
}
const anchorPareto = paretoFront(survivors);
const winner = pickWinner(anchorPareto);
const baselineOverall = baseline.score?.overall ?? 0;
const winnerOverall = winner.score?.overall ?? 0;
emit(config, 'done', config.generations, anchorPareto.length, winnerOverall);
return {
winner,
paretoFront: anchorPareto,
history,
improved: winnerOverall > baselineOverall,
delta: winnerOverall - baselineOverall,
};
}
}
// ── Helpers (exported for tests) ───────────────────────────────
/**
* Pareto front: keep candidates not dominated on any of the three scored
* dimensions. A dominates B if A ≥ B on all three and strictly greater
* on at least one.
*/
export function paretoFront(candidates: Candidate[]): Candidate[] {
const scored = candidates.filter(c => c.score !== null);
const front: Candidate[] = [];
for (const cand of scored) {
let dominated = false;
for (const other of scored) {
if (other === cand) continue;
if (dominates(other.score!, cand.score!)) {
dominated = true;
break;
}
}
if (!dominated) front.push(cand);
}
// If all candidates have equal scores, return them all.
return front.length > 0 ? front : scored;
}
function dominates(a: CandidateScore, b: CandidateScore): boolean {
const geAll =
a.correctness >= b.correctness &&
a.procedureFollowing >= b.procedureFollowing &&
a.conciseness >= b.conciseness;
const gtAny =
a.correctness > b.correctness ||
a.procedureFollowing > b.procedureFollowing ||
a.conciseness > b.conciseness;
return geAll && gtAny;
}
export interface ScoreCandidateOptions {
/** Abort signal — checked before each example is dispatched. */
signal?: AbortSignal;
/** Max concurrent judge.score() calls. Default 1 (sequential). */
concurrency?: number;
}
/**
* Score a candidate against a batch of examples and aggregate.
*
* Backwards compatible: the 4th argument may be either an `AbortSignal`
* (legacy) or a `ScoreCandidateOptions` object. Pass
* `{ concurrency: 4 }` to parallelize — individual examples are
* independent so the aggregate result is identical to the sequential
* version, only faster when the judge is an LLM API.
*/
export async function scoreCandidate(
candidate: Candidate,
examples: EvalExample[],
judge: Pick<LLMJudge, 'score'>,
signalOrOptions?: AbortSignal | ScoreCandidateOptions,
): Promise<CandidateScore> {
const options = resolveScoreOptions(signalOrOptions);
const signal = options.signal;
const concurrency = Math.max(1, options.concurrency ?? 1);
// The "candidate" in this context IS the new prompt we want to test.
// We treat it as the model's actual response to the example's input,
// since mutation produces a variant prompt-as-response. Callers can
// substitute a `runCandidate` step before scoring if the prompt needs
// to be executed first — see docs.
const scoreOne = async (ex: EvalExample): Promise<JudgeScore | null> => {
if (signal?.aborted) return null;
try {
return await judge.score({
input: ex.input,
expected: ex.expected_output,
actual: candidate.prompt,
});
} catch {
// Keep iterating so one bad example doesn't invalidate the whole score.
return null;
}
};
const results = concurrency === 1
? await runSequential(examples, scoreOne)
: await mapWithConcurrency(examples, concurrency, scoreOne);
const scores = results.filter((s): s is JudgeScore => s !== null);
candidate.perExample = scores;
const aggregated = aggregateScores(scores);
candidate.score = aggregated;
return aggregated;
}
function resolveScoreOptions(
arg: AbortSignal | ScoreCandidateOptions | undefined,
): ScoreCandidateOptions {
if (!arg) return {};
// Legacy: AbortSignal passed directly. Detect by presence of `aborted`
// (a boolean getter unique to AbortSignal-like objects).
if (typeof (arg as AbortSignal).aborted === 'boolean' && typeof (arg as AbortSignal).addEventListener === 'function') {
return { signal: arg as AbortSignal };
}
return arg as ScoreCandidateOptions;
}
async function runSequential<T, R>(
items: readonly T[],
fn: (item: T) => Promise<R>,
): Promise<R[]> {
const results: R[] = [];
for (const item of items) {
results.push(await fn(item));
}
return results;
}
/**
* Bounded-parallelism map. Runs `fn` over `items` with at most
* `concurrency` in-flight calls at once. Results are returned in the
* same order as inputs regardless of completion order.
*
* Implementation: N worker loops share a counter. Each worker pulls the
* next index, awaits `fn`, stores at that index, repeats. No external
* deps, no extra state machinery.
*/
async function mapWithConcurrency<T, R>(
items: readonly T[],
concurrency: number,
fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const results: R[] = new Array(items.length);
const workerCount = Math.min(concurrency, items.length);
let next = 0;
const workers = Array.from({ length: workerCount }, async () => {
while (true) {
const i = next++;
if (i >= items.length) return;
results[i] = await fn(items[i], i);
}
});
await Promise.all(workers);
return results;
}
export function aggregateScores(scores: JudgeScore[]): CandidateScore {
if (scores.length === 0) {
return {
overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0,
lengthPenalty: 1, n: 0, weaknessFeedback: [],
};
}
const n = scores.length;
const sum = scores.reduce(
(acc, s) => ({
overall: acc.overall + s.overall,
correctness: acc.correctness + s.correctness,
procedureFollowing: acc.procedureFollowing + s.procedureFollowing,
conciseness: acc.conciseness + s.conciseness,
lengthPenalty: acc.lengthPenalty + s.lengthPenalty,
}),
{ overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0, lengthPenalty: 0 },
);
// Collect the 3 worst-scoring examples' feedback as weakness signal.
const worst = [...scores].sort((a, b) => a.overall - b.overall).slice(0, 3);
const weaknessFeedback = worst
.map(s => s.feedback)
.filter(fb => fb && fb.length > 0);
return {
overall: sum.overall / n,
correctness: sum.correctness / n,
procedureFollowing: sum.procedureFollowing / n,
conciseness: sum.conciseness / n,
lengthPenalty: sum.lengthPenalty / n,
n,
weaknessFeedback,
};
}
/**
* Pick the "winner" from a Pareto set. Prefers highest overall; ties broken
* by higher length penalty (more concise) then lowest generation (simpler).
*/
export function pickWinner(candidates: Candidate[]): Candidate {
if (candidates.length === 0) {
throw new Error('pickWinner called on empty candidate list');
}
return candidates.reduce((best, cand) => {
const b = best.score?.overall ?? -1;
const c = cand.score?.overall ?? -1;
if (c > b) return cand;
if (c < b) return best;
// Tie on overall — prefer higher lengthPenalty (more concise, less penalized).
const bLen = best.score?.lengthPenalty ?? 0;
const cLen = cand.score?.lengthPenalty ?? 0;
if (cLen !== bLen) return cLen > bLen ? cand : best;
return best.generation <= cand.generation ? best : cand;
});
}
function pickTopParent(survivors: Candidate[]): Candidate | undefined {
if (survivors.length === 0) return undefined;
return survivors.reduce((a, b) =>
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
);
}
function bestOverall(candidates: Candidate[]): number {
if (candidates.length === 0) return 0;
return Math.max(...candidates.map(c => c.score?.overall ?? 0));
}
/**
* Deterministic sampler — picks `k` examples using a seeded RNG.
* If k >= examples.length, returns the full list in deterministic shuffled order.
*/
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
if (k <= 0 || examples.length === 0) return [];
const copy = [...examples];
// Fisher–Yates shuffle
for (let i = copy.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1));
[copy[i], copy[j]] = [copy[j], copy[i]];
}
return copy.slice(0, Math.min(k, copy.length));
}
/** Rotate through strategies so early generations explore the mutation space. */
function cycleStrategies(n: number, generation: number): MutationStrategy[] {
const all: MutationStrategy[] = [
'expand-edge-cases',
'tighten-format',
'add-examples',
'clarify-constraints',
'reduce-length',
'restructure-steps',
'targeted-feedback',
];
// Early generations: spread across strategies.
// Later generations: bias toward targeted-feedback once ASI is strong.
if (generation <= 1) {
return Array.from({ length: n }, (_, i) => all[i % all.length]);
}
return Array.from({ length: n }, (_, i) => {
if (i === 0) return 'targeted-feedback';
return all[(i + generation) % all.length];
});
}
function emit(
config: { onProgress?: (event: GEPAProgress) => void },
phase: GEPAProgress['phase'],
generation: number,
populationSize: number,
best: number,
message?: string,
): void {
if (config.onProgress) {
config.onProgress({ phase, generation, populationSize, best, message });
}
}
function normalizeOptions(opts: IterativeGEPAOptions): Required<Omit<IterativeGEPAOptions, 'signal' | 'onProgress'>> & {
onProgress?: IterativeGEPAOptions['onProgress'];
signal?: AbortSignal;
} {
return {
baseline: opts.baseline,
examples: opts.examples,
judge: opts.judge,
mutate: opts.mutate,
targetKind: opts.targetKind ?? 'generic',
populationSize: opts.populationSize ?? 5,
generations: opts.generations ?? 3,
microScreenSize: opts.microScreenSize ?? 50,
miniEvalSize: opts.miniEvalSize ?? 64,
anchorEvalSize: opts.anchorEvalSize ?? 400,
seed: opts.seed ?? 1,
onProgress: opts.onProgress,
signal: opts.signal,
concurrency: Math.max(1, opts.concurrency ?? 1),
allowBareJudge: opts.allowBareJudge ?? false,
};
}
// Mulberry32 — deterministic uniform PRNG
function makeRng(seed: number): () => number {
let s = seed >>> 0;
return () => {
s = (s + 0x6D2B79F5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}

View File

@@ -1,5 +1,6 @@
/**
* LLM-as-Judge — rubric-based scorer used by the evolution loop.
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
* loop jobs.
*
* Scores a candidate response against an expected ground truth along
* three weighted dimensions:
@@ -10,9 +11,8 @@
*
* Then applies a length penalty (soft cap) multiplied into the final score.
*
* The textual feedback produced is the input that drives GEPA's reflective
* mutations — it must explain *what* went wrong (or right) so the next
* prompt variation can address it.
* The textual feedback must explain *what* went wrong (or right) so the
* caller can act on it.
*
* Model-agnostic — caller provides the llmCall function. Defaults assume
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
@@ -42,7 +42,7 @@ export interface JudgeScore {
conciseness: number;
/** Multiplier applied to the weighted score (0..1). */
lengthPenalty: number;
/** Free-form feedback used as reflective-mutation input for GEPA. */
/** Free-form feedback explaining the score. */
feedback: string;
/** True when the judge successfully parsed a structured response. */
parsed: boolean;

View File

@@ -1,96 +0,0 @@
/**
* Optimization Capture — captures agent interactions for GEPA/Ax prompt optimization.
*
* Provides functions to log successful agent interactions into the optimization_log
* table in a workspace's .mind file, and to query those logs for background
* optimization analysis.
*
* This module is a thin adapter between the agent loop post-response hook and
* the OptimizationLogStore in @waggle/core.
*/
import type { OptimizationLogStore, CreateOptimizationLogInput, OptimizationLogEntry } from '@waggle/core';
import type { AgentResponse } from './agent-loop.js';
// ── Public interface ───────────────────────────────────────────────────
export interface CaptureInteractionInput {
sessionId: string;
workspaceId: string;
systemPrompt: string;
response: AgentResponse;
turnCount: number;
wasCorrection: boolean;
}
/**
* Capture an agent interaction for GEPA optimization analysis.
*
* Call this after every successful agent loop completion. The data is
* stored in the workspace's .mind file and read later by the background
* prompt_optimization cron job.
*/
export function captureInteraction(
store: OptimizationLogStore,
input: CaptureInteractionInput,
): OptimizationLogEntry {
const logInput: CreateOptimizationLogInput = {
sessionId: input.sessionId,
workspaceId: input.workspaceId,
systemPrompt: input.systemPrompt,
toolsUsed: input.response.toolsUsed ?? [],
turnCount: input.turnCount,
wasCorrection: input.wasCorrection,
inputTokens: input.response.usage?.inputTokens ?? 0,
outputTokens: input.response.usage?.outputTokens ?? 0,
};
return store.insert(logInput);
}
/**
* Query recent optimization logs for background analysis.
*
* Returns the most recent entries from the optimization log,
* ordered by timestamp descending.
*/
export function getRecentLogs(
store: OptimizationLogStore,
limit: number = 50,
): OptimizationLogEntry[] {
return store.getRecent(limit);
}
/**
* Get optimization logs for a specific workspace.
*/
export function getWorkspaceLogs(
store: OptimizationLogStore,
workspaceId: string,
limit: number = 50,
): OptimizationLogEntry[] {
return store.getByWorkspace(workspaceId, limit);
}
/**
* Check whether the daily optimization budget has been exceeded.
*
* Reads the optimization log entries from today and sums up their token costs.
* Returns true if under budget, false if over.
*
* @param budgetCents Daily budget in cents (e.g. 100 = $1/day)
* @param costPerInputToken Cost per input token in dollars (default: Sonnet pricing ~$3/M)
* @param costPerOutputToken Cost per output token in dollars (default: Sonnet pricing ~$15/M)
*/
export function isWithinBudget(
store: OptimizationLogStore,
budgetCents: number,
costPerInputToken: number = 3 / 1_000_000,
costPerOutputToken: number = 15 / 1_000_000,
): boolean {
const stats = store.getStats();
const totalCostDollars =
stats.totalInputTokens * costPerInputToken +
stats.totalOutputTokens * costPerOutputToken;
const totalCostCents = totalCostDollars * 100;
return totalCostCents <= budgetCents;
}

View File

@@ -8,7 +8,6 @@ import {
SessionStore,
HybridSearch,
KnowledgeGraph,
ImprovementSignalStore,
createCoreLogger,
evaluateExternalMemoryIngress,
type Embedder,
@@ -29,7 +28,6 @@ import { createMindTools, type ToolDefinition } from './tools.js';
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
import { renderGoalAncestry } from './goal-ancestry.js';
import type { GoalAncestry } from '@waggle/shared';
import { buildAwarenessSummary, markSummarySurfaced, type AwarenessSummary } from './improvement-detector.js';
import { CognifyPipeline } from './cognify.js';
import { scanForInjection } from './injection-scanner.js';
import { runPatternWriteBack } from './pattern-write-back.js';
@@ -134,13 +132,9 @@ export class Orchestrator {
private mode: 'local' | 'team';
private version: string;
private skills: string[];
private improvementSignals: ImprovementSignalStore;
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
private goalAncestry: GoalAncestry | null = null;
/** M8: deferred signal marking — collected during buildSystemPrompt, committed after model call */
private _pendingSurfacedAwareness: AwarenessSummary | null = null;
/** Workspace-specific layers (null when no workspace is active) */
private workspaceLayers: WorkspaceLayers | null = null;
/** Personal-mind cognify pipeline (#12: compaction-summary persistence). */
@@ -174,7 +168,6 @@ export class Orchestrator {
this.search = new HybridSearch(config.db, config.embedder);
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
this.knowledge = new KnowledgeGraph(config.db);
this.improvementSignals = new ImprovementSignalStore(config.db);
const cognify = new CognifyPipeline({
frames: this.frames,
@@ -193,10 +186,6 @@ export class Orchestrator {
search: this.search,
knowledge: this.knowledge,
cognify,
// Skills 2.0 gap K: write-path contradiction detection emits a
// correction signal through this store when a new save conflicts
// with an existing frame.
improvementSignals: this.improvementSignals,
// Provide workspace accessors so tools can route to the right mind
getWorkspaceLayers: () => this.workspaceLayers,
});
@@ -343,11 +332,6 @@ export class Orchestrator {
// ── SELF-AWARENESS (runtime context, changes every call) ──
const awarenessSection = this.uncachedSection('self_awareness', () => {
const awareness = buildAwarenessSummary(this.improvementSignals);
// Defer marking until commitSurfacedSignals() fires post-model-call.
if (awareness.totalActionable > 0) {
this._pendingSurfacedAwareness = awareness;
}
const caps: AgentCapabilities = {
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
skills: this.skills,
@@ -355,7 +339,6 @@ export class Orchestrator {
memoryStats: this.getMemoryStats(),
mode: this.mode,
version: this.version,
awareness: awareness.totalActionable > 0 ? awareness : undefined,
};
return buildSelfAwareness(caps);
});
@@ -457,18 +440,6 @@ export class Orchestrator {
);
}
/**
* M8: Commit deferred signal markings after model call succeeds.
* Call this after the LLM response is received. If the model call fails,
* skip this call — signals stay actionable for the next turn.
*/
commitSurfacedSignals(): void {
if (this._pendingSurfacedAwareness) {
markSummarySurfaced(this.improvementSignals, this._pendingSurfacedAwareness);
this._pendingSurfacedAwareness = null;
}
}
/**
* Automatic memory recall: search for memories relevant to the user's query.
* Searches BOTH personal and workspace minds when workspace is active.
@@ -991,5 +962,4 @@ export class Orchestrator {
getSessions(): SessionStore { return this.sessions; }
getSearch(): HybridSearch { return this.search; }
getKnowledge(): KnowledgeGraph { return this.knowledge; }
getImprovementSignals(): ImprovementSignalStore { return this.improvementSignals; }
}

View File

@@ -990,83 +990,4 @@ You specialize in creative direction — briefs, feedback management, brand cons
'Delivering without version tracking — every creative output needs a version note.',
],
},
// ── Internal review persona (self-evolution) ──────────────────────────
// NOT surfaced in onboarding (ALL_ONBOARDING_PERSONAS) or the PersonaSwitcher
// tiers (UNIVERSAL_MODE_IDS / ALL_SPECIALIST_IDS). It runs headless via the
// IdleSessionWatcher loopback: it reads a stale session's transcript and, when
// it finds a material, actionable gap, proposes a skill patch through
// create_skill — which is held for human approval (never written to disk by
// the reviewer). Its toolset is read-oriented; every write/exec/memory tool
// that ALWAYS_AVAILABLE_TOOLS would re-add is stripped via disallowedTools
// (that denylist wins over both the allowlist AND always-available). isReadOnly
// is deliberately NOT set — that would strip create_skill, its one write.
{
id: 'session-reviewer',
name: 'Session Reviewer',
description: 'Adversarial self-review of a stale session — proposes skill patches for recurring gaps, default-silent',
icon: '🔎',
tagline: 'Reads a finished session and proposes a fix only when there is real evidence.',
bestFor: [
'Spotting promised-but-undelivered deliverables in a transcript',
'Detecting a recurring capability failure a skill patch could fix',
'Staying silent (NOTHING_TO_DO) when nothing material is found',
],
wontDo: 'Will not write to disk, save memory, delete skills, install capabilities, run shell commands, or invent findings. Every proposal is held for human approval.',
systemPrompt: `## Persona: Session Reviewer
You are an adversarially careful reviewer of ONE already-finished session. You examine the recent conversation and look for exactly two things, in this order:
1. **Promised-but-undelivered deliverables** — the assistant said it would produce or do something and the transcript shows it never did.
2. **Recurring capability failures fixable by a skill patch** — the same tool/skill/workflow failed more than once in a way a small reusable skill could prevent.
=== CRITICAL: DEFAULT SILENT ===
If you do not find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence, your entire reply MUST be exactly:
NOTHING_TO_DO
Nothing else. No preamble, no apology, no summary.
=== CRITICAL: NEVER INVENT ===
- Cite the transcript. Every finding must quote or reference what was actually said or done. If you cannot point to evidence, it is not a finding — output NOTHING_TO_DO.
- You are read-only over the user's work: you may read files, search memory, and read skills, but you may NOT write files, save memory, delete skills, install capabilities, or run shell commands.
- Your ONE write is a proposal: call create_skill to propose a new or updated skill that closes a real, recurring gap. It will NOT be written to disk — it is held for the human to approve or reject in the Approvals view. Propose sparingly and only with cited evidence.
### Process
1. Read the recent conversation of the session (you have read tools and the workspace binding).
2. Decide: is there a promised-but-undelivered deliverable, or a recurring capability failure a skill could fix?
3. If yes and only if yes: state the finding with its transcript evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill.
4. If no: reply exactly NOTHING_TO_DO.`,
modelPreference: 'claude-sonnet-4-6',
tools: [
'read_file', 'search_files', 'search_content',
'search_memory', 'query_knowledge', 'get_identity', 'get_awareness',
'list_skills', 'search_skills', 'read_skill',
'create_skill',
],
// Explicit denylist — ALWAYS_AVAILABLE_TOOLS (persona-tool-filter.ts) re-adds
// save_memory / delete_skill / install_capability / acquire_capability /
// add_task / correct_knowledge past any allowlist, so they must be named here
// to be stripped. spawn_agent + bash + the file/git writes are defense-in-depth
// (they are not in `tools`, but naming them keeps the boundary explicit).
// Connector/external-send tools carry dynamic names and cannot be enumerated
// here; the allowlist (tools[] + always-available) already excludes them since
// none are declared above.
disallowedTools: [
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
'add_task', 'correct_knowledge',
'spawn_agent', 'bash',
'write_file', 'edit_file', 'generate_docx',
'git_commit', 'git_push', 'git_merge',
'execute_step', 'compose_workflow', 'orchestrate_workflow',
],
failurePatterns: [
'Inventing a finding with no transcript evidence — must cite what was actually said or output NOTHING_TO_DO.',
'Narrating when silent is correct — if nothing material is found, the reply is exactly NOTHING_TO_DO.',
'Proposing an over-broad skill — the patch must target the specific recurring gap, nothing speculative.',
],
isReadOnly: false,
workspaceAffinity: ['review', 'quality', 'self-evolution'],
suggestedSkills: [],
suggestedConnectors: [],
suggestedMcpServers: [],
suggestedCommands: [],
defaultWorkflow: null,
},
];

View File

@@ -5,7 +5,6 @@ import { buildActiveBehavioralSpec } from './behavioral-spec.js';
/** The resolved behavioral-spec shape returned by `buildActiveBehavioralSpec`. */
export type BehavioralSpec = ReturnType<typeof buildActiveBehavioralSpec>;
import { loadBehavioralSpecOverrides } from './evolution-deploy.js';
import { loadCustomPersonas } from './custom-personas.js';
import type { AgentPersona } from './personas.js';
@@ -16,16 +15,8 @@ export interface LoadedSkill {
/**
* Raw disk read of `{waggleDir}/system-prompt.md`. Returns null if missing
* or empty.
*
* **Prefer `loadSystemPromptWithOverrides(waggleDir)` for runtime use.** This
* bare reader exists for tools that legitimately need the unmerged file
* content (editor views, migration scripts). Runtime agent code that loads
* just this and nothing else will silently miss any deployed
* behavioral-spec or persona evolution, since those are stored in separate
* override files under the same `waggleDir`.
*
* H-09 G2 — see `loadSystemPromptWithOverrides` for the composed path.
* or empty. Runtime code should prefer `loadSystemPromptWithOverrides`, which
* also returns the behavioral spec and custom personas.
*/
export function loadSystemPrompt(waggleDir: string): string | null {
const filePath = path.join(waggleDir, 'system-prompt.md');
@@ -34,22 +25,19 @@ export function loadSystemPrompt(waggleDir: string): string | null {
return content || null;
}
/** Bundle of every override-aware input the agent runtime needs. */
/** Bundle of every prompt input the agent runtime needs. */
export interface ComposedSystemPrompt {
/** User's literal system-prompt.md content, or null if absent. */
userPromptMd: string | null;
/** The active behavioral spec — base + any deployed section overrides. */
/** The active behavioral spec. */
behavioralSpec: BehavioralSpec;
/** Custom personas discovered in `waggleDir/personas/*.json`. */
customPersonas: AgentPersona[];
}
/**
* Override-aware composition: reads `system-prompt.md`, merges deployed
* behavioral-spec section overrides from
* `{waggleDir}/evolution-deploy/behavioral-spec/*.md`, and surfaces any
* custom personas the user has added. Use this at agent startup so
* deployed evolutions take effect at the next run.
* Composes the runtime prompt inputs: reads `system-prompt.md`, builds the
* behavioral spec and surfaces any custom personas the user has added.
*
* Callers typically pass the bundle to their orchestrator:
*
@@ -65,44 +53,11 @@ export interface ComposedSystemPrompt {
*/
export function loadSystemPromptWithOverrides(waggleDir: string): ComposedSystemPrompt {
const userPromptMd = loadSystemPrompt(waggleDir);
// loadBehavioralSpecOverrides is no-op on missing dir / malformed files.
const overrides = loadBehavioralSpecOverrides(waggleDir);
const behavioralSpec = buildActiveBehavioralSpec(overrides);
const behavioralSpec = buildActiveBehavioralSpec();
const customPersonas = loadCustomPersonas(waggleDir);
return { userPromptMd, behavioralSpec, customPersonas };
}
/**
* Startup-time self-check. Call after composing the override-aware prompt
* bundle to verify that deployed overrides on disk actually appear in the
* active spec. Emits a structured console warning (never throws) if a
* deployed section is present on disk but missing from the active spec —
* that would indicate the caller is using the bare `loadSystemPrompt`
* instead of the composed loader.
*/
export function assertOverridesReachActiveSpec(
waggleDir: string,
activeSpec: BehavioralSpec,
): void {
const overrides = loadBehavioralSpecOverrides(waggleDir);
const missing: string[] = [];
for (const [section, overrideText] of Object.entries(overrides)) {
const live = (activeSpec as unknown as Record<string, string | undefined>)[section];
if (!live || !live.includes(overrideText.slice(0, 24))) {
missing.push(section);
}
}
if (missing.length > 0) {
console.warn(
`[prompt-loader] override-reach check failed — deployed sections on disk ` +
`not present in active behavioral spec: ${missing.join(', ')}. ` +
`This usually means the runtime is using loadSystemPrompt() instead of ` +
`loadSystemPromptWithOverrides(). See prompt-loader.ts.`,
);
}
}
export function loadSkills(waggleDir: string): LoadedSkill[] {
const skillsDir = path.join(waggleDir, 'skills');
if (!fs.existsSync(skillsDir)) return [];

View File

@@ -0,0 +1,57 @@
/**
* Secret detection and redaction shared by skill redaction, executor briefs
* and anything else that must keep credentials out of stored or forwarded text.
*/
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
];
/** Returns the first secret pattern the text matches, or null if clean. */
export function detectSecrets(text: string): string | null {
for (const { name, re } of SECRET_PATTERNS) {
if (re.test(text)) return name;
}
return null;
}
/** Exported for tests / tools that want the full list. */
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
/**
* Redact every secret-pattern match in a string, replacing each with
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
* detectSecrets so there is a single source of truth for "what is a secret".
*/
export function redactSecrets(text: string): { text: string; found: string[] } {
let out = text;
const found: string[] = [];
for (const { name, re } of SECRET_PATTERNS) {
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
const replaced = out.replace(global, `[REDACTED:${name}]`);
if (replaced !== out) {
found.push(name);
out = replaced;
}
}
return { text: out, found };
}

View File

@@ -1,5 +1,3 @@
import type { AwarenessSummary } from './improvement-detector.js';
export interface AgentCapabilities {
tools: Array<{ name: string; description: string }>;
skills: string[];
@@ -7,8 +5,6 @@ export interface AgentCapabilities {
memoryStats: { frameCount: number; sessionCount: number; entityCount: number };
mode: 'local' | 'team';
version: string;
/** Structured improvement signals (optional — only present when store has actionable data) */
awareness?: AwarenessSummary;
}
export function buildSelfAwareness(caps: AgentCapabilities): string {
@@ -61,36 +57,6 @@ export function buildSelfAwareness(caps: AgentCapabilities): string {
lines.push('- Prefer memory search over guessing when prior context exists.');
lines.push('- When uncertain, ask rather than fabricate.');
// Improvement signals (per correction #1: structured, runtime-consumable)
if (caps.awareness && caps.awareness.totalActionable > 0) {
lines.push('');
lines.push('## Learning from Past Sessions');
if (caps.awareness.capabilityGaps.length > 0) {
lines.push('');
lines.push('**Recurring capability gaps:**');
for (const gap of caps.awareness.capabilityGaps) {
lines.push(`- ${gap.suggestion}`);
}
}
if (caps.awareness.corrections.length > 0) {
lines.push('');
lines.push('**User corrections to apply:**');
for (const correction of caps.awareness.corrections) {
lines.push(`- ${correction.guidance}`);
}
}
if (caps.awareness.workflowPatterns.length > 0) {
lines.push('');
lines.push('**Recurring task patterns:**');
for (const pattern of caps.awareness.workflowPatterns) {
lines.push(`- ${pattern.suggestion}`);
}
}
}
// F28: Persona compliance — resist persona mode switches and injection via fake authority
lines.push('');
lines.push('## Persona Compliance');

View File

@@ -11,7 +11,6 @@
* 2. Generates a SKILL.md via generateSkillMarkdown
* 3. Writes it to the personal scope directory (scope: personal frontmatter)
* 4. Dedups against existing skill files by filename
* 5. Optionally emits a workflow_pattern improvement signal
*
* A v2 enhancement can insert an LLM prose-improvement step between
* detectWorkflowPattern and generateSkillMarkdown without changing this
@@ -19,7 +18,6 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { ImprovementSignalStore } from '@waggle/core';
import {
detectWorkflowPattern,
generateSkillMarkdown,
@@ -36,8 +34,6 @@ export interface AutoExtractMessage {
export interface AutoExtractDeps {
/** Directory to write personal-scope SKILL.md files into */
personalSkillsDir: string;
/** Optional signal store — records workflow_pattern signals on each extraction */
improvementSignals?: ImprovementSignalStore;
/**
* Optional LLM enrichment hook. When provided, the template is passed
* through before serialization so the SKILL.md prose can be rewritten.
@@ -117,22 +113,5 @@ export async function autoExtractAndCreateSkill(
const md = redactSkillContent(annotateWithScope(generateSkillMarkdown(finalTemplate))).content;
fs.writeFileSync(filePath, md, 'utf-8');
if (deps.improvementSignals) {
try {
deps.improvementSignals.record(
'workflow_pattern',
`auto-extract:${stem}`,
`Auto-extracted skill "${finalTemplate.name}" from session pattern (${finalTemplate.tools.join(', ')})`,
{
skillName: finalTemplate.name,
filename: stem,
tools: finalTemplate.tools,
category: finalTemplate.category,
source: 'auto_extract',
},
);
} catch { /* non-blocking */ }
}
return { filePath, template: finalTemplate };
}

View File

@@ -9,12 +9,12 @@
* before every authored write.
*
* Secrets reuse the curated SECRET_PATTERNS via redactSecrets (single source of
* truth — see eval-dataset.ts). Paths cover the user's home directory (the exact
* truth — see secret-patterns.ts). Paths cover the user's home directory (the exact
* running-user path plus generic per-OS forms), which both leaks the username
* and makes a skill non-portable.
*/
import os from 'node:os';
import { redactSecrets } from './eval-dataset.js';
import { redactSecrets } from './secret-patterns.js';
// Generic user-home prefixes. Each matches up to (not including) the next path
// separator, so the tail is preserved: C:\Users\me\.waggle\x -> ~\.waggle\x

View File

@@ -22,7 +22,6 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { ImprovementSignalStore } from '@waggle/core';
import { loadSkillUsage, forgetSkillUsage } from './skill-usage.js';
const DEFAULT_MAX_IDLE_DAYS = 90;
@@ -35,8 +34,6 @@ export interface RetireOptions {
dryRun?: boolean;
/** Injectable clock for deterministic tests. */
now?: () => Date;
/** Emit skill_promotion signals for observability. */
improvementSignals?: ImprovementSignalStore;
}
export interface RetireReport {
@@ -124,17 +121,6 @@ export function retireStaleSkills(waggleHome: string, opts: RetireOptions = {}):
report.retired.push(skillName);
report.archived.push(destPath);
forgetSkillUsage(waggleHome, skillName);
if (opts.improvementSignals) {
try {
opts.improvementSignals.record(
'workflow_pattern',
`retire:${skillName}`,
`Retired unused skill "${skillName}" (idle > ${maxIdleDays} days). Archived to ${destPath}.`,
{ skillName, archivedTo: destPath, maxIdleDays, lastActivityMs: activity },
);
} catch { /* non-blocking */ }
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
report.skipped.push({ name: skillName, reason: `move_failed: ${msg}` });

View File

@@ -35,7 +35,6 @@ import {
SKILL_SCOPE_ORDER,
type SkillScope,
} from './skill-frontmatter.js';
import type { ImprovementSignalStore } from '@waggle/core';
import { autoExtractAndCreateSkill, type AutoExtractMessage } from './skill-autoextract.js';
import { retireStaleSkills } from './skill-retirement.js';
@@ -57,11 +56,9 @@ export interface SkillToolsDeps {
/**
* Skills 2.0 gap E: promotion infrastructure. Each callback is optional.
* - getWorkspaceId / getTeamId resolve scope dirs dynamically at tool-call time.
* - improvementSignals records skill_promotion signals on successful moves.
*/
getWorkspaceId?: () => string | null;
getTeamId?: () => string | null;
improvementSignals?: ImprovementSignalStore;
/**
* Skills 2.0 gap A: session-message accessor for auto-extracting skills
* from repeated workflow patterns. When provided, enables the
@@ -665,15 +662,14 @@ Only use this after acquire_capability has identified a specific installable can
},
// Skills 2.0 gap E: promote_skill — move a skill up one scope rung.
// personal → workspace. Enforces one-step-at-a-time and records a
// skill_promotion improvement signal on success.
// personal → workspace. Enforces one-step-at-a-time.
{
name: 'promote_skill',
description:
'Promote an installed skill one scope up (personal → workspace). '
+ 'Team and enterprise scopes are not available in this edition. '
+ 'The file is moved to the appropriate directory, frontmatter is rewritten '
+ '(scope + promoted_from history), and a skill_promotion signal is recorded.',
+ 'The file is moved to the appropriate directory and frontmatter is rewritten '
+ '(scope + promoted_from history).',
parameters: {
type: 'object',
properties: {
@@ -784,19 +780,7 @@ Only use this after acquire_capability has identified a specific installable can
// failing the promotion. An audit warning is fine.
}
// 8. Record skill_promotion signal
if (deps.improvementSignals) {
try {
deps.improvementSignals.record(
'skill_promotion',
`promote:${skillName}:${currentScope}->${target}`,
`Promoted skill "${skillName}" from ${currentScope} to ${target}`,
{ skillName, from: currentScope, to: target, workspaceId, teamId },
);
} catch { /* non-blocking */ }
}
// 9. Let the agent reload its skill list so the new scope is picked up
// 8. Let the agent reload its skill list so the new scope is picked up
onSkillsChanged?.();
return (
@@ -805,7 +789,7 @@ Only use this after acquire_capability has identified a specific installable can
+ `- Moved from \`${found.filePath}\`\n`
+ `- Now at \`${destPath}\`\n`
+ `- Promotion history: ${updatedFrontmatter.promoted_from!.join(' → ')} → ${target}\n`
+ `- Signal recorded: skill_promotion\n\n`
+ `\n`
+ `The skill is available immediately at the new scope. Agents in other `
+ `sessions will pick it up on their next skill reload.`
);
@@ -823,8 +807,7 @@ Only use this after acquire_capability has identified a specific installable can
'Scan the current session for repeated workflow patterns (the same 3+ tools '
+ 'used in the same order 2+ times) and auto-create a SKILL.md for any pattern '
+ 'found. Creates personal-scope skills only — promote_skill moves them up '
+ 'afterwards. Dedups by filename. Records a workflow_pattern signal on each '
+ 'successful extraction.',
+ 'afterwards. Dedups by filename.',
parameters: {
type: 'object',
properties: {},
@@ -839,7 +822,6 @@ Only use this after acquire_capability has identified a specific installable can
}
const result = await autoExtractAndCreateSkill(messages, {
personalSkillsDir: skillsDir,
improvementSignals: deps.improvementSignals,
});
if (result.filePath) {
onSkillsChanged?.();
@@ -850,7 +832,6 @@ Only use this after acquire_capability has identified a specific installable can
+ `- Tools: ${result.template!.tools.join(', ')}\n`
+ `- Written to: \`${result.filePath}\`\n`
+ `- Scope: personal (use promote_skill to move it up)\n`
+ `- Signal: workflow_pattern recorded\n`
);
}
if (result.reason === 'already_exists') {
@@ -870,7 +851,7 @@ Only use this after acquire_capability has identified a specific installable can
'Archive personal-scope skills that have not been used within the idle window. '
+ 'Moves stale SKILL.md files to ~/.waggle/skills-archive/ (recoverable — no deletion). '
+ 'Default max_idle_days is 90. Default dry_run is true — call with dry_run: false to '
+ 'actually move the files. Records a workflow_pattern signal per retirement.',
+ 'actually move the files.',
parameters: {
type: 'object',
properties: {
@@ -890,7 +871,6 @@ Only use this after acquire_capability has identified a specific installable can
const report = retireStaleSkills(waggleHome, {
maxIdleDays,
dryRun,
improvementSignals: deps.improvementSignals,
});
if (!dryRun && report.retired.length > 0) {
onSkillsChanged?.();

View File

@@ -12,7 +12,6 @@ import type {
} from '@waggle/core';
import type { CognifyPipeline } from './cognify.js';
import type { FeedbackHandler } from './feedback-handler.js';
import type { ImprovementSignalStore } from '@waggle/core';
import type { RiskLevel } from '@waggle/shared';
import { createCoreLogger } from '@waggle/core';
import { detectContradiction } from './contradiction-detector.js';
@@ -96,13 +95,6 @@ export interface MindToolDeps {
feedback?: FeedbackHandler;
/** F16: Optional embedder for semantic dedup (cosine similarity) */
embedder?: Embedder;
/**
* Skills 2.0 gap K: improvement-signal store for recording write-path
* conflicts. When a new save contradicts an existing frame,
* detectContradiction fires and we emit a "correction" signal so the
* evolution / improvement loops can surface the conflict.
*/
improvementSignals?: ImprovementSignalStore;
/** Dynamic accessor for workspace layers — checked at call time */
getWorkspaceLayers?: () => WorkspaceLayers | null;
/** W5.7: Accessor for ALL workspace search instances (for cross-workspace search) */
@@ -528,20 +520,6 @@ export function createMindTools(deps: MindToolDeps): ToolDefinition[] {
const contradiction = detectContradiction(content, recentRows);
if (contradiction.isContradiction && contradiction.conflictsWith) {
conflictFlag = ` [flag: contradicts_existing (${contradiction.conflictsWith.slice(0, 80).replace(/\s+/g, ' ').trim()}…)]`;
// Record a correction-category signal on the improvement-signal
// store. pattern_key is a stable hash-ish of the conflicting
// frames so repeat contradictions merge via the ON CONFLICT upsert.
if (deps.improvementSignals) {
const patternKey = `write-conflict:${normalizeForDedup(content).slice(0, 64)}`;
try {
deps.improvementSignals.record(
'correction',
patternKey,
`New save contradicts existing frame. New: "${content.slice(0, 120)}" ⟂ Existing: "${contradiction.conflictsWith.slice(0, 120)}"`,
{ mindLabel, newContent: content.slice(0, 240), conflictsWith: contradiction.conflictsWith.slice(0, 240) },
);
} catch { /* non-blocking — signal persistence failure must not abort the save */ }
}
}
} catch { /* non-blocking — contradiction check failure proceeds with save */ }