refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ax-llm/ax": "^19.0.12",
|
||||
"@waggle/core": "*",
|
||||
"@waggle/marketplace": "*",
|
||||
"@waggle/shared": "*",
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
/**
|
||||
* Agent Learning — persistent behavioral adjustments from performance signals.
|
||||
*
|
||||
* Tracks what works and what doesn't across sessions, adjusts agent behavior
|
||||
* by building a learned context section injected into the system prompt.
|
||||
*
|
||||
* Three learning channels:
|
||||
* 1. Correction patterns — user corrections become persistent rules
|
||||
* 2. Success patterns — approaches that got positive feedback
|
||||
* 3. Persona effectiveness — per-persona quality signals
|
||||
*
|
||||
* Uses the improvement_signals table for persistence.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
|
||||
export interface LearnedBehavior {
|
||||
rule: string;
|
||||
source: 'correction' | 'success' | 'observation';
|
||||
confidence: number;
|
||||
occurrences: number;
|
||||
lastSeen: string;
|
||||
}
|
||||
|
||||
export interface PersonaEffectiveness {
|
||||
personaId: string;
|
||||
tasksCompleted: number;
|
||||
correctionsReceived: number;
|
||||
positiveSignals: number;
|
||||
effectivenessScore: number; // 0-100
|
||||
}
|
||||
|
||||
export interface LearningSnapshot {
|
||||
learnedBehaviors: LearnedBehavior[];
|
||||
personaStats: PersonaEffectiveness[];
|
||||
totalCorrections: number;
|
||||
totalSuccesses: number;
|
||||
adaptationLevel: 'new' | 'learning' | 'adapted' | 'expert';
|
||||
}
|
||||
|
||||
export class AgentLearning {
|
||||
private store: ImprovementSignalStore;
|
||||
|
||||
constructor(store: ImprovementSignalStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Record a successful interaction pattern. */
|
||||
recordSuccess(pattern: string, personaId?: string): void {
|
||||
this.store.record('workflow_pattern', `success:${pattern}`, `Approach worked well`, {
|
||||
type: 'success',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record that the user gave positive feedback. */
|
||||
recordPositiveFeedback(context: string, personaId?: string): void {
|
||||
this.store.record('correction', `positive:${context.slice(0, 50)}`, context, {
|
||||
type: 'positive',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record a persona's task completion. */
|
||||
recordPersonaTask(personaId: string, corrected: boolean): void {
|
||||
const key = corrected ? `persona:${personaId}:corrected` : `persona:${personaId}:completed`;
|
||||
this.store.record('workflow_pattern', key, personaId, {
|
||||
type: 'persona_task',
|
||||
personaId,
|
||||
corrected,
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a learning snapshot from accumulated signals. */
|
||||
getSnapshot(): LearningSnapshot {
|
||||
const all = this.store.getActionable();
|
||||
const learnedBehaviors: LearnedBehavior[] = [];
|
||||
const personaMap = new Map<string, PersonaEffectiveness>();
|
||||
let totalCorrections = 0;
|
||||
let totalSuccesses = 0;
|
||||
|
||||
for (const signal of all) {
|
||||
// Corrections become learned behavioral rules
|
||||
if (signal.category === 'correction' && !signal.pattern_key.startsWith('positive:')) {
|
||||
totalCorrections += signal.count;
|
||||
if (signal.count >= 2) { // Only learn from repeated corrections
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || signal.pattern_key,
|
||||
source: 'correction',
|
||||
confidence: Math.min(1.0, 0.5 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Positive feedback
|
||||
if (signal.pattern_key.startsWith('positive:')) {
|
||||
totalSuccesses += signal.count;
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || 'Positive approach',
|
||||
source: 'success',
|
||||
confidence: Math.min(1.0, 0.6 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
|
||||
// Success patterns
|
||||
if (signal.pattern_key.startsWith('success:')) {
|
||||
totalSuccesses += signal.count;
|
||||
}
|
||||
|
||||
// Persona stats
|
||||
if (signal.pattern_key.startsWith('persona:')) {
|
||||
const parts = signal.pattern_key.split(':');
|
||||
const personaId = parts[1];
|
||||
const isCorrected = parts[2] === 'corrected';
|
||||
|
||||
let stats = personaMap.get(personaId);
|
||||
if (!stats) {
|
||||
stats = { personaId, tasksCompleted: 0, correctionsReceived: 0, positiveSignals: 0, effectivenessScore: 50 };
|
||||
personaMap.set(personaId, stats);
|
||||
}
|
||||
|
||||
if (isCorrected) {
|
||||
stats.correctionsReceived += signal.count;
|
||||
} else {
|
||||
stats.tasksCompleted += signal.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate persona effectiveness
|
||||
for (const stats of personaMap.values()) {
|
||||
const total = stats.tasksCompleted + stats.correctionsReceived;
|
||||
stats.effectivenessScore = total > 0
|
||||
? Math.round((stats.tasksCompleted / total) * 100)
|
||||
: 50;
|
||||
}
|
||||
|
||||
// Determine adaptation level
|
||||
const totalSignals = totalCorrections + totalSuccesses;
|
||||
const adaptationLevel = totalSignals < 5 ? 'new'
|
||||
: totalSignals < 20 ? 'learning'
|
||||
: totalSignals < 50 ? 'adapted'
|
||||
: 'expert';
|
||||
|
||||
return {
|
||||
learnedBehaviors: learnedBehaviors.slice(0, 10), // Cap at 10 rules
|
||||
personaStats: Array.from(personaMap.values()),
|
||||
totalCorrections,
|
||||
totalSuccesses,
|
||||
adaptationLevel,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format learned behaviors as a system prompt section.
|
||||
* Only included when there are meaningful learned rules.
|
||||
*/
|
||||
formatLearningPrompt(): string | null {
|
||||
const snapshot = this.getSnapshot();
|
||||
if (snapshot.learnedBehaviors.length === 0) return null;
|
||||
|
||||
const lines: string[] = [
|
||||
'## Learned Behaviors',
|
||||
`*Adaptation level: ${snapshot.adaptationLevel} (${snapshot.totalCorrections} corrections, ${snapshot.totalSuccesses} successes)*`,
|
||||
'',
|
||||
];
|
||||
|
||||
const corrections = snapshot.learnedBehaviors.filter(b => b.source === 'correction');
|
||||
if (corrections.length > 0) {
|
||||
lines.push('**Avoid these patterns** (user has corrected before):');
|
||||
for (const b of corrections) {
|
||||
lines.push(`- ${b.rule} (corrected ${b.occurrences}x, confidence: ${(b.confidence * 100).toFixed(0)}%)`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const successes = snapshot.learnedBehaviors.filter(b => b.source === 'success');
|
||||
if (successes.length > 0) {
|
||||
lines.push('**Keep doing** (positive feedback received):');
|
||||
for (const b of successes) {
|
||||
lines.push(`- ${b.rule}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
}
|
||||
@@ -384,8 +384,7 @@ When approaching any task:
|
||||
};
|
||||
|
||||
/**
|
||||
* Section name literal type — matches the evolution-deploy module.
|
||||
* Kept minimal here so behavioral-spec.ts stays free of cross-imports.
|
||||
* Section name literal type for `buildActiveBehavioralSpec` overrides.
|
||||
*/
|
||||
export type BehavioralSpecSectionName =
|
||||
| 'coreLoop'
|
||||
@@ -402,8 +401,7 @@ export type BehavioralSpecSectionName =
|
||||
* the compiled baseline unchanged.
|
||||
*
|
||||
* Typical usage at server boot:
|
||||
* const overrides = loadBehavioralSpecOverrides(dataDir);
|
||||
* const spec = buildActiveBehavioralSpec(overrides);
|
||||
* const spec = buildActiveBehavioralSpec();
|
||||
* // ...pass spec.rules into the system prompt...
|
||||
*/
|
||||
export function buildActiveBehavioralSpec(
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
/**
|
||||
* Compose Evolution — Phase 3.2 of the self-evolution loop.
|
||||
*
|
||||
* Two-stage pipeline:
|
||||
*
|
||||
* Stage 1 — EvolveSchema
|
||||
* Evolve the output STRUCTURE (fields, order, types, constraints).
|
||||
* Returns a winner schema.
|
||||
*
|
||||
* Stage 2 — IterativeGEPA
|
||||
* Freeze the winner schema. Evolve the INSTRUCTION prompt that fills
|
||||
* that schema. Only instruction text changes here.
|
||||
*
|
||||
* Critical design detail (from Mikhail Pavlukhin's paper): **feedback
|
||||
* separation**. If the judge complains "missing reasoning field" during
|
||||
* Stage 2, and we let GEPA see that feedback, GEPA will mutate the
|
||||
* instruction to try to remove reasoning — undoing the schema Stage 1
|
||||
* just evolved. So Stage 2 receives a *filtered* judge that strips
|
||||
* structural complaints and keeps only value-level signals (correctness,
|
||||
* conciseness, format-of-values, tone).
|
||||
*
|
||||
* The composition is stateless: callers provide the baseline schema and
|
||||
* instruction separately; the result returns both winners + aggregate
|
||||
* accuracy improvement.
|
||||
*/
|
||||
|
||||
import {
|
||||
EvolveSchema,
|
||||
type EvolveSchemaOptions,
|
||||
type EvolveSchemaResult,
|
||||
type Schema,
|
||||
type SchemaExecuteFn,
|
||||
} from './evolve-schema.js';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
type IterativeGEPAOptions,
|
||||
type GEPARunResult,
|
||||
} from './iterative-optimizer.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import { RUNNING_JUDGE_BRAND, isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface ComposeEvolutionOptions {
|
||||
/** EvolveSchema stage configuration */
|
||||
schema: Omit<EvolveSchemaOptions, 'onProgress' | 'signal'>;
|
||||
/** IterativeGEPA stage configuration */
|
||||
instructions: Omit<IterativeGEPAOptions, 'onProgress' | 'signal'>;
|
||||
/**
|
||||
* Classifier — returns 'structural' to drop a feedback line from GEPA,
|
||||
* or 'value' to keep it. Default: `defaultFeedbackFilter`.
|
||||
*/
|
||||
feedbackFilter?: FeedbackFilter;
|
||||
/** Optional progress reporter across both stages */
|
||||
onProgress?: (event: ComposeProgress) => void;
|
||||
/** Optional abort signal (passed through to both stages) */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ComposeEvolutionResult {
|
||||
schema: EvolveSchemaResult;
|
||||
instructions: GEPARunResult;
|
||||
/**
|
||||
* Accuracy delta after both stages compared to the raw baseline schema +
|
||||
* baseline instructions. Positive = improvement.
|
||||
*/
|
||||
combinedDelta: number;
|
||||
/** True if both stages improved over their individual baselines. */
|
||||
fullyImproved: boolean;
|
||||
/** Frozen schema used as input to the GEPA stage — convenience pointer. */
|
||||
frozenSchema: Schema;
|
||||
}
|
||||
|
||||
export interface ComposeProgress {
|
||||
stage: 'schema' | 'instructions' | 'done';
|
||||
/** Underlying stage progress event, if available */
|
||||
detail?: unknown;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export type FeedbackFilter = (feedback: string) => 'structural' | 'value';
|
||||
|
||||
// ── Feedback filtering ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Default heuristic: classifies feedback lines as 'structural' when they
|
||||
* mention schema-level concepts (field, schema, missing/extra property,
|
||||
* wrong type, reorder, etc). Everything else is 'value'.
|
||||
*
|
||||
* Tight enough that ambiguous feedback still flows to GEPA — only obvious
|
||||
* structural complaints are dropped.
|
||||
*/
|
||||
export function defaultFeedbackFilter(feedback: string): 'structural' | 'value' {
|
||||
if (!feedback) return 'value';
|
||||
const lower = feedback.toLowerCase();
|
||||
|
||||
const structuralPatterns: RegExp[] = [
|
||||
/\bmissing\s+(?:the\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\badd(?:ed|ing)?\s+(?:a|the)?\s*(?:new\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\bextra\s+field\b/,
|
||||
/\bunexpected\s+field\b/,
|
||||
/\bwrong\s+(?:field\s+)?(?:type|order)\b/,
|
||||
/\bfield\s+type\s+(?:mismatch|wrong)/,
|
||||
/\b(?:should\s+)?(?:re)?order\s+(?:the\s+)?fields\b/,
|
||||
/\bschema\s+(?:mismatch|violation|error)\b/,
|
||||
/\binvalid\s+json\s+shape\b/,
|
||||
/\bmissing\s+required\s+property\b/,
|
||||
/\bproperty\s+missing\b/,
|
||||
/\bshape\s+is\s+wrong\b/,
|
||||
];
|
||||
|
||||
for (const pattern of structuralPatterns) {
|
||||
if (pattern.test(lower)) return 'structural';
|
||||
}
|
||||
return 'value';
|
||||
}
|
||||
|
||||
// ── Judge wrapping ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Wrap an LLMJudge-like object so the feedback it emits has structural
|
||||
* lines stripped according to `filter`. Only `feedback` is filtered —
|
||||
* the numerical scores (correctness, procedure, conciseness, overall)
|
||||
* are preserved as-is so GEPA sees the true accuracy signal.
|
||||
*
|
||||
* The multi-line feedback from the judge is split on newlines and each
|
||||
* line classified individually.
|
||||
*/
|
||||
export function filterJudgeFeedback(
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): Pick<LLMJudge, 'score'> {
|
||||
const wrapped: Pick<LLMJudge, 'score'> & Partial<Record<symbol, unknown>> = {
|
||||
async score(args) {
|
||||
const raw = await judge.score(args);
|
||||
const filteredFeedback = stripStructuralLines(raw.feedback, filter);
|
||||
const result: JudgeScore = { ...raw, feedback: filteredFeedback };
|
||||
return result;
|
||||
},
|
||||
};
|
||||
// H-09 G3: preserve the running-judge brand so IterativeGEPA's check
|
||||
// still passes after feedback filtering. Without this, wrapping a
|
||||
// running judge with filterJudgeFeedback would silently strip the
|
||||
// brand and GEPA would reject it.
|
||||
if (isRunningJudge(judge)) {
|
||||
wrapped[RUNNING_JUDGE_BRAND] = true;
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
/** Public helper — strips structural lines from a feedback string. */
|
||||
export function stripStructuralLines(
|
||||
feedback: string,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): string {
|
||||
if (!feedback) return feedback;
|
||||
return feedback
|
||||
.split(/\r?\n/)
|
||||
.filter(line => filter(line) === 'value')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────
|
||||
|
||||
export class ComposeEvolution {
|
||||
async run(options: ComposeEvolutionOptions): Promise<ComposeEvolutionResult> {
|
||||
const signal = options.signal;
|
||||
const filter = options.feedbackFilter ?? defaultFeedbackFilter;
|
||||
|
||||
// Stage 1 — evolve the schema.
|
||||
options.onProgress?.({ stage: 'schema', message: 'starting schema evolution' });
|
||||
const schemaResult = await new EvolveSchema().run({
|
||||
...options.schema,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'schema', detail: e }),
|
||||
});
|
||||
|
||||
if (signal?.aborted) {
|
||||
return assembleAbortedResult(schemaResult, options.instructions.baseline);
|
||||
}
|
||||
|
||||
const frozenSchema = schemaResult.winner.schema;
|
||||
|
||||
// Stage 2 — evolve the instructions, with a filtered judge so GEPA
|
||||
// never sees structural feedback.
|
||||
options.onProgress?.({ stage: 'instructions', message: 'starting instruction evolution' });
|
||||
const filteredJudge = filterJudgeFeedback(options.instructions.judge, filter);
|
||||
|
||||
const gepaResult = await new IterativeGEPA().run({
|
||||
...options.instructions,
|
||||
judge: filteredJudge,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'instructions', detail: e }),
|
||||
});
|
||||
|
||||
const combinedDelta =
|
||||
(gepaResult.winner.score?.overall ?? 0) -
|
||||
((schemaResult.history[0]?.score?.accuracy ?? 0));
|
||||
const fullyImproved = schemaResult.improved && gepaResult.improved;
|
||||
|
||||
options.onProgress?.({ stage: 'done' });
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: gepaResult,
|
||||
combinedDelta,
|
||||
fullyImproved,
|
||||
frozenSchema,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a convenience `SchemaExecuteFn` from a user-supplied runner that
|
||||
* only cares about the instruction prompt (not the schema itself). Used
|
||||
* by callers that want to compose — they supply one "execute" function
|
||||
* for the instructional stage and this helper wraps it for Stage 1.
|
||||
*
|
||||
* The wrapped executor serializes the schema into a short prefix the
|
||||
* model can follow (`Return JSON with fields: <field1>, <field2>, ...`)
|
||||
* before delegating to the caller's function.
|
||||
*/
|
||||
export function schemaExecutorFromInstructionRunner(
|
||||
runInstructions: (args: { prompt: string; input: string }) => Promise<string>,
|
||||
): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
const fieldList = schema.fields.map(f => `"${f.name}" (${f.type})`).join(', ');
|
||||
const prompt = `Return a JSON object with these fields: ${fieldList}.`;
|
||||
try {
|
||||
const actual = await runInstructions({ prompt, input });
|
||||
return { actual, parsed: actualLooksLikeJson(actual) };
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function actualLooksLikeJson(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
return (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
|
||||
(trimmed.startsWith('[') && trimmed.endsWith(']'));
|
||||
}
|
||||
|
||||
function assembleAbortedResult(
|
||||
schemaResult: EvolveSchemaResult,
|
||||
instructionBaseline: string,
|
||||
): ComposeEvolutionResult {
|
||||
// When aborted before Stage 2, produce a minimal GEPARunResult pointing
|
||||
// at the baseline instruction so callers have a stable shape.
|
||||
const baselineInstruction = {
|
||||
id: 'aborted',
|
||||
prompt: instructionBaseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'aborted',
|
||||
score: null,
|
||||
perExample: [],
|
||||
} as GEPARunResult['winner'];
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: {
|
||||
winner: baselineInstruction,
|
||||
paretoFront: [baselineInstruction],
|
||||
history: [baselineInstruction],
|
||||
improved: false,
|
||||
delta: 0,
|
||||
},
|
||||
combinedDelta: 0,
|
||||
fullyImproved: false,
|
||||
frozenSchema: schemaResult.winner.schema,
|
||||
};
|
||||
}
|
||||
|
||||
/** Re-export types so callers can `import { type Schema } from '...'`. */
|
||||
export type { EvolveSchemaResult, GEPARunResult };
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* Correction Detector — identifies when a user is correcting the agent
|
||||
* and classifies corrections as durable (behavior-changing) vs task-local.
|
||||
*
|
||||
* Per Slice 7 correction #2:
|
||||
* - Durable corrections: repeated patterns that should influence future behavior → improvement signals
|
||||
* - Task-local corrections: one-off adjustments within a session → NOT signals
|
||||
* - One-off disagreements: not corrections at all → ignored
|
||||
*/
|
||||
|
||||
export type CorrectionDurability = 'durable' | 'task_local' | 'not_correction';
|
||||
|
||||
export interface DetectedCorrection {
|
||||
isDurable: boolean;
|
||||
durability: CorrectionDurability;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
confidence: number; // 0-1
|
||||
}
|
||||
|
||||
// ── Correction signal patterns ──────────────────────────────
|
||||
|
||||
/** Strong correction signals — high confidence the user is correcting agent behavior */
|
||||
const STRONG_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /\bno[,.]?\s*(?:not that|don'?t|do not|stop|never)\b/i, weight: 3 },
|
||||
{ pattern: /\bI (?:said|told you|asked)\b/i, weight: 3 },
|
||||
{ pattern: /\bthat'?s (?:wrong|incorrect|not (?:what|right))\b/i, weight: 3 },
|
||||
{ pattern: /\bplease (?:don'?t|do not|stop|never)\b/i, weight: 2 },
|
||||
{ pattern: /\binstead[,.]?\s*(?:use|do|try|go with)\b/i, weight: 2 },
|
||||
{ pattern: /\bwrong (?:approach|way|format|style|tone)\b/i, weight: 2 },
|
||||
{ pattern: /\bnot what I (?:wanted|meant|asked)\b/i, weight: 3 },
|
||||
];
|
||||
|
||||
/** Moderate correction signals — may be correction or just refinement */
|
||||
const MODERATE_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /^no[,.:!]\s/i, weight: 2 }, // "No, ..." at start of message — strong disagreement
|
||||
{ pattern: /\bactually[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\brather[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\blet'?s (?:not|try|go with|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bprefer\b/i, weight: 1 },
|
||||
{ pattern: /\bshould (?:be|have been|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bchange (?:it|this|that) to\b/i, weight: 1 },
|
||||
{ pattern: /\btoo (?:verbose|long|short|formal|casual|technical|simple)\b/i, weight: 1 },
|
||||
{ pattern: /\bstop (?:doing|using|adding)\b/i, weight: 1 },
|
||||
{ pattern: /\bkeep (?:it|things) (?:simple|short|brief|casual|formal)\b/i, weight: 1 },
|
||||
];
|
||||
|
||||
// ── Durability classification patterns ──────────────────────
|
||||
|
||||
/** Durable signals — the correction applies beyond this specific task */
|
||||
const DURABLE_SIGNALS: RegExp[] = [
|
||||
/\balways\b/i,
|
||||
/\bnever\b/i,
|
||||
/\bfrom now on\b/i,
|
||||
/\bin (?:the )?future\b/i,
|
||||
/\bwhenever\b/i,
|
||||
/\bevery time\b/i,
|
||||
/\bin general\b/i,
|
||||
/\bI (?:always |usually )?prefer\b/i,
|
||||
/\bmy (?:preference|style|approach)\b/i,
|
||||
/\bdon'?t (?:ever|again)\b/i,
|
||||
/\bremember (?:to|that)\b/i,
|
||||
/\bkeep (?:it|things|this)\b/i,
|
||||
];
|
||||
|
||||
/** Task-local signals — the correction is specific to this task */
|
||||
const TASK_LOCAL_SIGNALS: RegExp[] = [
|
||||
/\bthis (?:time|one|specific|particular)\b/i,
|
||||
/\bfor (?:this|now)\b/i,
|
||||
/\bjust (?:here|this|now)\b/i,
|
||||
/\bin this (?:case|instance|response)\b/i,
|
||||
/\bright now\b/i,
|
||||
/\bhere\b/i,
|
||||
];
|
||||
|
||||
// ── Pattern key extraction ──────────────────────────────────
|
||||
|
||||
/** Categories of behavioral corrections we can extract pattern keys for */
|
||||
const PATTERN_KEY_EXTRACTORS: Array<{ category: string; pattern: RegExp }> = [
|
||||
{ category: 'tone', pattern: /\btoo (?:formal|casual|verbose|terse|technical|simple)\b/i },
|
||||
{ category: 'format', pattern: /\b(?:format|formatting|headers?|bullet|numbering|markdown)\b/i },
|
||||
{ category: 'length', pattern: /\btoo (?:long|short|brief|detailed)\b/i },
|
||||
{ category: 'approach', pattern: /\bwrong (?:approach|way|method|strategy)\b/i },
|
||||
{ category: 'scope', pattern: /\btoo (?:much|many|broad|narrow|specific|general)\b/i },
|
||||
{ category: 'accuracy', pattern: /\b(?:wrong|incorrect|inaccurate|mistake|error)\b/i },
|
||||
{ category: 'style', pattern: /\b(?:style|voice|writing|wording|phrasing)\b/i },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect whether a user message contains a correction and classify it.
|
||||
*
|
||||
* Returns null if no correction detected.
|
||||
* Returns DetectedCorrection with durability classification if correction found.
|
||||
*/
|
||||
export function detectCorrection(
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
if (!userMessage || userMessage.length < 5) return null;
|
||||
|
||||
// Score correction strength
|
||||
let correctionScore = 0;
|
||||
for (const { pattern, weight } of STRONG_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
for (const { pattern, weight } of MODERATE_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
|
||||
// Need minimum score to count as correction
|
||||
if (correctionScore < 2) return null;
|
||||
|
||||
// Classify durability
|
||||
const durability = classifyDurability(userMessage);
|
||||
|
||||
// Extract pattern key
|
||||
const patternKey = extractPatternKey(userMessage);
|
||||
|
||||
// Compute confidence (0-1)
|
||||
const confidence = Math.min(correctionScore / 6, 1);
|
||||
|
||||
// Build detail: first sentence or first 120 chars
|
||||
const detail = extractDetail(userMessage);
|
||||
|
||||
return {
|
||||
isDurable: durability === 'durable',
|
||||
durability,
|
||||
patternKey,
|
||||
detail,
|
||||
confidence,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyDurability(message: string): CorrectionDurability {
|
||||
let durableScore = 0;
|
||||
let taskLocalScore = 0;
|
||||
|
||||
for (const pattern of DURABLE_SIGNALS) {
|
||||
if (pattern.test(message)) durableScore++;
|
||||
}
|
||||
for (const pattern of TASK_LOCAL_SIGNALS) {
|
||||
if (pattern.test(message)) taskLocalScore++;
|
||||
}
|
||||
|
||||
// Explicit durable signals win
|
||||
if (durableScore > 0 && durableScore >= taskLocalScore) return 'durable';
|
||||
|
||||
// Explicit task-local signals
|
||||
if (taskLocalScore > 0) return 'task_local';
|
||||
|
||||
// No explicit signals — default to task_local (conservative; only promote to durable
|
||||
// when the same pattern_key recurs across sessions via ImprovementSignalStore)
|
||||
return 'task_local';
|
||||
}
|
||||
|
||||
function extractPatternKey(message: string): string {
|
||||
for (const { category, pattern } of PATTERN_KEY_EXTRACTORS) {
|
||||
const match = message.match(pattern);
|
||||
if (match) {
|
||||
const qualifier = match[0].toLowerCase().replace(/\s+/g, '_');
|
||||
return `${category}:${qualifier}`;
|
||||
}
|
||||
}
|
||||
// Fallback: generic correction key
|
||||
return 'general:correction';
|
||||
}
|
||||
|
||||
function extractDetail(message: string): string {
|
||||
// Take first sentence or first 120 chars
|
||||
const sentenceMatch = message.match(/^(.+?[.!?])\s/);
|
||||
if (sentenceMatch && sentenceMatch[1].length <= 120) {
|
||||
return sentenceMatch[1];
|
||||
}
|
||||
return message.length > 120 ? message.slice(0, 117) + '...' : message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a sequence of message pairs to detect corrections.
|
||||
* Useful for batch analysis of session history.
|
||||
*/
|
||||
export function detectCorrectionsInHistory(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
): DetectedCorrection[] {
|
||||
const corrections: DetectedCorrection[] = [];
|
||||
for (let i = 1; i < messages.length; i++) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== 'user') continue;
|
||||
|
||||
const prevAssistant = i > 0 ? messages[i - 1] : undefined;
|
||||
const correction = detectCorrection(
|
||||
msg.content,
|
||||
prevAssistant?.role === 'assistant' ? prevAssistant.content : undefined,
|
||||
);
|
||||
if (correction) {
|
||||
corrections.push(correction);
|
||||
}
|
||||
}
|
||||
return corrections;
|
||||
}
|
||||
@@ -1,441 +0,0 @@
|
||||
/**
|
||||
* Eval Dataset Builder — turns execution traces and harvest frames into
|
||||
* train/val/holdout splits for the self-evolution loop.
|
||||
*
|
||||
* Sources mined:
|
||||
* - ExecutionTraceStore (outcome = success | verified → positive examples;
|
||||
* outcome = corrected → negative example with correctionFeedback as ground truth)
|
||||
* - Optional harvest frames (DistilledKnowledge) for Q&A-style augmentation
|
||||
* - Optional corrections from ImprovementSignalStore
|
||||
*
|
||||
* Filter pipeline (in order):
|
||||
* 1. Secret scanning — reject any example containing credentials / tokens
|
||||
* 2. Keyword heuristic — min length, non-trivial content, no duplicate inputs
|
||||
* 3. Optional LLM-as-judge relevance filter (pass an llmCall callback)
|
||||
*
|
||||
* Split is deterministic given a seed — same traces + same seed = same split.
|
||||
* Default split ratio is 60/20/20 train/val/holdout.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
TraceQueryFilter,
|
||||
} from '@waggle/core';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface EvalExample {
|
||||
input: string;
|
||||
expected_output: string;
|
||||
metadata: EvalExampleMetadata;
|
||||
}
|
||||
|
||||
export interface EvalExampleMetadata {
|
||||
traceId?: number;
|
||||
personaId?: string | null;
|
||||
taskShape?: string | null;
|
||||
model?: string | null;
|
||||
outcome?: TraceOutcome;
|
||||
tags?: string[];
|
||||
/** Where this example came from */
|
||||
source: 'trace' | 'harvest' | 'correction';
|
||||
/** Optional opaque identifier from the source system */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface DatasetSplit {
|
||||
train: EvalExample[];
|
||||
val: EvalExample[];
|
||||
holdout: EvalExample[];
|
||||
/** Examples that were filtered out — useful for debugging */
|
||||
rejected: Array<{ reason: string; preview: string }>;
|
||||
/** Stats for logging / UI */
|
||||
stats: {
|
||||
sourced: number;
|
||||
acceptedAfterSecretScan: number;
|
||||
acceptedAfterHeuristic: number;
|
||||
acceptedAfterJudge: number;
|
||||
unique: number;
|
||||
total: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BuildOptions {
|
||||
/** Which trace outcomes count as positive examples (default: ['success', 'verified']). */
|
||||
positiveOutcomes?: TraceOutcome[];
|
||||
/** Whether to include correction traces as negative examples with feedback as expected_output (default true). */
|
||||
includeCorrections?: boolean;
|
||||
/** Filter passed straight through to ExecutionTraceStore.query */
|
||||
traceFilter?: Omit<TraceQueryFilter, 'outcome' | 'limit'> & { limit?: number };
|
||||
/** Optional external augmenters */
|
||||
harvestExamples?: EvalExample[];
|
||||
correctionExamples?: EvalExample[];
|
||||
/** Minimum input characters to keep the example (default 10). */
|
||||
minInputChars?: number;
|
||||
/** Minimum expected_output characters to keep the example (default 5). */
|
||||
minOutputChars?: number;
|
||||
/** Maximum input + output characters combined; over this, the example is rejected (default 16384). */
|
||||
maxCombinedChars?: number;
|
||||
/** Optional LLM-as-judge relevance filter. Pass null or omit to skip. */
|
||||
judge?: (example: EvalExample) => Promise<JudgeVerdict>;
|
||||
/** Ratio triple summing to 1.0 (default [0.6, 0.2, 0.2]). */
|
||||
splitRatios?: [number, number, number];
|
||||
/** Deterministic seed (default 1). */
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
export interface JudgeVerdict {
|
||||
keep: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
|
||||
// ── Deterministic PRNG (mulberry32) ─────────────────────────────
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
/** Fisher-Yates shuffle in-place using provided rng. */
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// ── Builder ─────────────────────────────────────────────────────
|
||||
|
||||
export class EvalDatasetBuilder {
|
||||
private store: ExecutionTraceStore;
|
||||
|
||||
constructor(store: ExecutionTraceStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Mine traces into a typed example list (before filtering). */
|
||||
sourceFromTraces(
|
||||
positiveOutcomes: TraceOutcome[],
|
||||
includeCorrections: boolean,
|
||||
filter: BuildOptions['traceFilter'] = {},
|
||||
): EvalExample[] {
|
||||
const outcomes: TraceOutcome[] = [...positiveOutcomes];
|
||||
if (includeCorrections) outcomes.push('corrected');
|
||||
|
||||
const rows = this.store.queryParsed({
|
||||
...filter,
|
||||
outcome: outcomes,
|
||||
limit: filter.limit ?? 10_000,
|
||||
});
|
||||
|
||||
return rows.map(row => traceToExample(row, includeCorrections));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the full dataset split.
|
||||
*
|
||||
* Filter order:
|
||||
* 1. Source (traces + augmenters)
|
||||
* 2. Secret scan
|
||||
* 3. Heuristic (length, non-trivial)
|
||||
* 4. Optional judge
|
||||
* 5. Dedup on input hash
|
||||
* 6. Shuffle + split
|
||||
*/
|
||||
async build(options: BuildOptions = {}): Promise<DatasetSplit> {
|
||||
const positiveOutcomes = options.positiveOutcomes ?? ['success', 'verified'];
|
||||
const includeCorrections = options.includeCorrections ?? true;
|
||||
const minInputChars = options.minInputChars ?? 10;
|
||||
const minOutputChars = options.minOutputChars ?? 5;
|
||||
const maxCombinedChars = options.maxCombinedChars ?? 16_384;
|
||||
const splitRatios = options.splitRatios ?? [0.6, 0.2, 0.2];
|
||||
const seed = options.seed ?? 1;
|
||||
|
||||
validateRatios(splitRatios);
|
||||
|
||||
const rejected: DatasetSplit['rejected'] = [];
|
||||
|
||||
// 1. Source
|
||||
const traceExamples = this.sourceFromTraces(
|
||||
positiveOutcomes,
|
||||
includeCorrections,
|
||||
options.traceFilter,
|
||||
);
|
||||
const sourced: EvalExample[] = [
|
||||
...traceExamples,
|
||||
...(options.harvestExamples ?? []),
|
||||
...(options.correctionExamples ?? []),
|
||||
];
|
||||
|
||||
// 2. Secret scan
|
||||
const afterSecretScan: EvalExample[] = [];
|
||||
for (const ex of sourced) {
|
||||
const hitInput = detectSecrets(ex.input);
|
||||
const hitOutput = detectSecrets(ex.expected_output);
|
||||
if (hitInput || hitOutput) {
|
||||
rejected.push({
|
||||
reason: `secret:${hitInput ?? hitOutput}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
afterSecretScan.push(ex);
|
||||
}
|
||||
|
||||
// 3. Heuristic
|
||||
const afterHeuristic: EvalExample[] = [];
|
||||
for (const ex of afterSecretScan) {
|
||||
const input = ex.input.trim();
|
||||
const output = ex.expected_output.trim();
|
||||
if (input.length < minInputChars) {
|
||||
rejected.push({ reason: 'too-short-input', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (output.length < minOutputChars) {
|
||||
rejected.push({ reason: 'too-short-output', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (input.length + output.length > maxCombinedChars) {
|
||||
rejected.push({ reason: 'too-long', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (isLowSignal(input) || isLowSignal(output)) {
|
||||
rejected.push({ reason: 'low-signal', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
afterHeuristic.push({ ...ex, input, expected_output: output });
|
||||
}
|
||||
|
||||
// 4. Optional judge
|
||||
let afterJudge = afterHeuristic;
|
||||
if (options.judge) {
|
||||
afterJudge = [];
|
||||
for (const ex of afterHeuristic) {
|
||||
try {
|
||||
const verdict = await options.judge(ex);
|
||||
if (verdict.keep) {
|
||||
afterJudge.push(ex);
|
||||
} else {
|
||||
rejected.push({
|
||||
reason: `judge:${verdict.reason ?? 'rejected'}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// On judge failure, keep the example — err on the side of more data.
|
||||
afterJudge.push(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Dedup on input hash
|
||||
const seen = new Set<string>();
|
||||
const unique: EvalExample[] = [];
|
||||
for (const ex of afterJudge) {
|
||||
const key = hashKey(ex.input);
|
||||
if (seen.has(key)) {
|
||||
rejected.push({ reason: 'duplicate', preview: ex.input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
unique.push(ex);
|
||||
}
|
||||
|
||||
// 6. Shuffle + split
|
||||
const rng = makeRng(seed);
|
||||
const shuffled = shuffle([...unique], rng);
|
||||
const { train, val, holdout } = splitExamples(shuffled, splitRatios);
|
||||
|
||||
return {
|
||||
train,
|
||||
val,
|
||||
holdout,
|
||||
rejected,
|
||||
stats: {
|
||||
sourced: sourced.length,
|
||||
acceptedAfterSecretScan: afterSecretScan.length,
|
||||
acceptedAfterHeuristic: afterHeuristic.length,
|
||||
acceptedAfterJudge: afterJudge.length,
|
||||
unique: unique.length,
|
||||
total: train.length + val.length + holdout.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── JSONL IO ────────────────────────────────────────────────────
|
||||
|
||||
/** Serialize examples as JSONL (one JSON object per line). */
|
||||
export function toJSONL(examples: EvalExample[]): string {
|
||||
return examples.map(ex => JSON.stringify(ex)).join('\n');
|
||||
}
|
||||
|
||||
/** Parse JSONL into examples. Silently skips unparseable lines. */
|
||||
export function fromJSONL(jsonl: string): EvalExample[] {
|
||||
const out: EvalExample[] = [];
|
||||
for (const line of jsonl.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed.input === 'string' &&
|
||||
typeof parsed.expected_output === 'string' &&
|
||||
parsed.metadata
|
||||
) {
|
||||
out.push(parsed as EvalExample);
|
||||
}
|
||||
} catch {
|
||||
// skip malformed line
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function traceToExample(
|
||||
trace: ParsedExecutionTrace,
|
||||
includeCorrections: boolean,
|
||||
): EvalExample {
|
||||
const isCorrected = trace.outcome === 'corrected';
|
||||
const expected = isCorrected && includeCorrections
|
||||
? (trace.payload.correctionFeedback ?? trace.payload.output)
|
||||
: trace.payload.output;
|
||||
|
||||
return {
|
||||
input: trace.payload.input,
|
||||
expected_output: expected,
|
||||
metadata: {
|
||||
traceId: trace.id,
|
||||
personaId: trace.persona_id,
|
||||
taskShape: trace.task_shape,
|
||||
model: trace.model,
|
||||
outcome: trace.outcome,
|
||||
tags: trace.payload.tags ?? [],
|
||||
source: isCorrected ? 'correction' : 'trace',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function validateRatios(ratios: [number, number, number]): void {
|
||||
const sum = ratios[0] + ratios[1] + ratios[2];
|
||||
if (Math.abs(sum - 1.0) > 1e-6) {
|
||||
throw new Error(`Split ratios must sum to 1.0, got ${sum}`);
|
||||
}
|
||||
if (ratios.some(r => r < 0)) {
|
||||
throw new Error('Split ratios must be non-negative');
|
||||
}
|
||||
}
|
||||
|
||||
function splitExamples(
|
||||
examples: EvalExample[],
|
||||
ratios: [number, number, number],
|
||||
): { train: EvalExample[]; val: EvalExample[]; holdout: EvalExample[] } {
|
||||
const n = examples.length;
|
||||
// Floor-then-assign-remainders so every example ends up in exactly one split.
|
||||
const trainN = Math.floor(n * ratios[0]);
|
||||
const valN = Math.floor(n * ratios[1]);
|
||||
const holdoutN = n - trainN - valN;
|
||||
|
||||
return {
|
||||
train: examples.slice(0, trainN),
|
||||
val: examples.slice(trainN, trainN + valN),
|
||||
holdout: examples.slice(trainN + valN, trainN + valN + holdoutN),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap signal check — reject examples that are mostly whitespace, repeated
|
||||
* punctuation, or otherwise meaningless. Not a language detector.
|
||||
*/
|
||||
function isLowSignal(text: string): boolean {
|
||||
if (!text) return true;
|
||||
const stripped = text.replace(/\s+/g, '');
|
||||
if (stripped.length < 3) return true;
|
||||
// 80%+ of the same character → reject
|
||||
const counts = new Map<string, number>();
|
||||
for (const ch of stripped) {
|
||||
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
||||
}
|
||||
const maxCount = Math.max(...counts.values());
|
||||
if (maxCount / stripped.length > 0.8) return true;
|
||||
// No alphanumeric at all → reject
|
||||
if (!/[A-Za-z0-9]/.test(stripped)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* FNV-1a 32-bit hash — stable across processes, no crypto import needed.
|
||||
* Used for input-level dedup.
|
||||
*/
|
||||
function hashKey(text: string): string {
|
||||
let h = 0x811c9dc5;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
h ^= text.charCodeAt(i);
|
||||
h = Math.imul(h, 0x01000193);
|
||||
}
|
||||
return (h >>> 0).toString(16);
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
/**
|
||||
* Evolution Deploy Helpers — Phase 6.1 of the self-evolution loop.
|
||||
*
|
||||
* When the user accepts an EvolutionRun, the orchestrator calls the
|
||||
* caller-supplied `deploy` callback. This module provides the built-in
|
||||
* building blocks for that callback:
|
||||
*
|
||||
* deployPersonaOverride — writes a persona JSON file that
|
||||
* shadows the built-in persona at
|
||||
* runtime via loadCustomPersonas.
|
||||
* deployBehavioralSpecOverride — writes a JSON file in
|
||||
* {dataDir}/behavioral-overrides/
|
||||
* that can be merged into
|
||||
* BEHAVIORAL_SPEC sections at load.
|
||||
* loadBehavioralSpecOverrides — reads those overrides from disk.
|
||||
*
|
||||
* Every writer:
|
||||
* - Creates the target directory if missing.
|
||||
* - Writes atomically (write to .tmp, rename) so a crash mid-write
|
||||
* never corrupts the config.
|
||||
* - Backs up the previous version to {file}.bak before overwriting.
|
||||
* - Returns a structured DeployResult with paths + timestamps.
|
||||
*
|
||||
* These helpers are pure filesystem operations — they don't talk to
|
||||
* any runtime singletons, so the server wiring is trivial:
|
||||
* orchestrator.accept(uuid) → onDeploy(run)
|
||||
* → deployPersonaOverride(dataDir, ...)
|
||||
* → reloadPersonas()
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
import { getPersona } from './personas.js';
|
||||
import { assertValidCustomPersonaId } from './custom-personas.js';
|
||||
|
||||
// ── Public result shape ────────────────────────────────────────
|
||||
|
||||
export interface DeployResult {
|
||||
/** Absolute path that was written */
|
||||
path: string;
|
||||
/** Path to the backup of the previous version (null if first write) */
|
||||
backupPath: string | null;
|
||||
/** When the deploy happened */
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
// ── Persona override ───────────────────────────────────────────
|
||||
|
||||
export interface DeployPersonaInput {
|
||||
/** Persona id — matches a built-in persona (the deploy SHADOWS it). */
|
||||
personaId: string;
|
||||
/** The evolved system prompt text */
|
||||
systemPrompt: string;
|
||||
/** Optional additional field overrides */
|
||||
overrides?: Partial<Omit<AgentPersona, 'id' | 'systemPrompt'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write an evolved persona system prompt to disk so the custom-personas
|
||||
* loader picks it up on next `listPersonas()` call.
|
||||
*
|
||||
* When a built-in persona with this id exists, the override inherits
|
||||
* all its properties (name, tools, tagline, etc) except systemPrompt
|
||||
* (replaced with the evolved one). When no built-in exists, a minimal
|
||||
* persona shell is built from the input.
|
||||
*/
|
||||
export function deployPersonaOverride(
|
||||
dataDir: string,
|
||||
input: DeployPersonaInput,
|
||||
): DeployResult {
|
||||
assertValidCustomPersonaId(input.personaId);
|
||||
const personasDir = path.join(dataDir, 'personas');
|
||||
if (!fs.existsSync(personasDir)) {
|
||||
fs.mkdirSync(personasDir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(personasDir, `${input.personaId}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const builtin = getPersona(input.personaId);
|
||||
const persona: AgentPersona = builtin
|
||||
? {
|
||||
...builtin,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
}
|
||||
: {
|
||||
name: input.personaId,
|
||||
description: input.overrides?.description ?? `${input.personaId} (evolved)`,
|
||||
icon: input.overrides?.icon ?? 'sparkles',
|
||||
modelPreference: input.overrides?.modelPreference ?? 'claude-sonnet-4-6',
|
||||
tools: input.overrides?.tools ?? [],
|
||||
workspaceAffinity: input.overrides?.workspaceAffinity ?? [],
|
||||
suggestedCommands: input.overrides?.suggestedCommands ?? [],
|
||||
defaultWorkflow: input.overrides?.defaultWorkflow ?? null,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(persona, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll back a previously-deployed persona override to the .bak version
|
||||
* (or remove the override entirely if there was no previous version).
|
||||
*/
|
||||
export function rollbackPersonaOverride(
|
||||
dataDir: string,
|
||||
personaId: string,
|
||||
): boolean {
|
||||
assertValidCustomPersonaId(personaId);
|
||||
const filePath = path.join(dataDir, 'personas', `${personaId}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Behavioral-spec override ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* BEHAVIORAL_SPEC is compiled into the bundle, so we cannot mutate it
|
||||
* on disk without rebuilding. Instead, overrides live as JSON files in
|
||||
* {dataDir}/behavioral-overrides/ and the runtime merges them at load.
|
||||
*
|
||||
* One file per section, filename = section name (coreLoop, qualityRules,
|
||||
* behavioralRules, workPatterns, intelligenceDefaults).
|
||||
*/
|
||||
|
||||
export type BehavioralSpecSection =
|
||||
| 'coreLoop'
|
||||
| 'qualityRules'
|
||||
| 'behavioralRules'
|
||||
| 'workPatterns'
|
||||
| 'intelligenceDefaults';
|
||||
|
||||
export const BEHAVIORAL_SPEC_SECTIONS: BehavioralSpecSection[] = [
|
||||
'coreLoop',
|
||||
'qualityRules',
|
||||
'behavioralRules',
|
||||
'workPatterns',
|
||||
'intelligenceDefaults',
|
||||
];
|
||||
|
||||
export interface DeployBehavioralSpecInput {
|
||||
/** Which section to override */
|
||||
section: BehavioralSpecSection;
|
||||
/** The evolved text for that section */
|
||||
text: string;
|
||||
/** Optional evolution-run uuid for audit */
|
||||
runUuid?: string;
|
||||
}
|
||||
|
||||
export interface BehavioralSpecOverride {
|
||||
section: BehavioralSpecSection;
|
||||
text: string;
|
||||
runUuid?: string;
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a behavioral-spec section override to disk. The runtime loader
|
||||
* (loadBehavioralSpecOverrides) reads these on startup and merges them
|
||||
* into the compiled BEHAVIORAL_SPEC at runtime.
|
||||
*/
|
||||
export function deployBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
input: DeployBehavioralSpecInput,
|
||||
): DeployResult {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(input.section)) {
|
||||
throw new Error(`Unknown behavioral-spec section: ${input.section}`);
|
||||
}
|
||||
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(dir, `${input.section}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const payload: BehavioralSpecOverride = {
|
||||
section: input.section,
|
||||
text: input.text,
|
||||
runUuid: input.runUuid,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(payload, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: payload.deployedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Roll back one section (restores .bak, or removes if no previous override). */
|
||||
export function rollbackBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
section: BehavioralSpecSection,
|
||||
): boolean {
|
||||
const filePath = path.join(dataDir, 'behavioral-overrides', `${section}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Read all behavioral-spec overrides from disk as a map section → text. */
|
||||
export function loadBehavioralSpecOverrides(
|
||||
dataDir: string,
|
||||
): Partial<Record<BehavioralSpecSection, string>> {
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) return {};
|
||||
|
||||
const out: Partial<Record<BehavioralSpecSection, string>> = {};
|
||||
try {
|
||||
for (const section of BEHAVIORAL_SPEC_SECTIONS) {
|
||||
const filePath = path.join(dir, `${section}.json`);
|
||||
if (!fs.existsSync(filePath)) continue;
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||||
const parsed = JSON.parse(raw) as Partial<BehavioralSpecOverride>;
|
||||
if (parsed && typeof parsed.text === 'string' && parsed.text.length > 0) {
|
||||
out[section] = parsed.text;
|
||||
}
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
} catch { /* dir unreadable */ }
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Merge overrides into a baseline spec object, preferring the override text. */
|
||||
export function applyBehavioralSpecOverrides(
|
||||
baseline: Record<BehavioralSpecSection, string>,
|
||||
overrides: Partial<Record<BehavioralSpecSection, string>>,
|
||||
): Record<BehavioralSpecSection, string> {
|
||||
return {
|
||||
...baseline,
|
||||
...Object.fromEntries(
|
||||
(Object.entries(overrides) as Array<[BehavioralSpecSection, string]>)
|
||||
.filter(([, v]) => typeof v === 'string' && v.length > 0),
|
||||
),
|
||||
} as Record<BehavioralSpecSection, string>;
|
||||
}
|
||||
|
||||
// ── Internal ───────────────────────────────────────────────────
|
||||
|
||||
/** Atomic write — write to .tmp then rename into place. */
|
||||
function writeAtomic(filePath: string, contents: string): void {
|
||||
const tmpPath = `${filePath}.tmp`;
|
||||
fs.writeFileSync(tmpPath, contents, 'utf-8');
|
||||
const waitBuffer = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
try {
|
||||
for (let attempt = 1; attempt <= 10; attempt++) {
|
||||
try {
|
||||
fs.renameSync(tmpPath, filePath);
|
||||
return;
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
const transient = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
|
||||
if (!transient || attempt === 10) throw error;
|
||||
// Windows antivirus and indexers can briefly hold an exclusive handle.
|
||||
Atomics.wait(waitBuffer, 0, 0, 25 * attempt);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
/**
|
||||
* Evolution Gates — Phase 2.2 of the self-evolution loop.
|
||||
*
|
||||
* Gatekeeping for prompts/skills produced by the iterative GEPA optimizer
|
||||
* before they reach production. A candidate must pass ALL gates; the first
|
||||
* failure short-circuits and returns a `fail` verdict with the reason.
|
||||
*
|
||||
* Four gate categories:
|
||||
*
|
||||
* 1. Size gates — hard character caps per target type (persona prompt
|
||||
* ≤ 3 000 chars, tool description ≤ 500, skill ≤ 15 KB,
|
||||
* behavioral-spec section ≤ 4 000).
|
||||
* 2. Growth gates — percentage cap over baseline (default +20%) prevents
|
||||
* runaway prompt bloat across generations.
|
||||
* 3. Structural — non-empty, no unresolved placeholders, balanced
|
||||
* markdown fences, no obvious template accidents
|
||||
* (e.g. "[PLACEHOLDER]", "TODO:" left in).
|
||||
* 4. Regression — score delta on a held-out eval set must not drop
|
||||
* below the allowed tolerance (default -0.02, i.e. a
|
||||
* 2% accuracy regression is the worst allowed).
|
||||
*
|
||||
* Each gate returns structured reasons so the orchestrator (phase 4)
|
||||
* can log WHY a candidate was rejected and surface it in the UI.
|
||||
*/
|
||||
|
||||
import type { EvolutionTarget } from './iterative-optimizer.js';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────
|
||||
|
||||
export type GateVerdict = 'pass' | 'fail';
|
||||
|
||||
export interface GateResult {
|
||||
gate: string;
|
||||
verdict: GateVerdict;
|
||||
reason: string;
|
||||
/** Optional measurable detail: e.g. `{ current: 3520, max: 3000 }` */
|
||||
detail?: Record<string, number | string>;
|
||||
}
|
||||
|
||||
export interface GateCheckResult {
|
||||
verdict: GateVerdict;
|
||||
/** First failing gate, or null if all passed */
|
||||
firstFailure: GateResult | null;
|
||||
/** All gate results in order */
|
||||
results: GateResult[];
|
||||
}
|
||||
|
||||
export interface SizeLimits {
|
||||
personaSystemPrompt: number;
|
||||
toolDescription: number;
|
||||
skillBody: number;
|
||||
behavioralSpecSection: number;
|
||||
generic: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_SIZE_LIMITS: SizeLimits = {
|
||||
personaSystemPrompt: 3_000,
|
||||
toolDescription: 500,
|
||||
skillBody: 15_000,
|
||||
behavioralSpecSection: 4_000,
|
||||
generic: 8_000,
|
||||
};
|
||||
|
||||
export interface GateOptions {
|
||||
/** What is being evolved — drives which size cap applies. Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Max allowed % growth over baseline. Default 0.2 (i.e. +20%). */
|
||||
maxGrowthRatio?: number;
|
||||
/** Largest allowed accuracy regression vs baseline. Default -0.02 (i.e. -2%). */
|
||||
maxRegression?: number;
|
||||
/** Override the default size limits */
|
||||
sizeLimits?: Partial<SizeLimits>;
|
||||
/** Allow empty candidates (rarely useful — default false) */
|
||||
allowEmpty?: boolean;
|
||||
}
|
||||
|
||||
export interface CheckInput {
|
||||
/** The new candidate text */
|
||||
candidate: string;
|
||||
/** The baseline (pre-evolution) text — used for growth + regression checks */
|
||||
baseline: string;
|
||||
/** Optional scores to enable the regression gate */
|
||||
scores?: {
|
||||
baseline: number;
|
||||
candidate: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run all gates in order. Returns a structured verdict. The overall
|
||||
* verdict is `pass` iff every gate passed; `fail` with `firstFailure`
|
||||
* populated otherwise.
|
||||
*
|
||||
* Callers can choose to short-circuit on the first failure or to
|
||||
* collect all of them — `runGates` returns every gate result either way.
|
||||
*/
|
||||
export function runGates(input: CheckInput, options: GateOptions = {}): GateCheckResult {
|
||||
const targetKind = options.targetKind ?? 'generic';
|
||||
const sizeLimits = { ...DEFAULT_SIZE_LIMITS, ...options.sizeLimits };
|
||||
const maxGrowthRatio = options.maxGrowthRatio ?? 0.2;
|
||||
const maxRegression = options.maxRegression ?? -0.02;
|
||||
const allowEmpty = options.allowEmpty ?? false;
|
||||
|
||||
const results: GateResult[] = [];
|
||||
|
||||
// 1. Non-empty
|
||||
if (!allowEmpty) {
|
||||
results.push(checkNonEmpty(input.candidate));
|
||||
}
|
||||
|
||||
// 2. Size
|
||||
results.push(checkSize(input.candidate, targetKind, sizeLimits));
|
||||
|
||||
// 3. Growth
|
||||
results.push(checkGrowth(input.candidate, input.baseline, maxGrowthRatio));
|
||||
|
||||
// 4. Structural integrity
|
||||
results.push(checkBalancedFences(input.candidate));
|
||||
results.push(checkNoPlaceholders(input.candidate));
|
||||
results.push(checkNoObviousTodos(input.candidate));
|
||||
|
||||
// 5. Regression (optional — skipped if scores not provided)
|
||||
if (input.scores) {
|
||||
results.push(checkRegression(input.scores.baseline, input.scores.candidate, maxRegression));
|
||||
}
|
||||
|
||||
const firstFailure = results.find(r => r.verdict === 'fail') ?? null;
|
||||
return {
|
||||
verdict: firstFailure ? 'fail' : 'pass',
|
||||
firstFailure,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Individual gates (exported for unit tests + fine-grained use) ──
|
||||
|
||||
export function checkNonEmpty(candidate: string): GateResult {
|
||||
const trimmed = candidate.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return {
|
||||
gate: 'non-empty',
|
||||
verdict: 'fail',
|
||||
reason: 'Candidate is empty or whitespace only',
|
||||
};
|
||||
}
|
||||
return { gate: 'non-empty', verdict: 'pass', reason: 'non-empty' };
|
||||
}
|
||||
|
||||
export function checkSize(
|
||||
candidate: string,
|
||||
targetKind: EvolutionTarget,
|
||||
limits: SizeLimits,
|
||||
): GateResult {
|
||||
const max = resolveSizeLimit(targetKind, limits);
|
||||
const len = candidate.length;
|
||||
if (len > max) {
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate is ${len} chars but ${targetKind} cap is ${max}`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'pass',
|
||||
reason: `Within ${targetKind} limit (${len}/${max})`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkGrowth(
|
||||
candidate: string,
|
||||
baseline: string,
|
||||
maxGrowthRatio: number,
|
||||
): GateResult {
|
||||
// A candidate shorter than baseline always passes the growth gate.
|
||||
if (candidate.length <= baseline.length) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'Candidate is not larger than baseline',
|
||||
detail: { baseline: baseline.length, candidate: candidate.length },
|
||||
};
|
||||
}
|
||||
|
||||
// Empty baseline means any candidate can't be measured as a ratio —
|
||||
// defer to the size gate instead.
|
||||
if (baseline.length === 0) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'No baseline to compare against',
|
||||
};
|
||||
}
|
||||
|
||||
const ratio = (candidate.length - baseline.length) / baseline.length;
|
||||
if (ratio > maxGrowthRatio) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate grew ${(ratio * 100).toFixed(1)}% over baseline (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: `Grew ${(ratio * 100).toFixed(1)}% (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkBalancedFences(candidate: string): GateResult {
|
||||
const fenceMatches = candidate.match(/```/g);
|
||||
const count = fenceMatches?.length ?? 0;
|
||||
if (count % 2 !== 0) {
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'fail',
|
||||
reason: `Odd number of markdown code fences (${count}) — unbalanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'pass',
|
||||
reason: `${count} fence(s), balanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates that still contain placeholder text like `[TODO]`,
|
||||
* `<placeholder>`, or `{{var}}` — these are almost always unfinished
|
||||
* generations.
|
||||
*/
|
||||
export function checkNoPlaceholders(candidate: string): GateResult {
|
||||
const patterns: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'bracket-placeholder', re: /\[(?:PLACEHOLDER|TODO|FIXME|INSERT|XXX|YOUR[_ ][A-Z ]+)\]/i },
|
||||
{ name: 'angle-placeholder', re: /<(?:placeholder|todo|your[-_ ][a-z ]+)>/i },
|
||||
{ name: 'handlebars-placeholder', re: /\{\{[a-zA-Z_][a-zA-Z0-9_.]*\}\}/ },
|
||||
];
|
||||
|
||||
for (const { name, re } of patterns) {
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-placeholders',
|
||||
verdict: 'fail',
|
||||
reason: `Found unresolved placeholder (${name}): "${m[0]}"`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { gate: 'no-placeholders', verdict: 'pass', reason: 'no unresolved placeholders' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates with "TODO:" or "FIXME:" headers — a prompt shouldn't
|
||||
* ship with unfinished author notes.
|
||||
*/
|
||||
export function checkNoObviousTodos(candidate: string): GateResult {
|
||||
// Only match lines that START with the marker (as a label), not prose
|
||||
// that happens to mention "todo" (e.g. a task-list prompt).
|
||||
const re = /(?:^|\n)\s*(?:TODO|FIXME|XXX)\s*:/i;
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-todos',
|
||||
verdict: 'fail',
|
||||
reason: `Found leftover author marker: "${m[0].trim()}"`,
|
||||
};
|
||||
}
|
||||
return { gate: 'no-todos', verdict: 'pass', reason: 'no leftover author markers' };
|
||||
}
|
||||
|
||||
export function checkRegression(
|
||||
baselineScore: number,
|
||||
candidateScore: number,
|
||||
maxRegression: number,
|
||||
): GateResult {
|
||||
const delta = candidateScore - baselineScore;
|
||||
// Small epsilon avoids float-precision false negatives when delta is
|
||||
// numerically equal to the tolerance (e.g. 0.78 - 0.8 === -0.02000002).
|
||||
const epsilon = 1e-9;
|
||||
if (delta < maxRegression - epsilon) {
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate score regressed by ${(delta * 100).toFixed(2)}pp (allowed ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'pass',
|
||||
reason: `Score delta ${(delta * 100).toFixed(2)}pp (allowed floor ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function resolveSizeLimit(target: EvolutionTarget, limits: SizeLimits): number {
|
||||
switch (target) {
|
||||
case 'persona-system-prompt': return limits.personaSystemPrompt;
|
||||
case 'tool-description': return limits.toolDescription;
|
||||
case 'skill-body': return limits.skillBody;
|
||||
case 'behavioral-spec-section': return limits.behavioralSpecSection;
|
||||
case 'generic':
|
||||
default:
|
||||
return limits.generic;
|
||||
}
|
||||
}
|
||||
@@ -1,43 +1,29 @@
|
||||
/**
|
||||
* Evolution LLM Wiring — binds real LLMs into the self-evolution loop.
|
||||
* LLM wiring for the skill audit and skill hygiene routes.
|
||||
*
|
||||
* The evolution primitives (judge, iterative-optimizer, evolve-schema) are
|
||||
* deliberately model-agnostic: they accept plain callables. This module
|
||||
* supplies production adapters that turn a Haiku-class LLM into those
|
||||
* callables, so the `/api/evolution/run` endpoint can perform a real run.
|
||||
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
|
||||
* This module supplies the production adapter that turns a Haiku-class LLM
|
||||
* into that callable, plus a retry wrapper for transient provider errors.
|
||||
*
|
||||
* Public surface:
|
||||
*
|
||||
* createAnthropicEvolutionLLM(apiKey, model?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax — same pattern as
|
||||
* `optimizer-service.ts`. Dynamic import to avoid startup cost when
|
||||
* evolution is not in use.
|
||||
* createAnthropicEvolutionLLM(apiKey, options?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
|
||||
* avoid startup cost when no audit runs.
|
||||
*
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
* buildGEPAMutateFn(llm) → MutateFn (for IterativeGEPA)
|
||||
* buildSchemaExecuteFn(llm) → SchemaExecuteFn (for EvolveSchema)
|
||||
* makeRunningJudge(base, llm) → Pick<LLMJudge,'score'>
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
*
|
||||
* The `makeRunningJudge` wrapper is what enables GEPA to evaluate real
|
||||
* prompt effectiveness: on each score() call it first executes the
|
||||
* candidate prompt against the example input via the LLM, then delegates
|
||||
* to the wrapped judge with the model's actual response. Without this,
|
||||
* GEPA compares prompt text to expected output directly — less useful.
|
||||
*
|
||||
* All adapters are pure wrappers over `EvolutionLLM.complete()`, which is a
|
||||
* minimal `(prompt: string) => Promise<string>` contract. Tests supply
|
||||
* in-memory mocks of that contract; production code goes through AxAI.
|
||||
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
|
||||
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
|
||||
* mocks of that contract; production code goes through AxAI.
|
||||
*/
|
||||
|
||||
import type { JudgeLLMCall, JudgeInput, JudgeScore } from './judge.js';
|
||||
import type { LLMJudge } from './judge.js';
|
||||
import type { MutateArgs, MutateFn, EvolutionTarget } from './iterative-optimizer.js';
|
||||
import type { SchemaExecuteFn, Schema, SchemaField } from './evolve-schema.js';
|
||||
import type { JudgeLLMCall } from './judge.js';
|
||||
|
||||
// ── Minimal LLM contract ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A minimal single-turn completion contract. All evolution adapters depend
|
||||
* A minimal single-turn completion contract. All adapters depend
|
||||
* on this interface rather than on @ax-llm/ax directly so they stay easy to
|
||||
* mock in unit tests.
|
||||
*/
|
||||
@@ -206,10 +192,10 @@ export interface CreateAnthropicEvolutionLLMOptions {
|
||||
|
||||
/**
|
||||
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
|
||||
* @ax-llm/ax so evolution is a zero-cost dependency until used.
|
||||
* @ax-llm/ax so it is a zero-cost dependency until used.
|
||||
*
|
||||
* Returns null when @ax-llm/ax is unavailable — callers should treat null
|
||||
* as "evolution disabled" and surface a clear error to the user.
|
||||
* as "LLM unavailable" and surface a clear error to the user.
|
||||
*/
|
||||
export async function createAnthropicEvolutionLLM(
|
||||
apiKey: string,
|
||||
@@ -240,8 +226,8 @@ export async function createAnthropicEvolutionLLM(
|
||||
const response = await ai.chat({
|
||||
chatPrompt: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
// chat() may return a ReadableStream in streaming mode — evolution
|
||||
// does not stream, so we only handle the non-stream case.
|
||||
// chat() may return a ReadableStream in streaming mode — audits
|
||||
// do not stream, so we only handle the non-stream case.
|
||||
const typed = response as {
|
||||
results?: readonly { content?: string }[];
|
||||
};
|
||||
@@ -249,7 +235,7 @@ export async function createAnthropicEvolutionLLM(
|
||||
return typeof first === 'string' ? first : '';
|
||||
},
|
||||
};
|
||||
// Apply retry policy — defaults are sized for background evolution runs
|
||||
// Apply retry policy — defaults are sized for background runs
|
||||
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
|
||||
// by passing `retry: { maxAttempts: 1 }`.
|
||||
return wrapWithRetry(inner, options.retry ?? {});
|
||||
@@ -268,217 +254,3 @@ export async function createAnthropicEvolutionLLM(
|
||||
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
|
||||
return (prompt: string) => llm.complete(prompt);
|
||||
}
|
||||
|
||||
// ── Reflective mutation (GEPA) ──────────────────────────────────
|
||||
|
||||
export interface BuildReflectiveMutationPromptArgs {
|
||||
parent: string;
|
||||
strategy: string;
|
||||
weaknessFeedback: string[];
|
||||
targetKind: EvolutionTarget;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reflective mutation prompt for GEPA. Returns the *mutated child*
|
||||
* text only — no commentary, no wrapping — per the instruction in the
|
||||
* prompt body. The LLM is told what to fix, using the weakness signal from
|
||||
* the parent's worst-scoring eval examples.
|
||||
*/
|
||||
export function buildReflectiveMutationPrompt(
|
||||
args: BuildReflectiveMutationPromptArgs,
|
||||
): string {
|
||||
const weaknessBullets = args.weaknessFeedback.length > 0
|
||||
? args.weaknessFeedback.map(fb => `- ${fb}`).join('\n')
|
||||
: '(no specific weakness signals yet)';
|
||||
|
||||
return `You are evolving an AI prompt. This is generation ${args.generation}.
|
||||
|
||||
TARGET KIND: ${args.targetKind}
|
||||
STRATEGY: ${args.strategy}
|
||||
|
||||
PARENT PROMPT (do not return this unchanged — produce a *better* variant):
|
||||
---
|
||||
${args.parent}
|
||||
---
|
||||
|
||||
WEAKNESS SIGNALS from the parent's lowest-scoring eval examples:
|
||||
${weaknessBullets}
|
||||
|
||||
Apply the strategy above to address the weakness signals. Keep the same
|
||||
role and overall structure — do not change the prompt's purpose. Produce
|
||||
a concrete variant that should score higher on the next eval.
|
||||
|
||||
Return ONLY the new prompt text. No preamble, no explanation, no markdown
|
||||
fences. Plain text only.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a GEPA MutateFn that uses an LLM to produce reflective mutations.
|
||||
* Graceful fallback: if the LLM throws or returns empty, returns the
|
||||
* parent prompt unchanged (GEPA will still evaluate it as part of the
|
||||
* Pareto population — no progress that generation, but no crash).
|
||||
*/
|
||||
export function buildGEPAMutateFn(llm: EvolutionLLM): MutateFn {
|
||||
return async (args: MutateArgs): Promise<string> => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: args.parent.prompt,
|
||||
strategy: args.strategy,
|
||||
weaknessFeedback: args.weaknessFeedback,
|
||||
targetKind: args.targetKind,
|
||||
generation: args.generation,
|
||||
});
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(prompt);
|
||||
} catch {
|
||||
return args.parent.prompt;
|
||||
}
|
||||
const cleaned = stripFences(raw).trim();
|
||||
if (cleaned.length === 0) return args.parent.prompt;
|
||||
return cleaned;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Schema execute (ES) ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a schema-filling prompt that asks the LLM to return JSON matching
|
||||
* the given schema for the given user input. Every field is documented by
|
||||
* name + type + description + required flag + constraints.
|
||||
*/
|
||||
export function buildSchemaFillPrompt(args: { schema: Schema; input: string }): string {
|
||||
const fieldLines = args.schema.fields.map(f => formatField(f)).join('\n');
|
||||
return `Fill the JSON schema below using the user input. Respond with
|
||||
valid JSON — no markdown, no prose, no trailing commas.
|
||||
|
||||
SCHEMA (name: ${args.schema.name}, version: ${args.schema.version}):
|
||||
${fieldLines}
|
||||
|
||||
USER INPUT:
|
||||
${args.input}
|
||||
|
||||
Return ONLY the JSON object.`;
|
||||
}
|
||||
|
||||
function formatField(field: SchemaField): string {
|
||||
const req = field.required ? 'required' : 'optional';
|
||||
const constraints = field.constraints.length > 0
|
||||
? ' constraints: ' + field.constraints.map(c => `${c.kind}=${stringifyConstraintValue(c.value)}`).join(', ')
|
||||
: '';
|
||||
return `- "${field.name}" (${field.type}, ${req}): ${field.description}${constraints}`;
|
||||
}
|
||||
|
||||
function stringifyConstraintValue(value: string | number | string[]): string {
|
||||
if (Array.isArray(value)) return `[${value.join(',')}]`;
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a SchemaExecuteFn that asks the LLM to fill a schema. Returns
|
||||
* `parsed: true` only when the response parses as JSON. On any error
|
||||
* (network, timeout, bad model), returns `{actual: '', parsed: false}`.
|
||||
*/
|
||||
export function buildSchemaExecuteFn(llm: EvolutionLLM): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(buildSchemaFillPrompt({ schema, input }));
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
const cleaned = stripFences(raw);
|
||||
return { actual: cleaned, parsed: looksLikeJSON(cleaned) };
|
||||
};
|
||||
}
|
||||
|
||||
// ── Running judge (executes candidate prompt first, then scores) ─
|
||||
|
||||
/**
|
||||
* Wrap a base judge so that each score() call first runs the candidate
|
||||
* prompt through the LLM on the example's input. The base judge then
|
||||
* receives the LLM's output as `actual`, giving a real behavioral signal
|
||||
* instead of a prompt-text-to-expected-output comparison.
|
||||
*
|
||||
* When the LLM call fails, returns a zero score with clear feedback so the
|
||||
* Pareto population treats the candidate as a loser without crashing the
|
||||
* whole generation.
|
||||
*/
|
||||
/**
|
||||
* Phantom property that marks a judge as running-capable (i.e. it executes
|
||||
* the candidate prompt against a real LLM before scoring). IterativeGEPA
|
||||
* checks for this to prevent regressions where a bare prompt-text-similarity
|
||||
* judge is passed by mistake.
|
||||
*
|
||||
* Symbol-keyed to avoid accidental conflicts with user-defined fields.
|
||||
*/
|
||||
export const RUNNING_JUDGE_BRAND = Symbol.for('waggle.runningJudge');
|
||||
|
||||
export interface RunningJudge extends Pick<LLMJudge, 'score'> {
|
||||
readonly [RUNNING_JUDGE_BRAND]: true;
|
||||
}
|
||||
|
||||
export function isRunningJudge(j: Pick<LLMJudge, 'score'> | RunningJudge): j is RunningJudge {
|
||||
return (j as Record<symbol, unknown>)[RUNNING_JUDGE_BRAND] === true;
|
||||
}
|
||||
|
||||
export function makeRunningJudge(
|
||||
baseJudge: Pick<LLMJudge, 'score'>,
|
||||
llm: EvolutionLLM,
|
||||
): RunningJudge {
|
||||
return {
|
||||
[RUNNING_JUDGE_BRAND]: true as const,
|
||||
async score(args: JudgeInput): Promise<JudgeScore> {
|
||||
// `args.actual` is the candidate prompt supplied by GEPA. Treat it as
|
||||
// a system prompt, run it against the example input via the LLM, and
|
||||
// use the LLM's reply as the new `actual` for the downstream judge.
|
||||
const runPrompt = buildRunPrompt(args.actual, args.input);
|
||||
let modelOutput = '';
|
||||
try {
|
||||
modelOutput = await llm.complete(runPrompt);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
overall: 0, weighted: 0,
|
||||
correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1,
|
||||
feedback: `Candidate execution failed: ${msg}`,
|
||||
parsed: false,
|
||||
};
|
||||
}
|
||||
return baseJudge.score({ ...args, actual: modelOutput });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildRunPrompt(candidatePrompt: string, userInput: string): string {
|
||||
return `${candidatePrompt}
|
||||
|
||||
USER INPUT:
|
||||
${userInput}
|
||||
|
||||
Respond now.`;
|
||||
}
|
||||
|
||||
// ── Shared helpers ──────────────────────────────────────────────
|
||||
|
||||
/** Strip ``` and ```json fences if present. Never throws. */
|
||||
function stripFences(raw: string): string {
|
||||
if (!raw) return '';
|
||||
return raw
|
||||
.replace(/```json\s*/gi, '')
|
||||
.replace(/```\s*/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function looksLikeJSON(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return false;
|
||||
try {
|
||||
JSON.parse(trimmed);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
/**
|
||||
* Evolution Orchestrator — Phase 4 of the self-evolution loop.
|
||||
*
|
||||
* Wires the primitives into a closed loop:
|
||||
*
|
||||
* traces → eval dataset → ComposeEvolution → gates → run store
|
||||
* ↓
|
||||
* user accepts/rejects
|
||||
* ↓
|
||||
* deploy callback fires
|
||||
* ↓
|
||||
* runStore.markDeployed()
|
||||
*
|
||||
* The orchestrator itself does NOT hot-reload personas or rewrite the
|
||||
* behavioral spec — it calls a pluggable `deploy` callback supplied by
|
||||
* the caller. This keeps the orchestrator pure (easy to unit-test) and
|
||||
* lets the server/desktop app wire up the real side-effects (file write,
|
||||
* persona reload, event emission) separately.
|
||||
*
|
||||
* Persistence is in @waggle/core's `EvolutionRunStore`. Every run — even
|
||||
* rejected or failed — is preserved for audit.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
EvolutionRun,
|
||||
EvolutionRunStore,
|
||||
EvolutionRunStatus,
|
||||
EvolutionRunTarget,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
} from '@waggle/core';
|
||||
import { ComposeEvolution, type ComposeEvolutionOptions, type ComposeEvolutionResult } from './compose-evolution.js';
|
||||
import { runGates, type GateResult, type GateOptions } from './evolution-gates.js';
|
||||
import { EvalDatasetBuilder, type EvalExample } from './eval-dataset.js';
|
||||
|
||||
// ── Dependency injection types ─────────────────────────────────
|
||||
|
||||
export interface EvolutionOrchestratorDeps {
|
||||
/** Source of traces (from @waggle/core) */
|
||||
traceStore: ExecutionTraceStore;
|
||||
/** Persistent run history (from @waggle/core) */
|
||||
runStore: EvolutionRunStore;
|
||||
/**
|
||||
* Function invoked when a run moves from accepted → (deployed | failed).
|
||||
* Returning normally marks the run deployed; throwing marks it failed
|
||||
* with the error message as reason.
|
||||
*
|
||||
* This is where the server writes to the persona file, reloads the
|
||||
* orchestrator cache, updates behavioral-spec.ts, etc.
|
||||
*/
|
||||
deploy?: (run: EvolutionRun) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface EvolutionAutoTriggerConfig {
|
||||
/** Minimum traces (with the given outcome filter) before a run is triggered */
|
||||
minTraces: number;
|
||||
/** Restrict trigger to a specific persona / workspace / taskShape */
|
||||
traceFilter?: {
|
||||
personaId?: string;
|
||||
workspaceId?: string;
|
||||
taskShape?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EvolutionOrchestratorOptions {
|
||||
/** What is being evolved (drives gate policy + persistence tag) */
|
||||
targetKind: EvolutionRunTarget;
|
||||
/** Stable identifier — e.g. persona id, spec-section name */
|
||||
targetName?: string;
|
||||
/** The current baseline text to evolve against */
|
||||
baseline: string;
|
||||
/** Compose options — already wired with examples, judge, execute, mutate */
|
||||
compose: Omit<ComposeEvolutionOptions, 'schema' | 'instructions'> & {
|
||||
schema: Omit<ComposeEvolutionOptions['schema'], 'baseline'>;
|
||||
instructions: Omit<ComposeEvolutionOptions['instructions'], 'baseline'>;
|
||||
};
|
||||
/** Optional gate overrides (size caps, growth ratio, etc) */
|
||||
gateOptions?: GateOptions;
|
||||
/** Improvement threshold required to create a proposal (default 0.02) */
|
||||
minDelta?: number;
|
||||
/** Auto-trigger configuration — when omitted, runOnce() is manual-only */
|
||||
autoTrigger?: EvolutionAutoTriggerConfig;
|
||||
/** Abort the run early */
|
||||
signal?: AbortSignal;
|
||||
/** Optional progress pass-through */
|
||||
onProgress?: (event: EvolutionProgress) => void;
|
||||
}
|
||||
|
||||
export interface EvolutionProgress {
|
||||
phase: 'trigger-check' | 'dataset' | 'compose' | 'gates' | 'persist' | 'skipped' | 'done';
|
||||
message?: string;
|
||||
detail?: unknown;
|
||||
}
|
||||
|
||||
// ── Schema baseline input ──────────────────────────────────────
|
||||
|
||||
/** Optional structural baseline when the target has one (e.g. persona's DSPy signature). */
|
||||
export interface SchemaBaselineInput {
|
||||
baseline: ComposeEvolutionOptions['schema']['baseline'];
|
||||
}
|
||||
|
||||
// ── Run result ─────────────────────────────────────────────────
|
||||
|
||||
export type OrchestratorOutcome =
|
||||
| 'proposed' // Run created, awaiting accept/reject
|
||||
| 'skipped-trigger'
|
||||
| 'skipped-gates' // Completed but gates failed
|
||||
| 'skipped-delta' // Improvement below minDelta
|
||||
| 'aborted';
|
||||
|
||||
export interface OrchestratorRunResult {
|
||||
outcome: OrchestratorOutcome;
|
||||
/** Populated for 'proposed' and 'skipped-gates' */
|
||||
run?: EvolutionRun;
|
||||
/** Raw ComposeEvolution result when the compose stage ran */
|
||||
compose?: ComposeEvolutionResult;
|
||||
/** Gate results for audit */
|
||||
gateResults?: GateResult[];
|
||||
/** Reason if outcome is skipped/aborted */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Orchestrator ──────────────────────────────────────────────
|
||||
|
||||
export class EvolutionOrchestrator {
|
||||
private deps: EvolutionOrchestratorDeps;
|
||||
|
||||
constructor(deps: EvolutionOrchestratorDeps) {
|
||||
this.deps = deps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full orchestration pipeline once. Returns structured outcome.
|
||||
* Does NOT deploy — that happens when the caller calls `accept(uuid)`.
|
||||
*/
|
||||
async runOnce(options: EvolutionOrchestratorOptions & {
|
||||
schemaBaseline: SchemaBaselineInput['baseline'];
|
||||
}): Promise<OrchestratorRunResult> {
|
||||
const emit = (phase: EvolutionProgress['phase'], message?: string, detail?: unknown) => {
|
||||
options.onProgress?.({ phase, message, detail });
|
||||
};
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted before start' };
|
||||
}
|
||||
|
||||
// Trigger check — skip if auto-trigger thresholds aren't met.
|
||||
if (options.autoTrigger) {
|
||||
emit('trigger-check');
|
||||
const ok = this.meetsAutoTrigger(options.autoTrigger);
|
||||
if (!ok) {
|
||||
emit('skipped');
|
||||
return {
|
||||
outcome: 'skipped-trigger',
|
||||
reason: `fewer than ${options.autoTrigger.minTraces} eligible traces`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Derive the eval dataset from finalized traces of the requested target.
|
||||
emit('dataset');
|
||||
const examples = this.buildExamplesFromTraces(options);
|
||||
if (examples.length === 0) {
|
||||
emit('skipped', 'no eval examples');
|
||||
return { outcome: 'skipped-trigger', reason: 'no eligible traces to form dataset' };
|
||||
}
|
||||
|
||||
// Run the compose pipeline.
|
||||
emit('compose');
|
||||
const composeResult = await new ComposeEvolution().run({
|
||||
schema: { ...options.compose.schema, baseline: options.schemaBaseline, examples },
|
||||
instructions: { ...options.compose.instructions, baseline: options.baseline, examples },
|
||||
feedbackFilter: options.compose.feedbackFilter,
|
||||
signal: options.signal,
|
||||
onProgress: (e) => emit('compose', undefined, e),
|
||||
});
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted during compose', compose: composeResult };
|
||||
}
|
||||
|
||||
const winnerText = composeResult.instructions.winner.prompt;
|
||||
// Apples-to-apples: compare the GEPA baseline candidate (history[0])
|
||||
// to the GEPA winner. Both are scored by the same judge on the same
|
||||
// eval stage, so the delta is a clean signal.
|
||||
const baselineOverall = composeResult.instructions.history[0]?.score?.overall ?? 0;
|
||||
const winnerOverall = composeResult.instructions.winner.score?.overall ?? 0;
|
||||
const delta = winnerOverall - baselineOverall;
|
||||
const minDelta = options.minDelta ?? 0.02;
|
||||
|
||||
// Gate check.
|
||||
emit('gates');
|
||||
const gateResult = runGates(
|
||||
{
|
||||
candidate: winnerText,
|
||||
baseline: options.baseline,
|
||||
scores: { baseline: baselineOverall, candidate: winnerOverall },
|
||||
},
|
||||
{
|
||||
targetKind: options.targetKind,
|
||||
...options.gateOptions,
|
||||
},
|
||||
);
|
||||
|
||||
if (delta < minDelta) {
|
||||
emit('skipped', `delta ${(delta * 100).toFixed(2)}pp below minimum ${(minDelta * 100).toFixed(2)}pp`);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-delta',
|
||||
reason: `winner improved only ${(delta * 100).toFixed(2)}pp (min ${(minDelta * 100).toFixed(2)}pp required)`,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
// Persist — create the proposed run even if gates failed, so there's audit.
|
||||
emit('persist');
|
||||
const run = this.deps.runStore.create({
|
||||
targetKind: options.targetKind,
|
||||
targetName: options.targetName ?? null,
|
||||
baselineText: options.baseline,
|
||||
winnerText,
|
||||
winnerSchema: composeResult.frozenSchema,
|
||||
deltaAccuracy: delta,
|
||||
gateVerdict: gateResult.verdict,
|
||||
gateReasons: gateResult.results.map(r => ({
|
||||
gate: r.gate, verdict: r.verdict, reason: r.reason,
|
||||
})),
|
||||
artifacts: {
|
||||
runSeed: options.compose.instructions.seed ?? null,
|
||||
generations: options.compose.instructions.generations ?? null,
|
||||
paretoFrontSize: composeResult.schema.paretoFront.length,
|
||||
exampleCount: examples.length,
|
||||
},
|
||||
});
|
||||
|
||||
if (gateResult.verdict === 'fail') {
|
||||
// Immediately reject runs that fail gates so the user doesn't see
|
||||
// dangerous candidates in the "review" queue. Run history is kept.
|
||||
const rejected = this.deps.runStore.reject(
|
||||
run.run_uuid,
|
||||
`gate failure: ${gateResult.firstFailure?.reason ?? 'unknown'}`,
|
||||
);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-gates',
|
||||
reason: gateResult.firstFailure?.reason ?? 'gate failure',
|
||||
run: rejected ?? run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'proposed',
|
||||
run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
/** Accept a proposed run and invoke the deploy callback. */
|
||||
async accept(runUuid: string, userNote?: string): Promise<EvolutionRun | undefined> {
|
||||
const accepted = this.deps.runStore.accept(runUuid, userNote);
|
||||
if (!accepted || accepted.status !== 'accepted') return accepted;
|
||||
|
||||
if (!this.deps.deploy) {
|
||||
// No deploy hook configured — leave as 'accepted' and let the caller
|
||||
// mark deployed manually.
|
||||
return accepted;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.deps.deploy(accepted);
|
||||
return this.deps.runStore.markDeployed(runUuid);
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
return this.deps.runStore.markFailed(runUuid, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reject a proposed run with optional reason. */
|
||||
reject(runUuid: string, reason?: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.reject(runUuid, reason);
|
||||
}
|
||||
|
||||
/** List runs (thin pass-through, handy for the UI). */
|
||||
list(filter?: Parameters<EvolutionRunStore['list']>[0]): EvolutionRun[] {
|
||||
return this.deps.runStore.list(filter);
|
||||
}
|
||||
|
||||
/** Get a single run by uuid. */
|
||||
get(runUuid: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.getByUuid(runUuid);
|
||||
}
|
||||
|
||||
// ── Internals ───────────────────────────────────────────────
|
||||
|
||||
private meetsAutoTrigger(trigger: EvolutionAutoTriggerConfig): boolean {
|
||||
const outcomes: TraceOutcome[] = ['success', 'verified', 'corrected'];
|
||||
const traces = this.deps.traceStore.query({
|
||||
...(trigger.traceFilter ?? {}),
|
||||
outcome: outcomes,
|
||||
limit: Math.max(trigger.minTraces * 2, 50),
|
||||
});
|
||||
return traces.length >= trigger.minTraces;
|
||||
}
|
||||
|
||||
private buildExamplesFromTraces(
|
||||
options: EvolutionOrchestratorOptions,
|
||||
): EvalExample[] {
|
||||
// If the caller already supplied examples via compose.schema/instructions,
|
||||
// we respect that. Otherwise we mine them from the trace store.
|
||||
const existing = options.compose.schema.examples ?? options.compose.instructions.examples;
|
||||
if (existing && existing.length > 0) return existing;
|
||||
|
||||
const builder = new EvalDatasetBuilder(this.deps.traceStore);
|
||||
const traceFilter = options.autoTrigger?.traceFilter ?? {};
|
||||
const traces = builder.sourceFromTraces(['success', 'verified'], true, {
|
||||
...traceFilter,
|
||||
limit: 500,
|
||||
});
|
||||
return traces;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Convenience helpers ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Filter a list of parsed traces down to those likely useful for evolution.
|
||||
* Keeps finalized (non-pending) traces with non-empty I/O. Used by callers
|
||||
* that want to manually pre-filter before handing to the orchestrator.
|
||||
*/
|
||||
export function eligibleForEvolution(traces: ParsedExecutionTrace[]): ParsedExecutionTrace[] {
|
||||
return traces.filter(t =>
|
||||
t.outcome !== 'pending' &&
|
||||
t.payload.input.trim().length > 0 &&
|
||||
(t.payload.output.trim().length > 0 || t.payload.correctionFeedback),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper that summarizes runs by status/target for dashboard display.
|
||||
*/
|
||||
export function summarizeRuns(runs: EvolutionRun[]): {
|
||||
total: number;
|
||||
byStatus: Record<EvolutionRunStatus, number>;
|
||||
byTargetKind: Record<string, number>;
|
||||
bestDelta: number;
|
||||
} {
|
||||
const byStatus: Record<EvolutionRunStatus, number> = {
|
||||
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
|
||||
};
|
||||
const byTargetKind: Record<string, number> = {};
|
||||
let bestDelta = 0;
|
||||
|
||||
for (const r of runs) {
|
||||
byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
|
||||
byTargetKind[r.target_kind] = (byTargetKind[r.target_kind] ?? 0) + 1;
|
||||
if (r.delta_accuracy > bestDelta) bestDelta = r.delta_accuracy;
|
||||
}
|
||||
|
||||
return {
|
||||
total: runs.length,
|
||||
byStatus,
|
||||
byTargetKind,
|
||||
bestDelta,
|
||||
};
|
||||
}
|
||||
@@ -1,852 +0,0 @@
|
||||
/**
|
||||
* EvolveSchema — Phase 3.1 of the self-evolution loop.
|
||||
*
|
||||
* Port of Mikhail Pavlukhin's EvolveSchema algorithm (Egzakta research).
|
||||
* Evolves the STRUCTURE of a typed output schema (DSPy-style signatures):
|
||||
* field names, types, descriptions, ordering, and constraints. This is
|
||||
* orthogonal to the GEPA optimizer which evolves INSTRUCTIONS against a
|
||||
* fixed schema.
|
||||
*
|
||||
* Three-phase per-generation loop:
|
||||
*
|
||||
* Phase A — Structure Discovery
|
||||
* Spawn variants that add, replace, or drop output fields. This is
|
||||
* the single biggest-impact mutation class (Mikhail's paper shows
|
||||
* ~74% of HotPotQA gains come from one structural mutation).
|
||||
*
|
||||
* Phase B — Field-Order Probes
|
||||
* Permute field order. LLMs condition on earlier fields when filling
|
||||
* later ones, so putting a "reasoning" field BEFORE "answer" routinely
|
||||
* outperforms the reverse.
|
||||
*
|
||||
* Phase C — Failure-Driven Refinement
|
||||
* Use the per-example judge feedback from the previous generation to
|
||||
* target specific fields — edit their descriptions, tighten / loosen
|
||||
* their constraints, or change their types where signals suggest.
|
||||
*
|
||||
* 8 typed mutations are available:
|
||||
* add_output_field, remove_field, edit_field_desc, change_field_type,
|
||||
* add_constraint, remove_constraint, reorder_fields, replace_output_fields
|
||||
*
|
||||
* Pareto selection is 2-dimensional: (accuracy, -complexity) — an
|
||||
* accurate-but-complex schema is not automatically preferred over a
|
||||
* slightly-less-accurate simpler one.
|
||||
*
|
||||
* Callers provide:
|
||||
* - `execute(schema, input) → actualOutput` the LLM runner
|
||||
* - `judge.score({input, expected, actual})` the LLMJudge from Phase 1.3
|
||||
* - `mutate` (optional) a custom LLM-driven mutator
|
||||
* for edit_field_desc; falls
|
||||
* back to a deterministic
|
||||
* rewriter when omitted.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
|
||||
// ── Schema types ───────────────────────────────────────────────
|
||||
|
||||
export type FieldType = 'string' | 'number' | 'boolean' | 'array' | 'object' | 'enum';
|
||||
|
||||
export interface FieldConstraint {
|
||||
kind: 'minLength' | 'maxLength' | 'pattern' | 'enum' | 'range' | 'custom';
|
||||
value: string | number | string[];
|
||||
}
|
||||
|
||||
export interface SchemaField {
|
||||
name: string;
|
||||
type: FieldType;
|
||||
description: string;
|
||||
required: boolean;
|
||||
constraints: FieldConstraint[];
|
||||
}
|
||||
|
||||
export interface Schema {
|
||||
name: string;
|
||||
fields: SchemaField[];
|
||||
version: number;
|
||||
}
|
||||
|
||||
// ── Mutation types ─────────────────────────────────────────────
|
||||
|
||||
export type MutationKind =
|
||||
| 'add_output_field'
|
||||
| 'remove_field'
|
||||
| 'edit_field_desc'
|
||||
| 'change_field_type'
|
||||
| 'add_constraint'
|
||||
| 'remove_constraint'
|
||||
| 'reorder_fields'
|
||||
| 'replace_output_fields';
|
||||
|
||||
export interface Mutation {
|
||||
kind: MutationKind;
|
||||
/** What changed, for logging + feedback */
|
||||
description: string;
|
||||
/** Functional transformer (pure) */
|
||||
apply: (schema: Schema) => Schema;
|
||||
}
|
||||
|
||||
// ── Candidate + result types ──────────────────────────────────
|
||||
|
||||
export interface SchemaCandidate {
|
||||
id: string;
|
||||
schema: Schema;
|
||||
generation: number;
|
||||
parent: string | null;
|
||||
mutation: MutationKind | 'baseline';
|
||||
mutationLabel: string;
|
||||
score: SchemaCandidateScore | null;
|
||||
perExample: SchemaExampleResult[];
|
||||
}
|
||||
|
||||
export interface SchemaExampleResult {
|
||||
input: string;
|
||||
expected: string;
|
||||
actual: string;
|
||||
score: JudgeScore;
|
||||
/** Diagnostic: did the output parse against the schema? */
|
||||
parsed: boolean;
|
||||
}
|
||||
|
||||
export interface SchemaCandidateScore {
|
||||
/** Mean overall judge score across the eval set (0..1) */
|
||||
accuracy: number;
|
||||
/** Schema "complexity" — a scalar that grows with fields, descriptions, constraints */
|
||||
complexity: number;
|
||||
/** Share of examples that produced parseable output against this schema (0..1) */
|
||||
parseRate: number;
|
||||
/** Weakness feedback taken from the worst-scoring 3 examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Number of examples scored */
|
||||
n: number;
|
||||
}
|
||||
|
||||
export type SchemaExecuteFn = (args: {
|
||||
schema: Schema;
|
||||
input: string;
|
||||
}) => Promise<{ actual: string; parsed: boolean }>;
|
||||
|
||||
export interface EvolveSchemaOptions {
|
||||
/** Starting schema */
|
||||
baseline: Schema;
|
||||
/** Eval dataset */
|
||||
examples: EvalExample[];
|
||||
/** LLM runner that takes a schema + input and returns an attempted output */
|
||||
execute: SchemaExecuteFn;
|
||||
/** Judge for scoring individual attempts */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Candidates per generation (default 5) */
|
||||
populationSize?: number;
|
||||
/** Generations (default 3) */
|
||||
generations?: number;
|
||||
/** Mix of phase mutations per generation */
|
||||
mutationMix?: { structure: number; order: number; refinement: number };
|
||||
/** Eval sample size per generation (default 32) */
|
||||
evalSize?: number;
|
||||
/** Anchor eval size at the end (default 100) */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for sampling (default 1) */
|
||||
seed?: number;
|
||||
/** Optional LLM helper for description edits — deterministic fallback used if omitted */
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>;
|
||||
/** Optional progress emitter */
|
||||
onProgress?: (event: EvolveSchemaProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaProgress {
|
||||
phase: 'start' | 'structure' | 'order' | 'refinement' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
bestAccuracy: number;
|
||||
bestComplexity: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaResult {
|
||||
winner: SchemaCandidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: SchemaCandidate[];
|
||||
/** All candidates across all generations */
|
||||
history: SchemaCandidate[];
|
||||
/** Accuracy delta: winner.accuracy - baseline.accuracy */
|
||||
deltaAccuracy: number;
|
||||
/** True if the winner's accuracy improved over baseline */
|
||||
improved: boolean;
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ───────────────────────────────────
|
||||
|
||||
export function addOutputField(
|
||||
schema: Schema,
|
||||
field: SchemaField,
|
||||
position?: number,
|
||||
): Schema {
|
||||
const pos = position ?? schema.fields.length;
|
||||
const fields = [...schema.fields];
|
||||
fields.splice(pos, 0, { ...field, constraints: [...field.constraints] });
|
||||
return { ...schema, fields, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function removeField(schema: Schema, fieldName: string): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.filter(f => f.name !== fieldName),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function editFieldDescription(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newDescription: string,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, description: newDescription } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function changeFieldType(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newType: FieldType,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, type: newType } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function addConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraint: FieldConstraint,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, constraints: [...f.constraints, constraint] } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function removeConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraintIndex: number,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f => {
|
||||
if (f.name !== fieldName) return f;
|
||||
const constraints = f.constraints.filter((_, i) => i !== constraintIndex);
|
||||
return { ...f, constraints };
|
||||
}),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function reorderFields(schema: Schema, newOrder: string[]): Schema {
|
||||
const byName = new Map(schema.fields.map(f => [f.name, f]));
|
||||
const reordered: SchemaField[] = [];
|
||||
for (const name of newOrder) {
|
||||
const field = byName.get(name);
|
||||
if (field) reordered.push(field);
|
||||
}
|
||||
// Append any fields that were omitted from newOrder (keeps their position stable).
|
||||
for (const field of schema.fields) {
|
||||
if (!newOrder.includes(field.name)) reordered.push(field);
|
||||
}
|
||||
return { ...schema, fields: reordered, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function replaceOutputFields(schema: Schema, newFields: SchemaField[]): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: newFields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Complexity + scoring helpers ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Scalar complexity measure: 1 per field + 0.3 per constraint +
|
||||
* 0.01 per character of description. Calibrated so a 3-field schema
|
||||
* with short descriptions ≈ 3.6, a 10-field schema with long
|
||||
* descriptions ≈ 15+.
|
||||
*/
|
||||
export function schemaComplexity(schema: Schema): number {
|
||||
let c = 0;
|
||||
for (const field of schema.fields) {
|
||||
c += 1;
|
||||
c += field.constraints.length * 0.3;
|
||||
c += (field.description?.length ?? 0) * 0.01;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
export function aggregateSchemaScores(results: SchemaExampleResult[]): SchemaCandidateScore {
|
||||
if (results.length === 0) {
|
||||
return {
|
||||
accuracy: 0, complexity: 0, parseRate: 0, weaknessFeedback: [], n: 0,
|
||||
};
|
||||
}
|
||||
const n = results.length;
|
||||
const accuracy = results.reduce((s, r) => s + r.score.overall, 0) / n;
|
||||
const parseRate = results.filter(r => r.parsed).length / n;
|
||||
const worst = [...results].sort((a, b) => a.score.overall - b.score.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(r => r.score.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
return { accuracy, complexity: 0, parseRate, weaknessFeedback, n };
|
||||
}
|
||||
|
||||
// ── Pareto (2D) ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front on (accuracy↑, complexity↓). A candidate dominates
|
||||
* another iff it is at least as accurate AND no more complex, and
|
||||
* strictly better on at least one dimension.
|
||||
*/
|
||||
export function paretoFrontSchema(candidates: SchemaCandidate[]): SchemaCandidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: SchemaCandidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominatesSchema(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominatesSchema(a: SchemaCandidateScore, b: SchemaCandidateScore): boolean {
|
||||
const geAll = a.accuracy >= b.accuracy && a.complexity <= b.complexity;
|
||||
const gtAny = a.accuracy > b.accuracy || a.complexity < b.complexity;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
// ── Candidate scoring ─────────────────────────────────────────
|
||||
|
||||
export async function scoreSchemaCandidate(
|
||||
candidate: SchemaCandidate,
|
||||
examples: EvalExample[],
|
||||
execute: SchemaExecuteFn,
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<SchemaCandidateScore> {
|
||||
const results: SchemaExampleResult[] = [];
|
||||
for (const ex of examples) {
|
||||
if (signal?.aborted) break;
|
||||
try {
|
||||
const { actual, parsed } = await execute({ schema: candidate.schema, input: ex.input });
|
||||
const score = await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual,
|
||||
});
|
||||
results.push({ input: ex.input, expected: ex.expected_output, actual, score, parsed });
|
||||
} catch {
|
||||
// Keep iterating — one broken example shouldn't poison the whole batch.
|
||||
}
|
||||
}
|
||||
const agg = aggregateSchemaScores(results);
|
||||
agg.complexity = schemaComplexity(candidate.schema);
|
||||
candidate.perExample = results;
|
||||
candidate.score = agg;
|
||||
return agg;
|
||||
}
|
||||
|
||||
// ── Mutation generators per phase ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Structure-Discovery mutations: add a new field, replace fields wholesale,
|
||||
* or drop one if the schema is too large. Returns up to `n` mutations.
|
||||
*/
|
||||
export function generateStructureMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Add a reasoning field if absent.
|
||||
if (!parent.fields.some(f => /reason|thought|scratch/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add reasoning field before answer',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Concise chain-of-thought explaining how the answer was derived.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
}, 0),
|
||||
});
|
||||
}
|
||||
|
||||
// Add a confidence field if absent.
|
||||
if (!parent.fields.some(f => /confidence|certainty|score/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add confidence field at end',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'confidence',
|
||||
type: 'number',
|
||||
description: 'Self-estimated confidence in the answer (0.0 - 1.0).',
|
||||
required: false,
|
||||
constraints: [{ kind: 'range', value: '0..1' }],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// Drop the lowest-priority field if > 4 fields.
|
||||
if (parent.fields.length > 4) {
|
||||
const candidate = parent.fields[parent.fields.length - 1];
|
||||
out.push({
|
||||
kind: 'remove_field',
|
||||
description: `drop trailing field "${candidate.name}"`,
|
||||
apply: (s) => removeField(s, candidate.name),
|
||||
});
|
||||
}
|
||||
|
||||
// Replace all output fields with a minimal skeleton (reasoning + answer).
|
||||
if (parent.fields.length >= 2) {
|
||||
out.push({
|
||||
kind: 'replace_output_fields',
|
||||
description: 'collapse to {reasoning, answer}',
|
||||
apply: (s) =>
|
||||
replaceOutputFields(s, [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Brief chain-of-thought.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'Final answer.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order-Probe mutations: generate permutations of field order. Limits
|
||||
* output to `n` by random sampling from the set of useful permutations.
|
||||
*/
|
||||
export function generateOrderMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const names = parent.fields.map(f => f.name);
|
||||
if (names.length < 2) return [];
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Heuristic 1: move any reasoning/thought field to the front.
|
||||
const reasoningIdx = parent.fields.findIndex(f => /reason|thought|scratch/i.test(f.name));
|
||||
if (reasoningIdx > 0) {
|
||||
const newOrder = [names[reasoningIdx], ...names.filter((_, i) => i !== reasoningIdx)];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[reasoningIdx]}" to front`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Heuristic 2: move any confidence/score field to the end.
|
||||
const confIdx = parent.fields.findIndex(f => /confidence|certainty/i.test(f.name));
|
||||
if (confIdx >= 0 && confIdx !== parent.fields.length - 1) {
|
||||
const newOrder = [...names.filter((_, i) => i !== confIdx), names[confIdx]];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[confIdx]}" to end`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Fill remaining with random permutations.
|
||||
while (out.length < n) {
|
||||
const shuffled = shuffle([...names], rng);
|
||||
if (!arraysEqual(shuffled, names)) {
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `reorder to [${shuffled.join(', ')}]`,
|
||||
apply: (s) => reorderFields(s, shuffled),
|
||||
});
|
||||
}
|
||||
if (out.length > n * 3) break; // safety — very short schemas
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refinement mutations driven by worst-scoring example feedback.
|
||||
* Edits descriptions, tightens constraints, or switches types based on
|
||||
* heuristic patterns in the feedback.
|
||||
*/
|
||||
export async function generateRefinementMutations(
|
||||
parent: Schema,
|
||||
weakness: string[],
|
||||
n: number,
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>,
|
||||
): Promise<Mutation[]> {
|
||||
const out: Mutation[] = [];
|
||||
const joined = weakness.join(' ').toLowerCase();
|
||||
|
||||
// Heuristic: if feedback mentions "too verbose" / "long" — add a maxLength to any string field.
|
||||
if (/verbose|too long|wordy/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `tighten "${f.name}" with maxLength: 200`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'maxLength', value: 200 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "too brief" / "incomplete" — add minLength.
|
||||
if (/brief|incomplete|missing|short/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `require "${f.name}" minLength: 20`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'minLength', value: 20 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "wrong format" / "parse" — clarify description.
|
||||
if (/format|parse|wrong type/i.test(joined)) {
|
||||
for (const field of parent.fields.slice(0, 2)) {
|
||||
const newDesc = editFieldDescription
|
||||
? await editFieldDescription({ field, feedback: weakness }).catch(() => field.description)
|
||||
: deterministicClarifyDescription(field);
|
||||
if (newDesc && newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: always consider one description rewrite on the first field.
|
||||
if (out.length === 0 && parent.fields.length > 0) {
|
||||
const field = parent.fields[0];
|
||||
const newDesc = deterministicClarifyDescription(field);
|
||||
if (newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out.slice(0, n);
|
||||
}
|
||||
|
||||
// ── EvolveSchema engine ──────────────────────────────────────
|
||||
|
||||
export class EvolveSchema {
|
||||
async run(options: EvolveSchemaOptions): Promise<EvolveSchemaResult> {
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
const baseline: SchemaCandidate = {
|
||||
id: 'g0-baseline',
|
||||
schema: cloneSchema(config.baseline),
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emitProgress(config, 'start', 0, 1, 0, 0, 'scoring baseline');
|
||||
const evalSample = pickSample(config.examples, config.evalSize, rng);
|
||||
await scoreSchemaCandidate(baseline, evalSample, config.execute, config.judge, config.signal);
|
||||
|
||||
const history: SchemaCandidate[] = [baseline];
|
||||
let survivors: SchemaCandidate[] = [baseline];
|
||||
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const topParent = pickTopByAccuracy(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
const children: SchemaCandidate[] = [];
|
||||
|
||||
// Phase A — Structure Discovery
|
||||
emitProgress(
|
||||
config, 'structure', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const structureMutations = generateStructureMutations(
|
||||
topParent.schema, config.mutationMix.structure, rng,
|
||||
);
|
||||
children.push(...applyMutations(structureMutations, topParent, gen));
|
||||
|
||||
// Phase B — Field-Order Probes
|
||||
emitProgress(
|
||||
config, 'order', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const orderMutations = generateOrderMutations(
|
||||
topParent.schema, config.mutationMix.order, rng,
|
||||
);
|
||||
children.push(...applyMutations(orderMutations, topParent, gen));
|
||||
|
||||
// Phase C — Failure-Driven Refinement
|
||||
emitProgress(
|
||||
config, 'refinement', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const refinementMutations = await generateRefinementMutations(
|
||||
topParent.schema,
|
||||
topParent.score?.weaknessFeedback ?? [],
|
||||
config.mutationMix.refinement,
|
||||
config.editFieldDescription,
|
||||
);
|
||||
children.push(...applyMutations(refinementMutations, topParent, gen));
|
||||
|
||||
// Score all children on the per-generation sample.
|
||||
const genSample = pickSample(config.examples, config.evalSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(child, genSample, config.execute, config.judge, config.signal);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// Pareto-select survivors: parent + children.
|
||||
survivors = paretoFrontSchema([topParent, ...children]);
|
||||
if (survivors.length > config.populationSize) {
|
||||
// Trim by picking the best-accuracy members.
|
||||
survivors = survivors
|
||||
.slice()
|
||||
.sort((a, b) => (b.score?.accuracy ?? 0) - (a.score?.accuracy ?? 0))
|
||||
.slice(0, config.populationSize);
|
||||
}
|
||||
}
|
||||
|
||||
// Anchor stage: larger eval on final survivors.
|
||||
emitProgress(
|
||||
config, 'anchor', config.generations, survivors.length,
|
||||
bestAccuracy(survivors), minComplexity(survivors),
|
||||
);
|
||||
const anchorSample = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(cand, anchorSample, config.execute, config.judge, config.signal);
|
||||
}
|
||||
|
||||
const paretoFront = paretoFrontSchema(survivors);
|
||||
const winner = pickSchemaWinner(paretoFront);
|
||||
const deltaAccuracy = (winner.score?.accuracy ?? 0) - (baseline.score?.accuracy ?? 0);
|
||||
|
||||
emitProgress(
|
||||
config, 'done', config.generations, paretoFront.length,
|
||||
winner.score?.accuracy ?? 0, winner.score?.complexity ?? 0,
|
||||
);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront,
|
||||
history,
|
||||
deltaAccuracy,
|
||||
improved: deltaAccuracy > 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
function applyMutations(
|
||||
mutations: Mutation[],
|
||||
parent: SchemaCandidate,
|
||||
generation: number,
|
||||
): SchemaCandidate[] {
|
||||
return mutations.map((m, i) => ({
|
||||
id: `g${generation}-${m.kind}-${i}`,
|
||||
schema: m.apply(parent.schema),
|
||||
generation,
|
||||
parent: parent.id,
|
||||
mutation: m.kind,
|
||||
mutationLabel: m.description,
|
||||
score: null,
|
||||
perExample: [],
|
||||
}));
|
||||
}
|
||||
|
||||
function pickTopByAccuracy(candidates: SchemaCandidate[]): SchemaCandidate | undefined {
|
||||
if (candidates.length === 0) return undefined;
|
||||
return candidates.reduce((a, b) =>
|
||||
(a.score?.accuracy ?? 0) >= (b.score?.accuracy ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
export function pickSchemaWinner(candidates: SchemaCandidate[]): SchemaCandidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickSchemaWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const bA = best.score?.accuracy ?? -1;
|
||||
const cA = cand.score?.accuracy ?? -1;
|
||||
if (cA > bA) return cand;
|
||||
if (cA < bA) return best;
|
||||
// Tie on accuracy → prefer lower complexity.
|
||||
const bC = best.score?.complexity ?? Infinity;
|
||||
const cC = cand.score?.complexity ?? Infinity;
|
||||
return cC < bC ? cand : best;
|
||||
});
|
||||
}
|
||||
|
||||
function bestAccuracy(cs: SchemaCandidate[]): number {
|
||||
if (cs.length === 0) return 0;
|
||||
return Math.max(...cs.map(c => c.score?.accuracy ?? 0));
|
||||
}
|
||||
|
||||
function minComplexity(cs: SchemaCandidate[]): number {
|
||||
const scored = cs.filter(c => c.score !== null);
|
||||
if (scored.length === 0) return 0;
|
||||
return Math.min(...scored.map(c => c.score!.complexity));
|
||||
}
|
||||
|
||||
function cloneSchema(s: Schema): Schema {
|
||||
return {
|
||||
...s,
|
||||
fields: s.fields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
};
|
||||
}
|
||||
|
||||
function deterministicClarifyDescription(field: SchemaField): string {
|
||||
const base = field.description.trim();
|
||||
if (!base) return `Return the value for "${field.name}" as ${field.type}.`;
|
||||
if (/\b(must|should|return)\b/i.test(base)) return base;
|
||||
return `${base} Must be a ${field.type}.`;
|
||||
}
|
||||
|
||||
// Disambiguation re-export — the generator calls this internally so mutations
|
||||
// can reference it without colliding with the public `editFieldDescription`
|
||||
// option callback in `EvolveSchemaOptions`.
|
||||
const editFieldDescription_pure = editFieldDescription;
|
||||
|
||||
function pickN<T>(arr: T[], n: number, rng: () => number): T[] {
|
||||
if (arr.length <= n) return arr.slice();
|
||||
const copy = arr.slice();
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, n);
|
||||
}
|
||||
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function arraysEqual<T>(a: T[], b: T[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
function emitProgress(
|
||||
config: { onProgress?: (e: EvolveSchemaProgress) => void },
|
||||
phase: EvolveSchemaProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
bestAcc: number,
|
||||
bestCmp: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({
|
||||
phase, generation, populationSize,
|
||||
bestAccuracy: bestAcc,
|
||||
bestComplexity: bestCmp,
|
||||
message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: EvolveSchemaOptions) {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
execute: opts.execute,
|
||||
judge: opts.judge,
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
mutationMix: opts.mutationMix ?? { structure: 3, order: 1, refinement: 1 },
|
||||
evalSize: opts.evalSize ?? 32,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 100,
|
||||
seed: opts.seed ?? 1,
|
||||
editFieldDescription: opts.editFieldDescription,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,19 +1,17 @@
|
||||
/**
|
||||
* HarnessTraceBridge — translates workflow-harness phase events into
|
||||
* execution traces so harnessed agent runs feed the self-evolution loop.
|
||||
* execution traces so harnessed agent runs show up in cost tracking and
|
||||
* run history.
|
||||
*
|
||||
* Without this bridge the workflow harness is an island: phases run,
|
||||
* gates pass or fail, but nothing flows into the `execution_traces`
|
||||
* table that the evaluation dataset builder mines. This module closes
|
||||
* that loop by subscribing to the shared `harnessEvents` emitter and
|
||||
* writing one trace per phase outcome:
|
||||
* table. This module subscribes to the shared `harnessEvents` emitter and
|
||||
* writes one trace per phase outcome:
|
||||
*
|
||||
* harness:phase:complete → trace outcome 'verified'
|
||||
* harness:phase:fail (aborted === true) → trace outcome 'abandoned'
|
||||
* harness:phase:fail (will retry) → skipped — only final
|
||||
* outcomes become training
|
||||
* signal, mid-retry noise
|
||||
* stays out of the dataset
|
||||
* outcomes are recorded
|
||||
*
|
||||
* Each trace records:
|
||||
* - input: the phase's instruction text
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
/**
|
||||
* Improvement Detector — produces structured, runtime-consumable awareness signals.
|
||||
*
|
||||
* Per correction #1: Returns actionable signals + surfaced suggestions as structured data,
|
||||
* not just prompt text changes.
|
||||
*
|
||||
* Per correction #5: Awareness is operational — route better, suggest better, adapt better,
|
||||
* avoid repeated mistakes.
|
||||
*
|
||||
* Per correction #3: Workflow suggestions require recency + pattern similarity,
|
||||
* not just raw shape count.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore, ActionableSignal, SignalCategory } from '@waggle/core';
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
|
||||
// ── Structured output types ──────────────────────────────────
|
||||
|
||||
export interface AwarenessSummary {
|
||||
/** Capability gaps the agent has encountered repeatedly */
|
||||
capabilityGaps: CapabilityGapSignal[];
|
||||
/** Behavioral corrections the user has made repeatedly */
|
||||
corrections: CorrectionSignal[];
|
||||
/** Workflow patterns that recur and could benefit from templates */
|
||||
workflowPatterns: WorkflowPatternSignal[];
|
||||
/** Total actionable signal count (for deciding whether to inject into prompt) */
|
||||
totalActionable: number;
|
||||
}
|
||||
|
||||
export interface CapabilityGapSignal {
|
||||
id: number;
|
||||
toolName: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
export interface CorrectionSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
occurrences: number;
|
||||
guidance: string;
|
||||
}
|
||||
|
||||
export interface WorkflowPatternSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
// ── Capability gap recording ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* Record a capability gap when a tool is not found or a skill is missing.
|
||||
* Called from agent-loop when tool lookup fails.
|
||||
*/
|
||||
export function recordCapabilityGap(
|
||||
store: ImprovementSignalStore,
|
||||
toolName: string,
|
||||
context?: string,
|
||||
): void {
|
||||
store.record('capability_gap', `missing:${toolName}`, context, {
|
||||
tool: toolName,
|
||||
lastContext: context,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Correction recording ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Analyze a user message for corrections and record if detected.
|
||||
* Only task-local and durable corrections are recorded (both go into the store;
|
||||
* the store's threshold mechanism handles promotion to actionable).
|
||||
*
|
||||
* Returns the detected correction (if any) for caller use.
|
||||
*/
|
||||
export function analyzeAndRecordCorrection(
|
||||
store: ImprovementSignalStore,
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
const correction = detectCorrection(userMessage, previousAssistantMessage);
|
||||
if (!correction) return null;
|
||||
|
||||
// Record in store — both durable and task-local go in.
|
||||
// Durable corrections get recorded; task-local ones also get recorded but
|
||||
// only become actionable if the same pattern_key recurs (count >= threshold).
|
||||
store.record('correction', correction.patternKey, correction.detail, {
|
||||
isDurable: correction.isDurable,
|
||||
confidence: correction.confidence,
|
||||
});
|
||||
|
||||
return correction;
|
||||
}
|
||||
|
||||
// ── Workflow pattern recording ────────────────────────────────
|
||||
|
||||
const RECENCY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
|
||||
|
||||
/**
|
||||
* Record a workflow pattern occurrence.
|
||||
* Per correction #3: requires recency context, not just raw shape count.
|
||||
*/
|
||||
export function recordWorkflowPattern(
|
||||
store: ImprovementSignalStore,
|
||||
taskShape: string,
|
||||
taskDescription: string,
|
||||
): void {
|
||||
store.record('workflow_pattern', `shape:${taskShape}`, taskDescription, {
|
||||
lastTask: taskDescription.slice(0, 100),
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Structured awareness builder ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a structured awareness summary from the signal store.
|
||||
* Returns runtime-consumable data that can be:
|
||||
* 1. Injected into system prompt as guidance
|
||||
* 2. Used by the agent loop for routing decisions
|
||||
* 3. Displayed in UI as improvement suggestions
|
||||
*
|
||||
* Per correction #6: capped at 3 actionable signals total, non-repeating.
|
||||
*/
|
||||
export function buildAwarenessSummary(store: ImprovementSignalStore): AwarenessSummary {
|
||||
const actionable = store.getActionable();
|
||||
|
||||
const capabilityGaps: CapabilityGapSignal[] = [];
|
||||
const corrections: CorrectionSignal[] = [];
|
||||
const workflowPatterns: WorkflowPatternSignal[] = [];
|
||||
|
||||
for (const signal of actionable) {
|
||||
switch (signal.category) {
|
||||
case 'capability_gap':
|
||||
capabilityGaps.push(formatCapabilityGap(signal));
|
||||
break;
|
||||
case 'correction':
|
||||
corrections.push(formatCorrection(signal));
|
||||
break;
|
||||
case 'workflow_pattern':
|
||||
// Per correction #3: only surface if recent (within 7 days)
|
||||
if (isRecent(signal.last_seen)) {
|
||||
workflowPatterns.push(formatWorkflowPattern(signal));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
capabilityGaps,
|
||||
corrections,
|
||||
workflowPatterns,
|
||||
totalActionable: capabilityGaps.length + corrections.length + workflowPatterns.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format awareness summary as a prompt section for system prompt injection.
|
||||
* Only included when there are actionable signals.
|
||||
*/
|
||||
export function formatAwarenessPrompt(summary: AwarenessSummary): string | null {
|
||||
if (summary.totalActionable === 0) return null;
|
||||
|
||||
const lines: string[] = ['## Improvement Signals'];
|
||||
|
||||
if (summary.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Missing capabilities** (user has needed these before):');
|
||||
for (const gap of summary.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Behavioral adjustments** (user has corrected these patterns):');
|
||||
for (const correction of summary.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring workflows** (consider suggesting a template):');
|
||||
for (const pattern of summary.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark all signals in a summary as surfaced.
|
||||
* Call this after the signals have been injected into a prompt or shown to the user.
|
||||
*/
|
||||
export function markSummarySurfaced(
|
||||
store: ImprovementSignalStore,
|
||||
summary: AwarenessSummary,
|
||||
): void {
|
||||
for (const gap of summary.capabilityGaps) store.markSurfaced(gap.id);
|
||||
for (const correction of summary.corrections) store.markSurfaced(correction.id);
|
||||
for (const pattern of summary.workflowPatterns) store.markSurfaced(pattern.id);
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────
|
||||
|
||||
function formatCapabilityGap(signal: ActionableSignal): CapabilityGapSignal {
|
||||
const toolName = signal.parsedMetadata.tool as string || signal.pattern_key.replace('missing:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
toolName,
|
||||
occurrences: signal.count,
|
||||
suggestion: `User has needed "${toolName}" ${signal.count} times. Consider using acquire_capability to find it.`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatCorrection(signal: ActionableSignal): CorrectionSignal {
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
detail: signal.detail,
|
||||
occurrences: signal.count,
|
||||
guidance: signal.detail
|
||||
? `${signal.detail} (corrected ${signal.count} times)`
|
||||
: `Correction pattern "${signal.pattern_key}" observed ${signal.count} times`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatWorkflowPattern(signal: ActionableSignal): WorkflowPatternSignal {
|
||||
const shape = signal.pattern_key.replace('shape:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
occurrences: signal.count,
|
||||
suggestion: `"${shape}" tasks recur frequently (${signal.count} times). A workflow template could streamline this.`,
|
||||
};
|
||||
}
|
||||
|
||||
function isRecent(dateStr: string): boolean {
|
||||
try {
|
||||
const date = new Date(dateStr);
|
||||
return Date.now() - date.getTime() < RECENCY_WINDOW_MS;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
/**
|
||||
* @internal Not exported from index.ts — future feature, tested but not wired.
|
||||
*
|
||||
* Improvement Wiring — processes each interaction for improvement signals.
|
||||
*
|
||||
* Bridges the gap between the chat route (which has raw messages) and the
|
||||
* existing correction-detector / improvement-detector / capability-acquisition
|
||||
* modules. Call processInteractionForImprovement after every agent response
|
||||
* to detect corrections, capability gaps, and recurring workflow patterns.
|
||||
*/
|
||||
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
import { detectTaskShape, type TaskShape } from './task-shape.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImprovementWiringParams {
|
||||
userMessage: string;
|
||||
agentResponse: string;
|
||||
toolsUsed: string[];
|
||||
workspaceId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
export interface ImprovementWiringResult {
|
||||
wasCorrection: boolean;
|
||||
correctionDetail?: string;
|
||||
capabilityGap?: string;
|
||||
workflowPattern?: string;
|
||||
}
|
||||
|
||||
// ── Well-known tool names (built-in tools the agent can use) ───────────
|
||||
|
||||
const KNOWN_TOOL_NAMES = new Set([
|
||||
'web_search', 'web_fetch', 'search_memory', 'save_memory',
|
||||
'get_identity', 'get_awareness', 'query_knowledge', 'add_task',
|
||||
'correct_knowledge', 'bash', 'read_file', 'write_file', 'edit_file',
|
||||
'search_files', 'search_content', 'git_status', 'git_diff', 'git_log',
|
||||
'git_commit', 'generate_docx', 'create_plan', 'add_plan_step',
|
||||
'execute_step', 'show_plan', 'list_skills', 'create_skill',
|
||||
'delete_skill', 'read_skill', 'search_skills', 'suggest_skill',
|
||||
'acquire_capability', 'install_capability', 'compose_workflow',
|
||||
'orchestrate_workflow', 'spawn_agent', 'list_agents', 'get_agent_result',
|
||||
]);
|
||||
|
||||
// ── Capability gap detection patterns ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Patterns in user messages that suggest the user wants a tool/capability
|
||||
* the agent doesn't have. These are request-like phrases paired with
|
||||
* tool/domain keywords.
|
||||
*/
|
||||
const GAP_REQUEST_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
|
||||
{ pattern: /\bcan you (?:send|post|publish|push) (?:to|on|via) (\w+)/i, domain: 'integration' },
|
||||
{ pattern: /\bconnect (?:to|with) (\w+)/i, domain: 'connector' },
|
||||
{ pattern: /\buse (\w+) (?:api|tool|service)/i, domain: 'integration' },
|
||||
{ pattern: /\b(?:read|open|parse|convert) (?:this |the |a )?(\w+) file/i, domain: 'file_format' },
|
||||
{ pattern: /\brun (?:this |the |a )?(\w+) (?:test|check|scan|lint)/i, domain: 'tooling' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect if the agent response itself indicates a tool was missing.
|
||||
* The agent often says things like "I don't have a tool for X" or
|
||||
* "Tool 'X' not found".
|
||||
*/
|
||||
const TOOL_NOT_FOUND_PATTERNS: RegExp[] = [
|
||||
/Tool "(.+?)" not found/,
|
||||
/I don't have (?:a |the )?(?:tool|capability|ability) (?:for|to) (.+?)(?:\.|$)/i,
|
||||
/no (?:tool|capability) available for (.+?)(?:\.|$)/i,
|
||||
];
|
||||
|
||||
// ── Workflow pattern detection ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Multi-step request indicators — the user is asking for a sequence of actions.
|
||||
*/
|
||||
const MULTI_STEP_PATTERNS: RegExp[] = [
|
||||
/\bthen\b.*\bthen\b/i, // "do X, then Y, then Z"
|
||||
/\bfirst\b.*\bthen\b/i, // "first do X, then Y"
|
||||
/\bstep\s*\d+/i, // "step 1, step 2"
|
||||
/\b(?:and then|after that|next|finally)\b/i, // sequence words
|
||||
];
|
||||
|
||||
// ── Main function ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Process an interaction for improvement signals.
|
||||
*
|
||||
* This is the main entry point called after every agent response.
|
||||
* It analyzes the user message and agent response to detect:
|
||||
* 1. Corrections (using the existing correction-detector)
|
||||
* 2. Capability gaps (tools requested but not available)
|
||||
* 3. Workflow patterns (recurring multi-step sequences)
|
||||
*
|
||||
* Returns structured signals for storage and display.
|
||||
*/
|
||||
export function processInteractionForImprovement(
|
||||
params: ImprovementWiringParams,
|
||||
): ImprovementWiringResult {
|
||||
const { userMessage, agentResponse, toolsUsed } = params;
|
||||
|
||||
const result: ImprovementWiringResult = {
|
||||
wasCorrection: false,
|
||||
};
|
||||
|
||||
// 1. Run correction detector on userMessage
|
||||
const correction = detectCorrection(userMessage);
|
||||
if (correction) {
|
||||
result.wasCorrection = true;
|
||||
result.correctionDetail = correction.detail;
|
||||
}
|
||||
|
||||
// 2. Check for capability gaps
|
||||
const gap = detectCapabilityGap(userMessage, agentResponse, toolsUsed);
|
||||
if (gap) {
|
||||
result.capabilityGap = gap;
|
||||
}
|
||||
|
||||
// 3. Check for workflow patterns (repeated multi-step sequences)
|
||||
const pattern = detectWorkflowPattern(userMessage);
|
||||
if (pattern) {
|
||||
result.workflowPattern = pattern;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Detect capability gaps from the interaction.
|
||||
* Checks:
|
||||
* - Agent response mentions a tool not found
|
||||
* - User message requests a tool/integration not in the known set
|
||||
*/
|
||||
function detectCapabilityGap(
|
||||
userMessage: string,
|
||||
agentResponse: string,
|
||||
toolsUsed: string[],
|
||||
): string | undefined {
|
||||
// Check agent response for explicit tool-not-found messages
|
||||
for (const pattern of TOOL_NOT_FOUND_PATTERNS) {
|
||||
const match = agentResponse.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Check user message for requests that imply missing capabilities
|
||||
for (const { pattern } of GAP_REQUEST_PATTERNS) {
|
||||
const match = userMessage.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const requested = match[1].toLowerCase();
|
||||
// Only flag as a gap if no tool was actually used for this domain
|
||||
if (toolsUsed.length === 0 || !toolsUsed.some(t => t.includes(requested))) {
|
||||
return requested;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect recurring workflow patterns from the user message.
|
||||
* Uses task-shape detection to identify the structural pattern,
|
||||
* and checks for multi-step indicators.
|
||||
*/
|
||||
function detectWorkflowPattern(userMessage: string): string | undefined {
|
||||
// Only flag multi-step requests as workflow patterns
|
||||
const isMultiStep = MULTI_STEP_PATTERNS.some(p => p.test(userMessage));
|
||||
if (!isMultiStep) return undefined;
|
||||
|
||||
// Use task-shape detector to identify the pattern type
|
||||
const shape = detectTaskShape(userMessage);
|
||||
if (shape.confidence >= 0.3) {
|
||||
return shape.type;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -203,7 +203,6 @@ export { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js'
|
||||
export {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
type ComposedSystemPrompt,
|
||||
type LoadedSkill,
|
||||
@@ -214,69 +213,11 @@ export { checkGrounding, extractClaimedSpecifics, type GroundingResult, type Cla
|
||||
export { CostTracker, DEFAULT_MODEL_PRICING, type ModelPricing, type UsageStats, type UsageEntry } from './cost-tracker.js';
|
||||
export { extractEntities, type ExtractedEntity } from './entity-extractor.js';
|
||||
export { CognifyPipeline, type CognifyConfig, type CognifyResult } from './cognify.js';
|
||||
export { AgentLearning, type LearnedBehavior, type PersonaEffectiveness, type LearningSnapshot } from './agent-learning.js';
|
||||
export {
|
||||
TraceRecorder, truncate as truncateTraceText, scrubSecrets,
|
||||
type TraceHandle, type FinalizeOptions as TraceFinalizeOptions,
|
||||
} from './trace-recorder.js';
|
||||
export {
|
||||
EvalDatasetBuilder, detectSecrets, redactSecrets, SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL, fromJSONL as evalFromJSONL,
|
||||
type EvalExample, type EvalExampleMetadata, type DatasetSplit,
|
||||
type BuildOptions as EvalBuildOptions, type JudgeVerdict,
|
||||
} from './eval-dataset.js';
|
||||
export {
|
||||
IterativeGEPA, paretoFront, scoreCandidate, aggregateScores,
|
||||
pickWinner, pickSample as pickGepaSample,
|
||||
type Candidate as GEPACandidate, type CandidateScore as GEPACandidateScore,
|
||||
type IterativeGEPAOptions, type GEPARunResult, type GEPAProgress,
|
||||
type ScoreCandidateOptions,
|
||||
type MutateArgs, type MutateFn, type MutationStrategy, type EvolutionTarget,
|
||||
} from './iterative-optimizer.js';
|
||||
export {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
type GateVerdict, type GateResult as EvolutionGateResult, type GateCheckResult,
|
||||
type SizeLimits, type GateOptions, type CheckInput as GateCheckInput,
|
||||
} from './evolution-gates.js';
|
||||
export {
|
||||
EvolveSchema,
|
||||
addOutputField, removeField, editFieldDescription,
|
||||
changeFieldType, addConstraint, removeConstraint,
|
||||
reorderFields, replaceOutputFields,
|
||||
schemaComplexity, aggregateSchemaScores, paretoFrontSchema,
|
||||
scoreSchemaCandidate, pickSchemaWinner,
|
||||
generateStructureMutations, generateOrderMutations, generateRefinementMutations,
|
||||
pickSample as pickSchemaSample,
|
||||
type Schema, type SchemaField, type FieldType, type FieldConstraint,
|
||||
type Mutation as SchemaMutation, type MutationKind as SchemaMutationKind,
|
||||
type SchemaCandidate, type SchemaCandidateScore, type SchemaExampleResult,
|
||||
type SchemaExecuteFn, type EvolveSchemaOptions, type EvolveSchemaResult,
|
||||
type EvolveSchemaProgress,
|
||||
} from './evolve-schema.js';
|
||||
export {
|
||||
ComposeEvolution, defaultFeedbackFilter, filterJudgeFeedback,
|
||||
stripStructuralLines, schemaExecutorFromInstructionRunner,
|
||||
type ComposeEvolutionOptions, type ComposeEvolutionResult,
|
||||
type ComposeProgress, type FeedbackFilter,
|
||||
} from './compose-evolution.js';
|
||||
export {
|
||||
EvolutionOrchestrator, eligibleForEvolution, summarizeRuns,
|
||||
type EvolutionOrchestratorDeps, type EvolutionOrchestratorOptions,
|
||||
type EvolutionAutoTriggerConfig, type OrchestratorRunResult,
|
||||
type OrchestratorOutcome, type EvolutionProgress,
|
||||
type SchemaBaselineInput,
|
||||
} from './evolution-orchestrator.js';
|
||||
export {
|
||||
deployPersonaOverride, rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride, rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides, applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type DeployResult, type DeployPersonaInput,
|
||||
type DeployBehavioralSpecInput, type BehavioralSpecOverride,
|
||||
type BehavioralSpecSection,
|
||||
} from './evolution-deploy.js';
|
||||
export { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from './secret-patterns.js';
|
||||
export {
|
||||
LLMJudge, DEFAULT_WEIGHTS, DEFAULT_RUBRIC,
|
||||
buildPrompt as buildJudgePrompt,
|
||||
@@ -285,17 +226,12 @@ export {
|
||||
type ParsedJudgeResponse,
|
||||
} from './judge.js';
|
||||
export {
|
||||
createAnthropicEvolutionLLM,
|
||||
buildJudgeLLMCall, buildGEPAMutateFn, buildSchemaExecuteFn,
|
||||
buildReflectiveMutationPrompt, buildSchemaFillPrompt,
|
||||
makeRunningJudge, isRunningJudge, RUNNING_JUDGE_BRAND,
|
||||
createAnthropicEvolutionLLM, buildJudgeLLMCall,
|
||||
retryWithBackoff, wrapWithRetry,
|
||||
isRetryableEvolutionError, computeRetryDelay,
|
||||
DEFAULT_RETRY_OPTIONS,
|
||||
type EvolutionLLM, type CreateAnthropicEvolutionLLMOptions,
|
||||
type BuildReflectiveMutationPromptArgs,
|
||||
type RetryOptions, type RetryInfo,
|
||||
type RunningJudge,
|
||||
} from './evolution-llm-wiring.js';
|
||||
export {
|
||||
createHarnessRun, advancePhase, getCurrentPhaseInstruction,
|
||||
@@ -346,7 +282,6 @@ export { createDocumentTools } from './document-tools.js';
|
||||
export { createSpreadsheetTools } from './spreadsheet-tools.js';
|
||||
export { createPresentationTools } from './presentation-tools.js';
|
||||
export { createPdfTools } from './pdf-tools.js';
|
||||
export { createInsightsTools, type InsightsDeps } from './insights-tools.js';
|
||||
export { createConnectorSearchTools } from './connector-search.js';
|
||||
export { createCrossWorkspaceTools, type CrossWorkspaceToolDeps } from './cross-workspace-tools.js';
|
||||
export { extractEntitiesWithLLM, type LLMCallFn as EntityLLMCallFn } from './entity-extractor.js';
|
||||
@@ -480,7 +415,6 @@ export {
|
||||
ComposioConnector, MSTeamsConnector, OutlookConnector, OneDriveConnector, OneNoteConnector,
|
||||
} from './connectors/index.js';
|
||||
export { IterationBudget, type IterationBudgetConfig } from './iteration-budget.js';
|
||||
export { captureInteraction, getRecentLogs, isWithinBudget, type CaptureInteractionInput } from './optimization-capture.js';
|
||||
export { routeMessage, type RoutingDecision } from './smart-router.js';
|
||||
export {
|
||||
compressConversation, estimateTokens, needsCompression,
|
||||
@@ -511,13 +445,7 @@ export {
|
||||
type DeliveryChannel, type DeliveryPreferences, type DeliveryMessage, type DeliveryResult,
|
||||
type DeliveryConnector, type DeliveryConnectorRegistry, type InAppEmitter,
|
||||
} from './cron-delivery-router.js';
|
||||
export { detectCorrection, detectCorrectionsInHistory, type DetectedCorrection, type CorrectionDurability } from './correction-detector.js';
|
||||
export { detectContradiction, type ContradictionResult } from './contradiction-detector.js';
|
||||
export {
|
||||
recordCapabilityGap, analyzeAndRecordCorrection, recordWorkflowPattern,
|
||||
buildAwarenessSummary, formatAwarenessPrompt, markSummarySurfaced,
|
||||
type AwarenessSummary, type CapabilityGapSignal, type CorrectionSignal, type WorkflowPatternSignal,
|
||||
} from './improvement-detector.js';
|
||||
export {
|
||||
lintMemoryWrite,
|
||||
type MemoryLintResult, type MemoryLintVerdict,
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
/**
|
||||
* Insights Tools — self-analytics for the agent (Hermes-inspired).
|
||||
*
|
||||
* Provides the agent with awareness of its own performance:
|
||||
* - Tool usage frequency and success rates
|
||||
* - Cost per model
|
||||
* - Correction frequency trends
|
||||
* - Improvement signal summary
|
||||
*
|
||||
* This enables the agent to reflect on what works and adapt strategy.
|
||||
*/
|
||||
|
||||
import type { ToolDefinition } from './tools.js';
|
||||
|
||||
export interface InsightsDeps {
|
||||
getOptimizationLogs: (limit: number) => Array<{
|
||||
model?: string;
|
||||
tool_names?: string;
|
||||
total_tokens?: number;
|
||||
estimated_cost?: number;
|
||||
was_correction?: number;
|
||||
created_at?: string;
|
||||
}>;
|
||||
getImprovementSignals: () => Array<{
|
||||
category: string;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export function createInsightsTools(deps: InsightsDeps): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: 'agent_insights',
|
||||
description: [
|
||||
'Analyze your own performance metrics. Returns: tool usage frequency,',
|
||||
'cost per model, correction rate, and active improvement signals.',
|
||||
'Use this to reflect on what approaches work and adapt your strategy.',
|
||||
'Call this when you want to understand your performance patterns.',
|
||||
].join(' '),
|
||||
parameters: {
|
||||
type: 'object' as const,
|
||||
required: [],
|
||||
properties: {
|
||||
lookbackDays: {
|
||||
type: 'number' as const,
|
||||
description: 'Number of days to analyze (default 30)',
|
||||
},
|
||||
},
|
||||
},
|
||||
execute: async (args: Record<string, unknown>) => {
|
||||
const lookbackDays = (args.lookbackDays as number) ?? 30;
|
||||
const limit = lookbackDays * 50; // ~50 interactions/day estimate
|
||||
|
||||
try {
|
||||
const logs = deps.getOptimizationLogs(limit);
|
||||
const signals = deps.getImprovementSignals();
|
||||
|
||||
// Tool usage frequency
|
||||
const toolCounts: Record<string, { total: number; successes: number }> = {};
|
||||
for (const log of logs) {
|
||||
const tools = log.tool_names?.split(',').map(t => t.trim()).filter(Boolean) ?? [];
|
||||
const isCorrection = log.was_correction === 1;
|
||||
for (const tool of tools) {
|
||||
if (!toolCounts[tool]) toolCounts[tool] = { total: 0, successes: 0 };
|
||||
toolCounts[tool].total++;
|
||||
if (!isCorrection) toolCounts[tool].successes++;
|
||||
}
|
||||
}
|
||||
|
||||
const toolStats = Object.entries(toolCounts)
|
||||
.sort((a, b) => b[1].total - a[1].total)
|
||||
.slice(0, 15)
|
||||
.map(([name, stats]) => ({
|
||||
tool: name,
|
||||
uses: stats.total,
|
||||
successRate: stats.total > 0 ? Math.round((stats.successes / stats.total) * 100) : 0,
|
||||
}));
|
||||
|
||||
// Cost per model
|
||||
const modelCosts: Record<string, { calls: number; tokens: number; cost: number }> = {};
|
||||
for (const log of logs) {
|
||||
const model = log.model ?? 'unknown';
|
||||
if (!modelCosts[model]) modelCosts[model] = { calls: 0, tokens: 0, cost: 0 };
|
||||
modelCosts[model].calls++;
|
||||
modelCosts[model].tokens += log.total_tokens ?? 0;
|
||||
modelCosts[model].cost += log.estimated_cost ?? 0;
|
||||
}
|
||||
|
||||
const modelStats = Object.entries(modelCosts)
|
||||
.sort((a, b) => b[1].calls - a[1].calls)
|
||||
.map(([model, stats]) => ({
|
||||
model,
|
||||
calls: stats.calls,
|
||||
tokens: stats.tokens,
|
||||
costUsd: Math.round(stats.cost * 100) / 100,
|
||||
}));
|
||||
|
||||
// Correction rate
|
||||
const totalInteractions = logs.length;
|
||||
const corrections = logs.filter(l => l.was_correction === 1).length;
|
||||
const correctionRate = totalInteractions > 0
|
||||
? Math.round((corrections / totalInteractions) * 100)
|
||||
: 0;
|
||||
|
||||
// Active improvement signals
|
||||
const activeSignals = signals
|
||||
.filter(s => s.count >= 2)
|
||||
.slice(0, 10)
|
||||
.map(s => ({
|
||||
category: s.category,
|
||||
pattern: s.pattern_key,
|
||||
detail: s.detail,
|
||||
occurrences: s.count,
|
||||
lastSeen: s.last_seen,
|
||||
}));
|
||||
|
||||
return JSON.stringify({
|
||||
period: `Last ${lookbackDays} days`,
|
||||
totalInteractions,
|
||||
correctionRate: `${correctionRate}%`,
|
||||
topTools: toolStats,
|
||||
modelUsage: modelStats,
|
||||
improvementSignals: activeSignals,
|
||||
recommendation: correctionRate > 15
|
||||
? 'High correction rate — review recent corrections and adjust approach.'
|
||||
: correctionRate > 5
|
||||
? 'Moderate corrections — some patterns may need attention.'
|
||||
: 'Low correction rate — current approach is working well.',
|
||||
}, null, 2);
|
||||
} catch (err: unknown) {
|
||||
return `Error generating insights: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -1,626 +0,0 @@
|
||||
/**
|
||||
* Iterative GEPA Optimizer — Phase 2.1 of the self-evolution loop.
|
||||
*
|
||||
* Upgrades the one-shot prompt expander (optimizer-service) to a multi-
|
||||
* generation population search with Pareto-frontier selection and
|
||||
* reflective mutations driven by LLMJudge feedback (Phase 1.3).
|
||||
*
|
||||
* Loop:
|
||||
* 1. Start with `baseline` prompt as the seed population (1 member).
|
||||
* 2. For each generation:
|
||||
* a. Micro-screen survivors on a small sample (default 50) to prune
|
||||
* obvious losers.
|
||||
* b. Mini-eval passing candidates on a larger sample (default 64).
|
||||
* c. Spawn `populationSize` reflective mutations of the top Pareto
|
||||
* candidate, using aggregated judge feedback as ASI.
|
||||
* 3. Anchor evaluation on the full eval set (default 400) picks the winner.
|
||||
*
|
||||
* "Actionable Side Information" (ASI) is the textual feedback the judge
|
||||
* produces — fed back into the `mutate` callback so the next generation
|
||||
* targets the identified weakness rather than mutating blindly.
|
||||
*
|
||||
* Pareto selection keeps candidates that are non-dominated on a
|
||||
* 3-dimensional fitness vector (correctness, procedureFollowing,
|
||||
* conciseness). A single scalar `overall` is also tracked for tie-breaking
|
||||
* and final ranking.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import { isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface Candidate {
|
||||
/** Stable id — e.g. `g0-baseline`, `g1-m2` */
|
||||
id: string;
|
||||
/** The prompt text being evolved */
|
||||
prompt: string;
|
||||
/** Generation index (0 = baseline) */
|
||||
generation: number;
|
||||
/** Parent candidate id (null for baseline) */
|
||||
parent: string | null;
|
||||
/** Short label describing the mutation strategy */
|
||||
strategy: string;
|
||||
/** Aggregated score across the most recent eval stage, or null if not yet scored */
|
||||
score: CandidateScore | null;
|
||||
/** Raw per-example scores used for Pareto analysis */
|
||||
perExample: JudgeScore[];
|
||||
}
|
||||
|
||||
/** Aggregated, mean-reduced score used for selection. */
|
||||
export interface CandidateScore {
|
||||
overall: number;
|
||||
correctness: number;
|
||||
procedureFollowing: number;
|
||||
conciseness: number;
|
||||
/** Mean length penalty applied across the eval set */
|
||||
lengthPenalty: number;
|
||||
/** Number of examples the candidate was scored on */
|
||||
n: number;
|
||||
/** Aggregated feedback lines from the lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
}
|
||||
|
||||
export type MutationStrategy =
|
||||
| 'expand-edge-cases'
|
||||
| 'tighten-format'
|
||||
| 'add-examples'
|
||||
| 'clarify-constraints'
|
||||
| 'reduce-length'
|
||||
| 'restructure-steps'
|
||||
| 'targeted-feedback';
|
||||
|
||||
export interface MutateArgs {
|
||||
parent: Candidate;
|
||||
strategy: MutationStrategy;
|
||||
/** Textual feedback taken from the parent's lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Target types we are mutating (for the mutate fn to tune style) */
|
||||
targetKind: EvolutionTarget;
|
||||
/** Generation index of the child */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export type MutateFn = (args: MutateArgs) => Promise<string>;
|
||||
|
||||
/** What is being evolved — informs mutation style and constraint policies. */
|
||||
export type EvolutionTarget =
|
||||
| 'persona-system-prompt'
|
||||
| 'behavioral-spec-section'
|
||||
| 'tool-description'
|
||||
| 'skill-body'
|
||||
| 'generic';
|
||||
|
||||
export interface GEPAProgress {
|
||||
phase: 'start' | 'micro-screen' | 'mini-eval' | 'mutate' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
/** The best-so-far overall score at this point */
|
||||
best: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface IterativeGEPAOptions {
|
||||
/** The starting prompt text. */
|
||||
baseline: string;
|
||||
/** Full eval dataset (train + val combined is typical). */
|
||||
examples: EvalExample[];
|
||||
/** Scorer — LLMJudge or anything that implements `score(input): Promise<JudgeScore>`. */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Function that produces a mutated prompt from a parent + weakness feedback. */
|
||||
mutate: MutateFn;
|
||||
/**
|
||||
* Opt-in escape hatch that disables the running-judge safety check.
|
||||
* Only set to true for unit tests with mock judges whose `score()` does
|
||||
* not need a real LLM call. Production code paths (evolving real prompts)
|
||||
* must always pass a `makeRunningJudge(...)`-wrapped judge; otherwise
|
||||
* GEPA optimizes prompt-text similarity to the expected output — a
|
||||
* meaningless gradient that produces nonsense winners.
|
||||
*/
|
||||
allowBareJudge?: boolean;
|
||||
/** What we're optimizing (tunes mutation + gate policy). Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Candidates per generation. Default 5. */
|
||||
populationSize?: number;
|
||||
/** How many generations to run. Default 3. */
|
||||
generations?: number;
|
||||
/** Micro-screen sample size. Default 50. */
|
||||
microScreenSize?: number;
|
||||
/** Mini-eval sample size. Default 64. */
|
||||
miniEvalSize?: number;
|
||||
/** Anchor evaluation sample size (applied to final Pareto set). Default 400. */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for deterministic sampling. Default 1. */
|
||||
seed?: number;
|
||||
/** Emits progress events across stages. */
|
||||
onProgress?: (event: GEPAProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Max concurrent `judge.score()` calls per candidate during eval stages.
|
||||
* Default 1 (fully sequential) — preserves legacy behavior. Bumping to
|
||||
* 4–8 cuts wall time dramatically when the judge is an LLM API with
|
||||
* plenty of concurrent capacity. Does not affect determinism: all
|
||||
* scores are aggregated after every example completes.
|
||||
*/
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface GEPARunResult {
|
||||
winner: Candidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: Candidate[];
|
||||
/** All candidates across all generations */
|
||||
history: Candidate[];
|
||||
/** True if the winner improved overall score over baseline */
|
||||
improved: boolean;
|
||||
/** Delta winner.score.overall - baseline.score.overall */
|
||||
delta: number;
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
export class IterativeGEPA {
|
||||
/**
|
||||
* Run the iterative optimization loop.
|
||||
*
|
||||
* Returns the winner candidate plus the full Pareto set. Winner is the
|
||||
* Pareto-set member with the highest `overall` score (ties broken by
|
||||
* length penalty, i.e. the more concise one wins).
|
||||
*/
|
||||
async run(options: IterativeGEPAOptions): Promise<GEPARunResult> {
|
||||
// H-09 G3 · reject bare judges — without makeRunningJudge the judge
|
||||
// scores prompt-text-vs-expected-output, which is a meaningless
|
||||
// gradient. Any caller evolving real prompts must wrap their judge.
|
||||
// Tests pass `allowBareJudge: true` to bypass.
|
||||
if (!options.allowBareJudge && !isRunningJudge(options.judge)) {
|
||||
throw new Error(
|
||||
'IterativeGEPA: judge is not a running judge. Wrap with ' +
|
||||
'makeRunningJudge(baseJudge, llm) before passing to .run(), or ' +
|
||||
'pass `allowBareJudge: true` for tests with mock judges. ' +
|
||||
'See evolution-llm-wiring.ts:makeRunningJudge for rationale.',
|
||||
);
|
||||
}
|
||||
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
// ── Seed population with baseline ────────────────────────
|
||||
const baseline: Candidate = {
|
||||
id: 'g0-baseline',
|
||||
prompt: config.baseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emit(config, 'start', 0, 1, 0, 'seeding baseline');
|
||||
|
||||
const history: Candidate[] = [baseline];
|
||||
let survivors: Candidate[] = [baseline];
|
||||
|
||||
// Score the baseline on the micro-screen so we have a comparison point.
|
||||
const microSample = pickSample(config.examples, config.microScreenSize, rng);
|
||||
await scoreCandidate(baseline, microSample, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
emit(config, 'micro-screen', 0, 1, baseline.score?.overall ?? 0);
|
||||
|
||||
// ── Generations ──────────────────────────────────────────
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
|
||||
// 1. Mutate top survivor(s) into N children.
|
||||
const topParent = pickTopParent(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
emit(config, 'mutate', gen, survivors.length, topParent.score?.overall ?? 0);
|
||||
|
||||
const children: Candidate[] = [];
|
||||
const strategies = cycleStrategies(config.populationSize, gen);
|
||||
for (let i = 0; i < config.populationSize; i++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const strategy = strategies[i];
|
||||
let childPrompt: string;
|
||||
try {
|
||||
childPrompt = await config.mutate({
|
||||
parent: topParent,
|
||||
strategy,
|
||||
weaknessFeedback: topParent.score?.weaknessFeedback ?? [],
|
||||
targetKind: config.targetKind,
|
||||
generation: gen,
|
||||
});
|
||||
} catch {
|
||||
// Mutation failed — reuse parent so the loop doesn't stall.
|
||||
childPrompt = topParent.prompt;
|
||||
}
|
||||
|
||||
const child: Candidate = {
|
||||
id: `g${gen}-m${i}`,
|
||||
prompt: childPrompt,
|
||||
generation: gen,
|
||||
parent: topParent.id,
|
||||
strategy,
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
children.push(child);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// 2. Micro-screen children on a small sample.
|
||||
const microSet = pickSample(config.examples, config.microScreenSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(child, microSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'micro-screen', gen, children.length,
|
||||
Math.max(...children.map(c => c.score?.overall ?? 0), topParent.score?.overall ?? 0),
|
||||
);
|
||||
|
||||
// 3. Keep children that at least match the parent's overall score.
|
||||
const threshold = topParent.score?.overall ?? 0;
|
||||
const microSurvivors = children.filter(c => (c.score?.overall ?? 0) >= threshold - 0.05);
|
||||
|
||||
// If everyone got filtered out, keep the single best child so we don't stall.
|
||||
if (microSurvivors.length === 0 && children.length > 0) {
|
||||
const best = children.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
microSurvivors.push(best);
|
||||
}
|
||||
|
||||
// 4. Mini-eval survivors on a larger sample.
|
||||
if (microSurvivors.length > 0) {
|
||||
const miniSet = pickSample(config.examples, config.miniEvalSize, rng);
|
||||
for (const cand of microSurvivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, miniSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'mini-eval', gen, microSurvivors.length,
|
||||
Math.max(...microSurvivors.map(c => c.score?.overall ?? 0)),
|
||||
);
|
||||
}
|
||||
|
||||
// 5. Pareto-select survivors for next generation.
|
||||
const candidatePool = [topParent, ...microSurvivors];
|
||||
survivors = paretoFront(candidatePool);
|
||||
}
|
||||
|
||||
// ── Anchor evaluation — big sample, final ranking ───────
|
||||
emit(config, 'anchor', config.generations, survivors.length, bestOverall(survivors));
|
||||
const anchorSet = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, anchorSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
|
||||
const anchorPareto = paretoFront(survivors);
|
||||
const winner = pickWinner(anchorPareto);
|
||||
const baselineOverall = baseline.score?.overall ?? 0;
|
||||
const winnerOverall = winner.score?.overall ?? 0;
|
||||
|
||||
emit(config, 'done', config.generations, anchorPareto.length, winnerOverall);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront: anchorPareto,
|
||||
history,
|
||||
improved: winnerOverall > baselineOverall,
|
||||
delta: winnerOverall - baselineOverall,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers (exported for tests) ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front: keep candidates not dominated on any of the three scored
|
||||
* dimensions. A dominates B if A ≥ B on all three and strictly greater
|
||||
* on at least one.
|
||||
*/
|
||||
export function paretoFront(candidates: Candidate[]): Candidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: Candidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominates(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
// If all candidates have equal scores, return them all.
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominates(a: CandidateScore, b: CandidateScore): boolean {
|
||||
const geAll =
|
||||
a.correctness >= b.correctness &&
|
||||
a.procedureFollowing >= b.procedureFollowing &&
|
||||
a.conciseness >= b.conciseness;
|
||||
const gtAny =
|
||||
a.correctness > b.correctness ||
|
||||
a.procedureFollowing > b.procedureFollowing ||
|
||||
a.conciseness > b.conciseness;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
export interface ScoreCandidateOptions {
|
||||
/** Abort signal — checked before each example is dispatched. */
|
||||
signal?: AbortSignal;
|
||||
/** Max concurrent judge.score() calls. Default 1 (sequential). */
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a candidate against a batch of examples and aggregate.
|
||||
*
|
||||
* Backwards compatible: the 4th argument may be either an `AbortSignal`
|
||||
* (legacy) or a `ScoreCandidateOptions` object. Pass
|
||||
* `{ concurrency: 4 }` to parallelize — individual examples are
|
||||
* independent so the aggregate result is identical to the sequential
|
||||
* version, only faster when the judge is an LLM API.
|
||||
*/
|
||||
export async function scoreCandidate(
|
||||
candidate: Candidate,
|
||||
examples: EvalExample[],
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signalOrOptions?: AbortSignal | ScoreCandidateOptions,
|
||||
): Promise<CandidateScore> {
|
||||
const options = resolveScoreOptions(signalOrOptions);
|
||||
const signal = options.signal;
|
||||
const concurrency = Math.max(1, options.concurrency ?? 1);
|
||||
|
||||
// The "candidate" in this context IS the new prompt we want to test.
|
||||
// We treat it as the model's actual response to the example's input,
|
||||
// since mutation produces a variant prompt-as-response. Callers can
|
||||
// substitute a `runCandidate` step before scoring if the prompt needs
|
||||
// to be executed first — see docs.
|
||||
const scoreOne = async (ex: EvalExample): Promise<JudgeScore | null> => {
|
||||
if (signal?.aborted) return null;
|
||||
try {
|
||||
return await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual: candidate.prompt,
|
||||
});
|
||||
} catch {
|
||||
// Keep iterating so one bad example doesn't invalidate the whole score.
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const results = concurrency === 1
|
||||
? await runSequential(examples, scoreOne)
|
||||
: await mapWithConcurrency(examples, concurrency, scoreOne);
|
||||
|
||||
const scores = results.filter((s): s is JudgeScore => s !== null);
|
||||
candidate.perExample = scores;
|
||||
const aggregated = aggregateScores(scores);
|
||||
candidate.score = aggregated;
|
||||
return aggregated;
|
||||
}
|
||||
|
||||
function resolveScoreOptions(
|
||||
arg: AbortSignal | ScoreCandidateOptions | undefined,
|
||||
): ScoreCandidateOptions {
|
||||
if (!arg) return {};
|
||||
// Legacy: AbortSignal passed directly. Detect by presence of `aborted`
|
||||
// (a boolean getter unique to AbortSignal-like objects).
|
||||
if (typeof (arg as AbortSignal).aborted === 'boolean' && typeof (arg as AbortSignal).addEventListener === 'function') {
|
||||
return { signal: arg as AbortSignal };
|
||||
}
|
||||
return arg as ScoreCandidateOptions;
|
||||
}
|
||||
|
||||
async function runSequential<T, R>(
|
||||
items: readonly T[],
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
for (const item of items) {
|
||||
results.push(await fn(item));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded-parallelism map. Runs `fn` over `items` with at most
|
||||
* `concurrency` in-flight calls at once. Results are returned in the
|
||||
* same order as inputs regardless of completion order.
|
||||
*
|
||||
* Implementation: N worker loops share a counter. Each worker pulls the
|
||||
* next index, awaits `fn`, stores at that index, repeats. No external
|
||||
* deps, no extra state machinery.
|
||||
*/
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
const workerCount = Math.min(concurrency, items.length);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: workerCount }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
export function aggregateScores(scores: JudgeScore[]): CandidateScore {
|
||||
if (scores.length === 0) {
|
||||
return {
|
||||
overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1, n: 0, weaknessFeedback: [],
|
||||
};
|
||||
}
|
||||
const n = scores.length;
|
||||
const sum = scores.reduce(
|
||||
(acc, s) => ({
|
||||
overall: acc.overall + s.overall,
|
||||
correctness: acc.correctness + s.correctness,
|
||||
procedureFollowing: acc.procedureFollowing + s.procedureFollowing,
|
||||
conciseness: acc.conciseness + s.conciseness,
|
||||
lengthPenalty: acc.lengthPenalty + s.lengthPenalty,
|
||||
}),
|
||||
{ overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0, lengthPenalty: 0 },
|
||||
);
|
||||
|
||||
// Collect the 3 worst-scoring examples' feedback as weakness signal.
|
||||
const worst = [...scores].sort((a, b) => a.overall - b.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(s => s.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
|
||||
return {
|
||||
overall: sum.overall / n,
|
||||
correctness: sum.correctness / n,
|
||||
procedureFollowing: sum.procedureFollowing / n,
|
||||
conciseness: sum.conciseness / n,
|
||||
lengthPenalty: sum.lengthPenalty / n,
|
||||
n,
|
||||
weaknessFeedback,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the "winner" from a Pareto set. Prefers highest overall; ties broken
|
||||
* by higher length penalty (more concise) then lowest generation (simpler).
|
||||
*/
|
||||
export function pickWinner(candidates: Candidate[]): Candidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const b = best.score?.overall ?? -1;
|
||||
const c = cand.score?.overall ?? -1;
|
||||
if (c > b) return cand;
|
||||
if (c < b) return best;
|
||||
// Tie on overall — prefer higher lengthPenalty (more concise, less penalized).
|
||||
const bLen = best.score?.lengthPenalty ?? 0;
|
||||
const cLen = cand.score?.lengthPenalty ?? 0;
|
||||
if (cLen !== bLen) return cLen > bLen ? cand : best;
|
||||
return best.generation <= cand.generation ? best : cand;
|
||||
});
|
||||
}
|
||||
|
||||
function pickTopParent(survivors: Candidate[]): Candidate | undefined {
|
||||
if (survivors.length === 0) return undefined;
|
||||
return survivors.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
function bestOverall(candidates: Candidate[]): number {
|
||||
if (candidates.length === 0) return 0;
|
||||
return Math.max(...candidates.map(c => c.score?.overall ?? 0));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic sampler — picks `k` examples using a seeded RNG.
|
||||
* If k >= examples.length, returns the full list in deterministic shuffled order.
|
||||
*/
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
// Fisher–Yates shuffle
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
/** Rotate through strategies so early generations explore the mutation space. */
|
||||
function cycleStrategies(n: number, generation: number): MutationStrategy[] {
|
||||
const all: MutationStrategy[] = [
|
||||
'expand-edge-cases',
|
||||
'tighten-format',
|
||||
'add-examples',
|
||||
'clarify-constraints',
|
||||
'reduce-length',
|
||||
'restructure-steps',
|
||||
'targeted-feedback',
|
||||
];
|
||||
// Early generations: spread across strategies.
|
||||
// Later generations: bias toward targeted-feedback once ASI is strong.
|
||||
if (generation <= 1) {
|
||||
return Array.from({ length: n }, (_, i) => all[i % all.length]);
|
||||
}
|
||||
return Array.from({ length: n }, (_, i) => {
|
||||
if (i === 0) return 'targeted-feedback';
|
||||
return all[(i + generation) % all.length];
|
||||
});
|
||||
}
|
||||
|
||||
function emit(
|
||||
config: { onProgress?: (event: GEPAProgress) => void },
|
||||
phase: GEPAProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
best: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({ phase, generation, populationSize, best, message });
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: IterativeGEPAOptions): Required<Omit<IterativeGEPAOptions, 'signal' | 'onProgress'>> & {
|
||||
onProgress?: IterativeGEPAOptions['onProgress'];
|
||||
signal?: AbortSignal;
|
||||
} {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
judge: opts.judge,
|
||||
mutate: opts.mutate,
|
||||
targetKind: opts.targetKind ?? 'generic',
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
microScreenSize: opts.microScreenSize ?? 50,
|
||||
miniEvalSize: opts.miniEvalSize ?? 64,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 400,
|
||||
seed: opts.seed ?? 1,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
concurrency: Math.max(1, opts.concurrency ?? 1),
|
||||
allowBareJudge: opts.allowBareJudge ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
// Mulberry32 — deterministic uniform PRNG
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* LLM-as-Judge — rubric-based scorer used by the evolution loop.
|
||||
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
|
||||
* loop jobs.
|
||||
*
|
||||
* Scores a candidate response against an expected ground truth along
|
||||
* three weighted dimensions:
|
||||
@@ -10,9 +11,8 @@
|
||||
*
|
||||
* Then applies a length penalty (soft cap) multiplied into the final score.
|
||||
*
|
||||
* The textual feedback produced is the input that drives GEPA's reflective
|
||||
* mutations — it must explain *what* went wrong (or right) so the next
|
||||
* prompt variation can address it.
|
||||
* The textual feedback must explain *what* went wrong (or right) so the
|
||||
* caller can act on it.
|
||||
*
|
||||
* Model-agnostic — caller provides the llmCall function. Defaults assume
|
||||
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
|
||||
@@ -42,7 +42,7 @@ export interface JudgeScore {
|
||||
conciseness: number;
|
||||
/** Multiplier applied to the weighted score (0..1). */
|
||||
lengthPenalty: number;
|
||||
/** Free-form feedback used as reflective-mutation input for GEPA. */
|
||||
/** Free-form feedback explaining the score. */
|
||||
feedback: string;
|
||||
/** True when the judge successfully parsed a structured response. */
|
||||
parsed: boolean;
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
/**
|
||||
* Optimization Capture — captures agent interactions for GEPA/Ax prompt optimization.
|
||||
*
|
||||
* Provides functions to log successful agent interactions into the optimization_log
|
||||
* table in a workspace's .mind file, and to query those logs for background
|
||||
* optimization analysis.
|
||||
*
|
||||
* This module is a thin adapter between the agent loop post-response hook and
|
||||
* the OptimizationLogStore in @waggle/core.
|
||||
*/
|
||||
|
||||
import type { OptimizationLogStore, CreateOptimizationLogInput, OptimizationLogEntry } from '@waggle/core';
|
||||
import type { AgentResponse } from './agent-loop.js';
|
||||
|
||||
// ── Public interface ───────────────────────────────────────────────────
|
||||
|
||||
export interface CaptureInteractionInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
response: AgentResponse;
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture an agent interaction for GEPA optimization analysis.
|
||||
*
|
||||
* Call this after every successful agent loop completion. The data is
|
||||
* stored in the workspace's .mind file and read later by the background
|
||||
* prompt_optimization cron job.
|
||||
*/
|
||||
export function captureInteraction(
|
||||
store: OptimizationLogStore,
|
||||
input: CaptureInteractionInput,
|
||||
): OptimizationLogEntry {
|
||||
const logInput: CreateOptimizationLogInput = {
|
||||
sessionId: input.sessionId,
|
||||
workspaceId: input.workspaceId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
toolsUsed: input.response.toolsUsed ?? [],
|
||||
turnCount: input.turnCount,
|
||||
wasCorrection: input.wasCorrection,
|
||||
inputTokens: input.response.usage?.inputTokens ?? 0,
|
||||
outputTokens: input.response.usage?.outputTokens ?? 0,
|
||||
};
|
||||
return store.insert(logInput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query recent optimization logs for background analysis.
|
||||
*
|
||||
* Returns the most recent entries from the optimization log,
|
||||
* ordered by timestamp descending.
|
||||
*/
|
||||
export function getRecentLogs(
|
||||
store: OptimizationLogStore,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getRecent(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimization logs for a specific workspace.
|
||||
*/
|
||||
export function getWorkspaceLogs(
|
||||
store: OptimizationLogStore,
|
||||
workspaceId: string,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getByWorkspace(workspaceId, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the daily optimization budget has been exceeded.
|
||||
*
|
||||
* Reads the optimization log entries from today and sums up their token costs.
|
||||
* Returns true if under budget, false if over.
|
||||
*
|
||||
* @param budgetCents Daily budget in cents (e.g. 100 = $1/day)
|
||||
* @param costPerInputToken Cost per input token in dollars (default: Sonnet pricing ~$3/M)
|
||||
* @param costPerOutputToken Cost per output token in dollars (default: Sonnet pricing ~$15/M)
|
||||
*/
|
||||
export function isWithinBudget(
|
||||
store: OptimizationLogStore,
|
||||
budgetCents: number,
|
||||
costPerInputToken: number = 3 / 1_000_000,
|
||||
costPerOutputToken: number = 15 / 1_000_000,
|
||||
): boolean {
|
||||
const stats = store.getStats();
|
||||
const totalCostDollars =
|
||||
stats.totalInputTokens * costPerInputToken +
|
||||
stats.totalOutputTokens * costPerOutputToken;
|
||||
const totalCostCents = totalCostDollars * 100;
|
||||
return totalCostCents <= budgetCents;
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
KnowledgeGraph,
|
||||
ImprovementSignalStore,
|
||||
createCoreLogger,
|
||||
evaluateExternalMemoryIngress,
|
||||
type Embedder,
|
||||
@@ -29,7 +28,6 @@ import { createMindTools, type ToolDefinition } from './tools.js';
|
||||
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
|
||||
import { renderGoalAncestry } from './goal-ancestry.js';
|
||||
import type { GoalAncestry } from '@waggle/shared';
|
||||
import { buildAwarenessSummary, markSummarySurfaced, type AwarenessSummary } from './improvement-detector.js';
|
||||
import { CognifyPipeline } from './cognify.js';
|
||||
import { scanForInjection } from './injection-scanner.js';
|
||||
import { runPatternWriteBack } from './pattern-write-back.js';
|
||||
@@ -134,13 +132,9 @@ export class Orchestrator {
|
||||
private mode: 'local' | 'team';
|
||||
private version: string;
|
||||
private skills: string[];
|
||||
private improvementSignals: ImprovementSignalStore;
|
||||
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
|
||||
private goalAncestry: GoalAncestry | null = null;
|
||||
|
||||
/** M8: deferred signal marking — collected during buildSystemPrompt, committed after model call */
|
||||
private _pendingSurfacedAwareness: AwarenessSummary | null = null;
|
||||
|
||||
/** Workspace-specific layers (null when no workspace is active) */
|
||||
private workspaceLayers: WorkspaceLayers | null = null;
|
||||
/** Personal-mind cognify pipeline (#12: compaction-summary persistence). */
|
||||
@@ -174,7 +168,6 @@ export class Orchestrator {
|
||||
this.search = new HybridSearch(config.db, config.embedder);
|
||||
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
|
||||
this.knowledge = new KnowledgeGraph(config.db);
|
||||
this.improvementSignals = new ImprovementSignalStore(config.db);
|
||||
|
||||
const cognify = new CognifyPipeline({
|
||||
frames: this.frames,
|
||||
@@ -193,10 +186,6 @@ export class Orchestrator {
|
||||
search: this.search,
|
||||
knowledge: this.knowledge,
|
||||
cognify,
|
||||
// Skills 2.0 gap K: write-path contradiction detection emits a
|
||||
// correction signal through this store when a new save conflicts
|
||||
// with an existing frame.
|
||||
improvementSignals: this.improvementSignals,
|
||||
// Provide workspace accessors so tools can route to the right mind
|
||||
getWorkspaceLayers: () => this.workspaceLayers,
|
||||
});
|
||||
@@ -343,11 +332,6 @@ export class Orchestrator {
|
||||
|
||||
// ── SELF-AWARENESS (runtime context, changes every call) ──
|
||||
const awarenessSection = this.uncachedSection('self_awareness', () => {
|
||||
const awareness = buildAwarenessSummary(this.improvementSignals);
|
||||
// Defer marking until commitSurfacedSignals() fires post-model-call.
|
||||
if (awareness.totalActionable > 0) {
|
||||
this._pendingSurfacedAwareness = awareness;
|
||||
}
|
||||
const caps: AgentCapabilities = {
|
||||
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
|
||||
skills: this.skills,
|
||||
@@ -355,7 +339,6 @@ export class Orchestrator {
|
||||
memoryStats: this.getMemoryStats(),
|
||||
mode: this.mode,
|
||||
version: this.version,
|
||||
awareness: awareness.totalActionable > 0 ? awareness : undefined,
|
||||
};
|
||||
return buildSelfAwareness(caps);
|
||||
});
|
||||
@@ -457,18 +440,6 @@ export class Orchestrator {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* M8: Commit deferred signal markings after model call succeeds.
|
||||
* Call this after the LLM response is received. If the model call fails,
|
||||
* skip this call — signals stay actionable for the next turn.
|
||||
*/
|
||||
commitSurfacedSignals(): void {
|
||||
if (this._pendingSurfacedAwareness) {
|
||||
markSummarySurfaced(this.improvementSignals, this._pendingSurfacedAwareness);
|
||||
this._pendingSurfacedAwareness = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic memory recall: search for memories relevant to the user's query.
|
||||
* Searches BOTH personal and workspace minds when workspace is active.
|
||||
@@ -991,5 +962,4 @@ export class Orchestrator {
|
||||
getSessions(): SessionStore { return this.sessions; }
|
||||
getSearch(): HybridSearch { return this.search; }
|
||||
getKnowledge(): KnowledgeGraph { return this.knowledge; }
|
||||
getImprovementSignals(): ImprovementSignalStore { return this.improvementSignals; }
|
||||
}
|
||||
|
||||
@@ -990,83 +990,4 @@ You specialize in creative direction — briefs, feedback management, brand cons
|
||||
'Delivering without version tracking — every creative output needs a version note.',
|
||||
],
|
||||
},
|
||||
|
||||
// ── Internal review persona (self-evolution) ──────────────────────────
|
||||
// NOT surfaced in onboarding (ALL_ONBOARDING_PERSONAS) or the PersonaSwitcher
|
||||
// tiers (UNIVERSAL_MODE_IDS / ALL_SPECIALIST_IDS). It runs headless via the
|
||||
// IdleSessionWatcher loopback: it reads a stale session's transcript and, when
|
||||
// it finds a material, actionable gap, proposes a skill patch through
|
||||
// create_skill — which is held for human approval (never written to disk by
|
||||
// the reviewer). Its toolset is read-oriented; every write/exec/memory tool
|
||||
// that ALWAYS_AVAILABLE_TOOLS would re-add is stripped via disallowedTools
|
||||
// (that denylist wins over both the allowlist AND always-available). isReadOnly
|
||||
// is deliberately NOT set — that would strip create_skill, its one write.
|
||||
{
|
||||
id: 'session-reviewer',
|
||||
name: 'Session Reviewer',
|
||||
description: 'Adversarial self-review of a stale session — proposes skill patches for recurring gaps, default-silent',
|
||||
icon: '🔎',
|
||||
tagline: 'Reads a finished session and proposes a fix only when there is real evidence.',
|
||||
bestFor: [
|
||||
'Spotting promised-but-undelivered deliverables in a transcript',
|
||||
'Detecting a recurring capability failure a skill patch could fix',
|
||||
'Staying silent (NOTHING_TO_DO) when nothing material is found',
|
||||
],
|
||||
wontDo: 'Will not write to disk, save memory, delete skills, install capabilities, run shell commands, or invent findings. Every proposal is held for human approval.',
|
||||
systemPrompt: `## Persona: Session Reviewer
|
||||
You are an adversarially careful reviewer of ONE already-finished session. You examine the recent conversation and look for exactly two things, in this order:
|
||||
1. **Promised-but-undelivered deliverables** — the assistant said it would produce or do something and the transcript shows it never did.
|
||||
2. **Recurring capability failures fixable by a skill patch** — the same tool/skill/workflow failed more than once in a way a small reusable skill could prevent.
|
||||
|
||||
=== CRITICAL: DEFAULT SILENT ===
|
||||
If you do not find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence, your entire reply MUST be exactly:
|
||||
NOTHING_TO_DO
|
||||
Nothing else. No preamble, no apology, no summary.
|
||||
|
||||
=== CRITICAL: NEVER INVENT ===
|
||||
- Cite the transcript. Every finding must quote or reference what was actually said or done. If you cannot point to evidence, it is not a finding — output NOTHING_TO_DO.
|
||||
- You are read-only over the user's work: you may read files, search memory, and read skills, but you may NOT write files, save memory, delete skills, install capabilities, or run shell commands.
|
||||
- Your ONE write is a proposal: call create_skill to propose a new or updated skill that closes a real, recurring gap. It will NOT be written to disk — it is held for the human to approve or reject in the Approvals view. Propose sparingly and only with cited evidence.
|
||||
|
||||
### Process
|
||||
1. Read the recent conversation of the session (you have read tools and the workspace binding).
|
||||
2. Decide: is there a promised-but-undelivered deliverable, or a recurring capability failure a skill could fix?
|
||||
3. If yes and only if yes: state the finding with its transcript evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill.
|
||||
4. If no: reply exactly NOTHING_TO_DO.`,
|
||||
modelPreference: 'claude-sonnet-4-6',
|
||||
tools: [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'list_skills', 'search_skills', 'read_skill',
|
||||
'create_skill',
|
||||
],
|
||||
// Explicit denylist — ALWAYS_AVAILABLE_TOOLS (persona-tool-filter.ts) re-adds
|
||||
// save_memory / delete_skill / install_capability / acquire_capability /
|
||||
// add_task / correct_knowledge past any allowlist, so they must be named here
|
||||
// to be stripped. spawn_agent + bash + the file/git writes are defense-in-depth
|
||||
// (they are not in `tools`, but naming them keeps the boundary explicit).
|
||||
// Connector/external-send tools carry dynamic names and cannot be enumerated
|
||||
// here; the allowlist (tools[] + always-available) already excludes them since
|
||||
// none are declared above.
|
||||
disallowedTools: [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'add_task', 'correct_knowledge',
|
||||
'spawn_agent', 'bash',
|
||||
'write_file', 'edit_file', 'generate_docx',
|
||||
'git_commit', 'git_push', 'git_merge',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
],
|
||||
failurePatterns: [
|
||||
'Inventing a finding with no transcript evidence — must cite what was actually said or output NOTHING_TO_DO.',
|
||||
'Narrating when silent is correct — if nothing material is found, the reply is exactly NOTHING_TO_DO.',
|
||||
'Proposing an over-broad skill — the patch must target the specific recurring gap, nothing speculative.',
|
||||
],
|
||||
isReadOnly: false,
|
||||
workspaceAffinity: ['review', 'quality', 'self-evolution'],
|
||||
suggestedSkills: [],
|
||||
suggestedConnectors: [],
|
||||
suggestedMcpServers: [],
|
||||
suggestedCommands: [],
|
||||
defaultWorkflow: null,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -5,7 +5,6 @@ import { buildActiveBehavioralSpec } from './behavioral-spec.js';
|
||||
|
||||
/** The resolved behavioral-spec shape returned by `buildActiveBehavioralSpec`. */
|
||||
export type BehavioralSpec = ReturnType<typeof buildActiveBehavioralSpec>;
|
||||
import { loadBehavioralSpecOverrides } from './evolution-deploy.js';
|
||||
import { loadCustomPersonas } from './custom-personas.js';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
|
||||
@@ -16,16 +15,8 @@ export interface LoadedSkill {
|
||||
|
||||
/**
|
||||
* Raw disk read of `{waggleDir}/system-prompt.md`. Returns null if missing
|
||||
* or empty.
|
||||
*
|
||||
* **Prefer `loadSystemPromptWithOverrides(waggleDir)` for runtime use.** This
|
||||
* bare reader exists for tools that legitimately need the unmerged file
|
||||
* content (editor views, migration scripts). Runtime agent code that loads
|
||||
* just this and nothing else will silently miss any deployed
|
||||
* behavioral-spec or persona evolution, since those are stored in separate
|
||||
* override files under the same `waggleDir`.
|
||||
*
|
||||
* H-09 G2 — see `loadSystemPromptWithOverrides` for the composed path.
|
||||
* or empty. Runtime code should prefer `loadSystemPromptWithOverrides`, which
|
||||
* also returns the behavioral spec and custom personas.
|
||||
*/
|
||||
export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
const filePath = path.join(waggleDir, 'system-prompt.md');
|
||||
@@ -34,22 +25,19 @@ export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
return content || null;
|
||||
}
|
||||
|
||||
/** Bundle of every override-aware input the agent runtime needs. */
|
||||
/** Bundle of every prompt input the agent runtime needs. */
|
||||
export interface ComposedSystemPrompt {
|
||||
/** User's literal system-prompt.md content, or null if absent. */
|
||||
userPromptMd: string | null;
|
||||
/** The active behavioral spec — base + any deployed section overrides. */
|
||||
/** The active behavioral spec. */
|
||||
behavioralSpec: BehavioralSpec;
|
||||
/** Custom personas discovered in `waggleDir/personas/*.json`. */
|
||||
customPersonas: AgentPersona[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Override-aware composition: reads `system-prompt.md`, merges deployed
|
||||
* behavioral-spec section overrides from
|
||||
* `{waggleDir}/evolution-deploy/behavioral-spec/*.md`, and surfaces any
|
||||
* custom personas the user has added. Use this at agent startup so
|
||||
* deployed evolutions take effect at the next run.
|
||||
* Composes the runtime prompt inputs: reads `system-prompt.md`, builds the
|
||||
* behavioral spec and surfaces any custom personas the user has added.
|
||||
*
|
||||
* Callers typically pass the bundle to their orchestrator:
|
||||
*
|
||||
@@ -65,44 +53,11 @@ export interface ComposedSystemPrompt {
|
||||
*/
|
||||
export function loadSystemPromptWithOverrides(waggleDir: string): ComposedSystemPrompt {
|
||||
const userPromptMd = loadSystemPrompt(waggleDir);
|
||||
// loadBehavioralSpecOverrides is no-op on missing dir / malformed files.
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const behavioralSpec = buildActiveBehavioralSpec(overrides);
|
||||
const behavioralSpec = buildActiveBehavioralSpec();
|
||||
const customPersonas = loadCustomPersonas(waggleDir);
|
||||
return { userPromptMd, behavioralSpec, customPersonas };
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup-time self-check. Call after composing the override-aware prompt
|
||||
* bundle to verify that deployed overrides on disk actually appear in the
|
||||
* active spec. Emits a structured console warning (never throws) if a
|
||||
* deployed section is present on disk but missing from the active spec —
|
||||
* that would indicate the caller is using the bare `loadSystemPrompt`
|
||||
* instead of the composed loader.
|
||||
*/
|
||||
export function assertOverridesReachActiveSpec(
|
||||
waggleDir: string,
|
||||
activeSpec: BehavioralSpec,
|
||||
): void {
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const missing: string[] = [];
|
||||
for (const [section, overrideText] of Object.entries(overrides)) {
|
||||
const live = (activeSpec as unknown as Record<string, string | undefined>)[section];
|
||||
if (!live || !live.includes(overrideText.slice(0, 24))) {
|
||||
missing.push(section);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
|
||||
console.warn(
|
||||
`[prompt-loader] override-reach check failed — deployed sections on disk ` +
|
||||
`not present in active behavioral spec: ${missing.join(', ')}. ` +
|
||||
`This usually means the runtime is using loadSystemPrompt() instead of ` +
|
||||
`loadSystemPromptWithOverrides(). See prompt-loader.ts.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function loadSkills(waggleDir: string): LoadedSkill[] {
|
||||
const skillsDir = path.join(waggleDir, 'skills');
|
||||
if (!fs.existsSync(skillsDir)) return [];
|
||||
|
||||
57
packages/agent/src/secret-patterns.ts
Normal file
57
packages/agent/src/secret-patterns.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Secret detection and redaction shared by skill redaction, executor briefs
|
||||
* and anything else that must keep credentials out of stored or forwarded text.
|
||||
*/
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { AwarenessSummary } from './improvement-detector.js';
|
||||
|
||||
export interface AgentCapabilities {
|
||||
tools: Array<{ name: string; description: string }>;
|
||||
skills: string[];
|
||||
@@ -7,8 +5,6 @@ export interface AgentCapabilities {
|
||||
memoryStats: { frameCount: number; sessionCount: number; entityCount: number };
|
||||
mode: 'local' | 'team';
|
||||
version: string;
|
||||
/** Structured improvement signals (optional — only present when store has actionable data) */
|
||||
awareness?: AwarenessSummary;
|
||||
}
|
||||
|
||||
export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
@@ -61,36 +57,6 @@ export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
lines.push('- Prefer memory search over guessing when prior context exists.');
|
||||
lines.push('- When uncertain, ask rather than fabricate.');
|
||||
|
||||
// Improvement signals (per correction #1: structured, runtime-consumable)
|
||||
if (caps.awareness && caps.awareness.totalActionable > 0) {
|
||||
lines.push('');
|
||||
lines.push('## Learning from Past Sessions');
|
||||
|
||||
if (caps.awareness.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring capability gaps:**');
|
||||
for (const gap of caps.awareness.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**User corrections to apply:**');
|
||||
for (const correction of caps.awareness.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring task patterns:**');
|
||||
for (const pattern of caps.awareness.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// F28: Persona compliance — resist persona mode switches and injection via fake authority
|
||||
lines.push('');
|
||||
lines.push('## Persona Compliance');
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
* 2. Generates a SKILL.md via generateSkillMarkdown
|
||||
* 3. Writes it to the personal scope directory (scope: personal frontmatter)
|
||||
* 4. Dedups against existing skill files by filename
|
||||
* 5. Optionally emits a workflow_pattern improvement signal
|
||||
*
|
||||
* A v2 enhancement can insert an LLM prose-improvement step between
|
||||
* detectWorkflowPattern and generateSkillMarkdown without changing this
|
||||
@@ -19,7 +18,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
detectWorkflowPattern,
|
||||
generateSkillMarkdown,
|
||||
@@ -36,8 +34,6 @@ export interface AutoExtractMessage {
|
||||
export interface AutoExtractDeps {
|
||||
/** Directory to write personal-scope SKILL.md files into */
|
||||
personalSkillsDir: string;
|
||||
/** Optional signal store — records workflow_pattern signals on each extraction */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Optional LLM enrichment hook. When provided, the template is passed
|
||||
* through before serialization so the SKILL.md prose can be rewritten.
|
||||
@@ -117,22 +113,5 @@ export async function autoExtractAndCreateSkill(
|
||||
const md = redactSkillContent(annotateWithScope(generateSkillMarkdown(finalTemplate))).content;
|
||||
fs.writeFileSync(filePath, md, 'utf-8');
|
||||
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`auto-extract:${stem}`,
|
||||
`Auto-extracted skill "${finalTemplate.name}" from session pattern (${finalTemplate.tools.join(', ')})`,
|
||||
{
|
||||
skillName: finalTemplate.name,
|
||||
filename: stem,
|
||||
tools: finalTemplate.tools,
|
||||
category: finalTemplate.category,
|
||||
source: 'auto_extract',
|
||||
},
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
return { filePath, template: finalTemplate };
|
||||
}
|
||||
|
||||
@@ -9,12 +9,12 @@
|
||||
* before every authored write.
|
||||
*
|
||||
* Secrets reuse the curated SECRET_PATTERNS via redactSecrets (single source of
|
||||
* truth — see eval-dataset.ts). Paths cover the user's home directory (the exact
|
||||
* truth — see secret-patterns.ts). Paths cover the user's home directory (the exact
|
||||
* running-user path plus generic per-OS forms), which both leaks the username
|
||||
* and makes a skill non-portable.
|
||||
*/
|
||||
import os from 'node:os';
|
||||
import { redactSecrets } from './eval-dataset.js';
|
||||
import { redactSecrets } from './secret-patterns.js';
|
||||
|
||||
// Generic user-home prefixes. Each matches up to (not including) the next path
|
||||
// separator, so the tail is preserved: C:\Users\me\.waggle\x -> ~\.waggle\x
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { loadSkillUsage, forgetSkillUsage } from './skill-usage.js';
|
||||
|
||||
const DEFAULT_MAX_IDLE_DAYS = 90;
|
||||
@@ -35,8 +34,6 @@ export interface RetireOptions {
|
||||
dryRun?: boolean;
|
||||
/** Injectable clock for deterministic tests. */
|
||||
now?: () => Date;
|
||||
/** Emit skill_promotion signals for observability. */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
}
|
||||
|
||||
export interface RetireReport {
|
||||
@@ -124,17 +121,6 @@ export function retireStaleSkills(waggleHome: string, opts: RetireOptions = {}):
|
||||
report.retired.push(skillName);
|
||||
report.archived.push(destPath);
|
||||
forgetSkillUsage(waggleHome, skillName);
|
||||
|
||||
if (opts.improvementSignals) {
|
||||
try {
|
||||
opts.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`retire:${skillName}`,
|
||||
`Retired unused skill "${skillName}" (idle > ${maxIdleDays} days). Archived to ${destPath}.`,
|
||||
{ skillName, archivedTo: destPath, maxIdleDays, lastActivityMs: activity },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
report.skipped.push({ name: skillName, reason: `move_failed: ${msg}` });
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
SKILL_SCOPE_ORDER,
|
||||
type SkillScope,
|
||||
} from './skill-frontmatter.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { autoExtractAndCreateSkill, type AutoExtractMessage } from './skill-autoextract.js';
|
||||
import { retireStaleSkills } from './skill-retirement.js';
|
||||
|
||||
@@ -57,11 +56,9 @@ export interface SkillToolsDeps {
|
||||
/**
|
||||
* Skills 2.0 gap E: promotion infrastructure. Each callback is optional.
|
||||
* - getWorkspaceId / getTeamId resolve scope dirs dynamically at tool-call time.
|
||||
* - improvementSignals records skill_promotion signals on successful moves.
|
||||
*/
|
||||
getWorkspaceId?: () => string | null;
|
||||
getTeamId?: () => string | null;
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Skills 2.0 gap A: session-message accessor for auto-extracting skills
|
||||
* from repeated workflow patterns. When provided, enables the
|
||||
@@ -665,15 +662,14 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
},
|
||||
|
||||
// Skills 2.0 gap E: promote_skill — move a skill up one scope rung.
|
||||
// personal → workspace. Enforces one-step-at-a-time and records a
|
||||
// skill_promotion improvement signal on success.
|
||||
// personal → workspace. Enforces one-step-at-a-time.
|
||||
{
|
||||
name: 'promote_skill',
|
||||
description:
|
||||
'Promote an installed skill one scope up (personal → workspace). '
|
||||
+ 'Team and enterprise scopes are not available in this edition. '
|
||||
+ 'The file is moved to the appropriate directory, frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history), and a skill_promotion signal is recorded.',
|
||||
+ 'The file is moved to the appropriate directory and frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -784,19 +780,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
// failing the promotion. An audit warning is fine.
|
||||
}
|
||||
|
||||
// 8. Record skill_promotion signal
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'skill_promotion',
|
||||
`promote:${skillName}:${currentScope}->${target}`,
|
||||
`Promoted skill "${skillName}" from ${currentScope} to ${target}`,
|
||||
{ skillName, from: currentScope, to: target, workspaceId, teamId },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
// 9. Let the agent reload its skill list so the new scope is picked up
|
||||
// 8. Let the agent reload its skill list so the new scope is picked up
|
||||
onSkillsChanged?.();
|
||||
|
||||
return (
|
||||
@@ -805,7 +789,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Moved from \`${found.filePath}\`\n`
|
||||
+ `- Now at \`${destPath}\`\n`
|
||||
+ `- Promotion history: ${updatedFrontmatter.promoted_from!.join(' → ')} → ${target}\n`
|
||||
+ `- Signal recorded: skill_promotion\n\n`
|
||||
+ `\n`
|
||||
+ `The skill is available immediately at the new scope. Agents in other `
|
||||
+ `sessions will pick it up on their next skill reload.`
|
||||
);
|
||||
@@ -823,8 +807,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Scan the current session for repeated workflow patterns (the same 3+ tools '
|
||||
+ 'used in the same order 2+ times) and auto-create a SKILL.md for any pattern '
|
||||
+ 'found. Creates personal-scope skills only — promote_skill moves them up '
|
||||
+ 'afterwards. Dedups by filename. Records a workflow_pattern signal on each '
|
||||
+ 'successful extraction.',
|
||||
+ 'afterwards. Dedups by filename.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
@@ -839,7 +822,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
}
|
||||
const result = await autoExtractAndCreateSkill(messages, {
|
||||
personalSkillsDir: skillsDir,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (result.filePath) {
|
||||
onSkillsChanged?.();
|
||||
@@ -850,7 +832,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Tools: ${result.template!.tools.join(', ')}\n`
|
||||
+ `- Written to: \`${result.filePath}\`\n`
|
||||
+ `- Scope: personal (use promote_skill to move it up)\n`
|
||||
+ `- Signal: workflow_pattern recorded\n`
|
||||
);
|
||||
}
|
||||
if (result.reason === 'already_exists') {
|
||||
@@ -870,7 +851,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Archive personal-scope skills that have not been used within the idle window. '
|
||||
+ 'Moves stale SKILL.md files to ~/.waggle/skills-archive/ (recoverable — no deletion). '
|
||||
+ 'Default max_idle_days is 90. Default dry_run is true — call with dry_run: false to '
|
||||
+ 'actually move the files. Records a workflow_pattern signal per retirement.',
|
||||
+ 'actually move the files.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -890,7 +871,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays,
|
||||
dryRun,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (!dryRun && report.retired.length > 0) {
|
||||
onSkillsChanged?.();
|
||||
|
||||
@@ -12,7 +12,6 @@ import type {
|
||||
} from '@waggle/core';
|
||||
import type { CognifyPipeline } from './cognify.js';
|
||||
import type { FeedbackHandler } from './feedback-handler.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import type { RiskLevel } from '@waggle/shared';
|
||||
import { createCoreLogger } from '@waggle/core';
|
||||
import { detectContradiction } from './contradiction-detector.js';
|
||||
@@ -96,13 +95,6 @@ export interface MindToolDeps {
|
||||
feedback?: FeedbackHandler;
|
||||
/** F16: Optional embedder for semantic dedup (cosine similarity) */
|
||||
embedder?: Embedder;
|
||||
/**
|
||||
* Skills 2.0 gap K: improvement-signal store for recording write-path
|
||||
* conflicts. When a new save contradicts an existing frame,
|
||||
* detectContradiction fires and we emit a "correction" signal so the
|
||||
* evolution / improvement loops can surface the conflict.
|
||||
*/
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/** Dynamic accessor for workspace layers — checked at call time */
|
||||
getWorkspaceLayers?: () => WorkspaceLayers | null;
|
||||
/** W5.7: Accessor for ALL workspace search instances (for cross-workspace search) */
|
||||
@@ -528,20 +520,6 @@ export function createMindTools(deps: MindToolDeps): ToolDefinition[] {
|
||||
const contradiction = detectContradiction(content, recentRows);
|
||||
if (contradiction.isContradiction && contradiction.conflictsWith) {
|
||||
conflictFlag = ` [flag: contradicts_existing (${contradiction.conflictsWith.slice(0, 80).replace(/\s+/g, ' ').trim()}…)]`;
|
||||
// Record a correction-category signal on the improvement-signal
|
||||
// store. pattern_key is a stable hash-ish of the conflicting
|
||||
// frames so repeat contradictions merge via the ON CONFLICT upsert.
|
||||
if (deps.improvementSignals) {
|
||||
const patternKey = `write-conflict:${normalizeForDedup(content).slice(0, 64)}`;
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'correction',
|
||||
patternKey,
|
||||
`New save contradicts existing frame. New: "${content.slice(0, 120)}" ⟂ Existing: "${contradiction.conflictsWith.slice(0, 120)}"`,
|
||||
{ mindLabel, newContent: content.slice(0, 240), conflictsWith: contradiction.conflictsWith.slice(0, 240) },
|
||||
);
|
||||
} catch { /* non-blocking — signal persistence failure must not abort the save */ }
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking — contradiction check failure proceeds with save */ }
|
||||
|
||||
|
||||
@@ -1,373 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
ComposeEvolution,
|
||||
defaultFeedbackFilter,
|
||||
filterJudgeFeedback,
|
||||
stripStructuralLines,
|
||||
schemaExecutorFromInstructionRunner,
|
||||
} from '../src/compose-evolution.js';
|
||||
import type {
|
||||
Schema,
|
||||
SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(name => ({
|
||||
name, type: 'string',
|
||||
description: `description of ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback: string): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFakeRunner(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ── defaultFeedbackFilter ──────────────────────────────────────
|
||||
|
||||
describe('defaultFeedbackFilter', () => {
|
||||
it('classifies "missing reasoning field" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Missing reasoning field.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "wrong field type" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Wrong field type for answer.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "schema mismatch" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Schema mismatch — expected 3 fields.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "should reorder fields" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Should reorder fields — put reasoning first.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies value-level complaints as value', () => {
|
||||
expect(defaultFeedbackFilter('The answer is too terse')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Response is too verbose')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Incorrect calculation')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Wrong tone — should be more formal')).toBe('value');
|
||||
});
|
||||
|
||||
it('classifies empty feedback as value (safe default)', () => {
|
||||
expect(defaultFeedbackFilter('')).toBe('value');
|
||||
});
|
||||
});
|
||||
|
||||
// ── stripStructuralLines ──────────────────────────────────────
|
||||
|
||||
describe('stripStructuralLines', () => {
|
||||
it('returns empty for empty input', () => {
|
||||
expect(stripStructuralLines('')).toBe('');
|
||||
});
|
||||
|
||||
it('drops only structural lines from multi-line feedback', () => {
|
||||
const input = [
|
||||
'Missing reasoning field.',
|
||||
'The answer is too terse.',
|
||||
'Wrong field type for confidence.',
|
||||
'Needs more detail.',
|
||||
].join('\n');
|
||||
const out = stripStructuralLines(input);
|
||||
expect(out).not.toContain('Missing reasoning field');
|
||||
expect(out).not.toContain('Wrong field type');
|
||||
expect(out).toContain('too terse');
|
||||
expect(out).toContain('Needs more detail');
|
||||
});
|
||||
|
||||
it('keeps all lines when none are structural', () => {
|
||||
const input = 'Too terse.\nNeeds more examples.';
|
||||
expect(stripStructuralLines(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('returns empty when all lines are structural', () => {
|
||||
const input = 'Missing reasoning field.\nSchema mismatch.';
|
||||
expect(stripStructuralLines(input)).toBe('');
|
||||
});
|
||||
|
||||
it('accepts a custom filter', () => {
|
||||
const custom = (line: string) => line.includes('DROP') ? 'structural' : 'value';
|
||||
const input = 'keep me\nDROP me\nkeep too';
|
||||
expect(stripStructuralLines(input, custom)).toBe('keep me\nkeep too');
|
||||
});
|
||||
});
|
||||
|
||||
// ── filterJudgeFeedback ───────────────────────────────────────
|
||||
|
||||
describe('filterJudgeFeedback', () => {
|
||||
it('strips structural feedback but preserves numeric scores', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.7, 'Missing reasoning field.\nResponse is too terse.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
|
||||
expect(score.overall).toBe(0.7);
|
||||
expect(score.correctness).toBe(0.7);
|
||||
expect(score.feedback).not.toContain('Missing reasoning field');
|
||||
expect(score.feedback).toContain('too terse');
|
||||
});
|
||||
|
||||
it('returns empty feedback when all lines are structural', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nSchema mismatch.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
expect(score.feedback).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
// ── schemaExecutorFromInstructionRunner ───────────────────────
|
||||
|
||||
describe('schemaExecutorFromInstructionRunner', () => {
|
||||
it('builds a schema prefix and delegates to the runner', async () => {
|
||||
let captured: { prompt: string; input: string } | null = null;
|
||||
const runner = async (args: { prompt: string; input: string }) => {
|
||||
captured = args;
|
||||
return '{"answer":"42"}';
|
||||
};
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(runner);
|
||||
const result = await schemaRunner({
|
||||
schema: makeSchema(['reasoning', 'answer']),
|
||||
input: 'What is 6*7?',
|
||||
});
|
||||
|
||||
expect(result.actual).toBe('{"answer":"42"}');
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(captured!.prompt).toContain('reasoning');
|
||||
expect(captured!.prompt).toContain('answer');
|
||||
expect(captured!.input).toBe('What is 6*7?');
|
||||
});
|
||||
|
||||
it('reports parsed=false for non-JSON output', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => 'just prose');
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
|
||||
it('swallows runner errors and reports parsed=false', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.actual).toBe('');
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── ComposeEvolution end-to-end ────────────────────────────────
|
||||
|
||||
describe('ComposeEvolution.run', () => {
|
||||
it('runs schema stage then instruction stage', async () => {
|
||||
const stages: string[] = [];
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
mutate: async ({ parent }) => `${parent.prompt} v2`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
onProgress: (e) => stages.push(e.stage),
|
||||
});
|
||||
|
||||
expect(stages).toContain('schema');
|
||||
expect(stages).toContain('instructions');
|
||||
expect(stages).toContain('done');
|
||||
expect(result.schema.winner).toBeDefined();
|
||||
expect(result.instructions.winner).toBeDefined();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
});
|
||||
|
||||
it('feedback separation: GEPA cannot see structural feedback from the judge', async () => {
|
||||
const capturedFeedbacks: string[] = [];
|
||||
|
||||
// Judge always emits structural + value feedback
|
||||
const mixedJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nResponse too terse.');
|
||||
},
|
||||
};
|
||||
|
||||
const mutateSpy = vi.fn(async ({ weaknessFeedback }: { parent: unknown; weaknessFeedback: string[]; strategy: string; targetKind: string; generation: number }) => {
|
||||
capturedFeedbacks.push(...weaknessFeedback);
|
||||
return 'mutated instruction prompt is a reasonable length';
|
||||
});
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: mixedJudge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: mixedJudge,
|
||||
mutate: mutateSpy,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
});
|
||||
|
||||
// GEPA must never receive "Missing reasoning field" in weaknessFeedback.
|
||||
for (const fb of capturedFeedbacks) {
|
||||
expect(fb.toLowerCase()).not.toContain('missing reasoning field');
|
||||
expect(fb.toLowerCase()).not.toContain('wrong field type');
|
||||
}
|
||||
// But it should see the value-level complaint
|
||||
expect(capturedFeedbacks.join(' ')).toContain('too terse');
|
||||
});
|
||||
|
||||
it('accepts a custom feedback filter', async () => {
|
||||
let filterCalls = 0;
|
||||
const customFilter = (feedback: string): 'structural' | 'value' => {
|
||||
filterCalls++;
|
||||
return feedback.includes('CUSTOM-DROP') ? 'structural' : 'value';
|
||||
};
|
||||
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'CUSTOM-DROP this line\nkeep this line');
|
||||
},
|
||||
};
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt text is long enough',
|
||||
examples: makeExamples(3),
|
||||
judge,
|
||||
mutate: async () => 'new prompt text that is long enough',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
feedbackFilter: customFilter,
|
||||
});
|
||||
|
||||
expect(filterCalls).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns a stable shape when aborted before instruction stage', async () => {
|
||||
const ctrl = new AbortController();
|
||||
let progressCount = 0;
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: async () => {
|
||||
// Abort during the schema stage
|
||||
if (progressCount === 1) ctrl.abort();
|
||||
progressCount++;
|
||||
return { actual: '', parsed: false };
|
||||
},
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt',
|
||||
examples: makeExamples(3),
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
mutate: async () => 'should not run',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
expect(result.schema).toBeDefined();
|
||||
expect(result.instructions).toBeDefined();
|
||||
expect(result.instructions.winner.prompt).toBe('baseline prompt');
|
||||
});
|
||||
|
||||
it('combinedDelta is a number; frozenSchema matches ES winner', async () => {
|
||||
const judge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(Math.min(1, args.actual.length / 50), 'ok');
|
||||
},
|
||||
};
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(8),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 4, anchorEvalSize: 4,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'short',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate: async ({ parent }) => `${parent.prompt} more tokens here`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 2,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 6,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.instructions.winner.score).not.toBeNull();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
expect(typeof result.combinedDelta).toBe('number');
|
||||
});
|
||||
});
|
||||
@@ -1,204 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
detectCorrection,
|
||||
detectCorrectionsInHistory,
|
||||
} from '../src/correction-detector.js';
|
||||
|
||||
describe('correction-detector', () => {
|
||||
// ── Basic detection ────────────────────────────────────────
|
||||
|
||||
describe('detectCorrection', () => {
|
||||
it('returns null for normal messages', () => {
|
||||
expect(detectCorrection('What is the weather today?')).toBeNull();
|
||||
expect(detectCorrection('Please write a function that sorts an array')).toBeNull();
|
||||
expect(detectCorrection('Thanks, that looks good')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for very short messages', () => {
|
||||
expect(detectCorrection('ok')).toBeNull();
|
||||
expect(detectCorrection('yes')).toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "no, not that"', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a bullet list, not paragraphs.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeGreaterThanOrEqual(0.3);
|
||||
});
|
||||
|
||||
it('detects strong correction: "I said / I told you"', () => {
|
||||
const result = detectCorrection('I said to use markdown headers, not plain text.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "that\'s wrong"', () => {
|
||||
const result = detectCorrection("That's wrong. The API endpoint should be POST, not GET.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects moderate correction: "actually, use X instead"', () => {
|
||||
const result = detectCorrection('Actually, instead use TypeScript for this module.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects tone correction', () => {
|
||||
const result = detectCorrection('No, that response was too formal. Keep it casual.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('detects format correction', () => {
|
||||
const result = detectCorrection("No, don't use those headers. Use bullet formatting instead.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects length correction', () => {
|
||||
const result = detectCorrection("That's not what I wanted. It's too long, make it shorter.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('length');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Durability classification ──────────────────────────────
|
||||
|
||||
describe('durability classification', () => {
|
||||
it('classifies "always" as durable', () => {
|
||||
const result = detectCorrection('No, always use bullet points instead of numbered lists.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
expect(result!.durability).toBe('durable');
|
||||
});
|
||||
|
||||
it('classifies "never" as durable', () => {
|
||||
const result = detectCorrection("No, never use code blocks for simple text. That's wrong.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "from now on" as durable', () => {
|
||||
const result = detectCorrection('From now on, please don\'t include timestamps in summaries.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "I prefer" as durable', () => {
|
||||
const result = detectCorrection("No, I prefer shorter responses. That's not what I wanted.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "this time" as task_local', () => {
|
||||
const result = detectCorrection("No, not that. This time just use a simple list.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(false);
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('classifies "for now" as task_local', () => {
|
||||
const result = detectCorrection("That's wrong for now. Instead use the old API endpoint.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('defaults to task_local when no explicit signal', () => {
|
||||
const result = detectCorrection('No, not that approach. Use the factory pattern.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pattern key extraction ─────────────────────────────────
|
||||
|
||||
describe('pattern key extraction', () => {
|
||||
it('extracts tone pattern key', () => {
|
||||
const result = detectCorrection("No, that's too verbose. Stop doing that.");
|
||||
expect(result).not.toBeNull();
|
||||
// "too verbose" maps to tone category
|
||||
// Note: might also match length depending on exact wording
|
||||
});
|
||||
|
||||
it('extracts accuracy pattern key for "wrong"', () => {
|
||||
const result = detectCorrection("That's incorrect. I said the opposite.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('accuracy');
|
||||
});
|
||||
|
||||
it('falls back to general:correction for unrecognized patterns', () => {
|
||||
const result = detectCorrection("No, not that. I told you to use something else entirely.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBe('general:correction');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Detail extraction ──────────────────────────────────────
|
||||
|
||||
describe('detail extraction', () => {
|
||||
it('extracts first sentence as detail', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a completely different structure. Also fix the naming.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail).toBe('No, not that.');
|
||||
});
|
||||
|
||||
it('truncates long messages', () => {
|
||||
const longMessage = 'No, not that. ' + 'A'.repeat(200);
|
||||
const result = detectCorrection(longMessage);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail.length).toBeLessThanOrEqual(120);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Confidence scoring ─────────────────────────────────────
|
||||
|
||||
describe('confidence', () => {
|
||||
it('gives higher confidence to strong corrections', () => {
|
||||
const strong = detectCorrection("No, that's wrong. I told you not to do that. Stop.");
|
||||
const moderate = detectCorrection('Actually, let\'s try a different approach instead.');
|
||||
expect(strong).not.toBeNull();
|
||||
expect(moderate).not.toBeNull();
|
||||
expect(strong!.confidence).toBeGreaterThan(moderate!.confidence);
|
||||
});
|
||||
|
||||
it('caps confidence at 1.0', () => {
|
||||
const result = detectCorrection(
|
||||
"No, that's wrong. I said don't do that. Not what I wanted. Please stop. Instead use something else.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeLessThanOrEqual(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── History analysis ───────────────────────────────────────
|
||||
|
||||
describe('detectCorrectionsInHistory', () => {
|
||||
it('detects corrections in a message sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Here is a very detailed formal report...' },
|
||||
{ role: 'user', content: "No, that's too formal. I said keep it casual." },
|
||||
{ role: 'assistant', content: 'Got it. Here is a casual version...' },
|
||||
{ role: 'user', content: 'Thanks, that looks better.' },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections).toHaveLength(1);
|
||||
expect(corrections[0].patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('returns empty array when no corrections found', () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Write a function to sort an array' },
|
||||
{ role: 'assistant', content: 'Here you go...' },
|
||||
{ role: 'user', content: 'Thanks!' },
|
||||
];
|
||||
expect(detectCorrectionsInHistory(messages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles multiple corrections in sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Draft version 1...' },
|
||||
{ role: 'user', content: "No, not that. That's wrong. Use a different approach." },
|
||||
{ role: 'assistant', content: 'Draft version 2...' },
|
||||
{ role: 'user', content: "I said don't use bullet points. Instead use paragraphs." },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,348 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore } from '@waggle/core';
|
||||
import {
|
||||
EvalDatasetBuilder,
|
||||
detectSecrets,
|
||||
SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL,
|
||||
fromJSONL as evalFromJSONL,
|
||||
type EvalExample,
|
||||
} from '../src/eval-dataset.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JSONL round-trip', () => {
|
||||
const examples: EvalExample[] = [
|
||||
{
|
||||
input: 'write a haiku',
|
||||
expected_output: 'Cherry blossoms fall / ...',
|
||||
metadata: { source: 'trace', personaId: 'writer', outcome: 'success' },
|
||||
},
|
||||
{
|
||||
input: 'explain closures',
|
||||
expected_output: 'A closure captures variables from its enclosing scope.',
|
||||
metadata: { source: 'trace', personaId: 'coder', outcome: 'verified' },
|
||||
},
|
||||
];
|
||||
|
||||
it('round-trips through JSONL', () => {
|
||||
const jsonl = evalToJSONL(examples);
|
||||
expect(jsonl.split('\n')).toHaveLength(2);
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toEqual(examples);
|
||||
});
|
||||
|
||||
it('skips malformed lines silently', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify(examples[0]),
|
||||
'{ this is not valid json',
|
||||
'',
|
||||
JSON.stringify(examples[1]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips lines missing required fields', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify({ input: 'x' }), // missing expected_output + metadata
|
||||
JSON.stringify(examples[0]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('EvalDatasetBuilder', () => {
|
||||
let db: MindDB;
|
||||
let store: ExecutionTraceStore;
|
||||
let builder: EvalDatasetBuilder;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ExecutionTraceStore(db);
|
||||
builder = new EvalDatasetBuilder(store);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function seedTrace(input: string, output: string, outcome: 'success' | 'verified' | 'corrected' | 'abandoned' = 'success', personaId = 'coder'): number {
|
||||
const id = store.start({ input, personaId });
|
||||
store.finalize(id, { outcome, output });
|
||||
return id;
|
||||
}
|
||||
|
||||
// ── sourceFromTraces ──
|
||||
|
||||
describe('sourceFromTraces', () => {
|
||||
it('pulls only traces matching the requested outcomes', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'verified');
|
||||
seedTrace('c?', 'C.', 'corrected');
|
||||
seedTrace('d?', 'D.', 'abandoned');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success', 'verified'], false);
|
||||
expect(examples).toHaveLength(2);
|
||||
expect(examples.map(e => e.input).sort()).toEqual(['a?', 'b?']);
|
||||
});
|
||||
|
||||
it('includes corrections when flag set', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'corrected');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
expect(examples).toHaveLength(2);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.metadata.source).toBe('correction');
|
||||
});
|
||||
|
||||
it('maps trace fields onto metadata', () => {
|
||||
seedTrace('hello', 'world', 'success', 'writer');
|
||||
const [example] = builder.sourceFromTraces(['success'], false);
|
||||
expect(example.metadata.personaId).toBe('writer');
|
||||
expect(example.metadata.traceId).toBeGreaterThan(0);
|
||||
expect(example.metadata.source).toBe('trace');
|
||||
});
|
||||
});
|
||||
|
||||
// ── build (end-to-end) ──
|
||||
|
||||
describe('build', () => {
|
||||
it('returns deterministic split with same seed', async () => {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
seedTrace(`question ${i}?`, `answer ${i}.`, 'success');
|
||||
}
|
||||
|
||||
const a = await builder.build({ seed: 42 });
|
||||
const b = await builder.build({ seed: 42 });
|
||||
|
||||
expect(a.train.map(e => e.input)).toEqual(b.train.map(e => e.input));
|
||||
expect(a.val.map(e => e.input)).toEqual(b.val.map(e => e.input));
|
||||
expect(a.holdout.map(e => e.input)).toEqual(b.holdout.map(e => e.input));
|
||||
});
|
||||
|
||||
it('different seeds produce different orderings', async () => {
|
||||
for (let i = 0; i < 30; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const a = await builder.build({ seed: 1 });
|
||||
const b = await builder.build({ seed: 999 });
|
||||
|
||||
const aIds = a.train.map(e => e.input).join('|');
|
||||
const bIds = b.train.map(e => e.input).join('|');
|
||||
expect(aIds).not.toBe(bIds);
|
||||
});
|
||||
|
||||
it('splits 60/20/20 when enough examples', async () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length).toBe(60);
|
||||
expect(split.val.length).toBe(20);
|
||||
expect(split.holdout.length).toBe(20);
|
||||
expect(split.stats.total).toBe(100);
|
||||
});
|
||||
|
||||
it('honors custom split ratios', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1, splitRatios: [0.8, 0.1, 0.1] });
|
||||
expect(split.train.length).toBe(40);
|
||||
expect(split.val.length).toBe(5);
|
||||
expect(split.holdout.length).toBe(5);
|
||||
});
|
||||
|
||||
it('rejects ratios that do not sum to 1', async () => {
|
||||
await expect(builder.build({ splitRatios: [0.5, 0.3, 0.3] })).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('filters out examples containing secrets', async () => {
|
||||
seedTrace('clean input', 'clean output', 'success');
|
||||
seedTrace('here is a key: AKIAIOSFODNN7EXAMPLE', 'sure', 'success');
|
||||
seedTrace('what is your token?', 'ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.sourced).toBe(3);
|
||||
expect(split.stats.acceptedAfterSecretScan).toBe(1);
|
||||
expect(split.rejected.filter(r => r.reason.startsWith('secret:'))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('filters examples by min length', async () => {
|
||||
seedTrace('?', 'ok', 'success'); // below both mins
|
||||
seedTrace('a proper question', 'a proper answer', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, minInputChars: 5, minOutputChars: 3 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-short-input')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters examples over max combined length', async () => {
|
||||
seedTrace('this is an ok prompt', 'this is an ok answer', 'success');
|
||||
seedTrace('x'.repeat(5000), 'y'.repeat(5000), 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, maxCombinedChars: 1000 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-long')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters low-signal garbage', async () => {
|
||||
seedTrace('.......................', '!!!!!!!!!!!!!!!!', 'success');
|
||||
seedTrace('real question?', 'real answer.', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'low-signal')).toBe(true);
|
||||
});
|
||||
|
||||
it('dedupes by input hash', async () => {
|
||||
seedTrace('same prompt', 'answer a', 'success');
|
||||
seedTrace('same prompt', 'answer b', 'success');
|
||||
seedTrace('different prompt', 'answer c', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.unique).toBe(2);
|
||||
expect(split.rejected.some(r => r.reason === 'duplicate')).toBe(true);
|
||||
});
|
||||
|
||||
it('applies optional LLM judge', async () => {
|
||||
seedTrace('keep this example please', 'kept response', 'success');
|
||||
seedTrace('drop this example please', 'dropped response', 'success');
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async (ex) => ({
|
||||
keep: !ex.input.includes('drop'),
|
||||
reason: ex.input.includes('drop') ? 'contains drop' : undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason.startsWith('judge:'))).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps example when judge throws (fail-open)', async () => {
|
||||
seedTrace('some prompt text', 'some answer text', 'success');
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async () => {
|
||||
throw new Error('llm down');
|
||||
},
|
||||
});
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.stats.total).toBe(1);
|
||||
});
|
||||
|
||||
it('accepts external harvest/correction examples', async () => {
|
||||
const harvest: EvalExample = {
|
||||
input: 'harvest prompt',
|
||||
expected_output: 'harvest answer',
|
||||
metadata: { source: 'harvest' },
|
||||
};
|
||||
const correction: EvalExample = {
|
||||
input: 'correction prompt',
|
||||
expected_output: 'correction answer',
|
||||
metadata: { source: 'correction' },
|
||||
};
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
harvestExamples: [harvest],
|
||||
correctionExamples: [correction],
|
||||
});
|
||||
|
||||
const allInputs = [...split.train, ...split.val, ...split.holdout].map(e => e.input);
|
||||
expect(allInputs).toContain('harvest prompt');
|
||||
expect(allInputs).toContain('correction prompt');
|
||||
});
|
||||
|
||||
it('maps correction trace output to correctionFeedback when available', async () => {
|
||||
const id = store.start({ input: 'wrong approach', personaId: 'coder' });
|
||||
store.finalize(id, { outcome: 'success', output: 'first try' });
|
||||
store.markCorrected(id, 'Should use async/await here');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.expected_output).toBe('Should use async/await here');
|
||||
});
|
||||
|
||||
it('returns empty splits when no traces match', async () => {
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(0);
|
||||
expect(split.train).toEqual([]);
|
||||
expect(split.val).toEqual([]);
|
||||
expect(split.holdout).toEqual([]);
|
||||
});
|
||||
|
||||
it('tiny datasets assign leftover to holdout', async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
// Ratios 0.6/0.2/0.2 with n=3 → floor(1.8)=1, floor(0.6)=0, remainder=2 → holdout
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length + split.val.length + split.holdout.length).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,372 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
deployPersonaOverride,
|
||||
rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride,
|
||||
rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides,
|
||||
applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
} from '../src/evolution-deploy.js';
|
||||
import { getPersona, listPersonas, setPersonaDataDir } from '../src/personas.js';
|
||||
import { loadCustomPersonas } from '../src/custom-personas.js';
|
||||
|
||||
// ── Test fixture: temp dir per-test ────────────────────────────
|
||||
|
||||
describe('evolution-deploy', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-deploy-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch { /* Windows lock cleanup racy; ignore */ }
|
||||
});
|
||||
|
||||
// ── deployPersonaOverride ──
|
||||
|
||||
describe('deployPersonaOverride', () => {
|
||||
it('writes a JSON file named {id}.json in personas/', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'You are the evolved coder.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('personas', 'coder.json'))).toBe(true);
|
||||
expect(result.backupPath).toBeNull();
|
||||
expect(result.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('inherits built-in persona fields when targeting a known id', () => {
|
||||
// 'coder' is a built-in persona
|
||||
const builtin = getPersona('coder');
|
||||
expect(builtin).not.toBeNull();
|
||||
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED coder prompt — tighter format.',
|
||||
});
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.id).toBe('coder');
|
||||
expect(loaded.name).toBe(builtin!.name);
|
||||
expect(loaded.icon).toBe(builtin!.icon);
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder prompt — tighter format.');
|
||||
expect(loaded.tools).toEqual(builtin!.tools);
|
||||
});
|
||||
|
||||
it('creates a backup on subsequent writes', () => {
|
||||
const first = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
|
||||
const second = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(fs.existsSync(second.backupPath!)).toBe(true);
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).systemPrompt).toBe('v1');
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).systemPrompt).toBe('v2');
|
||||
});
|
||||
|
||||
it('creates a minimal shell for unknown persona ids', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up-persona',
|
||||
systemPrompt: 'fresh prompt',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.id).toBe('made-up-persona');
|
||||
expect(loaded.name).toBe('made-up-persona');
|
||||
expect(loaded.systemPrompt).toBe('fresh prompt');
|
||||
expect(Array.isArray(loaded.tools)).toBe(true);
|
||||
});
|
||||
|
||||
it('applies caller-supplied overrides (description, icon, tools)', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up',
|
||||
systemPrompt: 'x',
|
||||
overrides: {
|
||||
description: 'custom description',
|
||||
icon: 'brain',
|
||||
tools: ['tool_a', 'tool_b'],
|
||||
},
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.description).toBe('custom description');
|
||||
expect(loaded.icon).toBe('brain');
|
||||
expect(loaded.tools).toEqual(['tool_a', 'tool_b']);
|
||||
});
|
||||
|
||||
it('is picked up by listPersonas after setPersonaDataDir', () => {
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED ROUTING',
|
||||
});
|
||||
|
||||
setPersonaDataDir(tmpDir);
|
||||
try {
|
||||
const personas = listPersonas();
|
||||
const evolved = personas.find(p => p.id === 'coder' && p.systemPrompt.includes('EVOLVED ROUTING'));
|
||||
expect(evolved).toBeDefined();
|
||||
} finally {
|
||||
setPersonaDataDir(''); // reset so other tests aren't polluted
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackPersonaOverride ──
|
||||
|
||||
describe('rollbackPersonaOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const second = deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v2' });
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.systemPrompt).toBe('v1');
|
||||
});
|
||||
|
||||
it('deletes the override entirely when no .bak exists', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadCustomPersonas(tmpDir)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('returns false for non-existent personas', () => {
|
||||
expect(rollbackPersonaOverride(tmpDir, 'ghost')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── deployBehavioralSpecOverride ──
|
||||
|
||||
describe('deployBehavioralSpecOverride', () => {
|
||||
it('writes a JSON file named {section}.json in behavioral-overrides/', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop',
|
||||
text: 'Evolved core loop instructions.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('behavioral-overrides', 'coreLoop.json'))).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.section).toBe('coreLoop');
|
||||
expect(loaded.text).toBe('Evolved core loop instructions.');
|
||||
expect(loaded.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('stores the originating run uuid for audit', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'qualityRules',
|
||||
text: 'new quality rules',
|
||||
runUuid: 'test-run-uuid-123',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.runUuid).toBe('test-run-uuid-123');
|
||||
});
|
||||
|
||||
it('creates backup on subsequent writes', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
});
|
||||
|
||||
it('retries transient Windows rename locks before replacing an override', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync')
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('busy'), { code: 'EBUSY' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked again'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('still busy'), { code: 'EBUSY' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(5);
|
||||
expect(wait).toHaveBeenCalledTimes(4);
|
||||
expect(JSON.parse(fs.readFileSync(second.path, 'utf-8')).text).toBe('v2');
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves the current override when transient rename retries are exhausted', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('still locked'), { code: 'EACCES' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/still locked/);
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(10);
|
||||
expect(wait).toHaveBeenCalledTimes(9);
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('does not retry non-transient rename failures', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('disk error'), { code: 'EIO' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/disk error/);
|
||||
|
||||
expect(rename).toHaveBeenCalledOnce();
|
||||
expect(wait).not.toHaveBeenCalled();
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects unknown sections', () => {
|
||||
expect(() =>
|
||||
deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'notASection' as BehavioralSpecSection,
|
||||
text: 'x',
|
||||
}),
|
||||
).toThrow(/Unknown behavioral-spec section/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackBehavioralSpecOverride ──
|
||||
|
||||
describe('rollbackBehavioralSpecOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v2' });
|
||||
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('v1');
|
||||
});
|
||||
|
||||
it('removes the override when no backup exists', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── loadBehavioralSpecOverrides ──
|
||||
|
||||
describe('loadBehavioralSpecOverrides', () => {
|
||||
it('returns empty object when no overrides exist', () => {
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('returns a section → text map for all written overrides', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'evolved core' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'qualityRules', text: 'evolved quality' });
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('evolved core');
|
||||
expect(overrides.qualityRules).toBe('evolved quality');
|
||||
expect(overrides.behavioralRules).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores malformed override files', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), '{ not valid json');
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('ignores files with empty text', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), JSON.stringify({
|
||||
section: 'coreLoop', text: '', deployedAt: '2026-04-14',
|
||||
}));
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── applyBehavioralSpecOverrides ──
|
||||
|
||||
describe('applyBehavioralSpecOverrides', () => {
|
||||
const baseline: Record<BehavioralSpecSection, string> = {
|
||||
coreLoop: 'base core',
|
||||
qualityRules: 'base quality',
|
||||
behavioralRules: 'base behavioral',
|
||||
workPatterns: 'base patterns',
|
||||
intelligenceDefaults: 'base intelligence',
|
||||
};
|
||||
|
||||
it('returns baseline unchanged when no overrides', () => {
|
||||
expect(applyBehavioralSpecOverrides(baseline, {})).toEqual(baseline);
|
||||
});
|
||||
|
||||
it('overlays provided sections', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: 'EVOLVED core',
|
||||
qualityRules: 'EVOLVED quality',
|
||||
});
|
||||
expect(merged.coreLoop).toBe('EVOLVED core');
|
||||
expect(merged.qualityRules).toBe('EVOLVED quality');
|
||||
expect(merged.behavioralRules).toBe('base behavioral');
|
||||
});
|
||||
|
||||
it('ignores empty/undefined overrides without clobbering baseline', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: '',
|
||||
qualityRules: undefined,
|
||||
});
|
||||
expect(merged.coreLoop).toBe('base core');
|
||||
expect(merged.qualityRules).toBe('base quality');
|
||||
});
|
||||
});
|
||||
|
||||
// ── BEHAVIORAL_SPEC_SECTIONS constant ──
|
||||
|
||||
describe('BEHAVIORAL_SPEC_SECTIONS', () => {
|
||||
it('contains the expected 5 sections in expected order', () => {
|
||||
expect(BEHAVIORAL_SPEC_SECTIONS).toEqual([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules',
|
||||
'workPatterns', 'intelligenceDefaults',
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,245 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
} from '../src/evolution-gates.js';
|
||||
|
||||
describe('checkNonEmpty', () => {
|
||||
it('fails on empty string', () => {
|
||||
expect(checkNonEmpty('').verdict).toBe('fail');
|
||||
});
|
||||
it('fails on whitespace-only string', () => {
|
||||
expect(checkNonEmpty(' \n\n ').verdict).toBe('fail');
|
||||
});
|
||||
it('passes on real content', () => {
|
||||
expect(checkNonEmpty('hello').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkSize', () => {
|
||||
it('passes within the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(2000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.current).toBe(2000);
|
||||
expect(result.detail?.max).toBe(DEFAULT_SIZE_LIMITS.personaSystemPrompt);
|
||||
});
|
||||
|
||||
it('fails over the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(4000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.reason).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('uses tool-description cap (500)', () => {
|
||||
expect(checkSize('x'.repeat(500), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(501), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses skill-body cap (15k)', () => {
|
||||
expect(checkSize('x'.repeat(15_000), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(15_001), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses generic cap for unknown target', () => {
|
||||
const result = checkSize('x'.repeat(5000), 'generic', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('accepts custom size limits', () => {
|
||||
const limits = { ...DEFAULT_SIZE_LIMITS, generic: 100 };
|
||||
const result = checkSize('x'.repeat(200), 'generic', limits);
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkGrowth', () => {
|
||||
it('passes when candidate is smaller than baseline', () => {
|
||||
const result = checkGrowth('abc', 'abcdefgh', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes when growth is within budget', () => {
|
||||
const result = checkGrowth('x'.repeat(110), 'x'.repeat(100), 0.2); // +10%
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('fails when growth exceeds budget', () => {
|
||||
const result = checkGrowth('x'.repeat(140), 'x'.repeat(100), 0.2); // +40%
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.ratio).toBeCloseTo(0.4, 5);
|
||||
});
|
||||
|
||||
it('passes when baseline is empty (cannot compute ratio)', () => {
|
||||
const result = checkGrowth('anything', '', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('boundary: exactly 20% growth passes with default budget', () => {
|
||||
// 100 → 120 is exactly 20%, not greater
|
||||
const result = checkGrowth('x'.repeat(120), 'x'.repeat(100), 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkBalancedFences', () => {
|
||||
it('passes no fences', () => {
|
||||
expect(checkBalancedFences('plain prose').verdict).toBe('pass');
|
||||
});
|
||||
it('passes two fences (one code block)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\n```\ndone';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
it('fails one fence (unbalanced)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\nstill going';
|
||||
const result = checkBalancedFences(text);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.fences).toBe(1);
|
||||
});
|
||||
it('passes four fences (two blocks)', () => {
|
||||
const text = '```a\nfoo\n```\n```b\nbar\n```';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoPlaceholders', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoPlaceholders('A prompt with normal prose.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails [TODO] bracket placeholder', () => {
|
||||
expect(checkNoPlaceholders('Write about [TODO] here.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [PLACEHOLDER]', () => {
|
||||
expect(checkNoPlaceholders('Fill in [PLACEHOLDER].').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [YOUR NAME] style', () => {
|
||||
expect(checkNoPlaceholders('Hi [YOUR NAME HERE]!').verdict).toBe('fail');
|
||||
});
|
||||
it('fails angle-brackets <placeholder>', () => {
|
||||
expect(checkNoPlaceholders('Set <placeholder> before sending.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails handlebars {{var}}', () => {
|
||||
expect(checkNoPlaceholders('Hello {{name}}.').verdict).toBe('fail');
|
||||
});
|
||||
it('does not false-positive normal bracketed citations', () => {
|
||||
expect(checkNoPlaceholders('See [1] and [Smith 2024] for details.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoObviousTodos', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoObviousTodos('Write clear documentation.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails lines starting with TODO:', () => {
|
||||
expect(checkNoObviousTodos('Intro paragraph.\nTODO: finish this').verdict).toBe('fail');
|
||||
});
|
||||
it('fails lines starting with FIXME:', () => {
|
||||
expect(checkNoObviousTodos('FIXME: tighten this up').verdict).toBe('fail');
|
||||
});
|
||||
it('does not fail mid-sentence "todo" mention', () => {
|
||||
expect(checkNoObviousTodos('Build a todo list UI.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkRegression', () => {
|
||||
it('passes when delta is within tolerance', () => {
|
||||
const result = checkRegression(0.8, 0.79, -0.02); // -1pp, floor -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes on improvement', () => {
|
||||
const result = checkRegression(0.7, 0.85, -0.02);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.delta).toBeCloseTo(0.15, 5);
|
||||
});
|
||||
|
||||
it('fails when delta exceeds tolerance', () => {
|
||||
const result = checkRegression(0.9, 0.7, -0.02); // -20pp
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('boundary: equal to tolerance passes', () => {
|
||||
const result = checkRegression(0.8, 0.78, -0.02); // exactly -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
// ── runGates end-to-end ────────────────────────────────────────
|
||||
|
||||
describe('runGates', () => {
|
||||
it('passes a clean candidate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'A concise persona prompt for the researcher.',
|
||||
baseline: 'A concise persona prompt for the researcher.',
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('pass');
|
||||
expect(res.firstFailure).toBeNull();
|
||||
});
|
||||
|
||||
it('fails an empty candidate on the first gate', () => {
|
||||
const res = runGates({ candidate: '', baseline: 'whatever' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('non-empty');
|
||||
});
|
||||
|
||||
it('fails oversized candidate on size gate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(4000),
|
||||
baseline: 'x'.repeat(3000),
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('size');
|
||||
});
|
||||
|
||||
it('fails runaway growth', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('growth');
|
||||
});
|
||||
|
||||
it('fails placeholder leakage', () => {
|
||||
const res = runGates({
|
||||
candidate: 'You are {{persona.name}} — answer questions.',
|
||||
baseline: 'You are the researcher — answer questions.',
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('no-placeholders');
|
||||
});
|
||||
|
||||
it('fails unbalanced markdown fences', () => {
|
||||
// Pad baseline so the growth gate doesn't trip before the fence gate.
|
||||
const padding = ' '.repeat(100);
|
||||
const res = runGates({
|
||||
candidate: `Instructions:\n\`\`\`js\nconst x = 1;\nmore prose${padding}`,
|
||||
baseline: `Instructions: write code clearly.${padding}`,
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('balanced-fences');
|
||||
});
|
||||
|
||||
it('applies the regression gate when scores are supplied', () => {
|
||||
const res = runGates({
|
||||
candidate: 'new prompt',
|
||||
baseline: 'old prompt',
|
||||
scores: { baseline: 0.85, candidate: 0.6 },
|
||||
}, { maxRegression: -0.02 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('regression');
|
||||
});
|
||||
|
||||
it('skips the regression gate when scores are omitted', () => {
|
||||
const res = runGates({ candidate: 'new', baseline: 'old' });
|
||||
expect(res.results.find(r => r.gate === 'regression')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns results for every gate even when one fails', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.results.length).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,6 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
buildReflectiveMutationPrompt,
|
||||
buildSchemaFillPrompt,
|
||||
retryWithBackoff,
|
||||
wrapWithRetry,
|
||||
isRetryableEvolutionError,
|
||||
@@ -15,9 +10,6 @@ import {
|
||||
type RetryOptions,
|
||||
type RetryInfo,
|
||||
} from '../src/evolution-llm-wiring.js';
|
||||
import type { MutateArgs, GEPACandidate } from '../src/index.js';
|
||||
import type { Schema } from '../src/evolve-schema.js';
|
||||
import type { JudgeInput } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
|
||||
return {
|
||||
id: 'g1-m0',
|
||||
prompt: 'You are a tester. Respond with the exact expected word.',
|
||||
generation: 1,
|
||||
parent: 'g0-baseline',
|
||||
strategy: 'expand-edge-cases',
|
||||
score: null,
|
||||
perExample: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(overrides: Partial<Schema> = {}): Schema {
|
||||
return {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
|
||||
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
|
||||
],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── buildJudgeLLMCall ─────────────────────────────────────────────
|
||||
|
||||
describe('buildJudgeLLMCall', () => {
|
||||
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildGEPAMutateFn ─────────────────────────────────────────────
|
||||
|
||||
describe('buildGEPAMutateFn', () => {
|
||||
const mutateArgs: MutateArgs = {
|
||||
parent: makeCandidate(),
|
||||
strategy: 'expand-edge-cases',
|
||||
weaknessFeedback: [
|
||||
'Fails to handle empty input',
|
||||
'Sometimes forgets to include the final answer',
|
||||
],
|
||||
targetKind: 'persona-system-prompt',
|
||||
generation: 2,
|
||||
};
|
||||
|
||||
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe('EVOLVED SYSTEM PROMPT');
|
||||
expect(calls).toHaveLength(1);
|
||||
const prompt = calls[0];
|
||||
expect(prompt).toContain(mutateArgs.parent.prompt);
|
||||
expect(prompt).toContain(mutateArgs.strategy);
|
||||
expect(prompt).toContain('Fails to handle empty input');
|
||||
expect(prompt).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
|
||||
const { llm } = makeMockLLM(() => ' \n ');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('strips markdown code fences from the LLM response', async () => {
|
||||
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result.trim()).toBe('HIDDEN PROMPT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildReflectiveMutationPrompt', () => {
|
||||
it('includes the generation number and weakness bullets', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'tighten-format',
|
||||
weaknessFeedback: ['a', 'b'],
|
||||
targetKind: 'behavioral-spec-section',
|
||||
generation: 3,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('generation 3');
|
||||
expect(prompt).toContain('- a');
|
||||
expect(prompt).toContain('- b');
|
||||
expect(prompt).toContain('tighten-format');
|
||||
expect(prompt).toContain('PARENT');
|
||||
});
|
||||
|
||||
it('handles missing weakness feedback gracefully', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'add-examples',
|
||||
weaknessFeedback: [],
|
||||
targetKind: 'generic',
|
||||
generation: 0,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('PARENT');
|
||||
expect(prompt).toContain('(no specific weakness signals yet)');
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildSchemaExecuteFn ──────────────────────────────────────────
|
||||
|
||||
describe('buildSchemaExecuteFn', () => {
|
||||
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
|
||||
expect(calls[0]).toContain('What is the answer?');
|
||||
expect(calls[0]).toContain('"reasoning"');
|
||||
expect(calls[0]).toContain('"answer"');
|
||||
});
|
||||
|
||||
it('returns parsed=false when LLM returns non-JSON', async () => {
|
||||
const { llm } = makeMockLLM(() => 'just some text');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('just some text');
|
||||
});
|
||||
|
||||
it('extracts JSON from markdown fences', async () => {
|
||||
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toContain('"reasoning"');
|
||||
});
|
||||
|
||||
it('returns parsed=false and empty actual when LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSchemaFillPrompt', () => {
|
||||
it('serializes each field with name + type + description', () => {
|
||||
const prompt = buildSchemaFillPrompt({
|
||||
schema: makeSchema(),
|
||||
input: 'What is 2+2?',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('"reasoning"');
|
||||
expect(prompt).toContain('"answer"');
|
||||
expect(prompt).toContain('step-by-step thinking');
|
||||
expect(prompt).toContain('the final answer');
|
||||
expect(prompt).toContain('What is 2+2?');
|
||||
});
|
||||
|
||||
it('notes required constraints', () => {
|
||||
const schema: Schema = {
|
||||
name: 'x', version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'score',
|
||||
type: 'number',
|
||||
description: 'a rating',
|
||||
required: true,
|
||||
constraints: [{ kind: 'range', value: '0-10' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
|
||||
expect(prompt).toContain('score');
|
||||
expect(prompt).toContain('number');
|
||||
expect(prompt).toContain('range');
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeRunningJudge ──────────────────────────────────────────────
|
||||
|
||||
describe('makeRunningJudge', () => {
|
||||
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
|
||||
const { llm, calls } = makeMockLLM((prompt) => {
|
||||
// Simulate running the candidate prompt and returning a model response.
|
||||
return `MODEL OUTPUT for: ${prompt}`;
|
||||
});
|
||||
|
||||
const baseScore = vi.fn(async (args: JudgeInput) => ({
|
||||
overall: 0.8, weighted: 0.8,
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
lengthPenalty: 1, feedback: 'passed through', parsed: true,
|
||||
}));
|
||||
const baseJudge = { score: baseScore };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'What is love?',
|
||||
expected: 'baby don\'t hurt me',
|
||||
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
|
||||
});
|
||||
|
||||
// Model was asked to run the candidate prompt against the example input.
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
|
||||
expect(calls[0]).toContain('What is love?');
|
||||
|
||||
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
|
||||
expect(baseScore).toHaveBeenCalledTimes(1);
|
||||
const passedArgs = baseScore.mock.calls[0][0];
|
||||
expect(passedArgs.input).toBe('What is love?');
|
||||
expect(passedArgs.expected).toBe('baby don\'t hurt me');
|
||||
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
|
||||
|
||||
expect(result.overall).toBe(0.8);
|
||||
});
|
||||
|
||||
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('rate limited'); },
|
||||
};
|
||||
const baseJudge = { score: vi.fn() };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'any', expected: 'any', actual: 'prompt',
|
||||
});
|
||||
|
||||
expect(result.overall).toBe(0);
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.feedback).toMatch(/execution failed/i);
|
||||
expect(baseJudge.score).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes context through to the base judge', async () => {
|
||||
const { llm } = makeMockLLM(() => 'EXECUTED');
|
||||
const baseScore = vi.fn(async () => ({
|
||||
overall: 0.5, weighted: 0.5,
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
lengthPenalty: 1, feedback: '', parsed: true,
|
||||
}));
|
||||
|
||||
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
|
||||
|
||||
await runningJudge.score({
|
||||
input: 'q', expected: 'a', actual: 'prompt',
|
||||
context: 'persona:coder',
|
||||
});
|
||||
|
||||
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
|
||||
});
|
||||
});
|
||||
|
||||
// ── isRetryableEvolutionError ─────────────────────────────────────
|
||||
|
||||
describe('isRetryableEvolutionError', () => {
|
||||
|
||||
@@ -1,447 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
eligibleForEvolution,
|
||||
summarizeRuns,
|
||||
} from '../src/evolution-orchestrator.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
|
||||
import type { MutateFn } from '../src/iterative-optimizer.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(n => ({
|
||||
name: n, type: 'string' as const,
|
||||
description: `${n} field`,
|
||||
required: true, constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Executor that produces longer output for bigger schemas.
|
||||
function makeExec(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Judge that prefers candidates containing the "plus" token so the
|
||||
// mutateAppend helper ("... plus") wins reliably. Creates a measurable
|
||||
// delta between baseline and winner regardless of absolute lengths.
|
||||
function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
|
||||
return {
|
||||
async score(args) {
|
||||
const base = Math.min(1, args.actual.length / 100) * 0.6;
|
||||
const bonus = args.actual.includes('plus') ? 0.3 : 0;
|
||||
return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Mutate appends ~10 chars so growth ratio stays below the default +20%.
|
||||
const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
|
||||
|
||||
// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
|
||||
const BASELINE =
|
||||
'Return a concise answer that directly addresses the question. ' +
|
||||
'Provide a brief explanation and the final answer. Be clear and accurate.';
|
||||
|
||||
function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
|
||||
for (let i = 0; i < n; i++) {
|
||||
const id = store.start({
|
||||
personaId,
|
||||
input: `test question ${i} with enough length to pass filters`,
|
||||
workspaceId: 'ws-1',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
store.finalize(id, {
|
||||
outcome: 'success',
|
||||
output: `a full answer for question ${i} that is long enough for use`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function baseComposeOptions() {
|
||||
return {
|
||||
schema: {
|
||||
examples: [] as never[],
|
||||
execute: makeExec(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
examples: [] as never[],
|
||||
judge: makeJudge(),
|
||||
mutate: mutateAppend,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── eligibleForEvolution ──
|
||||
|
||||
describe('eligibleForEvolution', () => {
|
||||
it('keeps finalized traces with non-empty input and output', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, session_id: 's', persona_id: null, workspace_id: null,
|
||||
model: null, task_shape: null, outcome: 'success',
|
||||
cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
|
||||
payload: {
|
||||
input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('drops pending traces', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'pending',
|
||||
payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops traces with empty input', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'success',
|
||||
payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'corrected',
|
||||
payload: {
|
||||
input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
correctionFeedback: 'use bullets',
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── summarizeRuns ──
|
||||
|
||||
describe('summarizeRuns', () => {
|
||||
it('returns zero-aggregates on empty input', () => {
|
||||
const s = summarizeRuns([]);
|
||||
expect(s.total).toBe(0);
|
||||
expect(s.byStatus.proposed).toBe(0);
|
||||
expect(s.bestDelta).toBe(0);
|
||||
});
|
||||
|
||||
it('counts by status and target kind, tracks best delta', () => {
|
||||
const runs = [
|
||||
{ status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
|
||||
{ status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
|
||||
{ status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
|
||||
{ status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
|
||||
] as unknown as EvolutionRun[];
|
||||
const s = summarizeRuns(runs);
|
||||
expect(s.total).toBe(4);
|
||||
expect(s.byStatus.proposed).toBe(1);
|
||||
expect(s.byStatus.accepted).toBe(1);
|
||||
expect(s.byStatus.rejected).toBe(1);
|
||||
expect(s.byStatus.deployed).toBe(1);
|
||||
expect(s.byTargetKind['persona-system-prompt']).toBe(3);
|
||||
expect(s.byTargetKind['tool-description']).toBe(1);
|
||||
expect(s.bestDelta).toBeCloseTo(0.12, 5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolutionOrchestrator end-to-end ───────────────────────────
|
||||
|
||||
describe('EvolutionOrchestrator', () => {
|
||||
let db: MindDB;
|
||||
let traceStore: ExecutionTraceStore;
|
||||
let runStore: EvolutionRunStore;
|
||||
let orchestrator: EvolutionOrchestrator;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
traceStore = new ExecutionTraceStore(db);
|
||||
runStore = new EvolutionRunStore(db);
|
||||
orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── runOnce ──
|
||||
|
||||
describe('runOnce', () => {
|
||||
it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.run?.status).toBe('proposed');
|
||||
expect(result.run?.delta_accuracy).toBeGreaterThan(0);
|
||||
expect(result.compose).toBeDefined();
|
||||
expect(result.gateResults).toBeDefined();
|
||||
});
|
||||
|
||||
it('persists compose artifacts (schema JSON, gates, delta)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const run = result.run!;
|
||||
expect(run.winner_schema_json).toBeTruthy();
|
||||
const schema = JSON.parse(run.winner_schema_json!);
|
||||
expect(schema.name).toBe('test');
|
||||
expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('skips with skipped-delta when improvement is below threshold', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'a reasonably detailed baseline prompt that already scores well',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
// Mutate returns the same text → zero delta
|
||||
mutate: async ({ parent }) => parent.prompt,
|
||||
},
|
||||
},
|
||||
minDelta: 0.5, // artificially high
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-delta');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: skips when too few traces', async () => {
|
||||
seedSuccessfulTraces(traceStore, 2);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 50 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-trigger');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: proceeds when threshold is met', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 5 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
});
|
||||
|
||||
it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
// Judge that prefers the mutated "y" text — ensures the GEPA winner
|
||||
// is the oversized candidate, not the baseline.
|
||||
const yPreferringJudge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
|
||||
},
|
||||
};
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'tool-description',
|
||||
baseline: 'x'.repeat(30),
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
judge: yPreferringJudge,
|
||||
// Mutate produces candidate that blows past tool-description cap (500)
|
||||
mutate: async () => 'y'.repeat(2000),
|
||||
},
|
||||
},
|
||||
minDelta: 0, // allow any delta so we reach the gate stage
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-gates');
|
||||
expect(result.run?.status).toBe('rejected');
|
||||
expect(result.run?.gate_verdict).toBe('fail');
|
||||
expect(result.run?.user_note).toMatch(/gate failure/);
|
||||
});
|
||||
|
||||
it('aborts cleanly when signal fires before start', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.outcome).toBe('aborted');
|
||||
});
|
||||
|
||||
it('emits progress events', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const phases: string[] = [];
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('compose');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
});
|
||||
|
||||
// ── accept ──
|
||||
|
||||
describe('accept', () => {
|
||||
it('invokes deploy callback and marks run deployed on success', async () => {
|
||||
const deploy = vi.fn(async () => { /* success */ });
|
||||
const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await deployedOrch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
expect(run.outcome).toBe('proposed');
|
||||
|
||||
const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
|
||||
expect(accepted?.status).toBe('deployed');
|
||||
expect(deploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy callback throws', async () => {
|
||||
const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
|
||||
const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await orch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const result = await orch.accept(run.run!.run_uuid);
|
||||
expect(result?.status).toBe('failed');
|
||||
expect(result?.failure_reason).toContain('persona write failed');
|
||||
});
|
||||
|
||||
it('leaves run as accepted when no deploy hook configured', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const accepted = await orchestrator.accept(run.run!.run_uuid);
|
||||
expect(accepted?.status).toBe('accepted');
|
||||
});
|
||||
|
||||
it('returns undefined for unknown uuid', async () => {
|
||||
expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── reject ──
|
||||
|
||||
describe('reject', () => {
|
||||
it('marks proposed run as rejected', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
|
||||
expect(rejected?.status).toBe('rejected');
|
||||
expect(rejected?.user_note).toBe('regressed on custom cases');
|
||||
});
|
||||
});
|
||||
|
||||
// ── list / get ──
|
||||
|
||||
describe('list / get', () => {
|
||||
it('returns runs in reverse-chronological order', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'a',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'b',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const list = orchestrator.list();
|
||||
expect(list.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('get returns a specific run by uuid', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const created = await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'short',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const fetched = orchestrator.get(created.run!.run_uuid);
|
||||
expect(fetched?.id).toBe(created.run!.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,545 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
EvolveSchema,
|
||||
addOutputField,
|
||||
removeField,
|
||||
editFieldDescription,
|
||||
changeFieldType,
|
||||
addConstraint,
|
||||
removeConstraint,
|
||||
reorderFields,
|
||||
replaceOutputFields,
|
||||
schemaComplexity,
|
||||
aggregateSchemaScores,
|
||||
paretoFrontSchema,
|
||||
pickSchemaWinner,
|
||||
generateStructureMutations,
|
||||
generateOrderMutations,
|
||||
generateRefinementMutations,
|
||||
pickSample,
|
||||
type Schema,
|
||||
type SchemaField,
|
||||
type SchemaCandidate,
|
||||
type SchemaCandidateScore,
|
||||
type SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeField(name: string, partial: Partial<SchemaField> = {}): SchemaField {
|
||||
return {
|
||||
name,
|
||||
type: 'string',
|
||||
description: `description for ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(f => makeField(f)),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, schema: Schema, score: SchemaCandidateScore | null): SchemaCandidate {
|
||||
return {
|
||||
id,
|
||||
schema,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake executor: returns a deterministic output string that scores higher
|
||||
// when schema has more fields (up to a cap) and scores lower when schema
|
||||
// is bloated beyond 5 fields.
|
||||
function makeFakeExecutor(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more'): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => `${f.name}=mock`).join('; '),
|
||||
parsed: schema.fields.length > 0,
|
||||
});
|
||||
}
|
||||
|
||||
function makeJudge(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const fieldCount = (args.actual.match(/=/g) || []).length;
|
||||
let overall: number;
|
||||
if (bias === 'prefers-more') {
|
||||
// More fields = better, up to 6 fields, then flat
|
||||
overall = Math.min(1, fieldCount / 6);
|
||||
} else if (bias === 'prefers-less') {
|
||||
overall = Math.max(0, 1 - fieldCount / 10);
|
||||
} else {
|
||||
overall = 0.5;
|
||||
}
|
||||
return makeJudgeScore(overall);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ────────────────────────────────────
|
||||
|
||||
describe('addOutputField', () => {
|
||||
it('appends when no position given', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = addOutputField(s, makeField('c'));
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('inserts at given position', () => {
|
||||
const s = makeSchema(['a', 'c']);
|
||||
const next = addOutputField(s, makeField('b'), 1);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('does not mutate source', () => {
|
||||
const s = makeSchema(['a']);
|
||||
addOutputField(s, makeField('b'));
|
||||
expect(s.fields.map(f => f.name)).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeField', () => {
|
||||
it('drops the named field', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = removeField(s, 'b');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = removeField(s, 'z');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('editFieldDescription', () => {
|
||||
it('updates the description of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = editFieldDescription(s, 'a', 'new');
|
||||
expect(next.fields[0].description).toBe('new');
|
||||
});
|
||||
});
|
||||
|
||||
describe('changeFieldType', () => {
|
||||
it('updates the type of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = changeFieldType(s, 'a', 'number');
|
||||
expect(next.fields[0].type).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('addConstraint / removeConstraint', () => {
|
||||
it('adds a constraint', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'a', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes the given constraint index', () => {
|
||||
const s = addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 });
|
||||
const next = removeConstraint(s, 'a', 0);
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field when adding', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'z', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reorderFields', () => {
|
||||
it('reorders by name', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['c', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
|
||||
it('appends fields omitted from newOrder', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['b']);
|
||||
// 'b' first, then the rest in original order
|
||||
expect(next.fields.map(f => f.name)).toEqual(['b', 'a', 'c']);
|
||||
});
|
||||
|
||||
it('ignores unknown field names in newOrder', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = reorderFields(s, ['unknown', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('replaceOutputFields', () => {
|
||||
it('replaces all fields', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = replaceOutputFields(s, [makeField('x'), makeField('y')]);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['x', 'y']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Complexity + scoring ───────────────────────────────────────
|
||||
|
||||
describe('schemaComplexity', () => {
|
||||
it('returns 0 for empty schema', () => {
|
||||
expect(schemaComplexity(makeSchema([]))).toBe(0);
|
||||
});
|
||||
|
||||
it('scales with field count, constraint count, and description length', () => {
|
||||
const simple = schemaComplexity(makeSchema(['a']));
|
||||
const twoField = schemaComplexity(makeSchema(['a', 'b']));
|
||||
expect(twoField).toBeGreaterThan(simple);
|
||||
|
||||
const withConstraint = schemaComplexity(
|
||||
addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 }),
|
||||
);
|
||||
expect(withConstraint).toBeGreaterThan(simple);
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregateSchemaScores', () => {
|
||||
it('returns zero aggregate on empty results', () => {
|
||||
const agg = aggregateSchemaScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.accuracy).toBe(0);
|
||||
expect(agg.parseRate).toBe(0);
|
||||
});
|
||||
|
||||
it('averages accuracy + computes parse rate', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.8), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.6), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'X', score: makeJudgeScore(0.2), parsed: false },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.n).toBe(3);
|
||||
expect(agg.accuracy).toBeCloseTo((0.8 + 0.6 + 0.2) / 3, 5);
|
||||
expect(agg.parseRate).toBeCloseTo(2 / 3, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 example feedback', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.9, 'best'), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.1, 'worst'), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'C', score: makeJudgeScore(0.5, 'middle'), parsed: true },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('middle');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pareto ─────────────────────────────────────────────────────
|
||||
|
||||
describe('paretoFrontSchema', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('removes strictly dominated candidates', () => {
|
||||
const dominated = makeCandidate('d', makeSchema(['a']), makeScore(0.5, 5));
|
||||
const better = makeCandidate('b', makeSchema(['a']), makeScore(0.8, 3));
|
||||
const front = paretoFrontSchema([dominated, better]);
|
||||
expect(front.map(c => c.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates (accurate vs simple)', () => {
|
||||
const accurate = makeCandidate('acc', makeSchema(['a', 'b']), makeScore(0.9, 10));
|
||||
const simple = makeCandidate('sim', makeSchema(['a']), makeScore(0.7, 2));
|
||||
const front = paretoFrontSchema([accurate, simple]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores unscored candidates', () => {
|
||||
const scored = makeCandidate('s', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const unscored = makeCandidate('u', makeSchema(['a']), null);
|
||||
const front = paretoFrontSchema([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSchemaWinner', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('picks highest accuracy', () => {
|
||||
const lo = makeCandidate('lo', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const hi = makeCandidate('hi', makeSchema(['a']), makeScore(0.9, 10));
|
||||
expect(pickSchemaWinner([lo, hi]).id).toBe('hi');
|
||||
});
|
||||
|
||||
it('ties on accuracy → prefers lower complexity', () => {
|
||||
const complex = makeCandidate('cx', makeSchema(['a', 'b']), makeScore(0.8, 10));
|
||||
const simple = makeCandidate('sm', makeSchema(['a']), makeScore(0.8, 3));
|
||||
expect(pickSchemaWinner([complex, simple]).id).toBe('sm');
|
||||
});
|
||||
|
||||
it('throws on empty', () => {
|
||||
expect(() => pickSchemaWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Mutation generators ────────────────────────────────────────
|
||||
|
||||
describe('generateStructureMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('suggests adding reasoning if absent', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.description.includes('reasoning'))).toBe(true);
|
||||
});
|
||||
|
||||
it('does not re-suggest reasoning if already present', () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
const hasReasoning = muts.some(m => m.description.includes('reasoning field'));
|
||||
expect(hasReasoning).toBe(false);
|
||||
});
|
||||
|
||||
it('suggests drop when schema is large', () => {
|
||||
const s = makeSchema(['a', 'b', 'c', 'd', 'e']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.kind === 'remove_field')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns at most n mutations', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 1, rng);
|
||||
expect(muts.length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateOrderMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty for 1-field schemas', () => {
|
||||
expect(generateOrderMutations(makeSchema(['a']), 3, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('moves a reasoning field to the front', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('answer'), makeField('reasoning'), makeField('confidence')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const applied = muts[0].apply(s);
|
||||
expect(applied.fields[0].name).toBe('reasoning');
|
||||
});
|
||||
|
||||
it('moves a confidence field to the end', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('confidence'), makeField('reasoning'), makeField('answer')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const last = muts[0].apply(s);
|
||||
// One of the generated mutations should put 'confidence' at the end
|
||||
const someMovesConfidenceToEnd = muts.some(m => {
|
||||
const applied = m.apply(s);
|
||||
return applied.fields[applied.fields.length - 1].name === 'confidence';
|
||||
});
|
||||
expect(someMovesConfidenceToEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateRefinementMutations', () => {
|
||||
it('adds maxLength when feedback mentions verbosity', async () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['too verbose, output wordy'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'add_constraint')).toBe(true);
|
||||
});
|
||||
|
||||
it('adds minLength when feedback mentions incompleteness', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['response is too brief, missing detail'], 5,
|
||||
);
|
||||
expect(muts.some(m =>
|
||||
m.description.includes('minLength'),
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
it('edits field descriptions when feedback mentions format issues', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format, could not parse'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'edit_field_desc')).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back to a single description rewrite when no heuristic matches', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(s, ['something unrelated'], 5);
|
||||
expect(muts.length).toBeGreaterThan(0);
|
||||
expect(muts[0].kind).toBe('edit_field_desc');
|
||||
});
|
||||
|
||||
it('uses LLM-provided editor when supplied', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format'], 5,
|
||||
async ({ field }) => `LLM-REWRITE of ${field.name}`,
|
||||
);
|
||||
const edit = muts.find(m => m.kind === 'edit_field_desc');
|
||||
const applied = edit!.apply(s);
|
||||
expect(applied.fields[0].description).toBe('LLM-REWRITE of answer');
|
||||
});
|
||||
});
|
||||
|
||||
// ── pickSample ─────────────────────────────────────────────────
|
||||
|
||||
describe('pickSample', () => {
|
||||
it('returns up to k items', () => {
|
||||
expect(pickSample(makeExamples(5), 3, () => 0.5)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(5), 0, () => 0.5)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolveSchema.run end-to-end ───────────────────────────────
|
||||
|
||||
describe('EvolveSchema.run', () => {
|
||||
it('runs through all phases and returns a winner', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('winner accuracy >= baseline accuracy when judge prefers richer schemas', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner.score!.accuracy).toBeGreaterThanOrEqual(
|
||||
result.history[0].score!.accuracy,
|
||||
);
|
||||
});
|
||||
|
||||
it('emits progress events for each phase', async () => {
|
||||
const phases: string[] = [];
|
||||
await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('start');
|
||||
expect(phases).toContain('structure');
|
||||
expect(phases).toContain('order');
|
||||
expect(phases).toContain('refinement');
|
||||
expect(phases).toContain('anchor');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const cfg = () => ({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 8,
|
||||
seed: 7,
|
||||
});
|
||||
const a = await new EvolveSchema().run(cfg());
|
||||
const b = await new EvolveSchema().run(cfg());
|
||||
expect(a.winner.schema.fields.map(f => f.name)).toEqual(
|
||||
b.winner.schema.fields.map(f => f.name),
|
||||
);
|
||||
});
|
||||
|
||||
it('respects abort signal', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
|
||||
it('records all candidates in history with generation numbers', async () => {
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
mutationMix: { structure: 2, order: 0, refinement: 0 },
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
const gen0 = result.history.filter(c => c.generation === 0);
|
||||
const gen1 = result.history.filter(c => c.generation === 1);
|
||||
expect(gen0.length).toBe(1);
|
||||
expect(gen1.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -1,226 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
recordCapabilityGap,
|
||||
analyzeAndRecordCorrection,
|
||||
recordWorkflowPattern,
|
||||
buildAwarenessSummary,
|
||||
formatAwarenessPrompt,
|
||||
markSummarySurfaced,
|
||||
} from '../src/improvement-detector.js';
|
||||
|
||||
describe('improvement-detector', () => {
|
||||
let db: MindDB;
|
||||
let store: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── recordCapabilityGap ────────────────────────────────────
|
||||
|
||||
describe('recordCapabilityGap', () => {
|
||||
it('records a capability gap signal', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader', 'user asked to read a PDF');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].pattern_key).toBe('missing:pdf_reader');
|
||||
expect(gaps[0].count).toBe(1);
|
||||
});
|
||||
|
||||
it('increments count on repeated gap', () => {
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].count).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── analyzeAndRecordCorrection ─────────────────────────────
|
||||
|
||||
describe('analyzeAndRecordCorrection', () => {
|
||||
it('returns null for non-correction messages', () => {
|
||||
const result = analyzeAndRecordCorrection(store, 'Please write a sorting function');
|
||||
expect(result).toBeNull();
|
||||
expect(store.getByCategory('correction')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('detects and records a correction', () => {
|
||||
const result = analyzeAndRecordCorrection(
|
||||
store,
|
||||
"No, that's wrong. I told you to use markdown headers.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBeDefined();
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records both durable and task-local corrections', () => {
|
||||
analyzeAndRecordCorrection(store, "No, always use shorter responses. I said keep it brief.");
|
||||
analyzeAndRecordCorrection(store, "No, not that. Just use a list this time.");
|
||||
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordWorkflowPattern ──────────────────────────────────
|
||||
|
||||
describe('recordWorkflowPattern', () => {
|
||||
it('records a workflow pattern', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].pattern_key).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('increments on repeated task shapes', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
recordWorkflowPattern(store, 'research', 'investigate market trends');
|
||||
recordWorkflowPattern(store, 'research', 'review latest papers');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].count).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildAwarenessSummary ──────────────────────────────────
|
||||
|
||||
describe('buildAwarenessSummary', () => {
|
||||
it('returns empty summary when no actionable signals', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBe(0);
|
||||
expect(summary.capabilityGaps).toEqual([]);
|
||||
expect(summary.corrections).toEqual([]);
|
||||
expect(summary.workflowPatterns).toEqual([]);
|
||||
});
|
||||
|
||||
it('includes capability gaps above threshold', () => {
|
||||
// Default threshold for capability_gap is 2
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
expect(summary.capabilityGaps[0].toolName).toBe('pdf_reader');
|
||||
expect(summary.capabilityGaps[0].occurrences).toBe(2);
|
||||
expect(summary.capabilityGaps[0].suggestion).toContain('pdf_reader');
|
||||
});
|
||||
|
||||
it('includes corrections above threshold', () => {
|
||||
// Default threshold for correction is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep responses casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.corrections).toHaveLength(1);
|
||||
expect(summary.corrections[0].patternKey).toBe('tone:too_formal');
|
||||
expect(summary.corrections[0].occurrences).toBe(3);
|
||||
});
|
||||
|
||||
it('includes recent workflow patterns above threshold', () => {
|
||||
// Default threshold for workflow_pattern is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'research', 'investigate topic');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(1);
|
||||
expect(summary.workflowPatterns[0].patternKey).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('excludes stale workflow patterns', () => {
|
||||
// Manually insert a workflow_pattern with old last_seen
|
||||
const raw = db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO improvement_signals (category, pattern_key, count, last_seen)
|
||||
VALUES ('workflow_pattern', 'shape:old_task', 5, datetime('now', '-30 days'))
|
||||
`).run();
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('respects total cap of 3 actionable signals', () => {
|
||||
// Create many signals above threshold
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const key = `missing:tool_${i}`;
|
||||
store.record('capability_gap', key);
|
||||
store.record('capability_gap', key);
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── formatAwarenessPrompt ──────────────────────────────────
|
||||
|
||||
describe('formatAwarenessPrompt', () => {
|
||||
it('returns null for empty summary', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(formatAwarenessPrompt(summary)).toBeNull();
|
||||
});
|
||||
|
||||
it('formats capability gaps into prompt text', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).not.toBeNull();
|
||||
expect(prompt).toContain('## Improvement Signals');
|
||||
expect(prompt).toContain('pdf_reader');
|
||||
expect(prompt).toContain('Missing capabilities');
|
||||
});
|
||||
|
||||
it('formats corrections into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep it casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Behavioral adjustments');
|
||||
expect(prompt).toContain('casual');
|
||||
});
|
||||
|
||||
it('formats workflow patterns into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'compare', 'compare options');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Recurring workflows');
|
||||
expect(prompt).toContain('compare');
|
||||
});
|
||||
});
|
||||
|
||||
// ── markSummarySurfaced ────────────────────────────────────
|
||||
|
||||
describe('markSummarySurfaced', () => {
|
||||
it('marks all signals in summary as surfaced', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
|
||||
markSummarySurfaced(store, summary);
|
||||
|
||||
// Should not appear in next summary
|
||||
const summary2 = buildAwarenessSummary(store);
|
||||
expect(summary2.capabilityGaps).toHaveLength(0);
|
||||
expect(summary2.totalActionable).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,153 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { processInteractionForImprovement } from '../src/improvement-wiring.js';
|
||||
|
||||
describe('improvement-wiring', () => {
|
||||
const baseParams = {
|
||||
workspaceId: 'ws-1',
|
||||
sessionId: 'sess-1',
|
||||
toolsUsed: [],
|
||||
agentResponse: 'Here is the result.',
|
||||
userMessage: '',
|
||||
};
|
||||
|
||||
// ── Correction detection ─────────────────────────────────────
|
||||
|
||||
describe('correction detection', () => {
|
||||
it('detects corrections in user messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use bullet points, not paragraphs.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.correctionDetail).toBeDefined();
|
||||
expect(result.correctionDetail!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for normal messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you help me write a function to sort an array?',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.correctionDetail).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for very short messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'ok',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
});
|
||||
|
||||
it('detects "I said" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'I said to use markdown headers, not plain text.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
|
||||
it('detects "actually instead use" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Actually, instead use TypeScript for this module.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Capability gap detection ─────────────────────────────────
|
||||
|
||||
describe('capability gap detection', () => {
|
||||
it('identifies capability gaps from tool-not-found in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Read this PDF for me.',
|
||||
agentResponse: 'Tool "pdf_reader" not found. I cannot read PDFs directly.',
|
||||
});
|
||||
expect(result.capabilityGap).toBe('pdf_reader');
|
||||
});
|
||||
|
||||
it('identifies capability gaps from "I don\'t have a tool" in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send this to Slack?',
|
||||
agentResponse: "I don't have a tool for sending Slack messages.",
|
||||
});
|
||||
expect(result.capabilityGap).toBe('sending Slack messages');
|
||||
});
|
||||
|
||||
it('identifies gaps from user requesting integration', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send to Slack the summary?',
|
||||
agentResponse: 'Let me try to do that.',
|
||||
toolsUsed: [],
|
||||
});
|
||||
expect(result.capabilityGap).toBe('slack');
|
||||
});
|
||||
|
||||
it('returns no gap for normal interactions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the weather today?',
|
||||
agentResponse: 'The weather today is sunny and 72F.',
|
||||
});
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Workflow pattern detection ───────────────────────────────
|
||||
|
||||
describe('workflow pattern detection', () => {
|
||||
it('detects multi-step workflow patterns with "then"', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'First research the competitors, then compare their features, then draft a summary report.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeDefined();
|
||||
});
|
||||
|
||||
it('does not flag single-step requests as workflow patterns', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Write a summary of this document.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for simple questions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the status of the project?',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Combined signals ─────────────────────────────────────────
|
||||
|
||||
describe('combined signals', () => {
|
||||
it('can detect correction and gap simultaneously', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use the Jira tool.",
|
||||
agentResponse: 'Tool "jira_create" not found.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.capabilityGap).toBe('jira_create');
|
||||
});
|
||||
|
||||
it('returns all-false for benign exchange', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Thanks, that looks great!',
|
||||
agentResponse: "You're welcome! Let me know if you need anything else.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,655 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
paretoFront,
|
||||
scoreCandidate,
|
||||
aggregateScores,
|
||||
pickWinner,
|
||||
pickSample,
|
||||
type Candidate,
|
||||
type CandidateScore,
|
||||
type IterativeGEPAOptions,
|
||||
type MutateFn,
|
||||
} from '../src/iterative-optimizer.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
/**
|
||||
* H-09 G3 — unit tests mock the judge, so they opt into the running-judge
|
||||
* safety-check bypass. Production code paths must still wrap with
|
||||
* makeRunningJudge(); the check in IterativeGEPA enforces that.
|
||||
*/
|
||||
function runGEPA(options: Omit<IterativeGEPAOptions, 'allowBareJudge'>) {
|
||||
const gepa = new IterativeGEPA();
|
||||
return gepa.run({ ...options, allowBareJudge: true });
|
||||
}
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `question ${i}`,
|
||||
expected_output: `answer ${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeScore(overall: number, extra: Partial<JudgeScore> = {}): JudgeScore {
|
||||
return {
|
||||
overall,
|
||||
weighted: overall,
|
||||
correctness: extra.correctness ?? overall,
|
||||
procedureFollowing: extra.procedureFollowing ?? overall,
|
||||
conciseness: extra.conciseness ?? overall,
|
||||
lengthPenalty: extra.lengthPenalty ?? 1,
|
||||
feedback: extra.feedback ?? `feedback for score ${overall}`,
|
||||
parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, score: CandidateScore | null): Candidate {
|
||||
return {
|
||||
id,
|
||||
prompt: `prompt-${id}`,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidateScore(
|
||||
overall: number,
|
||||
dims: Partial<{ correctness: number; procedureFollowing: number; conciseness: number; lengthPenalty: number; n: number; weaknessFeedback: string[] }> = {},
|
||||
): CandidateScore {
|
||||
return {
|
||||
overall,
|
||||
correctness: dims.correctness ?? overall,
|
||||
procedureFollowing: dims.procedureFollowing ?? overall,
|
||||
conciseness: dims.conciseness ?? overall,
|
||||
lengthPenalty: dims.lengthPenalty ?? 1,
|
||||
n: dims.n ?? 10,
|
||||
weaknessFeedback: dims.weaknessFeedback ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake judge that computes a deterministic score based on candidate text length.
|
||||
// Longer prompts → higher correctness, shorter → higher conciseness.
|
||||
function makeFakeJudge(bias: 'long' | 'short' | 'uniform' = 'uniform') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const len = args.actual.length;
|
||||
const corr = bias === 'long' ? Math.min(1, len / 50) : bias === 'short' ? Math.max(0, 1 - len / 100) : 0.5;
|
||||
const proc = 0.6;
|
||||
const conc = bias === 'short' ? 0.9 : 0.5;
|
||||
const overall = 0.5 * corr + 0.3 * proc + 0.2 * conc;
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: corr,
|
||||
procedureFollowing: proc,
|
||||
conciseness: conc,
|
||||
lengthPenalty: 1,
|
||||
feedback: `len=${len}`,
|
||||
parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────
|
||||
|
||||
describe('aggregateScores', () => {
|
||||
it('handles empty input', () => {
|
||||
const agg = aggregateScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.overall).toBe(0);
|
||||
expect(agg.weaknessFeedback).toEqual([]);
|
||||
});
|
||||
|
||||
it('averages component dimensions', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.8, { correctness: 0.9, procedureFollowing: 0.7, conciseness: 0.8 }),
|
||||
makeScore(0.6, { correctness: 0.7, procedureFollowing: 0.5, conciseness: 0.6 }),
|
||||
]);
|
||||
expect(agg.n).toBe(2);
|
||||
expect(agg.correctness).toBeCloseTo(0.8, 5);
|
||||
expect(agg.procedureFollowing).toBeCloseTo(0.6, 5);
|
||||
expect(agg.conciseness).toBeCloseTo(0.7, 5);
|
||||
expect(agg.overall).toBeCloseTo(0.7, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 feedback lines in weaknessFeedback', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.9, { feedback: 'best' }),
|
||||
makeScore(0.1, { feedback: 'worst' }),
|
||||
makeScore(0.2, { feedback: 'second-worst' }),
|
||||
makeScore(0.3, { feedback: 'third-worst' }),
|
||||
makeScore(0.8, { feedback: 'good' }),
|
||||
]);
|
||||
// Worst 3 should contain 'worst', 'second-worst', 'third-worst' in some order
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('second-worst');
|
||||
expect(agg.weaknessFeedback).toContain('third-worst');
|
||||
expect(agg.weaknessFeedback).not.toContain('best');
|
||||
});
|
||||
|
||||
it('drops empty feedback from weakness list', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.1, { feedback: '' }),
|
||||
makeScore(0.2, { feedback: 'real feedback' }),
|
||||
]);
|
||||
expect(agg.weaknessFeedback).toEqual(['real feedback']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('paretoFront', () => {
|
||||
it('keeps a single member when only one has a score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.8));
|
||||
const b = makeCandidate('b', null);
|
||||
expect(paretoFront([a, b])).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes strictly-dominated candidates', () => {
|
||||
const weak = makeCandidate('weak', makeCandidateScore(0.5, {
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
}));
|
||||
const strong = makeCandidate('strong', makeCandidateScore(0.8, {
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
}));
|
||||
const front = paretoFront([weak, strong]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('strong');
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates on the front', () => {
|
||||
const accurate = makeCandidate('accurate', makeCandidateScore(0.7, {
|
||||
correctness: 0.9, procedureFollowing: 0.6, conciseness: 0.5,
|
||||
}));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, {
|
||||
correctness: 0.6, procedureFollowing: 0.6, conciseness: 0.9,
|
||||
}));
|
||||
const front = paretoFront([accurate, concise]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('returns all candidates when they have identical scores', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.5));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.5));
|
||||
const front = paretoFront([a, b]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores candidates with null scores entirely', () => {
|
||||
const scored = makeCandidate('scored', makeCandidateScore(0.5));
|
||||
const unscored = makeCandidate('unscored', null);
|
||||
const front = paretoFront([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('scored');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickWinner', () => {
|
||||
it('picks highest overall score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.6));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.9));
|
||||
expect(pickWinner([a, b]).id).toBe('b');
|
||||
});
|
||||
|
||||
it('breaks ties using length penalty (higher = more concise, preferred)', () => {
|
||||
const verbose = makeCandidate('verbose', makeCandidateScore(0.7, { lengthPenalty: 0.6 }));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, { lengthPenalty: 0.9 }));
|
||||
expect(pickWinner([verbose, concise]).id).toBe('concise');
|
||||
});
|
||||
|
||||
it('breaks further ties by earliest generation', () => {
|
||||
const early = makeCandidate('early', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
const late = makeCandidate('late', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
late.generation = 3;
|
||||
expect(pickWinner([early, late]).id).toBe('early');
|
||||
});
|
||||
|
||||
it('throws on empty input', () => {
|
||||
expect(() => pickWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSample', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(10), 0, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns full set (shuffled) when k >= length', () => {
|
||||
const sample = pickSample(makeExamples(5), 10, rng);
|
||||
expect(sample).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('is deterministic with a fixed rng', () => {
|
||||
const rngA = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const rngB = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const a = pickSample(makeExamples(20), 5, rngA).map(e => e.input);
|
||||
const b = pickSample(makeExamples(20), 5, rngB).map(e => e.input);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreCandidate', () => {
|
||||
it('scores candidate against examples and populates the score field', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), makeFakeJudge('uniform'));
|
||||
expect(score.n).toBe(3);
|
||||
expect(cand.score).not.toBeNull();
|
||||
expect(cand.perExample).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns zero-score aggregation when examples list is empty', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, [], makeFakeJudge());
|
||||
expect(score.n).toBe(0);
|
||||
expect(score.overall).toBe(0);
|
||||
});
|
||||
|
||||
it('continues iterating if one score call throws', async () => {
|
||||
let callCount = 0;
|
||||
const flakyJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
callCount++;
|
||||
if (callCount === 2) throw new Error('transient');
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), flakyJudge);
|
||||
expect(score.n).toBe(2); // 2 succeeded, 1 failed
|
||||
});
|
||||
|
||||
it('stops early when abort signal fires', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const judge = makeFakeJudge();
|
||||
// Abort before any call
|
||||
ctrl.abort();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(10), judge, ctrl.signal);
|
||||
expect(cand.perExample.length).toBeLessThan(10);
|
||||
});
|
||||
|
||||
// ── Concurrency ────────────────────────────────────────────────
|
||||
|
||||
it('accepts an options object (new API) equivalently to AbortSignal (legacy)', async () => {
|
||||
const judge = makeFakeJudge('uniform');
|
||||
const candA = makeCandidate('a', null);
|
||||
const candB = makeCandidate('b', null);
|
||||
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
|
||||
await scoreCandidate(candA, makeExamples(5), judge, ctrl.signal);
|
||||
await scoreCandidate(candB, makeExamples(5), judge, { signal: ctrl.signal });
|
||||
|
||||
expect(candA.perExample.length).toBe(candB.perExample.length);
|
||||
});
|
||||
|
||||
it('concurrency=1 produces identical aggregate to no option (sequential baseline)', async () => {
|
||||
const examples = makeExamples(6);
|
||||
const candSeq = makeCandidate('seq', null);
|
||||
const candPar = makeCandidate('par', null);
|
||||
|
||||
const seqScore = await scoreCandidate(candSeq, examples, makeFakeJudge('uniform'));
|
||||
const parScore = await scoreCandidate(candPar, examples, makeFakeJudge('uniform'), { concurrency: 1 });
|
||||
|
||||
expect(parScore.overall).toBeCloseTo(seqScore.overall);
|
||||
expect(parScore.n).toBe(seqScore.n);
|
||||
});
|
||||
|
||||
it('with concurrency > 1, runs scores in parallel (observable via in-flight counter)', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
// Await a microtask + a real delay so parallel workers can accumulate.
|
||||
await new Promise(r => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(8), judge, { concurrency: 4 });
|
||||
|
||||
expect(peakInFlight).toBe(4);
|
||||
expect(cand.perExample).toHaveLength(8);
|
||||
});
|
||||
|
||||
it('with concurrency=1, exactly 1 in-flight call at a time', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 3));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(5), judge, { concurrency: 1 });
|
||||
expect(peakInFlight).toBe(1);
|
||||
});
|
||||
|
||||
it('concurrency is capped at examples.length (does not start idle workers)', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(3), judge, { concurrency: 100 });
|
||||
expect(peakInFlight).toBeLessThanOrEqual(3);
|
||||
});
|
||||
|
||||
it('parallel mode still filters out thrown-error results without corrupting the batch', async () => {
|
||||
let call = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
call++;
|
||||
if (call % 3 === 0) throw new Error('flaky');
|
||||
return makeScore(0.7);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(9), judge, { concurrency: 3 });
|
||||
// 9 total, every 3rd throws → 6 succeed.
|
||||
expect(score.n).toBe(6);
|
||||
});
|
||||
|
||||
it('parallel mode respects abort signal by not dispatching further workers', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const judge = makeFakeJudge();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(20), judge, {
|
||||
signal: ctrl.signal,
|
||||
concurrency: 4,
|
||||
});
|
||||
expect(cand.perExample.length).toBeLessThan(20);
|
||||
});
|
||||
});
|
||||
|
||||
// ── IterativeGEPA end-to-end with concurrency ────────────────────
|
||||
|
||||
describe('IterativeGEPA with concurrency', () => {
|
||||
it('threads options.concurrency into every scoreCandidate call', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt} :: ${strategy}`;
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 4,
|
||||
miniEvalSize: 4,
|
||||
anchorEvalSize: 4,
|
||||
concurrency: 3,
|
||||
});
|
||||
|
||||
expect(peakInFlight).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-09 G3 · running-judge guard ──────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run · running-judge guard (H-09 G3)', () => {
|
||||
it('throws when a bare judge is passed without allowBareJudge', async () => {
|
||||
const bareJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.5, weighted: 0.5, correctness: 0.5,
|
||||
procedureFollowing: 0.5, conciseness: 0.5, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} mutated`;
|
||||
await expect(
|
||||
new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: bareJudge,
|
||||
mutate,
|
||||
}),
|
||||
).rejects.toThrow(/not a running judge|makeRunningJudge/);
|
||||
});
|
||||
|
||||
it('accepts a judge wrapped with makeRunningJudge', async () => {
|
||||
const { makeRunningJudge } = await import('../src/evolution-llm-wiring.js');
|
||||
const baseJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.8, weighted: 0.8, correctness: 0.8,
|
||||
procedureFollowing: 0.8, conciseness: 0.8, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const fakeLLM = { complete: async () => 'LLM output' };
|
||||
const wrapped = makeRunningJudge(baseJudge, fakeLLM);
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} v2`;
|
||||
// Should not throw — the wrapped judge carries the brand.
|
||||
const result = await new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: wrapped,
|
||||
mutate,
|
||||
populationSize: 1,
|
||||
generations: 1,
|
||||
microScreenSize: 1, miniEvalSize: 1, anchorEvalSize: 1,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── End-to-end run ─────────────────────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run', () => {
|
||||
it('runs through all phases and produces a winner', async () => {
|
||||
const baseline = 'short baseline prompt';
|
||||
const examples = makeExamples(10);
|
||||
const judge = makeFakeJudge('uniform');
|
||||
|
||||
// Mutate appends text — simulates generation of children
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => {
|
||||
return `${parent.prompt} :: ${strategy}`;
|
||||
};
|
||||
|
||||
const progress: string[] = [];
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
onProgress: (e) => progress.push(`${e.phase}@g${e.generation}`),
|
||||
});
|
||||
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
expect(progress).toContain('start@g0');
|
||||
expect(progress.some(p => p.startsWith('anchor'))).toBe(true);
|
||||
expect(progress.some(p => p.startsWith('done'))).toBe(true);
|
||||
});
|
||||
|
||||
it('winner dominates or matches baseline on overall score', async () => {
|
||||
const baseline = 'x'; // very short
|
||||
const examples = makeExamples(20);
|
||||
// Long-biased judge: longer prompts score higher
|
||||
const judge = makeFakeJudge('long');
|
||||
|
||||
// Mutate doubles length each time
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} ${parent.prompt}more`;
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 15,
|
||||
});
|
||||
|
||||
// Winner should have evolved (not baseline prompt).
|
||||
expect(result.winner.prompt.length).toBeGreaterThan(baseline.length);
|
||||
expect(result.improved).toBe(true);
|
||||
expect(result.delta).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('passes weakness feedback from parent into mutate()', async () => {
|
||||
const captured: Array<{ strategy: string; feedbacks: string[] }> = [];
|
||||
const mutate: MutateFn = async ({ parent, strategy, weaknessFeedback }) => {
|
||||
captured.push({ strategy, feedbacks: weaknessFeedback });
|
||||
return `${parent.prompt} mutated`;
|
||||
};
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(10),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 5,
|
||||
});
|
||||
|
||||
// 2 mutations were spawned — each received the parent's feedback
|
||||
expect(captured.length).toBe(2);
|
||||
for (const c of captured) {
|
||||
expect(Array.isArray(c.feedbacks)).toBe(true);
|
||||
// Feedback array may be empty on first gen if baseline had no weaknesses;
|
||||
// we just assert the prop is present and is an array.
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers when mutate throws (falls back to parent prompt)', async () => {
|
||||
let mutations = 0;
|
||||
const mutate: MutateFn = async ({ parent }) => {
|
||||
mutations++;
|
||||
if (mutations === 1) throw new Error('boom');
|
||||
return `${parent.prompt} ok`;
|
||||
};
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
// Winner must be defined even though one mutation failed
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
});
|
||||
|
||||
it('respects abort signal and exits early without throwing', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const mutate: MutateFn = vi.fn(async () => 'should never run');
|
||||
|
||||
ctrl.abort(); // abort before starting generations
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
microScreenSize: 2,
|
||||
miniEvalSize: 2,
|
||||
anchorEvalSize: 2,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
// With abort before baseline scoring, no winner should evolve
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(mutate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const examples = makeExamples(20);
|
||||
const judge = makeFakeJudge();
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt}::${strategy}`;
|
||||
|
||||
const run = () => runGEPA({
|
||||
baseline: 'seed',
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
seed: 42,
|
||||
});
|
||||
|
||||
const r1 = await run();
|
||||
const r2 = await run();
|
||||
|
||||
expect(r1.winner.prompt).toBe(r2.winner.prompt);
|
||||
expect(r1.history.length).toBe(r2.history.length);
|
||||
});
|
||||
|
||||
it('produces history containing baseline and all mutated children', async () => {
|
||||
const result = await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate: async ({ parent, strategy }) => `${parent.prompt}_${strategy}`,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
expect(result.history[0].id).toBe('g0-baseline');
|
||||
expect(result.history[0].strategy).toBe('baseline');
|
||||
// 2 generations * 3 mutations + 1 baseline = 7 total
|
||||
expect(result.history.length).toBe(7);
|
||||
});
|
||||
});
|
||||
@@ -1,280 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import {
|
||||
captureInteraction,
|
||||
getRecentLogs,
|
||||
getWorkspaceLogs,
|
||||
isWithinBudget,
|
||||
} from '../src/optimization-capture.js';
|
||||
import type { AgentResponse } from '../src/agent-loop.js';
|
||||
|
||||
describe('optimization-capture', () => {
|
||||
let db: MindDB;
|
||||
let store: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function makeResponse(overrides?: Partial<AgentResponse>): AgentResponse {
|
||||
return {
|
||||
content: 'Hello, I can help with that.',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
usage: { inputTokens: 500, outputTokens: 200 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── captureInteraction ─────────────────────────────────────
|
||||
|
||||
describe('captureInteraction', () => {
|
||||
it('stores a log entry with correct fields', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'You are a helpful assistant.',
|
||||
response: makeResponse(),
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.id).toBeGreaterThan(0);
|
||||
expect(entry.session_id).toBe('session-1');
|
||||
expect(entry.workspace_id).toBe('ws-abc');
|
||||
expect(entry.system_prompt).toBe('You are a helpful assistant.');
|
||||
expect(JSON.parse(entry.tools_used)).toEqual(['search_memory', 'save_memory']);
|
||||
expect(entry.turn_count).toBe(3);
|
||||
expect(entry.was_correction).toBe(0);
|
||||
expect(entry.input_tokens).toBe(500);
|
||||
expect(entry.output_tokens).toBe(200);
|
||||
expect(entry.timestamp).toBeTruthy();
|
||||
});
|
||||
|
||||
it('records wasCorrection as 1 when true', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse(),
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
expect(entry.was_correction).toBe(1);
|
||||
});
|
||||
|
||||
it('handles empty toolsUsed', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse({ toolsUsed: [] }),
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(JSON.parse(entry.tools_used)).toEqual([]);
|
||||
});
|
||||
|
||||
it('defaults token counts to 0 when missing', () => {
|
||||
const response: AgentResponse = {
|
||||
content: 'response',
|
||||
toolsUsed: [],
|
||||
usage: { inputTokens: 0, outputTokens: 0 },
|
||||
};
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 's1',
|
||||
workspaceId: 'ws1',
|
||||
systemPrompt: 'prompt',
|
||||
response,
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.input_tokens).toBe(0);
|
||||
expect(entry.output_tokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getRecentLogs ──────────────────────────────────────────
|
||||
|
||||
describe('getRecentLogs', () => {
|
||||
it('retrieves stored entries ordered by most recent first', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p1',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p2',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const logs = getRecentLogs(store, 10);
|
||||
expect(logs).toHaveLength(2);
|
||||
// Ordered by timestamp DESC, id DESC — s2 was inserted second so it comes first
|
||||
expect(logs[0].session_id).toBe('s2');
|
||||
expect(logs[1].session_id).toBe('s1');
|
||||
});
|
||||
|
||||
it('respects the limit parameter', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
captureInteraction(store, {
|
||||
sessionId: `s${i}`, workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(store, 3);
|
||||
expect(logs).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty array when no entries exist', () => {
|
||||
const logs = getRecentLogs(store);
|
||||
expect(logs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getWorkspaceLogs ───────────────────────────────────────
|
||||
|
||||
describe('getWorkspaceLogs', () => {
|
||||
it('filters by workspace ID', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws-b', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's3', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const wsALogs = getWorkspaceLogs(store, 'ws-a');
|
||||
expect(wsALogs).toHaveLength(2);
|
||||
expect(wsALogs.every(l => l.workspace_id === 'ws-a')).toBe(true);
|
||||
|
||||
const wsBLogs = getWorkspaceLogs(store, 'ws-b');
|
||||
expect(wsBLogs).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Entries have correct shape ─────────────────────────────
|
||||
|
||||
describe('entry shape', () => {
|
||||
it('has all required fields with correct types', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-x',
|
||||
workspaceId: 'workspace-y',
|
||||
systemPrompt: 'You are helpful.',
|
||||
response: makeResponse({ toolsUsed: ['bash', 'read_file'] }),
|
||||
turnCount: 7,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
// Check all fields exist and are of expected types
|
||||
expect(typeof entry.id).toBe('number');
|
||||
expect(typeof entry.session_id).toBe('string');
|
||||
expect(typeof entry.workspace_id).toBe('string');
|
||||
expect(typeof entry.system_prompt).toBe('string');
|
||||
expect(typeof entry.tools_used).toBe('string');
|
||||
expect(typeof entry.turn_count).toBe('number');
|
||||
expect(typeof entry.was_correction).toBe('number');
|
||||
expect(typeof entry.input_tokens).toBe('number');
|
||||
expect(typeof entry.output_tokens).toBe('number');
|
||||
expect(typeof entry.timestamp).toBe('string');
|
||||
|
||||
// tools_used is valid JSON array
|
||||
const parsed = JSON.parse(entry.tools_used);
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed).toEqual(['bash', 'read_file']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Budget enforcement ─────────────────────────────────────
|
||||
|
||||
describe('isWithinBudget', () => {
|
||||
it('returns true when no entries exist (under budget)', () => {
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when token cost is under budget', () => {
|
||||
// Insert entry with small token count
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 100 input * $3/M + 50 output * $15/M = $0.0003 + $0.00075 = ~$0.00105 = ~0.1 cents
|
||||
// Budget is 100 cents = $1, so well within budget
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when token cost exceeds budget', () => {
|
||||
// Insert entry with large token count to exceed a tiny budget
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 1_000_000, outputTokens: 500_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 1M input * $3/M + 500K output * $15/M = $3 + $7.5 = $10.50 = 1050 cents
|
||||
// Budget is 1 cent, so way over budget
|
||||
expect(isWithinBudget(store, 1)).toBe(false);
|
||||
});
|
||||
|
||||
it('respects custom cost-per-token rates', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 10_000, outputTokens: 5_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// With very cheap rates: 10K * $0.0001/M + 5K * $0.0001/M ≈ ~0 cents
|
||||
expect(isWithinBudget(store, 1, 0.0001 / 1_000_000, 0.0001 / 1_000_000)).toBe(true);
|
||||
|
||||
// With very expensive rates: 10K * $100/M + 5K * $100/M = $1 + $0.5 = 150 cents
|
||||
expect(isWithinBudget(store, 1, 100 / 1_000_000, 100 / 1_000_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Store stats ────────────────────────────────────────────
|
||||
|
||||
describe('getStats (via store)', () => {
|
||||
it('computes correct aggregate stats', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 3, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 200, outputTokens: 100 } }),
|
||||
turnCount: 5, wasCorrection: true,
|
||||
});
|
||||
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.correctionRate).toBe(0.5); // 1/2
|
||||
expect(stats.avgTurnCount).toBe(4); // (3+5)/2
|
||||
expect(stats.totalInputTokens).toBe(300); // 100+200
|
||||
expect(stats.totalOutputTokens).toBe(150); // 50+100
|
||||
});
|
||||
|
||||
it('returns zeros when empty', () => {
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(0);
|
||||
expect(stats.correctionRate).toBe(0);
|
||||
expect(stats.avgTurnCount).toBe(0);
|
||||
expect(stats.totalInputTokens).toBe(0);
|
||||
expect(stats.totalOutputTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,10 +2,9 @@ import { describe, it, expect } from 'vitest';
|
||||
import { PERSONAS, getPersona, listPersonas, composePersonaPrompt } from '../src/personas.js';
|
||||
|
||||
describe('Agent Personas', () => {
|
||||
it('PERSONAS catalog has 23 entries', () => {
|
||||
// 22 user-facing (8 universal + 14 specialists) + 1 internal review persona
|
||||
// (session-reviewer) used only by the IdleSessionWatcher self-evolution loop.
|
||||
expect(PERSONAS).toHaveLength(23);
|
||||
it('PERSONAS catalog has 22 entries', () => {
|
||||
// 8 universal + 14 specialists.
|
||||
expect(PERSONAS).toHaveLength(22);
|
||||
});
|
||||
|
||||
it('each persona has required fields', () => {
|
||||
@@ -40,9 +39,9 @@ describe('Agent Personas', () => {
|
||||
expect(getPersona('')).toBeNull();
|
||||
});
|
||||
|
||||
it('listPersonas() returns all 23 personas', () => {
|
||||
it('listPersonas() returns all 22 personas', () => {
|
||||
const list = listPersonas();
|
||||
expect(list).toHaveLength(23);
|
||||
expect(list).toHaveLength(22);
|
||||
expect(list).not.toBe(PERSONAS); // Returns a copy
|
||||
});
|
||||
|
||||
@@ -208,19 +207,6 @@ describe('Soft tool model', () => {
|
||||
expect(v!.disallowedTools).toContain('write_file');
|
||||
expect(v!.isReadOnly).toBe(true);
|
||||
});
|
||||
|
||||
it('session-reviewer is a trust-boundary persona: reads + create_skill only, no writes/exec/memory', () => {
|
||||
const r = getPersona('session-reviewer');
|
||||
expect(r, 'session-reviewer not found').toBeTruthy();
|
||||
// create_skill is the proposal vehicle and must stay in the allowlist.
|
||||
expect(r!.tools).toContain('create_skill');
|
||||
// isReadOnly would strip create_skill, so it must be false.
|
||||
expect(r!.isReadOnly).toBe(false);
|
||||
// The critical tools ALWAYS_AVAILABLE_TOOLS re-adds must be explicitly denied.
|
||||
for (const t of ['save_memory', 'delete_skill', 'install_capability', 'acquire_capability', 'spawn_agent', 'bash']) {
|
||||
expect(r!.disallowedTools, `${t} must be disallowed`).toContain(t);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('New domain personas', () => {
|
||||
@@ -239,8 +225,7 @@ describe('New domain personas', () => {
|
||||
});
|
||||
}
|
||||
|
||||
it('total persona count is 23', () => {
|
||||
// 22 tiered + session-reviewer (internal self-evolution reviewer).
|
||||
expect(PERSONAS.length).toBe(23);
|
||||
it('total persona count is 22', () => {
|
||||
expect(PERSONAS.length).toBe(22);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
*
|
||||
* Tests the one-step promotion personal → workspace, the refusal of team and
|
||||
* enterprise scopes in this edition, frontmatter rewrite with promoted_from
|
||||
* history, file move, and skill_promotion improvement signal emission.
|
||||
* history and file move.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { createSkillTools, getSkillDirForScope } from '../src/skill-tools.js';
|
||||
import { parseSkillFrontmatter } from '../src/skill-frontmatter.js';
|
||||
|
||||
@@ -22,8 +21,6 @@ function seedSkill(dir: string, name: string, scope: string | null, body = 'body
|
||||
|
||||
describe('promote_skill — gap E', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
let reloadCount: number;
|
||||
|
||||
function createTools(overrides: Partial<Parameters<typeof createSkillTools>[0]> = {}) {
|
||||
@@ -32,7 +29,6 @@ describe('promote_skill — gap E', () => {
|
||||
onSkillsChanged: () => { reloadCount++; },
|
||||
getWorkspaceId: () => 'ws-alpha',
|
||||
getTeamId: () => 'team-bravo',
|
||||
improvementSignals: signals,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -45,18 +41,15 @@ describe('promote_skill — gap E', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-promote-test-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
reloadCount = 0;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('happy path', () => {
|
||||
it('promotes personal → workspace, moves file, appends history, records signal', async () => {
|
||||
it('promotes personal → workspace, moves file, appends history', async () => {
|
||||
const personalDir = getSkillDirForScope(tmpDir, 'personal', {})!;
|
||||
seedSkill(personalDir, 'deploy-helper', 'personal');
|
||||
const tools = createTools();
|
||||
@@ -64,7 +57,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(tools, 'promote_skill', { skill_name: 'deploy-helper' });
|
||||
|
||||
expect(out).toContain('personal → workspace');
|
||||
expect(out).toContain('Signal recorded');
|
||||
|
||||
// File moved
|
||||
expect(fs.existsSync(path.join(personalDir, 'deploy-helper.md'))).toBe(false);
|
||||
@@ -77,11 +69,6 @@ describe('promote_skill — gap E', () => {
|
||||
expect(parsed.frontmatter.scope).toBe('workspace');
|
||||
expect(parsed.frontmatter.promoted_from).toEqual(['personal']);
|
||||
|
||||
// Signal emitted
|
||||
const recorded = signals.getByCategory('skill_promotion');
|
||||
expect(recorded.length).toBe(1);
|
||||
expect(recorded[0].pattern_key).toBe('promote:deploy-helper:personal->workspace');
|
||||
|
||||
// onSkillsChanged fired
|
||||
expect(reloadCount).toBe(1);
|
||||
});
|
||||
@@ -125,7 +112,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(createTools(), 'promote_skill', { skill_name: 'shared-helper' });
|
||||
expect(out).toContain('not available in this edition');
|
||||
expect(fs.existsSync(path.join(wsDir, 'shared-helper.md'))).toBe(true);
|
||||
expect(signals.getByCategory('skill_promotion')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('refuses enterprise promotion', async () => {
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
} from '../src/prompt-loader.js';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -68,7 +67,6 @@ describe('Prompt Loader', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-08 G2 — override-aware loader ──────────────────────────
|
||||
describe('loadSystemPromptWithOverrides', () => {
|
||||
it('returns baseline when no overrides exist', () => {
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
@@ -86,55 +84,17 @@ describe('Prompt Loader', () => {
|
||||
expect(composed.userPromptMd).toBe('hello from user');
|
||||
});
|
||||
|
||||
it('applies behavioral-spec overrides into the active spec', () => {
|
||||
it('ignores override files left on disk by earlier versions', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'CUSTOM CORE LOOP CONTENT FOR TEST',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
JSON.stringify({ section: 'coreLoop', text: 'STALE OVERRIDE TEXT', deployedAt: new Date().toISOString() }),
|
||||
);
|
||||
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
const flattened = Object.values(composed.behavioralSpec).join('\n');
|
||||
expect(flattened).toContain('CUSTOM CORE LOOP CONTENT FOR TEST');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertOverridesReachActiveSpec', () => {
|
||||
it('is quiet when no overrides are deployed', () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
assertOverridesReachActiveSpec(tmpDir, composed.behavioralSpec);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('warns when disk has overrides but activeSpec does not contain them', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'UNIQUE MARKER STRING FOR G2 TEST — LONG ENOUGH TO MATCH',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
// Simulate using the BARE loader — activeSpec is the compile-time
|
||||
// baseline with no overrides folded in.
|
||||
const fakeActiveSpec = loadSystemPromptWithOverrides(path.join(tmpDir, 'nothing')).behavioralSpec;
|
||||
assertOverridesReachActiveSpec(tmpDir, fakeActiveSpec);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
const message = warnSpy.mock.calls[0]?.[0] as string;
|
||||
expect(message).toContain('coreLoop');
|
||||
warnSpy.mockRestore();
|
||||
expect(flattened).not.toContain('STALE OVERRIDE TEXT');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,13 +2,8 @@
|
||||
* Skills 2.0 gap K — write-path conflict detection.
|
||||
*
|
||||
* When save_memory writes a new frame that contradicts an existing frame
|
||||
* (detectContradiction), the save still succeeds but:
|
||||
* - the return string carries a [flag: contradicts_existing …] marker
|
||||
* - a correction-category improvement signal is recorded
|
||||
*
|
||||
* Previously detectContradiction existed but was never called on the write
|
||||
* path — write-path conflicts silently overwrote reality, degrading the
|
||||
* evolution-signal loop.
|
||||
* (detectContradiction), the save still succeeds but the return string
|
||||
* carries a [flag: contradicts_existing …] marker.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
@@ -19,7 +14,6 @@ import {
|
||||
SessionStore,
|
||||
KnowledgeGraph,
|
||||
HybridSearch,
|
||||
ImprovementSignalStore,
|
||||
} from '@waggle/core';
|
||||
import { createMindTools } from '../src/tools.js';
|
||||
import type { ToolDefinition } from '../src/tools.js';
|
||||
@@ -30,7 +24,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
let frames: FrameStore;
|
||||
let sessions: SessionStore;
|
||||
let tools: ToolDefinition[];
|
||||
let improvementSignals: ImprovementSignalStore;
|
||||
|
||||
function getSaveMemory(): ToolDefinition {
|
||||
const tool = tools.find(t => t.name === 'save_memory');
|
||||
@@ -45,7 +38,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions = new SessionStore(db);
|
||||
const knowledge = new KnowledgeGraph(db);
|
||||
const search = new HybridSearch(db, embedder);
|
||||
improvementSignals = new ImprovementSignalStore(db);
|
||||
|
||||
tools = createMindTools({
|
||||
db,
|
||||
@@ -55,7 +47,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions,
|
||||
search,
|
||||
knowledge,
|
||||
improvementSignals,
|
||||
// Intentionally NO cognify → exercises the raw-frame fallback path
|
||||
// (no LLM-dependent entity extraction in unit tests).
|
||||
});
|
||||
@@ -94,17 +85,9 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
expect(second).toContain('Memory saved');
|
||||
// But the return message is annotated
|
||||
expect(second).toContain('[flag: contradicts_existing');
|
||||
|
||||
// And a correction signal is recorded
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.length).toBeGreaterThan(0);
|
||||
const writeConflict = top.find(s => s.pattern_key.startsWith('write-conflict:'));
|
||||
expect(writeConflict).toBeDefined();
|
||||
expect(writeConflict!.detail).toMatch(/contradicts existing frame/i);
|
||||
expect(writeConflict!.category).toBe('correction');
|
||||
});
|
||||
|
||||
it('non-contradicting saves do not flag or emit a signal', async () => {
|
||||
it('non-contradicting saves are not flagged', async () => {
|
||||
const save = getSaveMemory();
|
||||
await save.execute({
|
||||
content: 'Decision: we chose PostgreSQL for analytics; approved and confirmed.',
|
||||
@@ -117,34 +100,5 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
source: 'user_stated',
|
||||
});
|
||||
expect(result).not.toContain('contradicts_existing');
|
||||
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.find(s => s.pattern_key.startsWith('write-conflict:'))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('works without improvementSignals dep (backwards-compatible)', async () => {
|
||||
// Rebuild tools without signals dep
|
||||
const db2 = new MindDB(':memory:');
|
||||
const embedder = new MockEmbedder();
|
||||
const frames2 = new FrameStore(db2);
|
||||
const sessions2 = new SessionStore(db2);
|
||||
const knowledge2 = new KnowledgeGraph(db2);
|
||||
const search2 = new HybridSearch(db2, embedder);
|
||||
const tools2 = createMindTools({
|
||||
db: db2,
|
||||
identity: new IdentityLayer(db2),
|
||||
awareness: new AwarenessLayer(db2),
|
||||
frames: frames2,
|
||||
sessions: sessions2,
|
||||
search: search2,
|
||||
knowledge: knowledge2,
|
||||
// No improvementSignals — should not throw
|
||||
});
|
||||
const save = tools2.find(t => t.name === 'save_memory')!;
|
||||
await save.execute({ content: 'Decision: enable feature flag rollout for users; proceed.', target: 'personal' });
|
||||
const result = await save.execute({ content: 'Decision: disable feature flag rollout for users; cancel.', target: 'personal' });
|
||||
// Flag still appears (detectContradiction doesn't need the signal store)
|
||||
expect(result).toContain('contradicts_existing');
|
||||
db2.close();
|
||||
});
|
||||
});
|
||||
|
||||
77
packages/agent/tests/secret-patterns.test.ts
Normal file
77
packages/agent/tests/secret-patterns.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from '../src/secret-patterns.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('redactSecrets', () => {
|
||||
it('replaces every match and reports the patterns that fired', () => {
|
||||
const { text, found } = redactSecrets('key AKIAIOSFODNN7EXAMPLE and PASSWORD=supersecret123');
|
||||
expect(text).toContain('[REDACTED:aws-access-key]');
|
||||
expect(text).toContain('[REDACTED:env-password]');
|
||||
expect(text).not.toContain('AKIAIOSFODNN7EXAMPLE');
|
||||
expect(found.sort()).toEqual(['aws-access-key', 'env-password']);
|
||||
});
|
||||
|
||||
it('leaves clean text untouched', () => {
|
||||
expect(redactSecrets('nothing to hide')).toEqual({ text: 'nothing to hide', found: [] });
|
||||
});
|
||||
});
|
||||
@@ -90,59 +90,4 @@ describe('Self-Awareness', () => {
|
||||
expect(result).toContain('acquire_capability');
|
||||
expect(result).toContain('uncertain');
|
||||
});
|
||||
|
||||
it('includes improvement signals when awareness has actionable data', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [{
|
||||
id: 1,
|
||||
toolName: 'pdf_reader',
|
||||
occurrences: 3,
|
||||
suggestion: 'User has needed "pdf_reader" 3 times. Consider using acquire_capability to find it.',
|
||||
}],
|
||||
corrections: [{
|
||||
id: 2,
|
||||
patternKey: 'tone:too_formal',
|
||||
detail: 'Keep it casual',
|
||||
occurrences: 4,
|
||||
guidance: 'Keep it casual (corrected 4 times)',
|
||||
}],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 2,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).toContain('Learning from Past Sessions');
|
||||
expect(result).toContain('pdf_reader');
|
||||
expect(result).toContain('Keep it casual');
|
||||
expect(result).toContain('Recurring capability gaps');
|
||||
expect(result).toContain('User corrections to apply');
|
||||
});
|
||||
|
||||
it('omits improvement section when awareness has zero actionable signals', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [],
|
||||
corrections: [],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 0,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).not.toContain('Learning from Past Sessions');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
autoExtractAndCreateSkill,
|
||||
skillFilename,
|
||||
@@ -35,24 +34,18 @@ function repeatedSession(): AutoExtractMessage[] {
|
||||
|
||||
describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-autoextract-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates a personal-scope SKILL.md from a repeated workflow', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(result.filePath).not.toBeNull();
|
||||
@@ -67,19 +60,6 @@ describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
expect(body).toContain('## Tools Used');
|
||||
});
|
||||
|
||||
it('records a workflow_pattern signal with auto-extract pattern_key', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
expect(result.filePath).not.toBeNull();
|
||||
|
||||
const recorded = signals.getByCategory('workflow_pattern');
|
||||
const autoExtract = recorded.find(s => s.pattern_key.startsWith('auto-extract:'));
|
||||
expect(autoExtract).toBeDefined();
|
||||
expect(autoExtract!.detail).toContain('Auto-extracted');
|
||||
});
|
||||
|
||||
it('returns no_pattern for short or random sessions', async () => {
|
||||
const messages: AutoExtractMessage[] = [
|
||||
{ role: 'user', content: 'Hi' },
|
||||
|
||||
@@ -5,7 +5,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { retireStaleSkills } from '../src/skill-retirement.js';
|
||||
import {
|
||||
recordSkillUsage,
|
||||
@@ -74,8 +73,6 @@ describe('skill-usage — gap F', () => {
|
||||
describe('retireStaleSkills — gap F', () => {
|
||||
let waggleHome: string;
|
||||
let skillsDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
const NOW = new Date('2026-04-15T00:00:00Z');
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
@@ -83,12 +80,9 @@ describe('retireStaleSkills — gap F', () => {
|
||||
beforeEach(() => {
|
||||
waggleHome = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-retire-'));
|
||||
skillsDir = path.join(waggleHome, 'skills');
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(waggleHome, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
@@ -110,7 +104,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays: 90,
|
||||
now: () => NOW,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(report.scanned).toBe(2);
|
||||
@@ -123,10 +116,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
// Usage entry for the retired skill is cleared
|
||||
expect(loadSkillUsage(waggleHome).stale).toBeUndefined();
|
||||
expect(loadSkillUsage(waggleHome).fresh).toBeDefined();
|
||||
|
||||
// Signal recorded
|
||||
const recorded = signals.getByCategory('workflow_pattern').find(s => s.pattern_key === 'retire:stale');
|
||||
expect(recorded).toBeDefined();
|
||||
});
|
||||
|
||||
it('falls back to file mtime when no usage entry exists', () => {
|
||||
|
||||
@@ -11,7 +11,7 @@ import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import chalk from 'chalk';
|
||||
import { MindDB, WaggleConfig, createLiteLLMEmbedder } from '@waggle/core';
|
||||
import { Orchestrator, ModelRouter, runAgentLoop, createSystemTools, createPlanTools, createGitTools, Workspace, ensureIdentity, loadSystemPromptWithOverrides, assertOverridesReachActiveSpec, loadSkills, CostTracker, HookRegistry, loadHooksFromConfig, needsConfirmation } from '@waggle/agent';
|
||||
import { Orchestrator, ModelRouter, runAgentLoop, createSystemTools, createPlanTools, createGitTools, Workspace, ensureIdentity, loadSystemPromptWithOverrides, loadSkills, CostTracker, HookRegistry, loadHooksFromConfig, needsConfirmation } from '@waggle/agent';
|
||||
import { parseCommand, COMMANDS } from './commands.js';
|
||||
import { renderMarkdown } from './renderer.js';
|
||||
import { AdminClient, formatTable } from './commands/admin.js';
|
||||
@@ -65,13 +65,10 @@ export async function startRepl(options: ReplOptions = {}): Promise<void> {
|
||||
const workspace = new Workspace(process.cwd());
|
||||
workspace.init();
|
||||
|
||||
// Load user customizations from ~/.waggle/ — H-08 G2: override-aware,
|
||||
// so any deployed behavioral-spec or persona evolution takes effect at
|
||||
// startup.
|
||||
// Load user customizations from ~/.waggle/.
|
||||
const waggleHome = path.join(os.homedir(), '.waggle');
|
||||
const composedPrompt = loadSystemPromptWithOverrides(waggleHome);
|
||||
const userSystemPrompt = composedPrompt.userPromptMd;
|
||||
assertOverridesReachActiveSpec(waggleHome, composedPrompt.behavioralSpec);
|
||||
const skills = loadSkills(waggleHome);
|
||||
|
||||
// Load user-configurable hooks from ~/.waggle/hooks.json
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@waggle/core",
|
||||
"version": "0.1.0",
|
||||
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import, optimization-log. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
|
||||
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
|
||||
@@ -12,15 +12,13 @@ import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'prompt_optimization' | 'monthly_assessment' | 'connector_fetch' | 'loop';
|
||||
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'connector_fetch' | 'loop';
|
||||
|
||||
export const VALID_JOB_TYPES: Set<string> = new Set([
|
||||
'agent_task',
|
||||
'memory_consolidation',
|
||||
'workspace_health',
|
||||
'proactive',
|
||||
'prompt_optimization',
|
||||
'monthly_assessment',
|
||||
'connector_fetch',
|
||||
// Loop v0: a stateful, memory-powered, report-only (L1) scheduled automation.
|
||||
// Composes recall + maker (toolless LLM) + checker (LLMJudge) + memory write.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
|
||||
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
|
||||
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
|
||||
// memory-import, optimization-log.
|
||||
// memory-import.
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
export {
|
||||
@@ -58,15 +58,10 @@ export {
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
|
||||
Ontology, validateEntity, type EntitySchema, type ValidationResult,
|
||||
ImprovementSignalStore,
|
||||
type ImprovementSignal, type ActionableSignal, type SignalCategory, type ActionableThresholds,
|
||||
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
|
||||
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
EvolutionRunStore, EVOLUTION_RUNS_TABLE_SQL,
|
||||
type EvolutionRun, type EvolutionRunStatus, type EvolutionRunTarget,
|
||||
type CreateEvolutionRunInput, type EvolutionRunFilter,
|
||||
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
@@ -144,7 +139,3 @@ export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge
|
||||
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
|
||||
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
|
||||
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';
|
||||
export {
|
||||
OptimizationLogStore, OPTIMIZATION_LOG_TABLE_SQL,
|
||||
type OptimizationLogEntry, type CreateOptimizationLogInput,
|
||||
} from './optimization-log.js';
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* failed) so there is a verifiable history of what was installed, when, why,
|
||||
* and by whom.
|
||||
*
|
||||
* Follows the same pattern as ImprovementSignalStore — operates on the .mind DB.
|
||||
* Operates on the .mind DB like the other stores in this package.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
/**
|
||||
* Optimization Log Store — SQLite persistence for GEPA prompt optimization data.
|
||||
*
|
||||
* Captures agent interaction metadata (system prompts, tools used, turn counts,
|
||||
* correction signals) for background analysis by the PromptOptimizer.
|
||||
*
|
||||
* Follows the same lazy-table pattern as CronStore and InstallAuditStore —
|
||||
* operates on the .mind DB with auto-table creation.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface OptimizationLogEntry {
|
||||
id: number;
|
||||
session_id: string;
|
||||
workspace_id: string;
|
||||
system_prompt: string;
|
||||
tools_used: string; // JSON array
|
||||
turn_count: number;
|
||||
was_correction: number; // SQLite integer boolean
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface CreateOptimizationLogInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
toolsUsed: string[];
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const OPTIMIZATION_LOG_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS optimization_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
workspace_id TEXT NOT NULL,
|
||||
system_prompt TEXT NOT NULL,
|
||||
tools_used TEXT NOT NULL DEFAULT '[]',
|
||||
turn_count INTEGER NOT NULL DEFAULT 0,
|
||||
was_correction INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_workspace ON optimization_log (workspace_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_timestamp ON optimization_log (timestamp DESC);
|
||||
`;
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class OptimizationLogStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='optimization_log'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(OPTIMIZATION_LOG_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Insert a new optimization log entry capturing an agent interaction. */
|
||||
insert(input: CreateOptimizationLogInput): OptimizationLogEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO optimization_log (session_id, workspace_id, system_prompt, tools_used, turn_count, was_correction, input_tokens, output_tokens)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.sessionId,
|
||||
input.workspaceId,
|
||||
input.systemPrompt,
|
||||
JSON.stringify(input.toolsUsed),
|
||||
input.turnCount,
|
||||
input.wasCorrection ? 1 : 0,
|
||||
input.inputTokens ?? 0,
|
||||
input.outputTokens ?? 0,
|
||||
);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM optimization_log WHERE id = ?',
|
||||
).get(result.lastInsertRowid) as OptimizationLogEntry;
|
||||
}
|
||||
|
||||
/** Get recent optimization log entries, ordered by most recent first. */
|
||||
getRecent(limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get log entries for a specific workspace. */
|
||||
getByWorkspace(workspaceId: string, limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log WHERE workspace_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(workspaceId, limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get aggregate stats: total entries, correction rate, avg turn count, total tokens. */
|
||||
getStats(): { total: number; correctionRate: number; avgTurnCount: number; totalInputTokens: number; totalOutputTokens: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate,
|
||||
COALESCE(AVG(turn_count), 0) as avg_turn_count,
|
||||
COALESCE(SUM(input_tokens), 0) as total_input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) as total_output_tokens
|
||||
FROM optimization_log
|
||||
`).get() as {
|
||||
total: number;
|
||||
correction_rate: number;
|
||||
avg_turn_count: number;
|
||||
total_input_tokens: number;
|
||||
total_output_tokens: number;
|
||||
};
|
||||
|
||||
return {
|
||||
total: row.total,
|
||||
correctionRate: row.correction_rate,
|
||||
avgTurnCount: row.avg_turn_count,
|
||||
totalInputTokens: row.total_input_tokens,
|
||||
totalOutputTokens: row.total_output_tokens,
|
||||
};
|
||||
}
|
||||
|
||||
/** Delete entries older than the given number of days. */
|
||||
pruneOlderThan(days: number): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
`DELETE FROM optimization_log WHERE timestamp < datetime('now', '-' || ? || ' days')`,
|
||||
).run(days);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Clear all entries (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM optimization_log').run();
|
||||
}
|
||||
}
|
||||
@@ -9,12 +9,6 @@ import { KnowledgeGraph } from '@waggle/hive-mind-core';
|
||||
import { MemoryWeaver } from '@waggle/weaver';
|
||||
import { Orchestrator } from '@waggle/agent';
|
||||
import { MockEmbedder } from '../mind/helpers/mock-embedder.js';
|
||||
import {
|
||||
PROGRAM_REGISTRY,
|
||||
createSummarizer,
|
||||
createClassifier,
|
||||
createPromptExpander,
|
||||
} from '@waggle/optimizer';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
@@ -373,16 +367,4 @@ describe('Full Integration Test', () => {
|
||||
expect(prompt).toContain('search_memory');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Optimizer integration', () => {
|
||||
it('all Ax programs are properly defined and constructable', () => {
|
||||
expect(PROGRAM_REGISTRY.length).toBe(3);
|
||||
for (const entry of PROGRAM_REGISTRY) {
|
||||
const program = entry.create();
|
||||
expect(program).toBeDefined();
|
||||
expect(entry.signature.getInputFields().length).toBeGreaterThan(0);
|
||||
expect(entry.signature.getOutputFields().length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
Copyright (c) 2026 Marko Markovic. All rights reserved.
|
||||
|
||||
This software is proprietary and confidential. Unauthorized copying,
|
||||
modification, distribution, or use of this software, via any medium,
|
||||
is strictly prohibited.
|
||||
@@ -1,41 +0,0 @@
|
||||
# @waggle/optimizer
|
||||
|
||||
Thin wrapper around [@ax-llm/ax](https://github.com/ax-llm/ax) that exposes a
|
||||
handful of typed LLM-program primitives (`summarizer`, `classifier`,
|
||||
`prompt_expander`). Consumed by the Waggle server's `optimizer-service.ts`
|
||||
to run Ax programs with vault-resolved API keys.
|
||||
|
||||
## Why this package still exists
|
||||
|
||||
The Skills 2.0 verification doc asked whether this package should be archived
|
||||
in favor of the evolution stack in `packages/agent/` (GEPA loop, evolve-schema,
|
||||
iterative-optimizer, judge, compose-evolution). The answer is **no** — the two
|
||||
systems solve different problems:
|
||||
|
||||
| `@waggle/optimizer` | `packages/agent/src/iterative-optimizer.ts` + friends |
|
||||
|---|---|
|
||||
| Runs a fixed Ax program (e.g. "summarize this text") once | Evolves a prompt across many trials by proposing mutations and scoring with a judge |
|
||||
| One-shot execution | Closed loop: generate → judge → gate → deploy |
|
||||
| API: `optimizer.summarize(text)` | API: `runEvolutionCycle()`, `iterateUntilBudget()` |
|
||||
| 132 lines | 500+ lines across 10+ files |
|
||||
|
||||
The agent-side GEPA / EvolveSchema stack is the "learn to write better prompts
|
||||
over time" system. This package is the "please execute this typed Ax program
|
||||
now" utility. The server uses the latter for deterministic summarization,
|
||||
classification, and prompt expansion tasks that do **not** need a feedback
|
||||
loop.
|
||||
|
||||
Production consumer: `packages/server/src/local/services/optimizer-service.ts`
|
||||
— see the `execute()` method.
|
||||
|
||||
## Programs
|
||||
|
||||
- **summarizer** — `textToSummarize → summaryText`
|
||||
- **classifier** — `textToClassify → intentCategory`
|
||||
- **prompt_expander** — `briefPrompt → expandedPrompt`
|
||||
|
||||
## When to extend
|
||||
|
||||
- **Need a new one-shot Ax program?** Add a signature in `src/signatures.ts`.
|
||||
- **Need a closed-loop optimizer that evolves prompts?** That's evolution —
|
||||
use `packages/agent/src/evolution-orchestrator.ts`, not this package.
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"name": "@waggle/optimizer",
|
||||
"version": "0.1.0",
|
||||
"description": "Waggle optimizer — GEPA prompt optimization with Ax signatures",
|
||||
"type": "module",
|
||||
"main": "src/optimizer.ts",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ax-llm/ax": "^19.0.12"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
export { PromptOptimizer, type OptimizerConfig, type ExecutionResult } from './optimizer.js';
|
||||
export {
|
||||
SUMMARIZER_SIGNATURE,
|
||||
CLASSIFIER_SIGNATURE,
|
||||
PROMPT_EXPANDER_SIGNATURE,
|
||||
createSummarizer,
|
||||
createClassifier,
|
||||
createPromptExpander,
|
||||
getProgram,
|
||||
PROGRAM_REGISTRY,
|
||||
type ProgramName,
|
||||
type ProgramEntry,
|
||||
} from './signatures.js';
|
||||
@@ -1,50 +0,0 @@
|
||||
import type { AxAIService } from '@ax-llm/ax';
|
||||
import { type ProgramName, getProgram, PROGRAM_REGISTRY } from './signatures.js';
|
||||
|
||||
export interface ExecutionResult {
|
||||
programName: ProgramName;
|
||||
input: Record<string, string>;
|
||||
output: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface OptimizerConfig {
|
||||
ai: AxAIService;
|
||||
}
|
||||
|
||||
export class PromptOptimizer {
|
||||
private ai: AxAIService;
|
||||
|
||||
constructor(config: OptimizerConfig) {
|
||||
this.ai = config.ai;
|
||||
}
|
||||
|
||||
async execute(programName: ProgramName, input: Record<string, string>): Promise<ExecutionResult> {
|
||||
const entry = getProgram(programName);
|
||||
const program = entry.create();
|
||||
const result = await program.forward(this.ai, input);
|
||||
return {
|
||||
programName,
|
||||
input,
|
||||
output: result as Record<string, string>,
|
||||
};
|
||||
}
|
||||
|
||||
async summarize(text: string): Promise<string> {
|
||||
const result = await this.execute('summarizer', { textToSummarize: text });
|
||||
return result.output.summaryText;
|
||||
}
|
||||
|
||||
async classify(text: string): Promise<string> {
|
||||
const result = await this.execute('classifier', { textToClassify: text });
|
||||
return result.output.intentCategory;
|
||||
}
|
||||
|
||||
async expandPrompt(text: string): Promise<string> {
|
||||
const result = await this.execute('prompt_expander', { briefPrompt: text });
|
||||
return result.output.expandedPrompt;
|
||||
}
|
||||
|
||||
listPrograms(): ProgramName[] {
|
||||
return PROGRAM_REGISTRY.map(p => p.name);
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { AxGen, AxSignature } from '@ax-llm/ax';
|
||||
import type { AxAIService } from '@ax-llm/ax';
|
||||
|
||||
// --- Signature Definitions ---
|
||||
|
||||
export const SUMMARIZER_SIGNATURE = new AxSignature(
|
||||
'textToSummarize:string "The text content to summarize" -> summaryText:string "A concise summary of the input text"'
|
||||
);
|
||||
|
||||
export const CLASSIFIER_SIGNATURE = new AxSignature(
|
||||
'textToClassify:string "Text to classify into a category" -> intentCategory:class "question, command, observation, request, greeting"'
|
||||
);
|
||||
|
||||
export const PROMPT_EXPANDER_SIGNATURE = new AxSignature(
|
||||
'briefPrompt:string "A brief or vague user prompt" -> expandedPrompt:string "A detailed, well-structured prompt with clear instructions"'
|
||||
);
|
||||
|
||||
// --- Program Definitions ---
|
||||
|
||||
export function createSummarizer(): AxGen {
|
||||
const gen = new AxGen(SUMMARIZER_SIGNATURE);
|
||||
gen.setInstruction(
|
||||
'You are a precise summarizer. Produce a concise summary that captures the key points of the input text. ' +
|
||||
'Keep the summary under 3 sentences.'
|
||||
);
|
||||
return gen;
|
||||
}
|
||||
|
||||
export function createClassifier(): AxGen {
|
||||
const gen = new AxGen(CLASSIFIER_SIGNATURE);
|
||||
gen.setInstruction(
|
||||
'Classify the user input into exactly one intent category. ' +
|
||||
'question = asking for information, command = directing an action, ' +
|
||||
'observation = stating a fact, request = asking for help, greeting = social pleasantry.'
|
||||
);
|
||||
return gen;
|
||||
}
|
||||
|
||||
export function createPromptExpander(): AxGen {
|
||||
const gen = new AxGen(PROMPT_EXPANDER_SIGNATURE);
|
||||
gen.setInstruction(
|
||||
'Take a brief or vague user prompt and expand it into a detailed, well-structured prompt. ' +
|
||||
'Add context, specify the desired format, and clarify ambiguities. ' +
|
||||
'The expanded prompt should be actionable by an AI assistant.'
|
||||
);
|
||||
return gen;
|
||||
}
|
||||
|
||||
// --- Program Registry ---
|
||||
|
||||
export type ProgramName = 'summarizer' | 'classifier' | 'prompt_expander';
|
||||
|
||||
export interface ProgramEntry {
|
||||
name: ProgramName;
|
||||
create: () => AxGen;
|
||||
signature: AxSignature;
|
||||
}
|
||||
|
||||
export const PROGRAM_REGISTRY: ProgramEntry[] = [
|
||||
{ name: 'summarizer', create: createSummarizer, signature: SUMMARIZER_SIGNATURE },
|
||||
{ name: 'classifier', create: createClassifier, signature: CLASSIFIER_SIGNATURE },
|
||||
{ name: 'prompt_expander', create: createPromptExpander, signature: PROMPT_EXPANDER_SIGNATURE },
|
||||
];
|
||||
|
||||
export function getProgram(name: ProgramName): ProgramEntry {
|
||||
const entry = PROGRAM_REGISTRY.find(p => p.name === name);
|
||||
if (!entry) throw new Error(`Unknown program: ${name}`);
|
||||
return entry;
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
SUMMARIZER_SIGNATURE,
|
||||
CLASSIFIER_SIGNATURE,
|
||||
PROMPT_EXPANDER_SIGNATURE,
|
||||
createSummarizer,
|
||||
createClassifier,
|
||||
createPromptExpander,
|
||||
getProgram,
|
||||
PROGRAM_REGISTRY,
|
||||
type ProgramName,
|
||||
} from '../src/signatures.js';
|
||||
import { PromptOptimizer } from '../src/optimizer.js';
|
||||
import type { AxAIService } from '@ax-llm/ax';
|
||||
|
||||
// Convert camelCase to Title Case (what Ax uses in prompts)
|
||||
function toTitleCase(camel: string): string {
|
||||
return camel.replace(/([A-Z])/g, ' $1').replace(/^./, s => s.toUpperCase()).trim();
|
||||
}
|
||||
|
||||
// Mock AI service that returns predictable outputs
|
||||
// Must implement enough of AxAIService for AxGen.forward() to work
|
||||
function createMockAI(responses: Record<string, string>): AxAIService {
|
||||
// Format response as Ax expects: Title Case field names followed by values
|
||||
const content = Object.entries(responses)
|
||||
.map(([k, v]) => `${toTitleCase(k)}: ${v}`)
|
||||
.join('\n');
|
||||
|
||||
return {
|
||||
chat: vi.fn().mockResolvedValue({
|
||||
results: [{ content, index: 0 }],
|
||||
modelUsage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
|
||||
}),
|
||||
embed: vi.fn(),
|
||||
getFeatures: vi.fn().mockReturnValue({
|
||||
functions: true,
|
||||
streaming: true,
|
||||
hasThinkingBudget: false,
|
||||
hasShowThoughts: false,
|
||||
structuredOutputs: false,
|
||||
media: {},
|
||||
caching: { supported: false, types: [] },
|
||||
thinking: false,
|
||||
multiTurn: true,
|
||||
}),
|
||||
getOptions: vi.fn().mockReturnValue({ debug: false, verbose: false }),
|
||||
setOptions: vi.fn(),
|
||||
getName: vi.fn().mockReturnValue('mock'),
|
||||
getId: vi.fn().mockReturnValue('mock-id'),
|
||||
getModelList: vi.fn().mockReturnValue([]),
|
||||
getLastUsedChatModel: vi.fn().mockReturnValue('mock-model'),
|
||||
getLastUsedEmbedModel: vi.fn().mockReturnValue(undefined),
|
||||
getLastUsedModelConfig: vi.fn().mockReturnValue(undefined),
|
||||
getMetrics: vi.fn().mockReturnValue(undefined),
|
||||
getLogger: vi.fn().mockReturnValue(undefined),
|
||||
} as unknown as AxAIService;
|
||||
}
|
||||
|
||||
describe('Prompt Optimization (Ax Integration)', () => {
|
||||
describe('Signature definitions', () => {
|
||||
it('summarizer signature has correct input/output fields', () => {
|
||||
const inputs = SUMMARIZER_SIGNATURE.getInputFields();
|
||||
const outputs = SUMMARIZER_SIGNATURE.getOutputFields();
|
||||
expect(inputs).toHaveLength(1);
|
||||
expect(inputs[0].name).toBe('textToSummarize');
|
||||
expect(outputs).toHaveLength(1);
|
||||
expect(outputs[0].name).toBe('summaryText');
|
||||
});
|
||||
|
||||
it('classifier signature has correct input/output fields', () => {
|
||||
const inputs = CLASSIFIER_SIGNATURE.getInputFields();
|
||||
const outputs = CLASSIFIER_SIGNATURE.getOutputFields();
|
||||
expect(inputs).toHaveLength(1);
|
||||
expect(inputs[0].name).toBe('textToClassify');
|
||||
expect(outputs).toHaveLength(1);
|
||||
expect(outputs[0].name).toBe('intentCategory');
|
||||
expect(outputs[0].type?.name).toBe('class');
|
||||
expect(outputs[0].type?.options).toEqual(['question', 'command', 'observation', 'request', 'greeting']);
|
||||
});
|
||||
|
||||
it('prompt expander signature has correct input/output fields', () => {
|
||||
const inputs = PROMPT_EXPANDER_SIGNATURE.getInputFields();
|
||||
const outputs = PROMPT_EXPANDER_SIGNATURE.getOutputFields();
|
||||
expect(inputs).toHaveLength(1);
|
||||
expect(inputs[0].name).toBe('briefPrompt');
|
||||
expect(outputs).toHaveLength(1);
|
||||
expect(outputs[0].name).toBe('expandedPrompt');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Program creation', () => {
|
||||
it('creates summarizer program with instruction', () => {
|
||||
const program = createSummarizer();
|
||||
expect(program).toBeDefined();
|
||||
expect(program.getInstruction()).toContain('summarizer');
|
||||
});
|
||||
|
||||
it('creates classifier program with instruction', () => {
|
||||
const program = createClassifier();
|
||||
expect(program).toBeDefined();
|
||||
expect(program.getInstruction()).toContain('Classify');
|
||||
});
|
||||
|
||||
it('creates prompt expander program with instruction', () => {
|
||||
const program = createPromptExpander();
|
||||
expect(program).toBeDefined();
|
||||
expect(program.getInstruction()).toContain('expand');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Program registry', () => {
|
||||
it('contains all three programs', () => {
|
||||
expect(PROGRAM_REGISTRY).toHaveLength(3);
|
||||
const names = PROGRAM_REGISTRY.map(p => p.name);
|
||||
expect(names).toContain('summarizer');
|
||||
expect(names).toContain('classifier');
|
||||
expect(names).toContain('prompt_expander');
|
||||
});
|
||||
|
||||
it('getProgram returns correct entry', () => {
|
||||
const entry = getProgram('summarizer');
|
||||
expect(entry.name).toBe('summarizer');
|
||||
expect(typeof entry.create).toBe('function');
|
||||
expect(entry.signature).toBe(SUMMARIZER_SIGNATURE);
|
||||
});
|
||||
|
||||
it('getProgram throws for unknown program', () => {
|
||||
expect(() => getProgram('nonexistent' as ProgramName)).toThrow('Unknown program: nonexistent');
|
||||
});
|
||||
|
||||
it('each registry entry creates a valid program', () => {
|
||||
for (const entry of PROGRAM_REGISTRY) {
|
||||
const program = entry.create();
|
||||
expect(program).toBeDefined();
|
||||
expect(program.getInstruction()).toBeTruthy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('PromptOptimizer', () => {
|
||||
it('lists all available programs', () => {
|
||||
const mockAI = createMockAI({});
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const programs = optimizer.listPrograms();
|
||||
expect(programs).toEqual(['summarizer', 'classifier', 'prompt_expander']);
|
||||
});
|
||||
});
|
||||
|
||||
// API-dependent tests - these test actual execution through the Ax framework
|
||||
// They use a mock AI service, but still exercise the full AxGen.forward() pipeline
|
||||
describe('Signature execution (API-dependent)', () => {
|
||||
it('summarizer produces string output type', async () => {
|
||||
const mockAI = createMockAI({ summaryText: 'This is a test summary.' });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const result = await optimizer.execute('summarizer', {
|
||||
textToSummarize: 'A long text about AI agents and their capabilities in modern software.',
|
||||
});
|
||||
expect(result.programName).toBe('summarizer');
|
||||
expect(result.input.textToSummarize).toContain('AI agents');
|
||||
expect(typeof result.output.summaryText).toBe('string');
|
||||
});
|
||||
|
||||
it('classifier returns valid category', async () => {
|
||||
const mockAI = createMockAI({ intentCategory: 'question' });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const result = await optimizer.execute('classifier', {
|
||||
textToClassify: 'What is the weather like today?',
|
||||
});
|
||||
expect(result.programName).toBe('classifier');
|
||||
expect(['question', 'command', 'observation', 'request', 'greeting']).toContain(
|
||||
result.output.intentCategory
|
||||
);
|
||||
});
|
||||
|
||||
it('prompt expander produces expanded output', async () => {
|
||||
const mockAI = createMockAI({ expandedPrompt: 'Please write a comprehensive blog post about AI, covering...' });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const result = await optimizer.execute('prompt_expander', {
|
||||
briefPrompt: 'Write about AI',
|
||||
});
|
||||
expect(result.programName).toBe('prompt_expander');
|
||||
expect(typeof result.output.expandedPrompt).toBe('string');
|
||||
expect(result.output.expandedPrompt.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Intent classification across categories', () => {
|
||||
const categories = [
|
||||
{ input: 'What time is it?', expected: 'question' },
|
||||
{ input: 'Delete all files now', expected: 'command' },
|
||||
{ input: 'The sky is blue', expected: 'observation' },
|
||||
{ input: 'Can you help me with this?', expected: 'request' },
|
||||
{ input: 'Hello, how are you?', expected: 'greeting' },
|
||||
];
|
||||
|
||||
for (const { input, expected } of categories) {
|
||||
it(`classifies "${input}" as ${expected}`, async () => {
|
||||
const mockAI = createMockAI({ intentCategory: expected });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const category = await optimizer.classify(input);
|
||||
expect(category).toBe(expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('Convenience methods', () => {
|
||||
it('summarize() returns summary string', async () => {
|
||||
const mockAI = createMockAI({ summaryText: 'Brief summary here.' });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const summary = await optimizer.summarize('Long text about many topics...');
|
||||
expect(typeof summary).toBe('string');
|
||||
expect(summary.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('expandPrompt() returns expanded string', async () => {
|
||||
const mockAI = createMockAI({ expandedPrompt: 'Detailed expanded prompt...' });
|
||||
const optimizer = new PromptOptimizer({ ai: mockAI });
|
||||
const expanded = await optimizer.expandPrompt('Fix bug');
|
||||
expect(typeof expanded).toBe('string');
|
||||
expect(expanded.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "tests"]
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
testTimeout: 30_000,
|
||||
include: ['tests/**/*.test.ts'],
|
||||
},
|
||||
});
|
||||
@@ -18,7 +18,6 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.78.0",
|
||||
"@ax-llm/ax": "^19.0.12",
|
||||
"@clerk/fastify": "^3.1.3",
|
||||
"@fastify/cors": "^10.0.0",
|
||||
"@fastify/static": "^10.1.2",
|
||||
|
||||
@@ -28,7 +28,7 @@ export interface ChatTurnRequest {
|
||||
/** Persisted session id — one per IM conversation. */
|
||||
session: string;
|
||||
timeoutMs?: number;
|
||||
/** Per-turn persona override (e.g. 'session-reviewer' for the idle watcher). */
|
||||
/** Per-turn persona override. */
|
||||
persona?: string;
|
||||
/**
|
||||
* Headless turn: hold gated proposable tools for human approval instead of
|
||||
|
||||
@@ -38,11 +38,11 @@ export function isProposableTool(tool: string): boolean {
|
||||
if (tool === 'send_email' || tool === 'write_file' || tool === 'edit_file' || tool === 'generate_docx') {
|
||||
return true;
|
||||
}
|
||||
// create_skill is the self-evolution proposal vehicle (session-reviewer). Held
|
||||
// here so a headless review turn can never write a skill to disk without human
|
||||
// approval; on approve, executeHeldAction runs the real create_skill tool from
|
||||
// the workspace pool, which persists through the sanctioned, backup-protected
|
||||
// writeSkill path (skill-tools.ts → skill-write-service.ts).
|
||||
// create_skill is held here so a headless turn can never write a skill to
|
||||
// disk without human approval; on approve, executeHeldAction runs the real
|
||||
// create_skill tool from the workspace pool, which persists through the
|
||||
// sanctioned, backup-protected writeSkill path (skill-tools.ts →
|
||||
// skill-write-service.ts).
|
||||
if (tool === 'create_skill') return true;
|
||||
// Connector WRITE actions (connector_<id>_<action> where action mutates).
|
||||
if (tool.startsWith('connector_') && /_(create|update|delete|send|post|transition|remove|add|set|put)(_|$)/.test(tool)) {
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
/**
|
||||
* IdleSessionWatcher — the runtime half of idle-triggered self-evolution
|
||||
* (CowAgent steal #3, v1 "review-before-apply only").
|
||||
*
|
||||
* A 60s daemon (mirrors LocalScheduler's start/stop/tick shape) that scans a
|
||||
* workspace's chat sessions for ones that have gone idle after a real
|
||||
* conversation, and — once per idle session — spawns a RESTRICTED reviewer
|
||||
* (persona `session-reviewer`) through a loopback chat turn. The reviewer reads
|
||||
* the transcript and either proposes a skill patch (held for human approval,
|
||||
* never written to disk autonomously) or replies NOTHING_TO_DO.
|
||||
*
|
||||
* Trust + cost guardrails:
|
||||
* - default OFF (self-evolution.json `enabled:false`) — founder opt-in.
|
||||
* - a session fires at most once per mtime (RAM fired-set keyed sessionId:mtime);
|
||||
* it only re-fires after the file advances (a new message lands).
|
||||
* - a per-day cap bounds spend.
|
||||
* - `channel-*` and `evolve-*` sessions are skipped (IM threads + the reviewer's
|
||||
* own loopback sessions must not be reviewed → no feedback loop).
|
||||
*
|
||||
* Enumeration is PURE (fs.statSync + raw line count). It deliberately does NOT
|
||||
* use readSessionMeta (session-utils.ts) — that lazily backfills a title/summary
|
||||
* onto disk, a write side effect a read-only scan must not trigger.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from './logger.js';
|
||||
import { materialFingerprint } from './notification-gate.js';
|
||||
import type { EmitNotificationOptions, EmitNotificationResult, NotificationEvent } from './routes/notifications.js';
|
||||
|
||||
const log = createLogger('idle-watcher');
|
||||
|
||||
/** Sentinel the reviewer emits when it finds nothing material — never notified. */
|
||||
export const NOTHING_TO_DO = 'NOTHING_TO_DO';
|
||||
|
||||
/** Config file under dataDir. Absent/corrupt ⇒ DEFAULT_CONFIG. */
|
||||
const CONFIG_FILE = 'self-evolution.json';
|
||||
|
||||
export interface SelfEvolutionConfig {
|
||||
/** Master switch — default OFF (founder opt-in). */
|
||||
enabled: boolean;
|
||||
/** Minutes a session must be idle before it is eligible for review. */
|
||||
idleMinutes: number;
|
||||
/** Minimum turns (message lines) — skip trivial/empty sessions. */
|
||||
minTurns: number;
|
||||
/** Hard ceiling on reviews launched per calendar day (spend guard). */
|
||||
maxReviewsPerDay: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: SelfEvolutionConfig = {
|
||||
enabled: false,
|
||||
idleMinutes: 15,
|
||||
minTurns: 6,
|
||||
maxReviewsPerDay: 5,
|
||||
};
|
||||
|
||||
/** Result of one review turn (from the injected runReviewTurn). */
|
||||
export interface ReviewTurnResult {
|
||||
content: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Everything the watcher needs, injected for testability. */
|
||||
export interface IdleSessionWatcherDeps {
|
||||
dataDir: string;
|
||||
/** Run one restricted review turn for a session; returns the reviewer's reply. */
|
||||
runReviewTurn: (input: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
|
||||
/** Bound emitNotification (server-scoped) — supports the anti-nag gate. */
|
||||
emitNotification: (
|
||||
event: Omit<NotificationEvent, 'type' | 'timestamp' | 'id' | 'read'>,
|
||||
options?: EmitNotificationOptions,
|
||||
) => EmitNotificationResult;
|
||||
log?: { info: (msg: string) => void; warn: (msg: string) => void };
|
||||
/** Clock injection point for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Newest N fired keys retained — bounds RAM for a long-running process. */
|
||||
const FIRED_SET_CAP = 500;
|
||||
|
||||
export class IdleSessionWatcher {
|
||||
private readonly deps: IdleSessionWatcherDeps;
|
||||
private readonly now: () => number;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private ticking = false;
|
||||
/** `${sessionId}:${mtimeMs}` — a session re-fires only after its file advances. */
|
||||
private readonly firedKeys = new Set<string>();
|
||||
/** Day-keyed review counter (RAM — acceptable v1; a restart resets the cap). */
|
||||
private dayCount = { day: '', count: 0 };
|
||||
|
||||
constructor(deps: IdleSessionWatcherDeps) {
|
||||
this.deps = deps;
|
||||
this.now = deps.now ?? Date.now;
|
||||
}
|
||||
|
||||
/** Start the tick loop. Default interval is 60 seconds. */
|
||||
start(intervalMs = 60_000): void {
|
||||
if (this.timer) return;
|
||||
this.timer = setInterval(() => {
|
||||
this.tick().catch(() => { /* tick swallows its own errors */ });
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/** Stop the tick loop. */
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.timer !== null;
|
||||
}
|
||||
|
||||
/** Read + validate config. Absent/corrupt/partial ⇒ merged onto DEFAULT_CONFIG. */
|
||||
private loadConfig(): SelfEvolutionConfig {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(path.join(this.deps.dataDir, CONFIG_FILE), 'utf-8')) as unknown;
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { ...DEFAULT_CONFIG };
|
||||
const r = raw as Partial<SelfEvolutionConfig>;
|
||||
return {
|
||||
enabled: typeof r.enabled === 'boolean' ? r.enabled : DEFAULT_CONFIG.enabled,
|
||||
idleMinutes: typeof r.idleMinutes === 'number' && r.idleMinutes > 0 ? r.idleMinutes : DEFAULT_CONFIG.idleMinutes,
|
||||
minTurns: typeof r.minTurns === 'number' && r.minTurns > 0 ? r.minTurns : DEFAULT_CONFIG.minTurns,
|
||||
maxReviewsPerDay: typeof r.maxReviewsPerDay === 'number' && r.maxReviewsPerDay >= 0 ? r.maxReviewsPerDay : DEFAULT_CONFIG.maxReviewsPerDay,
|
||||
};
|
||||
} catch {
|
||||
return { ...DEFAULT_CONFIG };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate eligible sessions across all workspaces. PURE — statSync + raw
|
||||
* line count, no readSessionMeta (that writes). Skips channel-/evolve- prefixes
|
||||
* and sub-threshold sessions.
|
||||
*/
|
||||
private findIdleSessions(cfg: SelfEvolutionConfig): Array<{ sessionId: string; workspaceId: string; mtimeMs: number }> {
|
||||
const workspacesDir = path.join(this.deps.dataDir, 'workspaces');
|
||||
let workspaceIds: string[];
|
||||
try {
|
||||
workspaceIds = fs.readdirSync(workspacesDir, { withFileTypes: true }).filter(d => d.isDirectory()).map(d => d.name);
|
||||
} catch {
|
||||
return []; // no workspaces dir yet
|
||||
}
|
||||
|
||||
const idleMs = cfg.idleMinutes * 60_000;
|
||||
const nowMs = this.now();
|
||||
const out: Array<{ sessionId: string; workspaceId: string; mtimeMs: number }> = [];
|
||||
|
||||
for (const workspaceId of workspaceIds) {
|
||||
const sessionsDir = path.join(workspacesDir, workspaceId, 'sessions');
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(sessionsDir).filter(f => f.endsWith('.jsonl'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const file of files) {
|
||||
const sessionId = file.slice(0, -'.jsonl'.length);
|
||||
// The reviewer's own loopback sessions (evolve-*) and IM threads
|
||||
// (channel-*) are never reviewed — reviewing evolve-* would loop.
|
||||
if (sessionId.startsWith('channel-') || sessionId.startsWith('evolve-')) continue;
|
||||
|
||||
const filePath = path.join(sessionsDir, file);
|
||||
let mtimeMs: number;
|
||||
let turnCount: number;
|
||||
try {
|
||||
mtimeMs = fs.statSync(filePath).mtimeMs;
|
||||
turnCount = countTurns(fs.readFileSync(filePath, 'utf-8'));
|
||||
} catch {
|
||||
continue; // unreadable — skip
|
||||
}
|
||||
|
||||
if (turnCount < cfg.minTurns) continue;
|
||||
if (nowMs - mtimeMs < idleMs) continue; // still active
|
||||
out.push({ sessionId, workspaceId, mtimeMs });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private incrementDayCounter(): void {
|
||||
const today = new Date(this.now()).toISOString().slice(0, 10);
|
||||
if (this.dayCount.day !== today) this.dayCount = { day: today, count: 0 };
|
||||
this.dayCount.count += 1;
|
||||
}
|
||||
|
||||
private dayCountRemaining(cfg: SelfEvolutionConfig): number {
|
||||
const today = new Date(this.now()).toISOString().slice(0, 10);
|
||||
const count = this.dayCount.day === today ? this.dayCount.count : 0;
|
||||
return cfg.maxReviewsPerDay - count;
|
||||
}
|
||||
|
||||
private markFired(key: string): void {
|
||||
this.firedKeys.add(key);
|
||||
while (this.firedKeys.size > FIRED_SET_CAP) {
|
||||
const oldest = this.firedKeys.values().next().value as string | undefined;
|
||||
if (oldest === undefined) break;
|
||||
this.firedKeys.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One tick: find idle sessions, review the ones not yet fired (within the day
|
||||
* cap), notify only on a material, non-empty finding. Single-flight guarded.
|
||||
*/
|
||||
async tick(): Promise<number> {
|
||||
if (this.ticking) return 0;
|
||||
this.ticking = true;
|
||||
let reviewed = 0;
|
||||
try {
|
||||
const cfg = this.loadConfig();
|
||||
if (!cfg.enabled) return 0; // opt-in — no-op when disabled
|
||||
|
||||
const candidates = this.findIdleSessions(cfg);
|
||||
for (const { sessionId, workspaceId, mtimeMs } of candidates) {
|
||||
const key = `${sessionId}:${mtimeMs}`;
|
||||
if (this.firedKeys.has(key)) continue; // already reviewed at this mtime
|
||||
if (this.dayCountRemaining(cfg) <= 0) {
|
||||
(this.deps.log ?? log).info(`[idle-watcher] daily review cap (${cfg.maxReviewsPerDay}) reached — deferring`);
|
||||
break;
|
||||
}
|
||||
|
||||
// Count the review against the cap and mark fired BEFORE the turn so a
|
||||
// NOTHING_TO_DO result still consumes its slot and won't immediately re-fire.
|
||||
this.incrementDayCounter();
|
||||
this.markFired(key);
|
||||
reviewed += 1;
|
||||
|
||||
let result: ReviewTurnResult;
|
||||
try {
|
||||
result = await this.deps.runReviewTurn({ sessionId, workspaceId });
|
||||
} catch (err) {
|
||||
(this.deps.log ?? log).warn(`[idle-watcher] review turn threw for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = (result.content ?? '').trim();
|
||||
if (result.error || content === '' || content === NOTHING_TO_DO) {
|
||||
continue; // nothing material — stay silent
|
||||
}
|
||||
|
||||
// Material finding — notify, anti-nag gated so an identical finding for
|
||||
// the same session on a later tick is suppressed.
|
||||
this.deps.emitNotification(
|
||||
{
|
||||
title: 'Self-evolution: a review found something',
|
||||
body: content.length > 200 ? `${content.slice(0, 197)}...` : content,
|
||||
category: 'agent',
|
||||
actionUrl: '/approvals',
|
||||
},
|
||||
{ dedupeKey: `self-evolution:${sessionId}`, materialHash: materialFingerprint(content) },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
this.ticking = false;
|
||||
}
|
||||
return reviewed;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a bounded, compact transcript of a session's most recent messages.
|
||||
* PURE (readFileSync only). Returns null if unreadable or too short.
|
||||
*
|
||||
* The reviewer runs in a SEPARATE `evolve-*` session and read_file is jailed to
|
||||
* the workspace `files/` dir (resolveSafe), so it cannot open the session JSONL
|
||||
* itself. We therefore embed the transcript in the review message. (Deviation
|
||||
* from the spec's "it has read tools + workspace binding" — the sandbox blocks a
|
||||
* cross-directory read, so providing the transcript is the reliable path.)
|
||||
*/
|
||||
export function readRecentTranscript(
|
||||
dataDir: string,
|
||||
workspaceId: string,
|
||||
sessionId: string,
|
||||
maxMessages = 40,
|
||||
perMessageChars = 1500,
|
||||
): string | null {
|
||||
const filePath = path.join(dataDir, 'workspaces', workspaceId, 'sessions', `${sessionId}.jsonl`);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(filePath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const lines = raw.split('\n').filter(l => l.trim());
|
||||
const messages: string[] = [];
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = JSON.parse(line) as { type?: string; role?: string; content?: string };
|
||||
if (parsed.type === 'meta') continue;
|
||||
if (!parsed.role || !parsed.content) continue;
|
||||
const role = parsed.role.toUpperCase();
|
||||
const content = parsed.content.length > perMessageChars
|
||||
? `${parsed.content.slice(0, perMessageChars)}…`
|
||||
: parsed.content;
|
||||
messages.push(`${role}: ${content}`);
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
if (messages.length === 0) return null;
|
||||
return messages.slice(-maxMessages).join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* The review instruction sent to the `session-reviewer` persona. Embeds the
|
||||
* transcript and states the two things to look for, the default-silent contract,
|
||||
* and the never-invent rule.
|
||||
*/
|
||||
export function buildReviewInstruction(sessionId: string, transcript: string): string {
|
||||
return [
|
||||
`You are reviewing the finished session "${sessionId}". Its recent conversation is below, between the markers.`,
|
||||
'',
|
||||
'--- BEGIN SESSION TRANSCRIPT ---',
|
||||
transcript,
|
||||
'--- END SESSION TRANSCRIPT ---',
|
||||
'',
|
||||
'Examine ONLY the transcript above and look for exactly two things:',
|
||||
'1. Promised-but-undelivered deliverables — the assistant said it would do or produce something and never did.',
|
||||
'2. Recurring capability failures fixable by a skill patch — the same tool/skill/workflow failed more than once.',
|
||||
'',
|
||||
'If — and only if — you find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence: state the finding with its evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill (it will be held for the human to approve — it is NOT written to disk now).',
|
||||
`Otherwise reply with exactly: ${NOTHING_TO_DO}`,
|
||||
'Never invent a finding. If you cannot cite the transcript, reply NOTHING_TO_DO.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn count for a session JSONL: non-empty lines, minus the leading meta line
|
||||
* if present. Pure — parses only the first line to detect meta, never writes.
|
||||
*/
|
||||
export function countTurns(raw: string): number {
|
||||
const lines = raw.split('\n').filter(l => l.trim());
|
||||
if (lines.length === 0) return 0;
|
||||
try {
|
||||
const first = JSON.parse(lines[0]) as { type?: string };
|
||||
if (first?.type === 'meta') return lines.length - 1;
|
||||
} catch {
|
||||
// First line isn't JSON meta — count every non-empty line.
|
||||
}
|
||||
return lines.length;
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import websocket from '@fastify/websocket';
|
||||
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, OptimizationLogStore, ImprovementSignalStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, EvolutionRunStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
|
||||
import { MindDB, MultiMind, MultiMindCache, WorkspaceManager, WaggleConfig, createEmbeddingProvider, type EmbeddingProviderConfig, type EmbeddingProviderInstance, FrameStore, SessionStore, SuppressionStore, InstallAuditStore, CronStore, AwarenessLayer, VaultStore, SkillHashStore, HarvestSourceStore, ClaudeCodeAdapter, reconcileIndexes, TeamSync, TelemetryStore, TELEMETRY_EVENTS, ExecutionTraceStore, harvestSetHash, type WorkspaceConfig } from '@waggle/core';
|
||||
import { corsOriginAllowed } from './cors-config.js';
|
||||
import { getBoundTeamServer } from './team-server-binding.js';
|
||||
import { fetchTeamServer } from './team-server-egress.js';
|
||||
@@ -46,18 +46,14 @@ import {
|
||||
createBrowserTools,
|
||||
createLspTools,
|
||||
createCliTools,
|
||||
createInsightsTools,
|
||||
createConnectorSearchTools,
|
||||
createCrossWorkspaceTools,
|
||||
McpRuntime,
|
||||
McpToolRetriever,
|
||||
isWithinBudget,
|
||||
getRecentLogs,
|
||||
setPersonaDataDir,
|
||||
deliverCronResult,
|
||||
createDefaultDeliveryPreferences,
|
||||
buildActiveBehavioralSpec,
|
||||
loadBehavioralSpecOverrides,
|
||||
TraceRecorder,
|
||||
HarnessTraceBridge,
|
||||
safeFetch,
|
||||
@@ -108,7 +104,6 @@ import { homeRoutes } from './routes/home.js';
|
||||
import { onboardingRoutes } from './routes/onboarding.js';
|
||||
import { cronRoutes } from './routes/cron.js';
|
||||
import { notificationRoutes, emitNotification, emitSubagentStatus } from './routes/notifications.js';
|
||||
import { materialFingerprint } from './notification-gate.js';
|
||||
import { marketplaceDevRoutes } from './routes/marketplace-dev.js';
|
||||
import { marketplaceRoutes } from './routes/marketplace.js';
|
||||
import {
|
||||
@@ -129,7 +124,6 @@ import { personaRoutes } from './routes/personas.js';
|
||||
import { feedbackRoutes } from './routes/feedback.js';
|
||||
import { workflowRoutes } from './routes/workflows.js';
|
||||
import { workspaceTemplateRoutes } from './routes/workspace-templates.js';
|
||||
import { evolutionRoutes } from './routes/evolution.js';
|
||||
import { exportRoutes } from './routes/export.js';
|
||||
import { dataEraseRoutes } from './routes/data-erase.js';
|
||||
import { costRoutes } from './routes/cost.js';
|
||||
@@ -154,7 +148,6 @@ import { runChannelChatTurn } from './channels/chat-client.js';
|
||||
import { channelRoutes } from './channels/routes.js';
|
||||
import { isChannelPlatform } from './channels/types.js';
|
||||
import { WorkspaceTurnCoordinator } from './workspace-turn-coordinator.js';
|
||||
import { IdleSessionWatcher, readRecentTranscript, buildReviewInstruction, NOTHING_TO_DO } from './idle-watcher.js';
|
||||
import { DreamJournal } from './dream-journal.js';
|
||||
import { dreamRoutes } from './routes/dreams.js';
|
||||
import { oauthRoutes } from './routes/oauth.js';
|
||||
@@ -183,7 +176,6 @@ import {
|
||||
securityMiddleware,
|
||||
} from './security-middleware.js';
|
||||
import { LocalScheduler, makeRecordExecutionCallback } from './cron.js';
|
||||
import { EvolutionService, isEvolutionAutoEnabled } from './services/evolution-service.js';
|
||||
import {
|
||||
generateMorningBriefing,
|
||||
checkStaleWorkspaces,
|
||||
@@ -192,7 +184,6 @@ import {
|
||||
type ProactiveContext,
|
||||
type ProactiveMessage,
|
||||
} from './proactive-handlers.js';
|
||||
import { generateMonthlyAssessment, saveAssessmentToMind } from './monthly-assessment.js';
|
||||
import { MAX_WORKSPACE_SESSIONS, WorkspaceSessionManager } from './workspace-sessions.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { LocalJobStore } from './job-store.js';
|
||||
@@ -392,21 +383,12 @@ declare module 'fastify' {
|
||||
auditStore: import('@waggle/core').InstallAuditStore;
|
||||
cronStore: import('@waggle/core').CronStore;
|
||||
traceStore: import('@waggle/core').ExecutionTraceStore;
|
||||
evolutionStore: import('@waggle/core').EvolutionRunStore;
|
||||
/**
|
||||
* Active behavioral spec — baseline `BEHAVIORAL_SPEC` with any
|
||||
* accepted-and-deployed evolution overrides applied. Re-derived on
|
||||
* the 'behavioral-spec:reloaded' event.
|
||||
*/
|
||||
/** Active behavioral spec — the compiled `BEHAVIORAL_SPEC` baseline. */
|
||||
activeBehavioralSpec: ReturnType<typeof import('@waggle/agent').buildActiveBehavioralSpec>;
|
||||
vault: import('@waggle/core').VaultStore;
|
||||
embeddingProvider: import('@waggle/core').EmbeddingProviderInstance;
|
||||
skillHashStore: import('@waggle/core').SkillHashStore;
|
||||
scheduler: import('./cron.js').LocalScheduler;
|
||||
/** Idle-session self-evolution watcher. Inert unless self-evolution.json enabled. */
|
||||
idleWatcher: import('./idle-watcher.js').IdleSessionWatcher;
|
||||
/** H-10 G1: autonomous evolution daemon. Inert unless WAGGLE_EVOLUTION_AUTO_ENABLED=1. */
|
||||
evolutionService: import('./services/evolution-service.js').EvolutionService;
|
||||
marketplace: import('@waggle/marketplace').MarketplaceDB | null;
|
||||
/** Shared per-workspace mind cache. Routes use this to reach an arbitrary
|
||||
* workspace's mind (e.g. FileIndexer for L-20 auto-indexing). */
|
||||
@@ -418,7 +400,7 @@ declare module 'fastify' {
|
||||
sessionManager: import('./workspace-sessions.js').WorkspaceSessionManager;
|
||||
/**
|
||||
* Bridges workflow-harness phase events into the execution trace store
|
||||
* so harnessed runs become training signal for the self-evolution loop.
|
||||
* so harnessed runs show up in cost tracking and run history.
|
||||
* Started at boot, stopped on close.
|
||||
*/
|
||||
harnessTraceBridge: HarnessTraceBridge;
|
||||
@@ -533,14 +515,14 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
|
||||
seedDefaultCrons(cronStore);
|
||||
|
||||
// Execution trace store — persistent log of agent runs (self-evolution substrate)
|
||||
// Execution trace store — persistent log of agent runs (cost tracking, run history)
|
||||
const traceStore = new ExecutionTraceStore(multiMind.personal);
|
||||
server.decorate('traceStore', traceStore);
|
||||
|
||||
// Workflow-harness → trace bridge. Any harness phase that completes or
|
||||
// aborts anywhere in the app emits on the shared `harnessEvents` emitter;
|
||||
// this bridge translates those into rows in `execution_traces` so harness
|
||||
// runs become training data for the self-evolution loop.
|
||||
// runs show up in cost tracking.
|
||||
const harnessTraceBridge = new HarnessTraceBridge({
|
||||
recorder: new TraceRecorder(traceStore),
|
||||
});
|
||||
@@ -550,22 +532,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
harnessTraceBridge.stop();
|
||||
});
|
||||
|
||||
// Evolution run store — audit trail of proposed/accepted/rejected self-evolution runs
|
||||
const evolutionStore = new EvolutionRunStore(multiMind.personal);
|
||||
server.decorate('evolutionStore', evolutionStore);
|
||||
|
||||
// Behavioral spec with deployed overrides applied (self-evolution deploy target).
|
||||
// Re-derived on the 'behavioral-spec:reloaded' event emitted by evolution routes.
|
||||
const initialOverrides = loadBehavioralSpecOverrides(fullConfig.dataDir);
|
||||
let activeBehavioralSpec = buildActiveBehavioralSpec(initialOverrides);
|
||||
server.decorate('activeBehavioralSpec', activeBehavioralSpec);
|
||||
eventBus.on('behavioral-spec:reloaded', () => {
|
||||
const latest = loadBehavioralSpecOverrides(fullConfig.dataDir);
|
||||
activeBehavioralSpec = buildActiveBehavioralSpec(latest);
|
||||
// Re-decorate so server.activeBehavioralSpec returns the fresh value.
|
||||
// Fastify's decorate() is idempotent for reassignment via the instance.
|
||||
(server as unknown as { activeBehavioralSpec: typeof activeBehavioralSpec }).activeBehavioralSpec = activeBehavioralSpec;
|
||||
});
|
||||
server.decorate('activeBehavioralSpec', buildActiveBehavioralSpec());
|
||||
|
||||
// Vault — encrypted secret storage
|
||||
const vault = new VaultStore(fullConfig.dataDir);
|
||||
@@ -819,86 +786,13 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// Dynamic connector tools (initial — regenerated per workspace in buildToolsForWorkspace)
|
||||
const defaultConnectorTools = connectorRegistry.generateTools();
|
||||
|
||||
// Insights tools — self-analytics so the agent can reflect on its own performance.
|
||||
// Aggregates optimization logs + improvement signals across the personal mind and
|
||||
// every workspace so the agent sees a complete picture regardless of where it runs.
|
||||
const insightsTools = createInsightsTools({
|
||||
getOptimizationLogs: (limit: number) => {
|
||||
const rows: Array<{
|
||||
tools_used: string;
|
||||
was_correction: number;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
timestamp: string;
|
||||
}> = [];
|
||||
try {
|
||||
rows.push(...new OptimizationLogStore(multiMind.personal).getRecent(limit));
|
||||
} catch { /* personal store unavailable — keep going */ }
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
try {
|
||||
rows.push(...new OptimizationLogStore(wsDb).getRecent(limit));
|
||||
} catch { /* workspace store unavailable — skip */ }
|
||||
}
|
||||
rows.sort((a, b) => (b.timestamp ?? '').localeCompare(a.timestamp ?? ''));
|
||||
return rows.slice(0, limit).map(row => {
|
||||
let toolNames = '';
|
||||
try {
|
||||
const parsed = JSON.parse(row.tools_used ?? '[]');
|
||||
if (Array.isArray(parsed)) toolNames = parsed.join(',');
|
||||
} catch { /* malformed — leave empty */ }
|
||||
return {
|
||||
model: undefined,
|
||||
tool_names: toolNames,
|
||||
total_tokens: (row.input_tokens ?? 0) + (row.output_tokens ?? 0),
|
||||
estimated_cost: 0,
|
||||
was_correction: row.was_correction,
|
||||
created_at: row.timestamp,
|
||||
};
|
||||
});
|
||||
},
|
||||
getImprovementSignals: () => {
|
||||
const collectFrom = (db: MindDB) => {
|
||||
const store = new ImprovementSignalStore(db);
|
||||
return [
|
||||
...store.getByCategory('capability_gap'),
|
||||
...store.getByCategory('correction'),
|
||||
...store.getByCategory('workflow_pattern'),
|
||||
];
|
||||
};
|
||||
const all: Array<{
|
||||
category: string;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
}> = [];
|
||||
try { all.push(...collectFrom(multiMind.personal)); } catch { /* skip */ }
|
||||
for (const ws of wsManager.list()) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
try { all.push(...collectFrom(wsDb)); } catch { /* skip */ }
|
||||
}
|
||||
return all.map(s => ({
|
||||
category: s.category,
|
||||
pattern_key: s.pattern_key,
|
||||
detail: s.detail,
|
||||
count: s.count,
|
||||
first_seen: s.first_seen,
|
||||
last_seen: s.last_seen,
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
// Connector search tools — give the agent natural-language discovery over
|
||||
// the 148-entry MCP catalog (find_connector, list_connector_categories).
|
||||
// Static data, zero I/O, no deps — cheap to always-include.
|
||||
const connectorSearchTools = createConnectorSearchTools();
|
||||
|
||||
// Collect all non-subagent tools first (sub-agent tools need the full list)
|
||||
const baseTools = [...mindTools, ...systemTools, ...planTools, ...gitTools, ...documentTools, ...skillTools, ...cronTools, ...searchTools, ...browserTools, ...lspTools, ...cliTools, ...insightsTools, ...connectorSearchTools, ...defaultConnectorTools];
|
||||
const baseTools = [...mindTools, ...systemTools, ...planTools, ...gitTools, ...documentTools, ...skillTools, ...cronTools, ...searchTools, ...browserTools, ...lspTools, ...cliTools, ...connectorSearchTools, ...defaultConnectorTools];
|
||||
|
||||
// PR4: expose the full native-tool name union to the agent-search route so
|
||||
// its "you already have a tool for this" lane isn't blind to search/browser/
|
||||
@@ -945,7 +839,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
const targetDb = wsDb ?? multiMind.personal;
|
||||
const mindLabel = wsDb ? `workspace:${wsId}` : 'personal';
|
||||
|
||||
// Structured content: tag provenance so search/evolution can filter
|
||||
// Structured content: tag provenance so search can filter
|
||||
// for sub-agent-derived frames and so humans can see where it came from.
|
||||
const content = `[Sub-agent: ${result.role}] ${result.agentName}\n\n${result.response}\n\n— tools: ${result.toolsUsed.join(', ') || 'none'} · ${(result.duration / 1000).toFixed(1)}s · ${result.usage.inputTokens + result.usage.outputTokens} tokens`;
|
||||
|
||||
@@ -1874,7 +1768,7 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
// W4.3d: extraction of the benchmark-proven recall lanes
|
||||
// (facts/events/profiles) over frames written since the last run.
|
||||
// Routes through the built-in proxy ('fast' tier = haiku) — the
|
||||
// same internal-LLM pattern as prompt_optimization above.
|
||||
// same internal-LLM pattern as the other cron LLM calls.
|
||||
try {
|
||||
const llmCall = async (prompt: string, _model: 'fast' | 'accurate'): Promise<string> => {
|
||||
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
@@ -2041,143 +1935,6 @@ export async function buildLocalServer(config: Partial<LocalConfig> = {}) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'prompt_optimization': {
|
||||
// Background GEPA/Ax prompt optimization — runs daily at 2 AM.
|
||||
// Iterates workspaces with optimizationEnabled, checks budget, reads logs,
|
||||
// and invokes PromptOptimizer for variant generation.
|
||||
try {
|
||||
const workspaces = wsManager.list().filter(ws => ws.optimizationEnabled);
|
||||
for (const ws of workspaces) {
|
||||
const wsDb = getWorkspaceMindDb(ws.id);
|
||||
if (!wsDb) continue;
|
||||
|
||||
const optStore = new OptimizationLogStore(wsDb);
|
||||
const budget = ws.optimizationBudget ?? 100; // cents, default $1/day
|
||||
|
||||
// Check budget before spending any tokens
|
||||
if (!isWithinBudget(optStore, budget)) {
|
||||
log.info(`[cron] Prompt optimization: workspace "${ws.name}" over budget, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Read recent logs for analysis
|
||||
const recentLogs = getRecentLogs(optStore, 100);
|
||||
if (recentLogs.length < 5) {
|
||||
// Not enough data to optimize yet
|
||||
continue;
|
||||
}
|
||||
|
||||
// Analyze patterns: high correction rate, common tools, avg turn count
|
||||
const stats = optStore.getStats();
|
||||
const correctionRate = stats.correctionRate;
|
||||
const avgTurns = stats.avgTurnCount;
|
||||
|
||||
// Only trigger optimization if correction rate is above threshold
|
||||
// or turn count is significantly above average (indicating inefficiency)
|
||||
if (correctionRate > 0.2 || avgTurns > 15) {
|
||||
const rate = (correctionRate * 100).toFixed(1);
|
||||
log.info(`[cron] Prompt optimization: workspace "${ws.name}" — correction rate ${rate}%, avg turns ${avgTurns.toFixed(1)}`);
|
||||
|
||||
// Re-check budget before the LLM call (another workspace may have consumed tokens)
|
||||
if (!isWithinBudget(optStore, budget)) {
|
||||
log.info(` GEPA optimization skipped — daily budget exceeded`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get the most recent system prompt from the workspace's logs
|
||||
const currentSystemPrompt = recentLogs[0]?.system_prompt ?? '';
|
||||
if (!currentSystemPrompt) {
|
||||
log.info(`[cron] Prompt optimization: no system prompt in logs, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate variant via the built-in Anthropic proxy (localhost)
|
||||
try {
|
||||
const proxyUrl = `http://127.0.0.1:${fullConfig.port ?? 3333}/v1/chat/completions`;
|
||||
const variantResponse = await fetch(proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'claude-sonnet-4-6',
|
||||
max_tokens: 4096,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You are a prompt optimization specialist. Your task is to improve an AI agent's system prompt based on performance signals.
|
||||
|
||||
The agent's correction rate is ${rate}% (target: under 20%) and average turn count is ${avgTurns.toFixed(1)} (target: under 15).
|
||||
|
||||
A high correction rate means the agent frequently misunderstands instructions or produces wrong outputs.
|
||||
A high turn count means the agent takes too many steps to complete tasks.
|
||||
|
||||
Analyze the current system prompt and generate an improved variant that:
|
||||
1. Adds clearer instructions for areas where corrections are common
|
||||
2. Reduces unnecessary verbosity that inflates turn count
|
||||
3. Preserves the core identity and capabilities
|
||||
4. Is production-ready (not a draft or explanation)
|
||||
|
||||
Return ONLY the improved system prompt text. No commentary, no markdown fences, no explanation.`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `Here is the current system prompt to improve:\n\n${currentSystemPrompt.slice(0, 12000)}`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
|
||||
if (!variantResponse.ok) {
|
||||
log.warn(`[cron] GEPA variant generation failed: HTTP ${variantResponse.status}`);
|
||||
} else {
|
||||
const variantBody = await variantResponse.json() as unknown;
|
||||
const variantText = parseOpenAiTextCompletion(variantBody).content;
|
||||
|
||||
if (variantText.length > 100) {
|
||||
// Store the variant in the optimization_log with a marker
|
||||
optStore.insert({
|
||||
sessionId: `gepa-variant-${Date.now()}`,
|
||||
workspaceId: ws.id,
|
||||
systemPrompt: variantText,
|
||||
toolsUsed: ['gepa_variant'],
|
||||
turnCount: 0,
|
||||
wasCorrection: false,
|
||||
inputTokens: currentSystemPrompt.length,
|
||||
outputTokens: variantText.length,
|
||||
});
|
||||
log.info(` GEPA generated prompt variant (correction_rate=${rate}%)`);
|
||||
} else {
|
||||
log.warn(`[cron] GEPA variant too short (${variantText.length} chars), discarding`);
|
||||
}
|
||||
}
|
||||
} catch (variantErr) {
|
||||
log.warn(`[cron] GEPA variant generation error: ${(variantErr as Error).message}`);
|
||||
}
|
||||
|
||||
// Anti-nag: one signal per (workspace, month, correction-rate bucket).
|
||||
// Re-runs on the same month with an unchanged rate bucket are suppressed.
|
||||
emitNotification(server, {
|
||||
title: 'Prompt optimization signal',
|
||||
body: `Workspace "${ws.name}" has ${(correctionRate * 100).toFixed(0)}% correction rate — optimization candidate`,
|
||||
category: 'agent',
|
||||
}, {
|
||||
dedupeKey: `prompt_optimization:${ws.id}`,
|
||||
materialHash: materialFingerprint({
|
||||
wsId: ws.id,
|
||||
period: new Date().toISOString().slice(0, 7),
|
||||
rateBucket: Math.round(correctionRate * 10),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// Prune old logs (>30 days) to keep the .mind file lean
|
||||
optStore.pruneOlderThan(30);
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Prompt optimization failed: ${(err as Error).message}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'agent_task': {
|
||||
// F9: Execute an agent task prompt in a workspace context
|
||||
const taskConfig = JSON.parse(schedule.job_config || '{}');
|
||||
@@ -2363,31 +2120,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'monthly_assessment': {
|
||||
try {
|
||||
const assessment = generateMonthlyAssessment(fullConfig, multiMind.personal);
|
||||
saveAssessmentToMind(multiMind.personal, assessment);
|
||||
log.info(`[cron] Monthly assessment for ${assessment.period}: ${assessment.totalInteractions} interactions, ${(assessment.correctionRate * 100).toFixed(1)}% correction rate`);
|
||||
emitNotification(server, {
|
||||
title: 'Monthly agent assessment ready',
|
||||
body: `${assessment.period} report: ${assessment.totalInteractions} interactions, ${(assessment.correctionRate * 100).toFixed(1)}% correction rate — check your workspace home`,
|
||||
category: 'agent',
|
||||
actionUrl: '/',
|
||||
}, {
|
||||
// Anti-nag: one per period + counters. Same month with unchanged
|
||||
// counters is suppressed; a new month or changed numbers re-fires.
|
||||
dedupeKey: 'monthly_assessment',
|
||||
materialHash: materialFingerprint({
|
||||
period: assessment.period,
|
||||
totalInteractions: assessment.totalInteractions,
|
||||
correctionRateBucket: Math.round(assessment.correctionRate * 100),
|
||||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`[cron] Monthly assessment failed: ${(err as Error).message}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'loop': {
|
||||
// Loop v0 (L1, report-only): a stateful, memory-powered scheduled
|
||||
// automation. Resolve the target mind (a specific workspace, else the
|
||||
@@ -2509,66 +2241,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
scheduler.start();
|
||||
server.decorate('scheduler', scheduler);
|
||||
|
||||
// Self-evolution: idle-session watcher (CowAgent steal #3, v1). Default OFF via
|
||||
// <dataDir>/self-evolution.json; when enabled, it reviews idle sessions through
|
||||
// a restricted `session-reviewer` loopback turn whose only write (create_skill)
|
||||
// is HELD for human approval. Skipped under test to keep unit runs hermetic
|
||||
// (mirrors the ChannelManager VITEST guard below).
|
||||
const idleWatcher = new IdleSessionWatcher({
|
||||
dataDir: fullConfig.dataDir,
|
||||
emitNotification: (event, options) => emitNotification(server, event, options),
|
||||
log: { info: (msg: string) => log.info(msg), warn: (msg: string) => log.warn(msg) },
|
||||
runReviewTurn: async ({ sessionId, workspaceId }) => {
|
||||
const transcript = readRecentTranscript(fullConfig.dataDir, workspaceId, sessionId);
|
||||
if (!transcript) return { content: NOTHING_TO_DO };
|
||||
const res = await runChannelChatTurn({
|
||||
port: server.localConfig.port,
|
||||
sessionToken: server.agentState.wsSessionToken,
|
||||
message: buildReviewInstruction(sessionId, transcript),
|
||||
workspace: workspaceId,
|
||||
session: `evolve-${sessionId}`,
|
||||
persona: 'session-reviewer',
|
||||
proposeHeld: true,
|
||||
origin: 'automation',
|
||||
});
|
||||
return { content: res.content, ...(res.error ? { error: res.error } : {}) };
|
||||
},
|
||||
});
|
||||
server.decorate('idleWatcher', idleWatcher);
|
||||
if (!process.env.VITEST && process.env.NODE_ENV !== 'test') {
|
||||
idleWatcher.start();
|
||||
}
|
||||
server.addHook('onClose', async () => {
|
||||
idleWatcher.stop();
|
||||
});
|
||||
|
||||
// H-10 G1: Evolution service — autonomous self-evolution daemon.
|
||||
// Disabled by default; opt-in via WAGGLE_EVOLUTION_AUTO_ENABLED=1 so
|
||||
// existing deployments don't start burning Anthropic credits silently.
|
||||
// Even when enabled, the service only produces `proposed` runs — accept
|
||||
// still happens manually via the Memory → Evolution UI.
|
||||
const evolutionService = new EvolutionService(
|
||||
{
|
||||
traceStore,
|
||||
runStore: evolutionStore,
|
||||
getApiKey: () => server.vault?.get('anthropic')?.value ?? null,
|
||||
getActiveBehavioralSpec: () => (server as unknown as { activeBehavioralSpec: Record<string, unknown> }).activeBehavioralSpec,
|
||||
log: (level, msg) => {
|
||||
if (level === 'warn') log.warn(`[evolution-service] ${msg}`);
|
||||
else log.info(`[evolution-service] ${msg}`);
|
||||
},
|
||||
},
|
||||
{
|
||||
tickIntervalMs: parseInt(process.env.WAGGLE_EVOLUTION_TICK_INTERVAL_MS ?? '', 10) || undefined,
|
||||
minTracesPerTarget: parseInt(process.env.WAGGLE_EVOLUTION_MIN_TRACES ?? '', 10) || undefined,
|
||||
},
|
||||
);
|
||||
server.decorate('evolutionService', evolutionService);
|
||||
if (isEvolutionAutoEnabled()) {
|
||||
evolutionService.start();
|
||||
log.info('[evolution-service] autonomous evolution enabled (WAGGLE_EVOLUTION_AUTO_ENABLED=1)');
|
||||
}
|
||||
|
||||
// F2: Daily audit event cleanup (retain 90 days by default)
|
||||
const auditRetentionDays = parseInt(process.env.WAGGLE_AUDIT_RETENTION_DAYS ?? '90', 10);
|
||||
const auditCleanupTimer = setInterval(() => {
|
||||
@@ -2807,7 +2479,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
await server.register(personaRoutes);
|
||||
await server.register(feedbackRoutes);
|
||||
await server.register(workspaceTemplateRoutes);
|
||||
await server.register(evolutionRoutes);
|
||||
await server.register(exportRoutes);
|
||||
await server.register(dataEraseRoutes);
|
||||
await server.register(costRoutes);
|
||||
@@ -3288,7 +2959,6 @@ Return ONLY the improved system prompt text. No commentary, no markdown fences,
|
||||
agentRunRegistry.close();
|
||||
stopMarketplaceBackgroundSync();
|
||||
scheduler.stop();
|
||||
evolutionService.stop();
|
||||
await offlineManager.stop();
|
||||
clearInterval(auditCleanupTimer);
|
||||
|
||||
|
||||
@@ -1,346 +0,0 @@
|
||||
/**
|
||||
* Monthly Self-Assessment — generates a structured report of agent performance.
|
||||
*
|
||||
* Reads optimization logs, feedback stats, and capability gap signals from
|
||||
* the personal mind to produce a monthly assessment. Designed to be called
|
||||
* by the cron scheduler on the 1st of each month.
|
||||
*
|
||||
* The assessment is saved as a memory frame (I-frame) in the personal mind
|
||||
* so it becomes part of the agent's long-term self-awareness.
|
||||
*/
|
||||
|
||||
import { MindDB, OptimizationLogStore, FrameStore, SessionStore, ImprovementSignalStore } from '@waggle/core';
|
||||
import type { LocalConfig } from './index.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface MonthlyAssessment {
|
||||
period: string; // "2026-03"
|
||||
totalInteractions: number;
|
||||
correctionRate: number;
|
||||
improvementTrend: string;
|
||||
topStrengths: string[];
|
||||
topWeaknesses: string[];
|
||||
capabilityGapsDetected: string[];
|
||||
skillsInstalled: number;
|
||||
recommendation: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a user-visible local calendar month to UTC SQLite bounds.
|
||||
* Stored timestamps use SQLite datetime('now') (UTC), while monthly reports
|
||||
* follow the same local calendar as the cron schedule.
|
||||
*/
|
||||
function getUtcMonthBounds(yearMonth: string): { startDate: string; endDate: string } {
|
||||
const [year, month] = yearMonth.split('-').map(Number);
|
||||
const toSqliteUtc = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');
|
||||
|
||||
return {
|
||||
startDate: toSqliteUtc(new Date(year, month - 1, 1)),
|
||||
endDate: toSqliteUtc(new Date(year, month, 1)),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compute a correction rate for a given month from optimization logs.
|
||||
*/
|
||||
function computeMonthCorrectionRate(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): { total: number; correctionRate: number } {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate
|
||||
FROM optimization_log
|
||||
WHERE timestamp >= ? AND timestamp < ?
|
||||
`).get(startDate, endDate) as { total: number; correction_rate: number } | undefined;
|
||||
|
||||
return {
|
||||
total: row?.total ?? 0,
|
||||
correctionRate: row?.correction_rate ?? 0,
|
||||
};
|
||||
} catch {
|
||||
return { total: 0, correctionRate: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a correction rate for the prior month for trend comparison.
|
||||
*/
|
||||
function computePriorMonthCorrectionRate(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): number {
|
||||
const [year, month] = yearMonth.split('-').map(Number);
|
||||
const priorMonth = month === 1 ? 12 : month - 1;
|
||||
const priorYear = month === 1 ? year - 1 : year;
|
||||
const priorYearMonth = `${priorYear}-${String(priorMonth).padStart(2, '0')}`;
|
||||
const { correctionRate } = computeMonthCorrectionRate(db, priorYearMonth);
|
||||
return correctionRate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get top feedback reasons from the feedback_entries table for the month.
|
||||
*/
|
||||
function getTopFeedbackReasons(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): { positiveReasons: string[]; negativeReasons: string[] } {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
const negativeReasons: string[] = [];
|
||||
const positiveReasons: string[] = [];
|
||||
|
||||
try {
|
||||
const negRows = db.prepare(`
|
||||
SELECT reason, COUNT(*) as count
|
||||
FROM feedback_entries
|
||||
WHERE rating = 'down' AND reason IS NOT NULL
|
||||
AND created_at >= ? AND created_at < ?
|
||||
GROUP BY reason
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
`).all(startDate, endDate) as Array<{ reason: string; count: number }>;
|
||||
negativeReasons.push(...negRows.map(r => r.reason));
|
||||
} catch {
|
||||
// feedback_entries table might not exist yet
|
||||
}
|
||||
|
||||
// Strengths: areas where positive feedback was given (no reason column for up, so we derive from low correction areas)
|
||||
try {
|
||||
const posCount = db.prepare(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM feedback_entries
|
||||
WHERE rating = 'up'
|
||||
AND created_at >= ? AND created_at < ?
|
||||
`).get(startDate, endDate) as { count: number } | undefined;
|
||||
if (posCount && posCount.count > 0) {
|
||||
positiveReasons.push('Consistent positive user feedback');
|
||||
}
|
||||
} catch {
|
||||
// table might not exist
|
||||
}
|
||||
|
||||
return { positiveReasons, negativeReasons };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get capability gaps detected from the improvement signals store.
|
||||
*/
|
||||
function getCapabilityGaps(signalStore: ImprovementSignalStore): string[] {
|
||||
try {
|
||||
const gaps = signalStore.getByCategory('capability_gap');
|
||||
return gaps
|
||||
.filter(g => g.count >= 2) // Only surface gaps that occurred multiple times
|
||||
.sort((a, b) => b.count - a.count)
|
||||
.slice(0, 5)
|
||||
.map(g => g.pattern_key.replace('missing:', ''));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count skills installed this month from the install audit trail.
|
||||
*/
|
||||
function countSkillsInstalled(
|
||||
db: import('better-sqlite3').Database,
|
||||
yearMonth: string,
|
||||
): number {
|
||||
const { startDate, endDate } = getUtcMonthBounds(yearMonth);
|
||||
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM install_audit
|
||||
WHERE action = 'installed'
|
||||
AND timestamp >= ? AND timestamp < ?
|
||||
`).get(startDate, endDate) as { count: number } | undefined;
|
||||
return row?.count ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a text recommendation based on the assessment data.
|
||||
*/
|
||||
function generateRecommendation(
|
||||
correctionRate: number,
|
||||
trend: string,
|
||||
gaps: string[],
|
||||
weaknesses: string[],
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (correctionRate > 0.3) {
|
||||
parts.push('High correction rate detected. Consider reviewing recurring feedback patterns and adjusting behavior accordingly.');
|
||||
} else if (correctionRate > 0.15) {
|
||||
parts.push('Moderate correction rate. Room for improvement in accuracy and user alignment.');
|
||||
} else if (correctionRate > 0) {
|
||||
parts.push('Low correction rate. Agent is performing well overall.');
|
||||
} else {
|
||||
parts.push('No corrections recorded. Consider encouraging more user feedback to track performance.');
|
||||
}
|
||||
|
||||
if (gaps.length > 0) {
|
||||
parts.push(`Capability gaps detected: ${gaps.slice(0, 3).join(', ')}. Consider installing relevant skills.`);
|
||||
}
|
||||
|
||||
if (weaknesses.length > 0) {
|
||||
parts.push(`Most common issues: ${weaknesses.slice(0, 3).map(w => w.replace(/_/g, ' ')).join(', ')}.`);
|
||||
}
|
||||
|
||||
if (trend.startsWith('+')) {
|
||||
parts.push('Positive trend — keep up the good work.');
|
||||
} else if (trend.startsWith('-')) {
|
||||
parts.push('Negative trend — attention needed on quality.');
|
||||
}
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Generate a monthly self-assessment from the personal mind data.
|
||||
*
|
||||
* @param config - Local server configuration (provides dataDir)
|
||||
* @param personalMind - The personal MindDB instance
|
||||
* @param periodOverride - Optional YYYY-MM override (defaults to previous month)
|
||||
*/
|
||||
export function generateMonthlyAssessment(
|
||||
config: LocalConfig,
|
||||
personalMind: MindDB,
|
||||
periodOverride?: string,
|
||||
): MonthlyAssessment {
|
||||
const db = personalMind.getDatabase();
|
||||
|
||||
// Default to previous month
|
||||
const now = new Date();
|
||||
const prevMonth = now.getMonth() === 0 ? 12 : now.getMonth(); // getMonth() is 0-based
|
||||
const prevYear = now.getMonth() === 0 ? now.getFullYear() - 1 : now.getFullYear();
|
||||
const period = periodOverride ?? `${prevYear}-${String(prevMonth).padStart(2, '0')}`;
|
||||
|
||||
// Correction stats for this month
|
||||
const { total: totalInteractions, correctionRate } = computeMonthCorrectionRate(db, period);
|
||||
|
||||
// Trend: compare to prior month
|
||||
const priorRate = computePriorMonthCorrectionRate(db, period);
|
||||
let improvementTrend = '0%';
|
||||
if (totalInteractions > 0 && priorRate > 0) {
|
||||
const diff = Math.round((priorRate - correctionRate) * 100); // positive = improvement (fewer corrections)
|
||||
improvementTrend = diff >= 0 ? `+${diff}%` : `${diff}%`;
|
||||
}
|
||||
|
||||
// Feedback analysis
|
||||
const { positiveReasons, negativeReasons } = getTopFeedbackReasons(db, period);
|
||||
|
||||
// Capability gaps
|
||||
const signalStore = new ImprovementSignalStore(personalMind);
|
||||
const capabilityGaps = getCapabilityGaps(signalStore);
|
||||
|
||||
// Skills installed
|
||||
const skillsInstalled = countSkillsInstalled(db, period);
|
||||
|
||||
// Derive strengths
|
||||
const topStrengths: string[] = [...positiveReasons];
|
||||
if (correctionRate < 0.15) topStrengths.push('Low correction rate');
|
||||
if (skillsInstalled > 0) topStrengths.push(`${skillsInstalled} new skills adopted`);
|
||||
if (topStrengths.length === 0) topStrengths.push('Stable operation');
|
||||
|
||||
// Derive weaknesses
|
||||
const topWeaknesses: string[] = negativeReasons.map(r => r.replace(/_/g, ' '));
|
||||
if (correctionRate > 0.25) topWeaknesses.push('High correction rate');
|
||||
if (capabilityGaps.length > 0) topWeaknesses.push('Missing capabilities requested');
|
||||
|
||||
// Generate recommendation
|
||||
const recommendation = generateRecommendation(correctionRate, improvementTrend, capabilityGaps, negativeReasons);
|
||||
|
||||
return {
|
||||
period,
|
||||
totalInteractions,
|
||||
correctionRate: +correctionRate.toFixed(3),
|
||||
improvementTrend,
|
||||
topStrengths: topStrengths.slice(0, 5),
|
||||
topWeaknesses: topWeaknesses.slice(0, 5),
|
||||
capabilityGapsDetected: capabilityGaps,
|
||||
skillsInstalled,
|
||||
recommendation,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure an "assessment" session exists in the personal mind.
|
||||
* FrameStore requires a valid gop_id in the sessions table (FK constraint).
|
||||
* We create one permanent session for all assessment frames.
|
||||
*/
|
||||
function ensureAssessmentSession(personalMind: MindDB): string {
|
||||
const sessions = new SessionStore(personalMind);
|
||||
const existing = sessions.getByGopId('assessment');
|
||||
if (existing) return existing.gop_id;
|
||||
|
||||
// Insert a permanent session with gop_id = 'assessment'
|
||||
const raw = personalMind.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO sessions (gop_id, project_id, status, started_at)
|
||||
VALUES ('assessment', NULL, 'active', datetime('now'))
|
||||
`).run();
|
||||
return 'assessment';
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a monthly assessment as an I-frame in the personal mind.
|
||||
* Uses a dedicated GOP (Group of Pictures) namespace: "assessment".
|
||||
*/
|
||||
export function saveAssessmentToMind(personalMind: MindDB, assessment: MonthlyAssessment): void {
|
||||
// A zero-data month has nothing to report — writing "Interactions: 0 /
|
||||
// Correction Rate: 0.0%" frames graded the agent on no data and polluted
|
||||
// the user's memory list with template noise (judge-verified).
|
||||
if (assessment.totalInteractions === 0 && assessment.skillsInstalled === 0) {
|
||||
return;
|
||||
}
|
||||
ensureAssessmentSession(personalMind);
|
||||
const frames = new FrameStore(personalMind);
|
||||
|
||||
// Replace-on-update: one assessment frame per period. A re-fired overdue
|
||||
// monthly job (every sidecar boot re-runs due schedules) must update the
|
||||
// month's report, not accumulate duplicates.
|
||||
frames.deleteByContentPrefix(`# Monthly Agent Assessment — ${assessment.period}`);
|
||||
|
||||
const content = [
|
||||
`# Monthly Agent Assessment — ${assessment.period}`,
|
||||
'',
|
||||
`**Interactions**: ${assessment.totalInteractions}`,
|
||||
`**Correction Rate**: ${(assessment.correctionRate * 100).toFixed(1)}%`,
|
||||
`**Improvement Trend**: ${assessment.improvementTrend}`,
|
||||
'',
|
||||
`## Strengths`,
|
||||
...assessment.topStrengths.map(s => `- ${s}`),
|
||||
'',
|
||||
`## Weaknesses`,
|
||||
...assessment.topWeaknesses.map(w => `- ${w}`),
|
||||
'',
|
||||
`## Capability Gaps`,
|
||||
...(assessment.capabilityGapsDetected.length > 0
|
||||
? assessment.capabilityGapsDetected.map(g => `- ${g}`)
|
||||
: ['- None detected']),
|
||||
'',
|
||||
`## Recommendation`,
|
||||
assessment.recommendation,
|
||||
'',
|
||||
`---`,
|
||||
`Skills installed this month: ${assessment.skillsInstalled}`,
|
||||
`Generated: ${new Date().toISOString()}`,
|
||||
].join('\n');
|
||||
|
||||
// 'system' source: this is an agent-generated report, not something the
|
||||
// user said — stamping it user_stated was a judge-verified provenance lie.
|
||||
frames.createIFrame('assessment', content, 'important', 'system');
|
||||
}
|
||||
@@ -85,20 +85,6 @@ function countMemoryFrames(ctx: ProactiveContext, workspaceId: string): number {
|
||||
}
|
||||
}
|
||||
|
||||
/** Count total tool uses logged in improvement_signals (proxy for usage patterns). */
|
||||
function countToolSignals(ctx: ProactiveContext): number {
|
||||
// Use personal mind to check for improvement signals — they track tool patterns
|
||||
try {
|
||||
const personalMindPath = path.join(ctx.dataDir, 'personal.mind');
|
||||
if (!fs.existsSync(personalMindPath)) return 0;
|
||||
// We don't re-open the personal mind here; just check if improvement_signals table exists
|
||||
// and count entries. This is a lightweight heuristic.
|
||||
return 0; // Will be enriched when improvement signals are populated
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if any capability packs are installed. */
|
||||
function hasInstalledCapabilities(dataDir: string): boolean {
|
||||
const skillsDir = path.join(dataDir, 'skills');
|
||||
|
||||
@@ -990,7 +990,6 @@ const PERSISTED_MEMORY_ACCESS_TOOL_NAMES = new Set([
|
||||
'promote_skill',
|
||||
'auto_extract_skills',
|
||||
'retire_skills',
|
||||
'agent_insights',
|
||||
'compose_workflow',
|
||||
'list_agents',
|
||||
'get_agent_result',
|
||||
|
||||
@@ -6,7 +6,7 @@ import { performance } from 'node:perf_hooks';
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import { createLogger } from '../logger.js';
|
||||
const log = createLogger('chat');
|
||||
import { runAgentLoop, needsConfirmation, needsConfirmationWithAutonomy, isCriticalNeverAutopass, classifyGatedToolRisk, CapabilityRouter, analyzeAndRecordCorrection, recordCapabilityGap, lintMemoryWrite, assessTrust, formatTrustSummary, scanForInjection, AGENT_LOOP_REROUTE_PREFIX, extractEntities, IterationBudget, routeMessage, compressConversation, createDefaultCompressionConfig, needsCompression, computeInputTokenBudget, getModelContextWindow, CredentialPool, loadCredentialPool, extractStatusCode, filterAvailableTools, shouldSuggestCapture, planSkillDistillation, selectAgentRunBudget, TraceRecorder, generateTurnId, logTurnEvent, checkGrounding, READONLY_TOOLS, type ToolDefinition, type TraceHandle } from '@waggle/agent';
|
||||
import { runAgentLoop, needsConfirmation, needsConfirmationWithAutonomy, isCriticalNeverAutopass, classifyGatedToolRisk, CapabilityRouter, lintMemoryWrite, assessTrust, formatTrustSummary, scanForInjection, AGENT_LOOP_REROUTE_PREFIX, extractEntities, IterationBudget, routeMessage, compressConversation, createDefaultCompressionConfig, needsCompression, computeInputTokenBudget, getModelContextWindow, CredentialPool, loadCredentialPool, extractStatusCode, filterAvailableTools, shouldSuggestCapture, planSkillDistillation, selectAgentRunBudget, TraceRecorder, generateTurnId, logTurnEvent, checkGrounding, READONLY_TOOLS, type ToolDefinition, type TraceHandle } from '@waggle/agent';
|
||||
import type { AgentLoopConfig, AgentResponse, Orchestrator, AutonomyLevel, HookRegistry } from '@waggle/agent';
|
||||
import type { WorkspaceSession } from '../workspace-sessions.js';
|
||||
import { buildWorkspaceNowBlock, formatWorkspaceNowPrompt } from './workspace-context.js';
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
stripCapabilityRequestMarker,
|
||||
} from './capability-proposals.js';
|
||||
import { resolveGrantRiskLevel } from '../approval-grants.js';
|
||||
import { getOptimizerService } from '../services/optimizer-service.js';
|
||||
import { validateOrigin } from '../cors-config.js';
|
||||
import { listPersonas, BEHAVIORAL_SPEC, isEnabled, detectTaskShape, isClosedWorldRewriteRequest, type AssembledPrompt } from '@waggle/agent';
|
||||
|
||||
@@ -33,10 +32,7 @@ const NON_RETAINED_TURN_CONTENT = '[Not retained: memory disabled for this turn]
|
||||
*
|
||||
* Previously `getPersona(id)` was used here, which only consulted the static
|
||||
* built-in PERSONAS array — `listPersonas()` adds the custom ones from disk
|
||||
* via `loadCustomPersonas()`. The deploy comment in evolution-deploy.ts
|
||||
* explicitly says "loader picks it up on next listPersonas() call", but
|
||||
* the chat consumer was reading the wrong function. See FR #3 in
|
||||
* docs/GEPA-SCOPE-AUDIT-2026-04-30.md.
|
||||
* via `loadCustomPersonas()`, so custom personas resolve too.
|
||||
*/
|
||||
function resolvePersona(id: string) {
|
||||
return listPersonas().find(p => p.id === id) ?? null;
|
||||
@@ -666,23 +662,13 @@ export const chatRoutes: FastifyPluginAsync = async (server) => {
|
||||
// that it is broken — a symptom memory rots the moment the tool is repaired.
|
||||
const lint = lintMemoryWrite(content, ctx.memoryType);
|
||||
if (lint.verdict === 'capability_symptom') {
|
||||
// Route the observation to the improvement-signal path so a recurring gap
|
||||
// still surfaces (via acquire_capability), rather than into durable memory.
|
||||
// The pre:memory-write ctx carries no workspaceId, so we record on the base
|
||||
// orchestrator's (personal) signal store — the least-invasive reachable
|
||||
// ImprovementSignalStore. Recording is non-blocking.
|
||||
try {
|
||||
recordCapabilityGap(orchestrator.getImprovementSignals(), lint.capability ?? 'capability', lint.reason);
|
||||
} catch {
|
||||
// Non-blocking — a signal-store failure must not swallow the cancel.
|
||||
}
|
||||
log.info('[memory-validation] Capability-failure symptom blocked from memory:', content.slice(0, 100));
|
||||
return {
|
||||
cancel: true,
|
||||
reason:
|
||||
`This looks like a capability failure symptom ("${lint.capability ?? 'a capability'}"), not a durable fact. ` +
|
||||
`Don't memorize that a tool/connector is broken — it will rot when the capability is fixed. ` +
|
||||
`Instead, fix or flag the capability (e.g. use acquire_capability). The gap has been recorded as an improvement signal.`,
|
||||
`Instead, fix or flag the capability (e.g. use acquire_capability).`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1022,8 +1008,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
`;
|
||||
}
|
||||
|
||||
// Behavioral rules from the ACTIVE spec — baseline with any deployed
|
||||
// self-evolution overrides applied. Falls back to the compiled
|
||||
// Behavioral rules from the active spec. Falls back to the compiled
|
||||
// BEHAVIORAL_SPEC when the server hasn't decorated activeBehavioralSpec
|
||||
// (legacy test harness).
|
||||
const activeSpec = server.activeBehavioralSpec ?? BEHAVIORAL_SPEC;
|
||||
@@ -1067,22 +1052,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// W3.3: Inject actionable correction signals — user corrections from prior sessions
|
||||
if (includePersistedMemory) {
|
||||
try {
|
||||
const signalStore = orch.getImprovementSignals();
|
||||
if (signalStore) {
|
||||
const actionable = signalStore.getActionable();
|
||||
if (actionable.length > 0) {
|
||||
prompt += '\n\n# User Corrections (from prior sessions — follow these)\n';
|
||||
for (const signal of actionable) {
|
||||
prompt += `- ${signal.detail} (observed ${signal.count}x)\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
// W1.3/W7.3: add persona only on the legacy path; assembler-owned persona
|
||||
// content stays singular. DOCX guidance always applies; persisted workspace
|
||||
// tone is withheld when the user disables memory reads for this turn.
|
||||
@@ -1130,22 +1099,19 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
*/
|
||||
retry?: boolean;
|
||||
/**
|
||||
* Self-evolution: set by the IdleSessionWatcher's loopback review turn.
|
||||
* A review turn runs headless — no interactive client watches the SSE
|
||||
* Set by headless loopback turns. No interactive client watches the SSE
|
||||
* stream, so a live approval prompt would auto-deny after the timeout and
|
||||
* the proposal would be lost. When true, a gated proposable tool (e.g.
|
||||
* create_skill) is HELD for durable human approval in Approvals instead of
|
||||
* a live SSE prompt, and any other gated tool is denied. The reviewer can
|
||||
* therefore never write to disk without approval.
|
||||
* a live SSE prompt, and any other gated tool is denied.
|
||||
*/
|
||||
proposeHeld?: boolean;
|
||||
/**
|
||||
* Steal #13: automation-origin memory write-back gate. Set ONLY by
|
||||
* headless/automated callers (idle-watcher review turns, scheduled
|
||||
* loops) — an automated turn re-analyzes existing transcripts, so its
|
||||
* post-response write-back (auto-save, skill distillation, KG
|
||||
* extraction, correction detection) would pollute memory with
|
||||
* re-detected "decisions" and false correction signals. IM channel
|
||||
* headless/automated callers (scheduled loops) — an automated turn
|
||||
* re-analyzes existing transcripts, so its post-response write-back
|
||||
* (auto-save, skill distillation, KG extraction) would pollute memory
|
||||
* with re-detected "decisions". IM channel
|
||||
* adapters must NOT set this: inbound IM messages are real user turns.
|
||||
*/
|
||||
origin?: 'automation' | 'router';
|
||||
@@ -1210,10 +1176,9 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
const requestedSessionId = session ?? sessionIdAlias;
|
||||
|
||||
// #13: automated turns skip the post-response memory write-back seams
|
||||
// below. `proposeHeld` is belt-and-braces — the shipped idle-watcher
|
||||
// already sets it, so its review turns are gated even without `origin`.
|
||||
// Compliance execution traces stay ungated. Learned memory, improvement
|
||||
// signals, and skill capture are gated below by the resolved turn policy.
|
||||
// below. `proposeHeld` is belt-and-braces, so held-action turns are gated
|
||||
// even without `origin`. Execution traces stay ungated. Learned memory and
|
||||
// skill capture are gated below by the resolved turn policy.
|
||||
const isAutomatedTurn = origin === 'automation' || !!proposeHeldTurn;
|
||||
|
||||
// Phase B.5: resolve the effective autonomy level for this request.
|
||||
@@ -1467,8 +1432,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
|
||||
// H-07 G4: hoist trace recorder/handle so the outer catch can finalize
|
||||
// aborted/errored traces with outcome='abandoned'. Without this, a failed
|
||||
// turn leaves its row in the 'pending' state and EvalDatasetBuilder skips
|
||||
// it, starving the evolution loop of negative examples.
|
||||
// turn leaves its row in the 'pending' state and its cost goes unrecorded.
|
||||
let traceRecorder: TraceRecorder | null = null;
|
||||
let traceHandle: TraceHandle | null = null;
|
||||
let traceFinalized = false;
|
||||
@@ -1967,57 +1931,17 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// Count prior user messages FIRST — needed by both GEPA and ambiguity guards.
|
||||
// Count prior user messages FIRST — needed by the ambiguity guard.
|
||||
// Q11:A — Mid-conversation follow-ups like "yes", "run it", "LGTM" are valid
|
||||
// replies and should NOT be expanded or flagged as vague.
|
||||
const priorUserMessages = history.filter((m: { role: string }) => m.role === 'user').length;
|
||||
const isFirstUserMessage = priorUserMessages <= 1; // history already includes current message
|
||||
|
||||
// ── GEPA Optimizer: classify + expand vague prompts ──────────
|
||||
// Uses @ax-llm/ax with cheapest model (Haiku) to optimize user input
|
||||
// before the main LLM call. Non-blocking — falls back gracefully.
|
||||
// CRITICAL: Only run on the FIRST user message in a session. Mid-conversation
|
||||
// replies ("yes", "the first three", "thats fine") must NOT be expanded —
|
||||
// GEPA has no conversation context and will misinterpret them as standalone
|
||||
// vague requests, generating phantom instructions the user never intended.
|
||||
let gepaExpanded: string | null = null;
|
||||
if (!hasCustomRunner
|
||||
&& isFirstUserMessage
|
||||
&& !closedWorldRewrite
|
||||
&& !toolFreeAdvisory
|
||||
&& allowDerivedPersistence
|
||||
&& turnMutationPolicy.contextScope === 'default') {
|
||||
try {
|
||||
const optimizer = await getOptimizerService(server);
|
||||
if (optimizer) {
|
||||
const result = await optimizer.expandWithChoices(agentMessage);
|
||||
throwIfTurnAborted();
|
||||
if (result.isVague && result.expanded) {
|
||||
gepaExpanded = result.expanded;
|
||||
sendEvent('step', { content: `GEPA: Expanded prompt for better results` });
|
||||
// Send expansion choices to frontend for ask-first mode
|
||||
if (result.clarifyingQuestions && result.clarifyingQuestions.length > 0) {
|
||||
sendEvent('gepa_choices', {
|
||||
original: agentMessage,
|
||||
expanded: result.expanded,
|
||||
clarifyingQuestions: result.clarifyingQuestions,
|
||||
intent: result.intent,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
throwIfTurnAborted();
|
||||
// Non-blocking — optimizer failure doesn't affect chat
|
||||
}
|
||||
}
|
||||
|
||||
// Build system prompt (with workspace path awareness + recalled memories)
|
||||
// GAP-006: Prepend ambiguity guard when user message is too brief/vague
|
||||
const shouldCheckAmbiguity = isFirstUserMessage
|
||||
&& !gepaExpanded
|
||||
&& !closedWorldRewrite
|
||||
&& turnMutationPolicy.contextScope === 'default'; // Skip when expansion or an explicit evidence boundary already resolves intent
|
||||
&& turnMutationPolicy.contextScope === 'default'; // Skip when an explicit evidence boundary already resolves intent
|
||||
const ambiguityPrefix = (!hasCustomRunner && shouldCheckAmbiguity && isAmbiguousMessage(agentMessage)) ? AMBIGUITY_PROMPT : '';
|
||||
|
||||
// M2-7: Track session start on first user message
|
||||
@@ -2141,8 +2065,8 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// timeout. Convert a gated proposable tool into a DURABLE held action
|
||||
// that ApprovalsApp shows; deny any other gated tool. This runs before the
|
||||
// grant-store shortcut on purpose \u2014 a saved "Always allow" grant must
|
||||
// NOT let a headless reviewer write a skill to disk. The trust boundary:
|
||||
// the reviewer can never persist a skill without explicit human approval.
|
||||
// NOT let a headless turn write a skill to disk. The trust boundary:
|
||||
// a headless turn can never persist a skill without explicit human approval.
|
||||
if (proposeHeldTurn) {
|
||||
if (!allowDerivedPersistence) {
|
||||
sendEvent('step', { content: 'Tool proposal denied because memory is disabled for this turn.' });
|
||||
@@ -2150,7 +2074,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
const heldSource = sessionId.startsWith('channel-')
|
||||
? `channel:${sessionId}`
|
||||
: `session-reviewer:${sessionId}`;
|
||||
: `session:${sessionId}`;
|
||||
const decision = decideReviewTurnTool(server, {
|
||||
workspaceId: effectiveWorkspace || null,
|
||||
source: heldSource,
|
||||
@@ -2573,17 +2497,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
windowedMessages = applyContextWindow(history);
|
||||
}
|
||||
|
||||
// GEPA: if the prompt was expanded, replace the last user message
|
||||
// so the LLM sees the optimized version (original stays in disk history)
|
||||
if (gepaExpanded) {
|
||||
windowedMessages = windowedMessages.map((m, i, arr) => {
|
||||
if (i === arr.length - 1 && m.role === 'user') {
|
||||
return { ...m, content: `${gepaExpanded}\n\n(Original: "${m.content}")` };
|
||||
}
|
||||
return m;
|
||||
});
|
||||
}
|
||||
|
||||
// Governance policies for team workspaces — direct call (no HTTP loopback)
|
||||
let governancePolicies: { blockedTools?: string[]; allowedSources?: string[] } | undefined;
|
||||
if (wsConfig?.teamId && effectiveWorkspace) {
|
||||
@@ -3046,7 +2959,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
const poolKey = credPool?.getKey();
|
||||
const effectiveApiKey = poolKey ?? server.agentState.litellmApiKey;
|
||||
|
||||
// ── Start execution trace (self-evolution substrate) ──
|
||||
// ── Start execution trace (cost tracking, run history) ──
|
||||
// Lazy-created per request so unit tests with no traceStore decorator
|
||||
// (legacy suites) still pass. Assigned to the hoisted outer-scope
|
||||
// variables so the outer catch can finalize with outcome='abandoned'
|
||||
@@ -3340,9 +3253,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
);
|
||||
}
|
||||
|
||||
// ── Finalize the execution trace (self-evolution substrate) ──
|
||||
// Default outcome is 'success'; correction-detector may downgrade to
|
||||
// 'corrected' on the next turn via traceStore.markCorrected().
|
||||
// ── Finalize the execution trace (cost tracking, run history) ──
|
||||
if (traceRecorder && traceHandle) {
|
||||
try {
|
||||
traceRecorder.finalize(traceHandle, {
|
||||
@@ -3359,9 +3270,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
} catch { /* tracing is best-effort — don't fail the response */ }
|
||||
}
|
||||
|
||||
// M8: commit deferred signal markings now that model call succeeded
|
||||
if (!hasCustomRunner && allowDerivedPersistence) sessionOrch.commitSurfacedSignals();
|
||||
|
||||
// ── Post-response memory write-back ──────────────────────
|
||||
// If the agent didn't save memory itself, check if the exchange
|
||||
// contains save-worthy content and auto-save it.
|
||||
@@ -3403,24 +3311,12 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// behavioral-spec prose — detects a successful ≥5-tool turn and
|
||||
// surfaces the distillation directive, so the agent reliably authors
|
||||
// a reusable skill via its own create_skill tool. R2-gated end to
|
||||
// end: a refusal / self-incapacity turn yields no plan. The signal
|
||||
// is recorded idempotently (skill_promotion) so recurring workflows
|
||||
// bubble up through the existing actionable-signal substrate.
|
||||
// end: a refusal / self-incapacity turn yields no plan.
|
||||
// Skipped whenever learned/derived persistence is disabled.
|
||||
if (!hasCustomRunner && allowDerivedPersistence) {
|
||||
const distillPlan = planSkillDistillation(result.toolsUsed ?? [], result.content ?? '');
|
||||
if (distillPlan) {
|
||||
sendEvent('step', { content: distillPlan.directive });
|
||||
try {
|
||||
sessionOrch.getImprovementSignals().record(
|
||||
'skill_promotion',
|
||||
distillPlan.patternKey,
|
||||
distillPlan.directive,
|
||||
{ sessionId, toolCalls: (result.toolsUsed ?? []).length },
|
||||
);
|
||||
} catch {
|
||||
// Signal recording is best-effort — never fail the response.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3459,29 +3355,6 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
}
|
||||
}
|
||||
|
||||
// ── Correction detection ──────────────────────────────────
|
||||
// Analyze user message for corrections and record improvement signals.
|
||||
// Non-blocking — detection failure shouldn't affect the response.
|
||||
// Skipped whenever learned/derived persistence is disabled; a review
|
||||
// instruction can quote old corrections that must not re-fire.
|
||||
if (!hasCustomRunner && allowDerivedPersistence) {
|
||||
try {
|
||||
const signalStore = sessionOrch.getImprovementSignals();
|
||||
analyzeAndRecordCorrection(signalStore, message);
|
||||
|
||||
// Record capability gaps from tool-not-found events
|
||||
const toolNotFoundPattern = /Tool "(.+?)" not found/;
|
||||
if (result.content) {
|
||||
const match = result.content.match(toolNotFoundPattern);
|
||||
if (match) {
|
||||
recordCapabilityGap(signalStore, match[1]);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-blocking
|
||||
}
|
||||
}
|
||||
|
||||
// ── Auto skill capture — detect repeatable workflow patterns ──
|
||||
if (!hasCustomRunner
|
||||
&& allowDerivedPersistence
|
||||
@@ -3698,9 +3571,8 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
);
|
||||
}
|
||||
}
|
||||
// H-07 G4: finalize aborted trace so the evolution dataset builder can
|
||||
// mine it as a negative example. Without this the row stays 'pending'
|
||||
// and GEPA never sees it — starving the loop of counterexamples.
|
||||
// H-07 G4: finalize the aborted trace so its cost is accounted for and
|
||||
// the row does not stay 'pending'.
|
||||
if (traceRecorder && traceHandle && !traceFinalized) {
|
||||
try {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
@@ -3822,7 +3694,7 @@ ${wsConfig?.templateId ? `- Workspace template: ${wsConfig.templateId} — tailo
|
||||
// H-07 G4: defensive trace finalization. Catches SSE-disconnect and any
|
||||
// exotic exit path where the outer catch didn't run. Outcome stays
|
||||
// 'abandoned' because we don't know if the agent produced a usable
|
||||
// output — the correction-detector can upgrade it later if appropriate.
|
||||
// output.
|
||||
if (traceRecorder && traceHandle && !traceFinalized) {
|
||||
try {
|
||||
traceRecorder.finalize(traceHandle, {
|
||||
|
||||
@@ -1,661 +0,0 @@
|
||||
/**
|
||||
* Evolution Routes — Phase 7 of the self-evolution loop.
|
||||
*
|
||||
* HTTP surface for the EvolutionOrchestrator + EvolutionRunStore. The
|
||||
* server owns the deploy callback (file write + cache invalidation);
|
||||
* callers only manipulate run state.
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /api/evolution/runs — list proposals (with filters)
|
||||
* GET /api/evolution/runs/:uuid — single run detail
|
||||
* POST /api/evolution/runs/:uuid/accept — deploy + mark deployed
|
||||
* POST /api/evolution/runs/:uuid/reject — mark rejected
|
||||
* GET /api/evolution/status — aggregate counts for dashboard
|
||||
*/
|
||||
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import type { EvolutionRun, EvolutionRunStatus } from '@waggle/core';
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
LLMJudge,
|
||||
deployPersonaOverride,
|
||||
deployBehavioralSpecOverride,
|
||||
createAnthropicEvolutionLLM,
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
listPersonas,
|
||||
getPersona,
|
||||
BEHAVIORAL_SPEC,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
type EvolutionLLM,
|
||||
type Schema,
|
||||
type SchemaBaselineInput,
|
||||
type EvolutionTarget,
|
||||
type GateOptions,
|
||||
} from '@waggle/agent';
|
||||
import { createLogger } from '../logger.js';
|
||||
|
||||
const log = createLogger('evolution');
|
||||
|
||||
// ── Deploy dispatcher ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Given an accepted run, write the evolved text to the right place on
|
||||
* disk. Returns a structured result for audit. Throws when the run's
|
||||
* target kind isn't one the deployer knows how to handle — the
|
||||
* orchestrator converts that throw into `failed` status.
|
||||
*/
|
||||
function deployFromRun(dataDir: string, run: EvolutionRun): { path: string } {
|
||||
switch (run.target_kind) {
|
||||
case 'persona-system-prompt': {
|
||||
if (!run.target_name) {
|
||||
throw new Error('persona-system-prompt deploy needs target_name (persona id)');
|
||||
}
|
||||
const result = deployPersonaOverride(dataDir, {
|
||||
personaId: run.target_name,
|
||||
systemPrompt: run.winner_text,
|
||||
});
|
||||
return { path: result.path };
|
||||
}
|
||||
case 'behavioral-spec-section': {
|
||||
if (!run.target_name) {
|
||||
throw new Error('behavioral-spec-section deploy needs target_name (section id)');
|
||||
}
|
||||
const result = deployBehavioralSpecOverride(dataDir, {
|
||||
section: run.target_name as BehavioralSpecSection,
|
||||
text: run.winner_text,
|
||||
runUuid: run.run_uuid,
|
||||
});
|
||||
return { path: result.path };
|
||||
}
|
||||
case 'tool-description':
|
||||
case 'skill-body':
|
||||
case 'generic':
|
||||
default:
|
||||
throw new Error(
|
||||
`Deploy for target_kind "${run.target_kind}" is not yet implemented`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Plugin ───────────────────────────────────────────────────────
|
||||
|
||||
export const evolutionRoutes: FastifyPluginAsync = async (server) => {
|
||||
/**
|
||||
* GET /api/evolution/runs — list runs (newest first).
|
||||
*
|
||||
* Query:
|
||||
* ?status=proposed|accepted|rejected|deployed|failed (repeat for multiple)
|
||||
* ?targetKind=persona-system-prompt|...
|
||||
* ?targetName=coder
|
||||
* ?since=2026-04-01T00:00:00Z
|
||||
* ?limit=50
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: {
|
||||
status?: string | string[];
|
||||
targetKind?: string;
|
||||
targetName?: string;
|
||||
since?: string;
|
||||
limit?: string;
|
||||
};
|
||||
}>('/api/evolution/runs', async (request) => {
|
||||
const q = request.query;
|
||||
const statusFilter = Array.isArray(q.status)
|
||||
? q.status as EvolutionRunStatus[]
|
||||
: q.status
|
||||
? [q.status as EvolutionRunStatus]
|
||||
: undefined;
|
||||
|
||||
const rows = server.evolutionStore.list({
|
||||
status: statusFilter,
|
||||
targetKind: q.targetKind as EvolutionRun['target_kind'] | undefined,
|
||||
targetName: q.targetName,
|
||||
since: q.since,
|
||||
limit: q.limit ? Math.max(1, Math.min(500, parseInt(q.limit, 10) || 50)) : 50,
|
||||
});
|
||||
|
||||
return { runs: rows, count: rows.length };
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/runs/:uuid — single run with parsed JSON blobs.
|
||||
*/
|
||||
server.get<{ Params: { uuid: string } }>(
|
||||
'/api/evolution/runs/:uuid',
|
||||
async (request, reply) => {
|
||||
const run = server.evolutionStore.getByUuid(request.params.uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
|
||||
let winnerSchema: unknown = null;
|
||||
if (run.winner_schema_json) {
|
||||
try { winnerSchema = JSON.parse(run.winner_schema_json); } catch { /* keep null */ }
|
||||
}
|
||||
let artifacts: unknown = null;
|
||||
if (run.artifacts_json) {
|
||||
try { artifacts = JSON.parse(run.artifacts_json); } catch { /* keep null */ }
|
||||
}
|
||||
let gateReasons: unknown = [];
|
||||
try { gateReasons = JSON.parse(run.gate_reasons_json); } catch { /* keep [] */ }
|
||||
|
||||
return { ...run, winnerSchema, artifacts, gateReasons };
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* POST /api/evolution/runs/:uuid/accept
|
||||
*
|
||||
* Body: { note?: string }
|
||||
*
|
||||
* Marks the run accepted, invokes the deploy dispatcher, then moves
|
||||
* to deployed (on success) or failed (on throw). Emits a
|
||||
* 'persona:reloaded' or 'behavioral-spec:reloaded' event on the
|
||||
* server event bus so other subsystems (chat route's
|
||||
* systemPromptCache) can invalidate their caches.
|
||||
*/
|
||||
server.post<{
|
||||
Params: { uuid: string };
|
||||
Body: { note?: string };
|
||||
}>('/api/evolution/runs/:uuid/accept', async (request, reply) => {
|
||||
const { uuid } = request.params;
|
||||
const note = (request.body ?? {}).note;
|
||||
|
||||
const run = server.evolutionStore.getByUuid(uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
if (run.status !== 'proposed') {
|
||||
return reply.status(409).send({
|
||||
error: `Run is in status "${run.status}" — only proposed runs can be accepted`,
|
||||
});
|
||||
}
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: server.traceStore,
|
||||
runStore: server.evolutionStore,
|
||||
deploy: async (acceptedRun) => {
|
||||
const result = deployFromRun(server.localConfig.dataDir, acceptedRun);
|
||||
log.info(
|
||||
`Evolution deployed: ${acceptedRun.target_kind}/${acceptedRun.target_name ?? '?'} → ${result.path}`,
|
||||
);
|
||||
// Signal subsystems that the persona / spec source changed so
|
||||
// they can drop any cached derivations.
|
||||
if (acceptedRun.target_kind === 'persona-system-prompt') {
|
||||
server.eventBus.emit('persona:reloaded', { personaId: acceptedRun.target_name });
|
||||
} else if (acceptedRun.target_kind === 'behavioral-spec-section') {
|
||||
server.eventBus.emit('behavioral-spec:reloaded', { section: acceptedRun.target_name });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const updated = await orchestrator.accept(uuid, note);
|
||||
if (!updated) return reply.status(500).send({ error: 'Accept returned no record' });
|
||||
|
||||
return reply.status(200).send(updated);
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/evolution/runs/:uuid/reject
|
||||
*
|
||||
* Body: { reason?: string }
|
||||
*/
|
||||
server.post<{
|
||||
Params: { uuid: string };
|
||||
Body: { reason?: string };
|
||||
}>('/api/evolution/runs/:uuid/reject', async (request, reply) => {
|
||||
const { uuid } = request.params;
|
||||
const reason = (request.body ?? {}).reason;
|
||||
|
||||
const run = server.evolutionStore.getByUuid(uuid);
|
||||
if (!run) return reply.status(404).send({ error: 'Run not found' });
|
||||
if (run.status !== 'proposed') {
|
||||
return reply.status(409).send({
|
||||
error: `Run is in status "${run.status}" — only proposed runs can be rejected`,
|
||||
});
|
||||
}
|
||||
|
||||
const updated = server.evolutionStore.reject(uuid, reason);
|
||||
return reply.status(200).send(updated);
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/targets — enumerate what the user can evolve.
|
||||
*
|
||||
* Returns the available persona ids + the fixed behavioral-spec section
|
||||
* list so the UI can populate its Run-form dropdowns without hardcoding.
|
||||
*/
|
||||
server.get('/api/evolution/targets', async () => {
|
||||
const personas = listPersonas().map(p => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
description: p.description,
|
||||
icon: p.icon,
|
||||
}));
|
||||
return {
|
||||
personas,
|
||||
sections: BEHAVIORAL_SPEC_SECTIONS,
|
||||
defaultSchema: defaultSchemaBaseline('generic'),
|
||||
};
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/baseline?kind=X&name=Y — fetch the current baseline.
|
||||
*
|
||||
* For `persona-system-prompt`, returns the persona's live systemPrompt.
|
||||
* For `behavioral-spec-section`, returns the active section text from
|
||||
* `server.activeBehavioralSpec` (which already accounts for any
|
||||
* deployed overrides). Also returns a sensible default schemaBaseline.
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: { kind?: string; name?: string };
|
||||
}>('/api/evolution/baseline', async (request, reply) => {
|
||||
const { kind, name } = request.query;
|
||||
if (!kind || !name) {
|
||||
return reply.status(400).send({ error: 'kind and name query params are required' });
|
||||
}
|
||||
|
||||
if (kind === 'persona-system-prompt') {
|
||||
const persona = getPersona(name);
|
||||
if (!persona) {
|
||||
return reply.status(404).send({ error: `Unknown persona: ${name}` });
|
||||
}
|
||||
return {
|
||||
baseline: persona.systemPrompt,
|
||||
schemaBaseline: defaultSchemaBaseline('persona-system-prompt'),
|
||||
};
|
||||
}
|
||||
|
||||
if (kind === 'behavioral-spec-section') {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(name as BehavioralSpecSection)) {
|
||||
return reply.status(404).send({ error: `Unknown section: ${name}` });
|
||||
}
|
||||
// Prefer the live active spec (accounts for any deployed overrides)
|
||||
// over BEHAVIORAL_SPEC's compile-time text.
|
||||
const section = name as BehavioralSpecSection;
|
||||
const activeSpec = server.activeBehavioralSpec as unknown as Record<string, string> | undefined;
|
||||
const activeText =
|
||||
activeSpec?.[section]
|
||||
?? (BEHAVIORAL_SPEC as unknown as Record<string, string>)[section];
|
||||
return {
|
||||
baseline: activeText ?? '',
|
||||
schemaBaseline: defaultSchemaBaseline('behavioral-spec-section'),
|
||||
};
|
||||
}
|
||||
|
||||
return reply.status(400).send({
|
||||
error: 'kind must be "persona-system-prompt" or "behavioral-spec-section"',
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/evolution/run
|
||||
*
|
||||
* Trigger a real evolution run synchronously. Uses the Anthropic key from
|
||||
* the vault to build Haiku-backed judge / mutate / execute functions and
|
||||
* invokes `EvolutionOrchestrator.runOnce`. The resulting proposal (if
|
||||
* any) is persisted to the evolution store and surfaced in the response
|
||||
* so the caller can immediately show it to the user.
|
||||
*
|
||||
* Body:
|
||||
* {
|
||||
* targetKind: 'persona-system-prompt' | 'behavioral-spec-section' | ...
|
||||
* targetName: string
|
||||
* baseline: string // current instruction text
|
||||
* schemaBaseline: Schema // current structural baseline (DSPy signature)
|
||||
* minDelta?: number // default 0.02
|
||||
* gepa?: { populationSize?, generations?, miniEvalSize?, anchorEvalSize?, seed? }
|
||||
* schema?: { populationSize?, generations?, evalSize?, anchorEvalSize?, seed? }
|
||||
* gateOptions?: GateOptions
|
||||
* }
|
||||
*
|
||||
* Status codes:
|
||||
* 200 — orchestrator ran (see body.outcome for proposed / skipped-*)
|
||||
* 400 — validation error (missing/invalid body fields)
|
||||
* 422 — no Anthropic API key configured in the vault
|
||||
* 503 — @ax-llm/ax unavailable or LLM init failed
|
||||
*/
|
||||
server.post<{
|
||||
Body: {
|
||||
targetKind?: EvolutionTarget;
|
||||
targetName?: string;
|
||||
baseline?: string;
|
||||
schemaBaseline?: Schema;
|
||||
minDelta?: number;
|
||||
gepa?: {
|
||||
populationSize?: number;
|
||||
generations?: number;
|
||||
miniEvalSize?: number;
|
||||
anchorEvalSize?: number;
|
||||
seed?: number;
|
||||
/**
|
||||
* Max concurrent judge.score() calls during eval stages. Default
|
||||
* 4 — a safe tradeoff between wall time and API politeness. Bump
|
||||
* higher for dedicated endpoints, drop to 1 for strict ordering.
|
||||
*/
|
||||
concurrency?: number;
|
||||
};
|
||||
schema?: {
|
||||
populationSize?: number;
|
||||
generations?: number;
|
||||
evalSize?: number;
|
||||
anchorEvalSize?: number;
|
||||
seed?: number;
|
||||
};
|
||||
gateOptions?: GateOptions;
|
||||
};
|
||||
}>('/api/evolution/run', async (request, reply) => {
|
||||
const body = request.body ?? {};
|
||||
const validation = validateRunBody(body);
|
||||
if (!validation.ok) {
|
||||
return reply.status(400).send({ error: validation.error });
|
||||
}
|
||||
const { targetKind, targetName, baseline, schemaBaseline } = validation;
|
||||
|
||||
// Resolve the Anthropic key from the vault. Evolution is opt-in — if
|
||||
// the user has not added their key we stop here with an actionable
|
||||
// message rather than burning CPU on stub LLMs.
|
||||
const apiKey = server.vault?.get('anthropic')?.value;
|
||||
if (!apiKey) {
|
||||
return reply.status(422).send({
|
||||
error: 'No Anthropic API key configured. Add one in Settings → Vault.',
|
||||
});
|
||||
}
|
||||
|
||||
const llm = await buildEvolutionLLM(apiKey, request.log);
|
||||
if (!llm) {
|
||||
return reply.status(503).send({
|
||||
error: '@ax-llm/ax is not available — cannot initialize the evolution LLM.',
|
||||
});
|
||||
}
|
||||
|
||||
// Compose adapters — shared judge for both stages, wrapped with a
|
||||
// running judge for GEPA so prompts are executed against real LLM
|
||||
// output before scoring. The ES stage has its own executor and uses
|
||||
// the base judge directly.
|
||||
const baseJudge = new LLMJudge(buildJudgeLLMCall(llm));
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
const schemaExecute = buildSchemaExecuteFn(llm);
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: server.traceStore,
|
||||
runStore: server.evolutionStore,
|
||||
// No deploy callback — accept endpoint handles that separately.
|
||||
});
|
||||
|
||||
const runOptions = {
|
||||
targetKind,
|
||||
targetName,
|
||||
baseline,
|
||||
schemaBaseline: schemaBaseline as SchemaBaselineInput['baseline'],
|
||||
minDelta: body.minDelta,
|
||||
gateOptions: body.gateOptions,
|
||||
compose: {
|
||||
schema: {
|
||||
execute: schemaExecute,
|
||||
judge: baseJudge,
|
||||
// Orchestrator mines examples from the trace store when empty;
|
||||
// the field is present for type-completeness only.
|
||||
examples: [],
|
||||
...(body.schema ?? {}),
|
||||
},
|
||||
instructions: {
|
||||
judge: runningJudge,
|
||||
mutate,
|
||||
targetKind,
|
||||
examples: [],
|
||||
// Parallelize the per-candidate mini-eval by default.
|
||||
// Historical sequential runs took 54 min for 10×3×2; a
|
||||
// concurrency of 4 gets that down to roughly a quarter of
|
||||
// the wall time without saturating the Anthropic API.
|
||||
concurrency: 4,
|
||||
...(body.gepa ?? {}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const buildResultPayload = (result: Awaited<ReturnType<typeof orchestrator.runOnce>>) => ({
|
||||
outcome: result.outcome,
|
||||
reason: result.reason,
|
||||
run: result.run,
|
||||
gateResults: result.gateResults,
|
||||
// Compose result can be large; only include the deltas + winner id.
|
||||
composeSummary: result.compose
|
||||
? {
|
||||
combinedDelta: result.compose.combinedDelta,
|
||||
fullyImproved: result.compose.fullyImproved,
|
||||
schemaImproved: result.compose.schema.improved,
|
||||
schemaDelta: result.compose.schema.deltaAccuracy,
|
||||
instructionImproved: result.compose.instructions.improved,
|
||||
instructionDelta: result.compose.instructions.delta,
|
||||
winnerId: result.compose.instructions.winner.id,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
|
||||
// Opt-in SSE: when the client sends Accept: text/event-stream, stream
|
||||
// per-phase progress events. Without the header the route returns a
|
||||
// single JSON body as before (backwards compat for tests + cron).
|
||||
const wantsSse = (request.headers.accept ?? '').toLowerCase().includes('text/event-stream');
|
||||
|
||||
if (wantsSse) {
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
// Disable reverse-proxy buffering (nginx, etc) — streamed data must
|
||||
// reach the client immediately, not in chunks.
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
reply.raw.flushHeaders?.();
|
||||
|
||||
const writeSse = (event: string, data: unknown): void => {
|
||||
// Guard against writes after the socket has already closed.
|
||||
if (reply.raw.writableEnded || reply.raw.destroyed) return;
|
||||
reply.raw.write(`event: ${event}\n`);
|
||||
reply.raw.write(`data: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
|
||||
// If the client disconnects, stop emitting. The orchestrator run
|
||||
// itself continues to completion — cancelling mid-flight LLM work
|
||||
// would waste spend already incurred.
|
||||
let clientClosed = false;
|
||||
request.raw.on('close', () => { clientClosed = true; });
|
||||
|
||||
// Emit an initial event so clients know the stream is live.
|
||||
writeSse('open', { targetKind, targetName });
|
||||
|
||||
try {
|
||||
const result = await orchestrator.runOnce({
|
||||
...runOptions,
|
||||
onProgress: (progress) => {
|
||||
if (!clientClosed) writeSse('progress', progress);
|
||||
},
|
||||
});
|
||||
|
||||
log.info(
|
||||
`Evolution run (SSE): ${targetKind}/${targetName ?? '?'} → ${result.outcome}` +
|
||||
(result.run ? ` (run ${result.run.run_uuid})` : ''),
|
||||
);
|
||||
writeSse('done', buildResultPayload(result));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`Evolution run failed (SSE): ${msg}`);
|
||||
writeSse('error', { error: `Evolution run failed: ${msg}` });
|
||||
} finally {
|
||||
if (!reply.raw.writableEnded) reply.raw.end();
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
// ── JSON path (legacy / non-streaming callers) ──────────────
|
||||
try {
|
||||
const result = await orchestrator.runOnce(runOptions);
|
||||
|
||||
log.info(
|
||||
`Evolution run: ${targetKind}/${targetName ?? '?'} → ${result.outcome}` +
|
||||
(result.run ? ` (run ${result.run.run_uuid})` : ''),
|
||||
);
|
||||
|
||||
return reply.status(200).send(buildResultPayload(result));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`Evolution run failed: ${msg}`);
|
||||
return reply.status(500).send({ error: `Evolution run failed: ${msg}` });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/evolution/status — aggregate counts for the dashboard.
|
||||
*/
|
||||
server.get<{
|
||||
Querystring: {
|
||||
targetKind?: string;
|
||||
targetName?: string;
|
||||
since?: string;
|
||||
};
|
||||
}>('/api/evolution/status', async (request) => {
|
||||
const q = request.query;
|
||||
const counts = server.evolutionStore.statusCounts({
|
||||
targetKind: q.targetKind as EvolutionRun['target_kind'] | undefined,
|
||||
targetName: q.targetName,
|
||||
since: q.since,
|
||||
});
|
||||
const pendingCount = counts.proposed;
|
||||
return { counts, pendingCount };
|
||||
});
|
||||
};
|
||||
|
||||
// ── /run helpers ────────────────────────────────────────────────
|
||||
|
||||
const VALID_TARGET_KINDS: EvolutionTarget[] = [
|
||||
'persona-system-prompt',
|
||||
'behavioral-spec-section',
|
||||
'tool-description',
|
||||
'skill-body',
|
||||
'generic',
|
||||
];
|
||||
|
||||
type ValidationOk = {
|
||||
ok: true;
|
||||
targetKind: EvolutionTarget;
|
||||
targetName: string;
|
||||
baseline: string;
|
||||
schemaBaseline: Schema;
|
||||
};
|
||||
type ValidationErr = { ok: false; error: string };
|
||||
|
||||
/**
|
||||
* Validate the POST /api/evolution/run body. Returns a discriminated union
|
||||
* so the handler can early-return with a 400 on any shape error.
|
||||
*/
|
||||
function validateRunBody(body: unknown): ValidationOk | ValidationErr {
|
||||
if (!body || typeof body !== 'object') {
|
||||
return { ok: false, error: 'Body must be a JSON object.' };
|
||||
}
|
||||
const b = body as Record<string, unknown>;
|
||||
|
||||
if (typeof b.targetKind !== 'string' || !VALID_TARGET_KINDS.includes(b.targetKind as EvolutionTarget)) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `targetKind must be one of: ${VALID_TARGET_KINDS.join(', ')}`,
|
||||
};
|
||||
}
|
||||
if (typeof b.targetName !== 'string' || b.targetName.trim().length === 0) {
|
||||
return { ok: false, error: 'targetName must be a non-empty string.' };
|
||||
}
|
||||
if (typeof b.baseline !== 'string' || b.baseline.trim().length === 0) {
|
||||
return { ok: false, error: 'baseline must be a non-empty string.' };
|
||||
}
|
||||
const sb = b.schemaBaseline;
|
||||
if (!sb || typeof sb !== 'object') {
|
||||
return { ok: false, error: 'schemaBaseline must be an object with {name, fields, version}.' };
|
||||
}
|
||||
const schemaObj = sb as Record<string, unknown>;
|
||||
if (typeof schemaObj.name !== 'string' || !Array.isArray(schemaObj.fields)) {
|
||||
return { ok: false, error: 'schemaBaseline requires string "name" and array "fields".' };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
targetKind: b.targetKind as EvolutionTarget,
|
||||
targetName: b.targetName,
|
||||
baseline: b.baseline,
|
||||
schemaBaseline: sb as Schema,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sensible default schema for a given target kind. Returning this from
|
||||
* the baseline endpoint lets the UI submit without forcing the user to
|
||||
* hand-write a DSPy signature up front.
|
||||
*/
|
||||
function defaultSchemaBaseline(kind: EvolutionTarget): Schema {
|
||||
return {
|
||||
name: kind.replace(/-/g, '_') + '_baseline',
|
||||
version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'short step-by-step justification before the answer',
|
||||
required: false,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'the assistant\u0027s direct response',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal logger shape — structured enough for Fastify's logger, noop-safe
|
||||
* when omitted. Kept narrow so we don't pull in Fastify types here.
|
||||
*/
|
||||
interface RunLogger {
|
||||
warn(payload: unknown, msg?: string): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook point for tests: override the LLM factory by setting
|
||||
* `(globalThis as any).__waggleEvolutionLlmFactory` to an `(apiKey) => EvolutionLLM`.
|
||||
* Production code flows through `createAnthropicEvolutionLLM` with the
|
||||
* default retry policy (5s → 15s → 45s → 135s → 150s, 6 attempts). When a
|
||||
* logger is supplied each backoff is surfaced as a structured warn event
|
||||
* so rate-limit backoffs show up in ops logs instead of looking like a
|
||||
* silent hang.
|
||||
*/
|
||||
async function buildEvolutionLLM(
|
||||
apiKey: string,
|
||||
logger?: RunLogger,
|
||||
): Promise<EvolutionLLM | null> {
|
||||
const override = (globalThis as unknown as {
|
||||
__waggleEvolutionLlmFactory?: (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
||||
}).__waggleEvolutionLlmFactory;
|
||||
if (override) {
|
||||
return await override(apiKey);
|
||||
}
|
||||
return createAnthropicEvolutionLLM(apiKey, {
|
||||
retry: logger
|
||||
? {
|
||||
onRetry: ({ attempt, delayMs, error }) => {
|
||||
const e = error as { status?: number; message?: string };
|
||||
logger.warn(
|
||||
{
|
||||
component: 'evolution.run',
|
||||
attempt,
|
||||
delayMs,
|
||||
status: e?.status,
|
||||
error: e?.message ?? String(error),
|
||||
},
|
||||
`evolution LLM rate-limited — backing off ${delayMs}ms (attempt ${attempt})`,
|
||||
);
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
/**
|
||||
* Feedback REST API Routes — records user feedback on agent responses
|
||||
* and surfaces improvement stats.
|
||||
* and surfaces feedback stats.
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /api/feedback — record user feedback (thumbs up/down + reason)
|
||||
* GET /api/feedback/stats — return improvement stats and trends
|
||||
* GET /api/feedback/stats — return feedback stats and trends
|
||||
*
|
||||
* Feedback is stored in the .mind database in a dedicated feedback_entries table.
|
||||
* Negative feedback is also cross-recorded as correction signals in the
|
||||
* improvement_signals table so the self-improvement loop can track recurring issues.
|
||||
*/
|
||||
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
@@ -86,13 +84,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(sessionId, messageIndex, rating, reason ?? null, detail ?? '');
|
||||
|
||||
// Also record negative feedback as a correction signal in the improvement store
|
||||
// so the self-improvement loop can track recurring issues
|
||||
if (rating === 'down' && reason) {
|
||||
const signalStore = fastify.agentState.orchestrator.getImprovementSignals();
|
||||
signalStore.record('correction', `feedback:${reason}`, detail ?? `User thumbs-down: ${reason}`);
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Unknown error';
|
||||
@@ -100,7 +91,7 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/feedback/stats — return improvement stats
|
||||
// GET /api/feedback/stats — return feedback stats
|
||||
fastify.get('/api/feedback/stats', async (_request, reply) => {
|
||||
try {
|
||||
const db = fastify.multiMind.personal.getDatabase();
|
||||
@@ -130,18 +121,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
`).all() as Array<{ reason: string; count: number }>;
|
||||
const topIssues = topIssuesRows.map(r => r.reason);
|
||||
|
||||
// Corrections this week (from improvement_signals table)
|
||||
let correctionsThisWeek = 0;
|
||||
try {
|
||||
const corrRow = db.prepare(`
|
||||
SELECT COUNT(*) as count FROM improvement_signals
|
||||
WHERE category = 'correction'
|
||||
AND last_seen >= datetime('now', '-7 days')
|
||||
`).get() as { count: number } | undefined;
|
||||
correctionsThisWeek = corrRow?.count ?? 0;
|
||||
} catch {
|
||||
// improvement_signals table might not exist yet
|
||||
}
|
||||
|
||||
// Improvement trend: compare last 7 days positive rate to prior 7 days
|
||||
let improvementTrend = '0%';
|
||||
@@ -173,7 +152,6 @@ export async function feedbackRoutes(fastify: FastifyInstance) {
|
||||
totalFeedback,
|
||||
positiveRate,
|
||||
topIssues,
|
||||
correctionsThisWeek,
|
||||
improvementTrend,
|
||||
};
|
||||
} catch (err) {
|
||||
|
||||
@@ -114,9 +114,10 @@ export interface CronScheduleLike {
|
||||
*/
|
||||
const SYSTEM_JOB_TYPES = new Set([
|
||||
'memory_consolidation',
|
||||
'workspace_health',
|
||||
// Self-learning job types seeded by earlier versions; they no longer run.
|
||||
'monthly_assessment',
|
||||
'prompt_optimization',
|
||||
'workspace_health',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -448,12 +448,6 @@ function cronScheduleWorkspaceIds(
|
||||
if (ALL_WORKSPACE_CRON_JOB_TYPES.has(jobType)) {
|
||||
return [...explicitOwnerIds, ...allWorkspaceIds(fastify)];
|
||||
}
|
||||
if (jobType === 'prompt_optimization') {
|
||||
const optimizedWorkspaceIds = fastify.workspaceManager?.list()
|
||||
.filter((workspace) => Boolean(workspace.optimizationEnabled))
|
||||
.map((workspace) => workspace.id) ?? [];
|
||||
return [...explicitOwnerIds, ...optimizedWorkspaceIds];
|
||||
}
|
||||
if (jobType === 'memory_consolidation') {
|
||||
const rawAction = parsedCronJobConfig(jobConfig, serializedJobConfig)?.action;
|
||||
const action = typeof rawAction === 'string' ? rawAction : undefined;
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
/**
|
||||
* H-10 G1 — Evolution Service.
|
||||
*
|
||||
* Autonomy layer for the self-evolution loop. Runs a quiet background daemon
|
||||
* that, on each tick:
|
||||
*
|
||||
* 1. Picks one eligible evolve target (persona or behavioral-spec section)
|
||||
* whose trace count since its last run passes a minimum-dataset gate.
|
||||
* 2. Invokes EvolutionOrchestrator.runOnce with Anthropic-backed judge /
|
||||
* mutate / execute adapters — same plumbing as POST /api/evolution/run.
|
||||
* 3. Produces a `proposed` EvolutionRun. Accept/reject stays manual via the
|
||||
* existing Memory → Evolution UI; the service never auto-deploys.
|
||||
*
|
||||
* Why a standalone setInterval rather than CronStore integration: CronStore's
|
||||
* job-type enum is a breaking-change surface and the existing cron executor
|
||||
* dispatches by enum. A standalone daemon keeps H-10 surgical, testable, and
|
||||
* independent — the Memory → Evolution "Run now" button shipped in Phase 8.5
|
||||
* still covers manual triggering through the HTTP route.
|
||||
*
|
||||
* Disabled by default. Opt-in via WAGGLE_EVOLUTION_AUTO_ENABLED=1.
|
||||
*/
|
||||
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
LLMJudge,
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
createAnthropicEvolutionLLM,
|
||||
listPersonas,
|
||||
getPersona,
|
||||
BEHAVIORAL_SPEC,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
type EvolutionLLM,
|
||||
type Schema,
|
||||
type EvolutionTarget,
|
||||
} from '@waggle/agent';
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
EvolutionRunStore,
|
||||
TraceOutcome,
|
||||
} from '@waggle/core';
|
||||
|
||||
// ── Public types ─────────────────────────────────────────────────
|
||||
|
||||
export interface EvolutionTargetId {
|
||||
kind: EvolutionTarget;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface EvolutionCandidate extends EvolutionTargetId {
|
||||
/** ISO timestamp of the most recent run for this target, or null if never run. */
|
||||
lastRunAt: string | null;
|
||||
/** Finalized traces recorded since `lastRunAt` (or total when never run). */
|
||||
newTraces: number;
|
||||
}
|
||||
|
||||
/** Result of a single tick — a structured report the caller can log. */
|
||||
export type TickResult =
|
||||
| { skipped: true; reason: string }
|
||||
| {
|
||||
skipped: false;
|
||||
targetKind: EvolutionTarget;
|
||||
targetName: string;
|
||||
outcome: string;
|
||||
runUuid: string | null;
|
||||
};
|
||||
|
||||
export interface ActiveBehavioralSpecLike {
|
||||
[section: string]: unknown;
|
||||
}
|
||||
|
||||
export interface EvolutionServiceDeps {
|
||||
traceStore: ExecutionTraceStore;
|
||||
runStore: EvolutionRunStore;
|
||||
/** Returns the current Anthropic API key or null when none configured. */
|
||||
getApiKey: () => string | null;
|
||||
/** Active behavioral spec (with deployed overrides applied). */
|
||||
getActiveBehavioralSpec: () => ActiveBehavioralSpecLike;
|
||||
/** Test hook — defaults to createAnthropicEvolutionLLM. */
|
||||
llmFactory?: (apiKey: string) => Promise<EvolutionLLM | null>;
|
||||
/**
|
||||
* Test hook — by default we build and invoke an EvolutionOrchestrator.
|
||||
* Overriding this lets unit tests inject a deterministic runner without
|
||||
* spinning up the real LLM plumbing.
|
||||
*/
|
||||
runner?: (target: EvolutionTargetId, baseline: string, apiKey: string) => Promise<TickResult>;
|
||||
/** Structured log sink — defaults to no-op. */
|
||||
log?: (level: 'info' | 'warn', msg: string) => void;
|
||||
}
|
||||
|
||||
export interface EvolutionServiceConfig {
|
||||
/** Milliseconds between tick attempts. Default 6h. Clamped to ≥ 60s. */
|
||||
tickIntervalMs?: number;
|
||||
/** Minimum finalized traces since a target's last run for it to be eligible. */
|
||||
minTracesPerTarget?: number;
|
||||
/** Explicit targets list. Defaults to all personas + all spec sections. */
|
||||
targets?: EvolutionTargetId[];
|
||||
/** Trace outcomes that count toward the dataset gate. */
|
||||
eligibleOutcomes?: TraceOutcome[];
|
||||
}
|
||||
|
||||
// ── Service ──────────────────────────────────────────────────────
|
||||
|
||||
const MIN_TICK_INTERVAL_MS = 60_000;
|
||||
const DEFAULT_TICK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6h
|
||||
const DEFAULT_MIN_TRACES = 20;
|
||||
const DEFAULT_ELIGIBLE_OUTCOMES: TraceOutcome[] = ['success', 'corrected', 'verified'];
|
||||
|
||||
export class EvolutionService {
|
||||
private readonly deps: EvolutionServiceDeps;
|
||||
private readonly cfg: Required<EvolutionServiceConfig>;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private tickInFlight = false;
|
||||
|
||||
constructor(deps: EvolutionServiceDeps, cfg: EvolutionServiceConfig = {}) {
|
||||
this.deps = deps;
|
||||
this.cfg = {
|
||||
tickIntervalMs: Math.max(MIN_TICK_INTERVAL_MS, cfg.tickIntervalMs ?? DEFAULT_TICK_INTERVAL_MS),
|
||||
minTracesPerTarget: Math.max(1, cfg.minTracesPerTarget ?? DEFAULT_MIN_TRACES),
|
||||
targets: cfg.targets ?? EvolutionService.defaultTargets(),
|
||||
eligibleOutcomes: cfg.eligibleOutcomes ?? DEFAULT_ELIGIBLE_OUTCOMES,
|
||||
};
|
||||
}
|
||||
|
||||
/** Default targets = every registered persona + every behavioral-spec section. */
|
||||
static defaultTargets(): EvolutionTargetId[] {
|
||||
const personas: EvolutionTargetId[] = listPersonas().map(p => ({
|
||||
kind: 'persona-system-prompt',
|
||||
name: p.id,
|
||||
}));
|
||||
const sections: EvolutionTargetId[] = BEHAVIORAL_SPEC_SECTIONS.map(s => ({
|
||||
kind: 'behavioral-spec-section',
|
||||
name: s,
|
||||
}));
|
||||
return [...personas, ...sections];
|
||||
}
|
||||
|
||||
/** Kick off the daemon. Fires one tick immediately, then on interval. */
|
||||
start(): void {
|
||||
if (this.timer) return;
|
||||
// Run an initial tick on the next event-loop turn so callers can await
|
||||
// startup side-effects deterministically.
|
||||
setImmediate(() => { void this.tick(); });
|
||||
this.timer = setInterval(() => { void this.tick(); }, this.cfg.tickIntervalMs);
|
||||
// Don't pin the process alive just for the evolution timer.
|
||||
if (typeof this.timer.unref === 'function') this.timer.unref();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.timer !== null;
|
||||
}
|
||||
|
||||
get config(): Readonly<Required<EvolutionServiceConfig>> {
|
||||
return this.cfg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one tick. Returns a structured report. Never throws — errors are
|
||||
* caught, logged, and surfaced through the return value.
|
||||
*/
|
||||
async tick(): Promise<TickResult> {
|
||||
if (this.tickInFlight) {
|
||||
return { skipped: true, reason: 'tick already in progress' };
|
||||
}
|
||||
this.tickInFlight = true;
|
||||
try {
|
||||
const apiKey = this.deps.getApiKey();
|
||||
if (!apiKey) {
|
||||
return { skipped: true, reason: 'no Anthropic API key in vault' };
|
||||
}
|
||||
|
||||
const target = this.pickNextTarget();
|
||||
if (!target) {
|
||||
return { skipped: true, reason: 'no target meets the dataset gate' };
|
||||
}
|
||||
|
||||
const baseline = this.resolveBaseline(target);
|
||||
if (!baseline) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: `no baseline for ${target.kind}/${target.name}`,
|
||||
};
|
||||
}
|
||||
|
||||
const runner = this.deps.runner ?? this.defaultRunner.bind(this);
|
||||
const result = await runner(target, baseline, apiKey);
|
||||
|
||||
if (!result.skipped) {
|
||||
this.deps.log?.(
|
||||
'info',
|
||||
`evolution tick: ${result.targetKind}/${result.targetName} → ${result.outcome}` +
|
||||
(result.runUuid ? ` (run ${result.runUuid})` : ''),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
this.deps.log?.('warn', `evolution tick failed: ${msg}`);
|
||||
return { skipped: true, reason: `tick failed: ${msg}` };
|
||||
} finally {
|
||||
this.tickInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate every configured target with its current dataset-gate status.
|
||||
* Exposed for inspection + the "status" UI; `pickNextTarget` consumes the
|
||||
* same data to choose a winner.
|
||||
*/
|
||||
enumerateCandidates(): EvolutionCandidate[] {
|
||||
return this.cfg.targets.map(t => {
|
||||
const lastRun = this.deps.runStore.list({
|
||||
targetKind: t.kind,
|
||||
targetName: t.name,
|
||||
limit: 1,
|
||||
})[0];
|
||||
const lastRunAt = lastRun?.created_at ?? null;
|
||||
|
||||
const traceFilter: {
|
||||
personaId?: string;
|
||||
since?: string;
|
||||
outcome: TraceOutcome[];
|
||||
} = { outcome: this.cfg.eligibleOutcomes };
|
||||
if (t.kind === 'persona-system-prompt') {
|
||||
traceFilter.personaId = t.name;
|
||||
}
|
||||
if (lastRunAt) traceFilter.since = lastRunAt;
|
||||
|
||||
const newTraces = this.deps.traceStore.count(traceFilter);
|
||||
return { kind: t.kind, name: t.name, lastRunAt, newTraces };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the best eligible target. "Eligible" = newTraces ≥ minTracesPerTarget.
|
||||
* "Best" = never-run first, then oldest lastRunAt, then most newTraces as a
|
||||
* tiebreaker. Returns null when no target clears the gate.
|
||||
*/
|
||||
pickNextTarget(): EvolutionCandidate | null {
|
||||
const eligible = this.enumerateCandidates().filter(
|
||||
c => c.newTraces >= this.cfg.minTracesPerTarget,
|
||||
);
|
||||
if (eligible.length === 0) return null;
|
||||
|
||||
eligible.sort((a, b) => {
|
||||
// Never-run first
|
||||
if (a.lastRunAt === null && b.lastRunAt !== null) return -1;
|
||||
if (a.lastRunAt !== null && b.lastRunAt === null) return 1;
|
||||
// Oldest last run first
|
||||
if (a.lastRunAt !== b.lastRunAt && a.lastRunAt && b.lastRunAt) {
|
||||
return a.lastRunAt.localeCompare(b.lastRunAt);
|
||||
}
|
||||
// Most new traces as tiebreaker
|
||||
return b.newTraces - a.newTraces;
|
||||
});
|
||||
return eligible[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the current baseline text for a target:
|
||||
* - persona-system-prompt: persona.systemPrompt from the live registry.
|
||||
* - behavioral-spec-section: the active spec (with overrides) with a
|
||||
* fallback to compile-time BEHAVIORAL_SPEC.
|
||||
*/
|
||||
resolveBaseline(target: EvolutionTargetId): string | null {
|
||||
if (target.kind === 'persona-system-prompt') {
|
||||
return getPersona(target.name)?.systemPrompt ?? null;
|
||||
}
|
||||
if (target.kind === 'behavioral-spec-section') {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(target.name as BehavioralSpecSection)) {
|
||||
return null;
|
||||
}
|
||||
const active = this.deps.getActiveBehavioralSpec() as Record<string, unknown>;
|
||||
const fallback = BEHAVIORAL_SPEC as unknown as Record<string, unknown>;
|
||||
const value = active?.[target.name] ?? fallback[target.name];
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
// tool-description / skill-body / generic aren't auto-evolvable yet —
|
||||
// they would need a baseline source we don't have on disk.
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default runner — builds the Anthropic LLM + adapter stack and invokes
|
||||
* EvolutionOrchestrator.runOnce. Kept as an instance method so tests can
|
||||
* bypass it entirely via `deps.runner`.
|
||||
*/
|
||||
private async defaultRunner(
|
||||
target: EvolutionTargetId,
|
||||
baseline: string,
|
||||
apiKey: string,
|
||||
): Promise<TickResult> {
|
||||
const factory = this.deps.llmFactory ?? ((key: string) => createAnthropicEvolutionLLM(key));
|
||||
const llm = await factory(apiKey);
|
||||
if (!llm) {
|
||||
return { skipped: true, reason: '@ax-llm/ax unavailable — evolution LLM not initialized' };
|
||||
}
|
||||
|
||||
const baseJudge = new LLMJudge(buildJudgeLLMCall(llm));
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
const schemaExecute = buildSchemaExecuteFn(llm);
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const orchestrator = new EvolutionOrchestrator({
|
||||
traceStore: this.deps.traceStore,
|
||||
runStore: this.deps.runStore,
|
||||
// No deploy — auto-accept is an explicit non-goal. User reviews proposals.
|
||||
});
|
||||
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: target.kind,
|
||||
targetName: target.name,
|
||||
baseline,
|
||||
schemaBaseline: defaultSchemaBaseline(target.kind),
|
||||
compose: {
|
||||
schema: {
|
||||
execute: schemaExecute,
|
||||
judge: baseJudge,
|
||||
examples: [],
|
||||
},
|
||||
instructions: {
|
||||
judge: runningJudge,
|
||||
mutate,
|
||||
targetKind: target.kind,
|
||||
examples: [],
|
||||
concurrency: 4,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
skipped: false,
|
||||
targetKind: target.kind,
|
||||
targetName: target.name,
|
||||
outcome: result.outcome,
|
||||
runUuid: result.run?.run_uuid ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Sensible default Schema when a target has no structural baseline on disk.
|
||||
* Mirrors the default in packages/server/src/local/routes/evolution.ts so the
|
||||
* service produces runs with the same shape as the manual route.
|
||||
*/
|
||||
function defaultSchemaBaseline(kind: EvolutionTarget): Schema {
|
||||
return {
|
||||
name: kind.replace(/-/g, '_') + '_baseline',
|
||||
version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'short step-by-step justification before the answer',
|
||||
required: false,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'the assistant\u0027s direct response',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the env flag that gates automatic evolution. Returns true only when
|
||||
* the value is an explicit opt-in ("1" / "true"). Anything else — unset,
|
||||
* empty, "false", "0" — leaves the daemon dormant.
|
||||
*/
|
||||
export function isEvolutionAutoEnabled(
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): boolean {
|
||||
const v = (env.WAGGLE_EVOLUTION_AUTO_ENABLED ?? '').toLowerCase().trim();
|
||||
return v === '1' || v === 'true' || v === 'yes';
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
/**
|
||||
* Optimizer Service — wraps @waggle/optimizer with vault key resolution,
|
||||
* caching, and graceful fallback.
|
||||
*
|
||||
* Uses @ax-llm/ax directly (no LiteLLM dependency) with the Anthropic
|
||||
* key from the vault.
|
||||
*
|
||||
* Usage in chat loop:
|
||||
* const optimizer = getOptimizerService(server);
|
||||
* if (optimizer) {
|
||||
* const intent = await optimizer.classify(message);
|
||||
* const expanded = await optimizer.expandIfVague(message, intent);
|
||||
* }
|
||||
*/
|
||||
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
let _optimizerInstance: OptimizerService | null = null;
|
||||
let _lastKeyHash: string | null = null;
|
||||
|
||||
export interface GEPAExpansionResult {
|
||||
expanded: string | null;
|
||||
clarifyingQuestions: string[] | null;
|
||||
intent: string;
|
||||
isVague: boolean;
|
||||
}
|
||||
|
||||
interface OptimizerService {
|
||||
classify(text: string): Promise<string>;
|
||||
expandIfVague(text: string, intent: string): Promise<string | null>;
|
||||
expandWithChoices(text: string): Promise<GEPAExpansionResult>;
|
||||
summarize(text: string): Promise<string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create the optimizer service. Returns null if no API key available.
|
||||
* Caches the instance and recreates if the vault key changes.
|
||||
*/
|
||||
export async function getOptimizerService(server: FastifyInstance): Promise<OptimizerService | null> {
|
||||
// Get Anthropic key from vault
|
||||
const apiKey = server.vault?.get('anthropic')?.value;
|
||||
if (!apiKey) return null;
|
||||
|
||||
// Check if key changed (user updated vault)
|
||||
const keyHash = apiKey.slice(-8);
|
||||
if (_optimizerInstance && _lastKeyHash === keyHash) {
|
||||
return _optimizerInstance;
|
||||
}
|
||||
|
||||
try {
|
||||
// Dynamic import to avoid loading @ax-llm/ax at startup if not needed
|
||||
const { AxAI, AxAIAnthropicModel } = await import('@ax-llm/ax');
|
||||
const { PromptOptimizer } = await import('@waggle/optimizer');
|
||||
|
||||
const ai = new AxAI({
|
||||
name: 'anthropic',
|
||||
apiKey,
|
||||
config: { model: AxAIAnthropicModel.Claude45Haiku }, // Use cheapest model for optimization
|
||||
});
|
||||
|
||||
const optimizer = new PromptOptimizer({ ai });
|
||||
|
||||
// Simple cache to avoid re-classifying the same message
|
||||
const classifyCache = new Map<string, string>();
|
||||
const expandCache = new Map<string, string>();
|
||||
|
||||
_optimizerInstance = {
|
||||
async classify(text: string): Promise<string> {
|
||||
const cacheKey = text.slice(0, 100);
|
||||
const cached = classifyCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const result = await optimizer.classify(text);
|
||||
classifyCache.set(cacheKey, result);
|
||||
// Cap cache size
|
||||
if (classifyCache.size > 200) {
|
||||
const firstKey = classifyCache.keys().next().value;
|
||||
if (firstKey) classifyCache.delete(firstKey);
|
||||
}
|
||||
return result;
|
||||
} catch {
|
||||
return 'request'; // Default fallback
|
||||
}
|
||||
},
|
||||
|
||||
async expandIfVague(text: string, intent: string): Promise<string | null> {
|
||||
// Only expand vague requests, not questions/greetings/commands
|
||||
if (intent === 'greeting' || intent === 'question') return null;
|
||||
|
||||
// Don't expand if message is already detailed (>100 chars)
|
||||
if (text.length > 100) return null;
|
||||
|
||||
// Don't expand commands (starting with /)
|
||||
if (text.startsWith('/')) return null;
|
||||
|
||||
const cacheKey = text.slice(0, 100);
|
||||
const cached = expandCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const expanded = await optimizer.expandPrompt(text);
|
||||
// Only use expansion if it's meaningfully different
|
||||
if (expanded && expanded.length > text.length * 1.5) {
|
||||
expandCache.set(cacheKey, expanded);
|
||||
if (expandCache.size > 100) {
|
||||
const firstKey = expandCache.keys().next().value;
|
||||
if (firstKey) expandCache.delete(firstKey);
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null; // Graceful fallback — use original message
|
||||
}
|
||||
},
|
||||
|
||||
async expandWithChoices(text: string): Promise<GEPAExpansionResult> {
|
||||
const intent = await this.classify(text);
|
||||
const isVague = intent !== 'greeting' && intent !== 'question' && text.length <= 100 && !text.startsWith('/');
|
||||
|
||||
if (!isVague) {
|
||||
return { expanded: null, clarifyingQuestions: null, intent, isVague: false };
|
||||
}
|
||||
|
||||
const expanded = await this.expandIfVague(text, intent);
|
||||
|
||||
// Generate clarifying questions using the optimizer's summarizer
|
||||
// (repurposed — it's a cheap Haiku call that can handle short prompts)
|
||||
let clarifyingQuestions: string[] | null = null;
|
||||
try {
|
||||
const questionsText = await optimizer.summarize(
|
||||
`Generate 3 short clarifying questions for this vague request: "${text}". One question per line, no numbering.`,
|
||||
);
|
||||
clarifyingQuestions = questionsText
|
||||
.split('\n')
|
||||
.map((q: string) => q.trim())
|
||||
.filter((q: string) => q.length > 5 && q.includes('?'))
|
||||
.slice(0, 3);
|
||||
if (clarifyingQuestions.length === 0) clarifyingQuestions = null;
|
||||
} catch {
|
||||
clarifyingQuestions = null;
|
||||
}
|
||||
|
||||
return { expanded, clarifyingQuestions, intent, isVague: true };
|
||||
},
|
||||
|
||||
async summarize(text: string): Promise<string> {
|
||||
try {
|
||||
return await optimizer.summarize(text);
|
||||
} catch {
|
||||
return text; // Fallback to original
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
_lastKeyHash = keyHash;
|
||||
return _optimizerInstance;
|
||||
} catch (err) {
|
||||
// @ax-llm/ax or @waggle/optimizer not available — graceful degradation
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,6 @@ export function seedDefaultCrons(cronStore: CronStore): void {
|
||||
ensure('Stale workspace check', '0 9 * * 1', 'proactive', { action: 'stale_workspace_check' });
|
||||
ensure('Task reminder', '30 8 * * *', 'proactive', { action: 'task_reminder' });
|
||||
ensure('Capability suggestion', '0 10 * * 3', 'proactive', { action: 'capability_suggestion' });
|
||||
ensure('Prompt optimization', '0 2 * * *', 'prompt_optimization', undefined, false);
|
||||
ensure('Monthly assessment', '0 6 1 * *', 'monthly_assessment');
|
||||
ensure('Index reconciliation', '0 4 * * 0', 'memory_consolidation', { action: 'index_reconcile' });
|
||||
|
||||
// Phase 7: Memory Harvest sync — scan local sources (Claude Code, Cursor) for new data
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC, deployBehavioralSpecOverride } from '@waggle/agent';
|
||||
import { BEHAVIORAL_SPEC } from '@waggle/agent';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
@@ -21,7 +21,7 @@ describe('activeBehavioralSpec integration', () => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
it('loads the compiled baseline when no overrides exist on disk', async () => {
|
||||
it('serves the compiled behavioral spec baseline', async () => {
|
||||
const server = await buildLocalServer({ dataDir: tmpDir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec).toBeDefined();
|
||||
@@ -31,104 +31,4 @@ describe('activeBehavioralSpec integration', () => {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('applies a pre-existing override at boot', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec2-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
// Write an override BEFORE the server boots.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'EVOLVED core loop — test fixture',
|
||||
});
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('EVOLVED core loop — test fixture');
|
||||
// Unaffected sections pass through
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe(BEHAVIORAL_SPEC.qualityRules);
|
||||
// rules reflects the override
|
||||
expect(server.activeBehavioralSpec.rules).toContain('EVOLVED core loop — test fixture');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('re-derives activeBehavioralSpec on behavioral-spec:reloaded event', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec3-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Sanity: no override yet
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe(BEHAVIORAL_SPEC.coreLoop);
|
||||
|
||||
// Write an override and fire the reload event.
|
||||
deployBehavioralSpecOverride(dir, {
|
||||
section: 'coreLoop',
|
||||
text: 'HOT-RELOADED core loop',
|
||||
});
|
||||
server.eventBus.emit('behavioral-spec:reloaded', { section: 'coreLoop' });
|
||||
|
||||
// Give the sync listener a tick to update (it's synchronous actually,
|
||||
// but await-nothing keeps us defensive).
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.coreLoop).toBe('HOT-RELOADED core loop');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
it('accepted evolution run causes activeBehavioralSpec to pick up the override', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-active-spec4-'));
|
||||
try {
|
||||
const personalPath = path.join(dir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
const server = await buildLocalServer({ dataDir: dir });
|
||||
try {
|
||||
// Propose a run directly on the store, then accept via HTTP
|
||||
const run = server.evolutionStore.create({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'qualityRules',
|
||||
baselineText: BEHAVIORAL_SPEC.qualityRules,
|
||||
winnerText: 'END-TO-END EVOLVED quality rules',
|
||||
deltaAccuracy: 0.1,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [],
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
headers: { 'content-type': 'application/json', authorization: 'Bearer test' },
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
// Event fires synchronously on emit (EventEmitter behavior); give
|
||||
// the listener a microtask just in case.
|
||||
await Promise.resolve();
|
||||
|
||||
expect(server.activeBehavioralSpec.qualityRules).toBe('END-TO-END EVOLVED quality rules');
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,386 +0,0 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
describe('Evolution Routes', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-routes-'));
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
function seedRun(overrides: Partial<Parameters<typeof server.evolutionStore.create>[0]> = {}) {
|
||||
return server.evolutionStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'baseline prompt that is long enough',
|
||||
winnerText: 'evolved prompt that is long enough to pass the gate',
|
||||
deltaAccuracy: 0.08,
|
||||
gateVerdict: 'pass',
|
||||
gateReasons: [{ gate: 'size', verdict: 'pass', reason: 'within limit' }],
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/evolution/runs ──
|
||||
|
||||
describe('GET /api/evolution/runs', () => {
|
||||
beforeAll(() => server.evolutionStore.clear());
|
||||
|
||||
it('returns an empty list when nothing has run', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toEqual([]);
|
||||
expect(body.count).toBe(0);
|
||||
});
|
||||
|
||||
it('returns seeded runs in reverse-chronological order', async () => {
|
||||
seedRun({ targetName: 'a' });
|
||||
seedRun({ targetName: 'b' });
|
||||
seedRun({ targetName: 'c' });
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.count).toBe(3);
|
||||
expect(body.runs[0].id).toBeGreaterThan(body.runs[body.runs.length - 1].id);
|
||||
});
|
||||
|
||||
it('filters by status', async () => {
|
||||
const a = seedRun({ targetName: 'x' });
|
||||
server.evolutionStore.reject(a.run_uuid, 'test reject');
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?status=rejected',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs.length).toBeGreaterThanOrEqual(1);
|
||||
expect(body.runs.every((r: { status: string }) => r.status === 'rejected')).toBe(true);
|
||||
});
|
||||
|
||||
it('respects the limit query param', async () => {
|
||||
for (let i = 0; i < 5; i++) seedRun({ targetName: `limit-${i}` });
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs?limit=2',
|
||||
});
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.runs).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/runs/:uuid ──
|
||||
|
||||
describe('GET /api/evolution/runs/:uuid', () => {
|
||||
it('returns a single run with parsed JSON blobs', async () => {
|
||||
const run = seedRun({
|
||||
winnerSchema: { name: 'test', fields: [{ name: 'answer', type: 'string' }] },
|
||||
artifacts: { seed: 42, generations: 2 },
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.run_uuid).toBe(run.run_uuid);
|
||||
expect(body.winnerSchema).toEqual({ name: 'test', fields: [{ name: 'answer', type: 'string' }] });
|
||||
expect(body.artifacts).toEqual({ seed: 42, generations: 2 });
|
||||
expect(body.gateReasons).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs/does-not-exist',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns run even when artifacts are null/malformed', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/runs/${run.run_uuid}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.winnerSchema).toBeNull();
|
||||
expect(body.artifacts).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/reject ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/reject', () => {
|
||||
it('moves proposed run to rejected with reason', async () => {
|
||||
const run = seedRun();
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: { reason: 'not great' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('rejected');
|
||||
expect(body.user_note).toBe('not great');
|
||||
});
|
||||
|
||||
it('rejects 404 for unknown uuid', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/runs/ghost/reject',
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('rejects 409 when run is not proposed', async () => {
|
||||
const run = seedRun();
|
||||
server.evolutionStore.reject(run.run_uuid);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/reject`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── POST /api/evolution/runs/:uuid/accept ──
|
||||
|
||||
describe('POST /api/evolution/runs/:uuid/accept', () => {
|
||||
it('writes a persona override to disk and moves to deployed', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
winnerText: 'EVOLVED coder system prompt',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: { note: 'looks good' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('deployed');
|
||||
|
||||
const overridePath = path.join(tmpDir, 'personas', 'coder.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder system prompt');
|
||||
});
|
||||
|
||||
it('writes a behavioral-spec override to disk', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'behavioral-spec-section',
|
||||
targetName: 'coreLoop',
|
||||
winnerText: 'EVOLVED core loop text',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
const overridePath = path.join(tmpDir, 'behavioral-overrides', 'coreLoop.json');
|
||||
expect(fs.existsSync(overridePath)).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(overridePath, 'utf-8'));
|
||||
expect(loaded.text).toBe('EVOLVED core loop text');
|
||||
expect(loaded.runUuid).toBe(run.run_uuid);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy throws (unsupported target_kind)', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'tool-description',
|
||||
targetName: 'some_tool',
|
||||
winnerText: 'new description',
|
||||
});
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.status).toBe('failed');
|
||||
expect(body.failure_reason).toMatch(/not yet implemented/);
|
||||
});
|
||||
|
||||
it('emits persona:reloaded event on persona deploy', async () => {
|
||||
const run = seedRun({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'writer',
|
||||
winnerText: 'EVOLVED writer',
|
||||
});
|
||||
|
||||
let emitted: unknown = null;
|
||||
const listener = (payload: unknown) => { emitted = payload; };
|
||||
server.eventBus.on('persona:reloaded', listener);
|
||||
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
|
||||
server.eventBus.off('persona:reloaded', listener);
|
||||
expect(emitted).toEqual({ personaId: 'writer' });
|
||||
});
|
||||
|
||||
it('returns 409 for already-accepted run', async () => {
|
||||
const run = seedRun();
|
||||
await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
const second = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: `/api/evolution/runs/${run.run_uuid}/accept`,
|
||||
payload: {},
|
||||
});
|
||||
expect(second.statusCode).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/status ──
|
||||
|
||||
describe('GET /api/evolution/status', () => {
|
||||
it('returns aggregate counts per status', async () => {
|
||||
server.evolutionStore.clear();
|
||||
const a = seedRun();
|
||||
const b = seedRun();
|
||||
server.evolutionStore.reject(a.run_uuid);
|
||||
server.evolutionStore.accept(b.run_uuid);
|
||||
server.evolutionStore.markDeployed(b.run_uuid);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/status',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.counts.rejected).toBe(1);
|
||||
expect(body.counts.deployed).toBe(1);
|
||||
expect(body.pendingCount).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/targets ──
|
||||
|
||||
describe('GET /api/evolution/targets', () => {
|
||||
it('returns personas + behavioral-spec sections + a default schema', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(Array.isArray(body.personas)).toBe(true);
|
||||
expect(body.personas.length).toBeGreaterThan(0);
|
||||
expect(body.personas[0]).toHaveProperty('id');
|
||||
expect(body.personas[0]).toHaveProperty('name');
|
||||
// systemPrompt is intentionally omitted from /targets to match /api/personas privacy policy
|
||||
expect(body.personas[0]).not.toHaveProperty('systemPrompt');
|
||||
|
||||
expect(Array.isArray(body.sections)).toBe(true);
|
||||
expect(body.sections).toEqual(expect.arrayContaining([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules', 'workPatterns', 'intelligenceDefaults',
|
||||
]));
|
||||
|
||||
expect(body.defaultSchema).toMatchObject({
|
||||
name: expect.any(String),
|
||||
version: expect.any(Number),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
expect(body.defaultSchema.fields.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── GET /api/evolution/baseline ──
|
||||
|
||||
describe('GET /api/evolution/baseline', () => {
|
||||
it('returns 400 when kind or name is missing', async () => {
|
||||
const a = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline',
|
||||
});
|
||||
expect(a.statusCode).toBe(400);
|
||||
|
||||
const b = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt',
|
||||
});
|
||||
expect(b.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('returns the persona systemPrompt for persona-system-prompt', async () => {
|
||||
// Pick whatever persona the registry lists first — avoids hardcoding
|
||||
// an id that might be renamed.
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/targets',
|
||||
});
|
||||
const { personas } = JSON.parse(listRes.body);
|
||||
const personaId = personas[0].id;
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: `/api/evolution/baseline?kind=persona-system-prompt&name=${encodeURIComponent(personaId)}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
expect(body.schemaBaseline).toMatchObject({
|
||||
name: expect.any(String),
|
||||
fields: expect.any(Array),
|
||||
});
|
||||
});
|
||||
|
||||
it('returns 404 for unknown persona', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=persona-system-prompt&name=ghost-persona',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns the active section text for behavioral-spec-section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=coreLoop',
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(typeof body.baseline).toBe('string');
|
||||
expect(body.baseline.length).toBeGreaterThan(0);
|
||||
// coreLoop has identifiable content
|
||||
expect(body.baseline).toMatch(/HOW YOU THINK|Core Loop|Step 1/i);
|
||||
});
|
||||
|
||||
it('returns 404 for unknown section', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=behavioral-spec-section&name=nopeSection',
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 400 for unknown kind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/baseline?kind=tool-description&name=x',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,397 +0,0 @@
|
||||
/**
|
||||
* Integration tests for POST /api/evolution/run.
|
||||
*
|
||||
* The endpoint needs an Anthropic key and an LLM. We stub the LLM by
|
||||
* installing a global factory override (`__waggleEvolutionLlmFactory`) so
|
||||
* every call goes through an in-memory responder — no network, no @ax-llm/ax.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, VaultStore } from '@waggle/core';
|
||||
import { buildLocalServer } from '../src/local/index.js';
|
||||
import { injectWithAuth } from './test-utils.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { EvolutionLLM } from '@waggle/agent';
|
||||
|
||||
// ── Global LLM override hook ─────────────────────────────────────
|
||||
|
||||
type LLMFactory = (apiKey: string) => EvolutionLLM | Promise<EvolutionLLM>;
|
||||
|
||||
function installLLMFactory(factory: LLMFactory): void {
|
||||
(globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory = factory;
|
||||
}
|
||||
|
||||
function clearLLMFactory(): void {
|
||||
delete (globalThis as unknown as { __waggleEvolutionLlmFactory?: LLMFactory })
|
||||
.__waggleEvolutionLlmFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stub LLM that returns a canned answer based on a pattern match on the
|
||||
* prompt. Falls back to echoing the input so the judge still produces a
|
||||
* score. Every call increments a counter so tests can assert the LLM was
|
||||
* actually invoked.
|
||||
*/
|
||||
function makeStubLLM(responses: Array<{ match: RegExp; reply: string }>): {
|
||||
llm: EvolutionLLM;
|
||||
callCount: () => number;
|
||||
calls: string[];
|
||||
} {
|
||||
const calls: string[] = [];
|
||||
return {
|
||||
callCount: () => calls.length,
|
||||
calls,
|
||||
llm: {
|
||||
async complete(prompt: string): Promise<string> {
|
||||
calls.push(prompt);
|
||||
for (const r of responses) {
|
||||
if (r.match.test(prompt)) return r.reply;
|
||||
}
|
||||
// Default response that parses as a valid judge verdict.
|
||||
return '{"correctness":8,"procedure":8,"conciseness":7,"feedback":"ok"}';
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Test scaffolding ─────────────────────────────────────────────
|
||||
|
||||
describe('POST /api/evolution/run', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evolution-run-'));
|
||||
|
||||
// Fresh mind database.
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
mind.close();
|
||||
|
||||
// Seed the vault with an Anthropic key before the server boots — the
|
||||
// server loads the vault from disk on startup.
|
||||
const vault = new VaultStore(tmpDir);
|
||||
vault.set('anthropic', 'sk-ant-stub-for-tests', { models: ['claude-haiku-4-5-20251001'] });
|
||||
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.close();
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
clearLLMFactory();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(() => {
|
||||
server.evolutionStore.clear();
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearLLMFactory();
|
||||
});
|
||||
|
||||
/** Seed a handful of finalized traces so the orchestrator's dataset miner has material. */
|
||||
function seedTraces(count = 8): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
const id = server.traceStore.start({
|
||||
input: `What is the capital of country ${i}?`,
|
||||
personaId: 'coder',
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
server.traceStore.finalize(id, {
|
||||
outcome: i % 3 === 0 ? 'verified' : 'success',
|
||||
output: `The capital is City${i}.`,
|
||||
tokens: { input: 40, output: 20 },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const baseBody = {
|
||||
targetKind: 'behavioral-spec-section' as const,
|
||||
targetName: 'coreLoop',
|
||||
baseline: 'You are a careful assistant. Follow instructions.',
|
||||
schemaBaseline: {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'answer', type: 'string', description: 'the answer', required: true, constraints: [] },
|
||||
],
|
||||
},
|
||||
// Keep populations tiny so the test stays fast even though we use a stub LLM.
|
||||
schema: { populationSize: 1, generations: 1, evalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
gepa: { populationSize: 1, generations: 1, miniEvalSize: 2, anchorEvalSize: 2, seed: 1 },
|
||||
};
|
||||
|
||||
// ── 422 when no API key ────────────────────────────────────────
|
||||
|
||||
it('returns 422 when no Anthropic key is in the vault', async () => {
|
||||
// Temporarily hide the key for just this request.
|
||||
const saved = server.vault?.get('anthropic');
|
||||
server.vault?.delete('anthropic');
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(422);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toMatch(/Anthropic API key/i);
|
||||
} finally {
|
||||
if (saved) {
|
||||
server.vault?.set('anthropic', saved.value, saved.metadata);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ── 400 on validation errors ───────────────────────────────────
|
||||
|
||||
describe('validation', () => {
|
||||
it('rejects missing body', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects invalid targetKind', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetKind: 'nope' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetKind/);
|
||||
});
|
||||
|
||||
it('rejects empty targetName', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, targetName: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/targetName/);
|
||||
});
|
||||
|
||||
it('rejects empty baseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, baseline: '' },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/baseline/);
|
||||
});
|
||||
|
||||
it('rejects malformed schemaBaseline', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run',
|
||||
payload: { ...baseBody, schemaBaseline: { name: 'x' } },
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(JSON.parse(res.body).error).toMatch(/schemaBaseline/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 503 when LLM factory fails ─────────────────────────────────
|
||||
|
||||
it('returns 503 when the LLM factory returns null', async () => {
|
||||
installLLMFactory(() => null as unknown as EvolutionLLM);
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(JSON.parse(res.body).error).toMatch(/ax-llm/);
|
||||
});
|
||||
|
||||
// ── Skip path: no traces ───────────────────────────────────────
|
||||
|
||||
it('returns outcome=skipped-trigger when the trace store has no eligible traces', async () => {
|
||||
const { llm } = makeStubLLM([]);
|
||||
installLLMFactory(() => llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.outcome).toBe('skipped-trigger');
|
||||
expect(body.reason).toMatch(/eligible traces/);
|
||||
});
|
||||
|
||||
// ── Happy path: runs + persists ────────────────────────────────
|
||||
|
||||
it('runs the full orchestrator when traces exist and returns a proposal summary', async () => {
|
||||
seedTraces(8);
|
||||
|
||||
const stub = makeStubLLM([
|
||||
// Mutation requests get a slightly different prompt text
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'You are an extremely careful assistant. Follow all instructions step by step.' },
|
||||
// Schema fill requests get valid JSON
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
// Running-judge execution requests get a short answer
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
// Any terminal outcome is acceptable — the point is that the orchestrator
|
||||
// ran end-to-end, talked to our stub LLM, and wrote something to the store.
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(body.outcome);
|
||||
expect(body.composeSummary).toBeTruthy();
|
||||
// Every run (including skipped-delta) returns a composeSummary; only
|
||||
// 'proposed' and 'skipped-gates' attach a persisted run.
|
||||
expect(stub.callCount()).toBeGreaterThan(0);
|
||||
|
||||
// If a run was persisted, check it shows up in the list endpoint.
|
||||
if (body.run) {
|
||||
const listRes = await injectWithAuth(server, {
|
||||
method: 'GET', url: '/api/evolution/runs',
|
||||
});
|
||||
const listed = JSON.parse(listRes.body);
|
||||
expect(listed.count).toBeGreaterThan(0);
|
||||
const matching = listed.runs.find((r: { run_uuid: string }) => r.run_uuid === body.run.run_uuid);
|
||||
expect(matching).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
// ── SSE streaming path ─────────────────────────────────────────
|
||||
|
||||
describe('SSE streaming (Accept: text/event-stream)', () => {
|
||||
/** Tiny SSE parser — takes the raw event-stream body and emits parsed events. */
|
||||
function parseSseEvents(raw: string): Array<{ event: string; data: unknown }> {
|
||||
const out: Array<{ event: string; data: unknown }> = [];
|
||||
for (const block of raw.split('\n\n')) {
|
||||
if (!block.trim()) continue;
|
||||
let event = 'message';
|
||||
const dataLines: string[] = [];
|
||||
for (const line of block.split('\n')) {
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||
else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
|
||||
}
|
||||
if (dataLines.length > 0) {
|
||||
try {
|
||||
out.push({ event, data: JSON.parse(dataLines.join('\n')) });
|
||||
} catch {
|
||||
out.push({ event, data: dataLines.join('\n') });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
it('emits open → progress* → done sequence for a successful run', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([
|
||||
{ match: /reflective|evolving an AI prompt/i, reply: 'A careful assistant.' },
|
||||
{ match: /schema/i, reply: '{"answer":"City-X"}' },
|
||||
{ match: /USER INPUT:/i, reply: 'The capital is City-X.' },
|
||||
]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/text\/event-stream/);
|
||||
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('done');
|
||||
|
||||
// Terminal event carries the same payload shape as the JSON path.
|
||||
const done = events.find(e => e.event === 'done');
|
||||
expect(done).toBeDefined();
|
||||
const donePayload = done!.data as { outcome: string; composeSummary: unknown };
|
||||
expect(['proposed', 'skipped-delta', 'skipped-gates']).toContain(donePayload.outcome);
|
||||
expect(donePayload.composeSummary).toBeTruthy();
|
||||
|
||||
// There should be at least one progress event reporting an orchestrator phase.
|
||||
const progresses = events.filter(e => e.event === 'progress');
|
||||
expect(progresses.length).toBeGreaterThan(0);
|
||||
const phases = progresses.map(p => (p.data as { phase: string }).phase);
|
||||
// Orchestrator emits at least 'dataset' before skipping or compose.
|
||||
expect(phases.some(p => ['dataset', 'compose', 'gates', 'persist', 'skipped', 'done'].includes(p))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
it('emits open → error when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
// Force a real error path like the JSON test does.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST',
|
||||
url: '/api/evolution/run',
|
||||
payload: baseBody,
|
||||
headers: { accept: 'text/event-stream' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200); // SSE streams return 200 even on internal errors; the error is an SSE event
|
||||
const events = parseSseEvents(res.body);
|
||||
const types = events.map(e => e.event);
|
||||
expect(types[0]).toBe('open');
|
||||
expect(types).toContain('error');
|
||||
const errEvent = events.find(e => e.event === 'error');
|
||||
expect((errEvent!.data as { error: string }).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
|
||||
it('falls back to JSON when Accept header is absent', async () => {
|
||||
seedTraces(8);
|
||||
const stub = makeStubLLM([]);
|
||||
installLLMFactory(() => stub.llm);
|
||||
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
// No accept header — should return JSON.
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/application\/json/);
|
||||
expect(() => JSON.parse(res.body)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Orchestrator error path ────────────────────────────────────
|
||||
|
||||
it('returns 500 when the orchestrator throws', async () => {
|
||||
seedTraces(4);
|
||||
installLLMFactory(() => ({
|
||||
async complete() { throw new Error('stub-llm-always-fails'); },
|
||||
}));
|
||||
|
||||
// Judge swallows its own LLM errors with a zero score, so we need to
|
||||
// make the mutate + execute paths noisy enough that *something* escapes.
|
||||
// Easiest: swap out the trace store to blow up during buildExamples.
|
||||
const originalQuery = server.traceStore.queryParsed.bind(server.traceStore);
|
||||
server.traceStore.queryParsed = () => { throw new Error('stub-trace-store-explodes'); };
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/evolution/run', payload: baseBody,
|
||||
});
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(JSON.parse(res.body).error).toMatch(/stub-trace-store-explodes/);
|
||||
} finally {
|
||||
server.traceStore.queryParsed = originalQuery;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -40,7 +40,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let includeViewerWorkspace: boolean;
|
||||
let literalGlobalIsViewer: boolean;
|
||||
let viewerOptimizationEnabled: boolean;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = new MindDB(':memory:');
|
||||
@@ -49,7 +48,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
failNext = false;
|
||||
includeViewerWorkspace = false;
|
||||
literalGlobalIsViewer = false;
|
||||
viewerOptimizationEnabled = 1 as unknown as boolean;
|
||||
scheduler = new LocalScheduler(
|
||||
cronStore,
|
||||
async (schedule) => {
|
||||
@@ -79,7 +77,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
};
|
||||
}
|
||||
if (id === 'global' && literalGlobalIsViewer) {
|
||||
@@ -96,7 +93,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
name: 'Viewer WS',
|
||||
teamId: 'team-1',
|
||||
teamRole: 'viewer',
|
||||
optimizationEnabled: viewerOptimizationEnabled,
|
||||
}]
|
||||
: []),
|
||||
],
|
||||
@@ -460,13 +456,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/cron/${id}/trigger` }),
|
||||
},
|
||||
{
|
||||
label: 'prompt optimization through automation run',
|
||||
jobType: 'prompt_optimization' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
request: (id: number) => ({ method: 'POST' as const, url: `/api/automations/${id}/run` }),
|
||||
},
|
||||
{
|
||||
label: 'memory compaction through cron trigger',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
@@ -506,25 +495,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
expect(executed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('prompt optimization ignores a listed viewer workspace without optimization opt-in', async () => {
|
||||
includeViewerWorkspace = true;
|
||||
viewerOptimizationEnabled = false;
|
||||
const row = cronStore.create({
|
||||
name: 'Prompt optimization',
|
||||
cronExpr: '0 2 * * *',
|
||||
jobType: 'prompt_optimization',
|
||||
enabled: false,
|
||||
});
|
||||
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
url: `/api/cron/${row.id}/trigger`,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(executed).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: 'cron',
|
||||
@@ -625,12 +595,6 @@ describe('Automations alias routes (Phase 3)', () => {
|
||||
jobConfig: { action: 'morning_briefing' },
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'personal monthly assessment',
|
||||
jobType: 'monthly_assessment' as const,
|
||||
jobConfig: {},
|
||||
workspaceId: undefined,
|
||||
},
|
||||
{
|
||||
label: 'non-matching spaced memory action',
|
||||
jobType: 'memory_consolidation' as const,
|
||||
|
||||
@@ -377,7 +377,6 @@ describe('classifyExplicitTurnMutationPolicy', () => {
|
||||
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity',
|
||||
'get_awareness', 'read_other_workspace', 'save_memory', 'add_task',
|
||||
'correct_knowledge', 'read_file', 'web_search', 'mcp_external_read',
|
||||
'agent_insights',
|
||||
].map(name => ({ name }));
|
||||
const filtered = filterToolsByTurnMutationPolicy(
|
||||
tools,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Tests the two feedback endpoints:
|
||||
* POST /api/feedback — record user feedback
|
||||
* GET /api/feedback/stats — get improvement stats
|
||||
* GET /api/feedback/stats — get feedback stats
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
@@ -11,35 +11,27 @@ import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import Fastify from 'fastify';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { MindDB } from '@waggle/core';
|
||||
import { feedbackRoutes } from '../../src/local/routes/feedback.js';
|
||||
|
||||
/** Minimal multiMind mock that provides a real MindDB for the personal mind. */
|
||||
function createTestServer(db: MindDB) {
|
||||
const server = Fastify({ logger: false });
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
server.decorate('multiMind', {
|
||||
personal: db,
|
||||
});
|
||||
server.decorate('agentState', {
|
||||
orchestrator: {
|
||||
getImprovementSignals: () => signalStore,
|
||||
},
|
||||
});
|
||||
server.register(feedbackRoutes);
|
||||
return { server, signalStore };
|
||||
return { server };
|
||||
}
|
||||
|
||||
describe('Feedback Routes', () => {
|
||||
let db: MindDB;
|
||||
let server: ReturnType<typeof Fastify>;
|
||||
let signalStore: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
const result = createTestServer(db);
|
||||
server = result.server;
|
||||
signalStore = result.signalStore;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -80,26 +72,6 @@ describe('Feedback Routes', () => {
|
||||
expect(res.json()).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('records negative feedback as correction signal', async () => {
|
||||
await server.inject({
|
||||
method: 'POST',
|
||||
url: '/api/feedback',
|
||||
payload: {
|
||||
sessionId: 'sess-1',
|
||||
messageIndex: 2,
|
||||
rating: 'down',
|
||||
reason: 'wrong_tool',
|
||||
detail: 'Should have used web_search',
|
||||
},
|
||||
});
|
||||
|
||||
// Check that it was recorded in the improvement signals store
|
||||
const corrections = signalStore.getByCategory('correction');
|
||||
const feedbackSignal = corrections.find(c => c.pattern_key === 'feedback:wrong_tool');
|
||||
expect(feedbackSignal).toBeDefined();
|
||||
expect(feedbackSignal!.count).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects missing sessionId', async () => {
|
||||
const res = await server.inject({
|
||||
method: 'POST',
|
||||
@@ -170,7 +142,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(0);
|
||||
expect(body.positiveRate).toBe(0);
|
||||
expect(body.topIssues).toEqual([]);
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
@@ -202,7 +173,6 @@ describe('Feedback Routes', () => {
|
||||
expect(body.totalFeedback).toBe(3);
|
||||
expect(body.positiveRate).toBeCloseTo(0.67, 1);
|
||||
expect(body.topIssues).toContain('too_verbose');
|
||||
expect(typeof body.correctionsThisWeek).toBe('number');
|
||||
expect(typeof body.improvementTrend).toBe('string');
|
||||
});
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
/**
|
||||
* GEPA Prompt Optimization Cron Handler Tests
|
||||
*
|
||||
* Tests the prompt_optimization cron handler logic:
|
||||
* - Calls LLM when correction rate > 20% or avg turns > 15
|
||||
* - Stores generated variant in optimization_log
|
||||
* - Skips when budget is exceeded
|
||||
* - Skips when not enough data
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import { isWithinBudget, getRecentLogs } from '@waggle/agent';
|
||||
|
||||
describe('GEPA Prompt Optimization', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let optStore: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-gepa-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
optStore = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('Signal detection', () => {
|
||||
it('detects high correction rate as optimization signal', () => {
|
||||
// Insert logs with high correction rate (>20%)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: i < 5, // 50% correction rate
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeGreaterThan(0.2);
|
||||
expect(stats.total).toBe(10);
|
||||
});
|
||||
|
||||
it('detects high avg turn count as optimization signal', () => {
|
||||
// Insert logs with high turn count (>15)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt for optimization',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 20, // >15 threshold
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.avgTurnCount).toBeGreaterThan(15);
|
||||
});
|
||||
|
||||
it('does not trigger when both metrics are within thresholds', () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test system prompt',
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = optStore.getStats();
|
||||
expect(stats.correctionRate).toBeLessThanOrEqual(0.2);
|
||||
expect(stats.avgTurnCount).toBeLessThanOrEqual(15);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Budget check', () => {
|
||||
it('reports within budget when token costs are low', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
});
|
||||
}
|
||||
|
||||
// Budget is 100 cents ($1). 500 input + 250 output tokens is far under budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('reports over budget when token costs exceed limit', () => {
|
||||
// Insert a log with massive token counts to exceed budget
|
||||
optStore.insert({
|
||||
sessionId: 'sess-big',
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
inputTokens: 10_000_000, // 10M input tokens at $3/M = $30
|
||||
outputTokens: 1_000_000, // 1M output tokens at $15/M = $15
|
||||
});
|
||||
|
||||
// Budget is 100 cents ($1). Total cost is ~$45 = 4500 cents. Way over budget.
|
||||
expect(isWithinBudget(optStore, 100)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Variant storage', () => {
|
||||
it('stores generated variant in optimization_log', () => {
|
||||
// Simulate what the cron handler does after generating a variant
|
||||
const variantText = 'Improved system prompt with better instructions for reducing corrections...'.repeat(5);
|
||||
|
||||
optStore.insert({
|
||||
sessionId: `gepa-variant-${Date.now()}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: variantText,
|
||||
toolsUsed: ['gepa_variant'],
|
||||
turnCount: 0,
|
||||
wasCorrection: false,
|
||||
inputTokens: 500,
|
||||
outputTokens: variantText.length,
|
||||
});
|
||||
|
||||
const recent = getRecentLogs(optStore, 10);
|
||||
const variant = recent.find(l => JSON.parse(l.tools_used).includes('gepa_variant'));
|
||||
expect(variant).toBeDefined();
|
||||
expect(variant!.system_prompt).toBe(variantText);
|
||||
expect(variant!.turn_count).toBe(0);
|
||||
expect(variant!.was_correction).toBe(0);
|
||||
});
|
||||
|
||||
it('retrieves recent logs for analysis', () => {
|
||||
for (let i = 0; i < 8; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: `Prompt v${i}`,
|
||||
toolsUsed: ['search_memory'],
|
||||
turnCount: i + 1,
|
||||
wasCorrection: i % 3 === 0,
|
||||
inputTokens: 100 * (i + 1),
|
||||
outputTokens: 50 * (i + 1),
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(optStore, 5);
|
||||
expect(logs).toHaveLength(5);
|
||||
// Most recent first
|
||||
expect(logs[0].session_id).toBe('sess-7');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Minimum data requirement', () => {
|
||||
it('skips optimization when fewer than 5 logs', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
optStore.insert({
|
||||
sessionId: `sess-${i}`,
|
||||
workspaceId: 'ws-test',
|
||||
systemPrompt: 'Test prompt',
|
||||
toolsUsed: [],
|
||||
turnCount: 20,
|
||||
wasCorrection: true,
|
||||
inputTokens: 100,
|
||||
outputTokens: 200,
|
||||
});
|
||||
}
|
||||
|
||||
const recentLogs = getRecentLogs(optStore, 100);
|
||||
// The cron handler checks: if (recentLogs.length < 5) continue;
|
||||
expect(recentLogs.length).toBeLessThan(5);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,287 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import {
|
||||
IdleSessionWatcher,
|
||||
countTurns,
|
||||
readRecentTranscript,
|
||||
buildReviewInstruction,
|
||||
NOTHING_TO_DO,
|
||||
DEFAULT_CONFIG,
|
||||
type SelfEvolutionConfig,
|
||||
type ReviewTurnResult,
|
||||
} from '../../src/local/idle-watcher.js';
|
||||
import { getNotificationGate } from '../../src/local/notification-gate.js';
|
||||
|
||||
// A fixed synthetic "now" so idle math is deterministic.
|
||||
const NOW = 2_000_000_000_000;
|
||||
const MIN = 60_000;
|
||||
|
||||
describe('idle-watcher', () => {
|
||||
let dataDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-idle-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ── Fixtures ─────────────────────────────────────────────────────────
|
||||
function writeConfig(cfg: Partial<SelfEvolutionConfig> | string): void {
|
||||
const body = typeof cfg === 'string' ? cfg : JSON.stringify(cfg);
|
||||
fs.writeFileSync(path.join(dataDir, 'self-evolution.json'), body);
|
||||
}
|
||||
|
||||
/** Create a session .jsonl with `turns` message lines and a given mtime (ms). */
|
||||
function writeSession(workspaceId: string, sessionId: string, turns: number, mtimeMs: number): void {
|
||||
const dir = path.join(dataDir, 'workspaces', workspaceId, 'sessions');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const lines = [JSON.stringify({ type: 'meta', title: sessionId })];
|
||||
for (let i = 0; i < turns; i++) {
|
||||
lines.push(JSON.stringify({ role: i % 2 === 0 ? 'user' : 'assistant', content: `msg ${i}` }));
|
||||
}
|
||||
const filePath = path.join(dir, `${sessionId}.jsonl`);
|
||||
fs.writeFileSync(filePath, lines.join('\n') + '\n');
|
||||
fs.utimesSync(filePath, new Date(mtimeMs), new Date(mtimeMs));
|
||||
}
|
||||
|
||||
function makeWatcher(over?: {
|
||||
runReviewTurn?: (i: { sessionId: string; workspaceId: string }) => Promise<ReviewTurnResult>;
|
||||
}) {
|
||||
const reviewed: Array<{ sessionId: string; workspaceId: string }> = [];
|
||||
const emitCalls: Array<{ suppressed: boolean; dedupeKey?: string }> = [];
|
||||
const runReviewTurn =
|
||||
over?.runReviewTurn ??
|
||||
(async (i: { sessionId: string; workspaceId: string }) => {
|
||||
reviewed.push(i);
|
||||
return { content: NOTHING_TO_DO };
|
||||
});
|
||||
const runSpy = vi.fn(runReviewTurn);
|
||||
const watcher = new IdleSessionWatcher({
|
||||
dataDir,
|
||||
now: () => NOW,
|
||||
runReviewTurn: runSpy,
|
||||
// Mirror production emitNotification's gate so suppression is genuinely tested.
|
||||
emitNotification: (_event, options) => {
|
||||
let suppressed = false;
|
||||
if (options?.dedupeKey && options.materialHash) {
|
||||
suppressed = !getNotificationGate(dataDir).shouldNotify(options.dedupeKey, options.materialHash);
|
||||
}
|
||||
emitCalls.push({ suppressed, dedupeKey: options?.dedupeKey });
|
||||
return { suppressed };
|
||||
},
|
||||
log: { info: () => {}, warn: () => {} },
|
||||
});
|
||||
return { watcher, reviewed, emitCalls, runSpy };
|
||||
}
|
||||
|
||||
// ── Fire-condition matrix ────────────────────────────────────────────
|
||||
describe('fire condition', () => {
|
||||
beforeEach(() => writeConfig({ enabled: true })); // rest of fields default
|
||||
|
||||
it('reviews an idle session with enough turns', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-a', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
it('does NOT review a still-active session (mtime too recent)', async () => {
|
||||
writeSession('w1', 'sess-active', 10, NOW - 5 * MIN); // < 15 min idle
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
|
||||
it('does NOT review a session below the turn threshold', async () => {
|
||||
writeSession('w1', 'sess-short', 3, NOW - 30 * MIN); // idle but only 3 turns
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(reviewed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Fired-set: no re-fire until mtime advances ───────────────────────
|
||||
it('does not re-review the same session until its file advances', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(await watcher.tick()).toBe(0); // same mtime — already fired
|
||||
expect(runSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// File advances (a new message lands) → new mtime key → re-fires once.
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Disabled / config handling ───────────────────────────────────────
|
||||
it('is a no-op when disabled (default)', async () => {
|
||||
writeConfig({ enabled: false });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is a no-op when the config file is absent (defaults ⇒ disabled)', async () => {
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to defaults on a corrupt config (⇒ disabled)', async () => {
|
||||
writeConfig('{ this is not json');
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).not.toHaveBeenCalled();
|
||||
expect(DEFAULT_CONFIG.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('applies partial config over defaults (enabled + custom idle threshold)', async () => {
|
||||
writeConfig({ enabled: true, idleMinutes: 60 }); // minTurns/cap default
|
||||
writeSession('w1', 'fresh', 6, NOW - 30 * MIN); // idle 30m < 60m ⇒ skipped
|
||||
writeSession('w1', 'old', 6, NOW - 90 * MIN); // idle 90m ⇒ reviewed
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'old', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Daily cap ────────────────────────────────────────────────────────
|
||||
it('enforces maxReviewsPerDay', async () => {
|
||||
writeConfig({ enabled: true, maxReviewsPerDay: 2 });
|
||||
writeSession('w1', 's1', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 's2', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 's3', 6, NOW - 22 * MIN);
|
||||
const { watcher, runSpy } = makeWatcher();
|
||||
const n = await watcher.tick();
|
||||
expect(n).toBe(2); // capped at 2 even though 3 are eligible
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
// A later tick in the same day stays capped (the 3rd never runs).
|
||||
expect(await watcher.tick()).toBe(0);
|
||||
expect(runSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// ── Enumeration skips channel-/evolve- prefixes ──────────────────────
|
||||
it('skips channel-* and evolve-* sessions', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'channel-telegram-123', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'evolve-sess-a', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'sess-real', 6, NOW - 20 * MIN);
|
||||
const { watcher, reviewed } = makeWatcher();
|
||||
expect(await watcher.tick()).toBe(1);
|
||||
expect(reviewed).toEqual([{ sessionId: 'sess-real', workspaceId: 'w1' }]);
|
||||
});
|
||||
|
||||
// ── Notification gating ──────────────────────────────────────────────
|
||||
it('does NOT notify on a NOTHING_TO_DO / empty / error result', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'ntd', 6, NOW - 20 * MIN);
|
||||
writeSession('w1', 'empty', 6, NOW - 21 * MIN);
|
||||
writeSession('w1', 'err', 6, NOW - 22 * MIN);
|
||||
const results: Record<string, ReviewTurnResult> = {
|
||||
ntd: { content: NOTHING_TO_DO },
|
||||
empty: { content: ' ' },
|
||||
err: { content: 'partial', error: 'boom' },
|
||||
};
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async ({ sessionId }) => results[sessionId],
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('notifies once on a material finding and suppresses an identical re-finding', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
const { watcher, emitCalls } = makeWatcher({
|
||||
runReviewTurn: async () => ({ content: 'Finding: the assistant promised a report and never produced it.' }),
|
||||
});
|
||||
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(1);
|
||||
expect(emitCalls[0].suppressed).toBe(false);
|
||||
expect(emitCalls[0].dedupeKey).toBe('self-evolution:sess-a');
|
||||
|
||||
// Session advances, same finding recurs → notification suppressed (anti-nag).
|
||||
writeSession('w1', 'sess-a', 7, NOW - 18 * MIN);
|
||||
await watcher.tick();
|
||||
expect(emitCalls).toHaveLength(2);
|
||||
expect(emitCalls[1].suppressed).toBe(true);
|
||||
});
|
||||
|
||||
// ── Single-flight guard ──────────────────────────────────────────────
|
||||
it('tick is single-flight (a re-entrant tick returns 0)', async () => {
|
||||
writeConfig({ enabled: true });
|
||||
writeSession('w1', 'sess-a', 6, NOW - 20 * MIN);
|
||||
let secondResult = -1;
|
||||
const { watcher } = makeWatcher({
|
||||
runReviewTurn: async () => {
|
||||
// Re-enter while the first tick is still in-flight.
|
||||
secondResult = await watcher.tick();
|
||||
return { content: NOTHING_TO_DO };
|
||||
},
|
||||
});
|
||||
await watcher.tick();
|
||||
expect(secondResult).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────────────
|
||||
describe('idle-watcher pure helpers', () => {
|
||||
it('countTurns excludes the meta line', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
});
|
||||
|
||||
it('countTurns counts every non-empty line when there is no meta', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ role: 'user', content: 'hi' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'yo' }),
|
||||
].join('\n');
|
||||
expect(countTurns(raw)).toBe(2);
|
||||
expect(countTurns('')).toBe(0);
|
||||
});
|
||||
|
||||
it('readRecentTranscript formats messages and skips meta; null when unreadable', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-tr-'));
|
||||
try {
|
||||
const sessionsDir = path.join(dir, 'workspaces', 'w1', 'sessions');
|
||||
fs.mkdirSync(sessionsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(sessionsDir, 'sess-a.jsonl'),
|
||||
[
|
||||
JSON.stringify({ type: 'meta', title: 't' }),
|
||||
JSON.stringify({ role: 'user', content: 'build me a report' }),
|
||||
JSON.stringify({ role: 'assistant', content: 'on it' }),
|
||||
].join('\n'),
|
||||
);
|
||||
const t = readRecentTranscript(dir, 'w1', 'sess-a');
|
||||
expect(t).toContain('USER: build me a report');
|
||||
expect(t).toContain('ASSISTANT: on it');
|
||||
expect(t).not.toContain('meta');
|
||||
expect(readRecentTranscript(dir, 'w1', 'missing')).toBeNull();
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('buildReviewInstruction embeds the transcript and the NOTHING_TO_DO contract', () => {
|
||||
const msg = buildReviewInstruction('sess-a', 'USER: hi');
|
||||
expect(msg).toContain('sess-a');
|
||||
expect(msg).toContain('USER: hi');
|
||||
expect(msg).toContain(NOTHING_TO_DO);
|
||||
expect(msg).toContain('create_skill');
|
||||
});
|
||||
});
|
||||
@@ -8,8 +8,8 @@ import { buildLocalServer } from '../../src/local/index.js';
|
||||
/**
|
||||
* Integration test for the pre:memory-write lint handler registered in chat.ts.
|
||||
* Fires the real hook through the server's shared HookRegistry and asserts that
|
||||
* capability-failure symptoms are cancelled (and routed to the improvement-signal
|
||||
* path) while normal memories and dramatic-but-legitimate claims pass through.
|
||||
* capability-failure symptoms are cancelled while normal memories and
|
||||
* dramatic-but-legitimate claims pass through.
|
||||
*/
|
||||
describe('pre:memory-write capability-symptom lint (handler integration)', () => {
|
||||
let server: FastifyInstance;
|
||||
@@ -30,10 +30,8 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels a capability-failure symptom write and records an improvement signal', async () => {
|
||||
const { hookRegistry, orchestrator } = server.agentState;
|
||||
const store = orchestrator.getImprovementSignals();
|
||||
const before = store.getByCategory('capability_gap').length;
|
||||
it('cancels a capability-failure symptom write', async () => {
|
||||
const { hookRegistry } = server.agentState;
|
||||
|
||||
const result = await hookRegistry.fire('pre:memory-write', {
|
||||
toolName: 'save_memory',
|
||||
@@ -43,9 +41,6 @@ describe('pre:memory-write capability-symptom lint (handler integration)', () =>
|
||||
expect(result.cancelled).toBe(true);
|
||||
expect(result.reason).toMatch(/capability/i);
|
||||
expect(result.reason).toMatch(/acquire_capability/);
|
||||
|
||||
const after = store.getByCategory('capability_gap');
|
||||
expect(after.length).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('allows a normal memory (tool preference) to pass', async () => {
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
/**
|
||||
* Monthly Self-Assessment Tests
|
||||
*
|
||||
* Verifies:
|
||||
* - Assessment shape and all required fields
|
||||
* - Cron registration (monthly_assessment job type is valid)
|
||||
* - Assessment saves to personal mind as I-frame
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
MindDB,
|
||||
CronStore,
|
||||
FrameStore,
|
||||
OptimizationLogStore,
|
||||
ImprovementSignalStore,
|
||||
InstallAuditStore,
|
||||
} from '@waggle/core';
|
||||
import { generateMonthlyAssessment, saveAssessmentToMind, type MonthlyAssessment } from '../../src/local/monthly-assessment.js';
|
||||
|
||||
describe('Monthly Self-Assessment', () => {
|
||||
let db: MindDB;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── Assessment shape ────────────────────────────────────────────────
|
||||
|
||||
describe('generateMonthlyAssessment', () => {
|
||||
it('returns a valid MonthlyAssessment shape', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment).toBeDefined();
|
||||
expect(assessment.period).toBe('2026-03');
|
||||
expect(typeof assessment.totalInteractions).toBe('number');
|
||||
expect(typeof assessment.correctionRate).toBe('number');
|
||||
expect(typeof assessment.improvementTrend).toBe('string');
|
||||
expect(Array.isArray(assessment.topStrengths)).toBe(true);
|
||||
expect(Array.isArray(assessment.topWeaknesses)).toBe(true);
|
||||
expect(Array.isArray(assessment.capabilityGapsDetected)).toBe(true);
|
||||
expect(typeof assessment.skillsInstalled).toBe('number');
|
||||
expect(typeof assessment.recommendation).toBe('string');
|
||||
expect(assessment.recommendation.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('defaults to previous month when no period override', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db);
|
||||
|
||||
// Period should be YYYY-MM format for the previous month
|
||||
expect(assessment.period).toMatch(/^\d{4}-\d{2}$/);
|
||||
});
|
||||
|
||||
it('computes correction rate from optimization logs', () => {
|
||||
// Insert some optimization log entries for March 2026
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
optStore.insert({
|
||||
sessionId: 's1',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['read_file'],
|
||||
turnCount: 5,
|
||||
wasCorrection: false,
|
||||
});
|
||||
optStore.insert({
|
||||
sessionId: 's2',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: ['write_file'],
|
||||
turnCount: 3,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
// Use current month since the entries use datetime('now')
|
||||
const now = new Date();
|
||||
const currentPeriod = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
||||
const assessment = generateMonthlyAssessment(config, db, currentPeriod);
|
||||
|
||||
expect(assessment.totalInteractions).toBe(2);
|
||||
expect(assessment.correctionRate).toBe(0.5);
|
||||
});
|
||||
|
||||
it('uses local calendar boundaries for UTC-backed monthly data', () => {
|
||||
const previousTimezone = process.env.TZ;
|
||||
process.env.TZ = 'Europe/Budapest';
|
||||
|
||||
try {
|
||||
const raw = db.getDatabase();
|
||||
const optStore = new OptimizationLogStore(db);
|
||||
const insideInteraction = optStore.insert({
|
||||
sessionId: 'inside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
const outsideInteraction = optStore.insert({
|
||||
sessionId: 'outside-august',
|
||||
workspaceId: 'w1',
|
||||
systemPrompt: 'test',
|
||||
toolsUsed: [],
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInteraction.id);
|
||||
raw.prepare('UPDATE optimization_log SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInteraction.id);
|
||||
|
||||
raw.exec(`
|
||||
CREATE TABLE feedback_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
message_index INTEGER NOT NULL,
|
||||
rating TEXT NOT NULL,
|
||||
reason TEXT,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('inside-august', 'up', null, '2026-07-31 22:30:00');
|
||||
raw.prepare(`
|
||||
INSERT INTO feedback_entries
|
||||
(session_id, message_index, rating, reason, created_at)
|
||||
VALUES (?, 0, ?, ?, ?)
|
||||
`).run('outside-august', 'down', 'wrong_answer', '2026-08-31 22:30:00');
|
||||
|
||||
const auditStore = new InstallAuditStore(db);
|
||||
const recordInstall = (capabilityName: string) => auditStore.record({
|
||||
capabilityName,
|
||||
capabilityType: 'skill',
|
||||
source: 'starter-pack',
|
||||
riskLevel: 'low',
|
||||
trustSource: 'starter_pack',
|
||||
approvalClass: 'standard',
|
||||
action: 'installed',
|
||||
initiator: 'agent',
|
||||
});
|
||||
const insideInstallA = recordInstall('inside-skill-a');
|
||||
const insideInstallB = recordInstall('inside-skill-b');
|
||||
const outsideInstall = recordInstall('outside-skill');
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 22:30:00', insideInstallA.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-07-31 23:30:00', insideInstallB.id);
|
||||
raw.prepare('UPDATE install_audit SET timestamp = ? WHERE id = ?')
|
||||
.run('2026-08-31 22:30:00', outsideInstall.id);
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-08');
|
||||
|
||||
expect(assessment.totalInteractions).toBe(1);
|
||||
expect(assessment.correctionRate).toBe(1);
|
||||
expect(assessment.topStrengths).toContain('Consistent positive user feedback');
|
||||
expect(assessment.topWeaknesses).not.toContain('wrong answer');
|
||||
expect(assessment.skillsInstalled).toBe(2);
|
||||
} finally {
|
||||
if (previousTimezone === undefined) delete process.env.TZ;
|
||||
else process.env.TZ = previousTimezone;
|
||||
}
|
||||
});
|
||||
|
||||
it('includes capability gaps from improvement signals', () => {
|
||||
const signalStore = new ImprovementSignalStore(db);
|
||||
// Record the same gap twice (threshold for capability_gap is 2)
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search');
|
||||
signalStore.record('capability_gap', 'missing:web_search', 'User needed web search again');
|
||||
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.capabilityGapsDetected).toContain('web_search');
|
||||
});
|
||||
|
||||
it('has non-empty top strengths even with zero data', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
// Should have at least one strength (e.g., "Stable operation")
|
||||
expect(assessment.topStrengths.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('limits top strengths and weaknesses to 5', () => {
|
||||
const config = { port: 3333, host: '127.0.0.1', dataDir: '/tmp/test', litellmUrl: 'http://localhost:4000' };
|
||||
const assessment = generateMonthlyAssessment(config, db, '2026-03');
|
||||
|
||||
expect(assessment.topStrengths.length).toBeLessThanOrEqual(5);
|
||||
expect(assessment.topWeaknesses.length).toBeLessThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Save to mind ────────────────────────────────────────────────────
|
||||
|
||||
describe('saveAssessmentToMind', () => {
|
||||
it('saves assessment as an I-frame in the personal mind', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate', 'Consistent positive user feedback'],
|
||||
topWeaknesses: ['wrong answer'],
|
||||
capabilityGapsDetected: ['web_search'],
|
||||
skillsInstalled: 2,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
|
||||
// Verify the frame was created
|
||||
const frames = new FrameStore(db);
|
||||
const allFrames = frames.getRecent(10);
|
||||
expect(allFrames.length).toBe(1);
|
||||
|
||||
const frame = allFrames[0];
|
||||
expect(frame.frame_type).toBe('I');
|
||||
expect(frame.gop_id).toBe('assessment');
|
||||
expect(frame.importance).toBe('important');
|
||||
expect(frame.content).toContain('Monthly Agent Assessment');
|
||||
expect(frame.content).toContain('2026-03');
|
||||
expect(frame.content).toContain('15.0%');
|
||||
expect(frame.content).toContain('+5%');
|
||||
expect(frame.content).toContain('web_search');
|
||||
});
|
||||
|
||||
it('a zero-data month writes nothing — no template-noise frames', () => {
|
||||
saveAssessmentToMind(db, {
|
||||
period: '2026-03', totalInteractions: 0, correctionRate: 0,
|
||||
improvementTrend: '0%', topStrengths: ['Stable operation'], topWeaknesses: [],
|
||||
capabilityGapsDetected: [], skillsInstalled: 0,
|
||||
recommendation: 'No corrections recorded.',
|
||||
});
|
||||
expect(new FrameStore(db).getRecent(10)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('re-running the same period replaces the frame instead of duplicating it', () => {
|
||||
const assessment: MonthlyAssessment = {
|
||||
period: '2026-03',
|
||||
totalInteractions: 100,
|
||||
correctionRate: 0.15,
|
||||
improvementTrend: '+5%',
|
||||
topStrengths: ['Low correction rate'],
|
||||
topWeaknesses: [],
|
||||
capabilityGapsDetected: [],
|
||||
skillsInstalled: 0,
|
||||
recommendation: 'Agent is performing well overall.',
|
||||
};
|
||||
|
||||
saveAssessmentToMind(db, assessment);
|
||||
saveAssessmentToMind(db, { ...assessment, totalInteractions: 120 });
|
||||
// A different period must NOT be replaced — one frame per month.
|
||||
saveAssessmentToMind(db, { ...assessment, period: '2026-04' });
|
||||
|
||||
const frames = new FrameStore(db);
|
||||
const assessments = frames
|
||||
.getRecent(10)
|
||||
.filter((f) => f.content.startsWith('# Monthly Agent Assessment'));
|
||||
expect(assessments).toHaveLength(2);
|
||||
const march = assessments.filter((f) => f.content.includes('2026-03'));
|
||||
expect(march).toHaveLength(1);
|
||||
expect(march[0].content).toContain('**Interactions**: 120');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Cron registration ───────────────────────────────────────────────
|
||||
|
||||
describe('cron registration', () => {
|
||||
it('monthly_assessment is a valid cron job type', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
|
||||
// This should not throw — monthly_assessment is in VALID_JOB_TYPES
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
expect(schedule.name).toBe('Monthly assessment');
|
||||
expect(schedule.cron_expr).toBe('0 6 1 * *');
|
||||
expect(schedule.job_type).toBe('monthly_assessment');
|
||||
});
|
||||
|
||||
it('cron expression fires on 1st of month at 6 AM', () => {
|
||||
const cronStore = new CronStore(db);
|
||||
const schedule = cronStore.create({
|
||||
name: 'Monthly assessment',
|
||||
cronExpr: '0 6 1 * *',
|
||||
jobType: 'monthly_assessment',
|
||||
});
|
||||
|
||||
// next_run_at should be set and valid
|
||||
expect(schedule.next_run_at).toBeDefined();
|
||||
expect(schedule.next_run_at).not.toBeNull();
|
||||
|
||||
// The next run should be on the 1st of some month at 6 AM local time
|
||||
// cron-parser interprets expressions in local timezone
|
||||
const nextRun = new Date(schedule.next_run_at!);
|
||||
expect(nextRun.getDate()).toBe(1);
|
||||
expect(nextRun.getHours()).toBe(6);
|
||||
expect(nextRun.getMinutes()).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -20,7 +20,6 @@ const testState = vi.hoisted(() => {
|
||||
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
||||
return {
|
||||
previousPromptAssembler,
|
||||
optimizerExpand: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
};
|
||||
});
|
||||
@@ -41,12 +40,6 @@ vi.mock('../../src/local/model-availability.js', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../src/local/services/optimizer-service.js', () => ({
|
||||
getOptimizerService: async () => ({
|
||||
expandWithChoices: testState.optimizerExpand,
|
||||
}),
|
||||
}));
|
||||
|
||||
function parseSse(raw: string): Array<{ event: string; data: Record<string, unknown> }> {
|
||||
return raw.split(/\n\n/)
|
||||
.filter(Boolean)
|
||||
@@ -151,12 +144,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
resetRateLimiter(server);
|
||||
testState.optimizerExpand.mockReset().mockResolvedValue({
|
||||
expanded: null,
|
||||
clarifyingQuestions: null,
|
||||
intent: 'request',
|
||||
isVague: false,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -657,7 +644,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
expect(response.body).toContain(toolResultMarker);
|
||||
expect(response.body).toContain(responseMarker);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
@@ -740,7 +726,6 @@ describe('persona acceptance prompt budget', () => {
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(capturedConfig?.traceRecording).toBeUndefined();
|
||||
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
||||
|
||||
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
||||
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
||||
|
||||
@@ -14,10 +14,8 @@ import { afterEach, describe, it, expect } from 'vitest';
|
||||
import Fastify from 'fastify';
|
||||
import {
|
||||
deleteCustomPersona,
|
||||
deployPersonaOverride,
|
||||
isValidCustomPersonaId,
|
||||
listPersonas,
|
||||
rollbackPersonaOverride,
|
||||
saveCustomPersona,
|
||||
type AgentPersona,
|
||||
} from '@waggle/agent';
|
||||
@@ -118,14 +116,12 @@ describe('Personas Routes', () => {
|
||||
await server.close();
|
||||
});
|
||||
|
||||
it('contains all 23 personas', async () => {
|
||||
it('contains all 22 personas', async () => {
|
||||
const server = createTestServer();
|
||||
const res = await server.inject({ method: 'GET', url: '/api/personas' });
|
||||
const { personas } = res.json();
|
||||
|
||||
// 22 tiered personas + session-reviewer (internal self-evolution reviewer;
|
||||
// returned by the raw list but excluded from onboarding + PersonaSwitcher).
|
||||
expect(personas.length).toBe(23);
|
||||
expect(personas.length).toBe(22);
|
||||
const ids = personas.map((p: { id: string }) => p.id);
|
||||
expect(ids).toContain('researcher');
|
||||
expect(ids).toContain('writer');
|
||||
@@ -251,40 +247,5 @@ describe('Personas Routes', () => {
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
|
||||
it('deployPersonaOverride rejects traversal before writing an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
|
||||
expect(() => deployPersonaOverride(dataDir, {
|
||||
personaId: '../evolved',
|
||||
systemPrompt: 'Escaped override',
|
||||
})).toThrow('Invalid custom persona ID');
|
||||
expect(fs.existsSync(outsidePath)).toBe(false);
|
||||
});
|
||||
|
||||
it('deployPersonaOverride keeps its validated ID authoritative over overrides', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const result = deployPersonaOverride(dataDir, {
|
||||
personaId: 'safe-persona',
|
||||
systemPrompt: 'Validated prompt',
|
||||
overrides: { id: '../evolved', systemPrompt: 'Spoofed prompt' } as never,
|
||||
});
|
||||
|
||||
const deployed = JSON.parse(fs.readFileSync(result.path, 'utf-8')) as AgentPersona;
|
||||
expect(deployed.id).toBe('safe-persona');
|
||||
expect(deployed.systemPrompt).toBe('Validated prompt');
|
||||
expect(fs.existsSync(path.join(dataDir, 'evolved.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rollbackPersonaOverride rejects traversal before deleting an override', () => {
|
||||
const dataDir = makeTempDir();
|
||||
const outsidePath = path.join(dataDir, 'evolved.json');
|
||||
fs.writeFileSync(outsidePath, 'sentinel', 'utf-8');
|
||||
|
||||
expect(() => rollbackPersonaOverride(dataDir, '../evolved'))
|
||||
.toThrow('Invalid custom persona ID');
|
||||
expect(fs.readFileSync(outsidePath, 'utf-8')).toBe('sentinel');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
/**
|
||||
* session-reviewer persona — trust-boundary tool policy.
|
||||
*
|
||||
* The reviewer runs headless; ALWAYS_AVAILABLE_TOOLS (persona-tool-filter) re-adds
|
||||
* write-side tools past any allowlist, so the persona's disallowedTools must strip
|
||||
* the dangerous ones while KEEPING create_skill (its one proposal write) and the
|
||||
* read tools. This test locks that against regression.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getPersona, type ToolDefinition } from '@waggle/agent';
|
||||
import { applyPersonaToolFilter } from '../../src/local/persona-tool-filter.js';
|
||||
|
||||
/** Minimal ToolDefinition stub — only `name` matters for the filter. */
|
||||
function tool(name: string): ToolDefinition {
|
||||
return { name, description: '', parameters: { type: 'object', properties: {} }, execute: async () => '' };
|
||||
}
|
||||
|
||||
// A representative superset spanning reads, writes, memory, exec, and skill tools.
|
||||
const POOL: ToolDefinition[] = [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'save_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'correct_knowledge', 'add_task',
|
||||
'list_skills', 'search_skills', 'read_skill', 'create_skill', 'delete_skill',
|
||||
'write_file', 'edit_file', 'generate_docx', 'bash',
|
||||
'git_commit', 'git_push',
|
||||
'spawn_agent', 'install_capability', 'acquire_capability',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
].map(tool);
|
||||
|
||||
describe('session-reviewer persona', () => {
|
||||
it('exists and is not read-only (so create_skill survives)', () => {
|
||||
const p = getPersona('session-reviewer');
|
||||
expect(p).toBeTruthy();
|
||||
expect(p!.isReadOnly).toBe(false);
|
||||
expect(p!.tools).toContain('create_skill');
|
||||
});
|
||||
|
||||
it('applyPersonaToolFilter keeps reads + create_skill, strips writes/exec/memory', () => {
|
||||
const p = getPersona('session-reviewer')!;
|
||||
const names = new Set(applyPersonaToolFilter(POOL, p).map(t => t.name));
|
||||
|
||||
// Kept — the reviewer's read surface + its one proposal write.
|
||||
for (const keep of ['read_file', 'search_content', 'search_memory', 'read_skill', 'list_skills', 'create_skill']) {
|
||||
expect(names.has(keep), `expected ${keep} to survive`).toBe(true);
|
||||
}
|
||||
|
||||
// Stripped — writes, exec, memory writes, and the ALWAYS_AVAILABLE re-adds.
|
||||
for (const drop of [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'bash', 'spawn_agent', 'write_file', 'edit_file', 'git_commit',
|
||||
'add_task', 'correct_knowledge', 'execute_step', 'compose_workflow',
|
||||
]) {
|
||||
expect(names.has(drop), `expected ${drop} to be stripped`).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -217,7 +217,7 @@ const SELECTOR_TOOL_NAMES = [
|
||||
'delete_schedule', 'trigger_schedule', 'perplexity_search', 'tavily_search', 'brave_search',
|
||||
'browser_navigate', 'browser_screenshot', 'browser_click', 'browser_fill', 'browser_evaluate',
|
||||
'browser_snapshot', 'lsp_diagnostics', 'lsp_definition', 'lsp_references', 'lsp_hover',
|
||||
'cli_discover', 'cli_execute', 'agent_insights', 'find_connector', 'list_connector_categories',
|
||||
'cli_discover', 'cli_execute', 'find_connector', 'list_connector_categories',
|
||||
'spawn_agent', 'list_agents', 'get_agent_result', 'compose_workflow', 'orchestrate_workflow',
|
||||
'list_harnesses', 'run_harness',
|
||||
] as const;
|
||||
|
||||
@@ -1,341 +0,0 @@
|
||||
/**
|
||||
* H-10 G1 tests — EvolutionService.
|
||||
*
|
||||
* Covers:
|
||||
* - minimum-dataset gate (skips when no target has enough new traces)
|
||||
* - api-key gate (skips when vault is empty)
|
||||
* - target selection (never-run first, then oldest lastRunAt, then most traces)
|
||||
* - baseline resolution (persona + behavioral-spec-section + overrides)
|
||||
* - start/stop/tickInFlight lifecycle
|
||||
* - env flag parsing
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import { BEHAVIORAL_SPEC_SECTIONS } from '@waggle/agent';
|
||||
import {
|
||||
EvolutionService,
|
||||
isEvolutionAutoEnabled,
|
||||
type EvolutionServiceDeps,
|
||||
type EvolutionTargetId,
|
||||
type TickResult,
|
||||
} from '../../src/local/services/evolution-service.js';
|
||||
|
||||
// ── Shared test fixture ──────────────────────────────────────────
|
||||
|
||||
interface Fixture {
|
||||
tmpDir: string;
|
||||
mind: MindDB;
|
||||
traceStore: ExecutionTraceStore;
|
||||
runStore: EvolutionRunStore;
|
||||
apiKey: string | null;
|
||||
activeSpec: Record<string, string>;
|
||||
makeDeps: (overrides?: Partial<EvolutionServiceDeps>) => EvolutionServiceDeps;
|
||||
}
|
||||
|
||||
function setupFixture(): Fixture {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-svc-'));
|
||||
const mind = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
const traceStore = new ExecutionTraceStore(mind);
|
||||
const runStore = new EvolutionRunStore(mind);
|
||||
|
||||
const fx: Fixture = {
|
||||
tmpDir,
|
||||
mind,
|
||||
traceStore,
|
||||
runStore,
|
||||
apiKey: 'sk-test',
|
||||
activeSpec: {},
|
||||
makeDeps: (overrides = {}) => ({
|
||||
traceStore,
|
||||
runStore,
|
||||
getApiKey: () => fx.apiKey,
|
||||
getActiveBehavioralSpec: () => fx.activeSpec,
|
||||
...overrides,
|
||||
}),
|
||||
};
|
||||
return fx;
|
||||
}
|
||||
|
||||
function teardown(fx: Fixture): void {
|
||||
fx.mind.close();
|
||||
try { fs.rmSync(fx.tmpDir, { recursive: true, force: true }); } catch { /* EBUSY on win32 */ }
|
||||
}
|
||||
|
||||
/** Seed `count` successful finalized traces for a given personaId. */
|
||||
function seedTraces(
|
||||
fx: Fixture,
|
||||
opts: { personaId?: string; count: number; outcome?: 'success' | 'corrected' | 'pending' },
|
||||
): void {
|
||||
for (let i = 0; i < opts.count; i++) {
|
||||
const id = fx.traceStore.start({
|
||||
personaId: opts.personaId ?? null,
|
||||
input: `q${i}`,
|
||||
});
|
||||
const outcome = opts.outcome ?? 'success';
|
||||
if (outcome !== 'pending') {
|
||||
fx.traceStore.finalize(id, { outcome, output: `a${i}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────
|
||||
|
||||
describe('EvolutionService — H-10 G1', () => {
|
||||
let fx: Fixture;
|
||||
beforeEach(() => { fx = setupFixture(); });
|
||||
afterEach(() => { teardown(fx); });
|
||||
|
||||
describe('dataset gate + target selection', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
{ kind: 'persona-system-prompt', name: 'writer' },
|
||||
];
|
||||
|
||||
it('pickNextTarget returns null when no persona has enough traces', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('pickNextTarget picks the never-run target that clears the gate', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 5 });
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const chosen = svc.pickNextTarget();
|
||||
expect(chosen?.name).toBe('coder');
|
||||
expect(chosen?.newTraces).toBe(25);
|
||||
expect(chosen?.lastRunAt).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers never-run targets over those with an older lastRunAt', () => {
|
||||
// Both qualify by trace count.
|
||||
seedTraces(fx, { personaId: 'coder', count: 30 });
|
||||
seedTraces(fx, { personaId: 'writer', count: 30 });
|
||||
// coder has a historical run; writer has none.
|
||||
fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
expect(svc.pickNextTarget()?.name).toBe('writer');
|
||||
});
|
||||
|
||||
it('only counts outcomes in eligibleOutcomes', () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 30, outcome: 'pending' });
|
||||
const svc = new EvolutionService(fx.makeDeps(), {
|
||||
targets,
|
||||
minTracesPerTarget: 20,
|
||||
eligibleOutcomes: ['success'],
|
||||
});
|
||||
// All pending — 0 eligible even though 30 exist.
|
||||
expect(svc.pickNextTarget()).toBeNull();
|
||||
});
|
||||
|
||||
it('counts only traces created since the last run when computing newTraces', () => {
|
||||
// Seed historical traces first.
|
||||
seedTraces(fx, { personaId: 'coder', count: 100 });
|
||||
|
||||
// Record a run with a created_at that's strictly AFTER the trace rows.
|
||||
// SQLite datetime('now') has second-level precision, so we need to bump
|
||||
// the run's timestamp forward by an hour to avoid a same-second
|
||||
// collision that would make "since >= lastRunAt" include the seeded
|
||||
// traces. Using raw SQL is the least invasive way to simulate time
|
||||
// advancing between seed and run creation.
|
||||
const run = fx.runStore.create({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
baselineText: 'b', winnerText: 'w',
|
||||
deltaAccuracy: 0.05, gateVerdict: 'pass', gateReasons: [],
|
||||
});
|
||||
const futureTs = new Date(Date.now() + 60 * 60 * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z$/, '');
|
||||
fx.mind.getDatabase()
|
||||
.prepare('UPDATE evolution_runs SET created_at = ? WHERE run_uuid = ?')
|
||||
.run(futureTs, run.run_uuid);
|
||||
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const candidates = svc.enumerateCandidates();
|
||||
const coder = candidates.find(c => c.name === 'coder');
|
||||
expect(coder?.lastRunAt).toBe(futureTs);
|
||||
expect(coder?.newTraces).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('baseline resolution', () => {
|
||||
it('resolves a real persona baseline from the registry', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for an unknown persona', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'persona-system-prompt', name: 'nope' })).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers the active behavioral spec when an override exists', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
fx.activeSpec[section] = 'OVERRIDDEN TEXT';
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section }))
|
||||
.toBe('OVERRIDDEN TEXT');
|
||||
});
|
||||
|
||||
it('falls back to compile-time BEHAVIORAL_SPEC when no override', () => {
|
||||
const section = BEHAVIORAL_SPEC_SECTIONS[0];
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
const baseline = svc.resolveBaseline({ kind: 'behavioral-spec-section', name: section });
|
||||
expect(baseline).toBeTruthy();
|
||||
expect(typeof baseline).toBe('string');
|
||||
});
|
||||
|
||||
it('returns null for tool-description / skill-body / generic (not auto-evolvable)', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [] });
|
||||
expect(svc.resolveBaseline({ kind: 'tool-description', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'skill-body', name: 'anything' })).toBeNull();
|
||||
expect(svc.resolveBaseline({ kind: 'generic', name: 'anything' })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('tick outcomes', () => {
|
||||
const targets: EvolutionTargetId[] = [
|
||||
{ kind: 'persona-system-prompt', name: 'coder' },
|
||||
];
|
||||
|
||||
it('skips when no API key is configured', async () => {
|
||||
fx.apiKey = null;
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 1 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/api key/i);
|
||||
});
|
||||
|
||||
it('skips when no target clears the gate', async () => {
|
||||
// Zero traces for any target.
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets, minTracesPerTarget: 20 });
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/dataset gate/);
|
||||
});
|
||||
|
||||
it('invokes the runner for the picked target when gate passes', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn<Parameters<NonNullable<EvolutionServiceDeps['runner']>>, Promise<TickResult>>()
|
||||
.mockResolvedValue({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: 'uuid-stub',
|
||||
});
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(runner).toHaveBeenCalledOnce();
|
||||
const [calledTarget, calledBaseline, calledKey] = runner.mock.calls[0];
|
||||
// The service passes the full candidate (incl. lastRunAt + newTraces),
|
||||
// not just the bare target id.
|
||||
expect(calledTarget).toMatchObject({ kind: 'persona-system-prompt', name: 'coder' });
|
||||
expect(calledBaseline).toBeTruthy();
|
||||
expect(calledKey).toBe('sk-test');
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.runUuid).toBe('uuid-stub');
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces runner errors through the tick return, not a throw', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
const runner = vi.fn().mockRejectedValue(new Error('boom'));
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const result = await svc.tick();
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toMatch(/tick failed: boom/);
|
||||
});
|
||||
|
||||
it('reentrancy guard: concurrent tick returns "already in progress"', async () => {
|
||||
seedTraces(fx, { personaId: 'coder', count: 25 });
|
||||
let resolve: ((v: TickResult) => void) | null = null;
|
||||
const runner = vi.fn(() => new Promise<TickResult>(r => { resolve = r; }));
|
||||
|
||||
const svc = new EvolutionService(
|
||||
fx.makeDeps({ runner }),
|
||||
{ targets, minTracesPerTarget: 20 },
|
||||
);
|
||||
|
||||
const first = svc.tick();
|
||||
const second = await svc.tick(); // before first resolves
|
||||
|
||||
expect(second.skipped).toBe(true);
|
||||
if (second.skipped) expect(second.reason).toMatch(/already in progress/);
|
||||
|
||||
// Unblock and let the first finish cleanly.
|
||||
resolve!({
|
||||
skipped: false,
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'coder',
|
||||
outcome: 'proposed',
|
||||
runUuid: null,
|
||||
});
|
||||
await first;
|
||||
});
|
||||
});
|
||||
|
||||
describe('lifecycle', () => {
|
||||
it('start/stop toggles isRunning', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
svc.start();
|
||||
expect(svc.isRunning()).toBe(true);
|
||||
svc.stop();
|
||||
expect(svc.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('start is idempotent — double-start leaves a single timer', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 60_000 });
|
||||
svc.start();
|
||||
const firstTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
svc.start();
|
||||
const secondTimer = (svc as unknown as { timer: NodeJS.Timeout | null }).timer;
|
||||
expect(firstTimer).toBe(secondTimer);
|
||||
svc.stop();
|
||||
});
|
||||
|
||||
it('clamps tickIntervalMs to at least 60s', () => {
|
||||
const svc = new EvolutionService(fx.makeDeps(), { targets: [], tickIntervalMs: 1 });
|
||||
expect(svc.config.tickIntervalMs).toBe(60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isEvolutionAutoEnabled', () => {
|
||||
it('returns false when the flag is missing or unrelated', () => {
|
||||
expect(isEvolutionAutoEnabled({})).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'false' })).toBe(false);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '0' })).toBe(false);
|
||||
});
|
||||
it('returns true for "1" / "true" / "yes"', () => {
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: '1' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'true' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'TRUE' })).toBe(true);
|
||||
expect(isEvolutionAutoEnabled({ WAGGLE_EVOLUTION_AUTO_ENABLED: 'yes' })).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user