refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ax-llm/ax": "^19.0.12",
|
||||
"@waggle/core": "*",
|
||||
"@waggle/marketplace": "*",
|
||||
"@waggle/shared": "*",
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
/**
|
||||
* Agent Learning — persistent behavioral adjustments from performance signals.
|
||||
*
|
||||
* Tracks what works and what doesn't across sessions, adjusts agent behavior
|
||||
* by building a learned context section injected into the system prompt.
|
||||
*
|
||||
* Three learning channels:
|
||||
* 1. Correction patterns — user corrections become persistent rules
|
||||
* 2. Success patterns — approaches that got positive feedback
|
||||
* 3. Persona effectiveness — per-persona quality signals
|
||||
*
|
||||
* Uses the improvement_signals table for persistence.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
|
||||
export interface LearnedBehavior {
|
||||
rule: string;
|
||||
source: 'correction' | 'success' | 'observation';
|
||||
confidence: number;
|
||||
occurrences: number;
|
||||
lastSeen: string;
|
||||
}
|
||||
|
||||
export interface PersonaEffectiveness {
|
||||
personaId: string;
|
||||
tasksCompleted: number;
|
||||
correctionsReceived: number;
|
||||
positiveSignals: number;
|
||||
effectivenessScore: number; // 0-100
|
||||
}
|
||||
|
||||
export interface LearningSnapshot {
|
||||
learnedBehaviors: LearnedBehavior[];
|
||||
personaStats: PersonaEffectiveness[];
|
||||
totalCorrections: number;
|
||||
totalSuccesses: number;
|
||||
adaptationLevel: 'new' | 'learning' | 'adapted' | 'expert';
|
||||
}
|
||||
|
||||
export class AgentLearning {
|
||||
private store: ImprovementSignalStore;
|
||||
|
||||
constructor(store: ImprovementSignalStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Record a successful interaction pattern. */
|
||||
recordSuccess(pattern: string, personaId?: string): void {
|
||||
this.store.record('workflow_pattern', `success:${pattern}`, `Approach worked well`, {
|
||||
type: 'success',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record that the user gave positive feedback. */
|
||||
recordPositiveFeedback(context: string, personaId?: string): void {
|
||||
this.store.record('correction', `positive:${context.slice(0, 50)}`, context, {
|
||||
type: 'positive',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record a persona's task completion. */
|
||||
recordPersonaTask(personaId: string, corrected: boolean): void {
|
||||
const key = corrected ? `persona:${personaId}:corrected` : `persona:${personaId}:completed`;
|
||||
this.store.record('workflow_pattern', key, personaId, {
|
||||
type: 'persona_task',
|
||||
personaId,
|
||||
corrected,
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a learning snapshot from accumulated signals. */
|
||||
getSnapshot(): LearningSnapshot {
|
||||
const all = this.store.getActionable();
|
||||
const learnedBehaviors: LearnedBehavior[] = [];
|
||||
const personaMap = new Map<string, PersonaEffectiveness>();
|
||||
let totalCorrections = 0;
|
||||
let totalSuccesses = 0;
|
||||
|
||||
for (const signal of all) {
|
||||
// Corrections become learned behavioral rules
|
||||
if (signal.category === 'correction' && !signal.pattern_key.startsWith('positive:')) {
|
||||
totalCorrections += signal.count;
|
||||
if (signal.count >= 2) { // Only learn from repeated corrections
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || signal.pattern_key,
|
||||
source: 'correction',
|
||||
confidence: Math.min(1.0, 0.5 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Positive feedback
|
||||
if (signal.pattern_key.startsWith('positive:')) {
|
||||
totalSuccesses += signal.count;
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || 'Positive approach',
|
||||
source: 'success',
|
||||
confidence: Math.min(1.0, 0.6 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
|
||||
// Success patterns
|
||||
if (signal.pattern_key.startsWith('success:')) {
|
||||
totalSuccesses += signal.count;
|
||||
}
|
||||
|
||||
// Persona stats
|
||||
if (signal.pattern_key.startsWith('persona:')) {
|
||||
const parts = signal.pattern_key.split(':');
|
||||
const personaId = parts[1];
|
||||
const isCorrected = parts[2] === 'corrected';
|
||||
|
||||
let stats = personaMap.get(personaId);
|
||||
if (!stats) {
|
||||
stats = { personaId, tasksCompleted: 0, correctionsReceived: 0, positiveSignals: 0, effectivenessScore: 50 };
|
||||
personaMap.set(personaId, stats);
|
||||
}
|
||||
|
||||
if (isCorrected) {
|
||||
stats.correctionsReceived += signal.count;
|
||||
} else {
|
||||
stats.tasksCompleted += signal.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate persona effectiveness
|
||||
for (const stats of personaMap.values()) {
|
||||
const total = stats.tasksCompleted + stats.correctionsReceived;
|
||||
stats.effectivenessScore = total > 0
|
||||
? Math.round((stats.tasksCompleted / total) * 100)
|
||||
: 50;
|
||||
}
|
||||
|
||||
// Determine adaptation level
|
||||
const totalSignals = totalCorrections + totalSuccesses;
|
||||
const adaptationLevel = totalSignals < 5 ? 'new'
|
||||
: totalSignals < 20 ? 'learning'
|
||||
: totalSignals < 50 ? 'adapted'
|
||||
: 'expert';
|
||||
|
||||
return {
|
||||
learnedBehaviors: learnedBehaviors.slice(0, 10), // Cap at 10 rules
|
||||
personaStats: Array.from(personaMap.values()),
|
||||
totalCorrections,
|
||||
totalSuccesses,
|
||||
adaptationLevel,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format learned behaviors as a system prompt section.
|
||||
* Only included when there are meaningful learned rules.
|
||||
*/
|
||||
formatLearningPrompt(): string | null {
|
||||
const snapshot = this.getSnapshot();
|
||||
if (snapshot.learnedBehaviors.length === 0) return null;
|
||||
|
||||
const lines: string[] = [
|
||||
'## Learned Behaviors',
|
||||
`*Adaptation level: ${snapshot.adaptationLevel} (${snapshot.totalCorrections} corrections, ${snapshot.totalSuccesses} successes)*`,
|
||||
'',
|
||||
];
|
||||
|
||||
const corrections = snapshot.learnedBehaviors.filter(b => b.source === 'correction');
|
||||
if (corrections.length > 0) {
|
||||
lines.push('**Avoid these patterns** (user has corrected before):');
|
||||
for (const b of corrections) {
|
||||
lines.push(`- ${b.rule} (corrected ${b.occurrences}x, confidence: ${(b.confidence * 100).toFixed(0)}%)`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const successes = snapshot.learnedBehaviors.filter(b => b.source === 'success');
|
||||
if (successes.length > 0) {
|
||||
lines.push('**Keep doing** (positive feedback received):');
|
||||
for (const b of successes) {
|
||||
lines.push(`- ${b.rule}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
}
|
||||
@@ -384,8 +384,7 @@ When approaching any task:
|
||||
};
|
||||
|
||||
/**
|
||||
* Section name literal type — matches the evolution-deploy module.
|
||||
* Kept minimal here so behavioral-spec.ts stays free of cross-imports.
|
||||
* Section name literal type for `buildActiveBehavioralSpec` overrides.
|
||||
*/
|
||||
export type BehavioralSpecSectionName =
|
||||
| 'coreLoop'
|
||||
@@ -402,8 +401,7 @@ export type BehavioralSpecSectionName =
|
||||
* the compiled baseline unchanged.
|
||||
*
|
||||
* Typical usage at server boot:
|
||||
* const overrides = loadBehavioralSpecOverrides(dataDir);
|
||||
* const spec = buildActiveBehavioralSpec(overrides);
|
||||
* const spec = buildActiveBehavioralSpec();
|
||||
* // ...pass spec.rules into the system prompt...
|
||||
*/
|
||||
export function buildActiveBehavioralSpec(
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
/**
|
||||
* Compose Evolution — Phase 3.2 of the self-evolution loop.
|
||||
*
|
||||
* Two-stage pipeline:
|
||||
*
|
||||
* Stage 1 — EvolveSchema
|
||||
* Evolve the output STRUCTURE (fields, order, types, constraints).
|
||||
* Returns a winner schema.
|
||||
*
|
||||
* Stage 2 — IterativeGEPA
|
||||
* Freeze the winner schema. Evolve the INSTRUCTION prompt that fills
|
||||
* that schema. Only instruction text changes here.
|
||||
*
|
||||
* Critical design detail (from Mikhail Pavlukhin's paper): **feedback
|
||||
* separation**. If the judge complains "missing reasoning field" during
|
||||
* Stage 2, and we let GEPA see that feedback, GEPA will mutate the
|
||||
* instruction to try to remove reasoning — undoing the schema Stage 1
|
||||
* just evolved. So Stage 2 receives a *filtered* judge that strips
|
||||
* structural complaints and keeps only value-level signals (correctness,
|
||||
* conciseness, format-of-values, tone).
|
||||
*
|
||||
* The composition is stateless: callers provide the baseline schema and
|
||||
* instruction separately; the result returns both winners + aggregate
|
||||
* accuracy improvement.
|
||||
*/
|
||||
|
||||
import {
|
||||
EvolveSchema,
|
||||
type EvolveSchemaOptions,
|
||||
type EvolveSchemaResult,
|
||||
type Schema,
|
||||
type SchemaExecuteFn,
|
||||
} from './evolve-schema.js';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
type IterativeGEPAOptions,
|
||||
type GEPARunResult,
|
||||
} from './iterative-optimizer.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import { RUNNING_JUDGE_BRAND, isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface ComposeEvolutionOptions {
|
||||
/** EvolveSchema stage configuration */
|
||||
schema: Omit<EvolveSchemaOptions, 'onProgress' | 'signal'>;
|
||||
/** IterativeGEPA stage configuration */
|
||||
instructions: Omit<IterativeGEPAOptions, 'onProgress' | 'signal'>;
|
||||
/**
|
||||
* Classifier — returns 'structural' to drop a feedback line from GEPA,
|
||||
* or 'value' to keep it. Default: `defaultFeedbackFilter`.
|
||||
*/
|
||||
feedbackFilter?: FeedbackFilter;
|
||||
/** Optional progress reporter across both stages */
|
||||
onProgress?: (event: ComposeProgress) => void;
|
||||
/** Optional abort signal (passed through to both stages) */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ComposeEvolutionResult {
|
||||
schema: EvolveSchemaResult;
|
||||
instructions: GEPARunResult;
|
||||
/**
|
||||
* Accuracy delta after both stages compared to the raw baseline schema +
|
||||
* baseline instructions. Positive = improvement.
|
||||
*/
|
||||
combinedDelta: number;
|
||||
/** True if both stages improved over their individual baselines. */
|
||||
fullyImproved: boolean;
|
||||
/** Frozen schema used as input to the GEPA stage — convenience pointer. */
|
||||
frozenSchema: Schema;
|
||||
}
|
||||
|
||||
export interface ComposeProgress {
|
||||
stage: 'schema' | 'instructions' | 'done';
|
||||
/** Underlying stage progress event, if available */
|
||||
detail?: unknown;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export type FeedbackFilter = (feedback: string) => 'structural' | 'value';
|
||||
|
||||
// ── Feedback filtering ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Default heuristic: classifies feedback lines as 'structural' when they
|
||||
* mention schema-level concepts (field, schema, missing/extra property,
|
||||
* wrong type, reorder, etc). Everything else is 'value'.
|
||||
*
|
||||
* Tight enough that ambiguous feedback still flows to GEPA — only obvious
|
||||
* structural complaints are dropped.
|
||||
*/
|
||||
export function defaultFeedbackFilter(feedback: string): 'structural' | 'value' {
|
||||
if (!feedback) return 'value';
|
||||
const lower = feedback.toLowerCase();
|
||||
|
||||
const structuralPatterns: RegExp[] = [
|
||||
/\bmissing\s+(?:the\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\badd(?:ed|ing)?\s+(?:a|the)?\s*(?:new\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\bextra\s+field\b/,
|
||||
/\bunexpected\s+field\b/,
|
||||
/\bwrong\s+(?:field\s+)?(?:type|order)\b/,
|
||||
/\bfield\s+type\s+(?:mismatch|wrong)/,
|
||||
/\b(?:should\s+)?(?:re)?order\s+(?:the\s+)?fields\b/,
|
||||
/\bschema\s+(?:mismatch|violation|error)\b/,
|
||||
/\binvalid\s+json\s+shape\b/,
|
||||
/\bmissing\s+required\s+property\b/,
|
||||
/\bproperty\s+missing\b/,
|
||||
/\bshape\s+is\s+wrong\b/,
|
||||
];
|
||||
|
||||
for (const pattern of structuralPatterns) {
|
||||
if (pattern.test(lower)) return 'structural';
|
||||
}
|
||||
return 'value';
|
||||
}
|
||||
|
||||
// ── Judge wrapping ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Wrap an LLMJudge-like object so the feedback it emits has structural
|
||||
* lines stripped according to `filter`. Only `feedback` is filtered —
|
||||
* the numerical scores (correctness, procedure, conciseness, overall)
|
||||
* are preserved as-is so GEPA sees the true accuracy signal.
|
||||
*
|
||||
* The multi-line feedback from the judge is split on newlines and each
|
||||
* line classified individually.
|
||||
*/
|
||||
export function filterJudgeFeedback(
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): Pick<LLMJudge, 'score'> {
|
||||
const wrapped: Pick<LLMJudge, 'score'> & Partial<Record<symbol, unknown>> = {
|
||||
async score(args) {
|
||||
const raw = await judge.score(args);
|
||||
const filteredFeedback = stripStructuralLines(raw.feedback, filter);
|
||||
const result: JudgeScore = { ...raw, feedback: filteredFeedback };
|
||||
return result;
|
||||
},
|
||||
};
|
||||
// H-09 G3: preserve the running-judge brand so IterativeGEPA's check
|
||||
// still passes after feedback filtering. Without this, wrapping a
|
||||
// running judge with filterJudgeFeedback would silently strip the
|
||||
// brand and GEPA would reject it.
|
||||
if (isRunningJudge(judge)) {
|
||||
wrapped[RUNNING_JUDGE_BRAND] = true;
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
/** Public helper — strips structural lines from a feedback string. */
|
||||
export function stripStructuralLines(
|
||||
feedback: string,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): string {
|
||||
if (!feedback) return feedback;
|
||||
return feedback
|
||||
.split(/\r?\n/)
|
||||
.filter(line => filter(line) === 'value')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────
|
||||
|
||||
export class ComposeEvolution {
|
||||
async run(options: ComposeEvolutionOptions): Promise<ComposeEvolutionResult> {
|
||||
const signal = options.signal;
|
||||
const filter = options.feedbackFilter ?? defaultFeedbackFilter;
|
||||
|
||||
// Stage 1 — evolve the schema.
|
||||
options.onProgress?.({ stage: 'schema', message: 'starting schema evolution' });
|
||||
const schemaResult = await new EvolveSchema().run({
|
||||
...options.schema,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'schema', detail: e }),
|
||||
});
|
||||
|
||||
if (signal?.aborted) {
|
||||
return assembleAbortedResult(schemaResult, options.instructions.baseline);
|
||||
}
|
||||
|
||||
const frozenSchema = schemaResult.winner.schema;
|
||||
|
||||
// Stage 2 — evolve the instructions, with a filtered judge so GEPA
|
||||
// never sees structural feedback.
|
||||
options.onProgress?.({ stage: 'instructions', message: 'starting instruction evolution' });
|
||||
const filteredJudge = filterJudgeFeedback(options.instructions.judge, filter);
|
||||
|
||||
const gepaResult = await new IterativeGEPA().run({
|
||||
...options.instructions,
|
||||
judge: filteredJudge,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'instructions', detail: e }),
|
||||
});
|
||||
|
||||
const combinedDelta =
|
||||
(gepaResult.winner.score?.overall ?? 0) -
|
||||
((schemaResult.history[0]?.score?.accuracy ?? 0));
|
||||
const fullyImproved = schemaResult.improved && gepaResult.improved;
|
||||
|
||||
options.onProgress?.({ stage: 'done' });
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: gepaResult,
|
||||
combinedDelta,
|
||||
fullyImproved,
|
||||
frozenSchema,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a convenience `SchemaExecuteFn` from a user-supplied runner that
|
||||
* only cares about the instruction prompt (not the schema itself). Used
|
||||
* by callers that want to compose — they supply one "execute" function
|
||||
* for the instructional stage and this helper wraps it for Stage 1.
|
||||
*
|
||||
* The wrapped executor serializes the schema into a short prefix the
|
||||
* model can follow (`Return JSON with fields: <field1>, <field2>, ...`)
|
||||
* before delegating to the caller's function.
|
||||
*/
|
||||
export function schemaExecutorFromInstructionRunner(
|
||||
runInstructions: (args: { prompt: string; input: string }) => Promise<string>,
|
||||
): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
const fieldList = schema.fields.map(f => `"${f.name}" (${f.type})`).join(', ');
|
||||
const prompt = `Return a JSON object with these fields: ${fieldList}.`;
|
||||
try {
|
||||
const actual = await runInstructions({ prompt, input });
|
||||
return { actual, parsed: actualLooksLikeJson(actual) };
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function actualLooksLikeJson(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
return (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
|
||||
(trimmed.startsWith('[') && trimmed.endsWith(']'));
|
||||
}
|
||||
|
||||
function assembleAbortedResult(
|
||||
schemaResult: EvolveSchemaResult,
|
||||
instructionBaseline: string,
|
||||
): ComposeEvolutionResult {
|
||||
// When aborted before Stage 2, produce a minimal GEPARunResult pointing
|
||||
// at the baseline instruction so callers have a stable shape.
|
||||
const baselineInstruction = {
|
||||
id: 'aborted',
|
||||
prompt: instructionBaseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'aborted',
|
||||
score: null,
|
||||
perExample: [],
|
||||
} as GEPARunResult['winner'];
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: {
|
||||
winner: baselineInstruction,
|
||||
paretoFront: [baselineInstruction],
|
||||
history: [baselineInstruction],
|
||||
improved: false,
|
||||
delta: 0,
|
||||
},
|
||||
combinedDelta: 0,
|
||||
fullyImproved: false,
|
||||
frozenSchema: schemaResult.winner.schema,
|
||||
};
|
||||
}
|
||||
|
||||
/** Re-export types so callers can `import { type Schema } from '...'`. */
|
||||
export type { EvolveSchemaResult, GEPARunResult };
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* Correction Detector — identifies when a user is correcting the agent
|
||||
* and classifies corrections as durable (behavior-changing) vs task-local.
|
||||
*
|
||||
* Per Slice 7 correction #2:
|
||||
* - Durable corrections: repeated patterns that should influence future behavior → improvement signals
|
||||
* - Task-local corrections: one-off adjustments within a session → NOT signals
|
||||
* - One-off disagreements: not corrections at all → ignored
|
||||
*/
|
||||
|
||||
export type CorrectionDurability = 'durable' | 'task_local' | 'not_correction';
|
||||
|
||||
export interface DetectedCorrection {
|
||||
isDurable: boolean;
|
||||
durability: CorrectionDurability;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
confidence: number; // 0-1
|
||||
}
|
||||
|
||||
// ── Correction signal patterns ──────────────────────────────
|
||||
|
||||
/** Strong correction signals — high confidence the user is correcting agent behavior */
|
||||
const STRONG_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /\bno[,.]?\s*(?:not that|don'?t|do not|stop|never)\b/i, weight: 3 },
|
||||
{ pattern: /\bI (?:said|told you|asked)\b/i, weight: 3 },
|
||||
{ pattern: /\bthat'?s (?:wrong|incorrect|not (?:what|right))\b/i, weight: 3 },
|
||||
{ pattern: /\bplease (?:don'?t|do not|stop|never)\b/i, weight: 2 },
|
||||
{ pattern: /\binstead[,.]?\s*(?:use|do|try|go with)\b/i, weight: 2 },
|
||||
{ pattern: /\bwrong (?:approach|way|format|style|tone)\b/i, weight: 2 },
|
||||
{ pattern: /\bnot what I (?:wanted|meant|asked)\b/i, weight: 3 },
|
||||
];
|
||||
|
||||
/** Moderate correction signals — may be correction or just refinement */
|
||||
const MODERATE_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /^no[,.:!]\s/i, weight: 2 }, // "No, ..." at start of message — strong disagreement
|
||||
{ pattern: /\bactually[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\brather[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\blet'?s (?:not|try|go with|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bprefer\b/i, weight: 1 },
|
||||
{ pattern: /\bshould (?:be|have been|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bchange (?:it|this|that) to\b/i, weight: 1 },
|
||||
{ pattern: /\btoo (?:verbose|long|short|formal|casual|technical|simple)\b/i, weight: 1 },
|
||||
{ pattern: /\bstop (?:doing|using|adding)\b/i, weight: 1 },
|
||||
{ pattern: /\bkeep (?:it|things) (?:simple|short|brief|casual|formal)\b/i, weight: 1 },
|
||||
];
|
||||
|
||||
// ── Durability classification patterns ──────────────────────
|
||||
|
||||
/** Durable signals — the correction applies beyond this specific task */
|
||||
const DURABLE_SIGNALS: RegExp[] = [
|
||||
/\balways\b/i,
|
||||
/\bnever\b/i,
|
||||
/\bfrom now on\b/i,
|
||||
/\bin (?:the )?future\b/i,
|
||||
/\bwhenever\b/i,
|
||||
/\bevery time\b/i,
|
||||
/\bin general\b/i,
|
||||
/\bI (?:always |usually )?prefer\b/i,
|
||||
/\bmy (?:preference|style|approach)\b/i,
|
||||
/\bdon'?t (?:ever|again)\b/i,
|
||||
/\bremember (?:to|that)\b/i,
|
||||
/\bkeep (?:it|things|this)\b/i,
|
||||
];
|
||||
|
||||
/** Task-local signals — the correction is specific to this task */
|
||||
const TASK_LOCAL_SIGNALS: RegExp[] = [
|
||||
/\bthis (?:time|one|specific|particular)\b/i,
|
||||
/\bfor (?:this|now)\b/i,
|
||||
/\bjust (?:here|this|now)\b/i,
|
||||
/\bin this (?:case|instance|response)\b/i,
|
||||
/\bright now\b/i,
|
||||
/\bhere\b/i,
|
||||
];
|
||||
|
||||
// ── Pattern key extraction ──────────────────────────────────
|
||||
|
||||
/** Categories of behavioral corrections we can extract pattern keys for */
|
||||
const PATTERN_KEY_EXTRACTORS: Array<{ category: string; pattern: RegExp }> = [
|
||||
{ category: 'tone', pattern: /\btoo (?:formal|casual|verbose|terse|technical|simple)\b/i },
|
||||
{ category: 'format', pattern: /\b(?:format|formatting|headers?|bullet|numbering|markdown)\b/i },
|
||||
{ category: 'length', pattern: /\btoo (?:long|short|brief|detailed)\b/i },
|
||||
{ category: 'approach', pattern: /\bwrong (?:approach|way|method|strategy)\b/i },
|
||||
{ category: 'scope', pattern: /\btoo (?:much|many|broad|narrow|specific|general)\b/i },
|
||||
{ category: 'accuracy', pattern: /\b(?:wrong|incorrect|inaccurate|mistake|error)\b/i },
|
||||
{ category: 'style', pattern: /\b(?:style|voice|writing|wording|phrasing)\b/i },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect whether a user message contains a correction and classify it.
|
||||
*
|
||||
* Returns null if no correction detected.
|
||||
* Returns DetectedCorrection with durability classification if correction found.
|
||||
*/
|
||||
export function detectCorrection(
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
if (!userMessage || userMessage.length < 5) return null;
|
||||
|
||||
// Score correction strength
|
||||
let correctionScore = 0;
|
||||
for (const { pattern, weight } of STRONG_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
for (const { pattern, weight } of MODERATE_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
|
||||
// Need minimum score to count as correction
|
||||
if (correctionScore < 2) return null;
|
||||
|
||||
// Classify durability
|
||||
const durability = classifyDurability(userMessage);
|
||||
|
||||
// Extract pattern key
|
||||
const patternKey = extractPatternKey(userMessage);
|
||||
|
||||
// Compute confidence (0-1)
|
||||
const confidence = Math.min(correctionScore / 6, 1);
|
||||
|
||||
// Build detail: first sentence or first 120 chars
|
||||
const detail = extractDetail(userMessage);
|
||||
|
||||
return {
|
||||
isDurable: durability === 'durable',
|
||||
durability,
|
||||
patternKey,
|
||||
detail,
|
||||
confidence,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyDurability(message: string): CorrectionDurability {
|
||||
let durableScore = 0;
|
||||
let taskLocalScore = 0;
|
||||
|
||||
for (const pattern of DURABLE_SIGNALS) {
|
||||
if (pattern.test(message)) durableScore++;
|
||||
}
|
||||
for (const pattern of TASK_LOCAL_SIGNALS) {
|
||||
if (pattern.test(message)) taskLocalScore++;
|
||||
}
|
||||
|
||||
// Explicit durable signals win
|
||||
if (durableScore > 0 && durableScore >= taskLocalScore) return 'durable';
|
||||
|
||||
// Explicit task-local signals
|
||||
if (taskLocalScore > 0) return 'task_local';
|
||||
|
||||
// No explicit signals — default to task_local (conservative; only promote to durable
|
||||
// when the same pattern_key recurs across sessions via ImprovementSignalStore)
|
||||
return 'task_local';
|
||||
}
|
||||
|
||||
function extractPatternKey(message: string): string {
|
||||
for (const { category, pattern } of PATTERN_KEY_EXTRACTORS) {
|
||||
const match = message.match(pattern);
|
||||
if (match) {
|
||||
const qualifier = match[0].toLowerCase().replace(/\s+/g, '_');
|
||||
return `${category}:${qualifier}`;
|
||||
}
|
||||
}
|
||||
// Fallback: generic correction key
|
||||
return 'general:correction';
|
||||
}
|
||||
|
||||
function extractDetail(message: string): string {
|
||||
// Take first sentence or first 120 chars
|
||||
const sentenceMatch = message.match(/^(.+?[.!?])\s/);
|
||||
if (sentenceMatch && sentenceMatch[1].length <= 120) {
|
||||
return sentenceMatch[1];
|
||||
}
|
||||
return message.length > 120 ? message.slice(0, 117) + '...' : message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a sequence of message pairs to detect corrections.
|
||||
* Useful for batch analysis of session history.
|
||||
*/
|
||||
export function detectCorrectionsInHistory(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
): DetectedCorrection[] {
|
||||
const corrections: DetectedCorrection[] = [];
|
||||
for (let i = 1; i < messages.length; i++) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== 'user') continue;
|
||||
|
||||
const prevAssistant = i > 0 ? messages[i - 1] : undefined;
|
||||
const correction = detectCorrection(
|
||||
msg.content,
|
||||
prevAssistant?.role === 'assistant' ? prevAssistant.content : undefined,
|
||||
);
|
||||
if (correction) {
|
||||
corrections.push(correction);
|
||||
}
|
||||
}
|
||||
return corrections;
|
||||
}
|
||||
@@ -1,441 +0,0 @@
|
||||
/**
|
||||
* Eval Dataset Builder — turns execution traces and harvest frames into
|
||||
* train/val/holdout splits for the self-evolution loop.
|
||||
*
|
||||
* Sources mined:
|
||||
* - ExecutionTraceStore (outcome = success | verified → positive examples;
|
||||
* outcome = corrected → negative example with correctionFeedback as ground truth)
|
||||
* - Optional harvest frames (DistilledKnowledge) for Q&A-style augmentation
|
||||
* - Optional corrections from ImprovementSignalStore
|
||||
*
|
||||
* Filter pipeline (in order):
|
||||
* 1. Secret scanning — reject any example containing credentials / tokens
|
||||
* 2. Keyword heuristic — min length, non-trivial content, no duplicate inputs
|
||||
* 3. Optional LLM-as-judge relevance filter (pass an llmCall callback)
|
||||
*
|
||||
* Split is deterministic given a seed — same traces + same seed = same split.
|
||||
* Default split ratio is 60/20/20 train/val/holdout.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
TraceQueryFilter,
|
||||
} from '@waggle/core';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface EvalExample {
|
||||
input: string;
|
||||
expected_output: string;
|
||||
metadata: EvalExampleMetadata;
|
||||
}
|
||||
|
||||
export interface EvalExampleMetadata {
|
||||
traceId?: number;
|
||||
personaId?: string | null;
|
||||
taskShape?: string | null;
|
||||
model?: string | null;
|
||||
outcome?: TraceOutcome;
|
||||
tags?: string[];
|
||||
/** Where this example came from */
|
||||
source: 'trace' | 'harvest' | 'correction';
|
||||
/** Optional opaque identifier from the source system */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface DatasetSplit {
|
||||
train: EvalExample[];
|
||||
val: EvalExample[];
|
||||
holdout: EvalExample[];
|
||||
/** Examples that were filtered out — useful for debugging */
|
||||
rejected: Array<{ reason: string; preview: string }>;
|
||||
/** Stats for logging / UI */
|
||||
stats: {
|
||||
sourced: number;
|
||||
acceptedAfterSecretScan: number;
|
||||
acceptedAfterHeuristic: number;
|
||||
acceptedAfterJudge: number;
|
||||
unique: number;
|
||||
total: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BuildOptions {
|
||||
/** Which trace outcomes count as positive examples (default: ['success', 'verified']). */
|
||||
positiveOutcomes?: TraceOutcome[];
|
||||
/** Whether to include correction traces as negative examples with feedback as expected_output (default true). */
|
||||
includeCorrections?: boolean;
|
||||
/** Filter passed straight through to ExecutionTraceStore.query */
|
||||
traceFilter?: Omit<TraceQueryFilter, 'outcome' | 'limit'> & { limit?: number };
|
||||
/** Optional external augmenters */
|
||||
harvestExamples?: EvalExample[];
|
||||
correctionExamples?: EvalExample[];
|
||||
/** Minimum input characters to keep the example (default 10). */
|
||||
minInputChars?: number;
|
||||
/** Minimum expected_output characters to keep the example (default 5). */
|
||||
minOutputChars?: number;
|
||||
/** Maximum input + output characters combined; over this, the example is rejected (default 16384). */
|
||||
maxCombinedChars?: number;
|
||||
/** Optional LLM-as-judge relevance filter. Pass null or omit to skip. */
|
||||
judge?: (example: EvalExample) => Promise<JudgeVerdict>;
|
||||
/** Ratio triple summing to 1.0 (default [0.6, 0.2, 0.2]). */
|
||||
splitRatios?: [number, number, number];
|
||||
/** Deterministic seed (default 1). */
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
export interface JudgeVerdict {
|
||||
keep: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
|
||||
// ── Deterministic PRNG (mulberry32) ─────────────────────────────
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
/** Fisher-Yates shuffle in-place using provided rng. */
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// ── Builder ─────────────────────────────────────────────────────
|
||||
|
||||
export class EvalDatasetBuilder {
|
||||
private store: ExecutionTraceStore;
|
||||
|
||||
constructor(store: ExecutionTraceStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Mine traces into a typed example list (before filtering). */
|
||||
sourceFromTraces(
|
||||
positiveOutcomes: TraceOutcome[],
|
||||
includeCorrections: boolean,
|
||||
filter: BuildOptions['traceFilter'] = {},
|
||||
): EvalExample[] {
|
||||
const outcomes: TraceOutcome[] = [...positiveOutcomes];
|
||||
if (includeCorrections) outcomes.push('corrected');
|
||||
|
||||
const rows = this.store.queryParsed({
|
||||
...filter,
|
||||
outcome: outcomes,
|
||||
limit: filter.limit ?? 10_000,
|
||||
});
|
||||
|
||||
return rows.map(row => traceToExample(row, includeCorrections));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the full dataset split.
|
||||
*
|
||||
* Filter order:
|
||||
* 1. Source (traces + augmenters)
|
||||
* 2. Secret scan
|
||||
* 3. Heuristic (length, non-trivial)
|
||||
* 4. Optional judge
|
||||
* 5. Dedup on input hash
|
||||
* 6. Shuffle + split
|
||||
*/
|
||||
async build(options: BuildOptions = {}): Promise<DatasetSplit> {
|
||||
const positiveOutcomes = options.positiveOutcomes ?? ['success', 'verified'];
|
||||
const includeCorrections = options.includeCorrections ?? true;
|
||||
const minInputChars = options.minInputChars ?? 10;
|
||||
const minOutputChars = options.minOutputChars ?? 5;
|
||||
const maxCombinedChars = options.maxCombinedChars ?? 16_384;
|
||||
const splitRatios = options.splitRatios ?? [0.6, 0.2, 0.2];
|
||||
const seed = options.seed ?? 1;
|
||||
|
||||
validateRatios(splitRatios);
|
||||
|
||||
const rejected: DatasetSplit['rejected'] = [];
|
||||
|
||||
// 1. Source
|
||||
const traceExamples = this.sourceFromTraces(
|
||||
positiveOutcomes,
|
||||
includeCorrections,
|
||||
options.traceFilter,
|
||||
);
|
||||
const sourced: EvalExample[] = [
|
||||
...traceExamples,
|
||||
...(options.harvestExamples ?? []),
|
||||
...(options.correctionExamples ?? []),
|
||||
];
|
||||
|
||||
// 2. Secret scan
|
||||
const afterSecretScan: EvalExample[] = [];
|
||||
for (const ex of sourced) {
|
||||
const hitInput = detectSecrets(ex.input);
|
||||
const hitOutput = detectSecrets(ex.expected_output);
|
||||
if (hitInput || hitOutput) {
|
||||
rejected.push({
|
||||
reason: `secret:${hitInput ?? hitOutput}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
afterSecretScan.push(ex);
|
||||
}
|
||||
|
||||
// 3. Heuristic
|
||||
const afterHeuristic: EvalExample[] = [];
|
||||
for (const ex of afterSecretScan) {
|
||||
const input = ex.input.trim();
|
||||
const output = ex.expected_output.trim();
|
||||
if (input.length < minInputChars) {
|
||||
rejected.push({ reason: 'too-short-input', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (output.length < minOutputChars) {
|
||||
rejected.push({ reason: 'too-short-output', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (input.length + output.length > maxCombinedChars) {
|
||||
rejected.push({ reason: 'too-long', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (isLowSignal(input) || isLowSignal(output)) {
|
||||
rejected.push({ reason: 'low-signal', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
afterHeuristic.push({ ...ex, input, expected_output: output });
|
||||
}
|
||||
|
||||
// 4. Optional judge
|
||||
let afterJudge = afterHeuristic;
|
||||
if (options.judge) {
|
||||
afterJudge = [];
|
||||
for (const ex of afterHeuristic) {
|
||||
try {
|
||||
const verdict = await options.judge(ex);
|
||||
if (verdict.keep) {
|
||||
afterJudge.push(ex);
|
||||
} else {
|
||||
rejected.push({
|
||||
reason: `judge:${verdict.reason ?? 'rejected'}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// On judge failure, keep the example — err on the side of more data.
|
||||
afterJudge.push(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Dedup on input hash
|
||||
const seen = new Set<string>();
|
||||
const unique: EvalExample[] = [];
|
||||
for (const ex of afterJudge) {
|
||||
const key = hashKey(ex.input);
|
||||
if (seen.has(key)) {
|
||||
rejected.push({ reason: 'duplicate', preview: ex.input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
unique.push(ex);
|
||||
}
|
||||
|
||||
// 6. Shuffle + split
|
||||
const rng = makeRng(seed);
|
||||
const shuffled = shuffle([...unique], rng);
|
||||
const { train, val, holdout } = splitExamples(shuffled, splitRatios);
|
||||
|
||||
return {
|
||||
train,
|
||||
val,
|
||||
holdout,
|
||||
rejected,
|
||||
stats: {
|
||||
sourced: sourced.length,
|
||||
acceptedAfterSecretScan: afterSecretScan.length,
|
||||
acceptedAfterHeuristic: afterHeuristic.length,
|
||||
acceptedAfterJudge: afterJudge.length,
|
||||
unique: unique.length,
|
||||
total: train.length + val.length + holdout.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── JSONL IO ────────────────────────────────────────────────────
|
||||
|
||||
/** Serialize examples as JSONL (one JSON object per line). */
|
||||
export function toJSONL(examples: EvalExample[]): string {
|
||||
return examples.map(ex => JSON.stringify(ex)).join('\n');
|
||||
}
|
||||
|
||||
/** Parse JSONL into examples. Silently skips unparseable lines. */
|
||||
export function fromJSONL(jsonl: string): EvalExample[] {
|
||||
const out: EvalExample[] = [];
|
||||
for (const line of jsonl.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed.input === 'string' &&
|
||||
typeof parsed.expected_output === 'string' &&
|
||||
parsed.metadata
|
||||
) {
|
||||
out.push(parsed as EvalExample);
|
||||
}
|
||||
} catch {
|
||||
// skip malformed line
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function traceToExample(
|
||||
trace: ParsedExecutionTrace,
|
||||
includeCorrections: boolean,
|
||||
): EvalExample {
|
||||
const isCorrected = trace.outcome === 'corrected';
|
||||
const expected = isCorrected && includeCorrections
|
||||
? (trace.payload.correctionFeedback ?? trace.payload.output)
|
||||
: trace.payload.output;
|
||||
|
||||
return {
|
||||
input: trace.payload.input,
|
||||
expected_output: expected,
|
||||
metadata: {
|
||||
traceId: trace.id,
|
||||
personaId: trace.persona_id,
|
||||
taskShape: trace.task_shape,
|
||||
model: trace.model,
|
||||
outcome: trace.outcome,
|
||||
tags: trace.payload.tags ?? [],
|
||||
source: isCorrected ? 'correction' : 'trace',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function validateRatios(ratios: [number, number, number]): void {
|
||||
const sum = ratios[0] + ratios[1] + ratios[2];
|
||||
if (Math.abs(sum - 1.0) > 1e-6) {
|
||||
throw new Error(`Split ratios must sum to 1.0, got ${sum}`);
|
||||
}
|
||||
if (ratios.some(r => r < 0)) {
|
||||
throw new Error('Split ratios must be non-negative');
|
||||
}
|
||||
}
|
||||
|
||||
function splitExamples(
|
||||
examples: EvalExample[],
|
||||
ratios: [number, number, number],
|
||||
): { train: EvalExample[]; val: EvalExample[]; holdout: EvalExample[] } {
|
||||
const n = examples.length;
|
||||
// Floor-then-assign-remainders so every example ends up in exactly one split.
|
||||
const trainN = Math.floor(n * ratios[0]);
|
||||
const valN = Math.floor(n * ratios[1]);
|
||||
const holdoutN = n - trainN - valN;
|
||||
|
||||
return {
|
||||
train: examples.slice(0, trainN),
|
||||
val: examples.slice(trainN, trainN + valN),
|
||||
holdout: examples.slice(trainN + valN, trainN + valN + holdoutN),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap signal check — reject examples that are mostly whitespace, repeated
|
||||
* punctuation, or otherwise meaningless. Not a language detector.
|
||||
*/
|
||||
function isLowSignal(text: string): boolean {
|
||||
if (!text) return true;
|
||||
const stripped = text.replace(/\s+/g, '');
|
||||
if (stripped.length < 3) return true;
|
||||
// 80%+ of the same character → reject
|
||||
const counts = new Map<string, number>();
|
||||
for (const ch of stripped) {
|
||||
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
||||
}
|
||||
const maxCount = Math.max(...counts.values());
|
||||
if (maxCount / stripped.length > 0.8) return true;
|
||||
// No alphanumeric at all → reject
|
||||
if (!/[A-Za-z0-9]/.test(stripped)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* FNV-1a 32-bit hash — stable across processes, no crypto import needed.
|
||||
* Used for input-level dedup.
|
||||
*/
|
||||
function hashKey(text: string): string {
|
||||
let h = 0x811c9dc5;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
h ^= text.charCodeAt(i);
|
||||
h = Math.imul(h, 0x01000193);
|
||||
}
|
||||
return (h >>> 0).toString(16);
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
/**
|
||||
* Evolution Deploy Helpers — Phase 6.1 of the self-evolution loop.
|
||||
*
|
||||
* When the user accepts an EvolutionRun, the orchestrator calls the
|
||||
* caller-supplied `deploy` callback. This module provides the built-in
|
||||
* building blocks for that callback:
|
||||
*
|
||||
* deployPersonaOverride — writes a persona JSON file that
|
||||
* shadows the built-in persona at
|
||||
* runtime via loadCustomPersonas.
|
||||
* deployBehavioralSpecOverride — writes a JSON file in
|
||||
* {dataDir}/behavioral-overrides/
|
||||
* that can be merged into
|
||||
* BEHAVIORAL_SPEC sections at load.
|
||||
* loadBehavioralSpecOverrides — reads those overrides from disk.
|
||||
*
|
||||
* Every writer:
|
||||
* - Creates the target directory if missing.
|
||||
* - Writes atomically (write to .tmp, rename) so a crash mid-write
|
||||
* never corrupts the config.
|
||||
* - Backs up the previous version to {file}.bak before overwriting.
|
||||
* - Returns a structured DeployResult with paths + timestamps.
|
||||
*
|
||||
* These helpers are pure filesystem operations — they don't talk to
|
||||
* any runtime singletons, so the server wiring is trivial:
|
||||
* orchestrator.accept(uuid) → onDeploy(run)
|
||||
* → deployPersonaOverride(dataDir, ...)
|
||||
* → reloadPersonas()
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
import { getPersona } from './personas.js';
|
||||
import { assertValidCustomPersonaId } from './custom-personas.js';
|
||||
|
||||
// ── Public result shape ────────────────────────────────────────
|
||||
|
||||
export interface DeployResult {
|
||||
/** Absolute path that was written */
|
||||
path: string;
|
||||
/** Path to the backup of the previous version (null if first write) */
|
||||
backupPath: string | null;
|
||||
/** When the deploy happened */
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
// ── Persona override ───────────────────────────────────────────
|
||||
|
||||
export interface DeployPersonaInput {
|
||||
/** Persona id — matches a built-in persona (the deploy SHADOWS it). */
|
||||
personaId: string;
|
||||
/** The evolved system prompt text */
|
||||
systemPrompt: string;
|
||||
/** Optional additional field overrides */
|
||||
overrides?: Partial<Omit<AgentPersona, 'id' | 'systemPrompt'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write an evolved persona system prompt to disk so the custom-personas
|
||||
* loader picks it up on next `listPersonas()` call.
|
||||
*
|
||||
* When a built-in persona with this id exists, the override inherits
|
||||
* all its properties (name, tools, tagline, etc) except systemPrompt
|
||||
* (replaced with the evolved one). When no built-in exists, a minimal
|
||||
* persona shell is built from the input.
|
||||
*/
|
||||
export function deployPersonaOverride(
|
||||
dataDir: string,
|
||||
input: DeployPersonaInput,
|
||||
): DeployResult {
|
||||
assertValidCustomPersonaId(input.personaId);
|
||||
const personasDir = path.join(dataDir, 'personas');
|
||||
if (!fs.existsSync(personasDir)) {
|
||||
fs.mkdirSync(personasDir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(personasDir, `${input.personaId}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const builtin = getPersona(input.personaId);
|
||||
const persona: AgentPersona = builtin
|
||||
? {
|
||||
...builtin,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
}
|
||||
: {
|
||||
name: input.personaId,
|
||||
description: input.overrides?.description ?? `${input.personaId} (evolved)`,
|
||||
icon: input.overrides?.icon ?? 'sparkles',
|
||||
modelPreference: input.overrides?.modelPreference ?? 'claude-sonnet-4-6',
|
||||
tools: input.overrides?.tools ?? [],
|
||||
workspaceAffinity: input.overrides?.workspaceAffinity ?? [],
|
||||
suggestedCommands: input.overrides?.suggestedCommands ?? [],
|
||||
defaultWorkflow: input.overrides?.defaultWorkflow ?? null,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(persona, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll back a previously-deployed persona override to the .bak version
|
||||
* (or remove the override entirely if there was no previous version).
|
||||
*/
|
||||
export function rollbackPersonaOverride(
|
||||
dataDir: string,
|
||||
personaId: string,
|
||||
): boolean {
|
||||
assertValidCustomPersonaId(personaId);
|
||||
const filePath = path.join(dataDir, 'personas', `${personaId}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Behavioral-spec override ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* BEHAVIORAL_SPEC is compiled into the bundle, so we cannot mutate it
|
||||
* on disk without rebuilding. Instead, overrides live as JSON files in
|
||||
* {dataDir}/behavioral-overrides/ and the runtime merges them at load.
|
||||
*
|
||||
* One file per section, filename = section name (coreLoop, qualityRules,
|
||||
* behavioralRules, workPatterns, intelligenceDefaults).
|
||||
*/
|
||||
|
||||
export type BehavioralSpecSection =
|
||||
| 'coreLoop'
|
||||
| 'qualityRules'
|
||||
| 'behavioralRules'
|
||||
| 'workPatterns'
|
||||
| 'intelligenceDefaults';
|
||||
|
||||
export const BEHAVIORAL_SPEC_SECTIONS: BehavioralSpecSection[] = [
|
||||
'coreLoop',
|
||||
'qualityRules',
|
||||
'behavioralRules',
|
||||
'workPatterns',
|
||||
'intelligenceDefaults',
|
||||
];
|
||||
|
||||
export interface DeployBehavioralSpecInput {
|
||||
/** Which section to override */
|
||||
section: BehavioralSpecSection;
|
||||
/** The evolved text for that section */
|
||||
text: string;
|
||||
/** Optional evolution-run uuid for audit */
|
||||
runUuid?: string;
|
||||
}
|
||||
|
||||
export interface BehavioralSpecOverride {
|
||||
section: BehavioralSpecSection;
|
||||
text: string;
|
||||
runUuid?: string;
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a behavioral-spec section override to disk. The runtime loader
|
||||
* (loadBehavioralSpecOverrides) reads these on startup and merges them
|
||||
* into the compiled BEHAVIORAL_SPEC at runtime.
|
||||
*/
|
||||
export function deployBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
input: DeployBehavioralSpecInput,
|
||||
): DeployResult {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(input.section)) {
|
||||
throw new Error(`Unknown behavioral-spec section: ${input.section}`);
|
||||
}
|
||||
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(dir, `${input.section}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const payload: BehavioralSpecOverride = {
|
||||
section: input.section,
|
||||
text: input.text,
|
||||
runUuid: input.runUuid,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(payload, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: payload.deployedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Roll back one section (restores .bak, or removes if no previous override). */
|
||||
export function rollbackBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
section: BehavioralSpecSection,
|
||||
): boolean {
|
||||
const filePath = path.join(dataDir, 'behavioral-overrides', `${section}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Read all behavioral-spec overrides from disk as a map section → text. */
|
||||
export function loadBehavioralSpecOverrides(
|
||||
dataDir: string,
|
||||
): Partial<Record<BehavioralSpecSection, string>> {
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) return {};
|
||||
|
||||
const out: Partial<Record<BehavioralSpecSection, string>> = {};
|
||||
try {
|
||||
for (const section of BEHAVIORAL_SPEC_SECTIONS) {
|
||||
const filePath = path.join(dir, `${section}.json`);
|
||||
if (!fs.existsSync(filePath)) continue;
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||||
const parsed = JSON.parse(raw) as Partial<BehavioralSpecOverride>;
|
||||
if (parsed && typeof parsed.text === 'string' && parsed.text.length > 0) {
|
||||
out[section] = parsed.text;
|
||||
}
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
} catch { /* dir unreadable */ }
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Merge overrides into a baseline spec object, preferring the override text. */
|
||||
export function applyBehavioralSpecOverrides(
|
||||
baseline: Record<BehavioralSpecSection, string>,
|
||||
overrides: Partial<Record<BehavioralSpecSection, string>>,
|
||||
): Record<BehavioralSpecSection, string> {
|
||||
return {
|
||||
...baseline,
|
||||
...Object.fromEntries(
|
||||
(Object.entries(overrides) as Array<[BehavioralSpecSection, string]>)
|
||||
.filter(([, v]) => typeof v === 'string' && v.length > 0),
|
||||
),
|
||||
} as Record<BehavioralSpecSection, string>;
|
||||
}
|
||||
|
||||
// ── Internal ───────────────────────────────────────────────────
|
||||
|
||||
/** Atomic write — write to .tmp then rename into place. */
|
||||
function writeAtomic(filePath: string, contents: string): void {
|
||||
const tmpPath = `${filePath}.tmp`;
|
||||
fs.writeFileSync(tmpPath, contents, 'utf-8');
|
||||
const waitBuffer = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
try {
|
||||
for (let attempt = 1; attempt <= 10; attempt++) {
|
||||
try {
|
||||
fs.renameSync(tmpPath, filePath);
|
||||
return;
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
const transient = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
|
||||
if (!transient || attempt === 10) throw error;
|
||||
// Windows antivirus and indexers can briefly hold an exclusive handle.
|
||||
Atomics.wait(waitBuffer, 0, 0, 25 * attempt);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
/**
|
||||
* Evolution Gates — Phase 2.2 of the self-evolution loop.
|
||||
*
|
||||
* Gatekeeping for prompts/skills produced by the iterative GEPA optimizer
|
||||
* before they reach production. A candidate must pass ALL gates; the first
|
||||
* failure short-circuits and returns a `fail` verdict with the reason.
|
||||
*
|
||||
* Four gate categories:
|
||||
*
|
||||
* 1. Size gates — hard character caps per target type (persona prompt
|
||||
* ≤ 3 000 chars, tool description ≤ 500, skill ≤ 15 KB,
|
||||
* behavioral-spec section ≤ 4 000).
|
||||
* 2. Growth gates — percentage cap over baseline (default +20%) prevents
|
||||
* runaway prompt bloat across generations.
|
||||
* 3. Structural — non-empty, no unresolved placeholders, balanced
|
||||
* markdown fences, no obvious template accidents
|
||||
* (e.g. "[PLACEHOLDER]", "TODO:" left in).
|
||||
* 4. Regression — score delta on a held-out eval set must not drop
|
||||
* below the allowed tolerance (default -0.02, i.e. a
|
||||
* 2% accuracy regression is the worst allowed).
|
||||
*
|
||||
* Each gate returns structured reasons so the orchestrator (phase 4)
|
||||
* can log WHY a candidate was rejected and surface it in the UI.
|
||||
*/
|
||||
|
||||
import type { EvolutionTarget } from './iterative-optimizer.js';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────
|
||||
|
||||
export type GateVerdict = 'pass' | 'fail';
|
||||
|
||||
export interface GateResult {
|
||||
gate: string;
|
||||
verdict: GateVerdict;
|
||||
reason: string;
|
||||
/** Optional measurable detail: e.g. `{ current: 3520, max: 3000 }` */
|
||||
detail?: Record<string, number | string>;
|
||||
}
|
||||
|
||||
export interface GateCheckResult {
|
||||
verdict: GateVerdict;
|
||||
/** First failing gate, or null if all passed */
|
||||
firstFailure: GateResult | null;
|
||||
/** All gate results in order */
|
||||
results: GateResult[];
|
||||
}
|
||||
|
||||
export interface SizeLimits {
|
||||
personaSystemPrompt: number;
|
||||
toolDescription: number;
|
||||
skillBody: number;
|
||||
behavioralSpecSection: number;
|
||||
generic: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_SIZE_LIMITS: SizeLimits = {
|
||||
personaSystemPrompt: 3_000,
|
||||
toolDescription: 500,
|
||||
skillBody: 15_000,
|
||||
behavioralSpecSection: 4_000,
|
||||
generic: 8_000,
|
||||
};
|
||||
|
||||
export interface GateOptions {
|
||||
/** What is being evolved — drives which size cap applies. Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Max allowed % growth over baseline. Default 0.2 (i.e. +20%). */
|
||||
maxGrowthRatio?: number;
|
||||
/** Largest allowed accuracy regression vs baseline. Default -0.02 (i.e. -2%). */
|
||||
maxRegression?: number;
|
||||
/** Override the default size limits */
|
||||
sizeLimits?: Partial<SizeLimits>;
|
||||
/** Allow empty candidates (rarely useful — default false) */
|
||||
allowEmpty?: boolean;
|
||||
}
|
||||
|
||||
export interface CheckInput {
|
||||
/** The new candidate text */
|
||||
candidate: string;
|
||||
/** The baseline (pre-evolution) text — used for growth + regression checks */
|
||||
baseline: string;
|
||||
/** Optional scores to enable the regression gate */
|
||||
scores?: {
|
||||
baseline: number;
|
||||
candidate: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run all gates in order. Returns a structured verdict. The overall
|
||||
* verdict is `pass` iff every gate passed; `fail` with `firstFailure`
|
||||
* populated otherwise.
|
||||
*
|
||||
* Callers can choose to short-circuit on the first failure or to
|
||||
* collect all of them — `runGates` returns every gate result either way.
|
||||
*/
|
||||
export function runGates(input: CheckInput, options: GateOptions = {}): GateCheckResult {
|
||||
const targetKind = options.targetKind ?? 'generic';
|
||||
const sizeLimits = { ...DEFAULT_SIZE_LIMITS, ...options.sizeLimits };
|
||||
const maxGrowthRatio = options.maxGrowthRatio ?? 0.2;
|
||||
const maxRegression = options.maxRegression ?? -0.02;
|
||||
const allowEmpty = options.allowEmpty ?? false;
|
||||
|
||||
const results: GateResult[] = [];
|
||||
|
||||
// 1. Non-empty
|
||||
if (!allowEmpty) {
|
||||
results.push(checkNonEmpty(input.candidate));
|
||||
}
|
||||
|
||||
// 2. Size
|
||||
results.push(checkSize(input.candidate, targetKind, sizeLimits));
|
||||
|
||||
// 3. Growth
|
||||
results.push(checkGrowth(input.candidate, input.baseline, maxGrowthRatio));
|
||||
|
||||
// 4. Structural integrity
|
||||
results.push(checkBalancedFences(input.candidate));
|
||||
results.push(checkNoPlaceholders(input.candidate));
|
||||
results.push(checkNoObviousTodos(input.candidate));
|
||||
|
||||
// 5. Regression (optional — skipped if scores not provided)
|
||||
if (input.scores) {
|
||||
results.push(checkRegression(input.scores.baseline, input.scores.candidate, maxRegression));
|
||||
}
|
||||
|
||||
const firstFailure = results.find(r => r.verdict === 'fail') ?? null;
|
||||
return {
|
||||
verdict: firstFailure ? 'fail' : 'pass',
|
||||
firstFailure,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Individual gates (exported for unit tests + fine-grained use) ──
|
||||
|
||||
export function checkNonEmpty(candidate: string): GateResult {
|
||||
const trimmed = candidate.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return {
|
||||
gate: 'non-empty',
|
||||
verdict: 'fail',
|
||||
reason: 'Candidate is empty or whitespace only',
|
||||
};
|
||||
}
|
||||
return { gate: 'non-empty', verdict: 'pass', reason: 'non-empty' };
|
||||
}
|
||||
|
||||
export function checkSize(
|
||||
candidate: string,
|
||||
targetKind: EvolutionTarget,
|
||||
limits: SizeLimits,
|
||||
): GateResult {
|
||||
const max = resolveSizeLimit(targetKind, limits);
|
||||
const len = candidate.length;
|
||||
if (len > max) {
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate is ${len} chars but ${targetKind} cap is ${max}`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'pass',
|
||||
reason: `Within ${targetKind} limit (${len}/${max})`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkGrowth(
|
||||
candidate: string,
|
||||
baseline: string,
|
||||
maxGrowthRatio: number,
|
||||
): GateResult {
|
||||
// A candidate shorter than baseline always passes the growth gate.
|
||||
if (candidate.length <= baseline.length) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'Candidate is not larger than baseline',
|
||||
detail: { baseline: baseline.length, candidate: candidate.length },
|
||||
};
|
||||
}
|
||||
|
||||
// Empty baseline means any candidate can't be measured as a ratio —
|
||||
// defer to the size gate instead.
|
||||
if (baseline.length === 0) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'No baseline to compare against',
|
||||
};
|
||||
}
|
||||
|
||||
const ratio = (candidate.length - baseline.length) / baseline.length;
|
||||
if (ratio > maxGrowthRatio) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate grew ${(ratio * 100).toFixed(1)}% over baseline (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: `Grew ${(ratio * 100).toFixed(1)}% (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkBalancedFences(candidate: string): GateResult {
|
||||
const fenceMatches = candidate.match(/```/g);
|
||||
const count = fenceMatches?.length ?? 0;
|
||||
if (count % 2 !== 0) {
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'fail',
|
||||
reason: `Odd number of markdown code fences (${count}) — unbalanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'pass',
|
||||
reason: `${count} fence(s), balanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates that still contain placeholder text like `[TODO]`,
|
||||
* `<placeholder>`, or `{{var}}` — these are almost always unfinished
|
||||
* generations.
|
||||
*/
|
||||
export function checkNoPlaceholders(candidate: string): GateResult {
|
||||
const patterns: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'bracket-placeholder', re: /\[(?:PLACEHOLDER|TODO|FIXME|INSERT|XXX|YOUR[_ ][A-Z ]+)\]/i },
|
||||
{ name: 'angle-placeholder', re: /<(?:placeholder|todo|your[-_ ][a-z ]+)>/i },
|
||||
{ name: 'handlebars-placeholder', re: /\{\{[a-zA-Z_][a-zA-Z0-9_.]*\}\}/ },
|
||||
];
|
||||
|
||||
for (const { name, re } of patterns) {
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-placeholders',
|
||||
verdict: 'fail',
|
||||
reason: `Found unresolved placeholder (${name}): "${m[0]}"`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { gate: 'no-placeholders', verdict: 'pass', reason: 'no unresolved placeholders' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates with "TODO:" or "FIXME:" headers — a prompt shouldn't
|
||||
* ship with unfinished author notes.
|
||||
*/
|
||||
export function checkNoObviousTodos(candidate: string): GateResult {
|
||||
// Only match lines that START with the marker (as a label), not prose
|
||||
// that happens to mention "todo" (e.g. a task-list prompt).
|
||||
const re = /(?:^|\n)\s*(?:TODO|FIXME|XXX)\s*:/i;
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-todos',
|
||||
verdict: 'fail',
|
||||
reason: `Found leftover author marker: "${m[0].trim()}"`,
|
||||
};
|
||||
}
|
||||
return { gate: 'no-todos', verdict: 'pass', reason: 'no leftover author markers' };
|
||||
}
|
||||
|
||||
export function checkRegression(
|
||||
baselineScore: number,
|
||||
candidateScore: number,
|
||||
maxRegression: number,
|
||||
): GateResult {
|
||||
const delta = candidateScore - baselineScore;
|
||||
// Small epsilon avoids float-precision false negatives when delta is
|
||||
// numerically equal to the tolerance (e.g. 0.78 - 0.8 === -0.02000002).
|
||||
const epsilon = 1e-9;
|
||||
if (delta < maxRegression - epsilon) {
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate score regressed by ${(delta * 100).toFixed(2)}pp (allowed ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'pass',
|
||||
reason: `Score delta ${(delta * 100).toFixed(2)}pp (allowed floor ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function resolveSizeLimit(target: EvolutionTarget, limits: SizeLimits): number {
|
||||
switch (target) {
|
||||
case 'persona-system-prompt': return limits.personaSystemPrompt;
|
||||
case 'tool-description': return limits.toolDescription;
|
||||
case 'skill-body': return limits.skillBody;
|
||||
case 'behavioral-spec-section': return limits.behavioralSpecSection;
|
||||
case 'generic':
|
||||
default:
|
||||
return limits.generic;
|
||||
}
|
||||
}
|
||||
@@ -1,43 +1,29 @@
|
||||
/**
|
||||
* Evolution LLM Wiring — binds real LLMs into the self-evolution loop.
|
||||
* LLM wiring for the skill audit and skill hygiene routes.
|
||||
*
|
||||
* The evolution primitives (judge, iterative-optimizer, evolve-schema) are
|
||||
* deliberately model-agnostic: they accept plain callables. This module
|
||||
* supplies production adapters that turn a Haiku-class LLM into those
|
||||
* callables, so the `/api/evolution/run` endpoint can perform a real run.
|
||||
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
|
||||
* This module supplies the production adapter that turns a Haiku-class LLM
|
||||
* into that callable, plus a retry wrapper for transient provider errors.
|
||||
*
|
||||
* Public surface:
|
||||
*
|
||||
* createAnthropicEvolutionLLM(apiKey, model?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax — same pattern as
|
||||
* `optimizer-service.ts`. Dynamic import to avoid startup cost when
|
||||
* evolution is not in use.
|
||||
* createAnthropicEvolutionLLM(apiKey, options?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
|
||||
* avoid startup cost when no audit runs.
|
||||
*
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
* buildGEPAMutateFn(llm) → MutateFn (for IterativeGEPA)
|
||||
* buildSchemaExecuteFn(llm) → SchemaExecuteFn (for EvolveSchema)
|
||||
* makeRunningJudge(base, llm) → Pick<LLMJudge,'score'>
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
*
|
||||
* The `makeRunningJudge` wrapper is what enables GEPA to evaluate real
|
||||
* prompt effectiveness: on each score() call it first executes the
|
||||
* candidate prompt against the example input via the LLM, then delegates
|
||||
* to the wrapped judge with the model's actual response. Without this,
|
||||
* GEPA compares prompt text to expected output directly — less useful.
|
||||
*
|
||||
* All adapters are pure wrappers over `EvolutionLLM.complete()`, which is a
|
||||
* minimal `(prompt: string) => Promise<string>` contract. Tests supply
|
||||
* in-memory mocks of that contract; production code goes through AxAI.
|
||||
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
|
||||
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
|
||||
* mocks of that contract; production code goes through AxAI.
|
||||
*/
|
||||
|
||||
import type { JudgeLLMCall, JudgeInput, JudgeScore } from './judge.js';
|
||||
import type { LLMJudge } from './judge.js';
|
||||
import type { MutateArgs, MutateFn, EvolutionTarget } from './iterative-optimizer.js';
|
||||
import type { SchemaExecuteFn, Schema, SchemaField } from './evolve-schema.js';
|
||||
import type { JudgeLLMCall } from './judge.js';
|
||||
|
||||
// ── Minimal LLM contract ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A minimal single-turn completion contract. All evolution adapters depend
|
||||
* A minimal single-turn completion contract. All adapters depend
|
||||
* on this interface rather than on @ax-llm/ax directly so they stay easy to
|
||||
* mock in unit tests.
|
||||
*/
|
||||
@@ -206,10 +192,10 @@ export interface CreateAnthropicEvolutionLLMOptions {
|
||||
|
||||
/**
|
||||
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
|
||||
* @ax-llm/ax so evolution is a zero-cost dependency until used.
|
||||
* @ax-llm/ax so it is a zero-cost dependency until used.
|
||||
*
|
||||
* Returns null when @ax-llm/ax is unavailable — callers should treat null
|
||||
* as "evolution disabled" and surface a clear error to the user.
|
||||
* as "LLM unavailable" and surface a clear error to the user.
|
||||
*/
|
||||
export async function createAnthropicEvolutionLLM(
|
||||
apiKey: string,
|
||||
@@ -240,8 +226,8 @@ export async function createAnthropicEvolutionLLM(
|
||||
const response = await ai.chat({
|
||||
chatPrompt: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
// chat() may return a ReadableStream in streaming mode — evolution
|
||||
// does not stream, so we only handle the non-stream case.
|
||||
// chat() may return a ReadableStream in streaming mode — audits
|
||||
// do not stream, so we only handle the non-stream case.
|
||||
const typed = response as {
|
||||
results?: readonly { content?: string }[];
|
||||
};
|
||||
@@ -249,7 +235,7 @@ export async function createAnthropicEvolutionLLM(
|
||||
return typeof first === 'string' ? first : '';
|
||||
},
|
||||
};
|
||||
// Apply retry policy — defaults are sized for background evolution runs
|
||||
// Apply retry policy — defaults are sized for background runs
|
||||
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
|
||||
// by passing `retry: { maxAttempts: 1 }`.
|
||||
return wrapWithRetry(inner, options.retry ?? {});
|
||||
@@ -268,217 +254,3 @@ export async function createAnthropicEvolutionLLM(
|
||||
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
|
||||
return (prompt: string) => llm.complete(prompt);
|
||||
}
|
||||
|
||||
// ── Reflective mutation (GEPA) ──────────────────────────────────
|
||||
|
||||
export interface BuildReflectiveMutationPromptArgs {
|
||||
parent: string;
|
||||
strategy: string;
|
||||
weaknessFeedback: string[];
|
||||
targetKind: EvolutionTarget;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reflective mutation prompt for GEPA. Returns the *mutated child*
|
||||
* text only — no commentary, no wrapping — per the instruction in the
|
||||
* prompt body. The LLM is told what to fix, using the weakness signal from
|
||||
* the parent's worst-scoring eval examples.
|
||||
*/
|
||||
export function buildReflectiveMutationPrompt(
|
||||
args: BuildReflectiveMutationPromptArgs,
|
||||
): string {
|
||||
const weaknessBullets = args.weaknessFeedback.length > 0
|
||||
? args.weaknessFeedback.map(fb => `- ${fb}`).join('\n')
|
||||
: '(no specific weakness signals yet)';
|
||||
|
||||
return `You are evolving an AI prompt. This is generation ${args.generation}.
|
||||
|
||||
TARGET KIND: ${args.targetKind}
|
||||
STRATEGY: ${args.strategy}
|
||||
|
||||
PARENT PROMPT (do not return this unchanged — produce a *better* variant):
|
||||
---
|
||||
${args.parent}
|
||||
---
|
||||
|
||||
WEAKNESS SIGNALS from the parent's lowest-scoring eval examples:
|
||||
${weaknessBullets}
|
||||
|
||||
Apply the strategy above to address the weakness signals. Keep the same
|
||||
role and overall structure — do not change the prompt's purpose. Produce
|
||||
a concrete variant that should score higher on the next eval.
|
||||
|
||||
Return ONLY the new prompt text. No preamble, no explanation, no markdown
|
||||
fences. Plain text only.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a GEPA MutateFn that uses an LLM to produce reflective mutations.
|
||||
* Graceful fallback: if the LLM throws or returns empty, returns the
|
||||
* parent prompt unchanged (GEPA will still evaluate it as part of the
|
||||
* Pareto population — no progress that generation, but no crash).
|
||||
*/
|
||||
export function buildGEPAMutateFn(llm: EvolutionLLM): MutateFn {
|
||||
return async (args: MutateArgs): Promise<string> => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: args.parent.prompt,
|
||||
strategy: args.strategy,
|
||||
weaknessFeedback: args.weaknessFeedback,
|
||||
targetKind: args.targetKind,
|
||||
generation: args.generation,
|
||||
});
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(prompt);
|
||||
} catch {
|
||||
return args.parent.prompt;
|
||||
}
|
||||
const cleaned = stripFences(raw).trim();
|
||||
if (cleaned.length === 0) return args.parent.prompt;
|
||||
return cleaned;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Schema execute (ES) ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a schema-filling prompt that asks the LLM to return JSON matching
|
||||
* the given schema for the given user input. Every field is documented by
|
||||
* name + type + description + required flag + constraints.
|
||||
*/
|
||||
export function buildSchemaFillPrompt(args: { schema: Schema; input: string }): string {
|
||||
const fieldLines = args.schema.fields.map(f => formatField(f)).join('\n');
|
||||
return `Fill the JSON schema below using the user input. Respond with
|
||||
valid JSON — no markdown, no prose, no trailing commas.
|
||||
|
||||
SCHEMA (name: ${args.schema.name}, version: ${args.schema.version}):
|
||||
${fieldLines}
|
||||
|
||||
USER INPUT:
|
||||
${args.input}
|
||||
|
||||
Return ONLY the JSON object.`;
|
||||
}
|
||||
|
||||
function formatField(field: SchemaField): string {
|
||||
const req = field.required ? 'required' : 'optional';
|
||||
const constraints = field.constraints.length > 0
|
||||
? ' constraints: ' + field.constraints.map(c => `${c.kind}=${stringifyConstraintValue(c.value)}`).join(', ')
|
||||
: '';
|
||||
return `- "${field.name}" (${field.type}, ${req}): ${field.description}${constraints}`;
|
||||
}
|
||||
|
||||
function stringifyConstraintValue(value: string | number | string[]): string {
|
||||
if (Array.isArray(value)) return `[${value.join(',')}]`;
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a SchemaExecuteFn that asks the LLM to fill a schema. Returns
|
||||
* `parsed: true` only when the response parses as JSON. On any error
|
||||
* (network, timeout, bad model), returns `{actual: '', parsed: false}`.
|
||||
*/
|
||||
export function buildSchemaExecuteFn(llm: EvolutionLLM): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(buildSchemaFillPrompt({ schema, input }));
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
const cleaned = stripFences(raw);
|
||||
return { actual: cleaned, parsed: looksLikeJSON(cleaned) };
|
||||
};
|
||||
}
|
||||
|
||||
// ── Running judge (executes candidate prompt first, then scores) ─
|
||||
|
||||
/**
|
||||
* Wrap a base judge so that each score() call first runs the candidate
|
||||
* prompt through the LLM on the example's input. The base judge then
|
||||
* receives the LLM's output as `actual`, giving a real behavioral signal
|
||||
* instead of a prompt-text-to-expected-output comparison.
|
||||
*
|
||||
* When the LLM call fails, returns a zero score with clear feedback so the
|
||||
* Pareto population treats the candidate as a loser without crashing the
|
||||
* whole generation.
|
||||
*/
|
||||
/**
|
||||
* Phantom property that marks a judge as running-capable (i.e. it executes
|
||||
* the candidate prompt against a real LLM before scoring). IterativeGEPA
|
||||
* checks for this to prevent regressions where a bare prompt-text-similarity
|
||||
* judge is passed by mistake.
|
||||
*
|
||||
* Symbol-keyed to avoid accidental conflicts with user-defined fields.
|
||||
*/
|
||||
export const RUNNING_JUDGE_BRAND = Symbol.for('waggle.runningJudge');
|
||||
|
||||
export interface RunningJudge extends Pick<LLMJudge, 'score'> {
|
||||
readonly [RUNNING_JUDGE_BRAND]: true;
|
||||
}
|
||||
|
||||
export function isRunningJudge(j: Pick<LLMJudge, 'score'> | RunningJudge): j is RunningJudge {
|
||||
return (j as Record<symbol, unknown>)[RUNNING_JUDGE_BRAND] === true;
|
||||
}
|
||||
|
||||
export function makeRunningJudge(
|
||||
baseJudge: Pick<LLMJudge, 'score'>,
|
||||
llm: EvolutionLLM,
|
||||
): RunningJudge {
|
||||
return {
|
||||
[RUNNING_JUDGE_BRAND]: true as const,
|
||||
async score(args: JudgeInput): Promise<JudgeScore> {
|
||||
// `args.actual` is the candidate prompt supplied by GEPA. Treat it as
|
||||
// a system prompt, run it against the example input via the LLM, and
|
||||
// use the LLM's reply as the new `actual` for the downstream judge.
|
||||
const runPrompt = buildRunPrompt(args.actual, args.input);
|
||||
let modelOutput = '';
|
||||
try {
|
||||
modelOutput = await llm.complete(runPrompt);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
overall: 0, weighted: 0,
|
||||
correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1,
|
||||
feedback: `Candidate execution failed: ${msg}`,
|
||||
parsed: false,
|
||||
};
|
||||
}
|
||||
return baseJudge.score({ ...args, actual: modelOutput });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildRunPrompt(candidatePrompt: string, userInput: string): string {
|
||||
return `${candidatePrompt}
|
||||
|
||||
USER INPUT:
|
||||
${userInput}
|
||||
|
||||
Respond now.`;
|
||||
}
|
||||
|
||||
// ── Shared helpers ──────────────────────────────────────────────
|
||||
|
||||
/** Strip ``` and ```json fences if present. Never throws. */
|
||||
function stripFences(raw: string): string {
|
||||
if (!raw) return '';
|
||||
return raw
|
||||
.replace(/```json\s*/gi, '')
|
||||
.replace(/```\s*/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function looksLikeJSON(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return false;
|
||||
try {
|
||||
JSON.parse(trimmed);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
/**
|
||||
* Evolution Orchestrator — Phase 4 of the self-evolution loop.
|
||||
*
|
||||
* Wires the primitives into a closed loop:
|
||||
*
|
||||
* traces → eval dataset → ComposeEvolution → gates → run store
|
||||
* ↓
|
||||
* user accepts/rejects
|
||||
* ↓
|
||||
* deploy callback fires
|
||||
* ↓
|
||||
* runStore.markDeployed()
|
||||
*
|
||||
* The orchestrator itself does NOT hot-reload personas or rewrite the
|
||||
* behavioral spec — it calls a pluggable `deploy` callback supplied by
|
||||
* the caller. This keeps the orchestrator pure (easy to unit-test) and
|
||||
* lets the server/desktop app wire up the real side-effects (file write,
|
||||
* persona reload, event emission) separately.
|
||||
*
|
||||
* Persistence is in @waggle/core's `EvolutionRunStore`. Every run — even
|
||||
* rejected or failed — is preserved for audit.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
EvolutionRun,
|
||||
EvolutionRunStore,
|
||||
EvolutionRunStatus,
|
||||
EvolutionRunTarget,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
} from '@waggle/core';
|
||||
import { ComposeEvolution, type ComposeEvolutionOptions, type ComposeEvolutionResult } from './compose-evolution.js';
|
||||
import { runGates, type GateResult, type GateOptions } from './evolution-gates.js';
|
||||
import { EvalDatasetBuilder, type EvalExample } from './eval-dataset.js';
|
||||
|
||||
// ── Dependency injection types ─────────────────────────────────
|
||||
|
||||
export interface EvolutionOrchestratorDeps {
|
||||
/** Source of traces (from @waggle/core) */
|
||||
traceStore: ExecutionTraceStore;
|
||||
/** Persistent run history (from @waggle/core) */
|
||||
runStore: EvolutionRunStore;
|
||||
/**
|
||||
* Function invoked when a run moves from accepted → (deployed | failed).
|
||||
* Returning normally marks the run deployed; throwing marks it failed
|
||||
* with the error message as reason.
|
||||
*
|
||||
* This is where the server writes to the persona file, reloads the
|
||||
* orchestrator cache, updates behavioral-spec.ts, etc.
|
||||
*/
|
||||
deploy?: (run: EvolutionRun) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface EvolutionAutoTriggerConfig {
|
||||
/** Minimum traces (with the given outcome filter) before a run is triggered */
|
||||
minTraces: number;
|
||||
/** Restrict trigger to a specific persona / workspace / taskShape */
|
||||
traceFilter?: {
|
||||
personaId?: string;
|
||||
workspaceId?: string;
|
||||
taskShape?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EvolutionOrchestratorOptions {
|
||||
/** What is being evolved (drives gate policy + persistence tag) */
|
||||
targetKind: EvolutionRunTarget;
|
||||
/** Stable identifier — e.g. persona id, spec-section name */
|
||||
targetName?: string;
|
||||
/** The current baseline text to evolve against */
|
||||
baseline: string;
|
||||
/** Compose options — already wired with examples, judge, execute, mutate */
|
||||
compose: Omit<ComposeEvolutionOptions, 'schema' | 'instructions'> & {
|
||||
schema: Omit<ComposeEvolutionOptions['schema'], 'baseline'>;
|
||||
instructions: Omit<ComposeEvolutionOptions['instructions'], 'baseline'>;
|
||||
};
|
||||
/** Optional gate overrides (size caps, growth ratio, etc) */
|
||||
gateOptions?: GateOptions;
|
||||
/** Improvement threshold required to create a proposal (default 0.02) */
|
||||
minDelta?: number;
|
||||
/** Auto-trigger configuration — when omitted, runOnce() is manual-only */
|
||||
autoTrigger?: EvolutionAutoTriggerConfig;
|
||||
/** Abort the run early */
|
||||
signal?: AbortSignal;
|
||||
/** Optional progress pass-through */
|
||||
onProgress?: (event: EvolutionProgress) => void;
|
||||
}
|
||||
|
||||
export interface EvolutionProgress {
|
||||
phase: 'trigger-check' | 'dataset' | 'compose' | 'gates' | 'persist' | 'skipped' | 'done';
|
||||
message?: string;
|
||||
detail?: unknown;
|
||||
}
|
||||
|
||||
// ── Schema baseline input ──────────────────────────────────────
|
||||
|
||||
/** Optional structural baseline when the target has one (e.g. persona's DSPy signature). */
|
||||
export interface SchemaBaselineInput {
|
||||
baseline: ComposeEvolutionOptions['schema']['baseline'];
|
||||
}
|
||||
|
||||
// ── Run result ─────────────────────────────────────────────────
|
||||
|
||||
export type OrchestratorOutcome =
|
||||
| 'proposed' // Run created, awaiting accept/reject
|
||||
| 'skipped-trigger'
|
||||
| 'skipped-gates' // Completed but gates failed
|
||||
| 'skipped-delta' // Improvement below minDelta
|
||||
| 'aborted';
|
||||
|
||||
export interface OrchestratorRunResult {
|
||||
outcome: OrchestratorOutcome;
|
||||
/** Populated for 'proposed' and 'skipped-gates' */
|
||||
run?: EvolutionRun;
|
||||
/** Raw ComposeEvolution result when the compose stage ran */
|
||||
compose?: ComposeEvolutionResult;
|
||||
/** Gate results for audit */
|
||||
gateResults?: GateResult[];
|
||||
/** Reason if outcome is skipped/aborted */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Orchestrator ──────────────────────────────────────────────
|
||||
|
||||
export class EvolutionOrchestrator {
|
||||
private deps: EvolutionOrchestratorDeps;
|
||||
|
||||
constructor(deps: EvolutionOrchestratorDeps) {
|
||||
this.deps = deps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full orchestration pipeline once. Returns structured outcome.
|
||||
* Does NOT deploy — that happens when the caller calls `accept(uuid)`.
|
||||
*/
|
||||
async runOnce(options: EvolutionOrchestratorOptions & {
|
||||
schemaBaseline: SchemaBaselineInput['baseline'];
|
||||
}): Promise<OrchestratorRunResult> {
|
||||
const emit = (phase: EvolutionProgress['phase'], message?: string, detail?: unknown) => {
|
||||
options.onProgress?.({ phase, message, detail });
|
||||
};
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted before start' };
|
||||
}
|
||||
|
||||
// Trigger check — skip if auto-trigger thresholds aren't met.
|
||||
if (options.autoTrigger) {
|
||||
emit('trigger-check');
|
||||
const ok = this.meetsAutoTrigger(options.autoTrigger);
|
||||
if (!ok) {
|
||||
emit('skipped');
|
||||
return {
|
||||
outcome: 'skipped-trigger',
|
||||
reason: `fewer than ${options.autoTrigger.minTraces} eligible traces`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Derive the eval dataset from finalized traces of the requested target.
|
||||
emit('dataset');
|
||||
const examples = this.buildExamplesFromTraces(options);
|
||||
if (examples.length === 0) {
|
||||
emit('skipped', 'no eval examples');
|
||||
return { outcome: 'skipped-trigger', reason: 'no eligible traces to form dataset' };
|
||||
}
|
||||
|
||||
// Run the compose pipeline.
|
||||
emit('compose');
|
||||
const composeResult = await new ComposeEvolution().run({
|
||||
schema: { ...options.compose.schema, baseline: options.schemaBaseline, examples },
|
||||
instructions: { ...options.compose.instructions, baseline: options.baseline, examples },
|
||||
feedbackFilter: options.compose.feedbackFilter,
|
||||
signal: options.signal,
|
||||
onProgress: (e) => emit('compose', undefined, e),
|
||||
});
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted during compose', compose: composeResult };
|
||||
}
|
||||
|
||||
const winnerText = composeResult.instructions.winner.prompt;
|
||||
// Apples-to-apples: compare the GEPA baseline candidate (history[0])
|
||||
// to the GEPA winner. Both are scored by the same judge on the same
|
||||
// eval stage, so the delta is a clean signal.
|
||||
const baselineOverall = composeResult.instructions.history[0]?.score?.overall ?? 0;
|
||||
const winnerOverall = composeResult.instructions.winner.score?.overall ?? 0;
|
||||
const delta = winnerOverall - baselineOverall;
|
||||
const minDelta = options.minDelta ?? 0.02;
|
||||
|
||||
// Gate check.
|
||||
emit('gates');
|
||||
const gateResult = runGates(
|
||||
{
|
||||
candidate: winnerText,
|
||||
baseline: options.baseline,
|
||||
scores: { baseline: baselineOverall, candidate: winnerOverall },
|
||||
},
|
||||
{
|
||||
targetKind: options.targetKind,
|
||||
...options.gateOptions,
|
||||
},
|
||||
);
|
||||
|
||||
if (delta < minDelta) {
|
||||
emit('skipped', `delta ${(delta * 100).toFixed(2)}pp below minimum ${(minDelta * 100).toFixed(2)}pp`);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-delta',
|
||||
reason: `winner improved only ${(delta * 100).toFixed(2)}pp (min ${(minDelta * 100).toFixed(2)}pp required)`,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
// Persist — create the proposed run even if gates failed, so there's audit.
|
||||
emit('persist');
|
||||
const run = this.deps.runStore.create({
|
||||
targetKind: options.targetKind,
|
||||
targetName: options.targetName ?? null,
|
||||
baselineText: options.baseline,
|
||||
winnerText,
|
||||
winnerSchema: composeResult.frozenSchema,
|
||||
deltaAccuracy: delta,
|
||||
gateVerdict: gateResult.verdict,
|
||||
gateReasons: gateResult.results.map(r => ({
|
||||
gate: r.gate, verdict: r.verdict, reason: r.reason,
|
||||
})),
|
||||
artifacts: {
|
||||
runSeed: options.compose.instructions.seed ?? null,
|
||||
generations: options.compose.instructions.generations ?? null,
|
||||
paretoFrontSize: composeResult.schema.paretoFront.length,
|
||||
exampleCount: examples.length,
|
||||
},
|
||||
});
|
||||
|
||||
if (gateResult.verdict === 'fail') {
|
||||
// Immediately reject runs that fail gates so the user doesn't see
|
||||
// dangerous candidates in the "review" queue. Run history is kept.
|
||||
const rejected = this.deps.runStore.reject(
|
||||
run.run_uuid,
|
||||
`gate failure: ${gateResult.firstFailure?.reason ?? 'unknown'}`,
|
||||
);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-gates',
|
||||
reason: gateResult.firstFailure?.reason ?? 'gate failure',
|
||||
run: rejected ?? run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'proposed',
|
||||
run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
/** Accept a proposed run and invoke the deploy callback. */
|
||||
async accept(runUuid: string, userNote?: string): Promise<EvolutionRun | undefined> {
|
||||
const accepted = this.deps.runStore.accept(runUuid, userNote);
|
||||
if (!accepted || accepted.status !== 'accepted') return accepted;
|
||||
|
||||
if (!this.deps.deploy) {
|
||||
// No deploy hook configured — leave as 'accepted' and let the caller
|
||||
// mark deployed manually.
|
||||
return accepted;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.deps.deploy(accepted);
|
||||
return this.deps.runStore.markDeployed(runUuid);
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
return this.deps.runStore.markFailed(runUuid, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reject a proposed run with optional reason. */
|
||||
reject(runUuid: string, reason?: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.reject(runUuid, reason);
|
||||
}
|
||||
|
||||
/** List runs (thin pass-through, handy for the UI). */
|
||||
list(filter?: Parameters<EvolutionRunStore['list']>[0]): EvolutionRun[] {
|
||||
return this.deps.runStore.list(filter);
|
||||
}
|
||||
|
||||
/** Get a single run by uuid. */
|
||||
get(runUuid: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.getByUuid(runUuid);
|
||||
}
|
||||
|
||||
// ── Internals ───────────────────────────────────────────────
|
||||
|
||||
private meetsAutoTrigger(trigger: EvolutionAutoTriggerConfig): boolean {
|
||||
const outcomes: TraceOutcome[] = ['success', 'verified', 'corrected'];
|
||||
const traces = this.deps.traceStore.query({
|
||||
...(trigger.traceFilter ?? {}),
|
||||
outcome: outcomes,
|
||||
limit: Math.max(trigger.minTraces * 2, 50),
|
||||
});
|
||||
return traces.length >= trigger.minTraces;
|
||||
}
|
||||
|
||||
private buildExamplesFromTraces(
|
||||
options: EvolutionOrchestratorOptions,
|
||||
): EvalExample[] {
|
||||
// If the caller already supplied examples via compose.schema/instructions,
|
||||
// we respect that. Otherwise we mine them from the trace store.
|
||||
const existing = options.compose.schema.examples ?? options.compose.instructions.examples;
|
||||
if (existing && existing.length > 0) return existing;
|
||||
|
||||
const builder = new EvalDatasetBuilder(this.deps.traceStore);
|
||||
const traceFilter = options.autoTrigger?.traceFilter ?? {};
|
||||
const traces = builder.sourceFromTraces(['success', 'verified'], true, {
|
||||
...traceFilter,
|
||||
limit: 500,
|
||||
});
|
||||
return traces;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Convenience helpers ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Filter a list of parsed traces down to those likely useful for evolution.
|
||||
* Keeps finalized (non-pending) traces with non-empty I/O. Used by callers
|
||||
* that want to manually pre-filter before handing to the orchestrator.
|
||||
*/
|
||||
export function eligibleForEvolution(traces: ParsedExecutionTrace[]): ParsedExecutionTrace[] {
|
||||
return traces.filter(t =>
|
||||
t.outcome !== 'pending' &&
|
||||
t.payload.input.trim().length > 0 &&
|
||||
(t.payload.output.trim().length > 0 || t.payload.correctionFeedback),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper that summarizes runs by status/target for dashboard display.
|
||||
*/
|
||||
export function summarizeRuns(runs: EvolutionRun[]): {
|
||||
total: number;
|
||||
byStatus: Record<EvolutionRunStatus, number>;
|
||||
byTargetKind: Record<string, number>;
|
||||
bestDelta: number;
|
||||
} {
|
||||
const byStatus: Record<EvolutionRunStatus, number> = {
|
||||
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
|
||||
};
|
||||
const byTargetKind: Record<string, number> = {};
|
||||
let bestDelta = 0;
|
||||
|
||||
for (const r of runs) {
|
||||
byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
|
||||
byTargetKind[r.target_kind] = (byTargetKind[r.target_kind] ?? 0) + 1;
|
||||
if (r.delta_accuracy > bestDelta) bestDelta = r.delta_accuracy;
|
||||
}
|
||||
|
||||
return {
|
||||
total: runs.length,
|
||||
byStatus,
|
||||
byTargetKind,
|
||||
bestDelta,
|
||||
};
|
||||
}
|
||||
@@ -1,852 +0,0 @@
|
||||
/**
|
||||
* EvolveSchema — Phase 3.1 of the self-evolution loop.
|
||||
*
|
||||
* Port of Mikhail Pavlukhin's EvolveSchema algorithm (Egzakta research).
|
||||
* Evolves the STRUCTURE of a typed output schema (DSPy-style signatures):
|
||||
* field names, types, descriptions, ordering, and constraints. This is
|
||||
* orthogonal to the GEPA optimizer which evolves INSTRUCTIONS against a
|
||||
* fixed schema.
|
||||
*
|
||||
* Three-phase per-generation loop:
|
||||
*
|
||||
* Phase A — Structure Discovery
|
||||
* Spawn variants that add, replace, or drop output fields. This is
|
||||
* the single biggest-impact mutation class (Mikhail's paper shows
|
||||
* ~74% of HotPotQA gains come from one structural mutation).
|
||||
*
|
||||
* Phase B — Field-Order Probes
|
||||
* Permute field order. LLMs condition on earlier fields when filling
|
||||
* later ones, so putting a "reasoning" field BEFORE "answer" routinely
|
||||
* outperforms the reverse.
|
||||
*
|
||||
* Phase C — Failure-Driven Refinement
|
||||
* Use the per-example judge feedback from the previous generation to
|
||||
* target specific fields — edit their descriptions, tighten / loosen
|
||||
* their constraints, or change their types where signals suggest.
|
||||
*
|
||||
* 8 typed mutations are available:
|
||||
* add_output_field, remove_field, edit_field_desc, change_field_type,
|
||||
* add_constraint, remove_constraint, reorder_fields, replace_output_fields
|
||||
*
|
||||
* Pareto selection is 2-dimensional: (accuracy, -complexity) — an
|
||||
* accurate-but-complex schema is not automatically preferred over a
|
||||
* slightly-less-accurate simpler one.
|
||||
*
|
||||
* Callers provide:
|
||||
* - `execute(schema, input) → actualOutput` the LLM runner
|
||||
* - `judge.score({input, expected, actual})` the LLMJudge from Phase 1.3
|
||||
* - `mutate` (optional) a custom LLM-driven mutator
|
||||
* for edit_field_desc; falls
|
||||
* back to a deterministic
|
||||
* rewriter when omitted.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
|
||||
// ── Schema types ───────────────────────────────────────────────
|
||||
|
||||
export type FieldType = 'string' | 'number' | 'boolean' | 'array' | 'object' | 'enum';
|
||||
|
||||
export interface FieldConstraint {
|
||||
kind: 'minLength' | 'maxLength' | 'pattern' | 'enum' | 'range' | 'custom';
|
||||
value: string | number | string[];
|
||||
}
|
||||
|
||||
export interface SchemaField {
|
||||
name: string;
|
||||
type: FieldType;
|
||||
description: string;
|
||||
required: boolean;
|
||||
constraints: FieldConstraint[];
|
||||
}
|
||||
|
||||
export interface Schema {
|
||||
name: string;
|
||||
fields: SchemaField[];
|
||||
version: number;
|
||||
}
|
||||
|
||||
// ── Mutation types ─────────────────────────────────────────────
|
||||
|
||||
export type MutationKind =
|
||||
| 'add_output_field'
|
||||
| 'remove_field'
|
||||
| 'edit_field_desc'
|
||||
| 'change_field_type'
|
||||
| 'add_constraint'
|
||||
| 'remove_constraint'
|
||||
| 'reorder_fields'
|
||||
| 'replace_output_fields';
|
||||
|
||||
export interface Mutation {
|
||||
kind: MutationKind;
|
||||
/** What changed, for logging + feedback */
|
||||
description: string;
|
||||
/** Functional transformer (pure) */
|
||||
apply: (schema: Schema) => Schema;
|
||||
}
|
||||
|
||||
// ── Candidate + result types ──────────────────────────────────
|
||||
|
||||
export interface SchemaCandidate {
|
||||
id: string;
|
||||
schema: Schema;
|
||||
generation: number;
|
||||
parent: string | null;
|
||||
mutation: MutationKind | 'baseline';
|
||||
mutationLabel: string;
|
||||
score: SchemaCandidateScore | null;
|
||||
perExample: SchemaExampleResult[];
|
||||
}
|
||||
|
||||
export interface SchemaExampleResult {
|
||||
input: string;
|
||||
expected: string;
|
||||
actual: string;
|
||||
score: JudgeScore;
|
||||
/** Diagnostic: did the output parse against the schema? */
|
||||
parsed: boolean;
|
||||
}
|
||||
|
||||
export interface SchemaCandidateScore {
|
||||
/** Mean overall judge score across the eval set (0..1) */
|
||||
accuracy: number;
|
||||
/** Schema "complexity" — a scalar that grows with fields, descriptions, constraints */
|
||||
complexity: number;
|
||||
/** Share of examples that produced parseable output against this schema (0..1) */
|
||||
parseRate: number;
|
||||
/** Weakness feedback taken from the worst-scoring 3 examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Number of examples scored */
|
||||
n: number;
|
||||
}
|
||||
|
||||
export type SchemaExecuteFn = (args: {
|
||||
schema: Schema;
|
||||
input: string;
|
||||
}) => Promise<{ actual: string; parsed: boolean }>;
|
||||
|
||||
export interface EvolveSchemaOptions {
|
||||
/** Starting schema */
|
||||
baseline: Schema;
|
||||
/** Eval dataset */
|
||||
examples: EvalExample[];
|
||||
/** LLM runner that takes a schema + input and returns an attempted output */
|
||||
execute: SchemaExecuteFn;
|
||||
/** Judge for scoring individual attempts */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Candidates per generation (default 5) */
|
||||
populationSize?: number;
|
||||
/** Generations (default 3) */
|
||||
generations?: number;
|
||||
/** Mix of phase mutations per generation */
|
||||
mutationMix?: { structure: number; order: number; refinement: number };
|
||||
/** Eval sample size per generation (default 32) */
|
||||
evalSize?: number;
|
||||
/** Anchor eval size at the end (default 100) */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for sampling (default 1) */
|
||||
seed?: number;
|
||||
/** Optional LLM helper for description edits — deterministic fallback used if omitted */
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>;
|
||||
/** Optional progress emitter */
|
||||
onProgress?: (event: EvolveSchemaProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaProgress {
|
||||
phase: 'start' | 'structure' | 'order' | 'refinement' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
bestAccuracy: number;
|
||||
bestComplexity: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaResult {
|
||||
winner: SchemaCandidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: SchemaCandidate[];
|
||||
/** All candidates across all generations */
|
||||
history: SchemaCandidate[];
|
||||
/** Accuracy delta: winner.accuracy - baseline.accuracy */
|
||||
deltaAccuracy: number;
|
||||
/** True if the winner's accuracy improved over baseline */
|
||||
improved: boolean;
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ───────────────────────────────────
|
||||
|
||||
export function addOutputField(
|
||||
schema: Schema,
|
||||
field: SchemaField,
|
||||
position?: number,
|
||||
): Schema {
|
||||
const pos = position ?? schema.fields.length;
|
||||
const fields = [...schema.fields];
|
||||
fields.splice(pos, 0, { ...field, constraints: [...field.constraints] });
|
||||
return { ...schema, fields, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function removeField(schema: Schema, fieldName: string): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.filter(f => f.name !== fieldName),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function editFieldDescription(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newDescription: string,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, description: newDescription } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function changeFieldType(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newType: FieldType,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, type: newType } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function addConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraint: FieldConstraint,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, constraints: [...f.constraints, constraint] } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function removeConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraintIndex: number,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f => {
|
||||
if (f.name !== fieldName) return f;
|
||||
const constraints = f.constraints.filter((_, i) => i !== constraintIndex);
|
||||
return { ...f, constraints };
|
||||
}),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function reorderFields(schema: Schema, newOrder: string[]): Schema {
|
||||
const byName = new Map(schema.fields.map(f => [f.name, f]));
|
||||
const reordered: SchemaField[] = [];
|
||||
for (const name of newOrder) {
|
||||
const field = byName.get(name);
|
||||
if (field) reordered.push(field);
|
||||
}
|
||||
// Append any fields that were omitted from newOrder (keeps their position stable).
|
||||
for (const field of schema.fields) {
|
||||
if (!newOrder.includes(field.name)) reordered.push(field);
|
||||
}
|
||||
return { ...schema, fields: reordered, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function replaceOutputFields(schema: Schema, newFields: SchemaField[]): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: newFields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Complexity + scoring helpers ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Scalar complexity measure: 1 per field + 0.3 per constraint +
|
||||
* 0.01 per character of description. Calibrated so a 3-field schema
|
||||
* with short descriptions ≈ 3.6, a 10-field schema with long
|
||||
* descriptions ≈ 15+.
|
||||
*/
|
||||
export function schemaComplexity(schema: Schema): number {
|
||||
let c = 0;
|
||||
for (const field of schema.fields) {
|
||||
c += 1;
|
||||
c += field.constraints.length * 0.3;
|
||||
c += (field.description?.length ?? 0) * 0.01;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
export function aggregateSchemaScores(results: SchemaExampleResult[]): SchemaCandidateScore {
|
||||
if (results.length === 0) {
|
||||
return {
|
||||
accuracy: 0, complexity: 0, parseRate: 0, weaknessFeedback: [], n: 0,
|
||||
};
|
||||
}
|
||||
const n = results.length;
|
||||
const accuracy = results.reduce((s, r) => s + r.score.overall, 0) / n;
|
||||
const parseRate = results.filter(r => r.parsed).length / n;
|
||||
const worst = [...results].sort((a, b) => a.score.overall - b.score.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(r => r.score.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
return { accuracy, complexity: 0, parseRate, weaknessFeedback, n };
|
||||
}
|
||||
|
||||
// ── Pareto (2D) ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front on (accuracy↑, complexity↓). A candidate dominates
|
||||
* another iff it is at least as accurate AND no more complex, and
|
||||
* strictly better on at least one dimension.
|
||||
*/
|
||||
export function paretoFrontSchema(candidates: SchemaCandidate[]): SchemaCandidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: SchemaCandidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominatesSchema(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominatesSchema(a: SchemaCandidateScore, b: SchemaCandidateScore): boolean {
|
||||
const geAll = a.accuracy >= b.accuracy && a.complexity <= b.complexity;
|
||||
const gtAny = a.accuracy > b.accuracy || a.complexity < b.complexity;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
// ── Candidate scoring ─────────────────────────────────────────
|
||||
|
||||
export async function scoreSchemaCandidate(
|
||||
candidate: SchemaCandidate,
|
||||
examples: EvalExample[],
|
||||
execute: SchemaExecuteFn,
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<SchemaCandidateScore> {
|
||||
const results: SchemaExampleResult[] = [];
|
||||
for (const ex of examples) {
|
||||
if (signal?.aborted) break;
|
||||
try {
|
||||
const { actual, parsed } = await execute({ schema: candidate.schema, input: ex.input });
|
||||
const score = await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual,
|
||||
});
|
||||
results.push({ input: ex.input, expected: ex.expected_output, actual, score, parsed });
|
||||
} catch {
|
||||
// Keep iterating — one broken example shouldn't poison the whole batch.
|
||||
}
|
||||
}
|
||||
const agg = aggregateSchemaScores(results);
|
||||
agg.complexity = schemaComplexity(candidate.schema);
|
||||
candidate.perExample = results;
|
||||
candidate.score = agg;
|
||||
return agg;
|
||||
}
|
||||
|
||||
// ── Mutation generators per phase ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Structure-Discovery mutations: add a new field, replace fields wholesale,
|
||||
* or drop one if the schema is too large. Returns up to `n` mutations.
|
||||
*/
|
||||
export function generateStructureMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Add a reasoning field if absent.
|
||||
if (!parent.fields.some(f => /reason|thought|scratch/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add reasoning field before answer',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Concise chain-of-thought explaining how the answer was derived.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
}, 0),
|
||||
});
|
||||
}
|
||||
|
||||
// Add a confidence field if absent.
|
||||
if (!parent.fields.some(f => /confidence|certainty|score/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add confidence field at end',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'confidence',
|
||||
type: 'number',
|
||||
description: 'Self-estimated confidence in the answer (0.0 - 1.0).',
|
||||
required: false,
|
||||
constraints: [{ kind: 'range', value: '0..1' }],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// Drop the lowest-priority field if > 4 fields.
|
||||
if (parent.fields.length > 4) {
|
||||
const candidate = parent.fields[parent.fields.length - 1];
|
||||
out.push({
|
||||
kind: 'remove_field',
|
||||
description: `drop trailing field "${candidate.name}"`,
|
||||
apply: (s) => removeField(s, candidate.name),
|
||||
});
|
||||
}
|
||||
|
||||
// Replace all output fields with a minimal skeleton (reasoning + answer).
|
||||
if (parent.fields.length >= 2) {
|
||||
out.push({
|
||||
kind: 'replace_output_fields',
|
||||
description: 'collapse to {reasoning, answer}',
|
||||
apply: (s) =>
|
||||
replaceOutputFields(s, [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Brief chain-of-thought.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'Final answer.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order-Probe mutations: generate permutations of field order. Limits
|
||||
* output to `n` by random sampling from the set of useful permutations.
|
||||
*/
|
||||
export function generateOrderMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const names = parent.fields.map(f => f.name);
|
||||
if (names.length < 2) return [];
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Heuristic 1: move any reasoning/thought field to the front.
|
||||
const reasoningIdx = parent.fields.findIndex(f => /reason|thought|scratch/i.test(f.name));
|
||||
if (reasoningIdx > 0) {
|
||||
const newOrder = [names[reasoningIdx], ...names.filter((_, i) => i !== reasoningIdx)];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[reasoningIdx]}" to front`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Heuristic 2: move any confidence/score field to the end.
|
||||
const confIdx = parent.fields.findIndex(f => /confidence|certainty/i.test(f.name));
|
||||
if (confIdx >= 0 && confIdx !== parent.fields.length - 1) {
|
||||
const newOrder = [...names.filter((_, i) => i !== confIdx), names[confIdx]];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[confIdx]}" to end`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Fill remaining with random permutations.
|
||||
while (out.length < n) {
|
||||
const shuffled = shuffle([...names], rng);
|
||||
if (!arraysEqual(shuffled, names)) {
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `reorder to [${shuffled.join(', ')}]`,
|
||||
apply: (s) => reorderFields(s, shuffled),
|
||||
});
|
||||
}
|
||||
if (out.length > n * 3) break; // safety — very short schemas
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refinement mutations driven by worst-scoring example feedback.
|
||||
* Edits descriptions, tightens constraints, or switches types based on
|
||||
* heuristic patterns in the feedback.
|
||||
*/
|
||||
export async function generateRefinementMutations(
|
||||
parent: Schema,
|
||||
weakness: string[],
|
||||
n: number,
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>,
|
||||
): Promise<Mutation[]> {
|
||||
const out: Mutation[] = [];
|
||||
const joined = weakness.join(' ').toLowerCase();
|
||||
|
||||
// Heuristic: if feedback mentions "too verbose" / "long" — add a maxLength to any string field.
|
||||
if (/verbose|too long|wordy/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `tighten "${f.name}" with maxLength: 200`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'maxLength', value: 200 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "too brief" / "incomplete" — add minLength.
|
||||
if (/brief|incomplete|missing|short/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `require "${f.name}" minLength: 20`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'minLength', value: 20 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "wrong format" / "parse" — clarify description.
|
||||
if (/format|parse|wrong type/i.test(joined)) {
|
||||
for (const field of parent.fields.slice(0, 2)) {
|
||||
const newDesc = editFieldDescription
|
||||
? await editFieldDescription({ field, feedback: weakness }).catch(() => field.description)
|
||||
: deterministicClarifyDescription(field);
|
||||
if (newDesc && newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: always consider one description rewrite on the first field.
|
||||
if (out.length === 0 && parent.fields.length > 0) {
|
||||
const field = parent.fields[0];
|
||||
const newDesc = deterministicClarifyDescription(field);
|
||||
if (newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out.slice(0, n);
|
||||
}
|
||||
|
||||
// ── EvolveSchema engine ──────────────────────────────────────
|
||||
|
||||
export class EvolveSchema {
|
||||
async run(options: EvolveSchemaOptions): Promise<EvolveSchemaResult> {
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
const baseline: SchemaCandidate = {
|
||||
id: 'g0-baseline',
|
||||
schema: cloneSchema(config.baseline),
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emitProgress(config, 'start', 0, 1, 0, 0, 'scoring baseline');
|
||||
const evalSample = pickSample(config.examples, config.evalSize, rng);
|
||||
await scoreSchemaCandidate(baseline, evalSample, config.execute, config.judge, config.signal);
|
||||
|
||||
const history: SchemaCandidate[] = [baseline];
|
||||
let survivors: SchemaCandidate[] = [baseline];
|
||||
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const topParent = pickTopByAccuracy(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
const children: SchemaCandidate[] = [];
|
||||
|
||||
// Phase A — Structure Discovery
|
||||
emitProgress(
|
||||
config, 'structure', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const structureMutations = generateStructureMutations(
|
||||
topParent.schema, config.mutationMix.structure, rng,
|
||||
);
|
||||
children.push(...applyMutations(structureMutations, topParent, gen));
|
||||
|
||||
// Phase B — Field-Order Probes
|
||||
emitProgress(
|
||||
config, 'order', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const orderMutations = generateOrderMutations(
|
||||
topParent.schema, config.mutationMix.order, rng,
|
||||
);
|
||||
children.push(...applyMutations(orderMutations, topParent, gen));
|
||||
|
||||
// Phase C — Failure-Driven Refinement
|
||||
emitProgress(
|
||||
config, 'refinement', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const refinementMutations = await generateRefinementMutations(
|
||||
topParent.schema,
|
||||
topParent.score?.weaknessFeedback ?? [],
|
||||
config.mutationMix.refinement,
|
||||
config.editFieldDescription,
|
||||
);
|
||||
children.push(...applyMutations(refinementMutations, topParent, gen));
|
||||
|
||||
// Score all children on the per-generation sample.
|
||||
const genSample = pickSample(config.examples, config.evalSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(child, genSample, config.execute, config.judge, config.signal);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// Pareto-select survivors: parent + children.
|
||||
survivors = paretoFrontSchema([topParent, ...children]);
|
||||
if (survivors.length > config.populationSize) {
|
||||
// Trim by picking the best-accuracy members.
|
||||
survivors = survivors
|
||||
.slice()
|
||||
.sort((a, b) => (b.score?.accuracy ?? 0) - (a.score?.accuracy ?? 0))
|
||||
.slice(0, config.populationSize);
|
||||
}
|
||||
}
|
||||
|
||||
// Anchor stage: larger eval on final survivors.
|
||||
emitProgress(
|
||||
config, 'anchor', config.generations, survivors.length,
|
||||
bestAccuracy(survivors), minComplexity(survivors),
|
||||
);
|
||||
const anchorSample = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(cand, anchorSample, config.execute, config.judge, config.signal);
|
||||
}
|
||||
|
||||
const paretoFront = paretoFrontSchema(survivors);
|
||||
const winner = pickSchemaWinner(paretoFront);
|
||||
const deltaAccuracy = (winner.score?.accuracy ?? 0) - (baseline.score?.accuracy ?? 0);
|
||||
|
||||
emitProgress(
|
||||
config, 'done', config.generations, paretoFront.length,
|
||||
winner.score?.accuracy ?? 0, winner.score?.complexity ?? 0,
|
||||
);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront,
|
||||
history,
|
||||
deltaAccuracy,
|
||||
improved: deltaAccuracy > 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
function applyMutations(
|
||||
mutations: Mutation[],
|
||||
parent: SchemaCandidate,
|
||||
generation: number,
|
||||
): SchemaCandidate[] {
|
||||
return mutations.map((m, i) => ({
|
||||
id: `g${generation}-${m.kind}-${i}`,
|
||||
schema: m.apply(parent.schema),
|
||||
generation,
|
||||
parent: parent.id,
|
||||
mutation: m.kind,
|
||||
mutationLabel: m.description,
|
||||
score: null,
|
||||
perExample: [],
|
||||
}));
|
||||
}
|
||||
|
||||
function pickTopByAccuracy(candidates: SchemaCandidate[]): SchemaCandidate | undefined {
|
||||
if (candidates.length === 0) return undefined;
|
||||
return candidates.reduce((a, b) =>
|
||||
(a.score?.accuracy ?? 0) >= (b.score?.accuracy ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
export function pickSchemaWinner(candidates: SchemaCandidate[]): SchemaCandidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickSchemaWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const bA = best.score?.accuracy ?? -1;
|
||||
const cA = cand.score?.accuracy ?? -1;
|
||||
if (cA > bA) return cand;
|
||||
if (cA < bA) return best;
|
||||
// Tie on accuracy → prefer lower complexity.
|
||||
const bC = best.score?.complexity ?? Infinity;
|
||||
const cC = cand.score?.complexity ?? Infinity;
|
||||
return cC < bC ? cand : best;
|
||||
});
|
||||
}
|
||||
|
||||
function bestAccuracy(cs: SchemaCandidate[]): number {
|
||||
if (cs.length === 0) return 0;
|
||||
return Math.max(...cs.map(c => c.score?.accuracy ?? 0));
|
||||
}
|
||||
|
||||
function minComplexity(cs: SchemaCandidate[]): number {
|
||||
const scored = cs.filter(c => c.score !== null);
|
||||
if (scored.length === 0) return 0;
|
||||
return Math.min(...scored.map(c => c.score!.complexity));
|
||||
}
|
||||
|
||||
function cloneSchema(s: Schema): Schema {
|
||||
return {
|
||||
...s,
|
||||
fields: s.fields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
};
|
||||
}
|
||||
|
||||
function deterministicClarifyDescription(field: SchemaField): string {
|
||||
const base = field.description.trim();
|
||||
if (!base) return `Return the value for "${field.name}" as ${field.type}.`;
|
||||
if (/\b(must|should|return)\b/i.test(base)) return base;
|
||||
return `${base} Must be a ${field.type}.`;
|
||||
}
|
||||
|
||||
// Disambiguation re-export — the generator calls this internally so mutations
|
||||
// can reference it without colliding with the public `editFieldDescription`
|
||||
// option callback in `EvolveSchemaOptions`.
|
||||
const editFieldDescription_pure = editFieldDescription;
|
||||
|
||||
function pickN<T>(arr: T[], n: number, rng: () => number): T[] {
|
||||
if (arr.length <= n) return arr.slice();
|
||||
const copy = arr.slice();
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, n);
|
||||
}
|
||||
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function arraysEqual<T>(a: T[], b: T[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
function emitProgress(
|
||||
config: { onProgress?: (e: EvolveSchemaProgress) => void },
|
||||
phase: EvolveSchemaProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
bestAcc: number,
|
||||
bestCmp: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({
|
||||
phase, generation, populationSize,
|
||||
bestAccuracy: bestAcc,
|
||||
bestComplexity: bestCmp,
|
||||
message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: EvolveSchemaOptions) {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
execute: opts.execute,
|
||||
judge: opts.judge,
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
mutationMix: opts.mutationMix ?? { structure: 3, order: 1, refinement: 1 },
|
||||
evalSize: opts.evalSize ?? 32,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 100,
|
||||
seed: opts.seed ?? 1,
|
||||
editFieldDescription: opts.editFieldDescription,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,19 +1,17 @@
|
||||
/**
|
||||
* HarnessTraceBridge — translates workflow-harness phase events into
|
||||
* execution traces so harnessed agent runs feed the self-evolution loop.
|
||||
* execution traces so harnessed agent runs show up in cost tracking and
|
||||
* run history.
|
||||
*
|
||||
* Without this bridge the workflow harness is an island: phases run,
|
||||
* gates pass or fail, but nothing flows into the `execution_traces`
|
||||
* table that the evaluation dataset builder mines. This module closes
|
||||
* that loop by subscribing to the shared `harnessEvents` emitter and
|
||||
* writing one trace per phase outcome:
|
||||
* table. This module subscribes to the shared `harnessEvents` emitter and
|
||||
* writes one trace per phase outcome:
|
||||
*
|
||||
* harness:phase:complete → trace outcome 'verified'
|
||||
* harness:phase:fail (aborted === true) → trace outcome 'abandoned'
|
||||
* harness:phase:fail (will retry) → skipped — only final
|
||||
* outcomes become training
|
||||
* signal, mid-retry noise
|
||||
* stays out of the dataset
|
||||
* outcomes are recorded
|
||||
*
|
||||
* Each trace records:
|
||||
* - input: the phase's instruction text
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
/**
|
||||
* Improvement Detector — produces structured, runtime-consumable awareness signals.
|
||||
*
|
||||
* Per correction #1: Returns actionable signals + surfaced suggestions as structured data,
|
||||
* not just prompt text changes.
|
||||
*
|
||||
* Per correction #5: Awareness is operational — route better, suggest better, adapt better,
|
||||
* avoid repeated mistakes.
|
||||
*
|
||||
* Per correction #3: Workflow suggestions require recency + pattern similarity,
|
||||
* not just raw shape count.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore, ActionableSignal, SignalCategory } from '@waggle/core';
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
|
||||
// ── Structured output types ──────────────────────────────────
|
||||
|
||||
export interface AwarenessSummary {
|
||||
/** Capability gaps the agent has encountered repeatedly */
|
||||
capabilityGaps: CapabilityGapSignal[];
|
||||
/** Behavioral corrections the user has made repeatedly */
|
||||
corrections: CorrectionSignal[];
|
||||
/** Workflow patterns that recur and could benefit from templates */
|
||||
workflowPatterns: WorkflowPatternSignal[];
|
||||
/** Total actionable signal count (for deciding whether to inject into prompt) */
|
||||
totalActionable: number;
|
||||
}
|
||||
|
||||
export interface CapabilityGapSignal {
|
||||
id: number;
|
||||
toolName: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
export interface CorrectionSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
occurrences: number;
|
||||
guidance: string;
|
||||
}
|
||||
|
||||
export interface WorkflowPatternSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
// ── Capability gap recording ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* Record a capability gap when a tool is not found or a skill is missing.
|
||||
* Called from agent-loop when tool lookup fails.
|
||||
*/
|
||||
export function recordCapabilityGap(
|
||||
store: ImprovementSignalStore,
|
||||
toolName: string,
|
||||
context?: string,
|
||||
): void {
|
||||
store.record('capability_gap', `missing:${toolName}`, context, {
|
||||
tool: toolName,
|
||||
lastContext: context,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Correction recording ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Analyze a user message for corrections and record if detected.
|
||||
* Only task-local and durable corrections are recorded (both go into the store;
|
||||
* the store's threshold mechanism handles promotion to actionable).
|
||||
*
|
||||
* Returns the detected correction (if any) for caller use.
|
||||
*/
|
||||
export function analyzeAndRecordCorrection(
|
||||
store: ImprovementSignalStore,
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
const correction = detectCorrection(userMessage, previousAssistantMessage);
|
||||
if (!correction) return null;
|
||||
|
||||
// Record in store — both durable and task-local go in.
|
||||
// Durable corrections get recorded; task-local ones also get recorded but
|
||||
// only become actionable if the same pattern_key recurs (count >= threshold).
|
||||
store.record('correction', correction.patternKey, correction.detail, {
|
||||
isDurable: correction.isDurable,
|
||||
confidence: correction.confidence,
|
||||
});
|
||||
|
||||
return correction;
|
||||
}
|
||||
|
||||
// ── Workflow pattern recording ────────────────────────────────
|
||||
|
||||
const RECENCY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
|
||||
|
||||
/**
|
||||
* Record a workflow pattern occurrence.
|
||||
* Per correction #3: requires recency context, not just raw shape count.
|
||||
*/
|
||||
export function recordWorkflowPattern(
|
||||
store: ImprovementSignalStore,
|
||||
taskShape: string,
|
||||
taskDescription: string,
|
||||
): void {
|
||||
store.record('workflow_pattern', `shape:${taskShape}`, taskDescription, {
|
||||
lastTask: taskDescription.slice(0, 100),
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Structured awareness builder ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a structured awareness summary from the signal store.
|
||||
* Returns runtime-consumable data that can be:
|
||||
* 1. Injected into system prompt as guidance
|
||||
* 2. Used by the agent loop for routing decisions
|
||||
* 3. Displayed in UI as improvement suggestions
|
||||
*
|
||||
* Per correction #6: capped at 3 actionable signals total, non-repeating.
|
||||
*/
|
||||
export function buildAwarenessSummary(store: ImprovementSignalStore): AwarenessSummary {
|
||||
const actionable = store.getActionable();
|
||||
|
||||
const capabilityGaps: CapabilityGapSignal[] = [];
|
||||
const corrections: CorrectionSignal[] = [];
|
||||
const workflowPatterns: WorkflowPatternSignal[] = [];
|
||||
|
||||
for (const signal of actionable) {
|
||||
switch (signal.category) {
|
||||
case 'capability_gap':
|
||||
capabilityGaps.push(formatCapabilityGap(signal));
|
||||
break;
|
||||
case 'correction':
|
||||
corrections.push(formatCorrection(signal));
|
||||
break;
|
||||
case 'workflow_pattern':
|
||||
// Per correction #3: only surface if recent (within 7 days)
|
||||
if (isRecent(signal.last_seen)) {
|
||||
workflowPatterns.push(formatWorkflowPattern(signal));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
capabilityGaps,
|
||||
corrections,
|
||||
workflowPatterns,
|
||||
totalActionable: capabilityGaps.length + corrections.length + workflowPatterns.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format awareness summary as a prompt section for system prompt injection.
|
||||
* Only included when there are actionable signals.
|
||||
*/
|
||||
export function formatAwarenessPrompt(summary: AwarenessSummary): string | null {
|
||||
if (summary.totalActionable === 0) return null;
|
||||
|
||||
const lines: string[] = ['## Improvement Signals'];
|
||||
|
||||
if (summary.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Missing capabilities** (user has needed these before):');
|
||||
for (const gap of summary.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Behavioral adjustments** (user has corrected these patterns):');
|
||||
for (const correction of summary.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring workflows** (consider suggesting a template):');
|
||||
for (const pattern of summary.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark all signals in a summary as surfaced.
|
||||
* Call this after the signals have been injected into a prompt or shown to the user.
|
||||
*/
|
||||
export function markSummarySurfaced(
|
||||
store: ImprovementSignalStore,
|
||||
summary: AwarenessSummary,
|
||||
): void {
|
||||
for (const gap of summary.capabilityGaps) store.markSurfaced(gap.id);
|
||||
for (const correction of summary.corrections) store.markSurfaced(correction.id);
|
||||
for (const pattern of summary.workflowPatterns) store.markSurfaced(pattern.id);
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────
|
||||
|
||||
function formatCapabilityGap(signal: ActionableSignal): CapabilityGapSignal {
|
||||
const toolName = signal.parsedMetadata.tool as string || signal.pattern_key.replace('missing:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
toolName,
|
||||
occurrences: signal.count,
|
||||
suggestion: `User has needed "${toolName}" ${signal.count} times. Consider using acquire_capability to find it.`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatCorrection(signal: ActionableSignal): CorrectionSignal {
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
detail: signal.detail,
|
||||
occurrences: signal.count,
|
||||
guidance: signal.detail
|
||||
? `${signal.detail} (corrected ${signal.count} times)`
|
||||
: `Correction pattern "${signal.pattern_key}" observed ${signal.count} times`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatWorkflowPattern(signal: ActionableSignal): WorkflowPatternSignal {
|
||||
const shape = signal.pattern_key.replace('shape:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
occurrences: signal.count,
|
||||
suggestion: `"${shape}" tasks recur frequently (${signal.count} times). A workflow template could streamline this.`,
|
||||
};
|
||||
}
|
||||
|
||||
function isRecent(dateStr: string): boolean {
|
||||
try {
|
||||
const date = new Date(dateStr);
|
||||
return Date.now() - date.getTime() < RECENCY_WINDOW_MS;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
/**
|
||||
* @internal Not exported from index.ts — future feature, tested but not wired.
|
||||
*
|
||||
* Improvement Wiring — processes each interaction for improvement signals.
|
||||
*
|
||||
* Bridges the gap between the chat route (which has raw messages) and the
|
||||
* existing correction-detector / improvement-detector / capability-acquisition
|
||||
* modules. Call processInteractionForImprovement after every agent response
|
||||
* to detect corrections, capability gaps, and recurring workflow patterns.
|
||||
*/
|
||||
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
import { detectTaskShape, type TaskShape } from './task-shape.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImprovementWiringParams {
|
||||
userMessage: string;
|
||||
agentResponse: string;
|
||||
toolsUsed: string[];
|
||||
workspaceId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
export interface ImprovementWiringResult {
|
||||
wasCorrection: boolean;
|
||||
correctionDetail?: string;
|
||||
capabilityGap?: string;
|
||||
workflowPattern?: string;
|
||||
}
|
||||
|
||||
// ── Well-known tool names (built-in tools the agent can use) ───────────
|
||||
|
||||
const KNOWN_TOOL_NAMES = new Set([
|
||||
'web_search', 'web_fetch', 'search_memory', 'save_memory',
|
||||
'get_identity', 'get_awareness', 'query_knowledge', 'add_task',
|
||||
'correct_knowledge', 'bash', 'read_file', 'write_file', 'edit_file',
|
||||
'search_files', 'search_content', 'git_status', 'git_diff', 'git_log',
|
||||
'git_commit', 'generate_docx', 'create_plan', 'add_plan_step',
|
||||
'execute_step', 'show_plan', 'list_skills', 'create_skill',
|
||||
'delete_skill', 'read_skill', 'search_skills', 'suggest_skill',
|
||||
'acquire_capability', 'install_capability', 'compose_workflow',
|
||||
'orchestrate_workflow', 'spawn_agent', 'list_agents', 'get_agent_result',
|
||||
]);
|
||||
|
||||
// ── Capability gap detection patterns ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Patterns in user messages that suggest the user wants a tool/capability
|
||||
* the agent doesn't have. These are request-like phrases paired with
|
||||
* tool/domain keywords.
|
||||
*/
|
||||
const GAP_REQUEST_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
|
||||
{ pattern: /\bcan you (?:send|post|publish|push) (?:to|on|via) (\w+)/i, domain: 'integration' },
|
||||
{ pattern: /\bconnect (?:to|with) (\w+)/i, domain: 'connector' },
|
||||
{ pattern: /\buse (\w+) (?:api|tool|service)/i, domain: 'integration' },
|
||||
{ pattern: /\b(?:read|open|parse|convert) (?:this |the |a )?(\w+) file/i, domain: 'file_format' },
|
||||
{ pattern: /\brun (?:this |the |a )?(\w+) (?:test|check|scan|lint)/i, domain: 'tooling' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect if the agent response itself indicates a tool was missing.
|
||||
* The agent often says things like "I don't have a tool for X" or
|
||||
* "Tool 'X' not found".
|
||||
*/
|
||||
const TOOL_NOT_FOUND_PATTERNS: RegExp[] = [
|
||||
/Tool "(.+?)" not found/,
|
||||
/I don't have (?:a |the )?(?:tool|capability|ability) (?:for|to) (.+?)(?:\.|$)/i,
|
||||
/no (?:tool|capability) available for (.+?)(?:\.|$)/i,
|
||||
];
|
||||
|
||||
// ── Workflow pattern detection ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Multi-step request indicators — the user is asking for a sequence of actions.
|
||||
*/
|
||||
const MULTI_STEP_PATTERNS: RegExp[] = [
|
||||
/\bthen\b.*\bthen\b/i, // "do X, then Y, then Z"
|
||||
/\bfirst\b.*\bthen\b/i, // "first do X, then Y"
|
||||
/\bstep\s*\d+/i, // "step 1, step 2"
|
||||
/\b(?:and then|after that|next|finally)\b/i, // sequence words
|
||||
];
|
||||
|
||||
// ── Main function ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Process an interaction for improvement signals.
|
||||
*
|
||||
* This is the main entry point called after every agent response.
|
||||
* It analyzes the user message and agent response to detect:
|
||||
* 1. Corrections (using the existing correction-detector)
|
||||
* 2. Capability gaps (tools requested but not available)
|
||||
* 3. Workflow patterns (recurring multi-step sequences)
|
||||
*
|
||||
* Returns structured signals for storage and display.
|
||||
*/
|
||||
export function processInteractionForImprovement(
|
||||
params: ImprovementWiringParams,
|
||||
): ImprovementWiringResult {
|
||||
const { userMessage, agentResponse, toolsUsed } = params;
|
||||
|
||||
const result: ImprovementWiringResult = {
|
||||
wasCorrection: false,
|
||||
};
|
||||
|
||||
// 1. Run correction detector on userMessage
|
||||
const correction = detectCorrection(userMessage);
|
||||
if (correction) {
|
||||
result.wasCorrection = true;
|
||||
result.correctionDetail = correction.detail;
|
||||
}
|
||||
|
||||
// 2. Check for capability gaps
|
||||
const gap = detectCapabilityGap(userMessage, agentResponse, toolsUsed);
|
||||
if (gap) {
|
||||
result.capabilityGap = gap;
|
||||
}
|
||||
|
||||
// 3. Check for workflow patterns (repeated multi-step sequences)
|
||||
const pattern = detectWorkflowPattern(userMessage);
|
||||
if (pattern) {
|
||||
result.workflowPattern = pattern;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Detect capability gaps from the interaction.
|
||||
* Checks:
|
||||
* - Agent response mentions a tool not found
|
||||
* - User message requests a tool/integration not in the known set
|
||||
*/
|
||||
function detectCapabilityGap(
|
||||
userMessage: string,
|
||||
agentResponse: string,
|
||||
toolsUsed: string[],
|
||||
): string | undefined {
|
||||
// Check agent response for explicit tool-not-found messages
|
||||
for (const pattern of TOOL_NOT_FOUND_PATTERNS) {
|
||||
const match = agentResponse.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Check user message for requests that imply missing capabilities
|
||||
for (const { pattern } of GAP_REQUEST_PATTERNS) {
|
||||
const match = userMessage.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const requested = match[1].toLowerCase();
|
||||
// Only flag as a gap if no tool was actually used for this domain
|
||||
if (toolsUsed.length === 0 || !toolsUsed.some(t => t.includes(requested))) {
|
||||
return requested;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect recurring workflow patterns from the user message.
|
||||
* Uses task-shape detection to identify the structural pattern,
|
||||
* and checks for multi-step indicators.
|
||||
*/
|
||||
function detectWorkflowPattern(userMessage: string): string | undefined {
|
||||
// Only flag multi-step requests as workflow patterns
|
||||
const isMultiStep = MULTI_STEP_PATTERNS.some(p => p.test(userMessage));
|
||||
if (!isMultiStep) return undefined;
|
||||
|
||||
// Use task-shape detector to identify the pattern type
|
||||
const shape = detectTaskShape(userMessage);
|
||||
if (shape.confidence >= 0.3) {
|
||||
return shape.type;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -203,7 +203,6 @@ export { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js'
|
||||
export {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
type ComposedSystemPrompt,
|
||||
type LoadedSkill,
|
||||
@@ -214,69 +213,11 @@ export { checkGrounding, extractClaimedSpecifics, type GroundingResult, type Cla
|
||||
export { CostTracker, DEFAULT_MODEL_PRICING, type ModelPricing, type UsageStats, type UsageEntry } from './cost-tracker.js';
|
||||
export { extractEntities, type ExtractedEntity } from './entity-extractor.js';
|
||||
export { CognifyPipeline, type CognifyConfig, type CognifyResult } from './cognify.js';
|
||||
export { AgentLearning, type LearnedBehavior, type PersonaEffectiveness, type LearningSnapshot } from './agent-learning.js';
|
||||
export {
|
||||
TraceRecorder, truncate as truncateTraceText, scrubSecrets,
|
||||
type TraceHandle, type FinalizeOptions as TraceFinalizeOptions,
|
||||
} from './trace-recorder.js';
|
||||
export {
|
||||
EvalDatasetBuilder, detectSecrets, redactSecrets, SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL, fromJSONL as evalFromJSONL,
|
||||
type EvalExample, type EvalExampleMetadata, type DatasetSplit,
|
||||
type BuildOptions as EvalBuildOptions, type JudgeVerdict,
|
||||
} from './eval-dataset.js';
|
||||
export {
|
||||
IterativeGEPA, paretoFront, scoreCandidate, aggregateScores,
|
||||
pickWinner, pickSample as pickGepaSample,
|
||||
type Candidate as GEPACandidate, type CandidateScore as GEPACandidateScore,
|
||||
type IterativeGEPAOptions, type GEPARunResult, type GEPAProgress,
|
||||
type ScoreCandidateOptions,
|
||||
type MutateArgs, type MutateFn, type MutationStrategy, type EvolutionTarget,
|
||||
} from './iterative-optimizer.js';
|
||||
export {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
type GateVerdict, type GateResult as EvolutionGateResult, type GateCheckResult,
|
||||
type SizeLimits, type GateOptions, type CheckInput as GateCheckInput,
|
||||
} from './evolution-gates.js';
|
||||
export {
|
||||
EvolveSchema,
|
||||
addOutputField, removeField, editFieldDescription,
|
||||
changeFieldType, addConstraint, removeConstraint,
|
||||
reorderFields, replaceOutputFields,
|
||||
schemaComplexity, aggregateSchemaScores, paretoFrontSchema,
|
||||
scoreSchemaCandidate, pickSchemaWinner,
|
||||
generateStructureMutations, generateOrderMutations, generateRefinementMutations,
|
||||
pickSample as pickSchemaSample,
|
||||
type Schema, type SchemaField, type FieldType, type FieldConstraint,
|
||||
type Mutation as SchemaMutation, type MutationKind as SchemaMutationKind,
|
||||
type SchemaCandidate, type SchemaCandidateScore, type SchemaExampleResult,
|
||||
type SchemaExecuteFn, type EvolveSchemaOptions, type EvolveSchemaResult,
|
||||
type EvolveSchemaProgress,
|
||||
} from './evolve-schema.js';
|
||||
export {
|
||||
ComposeEvolution, defaultFeedbackFilter, filterJudgeFeedback,
|
||||
stripStructuralLines, schemaExecutorFromInstructionRunner,
|
||||
type ComposeEvolutionOptions, type ComposeEvolutionResult,
|
||||
type ComposeProgress, type FeedbackFilter,
|
||||
} from './compose-evolution.js';
|
||||
export {
|
||||
EvolutionOrchestrator, eligibleForEvolution, summarizeRuns,
|
||||
type EvolutionOrchestratorDeps, type EvolutionOrchestratorOptions,
|
||||
type EvolutionAutoTriggerConfig, type OrchestratorRunResult,
|
||||
type OrchestratorOutcome, type EvolutionProgress,
|
||||
type SchemaBaselineInput,
|
||||
} from './evolution-orchestrator.js';
|
||||
export {
|
||||
deployPersonaOverride, rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride, rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides, applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type DeployResult, type DeployPersonaInput,
|
||||
type DeployBehavioralSpecInput, type BehavioralSpecOverride,
|
||||
type BehavioralSpecSection,
|
||||
} from './evolution-deploy.js';
|
||||
export { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from './secret-patterns.js';
|
||||
export {
|
||||
LLMJudge, DEFAULT_WEIGHTS, DEFAULT_RUBRIC,
|
||||
buildPrompt as buildJudgePrompt,
|
||||
@@ -285,17 +226,12 @@ export {
|
||||
type ParsedJudgeResponse,
|
||||
} from './judge.js';
|
||||
export {
|
||||
createAnthropicEvolutionLLM,
|
||||
buildJudgeLLMCall, buildGEPAMutateFn, buildSchemaExecuteFn,
|
||||
buildReflectiveMutationPrompt, buildSchemaFillPrompt,
|
||||
makeRunningJudge, isRunningJudge, RUNNING_JUDGE_BRAND,
|
||||
createAnthropicEvolutionLLM, buildJudgeLLMCall,
|
||||
retryWithBackoff, wrapWithRetry,
|
||||
isRetryableEvolutionError, computeRetryDelay,
|
||||
DEFAULT_RETRY_OPTIONS,
|
||||
type EvolutionLLM, type CreateAnthropicEvolutionLLMOptions,
|
||||
type BuildReflectiveMutationPromptArgs,
|
||||
type RetryOptions, type RetryInfo,
|
||||
type RunningJudge,
|
||||
} from './evolution-llm-wiring.js';
|
||||
export {
|
||||
createHarnessRun, advancePhase, getCurrentPhaseInstruction,
|
||||
@@ -346,7 +282,6 @@ export { createDocumentTools } from './document-tools.js';
|
||||
export { createSpreadsheetTools } from './spreadsheet-tools.js';
|
||||
export { createPresentationTools } from './presentation-tools.js';
|
||||
export { createPdfTools } from './pdf-tools.js';
|
||||
export { createInsightsTools, type InsightsDeps } from './insights-tools.js';
|
||||
export { createConnectorSearchTools } from './connector-search.js';
|
||||
export { createCrossWorkspaceTools, type CrossWorkspaceToolDeps } from './cross-workspace-tools.js';
|
||||
export { extractEntitiesWithLLM, type LLMCallFn as EntityLLMCallFn } from './entity-extractor.js';
|
||||
@@ -480,7 +415,6 @@ export {
|
||||
ComposioConnector, MSTeamsConnector, OutlookConnector, OneDriveConnector, OneNoteConnector,
|
||||
} from './connectors/index.js';
|
||||
export { IterationBudget, type IterationBudgetConfig } from './iteration-budget.js';
|
||||
export { captureInteraction, getRecentLogs, isWithinBudget, type CaptureInteractionInput } from './optimization-capture.js';
|
||||
export { routeMessage, type RoutingDecision } from './smart-router.js';
|
||||
export {
|
||||
compressConversation, estimateTokens, needsCompression,
|
||||
@@ -511,13 +445,7 @@ export {
|
||||
type DeliveryChannel, type DeliveryPreferences, type DeliveryMessage, type DeliveryResult,
|
||||
type DeliveryConnector, type DeliveryConnectorRegistry, type InAppEmitter,
|
||||
} from './cron-delivery-router.js';
|
||||
export { detectCorrection, detectCorrectionsInHistory, type DetectedCorrection, type CorrectionDurability } from './correction-detector.js';
|
||||
export { detectContradiction, type ContradictionResult } from './contradiction-detector.js';
|
||||
export {
|
||||
recordCapabilityGap, analyzeAndRecordCorrection, recordWorkflowPattern,
|
||||
buildAwarenessSummary, formatAwarenessPrompt, markSummarySurfaced,
|
||||
type AwarenessSummary, type CapabilityGapSignal, type CorrectionSignal, type WorkflowPatternSignal,
|
||||
} from './improvement-detector.js';
|
||||
export {
|
||||
lintMemoryWrite,
|
||||
type MemoryLintResult, type MemoryLintVerdict,
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
/**
|
||||
* Insights Tools — self-analytics for the agent (Hermes-inspired).
|
||||
*
|
||||
* Provides the agent with awareness of its own performance:
|
||||
* - Tool usage frequency and success rates
|
||||
* - Cost per model
|
||||
* - Correction frequency trends
|
||||
* - Improvement signal summary
|
||||
*
|
||||
* This enables the agent to reflect on what works and adapt strategy.
|
||||
*/
|
||||
|
||||
import type { ToolDefinition } from './tools.js';
|
||||
|
||||
export interface InsightsDeps {
|
||||
getOptimizationLogs: (limit: number) => Array<{
|
||||
model?: string;
|
||||
tool_names?: string;
|
||||
total_tokens?: number;
|
||||
estimated_cost?: number;
|
||||
was_correction?: number;
|
||||
created_at?: string;
|
||||
}>;
|
||||
getImprovementSignals: () => Array<{
|
||||
category: string;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export function createInsightsTools(deps: InsightsDeps): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: 'agent_insights',
|
||||
description: [
|
||||
'Analyze your own performance metrics. Returns: tool usage frequency,',
|
||||
'cost per model, correction rate, and active improvement signals.',
|
||||
'Use this to reflect on what approaches work and adapt your strategy.',
|
||||
'Call this when you want to understand your performance patterns.',
|
||||
].join(' '),
|
||||
parameters: {
|
||||
type: 'object' as const,
|
||||
required: [],
|
||||
properties: {
|
||||
lookbackDays: {
|
||||
type: 'number' as const,
|
||||
description: 'Number of days to analyze (default 30)',
|
||||
},
|
||||
},
|
||||
},
|
||||
execute: async (args: Record<string, unknown>) => {
|
||||
const lookbackDays = (args.lookbackDays as number) ?? 30;
|
||||
const limit = lookbackDays * 50; // ~50 interactions/day estimate
|
||||
|
||||
try {
|
||||
const logs = deps.getOptimizationLogs(limit);
|
||||
const signals = deps.getImprovementSignals();
|
||||
|
||||
// Tool usage frequency
|
||||
const toolCounts: Record<string, { total: number; successes: number }> = {};
|
||||
for (const log of logs) {
|
||||
const tools = log.tool_names?.split(',').map(t => t.trim()).filter(Boolean) ?? [];
|
||||
const isCorrection = log.was_correction === 1;
|
||||
for (const tool of tools) {
|
||||
if (!toolCounts[tool]) toolCounts[tool] = { total: 0, successes: 0 };
|
||||
toolCounts[tool].total++;
|
||||
if (!isCorrection) toolCounts[tool].successes++;
|
||||
}
|
||||
}
|
||||
|
||||
const toolStats = Object.entries(toolCounts)
|
||||
.sort((a, b) => b[1].total - a[1].total)
|
||||
.slice(0, 15)
|
||||
.map(([name, stats]) => ({
|
||||
tool: name,
|
||||
uses: stats.total,
|
||||
successRate: stats.total > 0 ? Math.round((stats.successes / stats.total) * 100) : 0,
|
||||
}));
|
||||
|
||||
// Cost per model
|
||||
const modelCosts: Record<string, { calls: number; tokens: number; cost: number }> = {};
|
||||
for (const log of logs) {
|
||||
const model = log.model ?? 'unknown';
|
||||
if (!modelCosts[model]) modelCosts[model] = { calls: 0, tokens: 0, cost: 0 };
|
||||
modelCosts[model].calls++;
|
||||
modelCosts[model].tokens += log.total_tokens ?? 0;
|
||||
modelCosts[model].cost += log.estimated_cost ?? 0;
|
||||
}
|
||||
|
||||
const modelStats = Object.entries(modelCosts)
|
||||
.sort((a, b) => b[1].calls - a[1].calls)
|
||||
.map(([model, stats]) => ({
|
||||
model,
|
||||
calls: stats.calls,
|
||||
tokens: stats.tokens,
|
||||
costUsd: Math.round(stats.cost * 100) / 100,
|
||||
}));
|
||||
|
||||
// Correction rate
|
||||
const totalInteractions = logs.length;
|
||||
const corrections = logs.filter(l => l.was_correction === 1).length;
|
||||
const correctionRate = totalInteractions > 0
|
||||
? Math.round((corrections / totalInteractions) * 100)
|
||||
: 0;
|
||||
|
||||
// Active improvement signals
|
||||
const activeSignals = signals
|
||||
.filter(s => s.count >= 2)
|
||||
.slice(0, 10)
|
||||
.map(s => ({
|
||||
category: s.category,
|
||||
pattern: s.pattern_key,
|
||||
detail: s.detail,
|
||||
occurrences: s.count,
|
||||
lastSeen: s.last_seen,
|
||||
}));
|
||||
|
||||
return JSON.stringify({
|
||||
period: `Last ${lookbackDays} days`,
|
||||
totalInteractions,
|
||||
correctionRate: `${correctionRate}%`,
|
||||
topTools: toolStats,
|
||||
modelUsage: modelStats,
|
||||
improvementSignals: activeSignals,
|
||||
recommendation: correctionRate > 15
|
||||
? 'High correction rate — review recent corrections and adjust approach.'
|
||||
: correctionRate > 5
|
||||
? 'Moderate corrections — some patterns may need attention.'
|
||||
: 'Low correction rate — current approach is working well.',
|
||||
}, null, 2);
|
||||
} catch (err: unknown) {
|
||||
return `Error generating insights: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -1,626 +0,0 @@
|
||||
/**
|
||||
* Iterative GEPA Optimizer — Phase 2.1 of the self-evolution loop.
|
||||
*
|
||||
* Upgrades the one-shot prompt expander (optimizer-service) to a multi-
|
||||
* generation population search with Pareto-frontier selection and
|
||||
* reflective mutations driven by LLMJudge feedback (Phase 1.3).
|
||||
*
|
||||
* Loop:
|
||||
* 1. Start with `baseline` prompt as the seed population (1 member).
|
||||
* 2. For each generation:
|
||||
* a. Micro-screen survivors on a small sample (default 50) to prune
|
||||
* obvious losers.
|
||||
* b. Mini-eval passing candidates on a larger sample (default 64).
|
||||
* c. Spawn `populationSize` reflective mutations of the top Pareto
|
||||
* candidate, using aggregated judge feedback as ASI.
|
||||
* 3. Anchor evaluation on the full eval set (default 400) picks the winner.
|
||||
*
|
||||
* "Actionable Side Information" (ASI) is the textual feedback the judge
|
||||
* produces — fed back into the `mutate` callback so the next generation
|
||||
* targets the identified weakness rather than mutating blindly.
|
||||
*
|
||||
* Pareto selection keeps candidates that are non-dominated on a
|
||||
* 3-dimensional fitness vector (correctness, procedureFollowing,
|
||||
* conciseness). A single scalar `overall` is also tracked for tie-breaking
|
||||
* and final ranking.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import { isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface Candidate {
|
||||
/** Stable id — e.g. `g0-baseline`, `g1-m2` */
|
||||
id: string;
|
||||
/** The prompt text being evolved */
|
||||
prompt: string;
|
||||
/** Generation index (0 = baseline) */
|
||||
generation: number;
|
||||
/** Parent candidate id (null for baseline) */
|
||||
parent: string | null;
|
||||
/** Short label describing the mutation strategy */
|
||||
strategy: string;
|
||||
/** Aggregated score across the most recent eval stage, or null if not yet scored */
|
||||
score: CandidateScore | null;
|
||||
/** Raw per-example scores used for Pareto analysis */
|
||||
perExample: JudgeScore[];
|
||||
}
|
||||
|
||||
/** Aggregated, mean-reduced score used for selection. */
|
||||
export interface CandidateScore {
|
||||
overall: number;
|
||||
correctness: number;
|
||||
procedureFollowing: number;
|
||||
conciseness: number;
|
||||
/** Mean length penalty applied across the eval set */
|
||||
lengthPenalty: number;
|
||||
/** Number of examples the candidate was scored on */
|
||||
n: number;
|
||||
/** Aggregated feedback lines from the lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
}
|
||||
|
||||
export type MutationStrategy =
|
||||
| 'expand-edge-cases'
|
||||
| 'tighten-format'
|
||||
| 'add-examples'
|
||||
| 'clarify-constraints'
|
||||
| 'reduce-length'
|
||||
| 'restructure-steps'
|
||||
| 'targeted-feedback';
|
||||
|
||||
export interface MutateArgs {
|
||||
parent: Candidate;
|
||||
strategy: MutationStrategy;
|
||||
/** Textual feedback taken from the parent's lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Target types we are mutating (for the mutate fn to tune style) */
|
||||
targetKind: EvolutionTarget;
|
||||
/** Generation index of the child */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export type MutateFn = (args: MutateArgs) => Promise<string>;
|
||||
|
||||
/** What is being evolved — informs mutation style and constraint policies. */
|
||||
export type EvolutionTarget =
|
||||
| 'persona-system-prompt'
|
||||
| 'behavioral-spec-section'
|
||||
| 'tool-description'
|
||||
| 'skill-body'
|
||||
| 'generic';
|
||||
|
||||
export interface GEPAProgress {
|
||||
phase: 'start' | 'micro-screen' | 'mini-eval' | 'mutate' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
/** The best-so-far overall score at this point */
|
||||
best: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface IterativeGEPAOptions {
|
||||
/** The starting prompt text. */
|
||||
baseline: string;
|
||||
/** Full eval dataset (train + val combined is typical). */
|
||||
examples: EvalExample[];
|
||||
/** Scorer — LLMJudge or anything that implements `score(input): Promise<JudgeScore>`. */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Function that produces a mutated prompt from a parent + weakness feedback. */
|
||||
mutate: MutateFn;
|
||||
/**
|
||||
* Opt-in escape hatch that disables the running-judge safety check.
|
||||
* Only set to true for unit tests with mock judges whose `score()` does
|
||||
* not need a real LLM call. Production code paths (evolving real prompts)
|
||||
* must always pass a `makeRunningJudge(...)`-wrapped judge; otherwise
|
||||
* GEPA optimizes prompt-text similarity to the expected output — a
|
||||
* meaningless gradient that produces nonsense winners.
|
||||
*/
|
||||
allowBareJudge?: boolean;
|
||||
/** What we're optimizing (tunes mutation + gate policy). Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Candidates per generation. Default 5. */
|
||||
populationSize?: number;
|
||||
/** How many generations to run. Default 3. */
|
||||
generations?: number;
|
||||
/** Micro-screen sample size. Default 50. */
|
||||
microScreenSize?: number;
|
||||
/** Mini-eval sample size. Default 64. */
|
||||
miniEvalSize?: number;
|
||||
/** Anchor evaluation sample size (applied to final Pareto set). Default 400. */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for deterministic sampling. Default 1. */
|
||||
seed?: number;
|
||||
/** Emits progress events across stages. */
|
||||
onProgress?: (event: GEPAProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Max concurrent `judge.score()` calls per candidate during eval stages.
|
||||
* Default 1 (fully sequential) — preserves legacy behavior. Bumping to
|
||||
* 4–8 cuts wall time dramatically when the judge is an LLM API with
|
||||
* plenty of concurrent capacity. Does not affect determinism: all
|
||||
* scores are aggregated after every example completes.
|
||||
*/
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface GEPARunResult {
|
||||
winner: Candidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: Candidate[];
|
||||
/** All candidates across all generations */
|
||||
history: Candidate[];
|
||||
/** True if the winner improved overall score over baseline */
|
||||
improved: boolean;
|
||||
/** Delta winner.score.overall - baseline.score.overall */
|
||||
delta: number;
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
export class IterativeGEPA {
|
||||
/**
|
||||
* Run the iterative optimization loop.
|
||||
*
|
||||
* Returns the winner candidate plus the full Pareto set. Winner is the
|
||||
* Pareto-set member with the highest `overall` score (ties broken by
|
||||
* length penalty, i.e. the more concise one wins).
|
||||
*/
|
||||
async run(options: IterativeGEPAOptions): Promise<GEPARunResult> {
|
||||
// H-09 G3 · reject bare judges — without makeRunningJudge the judge
|
||||
// scores prompt-text-vs-expected-output, which is a meaningless
|
||||
// gradient. Any caller evolving real prompts must wrap their judge.
|
||||
// Tests pass `allowBareJudge: true` to bypass.
|
||||
if (!options.allowBareJudge && !isRunningJudge(options.judge)) {
|
||||
throw new Error(
|
||||
'IterativeGEPA: judge is not a running judge. Wrap with ' +
|
||||
'makeRunningJudge(baseJudge, llm) before passing to .run(), or ' +
|
||||
'pass `allowBareJudge: true` for tests with mock judges. ' +
|
||||
'See evolution-llm-wiring.ts:makeRunningJudge for rationale.',
|
||||
);
|
||||
}
|
||||
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
// ── Seed population with baseline ────────────────────────
|
||||
const baseline: Candidate = {
|
||||
id: 'g0-baseline',
|
||||
prompt: config.baseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emit(config, 'start', 0, 1, 0, 'seeding baseline');
|
||||
|
||||
const history: Candidate[] = [baseline];
|
||||
let survivors: Candidate[] = [baseline];
|
||||
|
||||
// Score the baseline on the micro-screen so we have a comparison point.
|
||||
const microSample = pickSample(config.examples, config.microScreenSize, rng);
|
||||
await scoreCandidate(baseline, microSample, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
emit(config, 'micro-screen', 0, 1, baseline.score?.overall ?? 0);
|
||||
|
||||
// ── Generations ──────────────────────────────────────────
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
|
||||
// 1. Mutate top survivor(s) into N children.
|
||||
const topParent = pickTopParent(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
emit(config, 'mutate', gen, survivors.length, topParent.score?.overall ?? 0);
|
||||
|
||||
const children: Candidate[] = [];
|
||||
const strategies = cycleStrategies(config.populationSize, gen);
|
||||
for (let i = 0; i < config.populationSize; i++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const strategy = strategies[i];
|
||||
let childPrompt: string;
|
||||
try {
|
||||
childPrompt = await config.mutate({
|
||||
parent: topParent,
|
||||
strategy,
|
||||
weaknessFeedback: topParent.score?.weaknessFeedback ?? [],
|
||||
targetKind: config.targetKind,
|
||||
generation: gen,
|
||||
});
|
||||
} catch {
|
||||
// Mutation failed — reuse parent so the loop doesn't stall.
|
||||
childPrompt = topParent.prompt;
|
||||
}
|
||||
|
||||
const child: Candidate = {
|
||||
id: `g${gen}-m${i}`,
|
||||
prompt: childPrompt,
|
||||
generation: gen,
|
||||
parent: topParent.id,
|
||||
strategy,
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
children.push(child);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// 2. Micro-screen children on a small sample.
|
||||
const microSet = pickSample(config.examples, config.microScreenSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(child, microSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'micro-screen', gen, children.length,
|
||||
Math.max(...children.map(c => c.score?.overall ?? 0), topParent.score?.overall ?? 0),
|
||||
);
|
||||
|
||||
// 3. Keep children that at least match the parent's overall score.
|
||||
const threshold = topParent.score?.overall ?? 0;
|
||||
const microSurvivors = children.filter(c => (c.score?.overall ?? 0) >= threshold - 0.05);
|
||||
|
||||
// If everyone got filtered out, keep the single best child so we don't stall.
|
||||
if (microSurvivors.length === 0 && children.length > 0) {
|
||||
const best = children.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
microSurvivors.push(best);
|
||||
}
|
||||
|
||||
// 4. Mini-eval survivors on a larger sample.
|
||||
if (microSurvivors.length > 0) {
|
||||
const miniSet = pickSample(config.examples, config.miniEvalSize, rng);
|
||||
for (const cand of microSurvivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, miniSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'mini-eval', gen, microSurvivors.length,
|
||||
Math.max(...microSurvivors.map(c => c.score?.overall ?? 0)),
|
||||
);
|
||||
}
|
||||
|
||||
// 5. Pareto-select survivors for next generation.
|
||||
const candidatePool = [topParent, ...microSurvivors];
|
||||
survivors = paretoFront(candidatePool);
|
||||
}
|
||||
|
||||
// ── Anchor evaluation — big sample, final ranking ───────
|
||||
emit(config, 'anchor', config.generations, survivors.length, bestOverall(survivors));
|
||||
const anchorSet = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, anchorSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
|
||||
const anchorPareto = paretoFront(survivors);
|
||||
const winner = pickWinner(anchorPareto);
|
||||
const baselineOverall = baseline.score?.overall ?? 0;
|
||||
const winnerOverall = winner.score?.overall ?? 0;
|
||||
|
||||
emit(config, 'done', config.generations, anchorPareto.length, winnerOverall);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront: anchorPareto,
|
||||
history,
|
||||
improved: winnerOverall > baselineOverall,
|
||||
delta: winnerOverall - baselineOverall,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers (exported for tests) ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front: keep candidates not dominated on any of the three scored
|
||||
* dimensions. A dominates B if A ≥ B on all three and strictly greater
|
||||
* on at least one.
|
||||
*/
|
||||
export function paretoFront(candidates: Candidate[]): Candidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: Candidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominates(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
// If all candidates have equal scores, return them all.
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominates(a: CandidateScore, b: CandidateScore): boolean {
|
||||
const geAll =
|
||||
a.correctness >= b.correctness &&
|
||||
a.procedureFollowing >= b.procedureFollowing &&
|
||||
a.conciseness >= b.conciseness;
|
||||
const gtAny =
|
||||
a.correctness > b.correctness ||
|
||||
a.procedureFollowing > b.procedureFollowing ||
|
||||
a.conciseness > b.conciseness;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
export interface ScoreCandidateOptions {
|
||||
/** Abort signal — checked before each example is dispatched. */
|
||||
signal?: AbortSignal;
|
||||
/** Max concurrent judge.score() calls. Default 1 (sequential). */
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a candidate against a batch of examples and aggregate.
|
||||
*
|
||||
* Backwards compatible: the 4th argument may be either an `AbortSignal`
|
||||
* (legacy) or a `ScoreCandidateOptions` object. Pass
|
||||
* `{ concurrency: 4 }` to parallelize — individual examples are
|
||||
* independent so the aggregate result is identical to the sequential
|
||||
* version, only faster when the judge is an LLM API.
|
||||
*/
|
||||
export async function scoreCandidate(
|
||||
candidate: Candidate,
|
||||
examples: EvalExample[],
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signalOrOptions?: AbortSignal | ScoreCandidateOptions,
|
||||
): Promise<CandidateScore> {
|
||||
const options = resolveScoreOptions(signalOrOptions);
|
||||
const signal = options.signal;
|
||||
const concurrency = Math.max(1, options.concurrency ?? 1);
|
||||
|
||||
// The "candidate" in this context IS the new prompt we want to test.
|
||||
// We treat it as the model's actual response to the example's input,
|
||||
// since mutation produces a variant prompt-as-response. Callers can
|
||||
// substitute a `runCandidate` step before scoring if the prompt needs
|
||||
// to be executed first — see docs.
|
||||
const scoreOne = async (ex: EvalExample): Promise<JudgeScore | null> => {
|
||||
if (signal?.aborted) return null;
|
||||
try {
|
||||
return await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual: candidate.prompt,
|
||||
});
|
||||
} catch {
|
||||
// Keep iterating so one bad example doesn't invalidate the whole score.
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const results = concurrency === 1
|
||||
? await runSequential(examples, scoreOne)
|
||||
: await mapWithConcurrency(examples, concurrency, scoreOne);
|
||||
|
||||
const scores = results.filter((s): s is JudgeScore => s !== null);
|
||||
candidate.perExample = scores;
|
||||
const aggregated = aggregateScores(scores);
|
||||
candidate.score = aggregated;
|
||||
return aggregated;
|
||||
}
|
||||
|
||||
function resolveScoreOptions(
|
||||
arg: AbortSignal | ScoreCandidateOptions | undefined,
|
||||
): ScoreCandidateOptions {
|
||||
if (!arg) return {};
|
||||
// Legacy: AbortSignal passed directly. Detect by presence of `aborted`
|
||||
// (a boolean getter unique to AbortSignal-like objects).
|
||||
if (typeof (arg as AbortSignal).aborted === 'boolean' && typeof (arg as AbortSignal).addEventListener === 'function') {
|
||||
return { signal: arg as AbortSignal };
|
||||
}
|
||||
return arg as ScoreCandidateOptions;
|
||||
}
|
||||
|
||||
async function runSequential<T, R>(
|
||||
items: readonly T[],
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
for (const item of items) {
|
||||
results.push(await fn(item));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded-parallelism map. Runs `fn` over `items` with at most
|
||||
* `concurrency` in-flight calls at once. Results are returned in the
|
||||
* same order as inputs regardless of completion order.
|
||||
*
|
||||
* Implementation: N worker loops share a counter. Each worker pulls the
|
||||
* next index, awaits `fn`, stores at that index, repeats. No external
|
||||
* deps, no extra state machinery.
|
||||
*/
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
const workerCount = Math.min(concurrency, items.length);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: workerCount }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
export function aggregateScores(scores: JudgeScore[]): CandidateScore {
|
||||
if (scores.length === 0) {
|
||||
return {
|
||||
overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1, n: 0, weaknessFeedback: [],
|
||||
};
|
||||
}
|
||||
const n = scores.length;
|
||||
const sum = scores.reduce(
|
||||
(acc, s) => ({
|
||||
overall: acc.overall + s.overall,
|
||||
correctness: acc.correctness + s.correctness,
|
||||
procedureFollowing: acc.procedureFollowing + s.procedureFollowing,
|
||||
conciseness: acc.conciseness + s.conciseness,
|
||||
lengthPenalty: acc.lengthPenalty + s.lengthPenalty,
|
||||
}),
|
||||
{ overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0, lengthPenalty: 0 },
|
||||
);
|
||||
|
||||
// Collect the 3 worst-scoring examples' feedback as weakness signal.
|
||||
const worst = [...scores].sort((a, b) => a.overall - b.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(s => s.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
|
||||
return {
|
||||
overall: sum.overall / n,
|
||||
correctness: sum.correctness / n,
|
||||
procedureFollowing: sum.procedureFollowing / n,
|
||||
conciseness: sum.conciseness / n,
|
||||
lengthPenalty: sum.lengthPenalty / n,
|
||||
n,
|
||||
weaknessFeedback,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the "winner" from a Pareto set. Prefers highest overall; ties broken
|
||||
* by higher length penalty (more concise) then lowest generation (simpler).
|
||||
*/
|
||||
export function pickWinner(candidates: Candidate[]): Candidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const b = best.score?.overall ?? -1;
|
||||
const c = cand.score?.overall ?? -1;
|
||||
if (c > b) return cand;
|
||||
if (c < b) return best;
|
||||
// Tie on overall — prefer higher lengthPenalty (more concise, less penalized).
|
||||
const bLen = best.score?.lengthPenalty ?? 0;
|
||||
const cLen = cand.score?.lengthPenalty ?? 0;
|
||||
if (cLen !== bLen) return cLen > bLen ? cand : best;
|
||||
return best.generation <= cand.generation ? best : cand;
|
||||
});
|
||||
}
|
||||
|
||||
function pickTopParent(survivors: Candidate[]): Candidate | undefined {
|
||||
if (survivors.length === 0) return undefined;
|
||||
return survivors.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
function bestOverall(candidates: Candidate[]): number {
|
||||
if (candidates.length === 0) return 0;
|
||||
return Math.max(...candidates.map(c => c.score?.overall ?? 0));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic sampler — picks `k` examples using a seeded RNG.
|
||||
* If k >= examples.length, returns the full list in deterministic shuffled order.
|
||||
*/
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
// Fisher–Yates shuffle
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
/** Rotate through strategies so early generations explore the mutation space. */
|
||||
function cycleStrategies(n: number, generation: number): MutationStrategy[] {
|
||||
const all: MutationStrategy[] = [
|
||||
'expand-edge-cases',
|
||||
'tighten-format',
|
||||
'add-examples',
|
||||
'clarify-constraints',
|
||||
'reduce-length',
|
||||
'restructure-steps',
|
||||
'targeted-feedback',
|
||||
];
|
||||
// Early generations: spread across strategies.
|
||||
// Later generations: bias toward targeted-feedback once ASI is strong.
|
||||
if (generation <= 1) {
|
||||
return Array.from({ length: n }, (_, i) => all[i % all.length]);
|
||||
}
|
||||
return Array.from({ length: n }, (_, i) => {
|
||||
if (i === 0) return 'targeted-feedback';
|
||||
return all[(i + generation) % all.length];
|
||||
});
|
||||
}
|
||||
|
||||
function emit(
|
||||
config: { onProgress?: (event: GEPAProgress) => void },
|
||||
phase: GEPAProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
best: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({ phase, generation, populationSize, best, message });
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: IterativeGEPAOptions): Required<Omit<IterativeGEPAOptions, 'signal' | 'onProgress'>> & {
|
||||
onProgress?: IterativeGEPAOptions['onProgress'];
|
||||
signal?: AbortSignal;
|
||||
} {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
judge: opts.judge,
|
||||
mutate: opts.mutate,
|
||||
targetKind: opts.targetKind ?? 'generic',
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
microScreenSize: opts.microScreenSize ?? 50,
|
||||
miniEvalSize: opts.miniEvalSize ?? 64,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 400,
|
||||
seed: opts.seed ?? 1,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
concurrency: Math.max(1, opts.concurrency ?? 1),
|
||||
allowBareJudge: opts.allowBareJudge ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
// Mulberry32 — deterministic uniform PRNG
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* LLM-as-Judge — rubric-based scorer used by the evolution loop.
|
||||
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
|
||||
* loop jobs.
|
||||
*
|
||||
* Scores a candidate response against an expected ground truth along
|
||||
* three weighted dimensions:
|
||||
@@ -10,9 +11,8 @@
|
||||
*
|
||||
* Then applies a length penalty (soft cap) multiplied into the final score.
|
||||
*
|
||||
* The textual feedback produced is the input that drives GEPA's reflective
|
||||
* mutations — it must explain *what* went wrong (or right) so the next
|
||||
* prompt variation can address it.
|
||||
* The textual feedback must explain *what* went wrong (or right) so the
|
||||
* caller can act on it.
|
||||
*
|
||||
* Model-agnostic — caller provides the llmCall function. Defaults assume
|
||||
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
|
||||
@@ -42,7 +42,7 @@ export interface JudgeScore {
|
||||
conciseness: number;
|
||||
/** Multiplier applied to the weighted score (0..1). */
|
||||
lengthPenalty: number;
|
||||
/** Free-form feedback used as reflective-mutation input for GEPA. */
|
||||
/** Free-form feedback explaining the score. */
|
||||
feedback: string;
|
||||
/** True when the judge successfully parsed a structured response. */
|
||||
parsed: boolean;
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
/**
|
||||
* Optimization Capture — captures agent interactions for GEPA/Ax prompt optimization.
|
||||
*
|
||||
* Provides functions to log successful agent interactions into the optimization_log
|
||||
* table in a workspace's .mind file, and to query those logs for background
|
||||
* optimization analysis.
|
||||
*
|
||||
* This module is a thin adapter between the agent loop post-response hook and
|
||||
* the OptimizationLogStore in @waggle/core.
|
||||
*/
|
||||
|
||||
import type { OptimizationLogStore, CreateOptimizationLogInput, OptimizationLogEntry } from '@waggle/core';
|
||||
import type { AgentResponse } from './agent-loop.js';
|
||||
|
||||
// ── Public interface ───────────────────────────────────────────────────
|
||||
|
||||
export interface CaptureInteractionInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
response: AgentResponse;
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture an agent interaction for GEPA optimization analysis.
|
||||
*
|
||||
* Call this after every successful agent loop completion. The data is
|
||||
* stored in the workspace's .mind file and read later by the background
|
||||
* prompt_optimization cron job.
|
||||
*/
|
||||
export function captureInteraction(
|
||||
store: OptimizationLogStore,
|
||||
input: CaptureInteractionInput,
|
||||
): OptimizationLogEntry {
|
||||
const logInput: CreateOptimizationLogInput = {
|
||||
sessionId: input.sessionId,
|
||||
workspaceId: input.workspaceId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
toolsUsed: input.response.toolsUsed ?? [],
|
||||
turnCount: input.turnCount,
|
||||
wasCorrection: input.wasCorrection,
|
||||
inputTokens: input.response.usage?.inputTokens ?? 0,
|
||||
outputTokens: input.response.usage?.outputTokens ?? 0,
|
||||
};
|
||||
return store.insert(logInput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query recent optimization logs for background analysis.
|
||||
*
|
||||
* Returns the most recent entries from the optimization log,
|
||||
* ordered by timestamp descending.
|
||||
*/
|
||||
export function getRecentLogs(
|
||||
store: OptimizationLogStore,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getRecent(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimization logs for a specific workspace.
|
||||
*/
|
||||
export function getWorkspaceLogs(
|
||||
store: OptimizationLogStore,
|
||||
workspaceId: string,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getByWorkspace(workspaceId, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the daily optimization budget has been exceeded.
|
||||
*
|
||||
* Reads the optimization log entries from today and sums up their token costs.
|
||||
* Returns true if under budget, false if over.
|
||||
*
|
||||
* @param budgetCents Daily budget in cents (e.g. 100 = $1/day)
|
||||
* @param costPerInputToken Cost per input token in dollars (default: Sonnet pricing ~$3/M)
|
||||
* @param costPerOutputToken Cost per output token in dollars (default: Sonnet pricing ~$15/M)
|
||||
*/
|
||||
export function isWithinBudget(
|
||||
store: OptimizationLogStore,
|
||||
budgetCents: number,
|
||||
costPerInputToken: number = 3 / 1_000_000,
|
||||
costPerOutputToken: number = 15 / 1_000_000,
|
||||
): boolean {
|
||||
const stats = store.getStats();
|
||||
const totalCostDollars =
|
||||
stats.totalInputTokens * costPerInputToken +
|
||||
stats.totalOutputTokens * costPerOutputToken;
|
||||
const totalCostCents = totalCostDollars * 100;
|
||||
return totalCostCents <= budgetCents;
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
KnowledgeGraph,
|
||||
ImprovementSignalStore,
|
||||
createCoreLogger,
|
||||
evaluateExternalMemoryIngress,
|
||||
type Embedder,
|
||||
@@ -29,7 +28,6 @@ import { createMindTools, type ToolDefinition } from './tools.js';
|
||||
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
|
||||
import { renderGoalAncestry } from './goal-ancestry.js';
|
||||
import type { GoalAncestry } from '@waggle/shared';
|
||||
import { buildAwarenessSummary, markSummarySurfaced, type AwarenessSummary } from './improvement-detector.js';
|
||||
import { CognifyPipeline } from './cognify.js';
|
||||
import { scanForInjection } from './injection-scanner.js';
|
||||
import { runPatternWriteBack } from './pattern-write-back.js';
|
||||
@@ -134,13 +132,9 @@ export class Orchestrator {
|
||||
private mode: 'local' | 'team';
|
||||
private version: string;
|
||||
private skills: string[];
|
||||
private improvementSignals: ImprovementSignalStore;
|
||||
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
|
||||
private goalAncestry: GoalAncestry | null = null;
|
||||
|
||||
/** M8: deferred signal marking — collected during buildSystemPrompt, committed after model call */
|
||||
private _pendingSurfacedAwareness: AwarenessSummary | null = null;
|
||||
|
||||
/** Workspace-specific layers (null when no workspace is active) */
|
||||
private workspaceLayers: WorkspaceLayers | null = null;
|
||||
/** Personal-mind cognify pipeline (#12: compaction-summary persistence). */
|
||||
@@ -174,7 +168,6 @@ export class Orchestrator {
|
||||
this.search = new HybridSearch(config.db, config.embedder);
|
||||
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
|
||||
this.knowledge = new KnowledgeGraph(config.db);
|
||||
this.improvementSignals = new ImprovementSignalStore(config.db);
|
||||
|
||||
const cognify = new CognifyPipeline({
|
||||
frames: this.frames,
|
||||
@@ -193,10 +186,6 @@ export class Orchestrator {
|
||||
search: this.search,
|
||||
knowledge: this.knowledge,
|
||||
cognify,
|
||||
// Skills 2.0 gap K: write-path contradiction detection emits a
|
||||
// correction signal through this store when a new save conflicts
|
||||
// with an existing frame.
|
||||
improvementSignals: this.improvementSignals,
|
||||
// Provide workspace accessors so tools can route to the right mind
|
||||
getWorkspaceLayers: () => this.workspaceLayers,
|
||||
});
|
||||
@@ -343,11 +332,6 @@ export class Orchestrator {
|
||||
|
||||
// ── SELF-AWARENESS (runtime context, changes every call) ──
|
||||
const awarenessSection = this.uncachedSection('self_awareness', () => {
|
||||
const awareness = buildAwarenessSummary(this.improvementSignals);
|
||||
// Defer marking until commitSurfacedSignals() fires post-model-call.
|
||||
if (awareness.totalActionable > 0) {
|
||||
this._pendingSurfacedAwareness = awareness;
|
||||
}
|
||||
const caps: AgentCapabilities = {
|
||||
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
|
||||
skills: this.skills,
|
||||
@@ -355,7 +339,6 @@ export class Orchestrator {
|
||||
memoryStats: this.getMemoryStats(),
|
||||
mode: this.mode,
|
||||
version: this.version,
|
||||
awareness: awareness.totalActionable > 0 ? awareness : undefined,
|
||||
};
|
||||
return buildSelfAwareness(caps);
|
||||
});
|
||||
@@ -457,18 +440,6 @@ export class Orchestrator {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* M8: Commit deferred signal markings after model call succeeds.
|
||||
* Call this after the LLM response is received. If the model call fails,
|
||||
* skip this call — signals stay actionable for the next turn.
|
||||
*/
|
||||
commitSurfacedSignals(): void {
|
||||
if (this._pendingSurfacedAwareness) {
|
||||
markSummarySurfaced(this.improvementSignals, this._pendingSurfacedAwareness);
|
||||
this._pendingSurfacedAwareness = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic memory recall: search for memories relevant to the user's query.
|
||||
* Searches BOTH personal and workspace minds when workspace is active.
|
||||
@@ -991,5 +962,4 @@ export class Orchestrator {
|
||||
getSessions(): SessionStore { return this.sessions; }
|
||||
getSearch(): HybridSearch { return this.search; }
|
||||
getKnowledge(): KnowledgeGraph { return this.knowledge; }
|
||||
getImprovementSignals(): ImprovementSignalStore { return this.improvementSignals; }
|
||||
}
|
||||
|
||||
@@ -990,83 +990,4 @@ You specialize in creative direction — briefs, feedback management, brand cons
|
||||
'Delivering without version tracking — every creative output needs a version note.',
|
||||
],
|
||||
},
|
||||
|
||||
// ── Internal review persona (self-evolution) ──────────────────────────
|
||||
// NOT surfaced in onboarding (ALL_ONBOARDING_PERSONAS) or the PersonaSwitcher
|
||||
// tiers (UNIVERSAL_MODE_IDS / ALL_SPECIALIST_IDS). It runs headless via the
|
||||
// IdleSessionWatcher loopback: it reads a stale session's transcript and, when
|
||||
// it finds a material, actionable gap, proposes a skill patch through
|
||||
// create_skill — which is held for human approval (never written to disk by
|
||||
// the reviewer). Its toolset is read-oriented; every write/exec/memory tool
|
||||
// that ALWAYS_AVAILABLE_TOOLS would re-add is stripped via disallowedTools
|
||||
// (that denylist wins over both the allowlist AND always-available). isReadOnly
|
||||
// is deliberately NOT set — that would strip create_skill, its one write.
|
||||
{
|
||||
id: 'session-reviewer',
|
||||
name: 'Session Reviewer',
|
||||
description: 'Adversarial self-review of a stale session — proposes skill patches for recurring gaps, default-silent',
|
||||
icon: '🔎',
|
||||
tagline: 'Reads a finished session and proposes a fix only when there is real evidence.',
|
||||
bestFor: [
|
||||
'Spotting promised-but-undelivered deliverables in a transcript',
|
||||
'Detecting a recurring capability failure a skill patch could fix',
|
||||
'Staying silent (NOTHING_TO_DO) when nothing material is found',
|
||||
],
|
||||
wontDo: 'Will not write to disk, save memory, delete skills, install capabilities, run shell commands, or invent findings. Every proposal is held for human approval.',
|
||||
systemPrompt: `## Persona: Session Reviewer
|
||||
You are an adversarially careful reviewer of ONE already-finished session. You examine the recent conversation and look for exactly two things, in this order:
|
||||
1. **Promised-but-undelivered deliverables** — the assistant said it would produce or do something and the transcript shows it never did.
|
||||
2. **Recurring capability failures fixable by a skill patch** — the same tool/skill/workflow failed more than once in a way a small reusable skill could prevent.
|
||||
|
||||
=== CRITICAL: DEFAULT SILENT ===
|
||||
If you do not find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence, your entire reply MUST be exactly:
|
||||
NOTHING_TO_DO
|
||||
Nothing else. No preamble, no apology, no summary.
|
||||
|
||||
=== CRITICAL: NEVER INVENT ===
|
||||
- Cite the transcript. Every finding must quote or reference what was actually said or done. If you cannot point to evidence, it is not a finding — output NOTHING_TO_DO.
|
||||
- You are read-only over the user's work: you may read files, search memory, and read skills, but you may NOT write files, save memory, delete skills, install capabilities, or run shell commands.
|
||||
- Your ONE write is a proposal: call create_skill to propose a new or updated skill that closes a real, recurring gap. It will NOT be written to disk — it is held for the human to approve or reject in the Approvals view. Propose sparingly and only with cited evidence.
|
||||
|
||||
### Process
|
||||
1. Read the recent conversation of the session (you have read tools and the workspace binding).
|
||||
2. Decide: is there a promised-but-undelivered deliverable, or a recurring capability failure a skill could fix?
|
||||
3. If yes and only if yes: state the finding with its transcript evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill.
|
||||
4. If no: reply exactly NOTHING_TO_DO.`,
|
||||
modelPreference: 'claude-sonnet-4-6',
|
||||
tools: [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'list_skills', 'search_skills', 'read_skill',
|
||||
'create_skill',
|
||||
],
|
||||
// Explicit denylist — ALWAYS_AVAILABLE_TOOLS (persona-tool-filter.ts) re-adds
|
||||
// save_memory / delete_skill / install_capability / acquire_capability /
|
||||
// add_task / correct_knowledge past any allowlist, so they must be named here
|
||||
// to be stripped. spawn_agent + bash + the file/git writes are defense-in-depth
|
||||
// (they are not in `tools`, but naming them keeps the boundary explicit).
|
||||
// Connector/external-send tools carry dynamic names and cannot be enumerated
|
||||
// here; the allowlist (tools[] + always-available) already excludes them since
|
||||
// none are declared above.
|
||||
disallowedTools: [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'add_task', 'correct_knowledge',
|
||||
'spawn_agent', 'bash',
|
||||
'write_file', 'edit_file', 'generate_docx',
|
||||
'git_commit', 'git_push', 'git_merge',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
],
|
||||
failurePatterns: [
|
||||
'Inventing a finding with no transcript evidence — must cite what was actually said or output NOTHING_TO_DO.',
|
||||
'Narrating when silent is correct — if nothing material is found, the reply is exactly NOTHING_TO_DO.',
|
||||
'Proposing an over-broad skill — the patch must target the specific recurring gap, nothing speculative.',
|
||||
],
|
||||
isReadOnly: false,
|
||||
workspaceAffinity: ['review', 'quality', 'self-evolution'],
|
||||
suggestedSkills: [],
|
||||
suggestedConnectors: [],
|
||||
suggestedMcpServers: [],
|
||||
suggestedCommands: [],
|
||||
defaultWorkflow: null,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -5,7 +5,6 @@ import { buildActiveBehavioralSpec } from './behavioral-spec.js';
|
||||
|
||||
/** The resolved behavioral-spec shape returned by `buildActiveBehavioralSpec`. */
|
||||
export type BehavioralSpec = ReturnType<typeof buildActiveBehavioralSpec>;
|
||||
import { loadBehavioralSpecOverrides } from './evolution-deploy.js';
|
||||
import { loadCustomPersonas } from './custom-personas.js';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
|
||||
@@ -16,16 +15,8 @@ export interface LoadedSkill {
|
||||
|
||||
/**
|
||||
* Raw disk read of `{waggleDir}/system-prompt.md`. Returns null if missing
|
||||
* or empty.
|
||||
*
|
||||
* **Prefer `loadSystemPromptWithOverrides(waggleDir)` for runtime use.** This
|
||||
* bare reader exists for tools that legitimately need the unmerged file
|
||||
* content (editor views, migration scripts). Runtime agent code that loads
|
||||
* just this and nothing else will silently miss any deployed
|
||||
* behavioral-spec or persona evolution, since those are stored in separate
|
||||
* override files under the same `waggleDir`.
|
||||
*
|
||||
* H-09 G2 — see `loadSystemPromptWithOverrides` for the composed path.
|
||||
* or empty. Runtime code should prefer `loadSystemPromptWithOverrides`, which
|
||||
* also returns the behavioral spec and custom personas.
|
||||
*/
|
||||
export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
const filePath = path.join(waggleDir, 'system-prompt.md');
|
||||
@@ -34,22 +25,19 @@ export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
return content || null;
|
||||
}
|
||||
|
||||
/** Bundle of every override-aware input the agent runtime needs. */
|
||||
/** Bundle of every prompt input the agent runtime needs. */
|
||||
export interface ComposedSystemPrompt {
|
||||
/** User's literal system-prompt.md content, or null if absent. */
|
||||
userPromptMd: string | null;
|
||||
/** The active behavioral spec — base + any deployed section overrides. */
|
||||
/** The active behavioral spec. */
|
||||
behavioralSpec: BehavioralSpec;
|
||||
/** Custom personas discovered in `waggleDir/personas/*.json`. */
|
||||
customPersonas: AgentPersona[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Override-aware composition: reads `system-prompt.md`, merges deployed
|
||||
* behavioral-spec section overrides from
|
||||
* `{waggleDir}/evolution-deploy/behavioral-spec/*.md`, and surfaces any
|
||||
* custom personas the user has added. Use this at agent startup so
|
||||
* deployed evolutions take effect at the next run.
|
||||
* Composes the runtime prompt inputs: reads `system-prompt.md`, builds the
|
||||
* behavioral spec and surfaces any custom personas the user has added.
|
||||
*
|
||||
* Callers typically pass the bundle to their orchestrator:
|
||||
*
|
||||
@@ -65,44 +53,11 @@ export interface ComposedSystemPrompt {
|
||||
*/
|
||||
export function loadSystemPromptWithOverrides(waggleDir: string): ComposedSystemPrompt {
|
||||
const userPromptMd = loadSystemPrompt(waggleDir);
|
||||
// loadBehavioralSpecOverrides is no-op on missing dir / malformed files.
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const behavioralSpec = buildActiveBehavioralSpec(overrides);
|
||||
const behavioralSpec = buildActiveBehavioralSpec();
|
||||
const customPersonas = loadCustomPersonas(waggleDir);
|
||||
return { userPromptMd, behavioralSpec, customPersonas };
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup-time self-check. Call after composing the override-aware prompt
|
||||
* bundle to verify that deployed overrides on disk actually appear in the
|
||||
* active spec. Emits a structured console warning (never throws) if a
|
||||
* deployed section is present on disk but missing from the active spec —
|
||||
* that would indicate the caller is using the bare `loadSystemPrompt`
|
||||
* instead of the composed loader.
|
||||
*/
|
||||
export function assertOverridesReachActiveSpec(
|
||||
waggleDir: string,
|
||||
activeSpec: BehavioralSpec,
|
||||
): void {
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const missing: string[] = [];
|
||||
for (const [section, overrideText] of Object.entries(overrides)) {
|
||||
const live = (activeSpec as unknown as Record<string, string | undefined>)[section];
|
||||
if (!live || !live.includes(overrideText.slice(0, 24))) {
|
||||
missing.push(section);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
|
||||
console.warn(
|
||||
`[prompt-loader] override-reach check failed — deployed sections on disk ` +
|
||||
`not present in active behavioral spec: ${missing.join(', ')}. ` +
|
||||
`This usually means the runtime is using loadSystemPrompt() instead of ` +
|
||||
`loadSystemPromptWithOverrides(). See prompt-loader.ts.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function loadSkills(waggleDir: string): LoadedSkill[] {
|
||||
const skillsDir = path.join(waggleDir, 'skills');
|
||||
if (!fs.existsSync(skillsDir)) return [];
|
||||
|
||||
57
packages/agent/src/secret-patterns.ts
Normal file
57
packages/agent/src/secret-patterns.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Secret detection and redaction shared by skill redaction, executor briefs
|
||||
* and anything else that must keep credentials out of stored or forwarded text.
|
||||
*/
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { AwarenessSummary } from './improvement-detector.js';
|
||||
|
||||
export interface AgentCapabilities {
|
||||
tools: Array<{ name: string; description: string }>;
|
||||
skills: string[];
|
||||
@@ -7,8 +5,6 @@ export interface AgentCapabilities {
|
||||
memoryStats: { frameCount: number; sessionCount: number; entityCount: number };
|
||||
mode: 'local' | 'team';
|
||||
version: string;
|
||||
/** Structured improvement signals (optional — only present when store has actionable data) */
|
||||
awareness?: AwarenessSummary;
|
||||
}
|
||||
|
||||
export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
@@ -61,36 +57,6 @@ export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
lines.push('- Prefer memory search over guessing when prior context exists.');
|
||||
lines.push('- When uncertain, ask rather than fabricate.');
|
||||
|
||||
// Improvement signals (per correction #1: structured, runtime-consumable)
|
||||
if (caps.awareness && caps.awareness.totalActionable > 0) {
|
||||
lines.push('');
|
||||
lines.push('## Learning from Past Sessions');
|
||||
|
||||
if (caps.awareness.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring capability gaps:**');
|
||||
for (const gap of caps.awareness.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**User corrections to apply:**');
|
||||
for (const correction of caps.awareness.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring task patterns:**');
|
||||
for (const pattern of caps.awareness.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// F28: Persona compliance — resist persona mode switches and injection via fake authority
|
||||
lines.push('');
|
||||
lines.push('## Persona Compliance');
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
* 2. Generates a SKILL.md via generateSkillMarkdown
|
||||
* 3. Writes it to the personal scope directory (scope: personal frontmatter)
|
||||
* 4. Dedups against existing skill files by filename
|
||||
* 5. Optionally emits a workflow_pattern improvement signal
|
||||
*
|
||||
* A v2 enhancement can insert an LLM prose-improvement step between
|
||||
* detectWorkflowPattern and generateSkillMarkdown without changing this
|
||||
@@ -19,7 +18,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
detectWorkflowPattern,
|
||||
generateSkillMarkdown,
|
||||
@@ -36,8 +34,6 @@ export interface AutoExtractMessage {
|
||||
export interface AutoExtractDeps {
|
||||
/** Directory to write personal-scope SKILL.md files into */
|
||||
personalSkillsDir: string;
|
||||
/** Optional signal store — records workflow_pattern signals on each extraction */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Optional LLM enrichment hook. When provided, the template is passed
|
||||
* through before serialization so the SKILL.md prose can be rewritten.
|
||||
@@ -117,22 +113,5 @@ export async function autoExtractAndCreateSkill(
|
||||
const md = redactSkillContent(annotateWithScope(generateSkillMarkdown(finalTemplate))).content;
|
||||
fs.writeFileSync(filePath, md, 'utf-8');
|
||||
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`auto-extract:${stem}`,
|
||||
`Auto-extracted skill "${finalTemplate.name}" from session pattern (${finalTemplate.tools.join(', ')})`,
|
||||
{
|
||||
skillName: finalTemplate.name,
|
||||
filename: stem,
|
||||
tools: finalTemplate.tools,
|
||||
category: finalTemplate.category,
|
||||
source: 'auto_extract',
|
||||
},
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
return { filePath, template: finalTemplate };
|
||||
}
|
||||
|
||||
@@ -9,12 +9,12 @@
|
||||
* before every authored write.
|
||||
*
|
||||
* Secrets reuse the curated SECRET_PATTERNS via redactSecrets (single source of
|
||||
* truth — see eval-dataset.ts). Paths cover the user's home directory (the exact
|
||||
* truth — see secret-patterns.ts). Paths cover the user's home directory (the exact
|
||||
* running-user path plus generic per-OS forms), which both leaks the username
|
||||
* and makes a skill non-portable.
|
||||
*/
|
||||
import os from 'node:os';
|
||||
import { redactSecrets } from './eval-dataset.js';
|
||||
import { redactSecrets } from './secret-patterns.js';
|
||||
|
||||
// Generic user-home prefixes. Each matches up to (not including) the next path
|
||||
// separator, so the tail is preserved: C:\Users\me\.waggle\x -> ~\.waggle\x
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { loadSkillUsage, forgetSkillUsage } from './skill-usage.js';
|
||||
|
||||
const DEFAULT_MAX_IDLE_DAYS = 90;
|
||||
@@ -35,8 +34,6 @@ export interface RetireOptions {
|
||||
dryRun?: boolean;
|
||||
/** Injectable clock for deterministic tests. */
|
||||
now?: () => Date;
|
||||
/** Emit skill_promotion signals for observability. */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
}
|
||||
|
||||
export interface RetireReport {
|
||||
@@ -124,17 +121,6 @@ export function retireStaleSkills(waggleHome: string, opts: RetireOptions = {}):
|
||||
report.retired.push(skillName);
|
||||
report.archived.push(destPath);
|
||||
forgetSkillUsage(waggleHome, skillName);
|
||||
|
||||
if (opts.improvementSignals) {
|
||||
try {
|
||||
opts.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`retire:${skillName}`,
|
||||
`Retired unused skill "${skillName}" (idle > ${maxIdleDays} days). Archived to ${destPath}.`,
|
||||
{ skillName, archivedTo: destPath, maxIdleDays, lastActivityMs: activity },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
report.skipped.push({ name: skillName, reason: `move_failed: ${msg}` });
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
SKILL_SCOPE_ORDER,
|
||||
type SkillScope,
|
||||
} from './skill-frontmatter.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { autoExtractAndCreateSkill, type AutoExtractMessage } from './skill-autoextract.js';
|
||||
import { retireStaleSkills } from './skill-retirement.js';
|
||||
|
||||
@@ -57,11 +56,9 @@ export interface SkillToolsDeps {
|
||||
/**
|
||||
* Skills 2.0 gap E: promotion infrastructure. Each callback is optional.
|
||||
* - getWorkspaceId / getTeamId resolve scope dirs dynamically at tool-call time.
|
||||
* - improvementSignals records skill_promotion signals on successful moves.
|
||||
*/
|
||||
getWorkspaceId?: () => string | null;
|
||||
getTeamId?: () => string | null;
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Skills 2.0 gap A: session-message accessor for auto-extracting skills
|
||||
* from repeated workflow patterns. When provided, enables the
|
||||
@@ -665,15 +662,14 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
},
|
||||
|
||||
// Skills 2.0 gap E: promote_skill — move a skill up one scope rung.
|
||||
// personal → workspace. Enforces one-step-at-a-time and records a
|
||||
// skill_promotion improvement signal on success.
|
||||
// personal → workspace. Enforces one-step-at-a-time.
|
||||
{
|
||||
name: 'promote_skill',
|
||||
description:
|
||||
'Promote an installed skill one scope up (personal → workspace). '
|
||||
+ 'Team and enterprise scopes are not available in this edition. '
|
||||
+ 'The file is moved to the appropriate directory, frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history), and a skill_promotion signal is recorded.',
|
||||
+ 'The file is moved to the appropriate directory and frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -784,19 +780,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
// failing the promotion. An audit warning is fine.
|
||||
}
|
||||
|
||||
// 8. Record skill_promotion signal
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'skill_promotion',
|
||||
`promote:${skillName}:${currentScope}->${target}`,
|
||||
`Promoted skill "${skillName}" from ${currentScope} to ${target}`,
|
||||
{ skillName, from: currentScope, to: target, workspaceId, teamId },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
// 9. Let the agent reload its skill list so the new scope is picked up
|
||||
// 8. Let the agent reload its skill list so the new scope is picked up
|
||||
onSkillsChanged?.();
|
||||
|
||||
return (
|
||||
@@ -805,7 +789,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Moved from \`${found.filePath}\`\n`
|
||||
+ `- Now at \`${destPath}\`\n`
|
||||
+ `- Promotion history: ${updatedFrontmatter.promoted_from!.join(' → ')} → ${target}\n`
|
||||
+ `- Signal recorded: skill_promotion\n\n`
|
||||
+ `\n`
|
||||
+ `The skill is available immediately at the new scope. Agents in other `
|
||||
+ `sessions will pick it up on their next skill reload.`
|
||||
);
|
||||
@@ -823,8 +807,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Scan the current session for repeated workflow patterns (the same 3+ tools '
|
||||
+ 'used in the same order 2+ times) and auto-create a SKILL.md for any pattern '
|
||||
+ 'found. Creates personal-scope skills only — promote_skill moves them up '
|
||||
+ 'afterwards. Dedups by filename. Records a workflow_pattern signal on each '
|
||||
+ 'successful extraction.',
|
||||
+ 'afterwards. Dedups by filename.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
@@ -839,7 +822,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
}
|
||||
const result = await autoExtractAndCreateSkill(messages, {
|
||||
personalSkillsDir: skillsDir,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (result.filePath) {
|
||||
onSkillsChanged?.();
|
||||
@@ -850,7 +832,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Tools: ${result.template!.tools.join(', ')}\n`
|
||||
+ `- Written to: \`${result.filePath}\`\n`
|
||||
+ `- Scope: personal (use promote_skill to move it up)\n`
|
||||
+ `- Signal: workflow_pattern recorded\n`
|
||||
);
|
||||
}
|
||||
if (result.reason === 'already_exists') {
|
||||
@@ -870,7 +851,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Archive personal-scope skills that have not been used within the idle window. '
|
||||
+ 'Moves stale SKILL.md files to ~/.waggle/skills-archive/ (recoverable — no deletion). '
|
||||
+ 'Default max_idle_days is 90. Default dry_run is true — call with dry_run: false to '
|
||||
+ 'actually move the files. Records a workflow_pattern signal per retirement.',
|
||||
+ 'actually move the files.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -890,7 +871,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays,
|
||||
dryRun,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (!dryRun && report.retired.length > 0) {
|
||||
onSkillsChanged?.();
|
||||
|
||||
@@ -12,7 +12,6 @@ import type {
|
||||
} from '@waggle/core';
|
||||
import type { CognifyPipeline } from './cognify.js';
|
||||
import type { FeedbackHandler } from './feedback-handler.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import type { RiskLevel } from '@waggle/shared';
|
||||
import { createCoreLogger } from '@waggle/core';
|
||||
import { detectContradiction } from './contradiction-detector.js';
|
||||
@@ -96,13 +95,6 @@ export interface MindToolDeps {
|
||||
feedback?: FeedbackHandler;
|
||||
/** F16: Optional embedder for semantic dedup (cosine similarity) */
|
||||
embedder?: Embedder;
|
||||
/**
|
||||
* Skills 2.0 gap K: improvement-signal store for recording write-path
|
||||
* conflicts. When a new save contradicts an existing frame,
|
||||
* detectContradiction fires and we emit a "correction" signal so the
|
||||
* evolution / improvement loops can surface the conflict.
|
||||
*/
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/** Dynamic accessor for workspace layers — checked at call time */
|
||||
getWorkspaceLayers?: () => WorkspaceLayers | null;
|
||||
/** W5.7: Accessor for ALL workspace search instances (for cross-workspace search) */
|
||||
@@ -528,20 +520,6 @@ export function createMindTools(deps: MindToolDeps): ToolDefinition[] {
|
||||
const contradiction = detectContradiction(content, recentRows);
|
||||
if (contradiction.isContradiction && contradiction.conflictsWith) {
|
||||
conflictFlag = ` [flag: contradicts_existing (${contradiction.conflictsWith.slice(0, 80).replace(/\s+/g, ' ').trim()}…)]`;
|
||||
// Record a correction-category signal on the improvement-signal
|
||||
// store. pattern_key is a stable hash-ish of the conflicting
|
||||
// frames so repeat contradictions merge via the ON CONFLICT upsert.
|
||||
if (deps.improvementSignals) {
|
||||
const patternKey = `write-conflict:${normalizeForDedup(content).slice(0, 64)}`;
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'correction',
|
||||
patternKey,
|
||||
`New save contradicts existing frame. New: "${content.slice(0, 120)}" ⟂ Existing: "${contradiction.conflictsWith.slice(0, 120)}"`,
|
||||
{ mindLabel, newContent: content.slice(0, 240), conflictsWith: contradiction.conflictsWith.slice(0, 240) },
|
||||
);
|
||||
} catch { /* non-blocking — signal persistence failure must not abort the save */ }
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking — contradiction check failure proceeds with save */ }
|
||||
|
||||
|
||||
@@ -1,373 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
ComposeEvolution,
|
||||
defaultFeedbackFilter,
|
||||
filterJudgeFeedback,
|
||||
stripStructuralLines,
|
||||
schemaExecutorFromInstructionRunner,
|
||||
} from '../src/compose-evolution.js';
|
||||
import type {
|
||||
Schema,
|
||||
SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(name => ({
|
||||
name, type: 'string',
|
||||
description: `description of ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback: string): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFakeRunner(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ── defaultFeedbackFilter ──────────────────────────────────────
|
||||
|
||||
describe('defaultFeedbackFilter', () => {
|
||||
it('classifies "missing reasoning field" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Missing reasoning field.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "wrong field type" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Wrong field type for answer.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "schema mismatch" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Schema mismatch — expected 3 fields.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies "should reorder fields" as structural', () => {
|
||||
expect(defaultFeedbackFilter('Should reorder fields — put reasoning first.')).toBe('structural');
|
||||
});
|
||||
|
||||
it('classifies value-level complaints as value', () => {
|
||||
expect(defaultFeedbackFilter('The answer is too terse')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Response is too verbose')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Incorrect calculation')).toBe('value');
|
||||
expect(defaultFeedbackFilter('Wrong tone — should be more formal')).toBe('value');
|
||||
});
|
||||
|
||||
it('classifies empty feedback as value (safe default)', () => {
|
||||
expect(defaultFeedbackFilter('')).toBe('value');
|
||||
});
|
||||
});
|
||||
|
||||
// ── stripStructuralLines ──────────────────────────────────────
|
||||
|
||||
describe('stripStructuralLines', () => {
|
||||
it('returns empty for empty input', () => {
|
||||
expect(stripStructuralLines('')).toBe('');
|
||||
});
|
||||
|
||||
it('drops only structural lines from multi-line feedback', () => {
|
||||
const input = [
|
||||
'Missing reasoning field.',
|
||||
'The answer is too terse.',
|
||||
'Wrong field type for confidence.',
|
||||
'Needs more detail.',
|
||||
].join('\n');
|
||||
const out = stripStructuralLines(input);
|
||||
expect(out).not.toContain('Missing reasoning field');
|
||||
expect(out).not.toContain('Wrong field type');
|
||||
expect(out).toContain('too terse');
|
||||
expect(out).toContain('Needs more detail');
|
||||
});
|
||||
|
||||
it('keeps all lines when none are structural', () => {
|
||||
const input = 'Too terse.\nNeeds more examples.';
|
||||
expect(stripStructuralLines(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('returns empty when all lines are structural', () => {
|
||||
const input = 'Missing reasoning field.\nSchema mismatch.';
|
||||
expect(stripStructuralLines(input)).toBe('');
|
||||
});
|
||||
|
||||
it('accepts a custom filter', () => {
|
||||
const custom = (line: string) => line.includes('DROP') ? 'structural' : 'value';
|
||||
const input = 'keep me\nDROP me\nkeep too';
|
||||
expect(stripStructuralLines(input, custom)).toBe('keep me\nkeep too');
|
||||
});
|
||||
});
|
||||
|
||||
// ── filterJudgeFeedback ───────────────────────────────────────
|
||||
|
||||
describe('filterJudgeFeedback', () => {
|
||||
it('strips structural feedback but preserves numeric scores', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.7, 'Missing reasoning field.\nResponse is too terse.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
|
||||
expect(score.overall).toBe(0.7);
|
||||
expect(score.correctness).toBe(0.7);
|
||||
expect(score.feedback).not.toContain('Missing reasoning field');
|
||||
expect(score.feedback).toContain('too terse');
|
||||
});
|
||||
|
||||
it('returns empty feedback when all lines are structural', async () => {
|
||||
const rawJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nSchema mismatch.');
|
||||
},
|
||||
};
|
||||
const filtered = filterJudgeFeedback(rawJudge);
|
||||
const score = await filtered.score({ input: 'x', expected: 'y', actual: 'z' });
|
||||
expect(score.feedback).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
// ── schemaExecutorFromInstructionRunner ───────────────────────
|
||||
|
||||
describe('schemaExecutorFromInstructionRunner', () => {
|
||||
it('builds a schema prefix and delegates to the runner', async () => {
|
||||
let captured: { prompt: string; input: string } | null = null;
|
||||
const runner = async (args: { prompt: string; input: string }) => {
|
||||
captured = args;
|
||||
return '{"answer":"42"}';
|
||||
};
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(runner);
|
||||
const result = await schemaRunner({
|
||||
schema: makeSchema(['reasoning', 'answer']),
|
||||
input: 'What is 6*7?',
|
||||
});
|
||||
|
||||
expect(result.actual).toBe('{"answer":"42"}');
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(captured!.prompt).toContain('reasoning');
|
||||
expect(captured!.prompt).toContain('answer');
|
||||
expect(captured!.input).toBe('What is 6*7?');
|
||||
});
|
||||
|
||||
it('reports parsed=false for non-JSON output', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => 'just prose');
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
|
||||
it('swallows runner errors and reports parsed=false', async () => {
|
||||
const schemaRunner = schemaExecutorFromInstructionRunner(async () => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
const result = await schemaRunner({ schema: makeSchema(['answer']), input: 'x' });
|
||||
expect(result.actual).toBe('');
|
||||
expect(result.parsed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── ComposeEvolution end-to-end ────────────────────────────────
|
||||
|
||||
describe('ComposeEvolution.run', () => {
|
||||
it('runs schema stage then instruction stage', async () => {
|
||||
const stages: string[] = [];
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: { async score() { return makeJudgeScore(0.5, 'ok'); } },
|
||||
mutate: async ({ parent }) => `${parent.prompt} v2`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
onProgress: (e) => stages.push(e.stage),
|
||||
});
|
||||
|
||||
expect(stages).toContain('schema');
|
||||
expect(stages).toContain('instructions');
|
||||
expect(stages).toContain('done');
|
||||
expect(result.schema.winner).toBeDefined();
|
||||
expect(result.instructions.winner).toBeDefined();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
});
|
||||
|
||||
it('feedback separation: GEPA cannot see structural feedback from the judge', async () => {
|
||||
const capturedFeedbacks: string[] = [];
|
||||
|
||||
// Judge always emits structural + value feedback
|
||||
const mixedJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'Missing reasoning field.\nResponse too terse.');
|
||||
},
|
||||
};
|
||||
|
||||
const mutateSpy = vi.fn(async ({ weaknessFeedback }: { parent: unknown; weaknessFeedback: string[]; strategy: string; targetKind: string; generation: number }) => {
|
||||
capturedFeedbacks.push(...weaknessFeedback);
|
||||
return 'mutated instruction prompt is a reasonable length';
|
||||
});
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeRunner(),
|
||||
judge: mixedJudge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'a baseline instruction prompt for testing purposes',
|
||||
examples: makeExamples(5),
|
||||
judge: mixedJudge,
|
||||
mutate: mutateSpy,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
});
|
||||
|
||||
// GEPA must never receive "Missing reasoning field" in weaknessFeedback.
|
||||
for (const fb of capturedFeedbacks) {
|
||||
expect(fb.toLowerCase()).not.toContain('missing reasoning field');
|
||||
expect(fb.toLowerCase()).not.toContain('wrong field type');
|
||||
}
|
||||
// But it should see the value-level complaint
|
||||
expect(capturedFeedbacks.join(' ')).toContain('too terse');
|
||||
});
|
||||
|
||||
it('accepts a custom feedback filter', async () => {
|
||||
let filterCalls = 0;
|
||||
const customFilter = (feedback: string): 'structural' | 'value' => {
|
||||
filterCalls++;
|
||||
return feedback.includes('CUSTOM-DROP') ? 'structural' : 'value';
|
||||
};
|
||||
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return makeJudgeScore(0.5, 'CUSTOM-DROP this line\nkeep this line');
|
||||
},
|
||||
};
|
||||
|
||||
await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt text is long enough',
|
||||
examples: makeExamples(3),
|
||||
judge,
|
||||
mutate: async () => 'new prompt text that is long enough',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
feedbackFilter: customFilter,
|
||||
});
|
||||
|
||||
expect(filterCalls).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns a stable shape when aborted before instruction stage', async () => {
|
||||
const ctrl = new AbortController();
|
||||
let progressCount = 0;
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(3),
|
||||
execute: async () => {
|
||||
// Abort during the schema stage
|
||||
if (progressCount === 1) ctrl.abort();
|
||||
progressCount++;
|
||||
return { actual: '', parsed: false };
|
||||
},
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
populationSize: 1, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'baseline prompt',
|
||||
examples: makeExamples(3),
|
||||
judge: { async score() { return makeJudgeScore(0.2, 'ok'); } },
|
||||
mutate: async () => 'should not run',
|
||||
allowBareJudge: true,
|
||||
populationSize: 1, generations: 1,
|
||||
microScreenSize: 2, miniEvalSize: 2, anchorEvalSize: 2,
|
||||
},
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
expect(result.schema).toBeDefined();
|
||||
expect(result.instructions).toBeDefined();
|
||||
expect(result.instructions.winner.prompt).toBe('baseline prompt');
|
||||
});
|
||||
|
||||
it('combinedDelta is a number; frozenSchema matches ES winner', async () => {
|
||||
const judge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(Math.min(1, args.actual.length / 50), 'ok');
|
||||
},
|
||||
};
|
||||
|
||||
const result = await new ComposeEvolution().run({
|
||||
schema: {
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(8),
|
||||
execute: makeFakeRunner(),
|
||||
judge,
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 4, anchorEvalSize: 4,
|
||||
},
|
||||
instructions: {
|
||||
baseline: 'short',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate: async ({ parent }) => `${parent.prompt} more tokens here`,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 2,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 6,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.instructions.winner.score).not.toBeNull();
|
||||
expect(result.frozenSchema).toBe(result.schema.winner.schema);
|
||||
expect(typeof result.combinedDelta).toBe('number');
|
||||
});
|
||||
});
|
||||
@@ -1,204 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
detectCorrection,
|
||||
detectCorrectionsInHistory,
|
||||
} from '../src/correction-detector.js';
|
||||
|
||||
describe('correction-detector', () => {
|
||||
// ── Basic detection ────────────────────────────────────────
|
||||
|
||||
describe('detectCorrection', () => {
|
||||
it('returns null for normal messages', () => {
|
||||
expect(detectCorrection('What is the weather today?')).toBeNull();
|
||||
expect(detectCorrection('Please write a function that sorts an array')).toBeNull();
|
||||
expect(detectCorrection('Thanks, that looks good')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for very short messages', () => {
|
||||
expect(detectCorrection('ok')).toBeNull();
|
||||
expect(detectCorrection('yes')).toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "no, not that"', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a bullet list, not paragraphs.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeGreaterThanOrEqual(0.3);
|
||||
});
|
||||
|
||||
it('detects strong correction: "I said / I told you"', () => {
|
||||
const result = detectCorrection('I said to use markdown headers, not plain text.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects strong correction: "that\'s wrong"', () => {
|
||||
const result = detectCorrection("That's wrong. The API endpoint should be POST, not GET.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects moderate correction: "actually, use X instead"', () => {
|
||||
const result = detectCorrection('Actually, instead use TypeScript for this module.');
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects tone correction', () => {
|
||||
const result = detectCorrection('No, that response was too formal. Keep it casual.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('detects format correction', () => {
|
||||
const result = detectCorrection("No, don't use those headers. Use bullet formatting instead.");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it('detects length correction', () => {
|
||||
const result = detectCorrection("That's not what I wanted. It's too long, make it shorter.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('length');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Durability classification ──────────────────────────────
|
||||
|
||||
describe('durability classification', () => {
|
||||
it('classifies "always" as durable', () => {
|
||||
const result = detectCorrection('No, always use bullet points instead of numbered lists.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
expect(result!.durability).toBe('durable');
|
||||
});
|
||||
|
||||
it('classifies "never" as durable', () => {
|
||||
const result = detectCorrection("No, never use code blocks for simple text. That's wrong.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "from now on" as durable', () => {
|
||||
const result = detectCorrection('From now on, please don\'t include timestamps in summaries.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "I prefer" as durable', () => {
|
||||
const result = detectCorrection("No, I prefer shorter responses. That's not what I wanted.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(true);
|
||||
});
|
||||
|
||||
it('classifies "this time" as task_local', () => {
|
||||
const result = detectCorrection("No, not that. This time just use a simple list.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.isDurable).toBe(false);
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('classifies "for now" as task_local', () => {
|
||||
const result = detectCorrection("That's wrong for now. Instead use the old API endpoint.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
|
||||
it('defaults to task_local when no explicit signal', () => {
|
||||
const result = detectCorrection('No, not that approach. Use the factory pattern.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.durability).toBe('task_local');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pattern key extraction ─────────────────────────────────
|
||||
|
||||
describe('pattern key extraction', () => {
|
||||
it('extracts tone pattern key', () => {
|
||||
const result = detectCorrection("No, that's too verbose. Stop doing that.");
|
||||
expect(result).not.toBeNull();
|
||||
// "too verbose" maps to tone category
|
||||
// Note: might also match length depending on exact wording
|
||||
});
|
||||
|
||||
it('extracts accuracy pattern key for "wrong"', () => {
|
||||
const result = detectCorrection("That's incorrect. I said the opposite.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toContain('accuracy');
|
||||
});
|
||||
|
||||
it('falls back to general:correction for unrecognized patterns', () => {
|
||||
const result = detectCorrection("No, not that. I told you to use something else entirely.");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBe('general:correction');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Detail extraction ──────────────────────────────────────
|
||||
|
||||
describe('detail extraction', () => {
|
||||
it('extracts first sentence as detail', () => {
|
||||
const result = detectCorrection('No, not that. I wanted a completely different structure. Also fix the naming.');
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail).toBe('No, not that.');
|
||||
});
|
||||
|
||||
it('truncates long messages', () => {
|
||||
const longMessage = 'No, not that. ' + 'A'.repeat(200);
|
||||
const result = detectCorrection(longMessage);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.detail.length).toBeLessThanOrEqual(120);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Confidence scoring ─────────────────────────────────────
|
||||
|
||||
describe('confidence', () => {
|
||||
it('gives higher confidence to strong corrections', () => {
|
||||
const strong = detectCorrection("No, that's wrong. I told you not to do that. Stop.");
|
||||
const moderate = detectCorrection('Actually, let\'s try a different approach instead.');
|
||||
expect(strong).not.toBeNull();
|
||||
expect(moderate).not.toBeNull();
|
||||
expect(strong!.confidence).toBeGreaterThan(moderate!.confidence);
|
||||
});
|
||||
|
||||
it('caps confidence at 1.0', () => {
|
||||
const result = detectCorrection(
|
||||
"No, that's wrong. I said don't do that. Not what I wanted. Please stop. Instead use something else.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.confidence).toBeLessThanOrEqual(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── History analysis ───────────────────────────────────────
|
||||
|
||||
describe('detectCorrectionsInHistory', () => {
|
||||
it('detects corrections in a message sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Here is a very detailed formal report...' },
|
||||
{ role: 'user', content: "No, that's too formal. I said keep it casual." },
|
||||
{ role: 'assistant', content: 'Got it. Here is a casual version...' },
|
||||
{ role: 'user', content: 'Thanks, that looks better.' },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections).toHaveLength(1);
|
||||
expect(corrections[0].patternKey).toContain('tone');
|
||||
});
|
||||
|
||||
it('returns empty array when no corrections found', () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Write a function to sort an array' },
|
||||
{ role: 'assistant', content: 'Here you go...' },
|
||||
{ role: 'user', content: 'Thanks!' },
|
||||
];
|
||||
expect(detectCorrectionsInHistory(messages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles multiple corrections in sequence', () => {
|
||||
const messages = [
|
||||
{ role: 'assistant', content: 'Draft version 1...' },
|
||||
{ role: 'user', content: "No, not that. That's wrong. Use a different approach." },
|
||||
{ role: 'assistant', content: 'Draft version 2...' },
|
||||
{ role: 'user', content: "I said don't use bullet points. Instead use paragraphs." },
|
||||
];
|
||||
const corrections = detectCorrectionsInHistory(messages);
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,348 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore } from '@waggle/core';
|
||||
import {
|
||||
EvalDatasetBuilder,
|
||||
detectSecrets,
|
||||
SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL,
|
||||
fromJSONL as evalFromJSONL,
|
||||
type EvalExample,
|
||||
} from '../src/eval-dataset.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JSONL round-trip', () => {
|
||||
const examples: EvalExample[] = [
|
||||
{
|
||||
input: 'write a haiku',
|
||||
expected_output: 'Cherry blossoms fall / ...',
|
||||
metadata: { source: 'trace', personaId: 'writer', outcome: 'success' },
|
||||
},
|
||||
{
|
||||
input: 'explain closures',
|
||||
expected_output: 'A closure captures variables from its enclosing scope.',
|
||||
metadata: { source: 'trace', personaId: 'coder', outcome: 'verified' },
|
||||
},
|
||||
];
|
||||
|
||||
it('round-trips through JSONL', () => {
|
||||
const jsonl = evalToJSONL(examples);
|
||||
expect(jsonl.split('\n')).toHaveLength(2);
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toEqual(examples);
|
||||
});
|
||||
|
||||
it('skips malformed lines silently', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify(examples[0]),
|
||||
'{ this is not valid json',
|
||||
'',
|
||||
JSON.stringify(examples[1]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips lines missing required fields', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify({ input: 'x' }), // missing expected_output + metadata
|
||||
JSON.stringify(examples[0]),
|
||||
].join('\n');
|
||||
const parsed = evalFromJSONL(jsonl);
|
||||
expect(parsed).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('EvalDatasetBuilder', () => {
|
||||
let db: MindDB;
|
||||
let store: ExecutionTraceStore;
|
||||
let builder: EvalDatasetBuilder;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ExecutionTraceStore(db);
|
||||
builder = new EvalDatasetBuilder(store);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function seedTrace(input: string, output: string, outcome: 'success' | 'verified' | 'corrected' | 'abandoned' = 'success', personaId = 'coder'): number {
|
||||
const id = store.start({ input, personaId });
|
||||
store.finalize(id, { outcome, output });
|
||||
return id;
|
||||
}
|
||||
|
||||
// ── sourceFromTraces ──
|
||||
|
||||
describe('sourceFromTraces', () => {
|
||||
it('pulls only traces matching the requested outcomes', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'verified');
|
||||
seedTrace('c?', 'C.', 'corrected');
|
||||
seedTrace('d?', 'D.', 'abandoned');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success', 'verified'], false);
|
||||
expect(examples).toHaveLength(2);
|
||||
expect(examples.map(e => e.input).sort()).toEqual(['a?', 'b?']);
|
||||
});
|
||||
|
||||
it('includes corrections when flag set', () => {
|
||||
seedTrace('a?', 'A.', 'success');
|
||||
seedTrace('b?', 'B.', 'corrected');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
expect(examples).toHaveLength(2);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.metadata.source).toBe('correction');
|
||||
});
|
||||
|
||||
it('maps trace fields onto metadata', () => {
|
||||
seedTrace('hello', 'world', 'success', 'writer');
|
||||
const [example] = builder.sourceFromTraces(['success'], false);
|
||||
expect(example.metadata.personaId).toBe('writer');
|
||||
expect(example.metadata.traceId).toBeGreaterThan(0);
|
||||
expect(example.metadata.source).toBe('trace');
|
||||
});
|
||||
});
|
||||
|
||||
// ── build (end-to-end) ──
|
||||
|
||||
describe('build', () => {
|
||||
it('returns deterministic split with same seed', async () => {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
seedTrace(`question ${i}?`, `answer ${i}.`, 'success');
|
||||
}
|
||||
|
||||
const a = await builder.build({ seed: 42 });
|
||||
const b = await builder.build({ seed: 42 });
|
||||
|
||||
expect(a.train.map(e => e.input)).toEqual(b.train.map(e => e.input));
|
||||
expect(a.val.map(e => e.input)).toEqual(b.val.map(e => e.input));
|
||||
expect(a.holdout.map(e => e.input)).toEqual(b.holdout.map(e => e.input));
|
||||
});
|
||||
|
||||
it('different seeds produce different orderings', async () => {
|
||||
for (let i = 0; i < 30; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const a = await builder.build({ seed: 1 });
|
||||
const b = await builder.build({ seed: 999 });
|
||||
|
||||
const aIds = a.train.map(e => e.input).join('|');
|
||||
const bIds = b.train.map(e => e.input).join('|');
|
||||
expect(aIds).not.toBe(bIds);
|
||||
});
|
||||
|
||||
it('splits 60/20/20 when enough examples', async () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length).toBe(60);
|
||||
expect(split.val.length).toBe(20);
|
||||
expect(split.holdout.length).toBe(20);
|
||||
expect(split.stats.total).toBe(100);
|
||||
});
|
||||
|
||||
it('honors custom split ratios', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
const split = await builder.build({ seed: 1, splitRatios: [0.8, 0.1, 0.1] });
|
||||
expect(split.train.length).toBe(40);
|
||||
expect(split.val.length).toBe(5);
|
||||
expect(split.holdout.length).toBe(5);
|
||||
});
|
||||
|
||||
it('rejects ratios that do not sum to 1', async () => {
|
||||
await expect(builder.build({ splitRatios: [0.5, 0.3, 0.3] })).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('filters out examples containing secrets', async () => {
|
||||
seedTrace('clean input', 'clean output', 'success');
|
||||
seedTrace('here is a key: AKIAIOSFODNN7EXAMPLE', 'sure', 'success');
|
||||
seedTrace('what is your token?', 'ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.sourced).toBe(3);
|
||||
expect(split.stats.acceptedAfterSecretScan).toBe(1);
|
||||
expect(split.rejected.filter(r => r.reason.startsWith('secret:'))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('filters examples by min length', async () => {
|
||||
seedTrace('?', 'ok', 'success'); // below both mins
|
||||
seedTrace('a proper question', 'a proper answer', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, minInputChars: 5, minOutputChars: 3 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-short-input')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters examples over max combined length', async () => {
|
||||
seedTrace('this is an ok prompt', 'this is an ok answer', 'success');
|
||||
seedTrace('x'.repeat(5000), 'y'.repeat(5000), 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1, maxCombinedChars: 1000 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'too-long')).toBe(true);
|
||||
});
|
||||
|
||||
it('filters low-signal garbage', async () => {
|
||||
seedTrace('.......................', '!!!!!!!!!!!!!!!!', 'success');
|
||||
seedTrace('real question?', 'real answer.', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason === 'low-signal')).toBe(true);
|
||||
});
|
||||
|
||||
it('dedupes by input hash', async () => {
|
||||
seedTrace('same prompt', 'answer a', 'success');
|
||||
seedTrace('same prompt', 'answer b', 'success');
|
||||
seedTrace('different prompt', 'answer c', 'success');
|
||||
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.unique).toBe(2);
|
||||
expect(split.rejected.some(r => r.reason === 'duplicate')).toBe(true);
|
||||
});
|
||||
|
||||
it('applies optional LLM judge', async () => {
|
||||
seedTrace('keep this example please', 'kept response', 'success');
|
||||
seedTrace('drop this example please', 'dropped response', 'success');
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async (ex) => ({
|
||||
keep: !ex.input.includes('drop'),
|
||||
reason: ex.input.includes('drop') ? 'contains drop' : undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.rejected.some(r => r.reason.startsWith('judge:'))).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps example when judge throws (fail-open)', async () => {
|
||||
seedTrace('some prompt text', 'some answer text', 'success');
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
judge: async () => {
|
||||
throw new Error('llm down');
|
||||
},
|
||||
});
|
||||
expect(split.stats.acceptedAfterJudge).toBe(1);
|
||||
expect(split.stats.total).toBe(1);
|
||||
});
|
||||
|
||||
it('accepts external harvest/correction examples', async () => {
|
||||
const harvest: EvalExample = {
|
||||
input: 'harvest prompt',
|
||||
expected_output: 'harvest answer',
|
||||
metadata: { source: 'harvest' },
|
||||
};
|
||||
const correction: EvalExample = {
|
||||
input: 'correction prompt',
|
||||
expected_output: 'correction answer',
|
||||
metadata: { source: 'correction' },
|
||||
};
|
||||
|
||||
const split = await builder.build({
|
||||
seed: 1,
|
||||
harvestExamples: [harvest],
|
||||
correctionExamples: [correction],
|
||||
});
|
||||
|
||||
const allInputs = [...split.train, ...split.val, ...split.holdout].map(e => e.input);
|
||||
expect(allInputs).toContain('harvest prompt');
|
||||
expect(allInputs).toContain('correction prompt');
|
||||
});
|
||||
|
||||
it('maps correction trace output to correctionFeedback when available', async () => {
|
||||
const id = store.start({ input: 'wrong approach', personaId: 'coder' });
|
||||
store.finalize(id, { outcome: 'success', output: 'first try' });
|
||||
store.markCorrected(id, 'Should use async/await here');
|
||||
|
||||
const examples = builder.sourceFromTraces(['success'], true);
|
||||
const corrected = examples.find(e => e.metadata.outcome === 'corrected');
|
||||
expect(corrected?.expected_output).toBe('Should use async/await here');
|
||||
});
|
||||
|
||||
it('returns empty splits when no traces match', async () => {
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.stats.total).toBe(0);
|
||||
expect(split.train).toEqual([]);
|
||||
expect(split.val).toEqual([]);
|
||||
expect(split.holdout).toEqual([]);
|
||||
});
|
||||
|
||||
it('tiny datasets assign leftover to holdout', async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
seedTrace(`question ${i} here?`, `answer ${i} here.`, 'success');
|
||||
}
|
||||
// Ratios 0.6/0.2/0.2 with n=3 → floor(1.8)=1, floor(0.6)=0, remainder=2 → holdout
|
||||
const split = await builder.build({ seed: 1 });
|
||||
expect(split.train.length + split.val.length + split.holdout.length).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,372 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
deployPersonaOverride,
|
||||
rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride,
|
||||
rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides,
|
||||
applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type BehavioralSpecSection,
|
||||
} from '../src/evolution-deploy.js';
|
||||
import { getPersona, listPersonas, setPersonaDataDir } from '../src/personas.js';
|
||||
import { loadCustomPersonas } from '../src/custom-personas.js';
|
||||
|
||||
// ── Test fixture: temp dir per-test ────────────────────────────
|
||||
|
||||
describe('evolution-deploy', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-evo-deploy-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch { /* Windows lock cleanup racy; ignore */ }
|
||||
});
|
||||
|
||||
// ── deployPersonaOverride ──
|
||||
|
||||
describe('deployPersonaOverride', () => {
|
||||
it('writes a JSON file named {id}.json in personas/', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'You are the evolved coder.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('personas', 'coder.json'))).toBe(true);
|
||||
expect(result.backupPath).toBeNull();
|
||||
expect(result.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('inherits built-in persona fields when targeting a known id', () => {
|
||||
// 'coder' is a built-in persona
|
||||
const builtin = getPersona('coder');
|
||||
expect(builtin).not.toBeNull();
|
||||
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED coder prompt — tighter format.',
|
||||
});
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.id).toBe('coder');
|
||||
expect(loaded.name).toBe(builtin!.name);
|
||||
expect(loaded.icon).toBe(builtin!.icon);
|
||||
expect(loaded.systemPrompt).toBe('EVOLVED coder prompt — tighter format.');
|
||||
expect(loaded.tools).toEqual(builtin!.tools);
|
||||
});
|
||||
|
||||
it('creates a backup on subsequent writes', () => {
|
||||
const first = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
|
||||
const second = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder', systemPrompt: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(fs.existsSync(second.backupPath!)).toBe(true);
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).systemPrompt).toBe('v1');
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).systemPrompt).toBe('v2');
|
||||
});
|
||||
|
||||
it('creates a minimal shell for unknown persona ids', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up-persona',
|
||||
systemPrompt: 'fresh prompt',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.id).toBe('made-up-persona');
|
||||
expect(loaded.name).toBe('made-up-persona');
|
||||
expect(loaded.systemPrompt).toBe('fresh prompt');
|
||||
expect(Array.isArray(loaded.tools)).toBe(true);
|
||||
});
|
||||
|
||||
it('applies caller-supplied overrides (description, icon, tools)', () => {
|
||||
const result = deployPersonaOverride(tmpDir, {
|
||||
personaId: 'made-up',
|
||||
systemPrompt: 'x',
|
||||
overrides: {
|
||||
description: 'custom description',
|
||||
icon: 'brain',
|
||||
tools: ['tool_a', 'tool_b'],
|
||||
},
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.description).toBe('custom description');
|
||||
expect(loaded.icon).toBe('brain');
|
||||
expect(loaded.tools).toEqual(['tool_a', 'tool_b']);
|
||||
});
|
||||
|
||||
it('is picked up by listPersonas after setPersonaDataDir', () => {
|
||||
deployPersonaOverride(tmpDir, {
|
||||
personaId: 'coder',
|
||||
systemPrompt: 'EVOLVED ROUTING',
|
||||
});
|
||||
|
||||
setPersonaDataDir(tmpDir);
|
||||
try {
|
||||
const personas = listPersonas();
|
||||
const evolved = personas.find(p => p.id === 'coder' && p.systemPrompt.includes('EVOLVED ROUTING'));
|
||||
expect(evolved).toBeDefined();
|
||||
} finally {
|
||||
setPersonaDataDir(''); // reset so other tests aren't polluted
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackPersonaOverride ──
|
||||
|
||||
describe('rollbackPersonaOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const second = deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v2' });
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const [loaded] = loadCustomPersonas(tmpDir);
|
||||
expect(loaded.systemPrompt).toBe('v1');
|
||||
});
|
||||
|
||||
it('deletes the override entirely when no .bak exists', () => {
|
||||
deployPersonaOverride(tmpDir, { personaId: 'coder', systemPrompt: 'v1' });
|
||||
const ok = rollbackPersonaOverride(tmpDir, 'coder');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadCustomPersonas(tmpDir)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('returns false for non-existent personas', () => {
|
||||
expect(rollbackPersonaOverride(tmpDir, 'ghost')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── deployBehavioralSpecOverride ──
|
||||
|
||||
describe('deployBehavioralSpecOverride', () => {
|
||||
it('writes a JSON file named {section}.json in behavioral-overrides/', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop',
|
||||
text: 'Evolved core loop instructions.',
|
||||
});
|
||||
expect(fs.existsSync(result.path)).toBe(true);
|
||||
expect(result.path.endsWith(path.join('behavioral-overrides', 'coreLoop.json'))).toBe(true);
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.section).toBe('coreLoop');
|
||||
expect(loaded.text).toBe('Evolved core loop instructions.');
|
||||
expect(loaded.deployedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('stores the originating run uuid for audit', () => {
|
||||
const result = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'qualityRules',
|
||||
text: 'new quality rules',
|
||||
runUuid: 'test-run-uuid-123',
|
||||
});
|
||||
const loaded = JSON.parse(fs.readFileSync(result.path, 'utf-8'));
|
||||
expect(loaded.runUuid).toBe('test-run-uuid-123');
|
||||
});
|
||||
|
||||
it('creates backup on subsequent writes', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
expect(first.backupPath).toBeNull();
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
expect(second.backupPath).not.toBeNull();
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
});
|
||||
|
||||
it('retries transient Windows rename locks before replacing an override', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync')
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('busy'), { code: 'EBUSY' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('locked again'), { code: 'EPERM' });
|
||||
})
|
||||
.mockImplementationOnce(() => {
|
||||
throw Object.assign(new Error('still busy'), { code: 'EBUSY' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
const second = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
});
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(5);
|
||||
expect(wait).toHaveBeenCalledTimes(4);
|
||||
expect(JSON.parse(fs.readFileSync(second.path, 'utf-8')).text).toBe('v2');
|
||||
expect(JSON.parse(fs.readFileSync(second.backupPath!, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves the current override when transient rename retries are exhausted', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('still locked'), { code: 'EACCES' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/still locked/);
|
||||
|
||||
expect(rename).toHaveBeenCalledTimes(10);
|
||||
expect(wait).toHaveBeenCalledTimes(9);
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('does not retry non-transient rename failures', () => {
|
||||
const first = deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v1',
|
||||
});
|
||||
const rename = vi.spyOn(fs, 'renameSync').mockImplementation(() => {
|
||||
throw Object.assign(new Error('disk error'), { code: 'EIO' });
|
||||
});
|
||||
const wait = vi.spyOn(Atomics, 'wait').mockReturnValue('timed-out');
|
||||
|
||||
expect(() => deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'coreLoop', text: 'v2',
|
||||
})).toThrow(/disk error/);
|
||||
|
||||
expect(rename).toHaveBeenCalledOnce();
|
||||
expect(wait).not.toHaveBeenCalled();
|
||||
expect(JSON.parse(fs.readFileSync(first.path, 'utf-8')).text).toBe('v1');
|
||||
expect(fs.existsSync(`${first.path}.tmp`)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects unknown sections', () => {
|
||||
expect(() =>
|
||||
deployBehavioralSpecOverride(tmpDir, {
|
||||
section: 'notASection' as BehavioralSpecSection,
|
||||
text: 'x',
|
||||
}),
|
||||
).toThrow(/Unknown behavioral-spec section/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── rollbackBehavioralSpecOverride ──
|
||||
|
||||
describe('rollbackBehavioralSpecOverride', () => {
|
||||
it('restores .bak when present', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v2' });
|
||||
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('v1');
|
||||
});
|
||||
|
||||
it('removes the override when no backup exists', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'v1' });
|
||||
const ok = rollbackBehavioralSpecOverride(tmpDir, 'coreLoop');
|
||||
expect(ok).toBe(true);
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── loadBehavioralSpecOverrides ──
|
||||
|
||||
describe('loadBehavioralSpecOverrides', () => {
|
||||
it('returns empty object when no overrides exist', () => {
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('returns a section → text map for all written overrides', () => {
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'coreLoop', text: 'evolved core' });
|
||||
deployBehavioralSpecOverride(tmpDir, { section: 'qualityRules', text: 'evolved quality' });
|
||||
|
||||
const overrides = loadBehavioralSpecOverrides(tmpDir);
|
||||
expect(overrides.coreLoop).toBe('evolved core');
|
||||
expect(overrides.qualityRules).toBe('evolved quality');
|
||||
expect(overrides.behavioralRules).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores malformed override files', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), '{ not valid json');
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir)).toEqual({});
|
||||
});
|
||||
|
||||
it('ignores files with empty text', () => {
|
||||
const dir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'coreLoop.json'), JSON.stringify({
|
||||
section: 'coreLoop', text: '', deployedAt: '2026-04-14',
|
||||
}));
|
||||
|
||||
expect(loadBehavioralSpecOverrides(tmpDir).coreLoop).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── applyBehavioralSpecOverrides ──
|
||||
|
||||
describe('applyBehavioralSpecOverrides', () => {
|
||||
const baseline: Record<BehavioralSpecSection, string> = {
|
||||
coreLoop: 'base core',
|
||||
qualityRules: 'base quality',
|
||||
behavioralRules: 'base behavioral',
|
||||
workPatterns: 'base patterns',
|
||||
intelligenceDefaults: 'base intelligence',
|
||||
};
|
||||
|
||||
it('returns baseline unchanged when no overrides', () => {
|
||||
expect(applyBehavioralSpecOverrides(baseline, {})).toEqual(baseline);
|
||||
});
|
||||
|
||||
it('overlays provided sections', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: 'EVOLVED core',
|
||||
qualityRules: 'EVOLVED quality',
|
||||
});
|
||||
expect(merged.coreLoop).toBe('EVOLVED core');
|
||||
expect(merged.qualityRules).toBe('EVOLVED quality');
|
||||
expect(merged.behavioralRules).toBe('base behavioral');
|
||||
});
|
||||
|
||||
it('ignores empty/undefined overrides without clobbering baseline', () => {
|
||||
const merged = applyBehavioralSpecOverrides(baseline, {
|
||||
coreLoop: '',
|
||||
qualityRules: undefined,
|
||||
});
|
||||
expect(merged.coreLoop).toBe('base core');
|
||||
expect(merged.qualityRules).toBe('base quality');
|
||||
});
|
||||
});
|
||||
|
||||
// ── BEHAVIORAL_SPEC_SECTIONS constant ──
|
||||
|
||||
describe('BEHAVIORAL_SPEC_SECTIONS', () => {
|
||||
it('contains the expected 5 sections in expected order', () => {
|
||||
expect(BEHAVIORAL_SPEC_SECTIONS).toEqual([
|
||||
'coreLoop', 'qualityRules', 'behavioralRules',
|
||||
'workPatterns', 'intelligenceDefaults',
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,245 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
} from '../src/evolution-gates.js';
|
||||
|
||||
describe('checkNonEmpty', () => {
|
||||
it('fails on empty string', () => {
|
||||
expect(checkNonEmpty('').verdict).toBe('fail');
|
||||
});
|
||||
it('fails on whitespace-only string', () => {
|
||||
expect(checkNonEmpty(' \n\n ').verdict).toBe('fail');
|
||||
});
|
||||
it('passes on real content', () => {
|
||||
expect(checkNonEmpty('hello').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkSize', () => {
|
||||
it('passes within the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(2000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.current).toBe(2000);
|
||||
expect(result.detail?.max).toBe(DEFAULT_SIZE_LIMITS.personaSystemPrompt);
|
||||
});
|
||||
|
||||
it('fails over the persona cap', () => {
|
||||
const result = checkSize('x'.repeat(4000), 'persona-system-prompt', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.reason).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('uses tool-description cap (500)', () => {
|
||||
expect(checkSize('x'.repeat(500), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(501), 'tool-description', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses skill-body cap (15k)', () => {
|
||||
expect(checkSize('x'.repeat(15_000), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('pass');
|
||||
expect(checkSize('x'.repeat(15_001), 'skill-body', DEFAULT_SIZE_LIMITS).verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('uses generic cap for unknown target', () => {
|
||||
const result = checkSize('x'.repeat(5000), 'generic', DEFAULT_SIZE_LIMITS);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('accepts custom size limits', () => {
|
||||
const limits = { ...DEFAULT_SIZE_LIMITS, generic: 100 };
|
||||
const result = checkSize('x'.repeat(200), 'generic', limits);
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkGrowth', () => {
|
||||
it('passes when candidate is smaller than baseline', () => {
|
||||
const result = checkGrowth('abc', 'abcdefgh', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes when growth is within budget', () => {
|
||||
const result = checkGrowth('x'.repeat(110), 'x'.repeat(100), 0.2); // +10%
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('fails when growth exceeds budget', () => {
|
||||
const result = checkGrowth('x'.repeat(140), 'x'.repeat(100), 0.2); // +40%
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.ratio).toBeCloseTo(0.4, 5);
|
||||
});
|
||||
|
||||
it('passes when baseline is empty (cannot compute ratio)', () => {
|
||||
const result = checkGrowth('anything', '', 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('boundary: exactly 20% growth passes with default budget', () => {
|
||||
// 100 → 120 is exactly 20%, not greater
|
||||
const result = checkGrowth('x'.repeat(120), 'x'.repeat(100), 0.2);
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkBalancedFences', () => {
|
||||
it('passes no fences', () => {
|
||||
expect(checkBalancedFences('plain prose').verdict).toBe('pass');
|
||||
});
|
||||
it('passes two fences (one code block)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\n```\ndone';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
it('fails one fence (unbalanced)', () => {
|
||||
const text = 'here:\n```js\nconst x = 1;\nstill going';
|
||||
const result = checkBalancedFences(text);
|
||||
expect(result.verdict).toBe('fail');
|
||||
expect(result.detail?.fences).toBe(1);
|
||||
});
|
||||
it('passes four fences (two blocks)', () => {
|
||||
const text = '```a\nfoo\n```\n```b\nbar\n```';
|
||||
expect(checkBalancedFences(text).verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoPlaceholders', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoPlaceholders('A prompt with normal prose.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails [TODO] bracket placeholder', () => {
|
||||
expect(checkNoPlaceholders('Write about [TODO] here.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [PLACEHOLDER]', () => {
|
||||
expect(checkNoPlaceholders('Fill in [PLACEHOLDER].').verdict).toBe('fail');
|
||||
});
|
||||
it('fails [YOUR NAME] style', () => {
|
||||
expect(checkNoPlaceholders('Hi [YOUR NAME HERE]!').verdict).toBe('fail');
|
||||
});
|
||||
it('fails angle-brackets <placeholder>', () => {
|
||||
expect(checkNoPlaceholders('Set <placeholder> before sending.').verdict).toBe('fail');
|
||||
});
|
||||
it('fails handlebars {{var}}', () => {
|
||||
expect(checkNoPlaceholders('Hello {{name}}.').verdict).toBe('fail');
|
||||
});
|
||||
it('does not false-positive normal bracketed citations', () => {
|
||||
expect(checkNoPlaceholders('See [1] and [Smith 2024] for details.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkNoObviousTodos', () => {
|
||||
it('passes clean text', () => {
|
||||
expect(checkNoObviousTodos('Write clear documentation.').verdict).toBe('pass');
|
||||
});
|
||||
it('fails lines starting with TODO:', () => {
|
||||
expect(checkNoObviousTodos('Intro paragraph.\nTODO: finish this').verdict).toBe('fail');
|
||||
});
|
||||
it('fails lines starting with FIXME:', () => {
|
||||
expect(checkNoObviousTodos('FIXME: tighten this up').verdict).toBe('fail');
|
||||
});
|
||||
it('does not fail mid-sentence "todo" mention', () => {
|
||||
expect(checkNoObviousTodos('Build a todo list UI.').verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkRegression', () => {
|
||||
it('passes when delta is within tolerance', () => {
|
||||
const result = checkRegression(0.8, 0.79, -0.02); // -1pp, floor -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
it('passes on improvement', () => {
|
||||
const result = checkRegression(0.7, 0.85, -0.02);
|
||||
expect(result.verdict).toBe('pass');
|
||||
expect(result.detail?.delta).toBeCloseTo(0.15, 5);
|
||||
});
|
||||
|
||||
it('fails when delta exceeds tolerance', () => {
|
||||
const result = checkRegression(0.9, 0.7, -0.02); // -20pp
|
||||
expect(result.verdict).toBe('fail');
|
||||
});
|
||||
|
||||
it('boundary: equal to tolerance passes', () => {
|
||||
const result = checkRegression(0.8, 0.78, -0.02); // exactly -2pp
|
||||
expect(result.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
// ── runGates end-to-end ────────────────────────────────────────
|
||||
|
||||
describe('runGates', () => {
|
||||
it('passes a clean candidate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'A concise persona prompt for the researcher.',
|
||||
baseline: 'A concise persona prompt for the researcher.',
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('pass');
|
||||
expect(res.firstFailure).toBeNull();
|
||||
});
|
||||
|
||||
it('fails an empty candidate on the first gate', () => {
|
||||
const res = runGates({ candidate: '', baseline: 'whatever' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('non-empty');
|
||||
});
|
||||
|
||||
it('fails oversized candidate on size gate', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(4000),
|
||||
baseline: 'x'.repeat(3000),
|
||||
}, { targetKind: 'persona-system-prompt' });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('size');
|
||||
});
|
||||
|
||||
it('fails runaway growth', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('growth');
|
||||
});
|
||||
|
||||
it('fails placeholder leakage', () => {
|
||||
const res = runGates({
|
||||
candidate: 'You are {{persona.name}} — answer questions.',
|
||||
baseline: 'You are the researcher — answer questions.',
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('no-placeholders');
|
||||
});
|
||||
|
||||
it('fails unbalanced markdown fences', () => {
|
||||
// Pad baseline so the growth gate doesn't trip before the fence gate.
|
||||
const padding = ' '.repeat(100);
|
||||
const res = runGates({
|
||||
candidate: `Instructions:\n\`\`\`js\nconst x = 1;\nmore prose${padding}`,
|
||||
baseline: `Instructions: write code clearly.${padding}`,
|
||||
});
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('balanced-fences');
|
||||
});
|
||||
|
||||
it('applies the regression gate when scores are supplied', () => {
|
||||
const res = runGates({
|
||||
candidate: 'new prompt',
|
||||
baseline: 'old prompt',
|
||||
scores: { baseline: 0.85, candidate: 0.6 },
|
||||
}, { maxRegression: -0.02 });
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.firstFailure?.gate).toBe('regression');
|
||||
});
|
||||
|
||||
it('skips the regression gate when scores are omitted', () => {
|
||||
const res = runGates({ candidate: 'new', baseline: 'old' });
|
||||
expect(res.results.find(r => r.gate === 'regression')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns results for every gate even when one fails', () => {
|
||||
const res = runGates({
|
||||
candidate: 'x'.repeat(200),
|
||||
baseline: 'x'.repeat(100),
|
||||
}, { targetKind: 'generic', maxGrowthRatio: 0.2 });
|
||||
expect(res.results.length).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,6 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
buildJudgeLLMCall,
|
||||
buildGEPAMutateFn,
|
||||
buildSchemaExecuteFn,
|
||||
makeRunningJudge,
|
||||
buildReflectiveMutationPrompt,
|
||||
buildSchemaFillPrompt,
|
||||
retryWithBackoff,
|
||||
wrapWithRetry,
|
||||
isRetryableEvolutionError,
|
||||
@@ -15,9 +10,6 @@ import {
|
||||
type RetryOptions,
|
||||
type RetryInfo,
|
||||
} from '../src/evolution-llm-wiring.js';
|
||||
import type { MutateArgs, GEPACandidate } from '../src/index.js';
|
||||
import type { Schema } from '../src/evolve-schema.js';
|
||||
import type { JudgeInput } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -37,31 +29,6 @@ function makeMockLLM(handler: (prompt: string) => string | Promise<string>): {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(overrides: Partial<GEPACandidate> = {}): GEPACandidate {
|
||||
return {
|
||||
id: 'g1-m0',
|
||||
prompt: 'You are a tester. Respond with the exact expected word.',
|
||||
generation: 1,
|
||||
parent: 'g0-baseline',
|
||||
strategy: 'expand-edge-cases',
|
||||
score: null,
|
||||
perExample: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(overrides: Partial<Schema> = {}): Schema {
|
||||
return {
|
||||
name: 'answer',
|
||||
version: 1,
|
||||
fields: [
|
||||
{ name: 'reasoning', type: 'string', description: 'step-by-step thinking', required: true, constraints: [] },
|
||||
{ name: 'answer', type: 'string', description: 'the final answer', required: true, constraints: [] },
|
||||
],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── buildJudgeLLMCall ─────────────────────────────────────────────
|
||||
|
||||
describe('buildJudgeLLMCall', () => {
|
||||
@@ -85,255 +52,6 @@ describe('buildJudgeLLMCall', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildGEPAMutateFn ─────────────────────────────────────────────
|
||||
|
||||
describe('buildGEPAMutateFn', () => {
|
||||
const mutateArgs: MutateArgs = {
|
||||
parent: makeCandidate(),
|
||||
strategy: 'expand-edge-cases',
|
||||
weaknessFeedback: [
|
||||
'Fails to handle empty input',
|
||||
'Sometimes forgets to include the final answer',
|
||||
],
|
||||
targetKind: 'persona-system-prompt',
|
||||
generation: 2,
|
||||
};
|
||||
|
||||
it('builds a reflective mutation prompt that includes parent + strategy + feedback', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => 'EVOLVED SYSTEM PROMPT');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe('EVOLVED SYSTEM PROMPT');
|
||||
expect(calls).toHaveLength(1);
|
||||
const prompt = calls[0];
|
||||
expect(prompt).toContain(mutateArgs.parent.prompt);
|
||||
expect(prompt).toContain(mutateArgs.strategy);
|
||||
expect(prompt).toContain('Fails to handle empty input');
|
||||
expect(prompt).toContain('persona-system-prompt');
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM returns empty or whitespace', async () => {
|
||||
const { llm } = makeMockLLM(() => ' \n ');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('falls back to the parent prompt when the LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result).toBe(mutateArgs.parent.prompt);
|
||||
});
|
||||
|
||||
it('strips markdown code fences from the LLM response', async () => {
|
||||
const { llm } = makeMockLLM(() => '```\nHIDDEN PROMPT\n```');
|
||||
const mutate = buildGEPAMutateFn(llm);
|
||||
|
||||
const result = await mutate(mutateArgs);
|
||||
|
||||
expect(result.trim()).toBe('HIDDEN PROMPT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildReflectiveMutationPrompt', () => {
|
||||
it('includes the generation number and weakness bullets', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'tighten-format',
|
||||
weaknessFeedback: ['a', 'b'],
|
||||
targetKind: 'behavioral-spec-section',
|
||||
generation: 3,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('generation 3');
|
||||
expect(prompt).toContain('- a');
|
||||
expect(prompt).toContain('- b');
|
||||
expect(prompt).toContain('tighten-format');
|
||||
expect(prompt).toContain('PARENT');
|
||||
});
|
||||
|
||||
it('handles missing weakness feedback gracefully', () => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: 'PARENT',
|
||||
strategy: 'add-examples',
|
||||
weaknessFeedback: [],
|
||||
targetKind: 'generic',
|
||||
generation: 0,
|
||||
});
|
||||
|
||||
expect(prompt).toContain('PARENT');
|
||||
expect(prompt).toContain('(no specific weakness signals yet)');
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildSchemaExecuteFn ──────────────────────────────────────────
|
||||
|
||||
describe('buildSchemaExecuteFn', () => {
|
||||
it('returns parsed=true when LLM returns valid JSON matching schema shape', async () => {
|
||||
const { llm, calls } = makeMockLLM(() => '{"reasoning":"thought","answer":"42"}');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'What is the answer?' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toBe('{"reasoning":"thought","answer":"42"}');
|
||||
expect(calls[0]).toContain('What is the answer?');
|
||||
expect(calls[0]).toContain('"reasoning"');
|
||||
expect(calls[0]).toContain('"answer"');
|
||||
});
|
||||
|
||||
it('returns parsed=false when LLM returns non-JSON', async () => {
|
||||
const { llm } = makeMockLLM(() => 'just some text');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('just some text');
|
||||
});
|
||||
|
||||
it('extracts JSON from markdown fences', async () => {
|
||||
const { llm } = makeMockLLM(() => '```json\n{"reasoning":"x","answer":"y"}\n```');
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(true);
|
||||
expect(result.actual).toContain('"reasoning"');
|
||||
});
|
||||
|
||||
it('returns parsed=false and empty actual when LLM throws', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('boom'); },
|
||||
};
|
||||
const execute = buildSchemaExecuteFn(llm);
|
||||
|
||||
const result = await execute({ schema: makeSchema(), input: 'hi' });
|
||||
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.actual).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSchemaFillPrompt', () => {
|
||||
it('serializes each field with name + type + description', () => {
|
||||
const prompt = buildSchemaFillPrompt({
|
||||
schema: makeSchema(),
|
||||
input: 'What is 2+2?',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('"reasoning"');
|
||||
expect(prompt).toContain('"answer"');
|
||||
expect(prompt).toContain('step-by-step thinking');
|
||||
expect(prompt).toContain('the final answer');
|
||||
expect(prompt).toContain('What is 2+2?');
|
||||
});
|
||||
|
||||
it('notes required constraints', () => {
|
||||
const schema: Schema = {
|
||||
name: 'x', version: 1,
|
||||
fields: [
|
||||
{
|
||||
name: 'score',
|
||||
type: 'number',
|
||||
description: 'a rating',
|
||||
required: true,
|
||||
constraints: [{ kind: 'range', value: '0-10' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
const prompt = buildSchemaFillPrompt({ schema, input: 'rate this' });
|
||||
expect(prompt).toContain('score');
|
||||
expect(prompt).toContain('number');
|
||||
expect(prompt).toContain('range');
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeRunningJudge ──────────────────────────────────────────────
|
||||
|
||||
describe('makeRunningJudge', () => {
|
||||
it('executes the candidate prompt with the example input, then delegates to the base judge', async () => {
|
||||
const { llm, calls } = makeMockLLM((prompt) => {
|
||||
// Simulate running the candidate prompt and returning a model response.
|
||||
return `MODEL OUTPUT for: ${prompt}`;
|
||||
});
|
||||
|
||||
const baseScore = vi.fn(async (args: JudgeInput) => ({
|
||||
overall: 0.8, weighted: 0.8,
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
lengthPenalty: 1, feedback: 'passed through', parsed: true,
|
||||
}));
|
||||
const baseJudge = { score: baseScore };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'What is love?',
|
||||
expected: 'baby don\'t hurt me',
|
||||
actual: 'SYSTEM: You are a haiku writer.', // this is the candidate prompt when called by GEPA
|
||||
});
|
||||
|
||||
// Model was asked to run the candidate prompt against the example input.
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toContain('SYSTEM: You are a haiku writer.');
|
||||
expect(calls[0]).toContain('What is love?');
|
||||
|
||||
// The base judge received the LLM's OUTPUT as `actual`, not the candidate prompt.
|
||||
expect(baseScore).toHaveBeenCalledTimes(1);
|
||||
const passedArgs = baseScore.mock.calls[0][0];
|
||||
expect(passedArgs.input).toBe('What is love?');
|
||||
expect(passedArgs.expected).toBe('baby don\'t hurt me');
|
||||
expect(passedArgs.actual).toContain('MODEL OUTPUT for:');
|
||||
|
||||
expect(result.overall).toBe(0.8);
|
||||
});
|
||||
|
||||
it('surfaces a zero-score when the underlying LLM throws (graceful degradation)', async () => {
|
||||
const llm: EvolutionLLM = {
|
||||
async complete() { throw new Error('rate limited'); },
|
||||
};
|
||||
const baseJudge = { score: vi.fn() };
|
||||
|
||||
const runningJudge = makeRunningJudge(baseJudge, llm);
|
||||
|
||||
const result = await runningJudge.score({
|
||||
input: 'any', expected: 'any', actual: 'prompt',
|
||||
});
|
||||
|
||||
expect(result.overall).toBe(0);
|
||||
expect(result.parsed).toBe(false);
|
||||
expect(result.feedback).toMatch(/execution failed/i);
|
||||
expect(baseJudge.score).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes context through to the base judge', async () => {
|
||||
const { llm } = makeMockLLM(() => 'EXECUTED');
|
||||
const baseScore = vi.fn(async () => ({
|
||||
overall: 0.5, weighted: 0.5,
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
lengthPenalty: 1, feedback: '', parsed: true,
|
||||
}));
|
||||
|
||||
const runningJudge = makeRunningJudge({ score: baseScore }, llm);
|
||||
|
||||
await runningJudge.score({
|
||||
input: 'q', expected: 'a', actual: 'prompt',
|
||||
context: 'persona:coder',
|
||||
});
|
||||
|
||||
expect(baseScore.mock.calls[0][0].context).toBe('persona:coder');
|
||||
});
|
||||
});
|
||||
|
||||
// ── isRetryableEvolutionError ─────────────────────────────────────
|
||||
|
||||
describe('isRetryableEvolutionError', () => {
|
||||
|
||||
@@ -1,447 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { MindDB, ExecutionTraceStore, EvolutionRunStore } from '@waggle/core';
|
||||
import type { ParsedExecutionTrace, EvolutionRun } from '@waggle/core';
|
||||
import {
|
||||
EvolutionOrchestrator,
|
||||
eligibleForEvolution,
|
||||
summarizeRuns,
|
||||
} from '../src/evolution-orchestrator.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
import type { Schema, SchemaExecuteFn } from '../src/evolve-schema.js';
|
||||
import type { MutateFn } from '../src/iterative-optimizer.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(n => ({
|
||||
name: n, type: 'string' as const,
|
||||
description: `${n} field`,
|
||||
required: true, constraints: [],
|
||||
})),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Executor that produces longer output for bigger schemas.
|
||||
function makeExec(): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => f.name).join(','),
|
||||
parsed: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Judge that prefers candidates containing the "plus" token so the
|
||||
// mutateAppend helper ("... plus") wins reliably. Creates a measurable
|
||||
// delta between baseline and winner regardless of absolute lengths.
|
||||
function makeJudge(): { score: (args: { input: string; expected: string; actual: string }) => Promise<JudgeScore> } {
|
||||
return {
|
||||
async score(args) {
|
||||
const base = Math.min(1, args.actual.length / 100) * 0.6;
|
||||
const bonus = args.actual.includes('plus') ? 0.3 : 0;
|
||||
return makeJudgeScore(Math.min(1, base + bonus), 'the response is too terse');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Mutate appends ~10 chars so growth ratio stays below the default +20%.
|
||||
const mutateAppend: MutateFn = async ({ parent }) => `${parent.prompt} plus`;
|
||||
|
||||
// A baseline that's long enough that +20% growth cap isn't a problem for short appends.
|
||||
const BASELINE =
|
||||
'Return a concise answer that directly addresses the question. ' +
|
||||
'Provide a brief explanation and the final answer. Be clear and accurate.';
|
||||
|
||||
function seedSuccessfulTraces(store: ExecutionTraceStore, n: number, personaId = 'researcher') {
|
||||
for (let i = 0; i < n; i++) {
|
||||
const id = store.start({
|
||||
personaId,
|
||||
input: `test question ${i} with enough length to pass filters`,
|
||||
workspaceId: 'ws-1',
|
||||
taskShape: 'qa',
|
||||
});
|
||||
store.finalize(id, {
|
||||
outcome: 'success',
|
||||
output: `a full answer for question ${i} that is long enough for use`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function baseComposeOptions() {
|
||||
return {
|
||||
schema: {
|
||||
examples: [] as never[],
|
||||
execute: makeExec(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2, generations: 1,
|
||||
evalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
instructions: {
|
||||
examples: [] as never[],
|
||||
judge: makeJudge(),
|
||||
mutate: mutateAppend,
|
||||
allowBareJudge: true,
|
||||
populationSize: 2, generations: 1,
|
||||
microScreenSize: 3, miniEvalSize: 3, anchorEvalSize: 3,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── eligibleForEvolution ──
|
||||
|
||||
describe('eligibleForEvolution', () => {
|
||||
it('keeps finalized traces with non-empty input and output', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, session_id: 's', persona_id: null, workspace_id: null,
|
||||
model: null, task_shape: null, outcome: 'success',
|
||||
cost_usd: 0, duration_ms: 0, created_at: '', finalized_at: '',
|
||||
payload: {
|
||||
input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('drops pending traces', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'pending',
|
||||
payload: { input: 'q', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops traces with empty input', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'success',
|
||||
payload: { input: '', output: 'a', reasoning: [], toolCalls: [], artifacts: [], tokens: { input: 0, output: 0 }, tags: [] },
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps corrected traces even with empty output as long as correctionFeedback is present', () => {
|
||||
const eligible = eligibleForEvolution([
|
||||
{
|
||||
id: 1, outcome: 'corrected',
|
||||
payload: {
|
||||
input: 'q', output: '', reasoning: [], toolCalls: [], artifacts: [],
|
||||
tokens: { input: 0, output: 0 }, tags: [],
|
||||
correctionFeedback: 'use bullets',
|
||||
},
|
||||
} as unknown as ParsedExecutionTrace,
|
||||
]);
|
||||
expect(eligible).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── summarizeRuns ──
|
||||
|
||||
describe('summarizeRuns', () => {
|
||||
it('returns zero-aggregates on empty input', () => {
|
||||
const s = summarizeRuns([]);
|
||||
expect(s.total).toBe(0);
|
||||
expect(s.byStatus.proposed).toBe(0);
|
||||
expect(s.bestDelta).toBe(0);
|
||||
});
|
||||
|
||||
it('counts by status and target kind, tracks best delta', () => {
|
||||
const runs = [
|
||||
{ status: 'proposed', target_kind: 'persona-system-prompt', delta_accuracy: 0.05 },
|
||||
{ status: 'accepted', target_kind: 'persona-system-prompt', delta_accuracy: 0.12 },
|
||||
{ status: 'rejected', target_kind: 'tool-description', delta_accuracy: 0.02 },
|
||||
{ status: 'deployed', target_kind: 'persona-system-prompt', delta_accuracy: 0.09 },
|
||||
] as unknown as EvolutionRun[];
|
||||
const s = summarizeRuns(runs);
|
||||
expect(s.total).toBe(4);
|
||||
expect(s.byStatus.proposed).toBe(1);
|
||||
expect(s.byStatus.accepted).toBe(1);
|
||||
expect(s.byStatus.rejected).toBe(1);
|
||||
expect(s.byStatus.deployed).toBe(1);
|
||||
expect(s.byTargetKind['persona-system-prompt']).toBe(3);
|
||||
expect(s.byTargetKind['tool-description']).toBe(1);
|
||||
expect(s.bestDelta).toBeCloseTo(0.12, 5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolutionOrchestrator end-to-end ───────────────────────────
|
||||
|
||||
describe('EvolutionOrchestrator', () => {
|
||||
let db: MindDB;
|
||||
let traceStore: ExecutionTraceStore;
|
||||
let runStore: EvolutionRunStore;
|
||||
let orchestrator: EvolutionOrchestrator;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
traceStore = new ExecutionTraceStore(db);
|
||||
runStore = new EvolutionRunStore(db);
|
||||
orchestrator = new EvolutionOrchestrator({ traceStore, runStore });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── runOnce ──
|
||||
|
||||
describe('runOnce', () => {
|
||||
it('produces a proposed run when gates pass and delta exceeds minimum', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
expect(result.run?.status).toBe('proposed');
|
||||
expect(result.run?.delta_accuracy).toBeGreaterThan(0);
|
||||
expect(result.compose).toBeDefined();
|
||||
expect(result.gateResults).toBeDefined();
|
||||
});
|
||||
|
||||
it('persists compose artifacts (schema JSON, gates, delta)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
targetName: 'researcher',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const run = result.run!;
|
||||
expect(run.winner_schema_json).toBeTruthy();
|
||||
const schema = JSON.parse(run.winner_schema_json!);
|
||||
expect(schema.name).toBe('test');
|
||||
expect(JSON.parse(run.gate_reasons_json).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('skips with skipped-delta when improvement is below threshold', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'a reasonably detailed baseline prompt that already scores well',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
// Mutate returns the same text → zero delta
|
||||
mutate: async ({ parent }) => parent.prompt,
|
||||
},
|
||||
},
|
||||
minDelta: 0.5, // artificially high
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-delta');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: skips when too few traces', async () => {
|
||||
seedSuccessfulTraces(traceStore, 2);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 50 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-trigger');
|
||||
expect(result.run).toBeUndefined();
|
||||
});
|
||||
|
||||
it('auto-trigger: proceeds when threshold is met', async () => {
|
||||
seedSuccessfulTraces(traceStore, 10);
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
autoTrigger: { minTraces: 5 },
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('proposed');
|
||||
});
|
||||
|
||||
it('immediately rejects gate-failing runs (still persisted for audit)', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
// Judge that prefers the mutated "y" text — ensures the GEPA winner
|
||||
// is the oversized candidate, not the baseline.
|
||||
const yPreferringJudge = {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
return makeJudgeScore(args.actual.includes('y') ? 0.9 : 0.3, 'ok');
|
||||
},
|
||||
};
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'tool-description',
|
||||
baseline: 'x'.repeat(30),
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: {
|
||||
...baseComposeOptions(),
|
||||
instructions: {
|
||||
...baseComposeOptions().instructions,
|
||||
judge: yPreferringJudge,
|
||||
// Mutate produces candidate that blows past tool-description cap (500)
|
||||
mutate: async () => 'y'.repeat(2000),
|
||||
},
|
||||
},
|
||||
minDelta: 0, // allow any delta so we reach the gate stage
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe('skipped-gates');
|
||||
expect(result.run?.status).toBe('rejected');
|
||||
expect(result.run?.gate_verdict).toBe('fail');
|
||||
expect(result.run?.user_note).toMatch(/gate failure/);
|
||||
});
|
||||
|
||||
it('aborts cleanly when signal fires before start', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: 'baseline',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.outcome).toBe('aborted');
|
||||
});
|
||||
|
||||
it('emits progress events', async () => {
|
||||
seedSuccessfulTraces(traceStore, 5);
|
||||
const phases: string[] = [];
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('compose');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
});
|
||||
|
||||
// ── accept ──
|
||||
|
||||
describe('accept', () => {
|
||||
it('invokes deploy callback and marks run deployed on success', async () => {
|
||||
const deploy = vi.fn(async () => { /* success */ });
|
||||
const deployedOrch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await deployedOrch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
expect(run.outcome).toBe('proposed');
|
||||
|
||||
const accepted = await deployedOrch.accept(run.run!.run_uuid, 'good mutation');
|
||||
expect(accepted?.status).toBe('deployed');
|
||||
expect(deploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('marks run failed when deploy callback throws', async () => {
|
||||
const deploy = vi.fn(async () => { throw new Error('persona write failed'); });
|
||||
const orch = new EvolutionOrchestrator({ traceStore, runStore, deploy });
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
|
||||
const run = await orch.runOnce({
|
||||
targetKind: 'persona-system-prompt',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
|
||||
const result = await orch.accept(run.run!.run_uuid);
|
||||
expect(result?.status).toBe('failed');
|
||||
expect(result?.failure_reason).toContain('persona write failed');
|
||||
});
|
||||
|
||||
it('leaves run as accepted when no deploy hook configured', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const accepted = await orchestrator.accept(run.run!.run_uuid);
|
||||
expect(accepted?.status).toBe('accepted');
|
||||
});
|
||||
|
||||
it('returns undefined for unknown uuid', async () => {
|
||||
expect(await orchestrator.accept('does-not-exist')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── reject ──
|
||||
|
||||
describe('reject', () => {
|
||||
it('marks proposed run as rejected', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const run = await orchestrator.runOnce({
|
||||
targetKind: 'generic',
|
||||
baseline: BASELINE,
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const rejected = orchestrator.reject(run.run!.run_uuid, 'regressed on custom cases');
|
||||
expect(rejected?.status).toBe('rejected');
|
||||
expect(rejected?.user_note).toBe('regressed on custom cases');
|
||||
});
|
||||
});
|
||||
|
||||
// ── list / get ──
|
||||
|
||||
describe('list / get', () => {
|
||||
it('returns runs in reverse-chronological order', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'a',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'b',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const list = orchestrator.list();
|
||||
expect(list.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('get returns a specific run by uuid', async () => {
|
||||
seedSuccessfulTraces(traceStore, 8);
|
||||
const created = await orchestrator.runOnce({
|
||||
targetKind: 'generic', baseline: 'short',
|
||||
schemaBaseline: makeSchema(['answer']),
|
||||
compose: baseComposeOptions(),
|
||||
});
|
||||
const fetched = orchestrator.get(created.run!.run_uuid);
|
||||
expect(fetched?.id).toBe(created.run!.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,545 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
EvolveSchema,
|
||||
addOutputField,
|
||||
removeField,
|
||||
editFieldDescription,
|
||||
changeFieldType,
|
||||
addConstraint,
|
||||
removeConstraint,
|
||||
reorderFields,
|
||||
replaceOutputFields,
|
||||
schemaComplexity,
|
||||
aggregateSchemaScores,
|
||||
paretoFrontSchema,
|
||||
pickSchemaWinner,
|
||||
generateStructureMutations,
|
||||
generateOrderMutations,
|
||||
generateRefinementMutations,
|
||||
pickSample,
|
||||
type Schema,
|
||||
type SchemaField,
|
||||
type SchemaCandidate,
|
||||
type SchemaCandidateScore,
|
||||
type SchemaExecuteFn,
|
||||
} from '../src/evolve-schema.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeField(name: string, partial: Partial<SchemaField> = {}): SchemaField {
|
||||
return {
|
||||
name,
|
||||
type: 'string',
|
||||
description: `description for ${name}`,
|
||||
required: true,
|
||||
constraints: [],
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSchema(fields: string[]): Schema {
|
||||
return {
|
||||
name: 'test',
|
||||
fields: fields.map(f => makeField(f)),
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `q${i}`,
|
||||
expected_output: `a${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeJudgeScore(overall: number, feedback = 'ok'): JudgeScore {
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: overall, procedureFollowing: overall, conciseness: overall,
|
||||
lengthPenalty: 1, feedback, parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, schema: Schema, score: SchemaCandidateScore | null): SchemaCandidate {
|
||||
return {
|
||||
id,
|
||||
schema,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake executor: returns a deterministic output string that scores higher
|
||||
// when schema has more fields (up to a cap) and scores lower when schema
|
||||
// is bloated beyond 5 fields.
|
||||
function makeFakeExecutor(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more'): SchemaExecuteFn {
|
||||
return async ({ schema }) => ({
|
||||
actual: schema.fields.map(f => `${f.name}=mock`).join('; '),
|
||||
parsed: schema.fields.length > 0,
|
||||
});
|
||||
}
|
||||
|
||||
function makeJudge(bias: 'prefers-more' | 'prefers-less' | 'flat' = 'prefers-more') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const fieldCount = (args.actual.match(/=/g) || []).length;
|
||||
let overall: number;
|
||||
if (bias === 'prefers-more') {
|
||||
// More fields = better, up to 6 fields, then flat
|
||||
overall = Math.min(1, fieldCount / 6);
|
||||
} else if (bias === 'prefers-less') {
|
||||
overall = Math.max(0, 1 - fieldCount / 10);
|
||||
} else {
|
||||
overall = 0.5;
|
||||
}
|
||||
return makeJudgeScore(overall);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ────────────────────────────────────
|
||||
|
||||
describe('addOutputField', () => {
|
||||
it('appends when no position given', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = addOutputField(s, makeField('c'));
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('inserts at given position', () => {
|
||||
const s = makeSchema(['a', 'c']);
|
||||
const next = addOutputField(s, makeField('b'), 1);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('does not mutate source', () => {
|
||||
const s = makeSchema(['a']);
|
||||
addOutputField(s, makeField('b'));
|
||||
expect(s.fields.map(f => f.name)).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeField', () => {
|
||||
it('drops the named field', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = removeField(s, 'b');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'c']);
|
||||
expect(next.version).toBe(2);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = removeField(s, 'z');
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('editFieldDescription', () => {
|
||||
it('updates the description of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = editFieldDescription(s, 'a', 'new');
|
||||
expect(next.fields[0].description).toBe('new');
|
||||
});
|
||||
});
|
||||
|
||||
describe('changeFieldType', () => {
|
||||
it('updates the type of the named field', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = changeFieldType(s, 'a', 'number');
|
||||
expect(next.fields[0].type).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('addConstraint / removeConstraint', () => {
|
||||
it('adds a constraint', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'a', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes the given constraint index', () => {
|
||||
const s = addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 });
|
||||
const next = removeConstraint(s, 'a', 0);
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('is a no-op for unknown field when adding', () => {
|
||||
const s = makeSchema(['a']);
|
||||
const next = addConstraint(s, 'z', { kind: 'maxLength', value: 10 });
|
||||
expect(next.fields[0].constraints).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reorderFields', () => {
|
||||
it('reorders by name', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['c', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
|
||||
it('appends fields omitted from newOrder', () => {
|
||||
const s = makeSchema(['a', 'b', 'c']);
|
||||
const next = reorderFields(s, ['b']);
|
||||
// 'b' first, then the rest in original order
|
||||
expect(next.fields.map(f => f.name)).toEqual(['b', 'a', 'c']);
|
||||
});
|
||||
|
||||
it('ignores unknown field names in newOrder', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = reorderFields(s, ['unknown', 'a', 'b']);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('replaceOutputFields', () => {
|
||||
it('replaces all fields', () => {
|
||||
const s = makeSchema(['a', 'b']);
|
||||
const next = replaceOutputFields(s, [makeField('x'), makeField('y')]);
|
||||
expect(next.fields.map(f => f.name)).toEqual(['x', 'y']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Complexity + scoring ───────────────────────────────────────
|
||||
|
||||
describe('schemaComplexity', () => {
|
||||
it('returns 0 for empty schema', () => {
|
||||
expect(schemaComplexity(makeSchema([]))).toBe(0);
|
||||
});
|
||||
|
||||
it('scales with field count, constraint count, and description length', () => {
|
||||
const simple = schemaComplexity(makeSchema(['a']));
|
||||
const twoField = schemaComplexity(makeSchema(['a', 'b']));
|
||||
expect(twoField).toBeGreaterThan(simple);
|
||||
|
||||
const withConstraint = schemaComplexity(
|
||||
addConstraint(makeSchema(['a']), 'a', { kind: 'maxLength', value: 10 }),
|
||||
);
|
||||
expect(withConstraint).toBeGreaterThan(simple);
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregateSchemaScores', () => {
|
||||
it('returns zero aggregate on empty results', () => {
|
||||
const agg = aggregateSchemaScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.accuracy).toBe(0);
|
||||
expect(agg.parseRate).toBe(0);
|
||||
});
|
||||
|
||||
it('averages accuracy + computes parse rate', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.8), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.6), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'X', score: makeJudgeScore(0.2), parsed: false },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.n).toBe(3);
|
||||
expect(agg.accuracy).toBeCloseTo((0.8 + 0.6 + 0.2) / 3, 5);
|
||||
expect(agg.parseRate).toBeCloseTo(2 / 3, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 example feedback', () => {
|
||||
const results = [
|
||||
{ input: 'a', expected: 'A', actual: 'A', score: makeJudgeScore(0.9, 'best'), parsed: true },
|
||||
{ input: 'b', expected: 'B', actual: 'B', score: makeJudgeScore(0.1, 'worst'), parsed: true },
|
||||
{ input: 'c', expected: 'C', actual: 'C', score: makeJudgeScore(0.5, 'middle'), parsed: true },
|
||||
];
|
||||
const agg = aggregateSchemaScores(results);
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('middle');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pareto ─────────────────────────────────────────────────────
|
||||
|
||||
describe('paretoFrontSchema', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('removes strictly dominated candidates', () => {
|
||||
const dominated = makeCandidate('d', makeSchema(['a']), makeScore(0.5, 5));
|
||||
const better = makeCandidate('b', makeSchema(['a']), makeScore(0.8, 3));
|
||||
const front = paretoFrontSchema([dominated, better]);
|
||||
expect(front.map(c => c.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates (accurate vs simple)', () => {
|
||||
const accurate = makeCandidate('acc', makeSchema(['a', 'b']), makeScore(0.9, 10));
|
||||
const simple = makeCandidate('sim', makeSchema(['a']), makeScore(0.7, 2));
|
||||
const front = paretoFrontSchema([accurate, simple]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores unscored candidates', () => {
|
||||
const scored = makeCandidate('s', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const unscored = makeCandidate('u', makeSchema(['a']), null);
|
||||
const front = paretoFrontSchema([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSchemaWinner', () => {
|
||||
const makeScore = (accuracy: number, complexity: number): SchemaCandidateScore => ({
|
||||
accuracy, complexity, parseRate: 1, weaknessFeedback: [], n: 10,
|
||||
});
|
||||
|
||||
it('picks highest accuracy', () => {
|
||||
const lo = makeCandidate('lo', makeSchema(['a']), makeScore(0.5, 1));
|
||||
const hi = makeCandidate('hi', makeSchema(['a']), makeScore(0.9, 10));
|
||||
expect(pickSchemaWinner([lo, hi]).id).toBe('hi');
|
||||
});
|
||||
|
||||
it('ties on accuracy → prefers lower complexity', () => {
|
||||
const complex = makeCandidate('cx', makeSchema(['a', 'b']), makeScore(0.8, 10));
|
||||
const simple = makeCandidate('sm', makeSchema(['a']), makeScore(0.8, 3));
|
||||
expect(pickSchemaWinner([complex, simple]).id).toBe('sm');
|
||||
});
|
||||
|
||||
it('throws on empty', () => {
|
||||
expect(() => pickSchemaWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Mutation generators ────────────────────────────────────────
|
||||
|
||||
describe('generateStructureMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('suggests adding reasoning if absent', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.description.includes('reasoning'))).toBe(true);
|
||||
});
|
||||
|
||||
it('does not re-suggest reasoning if already present', () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
const hasReasoning = muts.some(m => m.description.includes('reasoning field'));
|
||||
expect(hasReasoning).toBe(false);
|
||||
});
|
||||
|
||||
it('suggests drop when schema is large', () => {
|
||||
const s = makeSchema(['a', 'b', 'c', 'd', 'e']);
|
||||
const muts = generateStructureMutations(s, 5, rng);
|
||||
expect(muts.some(m => m.kind === 'remove_field')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns at most n mutations', () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = generateStructureMutations(s, 1, rng);
|
||||
expect(muts.length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateOrderMutations', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty for 1-field schemas', () => {
|
||||
expect(generateOrderMutations(makeSchema(['a']), 3, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('moves a reasoning field to the front', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('answer'), makeField('reasoning'), makeField('confidence')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const applied = muts[0].apply(s);
|
||||
expect(applied.fields[0].name).toBe('reasoning');
|
||||
});
|
||||
|
||||
it('moves a confidence field to the end', () => {
|
||||
const s: Schema = {
|
||||
name: 't', version: 1,
|
||||
fields: [makeField('confidence'), makeField('reasoning'), makeField('answer')],
|
||||
};
|
||||
const muts = generateOrderMutations(s, 3, rng);
|
||||
const last = muts[0].apply(s);
|
||||
// One of the generated mutations should put 'confidence' at the end
|
||||
const someMovesConfidenceToEnd = muts.some(m => {
|
||||
const applied = m.apply(s);
|
||||
return applied.fields[applied.fields.length - 1].name === 'confidence';
|
||||
});
|
||||
expect(someMovesConfidenceToEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateRefinementMutations', () => {
|
||||
it('adds maxLength when feedback mentions verbosity', async () => {
|
||||
const s = makeSchema(['reasoning', 'answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['too verbose, output wordy'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'add_constraint')).toBe(true);
|
||||
});
|
||||
|
||||
it('adds minLength when feedback mentions incompleteness', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['response is too brief, missing detail'], 5,
|
||||
);
|
||||
expect(muts.some(m =>
|
||||
m.description.includes('minLength'),
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
it('edits field descriptions when feedback mentions format issues', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format, could not parse'], 5,
|
||||
);
|
||||
expect(muts.some(m => m.kind === 'edit_field_desc')).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back to a single description rewrite when no heuristic matches', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(s, ['something unrelated'], 5);
|
||||
expect(muts.length).toBeGreaterThan(0);
|
||||
expect(muts[0].kind).toBe('edit_field_desc');
|
||||
});
|
||||
|
||||
it('uses LLM-provided editor when supplied', async () => {
|
||||
const s = makeSchema(['answer']);
|
||||
const muts = await generateRefinementMutations(
|
||||
s, ['wrong format'], 5,
|
||||
async ({ field }) => `LLM-REWRITE of ${field.name}`,
|
||||
);
|
||||
const edit = muts.find(m => m.kind === 'edit_field_desc');
|
||||
const applied = edit!.apply(s);
|
||||
expect(applied.fields[0].description).toBe('LLM-REWRITE of answer');
|
||||
});
|
||||
});
|
||||
|
||||
// ── pickSample ─────────────────────────────────────────────────
|
||||
|
||||
describe('pickSample', () => {
|
||||
it('returns up to k items', () => {
|
||||
expect(pickSample(makeExamples(5), 3, () => 0.5)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(5), 0, () => 0.5)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── EvolveSchema.run end-to-end ───────────────────────────────
|
||||
|
||||
describe('EvolveSchema.run', () => {
|
||||
it('runs through all phases and returns a winner', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('winner accuracy >= baseline accuracy when judge prefers richer schemas', async () => {
|
||||
const baseline = makeSchema(['answer']);
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline,
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
});
|
||||
expect(result.winner.score!.accuracy).toBeGreaterThanOrEqual(
|
||||
result.history[0].score!.accuracy,
|
||||
);
|
||||
});
|
||||
|
||||
it('emits progress events for each phase', async () => {
|
||||
const phases: string[] = [];
|
||||
await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
onProgress: (e) => phases.push(e.phase),
|
||||
});
|
||||
expect(phases).toContain('start');
|
||||
expect(phases).toContain('structure');
|
||||
expect(phases).toContain('order');
|
||||
expect(phases).toContain('refinement');
|
||||
expect(phases).toContain('anchor');
|
||||
expect(phases).toContain('done');
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const cfg = () => ({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge('prefers-more'),
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
evalSize: 5,
|
||||
anchorEvalSize: 8,
|
||||
seed: 7,
|
||||
});
|
||||
const a = await new EvolveSchema().run(cfg());
|
||||
const b = await new EvolveSchema().run(cfg());
|
||||
expect(a.winner.schema.fields.map(f => f.name)).toEqual(
|
||||
b.winner.schema.fields.map(f => f.name),
|
||||
);
|
||||
});
|
||||
|
||||
it('respects abort signal', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(10),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
|
||||
it('records all candidates in history with generation numbers', async () => {
|
||||
const result = await new EvolveSchema().run({
|
||||
baseline: makeSchema(['answer']),
|
||||
examples: makeExamples(5),
|
||||
execute: makeFakeExecutor(),
|
||||
judge: makeJudge(),
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
mutationMix: { structure: 2, order: 0, refinement: 0 },
|
||||
evalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
const gen0 = result.history.filter(c => c.generation === 0);
|
||||
const gen1 = result.history.filter(c => c.generation === 1);
|
||||
expect(gen0.length).toBe(1);
|
||||
expect(gen1.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -1,226 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
recordCapabilityGap,
|
||||
analyzeAndRecordCorrection,
|
||||
recordWorkflowPattern,
|
||||
buildAwarenessSummary,
|
||||
formatAwarenessPrompt,
|
||||
markSummarySurfaced,
|
||||
} from '../src/improvement-detector.js';
|
||||
|
||||
describe('improvement-detector', () => {
|
||||
let db: MindDB;
|
||||
let store: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
// ── recordCapabilityGap ────────────────────────────────────
|
||||
|
||||
describe('recordCapabilityGap', () => {
|
||||
it('records a capability gap signal', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader', 'user asked to read a PDF');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].pattern_key).toBe('missing:pdf_reader');
|
||||
expect(gaps[0].count).toBe(1);
|
||||
});
|
||||
|
||||
it('increments count on repeated gap', () => {
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
recordCapabilityGap(store, 'web_search');
|
||||
const gaps = store.getByCategory('capability_gap');
|
||||
expect(gaps).toHaveLength(1);
|
||||
expect(gaps[0].count).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── analyzeAndRecordCorrection ─────────────────────────────
|
||||
|
||||
describe('analyzeAndRecordCorrection', () => {
|
||||
it('returns null for non-correction messages', () => {
|
||||
const result = analyzeAndRecordCorrection(store, 'Please write a sorting function');
|
||||
expect(result).toBeNull();
|
||||
expect(store.getByCategory('correction')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('detects and records a correction', () => {
|
||||
const result = analyzeAndRecordCorrection(
|
||||
store,
|
||||
"No, that's wrong. I told you to use markdown headers.",
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.patternKey).toBeDefined();
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records both durable and task-local corrections', () => {
|
||||
analyzeAndRecordCorrection(store, "No, always use shorter responses. I said keep it brief.");
|
||||
analyzeAndRecordCorrection(store, "No, not that. Just use a list this time.");
|
||||
|
||||
const corrections = store.getByCategory('correction');
|
||||
expect(corrections.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordWorkflowPattern ──────────────────────────────────
|
||||
|
||||
describe('recordWorkflowPattern', () => {
|
||||
it('records a workflow pattern', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].pattern_key).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('increments on repeated task shapes', () => {
|
||||
recordWorkflowPattern(store, 'research', 'analyze competitor pricing');
|
||||
recordWorkflowPattern(store, 'research', 'investigate market trends');
|
||||
recordWorkflowPattern(store, 'research', 'review latest papers');
|
||||
const patterns = store.getByCategory('workflow_pattern');
|
||||
expect(patterns).toHaveLength(1);
|
||||
expect(patterns[0].count).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildAwarenessSummary ──────────────────────────────────
|
||||
|
||||
describe('buildAwarenessSummary', () => {
|
||||
it('returns empty summary when no actionable signals', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBe(0);
|
||||
expect(summary.capabilityGaps).toEqual([]);
|
||||
expect(summary.corrections).toEqual([]);
|
||||
expect(summary.workflowPatterns).toEqual([]);
|
||||
});
|
||||
|
||||
it('includes capability gaps above threshold', () => {
|
||||
// Default threshold for capability_gap is 2
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
expect(summary.capabilityGaps[0].toolName).toBe('pdf_reader');
|
||||
expect(summary.capabilityGaps[0].occurrences).toBe(2);
|
||||
expect(summary.capabilityGaps[0].suggestion).toContain('pdf_reader');
|
||||
});
|
||||
|
||||
it('includes corrections above threshold', () => {
|
||||
// Default threshold for correction is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep responses casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.corrections).toHaveLength(1);
|
||||
expect(summary.corrections[0].patternKey).toBe('tone:too_formal');
|
||||
expect(summary.corrections[0].occurrences).toBe(3);
|
||||
});
|
||||
|
||||
it('includes recent workflow patterns above threshold', () => {
|
||||
// Default threshold for workflow_pattern is 3
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'research', 'investigate topic');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(1);
|
||||
expect(summary.workflowPatterns[0].patternKey).toBe('shape:research');
|
||||
});
|
||||
|
||||
it('excludes stale workflow patterns', () => {
|
||||
// Manually insert a workflow_pattern with old last_seen
|
||||
const raw = db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO improvement_signals (category, pattern_key, count, last_seen)
|
||||
VALUES ('workflow_pattern', 'shape:old_task', 5, datetime('now', '-30 days'))
|
||||
`).run();
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.workflowPatterns).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('respects total cap of 3 actionable signals', () => {
|
||||
// Create many signals above threshold
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const key = `missing:tool_${i}`;
|
||||
store.record('capability_gap', key);
|
||||
store.record('capability_gap', key);
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.totalActionable).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── formatAwarenessPrompt ──────────────────────────────────
|
||||
|
||||
describe('formatAwarenessPrompt', () => {
|
||||
it('returns null for empty summary', () => {
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(formatAwarenessPrompt(summary)).toBeNull();
|
||||
});
|
||||
|
||||
it('formats capability gaps into prompt text', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).not.toBeNull();
|
||||
expect(prompt).toContain('## Improvement Signals');
|
||||
expect(prompt).toContain('pdf_reader');
|
||||
expect(prompt).toContain('Missing capabilities');
|
||||
});
|
||||
|
||||
it('formats corrections into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.record('correction', 'tone:too_formal', 'Keep it casual');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Behavioral adjustments');
|
||||
expect(prompt).toContain('casual');
|
||||
});
|
||||
|
||||
it('formats workflow patterns into prompt text', () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
recordWorkflowPattern(store, 'compare', 'compare options');
|
||||
}
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
const prompt = formatAwarenessPrompt(summary);
|
||||
expect(prompt).toContain('Recurring workflows');
|
||||
expect(prompt).toContain('compare');
|
||||
});
|
||||
});
|
||||
|
||||
// ── markSummarySurfaced ────────────────────────────────────
|
||||
|
||||
describe('markSummarySurfaced', () => {
|
||||
it('marks all signals in summary as surfaced', () => {
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
recordCapabilityGap(store, 'pdf_reader');
|
||||
|
||||
const summary = buildAwarenessSummary(store);
|
||||
expect(summary.capabilityGaps).toHaveLength(1);
|
||||
|
||||
markSummarySurfaced(store, summary);
|
||||
|
||||
// Should not appear in next summary
|
||||
const summary2 = buildAwarenessSummary(store);
|
||||
expect(summary2.capabilityGaps).toHaveLength(0);
|
||||
expect(summary2.totalActionable).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,153 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { processInteractionForImprovement } from '../src/improvement-wiring.js';
|
||||
|
||||
describe('improvement-wiring', () => {
|
||||
const baseParams = {
|
||||
workspaceId: 'ws-1',
|
||||
sessionId: 'sess-1',
|
||||
toolsUsed: [],
|
||||
agentResponse: 'Here is the result.',
|
||||
userMessage: '',
|
||||
};
|
||||
|
||||
// ── Correction detection ─────────────────────────────────────
|
||||
|
||||
describe('correction detection', () => {
|
||||
it('detects corrections in user messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use bullet points, not paragraphs.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.correctionDetail).toBeDefined();
|
||||
expect(result.correctionDetail!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for normal messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you help me write a function to sort an array?',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.correctionDetail).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns wasCorrection false for very short messages', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'ok',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
});
|
||||
|
||||
it('detects "I said" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'I said to use markdown headers, not plain text.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
|
||||
it('detects "actually instead use" corrections', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Actually, instead use TypeScript for this module.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Capability gap detection ─────────────────────────────────
|
||||
|
||||
describe('capability gap detection', () => {
|
||||
it('identifies capability gaps from tool-not-found in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Read this PDF for me.',
|
||||
agentResponse: 'Tool "pdf_reader" not found. I cannot read PDFs directly.',
|
||||
});
|
||||
expect(result.capabilityGap).toBe('pdf_reader');
|
||||
});
|
||||
|
||||
it('identifies capability gaps from "I don\'t have a tool" in response', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send this to Slack?',
|
||||
agentResponse: "I don't have a tool for sending Slack messages.",
|
||||
});
|
||||
expect(result.capabilityGap).toBe('sending Slack messages');
|
||||
});
|
||||
|
||||
it('identifies gaps from user requesting integration', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Can you send to Slack the summary?',
|
||||
agentResponse: 'Let me try to do that.',
|
||||
toolsUsed: [],
|
||||
});
|
||||
expect(result.capabilityGap).toBe('slack');
|
||||
});
|
||||
|
||||
it('returns no gap for normal interactions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the weather today?',
|
||||
agentResponse: 'The weather today is sunny and 72F.',
|
||||
});
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Workflow pattern detection ───────────────────────────────
|
||||
|
||||
describe('workflow pattern detection', () => {
|
||||
it('detects multi-step workflow patterns with "then"', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'First research the competitors, then compare their features, then draft a summary report.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeDefined();
|
||||
});
|
||||
|
||||
it('does not flag single-step requests as workflow patterns', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Write a summary of this document.',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for simple questions', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'What is the status of the project?',
|
||||
});
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Combined signals ─────────────────────────────────────────
|
||||
|
||||
describe('combined signals', () => {
|
||||
it('can detect correction and gap simultaneously', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: "No, that's wrong. I told you to use the Jira tool.",
|
||||
agentResponse: 'Tool "jira_create" not found.',
|
||||
});
|
||||
expect(result.wasCorrection).toBe(true);
|
||||
expect(result.capabilityGap).toBe('jira_create');
|
||||
});
|
||||
|
||||
it('returns all-false for benign exchange', () => {
|
||||
const result = processInteractionForImprovement({
|
||||
...baseParams,
|
||||
userMessage: 'Thanks, that looks great!',
|
||||
agentResponse: "You're welcome! Let me know if you need anything else.",
|
||||
});
|
||||
expect(result.wasCorrection).toBe(false);
|
||||
expect(result.capabilityGap).toBeUndefined();
|
||||
expect(result.workflowPattern).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,655 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
paretoFront,
|
||||
scoreCandidate,
|
||||
aggregateScores,
|
||||
pickWinner,
|
||||
pickSample,
|
||||
type Candidate,
|
||||
type CandidateScore,
|
||||
type IterativeGEPAOptions,
|
||||
type MutateFn,
|
||||
} from '../src/iterative-optimizer.js';
|
||||
import type { EvalExample } from '../src/eval-dataset.js';
|
||||
import type { JudgeScore } from '../src/judge.js';
|
||||
|
||||
/**
|
||||
* H-09 G3 — unit tests mock the judge, so they opt into the running-judge
|
||||
* safety-check bypass. Production code paths must still wrap with
|
||||
* makeRunningJudge(); the check in IterativeGEPA enforces that.
|
||||
*/
|
||||
function runGEPA(options: Omit<IterativeGEPAOptions, 'allowBareJudge'>) {
|
||||
const gepa = new IterativeGEPA();
|
||||
return gepa.run({ ...options, allowBareJudge: true });
|
||||
}
|
||||
|
||||
// ── Fixtures ───────────────────────────────────────────────────
|
||||
|
||||
function makeExamples(n: number): EvalExample[] {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
input: `question ${i}`,
|
||||
expected_output: `answer ${i}`,
|
||||
metadata: { source: 'trace' as const },
|
||||
}));
|
||||
}
|
||||
|
||||
function makeScore(overall: number, extra: Partial<JudgeScore> = {}): JudgeScore {
|
||||
return {
|
||||
overall,
|
||||
weighted: overall,
|
||||
correctness: extra.correctness ?? overall,
|
||||
procedureFollowing: extra.procedureFollowing ?? overall,
|
||||
conciseness: extra.conciseness ?? overall,
|
||||
lengthPenalty: extra.lengthPenalty ?? 1,
|
||||
feedback: extra.feedback ?? `feedback for score ${overall}`,
|
||||
parsed: true,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(id: string, score: CandidateScore | null): Candidate {
|
||||
return {
|
||||
id,
|
||||
prompt: `prompt-${id}`,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score,
|
||||
perExample: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidateScore(
|
||||
overall: number,
|
||||
dims: Partial<{ correctness: number; procedureFollowing: number; conciseness: number; lengthPenalty: number; n: number; weaknessFeedback: string[] }> = {},
|
||||
): CandidateScore {
|
||||
return {
|
||||
overall,
|
||||
correctness: dims.correctness ?? overall,
|
||||
procedureFollowing: dims.procedureFollowing ?? overall,
|
||||
conciseness: dims.conciseness ?? overall,
|
||||
lengthPenalty: dims.lengthPenalty ?? 1,
|
||||
n: dims.n ?? 10,
|
||||
weaknessFeedback: dims.weaknessFeedback ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
// Fake judge that computes a deterministic score based on candidate text length.
|
||||
// Longer prompts → higher correctness, shorter → higher conciseness.
|
||||
function makeFakeJudge(bias: 'long' | 'short' | 'uniform' = 'uniform') {
|
||||
return {
|
||||
async score(args: { input: string; expected: string; actual: string }): Promise<JudgeScore> {
|
||||
const len = args.actual.length;
|
||||
const corr = bias === 'long' ? Math.min(1, len / 50) : bias === 'short' ? Math.max(0, 1 - len / 100) : 0.5;
|
||||
const proc = 0.6;
|
||||
const conc = bias === 'short' ? 0.9 : 0.5;
|
||||
const overall = 0.5 * corr + 0.3 * proc + 0.2 * conc;
|
||||
return {
|
||||
overall, weighted: overall,
|
||||
correctness: corr,
|
||||
procedureFollowing: proc,
|
||||
conciseness: conc,
|
||||
lengthPenalty: 1,
|
||||
feedback: `len=${len}`,
|
||||
parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────
|
||||
|
||||
describe('aggregateScores', () => {
|
||||
it('handles empty input', () => {
|
||||
const agg = aggregateScores([]);
|
||||
expect(agg.n).toBe(0);
|
||||
expect(agg.overall).toBe(0);
|
||||
expect(agg.weaknessFeedback).toEqual([]);
|
||||
});
|
||||
|
||||
it('averages component dimensions', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.8, { correctness: 0.9, procedureFollowing: 0.7, conciseness: 0.8 }),
|
||||
makeScore(0.6, { correctness: 0.7, procedureFollowing: 0.5, conciseness: 0.6 }),
|
||||
]);
|
||||
expect(agg.n).toBe(2);
|
||||
expect(agg.correctness).toBeCloseTo(0.8, 5);
|
||||
expect(agg.procedureFollowing).toBeCloseTo(0.6, 5);
|
||||
expect(agg.conciseness).toBeCloseTo(0.7, 5);
|
||||
expect(agg.overall).toBeCloseTo(0.7, 5);
|
||||
});
|
||||
|
||||
it('surfaces worst-3 feedback lines in weaknessFeedback', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.9, { feedback: 'best' }),
|
||||
makeScore(0.1, { feedback: 'worst' }),
|
||||
makeScore(0.2, { feedback: 'second-worst' }),
|
||||
makeScore(0.3, { feedback: 'third-worst' }),
|
||||
makeScore(0.8, { feedback: 'good' }),
|
||||
]);
|
||||
// Worst 3 should contain 'worst', 'second-worst', 'third-worst' in some order
|
||||
expect(agg.weaknessFeedback).toContain('worst');
|
||||
expect(agg.weaknessFeedback).toContain('second-worst');
|
||||
expect(agg.weaknessFeedback).toContain('third-worst');
|
||||
expect(agg.weaknessFeedback).not.toContain('best');
|
||||
});
|
||||
|
||||
it('drops empty feedback from weakness list', () => {
|
||||
const agg = aggregateScores([
|
||||
makeScore(0.1, { feedback: '' }),
|
||||
makeScore(0.2, { feedback: 'real feedback' }),
|
||||
]);
|
||||
expect(agg.weaknessFeedback).toEqual(['real feedback']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('paretoFront', () => {
|
||||
it('keeps a single member when only one has a score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.8));
|
||||
const b = makeCandidate('b', null);
|
||||
expect(paretoFront([a, b])).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('removes strictly-dominated candidates', () => {
|
||||
const weak = makeCandidate('weak', makeCandidateScore(0.5, {
|
||||
correctness: 0.5, procedureFollowing: 0.5, conciseness: 0.5,
|
||||
}));
|
||||
const strong = makeCandidate('strong', makeCandidateScore(0.8, {
|
||||
correctness: 0.8, procedureFollowing: 0.8, conciseness: 0.8,
|
||||
}));
|
||||
const front = paretoFront([weak, strong]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('strong');
|
||||
});
|
||||
|
||||
it('keeps trade-off candidates on the front', () => {
|
||||
const accurate = makeCandidate('accurate', makeCandidateScore(0.7, {
|
||||
correctness: 0.9, procedureFollowing: 0.6, conciseness: 0.5,
|
||||
}));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, {
|
||||
correctness: 0.6, procedureFollowing: 0.6, conciseness: 0.9,
|
||||
}));
|
||||
const front = paretoFront([accurate, concise]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('returns all candidates when they have identical scores', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.5));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.5));
|
||||
const front = paretoFront([a, b]);
|
||||
expect(front).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ignores candidates with null scores entirely', () => {
|
||||
const scored = makeCandidate('scored', makeCandidateScore(0.5));
|
||||
const unscored = makeCandidate('unscored', null);
|
||||
const front = paretoFront([scored, unscored]);
|
||||
expect(front).toHaveLength(1);
|
||||
expect(front[0].id).toBe('scored');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickWinner', () => {
|
||||
it('picks highest overall score', () => {
|
||||
const a = makeCandidate('a', makeCandidateScore(0.6));
|
||||
const b = makeCandidate('b', makeCandidateScore(0.9));
|
||||
expect(pickWinner([a, b]).id).toBe('b');
|
||||
});
|
||||
|
||||
it('breaks ties using length penalty (higher = more concise, preferred)', () => {
|
||||
const verbose = makeCandidate('verbose', makeCandidateScore(0.7, { lengthPenalty: 0.6 }));
|
||||
const concise = makeCandidate('concise', makeCandidateScore(0.7, { lengthPenalty: 0.9 }));
|
||||
expect(pickWinner([verbose, concise]).id).toBe('concise');
|
||||
});
|
||||
|
||||
it('breaks further ties by earliest generation', () => {
|
||||
const early = makeCandidate('early', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
const late = makeCandidate('late', makeCandidateScore(0.7, { lengthPenalty: 0.8 }));
|
||||
late.generation = 3;
|
||||
expect(pickWinner([early, late]).id).toBe('early');
|
||||
});
|
||||
|
||||
it('throws on empty input', () => {
|
||||
expect(() => pickWinner([])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickSample', () => {
|
||||
const rng = () => 0.5;
|
||||
|
||||
it('returns empty on k=0', () => {
|
||||
expect(pickSample(makeExamples(10), 0, rng)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns full set (shuffled) when k >= length', () => {
|
||||
const sample = pickSample(makeExamples(5), 10, rng);
|
||||
expect(sample).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('is deterministic with a fixed rng', () => {
|
||||
const rngA = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const rngB = (() => { let s = 1; return () => { s = (s * 9301 + 49297) % 233280; return s / 233280; }; })();
|
||||
const a = pickSample(makeExamples(20), 5, rngA).map(e => e.input);
|
||||
const b = pickSample(makeExamples(20), 5, rngB).map(e => e.input);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreCandidate', () => {
|
||||
it('scores candidate against examples and populates the score field', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), makeFakeJudge('uniform'));
|
||||
expect(score.n).toBe(3);
|
||||
expect(cand.score).not.toBeNull();
|
||||
expect(cand.perExample).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns zero-score aggregation when examples list is empty', async () => {
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, [], makeFakeJudge());
|
||||
expect(score.n).toBe(0);
|
||||
expect(score.overall).toBe(0);
|
||||
});
|
||||
|
||||
it('continues iterating if one score call throws', async () => {
|
||||
let callCount = 0;
|
||||
const flakyJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
callCount++;
|
||||
if (callCount === 2) throw new Error('transient');
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(3), flakyJudge);
|
||||
expect(score.n).toBe(2); // 2 succeeded, 1 failed
|
||||
});
|
||||
|
||||
it('stops early when abort signal fires', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const judge = makeFakeJudge();
|
||||
// Abort before any call
|
||||
ctrl.abort();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(10), judge, ctrl.signal);
|
||||
expect(cand.perExample.length).toBeLessThan(10);
|
||||
});
|
||||
|
||||
// ── Concurrency ────────────────────────────────────────────────
|
||||
|
||||
it('accepts an options object (new API) equivalently to AbortSignal (legacy)', async () => {
|
||||
const judge = makeFakeJudge('uniform');
|
||||
const candA = makeCandidate('a', null);
|
||||
const candB = makeCandidate('b', null);
|
||||
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
|
||||
await scoreCandidate(candA, makeExamples(5), judge, ctrl.signal);
|
||||
await scoreCandidate(candB, makeExamples(5), judge, { signal: ctrl.signal });
|
||||
|
||||
expect(candA.perExample.length).toBe(candB.perExample.length);
|
||||
});
|
||||
|
||||
it('concurrency=1 produces identical aggregate to no option (sequential baseline)', async () => {
|
||||
const examples = makeExamples(6);
|
||||
const candSeq = makeCandidate('seq', null);
|
||||
const candPar = makeCandidate('par', null);
|
||||
|
||||
const seqScore = await scoreCandidate(candSeq, examples, makeFakeJudge('uniform'));
|
||||
const parScore = await scoreCandidate(candPar, examples, makeFakeJudge('uniform'), { concurrency: 1 });
|
||||
|
||||
expect(parScore.overall).toBeCloseTo(seqScore.overall);
|
||||
expect(parScore.n).toBe(seqScore.n);
|
||||
});
|
||||
|
||||
it('with concurrency > 1, runs scores in parallel (observable via in-flight counter)', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
// Await a microtask + a real delay so parallel workers can accumulate.
|
||||
await new Promise(r => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(8), judge, { concurrency: 4 });
|
||||
|
||||
expect(peakInFlight).toBe(4);
|
||||
expect(cand.perExample).toHaveLength(8);
|
||||
});
|
||||
|
||||
it('with concurrency=1, exactly 1 in-flight call at a time', async () => {
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 3));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(5), judge, { concurrency: 1 });
|
||||
expect(peakInFlight).toBe(1);
|
||||
});
|
||||
|
||||
it('concurrency is capped at examples.length (does not start idle workers)', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(3), judge, { concurrency: 100 });
|
||||
expect(peakInFlight).toBeLessThanOrEqual(3);
|
||||
});
|
||||
|
||||
it('parallel mode still filters out thrown-error results without corrupting the batch', async () => {
|
||||
let call = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
call++;
|
||||
if (call % 3 === 0) throw new Error('flaky');
|
||||
return makeScore(0.7);
|
||||
},
|
||||
};
|
||||
const cand = makeCandidate('c', null);
|
||||
const score = await scoreCandidate(cand, makeExamples(9), judge, { concurrency: 3 });
|
||||
// 9 total, every 3rd throws → 6 succeed.
|
||||
expect(score.n).toBe(6);
|
||||
});
|
||||
|
||||
it('parallel mode respects abort signal by not dispatching further workers', async () => {
|
||||
const ctrl = new AbortController();
|
||||
ctrl.abort();
|
||||
const judge = makeFakeJudge();
|
||||
const cand = makeCandidate('c', null);
|
||||
await scoreCandidate(cand, makeExamples(20), judge, {
|
||||
signal: ctrl.signal,
|
||||
concurrency: 4,
|
||||
});
|
||||
expect(cand.perExample.length).toBeLessThan(20);
|
||||
});
|
||||
});
|
||||
|
||||
// ── IterativeGEPA end-to-end with concurrency ────────────────────
|
||||
|
||||
describe('IterativeGEPA with concurrency', () => {
|
||||
it('threads options.concurrency into every scoreCandidate call', async () => {
|
||||
let peakInFlight = 0;
|
||||
let inFlight = 0;
|
||||
const judge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
inFlight++;
|
||||
peakInFlight = Math.max(peakInFlight, inFlight);
|
||||
await new Promise(r => setTimeout(r, 2));
|
||||
inFlight--;
|
||||
return makeScore(0.5);
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt} :: ${strategy}`;
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(8),
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 4,
|
||||
miniEvalSize: 4,
|
||||
anchorEvalSize: 4,
|
||||
concurrency: 3,
|
||||
});
|
||||
|
||||
expect(peakInFlight).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-09 G3 · running-judge guard ──────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run · running-judge guard (H-09 G3)', () => {
|
||||
it('throws when a bare judge is passed without allowBareJudge', async () => {
|
||||
const bareJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.5, weighted: 0.5, correctness: 0.5,
|
||||
procedureFollowing: 0.5, conciseness: 0.5, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} mutated`;
|
||||
await expect(
|
||||
new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: bareJudge,
|
||||
mutate,
|
||||
}),
|
||||
).rejects.toThrow(/not a running judge|makeRunningJudge/);
|
||||
});
|
||||
|
||||
it('accepts a judge wrapped with makeRunningJudge', async () => {
|
||||
const { makeRunningJudge } = await import('../src/evolution-llm-wiring.js');
|
||||
const baseJudge = {
|
||||
async score(): Promise<JudgeScore> {
|
||||
return {
|
||||
overall: 0.8, weighted: 0.8, correctness: 0.8,
|
||||
procedureFollowing: 0.8, conciseness: 0.8, lengthPenalty: 1,
|
||||
feedback: '', parsed: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const fakeLLM = { complete: async () => 'LLM output' };
|
||||
const wrapped = makeRunningJudge(baseJudge, fakeLLM);
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} v2`;
|
||||
// Should not throw — the wrapped judge carries the brand.
|
||||
const result = await new IterativeGEPA().run({
|
||||
baseline: 'seed',
|
||||
examples: makeExamples(3),
|
||||
judge: wrapped,
|
||||
mutate,
|
||||
populationSize: 1,
|
||||
generations: 1,
|
||||
microScreenSize: 1, miniEvalSize: 1, anchorEvalSize: 1,
|
||||
});
|
||||
expect(result.winner).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── End-to-end run ─────────────────────────────────────────────
|
||||
|
||||
describe('IterativeGEPA.run', () => {
|
||||
it('runs through all phases and produces a winner', async () => {
|
||||
const baseline = 'short baseline prompt';
|
||||
const examples = makeExamples(10);
|
||||
const judge = makeFakeJudge('uniform');
|
||||
|
||||
// Mutate appends text — simulates generation of children
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => {
|
||||
return `${parent.prompt} :: ${strategy}`;
|
||||
};
|
||||
|
||||
const progress: string[] = [];
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
onProgress: (e) => progress.push(`${e.phase}@g${e.generation}`),
|
||||
});
|
||||
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
expect(result.history.length).toBeGreaterThan(1);
|
||||
expect(progress).toContain('start@g0');
|
||||
expect(progress.some(p => p.startsWith('anchor'))).toBe(true);
|
||||
expect(progress.some(p => p.startsWith('done'))).toBe(true);
|
||||
});
|
||||
|
||||
it('winner dominates or matches baseline on overall score', async () => {
|
||||
const baseline = 'x'; // very short
|
||||
const examples = makeExamples(20);
|
||||
// Long-biased judge: longer prompts score higher
|
||||
const judge = makeFakeJudge('long');
|
||||
|
||||
// Mutate doubles length each time
|
||||
const mutate: MutateFn = async ({ parent }) => `${parent.prompt} ${parent.prompt}more`;
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline,
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 15,
|
||||
});
|
||||
|
||||
// Winner should have evolved (not baseline prompt).
|
||||
expect(result.winner.prompt.length).toBeGreaterThan(baseline.length);
|
||||
expect(result.improved).toBe(true);
|
||||
expect(result.delta).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('passes weakness feedback from parent into mutate()', async () => {
|
||||
const captured: Array<{ strategy: string; feedbacks: string[] }> = [];
|
||||
const mutate: MutateFn = async ({ parent, strategy, weaknessFeedback }) => {
|
||||
captured.push({ strategy, feedbacks: weaknessFeedback });
|
||||
return `${parent.prompt} mutated`;
|
||||
};
|
||||
|
||||
await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(10),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 5,
|
||||
});
|
||||
|
||||
// 2 mutations were spawned — each received the parent's feedback
|
||||
expect(captured.length).toBe(2);
|
||||
for (const c of captured) {
|
||||
expect(Array.isArray(c.feedbacks)).toBe(true);
|
||||
// Feedback array may be empty on first gen if baseline had no weaknesses;
|
||||
// we just assert the prop is present and is an array.
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers when mutate throws (falls back to parent prompt)', async () => {
|
||||
let mutations = 0;
|
||||
const mutate: MutateFn = async ({ parent }) => {
|
||||
mutations++;
|
||||
if (mutations === 1) throw new Error('boom');
|
||||
return `${parent.prompt} ok`;
|
||||
};
|
||||
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 1,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
// Winner must be defined even though one mutation failed
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(result.winner.score).not.toBeNull();
|
||||
});
|
||||
|
||||
it('respects abort signal and exits early without throwing', async () => {
|
||||
const ctrl = new AbortController();
|
||||
const mutate: MutateFn = vi.fn(async () => 'should never run');
|
||||
|
||||
ctrl.abort(); // abort before starting generations
|
||||
const result = await runGEPA({
|
||||
baseline: 'baseline',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate,
|
||||
populationSize: 2,
|
||||
generations: 2,
|
||||
microScreenSize: 2,
|
||||
miniEvalSize: 2,
|
||||
anchorEvalSize: 2,
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
|
||||
// With abort before baseline scoring, no winner should evolve
|
||||
expect(result.winner).toBeDefined();
|
||||
expect(mutate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is deterministic with same seed', async () => {
|
||||
const examples = makeExamples(20);
|
||||
const judge = makeFakeJudge();
|
||||
const mutate: MutateFn = async ({ parent, strategy }) => `${parent.prompt}::${strategy}`;
|
||||
|
||||
const run = () => runGEPA({
|
||||
baseline: 'seed',
|
||||
examples,
|
||||
judge,
|
||||
mutate,
|
||||
populationSize: 3,
|
||||
generations: 1,
|
||||
microScreenSize: 5,
|
||||
miniEvalSize: 5,
|
||||
anchorEvalSize: 10,
|
||||
seed: 42,
|
||||
});
|
||||
|
||||
const r1 = await run();
|
||||
const r2 = await run();
|
||||
|
||||
expect(r1.winner.prompt).toBe(r2.winner.prompt);
|
||||
expect(r1.history.length).toBe(r2.history.length);
|
||||
});
|
||||
|
||||
it('produces history containing baseline and all mutated children', async () => {
|
||||
const result = await runGEPA({
|
||||
baseline: 'base',
|
||||
examples: makeExamples(5),
|
||||
judge: makeFakeJudge(),
|
||||
mutate: async ({ parent, strategy }) => `${parent.prompt}_${strategy}`,
|
||||
populationSize: 3,
|
||||
generations: 2,
|
||||
microScreenSize: 3,
|
||||
miniEvalSize: 3,
|
||||
anchorEvalSize: 3,
|
||||
});
|
||||
|
||||
expect(result.history[0].id).toBe('g0-baseline');
|
||||
expect(result.history[0].strategy).toBe('baseline');
|
||||
// 2 generations * 3 mutations + 1 baseline = 7 total
|
||||
expect(result.history.length).toBe(7);
|
||||
});
|
||||
});
|
||||
@@ -1,280 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB, OptimizationLogStore } from '@waggle/core';
|
||||
import {
|
||||
captureInteraction,
|
||||
getRecentLogs,
|
||||
getWorkspaceLogs,
|
||||
isWithinBudget,
|
||||
} from '../src/optimization-capture.js';
|
||||
import type { AgentResponse } from '../src/agent-loop.js';
|
||||
|
||||
describe('optimization-capture', () => {
|
||||
let db: MindDB;
|
||||
let store: OptimizationLogStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new OptimizationLogStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
function makeResponse(overrides?: Partial<AgentResponse>): AgentResponse {
|
||||
return {
|
||||
content: 'Hello, I can help with that.',
|
||||
toolsUsed: ['search_memory', 'save_memory'],
|
||||
usage: { inputTokens: 500, outputTokens: 200 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── captureInteraction ─────────────────────────────────────
|
||||
|
||||
describe('captureInteraction', () => {
|
||||
it('stores a log entry with correct fields', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'You are a helpful assistant.',
|
||||
response: makeResponse(),
|
||||
turnCount: 3,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.id).toBeGreaterThan(0);
|
||||
expect(entry.session_id).toBe('session-1');
|
||||
expect(entry.workspace_id).toBe('ws-abc');
|
||||
expect(entry.system_prompt).toBe('You are a helpful assistant.');
|
||||
expect(JSON.parse(entry.tools_used)).toEqual(['search_memory', 'save_memory']);
|
||||
expect(entry.turn_count).toBe(3);
|
||||
expect(entry.was_correction).toBe(0);
|
||||
expect(entry.input_tokens).toBe(500);
|
||||
expect(entry.output_tokens).toBe(200);
|
||||
expect(entry.timestamp).toBeTruthy();
|
||||
});
|
||||
|
||||
it('records wasCorrection as 1 when true', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse(),
|
||||
turnCount: 1,
|
||||
wasCorrection: true,
|
||||
});
|
||||
|
||||
expect(entry.was_correction).toBe(1);
|
||||
});
|
||||
|
||||
it('handles empty toolsUsed', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-abc',
|
||||
systemPrompt: 'test prompt',
|
||||
response: makeResponse({ toolsUsed: [] }),
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(JSON.parse(entry.tools_used)).toEqual([]);
|
||||
});
|
||||
|
||||
it('defaults token counts to 0 when missing', () => {
|
||||
const response: AgentResponse = {
|
||||
content: 'response',
|
||||
toolsUsed: [],
|
||||
usage: { inputTokens: 0, outputTokens: 0 },
|
||||
};
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 's1',
|
||||
workspaceId: 'ws1',
|
||||
systemPrompt: 'prompt',
|
||||
response,
|
||||
turnCount: 1,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
expect(entry.input_tokens).toBe(0);
|
||||
expect(entry.output_tokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getRecentLogs ──────────────────────────────────────────
|
||||
|
||||
describe('getRecentLogs', () => {
|
||||
it('retrieves stored entries ordered by most recent first', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p1',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p2',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const logs = getRecentLogs(store, 10);
|
||||
expect(logs).toHaveLength(2);
|
||||
// Ordered by timestamp DESC, id DESC — s2 was inserted second so it comes first
|
||||
expect(logs[0].session_id).toBe('s2');
|
||||
expect(logs[1].session_id).toBe('s1');
|
||||
});
|
||||
|
||||
it('respects the limit parameter', () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
captureInteraction(store, {
|
||||
sessionId: `s${i}`, workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
}
|
||||
|
||||
const logs = getRecentLogs(store, 3);
|
||||
expect(logs).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('returns empty array when no entries exist', () => {
|
||||
const logs = getRecentLogs(store);
|
||||
expect(logs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getWorkspaceLogs ───────────────────────────────────────
|
||||
|
||||
describe('getWorkspaceLogs', () => {
|
||||
it('filters by workspace ID', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws-b', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's3', workspaceId: 'ws-a', systemPrompt: 'p',
|
||||
response: makeResponse(), turnCount: 2, wasCorrection: true,
|
||||
});
|
||||
|
||||
const wsALogs = getWorkspaceLogs(store, 'ws-a');
|
||||
expect(wsALogs).toHaveLength(2);
|
||||
expect(wsALogs.every(l => l.workspace_id === 'ws-a')).toBe(true);
|
||||
|
||||
const wsBLogs = getWorkspaceLogs(store, 'ws-b');
|
||||
expect(wsBLogs).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Entries have correct shape ─────────────────────────────
|
||||
|
||||
describe('entry shape', () => {
|
||||
it('has all required fields with correct types', () => {
|
||||
const entry = captureInteraction(store, {
|
||||
sessionId: 'session-x',
|
||||
workspaceId: 'workspace-y',
|
||||
systemPrompt: 'You are helpful.',
|
||||
response: makeResponse({ toolsUsed: ['bash', 'read_file'] }),
|
||||
turnCount: 7,
|
||||
wasCorrection: false,
|
||||
});
|
||||
|
||||
// Check all fields exist and are of expected types
|
||||
expect(typeof entry.id).toBe('number');
|
||||
expect(typeof entry.session_id).toBe('string');
|
||||
expect(typeof entry.workspace_id).toBe('string');
|
||||
expect(typeof entry.system_prompt).toBe('string');
|
||||
expect(typeof entry.tools_used).toBe('string');
|
||||
expect(typeof entry.turn_count).toBe('number');
|
||||
expect(typeof entry.was_correction).toBe('number');
|
||||
expect(typeof entry.input_tokens).toBe('number');
|
||||
expect(typeof entry.output_tokens).toBe('number');
|
||||
expect(typeof entry.timestamp).toBe('string');
|
||||
|
||||
// tools_used is valid JSON array
|
||||
const parsed = JSON.parse(entry.tools_used);
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed).toEqual(['bash', 'read_file']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Budget enforcement ─────────────────────────────────────
|
||||
|
||||
describe('isWithinBudget', () => {
|
||||
it('returns true when no entries exist (under budget)', () => {
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when token cost is under budget', () => {
|
||||
// Insert entry with small token count
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 100 input * $3/M + 50 output * $15/M = $0.0003 + $0.00075 = ~$0.00105 = ~0.1 cents
|
||||
// Budget is 100 cents = $1, so well within budget
|
||||
expect(isWithinBudget(store, 100)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when token cost exceeds budget', () => {
|
||||
// Insert entry with large token count to exceed a tiny budget
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 1_000_000, outputTokens: 500_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// 1M input * $3/M + 500K output * $15/M = $3 + $7.5 = $10.50 = 1050 cents
|
||||
// Budget is 1 cent, so way over budget
|
||||
expect(isWithinBudget(store, 1)).toBe(false);
|
||||
});
|
||||
|
||||
it('respects custom cost-per-token rates', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 10_000, outputTokens: 5_000 } }),
|
||||
turnCount: 1, wasCorrection: false,
|
||||
});
|
||||
|
||||
// With very cheap rates: 10K * $0.0001/M + 5K * $0.0001/M ≈ ~0 cents
|
||||
expect(isWithinBudget(store, 1, 0.0001 / 1_000_000, 0.0001 / 1_000_000)).toBe(true);
|
||||
|
||||
// With very expensive rates: 10K * $100/M + 5K * $100/M = $1 + $0.5 = 150 cents
|
||||
expect(isWithinBudget(store, 1, 100 / 1_000_000, 100 / 1_000_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Store stats ────────────────────────────────────────────
|
||||
|
||||
describe('getStats (via store)', () => {
|
||||
it('computes correct aggregate stats', () => {
|
||||
captureInteraction(store, {
|
||||
sessionId: 's1', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 100, outputTokens: 50 } }),
|
||||
turnCount: 3, wasCorrection: false,
|
||||
});
|
||||
captureInteraction(store, {
|
||||
sessionId: 's2', workspaceId: 'ws1', systemPrompt: 'p',
|
||||
response: makeResponse({ usage: { inputTokens: 200, outputTokens: 100 } }),
|
||||
turnCount: 5, wasCorrection: true,
|
||||
});
|
||||
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.correctionRate).toBe(0.5); // 1/2
|
||||
expect(stats.avgTurnCount).toBe(4); // (3+5)/2
|
||||
expect(stats.totalInputTokens).toBe(300); // 100+200
|
||||
expect(stats.totalOutputTokens).toBe(150); // 50+100
|
||||
});
|
||||
|
||||
it('returns zeros when empty', () => {
|
||||
const stats = store.getStats();
|
||||
expect(stats.total).toBe(0);
|
||||
expect(stats.correctionRate).toBe(0);
|
||||
expect(stats.avgTurnCount).toBe(0);
|
||||
expect(stats.totalInputTokens).toBe(0);
|
||||
expect(stats.totalOutputTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,10 +2,9 @@ import { describe, it, expect } from 'vitest';
|
||||
import { PERSONAS, getPersona, listPersonas, composePersonaPrompt } from '../src/personas.js';
|
||||
|
||||
describe('Agent Personas', () => {
|
||||
it('PERSONAS catalog has 23 entries', () => {
|
||||
// 22 user-facing (8 universal + 14 specialists) + 1 internal review persona
|
||||
// (session-reviewer) used only by the IdleSessionWatcher self-evolution loop.
|
||||
expect(PERSONAS).toHaveLength(23);
|
||||
it('PERSONAS catalog has 22 entries', () => {
|
||||
// 8 universal + 14 specialists.
|
||||
expect(PERSONAS).toHaveLength(22);
|
||||
});
|
||||
|
||||
it('each persona has required fields', () => {
|
||||
@@ -40,9 +39,9 @@ describe('Agent Personas', () => {
|
||||
expect(getPersona('')).toBeNull();
|
||||
});
|
||||
|
||||
it('listPersonas() returns all 23 personas', () => {
|
||||
it('listPersonas() returns all 22 personas', () => {
|
||||
const list = listPersonas();
|
||||
expect(list).toHaveLength(23);
|
||||
expect(list).toHaveLength(22);
|
||||
expect(list).not.toBe(PERSONAS); // Returns a copy
|
||||
});
|
||||
|
||||
@@ -208,19 +207,6 @@ describe('Soft tool model', () => {
|
||||
expect(v!.disallowedTools).toContain('write_file');
|
||||
expect(v!.isReadOnly).toBe(true);
|
||||
});
|
||||
|
||||
it('session-reviewer is a trust-boundary persona: reads + create_skill only, no writes/exec/memory', () => {
|
||||
const r = getPersona('session-reviewer');
|
||||
expect(r, 'session-reviewer not found').toBeTruthy();
|
||||
// create_skill is the proposal vehicle and must stay in the allowlist.
|
||||
expect(r!.tools).toContain('create_skill');
|
||||
// isReadOnly would strip create_skill, so it must be false.
|
||||
expect(r!.isReadOnly).toBe(false);
|
||||
// The critical tools ALWAYS_AVAILABLE_TOOLS re-adds must be explicitly denied.
|
||||
for (const t of ['save_memory', 'delete_skill', 'install_capability', 'acquire_capability', 'spawn_agent', 'bash']) {
|
||||
expect(r!.disallowedTools, `${t} must be disallowed`).toContain(t);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('New domain personas', () => {
|
||||
@@ -239,8 +225,7 @@ describe('New domain personas', () => {
|
||||
});
|
||||
}
|
||||
|
||||
it('total persona count is 23', () => {
|
||||
// 22 tiered + session-reviewer (internal self-evolution reviewer).
|
||||
expect(PERSONAS.length).toBe(23);
|
||||
it('total persona count is 22', () => {
|
||||
expect(PERSONAS.length).toBe(22);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
*
|
||||
* Tests the one-step promotion personal → workspace, the refusal of team and
|
||||
* enterprise scopes in this edition, frontmatter rewrite with promoted_from
|
||||
* history, file move, and skill_promotion improvement signal emission.
|
||||
* history and file move.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { createSkillTools, getSkillDirForScope } from '../src/skill-tools.js';
|
||||
import { parseSkillFrontmatter } from '../src/skill-frontmatter.js';
|
||||
|
||||
@@ -22,8 +21,6 @@ function seedSkill(dir: string, name: string, scope: string | null, body = 'body
|
||||
|
||||
describe('promote_skill — gap E', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
let reloadCount: number;
|
||||
|
||||
function createTools(overrides: Partial<Parameters<typeof createSkillTools>[0]> = {}) {
|
||||
@@ -32,7 +29,6 @@ describe('promote_skill — gap E', () => {
|
||||
onSkillsChanged: () => { reloadCount++; },
|
||||
getWorkspaceId: () => 'ws-alpha',
|
||||
getTeamId: () => 'team-bravo',
|
||||
improvementSignals: signals,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -45,18 +41,15 @@ describe('promote_skill — gap E', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-promote-test-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
reloadCount = 0;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('happy path', () => {
|
||||
it('promotes personal → workspace, moves file, appends history, records signal', async () => {
|
||||
it('promotes personal → workspace, moves file, appends history', async () => {
|
||||
const personalDir = getSkillDirForScope(tmpDir, 'personal', {})!;
|
||||
seedSkill(personalDir, 'deploy-helper', 'personal');
|
||||
const tools = createTools();
|
||||
@@ -64,7 +57,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(tools, 'promote_skill', { skill_name: 'deploy-helper' });
|
||||
|
||||
expect(out).toContain('personal → workspace');
|
||||
expect(out).toContain('Signal recorded');
|
||||
|
||||
// File moved
|
||||
expect(fs.existsSync(path.join(personalDir, 'deploy-helper.md'))).toBe(false);
|
||||
@@ -77,11 +69,6 @@ describe('promote_skill — gap E', () => {
|
||||
expect(parsed.frontmatter.scope).toBe('workspace');
|
||||
expect(parsed.frontmatter.promoted_from).toEqual(['personal']);
|
||||
|
||||
// Signal emitted
|
||||
const recorded = signals.getByCategory('skill_promotion');
|
||||
expect(recorded.length).toBe(1);
|
||||
expect(recorded[0].pattern_key).toBe('promote:deploy-helper:personal->workspace');
|
||||
|
||||
// onSkillsChanged fired
|
||||
expect(reloadCount).toBe(1);
|
||||
});
|
||||
@@ -125,7 +112,6 @@ describe('promote_skill — gap E', () => {
|
||||
const out = await run(createTools(), 'promote_skill', { skill_name: 'shared-helper' });
|
||||
expect(out).toContain('not available in this edition');
|
||||
expect(fs.existsSync(path.join(wsDir, 'shared-helper.md'))).toBe(true);
|
||||
expect(signals.getByCategory('skill_promotion')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('refuses enterprise promotion', async () => {
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
} from '../src/prompt-loader.js';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -68,7 +67,6 @@ describe('Prompt Loader', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── H-08 G2 — override-aware loader ──────────────────────────
|
||||
describe('loadSystemPromptWithOverrides', () => {
|
||||
it('returns baseline when no overrides exist', () => {
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
@@ -86,55 +84,17 @@ describe('Prompt Loader', () => {
|
||||
expect(composed.userPromptMd).toBe('hello from user');
|
||||
});
|
||||
|
||||
it('applies behavioral-spec overrides into the active spec', () => {
|
||||
it('ignores override files left on disk by earlier versions', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'CUSTOM CORE LOOP CONTENT FOR TEST',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
JSON.stringify({ section: 'coreLoop', text: 'STALE OVERRIDE TEXT', deployedAt: new Date().toISOString() }),
|
||||
);
|
||||
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
const flattened = Object.values(composed.behavioralSpec).join('\n');
|
||||
expect(flattened).toContain('CUSTOM CORE LOOP CONTENT FOR TEST');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertOverridesReachActiveSpec', () => {
|
||||
it('is quiet when no overrides are deployed', () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const composed = loadSystemPromptWithOverrides(tmpDir);
|
||||
assertOverridesReachActiveSpec(tmpDir, composed.behavioralSpec);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('warns when disk has overrides but activeSpec does not contain them', () => {
|
||||
const overrideDir = path.join(tmpDir, 'behavioral-overrides');
|
||||
fs.mkdirSync(overrideDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(overrideDir, 'coreLoop.json'),
|
||||
JSON.stringify({
|
||||
section: 'coreLoop',
|
||||
text: 'UNIQUE MARKER STRING FOR G2 TEST — LONG ENOUGH TO MATCH',
|
||||
deployedAt: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
// Simulate using the BARE loader — activeSpec is the compile-time
|
||||
// baseline with no overrides folded in.
|
||||
const fakeActiveSpec = loadSystemPromptWithOverrides(path.join(tmpDir, 'nothing')).behavioralSpec;
|
||||
assertOverridesReachActiveSpec(tmpDir, fakeActiveSpec);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
const message = warnSpy.mock.calls[0]?.[0] as string;
|
||||
expect(message).toContain('coreLoop');
|
||||
warnSpy.mockRestore();
|
||||
expect(flattened).not.toContain('STALE OVERRIDE TEXT');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,13 +2,8 @@
|
||||
* Skills 2.0 gap K — write-path conflict detection.
|
||||
*
|
||||
* When save_memory writes a new frame that contradicts an existing frame
|
||||
* (detectContradiction), the save still succeeds but:
|
||||
* - the return string carries a [flag: contradicts_existing …] marker
|
||||
* - a correction-category improvement signal is recorded
|
||||
*
|
||||
* Previously detectContradiction existed but was never called on the write
|
||||
* path — write-path conflicts silently overwrote reality, degrading the
|
||||
* evolution-signal loop.
|
||||
* (detectContradiction), the save still succeeds but the return string
|
||||
* carries a [flag: contradicts_existing …] marker.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
@@ -19,7 +14,6 @@ import {
|
||||
SessionStore,
|
||||
KnowledgeGraph,
|
||||
HybridSearch,
|
||||
ImprovementSignalStore,
|
||||
} from '@waggle/core';
|
||||
import { createMindTools } from '../src/tools.js';
|
||||
import type { ToolDefinition } from '../src/tools.js';
|
||||
@@ -30,7 +24,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
let frames: FrameStore;
|
||||
let sessions: SessionStore;
|
||||
let tools: ToolDefinition[];
|
||||
let improvementSignals: ImprovementSignalStore;
|
||||
|
||||
function getSaveMemory(): ToolDefinition {
|
||||
const tool = tools.find(t => t.name === 'save_memory');
|
||||
@@ -45,7 +38,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions = new SessionStore(db);
|
||||
const knowledge = new KnowledgeGraph(db);
|
||||
const search = new HybridSearch(db, embedder);
|
||||
improvementSignals = new ImprovementSignalStore(db);
|
||||
|
||||
tools = createMindTools({
|
||||
db,
|
||||
@@ -55,7 +47,6 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
sessions,
|
||||
search,
|
||||
knowledge,
|
||||
improvementSignals,
|
||||
// Intentionally NO cognify → exercises the raw-frame fallback path
|
||||
// (no LLM-dependent entity extraction in unit tests).
|
||||
});
|
||||
@@ -94,17 +85,9 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
expect(second).toContain('Memory saved');
|
||||
// But the return message is annotated
|
||||
expect(second).toContain('[flag: contradicts_existing');
|
||||
|
||||
// And a correction signal is recorded
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.length).toBeGreaterThan(0);
|
||||
const writeConflict = top.find(s => s.pattern_key.startsWith('write-conflict:'));
|
||||
expect(writeConflict).toBeDefined();
|
||||
expect(writeConflict!.detail).toMatch(/contradicts existing frame/i);
|
||||
expect(writeConflict!.category).toBe('correction');
|
||||
});
|
||||
|
||||
it('non-contradicting saves do not flag or emit a signal', async () => {
|
||||
it('non-contradicting saves are not flagged', async () => {
|
||||
const save = getSaveMemory();
|
||||
await save.execute({
|
||||
content: 'Decision: we chose PostgreSQL for analytics; approved and confirmed.',
|
||||
@@ -117,34 +100,5 @@ describe('save_memory — gap K (write-path conflict detection)', () => {
|
||||
source: 'user_stated',
|
||||
});
|
||||
expect(result).not.toContain('contradicts_existing');
|
||||
|
||||
const top = improvementSignals.getByCategory('correction');
|
||||
expect(top.find(s => s.pattern_key.startsWith('write-conflict:'))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('works without improvementSignals dep (backwards-compatible)', async () => {
|
||||
// Rebuild tools without signals dep
|
||||
const db2 = new MindDB(':memory:');
|
||||
const embedder = new MockEmbedder();
|
||||
const frames2 = new FrameStore(db2);
|
||||
const sessions2 = new SessionStore(db2);
|
||||
const knowledge2 = new KnowledgeGraph(db2);
|
||||
const search2 = new HybridSearch(db2, embedder);
|
||||
const tools2 = createMindTools({
|
||||
db: db2,
|
||||
identity: new IdentityLayer(db2),
|
||||
awareness: new AwarenessLayer(db2),
|
||||
frames: frames2,
|
||||
sessions: sessions2,
|
||||
search: search2,
|
||||
knowledge: knowledge2,
|
||||
// No improvementSignals — should not throw
|
||||
});
|
||||
const save = tools2.find(t => t.name === 'save_memory')!;
|
||||
await save.execute({ content: 'Decision: enable feature flag rollout for users; proceed.', target: 'personal' });
|
||||
const result = await save.execute({ content: 'Decision: disable feature flag rollout for users; cancel.', target: 'personal' });
|
||||
// Flag still appears (detectContradiction doesn't need the signal store)
|
||||
expect(result).toContain('contradicts_existing');
|
||||
db2.close();
|
||||
});
|
||||
});
|
||||
|
||||
77
packages/agent/tests/secret-patterns.test.ts
Normal file
77
packages/agent/tests/secret-patterns.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from '../src/secret-patterns.js';
|
||||
|
||||
describe('detectSecrets', () => {
|
||||
it('returns null for clean text', () => {
|
||||
expect(detectSecrets('Hello world, no secrets here.')).toBeNull();
|
||||
expect(detectSecrets('pk_test_looks_partial')).toBeNull(); // too short
|
||||
});
|
||||
|
||||
it('detects AWS access keys', () => {
|
||||
expect(detectSecrets('AKIAIOSFODNN7EXAMPLE used for upload')).toBe('aws-access-key');
|
||||
});
|
||||
|
||||
it('detects GitHub PATs', () => {
|
||||
expect(detectSecrets('token=ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789')).toBe('github-pat');
|
||||
});
|
||||
|
||||
it('detects OpenAI keys', () => {
|
||||
expect(detectSecrets('sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa')).toBe('openai-key');
|
||||
});
|
||||
|
||||
it('detects Anthropic keys', () => {
|
||||
expect(detectSecrets('key = sk-ant-api03-aaaaaaaaaaaaaaaaaaa')).toBe('anthropic-key');
|
||||
});
|
||||
|
||||
it('detects Google API keys', () => {
|
||||
// Google API keys = literal "AIza" + exactly 35 chars
|
||||
expect(detectSecrets('AIzaSyA0B1c2d3e4F5G6H7I8J9K0L1M2N3O4P5Q')).toBe('google-api-key');
|
||||
});
|
||||
|
||||
it('detects Stripe keys', () => {
|
||||
expect(detectSecrets('sk_live_51AbCdEfGhIjKlMnOpQrStUvWxYz')).toBe('stripe-secret');
|
||||
});
|
||||
|
||||
it('detects private key blocks', () => {
|
||||
expect(detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMII...')).toBe('private-key-block');
|
||||
});
|
||||
|
||||
it('detects JWT tokens', () => {
|
||||
const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
|
||||
expect(detectSecrets(jwt)).toBe('jwt');
|
||||
});
|
||||
|
||||
it('detects basic-auth URLs', () => {
|
||||
expect(detectSecrets('Use https://user:hunter2@example.com/api'))
|
||||
.toBe('basic-auth-url');
|
||||
});
|
||||
|
||||
it('detects env-var password assignments', () => {
|
||||
expect(detectSecrets('PASSWORD=supersecret123'))
|
||||
.toBe('env-password');
|
||||
});
|
||||
|
||||
it('detects postgres connection strings with creds', () => {
|
||||
expect(detectSecrets('postgres://admin:pw1234@db.local:5432/prod'))
|
||||
.toBe('pgsql-url');
|
||||
});
|
||||
|
||||
it('SECRET_PATTERN_NAMES is non-empty and unique', () => {
|
||||
expect(SECRET_PATTERN_NAMES.length).toBeGreaterThan(10);
|
||||
expect(new Set(SECRET_PATTERN_NAMES).size).toBe(SECRET_PATTERN_NAMES.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('redactSecrets', () => {
|
||||
it('replaces every match and reports the patterns that fired', () => {
|
||||
const { text, found } = redactSecrets('key AKIAIOSFODNN7EXAMPLE and PASSWORD=supersecret123');
|
||||
expect(text).toContain('[REDACTED:aws-access-key]');
|
||||
expect(text).toContain('[REDACTED:env-password]');
|
||||
expect(text).not.toContain('AKIAIOSFODNN7EXAMPLE');
|
||||
expect(found.sort()).toEqual(['aws-access-key', 'env-password']);
|
||||
});
|
||||
|
||||
it('leaves clean text untouched', () => {
|
||||
expect(redactSecrets('nothing to hide')).toEqual({ text: 'nothing to hide', found: [] });
|
||||
});
|
||||
});
|
||||
@@ -90,59 +90,4 @@ describe('Self-Awareness', () => {
|
||||
expect(result).toContain('acquire_capability');
|
||||
expect(result).toContain('uncertain');
|
||||
});
|
||||
|
||||
it('includes improvement signals when awareness has actionable data', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [{
|
||||
id: 1,
|
||||
toolName: 'pdf_reader',
|
||||
occurrences: 3,
|
||||
suggestion: 'User has needed "pdf_reader" 3 times. Consider using acquire_capability to find it.',
|
||||
}],
|
||||
corrections: [{
|
||||
id: 2,
|
||||
patternKey: 'tone:too_formal',
|
||||
detail: 'Keep it casual',
|
||||
occurrences: 4,
|
||||
guidance: 'Keep it casual (corrected 4 times)',
|
||||
}],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 2,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).toContain('Learning from Past Sessions');
|
||||
expect(result).toContain('pdf_reader');
|
||||
expect(result).toContain('Keep it casual');
|
||||
expect(result).toContain('Recurring capability gaps');
|
||||
expect(result).toContain('User corrections to apply');
|
||||
});
|
||||
|
||||
it('omits improvement section when awareness has zero actionable signals', () => {
|
||||
const caps: AgentCapabilities = {
|
||||
tools: [],
|
||||
skills: [],
|
||||
model: 'test',
|
||||
memoryStats: { frameCount: 10, sessionCount: 2, entityCount: 3 },
|
||||
mode: 'local',
|
||||
version: '1.0.0',
|
||||
awareness: {
|
||||
capabilityGaps: [],
|
||||
corrections: [],
|
||||
workflowPatterns: [],
|
||||
totalActionable: 0,
|
||||
},
|
||||
};
|
||||
|
||||
const result = buildSelfAwareness(caps);
|
||||
expect(result).not.toContain('Learning from Past Sessions');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
autoExtractAndCreateSkill,
|
||||
skillFilename,
|
||||
@@ -35,24 +34,18 @@ function repeatedSession(): AutoExtractMessage[] {
|
||||
|
||||
describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-autoextract-'));
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates a personal-scope SKILL.md from a repeated workflow', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(result.filePath).not.toBeNull();
|
||||
@@ -67,19 +60,6 @@ describe('autoExtractAndCreateSkill — gap A', () => {
|
||||
expect(body).toContain('## Tools Used');
|
||||
});
|
||||
|
||||
it('records a workflow_pattern signal with auto-extract pattern_key', async () => {
|
||||
const result = await autoExtractAndCreateSkill(repeatedSession(), {
|
||||
personalSkillsDir: tmpDir,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
expect(result.filePath).not.toBeNull();
|
||||
|
||||
const recorded = signals.getByCategory('workflow_pattern');
|
||||
const autoExtract = recorded.find(s => s.pattern_key.startsWith('auto-extract:'));
|
||||
expect(autoExtract).toBeDefined();
|
||||
expect(autoExtract!.detail).toContain('Auto-extracted');
|
||||
});
|
||||
|
||||
it('returns no_pattern for short or random sessions', async () => {
|
||||
const messages: AutoExtractMessage[] = [
|
||||
{ role: 'user', content: 'Hi' },
|
||||
|
||||
@@ -5,7 +5,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB, ImprovementSignalStore } from '@waggle/core';
|
||||
import { retireStaleSkills } from '../src/skill-retirement.js';
|
||||
import {
|
||||
recordSkillUsage,
|
||||
@@ -74,8 +73,6 @@ describe('skill-usage — gap F', () => {
|
||||
describe('retireStaleSkills — gap F', () => {
|
||||
let waggleHome: string;
|
||||
let skillsDir: string;
|
||||
let db: MindDB;
|
||||
let signals: ImprovementSignalStore;
|
||||
|
||||
const NOW = new Date('2026-04-15T00:00:00Z');
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
@@ -83,12 +80,9 @@ describe('retireStaleSkills — gap F', () => {
|
||||
beforeEach(() => {
|
||||
waggleHome = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-retire-'));
|
||||
skillsDir = path.join(waggleHome, 'skills');
|
||||
db = new MindDB(':memory:');
|
||||
signals = new ImprovementSignalStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(waggleHome, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
@@ -110,7 +104,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays: 90,
|
||||
now: () => NOW,
|
||||
improvementSignals: signals,
|
||||
});
|
||||
|
||||
expect(report.scanned).toBe(2);
|
||||
@@ -123,10 +116,6 @@ describe('retireStaleSkills — gap F', () => {
|
||||
// Usage entry for the retired skill is cleared
|
||||
expect(loadSkillUsage(waggleHome).stale).toBeUndefined();
|
||||
expect(loadSkillUsage(waggleHome).fresh).toBeDefined();
|
||||
|
||||
// Signal recorded
|
||||
const recorded = signals.getByCategory('workflow_pattern').find(s => s.pattern_key === 'retire:stale');
|
||||
expect(recorded).toBeDefined();
|
||||
});
|
||||
|
||||
it('falls back to file mtime when no usage entry exists', () => {
|
||||
|
||||
Reference in New Issue
Block a user