refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the project owner reworks it separately. Cost tracking stays. - agent: delete behaviour evolution (evolve-schema, iterative GEPA, compose-evolution, evolution gates, orchestrator and deploy), improvement and correction detection, AgentLearning, the eval-dataset builder, optimization-capture and the agent_insights tool. Secret detection moves from eval-dataset to secret-patterns. The LLM wiring keeps only the Haiku adapter and judge call used by skill audit and hygiene, so @ax-llm/ax is now declared by agent instead of server. - server: delete /api/evolution/*, the evolution and optimizer services, GEPA expansion of the first chat message, the prompt_optimization and monthly_assessment jobs, the idle-session reviewer and its session-reviewer persona, prompt injection of past corrections and thumbs-down recorded as correction signals. Existing job rows of the removed types are ignored, not deleted. - core: delete OptimizationLogStore; stop re-exporting the improvement signal and evolution run stores. - web: delete the Evolution screen and the two job types in the automation editor. - delete packages/optimizer. Also logs UF-027 (self-learning loops read data nothing collects) and UF-028 (the Settings prompt-shape choice is ignored by chat).
This commit is contained in:
@@ -1,194 +0,0 @@
|
||||
/**
|
||||
* Agent Learning — persistent behavioral adjustments from performance signals.
|
||||
*
|
||||
* Tracks what works and what doesn't across sessions, adjusts agent behavior
|
||||
* by building a learned context section injected into the system prompt.
|
||||
*
|
||||
* Three learning channels:
|
||||
* 1. Correction patterns — user corrections become persistent rules
|
||||
* 2. Success patterns — approaches that got positive feedback
|
||||
* 3. Persona effectiveness — per-persona quality signals
|
||||
*
|
||||
* Uses the improvement_signals table for persistence.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
|
||||
export interface LearnedBehavior {
|
||||
rule: string;
|
||||
source: 'correction' | 'success' | 'observation';
|
||||
confidence: number;
|
||||
occurrences: number;
|
||||
lastSeen: string;
|
||||
}
|
||||
|
||||
export interface PersonaEffectiveness {
|
||||
personaId: string;
|
||||
tasksCompleted: number;
|
||||
correctionsReceived: number;
|
||||
positiveSignals: number;
|
||||
effectivenessScore: number; // 0-100
|
||||
}
|
||||
|
||||
export interface LearningSnapshot {
|
||||
learnedBehaviors: LearnedBehavior[];
|
||||
personaStats: PersonaEffectiveness[];
|
||||
totalCorrections: number;
|
||||
totalSuccesses: number;
|
||||
adaptationLevel: 'new' | 'learning' | 'adapted' | 'expert';
|
||||
}
|
||||
|
||||
export class AgentLearning {
|
||||
private store: ImprovementSignalStore;
|
||||
|
||||
constructor(store: ImprovementSignalStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Record a successful interaction pattern. */
|
||||
recordSuccess(pattern: string, personaId?: string): void {
|
||||
this.store.record('workflow_pattern', `success:${pattern}`, `Approach worked well`, {
|
||||
type: 'success',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record that the user gave positive feedback. */
|
||||
recordPositiveFeedback(context: string, personaId?: string): void {
|
||||
this.store.record('correction', `positive:${context.slice(0, 50)}`, context, {
|
||||
type: 'positive',
|
||||
personaId,
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/** Record a persona's task completion. */
|
||||
recordPersonaTask(personaId: string, corrected: boolean): void {
|
||||
const key = corrected ? `persona:${personaId}:corrected` : `persona:${personaId}:completed`;
|
||||
this.store.record('workflow_pattern', key, personaId, {
|
||||
type: 'persona_task',
|
||||
personaId,
|
||||
corrected,
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a learning snapshot from accumulated signals. */
|
||||
getSnapshot(): LearningSnapshot {
|
||||
const all = this.store.getActionable();
|
||||
const learnedBehaviors: LearnedBehavior[] = [];
|
||||
const personaMap = new Map<string, PersonaEffectiveness>();
|
||||
let totalCorrections = 0;
|
||||
let totalSuccesses = 0;
|
||||
|
||||
for (const signal of all) {
|
||||
// Corrections become learned behavioral rules
|
||||
if (signal.category === 'correction' && !signal.pattern_key.startsWith('positive:')) {
|
||||
totalCorrections += signal.count;
|
||||
if (signal.count >= 2) { // Only learn from repeated corrections
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || signal.pattern_key,
|
||||
source: 'correction',
|
||||
confidence: Math.min(1.0, 0.5 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Positive feedback
|
||||
if (signal.pattern_key.startsWith('positive:')) {
|
||||
totalSuccesses += signal.count;
|
||||
learnedBehaviors.push({
|
||||
rule: signal.detail || 'Positive approach',
|
||||
source: 'success',
|
||||
confidence: Math.min(1.0, 0.6 + signal.count * 0.1),
|
||||
occurrences: signal.count,
|
||||
lastSeen: signal.last_seen,
|
||||
});
|
||||
}
|
||||
|
||||
// Success patterns
|
||||
if (signal.pattern_key.startsWith('success:')) {
|
||||
totalSuccesses += signal.count;
|
||||
}
|
||||
|
||||
// Persona stats
|
||||
if (signal.pattern_key.startsWith('persona:')) {
|
||||
const parts = signal.pattern_key.split(':');
|
||||
const personaId = parts[1];
|
||||
const isCorrected = parts[2] === 'corrected';
|
||||
|
||||
let stats = personaMap.get(personaId);
|
||||
if (!stats) {
|
||||
stats = { personaId, tasksCompleted: 0, correctionsReceived: 0, positiveSignals: 0, effectivenessScore: 50 };
|
||||
personaMap.set(personaId, stats);
|
||||
}
|
||||
|
||||
if (isCorrected) {
|
||||
stats.correctionsReceived += signal.count;
|
||||
} else {
|
||||
stats.tasksCompleted += signal.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate persona effectiveness
|
||||
for (const stats of personaMap.values()) {
|
||||
const total = stats.tasksCompleted + stats.correctionsReceived;
|
||||
stats.effectivenessScore = total > 0
|
||||
? Math.round((stats.tasksCompleted / total) * 100)
|
||||
: 50;
|
||||
}
|
||||
|
||||
// Determine adaptation level
|
||||
const totalSignals = totalCorrections + totalSuccesses;
|
||||
const adaptationLevel = totalSignals < 5 ? 'new'
|
||||
: totalSignals < 20 ? 'learning'
|
||||
: totalSignals < 50 ? 'adapted'
|
||||
: 'expert';
|
||||
|
||||
return {
|
||||
learnedBehaviors: learnedBehaviors.slice(0, 10), // Cap at 10 rules
|
||||
personaStats: Array.from(personaMap.values()),
|
||||
totalCorrections,
|
||||
totalSuccesses,
|
||||
adaptationLevel,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format learned behaviors as a system prompt section.
|
||||
* Only included when there are meaningful learned rules.
|
||||
*/
|
||||
formatLearningPrompt(): string | null {
|
||||
const snapshot = this.getSnapshot();
|
||||
if (snapshot.learnedBehaviors.length === 0) return null;
|
||||
|
||||
const lines: string[] = [
|
||||
'## Learned Behaviors',
|
||||
`*Adaptation level: ${snapshot.adaptationLevel} (${snapshot.totalCorrections} corrections, ${snapshot.totalSuccesses} successes)*`,
|
||||
'',
|
||||
];
|
||||
|
||||
const corrections = snapshot.learnedBehaviors.filter(b => b.source === 'correction');
|
||||
if (corrections.length > 0) {
|
||||
lines.push('**Avoid these patterns** (user has corrected before):');
|
||||
for (const b of corrections) {
|
||||
lines.push(`- ${b.rule} (corrected ${b.occurrences}x, confidence: ${(b.confidence * 100).toFixed(0)}%)`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const successes = snapshot.learnedBehaviors.filter(b => b.source === 'success');
|
||||
if (successes.length > 0) {
|
||||
lines.push('**Keep doing** (positive feedback received):');
|
||||
for (const b of successes) {
|
||||
lines.push(`- ${b.rule}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
}
|
||||
@@ -384,8 +384,7 @@ When approaching any task:
|
||||
};
|
||||
|
||||
/**
|
||||
* Section name literal type — matches the evolution-deploy module.
|
||||
* Kept minimal here so behavioral-spec.ts stays free of cross-imports.
|
||||
* Section name literal type for `buildActiveBehavioralSpec` overrides.
|
||||
*/
|
||||
export type BehavioralSpecSectionName =
|
||||
| 'coreLoop'
|
||||
@@ -402,8 +401,7 @@ export type BehavioralSpecSectionName =
|
||||
* the compiled baseline unchanged.
|
||||
*
|
||||
* Typical usage at server boot:
|
||||
* const overrides = loadBehavioralSpecOverrides(dataDir);
|
||||
* const spec = buildActiveBehavioralSpec(overrides);
|
||||
* const spec = buildActiveBehavioralSpec();
|
||||
* // ...pass spec.rules into the system prompt...
|
||||
*/
|
||||
export function buildActiveBehavioralSpec(
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
/**
|
||||
* Compose Evolution — Phase 3.2 of the self-evolution loop.
|
||||
*
|
||||
* Two-stage pipeline:
|
||||
*
|
||||
* Stage 1 — EvolveSchema
|
||||
* Evolve the output STRUCTURE (fields, order, types, constraints).
|
||||
* Returns a winner schema.
|
||||
*
|
||||
* Stage 2 — IterativeGEPA
|
||||
* Freeze the winner schema. Evolve the INSTRUCTION prompt that fills
|
||||
* that schema. Only instruction text changes here.
|
||||
*
|
||||
* Critical design detail (from Mikhail Pavlukhin's paper): **feedback
|
||||
* separation**. If the judge complains "missing reasoning field" during
|
||||
* Stage 2, and we let GEPA see that feedback, GEPA will mutate the
|
||||
* instruction to try to remove reasoning — undoing the schema Stage 1
|
||||
* just evolved. So Stage 2 receives a *filtered* judge that strips
|
||||
* structural complaints and keeps only value-level signals (correctness,
|
||||
* conciseness, format-of-values, tone).
|
||||
*
|
||||
* The composition is stateless: callers provide the baseline schema and
|
||||
* instruction separately; the result returns both winners + aggregate
|
||||
* accuracy improvement.
|
||||
*/
|
||||
|
||||
import {
|
||||
EvolveSchema,
|
||||
type EvolveSchemaOptions,
|
||||
type EvolveSchemaResult,
|
||||
type Schema,
|
||||
type SchemaExecuteFn,
|
||||
} from './evolve-schema.js';
|
||||
import {
|
||||
IterativeGEPA,
|
||||
type IterativeGEPAOptions,
|
||||
type GEPARunResult,
|
||||
} from './iterative-optimizer.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import { RUNNING_JUDGE_BRAND, isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface ComposeEvolutionOptions {
|
||||
/** EvolveSchema stage configuration */
|
||||
schema: Omit<EvolveSchemaOptions, 'onProgress' | 'signal'>;
|
||||
/** IterativeGEPA stage configuration */
|
||||
instructions: Omit<IterativeGEPAOptions, 'onProgress' | 'signal'>;
|
||||
/**
|
||||
* Classifier — returns 'structural' to drop a feedback line from GEPA,
|
||||
* or 'value' to keep it. Default: `defaultFeedbackFilter`.
|
||||
*/
|
||||
feedbackFilter?: FeedbackFilter;
|
||||
/** Optional progress reporter across both stages */
|
||||
onProgress?: (event: ComposeProgress) => void;
|
||||
/** Optional abort signal (passed through to both stages) */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ComposeEvolutionResult {
|
||||
schema: EvolveSchemaResult;
|
||||
instructions: GEPARunResult;
|
||||
/**
|
||||
* Accuracy delta after both stages compared to the raw baseline schema +
|
||||
* baseline instructions. Positive = improvement.
|
||||
*/
|
||||
combinedDelta: number;
|
||||
/** True if both stages improved over their individual baselines. */
|
||||
fullyImproved: boolean;
|
||||
/** Frozen schema used as input to the GEPA stage — convenience pointer. */
|
||||
frozenSchema: Schema;
|
||||
}
|
||||
|
||||
export interface ComposeProgress {
|
||||
stage: 'schema' | 'instructions' | 'done';
|
||||
/** Underlying stage progress event, if available */
|
||||
detail?: unknown;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export type FeedbackFilter = (feedback: string) => 'structural' | 'value';
|
||||
|
||||
// ── Feedback filtering ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Default heuristic: classifies feedback lines as 'structural' when they
|
||||
* mention schema-level concepts (field, schema, missing/extra property,
|
||||
* wrong type, reorder, etc). Everything else is 'value'.
|
||||
*
|
||||
* Tight enough that ambiguous feedback still flows to GEPA — only obvious
|
||||
* structural complaints are dropped.
|
||||
*/
|
||||
export function defaultFeedbackFilter(feedback: string): 'structural' | 'value' {
|
||||
if (!feedback) return 'value';
|
||||
const lower = feedback.toLowerCase();
|
||||
|
||||
const structuralPatterns: RegExp[] = [
|
||||
/\bmissing\s+(?:the\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\badd(?:ed|ing)?\s+(?:a|the)?\s*(?:new\s+)?(?:["`']?\w+["`']?\s+)?field\b/,
|
||||
/\bextra\s+field\b/,
|
||||
/\bunexpected\s+field\b/,
|
||||
/\bwrong\s+(?:field\s+)?(?:type|order)\b/,
|
||||
/\bfield\s+type\s+(?:mismatch|wrong)/,
|
||||
/\b(?:should\s+)?(?:re)?order\s+(?:the\s+)?fields\b/,
|
||||
/\bschema\s+(?:mismatch|violation|error)\b/,
|
||||
/\binvalid\s+json\s+shape\b/,
|
||||
/\bmissing\s+required\s+property\b/,
|
||||
/\bproperty\s+missing\b/,
|
||||
/\bshape\s+is\s+wrong\b/,
|
||||
];
|
||||
|
||||
for (const pattern of structuralPatterns) {
|
||||
if (pattern.test(lower)) return 'structural';
|
||||
}
|
||||
return 'value';
|
||||
}
|
||||
|
||||
// ── Judge wrapping ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Wrap an LLMJudge-like object so the feedback it emits has structural
|
||||
* lines stripped according to `filter`. Only `feedback` is filtered —
|
||||
* the numerical scores (correctness, procedure, conciseness, overall)
|
||||
* are preserved as-is so GEPA sees the true accuracy signal.
|
||||
*
|
||||
* The multi-line feedback from the judge is split on newlines and each
|
||||
* line classified individually.
|
||||
*/
|
||||
export function filterJudgeFeedback(
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): Pick<LLMJudge, 'score'> {
|
||||
const wrapped: Pick<LLMJudge, 'score'> & Partial<Record<symbol, unknown>> = {
|
||||
async score(args) {
|
||||
const raw = await judge.score(args);
|
||||
const filteredFeedback = stripStructuralLines(raw.feedback, filter);
|
||||
const result: JudgeScore = { ...raw, feedback: filteredFeedback };
|
||||
return result;
|
||||
},
|
||||
};
|
||||
// H-09 G3: preserve the running-judge brand so IterativeGEPA's check
|
||||
// still passes after feedback filtering. Without this, wrapping a
|
||||
// running judge with filterJudgeFeedback would silently strip the
|
||||
// brand and GEPA would reject it.
|
||||
if (isRunningJudge(judge)) {
|
||||
wrapped[RUNNING_JUDGE_BRAND] = true;
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
/** Public helper — strips structural lines from a feedback string. */
|
||||
export function stripStructuralLines(
|
||||
feedback: string,
|
||||
filter: FeedbackFilter = defaultFeedbackFilter,
|
||||
): string {
|
||||
if (!feedback) return feedback;
|
||||
return feedback
|
||||
.split(/\r?\n/)
|
||||
.filter(line => filter(line) === 'value')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────
|
||||
|
||||
export class ComposeEvolution {
|
||||
async run(options: ComposeEvolutionOptions): Promise<ComposeEvolutionResult> {
|
||||
const signal = options.signal;
|
||||
const filter = options.feedbackFilter ?? defaultFeedbackFilter;
|
||||
|
||||
// Stage 1 — evolve the schema.
|
||||
options.onProgress?.({ stage: 'schema', message: 'starting schema evolution' });
|
||||
const schemaResult = await new EvolveSchema().run({
|
||||
...options.schema,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'schema', detail: e }),
|
||||
});
|
||||
|
||||
if (signal?.aborted) {
|
||||
return assembleAbortedResult(schemaResult, options.instructions.baseline);
|
||||
}
|
||||
|
||||
const frozenSchema = schemaResult.winner.schema;
|
||||
|
||||
// Stage 2 — evolve the instructions, with a filtered judge so GEPA
|
||||
// never sees structural feedback.
|
||||
options.onProgress?.({ stage: 'instructions', message: 'starting instruction evolution' });
|
||||
const filteredJudge = filterJudgeFeedback(options.instructions.judge, filter);
|
||||
|
||||
const gepaResult = await new IterativeGEPA().run({
|
||||
...options.instructions,
|
||||
judge: filteredJudge,
|
||||
signal,
|
||||
onProgress: (e) => options.onProgress?.({ stage: 'instructions', detail: e }),
|
||||
});
|
||||
|
||||
const combinedDelta =
|
||||
(gepaResult.winner.score?.overall ?? 0) -
|
||||
((schemaResult.history[0]?.score?.accuracy ?? 0));
|
||||
const fullyImproved = schemaResult.improved && gepaResult.improved;
|
||||
|
||||
options.onProgress?.({ stage: 'done' });
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: gepaResult,
|
||||
combinedDelta,
|
||||
fullyImproved,
|
||||
frozenSchema,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a convenience `SchemaExecuteFn` from a user-supplied runner that
|
||||
* only cares about the instruction prompt (not the schema itself). Used
|
||||
* by callers that want to compose — they supply one "execute" function
|
||||
* for the instructional stage and this helper wraps it for Stage 1.
|
||||
*
|
||||
* The wrapped executor serializes the schema into a short prefix the
|
||||
* model can follow (`Return JSON with fields: <field1>, <field2>, ...`)
|
||||
* before delegating to the caller's function.
|
||||
*/
|
||||
export function schemaExecutorFromInstructionRunner(
|
||||
runInstructions: (args: { prompt: string; input: string }) => Promise<string>,
|
||||
): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
const fieldList = schema.fields.map(f => `"${f.name}" (${f.type})`).join(', ');
|
||||
const prompt = `Return a JSON object with these fields: ${fieldList}.`;
|
||||
try {
|
||||
const actual = await runInstructions({ prompt, input });
|
||||
return { actual, parsed: actualLooksLikeJson(actual) };
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function actualLooksLikeJson(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
return (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
|
||||
(trimmed.startsWith('[') && trimmed.endsWith(']'));
|
||||
}
|
||||
|
||||
function assembleAbortedResult(
|
||||
schemaResult: EvolveSchemaResult,
|
||||
instructionBaseline: string,
|
||||
): ComposeEvolutionResult {
|
||||
// When aborted before Stage 2, produce a minimal GEPARunResult pointing
|
||||
// at the baseline instruction so callers have a stable shape.
|
||||
const baselineInstruction = {
|
||||
id: 'aborted',
|
||||
prompt: instructionBaseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'aborted',
|
||||
score: null,
|
||||
perExample: [],
|
||||
} as GEPARunResult['winner'];
|
||||
|
||||
return {
|
||||
schema: schemaResult,
|
||||
instructions: {
|
||||
winner: baselineInstruction,
|
||||
paretoFront: [baselineInstruction],
|
||||
history: [baselineInstruction],
|
||||
improved: false,
|
||||
delta: 0,
|
||||
},
|
||||
combinedDelta: 0,
|
||||
fullyImproved: false,
|
||||
frozenSchema: schemaResult.winner.schema,
|
||||
};
|
||||
}
|
||||
|
||||
/** Re-export types so callers can `import { type Schema } from '...'`. */
|
||||
export type { EvolveSchemaResult, GEPARunResult };
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* Correction Detector — identifies when a user is correcting the agent
|
||||
* and classifies corrections as durable (behavior-changing) vs task-local.
|
||||
*
|
||||
* Per Slice 7 correction #2:
|
||||
* - Durable corrections: repeated patterns that should influence future behavior → improvement signals
|
||||
* - Task-local corrections: one-off adjustments within a session → NOT signals
|
||||
* - One-off disagreements: not corrections at all → ignored
|
||||
*/
|
||||
|
||||
export type CorrectionDurability = 'durable' | 'task_local' | 'not_correction';
|
||||
|
||||
export interface DetectedCorrection {
|
||||
isDurable: boolean;
|
||||
durability: CorrectionDurability;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
confidence: number; // 0-1
|
||||
}
|
||||
|
||||
// ── Correction signal patterns ──────────────────────────────
|
||||
|
||||
/** Strong correction signals — high confidence the user is correcting agent behavior */
|
||||
const STRONG_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /\bno[,.]?\s*(?:not that|don'?t|do not|stop|never)\b/i, weight: 3 },
|
||||
{ pattern: /\bI (?:said|told you|asked)\b/i, weight: 3 },
|
||||
{ pattern: /\bthat'?s (?:wrong|incorrect|not (?:what|right))\b/i, weight: 3 },
|
||||
{ pattern: /\bplease (?:don'?t|do not|stop|never)\b/i, weight: 2 },
|
||||
{ pattern: /\binstead[,.]?\s*(?:use|do|try|go with)\b/i, weight: 2 },
|
||||
{ pattern: /\bwrong (?:approach|way|format|style|tone)\b/i, weight: 2 },
|
||||
{ pattern: /\bnot what I (?:wanted|meant|asked)\b/i, weight: 3 },
|
||||
];
|
||||
|
||||
/** Moderate correction signals — may be correction or just refinement */
|
||||
const MODERATE_CORRECTION_PATTERNS: Array<{ pattern: RegExp; weight: number }> = [
|
||||
{ pattern: /^no[,.:!]\s/i, weight: 2 }, // "No, ..." at start of message — strong disagreement
|
||||
{ pattern: /\bactually[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\brather[,.]?\s/i, weight: 1 },
|
||||
{ pattern: /\blet'?s (?:not|try|go with|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bprefer\b/i, weight: 1 },
|
||||
{ pattern: /\bshould (?:be|have been|use)\b/i, weight: 1 },
|
||||
{ pattern: /\bchange (?:it|this|that) to\b/i, weight: 1 },
|
||||
{ pattern: /\btoo (?:verbose|long|short|formal|casual|technical|simple)\b/i, weight: 1 },
|
||||
{ pattern: /\bstop (?:doing|using|adding)\b/i, weight: 1 },
|
||||
{ pattern: /\bkeep (?:it|things) (?:simple|short|brief|casual|formal)\b/i, weight: 1 },
|
||||
];
|
||||
|
||||
// ── Durability classification patterns ──────────────────────
|
||||
|
||||
/** Durable signals — the correction applies beyond this specific task */
|
||||
const DURABLE_SIGNALS: RegExp[] = [
|
||||
/\balways\b/i,
|
||||
/\bnever\b/i,
|
||||
/\bfrom now on\b/i,
|
||||
/\bin (?:the )?future\b/i,
|
||||
/\bwhenever\b/i,
|
||||
/\bevery time\b/i,
|
||||
/\bin general\b/i,
|
||||
/\bI (?:always |usually )?prefer\b/i,
|
||||
/\bmy (?:preference|style|approach)\b/i,
|
||||
/\bdon'?t (?:ever|again)\b/i,
|
||||
/\bremember (?:to|that)\b/i,
|
||||
/\bkeep (?:it|things|this)\b/i,
|
||||
];
|
||||
|
||||
/** Task-local signals — the correction is specific to this task */
|
||||
const TASK_LOCAL_SIGNALS: RegExp[] = [
|
||||
/\bthis (?:time|one|specific|particular)\b/i,
|
||||
/\bfor (?:this|now)\b/i,
|
||||
/\bjust (?:here|this|now)\b/i,
|
||||
/\bin this (?:case|instance|response)\b/i,
|
||||
/\bright now\b/i,
|
||||
/\bhere\b/i,
|
||||
];
|
||||
|
||||
// ── Pattern key extraction ──────────────────────────────────
|
||||
|
||||
/** Categories of behavioral corrections we can extract pattern keys for */
|
||||
const PATTERN_KEY_EXTRACTORS: Array<{ category: string; pattern: RegExp }> = [
|
||||
{ category: 'tone', pattern: /\btoo (?:formal|casual|verbose|terse|technical|simple)\b/i },
|
||||
{ category: 'format', pattern: /\b(?:format|formatting|headers?|bullet|numbering|markdown)\b/i },
|
||||
{ category: 'length', pattern: /\btoo (?:long|short|brief|detailed)\b/i },
|
||||
{ category: 'approach', pattern: /\bwrong (?:approach|way|method|strategy)\b/i },
|
||||
{ category: 'scope', pattern: /\btoo (?:much|many|broad|narrow|specific|general)\b/i },
|
||||
{ category: 'accuracy', pattern: /\b(?:wrong|incorrect|inaccurate|mistake|error)\b/i },
|
||||
{ category: 'style', pattern: /\b(?:style|voice|writing|wording|phrasing)\b/i },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect whether a user message contains a correction and classify it.
|
||||
*
|
||||
* Returns null if no correction detected.
|
||||
* Returns DetectedCorrection with durability classification if correction found.
|
||||
*/
|
||||
export function detectCorrection(
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
if (!userMessage || userMessage.length < 5) return null;
|
||||
|
||||
// Score correction strength
|
||||
let correctionScore = 0;
|
||||
for (const { pattern, weight } of STRONG_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
for (const { pattern, weight } of MODERATE_CORRECTION_PATTERNS) {
|
||||
if (pattern.test(userMessage)) correctionScore += weight;
|
||||
}
|
||||
|
||||
// Need minimum score to count as correction
|
||||
if (correctionScore < 2) return null;
|
||||
|
||||
// Classify durability
|
||||
const durability = classifyDurability(userMessage);
|
||||
|
||||
// Extract pattern key
|
||||
const patternKey = extractPatternKey(userMessage);
|
||||
|
||||
// Compute confidence (0-1)
|
||||
const confidence = Math.min(correctionScore / 6, 1);
|
||||
|
||||
// Build detail: first sentence or first 120 chars
|
||||
const detail = extractDetail(userMessage);
|
||||
|
||||
return {
|
||||
isDurable: durability === 'durable',
|
||||
durability,
|
||||
patternKey,
|
||||
detail,
|
||||
confidence,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyDurability(message: string): CorrectionDurability {
|
||||
let durableScore = 0;
|
||||
let taskLocalScore = 0;
|
||||
|
||||
for (const pattern of DURABLE_SIGNALS) {
|
||||
if (pattern.test(message)) durableScore++;
|
||||
}
|
||||
for (const pattern of TASK_LOCAL_SIGNALS) {
|
||||
if (pattern.test(message)) taskLocalScore++;
|
||||
}
|
||||
|
||||
// Explicit durable signals win
|
||||
if (durableScore > 0 && durableScore >= taskLocalScore) return 'durable';
|
||||
|
||||
// Explicit task-local signals
|
||||
if (taskLocalScore > 0) return 'task_local';
|
||||
|
||||
// No explicit signals — default to task_local (conservative; only promote to durable
|
||||
// when the same pattern_key recurs across sessions via ImprovementSignalStore)
|
||||
return 'task_local';
|
||||
}
|
||||
|
||||
function extractPatternKey(message: string): string {
|
||||
for (const { category, pattern } of PATTERN_KEY_EXTRACTORS) {
|
||||
const match = message.match(pattern);
|
||||
if (match) {
|
||||
const qualifier = match[0].toLowerCase().replace(/\s+/g, '_');
|
||||
return `${category}:${qualifier}`;
|
||||
}
|
||||
}
|
||||
// Fallback: generic correction key
|
||||
return 'general:correction';
|
||||
}
|
||||
|
||||
function extractDetail(message: string): string {
|
||||
// Take first sentence or first 120 chars
|
||||
const sentenceMatch = message.match(/^(.+?[.!?])\s/);
|
||||
if (sentenceMatch && sentenceMatch[1].length <= 120) {
|
||||
return sentenceMatch[1];
|
||||
}
|
||||
return message.length > 120 ? message.slice(0, 117) + '...' : message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a sequence of message pairs to detect corrections.
|
||||
* Useful for batch analysis of session history.
|
||||
*/
|
||||
export function detectCorrectionsInHistory(
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
): DetectedCorrection[] {
|
||||
const corrections: DetectedCorrection[] = [];
|
||||
for (let i = 1; i < messages.length; i++) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== 'user') continue;
|
||||
|
||||
const prevAssistant = i > 0 ? messages[i - 1] : undefined;
|
||||
const correction = detectCorrection(
|
||||
msg.content,
|
||||
prevAssistant?.role === 'assistant' ? prevAssistant.content : undefined,
|
||||
);
|
||||
if (correction) {
|
||||
corrections.push(correction);
|
||||
}
|
||||
}
|
||||
return corrections;
|
||||
}
|
||||
@@ -1,441 +0,0 @@
|
||||
/**
|
||||
* Eval Dataset Builder — turns execution traces and harvest frames into
|
||||
* train/val/holdout splits for the self-evolution loop.
|
||||
*
|
||||
* Sources mined:
|
||||
* - ExecutionTraceStore (outcome = success | verified → positive examples;
|
||||
* outcome = corrected → negative example with correctionFeedback as ground truth)
|
||||
* - Optional harvest frames (DistilledKnowledge) for Q&A-style augmentation
|
||||
* - Optional corrections from ImprovementSignalStore
|
||||
*
|
||||
* Filter pipeline (in order):
|
||||
* 1. Secret scanning — reject any example containing credentials / tokens
|
||||
* 2. Keyword heuristic — min length, non-trivial content, no duplicate inputs
|
||||
* 3. Optional LLM-as-judge relevance filter (pass an llmCall callback)
|
||||
*
|
||||
* Split is deterministic given a seed — same traces + same seed = same split.
|
||||
* Default split ratio is 60/20/20 train/val/holdout.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
TraceQueryFilter,
|
||||
} from '@waggle/core';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface EvalExample {
|
||||
input: string;
|
||||
expected_output: string;
|
||||
metadata: EvalExampleMetadata;
|
||||
}
|
||||
|
||||
export interface EvalExampleMetadata {
|
||||
traceId?: number;
|
||||
personaId?: string | null;
|
||||
taskShape?: string | null;
|
||||
model?: string | null;
|
||||
outcome?: TraceOutcome;
|
||||
tags?: string[];
|
||||
/** Where this example came from */
|
||||
source: 'trace' | 'harvest' | 'correction';
|
||||
/** Optional opaque identifier from the source system */
|
||||
sourceId?: string;
|
||||
}
|
||||
|
||||
export interface DatasetSplit {
|
||||
train: EvalExample[];
|
||||
val: EvalExample[];
|
||||
holdout: EvalExample[];
|
||||
/** Examples that were filtered out — useful for debugging */
|
||||
rejected: Array<{ reason: string; preview: string }>;
|
||||
/** Stats for logging / UI */
|
||||
stats: {
|
||||
sourced: number;
|
||||
acceptedAfterSecretScan: number;
|
||||
acceptedAfterHeuristic: number;
|
||||
acceptedAfterJudge: number;
|
||||
unique: number;
|
||||
total: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BuildOptions {
|
||||
/** Which trace outcomes count as positive examples (default: ['success', 'verified']). */
|
||||
positiveOutcomes?: TraceOutcome[];
|
||||
/** Whether to include correction traces as negative examples with feedback as expected_output (default true). */
|
||||
includeCorrections?: boolean;
|
||||
/** Filter passed straight through to ExecutionTraceStore.query */
|
||||
traceFilter?: Omit<TraceQueryFilter, 'outcome' | 'limit'> & { limit?: number };
|
||||
/** Optional external augmenters */
|
||||
harvestExamples?: EvalExample[];
|
||||
correctionExamples?: EvalExample[];
|
||||
/** Minimum input characters to keep the example (default 10). */
|
||||
minInputChars?: number;
|
||||
/** Minimum expected_output characters to keep the example (default 5). */
|
||||
minOutputChars?: number;
|
||||
/** Maximum input + output characters combined; over this, the example is rejected (default 16384). */
|
||||
maxCombinedChars?: number;
|
||||
/** Optional LLM-as-judge relevance filter. Pass null or omit to skip. */
|
||||
judge?: (example: EvalExample) => Promise<JudgeVerdict>;
|
||||
/** Ratio triple summing to 1.0 (default [0.6, 0.2, 0.2]). */
|
||||
splitRatios?: [number, number, number];
|
||||
/** Deterministic seed (default 1). */
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
export interface JudgeVerdict {
|
||||
keep: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
|
||||
// ── Deterministic PRNG (mulberry32) ─────────────────────────────
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
/** Fisher-Yates shuffle in-place using provided rng. */
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// ── Builder ─────────────────────────────────────────────────────
|
||||
|
||||
export class EvalDatasetBuilder {
|
||||
private store: ExecutionTraceStore;
|
||||
|
||||
constructor(store: ExecutionTraceStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Mine traces into a typed example list (before filtering). */
|
||||
sourceFromTraces(
|
||||
positiveOutcomes: TraceOutcome[],
|
||||
includeCorrections: boolean,
|
||||
filter: BuildOptions['traceFilter'] = {},
|
||||
): EvalExample[] {
|
||||
const outcomes: TraceOutcome[] = [...positiveOutcomes];
|
||||
if (includeCorrections) outcomes.push('corrected');
|
||||
|
||||
const rows = this.store.queryParsed({
|
||||
...filter,
|
||||
outcome: outcomes,
|
||||
limit: filter.limit ?? 10_000,
|
||||
});
|
||||
|
||||
return rows.map(row => traceToExample(row, includeCorrections));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the full dataset split.
|
||||
*
|
||||
* Filter order:
|
||||
* 1. Source (traces + augmenters)
|
||||
* 2. Secret scan
|
||||
* 3. Heuristic (length, non-trivial)
|
||||
* 4. Optional judge
|
||||
* 5. Dedup on input hash
|
||||
* 6. Shuffle + split
|
||||
*/
|
||||
async build(options: BuildOptions = {}): Promise<DatasetSplit> {
|
||||
const positiveOutcomes = options.positiveOutcomes ?? ['success', 'verified'];
|
||||
const includeCorrections = options.includeCorrections ?? true;
|
||||
const minInputChars = options.minInputChars ?? 10;
|
||||
const minOutputChars = options.minOutputChars ?? 5;
|
||||
const maxCombinedChars = options.maxCombinedChars ?? 16_384;
|
||||
const splitRatios = options.splitRatios ?? [0.6, 0.2, 0.2];
|
||||
const seed = options.seed ?? 1;
|
||||
|
||||
validateRatios(splitRatios);
|
||||
|
||||
const rejected: DatasetSplit['rejected'] = [];
|
||||
|
||||
// 1. Source
|
||||
const traceExamples = this.sourceFromTraces(
|
||||
positiveOutcomes,
|
||||
includeCorrections,
|
||||
options.traceFilter,
|
||||
);
|
||||
const sourced: EvalExample[] = [
|
||||
...traceExamples,
|
||||
...(options.harvestExamples ?? []),
|
||||
...(options.correctionExamples ?? []),
|
||||
];
|
||||
|
||||
// 2. Secret scan
|
||||
const afterSecretScan: EvalExample[] = [];
|
||||
for (const ex of sourced) {
|
||||
const hitInput = detectSecrets(ex.input);
|
||||
const hitOutput = detectSecrets(ex.expected_output);
|
||||
if (hitInput || hitOutput) {
|
||||
rejected.push({
|
||||
reason: `secret:${hitInput ?? hitOutput}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
afterSecretScan.push(ex);
|
||||
}
|
||||
|
||||
// 3. Heuristic
|
||||
const afterHeuristic: EvalExample[] = [];
|
||||
for (const ex of afterSecretScan) {
|
||||
const input = ex.input.trim();
|
||||
const output = ex.expected_output.trim();
|
||||
if (input.length < minInputChars) {
|
||||
rejected.push({ reason: 'too-short-input', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (output.length < minOutputChars) {
|
||||
rejected.push({ reason: 'too-short-output', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (input.length + output.length > maxCombinedChars) {
|
||||
rejected.push({ reason: 'too-long', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
if (isLowSignal(input) || isLowSignal(output)) {
|
||||
rejected.push({ reason: 'low-signal', preview: input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
afterHeuristic.push({ ...ex, input, expected_output: output });
|
||||
}
|
||||
|
||||
// 4. Optional judge
|
||||
let afterJudge = afterHeuristic;
|
||||
if (options.judge) {
|
||||
afterJudge = [];
|
||||
for (const ex of afterHeuristic) {
|
||||
try {
|
||||
const verdict = await options.judge(ex);
|
||||
if (verdict.keep) {
|
||||
afterJudge.push(ex);
|
||||
} else {
|
||||
rejected.push({
|
||||
reason: `judge:${verdict.reason ?? 'rejected'}`,
|
||||
preview: ex.input.slice(0, 80),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// On judge failure, keep the example — err on the side of more data.
|
||||
afterJudge.push(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Dedup on input hash
|
||||
const seen = new Set<string>();
|
||||
const unique: EvalExample[] = [];
|
||||
for (const ex of afterJudge) {
|
||||
const key = hashKey(ex.input);
|
||||
if (seen.has(key)) {
|
||||
rejected.push({ reason: 'duplicate', preview: ex.input.slice(0, 80) });
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
unique.push(ex);
|
||||
}
|
||||
|
||||
// 6. Shuffle + split
|
||||
const rng = makeRng(seed);
|
||||
const shuffled = shuffle([...unique], rng);
|
||||
const { train, val, holdout } = splitExamples(shuffled, splitRatios);
|
||||
|
||||
return {
|
||||
train,
|
||||
val,
|
||||
holdout,
|
||||
rejected,
|
||||
stats: {
|
||||
sourced: sourced.length,
|
||||
acceptedAfterSecretScan: afterSecretScan.length,
|
||||
acceptedAfterHeuristic: afterHeuristic.length,
|
||||
acceptedAfterJudge: afterJudge.length,
|
||||
unique: unique.length,
|
||||
total: train.length + val.length + holdout.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── JSONL IO ────────────────────────────────────────────────────
|
||||
|
||||
/** Serialize examples as JSONL (one JSON object per line). */
|
||||
export function toJSONL(examples: EvalExample[]): string {
|
||||
return examples.map(ex => JSON.stringify(ex)).join('\n');
|
||||
}
|
||||
|
||||
/** Parse JSONL into examples. Silently skips unparseable lines. */
|
||||
export function fromJSONL(jsonl: string): EvalExample[] {
|
||||
const out: EvalExample[] = [];
|
||||
for (const line of jsonl.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed.input === 'string' &&
|
||||
typeof parsed.expected_output === 'string' &&
|
||||
parsed.metadata
|
||||
) {
|
||||
out.push(parsed as EvalExample);
|
||||
}
|
||||
} catch {
|
||||
// skip malformed line
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function traceToExample(
|
||||
trace: ParsedExecutionTrace,
|
||||
includeCorrections: boolean,
|
||||
): EvalExample {
|
||||
const isCorrected = trace.outcome === 'corrected';
|
||||
const expected = isCorrected && includeCorrections
|
||||
? (trace.payload.correctionFeedback ?? trace.payload.output)
|
||||
: trace.payload.output;
|
||||
|
||||
return {
|
||||
input: trace.payload.input,
|
||||
expected_output: expected,
|
||||
metadata: {
|
||||
traceId: trace.id,
|
||||
personaId: trace.persona_id,
|
||||
taskShape: trace.task_shape,
|
||||
model: trace.model,
|
||||
outcome: trace.outcome,
|
||||
tags: trace.payload.tags ?? [],
|
||||
source: isCorrected ? 'correction' : 'trace',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function validateRatios(ratios: [number, number, number]): void {
|
||||
const sum = ratios[0] + ratios[1] + ratios[2];
|
||||
if (Math.abs(sum - 1.0) > 1e-6) {
|
||||
throw new Error(`Split ratios must sum to 1.0, got ${sum}`);
|
||||
}
|
||||
if (ratios.some(r => r < 0)) {
|
||||
throw new Error('Split ratios must be non-negative');
|
||||
}
|
||||
}
|
||||
|
||||
function splitExamples(
|
||||
examples: EvalExample[],
|
||||
ratios: [number, number, number],
|
||||
): { train: EvalExample[]; val: EvalExample[]; holdout: EvalExample[] } {
|
||||
const n = examples.length;
|
||||
// Floor-then-assign-remainders so every example ends up in exactly one split.
|
||||
const trainN = Math.floor(n * ratios[0]);
|
||||
const valN = Math.floor(n * ratios[1]);
|
||||
const holdoutN = n - trainN - valN;
|
||||
|
||||
return {
|
||||
train: examples.slice(0, trainN),
|
||||
val: examples.slice(trainN, trainN + valN),
|
||||
holdout: examples.slice(trainN + valN, trainN + valN + holdoutN),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap signal check — reject examples that are mostly whitespace, repeated
|
||||
* punctuation, or otherwise meaningless. Not a language detector.
|
||||
*/
|
||||
function isLowSignal(text: string): boolean {
|
||||
if (!text) return true;
|
||||
const stripped = text.replace(/\s+/g, '');
|
||||
if (stripped.length < 3) return true;
|
||||
// 80%+ of the same character → reject
|
||||
const counts = new Map<string, number>();
|
||||
for (const ch of stripped) {
|
||||
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
||||
}
|
||||
const maxCount = Math.max(...counts.values());
|
||||
if (maxCount / stripped.length > 0.8) return true;
|
||||
// No alphanumeric at all → reject
|
||||
if (!/[A-Za-z0-9]/.test(stripped)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* FNV-1a 32-bit hash — stable across processes, no crypto import needed.
|
||||
* Used for input-level dedup.
|
||||
*/
|
||||
function hashKey(text: string): string {
|
||||
let h = 0x811c9dc5;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
h ^= text.charCodeAt(i);
|
||||
h = Math.imul(h, 0x01000193);
|
||||
}
|
||||
return (h >>> 0).toString(16);
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
/**
|
||||
* Evolution Deploy Helpers — Phase 6.1 of the self-evolution loop.
|
||||
*
|
||||
* When the user accepts an EvolutionRun, the orchestrator calls the
|
||||
* caller-supplied `deploy` callback. This module provides the built-in
|
||||
* building blocks for that callback:
|
||||
*
|
||||
* deployPersonaOverride — writes a persona JSON file that
|
||||
* shadows the built-in persona at
|
||||
* runtime via loadCustomPersonas.
|
||||
* deployBehavioralSpecOverride — writes a JSON file in
|
||||
* {dataDir}/behavioral-overrides/
|
||||
* that can be merged into
|
||||
* BEHAVIORAL_SPEC sections at load.
|
||||
* loadBehavioralSpecOverrides — reads those overrides from disk.
|
||||
*
|
||||
* Every writer:
|
||||
* - Creates the target directory if missing.
|
||||
* - Writes atomically (write to .tmp, rename) so a crash mid-write
|
||||
* never corrupts the config.
|
||||
* - Backs up the previous version to {file}.bak before overwriting.
|
||||
* - Returns a structured DeployResult with paths + timestamps.
|
||||
*
|
||||
* These helpers are pure filesystem operations — they don't talk to
|
||||
* any runtime singletons, so the server wiring is trivial:
|
||||
* orchestrator.accept(uuid) → onDeploy(run)
|
||||
* → deployPersonaOverride(dataDir, ...)
|
||||
* → reloadPersonas()
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
import { getPersona } from './personas.js';
|
||||
import { assertValidCustomPersonaId } from './custom-personas.js';
|
||||
|
||||
// ── Public result shape ────────────────────────────────────────
|
||||
|
||||
export interface DeployResult {
|
||||
/** Absolute path that was written */
|
||||
path: string;
|
||||
/** Path to the backup of the previous version (null if first write) */
|
||||
backupPath: string | null;
|
||||
/** When the deploy happened */
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
// ── Persona override ───────────────────────────────────────────
|
||||
|
||||
export interface DeployPersonaInput {
|
||||
/** Persona id — matches a built-in persona (the deploy SHADOWS it). */
|
||||
personaId: string;
|
||||
/** The evolved system prompt text */
|
||||
systemPrompt: string;
|
||||
/** Optional additional field overrides */
|
||||
overrides?: Partial<Omit<AgentPersona, 'id' | 'systemPrompt'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write an evolved persona system prompt to disk so the custom-personas
|
||||
* loader picks it up on next `listPersonas()` call.
|
||||
*
|
||||
* When a built-in persona with this id exists, the override inherits
|
||||
* all its properties (name, tools, tagline, etc) except systemPrompt
|
||||
* (replaced with the evolved one). When no built-in exists, a minimal
|
||||
* persona shell is built from the input.
|
||||
*/
|
||||
export function deployPersonaOverride(
|
||||
dataDir: string,
|
||||
input: DeployPersonaInput,
|
||||
): DeployResult {
|
||||
assertValidCustomPersonaId(input.personaId);
|
||||
const personasDir = path.join(dataDir, 'personas');
|
||||
if (!fs.existsSync(personasDir)) {
|
||||
fs.mkdirSync(personasDir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(personasDir, `${input.personaId}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const builtin = getPersona(input.personaId);
|
||||
const persona: AgentPersona = builtin
|
||||
? {
|
||||
...builtin,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
}
|
||||
: {
|
||||
name: input.personaId,
|
||||
description: input.overrides?.description ?? `${input.personaId} (evolved)`,
|
||||
icon: input.overrides?.icon ?? 'sparkles',
|
||||
modelPreference: input.overrides?.modelPreference ?? 'claude-sonnet-4-6',
|
||||
tools: input.overrides?.tools ?? [],
|
||||
workspaceAffinity: input.overrides?.workspaceAffinity ?? [],
|
||||
suggestedCommands: input.overrides?.suggestedCommands ?? [],
|
||||
defaultWorkflow: input.overrides?.defaultWorkflow ?? null,
|
||||
...input.overrides,
|
||||
id: input.personaId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(persona, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll back a previously-deployed persona override to the .bak version
|
||||
* (or remove the override entirely if there was no previous version).
|
||||
*/
|
||||
export function rollbackPersonaOverride(
|
||||
dataDir: string,
|
||||
personaId: string,
|
||||
): boolean {
|
||||
assertValidCustomPersonaId(personaId);
|
||||
const filePath = path.join(dataDir, 'personas', `${personaId}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Behavioral-spec override ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* BEHAVIORAL_SPEC is compiled into the bundle, so we cannot mutate it
|
||||
* on disk without rebuilding. Instead, overrides live as JSON files in
|
||||
* {dataDir}/behavioral-overrides/ and the runtime merges them at load.
|
||||
*
|
||||
* One file per section, filename = section name (coreLoop, qualityRules,
|
||||
* behavioralRules, workPatterns, intelligenceDefaults).
|
||||
*/
|
||||
|
||||
export type BehavioralSpecSection =
|
||||
| 'coreLoop'
|
||||
| 'qualityRules'
|
||||
| 'behavioralRules'
|
||||
| 'workPatterns'
|
||||
| 'intelligenceDefaults';
|
||||
|
||||
export const BEHAVIORAL_SPEC_SECTIONS: BehavioralSpecSection[] = [
|
||||
'coreLoop',
|
||||
'qualityRules',
|
||||
'behavioralRules',
|
||||
'workPatterns',
|
||||
'intelligenceDefaults',
|
||||
];
|
||||
|
||||
export interface DeployBehavioralSpecInput {
|
||||
/** Which section to override */
|
||||
section: BehavioralSpecSection;
|
||||
/** The evolved text for that section */
|
||||
text: string;
|
||||
/** Optional evolution-run uuid for audit */
|
||||
runUuid?: string;
|
||||
}
|
||||
|
||||
export interface BehavioralSpecOverride {
|
||||
section: BehavioralSpecSection;
|
||||
text: string;
|
||||
runUuid?: string;
|
||||
deployedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a behavioral-spec section override to disk. The runtime loader
|
||||
* (loadBehavioralSpecOverrides) reads these on startup and merges them
|
||||
* into the compiled BEHAVIORAL_SPEC at runtime.
|
||||
*/
|
||||
export function deployBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
input: DeployBehavioralSpecInput,
|
||||
): DeployResult {
|
||||
if (!BEHAVIORAL_SPEC_SECTIONS.includes(input.section)) {
|
||||
throw new Error(`Unknown behavioral-spec section: ${input.section}`);
|
||||
}
|
||||
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
const filePath = path.join(dir, `${input.section}.json`);
|
||||
const backupPath = fs.existsSync(filePath) ? `${filePath}.bak` : null;
|
||||
if (backupPath) {
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
}
|
||||
|
||||
const payload: BehavioralSpecOverride = {
|
||||
section: input.section,
|
||||
text: input.text,
|
||||
runUuid: input.runUuid,
|
||||
deployedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
writeAtomic(filePath, JSON.stringify(payload, null, 2));
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
backupPath,
|
||||
deployedAt: payload.deployedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Roll back one section (restores .bak, or removes if no previous override). */
|
||||
export function rollbackBehavioralSpecOverride(
|
||||
dataDir: string,
|
||||
section: BehavioralSpecSection,
|
||||
): boolean {
|
||||
const filePath = path.join(dataDir, 'behavioral-overrides', `${section}.json`);
|
||||
const backupPath = `${filePath}.bak`;
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.renameSync(backupPath, filePath);
|
||||
return true;
|
||||
}
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Read all behavioral-spec overrides from disk as a map section → text. */
|
||||
export function loadBehavioralSpecOverrides(
|
||||
dataDir: string,
|
||||
): Partial<Record<BehavioralSpecSection, string>> {
|
||||
const dir = path.join(dataDir, 'behavioral-overrides');
|
||||
if (!fs.existsSync(dir)) return {};
|
||||
|
||||
const out: Partial<Record<BehavioralSpecSection, string>> = {};
|
||||
try {
|
||||
for (const section of BEHAVIORAL_SPEC_SECTIONS) {
|
||||
const filePath = path.join(dir, `${section}.json`);
|
||||
if (!fs.existsSync(filePath)) continue;
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||||
const parsed = JSON.parse(raw) as Partial<BehavioralSpecOverride>;
|
||||
if (parsed && typeof parsed.text === 'string' && parsed.text.length > 0) {
|
||||
out[section] = parsed.text;
|
||||
}
|
||||
} catch { /* skip malformed */ }
|
||||
}
|
||||
} catch { /* dir unreadable */ }
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Merge overrides into a baseline spec object, preferring the override text. */
|
||||
export function applyBehavioralSpecOverrides(
|
||||
baseline: Record<BehavioralSpecSection, string>,
|
||||
overrides: Partial<Record<BehavioralSpecSection, string>>,
|
||||
): Record<BehavioralSpecSection, string> {
|
||||
return {
|
||||
...baseline,
|
||||
...Object.fromEntries(
|
||||
(Object.entries(overrides) as Array<[BehavioralSpecSection, string]>)
|
||||
.filter(([, v]) => typeof v === 'string' && v.length > 0),
|
||||
),
|
||||
} as Record<BehavioralSpecSection, string>;
|
||||
}
|
||||
|
||||
// ── Internal ───────────────────────────────────────────────────
|
||||
|
||||
/** Atomic write — write to .tmp then rename into place. */
|
||||
function writeAtomic(filePath: string, contents: string): void {
|
||||
const tmpPath = `${filePath}.tmp`;
|
||||
fs.writeFileSync(tmpPath, contents, 'utf-8');
|
||||
const waitBuffer = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
try {
|
||||
for (let attempt = 1; attempt <= 10; attempt++) {
|
||||
try {
|
||||
fs.renameSync(tmpPath, filePath);
|
||||
return;
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
const transient = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
|
||||
if (!transient || attempt === 10) throw error;
|
||||
// Windows antivirus and indexers can briefly hold an exclusive handle.
|
||||
Atomics.wait(waitBuffer, 0, 0, 25 * attempt);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
/**
|
||||
* Evolution Gates — Phase 2.2 of the self-evolution loop.
|
||||
*
|
||||
* Gatekeeping for prompts/skills produced by the iterative GEPA optimizer
|
||||
* before they reach production. A candidate must pass ALL gates; the first
|
||||
* failure short-circuits and returns a `fail` verdict with the reason.
|
||||
*
|
||||
* Four gate categories:
|
||||
*
|
||||
* 1. Size gates — hard character caps per target type (persona prompt
|
||||
* ≤ 3 000 chars, tool description ≤ 500, skill ≤ 15 KB,
|
||||
* behavioral-spec section ≤ 4 000).
|
||||
* 2. Growth gates — percentage cap over baseline (default +20%) prevents
|
||||
* runaway prompt bloat across generations.
|
||||
* 3. Structural — non-empty, no unresolved placeholders, balanced
|
||||
* markdown fences, no obvious template accidents
|
||||
* (e.g. "[PLACEHOLDER]", "TODO:" left in).
|
||||
* 4. Regression — score delta on a held-out eval set must not drop
|
||||
* below the allowed tolerance (default -0.02, i.e. a
|
||||
* 2% accuracy regression is the worst allowed).
|
||||
*
|
||||
* Each gate returns structured reasons so the orchestrator (phase 4)
|
||||
* can log WHY a candidate was rejected and surface it in the UI.
|
||||
*/
|
||||
|
||||
import type { EvolutionTarget } from './iterative-optimizer.js';
|
||||
|
||||
// ── Public types ───────────────────────────────────────────────
|
||||
|
||||
export type GateVerdict = 'pass' | 'fail';
|
||||
|
||||
export interface GateResult {
|
||||
gate: string;
|
||||
verdict: GateVerdict;
|
||||
reason: string;
|
||||
/** Optional measurable detail: e.g. `{ current: 3520, max: 3000 }` */
|
||||
detail?: Record<string, number | string>;
|
||||
}
|
||||
|
||||
export interface GateCheckResult {
|
||||
verdict: GateVerdict;
|
||||
/** First failing gate, or null if all passed */
|
||||
firstFailure: GateResult | null;
|
||||
/** All gate results in order */
|
||||
results: GateResult[];
|
||||
}
|
||||
|
||||
export interface SizeLimits {
|
||||
personaSystemPrompt: number;
|
||||
toolDescription: number;
|
||||
skillBody: number;
|
||||
behavioralSpecSection: number;
|
||||
generic: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_SIZE_LIMITS: SizeLimits = {
|
||||
personaSystemPrompt: 3_000,
|
||||
toolDescription: 500,
|
||||
skillBody: 15_000,
|
||||
behavioralSpecSection: 4_000,
|
||||
generic: 8_000,
|
||||
};
|
||||
|
||||
export interface GateOptions {
|
||||
/** What is being evolved — drives which size cap applies. Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Max allowed % growth over baseline. Default 0.2 (i.e. +20%). */
|
||||
maxGrowthRatio?: number;
|
||||
/** Largest allowed accuracy regression vs baseline. Default -0.02 (i.e. -2%). */
|
||||
maxRegression?: number;
|
||||
/** Override the default size limits */
|
||||
sizeLimits?: Partial<SizeLimits>;
|
||||
/** Allow empty candidates (rarely useful — default false) */
|
||||
allowEmpty?: boolean;
|
||||
}
|
||||
|
||||
export interface CheckInput {
|
||||
/** The new candidate text */
|
||||
candidate: string;
|
||||
/** The baseline (pre-evolution) text — used for growth + regression checks */
|
||||
baseline: string;
|
||||
/** Optional scores to enable the regression gate */
|
||||
scores?: {
|
||||
baseline: number;
|
||||
candidate: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run all gates in order. Returns a structured verdict. The overall
|
||||
* verdict is `pass` iff every gate passed; `fail` with `firstFailure`
|
||||
* populated otherwise.
|
||||
*
|
||||
* Callers can choose to short-circuit on the first failure or to
|
||||
* collect all of them — `runGates` returns every gate result either way.
|
||||
*/
|
||||
export function runGates(input: CheckInput, options: GateOptions = {}): GateCheckResult {
|
||||
const targetKind = options.targetKind ?? 'generic';
|
||||
const sizeLimits = { ...DEFAULT_SIZE_LIMITS, ...options.sizeLimits };
|
||||
const maxGrowthRatio = options.maxGrowthRatio ?? 0.2;
|
||||
const maxRegression = options.maxRegression ?? -0.02;
|
||||
const allowEmpty = options.allowEmpty ?? false;
|
||||
|
||||
const results: GateResult[] = [];
|
||||
|
||||
// 1. Non-empty
|
||||
if (!allowEmpty) {
|
||||
results.push(checkNonEmpty(input.candidate));
|
||||
}
|
||||
|
||||
// 2. Size
|
||||
results.push(checkSize(input.candidate, targetKind, sizeLimits));
|
||||
|
||||
// 3. Growth
|
||||
results.push(checkGrowth(input.candidate, input.baseline, maxGrowthRatio));
|
||||
|
||||
// 4. Structural integrity
|
||||
results.push(checkBalancedFences(input.candidate));
|
||||
results.push(checkNoPlaceholders(input.candidate));
|
||||
results.push(checkNoObviousTodos(input.candidate));
|
||||
|
||||
// 5. Regression (optional — skipped if scores not provided)
|
||||
if (input.scores) {
|
||||
results.push(checkRegression(input.scores.baseline, input.scores.candidate, maxRegression));
|
||||
}
|
||||
|
||||
const firstFailure = results.find(r => r.verdict === 'fail') ?? null;
|
||||
return {
|
||||
verdict: firstFailure ? 'fail' : 'pass',
|
||||
firstFailure,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Individual gates (exported for unit tests + fine-grained use) ──
|
||||
|
||||
export function checkNonEmpty(candidate: string): GateResult {
|
||||
const trimmed = candidate.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return {
|
||||
gate: 'non-empty',
|
||||
verdict: 'fail',
|
||||
reason: 'Candidate is empty or whitespace only',
|
||||
};
|
||||
}
|
||||
return { gate: 'non-empty', verdict: 'pass', reason: 'non-empty' };
|
||||
}
|
||||
|
||||
export function checkSize(
|
||||
candidate: string,
|
||||
targetKind: EvolutionTarget,
|
||||
limits: SizeLimits,
|
||||
): GateResult {
|
||||
const max = resolveSizeLimit(targetKind, limits);
|
||||
const len = candidate.length;
|
||||
if (len > max) {
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate is ${len} chars but ${targetKind} cap is ${max}`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'size',
|
||||
verdict: 'pass',
|
||||
reason: `Within ${targetKind} limit (${len}/${max})`,
|
||||
detail: { current: len, max },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkGrowth(
|
||||
candidate: string,
|
||||
baseline: string,
|
||||
maxGrowthRatio: number,
|
||||
): GateResult {
|
||||
// A candidate shorter than baseline always passes the growth gate.
|
||||
if (candidate.length <= baseline.length) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'Candidate is not larger than baseline',
|
||||
detail: { baseline: baseline.length, candidate: candidate.length },
|
||||
};
|
||||
}
|
||||
|
||||
// Empty baseline means any candidate can't be measured as a ratio —
|
||||
// defer to the size gate instead.
|
||||
if (baseline.length === 0) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: 'No baseline to compare against',
|
||||
};
|
||||
}
|
||||
|
||||
const ratio = (candidate.length - baseline.length) / baseline.length;
|
||||
if (ratio > maxGrowthRatio) {
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate grew ${(ratio * 100).toFixed(1)}% over baseline (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'growth',
|
||||
verdict: 'pass',
|
||||
reason: `Grew ${(ratio * 100).toFixed(1)}% (cap ${(maxGrowthRatio * 100).toFixed(0)}%)`,
|
||||
detail: { baseline: baseline.length, candidate: candidate.length, ratio },
|
||||
};
|
||||
}
|
||||
|
||||
export function checkBalancedFences(candidate: string): GateResult {
|
||||
const fenceMatches = candidate.match(/```/g);
|
||||
const count = fenceMatches?.length ?? 0;
|
||||
if (count % 2 !== 0) {
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'fail',
|
||||
reason: `Odd number of markdown code fences (${count}) — unbalanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'balanced-fences',
|
||||
verdict: 'pass',
|
||||
reason: `${count} fence(s), balanced`,
|
||||
detail: { fences: count },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates that still contain placeholder text like `[TODO]`,
|
||||
* `<placeholder>`, or `{{var}}` — these are almost always unfinished
|
||||
* generations.
|
||||
*/
|
||||
export function checkNoPlaceholders(candidate: string): GateResult {
|
||||
const patterns: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'bracket-placeholder', re: /\[(?:PLACEHOLDER|TODO|FIXME|INSERT|XXX|YOUR[_ ][A-Z ]+)\]/i },
|
||||
{ name: 'angle-placeholder', re: /<(?:placeholder|todo|your[-_ ][a-z ]+)>/i },
|
||||
{ name: 'handlebars-placeholder', re: /\{\{[a-zA-Z_][a-zA-Z0-9_.]*\}\}/ },
|
||||
];
|
||||
|
||||
for (const { name, re } of patterns) {
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-placeholders',
|
||||
verdict: 'fail',
|
||||
reason: `Found unresolved placeholder (${name}): "${m[0]}"`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { gate: 'no-placeholders', verdict: 'pass', reason: 'no unresolved placeholders' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject candidates with "TODO:" or "FIXME:" headers — a prompt shouldn't
|
||||
* ship with unfinished author notes.
|
||||
*/
|
||||
export function checkNoObviousTodos(candidate: string): GateResult {
|
||||
// Only match lines that START with the marker (as a label), not prose
|
||||
// that happens to mention "todo" (e.g. a task-list prompt).
|
||||
const re = /(?:^|\n)\s*(?:TODO|FIXME|XXX)\s*:/i;
|
||||
const m = candidate.match(re);
|
||||
if (m) {
|
||||
return {
|
||||
gate: 'no-todos',
|
||||
verdict: 'fail',
|
||||
reason: `Found leftover author marker: "${m[0].trim()}"`,
|
||||
};
|
||||
}
|
||||
return { gate: 'no-todos', verdict: 'pass', reason: 'no leftover author markers' };
|
||||
}
|
||||
|
||||
export function checkRegression(
|
||||
baselineScore: number,
|
||||
candidateScore: number,
|
||||
maxRegression: number,
|
||||
): GateResult {
|
||||
const delta = candidateScore - baselineScore;
|
||||
// Small epsilon avoids float-precision false negatives when delta is
|
||||
// numerically equal to the tolerance (e.g. 0.78 - 0.8 === -0.02000002).
|
||||
const epsilon = 1e-9;
|
||||
if (delta < maxRegression - epsilon) {
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'fail',
|
||||
reason: `Candidate score regressed by ${(delta * 100).toFixed(2)}pp (allowed ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
return {
|
||||
gate: 'regression',
|
||||
verdict: 'pass',
|
||||
reason: `Score delta ${(delta * 100).toFixed(2)}pp (allowed floor ${(maxRegression * 100).toFixed(2)}pp)`,
|
||||
detail: { baseline: baselineScore, candidate: candidateScore, delta },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function resolveSizeLimit(target: EvolutionTarget, limits: SizeLimits): number {
|
||||
switch (target) {
|
||||
case 'persona-system-prompt': return limits.personaSystemPrompt;
|
||||
case 'tool-description': return limits.toolDescription;
|
||||
case 'skill-body': return limits.skillBody;
|
||||
case 'behavioral-spec-section': return limits.behavioralSpecSection;
|
||||
case 'generic':
|
||||
default:
|
||||
return limits.generic;
|
||||
}
|
||||
}
|
||||
@@ -1,43 +1,29 @@
|
||||
/**
|
||||
* Evolution LLM Wiring — binds real LLMs into the self-evolution loop.
|
||||
* LLM wiring for the skill audit and skill hygiene routes.
|
||||
*
|
||||
* The evolution primitives (judge, iterative-optimizer, evolve-schema) are
|
||||
* deliberately model-agnostic: they accept plain callables. This module
|
||||
* supplies production adapters that turn a Haiku-class LLM into those
|
||||
* callables, so the `/api/evolution/run` endpoint can perform a real run.
|
||||
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
|
||||
* This module supplies the production adapter that turns a Haiku-class LLM
|
||||
* into that callable, plus a retry wrapper for transient provider errors.
|
||||
*
|
||||
* Public surface:
|
||||
*
|
||||
* createAnthropicEvolutionLLM(apiKey, model?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax — same pattern as
|
||||
* `optimizer-service.ts`. Dynamic import to avoid startup cost when
|
||||
* evolution is not in use.
|
||||
* createAnthropicEvolutionLLM(apiKey, options?)
|
||||
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
|
||||
* avoid startup cost when no audit runs.
|
||||
*
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
* buildGEPAMutateFn(llm) → MutateFn (for IterativeGEPA)
|
||||
* buildSchemaExecuteFn(llm) → SchemaExecuteFn (for EvolveSchema)
|
||||
* makeRunningJudge(base, llm) → Pick<LLMJudge,'score'>
|
||||
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
|
||||
*
|
||||
* The `makeRunningJudge` wrapper is what enables GEPA to evaluate real
|
||||
* prompt effectiveness: on each score() call it first executes the
|
||||
* candidate prompt against the example input via the LLM, then delegates
|
||||
* to the wrapped judge with the model's actual response. Without this,
|
||||
* GEPA compares prompt text to expected output directly — less useful.
|
||||
*
|
||||
* All adapters are pure wrappers over `EvolutionLLM.complete()`, which is a
|
||||
* minimal `(prompt: string) => Promise<string>` contract. Tests supply
|
||||
* in-memory mocks of that contract; production code goes through AxAI.
|
||||
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
|
||||
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
|
||||
* mocks of that contract; production code goes through AxAI.
|
||||
*/
|
||||
|
||||
import type { JudgeLLMCall, JudgeInput, JudgeScore } from './judge.js';
|
||||
import type { LLMJudge } from './judge.js';
|
||||
import type { MutateArgs, MutateFn, EvolutionTarget } from './iterative-optimizer.js';
|
||||
import type { SchemaExecuteFn, Schema, SchemaField } from './evolve-schema.js';
|
||||
import type { JudgeLLMCall } from './judge.js';
|
||||
|
||||
// ── Minimal LLM contract ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A minimal single-turn completion contract. All evolution adapters depend
|
||||
* A minimal single-turn completion contract. All adapters depend
|
||||
* on this interface rather than on @ax-llm/ax directly so they stay easy to
|
||||
* mock in unit tests.
|
||||
*/
|
||||
@@ -206,10 +192,10 @@ export interface CreateAnthropicEvolutionLLMOptions {
|
||||
|
||||
/**
|
||||
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
|
||||
* @ax-llm/ax so evolution is a zero-cost dependency until used.
|
||||
* @ax-llm/ax so it is a zero-cost dependency until used.
|
||||
*
|
||||
* Returns null when @ax-llm/ax is unavailable — callers should treat null
|
||||
* as "evolution disabled" and surface a clear error to the user.
|
||||
* as "LLM unavailable" and surface a clear error to the user.
|
||||
*/
|
||||
export async function createAnthropicEvolutionLLM(
|
||||
apiKey: string,
|
||||
@@ -240,8 +226,8 @@ export async function createAnthropicEvolutionLLM(
|
||||
const response = await ai.chat({
|
||||
chatPrompt: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
// chat() may return a ReadableStream in streaming mode — evolution
|
||||
// does not stream, so we only handle the non-stream case.
|
||||
// chat() may return a ReadableStream in streaming mode — audits
|
||||
// do not stream, so we only handle the non-stream case.
|
||||
const typed = response as {
|
||||
results?: readonly { content?: string }[];
|
||||
};
|
||||
@@ -249,7 +235,7 @@ export async function createAnthropicEvolutionLLM(
|
||||
return typeof first === 'string' ? first : '';
|
||||
},
|
||||
};
|
||||
// Apply retry policy — defaults are sized for background evolution runs
|
||||
// Apply retry policy — defaults are sized for background runs
|
||||
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
|
||||
// by passing `retry: { maxAttempts: 1 }`.
|
||||
return wrapWithRetry(inner, options.retry ?? {});
|
||||
@@ -268,217 +254,3 @@ export async function createAnthropicEvolutionLLM(
|
||||
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
|
||||
return (prompt: string) => llm.complete(prompt);
|
||||
}
|
||||
|
||||
// ── Reflective mutation (GEPA) ──────────────────────────────────
|
||||
|
||||
export interface BuildReflectiveMutationPromptArgs {
|
||||
parent: string;
|
||||
strategy: string;
|
||||
weaknessFeedback: string[];
|
||||
targetKind: EvolutionTarget;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reflective mutation prompt for GEPA. Returns the *mutated child*
|
||||
* text only — no commentary, no wrapping — per the instruction in the
|
||||
* prompt body. The LLM is told what to fix, using the weakness signal from
|
||||
* the parent's worst-scoring eval examples.
|
||||
*/
|
||||
export function buildReflectiveMutationPrompt(
|
||||
args: BuildReflectiveMutationPromptArgs,
|
||||
): string {
|
||||
const weaknessBullets = args.weaknessFeedback.length > 0
|
||||
? args.weaknessFeedback.map(fb => `- ${fb}`).join('\n')
|
||||
: '(no specific weakness signals yet)';
|
||||
|
||||
return `You are evolving an AI prompt. This is generation ${args.generation}.
|
||||
|
||||
TARGET KIND: ${args.targetKind}
|
||||
STRATEGY: ${args.strategy}
|
||||
|
||||
PARENT PROMPT (do not return this unchanged — produce a *better* variant):
|
||||
---
|
||||
${args.parent}
|
||||
---
|
||||
|
||||
WEAKNESS SIGNALS from the parent's lowest-scoring eval examples:
|
||||
${weaknessBullets}
|
||||
|
||||
Apply the strategy above to address the weakness signals. Keep the same
|
||||
role and overall structure — do not change the prompt's purpose. Produce
|
||||
a concrete variant that should score higher on the next eval.
|
||||
|
||||
Return ONLY the new prompt text. No preamble, no explanation, no markdown
|
||||
fences. Plain text only.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a GEPA MutateFn that uses an LLM to produce reflective mutations.
|
||||
* Graceful fallback: if the LLM throws or returns empty, returns the
|
||||
* parent prompt unchanged (GEPA will still evaluate it as part of the
|
||||
* Pareto population — no progress that generation, but no crash).
|
||||
*/
|
||||
export function buildGEPAMutateFn(llm: EvolutionLLM): MutateFn {
|
||||
return async (args: MutateArgs): Promise<string> => {
|
||||
const prompt = buildReflectiveMutationPrompt({
|
||||
parent: args.parent.prompt,
|
||||
strategy: args.strategy,
|
||||
weaknessFeedback: args.weaknessFeedback,
|
||||
targetKind: args.targetKind,
|
||||
generation: args.generation,
|
||||
});
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(prompt);
|
||||
} catch {
|
||||
return args.parent.prompt;
|
||||
}
|
||||
const cleaned = stripFences(raw).trim();
|
||||
if (cleaned.length === 0) return args.parent.prompt;
|
||||
return cleaned;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Schema execute (ES) ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a schema-filling prompt that asks the LLM to return JSON matching
|
||||
* the given schema for the given user input. Every field is documented by
|
||||
* name + type + description + required flag + constraints.
|
||||
*/
|
||||
export function buildSchemaFillPrompt(args: { schema: Schema; input: string }): string {
|
||||
const fieldLines = args.schema.fields.map(f => formatField(f)).join('\n');
|
||||
return `Fill the JSON schema below using the user input. Respond with
|
||||
valid JSON — no markdown, no prose, no trailing commas.
|
||||
|
||||
SCHEMA (name: ${args.schema.name}, version: ${args.schema.version}):
|
||||
${fieldLines}
|
||||
|
||||
USER INPUT:
|
||||
${args.input}
|
||||
|
||||
Return ONLY the JSON object.`;
|
||||
}
|
||||
|
||||
function formatField(field: SchemaField): string {
|
||||
const req = field.required ? 'required' : 'optional';
|
||||
const constraints = field.constraints.length > 0
|
||||
? ' constraints: ' + field.constraints.map(c => `${c.kind}=${stringifyConstraintValue(c.value)}`).join(', ')
|
||||
: '';
|
||||
return `- "${field.name}" (${field.type}, ${req}): ${field.description}${constraints}`;
|
||||
}
|
||||
|
||||
function stringifyConstraintValue(value: string | number | string[]): string {
|
||||
if (Array.isArray(value)) return `[${value.join(',')}]`;
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a SchemaExecuteFn that asks the LLM to fill a schema. Returns
|
||||
* `parsed: true` only when the response parses as JSON. On any error
|
||||
* (network, timeout, bad model), returns `{actual: '', parsed: false}`.
|
||||
*/
|
||||
export function buildSchemaExecuteFn(llm: EvolutionLLM): SchemaExecuteFn {
|
||||
return async ({ schema, input }) => {
|
||||
let raw = '';
|
||||
try {
|
||||
raw = await llm.complete(buildSchemaFillPrompt({ schema, input }));
|
||||
} catch {
|
||||
return { actual: '', parsed: false };
|
||||
}
|
||||
const cleaned = stripFences(raw);
|
||||
return { actual: cleaned, parsed: looksLikeJSON(cleaned) };
|
||||
};
|
||||
}
|
||||
|
||||
// ── Running judge (executes candidate prompt first, then scores) ─
|
||||
|
||||
/**
|
||||
* Wrap a base judge so that each score() call first runs the candidate
|
||||
* prompt through the LLM on the example's input. The base judge then
|
||||
* receives the LLM's output as `actual`, giving a real behavioral signal
|
||||
* instead of a prompt-text-to-expected-output comparison.
|
||||
*
|
||||
* When the LLM call fails, returns a zero score with clear feedback so the
|
||||
* Pareto population treats the candidate as a loser without crashing the
|
||||
* whole generation.
|
||||
*/
|
||||
/**
|
||||
* Phantom property that marks a judge as running-capable (i.e. it executes
|
||||
* the candidate prompt against a real LLM before scoring). IterativeGEPA
|
||||
* checks for this to prevent regressions where a bare prompt-text-similarity
|
||||
* judge is passed by mistake.
|
||||
*
|
||||
* Symbol-keyed to avoid accidental conflicts with user-defined fields.
|
||||
*/
|
||||
export const RUNNING_JUDGE_BRAND = Symbol.for('waggle.runningJudge');
|
||||
|
||||
export interface RunningJudge extends Pick<LLMJudge, 'score'> {
|
||||
readonly [RUNNING_JUDGE_BRAND]: true;
|
||||
}
|
||||
|
||||
export function isRunningJudge(j: Pick<LLMJudge, 'score'> | RunningJudge): j is RunningJudge {
|
||||
return (j as Record<symbol, unknown>)[RUNNING_JUDGE_BRAND] === true;
|
||||
}
|
||||
|
||||
export function makeRunningJudge(
|
||||
baseJudge: Pick<LLMJudge, 'score'>,
|
||||
llm: EvolutionLLM,
|
||||
): RunningJudge {
|
||||
return {
|
||||
[RUNNING_JUDGE_BRAND]: true as const,
|
||||
async score(args: JudgeInput): Promise<JudgeScore> {
|
||||
// `args.actual` is the candidate prompt supplied by GEPA. Treat it as
|
||||
// a system prompt, run it against the example input via the LLM, and
|
||||
// use the LLM's reply as the new `actual` for the downstream judge.
|
||||
const runPrompt = buildRunPrompt(args.actual, args.input);
|
||||
let modelOutput = '';
|
||||
try {
|
||||
modelOutput = await llm.complete(runPrompt);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
overall: 0, weighted: 0,
|
||||
correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1,
|
||||
feedback: `Candidate execution failed: ${msg}`,
|
||||
parsed: false,
|
||||
};
|
||||
}
|
||||
return baseJudge.score({ ...args, actual: modelOutput });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildRunPrompt(candidatePrompt: string, userInput: string): string {
|
||||
return `${candidatePrompt}
|
||||
|
||||
USER INPUT:
|
||||
${userInput}
|
||||
|
||||
Respond now.`;
|
||||
}
|
||||
|
||||
// ── Shared helpers ──────────────────────────────────────────────
|
||||
|
||||
/** Strip ``` and ```json fences if present. Never throws. */
|
||||
function stripFences(raw: string): string {
|
||||
if (!raw) return '';
|
||||
return raw
|
||||
.replace(/```json\s*/gi, '')
|
||||
.replace(/```\s*/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function looksLikeJSON(s: string): boolean {
|
||||
const trimmed = s.trim();
|
||||
if (!trimmed) return false;
|
||||
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return false;
|
||||
try {
|
||||
JSON.parse(trimmed);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
/**
|
||||
* Evolution Orchestrator — Phase 4 of the self-evolution loop.
|
||||
*
|
||||
* Wires the primitives into a closed loop:
|
||||
*
|
||||
* traces → eval dataset → ComposeEvolution → gates → run store
|
||||
* ↓
|
||||
* user accepts/rejects
|
||||
* ↓
|
||||
* deploy callback fires
|
||||
* ↓
|
||||
* runStore.markDeployed()
|
||||
*
|
||||
* The orchestrator itself does NOT hot-reload personas or rewrite the
|
||||
* behavioral spec — it calls a pluggable `deploy` callback supplied by
|
||||
* the caller. This keeps the orchestrator pure (easy to unit-test) and
|
||||
* lets the server/desktop app wire up the real side-effects (file write,
|
||||
* persona reload, event emission) separately.
|
||||
*
|
||||
* Persistence is in @waggle/core's `EvolutionRunStore`. Every run — even
|
||||
* rejected or failed — is preserved for audit.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExecutionTraceStore,
|
||||
EvolutionRun,
|
||||
EvolutionRunStore,
|
||||
EvolutionRunStatus,
|
||||
EvolutionRunTarget,
|
||||
ParsedExecutionTrace,
|
||||
TraceOutcome,
|
||||
} from '@waggle/core';
|
||||
import { ComposeEvolution, type ComposeEvolutionOptions, type ComposeEvolutionResult } from './compose-evolution.js';
|
||||
import { runGates, type GateResult, type GateOptions } from './evolution-gates.js';
|
||||
import { EvalDatasetBuilder, type EvalExample } from './eval-dataset.js';
|
||||
|
||||
// ── Dependency injection types ─────────────────────────────────
|
||||
|
||||
export interface EvolutionOrchestratorDeps {
|
||||
/** Source of traces (from @waggle/core) */
|
||||
traceStore: ExecutionTraceStore;
|
||||
/** Persistent run history (from @waggle/core) */
|
||||
runStore: EvolutionRunStore;
|
||||
/**
|
||||
* Function invoked when a run moves from accepted → (deployed | failed).
|
||||
* Returning normally marks the run deployed; throwing marks it failed
|
||||
* with the error message as reason.
|
||||
*
|
||||
* This is where the server writes to the persona file, reloads the
|
||||
* orchestrator cache, updates behavioral-spec.ts, etc.
|
||||
*/
|
||||
deploy?: (run: EvolutionRun) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface EvolutionAutoTriggerConfig {
|
||||
/** Minimum traces (with the given outcome filter) before a run is triggered */
|
||||
minTraces: number;
|
||||
/** Restrict trigger to a specific persona / workspace / taskShape */
|
||||
traceFilter?: {
|
||||
personaId?: string;
|
||||
workspaceId?: string;
|
||||
taskShape?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EvolutionOrchestratorOptions {
|
||||
/** What is being evolved (drives gate policy + persistence tag) */
|
||||
targetKind: EvolutionRunTarget;
|
||||
/** Stable identifier — e.g. persona id, spec-section name */
|
||||
targetName?: string;
|
||||
/** The current baseline text to evolve against */
|
||||
baseline: string;
|
||||
/** Compose options — already wired with examples, judge, execute, mutate */
|
||||
compose: Omit<ComposeEvolutionOptions, 'schema' | 'instructions'> & {
|
||||
schema: Omit<ComposeEvolutionOptions['schema'], 'baseline'>;
|
||||
instructions: Omit<ComposeEvolutionOptions['instructions'], 'baseline'>;
|
||||
};
|
||||
/** Optional gate overrides (size caps, growth ratio, etc) */
|
||||
gateOptions?: GateOptions;
|
||||
/** Improvement threshold required to create a proposal (default 0.02) */
|
||||
minDelta?: number;
|
||||
/** Auto-trigger configuration — when omitted, runOnce() is manual-only */
|
||||
autoTrigger?: EvolutionAutoTriggerConfig;
|
||||
/** Abort the run early */
|
||||
signal?: AbortSignal;
|
||||
/** Optional progress pass-through */
|
||||
onProgress?: (event: EvolutionProgress) => void;
|
||||
}
|
||||
|
||||
export interface EvolutionProgress {
|
||||
phase: 'trigger-check' | 'dataset' | 'compose' | 'gates' | 'persist' | 'skipped' | 'done';
|
||||
message?: string;
|
||||
detail?: unknown;
|
||||
}
|
||||
|
||||
// ── Schema baseline input ──────────────────────────────────────
|
||||
|
||||
/** Optional structural baseline when the target has one (e.g. persona's DSPy signature). */
|
||||
export interface SchemaBaselineInput {
|
||||
baseline: ComposeEvolutionOptions['schema']['baseline'];
|
||||
}
|
||||
|
||||
// ── Run result ─────────────────────────────────────────────────
|
||||
|
||||
export type OrchestratorOutcome =
|
||||
| 'proposed' // Run created, awaiting accept/reject
|
||||
| 'skipped-trigger'
|
||||
| 'skipped-gates' // Completed but gates failed
|
||||
| 'skipped-delta' // Improvement below minDelta
|
||||
| 'aborted';
|
||||
|
||||
export interface OrchestratorRunResult {
|
||||
outcome: OrchestratorOutcome;
|
||||
/** Populated for 'proposed' and 'skipped-gates' */
|
||||
run?: EvolutionRun;
|
||||
/** Raw ComposeEvolution result when the compose stage ran */
|
||||
compose?: ComposeEvolutionResult;
|
||||
/** Gate results for audit */
|
||||
gateResults?: GateResult[];
|
||||
/** Reason if outcome is skipped/aborted */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ── Orchestrator ──────────────────────────────────────────────
|
||||
|
||||
export class EvolutionOrchestrator {
|
||||
private deps: EvolutionOrchestratorDeps;
|
||||
|
||||
constructor(deps: EvolutionOrchestratorDeps) {
|
||||
this.deps = deps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full orchestration pipeline once. Returns structured outcome.
|
||||
* Does NOT deploy — that happens when the caller calls `accept(uuid)`.
|
||||
*/
|
||||
async runOnce(options: EvolutionOrchestratorOptions & {
|
||||
schemaBaseline: SchemaBaselineInput['baseline'];
|
||||
}): Promise<OrchestratorRunResult> {
|
||||
const emit = (phase: EvolutionProgress['phase'], message?: string, detail?: unknown) => {
|
||||
options.onProgress?.({ phase, message, detail });
|
||||
};
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted before start' };
|
||||
}
|
||||
|
||||
// Trigger check — skip if auto-trigger thresholds aren't met.
|
||||
if (options.autoTrigger) {
|
||||
emit('trigger-check');
|
||||
const ok = this.meetsAutoTrigger(options.autoTrigger);
|
||||
if (!ok) {
|
||||
emit('skipped');
|
||||
return {
|
||||
outcome: 'skipped-trigger',
|
||||
reason: `fewer than ${options.autoTrigger.minTraces} eligible traces`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Derive the eval dataset from finalized traces of the requested target.
|
||||
emit('dataset');
|
||||
const examples = this.buildExamplesFromTraces(options);
|
||||
if (examples.length === 0) {
|
||||
emit('skipped', 'no eval examples');
|
||||
return { outcome: 'skipped-trigger', reason: 'no eligible traces to form dataset' };
|
||||
}
|
||||
|
||||
// Run the compose pipeline.
|
||||
emit('compose');
|
||||
const composeResult = await new ComposeEvolution().run({
|
||||
schema: { ...options.compose.schema, baseline: options.schemaBaseline, examples },
|
||||
instructions: { ...options.compose.instructions, baseline: options.baseline, examples },
|
||||
feedbackFilter: options.compose.feedbackFilter,
|
||||
signal: options.signal,
|
||||
onProgress: (e) => emit('compose', undefined, e),
|
||||
});
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
return { outcome: 'aborted', reason: 'aborted during compose', compose: composeResult };
|
||||
}
|
||||
|
||||
const winnerText = composeResult.instructions.winner.prompt;
|
||||
// Apples-to-apples: compare the GEPA baseline candidate (history[0])
|
||||
// to the GEPA winner. Both are scored by the same judge on the same
|
||||
// eval stage, so the delta is a clean signal.
|
||||
const baselineOverall = composeResult.instructions.history[0]?.score?.overall ?? 0;
|
||||
const winnerOverall = composeResult.instructions.winner.score?.overall ?? 0;
|
||||
const delta = winnerOverall - baselineOverall;
|
||||
const minDelta = options.minDelta ?? 0.02;
|
||||
|
||||
// Gate check.
|
||||
emit('gates');
|
||||
const gateResult = runGates(
|
||||
{
|
||||
candidate: winnerText,
|
||||
baseline: options.baseline,
|
||||
scores: { baseline: baselineOverall, candidate: winnerOverall },
|
||||
},
|
||||
{
|
||||
targetKind: options.targetKind,
|
||||
...options.gateOptions,
|
||||
},
|
||||
);
|
||||
|
||||
if (delta < minDelta) {
|
||||
emit('skipped', `delta ${(delta * 100).toFixed(2)}pp below minimum ${(minDelta * 100).toFixed(2)}pp`);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-delta',
|
||||
reason: `winner improved only ${(delta * 100).toFixed(2)}pp (min ${(minDelta * 100).toFixed(2)}pp required)`,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
// Persist — create the proposed run even if gates failed, so there's audit.
|
||||
emit('persist');
|
||||
const run = this.deps.runStore.create({
|
||||
targetKind: options.targetKind,
|
||||
targetName: options.targetName ?? null,
|
||||
baselineText: options.baseline,
|
||||
winnerText,
|
||||
winnerSchema: composeResult.frozenSchema,
|
||||
deltaAccuracy: delta,
|
||||
gateVerdict: gateResult.verdict,
|
||||
gateReasons: gateResult.results.map(r => ({
|
||||
gate: r.gate, verdict: r.verdict, reason: r.reason,
|
||||
})),
|
||||
artifacts: {
|
||||
runSeed: options.compose.instructions.seed ?? null,
|
||||
generations: options.compose.instructions.generations ?? null,
|
||||
paretoFrontSize: composeResult.schema.paretoFront.length,
|
||||
exampleCount: examples.length,
|
||||
},
|
||||
});
|
||||
|
||||
if (gateResult.verdict === 'fail') {
|
||||
// Immediately reject runs that fail gates so the user doesn't see
|
||||
// dangerous candidates in the "review" queue. Run history is kept.
|
||||
const rejected = this.deps.runStore.reject(
|
||||
run.run_uuid,
|
||||
`gate failure: ${gateResult.firstFailure?.reason ?? 'unknown'}`,
|
||||
);
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'skipped-gates',
|
||||
reason: gateResult.firstFailure?.reason ?? 'gate failure',
|
||||
run: rejected ?? run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
emit('done');
|
||||
return {
|
||||
outcome: 'proposed',
|
||||
run,
|
||||
compose: composeResult,
|
||||
gateResults: gateResult.results,
|
||||
};
|
||||
}
|
||||
|
||||
/** Accept a proposed run and invoke the deploy callback. */
|
||||
async accept(runUuid: string, userNote?: string): Promise<EvolutionRun | undefined> {
|
||||
const accepted = this.deps.runStore.accept(runUuid, userNote);
|
||||
if (!accepted || accepted.status !== 'accepted') return accepted;
|
||||
|
||||
if (!this.deps.deploy) {
|
||||
// No deploy hook configured — leave as 'accepted' and let the caller
|
||||
// mark deployed manually.
|
||||
return accepted;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.deps.deploy(accepted);
|
||||
return this.deps.runStore.markDeployed(runUuid);
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
return this.deps.runStore.markFailed(runUuid, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reject a proposed run with optional reason. */
|
||||
reject(runUuid: string, reason?: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.reject(runUuid, reason);
|
||||
}
|
||||
|
||||
/** List runs (thin pass-through, handy for the UI). */
|
||||
list(filter?: Parameters<EvolutionRunStore['list']>[0]): EvolutionRun[] {
|
||||
return this.deps.runStore.list(filter);
|
||||
}
|
||||
|
||||
/** Get a single run by uuid. */
|
||||
get(runUuid: string): EvolutionRun | undefined {
|
||||
return this.deps.runStore.getByUuid(runUuid);
|
||||
}
|
||||
|
||||
// ── Internals ───────────────────────────────────────────────
|
||||
|
||||
private meetsAutoTrigger(trigger: EvolutionAutoTriggerConfig): boolean {
|
||||
const outcomes: TraceOutcome[] = ['success', 'verified', 'corrected'];
|
||||
const traces = this.deps.traceStore.query({
|
||||
...(trigger.traceFilter ?? {}),
|
||||
outcome: outcomes,
|
||||
limit: Math.max(trigger.minTraces * 2, 50),
|
||||
});
|
||||
return traces.length >= trigger.minTraces;
|
||||
}
|
||||
|
||||
private buildExamplesFromTraces(
|
||||
options: EvolutionOrchestratorOptions,
|
||||
): EvalExample[] {
|
||||
// If the caller already supplied examples via compose.schema/instructions,
|
||||
// we respect that. Otherwise we mine them from the trace store.
|
||||
const existing = options.compose.schema.examples ?? options.compose.instructions.examples;
|
||||
if (existing && existing.length > 0) return existing;
|
||||
|
||||
const builder = new EvalDatasetBuilder(this.deps.traceStore);
|
||||
const traceFilter = options.autoTrigger?.traceFilter ?? {};
|
||||
const traces = builder.sourceFromTraces(['success', 'verified'], true, {
|
||||
...traceFilter,
|
||||
limit: 500,
|
||||
});
|
||||
return traces;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Convenience helpers ────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Filter a list of parsed traces down to those likely useful for evolution.
|
||||
* Keeps finalized (non-pending) traces with non-empty I/O. Used by callers
|
||||
* that want to manually pre-filter before handing to the orchestrator.
|
||||
*/
|
||||
export function eligibleForEvolution(traces: ParsedExecutionTrace[]): ParsedExecutionTrace[] {
|
||||
return traces.filter(t =>
|
||||
t.outcome !== 'pending' &&
|
||||
t.payload.input.trim().length > 0 &&
|
||||
(t.payload.output.trim().length > 0 || t.payload.correctionFeedback),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper that summarizes runs by status/target for dashboard display.
|
||||
*/
|
||||
export function summarizeRuns(runs: EvolutionRun[]): {
|
||||
total: number;
|
||||
byStatus: Record<EvolutionRunStatus, number>;
|
||||
byTargetKind: Record<string, number>;
|
||||
bestDelta: number;
|
||||
} {
|
||||
const byStatus: Record<EvolutionRunStatus, number> = {
|
||||
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
|
||||
};
|
||||
const byTargetKind: Record<string, number> = {};
|
||||
let bestDelta = 0;
|
||||
|
||||
for (const r of runs) {
|
||||
byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
|
||||
byTargetKind[r.target_kind] = (byTargetKind[r.target_kind] ?? 0) + 1;
|
||||
if (r.delta_accuracy > bestDelta) bestDelta = r.delta_accuracy;
|
||||
}
|
||||
|
||||
return {
|
||||
total: runs.length,
|
||||
byStatus,
|
||||
byTargetKind,
|
||||
bestDelta,
|
||||
};
|
||||
}
|
||||
@@ -1,852 +0,0 @@
|
||||
/**
|
||||
* EvolveSchema — Phase 3.1 of the self-evolution loop.
|
||||
*
|
||||
* Port of Mikhail Pavlukhin's EvolveSchema algorithm (Egzakta research).
|
||||
* Evolves the STRUCTURE of a typed output schema (DSPy-style signatures):
|
||||
* field names, types, descriptions, ordering, and constraints. This is
|
||||
* orthogonal to the GEPA optimizer which evolves INSTRUCTIONS against a
|
||||
* fixed schema.
|
||||
*
|
||||
* Three-phase per-generation loop:
|
||||
*
|
||||
* Phase A — Structure Discovery
|
||||
* Spawn variants that add, replace, or drop output fields. This is
|
||||
* the single biggest-impact mutation class (Mikhail's paper shows
|
||||
* ~74% of HotPotQA gains come from one structural mutation).
|
||||
*
|
||||
* Phase B — Field-Order Probes
|
||||
* Permute field order. LLMs condition on earlier fields when filling
|
||||
* later ones, so putting a "reasoning" field BEFORE "answer" routinely
|
||||
* outperforms the reverse.
|
||||
*
|
||||
* Phase C — Failure-Driven Refinement
|
||||
* Use the per-example judge feedback from the previous generation to
|
||||
* target specific fields — edit their descriptions, tighten / loosen
|
||||
* their constraints, or change their types where signals suggest.
|
||||
*
|
||||
* 8 typed mutations are available:
|
||||
* add_output_field, remove_field, edit_field_desc, change_field_type,
|
||||
* add_constraint, remove_constraint, reorder_fields, replace_output_fields
|
||||
*
|
||||
* Pareto selection is 2-dimensional: (accuracy, -complexity) — an
|
||||
* accurate-but-complex schema is not automatically preferred over a
|
||||
* slightly-less-accurate simpler one.
|
||||
*
|
||||
* Callers provide:
|
||||
* - `execute(schema, input) → actualOutput` the LLM runner
|
||||
* - `judge.score({input, expected, actual})` the LLMJudge from Phase 1.3
|
||||
* - `mutate` (optional) a custom LLM-driven mutator
|
||||
* for edit_field_desc; falls
|
||||
* back to a deterministic
|
||||
* rewriter when omitted.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
|
||||
// ── Schema types ───────────────────────────────────────────────
|
||||
|
||||
export type FieldType = 'string' | 'number' | 'boolean' | 'array' | 'object' | 'enum';
|
||||
|
||||
export interface FieldConstraint {
|
||||
kind: 'minLength' | 'maxLength' | 'pattern' | 'enum' | 'range' | 'custom';
|
||||
value: string | number | string[];
|
||||
}
|
||||
|
||||
export interface SchemaField {
|
||||
name: string;
|
||||
type: FieldType;
|
||||
description: string;
|
||||
required: boolean;
|
||||
constraints: FieldConstraint[];
|
||||
}
|
||||
|
||||
export interface Schema {
|
||||
name: string;
|
||||
fields: SchemaField[];
|
||||
version: number;
|
||||
}
|
||||
|
||||
// ── Mutation types ─────────────────────────────────────────────
|
||||
|
||||
export type MutationKind =
|
||||
| 'add_output_field'
|
||||
| 'remove_field'
|
||||
| 'edit_field_desc'
|
||||
| 'change_field_type'
|
||||
| 'add_constraint'
|
||||
| 'remove_constraint'
|
||||
| 'reorder_fields'
|
||||
| 'replace_output_fields';
|
||||
|
||||
export interface Mutation {
|
||||
kind: MutationKind;
|
||||
/** What changed, for logging + feedback */
|
||||
description: string;
|
||||
/** Functional transformer (pure) */
|
||||
apply: (schema: Schema) => Schema;
|
||||
}
|
||||
|
||||
// ── Candidate + result types ──────────────────────────────────
|
||||
|
||||
export interface SchemaCandidate {
|
||||
id: string;
|
||||
schema: Schema;
|
||||
generation: number;
|
||||
parent: string | null;
|
||||
mutation: MutationKind | 'baseline';
|
||||
mutationLabel: string;
|
||||
score: SchemaCandidateScore | null;
|
||||
perExample: SchemaExampleResult[];
|
||||
}
|
||||
|
||||
export interface SchemaExampleResult {
|
||||
input: string;
|
||||
expected: string;
|
||||
actual: string;
|
||||
score: JudgeScore;
|
||||
/** Diagnostic: did the output parse against the schema? */
|
||||
parsed: boolean;
|
||||
}
|
||||
|
||||
export interface SchemaCandidateScore {
|
||||
/** Mean overall judge score across the eval set (0..1) */
|
||||
accuracy: number;
|
||||
/** Schema "complexity" — a scalar that grows with fields, descriptions, constraints */
|
||||
complexity: number;
|
||||
/** Share of examples that produced parseable output against this schema (0..1) */
|
||||
parseRate: number;
|
||||
/** Weakness feedback taken from the worst-scoring 3 examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Number of examples scored */
|
||||
n: number;
|
||||
}
|
||||
|
||||
export type SchemaExecuteFn = (args: {
|
||||
schema: Schema;
|
||||
input: string;
|
||||
}) => Promise<{ actual: string; parsed: boolean }>;
|
||||
|
||||
export interface EvolveSchemaOptions {
|
||||
/** Starting schema */
|
||||
baseline: Schema;
|
||||
/** Eval dataset */
|
||||
examples: EvalExample[];
|
||||
/** LLM runner that takes a schema + input and returns an attempted output */
|
||||
execute: SchemaExecuteFn;
|
||||
/** Judge for scoring individual attempts */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Candidates per generation (default 5) */
|
||||
populationSize?: number;
|
||||
/** Generations (default 3) */
|
||||
generations?: number;
|
||||
/** Mix of phase mutations per generation */
|
||||
mutationMix?: { structure: number; order: number; refinement: number };
|
||||
/** Eval sample size per generation (default 32) */
|
||||
evalSize?: number;
|
||||
/** Anchor eval size at the end (default 100) */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for sampling (default 1) */
|
||||
seed?: number;
|
||||
/** Optional LLM helper for description edits — deterministic fallback used if omitted */
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>;
|
||||
/** Optional progress emitter */
|
||||
onProgress?: (event: EvolveSchemaProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaProgress {
|
||||
phase: 'start' | 'structure' | 'order' | 'refinement' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
bestAccuracy: number;
|
||||
bestComplexity: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface EvolveSchemaResult {
|
||||
winner: SchemaCandidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: SchemaCandidate[];
|
||||
/** All candidates across all generations */
|
||||
history: SchemaCandidate[];
|
||||
/** Accuracy delta: winner.accuracy - baseline.accuracy */
|
||||
deltaAccuracy: number;
|
||||
/** True if the winner's accuracy improved over baseline */
|
||||
improved: boolean;
|
||||
}
|
||||
|
||||
// ── Pure mutation functions ───────────────────────────────────
|
||||
|
||||
export function addOutputField(
|
||||
schema: Schema,
|
||||
field: SchemaField,
|
||||
position?: number,
|
||||
): Schema {
|
||||
const pos = position ?? schema.fields.length;
|
||||
const fields = [...schema.fields];
|
||||
fields.splice(pos, 0, { ...field, constraints: [...field.constraints] });
|
||||
return { ...schema, fields, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function removeField(schema: Schema, fieldName: string): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.filter(f => f.name !== fieldName),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function editFieldDescription(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newDescription: string,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, description: newDescription } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function changeFieldType(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
newType: FieldType,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, type: newType } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function addConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraint: FieldConstraint,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f =>
|
||||
f.name === fieldName ? { ...f, constraints: [...f.constraints, constraint] } : f,
|
||||
),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function removeConstraint(
|
||||
schema: Schema,
|
||||
fieldName: string,
|
||||
constraintIndex: number,
|
||||
): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: schema.fields.map(f => {
|
||||
if (f.name !== fieldName) return f;
|
||||
const constraints = f.constraints.filter((_, i) => i !== constraintIndex);
|
||||
return { ...f, constraints };
|
||||
}),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function reorderFields(schema: Schema, newOrder: string[]): Schema {
|
||||
const byName = new Map(schema.fields.map(f => [f.name, f]));
|
||||
const reordered: SchemaField[] = [];
|
||||
for (const name of newOrder) {
|
||||
const field = byName.get(name);
|
||||
if (field) reordered.push(field);
|
||||
}
|
||||
// Append any fields that were omitted from newOrder (keeps their position stable).
|
||||
for (const field of schema.fields) {
|
||||
if (!newOrder.includes(field.name)) reordered.push(field);
|
||||
}
|
||||
return { ...schema, fields: reordered, version: schema.version + 1 };
|
||||
}
|
||||
|
||||
export function replaceOutputFields(schema: Schema, newFields: SchemaField[]): Schema {
|
||||
return {
|
||||
...schema,
|
||||
fields: newFields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
version: schema.version + 1,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Complexity + scoring helpers ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Scalar complexity measure: 1 per field + 0.3 per constraint +
|
||||
* 0.01 per character of description. Calibrated so a 3-field schema
|
||||
* with short descriptions ≈ 3.6, a 10-field schema with long
|
||||
* descriptions ≈ 15+.
|
||||
*/
|
||||
export function schemaComplexity(schema: Schema): number {
|
||||
let c = 0;
|
||||
for (const field of schema.fields) {
|
||||
c += 1;
|
||||
c += field.constraints.length * 0.3;
|
||||
c += (field.description?.length ?? 0) * 0.01;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
export function aggregateSchemaScores(results: SchemaExampleResult[]): SchemaCandidateScore {
|
||||
if (results.length === 0) {
|
||||
return {
|
||||
accuracy: 0, complexity: 0, parseRate: 0, weaknessFeedback: [], n: 0,
|
||||
};
|
||||
}
|
||||
const n = results.length;
|
||||
const accuracy = results.reduce((s, r) => s + r.score.overall, 0) / n;
|
||||
const parseRate = results.filter(r => r.parsed).length / n;
|
||||
const worst = [...results].sort((a, b) => a.score.overall - b.score.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(r => r.score.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
return { accuracy, complexity: 0, parseRate, weaknessFeedback, n };
|
||||
}
|
||||
|
||||
// ── Pareto (2D) ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front on (accuracy↑, complexity↓). A candidate dominates
|
||||
* another iff it is at least as accurate AND no more complex, and
|
||||
* strictly better on at least one dimension.
|
||||
*/
|
||||
export function paretoFrontSchema(candidates: SchemaCandidate[]): SchemaCandidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: SchemaCandidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominatesSchema(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominatesSchema(a: SchemaCandidateScore, b: SchemaCandidateScore): boolean {
|
||||
const geAll = a.accuracy >= b.accuracy && a.complexity <= b.complexity;
|
||||
const gtAny = a.accuracy > b.accuracy || a.complexity < b.complexity;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
// ── Candidate scoring ─────────────────────────────────────────
|
||||
|
||||
export async function scoreSchemaCandidate(
|
||||
candidate: SchemaCandidate,
|
||||
examples: EvalExample[],
|
||||
execute: SchemaExecuteFn,
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<SchemaCandidateScore> {
|
||||
const results: SchemaExampleResult[] = [];
|
||||
for (const ex of examples) {
|
||||
if (signal?.aborted) break;
|
||||
try {
|
||||
const { actual, parsed } = await execute({ schema: candidate.schema, input: ex.input });
|
||||
const score = await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual,
|
||||
});
|
||||
results.push({ input: ex.input, expected: ex.expected_output, actual, score, parsed });
|
||||
} catch {
|
||||
// Keep iterating — one broken example shouldn't poison the whole batch.
|
||||
}
|
||||
}
|
||||
const agg = aggregateSchemaScores(results);
|
||||
agg.complexity = schemaComplexity(candidate.schema);
|
||||
candidate.perExample = results;
|
||||
candidate.score = agg;
|
||||
return agg;
|
||||
}
|
||||
|
||||
// ── Mutation generators per phase ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Structure-Discovery mutations: add a new field, replace fields wholesale,
|
||||
* or drop one if the schema is too large. Returns up to `n` mutations.
|
||||
*/
|
||||
export function generateStructureMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Add a reasoning field if absent.
|
||||
if (!parent.fields.some(f => /reason|thought|scratch/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add reasoning field before answer',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Concise chain-of-thought explaining how the answer was derived.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
}, 0),
|
||||
});
|
||||
}
|
||||
|
||||
// Add a confidence field if absent.
|
||||
if (!parent.fields.some(f => /confidence|certainty|score/i.test(f.name))) {
|
||||
out.push({
|
||||
kind: 'add_output_field',
|
||||
description: 'add confidence field at end',
|
||||
apply: (s) =>
|
||||
addOutputField(s, {
|
||||
name: 'confidence',
|
||||
type: 'number',
|
||||
description: 'Self-estimated confidence in the answer (0.0 - 1.0).',
|
||||
required: false,
|
||||
constraints: [{ kind: 'range', value: '0..1' }],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// Drop the lowest-priority field if > 4 fields.
|
||||
if (parent.fields.length > 4) {
|
||||
const candidate = parent.fields[parent.fields.length - 1];
|
||||
out.push({
|
||||
kind: 'remove_field',
|
||||
description: `drop trailing field "${candidate.name}"`,
|
||||
apply: (s) => removeField(s, candidate.name),
|
||||
});
|
||||
}
|
||||
|
||||
// Replace all output fields with a minimal skeleton (reasoning + answer).
|
||||
if (parent.fields.length >= 2) {
|
||||
out.push({
|
||||
kind: 'replace_output_fields',
|
||||
description: 'collapse to {reasoning, answer}',
|
||||
apply: (s) =>
|
||||
replaceOutputFields(s, [
|
||||
{
|
||||
name: 'reasoning',
|
||||
type: 'string',
|
||||
description: 'Brief chain-of-thought.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
{
|
||||
name: 'answer',
|
||||
type: 'string',
|
||||
description: 'Final answer.',
|
||||
required: true,
|
||||
constraints: [],
|
||||
},
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order-Probe mutations: generate permutations of field order. Limits
|
||||
* output to `n` by random sampling from the set of useful permutations.
|
||||
*/
|
||||
export function generateOrderMutations(
|
||||
parent: Schema,
|
||||
n: number,
|
||||
rng: () => number,
|
||||
): Mutation[] {
|
||||
const names = parent.fields.map(f => f.name);
|
||||
if (names.length < 2) return [];
|
||||
const out: Mutation[] = [];
|
||||
|
||||
// Heuristic 1: move any reasoning/thought field to the front.
|
||||
const reasoningIdx = parent.fields.findIndex(f => /reason|thought|scratch/i.test(f.name));
|
||||
if (reasoningIdx > 0) {
|
||||
const newOrder = [names[reasoningIdx], ...names.filter((_, i) => i !== reasoningIdx)];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[reasoningIdx]}" to front`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Heuristic 2: move any confidence/score field to the end.
|
||||
const confIdx = parent.fields.findIndex(f => /confidence|certainty/i.test(f.name));
|
||||
if (confIdx >= 0 && confIdx !== parent.fields.length - 1) {
|
||||
const newOrder = [...names.filter((_, i) => i !== confIdx), names[confIdx]];
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `move "${names[confIdx]}" to end`,
|
||||
apply: (s) => reorderFields(s, newOrder),
|
||||
});
|
||||
}
|
||||
|
||||
// Fill remaining with random permutations.
|
||||
while (out.length < n) {
|
||||
const shuffled = shuffle([...names], rng);
|
||||
if (!arraysEqual(shuffled, names)) {
|
||||
out.push({
|
||||
kind: 'reorder_fields',
|
||||
description: `reorder to [${shuffled.join(', ')}]`,
|
||||
apply: (s) => reorderFields(s, shuffled),
|
||||
});
|
||||
}
|
||||
if (out.length > n * 3) break; // safety — very short schemas
|
||||
}
|
||||
|
||||
return pickN(out, n, rng);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refinement mutations driven by worst-scoring example feedback.
|
||||
* Edits descriptions, tightens constraints, or switches types based on
|
||||
* heuristic patterns in the feedback.
|
||||
*/
|
||||
export async function generateRefinementMutations(
|
||||
parent: Schema,
|
||||
weakness: string[],
|
||||
n: number,
|
||||
editFieldDescription?: (args: {
|
||||
field: SchemaField;
|
||||
feedback: string[];
|
||||
}) => Promise<string>,
|
||||
): Promise<Mutation[]> {
|
||||
const out: Mutation[] = [];
|
||||
const joined = weakness.join(' ').toLowerCase();
|
||||
|
||||
// Heuristic: if feedback mentions "too verbose" / "long" — add a maxLength to any string field.
|
||||
if (/verbose|too long|wordy/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `tighten "${f.name}" with maxLength: 200`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'maxLength', value: 200 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "too brief" / "incomplete" — add minLength.
|
||||
if (/brief|incomplete|missing|short/i.test(joined)) {
|
||||
const stringFields = parent.fields.filter(f => f.type === 'string');
|
||||
for (const f of stringFields.slice(0, 2)) {
|
||||
out.push({
|
||||
kind: 'add_constraint',
|
||||
description: `require "${f.name}" minLength: 20`,
|
||||
apply: (s) => addConstraint(s, f.name, { kind: 'minLength', value: 20 }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: if feedback mentions "wrong format" / "parse" — clarify description.
|
||||
if (/format|parse|wrong type/i.test(joined)) {
|
||||
for (const field of parent.fields.slice(0, 2)) {
|
||||
const newDesc = editFieldDescription
|
||||
? await editFieldDescription({ field, feedback: weakness }).catch(() => field.description)
|
||||
: deterministicClarifyDescription(field);
|
||||
if (newDesc && newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: always consider one description rewrite on the first field.
|
||||
if (out.length === 0 && parent.fields.length > 0) {
|
||||
const field = parent.fields[0];
|
||||
const newDesc = deterministicClarifyDescription(field);
|
||||
if (newDesc !== field.description) {
|
||||
out.push({
|
||||
kind: 'edit_field_desc',
|
||||
description: `clarify "${field.name}" description`,
|
||||
apply: (s) => editFieldDescription_pure(s, field.name, newDesc),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out.slice(0, n);
|
||||
}
|
||||
|
||||
// ── EvolveSchema engine ──────────────────────────────────────
|
||||
|
||||
export class EvolveSchema {
|
||||
async run(options: EvolveSchemaOptions): Promise<EvolveSchemaResult> {
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
const baseline: SchemaCandidate = {
|
||||
id: 'g0-baseline',
|
||||
schema: cloneSchema(config.baseline),
|
||||
generation: 0,
|
||||
parent: null,
|
||||
mutation: 'baseline',
|
||||
mutationLabel: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emitProgress(config, 'start', 0, 1, 0, 0, 'scoring baseline');
|
||||
const evalSample = pickSample(config.examples, config.evalSize, rng);
|
||||
await scoreSchemaCandidate(baseline, evalSample, config.execute, config.judge, config.signal);
|
||||
|
||||
const history: SchemaCandidate[] = [baseline];
|
||||
let survivors: SchemaCandidate[] = [baseline];
|
||||
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const topParent = pickTopByAccuracy(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
const children: SchemaCandidate[] = [];
|
||||
|
||||
// Phase A — Structure Discovery
|
||||
emitProgress(
|
||||
config, 'structure', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const structureMutations = generateStructureMutations(
|
||||
topParent.schema, config.mutationMix.structure, rng,
|
||||
);
|
||||
children.push(...applyMutations(structureMutations, topParent, gen));
|
||||
|
||||
// Phase B — Field-Order Probes
|
||||
emitProgress(
|
||||
config, 'order', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const orderMutations = generateOrderMutations(
|
||||
topParent.schema, config.mutationMix.order, rng,
|
||||
);
|
||||
children.push(...applyMutations(orderMutations, topParent, gen));
|
||||
|
||||
// Phase C — Failure-Driven Refinement
|
||||
emitProgress(
|
||||
config, 'refinement', gen, survivors.length,
|
||||
topParent.score?.accuracy ?? 0, topParent.score?.complexity ?? 0,
|
||||
);
|
||||
const refinementMutations = await generateRefinementMutations(
|
||||
topParent.schema,
|
||||
topParent.score?.weaknessFeedback ?? [],
|
||||
config.mutationMix.refinement,
|
||||
config.editFieldDescription,
|
||||
);
|
||||
children.push(...applyMutations(refinementMutations, topParent, gen));
|
||||
|
||||
// Score all children on the per-generation sample.
|
||||
const genSample = pickSample(config.examples, config.evalSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(child, genSample, config.execute, config.judge, config.signal);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// Pareto-select survivors: parent + children.
|
||||
survivors = paretoFrontSchema([topParent, ...children]);
|
||||
if (survivors.length > config.populationSize) {
|
||||
// Trim by picking the best-accuracy members.
|
||||
survivors = survivors
|
||||
.slice()
|
||||
.sort((a, b) => (b.score?.accuracy ?? 0) - (a.score?.accuracy ?? 0))
|
||||
.slice(0, config.populationSize);
|
||||
}
|
||||
}
|
||||
|
||||
// Anchor stage: larger eval on final survivors.
|
||||
emitProgress(
|
||||
config, 'anchor', config.generations, survivors.length,
|
||||
bestAccuracy(survivors), minComplexity(survivors),
|
||||
);
|
||||
const anchorSample = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreSchemaCandidate(cand, anchorSample, config.execute, config.judge, config.signal);
|
||||
}
|
||||
|
||||
const paretoFront = paretoFrontSchema(survivors);
|
||||
const winner = pickSchemaWinner(paretoFront);
|
||||
const deltaAccuracy = (winner.score?.accuracy ?? 0) - (baseline.score?.accuracy ?? 0);
|
||||
|
||||
emitProgress(
|
||||
config, 'done', config.generations, paretoFront.length,
|
||||
winner.score?.accuracy ?? 0, winner.score?.complexity ?? 0,
|
||||
);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront,
|
||||
history,
|
||||
deltaAccuracy,
|
||||
improved: deltaAccuracy > 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
function applyMutations(
|
||||
mutations: Mutation[],
|
||||
parent: SchemaCandidate,
|
||||
generation: number,
|
||||
): SchemaCandidate[] {
|
||||
return mutations.map((m, i) => ({
|
||||
id: `g${generation}-${m.kind}-${i}`,
|
||||
schema: m.apply(parent.schema),
|
||||
generation,
|
||||
parent: parent.id,
|
||||
mutation: m.kind,
|
||||
mutationLabel: m.description,
|
||||
score: null,
|
||||
perExample: [],
|
||||
}));
|
||||
}
|
||||
|
||||
function pickTopByAccuracy(candidates: SchemaCandidate[]): SchemaCandidate | undefined {
|
||||
if (candidates.length === 0) return undefined;
|
||||
return candidates.reduce((a, b) =>
|
||||
(a.score?.accuracy ?? 0) >= (b.score?.accuracy ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
export function pickSchemaWinner(candidates: SchemaCandidate[]): SchemaCandidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickSchemaWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const bA = best.score?.accuracy ?? -1;
|
||||
const cA = cand.score?.accuracy ?? -1;
|
||||
if (cA > bA) return cand;
|
||||
if (cA < bA) return best;
|
||||
// Tie on accuracy → prefer lower complexity.
|
||||
const bC = best.score?.complexity ?? Infinity;
|
||||
const cC = cand.score?.complexity ?? Infinity;
|
||||
return cC < bC ? cand : best;
|
||||
});
|
||||
}
|
||||
|
||||
function bestAccuracy(cs: SchemaCandidate[]): number {
|
||||
if (cs.length === 0) return 0;
|
||||
return Math.max(...cs.map(c => c.score?.accuracy ?? 0));
|
||||
}
|
||||
|
||||
function minComplexity(cs: SchemaCandidate[]): number {
|
||||
const scored = cs.filter(c => c.score !== null);
|
||||
if (scored.length === 0) return 0;
|
||||
return Math.min(...scored.map(c => c.score!.complexity));
|
||||
}
|
||||
|
||||
function cloneSchema(s: Schema): Schema {
|
||||
return {
|
||||
...s,
|
||||
fields: s.fields.map(f => ({ ...f, constraints: [...f.constraints] })),
|
||||
};
|
||||
}
|
||||
|
||||
function deterministicClarifyDescription(field: SchemaField): string {
|
||||
const base = field.description.trim();
|
||||
if (!base) return `Return the value for "${field.name}" as ${field.type}.`;
|
||||
if (/\b(must|should|return)\b/i.test(base)) return base;
|
||||
return `${base} Must be a ${field.type}.`;
|
||||
}
|
||||
|
||||
// Disambiguation re-export — the generator calls this internally so mutations
|
||||
// can reference it without colliding with the public `editFieldDescription`
|
||||
// option callback in `EvolveSchemaOptions`.
|
||||
const editFieldDescription_pure = editFieldDescription;
|
||||
|
||||
function pickN<T>(arr: T[], n: number, rng: () => number): T[] {
|
||||
if (arr.length <= n) return arr.slice();
|
||||
const copy = arr.slice();
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, n);
|
||||
}
|
||||
|
||||
function shuffle<T>(arr: T[], rng: () => number): T[] {
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function arraysEqual<T>(a: T[], b: T[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
function emitProgress(
|
||||
config: { onProgress?: (e: EvolveSchemaProgress) => void },
|
||||
phase: EvolveSchemaProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
bestAcc: number,
|
||||
bestCmp: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({
|
||||
phase, generation, populationSize,
|
||||
bestAccuracy: bestAcc,
|
||||
bestComplexity: bestCmp,
|
||||
message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: EvolveSchemaOptions) {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
execute: opts.execute,
|
||||
judge: opts.judge,
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
mutationMix: opts.mutationMix ?? { structure: 3, order: 1, refinement: 1 },
|
||||
evalSize: opts.evalSize ?? 32,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 100,
|
||||
seed: opts.seed ?? 1,
|
||||
editFieldDescription: opts.editFieldDescription,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,19 +1,17 @@
|
||||
/**
|
||||
* HarnessTraceBridge — translates workflow-harness phase events into
|
||||
* execution traces so harnessed agent runs feed the self-evolution loop.
|
||||
* execution traces so harnessed agent runs show up in cost tracking and
|
||||
* run history.
|
||||
*
|
||||
* Without this bridge the workflow harness is an island: phases run,
|
||||
* gates pass or fail, but nothing flows into the `execution_traces`
|
||||
* table that the evaluation dataset builder mines. This module closes
|
||||
* that loop by subscribing to the shared `harnessEvents` emitter and
|
||||
* writing one trace per phase outcome:
|
||||
* table. This module subscribes to the shared `harnessEvents` emitter and
|
||||
* writes one trace per phase outcome:
|
||||
*
|
||||
* harness:phase:complete → trace outcome 'verified'
|
||||
* harness:phase:fail (aborted === true) → trace outcome 'abandoned'
|
||||
* harness:phase:fail (will retry) → skipped — only final
|
||||
* outcomes become training
|
||||
* signal, mid-retry noise
|
||||
* stays out of the dataset
|
||||
* outcomes are recorded
|
||||
*
|
||||
* Each trace records:
|
||||
* - input: the phase's instruction text
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
/**
|
||||
* Improvement Detector — produces structured, runtime-consumable awareness signals.
|
||||
*
|
||||
* Per correction #1: Returns actionable signals + surfaced suggestions as structured data,
|
||||
* not just prompt text changes.
|
||||
*
|
||||
* Per correction #5: Awareness is operational — route better, suggest better, adapt better,
|
||||
* avoid repeated mistakes.
|
||||
*
|
||||
* Per correction #3: Workflow suggestions require recency + pattern similarity,
|
||||
* not just raw shape count.
|
||||
*/
|
||||
|
||||
import type { ImprovementSignalStore, ActionableSignal, SignalCategory } from '@waggle/core';
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
|
||||
// ── Structured output types ──────────────────────────────────
|
||||
|
||||
export interface AwarenessSummary {
|
||||
/** Capability gaps the agent has encountered repeatedly */
|
||||
capabilityGaps: CapabilityGapSignal[];
|
||||
/** Behavioral corrections the user has made repeatedly */
|
||||
corrections: CorrectionSignal[];
|
||||
/** Workflow patterns that recur and could benefit from templates */
|
||||
workflowPatterns: WorkflowPatternSignal[];
|
||||
/** Total actionable signal count (for deciding whether to inject into prompt) */
|
||||
totalActionable: number;
|
||||
}
|
||||
|
||||
export interface CapabilityGapSignal {
|
||||
id: number;
|
||||
toolName: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
export interface CorrectionSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
detail: string;
|
||||
occurrences: number;
|
||||
guidance: string;
|
||||
}
|
||||
|
||||
export interface WorkflowPatternSignal {
|
||||
id: number;
|
||||
patternKey: string;
|
||||
occurrences: number;
|
||||
suggestion: string;
|
||||
}
|
||||
|
||||
// ── Capability gap recording ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* Record a capability gap when a tool is not found or a skill is missing.
|
||||
* Called from agent-loop when tool lookup fails.
|
||||
*/
|
||||
export function recordCapabilityGap(
|
||||
store: ImprovementSignalStore,
|
||||
toolName: string,
|
||||
context?: string,
|
||||
): void {
|
||||
store.record('capability_gap', `missing:${toolName}`, context, {
|
||||
tool: toolName,
|
||||
lastContext: context,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Correction recording ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Analyze a user message for corrections and record if detected.
|
||||
* Only task-local and durable corrections are recorded (both go into the store;
|
||||
* the store's threshold mechanism handles promotion to actionable).
|
||||
*
|
||||
* Returns the detected correction (if any) for caller use.
|
||||
*/
|
||||
export function analyzeAndRecordCorrection(
|
||||
store: ImprovementSignalStore,
|
||||
userMessage: string,
|
||||
previousAssistantMessage?: string,
|
||||
): DetectedCorrection | null {
|
||||
const correction = detectCorrection(userMessage, previousAssistantMessage);
|
||||
if (!correction) return null;
|
||||
|
||||
// Record in store — both durable and task-local go in.
|
||||
// Durable corrections get recorded; task-local ones also get recorded but
|
||||
// only become actionable if the same pattern_key recurs (count >= threshold).
|
||||
store.record('correction', correction.patternKey, correction.detail, {
|
||||
isDurable: correction.isDurable,
|
||||
confidence: correction.confidence,
|
||||
});
|
||||
|
||||
return correction;
|
||||
}
|
||||
|
||||
// ── Workflow pattern recording ────────────────────────────────
|
||||
|
||||
const RECENCY_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
|
||||
|
||||
/**
|
||||
* Record a workflow pattern occurrence.
|
||||
* Per correction #3: requires recency context, not just raw shape count.
|
||||
*/
|
||||
export function recordWorkflowPattern(
|
||||
store: ImprovementSignalStore,
|
||||
taskShape: string,
|
||||
taskDescription: string,
|
||||
): void {
|
||||
store.record('workflow_pattern', `shape:${taskShape}`, taskDescription, {
|
||||
lastTask: taskDescription.slice(0, 100),
|
||||
recordedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Structured awareness builder ─────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a structured awareness summary from the signal store.
|
||||
* Returns runtime-consumable data that can be:
|
||||
* 1. Injected into system prompt as guidance
|
||||
* 2. Used by the agent loop for routing decisions
|
||||
* 3. Displayed in UI as improvement suggestions
|
||||
*
|
||||
* Per correction #6: capped at 3 actionable signals total, non-repeating.
|
||||
*/
|
||||
export function buildAwarenessSummary(store: ImprovementSignalStore): AwarenessSummary {
|
||||
const actionable = store.getActionable();
|
||||
|
||||
const capabilityGaps: CapabilityGapSignal[] = [];
|
||||
const corrections: CorrectionSignal[] = [];
|
||||
const workflowPatterns: WorkflowPatternSignal[] = [];
|
||||
|
||||
for (const signal of actionable) {
|
||||
switch (signal.category) {
|
||||
case 'capability_gap':
|
||||
capabilityGaps.push(formatCapabilityGap(signal));
|
||||
break;
|
||||
case 'correction':
|
||||
corrections.push(formatCorrection(signal));
|
||||
break;
|
||||
case 'workflow_pattern':
|
||||
// Per correction #3: only surface if recent (within 7 days)
|
||||
if (isRecent(signal.last_seen)) {
|
||||
workflowPatterns.push(formatWorkflowPattern(signal));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
capabilityGaps,
|
||||
corrections,
|
||||
workflowPatterns,
|
||||
totalActionable: capabilityGaps.length + corrections.length + workflowPatterns.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format awareness summary as a prompt section for system prompt injection.
|
||||
* Only included when there are actionable signals.
|
||||
*/
|
||||
export function formatAwarenessPrompt(summary: AwarenessSummary): string | null {
|
||||
if (summary.totalActionable === 0) return null;
|
||||
|
||||
const lines: string[] = ['## Improvement Signals'];
|
||||
|
||||
if (summary.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Missing capabilities** (user has needed these before):');
|
||||
for (const gap of summary.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Behavioral adjustments** (user has corrected these patterns):');
|
||||
for (const correction of summary.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring workflows** (consider suggesting a template):');
|
||||
for (const pattern of summary.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark all signals in a summary as surfaced.
|
||||
* Call this after the signals have been injected into a prompt or shown to the user.
|
||||
*/
|
||||
export function markSummarySurfaced(
|
||||
store: ImprovementSignalStore,
|
||||
summary: AwarenessSummary,
|
||||
): void {
|
||||
for (const gap of summary.capabilityGaps) store.markSurfaced(gap.id);
|
||||
for (const correction of summary.corrections) store.markSurfaced(correction.id);
|
||||
for (const pattern of summary.workflowPatterns) store.markSurfaced(pattern.id);
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────
|
||||
|
||||
function formatCapabilityGap(signal: ActionableSignal): CapabilityGapSignal {
|
||||
const toolName = signal.parsedMetadata.tool as string || signal.pattern_key.replace('missing:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
toolName,
|
||||
occurrences: signal.count,
|
||||
suggestion: `User has needed "${toolName}" ${signal.count} times. Consider using acquire_capability to find it.`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatCorrection(signal: ActionableSignal): CorrectionSignal {
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
detail: signal.detail,
|
||||
occurrences: signal.count,
|
||||
guidance: signal.detail
|
||||
? `${signal.detail} (corrected ${signal.count} times)`
|
||||
: `Correction pattern "${signal.pattern_key}" observed ${signal.count} times`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatWorkflowPattern(signal: ActionableSignal): WorkflowPatternSignal {
|
||||
const shape = signal.pattern_key.replace('shape:', '');
|
||||
return {
|
||||
id: signal.id,
|
||||
patternKey: signal.pattern_key,
|
||||
occurrences: signal.count,
|
||||
suggestion: `"${shape}" tasks recur frequently (${signal.count} times). A workflow template could streamline this.`,
|
||||
};
|
||||
}
|
||||
|
||||
function isRecent(dateStr: string): boolean {
|
||||
try {
|
||||
const date = new Date(dateStr);
|
||||
return Date.now() - date.getTime() < RECENCY_WINDOW_MS;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
/**
|
||||
* @internal Not exported from index.ts — future feature, tested but not wired.
|
||||
*
|
||||
* Improvement Wiring — processes each interaction for improvement signals.
|
||||
*
|
||||
* Bridges the gap between the chat route (which has raw messages) and the
|
||||
* existing correction-detector / improvement-detector / capability-acquisition
|
||||
* modules. Call processInteractionForImprovement after every agent response
|
||||
* to detect corrections, capability gaps, and recurring workflow patterns.
|
||||
*/
|
||||
|
||||
import { detectCorrection, type DetectedCorrection } from './correction-detector.js';
|
||||
import { detectTaskShape, type TaskShape } from './task-shape.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImprovementWiringParams {
|
||||
userMessage: string;
|
||||
agentResponse: string;
|
||||
toolsUsed: string[];
|
||||
workspaceId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
export interface ImprovementWiringResult {
|
||||
wasCorrection: boolean;
|
||||
correctionDetail?: string;
|
||||
capabilityGap?: string;
|
||||
workflowPattern?: string;
|
||||
}
|
||||
|
||||
// ── Well-known tool names (built-in tools the agent can use) ───────────
|
||||
|
||||
const KNOWN_TOOL_NAMES = new Set([
|
||||
'web_search', 'web_fetch', 'search_memory', 'save_memory',
|
||||
'get_identity', 'get_awareness', 'query_knowledge', 'add_task',
|
||||
'correct_knowledge', 'bash', 'read_file', 'write_file', 'edit_file',
|
||||
'search_files', 'search_content', 'git_status', 'git_diff', 'git_log',
|
||||
'git_commit', 'generate_docx', 'create_plan', 'add_plan_step',
|
||||
'execute_step', 'show_plan', 'list_skills', 'create_skill',
|
||||
'delete_skill', 'read_skill', 'search_skills', 'suggest_skill',
|
||||
'acquire_capability', 'install_capability', 'compose_workflow',
|
||||
'orchestrate_workflow', 'spawn_agent', 'list_agents', 'get_agent_result',
|
||||
]);
|
||||
|
||||
// ── Capability gap detection patterns ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Patterns in user messages that suggest the user wants a tool/capability
|
||||
* the agent doesn't have. These are request-like phrases paired with
|
||||
* tool/domain keywords.
|
||||
*/
|
||||
const GAP_REQUEST_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
|
||||
{ pattern: /\bcan you (?:send|post|publish|push) (?:to|on|via) (\w+)/i, domain: 'integration' },
|
||||
{ pattern: /\bconnect (?:to|with) (\w+)/i, domain: 'connector' },
|
||||
{ pattern: /\buse (\w+) (?:api|tool|service)/i, domain: 'integration' },
|
||||
{ pattern: /\b(?:read|open|parse|convert) (?:this |the |a )?(\w+) file/i, domain: 'file_format' },
|
||||
{ pattern: /\brun (?:this |the |a )?(\w+) (?:test|check|scan|lint)/i, domain: 'tooling' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect if the agent response itself indicates a tool was missing.
|
||||
* The agent often says things like "I don't have a tool for X" or
|
||||
* "Tool 'X' not found".
|
||||
*/
|
||||
const TOOL_NOT_FOUND_PATTERNS: RegExp[] = [
|
||||
/Tool "(.+?)" not found/,
|
||||
/I don't have (?:a |the )?(?:tool|capability|ability) (?:for|to) (.+?)(?:\.|$)/i,
|
||||
/no (?:tool|capability) available for (.+?)(?:\.|$)/i,
|
||||
];
|
||||
|
||||
// ── Workflow pattern detection ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Multi-step request indicators — the user is asking for a sequence of actions.
|
||||
*/
|
||||
const MULTI_STEP_PATTERNS: RegExp[] = [
|
||||
/\bthen\b.*\bthen\b/i, // "do X, then Y, then Z"
|
||||
/\bfirst\b.*\bthen\b/i, // "first do X, then Y"
|
||||
/\bstep\s*\d+/i, // "step 1, step 2"
|
||||
/\b(?:and then|after that|next|finally)\b/i, // sequence words
|
||||
];
|
||||
|
||||
// ── Main function ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Process an interaction for improvement signals.
|
||||
*
|
||||
* This is the main entry point called after every agent response.
|
||||
* It analyzes the user message and agent response to detect:
|
||||
* 1. Corrections (using the existing correction-detector)
|
||||
* 2. Capability gaps (tools requested but not available)
|
||||
* 3. Workflow patterns (recurring multi-step sequences)
|
||||
*
|
||||
* Returns structured signals for storage and display.
|
||||
*/
|
||||
export function processInteractionForImprovement(
|
||||
params: ImprovementWiringParams,
|
||||
): ImprovementWiringResult {
|
||||
const { userMessage, agentResponse, toolsUsed } = params;
|
||||
|
||||
const result: ImprovementWiringResult = {
|
||||
wasCorrection: false,
|
||||
};
|
||||
|
||||
// 1. Run correction detector on userMessage
|
||||
const correction = detectCorrection(userMessage);
|
||||
if (correction) {
|
||||
result.wasCorrection = true;
|
||||
result.correctionDetail = correction.detail;
|
||||
}
|
||||
|
||||
// 2. Check for capability gaps
|
||||
const gap = detectCapabilityGap(userMessage, agentResponse, toolsUsed);
|
||||
if (gap) {
|
||||
result.capabilityGap = gap;
|
||||
}
|
||||
|
||||
// 3. Check for workflow patterns (repeated multi-step sequences)
|
||||
const pattern = detectWorkflowPattern(userMessage);
|
||||
if (pattern) {
|
||||
result.workflowPattern = pattern;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Detect capability gaps from the interaction.
|
||||
* Checks:
|
||||
* - Agent response mentions a tool not found
|
||||
* - User message requests a tool/integration not in the known set
|
||||
*/
|
||||
function detectCapabilityGap(
|
||||
userMessage: string,
|
||||
agentResponse: string,
|
||||
toolsUsed: string[],
|
||||
): string | undefined {
|
||||
// Check agent response for explicit tool-not-found messages
|
||||
for (const pattern of TOOL_NOT_FOUND_PATTERNS) {
|
||||
const match = agentResponse.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Check user message for requests that imply missing capabilities
|
||||
for (const { pattern } of GAP_REQUEST_PATTERNS) {
|
||||
const match = userMessage.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const requested = match[1].toLowerCase();
|
||||
// Only flag as a gap if no tool was actually used for this domain
|
||||
if (toolsUsed.length === 0 || !toolsUsed.some(t => t.includes(requested))) {
|
||||
return requested;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect recurring workflow patterns from the user message.
|
||||
* Uses task-shape detection to identify the structural pattern,
|
||||
* and checks for multi-step indicators.
|
||||
*/
|
||||
function detectWorkflowPattern(userMessage: string): string | undefined {
|
||||
// Only flag multi-step requests as workflow patterns
|
||||
const isMultiStep = MULTI_STEP_PATTERNS.some(p => p.test(userMessage));
|
||||
if (!isMultiStep) return undefined;
|
||||
|
||||
// Use task-shape detector to identify the pattern type
|
||||
const shape = detectTaskShape(userMessage);
|
||||
if (shape.confidence >= 0.3) {
|
||||
return shape.type;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -203,7 +203,6 @@ export { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js'
|
||||
export {
|
||||
loadSystemPrompt,
|
||||
loadSystemPromptWithOverrides,
|
||||
assertOverridesReachActiveSpec,
|
||||
loadSkills,
|
||||
type ComposedSystemPrompt,
|
||||
type LoadedSkill,
|
||||
@@ -214,69 +213,11 @@ export { checkGrounding, extractClaimedSpecifics, type GroundingResult, type Cla
|
||||
export { CostTracker, DEFAULT_MODEL_PRICING, type ModelPricing, type UsageStats, type UsageEntry } from './cost-tracker.js';
|
||||
export { extractEntities, type ExtractedEntity } from './entity-extractor.js';
|
||||
export { CognifyPipeline, type CognifyConfig, type CognifyResult } from './cognify.js';
|
||||
export { AgentLearning, type LearnedBehavior, type PersonaEffectiveness, type LearningSnapshot } from './agent-learning.js';
|
||||
export {
|
||||
TraceRecorder, truncate as truncateTraceText, scrubSecrets,
|
||||
type TraceHandle, type FinalizeOptions as TraceFinalizeOptions,
|
||||
} from './trace-recorder.js';
|
||||
export {
|
||||
EvalDatasetBuilder, detectSecrets, redactSecrets, SECRET_PATTERN_NAMES,
|
||||
toJSONL as evalToJSONL, fromJSONL as evalFromJSONL,
|
||||
type EvalExample, type EvalExampleMetadata, type DatasetSplit,
|
||||
type BuildOptions as EvalBuildOptions, type JudgeVerdict,
|
||||
} from './eval-dataset.js';
|
||||
export {
|
||||
IterativeGEPA, paretoFront, scoreCandidate, aggregateScores,
|
||||
pickWinner, pickSample as pickGepaSample,
|
||||
type Candidate as GEPACandidate, type CandidateScore as GEPACandidateScore,
|
||||
type IterativeGEPAOptions, type GEPARunResult, type GEPAProgress,
|
||||
type ScoreCandidateOptions,
|
||||
type MutateArgs, type MutateFn, type MutationStrategy, type EvolutionTarget,
|
||||
} from './iterative-optimizer.js';
|
||||
export {
|
||||
runGates, DEFAULT_SIZE_LIMITS,
|
||||
checkNonEmpty, checkSize, checkGrowth,
|
||||
checkBalancedFences, checkNoPlaceholders, checkNoObviousTodos, checkRegression,
|
||||
type GateVerdict, type GateResult as EvolutionGateResult, type GateCheckResult,
|
||||
type SizeLimits, type GateOptions, type CheckInput as GateCheckInput,
|
||||
} from './evolution-gates.js';
|
||||
export {
|
||||
EvolveSchema,
|
||||
addOutputField, removeField, editFieldDescription,
|
||||
changeFieldType, addConstraint, removeConstraint,
|
||||
reorderFields, replaceOutputFields,
|
||||
schemaComplexity, aggregateSchemaScores, paretoFrontSchema,
|
||||
scoreSchemaCandidate, pickSchemaWinner,
|
||||
generateStructureMutations, generateOrderMutations, generateRefinementMutations,
|
||||
pickSample as pickSchemaSample,
|
||||
type Schema, type SchemaField, type FieldType, type FieldConstraint,
|
||||
type Mutation as SchemaMutation, type MutationKind as SchemaMutationKind,
|
||||
type SchemaCandidate, type SchemaCandidateScore, type SchemaExampleResult,
|
||||
type SchemaExecuteFn, type EvolveSchemaOptions, type EvolveSchemaResult,
|
||||
type EvolveSchemaProgress,
|
||||
} from './evolve-schema.js';
|
||||
export {
|
||||
ComposeEvolution, defaultFeedbackFilter, filterJudgeFeedback,
|
||||
stripStructuralLines, schemaExecutorFromInstructionRunner,
|
||||
type ComposeEvolutionOptions, type ComposeEvolutionResult,
|
||||
type ComposeProgress, type FeedbackFilter,
|
||||
} from './compose-evolution.js';
|
||||
export {
|
||||
EvolutionOrchestrator, eligibleForEvolution, summarizeRuns,
|
||||
type EvolutionOrchestratorDeps, type EvolutionOrchestratorOptions,
|
||||
type EvolutionAutoTriggerConfig, type OrchestratorRunResult,
|
||||
type OrchestratorOutcome, type EvolutionProgress,
|
||||
type SchemaBaselineInput,
|
||||
} from './evolution-orchestrator.js';
|
||||
export {
|
||||
deployPersonaOverride, rollbackPersonaOverride,
|
||||
deployBehavioralSpecOverride, rollbackBehavioralSpecOverride,
|
||||
loadBehavioralSpecOverrides, applyBehavioralSpecOverrides,
|
||||
BEHAVIORAL_SPEC_SECTIONS,
|
||||
type DeployResult, type DeployPersonaInput,
|
||||
type DeployBehavioralSpecInput, type BehavioralSpecOverride,
|
||||
type BehavioralSpecSection,
|
||||
} from './evolution-deploy.js';
|
||||
export { detectSecrets, redactSecrets, SECRET_PATTERN_NAMES } from './secret-patterns.js';
|
||||
export {
|
||||
LLMJudge, DEFAULT_WEIGHTS, DEFAULT_RUBRIC,
|
||||
buildPrompt as buildJudgePrompt,
|
||||
@@ -285,17 +226,12 @@ export {
|
||||
type ParsedJudgeResponse,
|
||||
} from './judge.js';
|
||||
export {
|
||||
createAnthropicEvolutionLLM,
|
||||
buildJudgeLLMCall, buildGEPAMutateFn, buildSchemaExecuteFn,
|
||||
buildReflectiveMutationPrompt, buildSchemaFillPrompt,
|
||||
makeRunningJudge, isRunningJudge, RUNNING_JUDGE_BRAND,
|
||||
createAnthropicEvolutionLLM, buildJudgeLLMCall,
|
||||
retryWithBackoff, wrapWithRetry,
|
||||
isRetryableEvolutionError, computeRetryDelay,
|
||||
DEFAULT_RETRY_OPTIONS,
|
||||
type EvolutionLLM, type CreateAnthropicEvolutionLLMOptions,
|
||||
type BuildReflectiveMutationPromptArgs,
|
||||
type RetryOptions, type RetryInfo,
|
||||
type RunningJudge,
|
||||
} from './evolution-llm-wiring.js';
|
||||
export {
|
||||
createHarnessRun, advancePhase, getCurrentPhaseInstruction,
|
||||
@@ -346,7 +282,6 @@ export { createDocumentTools } from './document-tools.js';
|
||||
export { createSpreadsheetTools } from './spreadsheet-tools.js';
|
||||
export { createPresentationTools } from './presentation-tools.js';
|
||||
export { createPdfTools } from './pdf-tools.js';
|
||||
export { createInsightsTools, type InsightsDeps } from './insights-tools.js';
|
||||
export { createConnectorSearchTools } from './connector-search.js';
|
||||
export { createCrossWorkspaceTools, type CrossWorkspaceToolDeps } from './cross-workspace-tools.js';
|
||||
export { extractEntitiesWithLLM, type LLMCallFn as EntityLLMCallFn } from './entity-extractor.js';
|
||||
@@ -480,7 +415,6 @@ export {
|
||||
ComposioConnector, MSTeamsConnector, OutlookConnector, OneDriveConnector, OneNoteConnector,
|
||||
} from './connectors/index.js';
|
||||
export { IterationBudget, type IterationBudgetConfig } from './iteration-budget.js';
|
||||
export { captureInteraction, getRecentLogs, isWithinBudget, type CaptureInteractionInput } from './optimization-capture.js';
|
||||
export { routeMessage, type RoutingDecision } from './smart-router.js';
|
||||
export {
|
||||
compressConversation, estimateTokens, needsCompression,
|
||||
@@ -511,13 +445,7 @@ export {
|
||||
type DeliveryChannel, type DeliveryPreferences, type DeliveryMessage, type DeliveryResult,
|
||||
type DeliveryConnector, type DeliveryConnectorRegistry, type InAppEmitter,
|
||||
} from './cron-delivery-router.js';
|
||||
export { detectCorrection, detectCorrectionsInHistory, type DetectedCorrection, type CorrectionDurability } from './correction-detector.js';
|
||||
export { detectContradiction, type ContradictionResult } from './contradiction-detector.js';
|
||||
export {
|
||||
recordCapabilityGap, analyzeAndRecordCorrection, recordWorkflowPattern,
|
||||
buildAwarenessSummary, formatAwarenessPrompt, markSummarySurfaced,
|
||||
type AwarenessSummary, type CapabilityGapSignal, type CorrectionSignal, type WorkflowPatternSignal,
|
||||
} from './improvement-detector.js';
|
||||
export {
|
||||
lintMemoryWrite,
|
||||
type MemoryLintResult, type MemoryLintVerdict,
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
/**
|
||||
* Insights Tools — self-analytics for the agent (Hermes-inspired).
|
||||
*
|
||||
* Provides the agent with awareness of its own performance:
|
||||
* - Tool usage frequency and success rates
|
||||
* - Cost per model
|
||||
* - Correction frequency trends
|
||||
* - Improvement signal summary
|
||||
*
|
||||
* This enables the agent to reflect on what works and adapt strategy.
|
||||
*/
|
||||
|
||||
import type { ToolDefinition } from './tools.js';
|
||||
|
||||
export interface InsightsDeps {
|
||||
getOptimizationLogs: (limit: number) => Array<{
|
||||
model?: string;
|
||||
tool_names?: string;
|
||||
total_tokens?: number;
|
||||
estimated_cost?: number;
|
||||
was_correction?: number;
|
||||
created_at?: string;
|
||||
}>;
|
||||
getImprovementSignals: () => Array<{
|
||||
category: string;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export function createInsightsTools(deps: InsightsDeps): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: 'agent_insights',
|
||||
description: [
|
||||
'Analyze your own performance metrics. Returns: tool usage frequency,',
|
||||
'cost per model, correction rate, and active improvement signals.',
|
||||
'Use this to reflect on what approaches work and adapt your strategy.',
|
||||
'Call this when you want to understand your performance patterns.',
|
||||
].join(' '),
|
||||
parameters: {
|
||||
type: 'object' as const,
|
||||
required: [],
|
||||
properties: {
|
||||
lookbackDays: {
|
||||
type: 'number' as const,
|
||||
description: 'Number of days to analyze (default 30)',
|
||||
},
|
||||
},
|
||||
},
|
||||
execute: async (args: Record<string, unknown>) => {
|
||||
const lookbackDays = (args.lookbackDays as number) ?? 30;
|
||||
const limit = lookbackDays * 50; // ~50 interactions/day estimate
|
||||
|
||||
try {
|
||||
const logs = deps.getOptimizationLogs(limit);
|
||||
const signals = deps.getImprovementSignals();
|
||||
|
||||
// Tool usage frequency
|
||||
const toolCounts: Record<string, { total: number; successes: number }> = {};
|
||||
for (const log of logs) {
|
||||
const tools = log.tool_names?.split(',').map(t => t.trim()).filter(Boolean) ?? [];
|
||||
const isCorrection = log.was_correction === 1;
|
||||
for (const tool of tools) {
|
||||
if (!toolCounts[tool]) toolCounts[tool] = { total: 0, successes: 0 };
|
||||
toolCounts[tool].total++;
|
||||
if (!isCorrection) toolCounts[tool].successes++;
|
||||
}
|
||||
}
|
||||
|
||||
const toolStats = Object.entries(toolCounts)
|
||||
.sort((a, b) => b[1].total - a[1].total)
|
||||
.slice(0, 15)
|
||||
.map(([name, stats]) => ({
|
||||
tool: name,
|
||||
uses: stats.total,
|
||||
successRate: stats.total > 0 ? Math.round((stats.successes / stats.total) * 100) : 0,
|
||||
}));
|
||||
|
||||
// Cost per model
|
||||
const modelCosts: Record<string, { calls: number; tokens: number; cost: number }> = {};
|
||||
for (const log of logs) {
|
||||
const model = log.model ?? 'unknown';
|
||||
if (!modelCosts[model]) modelCosts[model] = { calls: 0, tokens: 0, cost: 0 };
|
||||
modelCosts[model].calls++;
|
||||
modelCosts[model].tokens += log.total_tokens ?? 0;
|
||||
modelCosts[model].cost += log.estimated_cost ?? 0;
|
||||
}
|
||||
|
||||
const modelStats = Object.entries(modelCosts)
|
||||
.sort((a, b) => b[1].calls - a[1].calls)
|
||||
.map(([model, stats]) => ({
|
||||
model,
|
||||
calls: stats.calls,
|
||||
tokens: stats.tokens,
|
||||
costUsd: Math.round(stats.cost * 100) / 100,
|
||||
}));
|
||||
|
||||
// Correction rate
|
||||
const totalInteractions = logs.length;
|
||||
const corrections = logs.filter(l => l.was_correction === 1).length;
|
||||
const correctionRate = totalInteractions > 0
|
||||
? Math.round((corrections / totalInteractions) * 100)
|
||||
: 0;
|
||||
|
||||
// Active improvement signals
|
||||
const activeSignals = signals
|
||||
.filter(s => s.count >= 2)
|
||||
.slice(0, 10)
|
||||
.map(s => ({
|
||||
category: s.category,
|
||||
pattern: s.pattern_key,
|
||||
detail: s.detail,
|
||||
occurrences: s.count,
|
||||
lastSeen: s.last_seen,
|
||||
}));
|
||||
|
||||
return JSON.stringify({
|
||||
period: `Last ${lookbackDays} days`,
|
||||
totalInteractions,
|
||||
correctionRate: `${correctionRate}%`,
|
||||
topTools: toolStats,
|
||||
modelUsage: modelStats,
|
||||
improvementSignals: activeSignals,
|
||||
recommendation: correctionRate > 15
|
||||
? 'High correction rate — review recent corrections and adjust approach.'
|
||||
: correctionRate > 5
|
||||
? 'Moderate corrections — some patterns may need attention.'
|
||||
: 'Low correction rate — current approach is working well.',
|
||||
}, null, 2);
|
||||
} catch (err: unknown) {
|
||||
return `Error generating insights: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -1,626 +0,0 @@
|
||||
/**
|
||||
* Iterative GEPA Optimizer — Phase 2.1 of the self-evolution loop.
|
||||
*
|
||||
* Upgrades the one-shot prompt expander (optimizer-service) to a multi-
|
||||
* generation population search with Pareto-frontier selection and
|
||||
* reflective mutations driven by LLMJudge feedback (Phase 1.3).
|
||||
*
|
||||
* Loop:
|
||||
* 1. Start with `baseline` prompt as the seed population (1 member).
|
||||
* 2. For each generation:
|
||||
* a. Micro-screen survivors on a small sample (default 50) to prune
|
||||
* obvious losers.
|
||||
* b. Mini-eval passing candidates on a larger sample (default 64).
|
||||
* c. Spawn `populationSize` reflective mutations of the top Pareto
|
||||
* candidate, using aggregated judge feedback as ASI.
|
||||
* 3. Anchor evaluation on the full eval set (default 400) picks the winner.
|
||||
*
|
||||
* "Actionable Side Information" (ASI) is the textual feedback the judge
|
||||
* produces — fed back into the `mutate` callback so the next generation
|
||||
* targets the identified weakness rather than mutating blindly.
|
||||
*
|
||||
* Pareto selection keeps candidates that are non-dominated on a
|
||||
* 3-dimensional fitness vector (correctness, procedureFollowing,
|
||||
* conciseness). A single scalar `overall` is also tracked for tie-breaking
|
||||
* and final ranking.
|
||||
*/
|
||||
|
||||
import type { EvalExample } from './eval-dataset.js';
|
||||
import type { LLMJudge, JudgeScore } from './judge.js';
|
||||
import { isRunningJudge } from './evolution-llm-wiring.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export interface Candidate {
|
||||
/** Stable id — e.g. `g0-baseline`, `g1-m2` */
|
||||
id: string;
|
||||
/** The prompt text being evolved */
|
||||
prompt: string;
|
||||
/** Generation index (0 = baseline) */
|
||||
generation: number;
|
||||
/** Parent candidate id (null for baseline) */
|
||||
parent: string | null;
|
||||
/** Short label describing the mutation strategy */
|
||||
strategy: string;
|
||||
/** Aggregated score across the most recent eval stage, or null if not yet scored */
|
||||
score: CandidateScore | null;
|
||||
/** Raw per-example scores used for Pareto analysis */
|
||||
perExample: JudgeScore[];
|
||||
}
|
||||
|
||||
/** Aggregated, mean-reduced score used for selection. */
|
||||
export interface CandidateScore {
|
||||
overall: number;
|
||||
correctness: number;
|
||||
procedureFollowing: number;
|
||||
conciseness: number;
|
||||
/** Mean length penalty applied across the eval set */
|
||||
lengthPenalty: number;
|
||||
/** Number of examples the candidate was scored on */
|
||||
n: number;
|
||||
/** Aggregated feedback lines from the lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
}
|
||||
|
||||
export type MutationStrategy =
|
||||
| 'expand-edge-cases'
|
||||
| 'tighten-format'
|
||||
| 'add-examples'
|
||||
| 'clarify-constraints'
|
||||
| 'reduce-length'
|
||||
| 'restructure-steps'
|
||||
| 'targeted-feedback';
|
||||
|
||||
export interface MutateArgs {
|
||||
parent: Candidate;
|
||||
strategy: MutationStrategy;
|
||||
/** Textual feedback taken from the parent's lowest-scoring examples */
|
||||
weaknessFeedback: string[];
|
||||
/** Target types we are mutating (for the mutate fn to tune style) */
|
||||
targetKind: EvolutionTarget;
|
||||
/** Generation index of the child */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export type MutateFn = (args: MutateArgs) => Promise<string>;
|
||||
|
||||
/** What is being evolved — informs mutation style and constraint policies. */
|
||||
export type EvolutionTarget =
|
||||
| 'persona-system-prompt'
|
||||
| 'behavioral-spec-section'
|
||||
| 'tool-description'
|
||||
| 'skill-body'
|
||||
| 'generic';
|
||||
|
||||
export interface GEPAProgress {
|
||||
phase: 'start' | 'micro-screen' | 'mini-eval' | 'mutate' | 'anchor' | 'done';
|
||||
generation: number;
|
||||
populationSize: number;
|
||||
/** The best-so-far overall score at this point */
|
||||
best: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface IterativeGEPAOptions {
|
||||
/** The starting prompt text. */
|
||||
baseline: string;
|
||||
/** Full eval dataset (train + val combined is typical). */
|
||||
examples: EvalExample[];
|
||||
/** Scorer — LLMJudge or anything that implements `score(input): Promise<JudgeScore>`. */
|
||||
judge: Pick<LLMJudge, 'score'>;
|
||||
/** Function that produces a mutated prompt from a parent + weakness feedback. */
|
||||
mutate: MutateFn;
|
||||
/**
|
||||
* Opt-in escape hatch that disables the running-judge safety check.
|
||||
* Only set to true for unit tests with mock judges whose `score()` does
|
||||
* not need a real LLM call. Production code paths (evolving real prompts)
|
||||
* must always pass a `makeRunningJudge(...)`-wrapped judge; otherwise
|
||||
* GEPA optimizes prompt-text similarity to the expected output — a
|
||||
* meaningless gradient that produces nonsense winners.
|
||||
*/
|
||||
allowBareJudge?: boolean;
|
||||
/** What we're optimizing (tunes mutation + gate policy). Default 'generic'. */
|
||||
targetKind?: EvolutionTarget;
|
||||
/** Candidates per generation. Default 5. */
|
||||
populationSize?: number;
|
||||
/** How many generations to run. Default 3. */
|
||||
generations?: number;
|
||||
/** Micro-screen sample size. Default 50. */
|
||||
microScreenSize?: number;
|
||||
/** Mini-eval sample size. Default 64. */
|
||||
miniEvalSize?: number;
|
||||
/** Anchor evaluation sample size (applied to final Pareto set). Default 400. */
|
||||
anchorEvalSize?: number;
|
||||
/** Seed for deterministic sampling. Default 1. */
|
||||
seed?: number;
|
||||
/** Emits progress events across stages. */
|
||||
onProgress?: (event: GEPAProgress) => void;
|
||||
/** Optional abort signal */
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Max concurrent `judge.score()` calls per candidate during eval stages.
|
||||
* Default 1 (fully sequential) — preserves legacy behavior. Bumping to
|
||||
* 4–8 cuts wall time dramatically when the judge is an LLM API with
|
||||
* plenty of concurrent capacity. Does not affect determinism: all
|
||||
* scores are aggregated after every example completes.
|
||||
*/
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface GEPARunResult {
|
||||
winner: Candidate;
|
||||
/** Full Pareto-non-dominated set from the anchor stage */
|
||||
paretoFront: Candidate[];
|
||||
/** All candidates across all generations */
|
||||
history: Candidate[];
|
||||
/** True if the winner improved overall score over baseline */
|
||||
improved: boolean;
|
||||
/** Delta winner.score.overall - baseline.score.overall */
|
||||
delta: number;
|
||||
}
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
export class IterativeGEPA {
|
||||
/**
|
||||
* Run the iterative optimization loop.
|
||||
*
|
||||
* Returns the winner candidate plus the full Pareto set. Winner is the
|
||||
* Pareto-set member with the highest `overall` score (ties broken by
|
||||
* length penalty, i.e. the more concise one wins).
|
||||
*/
|
||||
async run(options: IterativeGEPAOptions): Promise<GEPARunResult> {
|
||||
// H-09 G3 · reject bare judges — without makeRunningJudge the judge
|
||||
// scores prompt-text-vs-expected-output, which is a meaningless
|
||||
// gradient. Any caller evolving real prompts must wrap their judge.
|
||||
// Tests pass `allowBareJudge: true` to bypass.
|
||||
if (!options.allowBareJudge && !isRunningJudge(options.judge)) {
|
||||
throw new Error(
|
||||
'IterativeGEPA: judge is not a running judge. Wrap with ' +
|
||||
'makeRunningJudge(baseJudge, llm) before passing to .run(), or ' +
|
||||
'pass `allowBareJudge: true` for tests with mock judges. ' +
|
||||
'See evolution-llm-wiring.ts:makeRunningJudge for rationale.',
|
||||
);
|
||||
}
|
||||
|
||||
const config = normalizeOptions(options);
|
||||
const rng = makeRng(config.seed);
|
||||
|
||||
// ── Seed population with baseline ────────────────────────
|
||||
const baseline: Candidate = {
|
||||
id: 'g0-baseline',
|
||||
prompt: config.baseline,
|
||||
generation: 0,
|
||||
parent: null,
|
||||
strategy: 'baseline',
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
|
||||
emit(config, 'start', 0, 1, 0, 'seeding baseline');
|
||||
|
||||
const history: Candidate[] = [baseline];
|
||||
let survivors: Candidate[] = [baseline];
|
||||
|
||||
// Score the baseline on the micro-screen so we have a comparison point.
|
||||
const microSample = pickSample(config.examples, config.microScreenSize, rng);
|
||||
await scoreCandidate(baseline, microSample, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
emit(config, 'micro-screen', 0, 1, baseline.score?.overall ?? 0);
|
||||
|
||||
// ── Generations ──────────────────────────────────────────
|
||||
for (let gen = 1; gen <= config.generations; gen++) {
|
||||
if (config.signal?.aborted) break;
|
||||
|
||||
// 1. Mutate top survivor(s) into N children.
|
||||
const topParent = pickTopParent(survivors);
|
||||
if (!topParent) break;
|
||||
|
||||
emit(config, 'mutate', gen, survivors.length, topParent.score?.overall ?? 0);
|
||||
|
||||
const children: Candidate[] = [];
|
||||
const strategies = cycleStrategies(config.populationSize, gen);
|
||||
for (let i = 0; i < config.populationSize; i++) {
|
||||
if (config.signal?.aborted) break;
|
||||
const strategy = strategies[i];
|
||||
let childPrompt: string;
|
||||
try {
|
||||
childPrompt = await config.mutate({
|
||||
parent: topParent,
|
||||
strategy,
|
||||
weaknessFeedback: topParent.score?.weaknessFeedback ?? [],
|
||||
targetKind: config.targetKind,
|
||||
generation: gen,
|
||||
});
|
||||
} catch {
|
||||
// Mutation failed — reuse parent so the loop doesn't stall.
|
||||
childPrompt = topParent.prompt;
|
||||
}
|
||||
|
||||
const child: Candidate = {
|
||||
id: `g${gen}-m${i}`,
|
||||
prompt: childPrompt,
|
||||
generation: gen,
|
||||
parent: topParent.id,
|
||||
strategy,
|
||||
score: null,
|
||||
perExample: [],
|
||||
};
|
||||
children.push(child);
|
||||
history.push(child);
|
||||
}
|
||||
|
||||
// 2. Micro-screen children on a small sample.
|
||||
const microSet = pickSample(config.examples, config.microScreenSize, rng);
|
||||
for (const child of children) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(child, microSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'micro-screen', gen, children.length,
|
||||
Math.max(...children.map(c => c.score?.overall ?? 0), topParent.score?.overall ?? 0),
|
||||
);
|
||||
|
||||
// 3. Keep children that at least match the parent's overall score.
|
||||
const threshold = topParent.score?.overall ?? 0;
|
||||
const microSurvivors = children.filter(c => (c.score?.overall ?? 0) >= threshold - 0.05);
|
||||
|
||||
// If everyone got filtered out, keep the single best child so we don't stall.
|
||||
if (microSurvivors.length === 0 && children.length > 0) {
|
||||
const best = children.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
microSurvivors.push(best);
|
||||
}
|
||||
|
||||
// 4. Mini-eval survivors on a larger sample.
|
||||
if (microSurvivors.length > 0) {
|
||||
const miniSet = pickSample(config.examples, config.miniEvalSize, rng);
|
||||
for (const cand of microSurvivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, miniSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
emit(
|
||||
config, 'mini-eval', gen, microSurvivors.length,
|
||||
Math.max(...microSurvivors.map(c => c.score?.overall ?? 0)),
|
||||
);
|
||||
}
|
||||
|
||||
// 5. Pareto-select survivors for next generation.
|
||||
const candidatePool = [topParent, ...microSurvivors];
|
||||
survivors = paretoFront(candidatePool);
|
||||
}
|
||||
|
||||
// ── Anchor evaluation — big sample, final ranking ───────
|
||||
emit(config, 'anchor', config.generations, survivors.length, bestOverall(survivors));
|
||||
const anchorSet = pickSample(config.examples, config.anchorEvalSize, rng);
|
||||
for (const cand of survivors) {
|
||||
if (config.signal?.aborted) break;
|
||||
await scoreCandidate(cand, anchorSet, config.judge, {
|
||||
signal: config.signal,
|
||||
concurrency: config.concurrency,
|
||||
});
|
||||
}
|
||||
|
||||
const anchorPareto = paretoFront(survivors);
|
||||
const winner = pickWinner(anchorPareto);
|
||||
const baselineOverall = baseline.score?.overall ?? 0;
|
||||
const winnerOverall = winner.score?.overall ?? 0;
|
||||
|
||||
emit(config, 'done', config.generations, anchorPareto.length, winnerOverall);
|
||||
|
||||
return {
|
||||
winner,
|
||||
paretoFront: anchorPareto,
|
||||
history,
|
||||
improved: winnerOverall > baselineOverall,
|
||||
delta: winnerOverall - baselineOverall,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers (exported for tests) ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Pareto front: keep candidates not dominated on any of the three scored
|
||||
* dimensions. A dominates B if A ≥ B on all three and strictly greater
|
||||
* on at least one.
|
||||
*/
|
||||
export function paretoFront(candidates: Candidate[]): Candidate[] {
|
||||
const scored = candidates.filter(c => c.score !== null);
|
||||
const front: Candidate[] = [];
|
||||
for (const cand of scored) {
|
||||
let dominated = false;
|
||||
for (const other of scored) {
|
||||
if (other === cand) continue;
|
||||
if (dominates(other.score!, cand.score!)) {
|
||||
dominated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dominated) front.push(cand);
|
||||
}
|
||||
// If all candidates have equal scores, return them all.
|
||||
return front.length > 0 ? front : scored;
|
||||
}
|
||||
|
||||
function dominates(a: CandidateScore, b: CandidateScore): boolean {
|
||||
const geAll =
|
||||
a.correctness >= b.correctness &&
|
||||
a.procedureFollowing >= b.procedureFollowing &&
|
||||
a.conciseness >= b.conciseness;
|
||||
const gtAny =
|
||||
a.correctness > b.correctness ||
|
||||
a.procedureFollowing > b.procedureFollowing ||
|
||||
a.conciseness > b.conciseness;
|
||||
return geAll && gtAny;
|
||||
}
|
||||
|
||||
export interface ScoreCandidateOptions {
|
||||
/** Abort signal — checked before each example is dispatched. */
|
||||
signal?: AbortSignal;
|
||||
/** Max concurrent judge.score() calls. Default 1 (sequential). */
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a candidate against a batch of examples and aggregate.
|
||||
*
|
||||
* Backwards compatible: the 4th argument may be either an `AbortSignal`
|
||||
* (legacy) or a `ScoreCandidateOptions` object. Pass
|
||||
* `{ concurrency: 4 }` to parallelize — individual examples are
|
||||
* independent so the aggregate result is identical to the sequential
|
||||
* version, only faster when the judge is an LLM API.
|
||||
*/
|
||||
export async function scoreCandidate(
|
||||
candidate: Candidate,
|
||||
examples: EvalExample[],
|
||||
judge: Pick<LLMJudge, 'score'>,
|
||||
signalOrOptions?: AbortSignal | ScoreCandidateOptions,
|
||||
): Promise<CandidateScore> {
|
||||
const options = resolveScoreOptions(signalOrOptions);
|
||||
const signal = options.signal;
|
||||
const concurrency = Math.max(1, options.concurrency ?? 1);
|
||||
|
||||
// The "candidate" in this context IS the new prompt we want to test.
|
||||
// We treat it as the model's actual response to the example's input,
|
||||
// since mutation produces a variant prompt-as-response. Callers can
|
||||
// substitute a `runCandidate` step before scoring if the prompt needs
|
||||
// to be executed first — see docs.
|
||||
const scoreOne = async (ex: EvalExample): Promise<JudgeScore | null> => {
|
||||
if (signal?.aborted) return null;
|
||||
try {
|
||||
return await judge.score({
|
||||
input: ex.input,
|
||||
expected: ex.expected_output,
|
||||
actual: candidate.prompt,
|
||||
});
|
||||
} catch {
|
||||
// Keep iterating so one bad example doesn't invalidate the whole score.
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const results = concurrency === 1
|
||||
? await runSequential(examples, scoreOne)
|
||||
: await mapWithConcurrency(examples, concurrency, scoreOne);
|
||||
|
||||
const scores = results.filter((s): s is JudgeScore => s !== null);
|
||||
candidate.perExample = scores;
|
||||
const aggregated = aggregateScores(scores);
|
||||
candidate.score = aggregated;
|
||||
return aggregated;
|
||||
}
|
||||
|
||||
function resolveScoreOptions(
|
||||
arg: AbortSignal | ScoreCandidateOptions | undefined,
|
||||
): ScoreCandidateOptions {
|
||||
if (!arg) return {};
|
||||
// Legacy: AbortSignal passed directly. Detect by presence of `aborted`
|
||||
// (a boolean getter unique to AbortSignal-like objects).
|
||||
if (typeof (arg as AbortSignal).aborted === 'boolean' && typeof (arg as AbortSignal).addEventListener === 'function') {
|
||||
return { signal: arg as AbortSignal };
|
||||
}
|
||||
return arg as ScoreCandidateOptions;
|
||||
}
|
||||
|
||||
async function runSequential<T, R>(
|
||||
items: readonly T[],
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
for (const item of items) {
|
||||
results.push(await fn(item));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded-parallelism map. Runs `fn` over `items` with at most
|
||||
* `concurrency` in-flight calls at once. Results are returned in the
|
||||
* same order as inputs regardless of completion order.
|
||||
*
|
||||
* Implementation: N worker loops share a counter. Each worker pulls the
|
||||
* next index, awaits `fn`, stores at that index, repeats. No external
|
||||
* deps, no extra state machinery.
|
||||
*/
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
const workerCount = Math.min(concurrency, items.length);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: workerCount }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
export function aggregateScores(scores: JudgeScore[]): CandidateScore {
|
||||
if (scores.length === 0) {
|
||||
return {
|
||||
overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0,
|
||||
lengthPenalty: 1, n: 0, weaknessFeedback: [],
|
||||
};
|
||||
}
|
||||
const n = scores.length;
|
||||
const sum = scores.reduce(
|
||||
(acc, s) => ({
|
||||
overall: acc.overall + s.overall,
|
||||
correctness: acc.correctness + s.correctness,
|
||||
procedureFollowing: acc.procedureFollowing + s.procedureFollowing,
|
||||
conciseness: acc.conciseness + s.conciseness,
|
||||
lengthPenalty: acc.lengthPenalty + s.lengthPenalty,
|
||||
}),
|
||||
{ overall: 0, correctness: 0, procedureFollowing: 0, conciseness: 0, lengthPenalty: 0 },
|
||||
);
|
||||
|
||||
// Collect the 3 worst-scoring examples' feedback as weakness signal.
|
||||
const worst = [...scores].sort((a, b) => a.overall - b.overall).slice(0, 3);
|
||||
const weaknessFeedback = worst
|
||||
.map(s => s.feedback)
|
||||
.filter(fb => fb && fb.length > 0);
|
||||
|
||||
return {
|
||||
overall: sum.overall / n,
|
||||
correctness: sum.correctness / n,
|
||||
procedureFollowing: sum.procedureFollowing / n,
|
||||
conciseness: sum.conciseness / n,
|
||||
lengthPenalty: sum.lengthPenalty / n,
|
||||
n,
|
||||
weaknessFeedback,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the "winner" from a Pareto set. Prefers highest overall; ties broken
|
||||
* by higher length penalty (more concise) then lowest generation (simpler).
|
||||
*/
|
||||
export function pickWinner(candidates: Candidate[]): Candidate {
|
||||
if (candidates.length === 0) {
|
||||
throw new Error('pickWinner called on empty candidate list');
|
||||
}
|
||||
return candidates.reduce((best, cand) => {
|
||||
const b = best.score?.overall ?? -1;
|
||||
const c = cand.score?.overall ?? -1;
|
||||
if (c > b) return cand;
|
||||
if (c < b) return best;
|
||||
// Tie on overall — prefer higher lengthPenalty (more concise, less penalized).
|
||||
const bLen = best.score?.lengthPenalty ?? 0;
|
||||
const cLen = cand.score?.lengthPenalty ?? 0;
|
||||
if (cLen !== bLen) return cLen > bLen ? cand : best;
|
||||
return best.generation <= cand.generation ? best : cand;
|
||||
});
|
||||
}
|
||||
|
||||
function pickTopParent(survivors: Candidate[]): Candidate | undefined {
|
||||
if (survivors.length === 0) return undefined;
|
||||
return survivors.reduce((a, b) =>
|
||||
(a.score?.overall ?? 0) >= (b.score?.overall ?? 0) ? a : b,
|
||||
);
|
||||
}
|
||||
|
||||
function bestOverall(candidates: Candidate[]): number {
|
||||
if (candidates.length === 0) return 0;
|
||||
return Math.max(...candidates.map(c => c.score?.overall ?? 0));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic sampler — picks `k` examples using a seeded RNG.
|
||||
* If k >= examples.length, returns the full list in deterministic shuffled order.
|
||||
*/
|
||||
export function pickSample(examples: EvalExample[], k: number, rng: () => number): EvalExample[] {
|
||||
if (k <= 0 || examples.length === 0) return [];
|
||||
const copy = [...examples];
|
||||
// Fisher–Yates shuffle
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j], copy[i]];
|
||||
}
|
||||
return copy.slice(0, Math.min(k, copy.length));
|
||||
}
|
||||
|
||||
/** Rotate through strategies so early generations explore the mutation space. */
|
||||
function cycleStrategies(n: number, generation: number): MutationStrategy[] {
|
||||
const all: MutationStrategy[] = [
|
||||
'expand-edge-cases',
|
||||
'tighten-format',
|
||||
'add-examples',
|
||||
'clarify-constraints',
|
||||
'reduce-length',
|
||||
'restructure-steps',
|
||||
'targeted-feedback',
|
||||
];
|
||||
// Early generations: spread across strategies.
|
||||
// Later generations: bias toward targeted-feedback once ASI is strong.
|
||||
if (generation <= 1) {
|
||||
return Array.from({ length: n }, (_, i) => all[i % all.length]);
|
||||
}
|
||||
return Array.from({ length: n }, (_, i) => {
|
||||
if (i === 0) return 'targeted-feedback';
|
||||
return all[(i + generation) % all.length];
|
||||
});
|
||||
}
|
||||
|
||||
function emit(
|
||||
config: { onProgress?: (event: GEPAProgress) => void },
|
||||
phase: GEPAProgress['phase'],
|
||||
generation: number,
|
||||
populationSize: number,
|
||||
best: number,
|
||||
message?: string,
|
||||
): void {
|
||||
if (config.onProgress) {
|
||||
config.onProgress({ phase, generation, populationSize, best, message });
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptions(opts: IterativeGEPAOptions): Required<Omit<IterativeGEPAOptions, 'signal' | 'onProgress'>> & {
|
||||
onProgress?: IterativeGEPAOptions['onProgress'];
|
||||
signal?: AbortSignal;
|
||||
} {
|
||||
return {
|
||||
baseline: opts.baseline,
|
||||
examples: opts.examples,
|
||||
judge: opts.judge,
|
||||
mutate: opts.mutate,
|
||||
targetKind: opts.targetKind ?? 'generic',
|
||||
populationSize: opts.populationSize ?? 5,
|
||||
generations: opts.generations ?? 3,
|
||||
microScreenSize: opts.microScreenSize ?? 50,
|
||||
miniEvalSize: opts.miniEvalSize ?? 64,
|
||||
anchorEvalSize: opts.anchorEvalSize ?? 400,
|
||||
seed: opts.seed ?? 1,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
concurrency: Math.max(1, opts.concurrency ?? 1),
|
||||
allowBareJudge: opts.allowBareJudge ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
// Mulberry32 — deterministic uniform PRNG
|
||||
function makeRng(seed: number): () => number {
|
||||
let s = seed >>> 0;
|
||||
return () => {
|
||||
s = (s + 0x6D2B79F5) >>> 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* LLM-as-Judge — rubric-based scorer used by the evolution loop.
|
||||
* LLM-as-Judge — rubric-based scorer used by skill audit, skill hygiene and
|
||||
* loop jobs.
|
||||
*
|
||||
* Scores a candidate response against an expected ground truth along
|
||||
* three weighted dimensions:
|
||||
@@ -10,9 +11,8 @@
|
||||
*
|
||||
* Then applies a length penalty (soft cap) multiplied into the final score.
|
||||
*
|
||||
* The textual feedback produced is the input that drives GEPA's reflective
|
||||
* mutations — it must explain *what* went wrong (or right) so the next
|
||||
* prompt variation can address it.
|
||||
* The textual feedback must explain *what* went wrong (or right) so the
|
||||
* caller can act on it.
|
||||
*
|
||||
* Model-agnostic — caller provides the llmCall function. Defaults assume
|
||||
* a Haiku-class judge via LiteLLM, but any OpenAI-compatible backend works.
|
||||
@@ -42,7 +42,7 @@ export interface JudgeScore {
|
||||
conciseness: number;
|
||||
/** Multiplier applied to the weighted score (0..1). */
|
||||
lengthPenalty: number;
|
||||
/** Free-form feedback used as reflective-mutation input for GEPA. */
|
||||
/** Free-form feedback explaining the score. */
|
||||
feedback: string;
|
||||
/** True when the judge successfully parsed a structured response. */
|
||||
parsed: boolean;
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
/**
|
||||
* Optimization Capture — captures agent interactions for GEPA/Ax prompt optimization.
|
||||
*
|
||||
* Provides functions to log successful agent interactions into the optimization_log
|
||||
* table in a workspace's .mind file, and to query those logs for background
|
||||
* optimization analysis.
|
||||
*
|
||||
* This module is a thin adapter between the agent loop post-response hook and
|
||||
* the OptimizationLogStore in @waggle/core.
|
||||
*/
|
||||
|
||||
import type { OptimizationLogStore, CreateOptimizationLogInput, OptimizationLogEntry } from '@waggle/core';
|
||||
import type { AgentResponse } from './agent-loop.js';
|
||||
|
||||
// ── Public interface ───────────────────────────────────────────────────
|
||||
|
||||
export interface CaptureInteractionInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
response: AgentResponse;
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture an agent interaction for GEPA optimization analysis.
|
||||
*
|
||||
* Call this after every successful agent loop completion. The data is
|
||||
* stored in the workspace's .mind file and read later by the background
|
||||
* prompt_optimization cron job.
|
||||
*/
|
||||
export function captureInteraction(
|
||||
store: OptimizationLogStore,
|
||||
input: CaptureInteractionInput,
|
||||
): OptimizationLogEntry {
|
||||
const logInput: CreateOptimizationLogInput = {
|
||||
sessionId: input.sessionId,
|
||||
workspaceId: input.workspaceId,
|
||||
systemPrompt: input.systemPrompt,
|
||||
toolsUsed: input.response.toolsUsed ?? [],
|
||||
turnCount: input.turnCount,
|
||||
wasCorrection: input.wasCorrection,
|
||||
inputTokens: input.response.usage?.inputTokens ?? 0,
|
||||
outputTokens: input.response.usage?.outputTokens ?? 0,
|
||||
};
|
||||
return store.insert(logInput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query recent optimization logs for background analysis.
|
||||
*
|
||||
* Returns the most recent entries from the optimization log,
|
||||
* ordered by timestamp descending.
|
||||
*/
|
||||
export function getRecentLogs(
|
||||
store: OptimizationLogStore,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getRecent(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimization logs for a specific workspace.
|
||||
*/
|
||||
export function getWorkspaceLogs(
|
||||
store: OptimizationLogStore,
|
||||
workspaceId: string,
|
||||
limit: number = 50,
|
||||
): OptimizationLogEntry[] {
|
||||
return store.getByWorkspace(workspaceId, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the daily optimization budget has been exceeded.
|
||||
*
|
||||
* Reads the optimization log entries from today and sums up their token costs.
|
||||
* Returns true if under budget, false if over.
|
||||
*
|
||||
* @param budgetCents Daily budget in cents (e.g. 100 = $1/day)
|
||||
* @param costPerInputToken Cost per input token in dollars (default: Sonnet pricing ~$3/M)
|
||||
* @param costPerOutputToken Cost per output token in dollars (default: Sonnet pricing ~$15/M)
|
||||
*/
|
||||
export function isWithinBudget(
|
||||
store: OptimizationLogStore,
|
||||
budgetCents: number,
|
||||
costPerInputToken: number = 3 / 1_000_000,
|
||||
costPerOutputToken: number = 15 / 1_000_000,
|
||||
): boolean {
|
||||
const stats = store.getStats();
|
||||
const totalCostDollars =
|
||||
stats.totalInputTokens * costPerInputToken +
|
||||
stats.totalOutputTokens * costPerOutputToken;
|
||||
const totalCostCents = totalCostDollars * 100;
|
||||
return totalCostCents <= budgetCents;
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
SessionStore,
|
||||
HybridSearch,
|
||||
KnowledgeGraph,
|
||||
ImprovementSignalStore,
|
||||
createCoreLogger,
|
||||
evaluateExternalMemoryIngress,
|
||||
type Embedder,
|
||||
@@ -29,7 +28,6 @@ import { createMindTools, type ToolDefinition } from './tools.js';
|
||||
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
|
||||
import { renderGoalAncestry } from './goal-ancestry.js';
|
||||
import type { GoalAncestry } from '@waggle/shared';
|
||||
import { buildAwarenessSummary, markSummarySurfaced, type AwarenessSummary } from './improvement-detector.js';
|
||||
import { CognifyPipeline } from './cognify.js';
|
||||
import { scanForInjection } from './injection-scanner.js';
|
||||
import { runPatternWriteBack } from './pattern-write-back.js';
|
||||
@@ -134,13 +132,9 @@ export class Orchestrator {
|
||||
private mode: 'local' | 'team';
|
||||
private version: string;
|
||||
private skills: string[];
|
||||
private improvementSignals: ImprovementSignalStore;
|
||||
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
|
||||
private goalAncestry: GoalAncestry | null = null;
|
||||
|
||||
/** M8: deferred signal marking — collected during buildSystemPrompt, committed after model call */
|
||||
private _pendingSurfacedAwareness: AwarenessSummary | null = null;
|
||||
|
||||
/** Workspace-specific layers (null when no workspace is active) */
|
||||
private workspaceLayers: WorkspaceLayers | null = null;
|
||||
/** Personal-mind cognify pipeline (#12: compaction-summary persistence). */
|
||||
@@ -174,7 +168,6 @@ export class Orchestrator {
|
||||
this.search = new HybridSearch(config.db, config.embedder);
|
||||
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
|
||||
this.knowledge = new KnowledgeGraph(config.db);
|
||||
this.improvementSignals = new ImprovementSignalStore(config.db);
|
||||
|
||||
const cognify = new CognifyPipeline({
|
||||
frames: this.frames,
|
||||
@@ -193,10 +186,6 @@ export class Orchestrator {
|
||||
search: this.search,
|
||||
knowledge: this.knowledge,
|
||||
cognify,
|
||||
// Skills 2.0 gap K: write-path contradiction detection emits a
|
||||
// correction signal through this store when a new save conflicts
|
||||
// with an existing frame.
|
||||
improvementSignals: this.improvementSignals,
|
||||
// Provide workspace accessors so tools can route to the right mind
|
||||
getWorkspaceLayers: () => this.workspaceLayers,
|
||||
});
|
||||
@@ -343,11 +332,6 @@ export class Orchestrator {
|
||||
|
||||
// ── SELF-AWARENESS (runtime context, changes every call) ──
|
||||
const awarenessSection = this.uncachedSection('self_awareness', () => {
|
||||
const awareness = buildAwarenessSummary(this.improvementSignals);
|
||||
// Defer marking until commitSurfacedSignals() fires post-model-call.
|
||||
if (awareness.totalActionable > 0) {
|
||||
this._pendingSurfacedAwareness = awareness;
|
||||
}
|
||||
const caps: AgentCapabilities = {
|
||||
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
|
||||
skills: this.skills,
|
||||
@@ -355,7 +339,6 @@ export class Orchestrator {
|
||||
memoryStats: this.getMemoryStats(),
|
||||
mode: this.mode,
|
||||
version: this.version,
|
||||
awareness: awareness.totalActionable > 0 ? awareness : undefined,
|
||||
};
|
||||
return buildSelfAwareness(caps);
|
||||
});
|
||||
@@ -457,18 +440,6 @@ export class Orchestrator {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* M8: Commit deferred signal markings after model call succeeds.
|
||||
* Call this after the LLM response is received. If the model call fails,
|
||||
* skip this call — signals stay actionable for the next turn.
|
||||
*/
|
||||
commitSurfacedSignals(): void {
|
||||
if (this._pendingSurfacedAwareness) {
|
||||
markSummarySurfaced(this.improvementSignals, this._pendingSurfacedAwareness);
|
||||
this._pendingSurfacedAwareness = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic memory recall: search for memories relevant to the user's query.
|
||||
* Searches BOTH personal and workspace minds when workspace is active.
|
||||
@@ -991,5 +962,4 @@ export class Orchestrator {
|
||||
getSessions(): SessionStore { return this.sessions; }
|
||||
getSearch(): HybridSearch { return this.search; }
|
||||
getKnowledge(): KnowledgeGraph { return this.knowledge; }
|
||||
getImprovementSignals(): ImprovementSignalStore { return this.improvementSignals; }
|
||||
}
|
||||
|
||||
@@ -990,83 +990,4 @@ You specialize in creative direction — briefs, feedback management, brand cons
|
||||
'Delivering without version tracking — every creative output needs a version note.',
|
||||
],
|
||||
},
|
||||
|
||||
// ── Internal review persona (self-evolution) ──────────────────────────
|
||||
// NOT surfaced in onboarding (ALL_ONBOARDING_PERSONAS) or the PersonaSwitcher
|
||||
// tiers (UNIVERSAL_MODE_IDS / ALL_SPECIALIST_IDS). It runs headless via the
|
||||
// IdleSessionWatcher loopback: it reads a stale session's transcript and, when
|
||||
// it finds a material, actionable gap, proposes a skill patch through
|
||||
// create_skill — which is held for human approval (never written to disk by
|
||||
// the reviewer). Its toolset is read-oriented; every write/exec/memory tool
|
||||
// that ALWAYS_AVAILABLE_TOOLS would re-add is stripped via disallowedTools
|
||||
// (that denylist wins over both the allowlist AND always-available). isReadOnly
|
||||
// is deliberately NOT set — that would strip create_skill, its one write.
|
||||
{
|
||||
id: 'session-reviewer',
|
||||
name: 'Session Reviewer',
|
||||
description: 'Adversarial self-review of a stale session — proposes skill patches for recurring gaps, default-silent',
|
||||
icon: '🔎',
|
||||
tagline: 'Reads a finished session and proposes a fix only when there is real evidence.',
|
||||
bestFor: [
|
||||
'Spotting promised-but-undelivered deliverables in a transcript',
|
||||
'Detecting a recurring capability failure a skill patch could fix',
|
||||
'Staying silent (NOTHING_TO_DO) when nothing material is found',
|
||||
],
|
||||
wontDo: 'Will not write to disk, save memory, delete skills, install capabilities, run shell commands, or invent findings. Every proposal is held for human approval.',
|
||||
systemPrompt: `## Persona: Session Reviewer
|
||||
You are an adversarially careful reviewer of ONE already-finished session. You examine the recent conversation and look for exactly two things, in this order:
|
||||
1. **Promised-but-undelivered deliverables** — the assistant said it would produce or do something and the transcript shows it never did.
|
||||
2. **Recurring capability failures fixable by a skill patch** — the same tool/skill/workflow failed more than once in a way a small reusable skill could prevent.
|
||||
|
||||
=== CRITICAL: DEFAULT SILENT ===
|
||||
If you do not find a MATERIAL, ACTIONABLE finding backed by explicit transcript evidence, your entire reply MUST be exactly:
|
||||
NOTHING_TO_DO
|
||||
Nothing else. No preamble, no apology, no summary.
|
||||
|
||||
=== CRITICAL: NEVER INVENT ===
|
||||
- Cite the transcript. Every finding must quote or reference what was actually said or done. If you cannot point to evidence, it is not a finding — output NOTHING_TO_DO.
|
||||
- You are read-only over the user's work: you may read files, search memory, and read skills, but you may NOT write files, save memory, delete skills, install capabilities, or run shell commands.
|
||||
- Your ONE write is a proposal: call create_skill to propose a new or updated skill that closes a real, recurring gap. It will NOT be written to disk — it is held for the human to approve or reject in the Approvals view. Propose sparingly and only with cited evidence.
|
||||
|
||||
### Process
|
||||
1. Read the recent conversation of the session (you have read tools and the workspace binding).
|
||||
2. Decide: is there a promised-but-undelivered deliverable, or a recurring capability failure a skill could fix?
|
||||
3. If yes and only if yes: state the finding with its transcript evidence in one short paragraph, then propose the smallest skill that would prevent it via create_skill.
|
||||
4. If no: reply exactly NOTHING_TO_DO.`,
|
||||
modelPreference: 'claude-sonnet-4-6',
|
||||
tools: [
|
||||
'read_file', 'search_files', 'search_content',
|
||||
'search_memory', 'query_knowledge', 'get_identity', 'get_awareness',
|
||||
'list_skills', 'search_skills', 'read_skill',
|
||||
'create_skill',
|
||||
],
|
||||
// Explicit denylist — ALWAYS_AVAILABLE_TOOLS (persona-tool-filter.ts) re-adds
|
||||
// save_memory / delete_skill / install_capability / acquire_capability /
|
||||
// add_task / correct_knowledge past any allowlist, so they must be named here
|
||||
// to be stripped. spawn_agent + bash + the file/git writes are defense-in-depth
|
||||
// (they are not in `tools`, but naming them keeps the boundary explicit).
|
||||
// Connector/external-send tools carry dynamic names and cannot be enumerated
|
||||
// here; the allowlist (tools[] + always-available) already excludes them since
|
||||
// none are declared above.
|
||||
disallowedTools: [
|
||||
'save_memory', 'delete_skill', 'install_capability', 'acquire_capability',
|
||||
'add_task', 'correct_knowledge',
|
||||
'spawn_agent', 'bash',
|
||||
'write_file', 'edit_file', 'generate_docx',
|
||||
'git_commit', 'git_push', 'git_merge',
|
||||
'execute_step', 'compose_workflow', 'orchestrate_workflow',
|
||||
],
|
||||
failurePatterns: [
|
||||
'Inventing a finding with no transcript evidence — must cite what was actually said or output NOTHING_TO_DO.',
|
||||
'Narrating when silent is correct — if nothing material is found, the reply is exactly NOTHING_TO_DO.',
|
||||
'Proposing an over-broad skill — the patch must target the specific recurring gap, nothing speculative.',
|
||||
],
|
||||
isReadOnly: false,
|
||||
workspaceAffinity: ['review', 'quality', 'self-evolution'],
|
||||
suggestedSkills: [],
|
||||
suggestedConnectors: [],
|
||||
suggestedMcpServers: [],
|
||||
suggestedCommands: [],
|
||||
defaultWorkflow: null,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -5,7 +5,6 @@ import { buildActiveBehavioralSpec } from './behavioral-spec.js';
|
||||
|
||||
/** The resolved behavioral-spec shape returned by `buildActiveBehavioralSpec`. */
|
||||
export type BehavioralSpec = ReturnType<typeof buildActiveBehavioralSpec>;
|
||||
import { loadBehavioralSpecOverrides } from './evolution-deploy.js';
|
||||
import { loadCustomPersonas } from './custom-personas.js';
|
||||
import type { AgentPersona } from './personas.js';
|
||||
|
||||
@@ -16,16 +15,8 @@ export interface LoadedSkill {
|
||||
|
||||
/**
|
||||
* Raw disk read of `{waggleDir}/system-prompt.md`. Returns null if missing
|
||||
* or empty.
|
||||
*
|
||||
* **Prefer `loadSystemPromptWithOverrides(waggleDir)` for runtime use.** This
|
||||
* bare reader exists for tools that legitimately need the unmerged file
|
||||
* content (editor views, migration scripts). Runtime agent code that loads
|
||||
* just this and nothing else will silently miss any deployed
|
||||
* behavioral-spec or persona evolution, since those are stored in separate
|
||||
* override files under the same `waggleDir`.
|
||||
*
|
||||
* H-09 G2 — see `loadSystemPromptWithOverrides` for the composed path.
|
||||
* or empty. Runtime code should prefer `loadSystemPromptWithOverrides`, which
|
||||
* also returns the behavioral spec and custom personas.
|
||||
*/
|
||||
export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
const filePath = path.join(waggleDir, 'system-prompt.md');
|
||||
@@ -34,22 +25,19 @@ export function loadSystemPrompt(waggleDir: string): string | null {
|
||||
return content || null;
|
||||
}
|
||||
|
||||
/** Bundle of every override-aware input the agent runtime needs. */
|
||||
/** Bundle of every prompt input the agent runtime needs. */
|
||||
export interface ComposedSystemPrompt {
|
||||
/** User's literal system-prompt.md content, or null if absent. */
|
||||
userPromptMd: string | null;
|
||||
/** The active behavioral spec — base + any deployed section overrides. */
|
||||
/** The active behavioral spec. */
|
||||
behavioralSpec: BehavioralSpec;
|
||||
/** Custom personas discovered in `waggleDir/personas/*.json`. */
|
||||
customPersonas: AgentPersona[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Override-aware composition: reads `system-prompt.md`, merges deployed
|
||||
* behavioral-spec section overrides from
|
||||
* `{waggleDir}/evolution-deploy/behavioral-spec/*.md`, and surfaces any
|
||||
* custom personas the user has added. Use this at agent startup so
|
||||
* deployed evolutions take effect at the next run.
|
||||
* Composes the runtime prompt inputs: reads `system-prompt.md`, builds the
|
||||
* behavioral spec and surfaces any custom personas the user has added.
|
||||
*
|
||||
* Callers typically pass the bundle to their orchestrator:
|
||||
*
|
||||
@@ -65,44 +53,11 @@ export interface ComposedSystemPrompt {
|
||||
*/
|
||||
export function loadSystemPromptWithOverrides(waggleDir: string): ComposedSystemPrompt {
|
||||
const userPromptMd = loadSystemPrompt(waggleDir);
|
||||
// loadBehavioralSpecOverrides is no-op on missing dir / malformed files.
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const behavioralSpec = buildActiveBehavioralSpec(overrides);
|
||||
const behavioralSpec = buildActiveBehavioralSpec();
|
||||
const customPersonas = loadCustomPersonas(waggleDir);
|
||||
return { userPromptMd, behavioralSpec, customPersonas };
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup-time self-check. Call after composing the override-aware prompt
|
||||
* bundle to verify that deployed overrides on disk actually appear in the
|
||||
* active spec. Emits a structured console warning (never throws) if a
|
||||
* deployed section is present on disk but missing from the active spec —
|
||||
* that would indicate the caller is using the bare `loadSystemPrompt`
|
||||
* instead of the composed loader.
|
||||
*/
|
||||
export function assertOverridesReachActiveSpec(
|
||||
waggleDir: string,
|
||||
activeSpec: BehavioralSpec,
|
||||
): void {
|
||||
const overrides = loadBehavioralSpecOverrides(waggleDir);
|
||||
const missing: string[] = [];
|
||||
for (const [section, overrideText] of Object.entries(overrides)) {
|
||||
const live = (activeSpec as unknown as Record<string, string | undefined>)[section];
|
||||
if (!live || !live.includes(overrideText.slice(0, 24))) {
|
||||
missing.push(section);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
|
||||
console.warn(
|
||||
`[prompt-loader] override-reach check failed — deployed sections on disk ` +
|
||||
`not present in active behavioral spec: ${missing.join(', ')}. ` +
|
||||
`This usually means the runtime is using loadSystemPrompt() instead of ` +
|
||||
`loadSystemPromptWithOverrides(). See prompt-loader.ts.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function loadSkills(waggleDir: string): LoadedSkill[] {
|
||||
const skillsDir = path.join(waggleDir, 'skills');
|
||||
if (!fs.existsSync(skillsDir)) return [];
|
||||
|
||||
57
packages/agent/src/secret-patterns.ts
Normal file
57
packages/agent/src/secret-patterns.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Secret detection and redaction shared by skill redaction, executor briefs
|
||||
* and anything else that must keep credentials out of stored or forwarded text.
|
||||
*/
|
||||
|
||||
// ── Secret patterns (curated from GitHub/OWASP secret-scanning conventions) ──
|
||||
|
||||
const SECRET_PATTERNS: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'aws-access-key', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ name: 'aws-secret-key', re: /\baws(.{0,20})?['"`][0-9a-zA-Z/+]{40}['"`]/ },
|
||||
{ name: 'github-pat', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/ },
|
||||
{ name: 'github-fine-grained', re: /\bgithub_pat_[A-Za-z0-9_]{80,}\b/ },
|
||||
{ name: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'openai-key', re: /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ name: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
||||
{ name: 'stripe-secret', re: /\bsk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'stripe-publishable', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}\b/ },
|
||||
{ name: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
||||
{ name: 'private-key-block', re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ },
|
||||
{ name: 'bearer-token', re: /\bAuthorization:\s*Bearer\s+[A-Za-z0-9_.=-]{20,}/i },
|
||||
{ name: 'basic-auth-url', re: /https?:\/\/[^/:]+:[^@/]+@/ },
|
||||
{ name: 'env-password', re: /\b(?:PASSWORD|PASSWD|SECRET|API_KEY|PRIVATE_KEY)\s*=\s*['"]?[^\s'"]{8,}['"]?/i },
|
||||
{ name: 'pgsql-url', re: /\bpostgres(?:ql)?:\/\/[^:]+:[^@]+@[^\s]+/ },
|
||||
{ name: 'generic-high-entropy', re: /\b(?:secret|token|key)['"`\s]{0,3}[:=]['"`\s]{0,3}[A-Za-z0-9+/=]{32,}\b/i },
|
||||
];
|
||||
|
||||
/** Returns the first secret pattern the text matches, or null if clean. */
|
||||
export function detectSecrets(text: string): string | null {
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
if (re.test(text)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exported for tests / tools that want the full list. */
|
||||
export const SECRET_PATTERN_NAMES = SECRET_PATTERNS.map(p => p.name);
|
||||
|
||||
/**
|
||||
* Redact every secret-pattern match in a string, replacing each with
|
||||
* `[REDACTED:<pattern-name>]`. Returns the scrubbed text + the de-duplicated
|
||||
* list of pattern names that fired. Reuses the same curated SECRET_PATTERNS as
|
||||
* detectSecrets so there is a single source of truth for "what is a secret".
|
||||
*/
|
||||
export function redactSecrets(text: string): { text: string; found: string[] } {
|
||||
let out = text;
|
||||
const found: string[] = [];
|
||||
for (const { name, re } of SECRET_PATTERNS) {
|
||||
const global = new RegExp(re.source, re.flags.replace('g', '') + 'g');
|
||||
const replaced = out.replace(global, `[REDACTED:${name}]`);
|
||||
if (replaced !== out) {
|
||||
found.push(name);
|
||||
out = replaced;
|
||||
}
|
||||
}
|
||||
return { text: out, found };
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { AwarenessSummary } from './improvement-detector.js';
|
||||
|
||||
export interface AgentCapabilities {
|
||||
tools: Array<{ name: string; description: string }>;
|
||||
skills: string[];
|
||||
@@ -7,8 +5,6 @@ export interface AgentCapabilities {
|
||||
memoryStats: { frameCount: number; sessionCount: number; entityCount: number };
|
||||
mode: 'local' | 'team';
|
||||
version: string;
|
||||
/** Structured improvement signals (optional — only present when store has actionable data) */
|
||||
awareness?: AwarenessSummary;
|
||||
}
|
||||
|
||||
export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
@@ -61,36 +57,6 @@ export function buildSelfAwareness(caps: AgentCapabilities): string {
|
||||
lines.push('- Prefer memory search over guessing when prior context exists.');
|
||||
lines.push('- When uncertain, ask rather than fabricate.');
|
||||
|
||||
// Improvement signals (per correction #1: structured, runtime-consumable)
|
||||
if (caps.awareness && caps.awareness.totalActionable > 0) {
|
||||
lines.push('');
|
||||
lines.push('## Learning from Past Sessions');
|
||||
|
||||
if (caps.awareness.capabilityGaps.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring capability gaps:**');
|
||||
for (const gap of caps.awareness.capabilityGaps) {
|
||||
lines.push(`- ${gap.suggestion}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.corrections.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**User corrections to apply:**');
|
||||
for (const correction of caps.awareness.corrections) {
|
||||
lines.push(`- ${correction.guidance}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (caps.awareness.workflowPatterns.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('**Recurring task patterns:**');
|
||||
for (const pattern of caps.awareness.workflowPatterns) {
|
||||
lines.push(`- ${pattern.suggestion}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// F28: Persona compliance — resist persona mode switches and injection via fake authority
|
||||
lines.push('');
|
||||
lines.push('## Persona Compliance');
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
* 2. Generates a SKILL.md via generateSkillMarkdown
|
||||
* 3. Writes it to the personal scope directory (scope: personal frontmatter)
|
||||
* 4. Dedups against existing skill files by filename
|
||||
* 5. Optionally emits a workflow_pattern improvement signal
|
||||
*
|
||||
* A v2 enhancement can insert an LLM prose-improvement step between
|
||||
* detectWorkflowPattern and generateSkillMarkdown without changing this
|
||||
@@ -19,7 +18,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import {
|
||||
detectWorkflowPattern,
|
||||
generateSkillMarkdown,
|
||||
@@ -36,8 +34,6 @@ export interface AutoExtractMessage {
|
||||
export interface AutoExtractDeps {
|
||||
/** Directory to write personal-scope SKILL.md files into */
|
||||
personalSkillsDir: string;
|
||||
/** Optional signal store — records workflow_pattern signals on each extraction */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Optional LLM enrichment hook. When provided, the template is passed
|
||||
* through before serialization so the SKILL.md prose can be rewritten.
|
||||
@@ -117,22 +113,5 @@ export async function autoExtractAndCreateSkill(
|
||||
const md = redactSkillContent(annotateWithScope(generateSkillMarkdown(finalTemplate))).content;
|
||||
fs.writeFileSync(filePath, md, 'utf-8');
|
||||
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`auto-extract:${stem}`,
|
||||
`Auto-extracted skill "${finalTemplate.name}" from session pattern (${finalTemplate.tools.join(', ')})`,
|
||||
{
|
||||
skillName: finalTemplate.name,
|
||||
filename: stem,
|
||||
tools: finalTemplate.tools,
|
||||
category: finalTemplate.category,
|
||||
source: 'auto_extract',
|
||||
},
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
return { filePath, template: finalTemplate };
|
||||
}
|
||||
|
||||
@@ -9,12 +9,12 @@
|
||||
* before every authored write.
|
||||
*
|
||||
* Secrets reuse the curated SECRET_PATTERNS via redactSecrets (single source of
|
||||
* truth — see eval-dataset.ts). Paths cover the user's home directory (the exact
|
||||
* truth — see secret-patterns.ts). Paths cover the user's home directory (the exact
|
||||
* running-user path plus generic per-OS forms), which both leaks the username
|
||||
* and makes a skill non-portable.
|
||||
*/
|
||||
import os from 'node:os';
|
||||
import { redactSecrets } from './eval-dataset.js';
|
||||
import { redactSecrets } from './secret-patterns.js';
|
||||
|
||||
// Generic user-home prefixes. Each matches up to (not including) the next path
|
||||
// separator, so the tail is preserved: C:\Users\me\.waggle\x -> ~\.waggle\x
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { loadSkillUsage, forgetSkillUsage } from './skill-usage.js';
|
||||
|
||||
const DEFAULT_MAX_IDLE_DAYS = 90;
|
||||
@@ -35,8 +34,6 @@ export interface RetireOptions {
|
||||
dryRun?: boolean;
|
||||
/** Injectable clock for deterministic tests. */
|
||||
now?: () => Date;
|
||||
/** Emit skill_promotion signals for observability. */
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
}
|
||||
|
||||
export interface RetireReport {
|
||||
@@ -124,17 +121,6 @@ export function retireStaleSkills(waggleHome: string, opts: RetireOptions = {}):
|
||||
report.retired.push(skillName);
|
||||
report.archived.push(destPath);
|
||||
forgetSkillUsage(waggleHome, skillName);
|
||||
|
||||
if (opts.improvementSignals) {
|
||||
try {
|
||||
opts.improvementSignals.record(
|
||||
'workflow_pattern',
|
||||
`retire:${skillName}`,
|
||||
`Retired unused skill "${skillName}" (idle > ${maxIdleDays} days). Archived to ${destPath}.`,
|
||||
{ skillName, archivedTo: destPath, maxIdleDays, lastActivityMs: activity },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
report.skipped.push({ name: skillName, reason: `move_failed: ${msg}` });
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
SKILL_SCOPE_ORDER,
|
||||
type SkillScope,
|
||||
} from './skill-frontmatter.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import { autoExtractAndCreateSkill, type AutoExtractMessage } from './skill-autoextract.js';
|
||||
import { retireStaleSkills } from './skill-retirement.js';
|
||||
|
||||
@@ -57,11 +56,9 @@ export interface SkillToolsDeps {
|
||||
/**
|
||||
* Skills 2.0 gap E: promotion infrastructure. Each callback is optional.
|
||||
* - getWorkspaceId / getTeamId resolve scope dirs dynamically at tool-call time.
|
||||
* - improvementSignals records skill_promotion signals on successful moves.
|
||||
*/
|
||||
getWorkspaceId?: () => string | null;
|
||||
getTeamId?: () => string | null;
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/**
|
||||
* Skills 2.0 gap A: session-message accessor for auto-extracting skills
|
||||
* from repeated workflow patterns. When provided, enables the
|
||||
@@ -665,15 +662,14 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
},
|
||||
|
||||
// Skills 2.0 gap E: promote_skill — move a skill up one scope rung.
|
||||
// personal → workspace. Enforces one-step-at-a-time and records a
|
||||
// skill_promotion improvement signal on success.
|
||||
// personal → workspace. Enforces one-step-at-a-time.
|
||||
{
|
||||
name: 'promote_skill',
|
||||
description:
|
||||
'Promote an installed skill one scope up (personal → workspace). '
|
||||
+ 'Team and enterprise scopes are not available in this edition. '
|
||||
+ 'The file is moved to the appropriate directory, frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history), and a skill_promotion signal is recorded.',
|
||||
+ 'The file is moved to the appropriate directory and frontmatter is rewritten '
|
||||
+ '(scope + promoted_from history).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -784,19 +780,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
// failing the promotion. An audit warning is fine.
|
||||
}
|
||||
|
||||
// 8. Record skill_promotion signal
|
||||
if (deps.improvementSignals) {
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'skill_promotion',
|
||||
`promote:${skillName}:${currentScope}->${target}`,
|
||||
`Promoted skill "${skillName}" from ${currentScope} to ${target}`,
|
||||
{ skillName, from: currentScope, to: target, workspaceId, teamId },
|
||||
);
|
||||
} catch { /* non-blocking */ }
|
||||
}
|
||||
|
||||
// 9. Let the agent reload its skill list so the new scope is picked up
|
||||
// 8. Let the agent reload its skill list so the new scope is picked up
|
||||
onSkillsChanged?.();
|
||||
|
||||
return (
|
||||
@@ -805,7 +789,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Moved from \`${found.filePath}\`\n`
|
||||
+ `- Now at \`${destPath}\`\n`
|
||||
+ `- Promotion history: ${updatedFrontmatter.promoted_from!.join(' → ')} → ${target}\n`
|
||||
+ `- Signal recorded: skill_promotion\n\n`
|
||||
+ `\n`
|
||||
+ `The skill is available immediately at the new scope. Agents in other `
|
||||
+ `sessions will pick it up on their next skill reload.`
|
||||
);
|
||||
@@ -823,8 +807,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Scan the current session for repeated workflow patterns (the same 3+ tools '
|
||||
+ 'used in the same order 2+ times) and auto-create a SKILL.md for any pattern '
|
||||
+ 'found. Creates personal-scope skills only — promote_skill moves them up '
|
||||
+ 'afterwards. Dedups by filename. Records a workflow_pattern signal on each '
|
||||
+ 'successful extraction.',
|
||||
+ 'afterwards. Dedups by filename.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
@@ -839,7 +822,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
}
|
||||
const result = await autoExtractAndCreateSkill(messages, {
|
||||
personalSkillsDir: skillsDir,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (result.filePath) {
|
||||
onSkillsChanged?.();
|
||||
@@ -850,7 +832,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
+ `- Tools: ${result.template!.tools.join(', ')}\n`
|
||||
+ `- Written to: \`${result.filePath}\`\n`
|
||||
+ `- Scope: personal (use promote_skill to move it up)\n`
|
||||
+ `- Signal: workflow_pattern recorded\n`
|
||||
);
|
||||
}
|
||||
if (result.reason === 'already_exists') {
|
||||
@@ -870,7 +851,7 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
'Archive personal-scope skills that have not been used within the idle window. '
|
||||
+ 'Moves stale SKILL.md files to ~/.waggle/skills-archive/ (recoverable — no deletion). '
|
||||
+ 'Default max_idle_days is 90. Default dry_run is true — call with dry_run: false to '
|
||||
+ 'actually move the files. Records a workflow_pattern signal per retirement.',
|
||||
+ 'actually move the files.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -890,7 +871,6 @@ Only use this after acquire_capability has identified a specific installable can
|
||||
const report = retireStaleSkills(waggleHome, {
|
||||
maxIdleDays,
|
||||
dryRun,
|
||||
improvementSignals: deps.improvementSignals,
|
||||
});
|
||||
if (!dryRun && report.retired.length > 0) {
|
||||
onSkillsChanged?.();
|
||||
|
||||
@@ -12,7 +12,6 @@ import type {
|
||||
} from '@waggle/core';
|
||||
import type { CognifyPipeline } from './cognify.js';
|
||||
import type { FeedbackHandler } from './feedback-handler.js';
|
||||
import type { ImprovementSignalStore } from '@waggle/core';
|
||||
import type { RiskLevel } from '@waggle/shared';
|
||||
import { createCoreLogger } from '@waggle/core';
|
||||
import { detectContradiction } from './contradiction-detector.js';
|
||||
@@ -96,13 +95,6 @@ export interface MindToolDeps {
|
||||
feedback?: FeedbackHandler;
|
||||
/** F16: Optional embedder for semantic dedup (cosine similarity) */
|
||||
embedder?: Embedder;
|
||||
/**
|
||||
* Skills 2.0 gap K: improvement-signal store for recording write-path
|
||||
* conflicts. When a new save contradicts an existing frame,
|
||||
* detectContradiction fires and we emit a "correction" signal so the
|
||||
* evolution / improvement loops can surface the conflict.
|
||||
*/
|
||||
improvementSignals?: ImprovementSignalStore;
|
||||
/** Dynamic accessor for workspace layers — checked at call time */
|
||||
getWorkspaceLayers?: () => WorkspaceLayers | null;
|
||||
/** W5.7: Accessor for ALL workspace search instances (for cross-workspace search) */
|
||||
@@ -528,20 +520,6 @@ export function createMindTools(deps: MindToolDeps): ToolDefinition[] {
|
||||
const contradiction = detectContradiction(content, recentRows);
|
||||
if (contradiction.isContradiction && contradiction.conflictsWith) {
|
||||
conflictFlag = ` [flag: contradicts_existing (${contradiction.conflictsWith.slice(0, 80).replace(/\s+/g, ' ').trim()}…)]`;
|
||||
// Record a correction-category signal on the improvement-signal
|
||||
// store. pattern_key is a stable hash-ish of the conflicting
|
||||
// frames so repeat contradictions merge via the ON CONFLICT upsert.
|
||||
if (deps.improvementSignals) {
|
||||
const patternKey = `write-conflict:${normalizeForDedup(content).slice(0, 64)}`;
|
||||
try {
|
||||
deps.improvementSignals.record(
|
||||
'correction',
|
||||
patternKey,
|
||||
`New save contradicts existing frame. New: "${content.slice(0, 120)}" ⟂ Existing: "${contradiction.conflictsWith.slice(0, 120)}"`,
|
||||
{ mindLabel, newContent: content.slice(0, 240), conflictsWith: contradiction.conflictsWith.slice(0, 240) },
|
||||
);
|
||||
} catch { /* non-blocking — signal persistence failure must not abort the save */ }
|
||||
}
|
||||
}
|
||||
} catch { /* non-blocking — contradiction check failure proceeds with save */ }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user