Files
waggle-os/packages/agent/src/evolution-llm-wiring.ts
shumeiko.it 2e1999a3f6 refactor: remove self-learning and the optimizer package (P1.4)
Self-learning is excluded from the community edition (ADR-0004); the
project owner reworks it separately. Cost tracking stays.

- agent: delete behaviour evolution (evolve-schema, iterative GEPA,
  compose-evolution, evolution gates, orchestrator and deploy),
  improvement and correction detection, AgentLearning, the eval-dataset
  builder, optimization-capture and the agent_insights tool. Secret
  detection moves from eval-dataset to secret-patterns. The LLM wiring
  keeps only the Haiku adapter and judge call used by skill audit and
  hygiene, so @ax-llm/ax is now declared by agent instead of server.
- server: delete /api/evolution/*, the evolution and optimizer services,
  GEPA expansion of the first chat message, the prompt_optimization and
  monthly_assessment jobs, the idle-session reviewer and its
  session-reviewer persona, prompt injection of past corrections and
  thumbs-down recorded as correction signals. Existing job rows of the
  removed types are ignored, not deleted.
- core: delete OptimizationLogStore; stop re-exporting the improvement
  signal and evolution run stores.
- web: delete the Evolution screen and the two job types in the
  automation editor.
- delete packages/optimizer.

Also logs UF-027 (self-learning loops read data nothing collects) and
UF-028 (the Settings prompt-shape choice is ignored by chat).
2026-10-20 00:54:58 +02:00

257 lines
10 KiB
TypeScript

/**
* LLM wiring for the skill audit and skill hygiene routes.
*
* The judge in `judge.ts` is model-agnostic: it accepts a plain callable.
* This module supplies the production adapter that turns a Haiku-class LLM
* into that callable, plus a retry wrapper for transient provider errors.
*
* Public surface:
*
* createAnthropicEvolutionLLM(apiKey, options?)
* Builds the default Haiku-backed LLM via @ax-llm/ax. Dynamic import to
* avoid startup cost when no audit runs.
*
* buildJudgeLLMCall(llm) → JudgeLLMCall (for LLMJudge)
*
* Adapters are pure wrappers over `EvolutionLLM.complete()`, a minimal
* `(prompt: string) => Promise<string>` contract. Tests supply in-memory
* mocks of that contract; production code goes through AxAI.
*/
import type { JudgeLLMCall } from './judge.js';
// ── Minimal LLM contract ────────────────────────────────────────
/**
* A minimal single-turn completion contract. All adapters depend
* on this interface rather than on @ax-llm/ax directly so they stay easy to
* mock in unit tests.
*/
export interface EvolutionLLM {
complete(prompt: string): Promise<string>;
}
// ── Retry with exponential backoff ──────────────────────────────
/**
* Tuning knobs for `retryWithBackoff` and `wrapWithRetry`. All fields
* optional — defaults match the schedule proven out in the hypothesis
* test: 5s → 15s → 45s → 135s (capped at 150s) with ±3s jitter across
* six total attempts (five retries).
*/
export interface RetryOptions {
/** Total attempts including the first. Default 6 (= 5 retries). */
maxAttempts?: number;
/** Base backoff in ms. Default 5_000. */
baseMs?: number;
/** Cap on a single backoff in ms. Default 150_000. */
capMs?: number;
/** Multiplicative growth factor per attempt. Default 3. */
factor?: number;
/** Random jitter in ms added to each backoff. Default 3_000. */
jitterMs?: number;
/** Predicate that decides whether an error is retryable. */
isRetryable?: (err: unknown) => boolean;
/** Sleep override — tests can pass `async () => {}` for instant sleeps. */
sleep?: (ms: number) => Promise<void>;
/** Observability hook fired BEFORE each retry sleep. */
onRetry?: (info: RetryInfo) => void;
/** AbortSignal — interrupts the next sleep + causes the loop to throw. */
signal?: AbortSignal;
}
export interface RetryInfo {
/** 1-based attempt number that just failed. */
attempt: number;
/** Planned sleep duration in ms before the next attempt. */
delayMs: number;
/** The error that triggered the retry. */
error: unknown;
}
/** Immutable defaults — frozen so typos at call sites fail loudly. */
export const DEFAULT_RETRY_OPTIONS: Required<Omit<RetryOptions, 'onRetry' | 'signal' | 'sleep' | 'isRetryable'>> = Object.freeze({
maxAttempts: 6,
baseMs: 5_000,
capMs: 150_000,
factor: 3,
jitterMs: 3_000,
});
const DEFAULT_SLEEP = (ms: number): Promise<void> => new Promise(r => setTimeout(r, ms));
const RETRYABLE_STATUSES: ReadonlySet<number> = new Set([408, 425, 429, 500, 502, 503, 504, 529]);
const RETRYABLE_CODES: ReadonlySet<string> = new Set([
'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN',
'EPIPE', 'ECONNABORTED', 'ESOCKETTIMEDOUT', 'UND_ERR_SOCKET',
]);
const RETRYABLE_MESSAGE_RE = /\b(rate.?limit|too.many.requests|overloaded|timeout|timed.out|socket.hang.up|network.error|connection.reset|connection.refused|fetch.failed|retry|503|502|504|529|429)\b/i;
/**
* Default retryable-error detector. Looks at (in order):
* 1. Explicit HTTP status on the error object (`status`, `statusCode`).
* 2. Well-known Node/fetch failure codes (ETIMEDOUT, ECONNRESET, ...).
* 3. Message patterns that indicate a transient condition.
*
* Conservative on unknowns — returns `false` rather than retrying blind.
*/
export function isRetryableEvolutionError(err: unknown): boolean {
if (err === null || err === undefined) return false;
const e = err as { status?: unknown; statusCode?: unknown; code?: unknown; message?: unknown };
const status = typeof e.status === 'number' ? e.status
: typeof e.statusCode === 'number' ? e.statusCode
: undefined;
if (status !== undefined && RETRYABLE_STATUSES.has(status)) return true;
if (typeof e.code === 'string' && RETRYABLE_CODES.has(e.code)) return true;
const message = typeof e.message === 'string' ? e.message : '';
if (message && RETRYABLE_MESSAGE_RE.test(message)) return true;
return false;
}
/**
* Deterministic (when `jitterMs === 0`) backoff schedule. Attempt is
* 1-based: the first retry uses `attempt = 1`, giving `baseMs`.
*/
export function computeRetryDelay(attempt: number, options: RetryOptions = {}): number {
const baseMs = options.baseMs ?? DEFAULT_RETRY_OPTIONS.baseMs;
const capMs = options.capMs ?? DEFAULT_RETRY_OPTIONS.capMs;
const factor = options.factor ?? DEFAULT_RETRY_OPTIONS.factor;
const jitterMs = options.jitterMs ?? DEFAULT_RETRY_OPTIONS.jitterMs;
const clampedAttempt = Math.max(1, attempt);
const raw = baseMs * Math.pow(factor, clampedAttempt - 1);
const base = Math.min(capMs, raw);
const jitter = jitterMs > 0 ? Math.floor(Math.random() * jitterMs) : 0;
return base + jitter;
}
/**
* Run `op` with exponential-backoff retries. The closure receives the
* 1-based attempt number so callers can do per-attempt diagnostics.
*
* - Non-retryable errors throw on first occurrence.
* - After `maxAttempts` failed retryable attempts, the final error is
* rethrown unchanged (so callers can inspect `.status` / `.message`).
* - `signal` aborts the next sleep and causes the loop to throw — the
* op itself is not cancelled mid-flight.
*/
export async function retryWithBackoff<T>(
op: (attempt: number) => Promise<T>,
options: RetryOptions = {},
): Promise<T> {
const maxAttempts = options.maxAttempts ?? DEFAULT_RETRY_OPTIONS.maxAttempts;
const isRetryable = options.isRetryable ?? isRetryableEvolutionError;
const sleep = options.sleep ?? DEFAULT_SLEEP;
let lastError: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
if (options.signal?.aborted) {
throw options.signal.reason instanceof Error
? options.signal.reason
: new Error('retryWithBackoff aborted');
}
try {
return await op(attempt);
} catch (err) {
lastError = err;
if (!isRetryable(err) || attempt === maxAttempts) throw err;
const delayMs = computeRetryDelay(attempt, options);
options.onRetry?.({ attempt, delayMs, error: err });
await sleep(delayMs);
}
}
// Unreachable: the loop either returns, throws inside the catch, or exhausts
// maxAttempts (which is handled by the `attempt === maxAttempts` throw above).
throw lastError ?? new Error('retryWithBackoff: unreachable');
}
/**
* Wrap any `EvolutionLLM` so that each `complete()` call is retried on
* transient failures. Pure composition — the inner LLM is unaware.
*/
export function wrapWithRetry(llm: EvolutionLLM, options: RetryOptions = {}): EvolutionLLM {
return {
complete(prompt: string): Promise<string> {
return retryWithBackoff(() => llm.complete(prompt), options);
},
};
}
// ── Anthropic builder ───────────────────────────────────────────
export interface CreateAnthropicEvolutionLLMOptions {
/** Optional model override. Defaults to Claude 4.5 Haiku. */
model?: string;
/**
* Retry policy applied to every `complete()` call. Omit to use the
* library defaults; pass `{ maxAttempts: 1 }` to disable retries
* entirely.
*/
retry?: RetryOptions;
}
/**
* Build the default Anthropic-backed EvolutionLLM. Dynamic-imports
* @ax-llm/ax so it is a zero-cost dependency until used.
*
* Returns null when @ax-llm/ax is unavailable — callers should treat null
* as "LLM unavailable" and surface a clear error to the user.
*/
export async function createAnthropicEvolutionLLM(
apiKey: string,
options: CreateAnthropicEvolutionLLMOptions = {},
): Promise<EvolutionLLM | null> {
if (!apiKey) return null;
try {
const mod = await import('@ax-llm/ax');
const AxAI = (mod as unknown as { AxAI: new (config: unknown) => unknown }).AxAI;
const AxAIAnthropicModel = (mod as unknown as {
AxAIAnthropicModel: Record<string, string>;
}).AxAIAnthropicModel;
if (!AxAI) return null;
const model = options.model ?? AxAIAnthropicModel?.Claude45Haiku ?? 'claude-haiku-4-5-20251001';
const ai = new AxAI({
name: 'anthropic',
apiKey,
config: { model },
}) as {
chat(req: {
chatPrompt: { role: 'user' | 'system'; content: string }[];
}): Promise<unknown>;
};
const inner: EvolutionLLM = {
async complete(prompt: string): Promise<string> {
const response = await ai.chat({
chatPrompt: [{ role: 'user', content: prompt }],
});
// chat() may return a ReadableStream in streaming mode — audits
// do not stream, so we only handle the non-stream case.
const typed = response as {
results?: readonly { content?: string }[];
};
const first = typed.results?.[0]?.content;
return typeof first === 'string' ? first : '';
},
};
// Apply retry policy — defaults are sized for background runs
// (5s → 15s → 45s → 135s → 150s across 6 attempts). Callers can disable
// by passing `retry: { maxAttempts: 1 }`.
return wrapWithRetry(inner, options.retry ?? {});
} catch {
// @ax-llm/ax missing or init failure — graceful degradation.
return null;
}
}
// ── Judge adapter ───────────────────────────────────────────────
/**
* Turn an EvolutionLLM into a JudgeLLMCall. The rubric prompt is built by
* `LLMJudge` itself; this adapter only forwards the prompt text.
*/
export function buildJudgeLLMCall(llm: EvolutionLLM): JudgeLLMCall {
return (prompt: string) => llm.complete(prompt);
}