/** * WorkflowHarness — state-machine engine for phased workflow execution. * * Sits between workflow-composer and the agent loop. Provides deterministic * phase gates, checkpoints, validation, and retry logic. * * The harness does NOT run the agent loop itself — it provides state and * instructions that the agent loop consumes. All gate validation is * deterministic (no LLM calls inside the engine). * * Events emitted: * - harness:phase:start { harnessId, phaseId, phaseName } * - harness:phase:complete { harnessId, phaseId, gateResults } * - harness:phase:fail { harnessId, phaseId, gateResults, retryCount } * - harness:gate:pass { harnessId, phaseId, gateName } * - harness:gate:fail { harnessId, phaseId, gateName, reason } */ import { EventEmitter } from 'node:events'; // ── Types ─────────────────────────────────────────────────────── export type PhaseStatus = 'pending' | 'active' | 'validating' | 'passed' | 'failed' | 'skipped'; export interface PhaseGate { /** Human-readable name of the validation */ name: string; /** Validation function — receives the phase output, returns pass/fail + reason */ validate: (output: PhaseOutput) => Promise; } export interface GateResult { passed: boolean; reason: string; /** Evidence that the gate checked */ evidence?: string; } export interface HarnessPhase { /** Phase identifier */ id: string; /** Human-readable name */ name: string; /** What this phase does — injected into agent context */ instruction: string; /** Tools allowed in this phase (null = use persona default) */ allowedTools?: string[] | null; /** Validation gates that must ALL pass before moving to next phase */ gates: PhaseGate[]; /** Max retries if gates fail (default 1) */ maxRetries?: number; /** * Advisory: whether this phase requires human approval before proceeding. * Enforcement is the caller's responsibility — the harness engine cannot * gate approval itself because phase execution happens in the agent * loop outside `advancePhase`. Agents reading this flag should prompt * for approval (via the approvals inbox or equivalent) before running * the phase. */ requiresApproval?: boolean; /** * Advisory: suggested wall-clock timeout for the agent's work on this * phase (default 5 minutes). Not enforced by `advancePhase` — by the * time output is handed in, the work has already completed. Agents * executing phases should race their own work against this value if * they need hard timeouts. */ timeoutMs?: number; } export interface PhaseOutput { phaseId: string; /** The agent's output text for this phase */ content: string; /** Tool calls made during this phase */ toolCalls: Array<{ tool: string; args: Record; result: string }>; /** Files created or modified */ artifacts: string[]; /** Duration in ms */ durationMs: number; /** Tokens consumed */ tokens: { input: number; output: number }; } export interface HarnessCheckpoint { harnessId: string; phaseId: string; status: PhaseStatus; output?: PhaseOutput; gateResults: GateResult[]; retryCount: number; timestamp: number; } export interface WorkflowHarness { /** Unique ID for this harness definition */ id: string; /** Human-readable name */ name: string; /** When to auto-select this harness */ triggerPatterns: RegExp[]; /** Ordered phases */ phases: HarnessPhase[]; /** How to aggregate phase outputs into final result */ aggregation: 'concatenate' | 'last' | 'synthesize'; /** Feature flag that must be enabled (optional) */ featureFlag?: string; } export interface HarnessRunState { harnessId: string; /** Index of current phase */ currentPhase: number; /** Status of each phase */ phaseStatuses: Map; /** Checkpoints for each completed phase */ checkpoints: HarnessCheckpoint[]; /** Whether the harness completed successfully */ completed: boolean; /** Whether the harness was aborted */ aborted: boolean; /** Abort reason if applicable */ abortReason?: string; /** Total tokens across all phases */ totalTokens: { input: number; output: number }; /** Start timestamp */ startedAt: number; } // ── Harness Engine ────────────────────────────────────────────── export const harnessEvents = new EventEmitter(); /** Initialize a new harness run with all phases pending. */ export function createHarnessRun(harness: WorkflowHarness): HarnessRunState { const phaseStatuses = new Map(); for (const phase of harness.phases) { phaseStatuses.set(phase.id, 'pending'); } // Mark first phase as active if (harness.phases.length > 0) { phaseStatuses.set(harness.phases[0].id, 'active'); harnessEvents.emit('harness:phase:start', { harnessId: harness.id, phaseId: harness.phases[0].id, phaseName: harness.phases[0].name, }); } return { harnessId: harness.id, currentPhase: 0, phaseStatuses, checkpoints: [], completed: harness.phases.length === 0, aborted: false, totalTokens: { input: 0, output: 0 }, startedAt: Date.now(), }; } // ── Event payload types ────────────────────────────────────────── // // These shapes are what `harnessEvents.emit(...)` hands to listeners. // Exporting them lets downstream consumers (like HarnessTraceBridge) type // their subscribers properly instead of coercing `unknown`. export interface HarnessPhaseStartEvent { harnessId: string; phaseId: string; phaseName: string; } export interface HarnessPhaseCompleteEvent { harnessId: string; phaseId: string; phaseName: string; /** The phase's static instruction text — useful as trace input. */ phaseInstruction: string; output: PhaseOutput; gateResults: Array; } export interface HarnessPhaseFailEvent { harnessId: string; phaseId: string; phaseName: string; phaseInstruction: string; output: PhaseOutput; gateResults: Array; retryCount: number; aborted?: boolean; } export interface HarnessGatePassEvent { harnessId: string; phaseId: string; gateName: string; } export interface HarnessGateFailEvent { harnessId: string; phaseId: string; gateName: string; reason: string; } /** * Validate output against current phase gates, record checkpoint, * advance if all gates pass, retry or abort if they fail. */ export async function advancePhase( state: HarnessRunState, harness: WorkflowHarness, output: PhaseOutput, ): Promise { if (state.completed || state.aborted) return state; if (state.currentPhase >= harness.phases.length) return state; const phase = harness.phases[state.currentPhase]; const newState = { ...state, phaseStatuses: new Map(state.phaseStatuses) }; // Update tokens newState.totalTokens = { input: state.totalTokens.input + output.tokens.input, output: state.totalTokens.output + output.tokens.output, }; // Set phase to validating newState.phaseStatuses.set(phase.id, 'validating'); // Run all gates const gateResults: GateResult[] = []; let allPassed = true; for (const gate of phase.gates) { try { const result = await gate.validate(output); gateResults.push(result); if (result.passed) { harnessEvents.emit('harness:gate:pass', { harnessId: harness.id, phaseId: phase.id, gateName: gate.name, }); } else { allPassed = false; harnessEvents.emit('harness:gate:fail', { harnessId: harness.id, phaseId: phase.id, gateName: gate.name, reason: result.reason, }); } } catch (err) { const failResult: GateResult = { passed: false, reason: `Gate error: ${err instanceof Error ? err.message : String(err)}`, }; gateResults.push(failResult); allPassed = false; } } // Find existing checkpoint retry count const existingCheckpoints = state.checkpoints.filter(c => c.phaseId === phase.id); const retryCount = existingCheckpoints.length; // Record checkpoint const checkpoint: HarnessCheckpoint = { harnessId: harness.id, phaseId: phase.id, status: allPassed ? 'passed' : 'failed', output, gateResults, retryCount, timestamp: Date.now(), }; newState.checkpoints = [...state.checkpoints, checkpoint]; // Attach gate names so consumers don't have to zip back to phase.gates[i]. const namedGateResults: Array = gateResults.map((r, i) => ({ ...r, name: phase.gates[i]?.name ?? '(anonymous)', })); if (allPassed) { // Phase passed — advance to next newState.phaseStatuses.set(phase.id, 'passed'); const completeEvent: HarnessPhaseCompleteEvent = { harnessId: harness.id, phaseId: phase.id, phaseName: phase.name, phaseInstruction: phase.instruction, output, gateResults: namedGateResults, }; harnessEvents.emit('harness:phase:complete', completeEvent); const nextIdx = state.currentPhase + 1; if (nextIdx >= harness.phases.length) { // All phases complete newState.currentPhase = nextIdx; newState.completed = true; } else { // Advance to next phase newState.currentPhase = nextIdx; const nextPhase = harness.phases[nextIdx]; // Check if next phase should be skipped if (nextPhase.id.includes('verify') && shouldSkipVerify()) { newState.phaseStatuses.set(nextPhase.id, 'skipped'); // Try to advance past skipped phase const afterSkip = nextIdx + 1; if (afterSkip >= harness.phases.length) { newState.currentPhase = afterSkip; newState.completed = true; } else { newState.currentPhase = afterSkip; newState.phaseStatuses.set(harness.phases[afterSkip].id, 'active'); harnessEvents.emit('harness:phase:start', { harnessId: harness.id, phaseId: harness.phases[afterSkip].id, phaseName: harness.phases[afterSkip].name, }); } } else { newState.phaseStatuses.set(nextPhase.id, 'active'); harnessEvents.emit('harness:phase:start', { harnessId: harness.id, phaseId: nextPhase.id, phaseName: nextPhase.name, }); } } } else { // Phase failed const maxRetries = phase.maxRetries ?? 1; if (retryCount < maxRetries) { // Retry — keep phase active newState.phaseStatuses.set(phase.id, 'active'); const failEvent: HarnessPhaseFailEvent = { harnessId: harness.id, phaseId: phase.id, phaseName: phase.name, phaseInstruction: phase.instruction, output, gateResults: namedGateResults, retryCount: retryCount + 1, }; harnessEvents.emit('harness:phase:fail', failEvent); } else { // Max retries exceeded — abort newState.phaseStatuses.set(phase.id, 'failed'); newState.aborted = true; newState.abortReason = `Phase "${phase.name}" failed after ${retryCount + 1} attempts: ${gateResults.filter(g => !g.passed).map(g => g.reason).join('; ')}`; const abortEvent: HarnessPhaseFailEvent = { harnessId: harness.id, phaseId: phase.id, phaseName: phase.name, phaseInstruction: phase.instruction, output, gateResults: namedGateResults, retryCount: retryCount + 1, aborted: true, }; harnessEvents.emit('harness:phase:fail', abortEvent); } } return newState; } /** Get the instruction for the current active phase (null if completed/aborted). */ export function getCurrentPhaseInstruction( state: HarnessRunState, harness: WorkflowHarness, ): string | null { if (state.completed || state.aborted) return null; if (state.currentPhase >= harness.phases.length) return null; const phase = harness.phases[state.currentPhase]; const retryCount = state.checkpoints.filter(c => c.phaseId === phase.id).length; let instruction = `## Phase ${state.currentPhase + 1}/${harness.phases.length}: ${phase.name}\n\n${phase.instruction}`; if (retryCount > 0) { const lastCheckpoint = state.checkpoints.filter(c => c.phaseId === phase.id).pop(); const failedGates = lastCheckpoint?.gateResults.filter(g => !g.passed) ?? []; instruction += `\n\n**RETRY (attempt ${retryCount + 1})** — Previous attempt failed:\n`; for (const gate of failedGates) { instruction += `- ${gate.reason}\n`; } instruction += '\nPlease address these issues.'; } if (phase.gates.length > 0) { instruction += '\n\n**Completion criteria:**\n'; for (const gate of phase.gates) { instruction += `- ${gate.name}\n`; } } return instruction; } /** Check if current phase has retries remaining. */ export function canRetry(state: HarnessRunState, harness: WorkflowHarness): boolean { if (state.completed || state.aborted) return false; if (state.currentPhase >= harness.phases.length) return false; const phase = harness.phases[state.currentPhase]; const retryCount = state.checkpoints.filter(c => c.phaseId === phase.id).length; return retryCount < (phase.maxRetries ?? 1); } /** Generate a human-readable markdown summary of the harness run. */ export function getRunSummary(state: HarnessRunState, harness: WorkflowHarness): string { const lines: string[] = [ `# Workflow: ${harness.name}`, '', `**Status:** ${state.completed ? 'Completed' : state.aborted ? 'Aborted' : 'In Progress'}`, `**Duration:** ${Date.now() - state.startedAt}ms`, `**Tokens:** ${state.totalTokens.input} input / ${state.totalTokens.output} output`, '', '## Phases', '', '| # | Phase | Status | Gates | Retries |', '|---|-------|--------|-------|---------|', ]; for (let i = 0; i < harness.phases.length; i++) { const phase = harness.phases[i]; const status = state.phaseStatuses.get(phase.id) ?? 'pending'; const checkpoints = state.checkpoints.filter(c => c.phaseId === phase.id); const lastCheckpoint = checkpoints[checkpoints.length - 1]; const gatesSummary = lastCheckpoint ? `${lastCheckpoint.gateResults.filter(g => g.passed).length}/${lastCheckpoint.gateResults.length} passed` : '-'; const statusIcon = status === 'passed' ? 'PASS' : status === 'failed' ? 'FAIL' : status === 'skipped' ? 'SKIP' : status === 'active' ? 'ACTIVE' : status; lines.push(`| ${i + 1} | ${phase.name} | ${statusIcon} | ${gatesSummary} | ${checkpoints.length} |`); } if (state.abortReason) { lines.push('', `**Abort reason:** ${state.abortReason}`); } // Gate details for failed/passed phases const detailedCheckpoints = state.checkpoints.filter(c => c.gateResults.length > 0); if (detailedCheckpoints.length > 0) { lines.push('', '## Gate Details', ''); for (const cp of detailedCheckpoints) { const phase = harness.phases.find(p => p.id === cp.phaseId); lines.push(`### ${phase?.name ?? cp.phaseId} (attempt ${cp.retryCount + 1})`); for (const gate of cp.gateResults) { const icon = gate.passed ? 'PASS' : 'FAIL'; lines.push(`- **${icon}**: ${gate.reason}`); if (gate.evidence) { lines.push(` Evidence: ${gate.evidence}`); } } lines.push(''); } } return lines.join('\n'); } // ── Helpers ───────────────────────────────────────────────────── function shouldSkipVerify(): boolean { try { // Dynamic import to avoid circular dependency const env = process.env.WAGGLE_AUTO_VERIFY; return env !== 'true' && env !== '1'; } catch { return true; // Skip by default if flag can't be read } }