This commit is contained in:
482
packages/agent/src/workflow-harness.ts
Normal file
482
packages/agent/src/workflow-harness.ts
Normal file
@@ -0,0 +1,482 @@
|
||||
/**
|
||||
* WorkflowHarness — state-machine engine for phased workflow execution.
|
||||
*
|
||||
* Sits between workflow-composer and the agent loop. Provides deterministic
|
||||
* phase gates, checkpoints, validation, and retry logic.
|
||||
*
|
||||
* The harness does NOT run the agent loop itself — it provides state and
|
||||
* instructions that the agent loop consumes. All gate validation is
|
||||
* deterministic (no LLM calls inside the engine).
|
||||
*
|
||||
* Events emitted:
|
||||
* - harness:phase:start { harnessId, phaseId, phaseName }
|
||||
* - harness:phase:complete { harnessId, phaseId, gateResults }
|
||||
* - harness:phase:fail { harnessId, phaseId, gateResults, retryCount }
|
||||
* - harness:gate:pass { harnessId, phaseId, gateName }
|
||||
* - harness:gate:fail { harnessId, phaseId, gateName, reason }
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────
|
||||
|
||||
export type PhaseStatus = 'pending' | 'active' | 'validating' | 'passed' | 'failed' | 'skipped';
|
||||
|
||||
export interface PhaseGate {
|
||||
/** Human-readable name of the validation */
|
||||
name: string;
|
||||
/** Validation function — receives the phase output, returns pass/fail + reason */
|
||||
validate: (output: PhaseOutput) => Promise<GateResult>;
|
||||
}
|
||||
|
||||
export interface GateResult {
|
||||
passed: boolean;
|
||||
reason: string;
|
||||
/** Evidence that the gate checked */
|
||||
evidence?: string;
|
||||
}
|
||||
|
||||
export interface HarnessPhase {
|
||||
/** Phase identifier */
|
||||
id: string;
|
||||
/** Human-readable name */
|
||||
name: string;
|
||||
/** What this phase does — injected into agent context */
|
||||
instruction: string;
|
||||
/** Tools allowed in this phase (null = use persona default) */
|
||||
allowedTools?: string[] | null;
|
||||
/** Validation gates that must ALL pass before moving to next phase */
|
||||
gates: PhaseGate[];
|
||||
/** Max retries if gates fail (default 1) */
|
||||
maxRetries?: number;
|
||||
/**
|
||||
* Advisory: whether this phase requires human approval before proceeding.
|
||||
* Enforcement is the caller's responsibility — the harness engine cannot
|
||||
* gate approval itself because phase execution happens in the agent
|
||||
* loop outside `advancePhase`. Agents reading this flag should prompt
|
||||
* for approval (via the approvals inbox or equivalent) before running
|
||||
* the phase.
|
||||
*/
|
||||
requiresApproval?: boolean;
|
||||
/**
|
||||
* Advisory: suggested wall-clock timeout for the agent's work on this
|
||||
* phase (default 5 minutes). Not enforced by `advancePhase` — by the
|
||||
* time output is handed in, the work has already completed. Agents
|
||||
* executing phases should race their own work against this value if
|
||||
* they need hard timeouts.
|
||||
*/
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface PhaseOutput {
|
||||
phaseId: string;
|
||||
/** The agent's output text for this phase */
|
||||
content: string;
|
||||
/** Tool calls made during this phase */
|
||||
toolCalls: Array<{ tool: string; args: Record<string, unknown>; result: string }>;
|
||||
/** Files created or modified */
|
||||
artifacts: string[];
|
||||
/** Duration in ms */
|
||||
durationMs: number;
|
||||
/** Tokens consumed */
|
||||
tokens: { input: number; output: number };
|
||||
}
|
||||
|
||||
export interface HarnessCheckpoint {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
status: PhaseStatus;
|
||||
output?: PhaseOutput;
|
||||
gateResults: GateResult[];
|
||||
retryCount: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export interface WorkflowHarness {
|
||||
/** Unique ID for this harness definition */
|
||||
id: string;
|
||||
/** Human-readable name */
|
||||
name: string;
|
||||
/** When to auto-select this harness */
|
||||
triggerPatterns: RegExp[];
|
||||
/** Ordered phases */
|
||||
phases: HarnessPhase[];
|
||||
/** How to aggregate phase outputs into final result */
|
||||
aggregation: 'concatenate' | 'last' | 'synthesize';
|
||||
/** Feature flag that must be enabled (optional) */
|
||||
featureFlag?: string;
|
||||
}
|
||||
|
||||
export interface HarnessRunState {
|
||||
harnessId: string;
|
||||
/** Index of current phase */
|
||||
currentPhase: number;
|
||||
/** Status of each phase */
|
||||
phaseStatuses: Map<string, PhaseStatus>;
|
||||
/** Checkpoints for each completed phase */
|
||||
checkpoints: HarnessCheckpoint[];
|
||||
/** Whether the harness completed successfully */
|
||||
completed: boolean;
|
||||
/** Whether the harness was aborted */
|
||||
aborted: boolean;
|
||||
/** Abort reason if applicable */
|
||||
abortReason?: string;
|
||||
/** Total tokens across all phases */
|
||||
totalTokens: { input: number; output: number };
|
||||
/** Start timestamp */
|
||||
startedAt: number;
|
||||
}
|
||||
|
||||
// ── Harness Engine ──────────────────────────────────────────────
|
||||
|
||||
export const harnessEvents = new EventEmitter();
|
||||
|
||||
/** Initialize a new harness run with all phases pending. */
|
||||
export function createHarnessRun(harness: WorkflowHarness): HarnessRunState {
|
||||
const phaseStatuses = new Map<string, PhaseStatus>();
|
||||
for (const phase of harness.phases) {
|
||||
phaseStatuses.set(phase.id, 'pending');
|
||||
}
|
||||
|
||||
// Mark first phase as active
|
||||
if (harness.phases.length > 0) {
|
||||
phaseStatuses.set(harness.phases[0].id, 'active');
|
||||
harnessEvents.emit('harness:phase:start', {
|
||||
harnessId: harness.id,
|
||||
phaseId: harness.phases[0].id,
|
||||
phaseName: harness.phases[0].name,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
harnessId: harness.id,
|
||||
currentPhase: 0,
|
||||
phaseStatuses,
|
||||
checkpoints: [],
|
||||
completed: harness.phases.length === 0,
|
||||
aborted: false,
|
||||
totalTokens: { input: 0, output: 0 },
|
||||
startedAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Event payload types ──────────────────────────────────────────
|
||||
//
|
||||
// These shapes are what `harnessEvents.emit(...)` hands to listeners.
|
||||
// Exporting them lets downstream consumers (like HarnessTraceBridge) type
|
||||
// their subscribers properly instead of coercing `unknown`.
|
||||
|
||||
export interface HarnessPhaseStartEvent {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
phaseName: string;
|
||||
}
|
||||
|
||||
export interface HarnessPhaseCompleteEvent {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
phaseName: string;
|
||||
/** The phase's static instruction text — useful as trace input. */
|
||||
phaseInstruction: string;
|
||||
output: PhaseOutput;
|
||||
gateResults: Array<GateResult & { name: string }>;
|
||||
}
|
||||
|
||||
export interface HarnessPhaseFailEvent {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
phaseName: string;
|
||||
phaseInstruction: string;
|
||||
output: PhaseOutput;
|
||||
gateResults: Array<GateResult & { name: string }>;
|
||||
retryCount: number;
|
||||
aborted?: boolean;
|
||||
}
|
||||
|
||||
export interface HarnessGatePassEvent {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
gateName: string;
|
||||
}
|
||||
|
||||
export interface HarnessGateFailEvent {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
gateName: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate output against current phase gates, record checkpoint,
|
||||
* advance if all gates pass, retry or abort if they fail.
|
||||
*/
|
||||
export async function advancePhase(
|
||||
state: HarnessRunState,
|
||||
harness: WorkflowHarness,
|
||||
output: PhaseOutput,
|
||||
): Promise<HarnessRunState> {
|
||||
if (state.completed || state.aborted) return state;
|
||||
if (state.currentPhase >= harness.phases.length) return state;
|
||||
|
||||
const phase = harness.phases[state.currentPhase];
|
||||
const newState = { ...state, phaseStatuses: new Map(state.phaseStatuses) };
|
||||
|
||||
// Update tokens
|
||||
newState.totalTokens = {
|
||||
input: state.totalTokens.input + output.tokens.input,
|
||||
output: state.totalTokens.output + output.tokens.output,
|
||||
};
|
||||
|
||||
// Set phase to validating
|
||||
newState.phaseStatuses.set(phase.id, 'validating');
|
||||
|
||||
// Run all gates
|
||||
const gateResults: GateResult[] = [];
|
||||
let allPassed = true;
|
||||
|
||||
for (const gate of phase.gates) {
|
||||
try {
|
||||
const result = await gate.validate(output);
|
||||
gateResults.push(result);
|
||||
|
||||
if (result.passed) {
|
||||
harnessEvents.emit('harness:gate:pass', {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
gateName: gate.name,
|
||||
});
|
||||
} else {
|
||||
allPassed = false;
|
||||
harnessEvents.emit('harness:gate:fail', {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
gateName: gate.name,
|
||||
reason: result.reason,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
const failResult: GateResult = {
|
||||
passed: false,
|
||||
reason: `Gate error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
gateResults.push(failResult);
|
||||
allPassed = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Find existing checkpoint retry count
|
||||
const existingCheckpoints = state.checkpoints.filter(c => c.phaseId === phase.id);
|
||||
const retryCount = existingCheckpoints.length;
|
||||
|
||||
// Record checkpoint
|
||||
const checkpoint: HarnessCheckpoint = {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
status: allPassed ? 'passed' : 'failed',
|
||||
output,
|
||||
gateResults,
|
||||
retryCount,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
newState.checkpoints = [...state.checkpoints, checkpoint];
|
||||
|
||||
// Attach gate names so consumers don't have to zip back to phase.gates[i].
|
||||
const namedGateResults: Array<GateResult & { name: string }> = gateResults.map((r, i) => ({
|
||||
...r,
|
||||
name: phase.gates[i]?.name ?? '(anonymous)',
|
||||
}));
|
||||
|
||||
if (allPassed) {
|
||||
// Phase passed — advance to next
|
||||
newState.phaseStatuses.set(phase.id, 'passed');
|
||||
const completeEvent: HarnessPhaseCompleteEvent = {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
phaseName: phase.name,
|
||||
phaseInstruction: phase.instruction,
|
||||
output,
|
||||
gateResults: namedGateResults,
|
||||
};
|
||||
harnessEvents.emit('harness:phase:complete', completeEvent);
|
||||
|
||||
const nextIdx = state.currentPhase + 1;
|
||||
if (nextIdx >= harness.phases.length) {
|
||||
// All phases complete
|
||||
newState.currentPhase = nextIdx;
|
||||
newState.completed = true;
|
||||
} else {
|
||||
// Advance to next phase
|
||||
newState.currentPhase = nextIdx;
|
||||
const nextPhase = harness.phases[nextIdx];
|
||||
|
||||
// Check if next phase should be skipped
|
||||
if (nextPhase.id.includes('verify') && shouldSkipVerify()) {
|
||||
newState.phaseStatuses.set(nextPhase.id, 'skipped');
|
||||
// Try to advance past skipped phase
|
||||
const afterSkip = nextIdx + 1;
|
||||
if (afterSkip >= harness.phases.length) {
|
||||
newState.currentPhase = afterSkip;
|
||||
newState.completed = true;
|
||||
} else {
|
||||
newState.currentPhase = afterSkip;
|
||||
newState.phaseStatuses.set(harness.phases[afterSkip].id, 'active');
|
||||
harnessEvents.emit('harness:phase:start', {
|
||||
harnessId: harness.id,
|
||||
phaseId: harness.phases[afterSkip].id,
|
||||
phaseName: harness.phases[afterSkip].name,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
newState.phaseStatuses.set(nextPhase.id, 'active');
|
||||
harnessEvents.emit('harness:phase:start', {
|
||||
harnessId: harness.id,
|
||||
phaseId: nextPhase.id,
|
||||
phaseName: nextPhase.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Phase failed
|
||||
const maxRetries = phase.maxRetries ?? 1;
|
||||
if (retryCount < maxRetries) {
|
||||
// Retry — keep phase active
|
||||
newState.phaseStatuses.set(phase.id, 'active');
|
||||
const failEvent: HarnessPhaseFailEvent = {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
phaseName: phase.name,
|
||||
phaseInstruction: phase.instruction,
|
||||
output,
|
||||
gateResults: namedGateResults,
|
||||
retryCount: retryCount + 1,
|
||||
};
|
||||
harnessEvents.emit('harness:phase:fail', failEvent);
|
||||
} else {
|
||||
// Max retries exceeded — abort
|
||||
newState.phaseStatuses.set(phase.id, 'failed');
|
||||
newState.aborted = true;
|
||||
newState.abortReason = `Phase "${phase.name}" failed after ${retryCount + 1} attempts: ${gateResults.filter(g => !g.passed).map(g => g.reason).join('; ')}`;
|
||||
const abortEvent: HarnessPhaseFailEvent = {
|
||||
harnessId: harness.id,
|
||||
phaseId: phase.id,
|
||||
phaseName: phase.name,
|
||||
phaseInstruction: phase.instruction,
|
||||
output,
|
||||
gateResults: namedGateResults,
|
||||
retryCount: retryCount + 1,
|
||||
aborted: true,
|
||||
};
|
||||
harnessEvents.emit('harness:phase:fail', abortEvent);
|
||||
}
|
||||
}
|
||||
|
||||
return newState;
|
||||
}
|
||||
|
||||
/** Get the instruction for the current active phase (null if completed/aborted). */
|
||||
export function getCurrentPhaseInstruction(
|
||||
state: HarnessRunState,
|
||||
harness: WorkflowHarness,
|
||||
): string | null {
|
||||
if (state.completed || state.aborted) return null;
|
||||
if (state.currentPhase >= harness.phases.length) return null;
|
||||
|
||||
const phase = harness.phases[state.currentPhase];
|
||||
const retryCount = state.checkpoints.filter(c => c.phaseId === phase.id).length;
|
||||
|
||||
let instruction = `## Phase ${state.currentPhase + 1}/${harness.phases.length}: ${phase.name}\n\n${phase.instruction}`;
|
||||
|
||||
if (retryCount > 0) {
|
||||
const lastCheckpoint = state.checkpoints.filter(c => c.phaseId === phase.id).pop();
|
||||
const failedGates = lastCheckpoint?.gateResults.filter(g => !g.passed) ?? [];
|
||||
instruction += `\n\n**RETRY (attempt ${retryCount + 1})** — Previous attempt failed:\n`;
|
||||
for (const gate of failedGates) {
|
||||
instruction += `- ${gate.reason}\n`;
|
||||
}
|
||||
instruction += '\nPlease address these issues.';
|
||||
}
|
||||
|
||||
if (phase.gates.length > 0) {
|
||||
instruction += '\n\n**Completion criteria:**\n';
|
||||
for (const gate of phase.gates) {
|
||||
instruction += `- ${gate.name}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return instruction;
|
||||
}
|
||||
|
||||
/** Check if current phase has retries remaining. */
|
||||
export function canRetry(state: HarnessRunState, harness: WorkflowHarness): boolean {
|
||||
if (state.completed || state.aborted) return false;
|
||||
if (state.currentPhase >= harness.phases.length) return false;
|
||||
|
||||
const phase = harness.phases[state.currentPhase];
|
||||
const retryCount = state.checkpoints.filter(c => c.phaseId === phase.id).length;
|
||||
return retryCount < (phase.maxRetries ?? 1);
|
||||
}
|
||||
|
||||
/** Generate a human-readable markdown summary of the harness run. */
|
||||
export function getRunSummary(state: HarnessRunState, harness: WorkflowHarness): string {
|
||||
const lines: string[] = [
|
||||
`# Workflow: ${harness.name}`,
|
||||
'',
|
||||
`**Status:** ${state.completed ? 'Completed' : state.aborted ? 'Aborted' : 'In Progress'}`,
|
||||
`**Duration:** ${Date.now() - state.startedAt}ms`,
|
||||
`**Tokens:** ${state.totalTokens.input} input / ${state.totalTokens.output} output`,
|
||||
'',
|
||||
'## Phases',
|
||||
'',
|
||||
'| # | Phase | Status | Gates | Retries |',
|
||||
'|---|-------|--------|-------|---------|',
|
||||
];
|
||||
|
||||
for (let i = 0; i < harness.phases.length; i++) {
|
||||
const phase = harness.phases[i];
|
||||
const status = state.phaseStatuses.get(phase.id) ?? 'pending';
|
||||
const checkpoints = state.checkpoints.filter(c => c.phaseId === phase.id);
|
||||
const lastCheckpoint = checkpoints[checkpoints.length - 1];
|
||||
const gatesSummary = lastCheckpoint
|
||||
? `${lastCheckpoint.gateResults.filter(g => g.passed).length}/${lastCheckpoint.gateResults.length} passed`
|
||||
: '-';
|
||||
const statusIcon = status === 'passed' ? 'PASS' : status === 'failed' ? 'FAIL' : status === 'skipped' ? 'SKIP' : status === 'active' ? 'ACTIVE' : status;
|
||||
|
||||
lines.push(`| ${i + 1} | ${phase.name} | ${statusIcon} | ${gatesSummary} | ${checkpoints.length} |`);
|
||||
}
|
||||
|
||||
if (state.abortReason) {
|
||||
lines.push('', `**Abort reason:** ${state.abortReason}`);
|
||||
}
|
||||
|
||||
// Gate details for failed/passed phases
|
||||
const detailedCheckpoints = state.checkpoints.filter(c => c.gateResults.length > 0);
|
||||
if (detailedCheckpoints.length > 0) {
|
||||
lines.push('', '## Gate Details', '');
|
||||
for (const cp of detailedCheckpoints) {
|
||||
const phase = harness.phases.find(p => p.id === cp.phaseId);
|
||||
lines.push(`### ${phase?.name ?? cp.phaseId} (attempt ${cp.retryCount + 1})`);
|
||||
for (const gate of cp.gateResults) {
|
||||
const icon = gate.passed ? 'PASS' : 'FAIL';
|
||||
lines.push(`- **${icon}**: ${gate.reason}`);
|
||||
if (gate.evidence) {
|
||||
lines.push(` Evidence: ${gate.evidence}`);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function shouldSkipVerify(): boolean {
|
||||
try {
|
||||
// Dynamic import to avoid circular dependency
|
||||
const env = process.env.WAGGLE_AUTO_VERIFY;
|
||||
return env !== 'true' && env !== '1';
|
||||
} catch {
|
||||
return true; // Skip by default if flag can't be read
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user