Files
waggle-os/packages/agent/src/builtin-harnesses.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

260 lines
10 KiB
TypeScript

/**
* Built-in Workflow Harnesses — 3 pre-configured harness definitions.
*
* 1. research-verify: Gather → Synthesize → Verify
* 2. code-review-fix: Understand → Review → Fix → Verify
* 3. document-draft: Context → Draft → Self-review
*/
import type { WorkflowHarness, PhaseOutput, GateResult } from './workflow-harness.js';
// ── Gate Helpers ────────────────────────────────────────────────
function hasToolCalls(output: PhaseOutput, toolNames: string[], minCount: number): GateResult {
const matching = output.toolCalls.filter(tc =>
toolNames.some(name => tc.tool.toLowerCase().includes(name.toLowerCase())),
);
return {
passed: matching.length >= minCount,
reason: matching.length >= minCount
? `Found ${matching.length} matching tool call(s) (required: ${minCount})`
: `Only ${matching.length} matching tool call(s) found, need at least ${minCount}. Expected tools: ${toolNames.join(', ')}`,
evidence: matching.map(tc => `${tc.tool}(${JSON.stringify(tc.args).slice(0, 100)})`).join(', '),
};
}
function hasMinSections(output: PhaseOutput, minSections: number): GateResult {
// Count distinct sections (## headings, numbered lists, or bullet groups separated by blank lines)
const headings = (output.content.match(/^#{1,3}\s/gm) ?? []).length;
const bulletGroups = output.content.split(/\n\n+/).filter(block =>
block.trim().startsWith('-') || block.trim().startsWith('*') || /^\d+\./.test(block.trim()),
).length;
const sections = Math.max(headings, bulletGroups);
return {
passed: sections >= minSections,
reason: sections >= minSections
? `Found ${sections} distinct sections (required: ${minSections})`
: `Only ${sections} distinct sections found, need at least ${minSections}`,
evidence: `${headings} headings, ${bulletGroups} bullet groups`,
};
}
function hasPattern(output: PhaseOutput, pattern: RegExp, description: string): GateResult {
const match = pattern.test(output.content);
return {
passed: match,
reason: match
? `Output contains ${description}`
: `Output missing ${description}`,
};
}
function hasMinLength(output: PhaseOutput, minChars: number): GateResult {
const len = output.content.length;
return {
passed: len >= minChars,
reason: len >= minChars
? `Output is ${len} chars (required: ${minChars})`
: `Output is only ${len} chars, need at least ${minChars}`,
};
}
function hasSpecificImprovement(output: PhaseOutput): GateResult {
// Check if the review identifies at least one specific improvement
const improvementPatterns = [
/should|could|consider|improve|missing|add|change|fix|update|revise|clarify|expand/i,
/issue|problem|gap|inconsisten|contradict|unclear|vague|incomplete/i,
/recommend|suggest|better|instead|alternatively/i,
];
const lines = output.content.split('\n').filter(l => l.trim().length > 10);
const improvementLines = lines.filter(line =>
improvementPatterns.some(p => p.test(line)),
);
// Reject generic "looks good" responses
const isGenericApproval = /^(looks good|no issues|all good|perfect|great|lgtm)/i.test(output.content.trim());
const passed = improvementLines.length >= 1 && !isGenericApproval;
return {
passed,
reason: passed
? `Found ${improvementLines.length} specific improvement(s)`
: isGenericApproval
? 'Review is generic approval — must identify at least 1 specific improvement'
: 'No specific improvements identified',
evidence: improvementLines.slice(0, 3).join(' | '),
};
}
// ── Harness Definitions ─────────────────────────────────────────
/** Research → Synthesize → Verify */
export const researchVerifyHarness: WorkflowHarness = {
id: 'research-verify',
name: 'Research & Verify',
triggerPatterns: [
/\b(?:research|investigate|find out|look into|analyze)\b.*\b(?:and|then)\b.*\b(?:verify|check|validate|confirm)\b/i,
/\b(?:deep dive|thorough|comprehensive)\b.*\b(?:research|analysis|investigation)\b/i,
],
phases: [
{
id: 'gather',
name: 'Gather',
instruction: 'Search memory and available sources to collect relevant information. Use search_memory, recall_memory, or web_search to gather at least 2 distinct sources of information. Focus on breadth first — collect raw data before organizing.',
gates: [{
name: 'At least 2 search/recall tool calls',
validate: async (output) => hasToolCalls(output, ['search_memory', 'recall_memory', 'web_search', 'search_entities'], 2),
}],
maxRetries: 1,
},
{
id: 'synthesize',
name: 'Synthesize',
instruction: 'Organize findings into a structured summary. Create clear sections covering different aspects of the research. Include citations or references to sources where possible.',
gates: [{
name: 'At least 3 distinct sections in output',
validate: async (output) => hasMinSections(output, 3),
}],
maxRetries: 1,
},
{
id: 'verify',
name: 'Verify',
instruction: 'Review the synthesized findings for accuracy. Check claims against sources. Identify any contradictions, unsupported claims, or gaps. Output MUST include a "VERDICT:" line with your assessment (PASS, CONDITIONAL, or FAIL) followed by reasoning.',
gates: [{
name: 'Output contains VERDICT: assessment',
validate: async (output) => hasPattern(output, /VERDICT:\s*(PASS|CONDITIONAL|FAIL)/i, 'VERDICT: assessment'),
}],
maxRetries: 1,
},
],
aggregation: 'concatenate',
};
/** Understand → Review → Fix → Verify */
export const codeReviewFixHarness: WorkflowHarness = {
id: 'code-review-fix',
name: 'Code Review & Fix',
triggerPatterns: [
/\b(?:review|audit|check)\b.*\b(?:code|implementation|changes)\b.*\b(?:fix|resolve|address)\b/i,
/\b(?:find|identify)\b.*\b(?:bugs?|issues?|problems?)\b.*\b(?:fix|resolve)\b/i,
],
phases: [
{
id: 'understand',
name: 'Understand',
instruction: 'Read the code to understand the change and its context. Use read_file to examine the relevant files. Understand what the code does before evaluating it.',
gates: [{
name: 'At least 1 read_file tool call',
validate: async (output) => hasToolCalls(output, ['read_file', 'Read'], 1),
}],
maxRetries: 1,
},
{
id: 'review',
name: 'Review',
instruction: 'Identify issues with severity ratings. Categorize each issue as Critical, Warning, or Info. Be specific about what is wrong and why.',
gates: [{
name: 'Output contains structured issue list',
validate: async (output) => hasPattern(output, /(?:Critical|Warning|Info|HIGH|MEDIUM|LOW)\b/i, 'severity-rated issues'),
}],
maxRetries: 1,
},
{
id: 'fix',
name: 'Fix',
instruction: 'Apply fixes for the identified issues. Use write_file or edit_file to make changes. Address Critical issues first, then Warnings.',
gates: [{
name: 'At least 1 write/edit tool call',
validate: async (output) => hasToolCalls(output, ['write_file', 'edit_file', 'Write', 'Edit'], 1),
}],
maxRetries: 2,
},
{
id: 'verify',
name: 'Verify',
instruction: 'Run tests or type checking to verify the fixes. Use bash to run test commands (npm test, tsc --noEmit, etc.) and confirm the fixes work.',
gates: [{
name: 'At least 1 test/typecheck command',
validate: async (output) => hasToolCalls(output, ['bash', 'Bash', 'run_command'], 1),
}],
maxRetries: 1,
},
],
aggregation: 'last',
};
/** Context → Draft → Self-review */
export const documentDraftHarness: WorkflowHarness = {
id: 'document-draft',
name: 'Document Draft',
triggerPatterns: [
/\b(?:write|draft|create|compose)\b.*\b(?:document|doc|report|spec|proposal|brief|memo)\b/i,
/\b(?:document|write up|put together)\b.*\b(?:findings|analysis|results|plan)\b/i,
],
phases: [
{
id: 'context',
name: 'Context',
instruction: 'Gather requirements and prior context from memory. Use search_memory to find relevant background information, previous decisions, and any existing work on this topic.',
gates: [{
name: 'At least 1 memory search',
validate: async (output) => hasToolCalls(output, ['search_memory', 'recall_memory', 'search_entities'], 1),
}],
maxRetries: 1,
},
{
id: 'draft',
name: 'Draft',
instruction: 'Produce the document based on gathered context. Write comprehensive content that addresses all requirements. The draft should be substantial (500+ characters) or written to a file.',
gates: [{
name: 'Substantial output or file written',
validate: async (output) => {
const hasFile = output.toolCalls.some(tc =>
['write_file', 'Write', 'generate_docx', 'generate_pptx'].some(t =>
tc.tool.toLowerCase().includes(t.toLowerCase()),
),
);
if (hasFile) return { passed: true, reason: 'Document written to file' };
return hasMinLength(output, 500);
},
}],
maxRetries: 1,
},
{
id: 'self-review',
name: 'Self-Review',
instruction: 'Re-read your draft critically. Identify specific gaps, inconsistencies, missing sections, or areas that need improvement. Do NOT just say "looks good" — find at least one concrete improvement.',
gates: [{
name: 'Identifies at least 1 specific improvement',
validate: async (output) => hasSpecificImprovement(output),
}],
maxRetries: 2,
},
],
aggregation: 'concatenate',
};
// ── Registry ────────────────────────────────────────────────────
/** All built-in harnesses. */
export const BUILTIN_HARNESSES: WorkflowHarness[] = [
researchVerifyHarness,
codeReviewFixHarness,
documentDraftHarness,
];
/** Find a harness by ID. */
export function getHarnessById(id: string): WorkflowHarness | undefined {
return BUILTIN_HARNESSES.find(h => h.id === id);
}
/** Find harnesses whose trigger patterns match the given task. */
export function matchHarness(task: string): WorkflowHarness | undefined {
return BUILTIN_HARNESSES.find(h =>
h.triggerPatterns.some(p => p.test(task)),
);
}