260 lines
10 KiB
TypeScript
260 lines
10 KiB
TypeScript
/**
|
|
* Built-in Workflow Harnesses — 3 pre-configured harness definitions.
|
|
*
|
|
* 1. research-verify: Gather → Synthesize → Verify
|
|
* 2. code-review-fix: Understand → Review → Fix → Verify
|
|
* 3. document-draft: Context → Draft → Self-review
|
|
*/
|
|
|
|
import type { WorkflowHarness, PhaseOutput, GateResult } from './workflow-harness.js';
|
|
|
|
// ── Gate Helpers ────────────────────────────────────────────────
|
|
|
|
function hasToolCalls(output: PhaseOutput, toolNames: string[], minCount: number): GateResult {
|
|
const matching = output.toolCalls.filter(tc =>
|
|
toolNames.some(name => tc.tool.toLowerCase().includes(name.toLowerCase())),
|
|
);
|
|
return {
|
|
passed: matching.length >= minCount,
|
|
reason: matching.length >= minCount
|
|
? `Found ${matching.length} matching tool call(s) (required: ${minCount})`
|
|
: `Only ${matching.length} matching tool call(s) found, need at least ${minCount}. Expected tools: ${toolNames.join(', ')}`,
|
|
evidence: matching.map(tc => `${tc.tool}(${JSON.stringify(tc.args).slice(0, 100)})`).join(', '),
|
|
};
|
|
}
|
|
|
|
function hasMinSections(output: PhaseOutput, minSections: number): GateResult {
|
|
// Count distinct sections (## headings, numbered lists, or bullet groups separated by blank lines)
|
|
const headings = (output.content.match(/^#{1,3}\s/gm) ?? []).length;
|
|
const bulletGroups = output.content.split(/\n\n+/).filter(block =>
|
|
block.trim().startsWith('-') || block.trim().startsWith('*') || /^\d+\./.test(block.trim()),
|
|
).length;
|
|
const sections = Math.max(headings, bulletGroups);
|
|
|
|
return {
|
|
passed: sections >= minSections,
|
|
reason: sections >= minSections
|
|
? `Found ${sections} distinct sections (required: ${minSections})`
|
|
: `Only ${sections} distinct sections found, need at least ${minSections}`,
|
|
evidence: `${headings} headings, ${bulletGroups} bullet groups`,
|
|
};
|
|
}
|
|
|
|
function hasPattern(output: PhaseOutput, pattern: RegExp, description: string): GateResult {
|
|
const match = pattern.test(output.content);
|
|
return {
|
|
passed: match,
|
|
reason: match
|
|
? `Output contains ${description}`
|
|
: `Output missing ${description}`,
|
|
};
|
|
}
|
|
|
|
function hasMinLength(output: PhaseOutput, minChars: number): GateResult {
|
|
const len = output.content.length;
|
|
return {
|
|
passed: len >= minChars,
|
|
reason: len >= minChars
|
|
? `Output is ${len} chars (required: ${minChars})`
|
|
: `Output is only ${len} chars, need at least ${minChars}`,
|
|
};
|
|
}
|
|
|
|
function hasSpecificImprovement(output: PhaseOutput): GateResult {
|
|
// Check if the review identifies at least one specific improvement
|
|
const improvementPatterns = [
|
|
/should|could|consider|improve|missing|add|change|fix|update|revise|clarify|expand/i,
|
|
/issue|problem|gap|inconsisten|contradict|unclear|vague|incomplete/i,
|
|
/recommend|suggest|better|instead|alternatively/i,
|
|
];
|
|
|
|
const lines = output.content.split('\n').filter(l => l.trim().length > 10);
|
|
const improvementLines = lines.filter(line =>
|
|
improvementPatterns.some(p => p.test(line)),
|
|
);
|
|
|
|
// Reject generic "looks good" responses
|
|
const isGenericApproval = /^(looks good|no issues|all good|perfect|great|lgtm)/i.test(output.content.trim());
|
|
|
|
const passed = improvementLines.length >= 1 && !isGenericApproval;
|
|
return {
|
|
passed,
|
|
reason: passed
|
|
? `Found ${improvementLines.length} specific improvement(s)`
|
|
: isGenericApproval
|
|
? 'Review is generic approval — must identify at least 1 specific improvement'
|
|
: 'No specific improvements identified',
|
|
evidence: improvementLines.slice(0, 3).join(' | '),
|
|
};
|
|
}
|
|
|
|
// ── Harness Definitions ─────────────────────────────────────────
|
|
|
|
/** Research → Synthesize → Verify */
|
|
export const researchVerifyHarness: WorkflowHarness = {
|
|
id: 'research-verify',
|
|
name: 'Research & Verify',
|
|
triggerPatterns: [
|
|
/\b(?:research|investigate|find out|look into|analyze)\b.*\b(?:and|then)\b.*\b(?:verify|check|validate|confirm)\b/i,
|
|
/\b(?:deep dive|thorough|comprehensive)\b.*\b(?:research|analysis|investigation)\b/i,
|
|
],
|
|
phases: [
|
|
{
|
|
id: 'gather',
|
|
name: 'Gather',
|
|
instruction: 'Search memory and available sources to collect relevant information. Use search_memory, recall_memory, or web_search to gather at least 2 distinct sources of information. Focus on breadth first — collect raw data before organizing.',
|
|
gates: [{
|
|
name: 'At least 2 search/recall tool calls',
|
|
validate: async (output) => hasToolCalls(output, ['search_memory', 'recall_memory', 'web_search', 'search_entities'], 2),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'synthesize',
|
|
name: 'Synthesize',
|
|
instruction: 'Organize findings into a structured summary. Create clear sections covering different aspects of the research. Include citations or references to sources where possible.',
|
|
gates: [{
|
|
name: 'At least 3 distinct sections in output',
|
|
validate: async (output) => hasMinSections(output, 3),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'verify',
|
|
name: 'Verify',
|
|
instruction: 'Review the synthesized findings for accuracy. Check claims against sources. Identify any contradictions, unsupported claims, or gaps. Output MUST include a "VERDICT:" line with your assessment (PASS, CONDITIONAL, or FAIL) followed by reasoning.',
|
|
gates: [{
|
|
name: 'Output contains VERDICT: assessment',
|
|
validate: async (output) => hasPattern(output, /VERDICT:\s*(PASS|CONDITIONAL|FAIL)/i, 'VERDICT: assessment'),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
],
|
|
aggregation: 'concatenate',
|
|
};
|
|
|
|
/** Understand → Review → Fix → Verify */
|
|
export const codeReviewFixHarness: WorkflowHarness = {
|
|
id: 'code-review-fix',
|
|
name: 'Code Review & Fix',
|
|
triggerPatterns: [
|
|
/\b(?:review|audit|check)\b.*\b(?:code|implementation|changes)\b.*\b(?:fix|resolve|address)\b/i,
|
|
/\b(?:find|identify)\b.*\b(?:bugs?|issues?|problems?)\b.*\b(?:fix|resolve)\b/i,
|
|
],
|
|
phases: [
|
|
{
|
|
id: 'understand',
|
|
name: 'Understand',
|
|
instruction: 'Read the code to understand the change and its context. Use read_file to examine the relevant files. Understand what the code does before evaluating it.',
|
|
gates: [{
|
|
name: 'At least 1 read_file tool call',
|
|
validate: async (output) => hasToolCalls(output, ['read_file', 'Read'], 1),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'review',
|
|
name: 'Review',
|
|
instruction: 'Identify issues with severity ratings. Categorize each issue as Critical, Warning, or Info. Be specific about what is wrong and why.',
|
|
gates: [{
|
|
name: 'Output contains structured issue list',
|
|
validate: async (output) => hasPattern(output, /(?:Critical|Warning|Info|HIGH|MEDIUM|LOW)\b/i, 'severity-rated issues'),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'fix',
|
|
name: 'Fix',
|
|
instruction: 'Apply fixes for the identified issues. Use write_file or edit_file to make changes. Address Critical issues first, then Warnings.',
|
|
gates: [{
|
|
name: 'At least 1 write/edit tool call',
|
|
validate: async (output) => hasToolCalls(output, ['write_file', 'edit_file', 'Write', 'Edit'], 1),
|
|
}],
|
|
maxRetries: 2,
|
|
},
|
|
{
|
|
id: 'verify',
|
|
name: 'Verify',
|
|
instruction: 'Run tests or type checking to verify the fixes. Use bash to run test commands (npm test, tsc --noEmit, etc.) and confirm the fixes work.',
|
|
gates: [{
|
|
name: 'At least 1 test/typecheck command',
|
|
validate: async (output) => hasToolCalls(output, ['bash', 'Bash', 'run_command'], 1),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
],
|
|
aggregation: 'last',
|
|
};
|
|
|
|
/** Context → Draft → Self-review */
|
|
export const documentDraftHarness: WorkflowHarness = {
|
|
id: 'document-draft',
|
|
name: 'Document Draft',
|
|
triggerPatterns: [
|
|
/\b(?:write|draft|create|compose)\b.*\b(?:document|doc|report|spec|proposal|brief|memo)\b/i,
|
|
/\b(?:document|write up|put together)\b.*\b(?:findings|analysis|results|plan)\b/i,
|
|
],
|
|
phases: [
|
|
{
|
|
id: 'context',
|
|
name: 'Context',
|
|
instruction: 'Gather requirements and prior context from memory. Use search_memory to find relevant background information, previous decisions, and any existing work on this topic.',
|
|
gates: [{
|
|
name: 'At least 1 memory search',
|
|
validate: async (output) => hasToolCalls(output, ['search_memory', 'recall_memory', 'search_entities'], 1),
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'draft',
|
|
name: 'Draft',
|
|
instruction: 'Produce the document based on gathered context. Write comprehensive content that addresses all requirements. The draft should be substantial (500+ characters) or written to a file.',
|
|
gates: [{
|
|
name: 'Substantial output or file written',
|
|
validate: async (output) => {
|
|
const hasFile = output.toolCalls.some(tc =>
|
|
['write_file', 'Write', 'generate_docx', 'generate_pptx'].some(t =>
|
|
tc.tool.toLowerCase().includes(t.toLowerCase()),
|
|
),
|
|
);
|
|
if (hasFile) return { passed: true, reason: 'Document written to file' };
|
|
return hasMinLength(output, 500);
|
|
},
|
|
}],
|
|
maxRetries: 1,
|
|
},
|
|
{
|
|
id: 'self-review',
|
|
name: 'Self-Review',
|
|
instruction: 'Re-read your draft critically. Identify specific gaps, inconsistencies, missing sections, or areas that need improvement. Do NOT just say "looks good" — find at least one concrete improvement.',
|
|
gates: [{
|
|
name: 'Identifies at least 1 specific improvement',
|
|
validate: async (output) => hasSpecificImprovement(output),
|
|
}],
|
|
maxRetries: 2,
|
|
},
|
|
],
|
|
aggregation: 'concatenate',
|
|
};
|
|
|
|
// ── Registry ────────────────────────────────────────────────────
|
|
|
|
/** All built-in harnesses. */
|
|
export const BUILTIN_HARNESSES: WorkflowHarness[] = [
|
|
researchVerifyHarness,
|
|
codeReviewFixHarness,
|
|
documentDraftHarness,
|
|
];
|
|
|
|
/** Find a harness by ID. */
|
|
export function getHarnessById(id: string): WorkflowHarness | undefined {
|
|
return BUILTIN_HARNESSES.find(h => h.id === id);
|
|
}
|
|
|
|
/** Find harnesses whose trigger patterns match the given task. */
|
|
export function matchHarness(task: string): WorkflowHarness | undefined {
|
|
return BUILTIN_HARNESSES.find(h =>
|
|
h.triggerPatterns.some(p => p.test(task)),
|
|
);
|
|
}
|