Files
waggle-os/packages/agent/src/system-tools.ts
Oleg Maslov b20b138fe4 moving
2026-09-02 10:14:22 +02:00

1081 lines
42 KiB
TypeScript

import * as fs from 'node:fs';
import * as path from 'node:path';
import * as crypto from 'node:crypto';
import { execFile, type ChildProcess } from 'node:child_process';
import { glob } from 'glob';
import type { ToolDefinition } from './tools.js';
import { SearchCache, RateLimiter } from './web-search-utils.js';
import { dedupTextResults, truncateToTokenBudget } from './tool-output-compressor.js';
import { safeFetch, allowLocalFromEnv, EgressBlockedError } from './url-egress-guard.js';
import {
IMAGE_EXTENSIONS, DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
checkDeniedBinaries, createSanitizedEnv, execFileWithTreeTimeout, terminateProcessTree,
truncateOutput, resolveSafe, assertNonSensitiveFilePath,
} from './system-tools-helpers.js';
/**
* L-18: Storage abstraction the file-touching tools (read/write/edit) can
* optionally route through instead of `fs.*`. Satisfied structurally by
* `packages/server/src/local/storage`'s `StorageProvider`, so the server
* can pass its own provider through without a cross-package import.
*
* When `createSystemTools` is called with a `fileBackend`, workspaces
* backed by non-local storage (team S3/MinIO in particular) route file
* ops through the provider. When absent, the tools fall back to the
* legacy `fs.*` path so callers that haven't migrated keep working.
*/
export interface FileBackend {
read(filePath: string): Promise<Buffer>;
write(filePath: string, data: Buffer, mime?: string): Promise<void>;
exists(filePath: string): Promise<boolean>;
delete(filePath: string): Promise<void>;
}
export interface SystemToolDeps {
/** Absolute filesystem path used as cwd for bash + fallback fs ops. */
workspace: string;
/** Optional storage backend (team S3/MinIO). If present, file-content
* tools route through it instead of node:fs. */
fileBackend?: FileBackend;
/** Deny reads of well-known secret material for user-linked workspace roots. */
denySensitiveFiles?: boolean;
}
/** Prefer a repository README over GitHub navigation chrome for exact repo-root fetches. */
export function extractWebPageText(body: string, sourceUrl: string): string {
let content = body;
try {
const source = new URL(sourceUrl);
const pathSegments = source.pathname.split('/').filter(Boolean);
if (source.hostname.toLowerCase() === 'github.com' && pathSegments.length === 2) {
const readme = body.match(
/<article\b[^>]*class=(?:"[^"]*\bmarkdown-body\b[^"]*"|'[^']*\bmarkdown-body\b[^']*')[^>]*>([\s\S]*?)<\/article>/i,
)?.[1];
content = readme?.trim() ? readme : '';
}
} catch {
// URL validation happens in web_fetch; direct helper callers fall back to the full page.
}
return content
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
.replace(/<header[\s\S]*?<\/header>/gi, '')
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
.replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n')
.replace(/<[^>]+>/g, '')
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&nbsp;/g, ' ')
.replace(/[ \t]+/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
}
// Module-level instances — shared across all tool invocations
const searchCache = new SearchCache(300_000); // 5 min TTL
const searchRateLimiter = new RateLimiter(10, 60_000); // 10 searches per minute
/** Background task tracking */
interface BackgroundTask {
process: ChildProcess;
stdout: string;
stderr: string;
status: 'running' | 'completed' | 'failed' | 'killed';
exitCode?: number;
/** Timestamp when the task was created */
createdAt: number;
}
const backgroundTasks = new Map<string, BackgroundTask>();
/** Maximum number of completed background tasks to retain */
const MAX_BACKGROUND_TASKS = 100;
/** Maximum age (ms) for stale completed tasks — 30 minutes */
const STALE_TASK_THRESHOLD_MS = 30 * 60 * 1000;
/**
* Evict the oldest completed/failed/killed task when the map exceeds MAX_BACKGROUND_TASKS.
*/
function evictOldestTask(): void {
if (backgroundTasks.size <= MAX_BACKGROUND_TASKS) return;
let oldestKey: string | null = null;
let oldestTime = Infinity;
for (const [key, task] of backgroundTasks) {
if (task.status !== 'running' && task.createdAt < oldestTime) {
oldestTime = task.createdAt;
oldestKey = key;
}
}
if (oldestKey) {
backgroundTasks.delete(oldestKey);
}
}
/**
* Remove completed/failed/killed entries older than STALE_TASK_THRESHOLD_MS.
*/
export function cleanupStaleTasks(): number {
const cutoff = Date.now() - STALE_TASK_THRESHOLD_MS;
let removed = 0;
for (const [key, task] of backgroundTasks) {
if (task.status !== 'running' && task.createdAt < cutoff) {
backgroundTasks.delete(key);
removed++;
}
}
return removed;
}
export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefinition[] {
const deps: SystemToolDeps = typeof wsOrDeps === 'string' ? { workspace: wsOrDeps } : wsOrDeps;
const { workspace, fileBackend, denySensitiveFiles = false } = deps;
// Normalize a user-supplied path to a backend key. The fs-level resolveSafe
// can't be used here because backend keys are virtual paths (e.g. S3 object
// keys), not paths inside the process cwd. We still need path-traversal
// defense — reject any segment equal to `..`.
const resolveBackendKey = (userPath: string): string => {
if (typeof userPath !== 'string' || !userPath) {
throw new Error('Invalid path');
}
// Forward slashes throughout; providers expect POSIX-style keys.
const normalized = userPath.replace(/\\/g, '/');
const parts = normalized.split('/').filter(p => p.length > 0);
if (parts.some(p => p === '..')) {
throw new Error(`Path resolves outside workspace: ${userPath}`);
}
return '/' + parts.join('/');
};
const resolveReadableBackendKey = (userPath: string): string => {
if (denySensitiveFiles) assertNonSensitiveFilePath(userPath);
return resolveBackendKey(userPath);
};
const resolveReadablePath = (userPath: string): string => resolveSafe(
workspace,
userPath,
{ denySensitiveFiles },
);
const filterReadableSearchPaths = (filePaths: string[]): string[] => {
if (!denySensitiveFiles) {
for (const filePath of filePaths) resolveSafe(workspace, filePath);
return filePaths;
}
return filePaths.filter((filePath) => {
try {
resolveReadablePath(filePath);
return true;
} catch {
// Search must not disclose sensitive or link-escaped filenames.
return false;
}
});
};
const validateWorkspaceGlob = (pattern: string): string => {
if (typeof pattern !== 'string' || !pattern) throw new Error('Invalid glob pattern');
if (path.isAbsolute(pattern)) throw new Error('Glob pattern must be relative to the workspace');
resolveSafe(workspace, pattern);
return pattern;
};
return [
// 1. bash — Execute shell commands
{
name: 'bash',
description: 'Execute an explicitly approved host shell command starting in the workspace directory. This is host-wide execution, not an OS sandbox.',
riskLevel: 'high',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
command: { type: 'string', description: 'Shell command to execute' },
timeout: { type: 'number', description: 'Timeout in milliseconds (default: 120000)' },
run_in_background: { type: 'boolean', description: 'When true, start process and return immediately with a task ID (default: false)' },
},
required: ['command'],
},
execute: async (args) => {
const command = args.command as string;
const timeout = (args.timeout as number) ?? 120_000;
const runInBackground = (args.run_in_background as boolean) ?? false;
// Security: check denylist before executing
const deniedBin = checkDeniedBinaries(command);
if (deniedBin) {
return `Error: Blocked — '${deniedBin}' is not allowed for security reasons`;
}
const isWindows = process.platform === 'win32';
const shell = isWindows ? 'cmd.exe' : '/bin/sh';
const shellArgs = isWindows ? ['/c', command] : ['-c', command];
const sanitizedEnv = createSanitizedEnv();
if (runInBackground) {
const taskId = crypto.randomUUID();
const child = execFile(shell, shellArgs, {
cwd: workspace,
maxBuffer: 10 * 1024 * 1024,
env: sanitizedEnv,
windowsHide: true,
});
const task: BackgroundTask = {
process: child,
stdout: '',
stderr: '',
status: 'running',
createdAt: Date.now(),
};
backgroundTasks.set(taskId, task);
evictOldestTask();
child.stdout?.on('data', (data: string) => {
task.stdout += data;
if (task.stdout.length > MAX_OUTPUT_SIZE) {
task.stdout = task.stdout.slice(0, MAX_OUTPUT_SIZE) + '\n[output truncated — exceeded 1 MB limit]';
}
});
child.stderr?.on('data', (data: string) => {
task.stderr += data;
if (task.stderr.length > MAX_OUTPUT_SIZE) {
task.stderr = task.stderr.slice(0, MAX_OUTPUT_SIZE) + '\n[output truncated — exceeded 1 MB limit]';
}
});
child.on('close', (code) => {
task.exitCode = code ?? undefined;
task.status = code === 0 ? 'completed' : 'failed';
});
child.on('error', (err) => {
task.stderr += err.message;
task.status = 'failed';
});
return `Background task started. Task ID: ${taskId}`;
}
const result = await execFileWithTreeTimeout(shell, shellArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
env: sanitizedEnv,
windowsHide: true,
}, timeout);
if (result.timedOut) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
: '';
return `Error: Command timeout after ${timeout}ms.${cleanupWarning}`;
}
if (result.errorMessage) {
// Return stderr + stdout on non-zero exit (truncated)
const output = truncateOutput((result.stderr || '') + (result.stdout || ''));
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup was incomplete; descendants may still be running.'
: '';
return `Error: ${result.errorMessage}.${cleanupWarning}${output ? `\n${output}` : ''}`;
}
return output || `Error: ${result.errorMessage}`;
}
return truncateOutput(result.stdout);
},
},
// 2. read_file — Read file contents
{
name: 'read_file',
description: 'Read the contents of a file (path relative to workspace). Supports offset/limit for partial reads and line numbers.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
path: { type: 'string', description: 'File path relative to workspace' },
offset: { type: 'number', description: 'Line number to start reading from (1-based, default: 1)' },
limit: { type: 'number', description: 'Maximum number of lines to return (default: all)' },
line_numbers: { type: 'boolean', description: 'When true, prefix each line with right-aligned line number (default: false)' },
},
required: ['path'],
},
execute: async (args) => {
try {
const filePath = args.path as string;
const ext = path.extname(filePath).toLowerCase();
// Image / PDF branches are fs-only for v1. Team-storage users will see
// backend routing for text files; images/PDFs stay on local disk until
// Bucket 2 adds binary-stream support in the backend contract.
if (!fileBackend) {
const resolved = resolveReadablePath(filePath);
if (IMAGE_EXTENSIONS.has(ext)) {
const stat = fs.statSync(resolved);
return `[Image file: ${filePath}, ${stat.size} bytes]`;
}
if (ext === '.pdf') {
const stat = fs.statSync(resolved);
try {
// pdf-parse is an optional CJS module; describe only the call we make.
type PdfParseFn = (buf: Buffer) => Promise<{ text: string }>;
const pdfModule = (await import('pdf-parse')) as unknown as
PdfParseFn & { default?: PdfParseFn };
const buffer = fs.readFileSync(resolved);
const parseFn: PdfParseFn = pdfModule.default ?? pdfModule;
const data = await parseFn(buffer);
const text = data.text;
return text || `[PDF file: ${filePath}, ${stat.size} bytes, no text content extracted]`;
} catch {
return `[PDF file: ${filePath}, ${stat.size} bytes. Install pdf-parse for text extraction: npm install pdf-parse]`;
}
}
}
// Text read — branch on backend presence.
let content: string;
if (fileBackend) {
if (IMAGE_EXTENSIONS.has(ext)) {
return `[Image file: ${filePath}, binary content not read over storage backend]`;
}
if (ext === '.pdf') {
return `[PDF file: ${filePath}, backend-routed read does not yet extract PDF text. Download the file to inspect it.]`;
}
const key = resolveReadableBackendKey(filePath);
const buf = await fileBackend.read(key);
content = buf.toString('utf-8');
} else {
const resolved = resolveReadablePath(filePath);
content = fs.readFileSync(resolved, 'utf-8');
}
let lines = content.split('\n');
const offset = (args.offset as number) ?? 1;
const limit = args.limit as number | undefined;
const lineNumbers = (args.line_numbers as boolean) ?? false;
// Apply offset (1-based)
const startIdx = Math.max(0, offset - 1);
lines = lines.slice(startIdx);
// Apply limit
if (limit !== undefined && limit > 0) {
lines = lines.slice(0, limit);
}
// Apply line numbers
if (lineNumbers) {
const maxLineNum = startIdx + lines.length;
const padWidth = String(maxLineNum).length;
lines = lines.map((line, i) => {
const lineNum = String(startIdx + i + 1).padStart(padWidth, ' ');
return `${lineNum}\t${line}`;
});
}
return lines.join('\n');
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 3. write_file — Create/overwrite files
{
name: 'write_file',
description: 'Write content to a file, creating parent directories if needed',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
path: { type: 'string', description: 'File path relative to workspace' },
content: { type: 'string', description: 'Content to write' },
},
required: ['path', 'content'],
},
execute: async (args) => {
try {
const filePath = args.path as string;
const contentStr = args.content as string;
if (fileBackend) {
const key = resolveBackendKey(filePath);
await fileBackend.write(key, Buffer.from(contentStr, 'utf-8'));
} else {
const resolved = resolveSafe(workspace, filePath);
fs.mkdirSync(path.dirname(resolved), { recursive: true });
fs.writeFileSync(resolved, contentStr);
}
return `Successfully wrote ${filePath}`;
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 4. edit_file — Exact string replacement
{
name: 'edit_file',
description: 'Replace an exact string in a file. old_string must appear exactly once unless replace_all is true.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
path: { type: 'string', description: 'File path relative to workspace' },
old_string: { type: 'string', description: 'Exact string to find' },
new_string: { type: 'string', description: 'Replacement string' },
replace_all: { type: 'boolean', description: 'When true, replace ALL occurrences (default: false)' },
},
required: ['path', 'old_string', 'new_string'],
},
execute: async (args) => {
try {
const filePath = args.path as string;
const oldStr = args.old_string as string;
const newStr = args.new_string as string;
const replaceAll = (args.replace_all as boolean) ?? false;
let content: string;
if (fileBackend) {
const key = resolveBackendKey(filePath);
const buf = await fileBackend.read(key);
content = buf.toString('utf-8');
} else {
const resolved = resolveSafe(workspace, filePath);
content = fs.readFileSync(resolved, 'utf-8');
}
const occurrences = content.split(oldStr).length - 1;
if (occurrences === 0) {
return `Error: old_string not found in ${filePath}`;
}
if (!replaceAll && occurrences > 1) {
return `Error: old_string found multiple times (${occurrences}) in ${filePath}. Must appear exactly once. Use replace_all: true to replace all occurrences.`;
}
const updated = replaceAll
? content.split(oldStr).join(newStr)
: content.replace(oldStr, newStr);
if (fileBackend) {
const key = resolveBackendKey(filePath);
await fileBackend.write(key, Buffer.from(updated, 'utf-8'));
} else {
const resolved = resolveSafe(workspace, filePath);
fs.writeFileSync(resolved, updated);
}
const countMsg = replaceAll && occurrences > 1 ? ` (${occurrences} occurrences)` : '';
return `Successfully edited ${filePath}${countMsg}`;
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 5. search_files — Glob pattern matching
{
name: 'search_files',
description: 'Search for files matching a glob pattern in the workspace',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
pattern: { type: 'string', description: 'Glob pattern (e.g., "**/*.ts")' },
},
required: ['pattern'],
},
execute: async (args) => {
try {
const pattern = validateWorkspaceGlob(args.pattern as string);
const globMatches = await glob(pattern, {
cwd: workspace,
ignore: ['node_modules/**', '.git/**'],
nodir: true,
});
const matches = filterReadableSearchPaths(globMatches);
if (matches.length === 0) return 'No files found.';
// A3: Cap file list to prevent token overflow
const MAX_FILE_RESULTS = 200;
if (matches.length > MAX_FILE_RESULTS) {
return matches.slice(0, MAX_FILE_RESULTS).join('\n')
+ `\n\n[Showing ${MAX_FILE_RESULTS} of ${matches.length} files. Use a more specific pattern to narrow results.]`;
}
return matches.join('\n');
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 6. search_content — Regex search through file contents
{
name: 'search_content',
description: 'Search file contents using a regex pattern. Returns file:line: match format. Supports context lines, output modes, file type filter, and max results.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
pattern: { type: 'string', description: 'Regex pattern to search for' },
glob: { type: 'string', description: 'Glob pattern to filter files (default: "**/*")' },
context_before: { type: 'number', description: 'Number of lines to show before each match' },
context_after: { type: 'number', description: 'Number of lines to show after each match' },
output_mode: { type: 'string', description: 'Output mode: "content" (default), "files" (file paths only), "count" (match counts per file)' },
file_type: { type: 'string', description: 'Filter by file extension, e.g., "ts", "js", "py"' },
max_results: { type: 'number', description: 'Maximum total results to return' },
},
required: ['pattern'],
},
execute: async (args) => {
try {
let filePattern = (args.glob as string) ?? '**/*';
const regex = new RegExp(args.pattern as string);
const contextBefore = (args.context_before as number) ?? 0;
const contextAfter = (args.context_after as number) ?? 0;
const outputMode = (args.output_mode as string) ?? 'content';
const fileType = args.file_type as string | undefined;
// A3: Default cap on results to prevent token overflow (211K tokens from unbounded search)
const MAX_SEARCH_RESULTS = 50;
const MAX_SEARCH_OUTPUT_CHARS = 30_000;
const maxResults = Math.min((args.max_results as number) ?? MAX_SEARCH_RESULTS, MAX_SEARCH_RESULTS);
// If file_type is specified, override glob with extension-specific pattern
if (fileType) {
if (!/^[a-zA-Z0-9][a-zA-Z0-9.+-]*$/.test(fileType)) {
throw new Error('Invalid file_type extension');
}
filePattern = `**/*.${fileType}`;
}
validateWorkspaceGlob(filePattern);
const globMatches = await glob(filePattern, {
cwd: workspace,
ignore: ['node_modules/**', '.git/**'],
nodir: true,
});
const files = filterReadableSearchPaths(globMatches);
if (outputMode === 'files') {
// Return only file paths that contain matches
const matchingFiles: string[] = [];
for (const file of files) {
if (maxResults !== undefined && matchingFiles.length >= maxResults) break;
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
if (regex.test(content)) {
matchingFiles.push(file);
}
} catch {
// Skip binary/unreadable files
}
}
if (matchingFiles.length === 0) return 'No matches found.';
return matchingFiles.join('\n');
}
if (outputMode === 'count') {
// Return file paths with match counts
const counts: string[] = [];
for (const file of files) {
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
const lines = content.split('\n');
let count = 0;
for (const line of lines) {
if (regex.test(line)) count++;
}
if (count > 0) {
counts.push(`${file}: ${count}`);
if (maxResults !== undefined && counts.length >= maxResults) break;
}
} catch {
// Skip binary/unreadable files
}
}
if (counts.length === 0) return 'No matches found.';
return counts.join('\n');
}
// Default: content mode
const results: string[] = [];
let totalResults = 0;
for (const file of files) {
if (maxResults !== undefined && totalResults >= maxResults) break;
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
if (maxResults !== undefined && totalResults >= maxResults) break;
if (regex.test(lines[i])) {
if (contextBefore > 0 || contextAfter > 0) {
// Show context lines
const startCtx = Math.max(0, i - contextBefore);
const endCtx = Math.min(lines.length - 1, i + contextAfter);
for (let j = startCtx; j <= endCtx; j++) {
const marker = j === i ? '>' : ' ';
results.push(`${file}:${j + 1}:${marker} ${lines[j]}`);
}
results.push('---');
} else {
results.push(`${file}:${i + 1}: ${lines[i]}`);
}
totalResults++;
}
}
} catch {
// Skip binary/unreadable files
}
}
if (results.length === 0) return 'No matches found.';
let output = results.join('\n');
// A3: Hard output size cap to prevent token overflow
if (output.length > MAX_SEARCH_OUTPUT_CHARS) {
output = output.slice(0, MAX_SEARCH_OUTPUT_CHARS)
+ `\n\n[Truncated: showing first ${MAX_SEARCH_OUTPUT_CHARS} chars of ${output.length}. Use a more specific pattern or path to narrow results.]`;
}
if (totalResults >= maxResults) {
output += `\n\n[Showing ${maxResults} matches. Use max_results parameter or a more specific pattern for different results.]`;
}
return output;
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 7. web_search — Search the web (auto-routes to best available provider)
// Priority: Perplexity → Tavily → Brave → DuckDuckGo (free fallback)
{
name: 'web_search',
description: 'Search the web via DuckDuckGo (free fallback). PREFER using perplexity_search, tavily_search, or brave_search if available — they provide higher quality results. Only use this tool if no premium search tools are available.',
parameters: {
type: 'object',
properties: {
query: { type: 'string', description: 'Search query' },
max_results: { type: 'number', description: 'Max results (default: 5, max: 10)' },
},
required: ['query'],
},
execute: async (args) => {
try {
const query = args.query as string;
const maxResults = Math.min((args.max_results as number) ?? 5, 10);
// Check cache first
const cacheKey = `${query}|${maxResults}`;
const cached = searchCache.get(cacheKey);
if (cached) return cached;
// Rate limit check
if (!searchRateLimiter.canProceed()) {
return 'Search rate limit exceeded. Please wait a moment before searching again.';
}
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
const response = await fetch(url, {
headers: { 'User-Agent': 'Waggle/1.0 (AI Assistant)' },
});
if (!response.ok) {
return `Search failed (${response.status}): ${response.statusText}`;
}
const html = await response.text();
// Parse DuckDuckGo HTML results
const results: Array<{ title: string; url: string; snippet: string }> = [];
const resultRegex = /<a[^>]+class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
let match;
while ((match = resultRegex.exec(html)) !== null && results.length < maxResults) {
const rawUrl = match[1];
const title = match[2].replace(/<[^>]+>/g, '').trim();
const snippet = match[3].replace(/<[^>]+>/g, '').trim();
// DuckDuckGo wraps URLs in a redirect — extract the real URL
let realUrl = rawUrl;
const uddgMatch = rawUrl.match(/[?&]uddg=([^&]+)/);
if (uddgMatch) {
realUrl = decodeURIComponent(uddgMatch[1]);
}
if (title && realUrl) {
results.push({ title, url: realUrl, snippet });
}
}
if (results.length === 0) return 'No search results found.';
// Collapse near-duplicate results before formatting: exact-URL dupes
// first, then snippets that are >=85% trigram-similar to a kept one.
// Search engines routinely surface the same passage from several URLs;
// sending all of them just spends tokens (TokenJuice subset).
const seenUrls = new Set<string>();
const urlUnique = results.filter((r) => {
if (seenUrls.has(r.url)) return false;
seenUrls.add(r.url);
return true;
});
const deduped = dedupTextResults(urlUnique, (r) => r.snippet);
const output = deduped
.map((r, i) => `[${i + 1}] ${r.title}\n ${r.url}\n ${r.snippet}`)
.join('\n\n');
// Cache successful results
searchCache.set(cacheKey, output);
return output;
} catch (err: unknown) {
return `Search error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 8. web_fetch — Fetch and extract text from a URL
{
name: 'web_fetch',
description: 'Fetch a web page and extract its text content. Use to read documentation, articles, or any web page.',
parameters: {
type: 'object',
properties: {
url: { type: 'string', description: 'URL to fetch' },
max_length: { type: 'number', description: 'Approx. max output (default 10000), enforced as a token budget (~max_length/4 tokens), so dense scripts like CJK return fewer characters than the number suggests' },
},
required: ['url'],
},
execute: async (args) => {
try {
const url = args.url as string;
const maxLength = (args.max_length as number) ?? 10_000;
// Token-budgeted cut (~4 chars/token) instead of a blunt char slice:
// grapheme-safe and consistent across prose/code/JSON/CJK.
const maxTokens = Math.max(1, Math.ceil(maxLength / 4));
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return 'Error: Invalid URL';
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return 'Error: Only http and https URLs are supported';
}
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), 15_000);
// SSRF guard: resolve + reject loopback/private/link-local/reserved
// targets, and re-validate every redirect hop. Loopback is fetchable
// only when WAGGLE_ALLOW_LOCAL_FETCH is set (desktop dev servers).
let response: Response;
try {
response = await safeFetch(
url,
{
headers: {
'User-Agent': 'Waggle/1.0 (AI Assistant)',
Accept: 'text/html,application/xhtml+xml,text/plain,application/json',
},
signal: ac.signal,
},
{ allowLocal: allowLocalFromEnv() },
);
} finally {
clearTimeout(timer);
}
if (!response.ok) {
return `Fetch failed (${response.status}): ${response.statusText}`;
}
const contentType = response.headers.get('content-type') ?? '';
const body = await response.text();
// JSON — return formatted
if (contentType.includes('application/json')) {
try {
return truncateToTokenBudget(JSON.stringify(JSON.parse(body), null, 2), maxTokens);
} catch {
return truncateToTokenBudget(body, maxTokens);
}
}
// HTML — extract text
const text = extractWebPageText(body, url);
if (!text) return 'Page fetched but no text content found.';
return truncateToTokenBudget(text, maxTokens);
} catch (err: unknown) {
if (err instanceof EgressBlockedError) return `Error: blocked for safety — ${err.message}`;
if (err instanceof Error && err.name === 'AbortError') return 'Error: Request timed out (15s)';
return `Fetch error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 9. multi_edit — Atomic multi-file edits
{
name: 'multi_edit',
description: 'Apply multiple edits atomically across one or more files. If any edit fails validation, none are applied.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
edits: {
type: 'array',
description: 'Array of edits to apply',
items: {
type: 'object',
properties: {
file_path: { type: 'string', description: 'File path relative to workspace' },
old_string: { type: 'string', description: 'Exact string to find (must appear exactly once per file)' },
new_string: { type: 'string', description: 'Replacement string' },
},
required: ['file_path', 'old_string', 'new_string'],
},
},
},
required: ['edits'],
},
execute: async (args) => {
try {
const edits = args.edits as Array<{ file_path: string; old_string: string; new_string: string }>;
if (!edits || edits.length === 0) {
return 'Error: No edits provided';
}
// Phase 1: Read all files and validate all old_strings exist exactly once.
// Route through fileBackend when present; otherwise fs. Key in the maps
// is the logical identifier (backend key, or resolved fs path).
const fileContents = new Map<string, string>();
const resolvedKeys = new Map<string, string>();
for (const edit of edits) {
const key = fileBackend
? resolveBackendKey(edit.file_path)
: resolveSafe(workspace, edit.file_path);
resolvedKeys.set(edit.file_path, key);
if (!fileContents.has(key)) {
const content = fileBackend
? (await fileBackend.read(key)).toString('utf-8')
: fs.readFileSync(key, 'utf-8');
fileContents.set(key, content);
}
const content = fileContents.get(key)!;
const occurrences = content.split(edit.old_string).length - 1;
if (occurrences === 0) {
return `Error: old_string not found in ${edit.file_path}. No edits applied (atomic rollback).`;
}
if (occurrences > 1) {
return `Error: old_string found multiple times (${occurrences}) in ${edit.file_path}. No edits applied (atomic rollback).`;
}
}
// Phase 2: Apply all edits (in memory first)
for (const edit of edits) {
const key = resolvedKeys.get(edit.file_path)!;
const content = fileContents.get(key)!;
fileContents.set(key, content.replace(edit.old_string, edit.new_string));
}
// Phase 3: Write all files
for (const [key, content] of fileContents) {
if (fileBackend) {
await fileBackend.write(key, Buffer.from(content, 'utf-8'));
} else {
fs.writeFileSync(key, content);
}
}
// Summarize
const fileCounts = new Map<string, number>();
for (const edit of edits) {
fileCounts.set(edit.file_path, (fileCounts.get(edit.file_path) ?? 0) + 1);
}
const summary = Array.from(fileCounts.entries())
.map(([file, count]) => ` ${file}: ${count} edit(s)`)
.join('\n');
return `Successfully applied ${edits.length} edit(s) across ${fileCounts.size} file(s):\n${summary}`;
} catch (err: unknown) {
return `Error: ${err instanceof Error ? err.message : String(err)}`;
}
},
},
// 10. get_task_output — Get output from a background task
{
name: 'get_task_output',
description: 'Get the current output and status of a background task started with run_in_background.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
task_id: { type: 'string', description: 'Task ID returned by run_in_background' },
},
required: ['task_id'],
},
execute: async (args) => {
const taskId = args.task_id as string;
const task = backgroundTasks.get(taskId);
if (!task) {
return `Error: No background task found with ID: ${taskId}`;
}
const parts: string[] = [
`Status: ${task.status}`,
];
if (task.exitCode !== undefined) {
parts.push(`Exit code: ${task.exitCode}`);
}
if (task.stdout) {
parts.push(`\n--- stdout ---\n${task.stdout}`);
}
if (task.stderr) {
parts.push(`\n--- stderr ---\n${task.stderr}`);
}
return parts.join('\n');
},
},
// 11. run_code — Execute code in an explicitly approved host child process
{
name: 'run_code',
description: 'Execute an explicitly approved code snippet in a host child process starting in the workspace directory. This is host-wide execution, not an OS sandbox. Supports JavaScript/TypeScript and Python (if installed).',
riskLevel: 'high',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
code: { type: 'string', description: 'The code to execute' },
language: { type: 'string', enum: ['javascript', 'typescript', 'python'], description: 'Programming language' },
timeout: { type: 'number', description: 'Timeout in milliseconds (default 10000, max 30000)' },
},
required: ['code', 'language'],
},
execute: async (args) => {
const code = args.code as string;
const language = args.language as string;
const rawTimeout = (args.timeout as number) ?? 10_000;
const timeout = Math.min(Math.max(rawTimeout, 1000), 30_000);
let executable: string;
let runtimeArgs: string[];
if (language === 'javascript' || language === 'typescript') {
executable = process.execPath;
runtimeArgs = language === 'typescript'
? ['--experimental-strip-types', '-e', code]
: ['-e', code];
} else if (language === 'python') {
executable = process.platform === 'win32' ? 'python' : 'python3';
runtimeArgs = ['-c', code];
} else {
return `Error: Unsupported language "${language}". Supported: javascript, typescript, python.`;
}
const sanitizedEnv = createSanitizedEnv();
const result = await execFileWithTreeTimeout(executable, runtimeArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
env: sanitizedEnv,
windowsHide: true,
}, timeout);
const parts: string[] = [];
if (result.timedOut) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
: '';
return `Error: Code execution timed out after ${timeout}ms.${cleanupWarning}`;
}
if (result.errorMessage) {
// Check for runtime not found
if (result.errorCode === 'ENOENT' || result.errorMessage.includes('not found')) {
return `Error: ${language} runtime not found. Please ensure ${language === 'python' ? 'python3/python' : 'node'} is installed and on PATH.`;
}
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup was incomplete; descendants may still be running.'
: '';
parts.push(`--- error ---\n${result.errorMessage}.${cleanupWarning}`);
}
}
if (result.stdout) parts.push(`--- stdout ---\n${truncateOutput(result.stdout)}`);
if (result.stderr) parts.push(`--- stderr ---\n${truncateOutput(result.stderr)}`);
if (parts.length === 0 && result.errorMessage) return `Error: ${result.errorMessage}`;
if (parts.length === 0) return '(no output)';
return parts.join('\n');
},
},
// 12. kill_task — Kill a background task
{
name: 'kill_task',
description: 'Kill a running background task.',
offlineCapable: true,
parameters: {
type: 'object',
properties: {
task_id: { type: 'string', description: 'Task ID to kill' },
},
required: ['task_id'],
},
execute: async (args) => {
const taskId = args.task_id as string;
const task = backgroundTasks.get(taskId);
if (!task) {
return `Error: No background task found with ID: ${taskId}`;
}
if (task.status !== 'running') {
return `Task ${taskId} is already ${task.status}`;
}
terminateProcessTree(task.process);
task.status = 'killed';
return `Task ${taskId} has been killed`;
},
},
];
}
/** Expose internals for testing */
export { backgroundTasks, MAX_BACKGROUND_TASKS, STALE_TASK_THRESHOLD_MS };
/** Re-export helpers so existing consumers keep working */
export {
DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
checkDeniedBinaries, createSanitizedEnv, terminateProcessTree, truncateOutput,
resolveSafe,
} from './system-tools-helpers.js';