1081 lines
42 KiB
TypeScript
1081 lines
42 KiB
TypeScript
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import * as crypto from 'node:crypto';
|
|
import { execFile, type ChildProcess } from 'node:child_process';
|
|
import { glob } from 'glob';
|
|
import type { ToolDefinition } from './tools.js';
|
|
import { SearchCache, RateLimiter } from './web-search-utils.js';
|
|
import { dedupTextResults, truncateToTokenBudget } from './tool-output-compressor.js';
|
|
import { safeFetch, allowLocalFromEnv, EgressBlockedError } from './url-egress-guard.js';
|
|
import {
|
|
IMAGE_EXTENSIONS, DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
|
|
checkDeniedBinaries, createSanitizedEnv, execFileWithTreeTimeout, terminateProcessTree,
|
|
truncateOutput, resolveSafe, assertNonSensitiveFilePath,
|
|
} from './system-tools-helpers.js';
|
|
|
|
/**
|
|
* L-18: Storage abstraction the file-touching tools (read/write/edit) can
|
|
* optionally route through instead of `fs.*`. Satisfied structurally by
|
|
* `packages/server/src/local/storage`'s `StorageProvider`, so the server
|
|
* can pass its own provider through without a cross-package import.
|
|
*
|
|
* When `createSystemTools` is called with a `fileBackend`, workspaces
|
|
* backed by non-local storage (team S3/MinIO in particular) route file
|
|
* ops through the provider. When absent, the tools fall back to the
|
|
* legacy `fs.*` path so callers that haven't migrated keep working.
|
|
*/
|
|
export interface FileBackend {
|
|
read(filePath: string): Promise<Buffer>;
|
|
write(filePath: string, data: Buffer, mime?: string): Promise<void>;
|
|
exists(filePath: string): Promise<boolean>;
|
|
delete(filePath: string): Promise<void>;
|
|
}
|
|
|
|
export interface SystemToolDeps {
|
|
/** Absolute filesystem path used as cwd for bash + fallback fs ops. */
|
|
workspace: string;
|
|
/** Optional storage backend (team S3/MinIO). If present, file-content
|
|
* tools route through it instead of node:fs. */
|
|
fileBackend?: FileBackend;
|
|
/** Deny reads of well-known secret material for user-linked workspace roots. */
|
|
denySensitiveFiles?: boolean;
|
|
}
|
|
|
|
/** Prefer a repository README over GitHub navigation chrome for exact repo-root fetches. */
|
|
export function extractWebPageText(body: string, sourceUrl: string): string {
|
|
let content = body;
|
|
try {
|
|
const source = new URL(sourceUrl);
|
|
const pathSegments = source.pathname.split('/').filter(Boolean);
|
|
if (source.hostname.toLowerCase() === 'github.com' && pathSegments.length === 2) {
|
|
const readme = body.match(
|
|
/<article\b[^>]*class=(?:"[^"]*\bmarkdown-body\b[^"]*"|'[^']*\bmarkdown-body\b[^']*')[^>]*>([\s\S]*?)<\/article>/i,
|
|
)?.[1];
|
|
content = readme?.trim() ? readme : '';
|
|
}
|
|
} catch {
|
|
// URL validation happens in web_fetch; direct helper callers fall back to the full page.
|
|
}
|
|
|
|
return content
|
|
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
|
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
|
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
|
|
.replace(/<header[\s\S]*?<\/header>/gi, '')
|
|
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
|
|
.replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n')
|
|
.replace(/<[^>]+>/g, '')
|
|
.replace(/&/g, '&')
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>')
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/ /g, ' ')
|
|
.replace(/[ \t]+/g, ' ')
|
|
.replace(/\n{3,}/g, '\n\n')
|
|
.trim();
|
|
}
|
|
|
|
// Module-level instances — shared across all tool invocations
|
|
const searchCache = new SearchCache(300_000); // 5 min TTL
|
|
const searchRateLimiter = new RateLimiter(10, 60_000); // 10 searches per minute
|
|
|
|
/** Background task tracking */
|
|
interface BackgroundTask {
|
|
process: ChildProcess;
|
|
stdout: string;
|
|
stderr: string;
|
|
status: 'running' | 'completed' | 'failed' | 'killed';
|
|
exitCode?: number;
|
|
/** Timestamp when the task was created */
|
|
createdAt: number;
|
|
}
|
|
|
|
const backgroundTasks = new Map<string, BackgroundTask>();
|
|
|
|
/** Maximum number of completed background tasks to retain */
|
|
const MAX_BACKGROUND_TASKS = 100;
|
|
|
|
/** Maximum age (ms) for stale completed tasks — 30 minutes */
|
|
const STALE_TASK_THRESHOLD_MS = 30 * 60 * 1000;
|
|
|
|
/**
|
|
* Evict the oldest completed/failed/killed task when the map exceeds MAX_BACKGROUND_TASKS.
|
|
*/
|
|
function evictOldestTask(): void {
|
|
if (backgroundTasks.size <= MAX_BACKGROUND_TASKS) return;
|
|
let oldestKey: string | null = null;
|
|
let oldestTime = Infinity;
|
|
for (const [key, task] of backgroundTasks) {
|
|
if (task.status !== 'running' && task.createdAt < oldestTime) {
|
|
oldestTime = task.createdAt;
|
|
oldestKey = key;
|
|
}
|
|
}
|
|
if (oldestKey) {
|
|
backgroundTasks.delete(oldestKey);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Remove completed/failed/killed entries older than STALE_TASK_THRESHOLD_MS.
|
|
*/
|
|
export function cleanupStaleTasks(): number {
|
|
const cutoff = Date.now() - STALE_TASK_THRESHOLD_MS;
|
|
let removed = 0;
|
|
for (const [key, task] of backgroundTasks) {
|
|
if (task.status !== 'running' && task.createdAt < cutoff) {
|
|
backgroundTasks.delete(key);
|
|
removed++;
|
|
}
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefinition[] {
|
|
const deps: SystemToolDeps = typeof wsOrDeps === 'string' ? { workspace: wsOrDeps } : wsOrDeps;
|
|
const { workspace, fileBackend, denySensitiveFiles = false } = deps;
|
|
|
|
// Normalize a user-supplied path to a backend key. The fs-level resolveSafe
|
|
// can't be used here because backend keys are virtual paths (e.g. S3 object
|
|
// keys), not paths inside the process cwd. We still need path-traversal
|
|
// defense — reject any segment equal to `..`.
|
|
const resolveBackendKey = (userPath: string): string => {
|
|
if (typeof userPath !== 'string' || !userPath) {
|
|
throw new Error('Invalid path');
|
|
}
|
|
// Forward slashes throughout; providers expect POSIX-style keys.
|
|
const normalized = userPath.replace(/\\/g, '/');
|
|
const parts = normalized.split('/').filter(p => p.length > 0);
|
|
if (parts.some(p => p === '..')) {
|
|
throw new Error(`Path resolves outside workspace: ${userPath}`);
|
|
}
|
|
return '/' + parts.join('/');
|
|
};
|
|
|
|
const resolveReadableBackendKey = (userPath: string): string => {
|
|
if (denySensitiveFiles) assertNonSensitiveFilePath(userPath);
|
|
return resolveBackendKey(userPath);
|
|
};
|
|
|
|
const resolveReadablePath = (userPath: string): string => resolveSafe(
|
|
workspace,
|
|
userPath,
|
|
{ denySensitiveFiles },
|
|
);
|
|
|
|
const filterReadableSearchPaths = (filePaths: string[]): string[] => {
|
|
if (!denySensitiveFiles) {
|
|
for (const filePath of filePaths) resolveSafe(workspace, filePath);
|
|
return filePaths;
|
|
}
|
|
return filePaths.filter((filePath) => {
|
|
try {
|
|
resolveReadablePath(filePath);
|
|
return true;
|
|
} catch {
|
|
// Search must not disclose sensitive or link-escaped filenames.
|
|
return false;
|
|
}
|
|
});
|
|
};
|
|
|
|
const validateWorkspaceGlob = (pattern: string): string => {
|
|
if (typeof pattern !== 'string' || !pattern) throw new Error('Invalid glob pattern');
|
|
if (path.isAbsolute(pattern)) throw new Error('Glob pattern must be relative to the workspace');
|
|
resolveSafe(workspace, pattern);
|
|
return pattern;
|
|
};
|
|
|
|
return [
|
|
// 1. bash — Execute shell commands
|
|
{
|
|
name: 'bash',
|
|
description: 'Execute an explicitly approved host shell command starting in the workspace directory. This is host-wide execution, not an OS sandbox.',
|
|
riskLevel: 'high',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
command: { type: 'string', description: 'Shell command to execute' },
|
|
timeout: { type: 'number', description: 'Timeout in milliseconds (default: 120000)' },
|
|
run_in_background: { type: 'boolean', description: 'When true, start process and return immediately with a task ID (default: false)' },
|
|
},
|
|
required: ['command'],
|
|
},
|
|
execute: async (args) => {
|
|
const command = args.command as string;
|
|
const timeout = (args.timeout as number) ?? 120_000;
|
|
const runInBackground = (args.run_in_background as boolean) ?? false;
|
|
|
|
// Security: check denylist before executing
|
|
const deniedBin = checkDeniedBinaries(command);
|
|
if (deniedBin) {
|
|
return `Error: Blocked — '${deniedBin}' is not allowed for security reasons`;
|
|
}
|
|
|
|
const isWindows = process.platform === 'win32';
|
|
const shell = isWindows ? 'cmd.exe' : '/bin/sh';
|
|
const shellArgs = isWindows ? ['/c', command] : ['-c', command];
|
|
const sanitizedEnv = createSanitizedEnv();
|
|
|
|
if (runInBackground) {
|
|
const taskId = crypto.randomUUID();
|
|
const child = execFile(shell, shellArgs, {
|
|
cwd: workspace,
|
|
maxBuffer: 10 * 1024 * 1024,
|
|
env: sanitizedEnv,
|
|
windowsHide: true,
|
|
});
|
|
|
|
const task: BackgroundTask = {
|
|
process: child,
|
|
stdout: '',
|
|
stderr: '',
|
|
status: 'running',
|
|
createdAt: Date.now(),
|
|
};
|
|
backgroundTasks.set(taskId, task);
|
|
evictOldestTask();
|
|
|
|
child.stdout?.on('data', (data: string) => {
|
|
task.stdout += data;
|
|
if (task.stdout.length > MAX_OUTPUT_SIZE) {
|
|
task.stdout = task.stdout.slice(0, MAX_OUTPUT_SIZE) + '\n[output truncated — exceeded 1 MB limit]';
|
|
}
|
|
});
|
|
child.stderr?.on('data', (data: string) => {
|
|
task.stderr += data;
|
|
if (task.stderr.length > MAX_OUTPUT_SIZE) {
|
|
task.stderr = task.stderr.slice(0, MAX_OUTPUT_SIZE) + '\n[output truncated — exceeded 1 MB limit]';
|
|
}
|
|
});
|
|
child.on('close', (code) => {
|
|
task.exitCode = code ?? undefined;
|
|
task.status = code === 0 ? 'completed' : 'failed';
|
|
});
|
|
child.on('error', (err) => {
|
|
task.stderr += err.message;
|
|
task.status = 'failed';
|
|
});
|
|
|
|
return `Background task started. Task ID: ${taskId}`;
|
|
}
|
|
|
|
const result = await execFileWithTreeTimeout(shell, shellArgs, {
|
|
cwd: workspace,
|
|
maxBuffer: MAX_OUTPUT_SIZE,
|
|
env: sanitizedEnv,
|
|
windowsHide: true,
|
|
}, timeout);
|
|
if (result.timedOut) {
|
|
const cleanupWarning = result.cleanupDegraded
|
|
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
|
|
: '';
|
|
return `Error: Command timeout after ${timeout}ms.${cleanupWarning}`;
|
|
}
|
|
if (result.errorMessage) {
|
|
// Return stderr + stdout on non-zero exit (truncated)
|
|
const output = truncateOutput((result.stderr || '') + (result.stdout || ''));
|
|
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
|
|
const cleanupWarning = result.cleanupDegraded
|
|
? ' Process-tree cleanup was incomplete; descendants may still be running.'
|
|
: '';
|
|
return `Error: ${result.errorMessage}.${cleanupWarning}${output ? `\n${output}` : ''}`;
|
|
}
|
|
return output || `Error: ${result.errorMessage}`;
|
|
}
|
|
return truncateOutput(result.stdout);
|
|
},
|
|
},
|
|
|
|
// 2. read_file — Read file contents
|
|
{
|
|
name: 'read_file',
|
|
description: 'Read the contents of a file (path relative to workspace). Supports offset/limit for partial reads and line numbers.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
path: { type: 'string', description: 'File path relative to workspace' },
|
|
offset: { type: 'number', description: 'Line number to start reading from (1-based, default: 1)' },
|
|
limit: { type: 'number', description: 'Maximum number of lines to return (default: all)' },
|
|
line_numbers: { type: 'boolean', description: 'When true, prefix each line with right-aligned line number (default: false)' },
|
|
},
|
|
required: ['path'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const filePath = args.path as string;
|
|
const ext = path.extname(filePath).toLowerCase();
|
|
|
|
// Image / PDF branches are fs-only for v1. Team-storage users will see
|
|
// backend routing for text files; images/PDFs stay on local disk until
|
|
// Bucket 2 adds binary-stream support in the backend contract.
|
|
if (!fileBackend) {
|
|
const resolved = resolveReadablePath(filePath);
|
|
|
|
if (IMAGE_EXTENSIONS.has(ext)) {
|
|
const stat = fs.statSync(resolved);
|
|
return `[Image file: ${filePath}, ${stat.size} bytes]`;
|
|
}
|
|
if (ext === '.pdf') {
|
|
const stat = fs.statSync(resolved);
|
|
try {
|
|
// pdf-parse is an optional CJS module; describe only the call we make.
|
|
type PdfParseFn = (buf: Buffer) => Promise<{ text: string }>;
|
|
const pdfModule = (await import('pdf-parse')) as unknown as
|
|
PdfParseFn & { default?: PdfParseFn };
|
|
const buffer = fs.readFileSync(resolved);
|
|
const parseFn: PdfParseFn = pdfModule.default ?? pdfModule;
|
|
const data = await parseFn(buffer);
|
|
const text = data.text;
|
|
return text || `[PDF file: ${filePath}, ${stat.size} bytes, no text content extracted]`;
|
|
} catch {
|
|
return `[PDF file: ${filePath}, ${stat.size} bytes. Install pdf-parse for text extraction: npm install pdf-parse]`;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Text read — branch on backend presence.
|
|
let content: string;
|
|
if (fileBackend) {
|
|
if (IMAGE_EXTENSIONS.has(ext)) {
|
|
return `[Image file: ${filePath}, binary content not read over storage backend]`;
|
|
}
|
|
if (ext === '.pdf') {
|
|
return `[PDF file: ${filePath}, backend-routed read does not yet extract PDF text. Download the file to inspect it.]`;
|
|
}
|
|
const key = resolveReadableBackendKey(filePath);
|
|
const buf = await fileBackend.read(key);
|
|
content = buf.toString('utf-8');
|
|
} else {
|
|
const resolved = resolveReadablePath(filePath);
|
|
content = fs.readFileSync(resolved, 'utf-8');
|
|
}
|
|
|
|
let lines = content.split('\n');
|
|
|
|
const offset = (args.offset as number) ?? 1;
|
|
const limit = args.limit as number | undefined;
|
|
const lineNumbers = (args.line_numbers as boolean) ?? false;
|
|
|
|
// Apply offset (1-based)
|
|
const startIdx = Math.max(0, offset - 1);
|
|
lines = lines.slice(startIdx);
|
|
|
|
// Apply limit
|
|
if (limit !== undefined && limit > 0) {
|
|
lines = lines.slice(0, limit);
|
|
}
|
|
|
|
// Apply line numbers
|
|
if (lineNumbers) {
|
|
const maxLineNum = startIdx + lines.length;
|
|
const padWidth = String(maxLineNum).length;
|
|
lines = lines.map((line, i) => {
|
|
const lineNum = String(startIdx + i + 1).padStart(padWidth, ' ');
|
|
return `${lineNum}\t${line}`;
|
|
});
|
|
}
|
|
|
|
return lines.join('\n');
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 3. write_file — Create/overwrite files
|
|
{
|
|
name: 'write_file',
|
|
description: 'Write content to a file, creating parent directories if needed',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
path: { type: 'string', description: 'File path relative to workspace' },
|
|
content: { type: 'string', description: 'Content to write' },
|
|
},
|
|
required: ['path', 'content'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const filePath = args.path as string;
|
|
const contentStr = args.content as string;
|
|
if (fileBackend) {
|
|
const key = resolveBackendKey(filePath);
|
|
await fileBackend.write(key, Buffer.from(contentStr, 'utf-8'));
|
|
} else {
|
|
const resolved = resolveSafe(workspace, filePath);
|
|
fs.mkdirSync(path.dirname(resolved), { recursive: true });
|
|
fs.writeFileSync(resolved, contentStr);
|
|
}
|
|
return `Successfully wrote ${filePath}`;
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 4. edit_file — Exact string replacement
|
|
{
|
|
name: 'edit_file',
|
|
description: 'Replace an exact string in a file. old_string must appear exactly once unless replace_all is true.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
path: { type: 'string', description: 'File path relative to workspace' },
|
|
old_string: { type: 'string', description: 'Exact string to find' },
|
|
new_string: { type: 'string', description: 'Replacement string' },
|
|
replace_all: { type: 'boolean', description: 'When true, replace ALL occurrences (default: false)' },
|
|
},
|
|
required: ['path', 'old_string', 'new_string'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const filePath = args.path as string;
|
|
const oldStr = args.old_string as string;
|
|
const newStr = args.new_string as string;
|
|
const replaceAll = (args.replace_all as boolean) ?? false;
|
|
|
|
let content: string;
|
|
if (fileBackend) {
|
|
const key = resolveBackendKey(filePath);
|
|
const buf = await fileBackend.read(key);
|
|
content = buf.toString('utf-8');
|
|
} else {
|
|
const resolved = resolveSafe(workspace, filePath);
|
|
content = fs.readFileSync(resolved, 'utf-8');
|
|
}
|
|
|
|
const occurrences = content.split(oldStr).length - 1;
|
|
|
|
if (occurrences === 0) {
|
|
return `Error: old_string not found in ${filePath}`;
|
|
}
|
|
|
|
if (!replaceAll && occurrences > 1) {
|
|
return `Error: old_string found multiple times (${occurrences}) in ${filePath}. Must appear exactly once. Use replace_all: true to replace all occurrences.`;
|
|
}
|
|
|
|
const updated = replaceAll
|
|
? content.split(oldStr).join(newStr)
|
|
: content.replace(oldStr, newStr);
|
|
|
|
if (fileBackend) {
|
|
const key = resolveBackendKey(filePath);
|
|
await fileBackend.write(key, Buffer.from(updated, 'utf-8'));
|
|
} else {
|
|
const resolved = resolveSafe(workspace, filePath);
|
|
fs.writeFileSync(resolved, updated);
|
|
}
|
|
|
|
const countMsg = replaceAll && occurrences > 1 ? ` (${occurrences} occurrences)` : '';
|
|
return `Successfully edited ${filePath}${countMsg}`;
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 5. search_files — Glob pattern matching
|
|
{
|
|
name: 'search_files',
|
|
description: 'Search for files matching a glob pattern in the workspace',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
pattern: { type: 'string', description: 'Glob pattern (e.g., "**/*.ts")' },
|
|
},
|
|
required: ['pattern'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const pattern = validateWorkspaceGlob(args.pattern as string);
|
|
const globMatches = await glob(pattern, {
|
|
cwd: workspace,
|
|
ignore: ['node_modules/**', '.git/**'],
|
|
nodir: true,
|
|
});
|
|
const matches = filterReadableSearchPaths(globMatches);
|
|
if (matches.length === 0) return 'No files found.';
|
|
// A3: Cap file list to prevent token overflow
|
|
const MAX_FILE_RESULTS = 200;
|
|
if (matches.length > MAX_FILE_RESULTS) {
|
|
return matches.slice(0, MAX_FILE_RESULTS).join('\n')
|
|
+ `\n\n[Showing ${MAX_FILE_RESULTS} of ${matches.length} files. Use a more specific pattern to narrow results.]`;
|
|
}
|
|
return matches.join('\n');
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 6. search_content — Regex search through file contents
|
|
{
|
|
name: 'search_content',
|
|
description: 'Search file contents using a regex pattern. Returns file:line: match format. Supports context lines, output modes, file type filter, and max results.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
pattern: { type: 'string', description: 'Regex pattern to search for' },
|
|
glob: { type: 'string', description: 'Glob pattern to filter files (default: "**/*")' },
|
|
context_before: { type: 'number', description: 'Number of lines to show before each match' },
|
|
context_after: { type: 'number', description: 'Number of lines to show after each match' },
|
|
output_mode: { type: 'string', description: 'Output mode: "content" (default), "files" (file paths only), "count" (match counts per file)' },
|
|
file_type: { type: 'string', description: 'Filter by file extension, e.g., "ts", "js", "py"' },
|
|
max_results: { type: 'number', description: 'Maximum total results to return' },
|
|
},
|
|
required: ['pattern'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
let filePattern = (args.glob as string) ?? '**/*';
|
|
const regex = new RegExp(args.pattern as string);
|
|
const contextBefore = (args.context_before as number) ?? 0;
|
|
const contextAfter = (args.context_after as number) ?? 0;
|
|
const outputMode = (args.output_mode as string) ?? 'content';
|
|
const fileType = args.file_type as string | undefined;
|
|
// A3: Default cap on results to prevent token overflow (211K tokens from unbounded search)
|
|
const MAX_SEARCH_RESULTS = 50;
|
|
const MAX_SEARCH_OUTPUT_CHARS = 30_000;
|
|
const maxResults = Math.min((args.max_results as number) ?? MAX_SEARCH_RESULTS, MAX_SEARCH_RESULTS);
|
|
|
|
// If file_type is specified, override glob with extension-specific pattern
|
|
if (fileType) {
|
|
if (!/^[a-zA-Z0-9][a-zA-Z0-9.+-]*$/.test(fileType)) {
|
|
throw new Error('Invalid file_type extension');
|
|
}
|
|
filePattern = `**/*.${fileType}`;
|
|
}
|
|
|
|
validateWorkspaceGlob(filePattern);
|
|
|
|
const globMatches = await glob(filePattern, {
|
|
cwd: workspace,
|
|
ignore: ['node_modules/**', '.git/**'],
|
|
nodir: true,
|
|
});
|
|
const files = filterReadableSearchPaths(globMatches);
|
|
|
|
if (outputMode === 'files') {
|
|
// Return only file paths that contain matches
|
|
const matchingFiles: string[] = [];
|
|
for (const file of files) {
|
|
if (maxResults !== undefined && matchingFiles.length >= maxResults) break;
|
|
const absPath = resolveReadablePath(file);
|
|
try {
|
|
const content = fs.readFileSync(absPath, 'utf-8');
|
|
if (regex.test(content)) {
|
|
matchingFiles.push(file);
|
|
}
|
|
} catch {
|
|
// Skip binary/unreadable files
|
|
}
|
|
}
|
|
if (matchingFiles.length === 0) return 'No matches found.';
|
|
return matchingFiles.join('\n');
|
|
}
|
|
|
|
if (outputMode === 'count') {
|
|
// Return file paths with match counts
|
|
const counts: string[] = [];
|
|
for (const file of files) {
|
|
const absPath = resolveReadablePath(file);
|
|
try {
|
|
const content = fs.readFileSync(absPath, 'utf-8');
|
|
const lines = content.split('\n');
|
|
let count = 0;
|
|
for (const line of lines) {
|
|
if (regex.test(line)) count++;
|
|
}
|
|
if (count > 0) {
|
|
counts.push(`${file}: ${count}`);
|
|
if (maxResults !== undefined && counts.length >= maxResults) break;
|
|
}
|
|
} catch {
|
|
// Skip binary/unreadable files
|
|
}
|
|
}
|
|
if (counts.length === 0) return 'No matches found.';
|
|
return counts.join('\n');
|
|
}
|
|
|
|
// Default: content mode
|
|
const results: string[] = [];
|
|
let totalResults = 0;
|
|
|
|
for (const file of files) {
|
|
if (maxResults !== undefined && totalResults >= maxResults) break;
|
|
const absPath = resolveReadablePath(file);
|
|
try {
|
|
const content = fs.readFileSync(absPath, 'utf-8');
|
|
const lines = content.split('\n');
|
|
for (let i = 0; i < lines.length; i++) {
|
|
if (maxResults !== undefined && totalResults >= maxResults) break;
|
|
if (regex.test(lines[i])) {
|
|
if (contextBefore > 0 || contextAfter > 0) {
|
|
// Show context lines
|
|
const startCtx = Math.max(0, i - contextBefore);
|
|
const endCtx = Math.min(lines.length - 1, i + contextAfter);
|
|
for (let j = startCtx; j <= endCtx; j++) {
|
|
const marker = j === i ? '>' : ' ';
|
|
results.push(`${file}:${j + 1}:${marker} ${lines[j]}`);
|
|
}
|
|
results.push('---');
|
|
} else {
|
|
results.push(`${file}:${i + 1}: ${lines[i]}`);
|
|
}
|
|
totalResults++;
|
|
}
|
|
}
|
|
} catch {
|
|
// Skip binary/unreadable files
|
|
}
|
|
}
|
|
|
|
if (results.length === 0) return 'No matches found.';
|
|
let output = results.join('\n');
|
|
// A3: Hard output size cap to prevent token overflow
|
|
if (output.length > MAX_SEARCH_OUTPUT_CHARS) {
|
|
output = output.slice(0, MAX_SEARCH_OUTPUT_CHARS)
|
|
+ `\n\n[Truncated: showing first ${MAX_SEARCH_OUTPUT_CHARS} chars of ${output.length}. Use a more specific pattern or path to narrow results.]`;
|
|
}
|
|
if (totalResults >= maxResults) {
|
|
output += `\n\n[Showing ${maxResults} matches. Use max_results parameter or a more specific pattern for different results.]`;
|
|
}
|
|
return output;
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 7. web_search — Search the web (auto-routes to best available provider)
|
|
// Priority: Perplexity → Tavily → Brave → DuckDuckGo (free fallback)
|
|
{
|
|
name: 'web_search',
|
|
description: 'Search the web via DuckDuckGo (free fallback). PREFER using perplexity_search, tavily_search, or brave_search if available — they provide higher quality results. Only use this tool if no premium search tools are available.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
query: { type: 'string', description: 'Search query' },
|
|
max_results: { type: 'number', description: 'Max results (default: 5, max: 10)' },
|
|
},
|
|
required: ['query'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const query = args.query as string;
|
|
const maxResults = Math.min((args.max_results as number) ?? 5, 10);
|
|
|
|
// Check cache first
|
|
const cacheKey = `${query}|${maxResults}`;
|
|
const cached = searchCache.get(cacheKey);
|
|
if (cached) return cached;
|
|
|
|
// Rate limit check
|
|
if (!searchRateLimiter.canProceed()) {
|
|
return 'Search rate limit exceeded. Please wait a moment before searching again.';
|
|
}
|
|
|
|
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
|
|
const response = await fetch(url, {
|
|
headers: { 'User-Agent': 'Waggle/1.0 (AI Assistant)' },
|
|
});
|
|
|
|
if (!response.ok) {
|
|
return `Search failed (${response.status}): ${response.statusText}`;
|
|
}
|
|
|
|
const html = await response.text();
|
|
|
|
// Parse DuckDuckGo HTML results
|
|
const results: Array<{ title: string; url: string; snippet: string }> = [];
|
|
const resultRegex = /<a[^>]+class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
|
|
let match;
|
|
|
|
while ((match = resultRegex.exec(html)) !== null && results.length < maxResults) {
|
|
const rawUrl = match[1];
|
|
const title = match[2].replace(/<[^>]+>/g, '').trim();
|
|
const snippet = match[3].replace(/<[^>]+>/g, '').trim();
|
|
|
|
// DuckDuckGo wraps URLs in a redirect — extract the real URL
|
|
let realUrl = rawUrl;
|
|
const uddgMatch = rawUrl.match(/[?&]uddg=([^&]+)/);
|
|
if (uddgMatch) {
|
|
realUrl = decodeURIComponent(uddgMatch[1]);
|
|
}
|
|
|
|
if (title && realUrl) {
|
|
results.push({ title, url: realUrl, snippet });
|
|
}
|
|
}
|
|
|
|
if (results.length === 0) return 'No search results found.';
|
|
|
|
// Collapse near-duplicate results before formatting: exact-URL dupes
|
|
// first, then snippets that are >=85% trigram-similar to a kept one.
|
|
// Search engines routinely surface the same passage from several URLs;
|
|
// sending all of them just spends tokens (TokenJuice subset).
|
|
const seenUrls = new Set<string>();
|
|
const urlUnique = results.filter((r) => {
|
|
if (seenUrls.has(r.url)) return false;
|
|
seenUrls.add(r.url);
|
|
return true;
|
|
});
|
|
const deduped = dedupTextResults(urlUnique, (r) => r.snippet);
|
|
|
|
const output = deduped
|
|
.map((r, i) => `[${i + 1}] ${r.title}\n ${r.url}\n ${r.snippet}`)
|
|
.join('\n\n');
|
|
|
|
// Cache successful results
|
|
searchCache.set(cacheKey, output);
|
|
|
|
return output;
|
|
} catch (err: unknown) {
|
|
return `Search error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 8. web_fetch — Fetch and extract text from a URL
|
|
{
|
|
name: 'web_fetch',
|
|
description: 'Fetch a web page and extract its text content. Use to read documentation, articles, or any web page.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
url: { type: 'string', description: 'URL to fetch' },
|
|
max_length: { type: 'number', description: 'Approx. max output (default 10000), enforced as a token budget (~max_length/4 tokens), so dense scripts like CJK return fewer characters than the number suggests' },
|
|
},
|
|
required: ['url'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const url = args.url as string;
|
|
const maxLength = (args.max_length as number) ?? 10_000;
|
|
// Token-budgeted cut (~4 chars/token) instead of a blunt char slice:
|
|
// grapheme-safe and consistent across prose/code/JSON/CJK.
|
|
const maxTokens = Math.max(1, Math.ceil(maxLength / 4));
|
|
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(url);
|
|
} catch {
|
|
return 'Error: Invalid URL';
|
|
}
|
|
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
return 'Error: Only http and https URLs are supported';
|
|
}
|
|
|
|
const ac = new AbortController();
|
|
const timer = setTimeout(() => ac.abort(), 15_000);
|
|
|
|
// SSRF guard: resolve + reject loopback/private/link-local/reserved
|
|
// targets, and re-validate every redirect hop. Loopback is fetchable
|
|
// only when WAGGLE_ALLOW_LOCAL_FETCH is set (desktop dev servers).
|
|
let response: Response;
|
|
try {
|
|
response = await safeFetch(
|
|
url,
|
|
{
|
|
headers: {
|
|
'User-Agent': 'Waggle/1.0 (AI Assistant)',
|
|
Accept: 'text/html,application/xhtml+xml,text/plain,application/json',
|
|
},
|
|
signal: ac.signal,
|
|
},
|
|
{ allowLocal: allowLocalFromEnv() },
|
|
);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
|
|
if (!response.ok) {
|
|
return `Fetch failed (${response.status}): ${response.statusText}`;
|
|
}
|
|
|
|
const contentType = response.headers.get('content-type') ?? '';
|
|
const body = await response.text();
|
|
|
|
// JSON — return formatted
|
|
if (contentType.includes('application/json')) {
|
|
try {
|
|
return truncateToTokenBudget(JSON.stringify(JSON.parse(body), null, 2), maxTokens);
|
|
} catch {
|
|
return truncateToTokenBudget(body, maxTokens);
|
|
}
|
|
}
|
|
|
|
// HTML — extract text
|
|
const text = extractWebPageText(body, url);
|
|
|
|
if (!text) return 'Page fetched but no text content found.';
|
|
return truncateToTokenBudget(text, maxTokens);
|
|
} catch (err: unknown) {
|
|
if (err instanceof EgressBlockedError) return `Error: blocked for safety — ${err.message}`;
|
|
if (err instanceof Error && err.name === 'AbortError') return 'Error: Request timed out (15s)';
|
|
return `Fetch error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 9. multi_edit — Atomic multi-file edits
|
|
{
|
|
name: 'multi_edit',
|
|
description: 'Apply multiple edits atomically across one or more files. If any edit fails validation, none are applied.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
edits: {
|
|
type: 'array',
|
|
description: 'Array of edits to apply',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
file_path: { type: 'string', description: 'File path relative to workspace' },
|
|
old_string: { type: 'string', description: 'Exact string to find (must appear exactly once per file)' },
|
|
new_string: { type: 'string', description: 'Replacement string' },
|
|
},
|
|
required: ['file_path', 'old_string', 'new_string'],
|
|
},
|
|
},
|
|
},
|
|
required: ['edits'],
|
|
},
|
|
execute: async (args) => {
|
|
try {
|
|
const edits = args.edits as Array<{ file_path: string; old_string: string; new_string: string }>;
|
|
|
|
if (!edits || edits.length === 0) {
|
|
return 'Error: No edits provided';
|
|
}
|
|
|
|
// Phase 1: Read all files and validate all old_strings exist exactly once.
|
|
// Route through fileBackend when present; otherwise fs. Key in the maps
|
|
// is the logical identifier (backend key, or resolved fs path).
|
|
const fileContents = new Map<string, string>();
|
|
const resolvedKeys = new Map<string, string>();
|
|
|
|
for (const edit of edits) {
|
|
const key = fileBackend
|
|
? resolveBackendKey(edit.file_path)
|
|
: resolveSafe(workspace, edit.file_path);
|
|
resolvedKeys.set(edit.file_path, key);
|
|
|
|
if (!fileContents.has(key)) {
|
|
const content = fileBackend
|
|
? (await fileBackend.read(key)).toString('utf-8')
|
|
: fs.readFileSync(key, 'utf-8');
|
|
fileContents.set(key, content);
|
|
}
|
|
|
|
const content = fileContents.get(key)!;
|
|
const occurrences = content.split(edit.old_string).length - 1;
|
|
|
|
if (occurrences === 0) {
|
|
return `Error: old_string not found in ${edit.file_path}. No edits applied (atomic rollback).`;
|
|
}
|
|
if (occurrences > 1) {
|
|
return `Error: old_string found multiple times (${occurrences}) in ${edit.file_path}. No edits applied (atomic rollback).`;
|
|
}
|
|
}
|
|
|
|
// Phase 2: Apply all edits (in memory first)
|
|
for (const edit of edits) {
|
|
const key = resolvedKeys.get(edit.file_path)!;
|
|
const content = fileContents.get(key)!;
|
|
fileContents.set(key, content.replace(edit.old_string, edit.new_string));
|
|
}
|
|
|
|
// Phase 3: Write all files
|
|
for (const [key, content] of fileContents) {
|
|
if (fileBackend) {
|
|
await fileBackend.write(key, Buffer.from(content, 'utf-8'));
|
|
} else {
|
|
fs.writeFileSync(key, content);
|
|
}
|
|
}
|
|
|
|
// Summarize
|
|
const fileCounts = new Map<string, number>();
|
|
for (const edit of edits) {
|
|
fileCounts.set(edit.file_path, (fileCounts.get(edit.file_path) ?? 0) + 1);
|
|
}
|
|
const summary = Array.from(fileCounts.entries())
|
|
.map(([file, count]) => ` ${file}: ${count} edit(s)`)
|
|
.join('\n');
|
|
|
|
return `Successfully applied ${edits.length} edit(s) across ${fileCounts.size} file(s):\n${summary}`;
|
|
} catch (err: unknown) {
|
|
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
},
|
|
},
|
|
|
|
// 10. get_task_output — Get output from a background task
|
|
{
|
|
name: 'get_task_output',
|
|
description: 'Get the current output and status of a background task started with run_in_background.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
task_id: { type: 'string', description: 'Task ID returned by run_in_background' },
|
|
},
|
|
required: ['task_id'],
|
|
},
|
|
execute: async (args) => {
|
|
const taskId = args.task_id as string;
|
|
const task = backgroundTasks.get(taskId);
|
|
|
|
if (!task) {
|
|
return `Error: No background task found with ID: ${taskId}`;
|
|
}
|
|
|
|
const parts: string[] = [
|
|
`Status: ${task.status}`,
|
|
];
|
|
|
|
if (task.exitCode !== undefined) {
|
|
parts.push(`Exit code: ${task.exitCode}`);
|
|
}
|
|
|
|
if (task.stdout) {
|
|
parts.push(`\n--- stdout ---\n${task.stdout}`);
|
|
}
|
|
|
|
if (task.stderr) {
|
|
parts.push(`\n--- stderr ---\n${task.stderr}`);
|
|
}
|
|
|
|
return parts.join('\n');
|
|
},
|
|
},
|
|
|
|
// 11. run_code — Execute code in an explicitly approved host child process
|
|
{
|
|
name: 'run_code',
|
|
description: 'Execute an explicitly approved code snippet in a host child process starting in the workspace directory. This is host-wide execution, not an OS sandbox. Supports JavaScript/TypeScript and Python (if installed).',
|
|
riskLevel: 'high',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
code: { type: 'string', description: 'The code to execute' },
|
|
language: { type: 'string', enum: ['javascript', 'typescript', 'python'], description: 'Programming language' },
|
|
timeout: { type: 'number', description: 'Timeout in milliseconds (default 10000, max 30000)' },
|
|
},
|
|
required: ['code', 'language'],
|
|
},
|
|
execute: async (args) => {
|
|
const code = args.code as string;
|
|
const language = args.language as string;
|
|
const rawTimeout = (args.timeout as number) ?? 10_000;
|
|
const timeout = Math.min(Math.max(rawTimeout, 1000), 30_000);
|
|
|
|
let executable: string;
|
|
let runtimeArgs: string[];
|
|
|
|
if (language === 'javascript' || language === 'typescript') {
|
|
executable = process.execPath;
|
|
runtimeArgs = language === 'typescript'
|
|
? ['--experimental-strip-types', '-e', code]
|
|
: ['-e', code];
|
|
} else if (language === 'python') {
|
|
executable = process.platform === 'win32' ? 'python' : 'python3';
|
|
runtimeArgs = ['-c', code];
|
|
} else {
|
|
return `Error: Unsupported language "${language}". Supported: javascript, typescript, python.`;
|
|
}
|
|
|
|
const sanitizedEnv = createSanitizedEnv();
|
|
|
|
const result = await execFileWithTreeTimeout(executable, runtimeArgs, {
|
|
cwd: workspace,
|
|
maxBuffer: MAX_OUTPUT_SIZE,
|
|
env: sanitizedEnv,
|
|
windowsHide: true,
|
|
}, timeout);
|
|
const parts: string[] = [];
|
|
|
|
if (result.timedOut) {
|
|
const cleanupWarning = result.cleanupDegraded
|
|
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
|
|
: '';
|
|
return `Error: Code execution timed out after ${timeout}ms.${cleanupWarning}`;
|
|
}
|
|
|
|
if (result.errorMessage) {
|
|
// Check for runtime not found
|
|
if (result.errorCode === 'ENOENT' || result.errorMessage.includes('not found')) {
|
|
return `Error: ${language} runtime not found. Please ensure ${language === 'python' ? 'python3/python' : 'node'} is installed and on PATH.`;
|
|
}
|
|
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
|
|
const cleanupWarning = result.cleanupDegraded
|
|
? ' Process-tree cleanup was incomplete; descendants may still be running.'
|
|
: '';
|
|
parts.push(`--- error ---\n${result.errorMessage}.${cleanupWarning}`);
|
|
}
|
|
}
|
|
|
|
if (result.stdout) parts.push(`--- stdout ---\n${truncateOutput(result.stdout)}`);
|
|
if (result.stderr) parts.push(`--- stderr ---\n${truncateOutput(result.stderr)}`);
|
|
|
|
if (parts.length === 0 && result.errorMessage) return `Error: ${result.errorMessage}`;
|
|
if (parts.length === 0) return '(no output)';
|
|
return parts.join('\n');
|
|
},
|
|
},
|
|
|
|
// 12. kill_task — Kill a background task
|
|
{
|
|
name: 'kill_task',
|
|
description: 'Kill a running background task.',
|
|
offlineCapable: true,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
task_id: { type: 'string', description: 'Task ID to kill' },
|
|
},
|
|
required: ['task_id'],
|
|
},
|
|
execute: async (args) => {
|
|
const taskId = args.task_id as string;
|
|
const task = backgroundTasks.get(taskId);
|
|
|
|
if (!task) {
|
|
return `Error: No background task found with ID: ${taskId}`;
|
|
}
|
|
|
|
if (task.status !== 'running') {
|
|
return `Task ${taskId} is already ${task.status}`;
|
|
}
|
|
|
|
terminateProcessTree(task.process);
|
|
task.status = 'killed';
|
|
return `Task ${taskId} has been killed`;
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
/** Expose internals for testing */
|
|
export { backgroundTasks, MAX_BACKGROUND_TASKS, STALE_TASK_THRESHOLD_MS };
|
|
|
|
/** Re-export helpers so existing consumers keep working */
|
|
export {
|
|
DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
|
|
checkDeniedBinaries, createSanitizedEnv, terminateProcessTree, truncateOutput,
|
|
resolveSafe,
|
|
} from './system-tools-helpers.js';
|