This commit is contained in:
Oleg Maslov
2026-09-02 10:14:22 +02:00
parent 0c3e2ead3b
commit b20b138fe4
771 changed files with 161561 additions and 9027 deletions

View File

@@ -9,7 +9,8 @@ import { dedupTextResults, truncateToTokenBudget } from './tool-output-compresso
import { safeFetch, allowLocalFromEnv, EgressBlockedError } from './url-egress-guard.js';
import {
IMAGE_EXTENSIONS, DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
checkDeniedBinaries, createSanitizedEnv, truncateOutput, resolveSafe,
checkDeniedBinaries, createSanitizedEnv, execFileWithTreeTimeout, terminateProcessTree,
truncateOutput, resolveSafe, assertNonSensitiveFilePath,
} from './system-tools-helpers.js';
/**
@@ -36,6 +37,43 @@ export interface SystemToolDeps {
/** Optional storage backend (team S3/MinIO). If present, file-content
* tools route through it instead of node:fs. */
fileBackend?: FileBackend;
/** Deny reads of well-known secret material for user-linked workspace roots. */
denySensitiveFiles?: boolean;
}
/** Prefer a repository README over GitHub navigation chrome for exact repo-root fetches. */
export function extractWebPageText(body: string, sourceUrl: string): string {
let content = body;
try {
const source = new URL(sourceUrl);
const pathSegments = source.pathname.split('/').filter(Boolean);
if (source.hostname.toLowerCase() === 'github.com' && pathSegments.length === 2) {
const readme = body.match(
/<article\b[^>]*class=(?:"[^"]*\bmarkdown-body\b[^"]*"|'[^']*\bmarkdown-body\b[^']*')[^>]*>([\s\S]*?)<\/article>/i,
)?.[1];
content = readme?.trim() ? readme : '';
}
} catch {
// URL validation happens in web_fetch; direct helper callers fall back to the full page.
}
return content
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
.replace(/<header[\s\S]*?<\/header>/gi, '')
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
.replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n')
.replace(/<[^>]+>/g, '')
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&nbsp;/g, ' ')
.replace(/[ \t]+/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
}
// Module-level instances — shared across all tool invocations
@@ -96,7 +134,7 @@ export function cleanupStaleTasks(): number {
export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefinition[] {
const deps: SystemToolDeps = typeof wsOrDeps === 'string' ? { workspace: wsOrDeps } : wsOrDeps;
const { workspace, fileBackend } = deps;
const { workspace, fileBackend, denySensitiveFiles = false } = deps;
// Normalize a user-supplied path to a backend key. The fs-level resolveSafe
// can't be used here because backend keys are virtual paths (e.g. S3 object
@@ -115,11 +153,46 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
return '/' + parts.join('/');
};
const resolveReadableBackendKey = (userPath: string): string => {
if (denySensitiveFiles) assertNonSensitiveFilePath(userPath);
return resolveBackendKey(userPath);
};
const resolveReadablePath = (userPath: string): string => resolveSafe(
workspace,
userPath,
{ denySensitiveFiles },
);
const filterReadableSearchPaths = (filePaths: string[]): string[] => {
if (!denySensitiveFiles) {
for (const filePath of filePaths) resolveSafe(workspace, filePath);
return filePaths;
}
return filePaths.filter((filePath) => {
try {
resolveReadablePath(filePath);
return true;
} catch {
// Search must not disclose sensitive or link-escaped filenames.
return false;
}
});
};
const validateWorkspaceGlob = (pattern: string): string => {
if (typeof pattern !== 'string' || !pattern) throw new Error('Invalid glob pattern');
if (path.isAbsolute(pattern)) throw new Error('Glob pattern must be relative to the workspace');
resolveSafe(workspace, pattern);
return pattern;
};
return [
// 1. bash — Execute shell commands
{
name: 'bash',
description: 'Execute a shell command in the workspace directory',
description: 'Execute an explicitly approved host shell command starting in the workspace directory. This is host-wide execution, not an OS sandbox.',
riskLevel: 'high',
offlineCapable: true,
parameters: {
type: 'object',
@@ -152,6 +225,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
cwd: workspace,
maxBuffer: 10 * 1024 * 1024,
env: sanitizedEnv,
windowsHide: true,
});
const task: BackgroundTask = {
@@ -188,30 +262,30 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
return `Background task started. Task ID: ${taskId}`;
}
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), timeout);
return new Promise<string>((resolve) => {
execFile(shell, shellArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
signal: ac.signal,
env: sanitizedEnv,
}, (error, stdout, stderr) => {
clearTimeout(timer);
if (error) {
if (error.code === 'ABORT_ERR') {
resolve(`Error: Command timeout after ${timeout}ms`);
return;
}
// Return stderr + stdout on non-zero exit (truncated)
const output = truncateOutput((stderr || '') + (stdout || ''));
resolve(output || `Error: ${error.message}`);
return;
}
resolve(truncateOutput(stdout));
});
});
const result = await execFileWithTreeTimeout(shell, shellArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
env: sanitizedEnv,
windowsHide: true,
}, timeout);
if (result.timedOut) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
: '';
return `Error: Command timeout after ${timeout}ms.${cleanupWarning}`;
}
if (result.errorMessage) {
// Return stderr + stdout on non-zero exit (truncated)
const output = truncateOutput((result.stderr || '') + (result.stdout || ''));
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup was incomplete; descendants may still be running.'
: '';
return `Error: ${result.errorMessage}.${cleanupWarning}${output ? `\n${output}` : ''}`;
}
return output || `Error: ${result.errorMessage}`;
}
return truncateOutput(result.stdout);
},
},
@@ -239,7 +313,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
// backend routing for text files; images/PDFs stay on local disk until
// Bucket 2 adds binary-stream support in the backend contract.
if (!fileBackend) {
const resolved = resolveSafe(workspace, filePath);
const resolved = resolveReadablePath(filePath);
if (IMAGE_EXTENSIONS.has(ext)) {
const stat = fs.statSync(resolved);
@@ -272,11 +346,11 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
if (ext === '.pdf') {
return `[PDF file: ${filePath}, backend-routed read does not yet extract PDF text. Download the file to inspect it.]`;
}
const key = resolveBackendKey(filePath);
const key = resolveReadableBackendKey(filePath);
const buf = await fileBackend.read(key);
content = buf.toString('utf-8');
} else {
const resolved = resolveSafe(workspace, filePath);
const resolved = resolveReadablePath(filePath);
content = fs.readFileSync(resolved, 'utf-8');
}
@@ -420,11 +494,13 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
},
execute: async (args) => {
try {
const matches = await glob(args.pattern as string, {
const pattern = validateWorkspaceGlob(args.pattern as string);
const globMatches = await glob(pattern, {
cwd: workspace,
ignore: ['node_modules/**', '.git/**'],
nodir: true,
});
const matches = filterReadableSearchPaths(globMatches);
if (matches.length === 0) return 'No files found.';
// A3: Cap file list to prevent token overflow
const MAX_FILE_RESULTS = 200;
@@ -472,21 +548,27 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
// If file_type is specified, override glob with extension-specific pattern
if (fileType) {
if (!/^[a-zA-Z0-9][a-zA-Z0-9.+-]*$/.test(fileType)) {
throw new Error('Invalid file_type extension');
}
filePattern = `**/*.${fileType}`;
}
const files = await glob(filePattern, {
validateWorkspaceGlob(filePattern);
const globMatches = await glob(filePattern, {
cwd: workspace,
ignore: ['node_modules/**', '.git/**'],
nodir: true,
});
const files = filterReadableSearchPaths(globMatches);
if (outputMode === 'files') {
// Return only file paths that contain matches
const matchingFiles: string[] = [];
for (const file of files) {
if (maxResults !== undefined && matchingFiles.length >= maxResults) break;
const absPath = path.join(workspace, file);
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
if (regex.test(content)) {
@@ -504,7 +586,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
// Return file paths with match counts
const counts: string[] = [];
for (const file of files) {
const absPath = path.join(workspace, file);
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
const lines = content.split('\n');
@@ -530,7 +612,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
for (const file of files) {
if (maxResults !== undefined && totalResults >= maxResults) break;
const absPath = path.join(workspace, file);
const absPath = resolveReadablePath(file);
try {
const content = fs.readFileSync(absPath, 'utf-8');
const lines = content.split('\n');
@@ -734,23 +816,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
}
// HTML — extract text
const text = body
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
.replace(/<header[\s\S]*?<\/header>/gi, '')
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
.replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n')
.replace(/<[^>]+>/g, '')
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&nbsp;/g, ' ')
.replace(/[ \t]+/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
const text = extractWebPageText(body, url);
if (!text) return 'Page fetched but no text content found.';
return truncateToTokenBudget(text, maxTokens);
@@ -896,10 +962,11 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
},
},
// 11. run_code — Execute code in a sandboxed environment
// 11. run_code — Execute code in an explicitly approved host child process
{
name: 'run_code',
description: 'Execute a code snippet in a sandboxed environment. Supports JavaScript/TypeScript and Python (if installed).',
description: 'Execute an explicitly approved code snippet in a host child process starting in the workspace directory. This is host-wide execution, not an OS sandbox. Supports JavaScript/TypeScript and Python (if installed).',
riskLevel: 'high',
offlineCapable: true,
parameters: {
type: 'object',
@@ -916,66 +983,57 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
const rawTimeout = (args.timeout as number) ?? 10_000;
const timeout = Math.min(Math.max(rawTimeout, 1000), 30_000);
// Build the command depending on language
let shell: string;
let shellArgs: string[];
const isWindows = process.platform === 'win32';
let executable: string;
let runtimeArgs: string[];
if (language === 'javascript' || language === 'typescript') {
// Use node -e for both JS and TS (TS runs as JS via node — for full TS, tsx would be needed)
shell = isWindows ? 'cmd.exe' : '/bin/sh';
const nodeCmd = `node -e ${JSON.stringify(code)}`;
shellArgs = isWindows ? ['/c', nodeCmd] : ['-c', nodeCmd];
executable = process.execPath;
runtimeArgs = language === 'typescript'
? ['--experimental-strip-types', '-e', code]
: ['-e', code];
} else if (language === 'python') {
shell = isWindows ? 'cmd.exe' : '/bin/sh';
// Try python3 first on Unix, python on Windows
const pythonBin = isWindows ? 'python' : 'python3';
const pyCmd = `${pythonBin} -c ${JSON.stringify(code)}`;
shellArgs = isWindows ? ['/c', pyCmd] : ['-c', pyCmd];
executable = process.platform === 'win32' ? 'python' : 'python3';
runtimeArgs = ['-c', code];
} else {
return `Error: Unsupported language "${language}". Supported: javascript, typescript, python.`;
}
const sanitizedEnv = createSanitizedEnv();
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), timeout);
return new Promise<string>((resolve) => {
execFile(shell, shellArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
signal: ac.signal,
env: sanitizedEnv,
}, (error, stdout, stderr) => {
clearTimeout(timer);
const parts: string[] = [];
const result = await execFileWithTreeTimeout(executable, runtimeArgs, {
cwd: workspace,
maxBuffer: MAX_OUTPUT_SIZE,
env: sanitizedEnv,
windowsHide: true,
}, timeout);
const parts: string[] = [];
if (error) {
if (error.code === 'ABORT_ERR') {
resolve(`Error: Code execution timed out after ${timeout}ms`);
return;
}
// Check for runtime not found
if (error.code === 'ENOENT' || (error.message && error.message.includes('not found'))) {
resolve(`Error: ${language} runtime not found. Please ensure ${language === 'python' ? 'python3/python' : 'node'} is installed and on PATH.`);
return;
}
}
if (result.timedOut) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup degraded to the root process; descendants may still be running.'
: '';
return `Error: Code execution timed out after ${timeout}ms.${cleanupWarning}`;
}
if (stdout) parts.push(`--- stdout ---\n${truncateOutput(stdout)}`);
if (stderr) parts.push(`--- stderr ---\n${truncateOutput(stderr)}`);
if (result.errorMessage) {
// Check for runtime not found
if (result.errorCode === 'ENOENT' || result.errorMessage.includes('not found')) {
return `Error: ${language} runtime not found. Please ensure ${language === 'python' ? 'python3/python' : 'node'} is installed and on PATH.`;
}
if (result.errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || result.cleanupDegraded) {
const cleanupWarning = result.cleanupDegraded
? ' Process-tree cleanup was incomplete; descendants may still be running.'
: '';
parts.push(`--- error ---\n${result.errorMessage}.${cleanupWarning}`);
}
}
if (parts.length === 0 && error) {
resolve(`Error: ${error.message}`);
return;
}
if (parts.length === 0) {
resolve('(no output)');
return;
}
resolve(parts.join('\n'));
});
});
if (result.stdout) parts.push(`--- stdout ---\n${truncateOutput(result.stdout)}`);
if (result.stderr) parts.push(`--- stderr ---\n${truncateOutput(result.stderr)}`);
if (parts.length === 0 && result.errorMessage) return `Error: ${result.errorMessage}`;
if (parts.length === 0) return '(no output)';
return parts.join('\n');
},
},
@@ -1003,7 +1061,7 @@ export function createSystemTools(wsOrDeps: string | SystemToolDeps): ToolDefini
return `Task ${taskId} is already ${task.status}`;
}
task.process.kill();
terminateProcessTree(task.process);
task.status = 'killed';
return `Task ${taskId} has been killed`;
},
@@ -1017,6 +1075,6 @@ export { backgroundTasks, MAX_BACKGROUND_TASKS, STALE_TASK_THRESHOLD_MS };
/** Re-export helpers so existing consumers keep working */
export {
DENIED_BINARIES, SENSITIVE_ENV_VARS, MAX_OUTPUT_SIZE,
checkDeniedBinaries, createSanitizedEnv, truncateOutput,
checkDeniedBinaries, createSanitizedEnv, terminateProcessTree, truncateOutput,
resolveSafe,
} from './system-tools-helpers.js';