test: remove the Windows external-agent lane

The lane drove real Claude Code, Codex, Hermes and OpenClaw installs
through Waggle Room runs, the coding-agent hooks and the hive-mind CLI,
none of which is in the community edition.

- scripts/test-windows-external-agents.ps1 and
  scripts/test-windows-official-auth-canaries.ps1 (the runners and their
  receipt sealing), scripts/verify-codex-tool-denial.mjs
- tests/e2e/launcher-real-hook-lifecycle.spec.ts and
  tests/e2e/launcher-real-tool-lifecycle.spec.ts (Windows-only, skipped
  unless the runner set them up)
- tests/integration/external-agent-collaboration.live.test.ts
- the tauri-config tests that pinned those scripts
This commit is contained in:
shumeiko.it
2026-11-01 11:13:35 +01:00
parent 25f95c584b
commit 6d187c8da7
7 changed files with 0 additions and 8672 deletions

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,221 +0,0 @@
import { expect, test } from '@playwright/test';
import fs from 'node:fs';
import path from 'node:path';
type HookEnvelope = {
ok: boolean;
action: 'install' | 'verify' | 'uninstall';
packageName: string;
stdout: string;
stderr: string;
code: number;
error?: string;
};
type HookToolCase = {
id: 'claude-code' | 'claude-desktop' | 'codex' | 'codex-desktop' | 'cursor' | 'hermes' | 'openclaw';
packageName: string;
configPath: string;
pointerPath: string;
cleanupDirs: string[];
precreateConfig?: string;
managedHookDir?: string;
};
const HOOK_TOOL_CASES: HookToolCase[] = [
{
id: 'claude-code',
packageName: '@waggle/hive-mind-hooks-claude-code',
configPath: path.join('.claude', 'settings.json'),
pointerPath: path.join('.claude', 'hive-mind-install.json'),
cleanupDirs: ['.claude'],
precreateConfig: '{}\n',
},
{
id: 'claude-desktop',
packageName: '@waggle/hive-mind-hooks-claude-desktop',
configPath: path.join('AppData', 'Roaming', 'Claude', 'claude_desktop_config.json'),
pointerPath: path.join('.waggle', 'claude-desktop', 'hive-mind-install.json'),
cleanupDirs: [path.join('AppData', 'Roaming', 'Claude'), path.join('.waggle', 'claude-desktop')],
precreateConfig: '{}\n',
},
{
id: 'codex',
packageName: '@waggle/hive-mind-hooks-codex',
configPath: path.join('.codex', 'hooks.json'),
pointerPath: path.join('.codex', 'hive-mind-install.json'),
cleanupDirs: ['.codex'],
},
{
id: 'codex-desktop',
packageName: '@waggle/hive-mind-hooks-codex-desktop',
configPath: path.join('.codex', 'hooks.json'),
pointerPath: path.join('.codex', 'hive-mind-install.json'),
cleanupDirs: ['.codex'],
},
{
id: 'cursor',
packageName: '@waggle/hive-mind-hooks-cursor',
configPath: path.join('.cursor', 'hooks.json'),
pointerPath: path.join('.cursor', 'hive-mind-install.json'),
cleanupDirs: ['.cursor'],
},
{
id: 'hermes',
packageName: '@waggle/hive-mind-hooks-hermes',
configPath: path.join('.hermes', 'config.yaml'),
pointerPath: path.join('.hermes', 'hive-mind-install.json'),
cleanupDirs: ['.hermes'],
},
{
id: 'openclaw',
packageName: '@waggle/hive-mind-hooks-openclaw',
configPath: path.join('.openclaw', 'openclaw.json'),
pointerPath: path.join('.openclaw', 'hive-mind-install.json'),
cleanupDirs: ['.openclaw'],
managedHookDir: path.join('.openclaw', 'hooks', 'hive-mind'),
},
];
function selectRequestedHookTools(tools: readonly HookToolCase[]): HookToolCase[] {
const raw = process.env.WAGGLE_E2E_HOST_IDS;
if (raw === undefined) return [...tools];
const rawIds = raw.split(',');
if (rawIds.some(id => id.trim().length === 0)) {
throw new Error('Invalid WAGGLE_E2E_HOST_IDS: empty host ID.');
}
const requestedIds = rawIds.map(id => id.trim());
const duplicateIds = requestedIds.filter(
(id, index) => requestedIds.indexOf(id) !== index,
);
if (duplicateIds.length > 0) {
throw new Error(`Duplicate WAGGLE_E2E_HOST_IDS: ${[...new Set(duplicateIds)].join(', ')}`);
}
const availableIds = new Set(tools.map(tool => tool.id));
const unknownIds = requestedIds.filter(
id => !availableIds.has(id as HookToolCase['id']),
);
if (unknownIds.length > 0) {
throw new Error(`Unknown WAGGLE_E2E_HOST_IDS: ${unknownIds.join(', ')}`);
}
const requested = new Set(requestedIds);
return tools.filter(tool => requested.has(tool.id));
}
function normalized(value: string): string {
return path.resolve(value).toLowerCase();
}
function inside(root: string, relativePath: string): string {
const candidate = path.resolve(root, relativePath);
const relative = path.relative(path.resolve(root), candidate);
if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) {
throw new Error(`Refusing path outside isolated hook profile: ${candidate}`);
}
return candidate;
}
function safeRemove(root: string, relativePath: string): void {
fs.rmSync(inside(root, relativePath), { recursive: true, force: true });
}
function assertIsolatedWindowsProfile(hookHome: string): void {
const guardedRoot = process.env.WAGGLE_E2E_TEMP_ROOT;
expect(guardedRoot, 'guarded runner must issue WAGGLE_E2E_TEMP_ROOT').toBeTruthy();
const relativeToRoot = path.relative(path.resolve(guardedRoot!), path.resolve(hookHome));
expect(relativeToRoot, 'hook profile must be a child of the guarded temp root').not.toMatch(/^\.\.|^[\\/]/);
expect(relativeToRoot, 'hook profile must not be the guarded temp root itself').not.toBe('');
expect(normalized(process.env.USERPROFILE ?? ''), 'isolated USERPROFILE').toBe(normalized(hookHome));
expect(normalized(process.env.HOME ?? ''), 'isolated HOME').toBe(normalized(hookHome));
expect(normalized(process.env.APPDATA ?? ''), 'isolated APPDATA').toBe(
normalized(path.join(hookHome, 'AppData', 'Roaming')),
);
expect(normalized(process.env.LOCALAPPDATA ?? ''), 'isolated LOCALAPPDATA').toBe(
normalized(path.join(hookHome, 'AppData', 'Local')),
);
}
test.describe('Launcher real Windows hook lifecycle', () => {
test('runs requested packaged hook routes with server-owned CLI wiring and reversible temp-profile cleanup', async ({ request }, testInfo) => {
test.setTimeout(240_000);
test.skip(process.platform !== 'win32', 'This real-host safety lane is Windows-specific.');
test.skip(
process.env.WAGGLE_E2E_REAL_HOOKS !== '1' || !process.env.WAGGLE_E2E_HOOK_HOME,
'Use scripts/test-windows-external-agents.ps1 to provide a throwaway Windows profile.',
);
const hookToolCases = selectRequestedHookTools(HOOK_TOOL_CASES);
const hookHome = path.resolve(process.env.WAGGLE_E2E_HOOK_HOME!);
assertIsolatedWindowsProfile(hookHome);
fs.mkdirSync(hookHome, { recursive: true });
const completed: Array<{ id: HookToolCase['id']; actions: string[]; packagedCli: string }> = [];
const postHook = async (tool: HookToolCase, action: HookEnvelope['action']) => {
const response = await request.post('/api/tools/hooks', {
data: { id: tool.id, action },
});
expect(response.status(), await response.text()).toBe(200);
const body = await response.json() as HookEnvelope;
expect(body, body.error ?? body.stderr).toMatchObject({
ok: true,
action,
packageName: tool.packageName,
code: 0,
});
return body;
};
try {
for (const tool of hookToolCases) {
const configPath = inside(hookHome, tool.configPath);
const pointerPath = inside(hookHome, tool.pointerPath);
const managedHookDir = tool.managedHookDir ? inside(hookHome, tool.managedHookDir) : null;
await test.step(`${tool.id} hook install -> verify -> uninstall`, async () => {
for (const cleanupDir of tool.cleanupDirs) safeRemove(hookHome, cleanupDir);
if (tool.precreateConfig !== undefined) {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(configPath, tool.precreateConfig, 'utf8');
}
const install = await postHook(tool, 'install');
expect(install.stdout).toContain('install');
expect(fs.existsSync(configPath)).toBe(true);
expect(fs.existsSync(pointerPath)).toBe(true);
if (managedHookDir) expect(fs.existsSync(managedHookDir)).toBe(true);
const pointer = JSON.parse(fs.readFileSync(pointerPath, 'utf8')) as { cli_path?: unknown };
expect(pointer.cli_path, 'route pins the packaged hive-mind CLI').toEqual(expect.any(String));
const packagedCli = path.resolve(String(pointer.cli_path));
expect(fs.existsSync(packagedCli), `packaged CLI exists: ${packagedCli}`).toBe(true);
expect(packagedCli.replace(/\\/g, '/')).toMatch(/hive-mind-cli\/dist\/index\.js$/);
const verify = await postHook(tool, 'verify');
expect(verify.stdout).toContain('All checks passed.');
const uninstall = await postHook(tool, 'uninstall');
expect(uninstall.stdout).toContain('uninstall');
expect(fs.existsSync(pointerPath)).toBe(false);
if (tool.precreateConfig !== undefined) {
expect(fs.readFileSync(configPath, 'utf8')).toBe(tool.precreateConfig);
} else {
expect(fs.existsSync(configPath)).toBe(false);
}
if (managedHookDir) expect(fs.existsSync(managedHookDir)).toBe(false);
completed.push({ id: tool.id, actions: ['install', 'verify', 'uninstall'], packagedCli });
});
}
await testInfo.attach('windows-hook-route-summary', {
body: Buffer.from(JSON.stringify({ hookHome, completed }, null, 2)),
contentType: 'application/json',
});
expect(completed.map(item => item.id)).toEqual(hookToolCases.map(tool => tool.id));
} finally {
for (const tool of hookToolCases) {
for (const cleanupDir of tool.cleanupDirs) safeRemove(hookHome, cleanupDir);
}
}
});
});

View File

@@ -1,357 +0,0 @@
import { expect, test, type APIRequestContext } from '@playwright/test';
import { SUPPORTED_TOOLS } from '@waggle/shared';
import { randomUUID } from 'node:crypto';
import path from 'node:path';
const SKIP_PARAMS = 'skipOnboarding=true&skipBoot=true&skipBriefing=true&tier=power';
type ToolId = typeof SUPPORTED_TOOLS[number];
const SAFE_VERSION_ARGS: Partial<Record<ToolId, string[]>> = {
'claude-code': ['--version'],
codex: ['--version'],
hermes: ['--version'],
openclaw: ['--version'],
};
const SECRET_ENV_NAMES = [
'ANTHROPIC_API_KEY',
'ANTHROPIC_AUTH_TOKEN',
'CLAUDE_CODE_OAUTH_TOKEN',
'OPENAI_API_KEY',
'OPENAI_ACCESS_TOKEN',
'OPENROUTER_API_KEY',
'GOOGLE_API_KEY',
'GEMINI_API_KEY',
'GROQ_API_KEY',
'XAI_API_KEY',
'MISTRAL_API_KEY',
'COHERE_API_KEY',
'DEEPSEEK_API_KEY',
'AZURE_OPENAI_API_KEY',
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'AWS_SESSION_TOKEN',
'AWS_PROFILE',
'AWS_CONFIG_FILE',
'AWS_SHARED_CREDENTIALS_FILE',
'GOOGLE_APPLICATION_CREDENTIALS',
'CLOUDSDK_CONFIG',
'AZURE_CONFIG_DIR',
'KUBECONFIG',
'DOCKER_CONFIG',
'DOCKER_HOST',
'GITHUB_TOKEN',
'GH_TOKEN',
'STRIPE_SECRET_KEY',
'DATABASE_URL',
'NPM_TOKEN',
'HF_TOKEN',
'HUGGING_FACE_HUB_TOKEN',
'RENDER_API_KEY',
'SSH_AUTH_SOCK',
'GIT_ASKPASS',
'SSH_ASKPASS',
'GIT_SSH_COMMAND',
'HTTP_PROXY',
'HTTPS_PROXY',
'ALL_PROXY',
'NODE_OPTIONS',
] as const;
function selectRequestedToolIds(tools: readonly ToolId[]): ToolId[] {
const raw = process.env.WAGGLE_E2E_HOST_IDS;
if (raw === undefined) return [...tools];
const rawIds = raw.split(',');
if (rawIds.some(id => id.trim().length === 0)) {
throw new Error('Invalid WAGGLE_E2E_HOST_IDS: empty host ID.');
}
const requestedIds = rawIds.map(id => id.trim());
const duplicateIds = requestedIds.filter(
(id, index) => requestedIds.indexOf(id) !== index,
);
if (duplicateIds.length > 0) {
throw new Error(`Duplicate WAGGLE_E2E_HOST_IDS: ${[...new Set(duplicateIds)].join(', ')}`);
}
const unknownIds = requestedIds.filter(id => !tools.includes(id as ToolId));
if (unknownIds.length > 0) {
throw new Error(`Unknown WAGGLE_E2E_HOST_IDS: ${unknownIds.join(', ')}`);
}
const requested = new Set(requestedIds);
return tools.filter(tool => requested.has(tool));
}
type DetectedTool = {
id: string;
displayName: string;
installed: boolean;
installedPath: string | null;
launchable?: boolean;
builtin?: boolean;
capabilities?: {
interactiveLaunch: boolean;
headlessTask: boolean;
};
};
type DetectionEnvelope = {
platform: string;
tools: DetectedTool[];
};
type LaunchEnvelope = {
ok: boolean;
pid: number | null;
roomId?: string;
runId?: string;
error?: string;
};
type WorkspaceEnvelope = {
id?: string;
name?: string;
storageType?: string;
};
type RouteResult = {
id: string;
displayName: string;
status: 'unavailable' | 'safe-version-exit' | 'interactive-only-rejected';
installedPath: string | null;
exitCode?: number | null;
output?: string;
route?: string;
};
function routeWithSkip(route: string): string {
const sep = route.includes('?') ? '&' : '?';
return `${route}${sep}${SKIP_PARAMS}`;
}
function assertTemporaryDataDir(): void {
const dataDir = process.env.WAGGLE_E2E_DATA_DIR;
const guardedRoot = process.env.WAGGLE_E2E_TEMP_ROOT;
expect(dataDir, 'WAGGLE_E2E_DATA_DIR must be explicitly isolated').toBeTruthy();
expect(guardedRoot, 'guarded runner must issue WAGGLE_E2E_TEMP_ROOT').toBeTruthy();
const relative = path.relative(path.resolve(guardedRoot!), path.resolve(dataDir!));
expect(relative, 'E2E data dir must be a child of the guarded temp root').not.toMatch(/^\.\.|^[\\/]/);
expect(relative, 'E2E data dir must not be the guarded temp root itself').not.toBe('');
}
async function createManagedWorkspace(request: APIRequestContext): Promise<WorkspaceEnvelope & { id: string }> {
const createResponse = await request.post('/api/workspaces', {
data: {
name: `Windows external-agent route ${randomUUID().slice(0, 8)}`,
group: 'external-agent-e2e',
icon: 'Terminal',
tone: 'technical',
storageType: 'virtual',
},
});
expect(createResponse.status(), await createResponse.text()).toBe(201);
const created = await createResponse.json() as WorkspaceEnvelope;
expect(created.id, 'managed workspace id from POST /api/workspaces').toMatch(/\S/);
const persistedResponse = await request.get(`/api/workspaces/${encodeURIComponent(created.id!)}`);
expect(persistedResponse.status(), await persistedResponse.text()).toBe(200);
const persisted = await persistedResponse.json() as WorkspaceEnvelope;
expect(persisted).toMatchObject({ id: created.id, name: created.name });
return { ...created, id: created.id! };
}
async function readObservedStream(
baseURL: string,
pid: number,
): Promise<{ lines: string[]; exitCode: number | null | undefined }> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30_000);
const lines: string[] = [];
let exitCode: number | null | undefined;
try {
const response = await fetch(new URL(`/api/tools/stream?pid=${pid}`, baseURL), {
signal: controller.signal,
});
expect(response.ok).toBe(true);
expect(response.body).toBeTruthy();
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let buffer = '';
while (exitCode === undefined) {
const chunk = await reader.read();
if (chunk.done) break;
buffer += decoder.decode(chunk.value, { stream: true }).replace(/\r\n/g, '\n');
let eventEnd = buffer.indexOf('\n\n');
while (eventEnd >= 0) {
const rawEvent = buffer.slice(0, eventEnd);
buffer = buffer.slice(eventEnd + 2);
eventEnd = buffer.indexOf('\n\n');
let event = 'message';
let data = '';
for (const line of rawEvent.split('\n')) {
if (line.startsWith('event:')) event = line.slice('event:'.length).trim();
if (line.startsWith('data:')) data += line.slice('data:'.length).trim();
}
if (event === 'line') lines.push((JSON.parse(data) as { line: string }).line);
if (event === 'exit') {
exitCode = (JSON.parse(data) as { code: number | null }).code;
break;
}
}
}
} finally {
clearTimeout(timeout);
}
return { lines, exitCode };
}
async function waitForProcessClear(request: APIRequestContext, pid: number): Promise<void> {
await expect.poll(async () => {
const processesResponse = await request.get('/api/tools/processes');
expect(processesResponse.ok()).toBe(true);
const body = await processesResponse.json() as { processes: Array<{ pid: number }> };
return body.processes.some(process => process.pid === pid);
}, { timeout: 10_000 }).toBe(false);
}
test.describe('Launcher real Windows supported-route lifecycle', () => {
test('covers requested built-in tools without credentials, unsafe GUI launch, or fabricated workspace ids', async ({ baseURL, page, request }, testInfo) => {
test.setTimeout(240_000);
test.skip(process.platform !== 'win32', 'This real-host route lane is Windows-specific.');
test.skip(
process.env.WAGGLE_E2E_REAL_TOOLS !== '1',
'Use scripts/test-windows-external-agents.ps1 to run the guarded real-tool lane.',
);
assertTemporaryDataDir();
expect(
process.env.WAGGLE_E2E_REUSE_EXISTING_SERVER,
'guarded runner must forbid reuse of a pre-existing app server',
).toBe('0');
expect(
SECRET_ENV_NAMES.filter(name => Boolean(process.env[name])),
'provider and cloud credentials must be scrubbed by the guarded runner',
).toEqual([]);
const toolIds = selectRequestedToolIds(SUPPORTED_TOOLS);
const root = baseURL ?? process.env.WAGGLE_E2E_BASE_URL ?? 'http://127.0.0.1:3333';
const detectionResponse = await request.get('/api/tools/detect');
expect(detectionResponse.ok()).toBe(true);
const detection = await detectionResponse.json() as DetectionEnvelope;
expect(detection.platform).toBe('win32');
expect(
detection.tools.filter(tool => tool.builtin === true).map(tool => tool.id),
).toEqual(SUPPORTED_TOOLS);
await page.goto(routeWithSkip('/launcher?watch=1'), { waitUntil: 'domcontentloaded' });
await expect(page.getByText('Tool Launcher', { exact: true })).toBeVisible({ timeout: 15_000 });
for (const tool of detection.tools) {
await expect(page.getByText(tool.displayName, { exact: true }).first()).toBeVisible({ timeout: 10_000 });
}
const workspace = await createManagedWorkspace(request);
const activePids = new Set<number>();
const results: RouteResult[] = [];
try {
for (const toolId of toolIds) {
const tool = detection.tools.find(candidate => candidate.id === toolId)!;
if (!tool.installed || !tool.installedPath) {
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'unavailable',
installedPath: null,
});
testInfo.annotations.push({ type: 'tool-unavailable', description: tool.displayName });
continue;
}
await test.step(`${tool.displayName} supported route`, async () => {
if (tool.capabilities?.headlessTask !== true) {
const unsupported = await request.post('/api/tools/run', {
data: {
toolId: tool.id,
workspaceIds: [workspace.id],
prompt: 'Do not run. This request must be rejected as interactive-only.',
timeoutMs: 10_000,
},
});
expect(unsupported.status(), await unsupported.text()).toBe(409);
expect(await unsupported.json()).toMatchObject({
error: 'TOOL_NOT_HEADLESS',
toolId: tool.id,
});
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'interactive-only-rejected',
installedPath: tool.installedPath,
route: '/api/tools/run',
});
return;
}
const args = SAFE_VERSION_ARGS[tool.id as ToolId];
expect(args, `${tool.displayName} must have an audited no-network version command`).toBeTruthy();
const launchResponse = await request.post('/api/tools/launch', {
data: {
id: tool.id,
args,
workspaceId: workspace.id,
observe: true,
},
});
expect(launchResponse.status(), await launchResponse.text()).toBe(202);
const launch = await launchResponse.json() as LaunchEnvelope;
expect(launch, launch.error).toMatchObject({ ok: true });
expect(launch.pid).toEqual(expect.any(Number));
expect(launch.roomId).toMatch(/\S/);
expect(launch.runId).toMatch(/\S/);
activePids.add(launch.pid!);
const stream = await readObservedStream(root, launch.pid!);
expect(stream.exitCode).toBe(0);
expect(stream.lines.join('\n').trim().length).toBeGreaterThan(0);
await waitForProcessClear(request, launch.pid!);
activePids.delete(launch.pid!);
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'safe-version-exit',
installedPath: tool.installedPath,
exitCode: stream.exitCode,
output: stream.lines.join('\n').trim(),
route: '/api/tools/launch',
});
});
}
expect(results.map(result => result.id)).toEqual(toolIds);
if (process.env.WAGGLE_E2E_HOST_IDS) {
expect(
results.filter(result => result.status === 'unavailable').map(result => result.id),
'every explicitly requested host must be installed and healthy',
).toEqual([]);
} else {
expect(results.some(result => result.status !== 'unavailable'), 'at least one real installed tool route').toBe(true);
}
} finally {
try {
await testInfo.attach('windows-external-tool-route-summary', {
body: Buffer.from(JSON.stringify({ workspace, results }, null, 2)),
contentType: 'application/json',
});
} finally {
for (const pid of activePids) {
await request.post('/api/tools/kill', { data: { pid } }).catch(() => null);
}
const deleteResponse = await request.delete(`/api/workspaces/${encodeURIComponent(workspace.id)}`).catch(() => null);
if (deleteResponse) expect([204, 404]).toContain(deleteResponse.status());
}
}
});
});

View File

@@ -1,344 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createHash, randomBytes } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { FastifyInstance } from 'fastify';
import { detectInstalledTools, getToolRegistry, runExternalTool } from '@waggle/agent';
import { scopeKey, workspaceScope } from '@waggle/memory-contract';
import { AgentRunRegistry } from '../../packages/server/src/local/agent-run-registry.js';
import { buildLocalServer } from '../../packages/server/src/local/index.js';
import { injectWithAuth } from '../../packages/server/tests/test-utils.js';
const LIVE = process.env.WAGGLE_LIVE_EXTERNAL_AGENTS === '1';
const describeLive = LIVE ? describe : describe.skip;
const REQUIRED_TOOLS = ['claude-code', 'codex', 'hermes'] as const;
async function waitFor(
predicate: () => boolean,
timeoutMs: number,
message: string,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 250));
}
throw new Error(message);
}
describeLive('live external-agent collaboration', () => {
let server: FastifyInstance;
let dataDir: string;
let sourceWorkspaceDir: string;
let sourceWorkspaceId: string;
let synthesisWorkspaceDir: string;
let synthesisWorkspaceId: string;
let expectedCanaryLine: string;
beforeAll(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-live-collab-'));
sourceWorkspaceDir = path.join(dataDir, 'source-workspace');
synthesisWorkspaceDir = path.join(dataDir, 'synthesis-workspace');
fs.mkdirSync(sourceWorkspaceDir, { recursive: true });
fs.mkdirSync(synthesisWorkspaceDir, { recursive: true });
expectedCanaryLine = `CANARY=${randomBytes(24).toString('hex')}`;
fs.writeFileSync(path.join(sourceWorkspaceDir, 'CANARY.txt'), `${expectedCanaryLine}\n`, 'utf8');
fs.writeFileSync(path.join(synthesisWorkspaceDir, 'SENTINEL.txt'), 'workspace must remain unchanged\n', 'utf8');
server = await buildLocalServer({ dataDir });
const hermesProvider = process.env.WAGGLE_LIVE_HERMES_PROVIDER?.trim();
const hermesModel = process.env.WAGGLE_LIVE_HERMES_MODEL?.trim();
if (Boolean(hermesProvider) !== Boolean(hermesModel)) {
throw new Error('Set both WAGGLE_LIVE_HERMES_PROVIDER and WAGGLE_LIVE_HERMES_MODEL');
}
server.decorate('externalToolRunner', (request) => {
const task = request.manifest.task;
if (request.manifest.id !== 'hermes' || !task || !hermesProvider || !hermesModel) {
return runExternalTool(request);
}
const override = ['--provider', hermesProvider, '-m', hermesModel];
return runExternalTool({
...request,
manifest: {
...request.manifest,
task: {
...task,
argvTemplate: [...task.argvTemplate, ...override],
...(task.resumeArgvTemplate
? { resumeArgvTemplate: [...task.resumeArgvTemplate, ...override] }
: {}),
},
},
});
});
const sourceWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Source',
group: 'live-test',
directory: sourceWorkspaceDir,
});
const synthesisWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Synthesis',
group: 'live-test',
directory: synthesisWorkspaceDir,
});
sourceWorkspaceId = sourceWorkspace.id;
synthesisWorkspaceId = synthesisWorkspace.id;
await server.listen({ host: '127.0.0.1', port: 0 });
}, 120_000);
afterAll(async () => {
if (server) await server.close();
if (dataDir) {
let lastError: unknown;
let removed = false;
for (let attempt = 0; attempt < 10; attempt++) {
try {
fs.rmSync(dataDir, { recursive: true, force: true });
removed = true;
break;
} catch (err) {
lastError = err;
await new Promise((resolve) => setTimeout(resolve, 250));
}
}
if (!removed) throw lastError;
}
}, 90_000);
it('runs the authenticated supported-agent cohort and delivers peer evidence through WaggleDance', async () => {
const detection = await detectInstalledTools();
const manifests = new Map(getToolRegistry().map((manifest) => [manifest.id, manifest]));
const unavailable = REQUIRED_TOOLS.filter((id) => !detection.tools.some(
(tool) => tool.id === id && tool.installed && tool.installedPath,
) || manifests.get(id)?.capabilities?.headlessTask !== true);
expect(unavailable, `Missing authenticated headless tools: ${unavailable.join(', ')}`).toEqual([]);
expect(server.workspaceManager.get(sourceWorkspaceId)?.directory).toBe(sourceWorkspaceDir);
expect(server.workspaceManager.get(synthesisWorkspaceId)?.directory).toBe(synthesisWorkspaceDir);
const sourceDigestBefore = workspaceDigest(sourceWorkspaceDir);
const synthesisDigestBefore = workspaceDigest(synthesisWorkspaceDir);
const response = await injectWithAuth(server, {
method: 'POST',
url: '/api/tools/run',
payload: {
prompt: [
'This is a read-only acceptance test. Do not modify any file.',
'Look only in the assigned workspace for ./CANARY.txt.',
'If it exists, copy its entire single line verbatim, including the literal CANARY= prefix, and do not add commentary.',
'A source-file answer must start with the literal CANARY= prefix followed by exactly 48 lowercase hexadecimal characters.',
'If it does not exist, return exactly NO_LOCAL_CANARY and never guess its content.',
'If peer findings are supplied in a later collaboration round, copy the complete matching CANARY= line from those findings verbatim as the source of truth.',
].join(' '),
participants: [
{ toolId: 'claude-code', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'codex', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'hermes', workspaceIds: [sourceWorkspaceId], access: 'native' },
{ toolId: 'hermes', workspaceIds: [synthesisWorkspaceId], access: 'native' },
],
timeoutMs: 300_000,
},
});
expect(response.statusCode).toBe(202);
const body = response.json() as {
roomId: string;
runs: Array<{ runId: string; toolId: string; workspaceId: string }>;
};
expect(body.runs).toHaveLength(5);
await waitFor(
() => {
return [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
['completed', 'failed', 'cancelled', 'interrupted'].includes(
server.agentRunRegistry.get(id)?.status ?? '',
));
},
390_000,
'live external-agent Room did not settle',
);
const terminalRoom = server.agentRunRegistry.get(body.roomId);
const terminalRuns = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
if (terminalRoom?.status === 'completed' && terminalRuns.every((run) => run.status === 'completed')) {
await waitFor(
() => [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
server.agentRunRegistry.get(id)?.memoryRefs.status === 'complete'),
30_000,
'live external-agent memory receipts did not settle',
);
}
const room = server.agentRunRegistry.get(body.roomId);
const runs = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
const diagnostic = JSON.stringify({
room: { status: room?.status, memoryStatus: room?.memoryRefs.status },
versions: Object.fromEntries(detection.tools
.filter((tool) => REQUIRED_TOOLS.includes(tool.id as typeof REQUIRED_TOOLS[number]))
.map((tool) => [tool.id, tool.version ?? null])),
runs: runs.map((run) => ({
tool: run.executor.toolId, status: run.status, exitCode: run.result?.exitCode,
memoryStatus: run.memoryRefs.status,
error: run.result?.error,
summary: run.result?.summary.slice(0, 500),
summaryHash: hashText(run.result?.summary ?? ''),
})),
});
expect(room?.status, diagnostic).toBe('completed');
expect(room?.memoryRefs.status, diagnostic).toBe('complete');
for (const run of runs) {
expect(run.status, diagnostic).toBe('completed');
expect(run.result?.exitCode, diagnostic).toBe(0);
expect(run.memoryRefs.status, diagnostic).toBe('complete');
expect(run.memoryRefs.personalFrameIds?.length).toBeGreaterThan(0);
expect(run.kind).toBe('worker');
if (run.kind === 'worker') {
expect(run.memoryRefs.workspaceFrameIds?.[run.workspaceId]?.length).toBeGreaterThan(0);
}
}
const claude = runs.find((run) => run.executor.toolId === 'claude-code');
const codex = runs.find((run) => run.executor.toolId === 'codex');
const hermes = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === sourceWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const hermesNoCanary = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === synthesisWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const synthesis = runs.find((run) => run.title.startsWith('WaggleDance synthesis'));
for (const run of [claude, codex, hermes]) {
expect(run?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
}
expect(hermesNoCanary?.result?.summary.trim(), diagnostic).toBe('NO_LOCAL_CANARY');
expect(synthesis?.kind).toBe('worker');
expect(synthesis?.executor.toolId).toBe('hermes');
if (synthesis?.kind === 'worker') expect(synthesis.workspaceId).toBe(synthesisWorkspaceId);
expect(synthesis?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
expect(workspaceDigest(sourceWorkspaceDir)).toBe(sourceDigestBefore);
expect(workspaceDigest(synthesisWorkspaceDir)).toBe(synthesisDigestBefore);
const durableRegistry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
for (const id of [body.roomId, ...body.runs.map(({ runId }) => runId)]) {
const durable = durableRegistry.get(id);
expect(durable?.status, diagnostic).toBe('completed');
expect(durable?.memoryRefs.status, diagnostic).toBe('complete');
expect(durable?.memoryRefs, diagnostic).toEqual(server.agentRunRegistry.get(id)?.memoryRefs);
}
for (const run of runs) {
const workspaceId = run.kind === 'worker' ? run.workspaceId : undefined;
expect(Object.keys(run.memoryRefs.workspaceFrameIds), diagnostic).toEqual([workspaceId]);
const personal = await Promise.all(
run.memoryRefs.personalFrameIds.map((id) => server.memory.get(id)),
);
const workspace = workspaceId
? await Promise.all(
run.memoryRefs.workspaceFrameIds[workspaceId].map((id) => server.memory.get(id)),
)
: [];
expect(personal.length, diagnostic).toBeGreaterThan(0);
expect(workspace.length, diagnostic).toBeGreaterThan(0);
for (const memory of personal) {
expect(memory?.scope.kind, diagnostic).toBe('personal');
}
for (const memory of workspace) {
expect(memory && scopeKey(memory.scope), diagnostic).toBe(scopeKey(workspaceScope(workspaceId!)));
}
for (const memory of [...personal, ...workspace]) {
expect(memory, diagnostic).not.toBeNull();
const metadata = memory?.metadata ?? {};
expect(metadata).toMatchObject({
runId: run.id,
roomId: run.roomId,
workspaceId: run.kind === 'worker' ? run.workspaceId : undefined,
toolId: run.executor.toolId,
});
expect(memory?.text.includes(run.id), diagnostic).toBe(true);
for (const forbiddenPath of [dataDir, sourceWorkspaceDir, synthesisWorkspaceDir]) {
expect(memory?.text.includes(forbiddenPath), diagnostic).toBe(false);
expect(JSON.stringify(metadata).includes(forbiddenPath), diagnostic).toBe(false);
}
}
const expectedSummary = run.result?.summary.trim() ?? '';
expect(personal.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
expect(workspace.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
}
expect(new Set(room?.memoryRefs.personalFrameIds)).toEqual(
new Set(runs.flatMap((run) => run.memoryRefs.personalFrameIds)),
);
expect(room?.memoryRefs.workspaceFrameIds).toEqual(Object.fromEntries(
[...new Set(runs.map((run) => run.workspaceId))].map((workspaceId) => [
workspaceId,
[...new Set(runs.flatMap((run) => run.memoryRefs.workspaceFrameIds[workspaceId] ?? []))],
]),
));
const roomSignals = server.signalBus?.query({ teamId: `room::${body.roomId}`, limit: 1_000 }) ?? [];
const subtypes = roomSignals.map((message) => message.subtype);
expect(subtypes).toEqual(expect.arrayContaining(['task_delegation', 'task_claim', 'routed_share', 'knowledge_match']));
for (const run of runs) {
const delegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === run.id);
const claim = roomSignals.find((message) =>
message.subtype === 'task_claim' && message.content.runId === run.id);
const share = roomSignals.find((message) =>
message.subtype === 'routed_share' && message.content.runId === run.id
&& message.content.phase === 'completed');
expect(delegation, diagnostic).toBeDefined();
expect(claim?.referenceId, diagnostic).toBe(delegation?.id);
expect(share?.referenceId, diagnostic).toBe(delegation?.id);
}
const synthesisDelegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === synthesis?.id);
const peerDelivery = roomSignals.find((message) =>
message.subtype === 'knowledge_match' && message.content.runId === synthesis?.id);
expect(peerDelivery?.referenceId, diagnostic).toBe(synthesisDelegation?.id);
expect(peerDelivery?.content.tool, diagnostic).toBe('hermes');
expect(peerDelivery?.content.peerFindings).toEqual(expect.arrayContaining([
expect.stringContaining(expectedCanaryLine),
expect.stringContaining('NO_LOCAL_CANARY'),
]));
const initialRunIds = new Set(runs.filter((run) => run.id !== synthesis?.id).map((run) => run.id));
const expectedSourceMessageIds = new Set(roomSignals.filter((message) =>
message.subtype === 'routed_share'
&& initialRunIds.has(String(message.content.runId))
&& message.content.phase === 'completed').map((message) => message.id));
expect(new Set(peerDelivery?.content.sourceMessageIds as string[] | undefined)).toEqual(expectedSourceMessageIds);
expect(new Set(synthesisDelegation?.content.sourceRunIds as string[] | undefined)).toEqual(initialRunIds);
const peerFindings = peerDelivery?.content.peerFindings as string[] | undefined;
expect(peerFindings, diagnostic).toHaveLength(initialRunIds.size);
for (const runId of initialRunIds) {
expect(peerFindings?.filter((finding) => finding.includes(`· run ${runId}]`)), diagnostic).toHaveLength(1);
}
expect(new Set(roomSignals.map((message) => message.content.runId))).toEqual(
new Set(body.runs.map((run) => run.runId)),
);
}, 600_000);
});
function hashText(value: string): string {
return createHash('sha256').update(value).digest('hex');
}
function workspaceDigest(root: string): string {
const hash = createHash('sha256');
const visit = (directory: string, relativeDirectory: string): void => {
const entries = fs.readdirSync(directory, { withFileTypes: true })
.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const relative = path.join(relativeDirectory, entry.name).replaceAll('\\', '/');
const absolute = path.join(directory, entry.name);
if (entry.isDirectory()) {
hash.update(`D\0${relative}\0`);
visit(absolute, relative);
} else if (entry.isFile()) {
hash.update(`F\0${relative}\0`);
hash.update(fs.readFileSync(absolute));
hash.update('\0');
} else if (entry.isSymbolicLink()) {
hash.update(`L\0${relative}\0${fs.readlinkSync(absolute)}\0`);
} else {
hash.update(`X\0${relative}\0`);
}
}
};
visit(root, '');
return hash.digest('hex');
}