test: remove the Windows external-agent lane

The lane drove real Claude Code, Codex, Hermes and OpenClaw installs
through Waggle Room runs, the coding-agent hooks and the hive-mind CLI,
none of which is in the community edition.

- scripts/test-windows-external-agents.ps1 and
  scripts/test-windows-official-auth-canaries.ps1 (the runners and their
  receipt sealing), scripts/verify-codex-tool-denial.mjs
- tests/e2e/launcher-real-hook-lifecycle.spec.ts and
  tests/e2e/launcher-real-tool-lifecycle.spec.ts (Windows-only, skipped
  unless the runner set them up)
- tests/integration/external-agent-collaboration.live.test.ts
- the tauri-config tests that pinned those scripts
This commit is contained in:
shumeiko.it
2026-11-01 11:13:35 +01:00
parent 25f95c584b
commit 6d187c8da7
7 changed files with 0 additions and 8672 deletions

View File

@@ -1,221 +0,0 @@
import { expect, test } from '@playwright/test';
import fs from 'node:fs';
import path from 'node:path';
type HookEnvelope = {
ok: boolean;
action: 'install' | 'verify' | 'uninstall';
packageName: string;
stdout: string;
stderr: string;
code: number;
error?: string;
};
type HookToolCase = {
id: 'claude-code' | 'claude-desktop' | 'codex' | 'codex-desktop' | 'cursor' | 'hermes' | 'openclaw';
packageName: string;
configPath: string;
pointerPath: string;
cleanupDirs: string[];
precreateConfig?: string;
managedHookDir?: string;
};
const HOOK_TOOL_CASES: HookToolCase[] = [
{
id: 'claude-code',
packageName: '@waggle/hive-mind-hooks-claude-code',
configPath: path.join('.claude', 'settings.json'),
pointerPath: path.join('.claude', 'hive-mind-install.json'),
cleanupDirs: ['.claude'],
precreateConfig: '{}\n',
},
{
id: 'claude-desktop',
packageName: '@waggle/hive-mind-hooks-claude-desktop',
configPath: path.join('AppData', 'Roaming', 'Claude', 'claude_desktop_config.json'),
pointerPath: path.join('.waggle', 'claude-desktop', 'hive-mind-install.json'),
cleanupDirs: [path.join('AppData', 'Roaming', 'Claude'), path.join('.waggle', 'claude-desktop')],
precreateConfig: '{}\n',
},
{
id: 'codex',
packageName: '@waggle/hive-mind-hooks-codex',
configPath: path.join('.codex', 'hooks.json'),
pointerPath: path.join('.codex', 'hive-mind-install.json'),
cleanupDirs: ['.codex'],
},
{
id: 'codex-desktop',
packageName: '@waggle/hive-mind-hooks-codex-desktop',
configPath: path.join('.codex', 'hooks.json'),
pointerPath: path.join('.codex', 'hive-mind-install.json'),
cleanupDirs: ['.codex'],
},
{
id: 'cursor',
packageName: '@waggle/hive-mind-hooks-cursor',
configPath: path.join('.cursor', 'hooks.json'),
pointerPath: path.join('.cursor', 'hive-mind-install.json'),
cleanupDirs: ['.cursor'],
},
{
id: 'hermes',
packageName: '@waggle/hive-mind-hooks-hermes',
configPath: path.join('.hermes', 'config.yaml'),
pointerPath: path.join('.hermes', 'hive-mind-install.json'),
cleanupDirs: ['.hermes'],
},
{
id: 'openclaw',
packageName: '@waggle/hive-mind-hooks-openclaw',
configPath: path.join('.openclaw', 'openclaw.json'),
pointerPath: path.join('.openclaw', 'hive-mind-install.json'),
cleanupDirs: ['.openclaw'],
managedHookDir: path.join('.openclaw', 'hooks', 'hive-mind'),
},
];
function selectRequestedHookTools(tools: readonly HookToolCase[]): HookToolCase[] {
const raw = process.env.WAGGLE_E2E_HOST_IDS;
if (raw === undefined) return [...tools];
const rawIds = raw.split(',');
if (rawIds.some(id => id.trim().length === 0)) {
throw new Error('Invalid WAGGLE_E2E_HOST_IDS: empty host ID.');
}
const requestedIds = rawIds.map(id => id.trim());
const duplicateIds = requestedIds.filter(
(id, index) => requestedIds.indexOf(id) !== index,
);
if (duplicateIds.length > 0) {
throw new Error(`Duplicate WAGGLE_E2E_HOST_IDS: ${[...new Set(duplicateIds)].join(', ')}`);
}
const availableIds = new Set(tools.map(tool => tool.id));
const unknownIds = requestedIds.filter(
id => !availableIds.has(id as HookToolCase['id']),
);
if (unknownIds.length > 0) {
throw new Error(`Unknown WAGGLE_E2E_HOST_IDS: ${unknownIds.join(', ')}`);
}
const requested = new Set(requestedIds);
return tools.filter(tool => requested.has(tool.id));
}
function normalized(value: string): string {
return path.resolve(value).toLowerCase();
}
function inside(root: string, relativePath: string): string {
const candidate = path.resolve(root, relativePath);
const relative = path.relative(path.resolve(root), candidate);
if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) {
throw new Error(`Refusing path outside isolated hook profile: ${candidate}`);
}
return candidate;
}
function safeRemove(root: string, relativePath: string): void {
fs.rmSync(inside(root, relativePath), { recursive: true, force: true });
}
function assertIsolatedWindowsProfile(hookHome: string): void {
const guardedRoot = process.env.WAGGLE_E2E_TEMP_ROOT;
expect(guardedRoot, 'guarded runner must issue WAGGLE_E2E_TEMP_ROOT').toBeTruthy();
const relativeToRoot = path.relative(path.resolve(guardedRoot!), path.resolve(hookHome));
expect(relativeToRoot, 'hook profile must be a child of the guarded temp root').not.toMatch(/^\.\.|^[\\/]/);
expect(relativeToRoot, 'hook profile must not be the guarded temp root itself').not.toBe('');
expect(normalized(process.env.USERPROFILE ?? ''), 'isolated USERPROFILE').toBe(normalized(hookHome));
expect(normalized(process.env.HOME ?? ''), 'isolated HOME').toBe(normalized(hookHome));
expect(normalized(process.env.APPDATA ?? ''), 'isolated APPDATA').toBe(
normalized(path.join(hookHome, 'AppData', 'Roaming')),
);
expect(normalized(process.env.LOCALAPPDATA ?? ''), 'isolated LOCALAPPDATA').toBe(
normalized(path.join(hookHome, 'AppData', 'Local')),
);
}
test.describe('Launcher real Windows hook lifecycle', () => {
test('runs requested packaged hook routes with server-owned CLI wiring and reversible temp-profile cleanup', async ({ request }, testInfo) => {
test.setTimeout(240_000);
test.skip(process.platform !== 'win32', 'This real-host safety lane is Windows-specific.');
test.skip(
process.env.WAGGLE_E2E_REAL_HOOKS !== '1' || !process.env.WAGGLE_E2E_HOOK_HOME,
'Use scripts/test-windows-external-agents.ps1 to provide a throwaway Windows profile.',
);
const hookToolCases = selectRequestedHookTools(HOOK_TOOL_CASES);
const hookHome = path.resolve(process.env.WAGGLE_E2E_HOOK_HOME!);
assertIsolatedWindowsProfile(hookHome);
fs.mkdirSync(hookHome, { recursive: true });
const completed: Array<{ id: HookToolCase['id']; actions: string[]; packagedCli: string }> = [];
const postHook = async (tool: HookToolCase, action: HookEnvelope['action']) => {
const response = await request.post('/api/tools/hooks', {
data: { id: tool.id, action },
});
expect(response.status(), await response.text()).toBe(200);
const body = await response.json() as HookEnvelope;
expect(body, body.error ?? body.stderr).toMatchObject({
ok: true,
action,
packageName: tool.packageName,
code: 0,
});
return body;
};
try {
for (const tool of hookToolCases) {
const configPath = inside(hookHome, tool.configPath);
const pointerPath = inside(hookHome, tool.pointerPath);
const managedHookDir = tool.managedHookDir ? inside(hookHome, tool.managedHookDir) : null;
await test.step(`${tool.id} hook install -> verify -> uninstall`, async () => {
for (const cleanupDir of tool.cleanupDirs) safeRemove(hookHome, cleanupDir);
if (tool.precreateConfig !== undefined) {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(configPath, tool.precreateConfig, 'utf8');
}
const install = await postHook(tool, 'install');
expect(install.stdout).toContain('install');
expect(fs.existsSync(configPath)).toBe(true);
expect(fs.existsSync(pointerPath)).toBe(true);
if (managedHookDir) expect(fs.existsSync(managedHookDir)).toBe(true);
const pointer = JSON.parse(fs.readFileSync(pointerPath, 'utf8')) as { cli_path?: unknown };
expect(pointer.cli_path, 'route pins the packaged hive-mind CLI').toEqual(expect.any(String));
const packagedCli = path.resolve(String(pointer.cli_path));
expect(fs.existsSync(packagedCli), `packaged CLI exists: ${packagedCli}`).toBe(true);
expect(packagedCli.replace(/\\/g, '/')).toMatch(/hive-mind-cli\/dist\/index\.js$/);
const verify = await postHook(tool, 'verify');
expect(verify.stdout).toContain('All checks passed.');
const uninstall = await postHook(tool, 'uninstall');
expect(uninstall.stdout).toContain('uninstall');
expect(fs.existsSync(pointerPath)).toBe(false);
if (tool.precreateConfig !== undefined) {
expect(fs.readFileSync(configPath, 'utf8')).toBe(tool.precreateConfig);
} else {
expect(fs.existsSync(configPath)).toBe(false);
}
if (managedHookDir) expect(fs.existsSync(managedHookDir)).toBe(false);
completed.push({ id: tool.id, actions: ['install', 'verify', 'uninstall'], packagedCli });
});
}
await testInfo.attach('windows-hook-route-summary', {
body: Buffer.from(JSON.stringify({ hookHome, completed }, null, 2)),
contentType: 'application/json',
});
expect(completed.map(item => item.id)).toEqual(hookToolCases.map(tool => tool.id));
} finally {
for (const tool of hookToolCases) {
for (const cleanupDir of tool.cleanupDirs) safeRemove(hookHome, cleanupDir);
}
}
});
});

View File

@@ -1,357 +0,0 @@
import { expect, test, type APIRequestContext } from '@playwright/test';
import { SUPPORTED_TOOLS } from '@waggle/shared';
import { randomUUID } from 'node:crypto';
import path from 'node:path';
const SKIP_PARAMS = 'skipOnboarding=true&skipBoot=true&skipBriefing=true&tier=power';
type ToolId = typeof SUPPORTED_TOOLS[number];
const SAFE_VERSION_ARGS: Partial<Record<ToolId, string[]>> = {
'claude-code': ['--version'],
codex: ['--version'],
hermes: ['--version'],
openclaw: ['--version'],
};
const SECRET_ENV_NAMES = [
'ANTHROPIC_API_KEY',
'ANTHROPIC_AUTH_TOKEN',
'CLAUDE_CODE_OAUTH_TOKEN',
'OPENAI_API_KEY',
'OPENAI_ACCESS_TOKEN',
'OPENROUTER_API_KEY',
'GOOGLE_API_KEY',
'GEMINI_API_KEY',
'GROQ_API_KEY',
'XAI_API_KEY',
'MISTRAL_API_KEY',
'COHERE_API_KEY',
'DEEPSEEK_API_KEY',
'AZURE_OPENAI_API_KEY',
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'AWS_SESSION_TOKEN',
'AWS_PROFILE',
'AWS_CONFIG_FILE',
'AWS_SHARED_CREDENTIALS_FILE',
'GOOGLE_APPLICATION_CREDENTIALS',
'CLOUDSDK_CONFIG',
'AZURE_CONFIG_DIR',
'KUBECONFIG',
'DOCKER_CONFIG',
'DOCKER_HOST',
'GITHUB_TOKEN',
'GH_TOKEN',
'STRIPE_SECRET_KEY',
'DATABASE_URL',
'NPM_TOKEN',
'HF_TOKEN',
'HUGGING_FACE_HUB_TOKEN',
'RENDER_API_KEY',
'SSH_AUTH_SOCK',
'GIT_ASKPASS',
'SSH_ASKPASS',
'GIT_SSH_COMMAND',
'HTTP_PROXY',
'HTTPS_PROXY',
'ALL_PROXY',
'NODE_OPTIONS',
] as const;
function selectRequestedToolIds(tools: readonly ToolId[]): ToolId[] {
const raw = process.env.WAGGLE_E2E_HOST_IDS;
if (raw === undefined) return [...tools];
const rawIds = raw.split(',');
if (rawIds.some(id => id.trim().length === 0)) {
throw new Error('Invalid WAGGLE_E2E_HOST_IDS: empty host ID.');
}
const requestedIds = rawIds.map(id => id.trim());
const duplicateIds = requestedIds.filter(
(id, index) => requestedIds.indexOf(id) !== index,
);
if (duplicateIds.length > 0) {
throw new Error(`Duplicate WAGGLE_E2E_HOST_IDS: ${[...new Set(duplicateIds)].join(', ')}`);
}
const unknownIds = requestedIds.filter(id => !tools.includes(id as ToolId));
if (unknownIds.length > 0) {
throw new Error(`Unknown WAGGLE_E2E_HOST_IDS: ${unknownIds.join(', ')}`);
}
const requested = new Set(requestedIds);
return tools.filter(tool => requested.has(tool));
}
type DetectedTool = {
id: string;
displayName: string;
installed: boolean;
installedPath: string | null;
launchable?: boolean;
builtin?: boolean;
capabilities?: {
interactiveLaunch: boolean;
headlessTask: boolean;
};
};
type DetectionEnvelope = {
platform: string;
tools: DetectedTool[];
};
type LaunchEnvelope = {
ok: boolean;
pid: number | null;
roomId?: string;
runId?: string;
error?: string;
};
type WorkspaceEnvelope = {
id?: string;
name?: string;
storageType?: string;
};
type RouteResult = {
id: string;
displayName: string;
status: 'unavailable' | 'safe-version-exit' | 'interactive-only-rejected';
installedPath: string | null;
exitCode?: number | null;
output?: string;
route?: string;
};
function routeWithSkip(route: string): string {
const sep = route.includes('?') ? '&' : '?';
return `${route}${sep}${SKIP_PARAMS}`;
}
function assertTemporaryDataDir(): void {
const dataDir = process.env.WAGGLE_E2E_DATA_DIR;
const guardedRoot = process.env.WAGGLE_E2E_TEMP_ROOT;
expect(dataDir, 'WAGGLE_E2E_DATA_DIR must be explicitly isolated').toBeTruthy();
expect(guardedRoot, 'guarded runner must issue WAGGLE_E2E_TEMP_ROOT').toBeTruthy();
const relative = path.relative(path.resolve(guardedRoot!), path.resolve(dataDir!));
expect(relative, 'E2E data dir must be a child of the guarded temp root').not.toMatch(/^\.\.|^[\\/]/);
expect(relative, 'E2E data dir must not be the guarded temp root itself').not.toBe('');
}
async function createManagedWorkspace(request: APIRequestContext): Promise<WorkspaceEnvelope & { id: string }> {
const createResponse = await request.post('/api/workspaces', {
data: {
name: `Windows external-agent route ${randomUUID().slice(0, 8)}`,
group: 'external-agent-e2e',
icon: 'Terminal',
tone: 'technical',
storageType: 'virtual',
},
});
expect(createResponse.status(), await createResponse.text()).toBe(201);
const created = await createResponse.json() as WorkspaceEnvelope;
expect(created.id, 'managed workspace id from POST /api/workspaces').toMatch(/\S/);
const persistedResponse = await request.get(`/api/workspaces/${encodeURIComponent(created.id!)}`);
expect(persistedResponse.status(), await persistedResponse.text()).toBe(200);
const persisted = await persistedResponse.json() as WorkspaceEnvelope;
expect(persisted).toMatchObject({ id: created.id, name: created.name });
return { ...created, id: created.id! };
}
async function readObservedStream(
baseURL: string,
pid: number,
): Promise<{ lines: string[]; exitCode: number | null | undefined }> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30_000);
const lines: string[] = [];
let exitCode: number | null | undefined;
try {
const response = await fetch(new URL(`/api/tools/stream?pid=${pid}`, baseURL), {
signal: controller.signal,
});
expect(response.ok).toBe(true);
expect(response.body).toBeTruthy();
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let buffer = '';
while (exitCode === undefined) {
const chunk = await reader.read();
if (chunk.done) break;
buffer += decoder.decode(chunk.value, { stream: true }).replace(/\r\n/g, '\n');
let eventEnd = buffer.indexOf('\n\n');
while (eventEnd >= 0) {
const rawEvent = buffer.slice(0, eventEnd);
buffer = buffer.slice(eventEnd + 2);
eventEnd = buffer.indexOf('\n\n');
let event = 'message';
let data = '';
for (const line of rawEvent.split('\n')) {
if (line.startsWith('event:')) event = line.slice('event:'.length).trim();
if (line.startsWith('data:')) data += line.slice('data:'.length).trim();
}
if (event === 'line') lines.push((JSON.parse(data) as { line: string }).line);
if (event === 'exit') {
exitCode = (JSON.parse(data) as { code: number | null }).code;
break;
}
}
}
} finally {
clearTimeout(timeout);
}
return { lines, exitCode };
}
async function waitForProcessClear(request: APIRequestContext, pid: number): Promise<void> {
await expect.poll(async () => {
const processesResponse = await request.get('/api/tools/processes');
expect(processesResponse.ok()).toBe(true);
const body = await processesResponse.json() as { processes: Array<{ pid: number }> };
return body.processes.some(process => process.pid === pid);
}, { timeout: 10_000 }).toBe(false);
}
test.describe('Launcher real Windows supported-route lifecycle', () => {
test('covers requested built-in tools without credentials, unsafe GUI launch, or fabricated workspace ids', async ({ baseURL, page, request }, testInfo) => {
test.setTimeout(240_000);
test.skip(process.platform !== 'win32', 'This real-host route lane is Windows-specific.');
test.skip(
process.env.WAGGLE_E2E_REAL_TOOLS !== '1',
'Use scripts/test-windows-external-agents.ps1 to run the guarded real-tool lane.',
);
assertTemporaryDataDir();
expect(
process.env.WAGGLE_E2E_REUSE_EXISTING_SERVER,
'guarded runner must forbid reuse of a pre-existing app server',
).toBe('0');
expect(
SECRET_ENV_NAMES.filter(name => Boolean(process.env[name])),
'provider and cloud credentials must be scrubbed by the guarded runner',
).toEqual([]);
const toolIds = selectRequestedToolIds(SUPPORTED_TOOLS);
const root = baseURL ?? process.env.WAGGLE_E2E_BASE_URL ?? 'http://127.0.0.1:3333';
const detectionResponse = await request.get('/api/tools/detect');
expect(detectionResponse.ok()).toBe(true);
const detection = await detectionResponse.json() as DetectionEnvelope;
expect(detection.platform).toBe('win32');
expect(
detection.tools.filter(tool => tool.builtin === true).map(tool => tool.id),
).toEqual(SUPPORTED_TOOLS);
await page.goto(routeWithSkip('/launcher?watch=1'), { waitUntil: 'domcontentloaded' });
await expect(page.getByText('Tool Launcher', { exact: true })).toBeVisible({ timeout: 15_000 });
for (const tool of detection.tools) {
await expect(page.getByText(tool.displayName, { exact: true }).first()).toBeVisible({ timeout: 10_000 });
}
const workspace = await createManagedWorkspace(request);
const activePids = new Set<number>();
const results: RouteResult[] = [];
try {
for (const toolId of toolIds) {
const tool = detection.tools.find(candidate => candidate.id === toolId)!;
if (!tool.installed || !tool.installedPath) {
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'unavailable',
installedPath: null,
});
testInfo.annotations.push({ type: 'tool-unavailable', description: tool.displayName });
continue;
}
await test.step(`${tool.displayName} supported route`, async () => {
if (tool.capabilities?.headlessTask !== true) {
const unsupported = await request.post('/api/tools/run', {
data: {
toolId: tool.id,
workspaceIds: [workspace.id],
prompt: 'Do not run. This request must be rejected as interactive-only.',
timeoutMs: 10_000,
},
});
expect(unsupported.status(), await unsupported.text()).toBe(409);
expect(await unsupported.json()).toMatchObject({
error: 'TOOL_NOT_HEADLESS',
toolId: tool.id,
});
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'interactive-only-rejected',
installedPath: tool.installedPath,
route: '/api/tools/run',
});
return;
}
const args = SAFE_VERSION_ARGS[tool.id as ToolId];
expect(args, `${tool.displayName} must have an audited no-network version command`).toBeTruthy();
const launchResponse = await request.post('/api/tools/launch', {
data: {
id: tool.id,
args,
workspaceId: workspace.id,
observe: true,
},
});
expect(launchResponse.status(), await launchResponse.text()).toBe(202);
const launch = await launchResponse.json() as LaunchEnvelope;
expect(launch, launch.error).toMatchObject({ ok: true });
expect(launch.pid).toEqual(expect.any(Number));
expect(launch.roomId).toMatch(/\S/);
expect(launch.runId).toMatch(/\S/);
activePids.add(launch.pid!);
const stream = await readObservedStream(root, launch.pid!);
expect(stream.exitCode).toBe(0);
expect(stream.lines.join('\n').trim().length).toBeGreaterThan(0);
await waitForProcessClear(request, launch.pid!);
activePids.delete(launch.pid!);
results.push({
id: tool.id,
displayName: tool.displayName,
status: 'safe-version-exit',
installedPath: tool.installedPath,
exitCode: stream.exitCode,
output: stream.lines.join('\n').trim(),
route: '/api/tools/launch',
});
});
}
expect(results.map(result => result.id)).toEqual(toolIds);
if (process.env.WAGGLE_E2E_HOST_IDS) {
expect(
results.filter(result => result.status === 'unavailable').map(result => result.id),
'every explicitly requested host must be installed and healthy',
).toEqual([]);
} else {
expect(results.some(result => result.status !== 'unavailable'), 'at least one real installed tool route').toBe(true);
}
} finally {
try {
await testInfo.attach('windows-external-tool-route-summary', {
body: Buffer.from(JSON.stringify({ workspace, results }, null, 2)),
contentType: 'application/json',
});
} finally {
for (const pid of activePids) {
await request.post('/api/tools/kill', { data: { pid } }).catch(() => null);
}
const deleteResponse = await request.delete(`/api/workspaces/${encodeURIComponent(workspace.id)}`).catch(() => null);
if (deleteResponse) expect([204, 404]).toContain(deleteResponse.status());
}
}
});
});

View File

@@ -1,344 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createHash, randomBytes } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { FastifyInstance } from 'fastify';
import { detectInstalledTools, getToolRegistry, runExternalTool } from '@waggle/agent';
import { scopeKey, workspaceScope } from '@waggle/memory-contract';
import { AgentRunRegistry } from '../../packages/server/src/local/agent-run-registry.js';
import { buildLocalServer } from '../../packages/server/src/local/index.js';
import { injectWithAuth } from '../../packages/server/tests/test-utils.js';
const LIVE = process.env.WAGGLE_LIVE_EXTERNAL_AGENTS === '1';
const describeLive = LIVE ? describe : describe.skip;
const REQUIRED_TOOLS = ['claude-code', 'codex', 'hermes'] as const;
async function waitFor(
predicate: () => boolean,
timeoutMs: number,
message: string,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 250));
}
throw new Error(message);
}
describeLive('live external-agent collaboration', () => {
let server: FastifyInstance;
let dataDir: string;
let sourceWorkspaceDir: string;
let sourceWorkspaceId: string;
let synthesisWorkspaceDir: string;
let synthesisWorkspaceId: string;
let expectedCanaryLine: string;
beforeAll(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-live-collab-'));
sourceWorkspaceDir = path.join(dataDir, 'source-workspace');
synthesisWorkspaceDir = path.join(dataDir, 'synthesis-workspace');
fs.mkdirSync(sourceWorkspaceDir, { recursive: true });
fs.mkdirSync(synthesisWorkspaceDir, { recursive: true });
expectedCanaryLine = `CANARY=${randomBytes(24).toString('hex')}`;
fs.writeFileSync(path.join(sourceWorkspaceDir, 'CANARY.txt'), `${expectedCanaryLine}\n`, 'utf8');
fs.writeFileSync(path.join(synthesisWorkspaceDir, 'SENTINEL.txt'), 'workspace must remain unchanged\n', 'utf8');
server = await buildLocalServer({ dataDir });
const hermesProvider = process.env.WAGGLE_LIVE_HERMES_PROVIDER?.trim();
const hermesModel = process.env.WAGGLE_LIVE_HERMES_MODEL?.trim();
if (Boolean(hermesProvider) !== Boolean(hermesModel)) {
throw new Error('Set both WAGGLE_LIVE_HERMES_PROVIDER and WAGGLE_LIVE_HERMES_MODEL');
}
server.decorate('externalToolRunner', (request) => {
const task = request.manifest.task;
if (request.manifest.id !== 'hermes' || !task || !hermesProvider || !hermesModel) {
return runExternalTool(request);
}
const override = ['--provider', hermesProvider, '-m', hermesModel];
return runExternalTool({
...request,
manifest: {
...request.manifest,
task: {
...task,
argvTemplate: [...task.argvTemplate, ...override],
...(task.resumeArgvTemplate
? { resumeArgvTemplate: [...task.resumeArgvTemplate, ...override] }
: {}),
},
},
});
});
const sourceWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Source',
group: 'live-test',
directory: sourceWorkspaceDir,
});
const synthesisWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Synthesis',
group: 'live-test',
directory: synthesisWorkspaceDir,
});
sourceWorkspaceId = sourceWorkspace.id;
synthesisWorkspaceId = synthesisWorkspace.id;
await server.listen({ host: '127.0.0.1', port: 0 });
}, 120_000);
afterAll(async () => {
if (server) await server.close();
if (dataDir) {
let lastError: unknown;
let removed = false;
for (let attempt = 0; attempt < 10; attempt++) {
try {
fs.rmSync(dataDir, { recursive: true, force: true });
removed = true;
break;
} catch (err) {
lastError = err;
await new Promise((resolve) => setTimeout(resolve, 250));
}
}
if (!removed) throw lastError;
}
}, 90_000);
it('runs the authenticated supported-agent cohort and delivers peer evidence through WaggleDance', async () => {
const detection = await detectInstalledTools();
const manifests = new Map(getToolRegistry().map((manifest) => [manifest.id, manifest]));
const unavailable = REQUIRED_TOOLS.filter((id) => !detection.tools.some(
(tool) => tool.id === id && tool.installed && tool.installedPath,
) || manifests.get(id)?.capabilities?.headlessTask !== true);
expect(unavailable, `Missing authenticated headless tools: ${unavailable.join(', ')}`).toEqual([]);
expect(server.workspaceManager.get(sourceWorkspaceId)?.directory).toBe(sourceWorkspaceDir);
expect(server.workspaceManager.get(synthesisWorkspaceId)?.directory).toBe(synthesisWorkspaceDir);
const sourceDigestBefore = workspaceDigest(sourceWorkspaceDir);
const synthesisDigestBefore = workspaceDigest(synthesisWorkspaceDir);
const response = await injectWithAuth(server, {
method: 'POST',
url: '/api/tools/run',
payload: {
prompt: [
'This is a read-only acceptance test. Do not modify any file.',
'Look only in the assigned workspace for ./CANARY.txt.',
'If it exists, copy its entire single line verbatim, including the literal CANARY= prefix, and do not add commentary.',
'A source-file answer must start with the literal CANARY= prefix followed by exactly 48 lowercase hexadecimal characters.',
'If it does not exist, return exactly NO_LOCAL_CANARY and never guess its content.',
'If peer findings are supplied in a later collaboration round, copy the complete matching CANARY= line from those findings verbatim as the source of truth.',
].join(' '),
participants: [
{ toolId: 'claude-code', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'codex', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'hermes', workspaceIds: [sourceWorkspaceId], access: 'native' },
{ toolId: 'hermes', workspaceIds: [synthesisWorkspaceId], access: 'native' },
],
timeoutMs: 300_000,
},
});
expect(response.statusCode).toBe(202);
const body = response.json() as {
roomId: string;
runs: Array<{ runId: string; toolId: string; workspaceId: string }>;
};
expect(body.runs).toHaveLength(5);
await waitFor(
() => {
return [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
['completed', 'failed', 'cancelled', 'interrupted'].includes(
server.agentRunRegistry.get(id)?.status ?? '',
));
},
390_000,
'live external-agent Room did not settle',
);
const terminalRoom = server.agentRunRegistry.get(body.roomId);
const terminalRuns = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
if (terminalRoom?.status === 'completed' && terminalRuns.every((run) => run.status === 'completed')) {
await waitFor(
() => [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
server.agentRunRegistry.get(id)?.memoryRefs.status === 'complete'),
30_000,
'live external-agent memory receipts did not settle',
);
}
const room = server.agentRunRegistry.get(body.roomId);
const runs = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
const diagnostic = JSON.stringify({
room: { status: room?.status, memoryStatus: room?.memoryRefs.status },
versions: Object.fromEntries(detection.tools
.filter((tool) => REQUIRED_TOOLS.includes(tool.id as typeof REQUIRED_TOOLS[number]))
.map((tool) => [tool.id, tool.version ?? null])),
runs: runs.map((run) => ({
tool: run.executor.toolId, status: run.status, exitCode: run.result?.exitCode,
memoryStatus: run.memoryRefs.status,
error: run.result?.error,
summary: run.result?.summary.slice(0, 500),
summaryHash: hashText(run.result?.summary ?? ''),
})),
});
expect(room?.status, diagnostic).toBe('completed');
expect(room?.memoryRefs.status, diagnostic).toBe('complete');
for (const run of runs) {
expect(run.status, diagnostic).toBe('completed');
expect(run.result?.exitCode, diagnostic).toBe(0);
expect(run.memoryRefs.status, diagnostic).toBe('complete');
expect(run.memoryRefs.personalFrameIds?.length).toBeGreaterThan(0);
expect(run.kind).toBe('worker');
if (run.kind === 'worker') {
expect(run.memoryRefs.workspaceFrameIds?.[run.workspaceId]?.length).toBeGreaterThan(0);
}
}
const claude = runs.find((run) => run.executor.toolId === 'claude-code');
const codex = runs.find((run) => run.executor.toolId === 'codex');
const hermes = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === sourceWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const hermesNoCanary = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === synthesisWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const synthesis = runs.find((run) => run.title.startsWith('WaggleDance synthesis'));
for (const run of [claude, codex, hermes]) {
expect(run?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
}
expect(hermesNoCanary?.result?.summary.trim(), diagnostic).toBe('NO_LOCAL_CANARY');
expect(synthesis?.kind).toBe('worker');
expect(synthesis?.executor.toolId).toBe('hermes');
if (synthesis?.kind === 'worker') expect(synthesis.workspaceId).toBe(synthesisWorkspaceId);
expect(synthesis?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
expect(workspaceDigest(sourceWorkspaceDir)).toBe(sourceDigestBefore);
expect(workspaceDigest(synthesisWorkspaceDir)).toBe(synthesisDigestBefore);
const durableRegistry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
for (const id of [body.roomId, ...body.runs.map(({ runId }) => runId)]) {
const durable = durableRegistry.get(id);
expect(durable?.status, diagnostic).toBe('completed');
expect(durable?.memoryRefs.status, diagnostic).toBe('complete');
expect(durable?.memoryRefs, diagnostic).toEqual(server.agentRunRegistry.get(id)?.memoryRefs);
}
for (const run of runs) {
const workspaceId = run.kind === 'worker' ? run.workspaceId : undefined;
expect(Object.keys(run.memoryRefs.workspaceFrameIds), diagnostic).toEqual([workspaceId]);
const personal = await Promise.all(
run.memoryRefs.personalFrameIds.map((id) => server.memory.get(id)),
);
const workspace = workspaceId
? await Promise.all(
run.memoryRefs.workspaceFrameIds[workspaceId].map((id) => server.memory.get(id)),
)
: [];
expect(personal.length, diagnostic).toBeGreaterThan(0);
expect(workspace.length, diagnostic).toBeGreaterThan(0);
for (const memory of personal) {
expect(memory?.scope.kind, diagnostic).toBe('personal');
}
for (const memory of workspace) {
expect(memory && scopeKey(memory.scope), diagnostic).toBe(scopeKey(workspaceScope(workspaceId!)));
}
for (const memory of [...personal, ...workspace]) {
expect(memory, diagnostic).not.toBeNull();
const metadata = memory?.metadata ?? {};
expect(metadata).toMatchObject({
runId: run.id,
roomId: run.roomId,
workspaceId: run.kind === 'worker' ? run.workspaceId : undefined,
toolId: run.executor.toolId,
});
expect(memory?.text.includes(run.id), diagnostic).toBe(true);
for (const forbiddenPath of [dataDir, sourceWorkspaceDir, synthesisWorkspaceDir]) {
expect(memory?.text.includes(forbiddenPath), diagnostic).toBe(false);
expect(JSON.stringify(metadata).includes(forbiddenPath), diagnostic).toBe(false);
}
}
const expectedSummary = run.result?.summary.trim() ?? '';
expect(personal.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
expect(workspace.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
}
expect(new Set(room?.memoryRefs.personalFrameIds)).toEqual(
new Set(runs.flatMap((run) => run.memoryRefs.personalFrameIds)),
);
expect(room?.memoryRefs.workspaceFrameIds).toEqual(Object.fromEntries(
[...new Set(runs.map((run) => run.workspaceId))].map((workspaceId) => [
workspaceId,
[...new Set(runs.flatMap((run) => run.memoryRefs.workspaceFrameIds[workspaceId] ?? []))],
]),
));
const roomSignals = server.signalBus?.query({ teamId: `room::${body.roomId}`, limit: 1_000 }) ?? [];
const subtypes = roomSignals.map((message) => message.subtype);
expect(subtypes).toEqual(expect.arrayContaining(['task_delegation', 'task_claim', 'routed_share', 'knowledge_match']));
for (const run of runs) {
const delegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === run.id);
const claim = roomSignals.find((message) =>
message.subtype === 'task_claim' && message.content.runId === run.id);
const share = roomSignals.find((message) =>
message.subtype === 'routed_share' && message.content.runId === run.id
&& message.content.phase === 'completed');
expect(delegation, diagnostic).toBeDefined();
expect(claim?.referenceId, diagnostic).toBe(delegation?.id);
expect(share?.referenceId, diagnostic).toBe(delegation?.id);
}
const synthesisDelegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === synthesis?.id);
const peerDelivery = roomSignals.find((message) =>
message.subtype === 'knowledge_match' && message.content.runId === synthesis?.id);
expect(peerDelivery?.referenceId, diagnostic).toBe(synthesisDelegation?.id);
expect(peerDelivery?.content.tool, diagnostic).toBe('hermes');
expect(peerDelivery?.content.peerFindings).toEqual(expect.arrayContaining([
expect.stringContaining(expectedCanaryLine),
expect.stringContaining('NO_LOCAL_CANARY'),
]));
const initialRunIds = new Set(runs.filter((run) => run.id !== synthesis?.id).map((run) => run.id));
const expectedSourceMessageIds = new Set(roomSignals.filter((message) =>
message.subtype === 'routed_share'
&& initialRunIds.has(String(message.content.runId))
&& message.content.phase === 'completed').map((message) => message.id));
expect(new Set(peerDelivery?.content.sourceMessageIds as string[] | undefined)).toEqual(expectedSourceMessageIds);
expect(new Set(synthesisDelegation?.content.sourceRunIds as string[] | undefined)).toEqual(initialRunIds);
const peerFindings = peerDelivery?.content.peerFindings as string[] | undefined;
expect(peerFindings, diagnostic).toHaveLength(initialRunIds.size);
for (const runId of initialRunIds) {
expect(peerFindings?.filter((finding) => finding.includes(`· run ${runId}]`)), diagnostic).toHaveLength(1);
}
expect(new Set(roomSignals.map((message) => message.content.runId))).toEqual(
new Set(body.runs.map((run) => run.runId)),
);
}, 600_000);
});
function hashText(value: string): string {
return createHash('sha256').update(value).digest('hex');
}
function workspaceDigest(root: string): string {
const hash = createHash('sha256');
const visit = (directory: string, relativeDirectory: string): void => {
const entries = fs.readdirSync(directory, { withFileTypes: true })
.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const relative = path.join(relativeDirectory, entry.name).replaceAll('\\', '/');
const absolute = path.join(directory, entry.name);
if (entry.isDirectory()) {
hash.update(`D\0${relative}\0`);
visit(absolute, relative);
} else if (entry.isFile()) {
hash.update(`F\0${relative}\0`);
hash.update(fs.readFileSync(absolute));
hash.update('\0');
} else if (entry.isSymbolicLink()) {
hash.update(`L\0${relative}\0${fs.readlinkSync(absolute)}\0`);
} else {
hash.update(`X\0${relative}\0`);
}
}
};
visit(root, '');
return hash.digest('hex');
}