Files
waggle-os/tests/integration/external-agent-collaboration.live.test.ts
shumeiko.it 3308263613 test: read run memories through the contract in the live collaboration test (P1.5)
The live external-agent test (skipped unless WAGGLE_LIVE_EXTERNAL_AGENTS=1)
looked up run memory receipts as frames in server.multiMind and
server.mindCache, which the server no longer has. Look them up with
server.memory.get and check that personal and workspace receipts sit in
their own scopes. Also type the prompt assembler's bulk fixture as
RecallFrame.
2026-10-24 17:43:27 +02:00

345 lines
16 KiB
TypeScript

import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createHash, randomBytes } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { FastifyInstance } from 'fastify';
import { detectInstalledTools, getToolRegistry, runExternalTool } from '@waggle/agent';
import { scopeKey, workspaceScope } from '@waggle/memory-contract';
import { AgentRunRegistry } from '../../packages/server/src/local/agent-run-registry.js';
import { buildLocalServer } from '../../packages/server/src/local/index.js';
import { injectWithAuth } from '../../packages/server/tests/test-utils.js';
const LIVE = process.env.WAGGLE_LIVE_EXTERNAL_AGENTS === '1';
const describeLive = LIVE ? describe : describe.skip;
const REQUIRED_TOOLS = ['claude-code', 'codex', 'hermes'] as const;
async function waitFor(
predicate: () => boolean,
timeoutMs: number,
message: string,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 250));
}
throw new Error(message);
}
describeLive('live external-agent collaboration', () => {
let server: FastifyInstance;
let dataDir: string;
let sourceWorkspaceDir: string;
let sourceWorkspaceId: string;
let synthesisWorkspaceDir: string;
let synthesisWorkspaceId: string;
let expectedCanaryLine: string;
beforeAll(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-live-collab-'));
sourceWorkspaceDir = path.join(dataDir, 'source-workspace');
synthesisWorkspaceDir = path.join(dataDir, 'synthesis-workspace');
fs.mkdirSync(sourceWorkspaceDir, { recursive: true });
fs.mkdirSync(synthesisWorkspaceDir, { recursive: true });
expectedCanaryLine = `CANARY=${randomBytes(24).toString('hex')}`;
fs.writeFileSync(path.join(sourceWorkspaceDir, 'CANARY.txt'), `${expectedCanaryLine}\n`, 'utf8');
fs.writeFileSync(path.join(synthesisWorkspaceDir, 'SENTINEL.txt'), 'workspace must remain unchanged\n', 'utf8');
server = await buildLocalServer({ dataDir });
const hermesProvider = process.env.WAGGLE_LIVE_HERMES_PROVIDER?.trim();
const hermesModel = process.env.WAGGLE_LIVE_HERMES_MODEL?.trim();
if (Boolean(hermesProvider) !== Boolean(hermesModel)) {
throw new Error('Set both WAGGLE_LIVE_HERMES_PROVIDER and WAGGLE_LIVE_HERMES_MODEL');
}
server.decorate('externalToolRunner', (request) => {
const task = request.manifest.task;
if (request.manifest.id !== 'hermes' || !task || !hermesProvider || !hermesModel) {
return runExternalTool(request);
}
const override = ['--provider', hermesProvider, '-m', hermesModel];
return runExternalTool({
...request,
manifest: {
...request.manifest,
task: {
...task,
argvTemplate: [...task.argvTemplate, ...override],
...(task.resumeArgvTemplate
? { resumeArgvTemplate: [...task.resumeArgvTemplate, ...override] }
: {}),
},
},
});
});
const sourceWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Source',
group: 'live-test',
directory: sourceWorkspaceDir,
});
const synthesisWorkspace = server.workspaceManager.create({
name: 'Live Collaboration Synthesis',
group: 'live-test',
directory: synthesisWorkspaceDir,
});
sourceWorkspaceId = sourceWorkspace.id;
synthesisWorkspaceId = synthesisWorkspace.id;
await server.listen({ host: '127.0.0.1', port: 0 });
}, 120_000);
afterAll(async () => {
if (server) await server.close();
if (dataDir) {
let lastError: unknown;
let removed = false;
for (let attempt = 0; attempt < 10; attempt++) {
try {
fs.rmSync(dataDir, { recursive: true, force: true });
removed = true;
break;
} catch (err) {
lastError = err;
await new Promise((resolve) => setTimeout(resolve, 250));
}
}
if (!removed) throw lastError;
}
}, 90_000);
it('runs the authenticated supported-agent cohort and delivers peer evidence through WaggleDance', async () => {
const detection = await detectInstalledTools();
const manifests = new Map(getToolRegistry().map((manifest) => [manifest.id, manifest]));
const unavailable = REQUIRED_TOOLS.filter((id) => !detection.tools.some(
(tool) => tool.id === id && tool.installed && tool.installedPath,
) || manifests.get(id)?.capabilities?.headlessTask !== true);
expect(unavailable, `Missing authenticated headless tools: ${unavailable.join(', ')}`).toEqual([]);
expect(server.workspaceManager.get(sourceWorkspaceId)?.directory).toBe(sourceWorkspaceDir);
expect(server.workspaceManager.get(synthesisWorkspaceId)?.directory).toBe(synthesisWorkspaceDir);
const sourceDigestBefore = workspaceDigest(sourceWorkspaceDir);
const synthesisDigestBefore = workspaceDigest(synthesisWorkspaceDir);
const response = await injectWithAuth(server, {
method: 'POST',
url: '/api/tools/run',
payload: {
prompt: [
'This is a read-only acceptance test. Do not modify any file.',
'Look only in the assigned workspace for ./CANARY.txt.',
'If it exists, copy its entire single line verbatim, including the literal CANARY= prefix, and do not add commentary.',
'A source-file answer must start with the literal CANARY= prefix followed by exactly 48 lowercase hexadecimal characters.',
'If it does not exist, return exactly NO_LOCAL_CANARY and never guess its content.',
'If peer findings are supplied in a later collaboration round, copy the complete matching CANARY= line from those findings verbatim as the source of truth.',
].join(' '),
participants: [
{ toolId: 'claude-code', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'codex', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
{ toolId: 'hermes', workspaceIds: [sourceWorkspaceId], access: 'native' },
{ toolId: 'hermes', workspaceIds: [synthesisWorkspaceId], access: 'native' },
],
timeoutMs: 300_000,
},
});
expect(response.statusCode).toBe(202);
const body = response.json() as {
roomId: string;
runs: Array<{ runId: string; toolId: string; workspaceId: string }>;
};
expect(body.runs).toHaveLength(5);
await waitFor(
() => {
return [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
['completed', 'failed', 'cancelled', 'interrupted'].includes(
server.agentRunRegistry.get(id)?.status ?? '',
));
},
390_000,
'live external-agent Room did not settle',
);
const terminalRoom = server.agentRunRegistry.get(body.roomId);
const terminalRuns = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
if (terminalRoom?.status === 'completed' && terminalRuns.every((run) => run.status === 'completed')) {
await waitFor(
() => [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
server.agentRunRegistry.get(id)?.memoryRefs.status === 'complete'),
30_000,
'live external-agent memory receipts did not settle',
);
}
const room = server.agentRunRegistry.get(body.roomId);
const runs = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
const diagnostic = JSON.stringify({
room: { status: room?.status, memoryStatus: room?.memoryRefs.status },
versions: Object.fromEntries(detection.tools
.filter((tool) => REQUIRED_TOOLS.includes(tool.id as typeof REQUIRED_TOOLS[number]))
.map((tool) => [tool.id, tool.version ?? null])),
runs: runs.map((run) => ({
tool: run.executor.toolId, status: run.status, exitCode: run.result?.exitCode,
memoryStatus: run.memoryRefs.status,
error: run.result?.error,
summary: run.result?.summary.slice(0, 500),
summaryHash: hashText(run.result?.summary ?? ''),
})),
});
expect(room?.status, diagnostic).toBe('completed');
expect(room?.memoryRefs.status, diagnostic).toBe('complete');
for (const run of runs) {
expect(run.status, diagnostic).toBe('completed');
expect(run.result?.exitCode, diagnostic).toBe(0);
expect(run.memoryRefs.status, diagnostic).toBe('complete');
expect(run.memoryRefs.personalFrameIds?.length).toBeGreaterThan(0);
expect(run.kind).toBe('worker');
if (run.kind === 'worker') {
expect(run.memoryRefs.workspaceFrameIds?.[run.workspaceId]?.length).toBeGreaterThan(0);
}
}
const claude = runs.find((run) => run.executor.toolId === 'claude-code');
const codex = runs.find((run) => run.executor.toolId === 'codex');
const hermes = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === sourceWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const hermesNoCanary = runs.find((run) => run.executor.toolId === 'hermes'
&& run.workspaceId === synthesisWorkspaceId
&& !run.title.startsWith('WaggleDance synthesis'));
const synthesis = runs.find((run) => run.title.startsWith('WaggleDance synthesis'));
for (const run of [claude, codex, hermes]) {
expect(run?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
}
expect(hermesNoCanary?.result?.summary.trim(), diagnostic).toBe('NO_LOCAL_CANARY');
expect(synthesis?.kind).toBe('worker');
expect(synthesis?.executor.toolId).toBe('hermes');
if (synthesis?.kind === 'worker') expect(synthesis.workspaceId).toBe(synthesisWorkspaceId);
expect(synthesis?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
expect(workspaceDigest(sourceWorkspaceDir)).toBe(sourceDigestBefore);
expect(workspaceDigest(synthesisWorkspaceDir)).toBe(synthesisDigestBefore);
const durableRegistry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
for (const id of [body.roomId, ...body.runs.map(({ runId }) => runId)]) {
const durable = durableRegistry.get(id);
expect(durable?.status, diagnostic).toBe('completed');
expect(durable?.memoryRefs.status, diagnostic).toBe('complete');
expect(durable?.memoryRefs, diagnostic).toEqual(server.agentRunRegistry.get(id)?.memoryRefs);
}
for (const run of runs) {
const workspaceId = run.kind === 'worker' ? run.workspaceId : undefined;
expect(Object.keys(run.memoryRefs.workspaceFrameIds), diagnostic).toEqual([workspaceId]);
const personal = await Promise.all(
run.memoryRefs.personalFrameIds.map((id) => server.memory.get(id)),
);
const workspace = workspaceId
? await Promise.all(
run.memoryRefs.workspaceFrameIds[workspaceId].map((id) => server.memory.get(id)),
)
: [];
expect(personal.length, diagnostic).toBeGreaterThan(0);
expect(workspace.length, diagnostic).toBeGreaterThan(0);
for (const memory of personal) {
expect(memory?.scope.kind, diagnostic).toBe('personal');
}
for (const memory of workspace) {
expect(memory && scopeKey(memory.scope), diagnostic).toBe(scopeKey(workspaceScope(workspaceId!)));
}
for (const memory of [...personal, ...workspace]) {
expect(memory, diagnostic).not.toBeNull();
const metadata = memory?.metadata ?? {};
expect(metadata).toMatchObject({
runId: run.id,
roomId: run.roomId,
workspaceId: run.kind === 'worker' ? run.workspaceId : undefined,
toolId: run.executor.toolId,
});
expect(memory?.text.includes(run.id), diagnostic).toBe(true);
for (const forbiddenPath of [dataDir, sourceWorkspaceDir, synthesisWorkspaceDir]) {
expect(memory?.text.includes(forbiddenPath), diagnostic).toBe(false);
expect(JSON.stringify(metadata).includes(forbiddenPath), diagnostic).toBe(false);
}
}
const expectedSummary = run.result?.summary.trim() ?? '';
expect(personal.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
expect(workspace.some((memory) => memory?.text.includes(expectedSummary)), diagnostic).toBe(true);
}
expect(new Set(room?.memoryRefs.personalFrameIds)).toEqual(
new Set(runs.flatMap((run) => run.memoryRefs.personalFrameIds)),
);
expect(room?.memoryRefs.workspaceFrameIds).toEqual(Object.fromEntries(
[...new Set(runs.map((run) => run.workspaceId))].map((workspaceId) => [
workspaceId,
[...new Set(runs.flatMap((run) => run.memoryRefs.workspaceFrameIds[workspaceId] ?? []))],
]),
));
const roomSignals = server.signalBus?.query({ teamId: `room::${body.roomId}`, limit: 1_000 }) ?? [];
const subtypes = roomSignals.map((message) => message.subtype);
expect(subtypes).toEqual(expect.arrayContaining(['task_delegation', 'task_claim', 'routed_share', 'knowledge_match']));
for (const run of runs) {
const delegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === run.id);
const claim = roomSignals.find((message) =>
message.subtype === 'task_claim' && message.content.runId === run.id);
const share = roomSignals.find((message) =>
message.subtype === 'routed_share' && message.content.runId === run.id
&& message.content.phase === 'completed');
expect(delegation, diagnostic).toBeDefined();
expect(claim?.referenceId, diagnostic).toBe(delegation?.id);
expect(share?.referenceId, diagnostic).toBe(delegation?.id);
}
const synthesisDelegation = roomSignals.find((message) =>
message.subtype === 'task_delegation' && message.content.runId === synthesis?.id);
const peerDelivery = roomSignals.find((message) =>
message.subtype === 'knowledge_match' && message.content.runId === synthesis?.id);
expect(peerDelivery?.referenceId, diagnostic).toBe(synthesisDelegation?.id);
expect(peerDelivery?.content.tool, diagnostic).toBe('hermes');
expect(peerDelivery?.content.peerFindings).toEqual(expect.arrayContaining([
expect.stringContaining(expectedCanaryLine),
expect.stringContaining('NO_LOCAL_CANARY'),
]));
const initialRunIds = new Set(runs.filter((run) => run.id !== synthesis?.id).map((run) => run.id));
const expectedSourceMessageIds = new Set(roomSignals.filter((message) =>
message.subtype === 'routed_share'
&& initialRunIds.has(String(message.content.runId))
&& message.content.phase === 'completed').map((message) => message.id));
expect(new Set(peerDelivery?.content.sourceMessageIds as string[] | undefined)).toEqual(expectedSourceMessageIds);
expect(new Set(synthesisDelegation?.content.sourceRunIds as string[] | undefined)).toEqual(initialRunIds);
const peerFindings = peerDelivery?.content.peerFindings as string[] | undefined;
expect(peerFindings, diagnostic).toHaveLength(initialRunIds.size);
for (const runId of initialRunIds) {
expect(peerFindings?.filter((finding) => finding.includes(`ยท run ${runId}]`)), diagnostic).toHaveLength(1);
}
expect(new Set(roomSignals.map((message) => message.content.runId))).toEqual(
new Set(body.runs.map((run) => run.runId)),
);
}, 600_000);
});
function hashText(value: string): string {
return createHash('sha256').update(value).digest('hex');
}
function workspaceDigest(root: string): string {
const hash = createHash('sha256');
const visit = (directory: string, relativeDirectory: string): void => {
const entries = fs.readdirSync(directory, { withFileTypes: true })
.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const relative = path.join(relativeDirectory, entry.name).replaceAll('\\', '/');
const absolute = path.join(directory, entry.name);
if (entry.isDirectory()) {
hash.update(`D\0${relative}\0`);
visit(absolute, relative);
} else if (entry.isFile()) {
hash.update(`F\0${relative}\0`);
hash.update(fs.readFileSync(absolute));
hash.update('\0');
} else if (entry.isSymbolicLink()) {
hash.update(`L\0${relative}\0${fs.readlinkSync(absolute)}\0`);
} else {
hash.update(`X\0${relative}\0`);
}
}
};
visit(root, '');
return hash.digest('hex');
}