moving

tests/integration/external-agent-collaboration.live.test.ts

@@ -1,16 +1,18 @@
 import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { createHash } from 'node:crypto';
+import { createHash, randomBytes } from 'node:crypto';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import type { FastifyInstance } from 'fastify';
-import { detectInstalledTools, runExternalTool } from '@waggle/agent';
-import type { ToolManifest } from '@waggle/shared';
+import { detectInstalledTools, getToolRegistry, runExternalTool } from '@waggle/agent';
+import { FrameStore } from '@waggle/core';
+import { AgentRunRegistry } from '../../packages/server/src/local/agent-run-registry.js';
 import { buildLocalServer } from '../../packages/server/src/local/index.js';
 import { injectWithAuth } from '../../packages/server/tests/test-utils.js';
 
 const LIVE = process.env.WAGGLE_LIVE_EXTERNAL_AGENTS === '1';
 const describeLive = LIVE ? describe : describe.skip;
+const REQUIRED_TOOLS = ['claude-code', 'codex', 'hermes'] as const;
 
 async function waitFor(
   predicate: () => boolean,
@@ -32,7 +34,7 @@ describeLive('live external-agent collaboration', () => {
   let sourceWorkspaceId: string;
   let synthesisWorkspaceDir: string;
   let synthesisWorkspaceId: string;
-  let openClawBinary: string | undefined;
+  let expectedCanaryLine: string;
 
   beforeAll(async () => {
     dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-live-collab-'));
@@ -40,12 +42,35 @@ describeLive('live external-agent collaboration', () => {
     synthesisWorkspaceDir = path.join(dataDir, 'synthesis-workspace');
     fs.mkdirSync(sourceWorkspaceDir, { recursive: true });
     fs.mkdirSync(synthesisWorkspaceDir, { recursive: true });
-    fs.writeFileSync(
-      path.join(sourceWorkspaceDir, 'FACTS.md'),
-      '# Collaboration fixture\n\nAlpha code: HONEY-17\nBeta code: WAGGLE-42\n',
-      'utf8',
-    );
+    expectedCanaryLine = `CANARY=${randomBytes(24).toString('hex')}`;
+    fs.writeFileSync(path.join(sourceWorkspaceDir, 'CANARY.txt'), `${expectedCanaryLine}\n`, 'utf8');
+    fs.writeFileSync(path.join(synthesisWorkspaceDir, 'SENTINEL.txt'), 'workspace must remain unchanged\n', 'utf8');
     server = await buildLocalServer({ dataDir });
+    const hermesProvider = process.env.WAGGLE_LIVE_HERMES_PROVIDER?.trim();
+    const hermesModel = process.env.WAGGLE_LIVE_HERMES_MODEL?.trim();
+    if (Boolean(hermesProvider) !== Boolean(hermesModel)) {
+      throw new Error('Set both WAGGLE_LIVE_HERMES_PROVIDER and WAGGLE_LIVE_HERMES_MODEL');
+    }
+    server.decorate('externalToolRunner', (request) => {
+      const task = request.manifest.task;
+      if (request.manifest.id !== 'hermes' || !task || !hermesProvider || !hermesModel) {
+        return runExternalTool(request);
+      }
+      const override = ['--provider', hermesProvider, '-m', hermesModel];
+      return runExternalTool({
+        ...request,
+        manifest: {
+          ...request.manifest,
+          task: {
+            ...task,
+            argvTemplate: [...task.argvTemplate, ...override],
+            ...(task.resumeArgvTemplate
+              ? { resumeArgvTemplate: [...task.resumeArgvTemplate, ...override] }
+              : {}),
+          },
+        },
+      });
+    });
     const sourceWorkspace = server.workspaceManager.create({
       name: 'Live Collaboration Source',
       group: 'live-test',
@@ -62,33 +87,6 @@ describeLive('live external-agent collaboration', () => {
   }, 120_000);
 
   afterAll(async () => {
-    let cleanupError: unknown;
-    if (openClawBinary && synthesisWorkspaceDir && synthesisWorkspaceId && fs.existsSync(synthesisWorkspaceDir)) {
-      try {
-        const agentId = openClawAgentId(synthesisWorkspaceId, fs.realpathSync(synthesisWorkspaceDir));
-        const cleanupManifest: ToolManifest = {
-          id: 'openclaw-cleanup', displayName: 'OpenClaw cleanup', launchable: false,
-          hookCapable: false, hookPointer: '.openclaw/test-cleanup',
-          detect: { kind: 'path', binaryName: 'openclaw' },
-          capabilities: { interactiveLaunch: false, headlessTask: true, structuredProgress: true, resumable: false, liveWaggleDance: false },
-          task: {
-            argvTemplate: ['agents', 'delete', agentId, '--force', '--json'],
-            accessArgs: { native: [] }, promptTransport: 'stdin', outputDialect: 'json',
-            workspaceBinding: 'cwd', permissionModes: ['native'], resumable: false,
-          },
-        };
-        const cleanup = await runExternalTool({
-          manifest: cleanupManifest, binary: openClawBinary,
-          workspaceId: synthesisWorkspaceId, workspacePath: synthesisWorkspaceDir,
-          runId: 'live-cleanup', roomId: 'live-cleanup', prompt: '', access: 'native', timeoutMs: 30_000,
-        });
-        if (cleanup.status !== 'completed') {
-          cleanupError = new Error(`OpenClaw live-test cleanup failed: ${cleanup.stderrTail || cleanup.summary}`);
-        }
-      } catch (err) {
-        cleanupError = err;
-      }
-    }
     if (server) await server.close();
     if (dataDir) {
       let lastError: unknown;
@@ -105,22 +103,19 @@ describeLive('live external-agent collaboration', () => {
       }
       if (!removed) throw lastError;
     }
-    if (cleanupError) throw cleanupError;
-  }, 30_000);
+  }, 90_000);
 
-  it('delivers an asymmetric Hermes finding to OpenClaw through WaggleDance', async (context) => {
+  it('runs the authenticated supported-agent cohort and delivers peer evidence through WaggleDance', async () => {
     const detection = await detectInstalledTools();
-    const required = ['hermes', 'openclaw'];
-    const unavailable = required.filter((id) => !detection.tools.some(
+    const manifests = new Map(getToolRegistry().map((manifest) => [manifest.id, manifest]));
+    const unavailable = REQUIRED_TOOLS.filter((id) => !detection.tools.some(
       (tool) => tool.id === id && tool.installed && tool.installedPath,
-    ));
-    if (unavailable.length > 0) {
-      context.skip(`Missing live tools: ${unavailable.join(', ')}`);
-      return;
-    }
-    openClawBinary = detection.tools.find((tool) => tool.id === 'openclaw')?.installedPath ?? undefined;
+    ) || manifests.get(id)?.capabilities?.headlessTask !== true);
+    expect(unavailable, `Missing authenticated headless tools: ${unavailable.join(', ')}`).toEqual([]);
     expect(server.workspaceManager.get(sourceWorkspaceId)?.directory).toBe(sourceWorkspaceDir);
     expect(server.workspaceManager.get(synthesisWorkspaceId)?.directory).toBe(synthesisWorkspaceDir);
+    const sourceDigestBefore = workspaceDigest(sourceWorkspaceDir);
+    const synthesisDigestBefore = workspaceDigest(synthesisWorkspaceDir);
 
     const response = await injectWithAuth(server, {
       method: 'POST',
@@ -128,16 +123,19 @@ describeLive('live external-agent collaboration', () => {
       payload: {
         prompt: [
           'This is a read-only acceptance test. Do not modify any file.',
-          'Look only in the assigned workspace for FACTS.md.',
-          'If it exists, return one line: ALPHA=<alpha code> BETA=<beta code>.',
-          'If it does not exist, return exactly NO_LOCAL_FACTS and never guess the codes.',
-          'If peer findings are supplied in a later collaboration round, use those findings as the source of truth.',
+          'Look only in the assigned workspace for ./CANARY.txt.',
+          'If it exists, copy its entire single line verbatim, including the literal CANARY= prefix, and do not add commentary.',
+          'A source-file answer must start with the literal CANARY= prefix followed by exactly 48 lowercase hexadecimal characters.',
+          'If it does not exist, return exactly NO_LOCAL_CANARY and never guess its content.',
+          'If peer findings are supplied in a later collaboration round, copy the complete matching CANARY= line from those findings verbatim as the source of truth.',
         ].join(' '),
         participants: [
+          { toolId: 'claude-code', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
+          { toolId: 'codex', workspaceIds: [sourceWorkspaceId], access: 'read-only' },
           { toolId: 'hermes', workspaceIds: [sourceWorkspaceId], access: 'native' },
-          { toolId: 'openclaw', workspaceIds: [synthesisWorkspaceId], access: 'native' },
+          { toolId: 'hermes', workspaceIds: [synthesisWorkspaceId], access: 'native' },
         ],
-        timeoutMs: 180_000,
+        timeoutMs: 300_000,
       },
     });
     expect(response.statusCode).toBe(202);
@@ -145,28 +143,49 @@ describeLive('live external-agent collaboration', () => {
       roomId: string;
       runs: Array<{ runId: string; toolId: string; workspaceId: string }>;
     };
-    expect(body.runs).toHaveLength(3);
+    expect(body.runs).toHaveLength(5);
 
     await waitFor(
-      () => ['completed', 'failed', 'cancelled', 'interrupted'].includes(
-        server.agentRunRegistry.get(body.roomId)?.status ?? '',
-      ),
-      210_000,
+      () => {
+        return [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
+          ['completed', 'failed', 'cancelled', 'interrupted'].includes(
+            server.agentRunRegistry.get(id)?.status ?? '',
+          ));
+      },
+      390_000,
       'live external-agent Room did not settle',
     );
+    const terminalRoom = server.agentRunRegistry.get(body.roomId);
+    const terminalRuns = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
+    if (terminalRoom?.status === 'completed' && terminalRuns.every((run) => run.status === 'completed')) {
+      await waitFor(
+        () => [body.roomId, ...body.runs.map(({ runId }) => runId)].every((id) =>
+          server.agentRunRegistry.get(id)?.memoryRefs.status === 'complete'),
+        30_000,
+        'live external-agent memory receipts did not settle',
+      );
+    }
 
     const room = server.agentRunRegistry.get(body.roomId);
     const runs = body.runs.map(({ runId }) => server.agentRunRegistry.get(runId)!);
     const diagnostic = JSON.stringify({
-      room: { status: room?.status, result: room?.result },
+      room: { status: room?.status, memoryStatus: room?.memoryRefs.status },
+      versions: Object.fromEntries(detection.tools
+        .filter((tool) => REQUIRED_TOOLS.includes(tool.id as typeof REQUIRED_TOOLS[number]))
+        .map((tool) => [tool.id, tool.version ?? null])),
       runs: runs.map((run) => ({
-        id: run.id, tool: run.executor.toolId, status: run.status,
-        result: run.result, progress: run.progress, memoryRefs: run.memoryRefs,
+        tool: run.executor.toolId, status: run.status, exitCode: run.result?.exitCode,
+        memoryStatus: run.memoryRefs.status,
+        error: run.result?.error,
+        summary: run.result?.summary.slice(0, 500),
+        summaryHash: hashText(run.result?.summary ?? ''),
       })),
     });
     expect(room?.status, diagnostic).toBe('completed');
+    expect(room?.memoryRefs.status, diagnostic).toBe('complete');
     for (const run of runs) {
       expect(run.status, diagnostic).toBe('completed');
+      expect(run.result?.exitCode, diagnostic).toBe(0);
       expect(run.memoryRefs.status, diagnostic).toBe('complete');
       expect(run.memoryRefs.personalFrameIds?.length).toBeGreaterThan(0);
       expect(run.kind).toBe('worker');
@@ -174,32 +193,150 @@ describeLive('live external-agent collaboration', () => {
         expect(run.memoryRefs.workspaceFrameIds?.[run.workspaceId]?.length).toBeGreaterThan(0);
       }
     }
-    const hermes = runs.find((run) => run.executor.toolId === 'hermes');
-    const openClawFirstWave = runs.find((run) => run.executor.toolId === 'openclaw'
+    const claude = runs.find((run) => run.executor.toolId === 'claude-code');
+    const codex = runs.find((run) => run.executor.toolId === 'codex');
+    const hermes = runs.find((run) => run.executor.toolId === 'hermes'
+      && run.workspaceId === sourceWorkspaceId
+      && !run.title.startsWith('WaggleDance synthesis'));
+    const hermesNoCanary = runs.find((run) => run.executor.toolId === 'hermes'
+      && run.workspaceId === synthesisWorkspaceId
       && !run.title.startsWith('WaggleDance synthesis'));
     const synthesis = runs.find((run) => run.title.startsWith('WaggleDance synthesis'));
-    expect(hermes?.result?.summary, diagnostic).toContain('ALPHA=HONEY-17 BETA=WAGGLE-42');
-    expect(openClawFirstWave?.result?.summary, diagnostic).toContain('NO_LOCAL_FACTS');
-    expect(openClawFirstWave?.result?.summary, diagnostic).not.toContain('HONEY-17');
+    for (const run of [claude, codex, hermes]) {
+      expect(run?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
+    }
+    expect(hermesNoCanary?.result?.summary.trim(), diagnostic).toBe('NO_LOCAL_CANARY');
     expect(synthesis?.kind).toBe('worker');
+    expect(synthesis?.executor.toolId).toBe('hermes');
     if (synthesis?.kind === 'worker') expect(synthesis.workspaceId).toBe(synthesisWorkspaceId);
-    expect(synthesis?.result?.summary, diagnostic).toContain('ALPHA=HONEY-17 BETA=WAGGLE-42');
+    expect(synthesis?.result?.summary.trim(), diagnostic).toBe(expectedCanaryLine);
+
+    expect(workspaceDigest(sourceWorkspaceDir)).toBe(sourceDigestBefore);
+    expect(workspaceDigest(synthesisWorkspaceDir)).toBe(synthesisDigestBefore);
+
+    const durableRegistry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
+    for (const id of [body.roomId, ...body.runs.map(({ runId }) => runId)]) {
+      const durable = durableRegistry.get(id);
+      expect(durable?.status, diagnostic).toBe('completed');
+      expect(durable?.memoryRefs.status, diagnostic).toBe('complete');
+      expect(durable?.memoryRefs, diagnostic).toEqual(server.agentRunRegistry.get(id)?.memoryRefs);
+    }
+
+    const personalFrames = new FrameStore(server.multiMind.personal);
+    for (const run of runs) {
+      const workspaceId = run.kind === 'worker' ? run.workspaceId : undefined;
+      expect(Object.keys(run.memoryRefs.workspaceFrameIds), diagnostic).toEqual([workspaceId]);
+      const personal = run.memoryRefs.personalFrameIds.map((id) => personalFrames.getById(id));
+      let workspace = [] as ReturnType<FrameStore['getById']>[];
+      if (workspaceId) {
+        const workspaceMind = server.mindCache.acquire(workspaceId);
+        try {
+          const frames = new FrameStore(workspaceMind);
+          workspace = run.memoryRefs.workspaceFrameIds[workspaceId].map((id) => frames.getById(id));
+        } finally {
+          server.mindCache.release(workspaceId);
+        }
+      }
+      expect(personal.length, diagnostic).toBeGreaterThan(0);
+      expect(workspace.length, diagnostic).toBeGreaterThan(0);
+      for (const frame of [...personal, ...workspace]) {
+        expect(frame, diagnostic).toBeDefined();
+        const metadata = JSON.parse(frame?.metadata ?? '{}') as Record<string, unknown>;
+        expect(metadata).toMatchObject({
+          runId: run.id,
+          roomId: run.roomId,
+          workspaceId: run.kind === 'worker' ? run.workspaceId : undefined,
+          toolId: run.executor.toolId,
+        });
+        expect(frame?.content.includes(run.id), diagnostic).toBe(true);
+        for (const forbiddenPath of [dataDir, sourceWorkspaceDir, synthesisWorkspaceDir]) {
+          expect(frame?.content.includes(forbiddenPath), diagnostic).toBe(false);
+          expect(JSON.stringify(metadata).includes(forbiddenPath), diagnostic).toBe(false);
+        }
+      }
+      const expectedSummary = run.result?.summary.trim() ?? '';
+      expect(personal.some((frame) => frame?.content.includes(expectedSummary)), diagnostic).toBe(true);
+      expect(workspace.some((frame) => frame?.content.includes(expectedSummary)), diagnostic).toBe(true);
+    }
+    expect(new Set(room?.memoryRefs.personalFrameIds)).toEqual(
+      new Set(runs.flatMap((run) => run.memoryRefs.personalFrameIds)),
+    );
+    expect(room?.memoryRefs.workspaceFrameIds).toEqual(Object.fromEntries(
+      [...new Set(runs.map((run) => run.workspaceId))].map((workspaceId) => [
+        workspaceId,
+        [...new Set(runs.flatMap((run) => run.memoryRefs.workspaceFrameIds[workspaceId] ?? []))],
+      ]),
+    ));
 
     const roomSignals = server.signalBus?.query({ teamId: `room::${body.roomId}`, limit: 1_000 }) ?? [];
     const subtypes = roomSignals.map((message) => message.subtype);
     expect(subtypes).toEqual(expect.arrayContaining(['task_delegation', 'task_claim', 'routed_share', 'knowledge_match']));
-    const peerDelivery = roomSignals.find((message) => message.subtype === 'knowledge_match');
+    for (const run of runs) {
+      const delegation = roomSignals.find((message) =>
+        message.subtype === 'task_delegation' && message.content.runId === run.id);
+      const claim = roomSignals.find((message) =>
+        message.subtype === 'task_claim' && message.content.runId === run.id);
+      const share = roomSignals.find((message) =>
+        message.subtype === 'routed_share' && message.content.runId === run.id
+          && message.content.phase === 'completed');
+      expect(delegation, diagnostic).toBeDefined();
+      expect(claim?.referenceId, diagnostic).toBe(delegation?.id);
+      expect(share?.referenceId, diagnostic).toBe(delegation?.id);
+    }
+    const synthesisDelegation = roomSignals.find((message) =>
+      message.subtype === 'task_delegation' && message.content.runId === synthesis?.id);
+    const peerDelivery = roomSignals.find((message) =>
+      message.subtype === 'knowledge_match' && message.content.runId === synthesis?.id);
+    expect(peerDelivery?.referenceId, diagnostic).toBe(synthesisDelegation?.id);
+    expect(peerDelivery?.content.tool, diagnostic).toBe('hermes');
     expect(peerDelivery?.content.peerFindings).toEqual(expect.arrayContaining([
-      expect.stringContaining('ALPHA=HONEY-17 BETA=WAGGLE-42'),
+      expect.stringContaining(expectedCanaryLine),
+      expect.stringContaining('NO_LOCAL_CANARY'),
     ]));
+    const initialRunIds = new Set(runs.filter((run) => run.id !== synthesis?.id).map((run) => run.id));
+    const expectedSourceMessageIds = new Set(roomSignals.filter((message) =>
+      message.subtype === 'routed_share'
+        && initialRunIds.has(String(message.content.runId))
+        && message.content.phase === 'completed').map((message) => message.id));
+    expect(new Set(peerDelivery?.content.sourceMessageIds as string[] | undefined)).toEqual(expectedSourceMessageIds);
+    expect(new Set(synthesisDelegation?.content.sourceRunIds as string[] | undefined)).toEqual(initialRunIds);
+    const peerFindings = peerDelivery?.content.peerFindings as string[] | undefined;
+    expect(peerFindings, diagnostic).toHaveLength(initialRunIds.size);
+    for (const runId of initialRunIds) {
+      expect(peerFindings?.filter((finding) => finding.includes(`· run ${runId}]`)), diagnostic).toHaveLength(1);
+    }
     expect(new Set(roomSignals.map((message) => message.content.runId))).toEqual(
       new Set(body.runs.map((run) => run.runId)),
     );
-  }, 360_000);
+  }, 600_000);
 });
 
-function openClawAgentId(workspaceId: string, cwd: string): string {
-  const digest = createHash('sha256').update(`${workspaceId}\0${cwd}`).digest('hex').slice(0, 8);
-  const base = workspaceId.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-|-$/g, '').slice(0, 32) || 'workspace';
-  return `waggle-${base}-${digest}`;
+function hashText(value: string): string {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function workspaceDigest(root: string): string {
+  const hash = createHash('sha256');
+  const visit = (directory: string, relativeDirectory: string): void => {
+    const entries = fs.readdirSync(directory, { withFileTypes: true })
+      .sort((left, right) => left.name.localeCompare(right.name));
+    for (const entry of entries) {
+      const relative = path.join(relativeDirectory, entry.name).replaceAll('\\', '/');
+      const absolute = path.join(directory, entry.name);
+      if (entry.isDirectory()) {
+        hash.update(`D\0${relative}\0`);
+        visit(absolute, relative);
+      } else if (entry.isFile()) {
+        hash.update(`F\0${relative}\0`);
+        hash.update(fs.readFileSync(absolute));
+        hash.update('\0');
+      } else if (entry.isSymbolicLink()) {
+        hash.update(`L\0${relative}\0${fs.readlinkSync(absolute)}\0`);
+      } else {
+        hash.update(`X\0${relative}\0`);
+      }
+    }
+  };
+  visit(root, '');
+  return hash.digest('hex');
 }