test(server): move the remaining server tests onto the memory contract (P1.5)

- Workspace sessions, fleet runs, agent groups, loops, the file indexer,
  external tool runs, chat and the health endpoint are checked against the
  memory store instead of .mind files. Assertions on mind-cache pins and
  releases go with the pins; the drain and cancellation checks stay.
- Delete the sticky-erasure test (harvest import was removed) and the
  session-distillation test (distillation on workspace activation was
  removed).
- The saved-history retry test checks the conversation only: the failed
  user turn is kept in workspace memory by design (#3).
This commit is contained in:
shumeiko.it
2026-10-24 12:50:23 +02:00
parent 74a4e55be0
commit 122b45990e
12 changed files with 274 additions and 785 deletions

View File

@@ -3,7 +3,13 @@ import Fastify from 'fastify';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { FrameStore, MindDB, MultiMindCache } from '@waggle/core';
import {
createInMemoryMemoryStore,
PERSONAL_SCOPE,
workspaceScope,
type MemoryScope,
type MemoryStore,
} from '@waggle/memory-contract';
import { createWorkflowTools, type AgentLoopConfig, type AgentResponse, type ToolDefinition } from '@waggle/agent';
import { AgentRunRegistry } from '../../src/local/agent-run-registry.js';
import {
@@ -71,6 +77,12 @@ async function waitFor(predicate: () => boolean, message: string): Promise<void>
throw new Error(message);
}
/** What a scope's memories hold: text and metadata, oldest first. */
async function memoryContents(memory: MemoryStore, scope: MemoryScope): Promise<unknown[]> {
const records = await memory.list({ scopes: [scope], limit: 500 });
return records.reverse().map((record) => ({ text: record.text, metadata: record.metadata }));
}
describe('local agent group execution', () => {
let server: ReturnType<typeof Fastify> | undefined;
@@ -217,7 +229,7 @@ describe('local agent group execution', () => {
expect(synthesizer.tools).toEqual([]);
});
it('runs a parallel group in one durable Room with Dance events and two-mind result attribution', async () => {
it('runs a parallel group in one durable Room with Dance events and two-scope result attribution', async () => {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-agent-group-room-'));
const workspaceDir = path.join(dataDir, 'project');
const localModel = 'ollama/qwen2.5:0.5b';
@@ -232,8 +244,7 @@ describe('local agent group execution', () => {
fs.mkdirSync(workspaceDir);
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const signalBus = new SignalBus();
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const calls: Array<{ config: AgentLoopConfig; finish: ReturnType<typeof deferred<AgentResponse>> }> = [];
const toolBuilds: Array<{ cwd: string; workspaceId?: string }> = [];
const sessionOnlyTool = {
@@ -250,7 +261,7 @@ describe('local agent group execution', () => {
server.decorate('localJobStore', new LocalJobStore());
server.decorate('agentRunRegistry', registry);
server.decorate('signalBus', signalBus);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('workspaceManager', {
getDefault: () => 'workspace-1',
list: () => [{ id: 'workspace-1' }],
@@ -261,10 +272,6 @@ describe('local agent group execution', () => {
}
: undefined,
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => {},
} as never);
server.decorate('agentState', {
allTools: [],
currentModel: 'anthropic/cloud-default',
@@ -286,8 +293,6 @@ describe('local agent group execution', () => {
});
server.addHook('onClose', async () => {
registry.close();
personalMind.close();
workspaceMind.close();
});
server.register(agentGroupRoutes);
server.register(localJobRoutes);
@@ -333,7 +338,7 @@ describe('local agent group execution', () => {
await waitFor(
() => startBody.runIds.every((id) => registry.get(id)?.memoryRefs.status === 'complete')
&& registry.get(startBody.roomId)?.memoryRefs.status === 'complete',
'worker and Room results were not attributed to both minds',
'worker and Room results were not attributed to both scopes',
);
const room = registry.get(startBody.roomId)!;
const workerRuns = startBody.runIds.map((id) => registry.get(id)!);
@@ -342,8 +347,6 @@ describe('local agent group execution', () => {
expect(room.status).toBe('completed');
expect(room.memoryRefs.status).toBe('complete');
const personalStore = new FrameStore(personalMind);
const workspaceStore = new FrameStore(workspaceMind);
const workerPersonalIds = workerRuns.flatMap((run) => run.memoryRefs.personalFrameIds);
const workerWorkspaceIds = workerRuns.flatMap((run) => run.memoryRefs.workspaceFrameIds['workspace-1'] ?? []);
const roomPersonalIds = room.memoryRefs.personalFrameIds;
@@ -358,22 +361,24 @@ describe('local agent group execution', () => {
for (const run of workerRuns) {
const expected = run.executor.personaId === 'researcher' ? 'Research result' : 'Draft result';
const other = expected === 'Research result' ? 'Draft result' : 'Research result';
const personalFrame = personalStore.getById(run.memoryRefs.personalFrameIds[0]!);
const workspaceFrame = workspaceStore.getById(run.memoryRefs.workspaceFrameIds['workspace-1']![0]!);
expect(personalFrame?.content).toContain(`Run: ${run.id}`);
expect(personalFrame?.content).toContain(expected);
expect(personalFrame?.content).not.toContain(other);
expect(workspaceFrame?.content).toContain(`Run: ${run.id}`);
expect(workspaceFrame?.content).toContain(expected);
expect(workspaceFrame?.content).not.toContain(other);
const personalMemory = await memory.get(run.memoryRefs.personalFrameIds[0]!);
const workspaceMemory = await memory.get(run.memoryRefs.workspaceFrameIds['workspace-1']![0]!);
expect(personalMemory?.scope).toEqual(PERSONAL_SCOPE);
expect(personalMemory?.text).toContain(`Run: ${run.id}`);
expect(personalMemory?.text).toContain(expected);
expect(personalMemory?.text).not.toContain(other);
expect(workspaceMemory?.scope).toEqual(workspaceScope('workspace-1'));
expect(workspaceMemory?.text).toContain(`Run: ${run.id}`);
expect(workspaceMemory?.text).toContain(expected);
expect(workspaceMemory?.text).not.toContain(other);
}
const roomPersonalFrame = personalStore.getById(roomPersonalIds[0]!);
const roomWorkspaceFrame = workspaceStore.getById(roomWorkspaceIds[0]!);
expect(roomPersonalFrame?.content).toContain('Research result');
expect(roomPersonalFrame?.content).toContain('Draft result');
expect(roomWorkspaceFrame?.content).toContain('Research result');
expect(roomWorkspaceFrame?.content).toContain('Draft result');
const roomPersonalMemory = await memory.get(roomPersonalIds[0]!);
const roomWorkspaceMemory = await memory.get(roomWorkspaceIds[0]!);
expect(roomPersonalMemory?.text).toContain('Research result');
expect(roomPersonalMemory?.text).toContain('Draft result');
expect(roomWorkspaceMemory?.text).toContain('Research result');
expect(roomWorkspaceMemory?.text).toContain('Draft result');
expect(signalBus.query({ teamId: `room::${startBody.roomId}` }).filter((signal) => signal.subtype === 'routed_share')).toHaveLength(2);
const runCountBeforeUnavailableModel = registry.snapshot().runs.length;
@@ -424,8 +429,7 @@ describe('local agent group execution', () => {
directory: workspaceDir, model: 'test-model',
}),
} as never);
server.decorate('mindCache', { acquire: () => ({}), release: () => {} } as never);
server.decorate('multiMind', { personal: {} } as never);
server.decorate('memory', createInMemoryMemoryStore());
server.decorate('agentState', {
allTools: checkoutTools,
currentModel: 'test-model',
@@ -618,8 +622,7 @@ describe('local agent group execution', () => {
directory: workspaceDir, model: 'test-model',
}),
} as never);
server.decorate('mindCache', { acquire: () => ({}), release: () => {} } as never);
server.decorate('multiMind', { personal: {} } as never);
server.decorate('memory', createInMemoryMemoryStore());
server.decorate('agentState', {
allTools: checkoutTools,
currentModel: 'test-model',
@@ -701,8 +704,7 @@ describe('local agent group execution', () => {
const registryPath = path.join(dataDir, 'agent-runs.json');
const registry = new AgentRunRegistry(registryPath);
const signalBus = new SignalBus();
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const calls: Array<{ finish: ReturnType<typeof deferred<AgentResponse>> }> = [];
const autoSaves: Array<{ user: string; assistant: string }> = [];
@@ -713,7 +715,7 @@ describe('local agent group execution', () => {
server.decorate('localJobStore', new LocalJobStore());
server.decorate('agentRunRegistry', registry);
server.decorate('signalBus', signalBus);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('workspaceManager', {
getDefault: () => 'workspace-1',
list: () => [{ id: 'workspace-1' }],
@@ -722,10 +724,6 @@ describe('local agent group execution', () => {
directory: workspaceDir, model: 'test-model',
}),
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => {},
} as never);
server.decorate('agentState', {
allTools: [],
currentModel: 'fallback-model',
@@ -746,8 +744,6 @@ describe('local agent group execution', () => {
});
server.addHook('onClose', async () => {
registry.close();
personalMind.close();
workspaceMind.close();
});
server.register(agentGroupRoutes);
server.register(localJobRoutes);
@@ -815,14 +811,8 @@ describe('local agent group execution', () => {
const jobProjection = JSON.stringify(job.output);
const registryProjection = fs.readFileSync(registryPath, 'utf-8');
const danceProjection = JSON.stringify(signalBus.query({ teamId: `room::${body.roomId}` }));
const personalProjection = JSON.stringify({
frames: personalMind.getDatabase().prepare('SELECT content FROM memory_frames ORDER BY id').all(),
fts: personalMind.getDatabase().prepare('SELECT content FROM memory_frames_fts ORDER BY rowid').all(),
});
const workspaceProjection = JSON.stringify({
frames: workspaceMind.getDatabase().prepare('SELECT content FROM memory_frames ORDER BY id').all(),
fts: workspaceMind.getDatabase().prepare('SELECT content FROM memory_frames_fts ORDER BY rowid').all(),
});
const personalProjection = JSON.stringify(await memoryContents(memory, PERSONAL_SCOPE));
const workspaceProjection = JSON.stringify(await memoryContents(memory, workspaceScope('workspace-1')));
const autoSaveProjection = JSON.stringify(autoSaves);
expect(jobProjection).toContain('[Quarantined agent result: unsafe external content]');
expect(jobProjection).toContain('[Quarantined agent error: unsafe external content]');
@@ -870,8 +860,7 @@ describe('local agent group execution', () => {
directory: workspaceDir, model: 'test-model',
}),
} as never);
server.decorate('mindCache', { acquire: () => ({}), release: () => {} } as never);
server.decorate('multiMind', { personal: {} } as never);
server.decorate('memory', createInMemoryMemoryStore());
server.decorate('agentState', {
allTools: [], currentModel: 'model', litellmApiKey: 'key', hookRegistry: undefined,
spawnSecurityContext: null,
@@ -910,17 +899,12 @@ describe('local agent group execution', () => {
expect(calls.every((call) => call.signal?.aborted)).toBe(true);
});
it('does not resolve Room cancellation before group workers release the real workspace mind pin', async () => {
it('does not resolve Room cancellation before group workers drain', async () => {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-agent-group-cancel-drain-'));
const workspaceDir = path.join(dataDir, 'project');
fs.mkdirSync(workspaceDir);
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const mindCache = new MultiMindCache({
maxOpen: 1,
allowedRoot: dataDir,
getMindPath: (workspaceId) => path.join(dataDir, `${workspaceId}.mind`),
});
const personalMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const firstRunnerMayFinish = deferred<void>();
const activeAbortObserved = deferred<void>();
const activeRunnerMayFinish = deferred<void>();
@@ -952,8 +936,7 @@ describe('local agent group execution', () => {
model: 'test-model',
}),
} as never);
server.decorate('mindCache', mindCache);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('agentState', {
allTools: checkoutTools,
currentModel: 'test-model',
@@ -988,8 +971,6 @@ describe('local agent group execution', () => {
});
server.addHook('onClose', async () => {
registry.close();
mindCache.closeAll();
personalMind.close();
});
server.register(agentGroupRoutes);
@@ -1051,9 +1032,6 @@ describe('local agent group execution', () => {
.toEqual(['cancelling', 'completed']);
expect(competingAcquired).toBe(false);
mindCache.getOrOpen('pressure-1');
expect(mindCache.has('workspace-1')).toBe(true);
activeRunnerMayFinish.resolve(undefined);
await Promise.all([cancel, concurrentCancel]);
expect(concurrentCancelError).toBeUndefined();
@@ -1067,9 +1045,6 @@ describe('local agent group execution', () => {
expect(terminalRoomEvents).toEqual(['cancelled']);
await competing;
expect(competingAcquired).toBe(true);
mindCache.getOrOpen('pressure-2');
expect(mindCache.has('workspace-1')).toBe(false);
} finally {
firstRunnerMayFinish.resolve(undefined);
activeRunnerMayFinish.resolve(undefined);
@@ -1083,11 +1058,10 @@ describe('local agent group execution', () => {
const workspaceDir = path.join(dataDir, 'project');
fs.mkdirSync(workspaceDir);
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const autoSaveEntered = deferred<void>();
const autoSaveMayFinish = deferred<void>();
let workspaceMindReleased = false;
let autoSaveFinished = false;
server = Fastify({ logger: false });
server.decorate('localConfig', {
@@ -1107,11 +1081,7 @@ describe('local agent group execution', () => {
model: 'test-model',
}),
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => { workspaceMindReleased = true; },
} as never);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('agentState', {
allTools: [],
currentModel: 'test-model',
@@ -1122,6 +1092,7 @@ describe('local agent group execution', () => {
autoSaveFromExchange: async () => {
autoSaveEntered.resolve(undefined);
await autoSaveMayFinish.promise;
autoSaveFinished = true;
},
}),
buildToolsForSession: () => [],
@@ -1135,8 +1106,6 @@ describe('local agent group execution', () => {
}));
server.addHook('onClose', async () => {
registry.close();
workspaceMind.close();
personalMind.close();
});
server.register(agentGroupRoutes);
server.register(localJobRoutes);
@@ -1177,25 +1146,22 @@ describe('local agent group execution', () => {
expect(registry.get(body.roomId)?.status).toBe('completed');
} finally {
autoSaveMayFinish.resolve(undefined);
await waitFor(() => workspaceMindReleased, 'group execution did not release the workspace mind');
await waitFor(() => autoSaveFinished, 'group execution did not finish its autosave');
}
});
it('waits for active group workers and workspace release before server shutdown cleanup', async () => {
it('waits for active group workers before server shutdown cleanup', async () => {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-agent-group-shutdown-drain-'));
const workspaceDir = path.join(dataDir, 'project');
fs.mkdirSync(workspaceDir);
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const runnersMayFinish = deferred<void>();
const allRunnersAborted = deferred<void>();
let activeWorkers = 0;
let abortedWorkers = 0;
let workspaceMindReleaseCount = 0;
let cleanupHookRan = false;
let workersAtCleanup = -1;
let mindReleasedAtCleanup = false;
let closeSettled = false;
let closing: Promise<void> | undefined;
@@ -1217,11 +1183,7 @@ describe('local agent group execution', () => {
model: 'test-model',
}),
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => { workspaceMindReleaseCount += 1; },
} as never);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('agentState', {
allTools: [],
currentModel: 'test-model',
@@ -1255,7 +1217,6 @@ describe('local agent group execution', () => {
server!.localJobStore.close();
cleanupHookRan = true;
workersAtCleanup = activeWorkers;
mindReleasedAtCleanup = workspaceMindReleaseCount === 2;
});
try {
@@ -1298,17 +1259,10 @@ describe('local agent group execution', () => {
expect(activeWorkers).toBe(0);
expect(cleanupHookRan).toBe(true);
expect(workersAtCleanup).toBe(0);
expect(mindReleasedAtCleanup).toBe(true);
} finally {
runnersMayFinish.resolve(undefined);
if (closing) await closing.catch(() => undefined);
await waitFor(
() => workspaceMindReleaseCount === 2,
'group executions did not release both mind references during shutdown',
);
registry.close();
workspaceMind.close();
personalMind.close();
}
});
@@ -1318,14 +1272,12 @@ describe('local agent group execution', () => {
fs.mkdirSync(workspaceDir);
const localModel = 'ollama/qwen2.5:0.5b';
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const modelLookupStarted = deferred<void>();
const modelLookupMayFinish = deferred<void>();
const jobStore = new LocalJobStore();
const createJob = vi.spyOn(jobStore, 'create');
let runnerCalls = 0;
let workspaceMindReleased = false;
let closeSettled = false;
let closing: Promise<void> | undefined;
process.env.OLLAMA_HOST = 'http://127.0.0.1:11461';
@@ -1358,11 +1310,7 @@ describe('local agent group execution', () => {
model: localModel,
}),
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => { workspaceMindReleased = true; },
} as never);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('agentState', {
allTools: [],
currentModel: localModel,
@@ -1419,17 +1367,11 @@ describe('local agent group execution', () => {
expect(runnerCalls).toBe(0);
expect(createJob).not.toHaveBeenCalled();
expect(registry.snapshot().runs).toHaveLength(0);
expect(workspaceMindReleased).toBe(false);
} finally {
modelLookupMayFinish.resolve(undefined);
if (pendingStart) await pendingStart.catch(() => undefined);
if (closing) await closing.catch(() => undefined);
if (runnerCalls > 0) {
await waitFor(() => workspaceMindReleased, 'late group execution did not release its workspace mind');
}
registry.close();
workspaceMind.close();
personalMind.close();
}
});
@@ -1438,12 +1380,20 @@ describe('local agent group execution', () => {
const workspaceDir = path.join(dataDir, 'project');
fs.mkdirSync(workspaceDir);
const registry = new AgentRunRegistry(path.join(dataDir, 'agent-runs.json'));
const personalMind = new MindDB(':memory:');
const workspaceMind = new MindDB(':memory:');
const memory = createInMemoryMemoryStore();
const slowWorkersMayFinish = deferred<void>();
const firstReleaseFailed = deferred<void>();
const firstCleanupFailed = deferred<void>();
let activeWorkers = 0;
let releaseCalls = 0;
let cleanupCalls = 0;
const finalizeRoomCancellation = registry.finalizeRoomCancellation.bind(registry);
vi.spyOn(registry, 'finalizeRoomCancellation').mockImplementation((roomId) => {
cleanupCalls += 1;
if (cleanupCalls === 1) {
firstCleanupFailed.resolve(undefined);
throw new Error('simulated Room cleanup failure');
}
return finalizeRoomCancellation(roomId);
});
let cleanupHookRan = false;
let closeSettled = false;
let closeError: unknown;
@@ -1467,17 +1417,7 @@ describe('local agent group execution', () => {
model: 'test-model',
}),
} as never);
server.decorate('mindCache', {
acquire: () => workspaceMind,
release: () => {
releaseCalls += 1;
if (releaseCalls === 1) {
firstReleaseFailed.resolve(undefined);
throw new Error('simulated workspace release failure');
}
},
} as never);
server.decorate('multiMind', { personal: personalMind } as never);
server.decorate('memory', memory);
server.decorate('agentState', {
allTools: [],
currentModel: 'test-model',
@@ -1515,8 +1455,6 @@ describe('local agent group execution', () => {
cleanupHookRan = true;
server!.localJobStore.close();
registry.close();
workspaceMind.close();
personalMind.close();
});
const created = await server.inject({
@@ -1555,11 +1493,11 @@ describe('local agent group execution', () => {
closeError = error;
},
);
await firstReleaseFailed.promise;
await firstCleanupFailed.promise;
await new Promise((resolve) => setTimeout(resolve, 0));
expect(closeSettled).toBe(false);
expect(cleanupHookRan).toBe(false);
expect(releaseCalls).toBe(1);
expect(cleanupCalls).toBe(1);
expect(activeWorkers).toBe(2);
slowWorkersMayFinish.resolve(undefined);
@@ -1568,7 +1506,7 @@ describe('local agent group execution', () => {
expect((closeError as Error).message)
.toContain('Agent group execution cleanup failed during shutdown');
expect(activeWorkers).toBe(0);
expect(releaseCalls).toBe(2);
expect(cleanupCalls).toBe(2);
expect(cleanupHookRan).toBe(true);
} finally {
slowWorkersMayFinish.resolve(undefined);