Files
waggle-os/packages/server/tests/workspace-sessions-concurrency.test.ts
shumeiko.it 122b45990e test(server): move the remaining server tests onto the memory contract (P1.5)
- Workspace sessions, fleet runs, agent groups, loops, the file indexer,
  external tool runs, chat and the health endpoint are checked against the
  memory store instead of .mind files. Assertions on mind-cache pins and
  releases go with the pins; the drain and cancellation checks stay.
- Delete the sticky-erasure test (harvest import was removed) and the
  session-distillation test (distillation on workspace activation was
  removed).
- The saved-history retry test checks the conversation only: the failed
  user turn is kept in workspace memory by design (#3).
2026-10-24 13:19:06 +02:00

518 lines
20 KiB
TypeScript

/**
* WorkspaceSessionManager concurrency test.
*
* Proves the Phase A.1 invariants: per-session orchestrator instances
* cannot corrupt each other, even when two sessions are "active" at the
* same time on different workspaces.
*
* Before A.1, the sidecar held a single `orchestrator` whose workspace
* got overwritten on every switch. Two chat requests targeting different
* workspaces would interleave and silently corrupt each other. Memory is
* one shared store; each session's orchestrator selects its own scope.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import path from 'node:path';
import os from 'node:os';
import fs from 'node:fs';
import { WaggleDB } from '@waggle/core';
import { createInMemoryMemoryStore, workspaceScope, type MemoryStore } from '@waggle/memory-contract';
import { Orchestrator } from '@waggle/agent';
import { buildLocalServer } from '../src/local/index.js';
import { chatSessionStateKey } from '../src/local/routes/chat-persistence.js';
import { WorkspaceSessionManager } from '../src/local/workspace-sessions.js';
import { injectWithAuth, resetRateLimiter } from './test-utils.js';
// ── Tests ────────────────────────────────────────────────────────────
describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
let memory: MemoryStore;
let waggleDb: WaggleDB;
function buildOrchestrator(workspaceId: string): Orchestrator {
const orch = new Orchestrator({ memory, waggleDb, mode: 'local', version: 'test' });
orch.setWorkspace(workspaceId);
return orch;
}
beforeEach(() => {
memory = createInMemoryMemoryStore();
waggleDb = new WaggleDB(':memory:');
});
afterEach(() => {
waggleDb.close();
});
it('keeps per-session orchestrators isolated from each other', () => {
const manager = new WorkspaceSessionManager(3);
const sessionA = manager.create('workspace-A', buildOrchestrator('workspace-A'), []);
const sessionB = manager.create('workspace-B', buildOrchestrator('workspace-B'), []);
// The invariant: session A's orchestrator must hold workspace A,
// session B's must hold workspace B. In the broken pre-A.1 code they'd
// both point at whichever workspace was set last.
expect(sessionA.orchestrator).not.toBe(sessionB.orchestrator);
expect(sessionA.orchestrator.getWorkspaceId()).toBe('workspace-A');
expect(sessionB.orchestrator.getWorkspaceId()).toBe('workspace-B');
expect(manager.size).toBe(2);
});
it('allows a third concurrent session but blocks the fourth at default cap', () => {
const manager = new WorkspaceSessionManager(3);
const build = (label: string) => manager.create(label, buildOrchestrator(label), []);
build('A');
build('B');
build('C');
expect(manager.size).toBe(3);
expect(() => build('D')).toThrow(/Max concurrent sessions reached/);
});
it('honors setMaxSessions raises and subsequent creates', () => {
const manager = new WorkspaceSessionManager(2);
expect(manager.getMaxSessions()).toBe(2);
const build = (label: string) => manager.create(label, buildOrchestrator(label), []);
build('A');
build('B');
expect(() => build('C')).toThrow(/Max concurrent sessions reached/);
manager.setMaxSessions(4);
expect(manager.getMaxSessions()).toBe(4);
// Now the third and fourth sessions succeed
build('C');
build('D');
expect(manager.size).toBe(4);
});
it('rejects invalid setMaxSessions values', () => {
const manager = new WorkspaceSessionManager(3);
expect(() => manager.setMaxSessions(0)).toThrow(/finite integer/);
expect(() => manager.setMaxSessions(-1)).toThrow(/finite integer/);
expect(() => manager.setMaxSessions(NaN)).toThrow(/finite integer/);
});
it('pause and resume flip status without affecting other sessions', () => {
const manager = new WorkspaceSessionManager(3);
const sessA = manager.create('A', buildOrchestrator('A'), []);
const sessB = manager.create('B', buildOrchestrator('B'), []);
expect(sessA.status).toBe('active');
expect(sessB.status).toBe('active');
expect(manager.pause('A')).toBe(true);
expect(sessA.status).toBe('paused');
expect(sessB.status).toBe('active'); // unaffected
expect(manager.resume('A')).toBe(true);
expect(sessA.status).toBe('active');
// Double-pause returns true but status is idempotent
expect(manager.pause('A')).toBe(true);
expect(sessA.status).toBe('paused');
// Resume on a non-paused session returns false
expect(manager.resume('B')).toBe(false);
});
it('close() closes one session cleanly and leaves others intact', () => {
const manager = new WorkspaceSessionManager(3);
manager.create('A', buildOrchestrator('A'), []);
const sessB = manager.create('B', buildOrchestrator('B'), []);
expect(manager.size).toBe(2);
expect(manager.close('A')).toBe(true);
expect(manager.size).toBe(1);
expect(manager.has('A')).toBe(false);
expect(manager.has('B')).toBe(true);
expect(sessB.status).toBe('active');
// Closing an already-closed session returns false
expect(manager.close('A')).toBe(false);
});
it('getOrCreate reuses existing sessions with factories called lazily', () => {
const manager = new WorkspaceSessionManager(3);
let orchFactoryCallCount = 0;
let toolsFactoryCallCount = 0;
const factories = {
orch: () => { orchFactoryCallCount++; return buildOrchestrator('A'); },
tools: (_o: Orchestrator) => { toolsFactoryCallCount++; return []; },
};
const first = manager.getOrCreate('A', factories.orch, factories.tools);
expect(orchFactoryCallCount).toBe(1);
expect(toolsFactoryCallCount).toBe(1);
// Second call should return the SAME session, not invoke factories
const second = manager.getOrCreate('A', factories.orch, factories.tools);
expect(second).toBe(first);
expect(orchFactoryCallCount).toBe(1);
expect(toolsFactoryCallCount).toBe(1);
// Touch should update lastActivity
const firstActivity = second.lastActivity;
// A tiny delay is not guaranteed by the test runner, so we don't assert
// strict inequality — just that getOrCreate didn't error.
expect(firstActivity).toBeGreaterThan(0);
});
it('closeAll clears every session at once', () => {
const manager = new WorkspaceSessionManager(4);
manager.create('A', buildOrchestrator('A'), []);
manager.create('B', buildOrchestrator('B'), []);
manager.create('C', buildOrchestrator('C'), []);
expect(manager.size).toBe(3);
manager.closeAll();
expect(manager.size).toBe(0);
expect(manager.has('A')).toBe(false);
expect(manager.has('B')).toBe(false);
expect(manager.has('C')).toBe(false);
});
it('two concurrent orchestrators on different workspaces do not cross-contaminate', async () => {
const manager = new WorkspaceSessionManager(3);
// Seed each workspace with content specific to that workspace
await memory.remember({
scope: workspaceScope('A'), text: 'A-specific: blueprint for launch campaign.', importance: 'important',
});
await memory.remember({
scope: workspaceScope('B'), text: 'B-specific: competitor analysis for fintech vertical.', importance: 'important',
});
const orchA = buildOrchestrator('A');
const orchB = buildOrchestrator('B');
manager.create('A', orchA, []);
manager.create('B', orchB, []);
// Fire two recalls in parallel. The invariant: each orchestrator only
// sees its own workspace's memories. On the pre-A.1 shared-orchestrator
// code, one would overwrite the other before both resolve.
const [recallA, recallB] = await Promise.all([
orchA.recallMemory('launch campaign blueprint', 5),
orchB.recallMemory('fintech competitor analysis', 5),
]);
const recalledA = (recallA.recalled ?? []).join('\n');
const recalledB = (recallB.recalled ?? []).join('\n');
expect(recalledA).toContain('A-specific');
expect(recalledA).not.toContain('B-specific');
expect(recalledB).toContain('B-specific');
expect(recalledB).not.toContain('A-specific');
// And crucially: after both recalls, both orchestrators still hold the
// RIGHT workspace (not crossed over).
expect(orchA.getWorkspaceId()).toBe('A');
expect(orchB.getWorkspaceId()).toBe('B');
expect((await orchA.getMemoryStats()).memoryCount).toBe(1);
expect((await orchB.getMemoryStats()).memoryCount).toBe(1);
});
it('composes the session cap, concurrent same-workspace planning isolation, and cleanup', async () => {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-solo-session-composition-'));
const server = await buildLocalServer({ dataDir });
// A small cap keeps the overflow check cheap; the sidecar default is MAX_WORKSPACE_SESSIONS.
server.sessionManager.setMaxSessions(10);
const workspaceId = server.agentState.activeWorkspaceId;
expect(workspaceId).toBeTruthy();
const sessionA = 'solo-concurrent-a';
const sessionB = 'solo-concurrent-b';
const markerA = 'SOLO_SESSION_A';
const markerB = 'SOLO_SESSION_B';
const previousOllamaHost = process.env.OLLAMA_HOST;
const previousReranker = process.env.WAGGLE_RERANKER;
const createdWorkspaceIds: string[] = [workspaceId!];
const deferred = () => {
let resolve!: () => void;
const promise = new Promise<void>((done) => { resolve = done; });
return { promise, resolve };
};
const aFirstProviderArrived = deferred();
const releaseAFirstProvider = deferred();
const providerOrder: string[] = [];
const providerBodies = new Map<'A' | 'B', Array<{
messages?: Array<{ role?: string; content?: string }>;
}>>([['A', []], ['B', []]]);
let requestA: Promise<Awaited<ReturnType<typeof injectWithAuth>>> | undefined;
let requestB: Promise<Awaited<ReturnType<typeof injectWithAuth>>> | undefined;
const coordinator = server.agentState.workspaceTurnCoordinator;
const acquireSpy = vi.spyOn(coordinator, 'acquire');
const streamResponse = (chunks: unknown[]) => new Response(
`${chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join('')}data: [DONE]\n\n`,
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
);
const toolResponse = (id: string, name: string, args: Record<string, unknown>) => streamResponse([
{
choices: [{
delta: {
tool_calls: [{
index: 0,
id,
type: 'function',
function: { name, arguments: JSON.stringify(args) },
}],
},
}],
},
{
choices: [{ delta: {}, finish_reason: 'tool_calls' }],
usage: { prompt_tokens: 10, completion_tokens: 2 },
},
]);
const finalResponse = (content: string) => streamResponse([
{ choices: [{ delta: { content } }] },
{
choices: [{ delta: {}, finish_reason: 'stop' }],
usage: { prompt_tokens: 10, completion_tokens: 2 },
},
]);
const stepPhases = (body: string): string[] => body
.split(/\n\n/)
.filter(block => block.split('\n').includes('event: step'))
.flatMap(block => {
const dataLine = block.split('\n').find(line => line.startsWith('data: '));
if (!dataLine) return [];
const event = JSON.parse(dataLine.slice(6)) as { phase?: unknown };
return typeof event.phase === 'string' ? [event.phase] : [];
});
server.agentState.llmProvider = {
provider: 'ollama',
health: 'healthy',
detail: 'Deterministic Solo concurrency fixture',
checkedAt: new Date().toISOString(),
};
server.agentState.currentModel = 'ollama/solo-local';
server.localConfig.litellmUrl = 'http://proxy.test/v1';
process.env.OLLAMA_HOST = 'http://ollama.test';
process.env.WAGGLE_RERANKER = '0';
resetRateLimiter(server);
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => {
const url = String(input);
if (url.endsWith('/api/tags')) {
return new Response(JSON.stringify({ models: [{ name: 'solo-local' }] }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
if (!url.endsWith('/chat/completions')) return new Response('', { status: 503 });
const body = JSON.parse(String(init?.body ?? '{}')) as {
messages?: Array<{ role?: string; content?: string }>;
};
const contents = (body.messages ?? []).map(message => message.content ?? '').join('\n');
const logicalSession = contents.includes(markerA)
? 'A'
: contents.includes(markerB) ? 'B' : null;
if (!logicalSession) throw new Error('Provider request did not contain a Solo session marker');
const bodies = providerBodies.get(logicalSession)!;
bodies.push(body);
const call = bodies.length;
providerOrder.push(`${logicalSession}#${call}`);
if (logicalSession === 'A' && call === 1) {
aFirstProviderArrived.resolve();
await releaseAFirstProvider.promise;
return toolResponse('solo-a-create', 'create_plan', { title: 'Plan A' });
}
if (logicalSession === 'A' && call === 2) {
return toolResponse('solo-a-add', 'add_plan_step', { title: 'A_ONLY' });
}
if (logicalSession === 'A' && call === 3) {
return finalResponse(`${markerA} complete`);
}
if (logicalSession === 'B' && call === 1) {
return toolResponse('solo-b-create', 'create_plan', { title: 'Plan B' });
}
if (logicalSession === 'B' && call === 2) {
return toolResponse('solo-b-show', 'show_plan', {});
}
if (logicalSession === 'B' && call === 3) {
return finalResponse(`${markerB} complete`);
}
throw new Error(`Unexpected provider call ${logicalSession}#${call}`);
});
try {
expect(server.sessionManager.getMaxSessions()).toBe(10);
expect(server.sessionManager.size).toBe(0);
requestA = injectWithAuth(server, {
method: 'POST',
url: '/api/chat',
payload: {
workspace: workspaceId,
session: sessionA,
persona: 'project-manager',
model: 'ollama/solo-local',
autonomy: { level: 'yolo' },
message: `Use create_plan, then add_plan_step with A_ONLY. Correlation: ${markerA}.`,
},
});
await Promise.race([
aFirstProviderArrived.promise,
requestA.then(() => { throw new Error('Session A completed before reaching the provider'); }),
]);
requestB = injectWithAuth(server, {
method: 'POST',
url: '/api/chat',
payload: {
workspace: workspaceId,
session: sessionB,
persona: 'project-manager',
model: 'ollama/solo-local',
autonomy: { level: 'yolo' },
message: `Execute create_plan for Plan B, then execute show_plan immediately. Correlation: ${markerB}.`,
},
});
let bCompletionTimeout: ReturnType<typeof setTimeout> | undefined;
const responseB = await Promise.race([
requestB,
new Promise<never>((_, reject) => {
bCompletionTimeout = setTimeout(
() => reject(new Error('Independent Session B did not complete concurrently')),
5_000,
);
}),
]).finally(() => {
if (bCompletionTimeout) clearTimeout(bCompletionTimeout);
});
expect(responseB.statusCode).toBe(200);
expect(responseB.body).toContain(`${markerB} complete`);
expect(providerBodies.get('B')).toHaveLength(3);
expect(providerOrder).toEqual(['A#1', 'B#1', 'B#2', 'B#3']);
expect(acquireSpy).not.toHaveBeenCalled();
releaseAFirstProvider.resolve();
const responseA = await requestA;
expect(responseA.statusCode).toBe(200);
expect(responseA.body).toContain(`${markerA} complete`);
expect(providerOrder).toEqual(['A#1', 'B#1', 'B#2', 'B#3', 'A#2', 'A#3']);
const aPhases = stepPhases(responseA.body);
const bPhases = stepPhases(responseB.body);
expect(aPhases).not.toContain('workspace_queue');
expect(bPhases).not.toContain('workspace_queue');
expect(bPhases).not.toContain('workspace_acquired');
expect(server.sessionManager.size).toBe(1);
const bShowResult = providerBodies.get('B')![2]?.messages
?.filter(message => message.role === 'tool')
.at(-1)?.content;
expect(bShowResult).toContain('Plan has no steps.');
expect(bShowResult).not.toContain('A_ONLY');
expect(server.agentState.sessionHistories.get(chatSessionStateKey(workspaceId!, sessionA)))
.toEqual([
{ role: 'user', content: `Use create_plan, then add_plan_step with A_ONLY. Correlation: ${markerA}.` },
expect.objectContaining({ role: 'assistant', content: `${markerA} complete` }),
]);
expect(server.agentState.sessionHistories.get(chatSessionStateKey(workspaceId!, sessionB)))
.toEqual([
{ role: 'user', content: `Execute create_plan for Plan B, then execute show_plan immediately. Correlation: ${markerB}.` },
expect.objectContaining({ role: 'assistant', content: `${markerB} complete` }),
]);
for (let index = 1; index < 10; index += 1) {
const workspace = server.workspaceManager.create({
name: `Solo cap ${index}`,
group: 'Test',
});
createdWorkspaceIds.push(workspace.id);
const workspacePath = path.join(dataDir, 'workspaces', workspace.id, 'files');
server.sessionManager.getOrCreate(
workspace.id,
() => server.agentState.createSessionOrchestrator(workspace.id),
orchestrator => server.agentState.buildToolsForSession(
orchestrator,
workspacePath,
workspace.id,
),
);
}
expect(server.sessionManager.size).toBe(10);
expect(new Set(server.sessionManager.getActive().map(session => session.workspaceId)))
.toEqual(new Set(createdWorkspaceIds));
const fleet = await injectWithAuth(server, { method: 'GET', url: '/api/fleet' });
expect(fleet.statusCode).toBe(200);
expect(fleet.json()).toMatchObject({ count: 10, maxSessions: 10 });
const anchor = server.sessionManager.get(workspaceId!)!;
expect(() => server.sessionManager.create(
'solo-overflow',
anchor.orchestrator,
anchor.tools,
)).toThrow('Max concurrent sessions reached (10)');
expect(server.sessionManager.size).toBe(10);
for (const sessionId of [sessionA, sessionB]) {
const cleared = await injectWithAuth(server, {
method: 'DELETE',
url: `/api/chat/history?workspace=${workspaceId}&session=${sessionId}`,
});
expect(cleared.statusCode).toBe(200);
expect(server.agentState.sessionHistories.has(
chatSessionStateKey(workspaceId!, sessionId),
)).toBe(false);
expect(fs.existsSync(path.join(
dataDir,
'workspaces',
workspaceId!,
'sessions',
`${sessionId}.jsonl`,
))).toBe(false);
}
const killed = await injectWithAuth(server, {
method: 'POST',
url: `/api/fleet/${workspaceId}/kill`,
});
expect(killed.statusCode).toBe(200);
expect(killed.json()).toEqual({ killed: true, workspaceId });
expect(server.sessionManager.has(workspaceId!)).toBe(false);
for (const id of createdWorkspaceIds) {
const deleted = await injectWithAuth(server, {
method: 'DELETE',
url: `/api/workspaces/${id}`,
});
expect(deleted.statusCode).toBe(204);
}
expect(server.sessionManager.size).toBe(0);
} finally {
aFirstProviderArrived.resolve();
releaseAFirstProvider.resolve();
await Promise.allSettled([requestA, requestB].filter(Boolean) as Promise<unknown>[]);
fetchSpy.mockRestore();
acquireSpy.mockRestore();
await server.close();
if (previousOllamaHost === undefined) delete process.env.OLLAMA_HOST;
else process.env.OLLAMA_HOST = previousOllamaHost;
if (previousReranker === undefined) delete process.env.WAGGLE_RERANKER;
else process.env.WAGGLE_RERANKER = previousReranker;
await new Promise(resolve => setTimeout(resolve, 100));
fs.rmSync(dataDir, { recursive: true, force: true });
}
}, 30_000);
});