test(server): move the remaining server tests onto the memory contract (P1.5)

- Workspace sessions, fleet runs, agent groups, loops, the file indexer,
  external tool runs, chat and the health endpoint are checked against the
  memory store instead of .mind files. Assertions on mind-cache pins and
  releases go with the pins; the drain and cancellation checks stay.
- Delete the sticky-erasure test (harvest import was removed) and the
  session-distillation test (distillation on workspace activation was
  removed).
- The saved-history retry test checks the conversation only: the failed
  user turn is kept in workspace memory by design (#3).
This commit is contained in:
shumeiko.it
2026-10-24 12:50:23 +02:00
parent 74a4e55be0
commit 122b45990e
12 changed files with 274 additions and 785 deletions

View File

@@ -3,175 +3,93 @@
*
* Proves the Phase A.1 invariants: per-session orchestrator instances
* cannot corrupt each other, even when two sessions are "active" at the
* same time on different workspace minds.
* same time on different workspaces.
*
* Before A.1, the sidecar held a single `orchestrator` whose workspace
* layers got overwritten on every `setWorkspaceMind()` call. Two chat
* requests targeting different workspaces would interleave and silently
* corrupt each other. This test would fail on the pre-A.1 codebase.
* got overwritten on every switch. Two chat requests targeting different
* workspaces would interleave and silently corrupt each other. Memory is
* one shared store; each session's orchestrator selects its own scope.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { randomUUID } from 'node:crypto';
import path from 'node:path';
import os from 'node:os';
import fs from 'node:fs';
import { MindDB, type Embedder } from '@waggle/core';
import { WaggleDB } from '@waggle/core';
import { createInMemoryMemoryStore, workspaceScope, type MemoryStore } from '@waggle/memory-contract';
import { Orchestrator } from '@waggle/agent';
import { buildLocalServer } from '../src/local/index.js';
import { chatSessionStateKey } from '../src/local/routes/chat-persistence.js';
import { WorkspaceSessionManager } from '../src/local/workspace-sessions.js';
import { injectWithAuth, resetRateLimiter } from './test-utils.js';
// ── Deterministic fake embedder ──────────────────────────────────────
// Produces a fixed-dimension zero vector so sqlite-vec stays happy but
// semantic search always returns zero matches. We don't need vector
// similarity for these tests — we're checking that workspace *identity*
// routing works, and frame content can be checked via FTS5 instead.
class FakeEmbedder implements Embedder {
dimensions = 384;
async embed(_text: string): Promise<Float32Array> {
return new Float32Array(this.dimensions);
}
async embedBatch(texts: string[]): Promise<Float32Array[]> {
return texts.map(() => new Float32Array(this.dimensions));
}
}
// ── Test harness ─────────────────────────────────────────────────────
interface TestWorkspace {
id: string;
mind: MindDB;
mindPath: string;
}
function createTempMind(label: string): TestWorkspace {
const id = `${label}-${randomUUID().slice(0, 8)}`;
const mindPath = path.join(os.tmpdir(), `waggle-test-${id}.mind`);
const mind = new MindDB(mindPath);
return { id, mind, mindPath };
}
function cleanupMind(ws: TestWorkspace): void {
try { ws.mind.close(); } catch { /* already closed */ }
for (const suffix of ['', '-shm', '-wal']) {
try { fs.unlinkSync(ws.mindPath + suffix); } catch { /* missing is fine */ }
}
}
function seedFrame(ws: TestWorkspace, content: string, importance = 'normal'): void {
const db = ws.mind.getDatabase();
// Ensure a GOP session exists so the FK check passes
db.prepare(`
INSERT INTO sessions (gop_id, started_at)
VALUES (?, ?)
ON CONFLICT(gop_id) DO NOTHING
`).run('test', new Date().toISOString());
db.prepare(`
INSERT INTO memory_frames (frame_type, gop_id, t, content, importance, source)
VALUES ('I', ?, 0, ?, ?, 'user_stated')
`).run('test', content, importance);
}
function buildOrchestrator(personal: MindDB, workspace: MindDB): Orchestrator {
const orch = new Orchestrator({
db: personal,
embedder: new FakeEmbedder(),
mode: 'local',
version: 'test',
});
orch.setWorkspaceMind(workspace);
return orch;
}
// ── Tests ────────────────────────────────────────────────────────────
describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
let personal: TestWorkspace;
let wsA: TestWorkspace;
let wsB: TestWorkspace;
let wsC: TestWorkspace;
let wsD: TestWorkspace;
let memory: MemoryStore;
let waggleDb: WaggleDB;
function buildOrchestrator(workspaceId: string): Orchestrator {
const orch = new Orchestrator({ memory, waggleDb, mode: 'local', version: 'test' });
orch.setWorkspace(workspaceId);
return orch;
}
beforeEach(() => {
personal = createTempMind('personal');
wsA = createTempMind('wsA');
wsB = createTempMind('wsB');
wsC = createTempMind('wsC');
wsD = createTempMind('wsD');
memory = createInMemoryMemoryStore();
waggleDb = new WaggleDB(':memory:');
});
afterEach(() => {
[personal, wsA, wsB, wsC, wsD].forEach(cleanupMind);
waggleDb.close();
});
it('keeps per-session orchestrators isolated from each other', () => {
const manager = new WorkspaceSessionManager(3);
// Seed each workspace with distinctive content so we can prove routing.
seedFrame(wsA, 'Workspace A decided to use Postgres.', 'important');
seedFrame(wsB, 'Workspace B prefers MongoDB.', 'important');
const sessionA = manager.create('workspace-A', buildOrchestrator('workspace-A'), []);
const sessionB = manager.create('workspace-B', buildOrchestrator('workspace-B'), []);
const orchA = buildOrchestrator(personal.mind, wsA.mind);
const orchB = buildOrchestrator(personal.mind, wsB.mind);
const sessionA = manager.create('workspace-A', wsA.mind, orchA, [], null);
const sessionB = manager.create('workspace-B', wsB.mind, orchB, [], null);
// The invariant: session A's orchestrator must hold workspace A's layers,
// session B's must hold workspace B's. In the broken pre-A.1 code they'd
// The invariant: session A's orchestrator must hold workspace A,
// session B's must hold workspace B. In the broken pre-A.1 code they'd
// both point at whichever workspace was set last.
expect(sessionA.orchestrator).not.toBe(sessionB.orchestrator);
expect(sessionA.orchestrator.hasWorkspaceMind()).toBe(true);
expect(sessionB.orchestrator.hasWorkspaceMind()).toBe(true);
expect(sessionA.orchestrator.getWorkspaceId()).toBe('workspace-A');
expect(sessionB.orchestrator.getWorkspaceId()).toBe('workspace-B');
expect(manager.size).toBe(2);
});
it('allows a third concurrent session but blocks the fourth at default cap', () => {
const manager = new WorkspaceSessionManager(3);
const build = (label: string, ws: TestWorkspace) => manager.create(
label,
ws.mind,
buildOrchestrator(personal.mind, ws.mind),
[],
null,
);
const build = (label: string) => manager.create(label, buildOrchestrator(label), []);
build('A', wsA);
build('B', wsB);
build('C', wsC);
build('A');
build('B');
build('C');
expect(manager.size).toBe(3);
expect(() => build('D', wsD)).toThrow(/Max concurrent sessions reached/);
expect(() => build('D')).toThrow(/Max concurrent sessions reached/);
});
it('honors setMaxSessions raises and subsequent creates', () => {
const manager = new WorkspaceSessionManager(2);
expect(manager.getMaxSessions()).toBe(2);
const build = (label: string, ws: TestWorkspace) => manager.create(
label,
ws.mind,
buildOrchestrator(personal.mind, ws.mind),
[],
null,
);
const build = (label: string) => manager.create(label, buildOrchestrator(label), []);
build('A', wsA);
build('B', wsB);
expect(() => build('C', wsC)).toThrow(/Max concurrent sessions reached/);
build('A');
build('B');
expect(() => build('C')).toThrow(/Max concurrent sessions reached/);
manager.setMaxSessions(4);
expect(manager.getMaxSessions()).toBe(4);
// Now the third and fourth sessions succeed
build('C', wsC);
build('D', wsD);
build('C');
build('D');
expect(manager.size).toBe(4);
});
@@ -185,8 +103,8 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
it('pause and resume flip status without affecting other sessions', () => {
const manager = new WorkspaceSessionManager(3);
const sessA = manager.create('A', wsA.mind, buildOrchestrator(personal.mind, wsA.mind), [], null);
const sessB = manager.create('B', wsB.mind, buildOrchestrator(personal.mind, wsB.mind), [], null);
const sessA = manager.create('A', buildOrchestrator('A'), []);
const sessB = manager.create('B', buildOrchestrator('B'), []);
expect(sessA.status).toBe('active');
expect(sessB.status).toBe('active');
@@ -209,8 +127,8 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
it('close() closes one session cleanly and leaves others intact', () => {
const manager = new WorkspaceSessionManager(3);
manager.create('A', wsA.mind, buildOrchestrator(personal.mind, wsA.mind), [], null);
const sessB = manager.create('B', wsB.mind, buildOrchestrator(personal.mind, wsB.mind), [], null);
manager.create('A', buildOrchestrator('A'), []);
const sessB = manager.create('B', buildOrchestrator('B'), []);
expect(manager.size).toBe(2);
expect(manager.close('A')).toBe(true);
@@ -225,25 +143,21 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
it('getOrCreate reuses existing sessions with factories called lazily', () => {
const manager = new WorkspaceSessionManager(3);
let mindFactoryCallCount = 0;
let orchFactoryCallCount = 0;
let toolsFactoryCallCount = 0;
const factories = {
mind: () => { mindFactoryCallCount++; return wsA.mind; },
orch: (m: MindDB) => { orchFactoryCallCount++; return buildOrchestrator(personal.mind, m); },
tools: (_m: MindDB, _o: Orchestrator) => { toolsFactoryCallCount++; return []; },
orch: () => { orchFactoryCallCount++; return buildOrchestrator('A'); },
tools: (_o: Orchestrator) => { toolsFactoryCallCount++; return []; },
};
const first = manager.getOrCreate('A', factories.mind, factories.orch, factories.tools, null);
expect(mindFactoryCallCount).toBe(1);
const first = manager.getOrCreate('A', factories.orch, factories.tools);
expect(orchFactoryCallCount).toBe(1);
expect(toolsFactoryCallCount).toBe(1);
// Second call should return the SAME session, not invoke factories
const second = manager.getOrCreate('A', factories.mind, factories.orch, factories.tools, null);
const second = manager.getOrCreate('A', factories.orch, factories.tools);
expect(second).toBe(first);
expect(mindFactoryCallCount).toBe(1);
expect(orchFactoryCallCount).toBe(1);
expect(toolsFactoryCallCount).toBe(1);
@@ -257,9 +171,9 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
it('closeAll clears every session at once', () => {
const manager = new WorkspaceSessionManager(4);
manager.create('A', wsA.mind, buildOrchestrator(personal.mind, wsA.mind), [], null);
manager.create('B', wsB.mind, buildOrchestrator(personal.mind, wsB.mind), [], null);
manager.create('C', wsC.mind, buildOrchestrator(personal.mind, wsC.mind), [], null);
manager.create('A', buildOrchestrator('A'), []);
manager.create('B', buildOrchestrator('B'), []);
manager.create('C', buildOrchestrator('C'), []);
expect(manager.size).toBe(3);
manager.closeAll();
@@ -273,44 +187,43 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
const manager = new WorkspaceSessionManager(3);
// Seed each workspace with content specific to that workspace
seedFrame(wsA, 'A-specific: blueprint for launch campaign.', 'important');
seedFrame(wsB, 'B-specific: competitor analysis for fintech vertical.', 'important');
await memory.remember({
scope: workspaceScope('A'), text: 'A-specific: blueprint for launch campaign.', importance: 'important',
});
await memory.remember({
scope: workspaceScope('B'), text: 'B-specific: competitor analysis for fintech vertical.', importance: 'important',
});
const orchA = buildOrchestrator(personal.mind, wsA.mind);
const orchB = buildOrchestrator(personal.mind, wsB.mind);
const orchA = buildOrchestrator('A');
const orchB = buildOrchestrator('B');
manager.create('A', wsA.mind, orchA, [], null);
manager.create('B', wsB.mind, orchB, [], null);
manager.create('A', orchA, []);
manager.create('B', orchB, []);
// Fire two recalls in parallel. The invariant: each orchestrator only
// sees its own workspace's frames. On the pre-A.1 shared-orchestrator
// sees its own workspace's memories. On the pre-A.1 shared-orchestrator
// code, one would overwrite the other before both resolve.
const [recallA, recallB] = await Promise.all([
orchA.recallMemory('launch', 5),
orchB.recallMemory('fintech', 5),
orchA.recallMemory('launch campaign blueprint', 5),
orchB.recallMemory('fintech competitor analysis', 5),
]);
// Because the embedder is a zero-vector stub, semantic search returns
// nothing — but the key invariant we check is that recall didn't throw
// and that each orchestrator still points at its own workspace mind.
// The no-crash + no-exception path is the core regression we're
// guarding against.
expect(recallA).toBeDefined();
expect(recallB).toBeDefined();
expect(orchA.hasWorkspaceMind()).toBe(true);
expect(orchB.hasWorkspaceMind()).toBe(true);
const recalledA = (recallA.recalled ?? []).join('\n');
const recalledB = (recallB.recalled ?? []).join('\n');
expect(recalledA).toContain('A-specific');
expect(recalledA).not.toContain('B-specific');
expect(recalledB).toContain('B-specific');
expect(recalledB).not.toContain('A-specific');
// And crucially: after both recalls, both orchestrators still have
// the RIGHT workspace mind (not crossed over).
const statsA = orchA.getMemoryStats();
const statsB = orchB.getMemoryStats();
// Both should have non-zero frame counts (they have different content)
// and the personal-mind fragment is shared, so totals are at least 1.
expect(statsA.frameCount).toBeGreaterThanOrEqual(1);
expect(statsB.frameCount).toBeGreaterThanOrEqual(1);
// And crucially: after both recalls, both orchestrators still hold the
// RIGHT workspace (not crossed over).
expect(orchA.getWorkspaceId()).toBe('A');
expect(orchB.getWorkspaceId()).toBe('B');
expect((await orchA.getMemoryStats()).memoryCount).toBe(1);
expect((await orchB.getMemoryStats()).memoryCount).toBe(1);
});
it('composes the session cap, concurrent same-workspace planning isolation, bounded cache, and cleanup', async () => {
it('composes the session cap, concurrent same-workspace planning isolation, and cleanup', async () => {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-solo-session-composition-'));
const server = await buildLocalServer({ dataDir });
// A small cap keeps the overflow check cheap; the sidecar default is MAX_WORKSPACE_SESSIONS.
@@ -324,7 +237,6 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
const markerB = 'SOLO_SESSION_B';
const previousOllamaHost = process.env.OLLAMA_HOST;
const previousReranker = process.env.WAGGLE_RERANKER;
const releaseSpy = vi.spyOn(server.mindCache, 'release');
const createdWorkspaceIds: string[] = [workspaceId!];
const deferred = () => {
@@ -444,7 +356,6 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
try {
expect(server.sessionManager.getMaxSessions()).toBe(10);
expect(server.sessionManager.size).toBe(0);
expect(server.mindCache.keys()).toEqual([workspaceId]);
requestA = injectWithAuth(server, {
method: 'POST',
@@ -504,7 +415,6 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
expect(bPhases).not.toContain('workspace_queue');
expect(bPhases).not.toContain('workspace_acquired');
expect(server.sessionManager.size).toBe(1);
expect(server.mindCache.size).toBe(1);
const bShowResult = providerBodies.get('B')![2]?.messages
?.filter(message => message.role === 'tool')
@@ -531,22 +441,18 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
const workspacePath = path.join(dataDir, 'workspaces', workspace.id, 'files');
server.sessionManager.getOrCreate(
workspace.id,
() => server.mindCache.acquire(workspace.id),
mind => server.agentState.createSessionOrchestrator(mind),
(_mind, orchestrator) => server.agentState.buildToolsForSession(
() => server.agentState.createSessionOrchestrator(workspace.id),
orchestrator => server.agentState.buildToolsForSession(
orchestrator,
workspacePath,
workspace.id,
),
undefined,
() => server.mindCache.release(workspace.id),
);
expect(server.mindCache.size).toBeLessThanOrEqual(10);
}
expect(server.sessionManager.size).toBe(10);
expect(server.mindCache.size).toBe(10);
expect(new Set(server.mindCache.keys())).toEqual(new Set(createdWorkspaceIds));
expect(new Set(server.sessionManager.getActive().map(session => session.workspaceId)))
.toEqual(new Set(createdWorkspaceIds));
const fleet = await injectWithAuth(server, { method: 'GET', url: '/api/fleet' });
expect(fleet.statusCode).toBe(200);
expect(fleet.json()).toMatchObject({ count: 10, maxSessions: 10 });
@@ -554,12 +460,10 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
const anchor = server.sessionManager.get(workspaceId!)!;
expect(() => server.sessionManager.create(
'solo-overflow',
anchor.mind,
anchor.orchestrator,
anchor.tools,
)).toThrow('Max concurrent sessions reached (10)');
expect(server.sessionManager.size).toBe(10);
expect(server.mindCache.size).toBe(10);
for (const sessionId of [sessionA, sessionB]) {
const cleared = await injectWithAuth(server, {
@@ -579,9 +483,6 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
))).toBe(false);
}
const turnReleases = releaseSpy.mock.calls
.filter(([releasedWorkspaceId]) => releasedWorkspaceId === workspaceId);
expect(turnReleases).toHaveLength(2);
const killed = await injectWithAuth(server, {
method: 'POST',
url: `/api/fleet/${workspaceId}/kill`,
@@ -589,8 +490,6 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
expect(killed.statusCode).toBe(200);
expect(killed.json()).toEqual({ killed: true, workspaceId });
expect(server.sessionManager.has(workspaceId!)).toBe(false);
expect(releaseSpy.mock.calls
.filter(([releasedWorkspaceId]) => releasedWorkspaceId === workspaceId)).toHaveLength(3);
for (const id of createdWorkspaceIds) {
const deleted = await injectWithAuth(server, {
@@ -598,21 +497,14 @@ describe('WorkspaceSessionManager — Phase A.1 concurrency invariants', () => {
url: `/api/workspaces/${id}`,
});
expect(deleted.statusCode).toBe(204);
expect(server.mindCache.has(id)).toBe(false);
}
expect(server.sessionManager.size).toBe(0);
expect(server.mindCache.size).toBe(0);
for (const id of createdWorkspaceIds.slice(1)) {
expect(releaseSpy.mock.calls
.filter(([releasedWorkspaceId]) => releasedWorkspaceId === id)).toHaveLength(1);
}
} finally {
aFirstProviderArrived.resolve();
releaseAFirstProvider.resolve();
await Promise.allSettled([requestA, requestB].filter(Boolean) as Promise<unknown>[]);
fetchSpy.mockRestore();
acquireSpy.mockRestore();
releaseSpy.mockRestore();
await server.close();
if (previousOllamaHost === undefined) delete process.env.OLLAMA_HOST;
else process.env.OLLAMA_HOST = previousOllamaHost;