Files
waggle-os/packages/cli/tests/comprehensive-e2e.test.ts
shumeiko.it 424b46db19 test(cli): run the CLI suites on the mem0 store and waggle.db (P1.5)
The comprehensive and hard-persistence suites open a data directory the
way the REPL does: mem0 in <dir>/memory (word embedder, no model download)
and waggle.db for the profile card and tasks. Persistence across sessions
now also checks identity and tasks after reopening.

The packaged-runtime tests install @waggle/memory-contract and
@waggle/memory-mem0 with the CLI (the weaver package is gone) and check
that the REPL creates the memory store and waggle.db instead of
default.mind.

Deleted: T4 and T18 (knowledge graph entities and their persistence) and
the knowledge-graph steps of the hard-persistence run; the graph is not in
v1 (ADR-0002).
2026-10-24 13:19:06 +02:00

351 lines
14 KiB
TypeScript

/**
* Comprehensive E2E test — exercises the same code paths as the CLI REPL
* but programmatically (no LiteLLM/API needed).
*
* Covers identity, awareness, memory persistence, cross-session recall, tool
* execution, system prompt, hooks, permissions, and more. Memory is the mem0
* store the REPL opens (ADR-0007), with the word embedder so no model is
* downloaded; the profile card and awareness live in waggle.db.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { WaggleDB } from '@waggle/core';
import { PERSONAL_SCOPE, type MemoryStore } from '@waggle/memory-contract';
import { createMem0MemoryStore, createWordEmbedder } from '@waggle/memory-mem0';
import {
Orchestrator,
createSystemTools,
createPlanTools,
createGitTools,
HookRegistry,
PermissionManager,
filterToolsForContext,
needsConfirmation,
} from '@waggle/agent';
interface Session {
memory: MemoryStore;
waggleDb: WaggleDB;
orchestrator: Orchestrator;
}
/** Opens a data directory the way the REPL does: memory in <dir>/memory, waggle.db next to it. */
async function openSession(dataDir: string): Promise<Session> {
const memory = await createMem0MemoryStore({
directory: path.join(dataDir, 'memory'),
embedder: createWordEmbedder(),
});
const waggleDb = WaggleDB.openInDataDir(dataDir);
return { memory, waggleDb, orchestrator: new Orchestrator({ memory, waggleDb }) };
}
async function closeSession(session: Session): Promise<void> {
await session.memory.close();
session.waggleDb.close();
}
describe('Comprehensive CLI E2E Test', () => {
let dataDir: string;
let session: Session;
let orchestrator: Orchestrator;
beforeEach(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-e2e-'));
session = await openSession(dataDir);
orchestrator = session.orchestrator;
});
afterEach(async () => {
await closeSession(session);
fs.rmSync(dataDir, { recursive: true, force: true });
});
// ─── Test 1: Identity CRUD ───
it('T1: Create and retrieve agent identity', async () => {
orchestrator.getIdentity().create({
name: 'Waggle',
role: 'Personal AI Assistant',
department: 'Engineering',
personality: 'Helpful and precise',
capabilities: 'Memory, search',
system_prompt: 'You are Waggle.',
});
const result = await orchestrator.executeTool('get_identity', {});
expect(result).toContain('Waggle');
expect(result).toContain('Personal AI Assistant');
expect(result).toContain('Engineering');
});
// ─── Test 2: Awareness Layer ───
it('T2: Add tasks and flags to awareness', async () => {
await orchestrator.executeTool('add_task', { content: 'Review PR #42', priority: 9 });
await orchestrator.executeTool('add_task', { content: 'Deploy staging', priority: 5 });
orchestrator.getAwareness().add('flag', 'User prefers dark mode', 10);
const result = await orchestrator.executeTool('get_awareness', {});
expect(result).toContain('Review PR #42');
expect(result).toContain('Deploy staging');
expect(result).toContain('dark mode');
});
// ─── Test 3: Save and Search Memory (within session) ───
it('T3: Save memory and search it back', async () => {
await orchestrator.executeTool('save_memory', {
content: 'The quarterly report deadline is March 15th',
importance: 'important',
});
await orchestrator.executeTool('save_memory', {
content: 'Project Alpha uses React and TypeScript',
importance: 'normal',
});
await orchestrator.executeTool('save_memory', {
content: 'Team standup is at 9:30 AM every day',
importance: 'normal',
});
// Search should find the memory
const result = await orchestrator.executeTool('search_memory', { query: 'quarterly report' });
expect(result).toContain('quarterly report');
expect(result).toContain('March 15th');
});
// ─── Test 5: Cross-Session Memory Persistence ───
it('T5: Memories, identity and tasks persist across sessions (file-backed data directory)', async () => {
orchestrator.getIdentity().create({
name: 'Waggle',
role: 'Personal AI Assistant',
department: '',
personality: '',
capabilities: '',
system_prompt: '',
});
await orchestrator.executeTool('add_task', { content: 'Prepare the quarterly review', priority: 7 });
// Session 1: save memories
await orchestrator.executeTool('save_memory', {
content: 'My name is Marko and I work on the Waggle project',
importance: 'critical',
});
await orchestrator.executeTool('save_memory', {
content: 'The API key for production is stored in 1Password',
importance: 'important',
});
await orchestrator.executeTool('save_memory', {
content: 'Python is used for data processing scripts',
importance: 'normal',
});
// Close everything (simulates CLI exit)
await closeSession(session);
// Session 2: reopen the same data directory
session = await openSession(dataDir);
const orchestrator2 = session.orchestrator;
// Search for memories from previous session
const result1 = await orchestrator2.executeTool('search_memory', { query: 'Marko Waggle' });
expect(result1).toContain('Marko');
expect(result1).toContain('Waggle');
const result2 = await orchestrator2.executeTool('search_memory', { query: 'API key production' });
expect(result2).toContain('1Password');
const result3 = await orchestrator2.executeTool('search_memory', { query: 'Python data' });
expect(result3).toContain('Python');
expect(await orchestrator2.executeTool('get_identity', {})).toContain('Personal AI Assistant');
expect(await orchestrator2.executeTool('get_awareness', {})).toContain('Prepare the quarterly review');
expect((await orchestrator2.getMemoryStats()).memoryCount).toBe(3);
});
// ─── Test 6: System Prompt Builder ───
it('T6: System prompt includes identity, awareness, and tools', async () => {
orchestrator.getIdentity().create({
name: 'TestBot',
role: 'Tester',
department: '',
personality: 'Thorough',
capabilities: 'Testing',
system_prompt: 'You run tests.',
});
orchestrator.getAwareness().add('task', 'Run integration tests', 10);
const prompt = await orchestrator.buildSystemPrompt();
expect(prompt).toContain('TestBot');
expect(prompt).toContain('Run integration tests');
// System prompt includes self-awareness block with tool summary
expect(prompt).toContain('# Self-Awareness');
expect(prompt).toContain('tools available');
});
// ─── Test 7: Tool Definitions ───
it('T7: All required tools are defined with correct shape', () => {
const tools = orchestrator.getTools();
const required = ['get_identity', 'get_awareness', 'search_memory', 'search_all_workspaces', 'save_memory', 'add_task'];
for (const name of required) {
const tool = tools.find(t => t.name === name);
expect(tool, `Tool ${name} should exist`).toBeDefined();
expect(tool!.description).toBeTruthy();
expect(typeof tool!.execute).toBe('function');
}
// The knowledge graph is not part of v1 (ADR-0002).
expect(tools.map(t => t.name)).not.toContain('query_knowledge');
});
// ─── Test 8: System Tools ───
it('T8: System tools (bash, read_file, etc.) are created', () => {
const systemTools = createSystemTools(process.cwd());
const names = systemTools.map(t => t.name);
expect(names).toContain('bash');
expect(names).toContain('read_file');
expect(names).toContain('write_file');
expect(names).toContain('edit_file');
expect(names).toContain('search_files');
expect(names).toContain('search_content');
});
// ─── Test 9: Plan Tools ───
it('T9: Plan tools are created', () => {
const planTools = createPlanTools(process.cwd());
const names = planTools.map(t => t.name);
expect(names).toContain('create_plan');
expect(names).toContain('add_plan_step');
expect(names).toContain('show_plan');
});
// ─── Test 10: Git Tools ───
it('T10: Git tools are created', () => {
const gitTools = createGitTools(process.cwd());
const names = gitTools.map(t => t.name);
expect(names).toContain('git_status');
expect(names).toContain('git_diff');
expect(names).toContain('git_log');
expect(names).toContain('git_commit');
});
// ─── Test 11: Hook Registry ───
it('T11: Hook registry fires pre/post hooks', async () => {
const hooks = new HookRegistry();
const events: string[] = [];
hooks.on('pre:tool', async (ctx) => { events.push(`pre:${ctx.toolName}`); });
hooks.on('post:tool', async (ctx) => { events.push(`post:${ctx.toolName}`); });
await hooks.fire('pre:tool', { toolName: 'bash', args: {} });
await hooks.fire('post:tool', { toolName: 'bash', args: {}, result: 'ok' });
expect(events).toEqual(['pre:bash', 'post:bash']);
});
// ─── Test 12: Permission Manager ───
it('T12: Permission manager filters tools', () => {
const perms = new PermissionManager({
blacklist: ['bash', 'write_file'],
});
expect(perms.isAllowed('bash')).toBe(false);
expect(perms.isAllowed('write_file')).toBe(false);
expect(perms.isAllowed('read_file')).toBe(true);
expect(perms.isAllowed('search_memory')).toBe(true);
// Sandbox mode only allows readonly tools
const sandbox = PermissionManager.sandbox();
expect(sandbox.isAllowed('bash')).toBe(false);
expect(sandbox.isAllowed('write_file')).toBe(false);
expect(sandbox.isAllowed('read_file')).toBe(true);
expect(sandbox.isAllowed('search_memory')).toBe(true);
});
// ─── Test 13: Confirmation Gate ───
it('T13: Confirmation gate identifies sensitive tools', () => {
// Non-bash tools
expect(needsConfirmation('write_file')).toBe(true);
expect(needsConfirmation('edit_file')).toBe(true);
expect(needsConfirmation('git_commit')).toBe(true);
expect(needsConfirmation('read_file')).toBe(false);
expect(needsConfirmation('search_memory')).toBe(false);
// Bash without args = unknown command = confirm
expect(needsConfirmation('bash')).toBe(true);
// Safe bash commands (read-only)
expect(needsConfirmation('bash', { command: 'date' })).toBe(false);
expect(needsConfirmation('bash', { command: 'ls -la' })).toBe(false);
expect(needsConfirmation('bash', { command: 'git status' })).toBe(false);
expect(needsConfirmation('bash', { command: 'git log --oneline' })).toBe(false);
expect(needsConfirmation('bash', { command: 'whoami' })).toBe(false);
// Destructive bash commands
expect(needsConfirmation('bash', { command: 'rm -rf /tmp/foo' })).toBe(true);
expect(needsConfirmation('bash', { command: 'git push origin main' })).toBe(true);
expect(needsConfirmation('bash', { command: 'sudo apt install foo' })).toBe(true);
});
// ─── Test 14: Memory Stats ───
it('T14: Memory stats reflect actual data', async () => {
// Start with empty
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(0);
// Save some memories
await orchestrator.executeTool('save_memory', { content: 'Memory one about the roadmap' });
await orchestrator.executeTool('save_memory', { content: 'Memory two about the hiring plan' });
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(2);
expect(await session.memory.count({ scopes: [PERSONAL_SCOPE] })).toBe(2);
});
// ─── Test 15: Empty state handling ───
it('T15: Graceful handling of empty state', async () => {
const id = await orchestrator.executeTool('get_identity', {});
expect(id).toContain('No identity configured');
const aw = await orchestrator.executeTool('get_awareness', {});
expect(aw).toContain('No active awareness items');
const search = await orchestrator.executeTool('search_memory', { query: 'anything' });
expect(search).toContain('No relevant memories');
expect(await orchestrator.recallMemory('anything')).toEqual({ text: '', count: 0, recalled: [], recalledFrames: [] });
});
// ─── Test 16: Unknown tool throws ───
it('T16: Unknown tool name throws error', async () => {
await expect(orchestrator.executeTool('nonexistent_tool', {})).rejects.toThrow('Unknown tool');
});
// ─── Test 17: Heavy memory load (50 per session, rate-limited by W2.10) ───
it('T17: memories up to session rate limit are all searchable', async () => {
// W2.10: save_memory is rate-limited to 50 per session to prevent flooding.
// Save 60 — first 50 succeed, remaining are rate-limited.
const RATE_LIMIT = 50;
const TOPICS = ['alpha', 'bravo', 'charlie', 'delta', 'echo', 'foxtrot', 'golf', 'hotel', 'india', 'juliett'];
const AREAS = ['billing', 'search', 'onboarding', 'reporting', 'security'];
const results: string[] = [];
for (let i = 0; i < 60; i++) {
results.push(await orchestrator.executeTool('save_memory', {
content: `Observation ${i}: topic ${TOPICS[i % 10]} with detail about ${AREAS[i % 5]}`,
importance: i % 20 === 0 ? 'important' : 'normal',
}));
}
expect(results.filter(r => r.startsWith('Memory saved'))).toHaveLength(RATE_LIMIT);
expect(results.slice(RATE_LIMIT).every(r => r.includes('rate limit reached'))).toBe(true);
// Search by topic and by area
const result = await orchestrator.executeTool('search_memory', { query: 'topic hotel' });
expect(result).toContain('topic hotel');
const result2 = await orchestrator.executeTool('search_memory', { query: 'reporting' });
expect(result2).toContain('reporting');
// Exactly the rate limit count was saved, the rest blocked
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(RATE_LIMIT);
});
});