Files
waggle-os/packages/cli/tests/comprehensive-e2e.test.ts
shumeiko.it 16eaf01e14 chore: scrub personal names, maintainer URLs and local paths (P2.2)
Sample data in tests, prompts and UI placeholders uses Alex Morgan
instead of the maintainer's name; decision attributions say "owner".
Personal details in two test fixtures are replaced. Local paths
(D:/Projects/...) and paths into the maintainer's planning repository
are reworded. Clone and install URLs in CONTRIBUTING.md and the getting
started guide use placeholders until the public repository location is
decided (Q9). The Benchmarks page no longer names the maintainer's
hive-mind repository, and the upstream fixes log names the main Waggle
repository instead of an internal git host.
2026-11-01 00:50:13 +01:00

351 lines
14 KiB
TypeScript

/**
* Comprehensive E2E test — exercises the same code paths as the CLI REPL
* but programmatically (no LiteLLM/API needed).
*
* Covers identity, awareness, memory persistence, cross-session recall, tool
* execution, system prompt, hooks, permissions, and more. Memory is the mem0
* store the REPL opens (ADR-0007), with the word embedder so no model is
* downloaded; the profile card and awareness live in waggle.db.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { WaggleDB } from '@waggle/core';
import { PERSONAL_SCOPE, type MemoryStore } from '@waggle/memory-contract';
import { createMem0MemoryStore, createWordEmbedder } from '@waggle/memory-mem0';
import {
Orchestrator,
createSystemTools,
createPlanTools,
createGitTools,
HookRegistry,
PermissionManager,
filterToolsForContext,
needsConfirmation,
} from '@waggle/agent';
interface Session {
memory: MemoryStore;
waggleDb: WaggleDB;
orchestrator: Orchestrator;
}
/** Opens a data directory the way the REPL does: memory in <dir>/memory, waggle.db next to it. */
async function openSession(dataDir: string): Promise<Session> {
const memory = await createMem0MemoryStore({
directory: path.join(dataDir, 'memory'),
embedder: createWordEmbedder(),
});
const waggleDb = WaggleDB.openInDataDir(dataDir);
return { memory, waggleDb, orchestrator: new Orchestrator({ memory, waggleDb }) };
}
async function closeSession(session: Session): Promise<void> {
await session.memory.close();
session.waggleDb.close();
}
describe('Comprehensive CLI E2E Test', () => {
let dataDir: string;
let session: Session;
let orchestrator: Orchestrator;
beforeEach(async () => {
dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-e2e-'));
session = await openSession(dataDir);
orchestrator = session.orchestrator;
});
afterEach(async () => {
await closeSession(session);
fs.rmSync(dataDir, { recursive: true, force: true });
});
// ─── Test 1: Identity CRUD ───
it('T1: Create and retrieve agent identity', async () => {
orchestrator.getIdentity().create({
name: 'Waggle',
role: 'Personal AI Assistant',
department: 'Engineering',
personality: 'Helpful and precise',
capabilities: 'Memory, search',
system_prompt: 'You are Waggle.',
});
const result = await orchestrator.executeTool('get_identity', {});
expect(result).toContain('Waggle');
expect(result).toContain('Personal AI Assistant');
expect(result).toContain('Engineering');
});
// ─── Test 2: Awareness Layer ───
it('T2: Add tasks and flags to awareness', async () => {
await orchestrator.executeTool('add_task', { content: 'Review PR #42', priority: 9 });
await orchestrator.executeTool('add_task', { content: 'Deploy staging', priority: 5 });
orchestrator.getAwareness().add('flag', 'User prefers dark mode', 10);
const result = await orchestrator.executeTool('get_awareness', {});
expect(result).toContain('Review PR #42');
expect(result).toContain('Deploy staging');
expect(result).toContain('dark mode');
});
// ─── Test 3: Save and Search Memory (within session) ───
it('T3: Save memory and search it back', async () => {
await orchestrator.executeTool('save_memory', {
content: 'The quarterly report deadline is March 15th',
importance: 'important',
});
await orchestrator.executeTool('save_memory', {
content: 'Project Alpha uses React and TypeScript',
importance: 'normal',
});
await orchestrator.executeTool('save_memory', {
content: 'Team standup is at 9:30 AM every day',
importance: 'normal',
});
// Search should find the memory
const result = await orchestrator.executeTool('search_memory', { query: 'quarterly report' });
expect(result).toContain('quarterly report');
expect(result).toContain('March 15th');
});
// ─── Test 5: Cross-Session Memory Persistence ───
it('T5: Memories, identity and tasks persist across sessions (file-backed data directory)', async () => {
orchestrator.getIdentity().create({
name: 'Waggle',
role: 'Personal AI Assistant',
department: '',
personality: '',
capabilities: '',
system_prompt: '',
});
await orchestrator.executeTool('add_task', { content: 'Prepare the quarterly review', priority: 7 });
// Session 1: save memories
await orchestrator.executeTool('save_memory', {
content: 'My name is Alex and I work on the Waggle project',
importance: 'critical',
});
await orchestrator.executeTool('save_memory', {
content: 'The API key for production is stored in 1Password',
importance: 'important',
});
await orchestrator.executeTool('save_memory', {
content: 'Python is used for data processing scripts',
importance: 'normal',
});
// Close everything (simulates CLI exit)
await closeSession(session);
// Session 2: reopen the same data directory
session = await openSession(dataDir);
const orchestrator2 = session.orchestrator;
// Search for memories from previous session
const result1 = await orchestrator2.executeTool('search_memory', { query: 'Alex Waggle' });
expect(result1).toContain('Alex');
expect(result1).toContain('Waggle');
const result2 = await orchestrator2.executeTool('search_memory', { query: 'API key production' });
expect(result2).toContain('1Password');
const result3 = await orchestrator2.executeTool('search_memory', { query: 'Python data' });
expect(result3).toContain('Python');
expect(await orchestrator2.executeTool('get_identity', {})).toContain('Personal AI Assistant');
expect(await orchestrator2.executeTool('get_awareness', {})).toContain('Prepare the quarterly review');
expect((await orchestrator2.getMemoryStats()).memoryCount).toBe(3);
});
// ─── Test 6: System Prompt Builder ───
it('T6: System prompt includes identity, awareness, and tools', async () => {
orchestrator.getIdentity().create({
name: 'TestBot',
role: 'Tester',
department: '',
personality: 'Thorough',
capabilities: 'Testing',
system_prompt: 'You run tests.',
});
orchestrator.getAwareness().add('task', 'Run integration tests', 10);
const prompt = await orchestrator.buildSystemPrompt();
expect(prompt).toContain('TestBot');
expect(prompt).toContain('Run integration tests');
// System prompt includes self-awareness block with tool summary
expect(prompt).toContain('# Self-Awareness');
expect(prompt).toContain('tools available');
});
// ─── Test 7: Tool Definitions ───
it('T7: All required tools are defined with correct shape', () => {
const tools = orchestrator.getTools();
const required = ['get_identity', 'get_awareness', 'search_memory', 'search_all_workspaces', 'save_memory', 'add_task'];
for (const name of required) {
const tool = tools.find(t => t.name === name);
expect(tool, `Tool ${name} should exist`).toBeDefined();
expect(tool!.description).toBeTruthy();
expect(typeof tool!.execute).toBe('function');
}
// The knowledge graph is not part of v1 (ADR-0002).
expect(tools.map(t => t.name)).not.toContain('query_knowledge');
});
// ─── Test 8: System Tools ───
it('T8: System tools (bash, read_file, etc.) are created', () => {
const systemTools = createSystemTools(process.cwd());
const names = systemTools.map(t => t.name);
expect(names).toContain('bash');
expect(names).toContain('read_file');
expect(names).toContain('write_file');
expect(names).toContain('edit_file');
expect(names).toContain('search_files');
expect(names).toContain('search_content');
});
// ─── Test 9: Plan Tools ───
it('T9: Plan tools are created', () => {
const planTools = createPlanTools(process.cwd());
const names = planTools.map(t => t.name);
expect(names).toContain('create_plan');
expect(names).toContain('add_plan_step');
expect(names).toContain('show_plan');
});
// ─── Test 10: Git Tools ───
it('T10: Git tools are created', () => {
const gitTools = createGitTools(process.cwd());
const names = gitTools.map(t => t.name);
expect(names).toContain('git_status');
expect(names).toContain('git_diff');
expect(names).toContain('git_log');
expect(names).toContain('git_commit');
});
// ─── Test 11: Hook Registry ───
it('T11: Hook registry fires pre/post hooks', async () => {
const hooks = new HookRegistry();
const events: string[] = [];
hooks.on('pre:tool', async (ctx) => { events.push(`pre:${ctx.toolName}`); });
hooks.on('post:tool', async (ctx) => { events.push(`post:${ctx.toolName}`); });
await hooks.fire('pre:tool', { toolName: 'bash', args: {} });
await hooks.fire('post:tool', { toolName: 'bash', args: {}, result: 'ok' });
expect(events).toEqual(['pre:bash', 'post:bash']);
});
// ─── Test 12: Permission Manager ───
it('T12: Permission manager filters tools', () => {
const perms = new PermissionManager({
blacklist: ['bash', 'write_file'],
});
expect(perms.isAllowed('bash')).toBe(false);
expect(perms.isAllowed('write_file')).toBe(false);
expect(perms.isAllowed('read_file')).toBe(true);
expect(perms.isAllowed('search_memory')).toBe(true);
// Sandbox mode only allows readonly tools
const sandbox = PermissionManager.sandbox();
expect(sandbox.isAllowed('bash')).toBe(false);
expect(sandbox.isAllowed('write_file')).toBe(false);
expect(sandbox.isAllowed('read_file')).toBe(true);
expect(sandbox.isAllowed('search_memory')).toBe(true);
});
// ─── Test 13: Confirmation Gate ───
it('T13: Confirmation gate identifies sensitive tools', () => {
// Non-bash tools
expect(needsConfirmation('write_file')).toBe(true);
expect(needsConfirmation('edit_file')).toBe(true);
expect(needsConfirmation('git_commit')).toBe(true);
expect(needsConfirmation('read_file')).toBe(false);
expect(needsConfirmation('search_memory')).toBe(false);
// Bash without args = unknown command = confirm
expect(needsConfirmation('bash')).toBe(true);
// Safe bash commands (read-only)
expect(needsConfirmation('bash', { command: 'date' })).toBe(false);
expect(needsConfirmation('bash', { command: 'ls -la' })).toBe(false);
expect(needsConfirmation('bash', { command: 'git status' })).toBe(false);
expect(needsConfirmation('bash', { command: 'git log --oneline' })).toBe(false);
expect(needsConfirmation('bash', { command: 'whoami' })).toBe(false);
// Destructive bash commands
expect(needsConfirmation('bash', { command: 'rm -rf /tmp/foo' })).toBe(true);
expect(needsConfirmation('bash', { command: 'git push origin main' })).toBe(true);
expect(needsConfirmation('bash', { command: 'sudo apt install foo' })).toBe(true);
});
// ─── Test 14: Memory Stats ───
it('T14: Memory stats reflect actual data', async () => {
// Start with empty
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(0);
// Save some memories
await orchestrator.executeTool('save_memory', { content: 'Memory one about the roadmap' });
await orchestrator.executeTool('save_memory', { content: 'Memory two about the hiring plan' });
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(2);
expect(await session.memory.count({ scopes: [PERSONAL_SCOPE] })).toBe(2);
});
// ─── Test 15: Empty state handling ───
it('T15: Graceful handling of empty state', async () => {
const id = await orchestrator.executeTool('get_identity', {});
expect(id).toContain('No identity configured');
const aw = await orchestrator.executeTool('get_awareness', {});
expect(aw).toContain('No active awareness items');
const search = await orchestrator.executeTool('search_memory', { query: 'anything' });
expect(search).toContain('No relevant memories');
expect(await orchestrator.recallMemory('anything')).toEqual({ text: '', count: 0, recalled: [], recalledFrames: [] });
});
// ─── Test 16: Unknown tool throws ───
it('T16: Unknown tool name throws error', async () => {
await expect(orchestrator.executeTool('nonexistent_tool', {})).rejects.toThrow('Unknown tool');
});
// ─── Test 17: Heavy memory load (50 per session, rate-limited by W2.10) ───
it('T17: memories up to session rate limit are all searchable', async () => {
// W2.10: save_memory is rate-limited to 50 per session to prevent flooding.
// Save 60 — first 50 succeed, remaining are rate-limited.
const RATE_LIMIT = 50;
const TOPICS = ['alpha', 'bravo', 'charlie', 'delta', 'echo', 'foxtrot', 'golf', 'hotel', 'india', 'juliett'];
const AREAS = ['billing', 'search', 'onboarding', 'reporting', 'security'];
const results: string[] = [];
for (let i = 0; i < 60; i++) {
results.push(await orchestrator.executeTool('save_memory', {
content: `Observation ${i}: topic ${TOPICS[i % 10]} with detail about ${AREAS[i % 5]}`,
importance: i % 20 === 0 ? 'important' : 'normal',
}));
}
expect(results.filter(r => r.startsWith('Memory saved'))).toHaveLength(RATE_LIMIT);
expect(results.slice(RATE_LIMIT).every(r => r.includes('rate limit reached'))).toBe(true);
// Search by topic and by area
const result = await orchestrator.executeTool('search_memory', { query: 'topic hotel' });
expect(result).toContain('topic hotel');
const result2 = await orchestrator.executeTool('search_memory', { query: 'reporting' });
expect(result2).toContain('reporting');
// Exactly the rate limit count was saved, the rest blocked
expect((await orchestrator.getMemoryStats()).memoryCount).toBe(RATE_LIMIT);
});
});