374 lines
14 KiB
TypeScript
374 lines
14 KiB
TypeScript
/**
|
|
* Comprehensive E2E test — exercises the same code paths as the CLI REPL
|
|
* but programmatically (no LiteLLM/API needed).
|
|
*
|
|
* Tests 14 scenarios covering identity, awareness, memory persistence,
|
|
* knowledge graph, cross-session recall, tool execution, system prompt,
|
|
* hooks, permissions, and more.
|
|
*/
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import * as os from 'node:os';
|
|
import { MindDB } from '@waggle/core';
|
|
import {
|
|
Orchestrator,
|
|
createSystemTools,
|
|
createPlanTools,
|
|
createGitTools,
|
|
HookRegistry,
|
|
PermissionManager,
|
|
filterToolsForContext,
|
|
needsConfirmation,
|
|
} from '@waggle/agent';
|
|
import { MockEmbedder } from '../../hive-mind-core/tests/mind/helpers/mock-embedder.js';
|
|
|
|
// Helper: create a file-backed .mind DB in temp dir
|
|
function createTmpMind(): { path: string; db: MindDB } {
|
|
const p = path.join(os.tmpdir(), `waggle-e2e-${Date.now()}-${Math.random().toString(36).slice(2)}.mind`);
|
|
return { path: p, db: new MindDB(p) };
|
|
}
|
|
|
|
function cleanup(filePath: string) {
|
|
for (const f of [filePath, filePath + '-wal', filePath + '-shm']) {
|
|
if (fs.existsSync(f)) fs.unlinkSync(f);
|
|
}
|
|
}
|
|
|
|
describe('Comprehensive CLI E2E Test', () => {
|
|
let mindPath: string;
|
|
let db: MindDB;
|
|
let orchestrator: Orchestrator;
|
|
const embedder = new MockEmbedder();
|
|
|
|
beforeEach(() => {
|
|
const tmp = createTmpMind();
|
|
mindPath = tmp.path;
|
|
db = tmp.db;
|
|
orchestrator = new Orchestrator({ db, embedder });
|
|
});
|
|
|
|
afterEach(() => {
|
|
db.close();
|
|
cleanup(mindPath);
|
|
});
|
|
|
|
// ─── Test 1: Identity CRUD ───
|
|
it('T1: Create and retrieve agent identity', async () => {
|
|
orchestrator.getIdentity().create({
|
|
name: 'Waggle',
|
|
role: 'Personal AI Assistant',
|
|
department: 'Engineering',
|
|
personality: 'Helpful and precise',
|
|
capabilities: 'Memory, search, knowledge graph',
|
|
system_prompt: 'You are Waggle.',
|
|
});
|
|
|
|
const result = await orchestrator.executeTool('get_identity', {});
|
|
expect(result).toContain('Waggle');
|
|
expect(result).toContain('Personal AI Assistant');
|
|
expect(result).toContain('Engineering');
|
|
});
|
|
|
|
// ─── Test 2: Awareness Layer ───
|
|
it('T2: Add tasks and flags to awareness', async () => {
|
|
await orchestrator.executeTool('add_task', { content: 'Review PR #42', priority: 9 });
|
|
await orchestrator.executeTool('add_task', { content: 'Deploy staging', priority: 5 });
|
|
orchestrator.getAwareness().add('flag', 'User prefers dark mode', 10);
|
|
|
|
const result = await orchestrator.executeTool('get_awareness', {});
|
|
expect(result).toContain('Review PR #42');
|
|
expect(result).toContain('Deploy staging');
|
|
expect(result).toContain('dark mode');
|
|
});
|
|
|
|
// ─── Test 3: Save and Search Memory (within session) ───
|
|
it('T3: Save memory and search it back', async () => {
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'The quarterly report deadline is March 15th',
|
|
importance: 'important',
|
|
});
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'Project Alpha uses React and TypeScript',
|
|
importance: 'normal',
|
|
});
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'Team standup is at 9:30 AM every day',
|
|
importance: 'normal',
|
|
});
|
|
|
|
// Search should find the memory
|
|
const result = await orchestrator.executeTool('search_memory', { query: 'quarterly report' });
|
|
expect(result).toContain('quarterly report');
|
|
expect(result).toContain('March 15th');
|
|
});
|
|
|
|
// ─── Test 4: Knowledge Graph ───
|
|
it('T4: Create entities and relations, query them', async () => {
|
|
const kg = orchestrator.getKnowledge();
|
|
const alice = kg.createEntity('person', 'Alice', { role: 'Tech Lead' });
|
|
const project = kg.createEntity('project', 'Phoenix', { status: 'active' });
|
|
const react = kg.createEntity('technology', 'React', { version: '18' });
|
|
|
|
kg.createRelation(alice.id, project.id, 'leads', 0.95);
|
|
kg.createRelation(project.id, react.id, 'uses', 0.9);
|
|
|
|
const result = await orchestrator.executeTool('query_knowledge', { query: 'Alice' });
|
|
expect(result).toContain('Alice');
|
|
expect(result).toContain('leads');
|
|
expect(result).toContain('Phoenix');
|
|
});
|
|
|
|
// ─── Test 5: Cross-Session Memory Persistence ───
|
|
it('T5: Memories persist across sessions (file-backed .mind)', async () => {
|
|
// Session 1: save memories
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'My name is Marko and I work on the Waggle project',
|
|
importance: 'critical',
|
|
});
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'The API key for production is stored in 1Password',
|
|
importance: 'important',
|
|
});
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: 'Python is used for data processing scripts',
|
|
importance: 'normal',
|
|
});
|
|
|
|
// Close DB (simulates CLI exit)
|
|
db.close();
|
|
|
|
// Session 2: reopen same .mind file
|
|
const db2 = new MindDB(mindPath);
|
|
const orchestrator2 = new Orchestrator({ db: db2, embedder });
|
|
|
|
// Search for memories from previous session
|
|
const result1 = await orchestrator2.executeTool('search_memory', { query: 'Marko Waggle' });
|
|
expect(result1).toContain('Marko');
|
|
expect(result1).toContain('Waggle');
|
|
|
|
const result2 = await orchestrator2.executeTool('search_memory', { query: 'API key production' });
|
|
expect(result2).toContain('1Password');
|
|
|
|
const result3 = await orchestrator2.executeTool('search_memory', { query: 'Python data' });
|
|
expect(result3).toContain('Python');
|
|
|
|
db2.close();
|
|
|
|
// Reassign so afterEach cleanup works
|
|
db = new MindDB(mindPath);
|
|
orchestrator = new Orchestrator({ db, embedder });
|
|
});
|
|
|
|
// ─── Test 6: System Prompt Builder ───
|
|
it('T6: System prompt includes identity, awareness, and tools', () => {
|
|
orchestrator.getIdentity().create({
|
|
name: 'TestBot',
|
|
role: 'Tester',
|
|
department: '',
|
|
personality: 'Thorough',
|
|
capabilities: 'Testing',
|
|
system_prompt: 'You run tests.',
|
|
});
|
|
orchestrator.getAwareness().add('task', 'Run integration tests', 10);
|
|
|
|
const prompt = orchestrator.buildSystemPrompt();
|
|
expect(prompt).toContain('TestBot');
|
|
expect(prompt).toContain('Run integration tests');
|
|
// System prompt includes self-awareness block with tool summary
|
|
expect(prompt).toContain('# Self-Awareness');
|
|
expect(prompt).toContain('tools available');
|
|
});
|
|
|
|
// ─── Test 7: Tool Definitions ───
|
|
it('T7: All required tools are defined with correct shape', () => {
|
|
const tools = orchestrator.getTools();
|
|
const required = ['get_identity', 'get_awareness', 'search_memory', 'save_memory', 'query_knowledge', 'add_task', 'correct_knowledge'];
|
|
|
|
for (const name of required) {
|
|
const tool = tools.find(t => t.name === name);
|
|
expect(tool, `Tool ${name} should exist`).toBeDefined();
|
|
expect(tool!.description).toBeTruthy();
|
|
expect(typeof tool!.execute).toBe('function');
|
|
}
|
|
});
|
|
|
|
// ─── Test 8: System Tools ───
|
|
it('T8: System tools (bash, read_file, etc.) are created', () => {
|
|
const systemTools = createSystemTools(process.cwd());
|
|
const names = systemTools.map(t => t.name);
|
|
|
|
expect(names).toContain('bash');
|
|
expect(names).toContain('read_file');
|
|
expect(names).toContain('write_file');
|
|
expect(names).toContain('edit_file');
|
|
expect(names).toContain('search_files');
|
|
expect(names).toContain('search_content');
|
|
});
|
|
|
|
// ─── Test 9: Plan Tools ───
|
|
it('T9: Plan tools are created', () => {
|
|
const planTools = createPlanTools(process.cwd());
|
|
const names = planTools.map(t => t.name);
|
|
|
|
expect(names).toContain('create_plan');
|
|
expect(names).toContain('add_plan_step');
|
|
expect(names).toContain('show_plan');
|
|
});
|
|
|
|
// ─── Test 10: Git Tools ───
|
|
it('T10: Git tools are created', () => {
|
|
const gitTools = createGitTools(process.cwd());
|
|
const names = gitTools.map(t => t.name);
|
|
|
|
expect(names).toContain('git_status');
|
|
expect(names).toContain('git_diff');
|
|
expect(names).toContain('git_log');
|
|
expect(names).toContain('git_commit');
|
|
});
|
|
|
|
// ─── Test 11: Hook Registry ───
|
|
it('T11: Hook registry fires pre/post hooks', async () => {
|
|
const hooks = new HookRegistry();
|
|
const events: string[] = [];
|
|
|
|
hooks.on('pre:tool', async (ctx) => { events.push(`pre:${ctx.toolName}`); });
|
|
hooks.on('post:tool', async (ctx) => { events.push(`post:${ctx.toolName}`); });
|
|
|
|
await hooks.fire('pre:tool', { toolName: 'bash', args: {} });
|
|
await hooks.fire('post:tool', { toolName: 'bash', args: {}, result: 'ok' });
|
|
|
|
expect(events).toEqual(['pre:bash', 'post:bash']);
|
|
});
|
|
|
|
// ─── Test 12: Permission Manager ───
|
|
it('T12: Permission manager filters tools', () => {
|
|
const perms = new PermissionManager({
|
|
blacklist: ['bash', 'write_file'],
|
|
});
|
|
|
|
expect(perms.isAllowed('bash')).toBe(false);
|
|
expect(perms.isAllowed('write_file')).toBe(false);
|
|
expect(perms.isAllowed('read_file')).toBe(true);
|
|
expect(perms.isAllowed('search_memory')).toBe(true);
|
|
|
|
// Sandbox mode only allows readonly tools
|
|
const sandbox = PermissionManager.sandbox();
|
|
expect(sandbox.isAllowed('bash')).toBe(false);
|
|
expect(sandbox.isAllowed('write_file')).toBe(false);
|
|
expect(sandbox.isAllowed('read_file')).toBe(true);
|
|
expect(sandbox.isAllowed('search_memory')).toBe(true);
|
|
});
|
|
|
|
// ─── Test 13: Confirmation Gate ───
|
|
it('T13: Confirmation gate identifies sensitive tools', () => {
|
|
// Non-bash tools
|
|
expect(needsConfirmation('write_file')).toBe(true);
|
|
expect(needsConfirmation('edit_file')).toBe(true);
|
|
expect(needsConfirmation('git_commit')).toBe(true);
|
|
expect(needsConfirmation('read_file')).toBe(false);
|
|
expect(needsConfirmation('search_memory')).toBe(false);
|
|
|
|
// Bash without args = unknown command = confirm
|
|
expect(needsConfirmation('bash')).toBe(true);
|
|
|
|
// Safe bash commands (read-only)
|
|
expect(needsConfirmation('bash', { command: 'date' })).toBe(false);
|
|
expect(needsConfirmation('bash', { command: 'ls -la' })).toBe(false);
|
|
expect(needsConfirmation('bash', { command: 'git status' })).toBe(false);
|
|
expect(needsConfirmation('bash', { command: 'git log --oneline' })).toBe(false);
|
|
expect(needsConfirmation('bash', { command: 'whoami' })).toBe(false);
|
|
|
|
// Destructive bash commands
|
|
expect(needsConfirmation('bash', { command: 'rm -rf /tmp/foo' })).toBe(true);
|
|
expect(needsConfirmation('bash', { command: 'git push origin main' })).toBe(true);
|
|
expect(needsConfirmation('bash', { command: 'sudo apt install foo' })).toBe(true);
|
|
});
|
|
|
|
// ─── Test 14: Memory Stats ───
|
|
it('T14: Memory stats reflect actual data', async () => {
|
|
// Start with empty
|
|
let stats = orchestrator.getMemoryStats();
|
|
expect(stats.frameCount).toBe(0);
|
|
expect(stats.sessionCount).toBe(0);
|
|
expect(stats.entityCount).toBe(0);
|
|
|
|
// Save some memories
|
|
await orchestrator.executeTool('save_memory', { content: 'Memory one' });
|
|
await orchestrator.executeTool('save_memory', { content: 'Memory two' });
|
|
orchestrator.getKnowledge().createEntity('test', 'Entity1', {});
|
|
|
|
stats = orchestrator.getMemoryStats();
|
|
expect(stats.frameCount).toBeGreaterThanOrEqual(2);
|
|
expect(stats.sessionCount).toBeGreaterThanOrEqual(1);
|
|
expect(stats.entityCount).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
// ─── Test 15: Empty state handling ───
|
|
it('T15: Graceful handling of empty state', async () => {
|
|
const id = await orchestrator.executeTool('get_identity', {});
|
|
expect(id).toContain('No identity configured');
|
|
|
|
const aw = await orchestrator.executeTool('get_awareness', {});
|
|
expect(aw).toContain('No active awareness items');
|
|
|
|
const search = await orchestrator.executeTool('search_memory', { query: 'anything' });
|
|
expect(search).toContain('No relevant memories');
|
|
|
|
const kg = await orchestrator.executeTool('query_knowledge', { query: 'nobody' });
|
|
expect(kg).toContain('No entities found');
|
|
});
|
|
|
|
// ─── Test 16: Unknown tool throws ───
|
|
it('T16: Unknown tool name throws error', async () => {
|
|
await expect(orchestrator.executeTool('nonexistent_tool', {})).rejects.toThrow('Unknown tool');
|
|
});
|
|
|
|
// ─── Test 17: Heavy memory load (50 per session, rate-limited by W2.10) ───
|
|
it('T17: memories up to session rate limit are all searchable', async () => {
|
|
// W2.10: save_memory is rate-limited to 50 per session to prevent flooding.
|
|
// Save 60 — first 50 succeed, remaining are rate-limited.
|
|
const RATE_LIMIT = 50;
|
|
for (let i = 0; i < 60; i++) {
|
|
await orchestrator.executeTool('save_memory', {
|
|
content: `Observation ${i}: topic-${i % 10} with detail about area-${i % 5}`,
|
|
importance: i % 20 === 0 ? 'important' : 'normal',
|
|
});
|
|
}
|
|
|
|
// Search by topic (hyphens in queries should work after FTS5 sanitization)
|
|
const result = await orchestrator.executeTool('search_memory', { query: 'topic-7' });
|
|
expect(result).toContain('topic-7');
|
|
|
|
// Search by area
|
|
const result2 = await orchestrator.executeTool('search_memory', { query: 'area-3' });
|
|
expect(result2).toContain('area-3');
|
|
|
|
// Stats should show exactly the rate limit count (50 saved, rest blocked)
|
|
const stats = orchestrator.getMemoryStats();
|
|
expect(stats.frameCount).toBeGreaterThanOrEqual(RATE_LIMIT);
|
|
});
|
|
|
|
// ─── Test 18: Cross-session knowledge graph persistence ───
|
|
it('T18: Knowledge graph persists across sessions', async () => {
|
|
const kg = orchestrator.getKnowledge();
|
|
const alice = kg.createEntity('person', 'Alice', { role: 'Engineer' });
|
|
const bob = kg.createEntity('person', 'Bob', { role: 'Designer' });
|
|
kg.createRelation(alice.id, bob.id, 'collaborates_with', 0.85);
|
|
|
|
// Close and reopen
|
|
db.close();
|
|
const db2 = new MindDB(mindPath);
|
|
const orchestrator2 = new Orchestrator({ db: db2, embedder });
|
|
|
|
const result = await orchestrator2.executeTool('query_knowledge', { query: 'Alice' });
|
|
expect(result).toContain('Alice');
|
|
expect(result).toContain('collaborates_with');
|
|
expect(result).toContain('Bob');
|
|
|
|
db2.close();
|
|
db = new MindDB(mindPath);
|
|
orchestrator = new Orchestrator({ db, embedder });
|
|
});
|
|
});
|