Files
waggle-os/packages/server/tests/performance/benchmarks.test.ts
shumeiko.it ad7b90bebd test(server): drop tests of the legacy memory engine (P1.5)
- harvest-provenance: tested the removed harvest pipeline (RawArchive and
  frame metadata), which no longer exists.
- benchmarks: drop the FTS5 frame search and frame/session write
  benchmarks; they measured the legacy engine only.
2026-10-24 17:43:27 +02:00

274 lines
9.9 KiB
TypeScript

/**
* Performance Benchmarks
*
* Tests critical performance paths with generous thresholds (CI is slower).
* All tests use temp directories — no real user data is touched.
*
* Coverage:
* 1. Cold start: server boot → first response
* 2. Workspace list with 50 workspaces
* 3. Session load with 500 messages
* 4. Marketplace FTS5 search (bundled DB)
* 5. Vault set+get cycle
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { VaultStore, WorkspaceManager } from '@waggle/core';
import { buildLocalServer } from '../../src/local/index.js';
import type { FastifyInstance } from 'fastify';
import { injectWithAuth } from '../test-utils.js';
import Database from 'better-sqlite3';
// ── Helpers ─────────────────────────────────────────────────────────────
// Wall-clock perf thresholds are environment-sensitive: shared CI runners (and a
// dev box running other suites in parallel) vary wildly, so tight literal bounds
// produce false failures under load. Scale the budget in CI so only GROSS
// regressions (a sync-FS-per-item or an accidental O(n²)) trip the gate, while
// local runs keep the tight bounds that make the numbers meaningful. `CI` is set
// by GitHub Actions (and most CI providers).
const PERF_SCALE = process.env.CI ? 6 : 1;
/** Scale a base millisecond budget for the current environment. */
function perfBudget(baseMs: number): number {
return baseMs * PERF_SCALE;
}
function createTmpDir(prefix: string): string {
return fs.mkdtempSync(path.join(os.tmpdir(), `waggle-perf-${prefix}-`));
}
function cleanupDir(dir: string): void {
try {
fs.rmSync(dir, { recursive: true, force: true });
} catch { /* best-effort cleanup */ }
}
// ── 1. Cold Start Benchmark ─────────────────────────────────────────────
describe('Performance Benchmarks', () => {
describe('Cold Start', () => {
it('server boots and responds within 5 seconds', async () => {
const tmpDir = createTmpDir('coldstart');
try {
const start = Date.now();
const server = await buildLocalServer({ dataDir: tmpDir });
const res = await injectWithAuth(server, { method: 'GET', url: '/health' });
const elapsed = Date.now() - start;
expect(res.statusCode).toBe(200);
// Generous threshold: CI machines can be slow. 5s is the safety margin.
expect(elapsed).toBeLessThan(perfBudget(5000));
await server.close();
} finally {
cleanupDir(tmpDir);
}
}, 15_000);
});
// ── 2. Workspace List Performance ───────────────────────────────────
describe('Workspace Listing with 50 workspaces', () => {
let tmpDir: string;
let server: FastifyInstance;
beforeAll(async () => {
tmpDir = createTmpDir('ws-list');
// Create 50 workspaces via WorkspaceManager
const wsManager = new WorkspaceManager(tmpDir);
for (let i = 0; i < 50; i++) {
wsManager.create({
name: `Project ${i}`,
group: `Group ${i % 5}`,
icon: 'folder',
});
}
server = await buildLocalServer({ dataDir: tmpDir });
}, 30_000);
afterAll(async () => {
await server.close();
cleanupDir(tmpDir);
});
it('listing 50 workspaces via API completes within 500ms', async () => {
const start = Date.now();
const res = await injectWithAuth(server, { method: 'GET', url: '/api/workspaces' });
const elapsed = Date.now() - start;
expect(res.statusCode).toBe(200);
const body = JSON.parse(res.body);
expect(body.length).toBe(50);
// 500ms is generous for reading 50 JSON files from disk
expect(elapsed).toBeLessThan(perfBudget(500));
});
});
// ── 3. Session Load Performance ─────────────────────────────────────
describe('Session File Loading', () => {
let tmpDir: string;
let sessionFilePath: string;
beforeAll(() => {
tmpDir = createTmpDir('session-load');
// Create a session JSONL file with 500 messages
const sessionsDir = path.join(tmpDir, 'workspaces', 'test-ws', 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
sessionFilePath = path.join(sessionsDir, '2026-03-19-benchmark.jsonl');
const lines: string[] = [];
for (let i = 0; i < 500; i++) {
const role = i % 2 === 0 ? 'user' : 'assistant';
const content = `${role === 'user' ? 'Question' : 'Answer'} number ${i}: ${
'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(3)
}`;
lines.push(JSON.stringify({
role,
content,
timestamp: new Date(Date.now() - (500 - i) * 60000).toISOString(),
}));
}
fs.writeFileSync(sessionFilePath, lines.join('\n'));
});
afterAll(() => {
cleanupDir(tmpDir);
});
it('loading and parsing a 500-message JSONL file completes within 500ms', () => {
const start = Date.now();
const raw = fs.readFileSync(sessionFilePath, 'utf-8');
const messages = raw.trim().split('\n').map(line => JSON.parse(line));
const elapsed = Date.now() - start;
expect(messages.length).toBe(500);
expect(messages[0].role).toBe('user');
expect(messages[499].role).toBe('assistant');
// 500ms is generous — 500 lines of JSONL should parse in <50ms
expect(elapsed).toBeLessThan(perfBudget(500));
});
});
// ── 4. Marketplace FTS5 Search ──────────────────────────────────────
describe('Marketplace Search', () => {
it('marketplace FTS5 search completes within 200ms (bundled DB)', () => {
// Try to find the bundled marketplace.db
// Use multiple resolution strategies (relative to test file and repo root)
const possiblePaths = [
path.resolve(__dirname, '../../../../marketplace/marketplace.db'),
path.resolve(__dirname, '../../../../../packages/marketplace/marketplace.db'),
path.resolve(process.cwd(), 'packages/marketplace/marketplace.db'),
];
let dbPath: string | null = null;
for (const p of possiblePaths) {
if (fs.existsSync(p)) {
dbPath = p;
break;
}
}
if (!dbPath) {
// Skip if bundled DB not available (e.g., fresh clone without seed)
console.log(' [skip] marketplace.db not found — skipping marketplace benchmark');
return;
}
// Open the bundled marketplace.db directly in read-only mode.
// SQLite supports concurrent readers, and the earlier copy-first
// approach was racy when other tests (backup-*) also touched the
// source file — a torn copy produced a malformed temp DB.
try {
const db = new Database(dbPath, { readonly: true, fileMustExist: true });
const start = Date.now();
const results = db.prepare(`
SELECT p.* FROM packages p
INNER JOIN packages_fts fts ON p.id = fts.rowid
WHERE packages_fts MATCH ?
LIMIT 20
`).all('code review');
const elapsed = Date.now() - start;
expect(elapsed).toBeLessThan(perfBudget(200));
// The marketplace should have at least some results for "code review"
expect(results.length).toBeGreaterThanOrEqual(0);
db.close();
} catch (err) {
// Source DB may be transiently malformed if another test is mid-write.
// Treat as a skip rather than a hard failure for this best-effort perf check.
if (String(err).includes('malformed') || String(err).includes('not a database')) {
console.log(' [skip] marketplace.db transient write — skipping');
return;
}
throw err;
}
});
});
// ── 5. Vault Operations ─────────────────────────────────────────────
describe('Vault Encrypt/Decrypt Cycle', () => {
let tmpDir: string;
let vault: VaultStore;
beforeEach(() => {
tmpDir = createTmpDir('vault');
vault = new VaultStore(tmpDir);
});
afterEach(() => {
cleanupDir(tmpDir);
});
it('vault set+get cycle completes within 100ms', () => {
const start = Date.now();
vault.set('BENCHMARK_KEY', 'sk-benchmark-secret-value-12345');
const result = vault.get('BENCHMARK_KEY');
const elapsed = Date.now() - start;
expect(result).not.toBeNull();
expect(result!.value).toBe('sk-benchmark-secret-value-12345');
// 100ms is generous — AES-256-GCM + file I/O should be <20ms
expect(elapsed).toBeLessThan(perfBudget(100));
});
it('10 sequential set+get cycles complete within 500ms', () => {
const start = Date.now();
for (let i = 0; i < 10; i++) {
vault.set(`KEY_${i}`, `secret-value-${i}-${'x'.repeat(100)}`);
const result = vault.get(`KEY_${i}`);
expect(result).not.toBeNull();
expect(result!.value).toBe(`secret-value-${i}-${'x'.repeat(100)}`);
}
const elapsed = Date.now() - start;
// 500ms for 10 encrypt+decrypt+file-write cycles
expect(elapsed).toBeLessThan(perfBudget(500));
});
it('vault list with 20 secrets completes within 50ms', () => {
// Pre-populate
for (let i = 0; i < 20; i++) {
vault.set(`LIST_KEY_${i}`, `value-${i}`);
}
const start = Date.now();
const secrets = vault.list();
const elapsed = Date.now() - start;
expect(secrets.length).toBe(20);
expect(elapsed).toBeLessThan(perfBudget(50));
});
});
});