Files
waggle-os/packages/server/tests/data-export.test.ts
shumeiko.it 06d4532598 refactor(server): serve memory and Memory Center from the memory contract (P1.5)
The memory HTTP API reads and writes server.memory instead of the legacy
MultiMind, FrameStore, SessionStore, KnowledgeGraph, MindErasure,
SuppressionStore and RawArchive objects.

- memory.ts: search (personal, workspace, all, global), frames list, write,
  edit and delete, stats and quick capture on the contract. Ids are strings.
  The access-count route is gone. New GET /api/memory/capabilities and
  PUT /api/memory/settings { extraction } for background fact extraction.
- memory-center.ts: list, get, create, patch, archive, confirm, trace,
  delete, erase and merge on the contract. Archive sets importance
  'temporary' (skipped by recall) and remembers the old importance; confirm
  stamps metadata.confirmedAt; 'unreviewed' means extracted in the
  background and not confirmed. Erase forgets one memory or every memory of
  a chat session. The original-source and suppression routes are removed:
  there is no raw archive and no harvest to re-import from.
- knowledge.ts returns an empty graph (not part of v1).
- export.ts writes memories/personal.json and memories/workspace-<id>.json
  from the contract.
- identity.ts loses a dead resolver; the erase safety check recognizes
  waggle.db.
- memory-records.ts holds the scope, status and creation-time helpers.
- normalizeToMemory stays in memory-center-legacy.ts until home.ts and
  artifacts.ts read memory through the contract.
- @waggle/shared MemoryKind follows the contract's kinds; Memory gains
  origin and sessionId.

Tests: new memory-routes and memory-center tests on the in-memory store;
local-mode and data-export tests seed memory through the contract; tests of
removed behaviour (source, suppression, sticky erasure, graph projection,
legacy normalizer, old erase and stats) are deleted.
2026-10-24 13:19:10 +02:00

213 lines
8.8 KiB
TypeScript

import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { Readable } from 'node:stream';
import { WorkspaceManager, WaggleConfig, VaultStore } from '@waggle/core';
import { PERSONAL_SCOPE, workspaceScope } from '@waggle/memory-contract';
import { buildLocalServer } from '../src/local/index.js';
import type { FastifyInstance } from 'fastify';
import { injectWithAuth } from './test-utils.js';
/**
* Minimal ZIP parser — reads the central directory to extract file names.
* ZIP files have a central directory at the end with all entry metadata.
*/
function extractZipFileNames(buffer: Buffer): string[] {
const names: string[] = [];
// Scan for central directory file headers (signature 0x02014b50)
for (let i = 0; i < buffer.length - 46; i++) {
if (
buffer[i] === 0x50 &&
buffer[i + 1] === 0x4b &&
buffer[i + 2] === 0x01 &&
buffer[i + 3] === 0x02
) {
const nameLen = buffer.readUInt16LE(i + 28);
const extraLen = buffer.readUInt16LE(i + 30);
const commentLen = buffer.readUInt16LE(i + 32);
const nameStart = i + 46;
if (nameStart + nameLen <= buffer.length) {
const name = buffer.toString('utf-8', nameStart, nameStart + nameLen);
names.push(name);
}
// Skip past this entry
i += 45 + nameLen + extraLen + commentLen;
}
}
return names;
}
/**
* Extract a specific file's content from a ZIP buffer.
* Reads local file headers (signature 0x04034b50) for uncompressed entries.
*/
function extractZipFileContent(buffer: Buffer, targetName: string): string | null {
for (let i = 0; i < buffer.length - 30; i++) {
if (
buffer[i] === 0x50 &&
buffer[i + 1] === 0x4b &&
buffer[i + 2] === 0x03 &&
buffer[i + 3] === 0x04
) {
const compressionMethod = buffer.readUInt16LE(i + 8);
const compressedSize = buffer.readUInt32LE(i + 18);
const uncompressedSize = buffer.readUInt32LE(i + 22);
const nameLen = buffer.readUInt16LE(i + 26);
const extraLen = buffer.readUInt16LE(i + 28);
const nameStart = i + 30;
if (nameStart + nameLen <= buffer.length) {
const name = buffer.toString('utf-8', nameStart, nameStart + nameLen);
if (name === targetName && compressionMethod === 0) {
// Stored (uncompressed) — read directly
const dataStart = nameStart + nameLen + extraLen;
if (dataStart + uncompressedSize <= buffer.length) {
return buffer.toString('utf-8', dataStart, dataStart + uncompressedSize);
}
}
}
}
}
return null;
}
describe('Data Export (GDPR)', () => {
let server: FastifyInstance;
let tmpDir: string;
let workspaceId: string;
beforeAll(async () => {
// Create a temp directory for test data
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-export-test-'));
// Create a workspace with a session
const wsManager = new WorkspaceManager(tmpDir);
const ws = wsManager.create({ name: 'Test Export WS', group: 'Testing' });
workspaceId = ws.id;
const sessionsDir = path.join(tmpDir, 'workspaces', ws.id, 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
fs.writeFileSync(
path.join(sessionsDir, 'test-session.jsonl'),
[
JSON.stringify({ type: 'meta', title: 'Export Test Session', created: new Date().toISOString() }),
JSON.stringify({ role: 'user', content: 'Hello, test export!', timestamp: new Date().toISOString() }),
JSON.stringify({ role: 'assistant', content: 'This is a test response for export.', timestamp: new Date().toISOString() }),
].join('\n') + '\n',
'utf-8',
);
// Create config.json with a test provider
const config = new WaggleConfig(tmpDir);
config.setDefaultModel('claude-sonnet-4-6');
config.setProvider('anthropic', { apiKey: 'sk-ant-secret-key-12345678', models: ['claude-sonnet-4-6'] });
config.save();
// Create vault with a test entry
const vault = new VaultStore(tmpDir);
vault.set('anthropic', 'sk-ant-secret-key-12345678', { models: ['claude-sonnet-4-6'] });
// Build the local server, then write memories through the contract
server = await buildLocalServer({ dataDir: tmpDir });
await server.memory.remember({ scope: PERSONAL_SCOPE, text: 'Export test memory', importance: 'normal' });
await server.memory.remember({ scope: PERSONAL_SCOPE, text: 'Another memory for export', importance: 'important' });
await server.memory.remember({ scope: workspaceScope(workspaceId), text: 'Workspace memory for export' });
}, 60_000);
afterAll(async () => {
await server?.close();
if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('POST /api/export returns a ZIP with correct Content-Type', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export' });
expect(res.statusCode).toBe(200);
expect(res.headers['content-type']).toBe('application/zip');
expect(res.headers['content-disposition']).toMatch(/^attachment; filename="waggle-export-\d{4}-\d{2}-\d{2}\.zip"$/);
});
it('ZIP contains expected directories (memories, sessions, workspaces)', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export' });
const buffer = Buffer.from(res.rawPayload);
const fileNames = extractZipFileNames(buffer);
// Check that key directories/files are present
const hasMemories = fileNames.some(n => n.startsWith('memories/'));
const hasSessions = fileNames.some(n => n.startsWith('sessions/'));
const hasWorkspaces = fileNames.some(n => n.startsWith('workspaces/'));
const hasSettings = fileNames.some(n => n === 'settings.json');
const hasVaultMeta = fileNames.some(n => n === 'vault-metadata.json');
expect(hasMemories).toBe(true);
expect(fileNames).toContain('memories/personal.json');
expect(fileNames).toContain(`memories/workspace-${workspaceId}.json`);
expect(hasSessions).toBe(true);
expect(hasWorkspaces).toBe(true);
expect(hasSettings).toBe(true);
expect(hasVaultMeta).toBe(true);
});
it('settings in export have masked API keys', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export' });
const buffer = Buffer.from(res.rawPayload);
const fileNames = extractZipFileNames(buffer);
// settings.json must exist
expect(fileNames).toContain('settings.json');
// For compressed entries, we verify by checking the raw buffer
// does NOT contain the original key as plaintext
const rawContent = buffer.toString('utf-8');
expect(rawContent).not.toContain('sk-ant-secret-key-12345678');
});
it('vault metadata excludes secret values', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export' });
const buffer = Buffer.from(res.rawPayload);
const fileNames = extractZipFileNames(buffer);
expect(fileNames).toContain('vault-metadata.json');
// Ensure the full ZIP content does NOT contain the decrypted secret value
const rawContent = buffer.toString('utf-8');
expect(rawContent).not.toContain('sk-ant-secret-key-12345678');
});
it('export works with empty data (new installation)', async () => {
// Create a fresh server with empty data
const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-export-empty-'));
const emptyServer = await buildLocalServer({ dataDir: emptyDir });
try {
const res = await injectWithAuth(emptyServer, { method: 'POST', url: '/api/export' });
expect(res.statusCode).toBe(200);
expect(res.headers['content-type']).toBe('application/zip');
const buffer = Buffer.from(res.rawPayload);
const fileNames = extractZipFileNames(buffer);
// Even with empty data, core files should exist
expect(fileNames).toContain('memories/personal.json');
expect(fileNames).toContain('settings.json');
expect(fileNames).toContain('vault-metadata.json');
} finally {
await emptyServer.close();
fs.rmSync(emptyDir, { recursive: true, force: true });
}
});
it('a workspace export holds only that workspace\'s memories', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export', payload: { workspaceId } });
expect(res.statusCode).toBe(200);
const fileNames = extractZipFileNames(Buffer.from(res.rawPayload));
expect(fileNames).toContain(`memories/workspace-${workspaceId}.json`);
expect(fileNames).not.toContain('memories/personal.json');
});
it('export ZIP has reasonable size (not empty)', async () => {
const res = await injectWithAuth(server, { method: 'POST', url: '/api/export' });
const buffer = Buffer.from(res.rawPayload);
// ZIP should be at least a few hundred bytes (headers + content)
expect(buffer.length).toBeGreaterThan(100);
});
});