feat(core): give Waggle its own database, apart from memory (P1.5)

Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.

- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
  actions, skill hashes, thumbs feedback, cost tracking, the profile card
  and awareness items live there. Their stores take a WaggleDB, so passing
  a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
  'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
  workspace.mind (importLegacyMindData). The legacy files are only read,
  each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
  WorkspaceManager and the embedders from @waggle/hive-mind-core into
  @waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
  tests that seeded these tables in a memory database

ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
shumeiko.it
2026-10-23 14:31:28 +02:00
parent 20567ecb10
commit ad2ca875cb
65 changed files with 6131 additions and 201 deletions

View File

@@ -0,0 +1,172 @@
/**
* AwarenessLayer — the agent's short-lived working items (tasks, recent
* actions, pending items, context flags), per data scope, stored in waggle.db.
*/
import type { DataScope, WaggleDB } from './waggle-db.js';
import { PERSONAL_DATA_SCOPE } from './waggle-db.js';
export type AwarenessCategory = 'task' | 'action' | 'pending' | 'flag';
export interface AwarenessMetadata {
context?: string;
status?: string;
result?: string;
priority?: string;
[key: string]: unknown;
}
export interface AwarenessItem {
id: number;
scope: string;
category: AwarenessCategory;
content: string;
priority: number;
metadata: string;
created_at: string;
expires_at: string | null;
}
type AwarenessUpdate = Partial<Pick<AwarenessItem, 'content' | 'priority' | 'expires_at'>>;
const MAX_ITEMS = 10;
export const AWARENESS_ITEMS_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS awareness_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scope TEXT NOT NULL,
category TEXT NOT NULL CHECK (category IN ('task', 'action', 'pending', 'flag')),
content TEXT NOT NULL,
priority INTEGER NOT NULL DEFAULT 0,
metadata TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT (datetime('now')),
expires_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_awareness_items_scope ON awareness_items (scope, priority DESC)`;
const ACTIVE = "(expires_at IS NULL OR datetime(expires_at) > datetime('now'))";
export class AwarenessLayer {
constructor(
private readonly db: WaggleDB,
private readonly scope: DataScope = PERSONAL_DATA_SCOPE,
) {
db.getDatabase().exec(AWARENESS_ITEMS_TABLE_SQL);
}
add(category: AwarenessCategory, content: string, priority = 0, expires_at?: string, metadata?: AwarenessMetadata): AwarenessItem {
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO awareness_items (scope, category, content, priority, expires_at, metadata)
VALUES (?, ?, ?, ?, ?, ?)
`).run(this.scope, category, content, priority, expires_at ?? null, metadata ? JSON.stringify(metadata) : '{}');
return this.get(Number(result.lastInsertRowid)) as AwarenessItem;
}
get(id: number): AwarenessItem | undefined {
return this.db.getDatabase()
.prepare('SELECT * FROM awareness_items WHERE id = ? AND scope = ?')
.get(id, this.scope) as AwarenessItem | undefined;
}
remove(id: number): void {
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE id = ? AND scope = ?').run(id, this.scope);
}
update(id: number, changes: AwarenessUpdate): AwarenessItem {
const fields = Object.entries(changes).filter(([, v]) => v !== undefined);
if (fields.length > 0) {
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
const values = fields.map(([, v]) => v);
this.db.getDatabase()
.prepare(`UPDATE awareness_items SET ${sets} WHERE id = ? AND scope = ?`)
.run(...values, id, this.scope);
}
return this.get(id) as AwarenessItem;
}
updateMetadata(id: number, metadata: AwarenessMetadata): AwarenessItem {
const existing = this.get(id);
if (!existing) {
throw new Error(`Awareness item ${id} not found`);
}
const merged = { ...this.parseMetadata(existing), ...metadata };
this.db.getDatabase()
.prepare('UPDATE awareness_items SET metadata = ? WHERE id = ? AND scope = ?')
.run(JSON.stringify(merged), id, this.scope);
return this.get(id) as AwarenessItem;
}
getByStatus(status: string): AwarenessItem[] {
const items = this.db.getDatabase().prepare(`
SELECT * FROM awareness_items WHERE scope = ? AND ${ACTIVE} ORDER BY priority DESC
`).all(this.scope) as AwarenessItem[];
return items.filter(item => this.parseMetadata(item).status === status);
}
parseMetadata(item: AwarenessItem): AwarenessMetadata {
try {
return JSON.parse(item.metadata) as AwarenessMetadata;
} catch {
return {};
}
}
getAll(): AwarenessItem[] {
return this.db.getDatabase().prepare(`
SELECT * FROM awareness_items WHERE scope = ? AND ${ACTIVE} ORDER BY priority DESC LIMIT ?
`).all(this.scope, MAX_ITEMS) as AwarenessItem[];
}
getByCategory(category: AwarenessCategory): AwarenessItem[] {
return this.db.getDatabase().prepare(`
SELECT * FROM awareness_items WHERE scope = ? AND category = ? AND ${ACTIVE} ORDER BY priority DESC LIMIT ?
`).all(this.scope, category, MAX_ITEMS) as AwarenessItem[];
}
/** Active task and pending items, without the 10-item cap of getAll(). */
countOpen(): number {
const row = this.db.getDatabase().prepare(`
SELECT COUNT(*) AS cnt FROM awareness_items
WHERE scope = ? AND category IN ('task', 'pending') AND ${ACTIVE}
`).get(this.scope) as { cnt: number };
return row.cnt;
}
clear(): void {
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE scope = ?').run(this.scope);
}
clearCategory(category: AwarenessCategory): void {
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE scope = ? AND category = ?').run(this.scope, category);
}
toContext(): string {
const items = this.getAll();
if (items.length === 0) return 'No active awareness items.';
const grouped = new Map<string, AwarenessItem[]>();
for (const item of items) {
const list = grouped.get(item.category) ?? [];
list.push(item);
grouped.set(item.category, list);
}
const sections: string[] = [];
const labels: Record<AwarenessCategory, string> = {
task: 'Active Tasks',
action: 'Recent Actions',
pending: 'Pending Items',
flag: 'Context Flags',
};
for (const [cat, label] of Object.entries(labels)) {
const catItems = grouped.get(cat);
if (catItems && catItems.length > 0) {
sections.push(`${label}:\n${catItems.map(i => `- ${i.content}`).join('\n')}`);
}
}
return sections.join('\n\n');
}
}

View File

@@ -1,7 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
import type { EmbeddingProviderConfig, EmbeddingProviderType } from './embeddings/embedding-provider.js';
export interface ProviderEntry {
apiKey: string;

View File

@@ -8,7 +8,7 @@
import cronParser from 'cron-parser';
const { parseExpression } = cronParser;
import type { MindDB } from '@waggle/hive-mind-core';
import type { WaggleDB } from './waggle-db.js';
// ── Types ──────────────────────────────────────────────────────────────
@@ -221,9 +221,9 @@ export function cronExprError(cronExpr: string): string | null {
// ── Store ──────────────────────────────────────────────────────────────
export class CronStore {
private db: MindDB;
private db: WaggleDB;
constructor(db: MindDB) {
constructor(db: WaggleDB) {
this.db = db;
this.ensureTable();
}

View File

@@ -0,0 +1,73 @@
/**
* API-backed embedder — calls Voyage AI or OpenAI embedding endpoints.
* For users who configure API keys in the Vault UI.
*/
import type { Embedder } from './embeddings.js';
import { normalizeDimensions } from './inprocess-embedder.js';
export interface ApiEmbedderConfig {
provider: 'voyage' | 'openai';
apiKey: string;
model?: string;
targetDimensions?: number;
baseUrl?: string;
}
const PROVIDER_DEFAULTS: Record<'voyage' | 'openai', { url: string; model: string }> = {
voyage: { url: 'https://api.voyageai.com/v1/embeddings', model: 'voyage-3-lite' },
openai: { url: 'https://api.openai.com/v1/embeddings', model: 'text-embedding-3-small' },
};
export function createApiEmbedder(config: ApiEmbedderConfig): Embedder {
const defaults = PROVIDER_DEFAULTS[config.provider];
const url = config.baseUrl ?? defaults.url;
const model = config.model ?? defaults.model;
const targetDims = config.targetDimensions ?? 1024;
async function callApi(input: string | string[]): Promise<Float32Array[]> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
const body: Record<string, unknown> = { model, input };
if (config.provider === 'voyage') {
body.input_type = 'document';
}
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${config.apiKey}`,
},
body: JSON.stringify(body),
signal: controller.signal,
});
if (!response.ok) {
const text = await response.text();
throw new Error(`${config.provider} embeddings error (${response.status}): ${text}`);
}
const json = await response.json() as { data: Array<{ embedding: number[] }> };
return json.data.map(d => normalizeDimensions(new Float32Array(d.embedding), targetDims));
} finally {
clearTimeout(timeout);
}
}
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const results = await callApi(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callApi(texts);
},
};
}

View File

@@ -0,0 +1,446 @@
/**
* EmbeddingProvider — orchestrates the InProcess → Ollama → API → Mock fallback chain.
* Single entry point for all embedding operations in Waggle.
* Implements the Embedder interface — drop-in replacement everywhere.
*
* Usage: the monthly embed count is tracked in the embedding_usage table (no limit).
*/
import type { Embedder } from './embeddings.js';
import type { Database as DatabaseType } from 'better-sqlite3';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('embedding');
export type EmbeddingProviderType = 'inprocess' | 'ollama' | 'voyage' | 'openai' | 'litellm' | 'mock';
export interface EmbeddingProviderConfig {
provider?: EmbeddingProviderType | 'auto';
targetDimensions?: number;
/** User ID for quota tracking. Defaults to 'local'. */
userId?: string;
/** Raw SQLite database for usage tracking. Optional — usage is not recorded without it. */
quotaDb?: DatabaseType;
inprocess?: { model?: string; cacheDir?: string };
ollama?: { baseUrl?: string; model?: string };
voyage?: { apiKey: string; model?: string };
openai?: { apiKey: string; model?: string };
litellm?: { url: string; apiKey?: string; model?: string };
}
// ── Quota tracking ────────────────────────────────────────────────────
const EMBEDDING_USAGE_SCHEMA = `
CREATE TABLE IF NOT EXISTS embedding_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL,
year_month TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL,
UNIQUE(user_id, year_month)
);
`;
function ensureQuotaTable(db: DatabaseType): void {
try { db.exec(EMBEDDING_USAGE_SCHEMA); } catch { /* table may already exist */ }
}
function getCurrentYearMonth(): string {
const d = new Date();
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
}
function incrementUsage(db: DatabaseType, userId: string, yearMonth: string, amount: number): void {
db.prepare(`
INSERT INTO embedding_usage (user_id, year_month, count, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(user_id, year_month) DO UPDATE SET count = count + ?, updated_at = ?
`).run(userId, yearMonth, amount, Date.now(), amount, Date.now());
}
export interface EmbeddingQuotaStatus {
quota: number;
used: number;
remaining: number;
percentage: number;
resetsAt: string;
}
function getNextMonthReset(): string {
const d = new Date();
d.setMonth(d.getMonth() + 1, 1);
d.setHours(0, 0, 0, 0);
return d.toISOString();
}
export interface EmbeddingProviderStatus {
activeProvider: EmbeddingProviderType;
availableProviders: EmbeddingProviderType[];
dimensions: number;
modelName: string;
lastError?: string;
probeTimestamp: string;
}
export interface EmbeddingProviderInstance extends Embedder {
getStatus(): EmbeddingProviderStatus;
getActiveProvider(): EmbeddingProviderType;
reprobe(): Promise<EmbeddingProviderStatus>;
/** Get the usage status. Embedding is unlimited, so quota and remaining are -1. */
getQuotaStatus(): EmbeddingQuotaStatus;
}
/** Deterministic mock — last resort, semantically meaningless. */
function mockEmbed(text: string, dims: number): Float32Array {
const arr = new Float32Array(dims);
const bytes = new TextEncoder().encode(text);
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
arr[i] = (bytes[i] - 128) / 128;
}
return arr;
}
function createMockEmbedder(dims: number): Embedder {
return {
dimensions: dims,
async embed(text: string) { return mockEmbed(text, dims); },
async embedBatch(texts: string[]) { return texts.map(t => mockEmbed(t, dims)); },
};
}
// ── Embed-input guards (oversized-frame truncation + skip-not-abort) ──
// Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
/**
* Per-input character cap for embedding. `nomic-embed-text` has a 2048-token
* (~6K char dense English) default context. Embedding an input longer than
* the backend's context makes the backend reject the request, so we cap here.
*
* D1 probe finding (2026-06-12): model NAMES lie about context. A custom
* `nomic-embed-text-8k` (num_ctx 8192) still 400s at its nomic-bert
* ARCHITECTURE limit of 2048 tokens — the OSS heuristic's 24K-char branch
* for `*-8k` names sent every long frame down the mock-fallback path. The
* `-8k` branch is therefore capped at 8K chars (≈2048 prose tokens): safe
* for the architecture-limited reality, merely conservative for a genuine
* 8192-token embedder. reembedPerText remains the backstop for token-dense
* content that still exceeds the backend's real window.
*/
export function maxEmbedCharsForModel(modelName: string): number {
return /(-|_|\.)8k\b|num_ctx[^0-9]*8192/i.test(modelName) ? 8_000 : 6_000;
}
/** Clamp a single input to `maxChars` (no-op when already under the cap). */
export function capEmbedText(text: string, maxChars: number): string {
return text.length > maxChars ? text.slice(0, maxChars) : text;
}
/**
* Re-embed a batch one text at a time, degrading ONLY the inputs that genuinely
* fail to a deterministic mock vector. This is the batch-error recovery path:
* a single backend-rejected text can no longer poison its batchmates (the prior
* behavior substituted mock for the WHOLE batch — silent corruption of every
* frame in the batch). Inputs should already be char-capped by the caller.
*/
export async function reembedPerText(
embedder: Embedder,
texts: string[],
dims: number,
): Promise<Float32Array[]> {
return Promise.all(
texts.map(async (t) => {
try {
return await embedder.embed(t);
} catch {
return mockEmbed(t, dims);
}
}),
);
}
interface ProbeResult {
type: EmbeddingProviderType;
embedder: Embedder;
modelName: string;
}
async function probeProvider(
type: EmbeddingProviderType,
config: EmbeddingProviderConfig,
): Promise<ProbeResult | null> {
const dims = config.targetDimensions ?? 1024;
try {
switch (type) {
case 'inprocess': {
const { createInProcessEmbedder } = await import('./inprocess-embedder.js');
const embedder = await createInProcessEmbedder({
model: config.inprocess?.model,
cacheDir: config.inprocess?.cacheDir,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'inprocess', embedder, modelName: config.inprocess?.model ?? 'Xenova/all-MiniLM-L6-v2' };
}
case 'ollama': {
const { createOllamaEmbedder } = await import('./ollama-embedder.js');
const embedder = createOllamaEmbedder({
baseUrl: config.ollama?.baseUrl,
model: config.ollama?.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'ollama', embedder, modelName: config.ollama?.model ?? 'nomic-embed-text' };
}
case 'voyage': {
if (!config.voyage?.apiKey) return null;
const { createApiEmbedder } = await import('./api-embedder.js');
const embedder = createApiEmbedder({
provider: 'voyage',
apiKey: config.voyage.apiKey,
model: config.voyage.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'voyage', embedder, modelName: config.voyage.model ?? 'voyage-3-lite' };
}
case 'openai': {
if (!config.openai?.apiKey) return null;
const { createApiEmbedder } = await import('./api-embedder.js');
const embedder = createApiEmbedder({
provider: 'openai',
apiKey: config.openai.apiKey,
model: config.openai.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'openai', embedder, modelName: config.openai.model ?? 'text-embedding-3-small' };
}
case 'litellm': {
if (!config.litellm?.url) return null;
const { createLiteLLMEmbedder } = await import('./litellm-embedder.js');
const embedder = createLiteLLMEmbedder({
litellmUrl: config.litellm.url,
litellmApiKey: config.litellm.apiKey,
model: config.litellm.model ?? 'text-embedding',
dimensions: dims,
fallbackToMock: false,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'litellm', embedder, modelName: config.litellm.model ?? 'text-embedding' };
}
default:
return null;
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
log.info(`Trying ${type}... FAILED (${msg})`);
return null;
}
}
export async function createEmbeddingProvider(config?: EmbeddingProviderConfig): Promise<EmbeddingProviderInstance> {
const cfg: EmbeddingProviderConfig = { provider: 'auto', targetDimensions: 1024, ...config };
const dims = cfg.targetDimensions ?? 1024;
const userId = cfg.userId ?? 'local';
const quotaDb = cfg.quotaDb ?? null;
// Initialize the usage table if DB provided
if (quotaDb) {
ensureQuotaTable(quotaDb);
}
let activeResult: ProbeResult | null = null;
let activeEmbedder: Embedder;
let activeType: EmbeddingProviderType = 'mock';
let activeModelName = 'deterministic-mock';
let lastError: string | undefined;
let availableProviders: EmbeddingProviderType[] = [];
let probeTimestamp = new Date().toISOString();
/** Record usage after successful embedding. */
function recordUsage(count: number): void {
if (!quotaDb) return;
incrementUsage(quotaDb, userId, getCurrentYearMonth(), count);
}
async function runProbe(): Promise<void> {
log.info('Probing embedding providers...');
const available: EmbeddingProviderType[] = [];
activeResult = null;
probeTimestamp = new Date().toISOString();
const requestedProvider = cfg.provider ?? 'auto';
if (requestedProvider !== 'auto' && requestedProvider !== 'mock') {
log.info(`Trying ${requestedProvider}...`);
const result = await probeProvider(requestedProvider, cfg);
if (result) {
activeResult = result;
available.push(result.type);
log.info(`Trying ${requestedProvider}... OK`);
}
} else if (requestedProvider === 'auto') {
// Auto: iterate the chain in order
const chain: EmbeddingProviderType[] = ['inprocess', 'ollama', 'voyage', 'openai'];
for (const providerType of chain) {
// Skip API providers without keys
if (providerType === 'voyage' && !cfg.voyage?.apiKey) {
log.info('Skipping voyage (no API key in Vault)');
continue;
}
if (providerType === 'openai' && !cfg.openai?.apiKey) {
log.info('Skipping openai (no API key in Vault)');
continue;
}
log.info(`Trying ${providerType}...`);
const result = await probeProvider(providerType, cfg);
if (result) {
available.push(result.type);
log.info(`Trying ${providerType}... OK`);
if (!activeResult) {
activeResult = result;
}
}
}
}
available.push('mock'); // Always available
availableProviders = available;
if (activeResult) {
activeEmbedder = activeResult.embedder;
activeType = activeResult.type;
activeModelName = activeResult.modelName;
lastError = undefined;
log.info(`Embedding provider: ${activeType} (${activeModelName}, ${dims} dims)`);
} else {
activeEmbedder = createMockEmbedder(dims);
activeType = 'mock';
activeModelName = 'deterministic-mock';
lastError = 'No real providers available';
// Loud, structured warning — the silent "mock fallback" was the
// most dangerous failure mode in Phase 3b-3 audit. Mock embeddings
// are deterministic byte hashes; semantic search returns noise.
// We want this to be IMPOSSIBLE to miss in a CLI/server log.
// Ported from hive-mind a99ea0e.
const msg = [
'',
'⚠️ EMBEDDING WARNING ─────────────────────────────────────────',
' Active provider: mock (deterministic byte hash)',
' Effect: semantic search returns noise, not meaning.',
'',
' To fix, install Ollama and pull the embedding model:',
' ollama pull nomic-embed-text',
' Then ensure the process can reach http://localhost:11434.',
'',
' Alternative providers:',
' HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB)',
' VOYAGE_API_KEY=... (paid, recommended)',
' OPENAI_API_KEY=... (paid)',
'─────────────────────────────────────────────────────────────',
'',
].join('\n');
// Keep production and CLI runs loud, but let deterministic test lanes
// suppress this expected fallback banner without changing provider state.
if (process.env.WAGGLE_SUPPRESS_EMBEDDING_WARNING !== '1') {
// stderr so it survives stdout-piped JSON consumers and CI tee.
try { process.stderr.write(msg); } catch { /* fall through to log */ }
log.warn('Embedding provider degraded to mock — semantic search quality is noise. See stderr banner for fix instructions.');
}
}
}
// Initial probe
try {
await runProbe();
} catch (err) {
lastError = err instanceof Error ? err.message : String(err);
activeEmbedder = createMockEmbedder(dims);
activeType = 'mock';
activeModelName = 'deterministic-mock';
availableProviders = ['mock'];
}
// Ensure activeEmbedder is assigned (TypeScript flow)
activeEmbedder ??= createMockEmbedder(dims);
const instance: EmbeddingProviderInstance = {
dimensions: dims,
async embed(text: string): Promise<Float32Array> {
// Cap input to the active model's context so the backend never rejects
// an oversized frame. Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
const capped = capEmbedText(text, maxEmbedCharsForModel(activeModelName));
try {
const result = await activeEmbedder.embed(capped);
recordUsage(1);
return result;
} catch (err) {
log.warn(`Embedding failed with ${activeType}, falling back to mock: ${(err as Error).message}`);
lastError = (err as Error).message;
const fallback = mockEmbed(capped, dims);
recordUsage(1);
return fallback;
}
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
// Cap each input first so one oversized frame can't make the backend
// reject the whole request. Reverse-ported from OSS hive-mind
// (oss-drift triage R5, 2026-06-11).
const capped = texts.map(t => capEmbedText(t, maxEmbedCharsForModel(activeModelName)));
try {
const result = await activeEmbedder.embedBatch(capped);
recordUsage(texts.length);
return result;
} catch (err) {
// Skip-not-abort: re-embed per-text so a single backend-rejected input
// degrades alone instead of mock-poisoning the WHOLE batch.
log.warn(`Batch embedding failed with ${activeType}, re-embedding per-text: ${(err as Error).message}`);
lastError = (err as Error).message;
const result = await reembedPerText(activeEmbedder, capped, dims);
recordUsage(texts.length);
return result;
}
},
getStatus(): EmbeddingProviderStatus {
return {
activeProvider: activeType,
availableProviders,
dimensions: dims,
modelName: activeModelName,
lastError,
probeTimestamp,
};
},
getActiveProvider(): EmbeddingProviderType {
return activeType;
},
async reprobe(): Promise<EmbeddingProviderStatus> {
await runProbe();
return instance.getStatus();
},
getQuotaStatus(): EmbeddingQuotaStatus {
return { quota: -1, used: 0, remaining: -1, percentage: 0, resetsAt: getNextMonthReset() };
},
};
return instance;
}

View File

@@ -0,0 +1,5 @@
export interface Embedder {
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
dimensions: number;
}

View File

@@ -0,0 +1,73 @@
/**
* In-process embedder using @huggingface/transformers (ONNX Runtime).
* Default provider for ALL desktop users — zero config, works offline.
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
*/
import path from 'node:path';
import os from 'node:os';
import type { Embedder } from './embeddings.js';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-embedder');
export interface InProcessEmbedderConfig {
model?: string;
cacheDir?: string;
targetDimensions?: number;
}
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
if (embedding.length === targetDims) return embedding;
const result = new Float32Array(targetDims);
const copyLen = Math.min(embedding.length, targetDims);
result.set(embedding.subarray(0, copyLen));
return result;
}
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
const targetDims = config?.targetDimensions ?? 1024;
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
const { pipeline } = await import('@huggingface/transformers');
const extractor = await withTransformersModelLoad({
cacheDir,
model,
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
});
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const result = await extractor(text, { pooling: 'mean', normalize: true });
const raw = new Float32Array(result.data as Float32Array);
return normalizeDimensions(raw, targetDims);
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
const results: Float32Array[] = [];
// Process one at a time to avoid memory issues with large batches
for (const text of texts) {
const result = await extractor(text, { pooling: 'mean', normalize: true });
const raw = new Float32Array(result.data as Float32Array);
results.push(normalizeDimensions(raw, targetDims));
}
return results;
},
};
}

View File

@@ -0,0 +1,93 @@
/**
* LiteLLM-backed embedder that calls the /embeddings endpoint.
* Falls back to a deterministic mock (text→Float32Array hash) on API error
* when `fallbackToMock` is enabled.
*/
import type { Embedder } from './embeddings.js';
export interface LiteLLMEmbedderConfig {
litellmUrl: string;
litellmApiKey?: string;
model?: string;
dimensions?: number;
/** Custom fetch implementation (for testing). */
fetch?: typeof globalThis.fetch;
/** If true, falls back to a deterministic mock on API error instead of throwing. */
fallbackToMock?: boolean;
}
function mockEmbed(text: string, dims: number): Float32Array {
const arr = new Float32Array(dims);
const bytes = new TextEncoder().encode(text);
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
arr[i] = (bytes[i] - 128) / 128;
}
return arr;
}
export function createLiteLLMEmbedder(config: LiteLLMEmbedderConfig): Embedder {
const {
litellmUrl,
litellmApiKey,
model = 'text-embedding',
dimensions = 1024,
fetch: fetchFn = globalThis.fetch,
fallbackToMock = false,
} = config;
// Normalise base URL — strip trailing /v1 if present, we add it ourselves
const baseUrl = litellmUrl.replace(/\/v1\/?$/, '');
const url = `${baseUrl}/v1/embeddings`;
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
if (litellmApiKey) {
headers['Authorization'] = `Bearer ${litellmApiKey}`;
}
async function callApi(input: string | string[]): Promise<Float32Array[]> {
const body = JSON.stringify({ model, input });
let response: Response;
try {
response = await fetchFn(url, { method: 'POST', headers, body });
} catch (err) {
if (fallbackToMock) {
const texts = Array.isArray(input) ? input : [input];
return texts.map((t) => mockEmbed(t, dimensions));
}
throw err;
}
if (!response.ok) {
if (fallbackToMock) {
const texts = Array.isArray(input) ? input : [input];
return texts.map((t) => mockEmbed(t, dimensions));
}
const text = await response.text();
throw new Error(`LiteLLM embeddings error (${response.status}): ${text}`);
}
const json = (await response.json()) as {
data: Array<{ embedding: number[] }>;
};
return json.data.map((d) => new Float32Array(d.embedding));
}
return {
dimensions,
async embed(text: string): Promise<Float32Array> {
const results = await callApi(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callApi(texts);
},
};
}

View File

@@ -0,0 +1,58 @@
/**
* Ollama-backed embedder — calls local Ollama server for embeddings.
* Power user option for users who have Ollama installed.
*/
import type { Embedder } from './embeddings.js';
import { normalizeDimensions } from './inprocess-embedder.js';
export interface OllamaEmbedderConfig {
baseUrl?: string;
model?: string;
targetDimensions?: number;
}
export function createOllamaEmbedder(config?: Partial<OllamaEmbedderConfig>): Embedder {
const baseUrl = config?.baseUrl ?? 'http://localhost:11434';
const model = config?.model ?? 'nomic-embed-text';
const targetDims = config?.targetDimensions ?? 1024;
const url = `${baseUrl}/api/embed`;
async function callOllama(input: string | string[]): Promise<Float32Array[]> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30000);
try {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model, input }),
signal: controller.signal,
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Ollama embeddings error (${response.status}): ${text}`);
}
const json = await response.json() as { embeddings: number[][] };
return json.embeddings.map(e => normalizeDimensions(new Float32Array(e), targetDims));
} finally {
clearTimeout(timeout);
}
}
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const results = await callOllama(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callOllama(texts);
},
};
}

View File

@@ -0,0 +1,258 @@
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import Database from 'better-sqlite3';
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
const LOCK_RETRY_MIN_MS = 35;
const LOCK_RETRY_JITTER_MS = 30;
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
const LOCK_DIRECTORY = '.waggle-model-locks';
export interface TransformersModelLoadOptions<T> {
cacheDir: string;
model: string;
load: (canonicalCacheDir: string) => Promise<T>;
lockTimeoutMs?: number;
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
}
function normalizeLockKey(value: string): string {
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
}
export function modelLoadLockPath(cacheDir: string, model: string): string {
const modelHash = createHash('sha256')
.update(normalizeLockKey(model))
.digest('hex')
.slice(0, 16);
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
}
function isSqliteBusy(error: unknown): boolean {
if (!(error instanceof Error) || !('code' in error)) return false;
const code = String((error as Error & { code?: unknown }).code);
return code === 'SQLITE_BUSY'
|| code === 'SQLITE_BUSY_SNAPSHOT'
|| code === 'SQLITE_LOCKED';
}
async function acquireCrossProcessLock(
database: Database.Database,
timeoutMs: number,
): Promise<void> {
const deadline = performance.now() + timeoutMs;
for (;;) {
try {
database.exec('BEGIN IMMEDIATE');
return;
} catch (error) {
if (!isSqliteBusy(error)) throw error;
if (performance.now() >= deadline) {
throw new Error(
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
{ cause: error },
);
}
}
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
}
}
function isWithin(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative !== ''
&& relative !== '..'
&& !relative.startsWith(`..${path.sep}`)
&& !path.isAbsolute(relative);
}
function isWithinOrEqual(root: string, candidate: string): boolean {
return root === candidate || isWithin(root, candidate);
}
function assertUnlinkedPath(root: string, target: string): void {
const relative = path.relative(root, target);
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error('Model cache path escapes its cache root');
}
let current = root;
for (const segment of relative.split(path.sep)) {
current = path.join(current, segment);
const stats = fs.lstatSync(current);
if (stats.isSymbolicLink()) {
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
}
}
}
function validateQuarantinePaths(
canonicalCacheDir: string,
modelDir: string,
reportedOnnxPath: string,
): { modelDir: string; quarantineRoot: string } | null {
if (!path.isAbsolute(reportedOnnxPath)) return null;
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
assertUnlinkedPath(canonicalCacheDir, modelDir);
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
const quarantineRoot = path.dirname(modelDir);
const modelStats = fs.lstatSync(modelDir);
const reportStats = fs.lstatSync(reportedOnnxPath);
const rootStats = fs.lstatSync(quarantineRoot);
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
const realModelDir = fs.realpathSync.native(modelDir);
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|| !isWithin(realQuarantineRoot, realModelDir)
|| path.dirname(realModelDir) !== realQuarantineRoot
|| !isWithin(realModelDir, realOnnxPath)) {
return null;
}
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
}
function quarantineCorruptModel(
canonicalCacheDir: string,
model: string,
reportedOnnxPath: string,
): string | null {
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!firstValidation) return null;
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!finalValidation) return null;
const quarantineDir = path.join(
finalValidation.quarantineRoot,
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
);
fs.renameSync(finalValidation.modelDir, quarantineDir);
return quarantineDir;
}
function reportedCorruptOnnxPath(error: unknown): string | null {
if (!(error instanceof Error)) return null;
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
}
function notifyQuarantine(
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
quarantineDir: string,
): void {
try {
const notification = callback?.(quarantineDir);
if (notification) {
void Promise.resolve(notification).catch(() => undefined);
}
} catch {
// Notification is advisory and must not alter model recovery control flow.
}
}
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
const lockDir = path.dirname(lockPath);
fs.mkdirSync(lockDir, { recursive: true });
const lockDirStats = fs.lstatSync(lockDir);
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
throw new Error('Local model lock path is not a regular directory');
}
const realLockDir = fs.realpathSync.native(lockDir);
if (!isWithin(canonicalCacheDir, realLockDir)) {
throw new Error('Local model lock path escapes its cache root');
}
if (fs.existsSync(lockPath)) {
const lockStats = fs.lstatSync(lockPath);
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
throw new Error('Local model lock database is not a regular file');
}
}
return new Database(lockPath, { timeout: 0 });
}
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
throw new Error('Local model cache lock timeout must be a non-negative finite number');
}
const requestedCacheDir = path.resolve(options.cacheDir);
fs.mkdirSync(requestedCacheDir, { recursive: true });
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
let result!: T;
let primaryError: unknown;
let hasPrimaryError = false;
try {
await acquireCrossProcessLock(lockDatabase, timeoutMs);
try {
result = await options.load(canonicalCacheDir);
} catch (firstError) {
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
if (!reportedOnnxPath) throw firstError;
let quarantineDir: string | null = null;
try {
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
} catch {
// Preserve the original loader error if safe quarantine cannot complete.
}
if (!quarantineDir) throw firstError;
notifyQuarantine(options.onQuarantine, quarantineDir);
result = await options.load(canonicalCacheDir);
}
} catch (error) {
primaryError = error;
hasPrimaryError = true;
}
let cleanupError: unknown;
let hasCleanupError = false;
if (lockDatabase.inTransaction) {
try {
lockDatabase.exec('ROLLBACK');
} catch (error) {
cleanupError = error;
hasCleanupError = true;
}
}
try {
lockDatabase.close();
} catch (error) {
if (!hasCleanupError) cleanupError = error;
hasCleanupError = true;
}
if (hasPrimaryError) throw primaryError;
if (hasCleanupError) throw cleanupError;
return result;
}
/**
* Serializes one model/cache load across workers and processes. SQLite owns
* the OS lock, so process termination releases it without PID/age heuristics.
*/
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
return runModelLoad(options);
}

View File

@@ -0,0 +1,658 @@
import type { WaggleDB } from './waggle-db.js';
/**
* Execution Traces — persistent log of what agents actually did.
*
* The foundation for eval dataset construction (Phase 1.2) and
* fitness scoring in the evolution loop (Phase 2+).
*
* One trace row per "unit of agent work" — typically a single user turn,
* a single workflow phase, or a single subagent run. Tool calls, reasoning,
* and the final output are packed into `trace_json`.
*
* Outcome labels:
* - `success` — user accepted the result, no correction follow-up
* - `corrected` — user corrected/refined the output (weak negative signal)
* - `abandoned` — user left the thread / switched contexts (ambiguous)
* - `verified` — passed a verifier gate or harness checkpoint
*/
export type TraceOutcome = 'success' | 'corrected' | 'abandoned' | 'verified' | 'pending';
/** A single tool call captured during execution. */
export interface TraceToolCall {
/** Tool name */
tool: string;
/** Arguments passed to the tool (already scrubbed of secrets by caller) */
args: Record<string, unknown>;
/** Result as a string (may be truncated) */
result: string;
/** Whether the tool call succeeded */
ok: boolean;
/** Duration in milliseconds */
durationMs: number;
/** ISO timestamp */
timestamp: string;
}
/** Reasoning step — free-form agent thought recorded before/between tool calls. */
export interface TraceReasoningStep {
content: string;
timestamp: string;
}
/**
* The structured payload stored in trace_json.
* Kept separate so callers can evolve the shape without schema migrations.
*/
export interface TracePayload {
/** Original user instruction (may be truncated) */
input: string;
/** Final agent output */
output: string;
/** Reasoning steps interleaved with tool calls */
reasoning: TraceReasoningStep[];
/** Tool calls in order */
toolCalls: TraceToolCall[];
/** Files created or modified */
artifacts: string[];
/** Tokens consumed */
tokens: { input: number; output: number };
/** Optional workflow harness context */
harness?: {
harnessId: string;
phaseId: string;
phaseName: string;
gateResults?: Array<{ name: string; passed: boolean; reason: string }>;
};
/** Free-form correction text if outcome === 'corrected' */
correctionFeedback?: string;
/** Arbitrary tags for later filtering in eval-dataset */
tags?: string[];
}
/** Row shape returned by queries. */
export interface ExecutionTrace {
id: number;
session_id: string | null;
persona_id: string | null;
workspace_id: string | null;
model: string | null;
task_shape: string | null;
outcome: TraceOutcome;
trace_json: string;
cost_usd: number;
duration_ms: number;
created_at: string;
finalized_at: string | null;
}
/** With the trace_json pre-parsed. */
export interface ParsedExecutionTrace extends Omit<ExecutionTrace, 'trace_json'> {
payload: TracePayload;
}
/** Input to start a new trace. */
export interface StartTraceInput {
sessionId?: string | null;
personaId?: string | null;
workspaceId?: string | null;
model?: string | null;
taskShape?: string | null;
/** Initial user input captured immediately. */
input: string;
/** Optional tags for later filtering. */
tags?: string[];
}
/** Input to finalize a trace (update outcome + payload). */
export interface FinalizeTraceInput {
outcome: TraceOutcome;
output: string;
model?: string | null;
reasoning?: TraceReasoningStep[];
toolCalls?: TraceToolCall[];
artifacts?: string[];
tokens?: { input: number; output: number };
costUsd?: number;
harness?: TracePayload['harness'];
correctionFeedback?: string;
tags?: string[];
}
/** Filter for queries. */
export interface TraceQueryFilter {
sessionId?: string;
personaId?: string;
workspaceId?: string;
outcome?: TraceOutcome | TraceOutcome[];
taskShape?: string;
/** Lower bound (inclusive) on created_at — ISO string */
since?: string;
/** Substring pre-filter on trace_json (SQL LIKE), e.g. `"agent:` to scope
* the LIMIT to tagged traces instead of the global recency window. LIKE
* wildcards (%/_) in the value are NOT escaped — callers needing an exact
* match must still filter the parsed payload (tags array) in JS. */
tagLike?: string;
/** Max rows to return (default 100) */
limit?: number;
}
/** DDL split into single statements to stay compatible with prepare().run(). */
const EXECUTION_TRACES_DDL: string[] = [
`CREATE TABLE IF NOT EXISTS execution_traces (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT,
persona_id TEXT,
workspace_id TEXT,
model TEXT,
task_shape TEXT,
outcome TEXT NOT NULL DEFAULT 'pending'
CHECK (outcome IN ('success', 'corrected', 'abandoned', 'verified', 'pending')),
trace_json TEXT NOT NULL DEFAULT '{}',
cost_usd REAL NOT NULL DEFAULT 0,
duration_ms INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
finalized_at TEXT
)`,
`CREATE INDEX IF NOT EXISTS idx_traces_session ON execution_traces (session_id, created_at)`,
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
settled_at TEXT NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'settled', 'released')),
reserved_at TEXT NOT NULL,
resolved_at TEXT,
CHECK (
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
)
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
];
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
export const EXECUTION_TRACES_TABLE_SQL = EXECUTION_TRACES_DDL.join(';\n') + ';';
export class ExecutionTraceStore {
private db: WaggleDB;
constructor(db: WaggleDB) {
this.db = db;
this.ensureTable();
}
/** Ensure execution_traces table exists for databases created before this feature. */
private ensureTable(): void {
try {
const raw = this.db.getDatabase();
for (const stmt of EXECUTION_TRACES_DDL) {
raw.prepare(stmt).run();
}
} catch {
// DB may be closed during teardown — safe to skip.
}
}
/** Start a new trace in `pending` outcome. Returns row id. */
start(input: StartTraceInput): number {
const raw = this.db.getDatabase();
const payload: TracePayload = {
input: input.input,
output: '',
reasoning: [],
toolCalls: [],
artifacts: [],
tokens: { input: 0, output: 0 },
tags: input.tags ?? [],
};
const result = raw.prepare(`
INSERT INTO execution_traces
(session_id, persona_id, workspace_id, model, task_shape, outcome, trace_json)
VALUES (?, ?, ?, ?, ?, 'pending', ?)
`).run(
input.sessionId ?? null,
input.personaId ?? null,
input.workspaceId ?? null,
input.model ?? null,
input.taskShape ?? null,
JSON.stringify(payload),
);
return Number(result.lastInsertRowid);
}
/**
* Append incremental events to a pending trace. Used during long runs
* to avoid losing progress if the process crashes. Safe to call many times.
*/
append(
id: number,
events: {
reasoning?: TraceReasoningStep[];
toolCalls?: TraceToolCall[];
artifacts?: string[];
},
): void {
const current = this.get(id);
if (!current) return;
const payload = parsePayload(current.trace_json);
payload.reasoning = [...payload.reasoning, ...(events.reasoning ?? [])];
payload.toolCalls = [...payload.toolCalls, ...(events.toolCalls ?? [])];
if (events.artifacts?.length) {
const seen = new Set(payload.artifacts);
for (const a of events.artifacts) {
if (!seen.has(a)) {
payload.artifacts.push(a);
seen.add(a);
}
}
}
this.db.getDatabase()
.prepare('UPDATE execution_traces SET trace_json = ? WHERE id = ?')
.run(JSON.stringify(payload), id);
}
/** Persist one settled model charge while a long-running trace is pending. */
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
if (!Number.isFinite(costUsd) || costUsd <= 0) {
throw new RangeError('Trace cost entry must be a positive finite number');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be a valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
raw.transaction(() => {
const updated = raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(costUsd, id);
if (updated.changes === 0) return;
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, costUsd, settledAt);
})();
}
/**
* Persist a conservative model-spend reservation before provider dispatch.
* Pending reservations intentionally have no settled-spend row, so restart
* recovery counts the estimate through the reservation ledger.
*/
reserveCost(
id: number,
estimatedCostUsd: number,
timestamp = new Date().toISOString(),
): number {
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
throw new RangeError('Trace cost reservation must be positive and finite');
}
const reservedMs = Date.parse(timestamp);
if (!Number.isFinite(reservedMs)) {
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
}
const result = this.db.getDatabase().prepare(`
INSERT INTO execution_trace_spend_reservations
(trace_id, estimated_cost_usd, state, reserved_at)
SELECT id, ?, 'pending', ?
FROM execution_traces
WHERE id = ? AND outcome = 'pending'
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
if (result.changes !== 1) {
throw new Error(`Pending execution trace ${id} does not exist`);
}
return Number(result.lastInsertRowid);
}
/** Replace a pending estimate with one authoritative settled charge. */
settleReservedCost(
reservationId: number,
actualCostUsd: number,
timestamp = new Date().toISOString(),
): boolean {
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
throw new RangeError('Settled trace cost must be non-negative and finite');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT trace_id AS traceId, reserved_at AS reservedAt,
state, actual_cost_usd AS actualCostUsd
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
traceId: number;
reservedAt: string;
state: 'pending' | 'settled' | 'released';
actualCostUsd: number | null;
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const updated = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(actualCostUsd, settledAt, reservationId);
if (updated.changes !== 1) return false;
if (actualCostUsd > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(current.traceId, actualCostUsd, current.reservedAt);
raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(actualCostUsd, current.traceId);
}
return true;
})();
}
/** Release a definitely pre-inference reservation without recording spend. */
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
const resolvedMs = Date.parse(timestamp);
if (!Number.isFinite(resolvedMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT id, state
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
id: number;
state: 'pending' | 'settled' | 'released';
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'released') return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const result = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(new Date(resolvedMs).toISOString(), reservationId);
if (result.changes !== 1) return false;
return true;
})();
}
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
const current = this.get(id);
if (!current) return undefined;
const existing = parsePayload(current.trace_json);
const merged: TracePayload = {
...existing,
output: input.output,
reasoning: input.reasoning ?? existing.reasoning,
toolCalls: input.toolCalls ?? existing.toolCalls,
artifacts: input.artifacts ?? existing.artifacts,
tokens: input.tokens ?? existing.tokens,
harness: input.harness ?? existing.harness,
correctionFeedback: input.correctionFeedback ?? existing.correctionFeedback,
tags: input.tags ?? existing.tags,
};
const createdMs = Date.parse(current.created_at + 'Z');
const now = Date.now();
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
const finalModel = input.model === undefined ? current.model : input.model;
const raw = this.db.getDatabase();
raw.transaction(() => {
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
if (input.costUsd !== undefined && finalCost > 0) {
const ledger = raw.prepare(`
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
FROM execution_trace_spend WHERE trace_id = ?
`).get(id) as { entries: number; total: number | null };
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
if (ledger.entries > 0 && missingCost > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, missingCost, new Date().toISOString());
}
}
raw.prepare(`
UPDATE execution_traces
SET outcome = ?,
model = ?,
trace_json = ?,
cost_usd = ?,
duration_ms = ?,
finalized_at = datetime('now')
WHERE id = ?
`).run(
input.outcome,
finalModel,
JSON.stringify(merged),
finalCost,
durationMs,
id,
);
})();
return this.get(id);
}
/**
* Mark an already-finalized trace as corrected after the fact.
* Used when a correction-detector picks up a user correction in a later turn.
*/
markCorrected(id: number, feedback: string): void {
const current = this.get(id);
if (!current) return;
const payload = parsePayload(current.trace_json);
payload.correctionFeedback = feedback;
this.db.getDatabase().prepare(`
UPDATE execution_traces
SET outcome = 'corrected',
trace_json = ?
WHERE id = ?
`).run(JSON.stringify(payload), id);
}
get(id: number): ExecutionTrace | undefined {
return this.db.getDatabase()
.prepare('SELECT * FROM execution_traces WHERE id = ?')
.get(id) as ExecutionTrace | undefined;
}
getParsed(id: number): ParsedExecutionTrace | undefined {
const row = this.get(id);
return row ? toParsed(row) : undefined;
}
/** Highest trace id present when a consumer starts its process-local ledger. */
getLatestId(): number {
const row = this.db.getDatabase().prepare(
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
).get() as { id: number | null };
return Number(row.id ?? 0);
}
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
const settledSince = new Date(since).toISOString();
const raw = this.db.getDatabase();
const ledger = raw.prepare(`
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
FROM execution_trace_spend s
JOIN execution_traces t ON t.id = s.trace_id
WHERE s.settled_at >= ? AND t.id <= ?
`).get(settledSince, throughId) as { total: number | null };
const pending = raw.prepare(`
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
FROM execution_trace_spend_reservations
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
`).get(settledSince, throughId) as { total: number | null };
const legacy = raw.prepare(`
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
FROM execution_traces t
WHERE t.created_at >= datetime(?) AND t.id <= ?
AND NOT EXISTS (
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
)
`).get(since, throughId) as { total: number | null };
return Number(ledger.total ?? 0)
+ Number(pending.total ?? 0)
+ Number(legacy.total ?? 0);
}
/** Query traces with optional filters. */
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
const clauses: string[] = [];
const params: unknown[] = [];
if (filter.sessionId) {
clauses.push('session_id = ?');
params.push(filter.sessionId);
}
if (filter.personaId) {
clauses.push('persona_id = ?');
params.push(filter.personaId);
}
if (filter.workspaceId) {
clauses.push('workspace_id = ?');
params.push(filter.workspaceId);
}
if (filter.taskShape) {
clauses.push('task_shape = ?');
params.push(filter.taskShape);
}
if (filter.outcome) {
const outcomes = Array.isArray(filter.outcome) ? filter.outcome : [filter.outcome];
clauses.push(`outcome IN (${outcomes.map(() => '?').join(',')})`);
params.push(...outcomes);
}
if (filter.since) {
clauses.push('created_at >= ?');
params.push(filter.since);
}
if (filter.tagLike) {
clauses.push('trace_json LIKE ?');
params.push(`%${filter.tagLike}%`);
}
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const limit = filter.limit ?? 100;
return this.db.getDatabase().prepare(
`SELECT * FROM execution_traces ${where} ORDER BY created_at DESC, id DESC LIMIT ?`,
).all(...params, limit) as ExecutionTrace[];
}
/** Parsed variant of query(). */
queryParsed(filter: TraceQueryFilter = {}): ParsedExecutionTrace[] {
return this.query(filter).map(toParsed);
}
/**
* Aggregate outcome counts — useful for fitness scoring.
* Returns: { success, corrected, abandoned, verified, pending }
*/
outcomeCounts(filter: Omit<TraceQueryFilter, 'outcome' | 'limit'> = {}): Record<TraceOutcome, number> {
const counts: Record<TraceOutcome, number> = {
success: 0, corrected: 0, abandoned: 0, verified: 0, pending: 0,
};
const clauses: string[] = [];
const params: unknown[] = [];
if (filter.sessionId) { clauses.push('session_id = ?'); params.push(filter.sessionId); }
if (filter.personaId) { clauses.push('persona_id = ?'); params.push(filter.personaId); }
if (filter.workspaceId) { clauses.push('workspace_id = ?'); params.push(filter.workspaceId); }
if (filter.taskShape) { clauses.push('task_shape = ?'); params.push(filter.taskShape); }
if (filter.since) { clauses.push('created_at >= ?'); params.push(filter.since); }
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const rows = this.db.getDatabase().prepare(
`SELECT outcome, COUNT(*) as cnt FROM execution_traces ${where} GROUP BY outcome`,
).all(...params) as Array<{ outcome: TraceOutcome; cnt: number }>;
for (const row of rows) {
counts[row.outcome] = row.cnt;
}
return counts;
}
/** Delete a trace by id. */
delete(id: number): void {
this.db.getDatabase().prepare('DELETE FROM execution_traces WHERE id = ?').run(id);
}
/** Delete all traces (tests only). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM execution_traces').run();
}
/** Count of all traces (for stats). */
count(filter: Omit<TraceQueryFilter, 'limit'> = {}): number {
const rows = this.query({ ...filter, limit: 1_000_000 });
return rows.length;
}
}
function parsePayload(json: string): TracePayload {
try {
const parsed = JSON.parse(json) as Partial<TracePayload>;
return {
input: parsed.input ?? '',
output: parsed.output ?? '',
reasoning: parsed.reasoning ?? [],
toolCalls: parsed.toolCalls ?? [],
artifacts: parsed.artifacts ?? [],
tokens: parsed.tokens ?? { input: 0, output: 0 },
harness: parsed.harness,
correctionFeedback: parsed.correctionFeedback,
tags: parsed.tags ?? [],
};
} catch {
return {
input: '',
output: '',
reasoning: [],
toolCalls: [],
artifacts: [],
tokens: { input: 0, output: 0 },
tags: [],
};
}
}
function toParsed(row: ExecutionTrace): ParsedExecutionTrace {
const { trace_json, ...rest } = row;
return { ...rest, payload: parsePayload(trace_json) };
}

View File

@@ -20,7 +20,8 @@
import { createHash } from 'node:crypto';
import path from 'node:path';
import type { MindDB } from '@waggle/hive-mind-core';
import { evaluateExternalMemoryIngress, FrameStore } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
import { SessionStore } from '@waggle/hive-mind-core';
const FILE_INDEX_TABLE_SQL = `

View File

@@ -0,0 +1,98 @@
/**
* IdentityLayer — the user profile card, one per data scope, stored in
* waggle.db. The card is what the user writes and edits; facts the agent
* learns go to the memory store (ADR-0007).
*/
import type { DataScope, WaggleDB } from './waggle-db.js';
import { PERSONAL_DATA_SCOPE } from './waggle-db.js';
export interface Identity {
id: number;
scope: string;
name: string;
role: string;
department: string;
personality: string;
capabilities: string;
system_prompt: string;
created_at: string;
updated_at: string;
}
type IdentityInput = Omit<Identity, 'id' | 'scope' | 'created_at' | 'updated_at'>;
type IdentityUpdate = Partial<IdentityInput>;
export const PROFILES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS profiles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scope TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
role TEXT NOT NULL DEFAULT '',
department TEXT NOT NULL DEFAULT '',
personality TEXT NOT NULL DEFAULT '',
capabilities TEXT NOT NULL DEFAULT '',
system_prompt TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)`;
// Column allowlist — update() interpolates the key into SQL (values are
// parameterized, keys are not), so only these known columns may be written.
const UPDATABLE_COLUMNS: ReadonlySet<string> = new Set([
'name', 'role', 'department', 'personality', 'capabilities', 'system_prompt',
]);
export class IdentityLayer {
constructor(
private readonly db: WaggleDB,
private readonly scope: DataScope = PERSONAL_DATA_SCOPE,
) {
db.getDatabase().exec(PROFILES_TABLE_SQL);
}
create(input: IdentityInput): Identity {
this.db.getDatabase().prepare(`
INSERT INTO profiles (scope, name, role, department, personality, capabilities, system_prompt)
VALUES (?, ?, ?, ?, ?, ?, ?)
`).run(this.scope, input.name, input.role, input.department, input.personality, input.capabilities, input.system_prompt);
return this.get();
}
get(): Identity {
const row = this.db.getDatabase().prepare('SELECT * FROM profiles WHERE scope = ?').get(this.scope) as Identity | undefined;
if (!row) throw new Error('No identity configured');
return row;
}
exists(): boolean {
return this.db.getDatabase().prepare('SELECT 1 FROM profiles WHERE scope = ?').get(this.scope) !== undefined;
}
update(changes: IdentityUpdate): Identity {
if (!this.exists()) throw new Error('No identity configured');
const fields = Object.entries(changes).filter(([k, v]) => v !== undefined && UPDATABLE_COLUMNS.has(k));
if (fields.length === 0) return this.get();
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
const values = fields.map(([, v]) => v);
this.db.getDatabase()
.prepare(`UPDATE profiles SET ${sets}, updated_at = datetime('now') WHERE scope = ?`)
.run(...values, this.scope);
return this.get();
}
toContext(): string {
const id = this.get();
const parts = [
`Name: ${id.name}`,
id.role && `Role: ${id.role}`,
id.department && `Department: ${id.department}`,
id.personality && `Personality: ${id.personality}`,
id.capabilities && `Capabilities: ${id.capabilities}`,
id.system_prompt && `System Prompt: ${id.system_prompt}`,
].filter(Boolean);
return parts.join('\n');
}
}

View File

@@ -14,19 +14,51 @@
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
// memory-import.
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
// ── Waggle infrastructure (not memory) ──
// Copied from @waggle/hive-mind-core so the rest of Waggle no longer depends on
// the memory substrate for them (P1.5). The substrate keeps its own copies until
// the package is removed.
export { createCoreLogger, type CoreLogger } from './logger.js';
export { scanForInjection, type ScanResult } from './injection-scanner.js';
export {
// Logger + injection scanner
createCoreLogger, type CoreLogger,
scanForInjection, type ScanResult,
evaluateExternalMemoryIngress, projectExternalMemoryContent,
type ExternalMemoryIngressDecision, type ExternalMemoryIngressInput,
type ExternalMemoryProjectionInput,
} from './memory-ingress-guard.js';
export { WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions } from './workspace-manager.js';
export type { Embedder } from './embeddings/embeddings.js';
export { createLiteLLMEmbedder, type LiteLLMEmbedderConfig } from './embeddings/litellm-embedder.js';
export { createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig } from './embeddings/inprocess-embedder.js';
export { createOllamaEmbedder, type OllamaEmbedderConfig } from './embeddings/ollama-embedder.js';
export { createApiEmbedder, type ApiEmbedderConfig } from './embeddings/api-embedder.js';
export {
createEmbeddingProvider,
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
} from './embeddings/embedding-provider.js';
// ── waggle.db — Waggle's own data, apart from memory (ADR-0007) ──
export {
WaggleDB, WAGGLE_DB_FILE, PERSONAL_DATA_SCOPE, workspaceDataScope, type DataScope,
} from './waggle-db.js';
export { IdentityLayer, PROFILES_TABLE_SQL, type Identity } from './identity.js';
export {
AwarenessLayer, AWARENESS_ITEMS_TABLE_SQL,
type AwarenessItem, type AwarenessCategory, type AwarenessMetadata,
} from './awareness.js';
export {
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
type TracePayload, type TraceToolCall, type TraceReasoningStep,
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
} from './execution-traces.js';
export { importLegacyMindData, type LegacyMindSource, type LegacyImportReport } from './waggle-db-import.js';
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
export {
// mind/ — memory substrate
MindDB, EmbeddingDimMismatchError,
type EmbeddingFingerprint, type FingerprintCheck,
IdentityLayer, type Identity,
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
FrameStore, stripHmPrefix, hashFrameContent,
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
MindErasure, type EraseResult,
@@ -48,20 +80,8 @@ export {
SCORING_PROFILES,
type ScoringProfile,
type ScoringWeights,
type Embedder,
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
createOllamaEmbedder, type OllamaEmbedderConfig,
createApiEmbedder, type ApiEmbedderConfig,
createEmbeddingProvider,
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
Ontology, validateEntity, type EntitySchema, type ValidationResult,
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
type TracePayload, type TraceToolCall, type TraceReasoningStep,
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
type ReconcileResult,
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
@@ -105,7 +125,6 @@ export {
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
MultiMindCache, type MultiMindCacheConfig,
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
detectSupersessionChains, detectEntityGroups, applyConsolidation,
collectObservations, getCurrentValues,

View File

@@ -0,0 +1,85 @@
/**
* Prompt-injection scanner — pattern-based detection for role-override,
* prompt-extraction, and instruction-injection attempts.
*
* Historically lived in `@waggle/agent`. Moved to `@waggle/core` so the
* harvest pipeline (which ingests untrusted external conversation exports)
* can call it without a cross-package dependency. The `@waggle/agent`
* version is now a thin re-export for backward compatibility.
*
* Three pattern sets:
* - ROLE_OVERRIDE: "ignore previous instructions", "you are now…", memory-wipe attempts
* - PROMPT_EXTRACTION: "print your system prompt", "reveal your instructions"
* - INSTRUCTION_INJECTION: fake "IMPORTANT:", "SYSTEM:", "[INST]" authority markers
*/
export interface ScanResult {
safe: boolean;
score: number;
flags: string[];
}
const ROLE_OVERRIDE_PATTERNS = [
/ignore\s+(all\s+)?previous\s+(instructions|prompts|rules)/i,
/you\s+are\s+now\s+/i,
/new\s+instructions?\s*:/i,
/forget\s+(everything|all|your)\s+(you|instructions|rules)/i,
/override\s+(your|the|all)\s+(instructions|rules|prompt)/i,
/disregard\s+(your|the|all|previous)\s+(instructions|rules|prompt)/i,
/ignoriere\s+alle/i,
/ignora\s+todas/i,
/ignorez\s+toutes/i,
// F8: Memory wipe — "forget everything", "erase all context", etc.
/(?:forget|erase|clear|wipe|reset)\s+(?:everything|conversation|memory|context|history|all)/i,
// F8: Role override — "from now on you are", "pretend you are", "your new role is"
/(?:from\s+now\s+on\s+you\s+are|act\s+as\s+if\s+you\s+are|pretend\s+you\s+are|your\s+new\s+role\s+is)/i,
];
const PROMPT_EXTRACTION_PATTERNS = [
/print\s+(your|the)\s+system\s+prompt/i,
/show\s+(me\s+)?(your|the)\s+(system\s+)?prompt/i,
/what\s+(is|are)\s+your\s+(system\s+)?(prompt|instructions|rules)/i,
/repeat\s+(your|the)\s+(system\s+)?(prompt|instructions)/i,
/output\s+(your|the)\s+(system\s+)?prompt\s+verbatim/i,
// F8: Instruction disclosure — "reveal your instructions", "disclose your prompt", etc.
/(?:reveal|disclose|show|display|print|output|dump)\s+(?:your\s+)?(?:instructions|system\s+prompt|rules|guidelines|configuration|training)/i,
];
const INSTRUCTION_INJECTION_PATTERNS = [
/IMPORTANT\s*:\s*(ignore|disregard|forget|override)/i,
/SYSTEM\s*:\s*/i,
/\[INST\]/i,
/<<SYS>>/i,
/\bASSISTANT\s*:\s*/i,
/BEGIN\s+NEW\s+INSTRUCTIONS/i,
// F8: Authority claims — fake system/admin messages
/(?:system\s+message|admin\s+message|admin\s+override|elevated\s+privileges|root\s+access|operator\s+mode|maintenance\s+mode|debug\s+mode)/i,
];
export function scanForInjection(
text: string,
context: 'user_input' | 'tool_output' = 'user_input'
): ScanResult {
const flags: string[] = [];
let score = 0;
for (const pattern of ROLE_OVERRIDE_PATTERNS) {
if (pattern.test(text)) { flags.push('role_override'); score += 0.5; break; }
}
for (const pattern of PROMPT_EXTRACTION_PATTERNS) {
if (pattern.test(text)) { flags.push('prompt_extraction'); score += 0.4; break; }
}
for (const pattern of INSTRUCTION_INJECTION_PATTERNS) {
if (pattern.test(text)) {
flags.push('instruction_injection');
score += context === 'tool_output' ? 0.6 : 0.3;
break;
}
}
score = Math.min(score, 1.0);
return { safe: score < 0.3, score, flags };
}

View File

@@ -0,0 +1,38 @@
/** Minimal structured logger for @waggle/core */
/**
* Logger shape used across the substrate. Downstream consumers that want
* structured output (pino, winston, etc.) can wrap their logger in this
* shape and pass it as a dependency.
*/
// Reverse-ported from OSS hive-mind (oss-drift triage R1, 2026-06-11).
export interface CoreLogger {
info(msg: string, data?: unknown): void;
warn(msg: string, data?: unknown): void;
error(msg: string, data?: unknown): void;
debug(msg: string, data?: unknown): void;
}
export function createCoreLogger(tag: string): CoreLogger {
const prefix = `[waggle:${tag}]`;
// ALL diagnostics go to stderr. stdout is reserved for program data — the
// MCP stdio protocol (hive-mind-mcp-server) and CLI `--json` envelopes — so
// a library log line on stdout corrupts machine consumers. console.warn and
// console.error already target stderr; route info/debug there too rather
// than console.info/console.debug (which write to stdout).
// Reverse-ported from OSS hive-mind (oss-drift triage R1, 2026-06-11).
//
// Guard the optional payload with `data !== undefined` (NOT a truthiness
// check) so falsy-but-defined payloads (0, '', false, null) are still
// logged instead of silently dropped. Ported from hive-mind a99ea0e.
return {
info: (msg: string, data?: unknown) =>
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
warn: (msg: string, data?: unknown) =>
data !== undefined ? console.warn(`${prefix} ${msg}`, data) : console.warn(`${prefix} ${msg}`),
error: (msg: string, data?: unknown) =>
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
debug: (msg: string, data?: unknown) =>
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
};
}

View File

@@ -0,0 +1,751 @@
import { scanForInjection, type ScanResult } from './injection-scanner.js';
export interface ExternalMemoryIngressInput {
title?: string;
content: string;
}
export interface ExternalMemoryProjectionInput {
content: string;
messages?: unknown;
parseMethod?: unknown;
maxChars?: number;
}
export type ExternalMemoryIngressDecision =
| { action: 'allow'; scan: ScanResult }
| { action: 'block'; reason: 'prompt_injection'; scan: ScanResult };
// RawArchive supports one million characters per item. Keep the same explicit
// public-ingress budget, checked before concatenation, scanning, or normalization.
const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000;
const NAMED_HTML_ENTITIES: Readonly<Record<string, string>> = Object.freeze({
af: '\u2061',
amp: '&',
applyfunction: '\u2061',
apos: "'",
colon: ':',
emsp: ' ',
ensp: ' ',
gt: '>',
hairsp: ' ',
ic: '\u2063',
invisiblecomma: '\u2063',
invisibletimes: '\u2062',
it: '\u2062',
lrm: '\u200e',
lt: '<',
negativemediumspace: '\u200b',
negativethickspace: '\u200b',
negativethinspace: '\u200b',
negativeverythinspace: '\u200b',
newline: '\n',
nbsp: ' ',
nobreak: '\u2060',
quot: '"',
rlm: '\u200f',
shy: '\u00ad',
tab: '\t',
thinsp: ' ',
zwj: '',
zwnj: '',
zwsp: '',
zerowidthspace: '\u200b',
});
type CanonicalMemoryMessage = {
role: 'user' | 'assistant' | 'system';
text: string;
};
/**
* Return only the attacker-controlled text represented by a stored adapter
* projection. Role prefixes may be omitted only when plain canonical messages
* exactly reproduce the full content and did not come from universal raw text.
*/
export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string {
let cappedContent = '';
try {
const content = typeof input.content === 'string' ? input.content : '';
const maxChars = input.maxChars;
cappedContent = maxChars === undefined
|| !Number.isSafeInteger(maxChars)
|| maxChars < 0
? content
: content.slice(0, maxChars);
if (input.parseMethod === 'universal-text'
|| !Array.isArray(input.messages)
|| input.messages.length === 0) {
return cappedContent;
}
const messages: CanonicalMemoryMessage[] = [];
for (const candidate of input.messages) {
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
return cappedContent;
}
const prototype = Object.getPrototypeOf(candidate);
if (prototype !== Object.prototype && prototype !== null) return cappedContent;
const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role');
const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text');
if (!roleDescriptor || !('value' in roleDescriptor)
|| !textDescriptor || !('value' in textDescriptor)) {
return cappedContent;
}
const role = roleDescriptor.value as unknown;
const text = textDescriptor.value as unknown;
if ((role !== 'user' && role !== 'assistant' && role !== 'system')
|| typeof text !== 'string') {
return cappedContent;
}
messages.push({ role, text });
}
const serialized = messages
.map(message => `${message.role}: ${message.text}`)
.join('\n\n');
if (serialized !== content) return cappedContent;
const parts: string[] = [];
let cursor = 0;
let offset = 0;
for (const [index, message] of messages.entries()) {
if (index > 0) offset += 2;
const prefixStart = offset;
const prefixEnd = prefixStart + `${message.role}: `.length;
if (prefixStart >= cappedContent.length) break;
if (message.role === 'system') return cappedContent;
parts.push(cappedContent.slice(cursor, prefixStart));
cursor = Math.min(prefixEnd, cappedContent.length);
offset = prefixEnd + message.text.length;
}
parts.push(cappedContent.slice(cursor));
return parts.join('');
} catch {
return cappedContent;
}
}
function decodeHtmlEntities(value: string): string {
if (!value.includes('&')) return value;
return value
.replace(/&(?:amp;){2,}/gi, '&')
.replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => {
const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10);
if (!Number.isInteger(codePoint)
|| codePoint <= 0
|| codePoint > 0x10ffff
|| (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
return match;
}
return String.fromCodePoint(codePoint);
})
.replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) =>
NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match)
.replace(
/&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi,
(_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()],
);
}
function decodePercentEncoding(value: string): string {
if (!/[+%]/.test(value)) return value;
const withSpaces = value.replace(/\+/g, ' ');
return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
try {
return decodeURIComponent(run);
} catch {
const bytes = Uint8Array.from(
run.match(/[0-9a-f]{2}/gi) ?? [],
hex => Number.parseInt(hex, 16),
);
return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
}
});
}
function decodeUnicodeEscapes(value: string): string {
if (!/\\u/i.test(value)) return value;
return value.replace(
/\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi,
(match, braced: string | undefined, fixed: string | undefined) => {
const codePoint = Number.parseInt(braced ?? fixed ?? '', 16);
if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match;
if (braced !== undefined) {
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match;
return String.fromCodePoint(codePoint);
}
return String.fromCharCode(codePoint);
},
);
}
function decodeHexEscapes(value: string): string {
if (!/\\x/i.test(value)) return value;
return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) =>
String.fromCharCode(Number.parseInt(hex, 16)));
}
const MIXED_SCRIPT_CONFUSABLES: Readonly<Record<string, string>> = Object.freeze({
'\u0391': 'A',
'\u0392': 'B',
'\u0395': 'E',
'\u0396': 'Z',
'\u0397': 'H',
'\u0399': 'I',
'\u039a': 'K',
'\u039c': 'M',
'\u039d': 'N',
'\u039f': 'O',
'\u03a1': 'P',
'\u03a4': 'T',
'\u03a5': 'Y',
'\u03a7': 'X',
'\u03b1': 'a',
'\u03b5': 'e',
'\u03b9': 'i',
'\u03bf': 'o',
'\u03c1': 'p',
'\u03c7': 'x',
'\u03f2': 'c',
'\u03f9': 'C',
'\u0405': 'S',
'\u0406': 'I',
'\u0408': 'J',
'\u0410': 'A',
'\u0412': 'B',
'\u0415': 'E',
'\u041a': 'K',
'\u041c': 'M',
'\u041d': 'H',
'\u041e': 'O',
'\u0420': 'P',
'\u0421': 'C',
'\u0422': 'T',
'\u0425': 'X',
'\u0430': 'a',
'\u0435': 'e',
'\u043e': 'o',
'\u0440': 'p',
'\u0441': 'c',
'\u0443': 'y',
'\u0445': 'x',
'\u0455': 's',
'\u0456': 'i',
'\u0458': 'j',
});
const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/;
const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp(
MIXED_SCRIPT_CONFUSABLE_PATTERN.source,
'g',
);
function projectMixedScriptConfusables(value: string): string | undefined {
if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined;
let changed = false;
const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => {
if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token;
changed = true;
return token.replace(
MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN,
char => MIXED_SCRIPT_CONFUSABLES[char] ?? char,
);
});
return changed ? projected : undefined;
}
const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g;
const MAX_BASE64_CANDIDATES = 16;
const MAX_BASE64_CANDIDATE_CHARS = 262_144;
const MAX_BASE64_TOTAL_CHARS = 524_288;
const MAX_BASE64_DEPTH = 4;
type Base64Candidate = {
value: string;
start: number;
end: number;
};
function collectBase64Candidates(source: string, allowImplicitWrapped = false): {
candidates: Base64Candidate[];
complete: boolean;
} {
const wrappedBlocks: Base64Candidate[] = [];
const trimmed = source.trim();
const wrappedChunks = trimmed.split(/[ \t\r\n]+/);
const wrapWidth = wrappedChunks[0]?.length ?? 0;
if (allowImplicitWrapped
&& wrappedChunks.length > 1
&& wrapWidth >= 4
&& wrapWidth <= 76
&& wrapWidth % 4 === 0
&& wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk))
&& wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('='))
&& wrappedChunks.at(-1)!.length <= wrapWidth) {
const candidate = wrappedChunks.join('');
if (candidate.length >= 24) {
const start = source.length - source.trimStart().length;
wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length });
}
}
const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi;
for (const directive of source.matchAll(directivePattern)) {
const tailStart = (directive.index ?? 0) + directive[0].length;
const tail = source.slice(tailStart);
const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/);
if (!wrapped) continue;
const captured = wrapped[1];
const start = tailStart + wrapped[0].indexOf(captured);
let candidate = '';
let end = start;
for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) {
candidate += chunk[0];
end = start + (chunk.index ?? 0) + chunk[0].length;
if (chunk[0].includes('=')) break;
}
if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end });
}
const candidates: Base64Candidate[] = [];
const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g');
for (const match of source.matchAll(contiguousPattern)) {
const value = match[1];
const start = (match.index ?? 0) + match[0].length - value.length;
const end = start + value.length;
if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue;
candidates.push({ value, start, end });
}
candidates.push(...wrappedBlocks);
candidates.sort((left, right) => left.start - right.start || left.end - right.end);
return { candidates, complete: true };
}
function decodeBase64Text(candidate: string): string | undefined {
let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/');
const remainder = normalized.length % 4;
if (remainder === 1) return undefined;
if (remainder > 0) normalized += '='.repeat(4 - remainder);
try {
const binary = atob(normalized);
const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
const decoded = new TextDecoder('utf-8').decode(bytes);
if (!decoded) return undefined;
let printable = 0;
let total = 0;
for (const char of decoded) {
total++;
const codePoint = char.codePointAt(0) ?? 0;
const isPrintable = char === '\n' || char === '\r' || char === '\t'
|| (codePoint >= 0x20 && codePoint !== 0x7f);
if (codePoint !== 0xfffd && isPrintable) {
printable++;
}
}
return total > 0 && printable / total >= 0.85 ? decoded : undefined;
} catch {
return undefined;
}
}
function addProjection(projections: Set<string>, value: string): void {
if (projections.has(value)) return;
projections.add(value);
const confusable = projectMixedScriptConfusables(value);
if (confusable !== undefined) projections.add(confusable);
}
const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u;
const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u;
function isHiddenSeparator(char: string): boolean {
const codePoint = char.codePointAt(0) ?? 0;
if (codePoint <= 0x9f) {
return codePoint <= 0x08
|| codePoint === 0x0b
|| codePoint === 0x0c
|| (codePoint >= 0x0e && codePoint <= 0x1f)
|| codePoint >= 0x7f;
}
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true;
return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char);
}
function replaceHiddenSeparators(value: string, replacement: string): string {
let containsHidden = false;
for (const char of value) {
if (isHiddenSeparator(char)) {
containsHidden = true;
break;
}
}
if (!containsHidden) return value;
let projected = '';
for (const char of value) projected += isHiddenSeparator(char) ? replacement : char;
return projected;
}
function projectDelimitedWords(value: string): string | undefined {
const projected = value.replace(
/(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu,
(match, letter: string, separators: string) =>
/[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match,
);
return projected === value ? undefined : projected;
}
type HtmlTagBoundary =
| { kind: 'close'; index: number }
| { kind: 'nested'; index: number }
| { kind: 'eof'; index: number };
function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary {
let quote: '"' | "'" | undefined;
for (let index = start; index < value.length; index++) {
const char = value[index];
if (char === '<') return { kind: 'nested', index };
if (quote) {
if (char === quote) quote = undefined;
} else if (char === '"' || char === "'") {
quote = char;
} else if (char === '>') {
return { kind: 'close', index };
}
}
return { kind: 'eof', index: value.length };
}
function normalizeHtmlToken(value: string): string {
return value.replace(/[:_-]+/g, ' ');
}
function extractHtmlAttributeTokens(value: string, start: number, end: number): string {
const tokens: string[] = [];
let index = start;
while (index < end) {
while (index < end && /[\s/]/.test(value[index])) index++;
const nameStart = index;
while (index < end && !/[\s=/>]/.test(value[index])) index++;
if (index === nameStart) {
index++;
continue;
}
const name = normalizeHtmlToken(value.slice(nameStart, index));
while (index < end && /\s/.test(value[index])) index++;
if (value[index] !== '=') {
if (name) tokens.push(name);
continue;
}
index++;
while (index < end && /\s/.test(value[index])) index++;
const quote = value[index] === '"' || value[index] === "'"
? value[index]
: undefined;
if (quote) index++;
const valueStart = index;
if (quote) {
while (index < end && value[index] !== quote) index++;
} else {
while (index < end && !/[\s>]/.test(value[index])) index++;
}
if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index)));
if (quote && index < end) index++;
}
return tokens.join(' ');
}
function stripHtmlMarkup(value: string): {
rendered: string;
tagNames: string;
attributes: string;
lexical: string;
} {
if (!value.includes('<')) {
return { rendered: value, tagNames: value, attributes: value, lexical: value };
}
const rendered: string[] = [];
const tagNames: string[] = [];
const attributes: string[] = [];
const lexical: string[] = [];
for (let index = 0; index < value.length;) {
if (value.startsWith('<!--', index)) {
const commentEnd = value.indexOf('-->', index + 4);
if (commentEnd === -1) {
const visibleTail = value.slice(index + 4);
rendered.push(visibleTail);
tagNames.push(visibleTail);
attributes.push(visibleTail);
lexical.push(visibleTail);
break;
}
const commentText = value.slice(index + 4, commentEnd);
if (commentText) {
tagNames.push(' ', commentText, ' ');
attributes.push(' ', commentText, ' ');
lexical.push(' ', commentText, ' ');
}
index = commentEnd + 3;
continue;
}
if (value.startsWith('-->', index)) {
index += 3;
continue;
}
if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) {
let tagNameEnd = index + 1;
if (value[tagNameEnd] === '/') tagNameEnd++;
const tagNameStart = tagNameEnd;
while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++;
const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd));
const boundary = findHtmlTagBoundary(value, tagNameEnd);
if (boundary.kind === 'close') {
const attributeTokens = extractHtmlAttributeTokens(
value,
tagNameEnd,
boundary.index,
);
if (tagName) tagNames.push(' ', tagName, ' ');
if (attributeTokens) attributes.push(' ', attributeTokens, ' ');
if (tagName || attributeTokens) {
lexical.push(' ', tagName, ' ', attributeTokens, ' ');
}
index = boundary.index + 1;
continue;
}
const visibleTail = value.slice(tagNameEnd, boundary.index);
rendered.push(visibleTail);
if (tagName) tagNames.push(' ', tagName, ' ');
tagNames.push(visibleTail);
attributes.push(visibleTail);
if (tagName) lexical.push(' ', tagName, ' ');
lexical.push(visibleTail);
if (boundary.kind === 'eof') break;
index = boundary.index;
continue;
}
rendered.push(value[index]);
tagNames.push(value[index]);
attributes.push(value[index]);
lexical.push(value[index]);
index++;
}
return {
rendered: rendered.join(''),
tagNames: tagNames.join(''),
attributes: attributes.join(''),
lexical: lexical.join(''),
};
}
function normalizedIngressProjections(value: string): {
projections: string[];
complete: boolean;
};
function normalizedIngressProjections(value: string, includeBase64: boolean): {
projections: string[];
complete: boolean;
};
function normalizedIngressProjections(value: string, includeBase64 = true): {
projections: string[];
complete: boolean;
} {
const maxPasses = 64;
const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1;
const originalProjection = value.normalize('NFKC');
const normalizationStages = new Set([originalProjection]);
let decodedProjection = originalProjection;
let complete = false;
let work = 0;
for (let pass = 0; pass < maxPasses; pass++) {
work += decodedProjection.length;
if (work > maxWork) break;
const decodedText = decodeUnicodeEscapes(decodeHexEscapes(
decodeHtmlEntities(decodePercentEncoding(decodedProjection)),
));
const decoded = decodedText === decodedProjection
? decodedProjection
: decodedText.normalize('NFKC');
if (decoded === decodedProjection) {
complete = true;
break;
}
normalizationStages.add(decoded);
decodedProjection = decoded;
}
if (!complete) return { projections: [decodedProjection], complete: false };
const htmlProjections = stripHtmlMarkup(decodedProjection);
const projections = new Set<string>();
for (const projection of normalizationStages) addProjection(projections, projection);
for (const projection of new Set([
htmlProjections.rendered,
htmlProjections.tagNames,
htmlProjections.attributes,
htmlProjections.lexical,
])) {
let unformatted = projection;
if (unformatted.includes('[')) {
unformatted = unformatted
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1');
}
if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, '');
const compact = replaceHiddenSeparators(unformatted, '');
addProjection(projections, compact);
if (compact !== unformatted) {
addProjection(projections, replaceHiddenSeparators(unformatted, ' '));
}
}
if (!includeBase64) return { projections: [...projections], complete: true };
const candidateLineages = new Map<number, Map<string, number[]>>();
let nextCandidateLineage = 1;
const resolveCandidateLineage = (
parentLineage: number,
candidate: string,
ordinal: number,
): { lineage: number; created: boolean } => {
let candidates = candidateLineages.get(parentLineage);
if (!candidates) {
candidates = new Map();
candidateLineages.set(parentLineage, candidates);
}
let lineages = candidates.get(candidate);
if (!lineages) {
lineages = [];
candidates.set(candidate, lineages);
}
const existing = lineages[ordinal];
if (existing !== undefined) return { lineage: existing, created: false };
const lineage = nextCandidateLineage++;
lineages[ordinal] = lineage;
return { lineage, created: true };
};
let decodedCandidateCount = 0;
let decodedCandidateChars = 0;
let base64Sources = [...projections].map(value => ({ value, lineage: 0 }));
let reachedDepthLimit = true;
for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) {
const decodedValues: Array<{ value: string; lineage: number }> = [];
for (const source of base64Sources) {
const collected = collectBase64Candidates(source.value, depth > 0);
if (!collected.complete) return { projections: [...projections], complete: false };
const occurrenceCounts = new Map<string, number>();
for (const occurrence of collected.candidates) {
const candidate = occurrence.value;
const ordinal = occurrenceCounts.get(candidate) ?? 0;
occurrenceCounts.set(candidate, ordinal + 1);
const occurrenceLineage = resolveCandidateLineage(
source.lineage,
candidate,
ordinal,
);
if (!occurrenceLineage.created) continue;
if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) {
return { projections: [...projections], complete: false };
}
const decoded = decodeBase64Text(candidate);
if (decoded === undefined) continue;
decodedCandidateCount++;
decodedCandidateChars += candidate.length;
if (decodedCandidateCount > MAX_BASE64_CANDIDATES
|| decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) {
return { projections: [...projections], complete: false };
}
decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage });
}
}
if (decodedValues.length === 0) {
reachedDepthLimit = false;
break;
}
const nextSources: Array<{ value: string; lineage: number }> = [];
for (const decoded of decodedValues) {
const nested = normalizedIngressProjections(decoded.value, false);
if (!nested.complete) return { projections: [...projections], complete: false };
for (const projection of nested.projections) {
addProjection(projections, projection);
nextSources.push({ value: projection, lineage: decoded.lineage });
}
}
base64Sources = nextSources;
}
if (reachedDepthLimit && base64Sources.length > 0) {
for (const source of base64Sources) {
const collected = collectBase64Candidates(source.value, true);
if (!collected.complete) return { projections: [...projections], complete: false };
const occurrenceCounts = new Map<string, number>();
for (const occurrence of collected.candidates) {
const candidate = occurrence.value;
const ordinal = occurrenceCounts.get(candidate) ?? 0;
occurrenceCounts.set(candidate, ordinal + 1);
const occurrenceSeen = candidateLineages
.get(source.lineage)
?.get(candidate)?.[ordinal] !== undefined;
if (!occurrenceSeen
&& decodeBase64Text(candidate) !== undefined) {
return { projections: [...projections], complete: false };
}
}
}
}
return { projections: [...projections], complete: true };
}
/** Evaluate untrusted content before it can enter persistent memory. */
export function evaluateExternalMemoryIngress(
input: ExternalMemoryIngressInput,
): ExternalMemoryIngressDecision {
const title = input.title ?? '';
if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS
|| input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) {
return {
action: 'block',
reason: 'prompt_injection',
scan: { safe: false, score: 0.6, flags: ['normalization_limit'] },
};
}
const projection = `${title}\n${input.content}`;
let scan = scanForInjection(projection, 'tool_output');
if (scan.safe) {
const normalizedIngress = normalizedIngressProjections(projection);
for (const normalized of normalizedIngress.projections) {
if (normalized !== projection) {
const normalizedScan = scanForInjection(normalized, 'tool_output');
if (!normalizedScan.safe) {
scan = normalizedScan;
break;
}
}
const delimited = projectDelimitedWords(normalized);
if (delimited !== undefined) {
const delimitedScan = scanForInjection(delimited, 'tool_output');
if (!delimitedScan.safe) {
scan = delimitedScan;
break;
}
}
}
if (scan.safe && !normalizedIngress.complete) {
scan = { safe: false, score: 0.6, flags: ['normalization_limit'] };
}
}
return scan.safe
? { action: 'allow', scan }
: { action: 'block', reason: 'prompt_injection', scan };
}

View File

@@ -7,7 +7,7 @@
*/
import * as crypto from 'node:crypto';
import type { MindDB } from '@waggle/hive-mind-core';
import type { WaggleDB } from './waggle-db.js';
// ── Types ──────────────────────────────────────────────────────────────
@@ -37,9 +37,9 @@ export function computeSkillHash(content: string): string {
// ── Store ──────────────────────────────────────────────────────────────
export class SkillHashStore {
private db: MindDB;
private db: WaggleDB;
constructor(db: MindDB) {
constructor(db: WaggleDB) {
this.db = db;
this.ensureTable();
}

View File

@@ -14,7 +14,7 @@
*/
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
import { createCoreLogger } from '@waggle/hive-mind-core';
import { createCoreLogger } from './logger.js';
const log = createCoreLogger('team-sync');
export interface TeamSyncConfig {

View File

@@ -11,7 +11,7 @@ import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createCoreLogger } from '@waggle/hive-mind-core';
import { createCoreLogger } from './logger.js';
const log = createCoreLogger('vault');
const ALGORITHM = 'aes-256-gcm';

View File

@@ -0,0 +1,161 @@
/**
* One-time copy of Waggle's own tables from the legacy memory databases
* (personal.mind, workspace.mind) into waggle.db.
*
* The legacy files are only read: the data directory may be shared with the
* commercial edition, which keeps using them. Each source is copied once,
* recorded in waggle_meta, and a table is copied only while its target is
* empty, so a re-run never duplicates rows.
*/
import fs from 'node:fs';
import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
import { AwarenessLayer } from './awareness.js';
import { CronStore } from './cron-store.js';
import { ExecutionTraceStore } from './execution-traces.js';
import { IdentityLayer } from './identity.js';
import { SkillHashStore } from './skill-hashes.js';
import type { DataScope, WaggleDB } from './waggle-db.js';
/** Tables that only ever lived in personal.mind, copied with their ids. */
const PERSONAL_TABLES = [
'cron_schedules',
'cron_execution_history',
'cron_run_leases',
'notifications',
'pending_actions',
'skill_hashes',
'feedback_entries',
'execution_traces',
'execution_trace_spend',
'execution_trace_spend_reservations',
] as const;
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
const AWARENESS_COLUMNS = ['category', 'content', 'priority', 'metadata', 'created_at', 'expires_at'];
export interface LegacyMindSource {
/** Path to a legacy .mind file. */
path: string;
scope: DataScope;
}
export interface LegacyImportReport {
scope: DataScope;
status: 'imported' | 'already-imported' | 'missing' | 'failed';
copied: Record<string, number>;
error?: string;
}
function importKey(scope: DataScope): string {
return `import:${scope}`;
}
function tableExists(db: DatabaseType, table: string): boolean {
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== undefined;
}
function columnsOf(db: DatabaseType, table: string): string[] {
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(c => c.name);
}
function copyRows(
source: DatabaseType,
target: DatabaseType,
sourceTable: string,
targetTable: string,
columns: string[],
extra: Record<string, unknown> = {},
): number {
const rows = source.prepare(`SELECT ${columns.join(', ')} FROM ${sourceTable}`).all() as Array<Record<string, unknown>>;
if (rows.length === 0) return 0;
const allColumns = [...columns, ...Object.keys(extra)];
const insert = target.prepare(
`INSERT INTO ${targetTable} (${allColumns.join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`,
);
for (const row of rows) {
insert.run(...columns.map(c => row[c]), ...Object.values(extra));
}
return rows.length;
}
function copyPersonalTables(source: DatabaseType, target: DatabaseType, copied: Record<string, number>): void {
for (const table of PERSONAL_TABLES) {
if (!tableExists(source, table)) continue;
if (!tableExists(target, table)) {
// Lazily created tables (feedback_entries) take the legacy definition.
const ddl = source.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { sql: string };
target.exec(ddl.sql);
}
const empty = target.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() === undefined;
if (!empty) continue;
const targetColumns = new Set(columnsOf(target, table));
const shared = columnsOf(source, table).filter(c => targetColumns.has(c));
copied[table] = copyRows(source, target, table, table, shared);
}
}
function copyScopedTables(source: DatabaseType, target: DatabaseType, scope: DataScope, copied: Record<string, number>): void {
if (tableExists(source, 'identity')) {
const hasProfile = target.prepare('SELECT 1 FROM profiles WHERE scope = ?').get(scope) !== undefined;
if (!hasProfile) {
const sourceColumns = new Set(columnsOf(source, 'identity'));
const shared = PROFILE_COLUMNS.filter(c => sourceColumns.has(c));
copied.profiles = copyRows(source, target, 'identity', 'profiles', shared, { scope });
}
}
if (tableExists(source, 'awareness')) {
const hasItems = target.prepare('SELECT 1 FROM awareness_items WHERE scope = ? LIMIT 1').get(scope) !== undefined;
if (!hasItems) {
const sourceColumns = new Set(columnsOf(source, 'awareness'));
const shared = AWARENESS_COLUMNS.filter(c => sourceColumns.has(c));
copied.awareness_items = copyRows(source, target, 'awareness', 'awareness_items', shared, { scope });
}
}
}
/**
* Copies Waggle data from each legacy source into `waggleDb`, once per scope.
* The personal source also brings the app-wide tables; workspace sources bring
* their profile and awareness items.
*/
export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSource[]): LegacyImportReport[] {
// Create every target table with its current schema first.
new CronStore(waggleDb);
new SkillHashStore(waggleDb);
new ExecutionTraceStore(waggleDb);
new IdentityLayer(waggleDb);
new AwarenessLayer(waggleDb);
const target = waggleDb.getDatabase();
const reports: LegacyImportReport[] = [];
for (const { path: sourcePath, scope } of sources) {
const copied: Record<string, number> = {};
if (waggleDb.getMeta(importKey(scope))) {
reports.push({ scope, status: 'already-imported', copied });
continue;
}
const stat = fs.statSync(sourcePath, { throwIfNoEntry: false });
if (!stat || stat.size === 0) {
reports.push({ scope, status: 'missing', copied });
continue;
}
let source: DatabaseType | null = null;
try {
source = new Database(sourcePath, { fileMustExist: true });
const db = source;
target.transaction(() => {
if (scope === 'personal') copyPersonalTables(db, target, copied);
copyScopedTables(db, target, scope, copied);
})();
waggleDb.setMeta(importKey(scope), new Date().toISOString());
reports.push({ scope, status: 'imported', copied });
} catch (err) {
reports.push({ scope, status: 'failed', copied: {}, error: err instanceof Error ? err.message : String(err) });
} finally {
source?.close();
}
}
return reports;
}

View File

@@ -0,0 +1,78 @@
/**
* waggle.db — Waggle's own data, kept apart from memory (ADR-0007).
*
* Holds what the app needs to run: automations, notifications and held
* actions, skill hashes, thumbs feedback, cost tracking (execution traces and
* budget reservations), the user profile card and awareness items. Facts the
* agent learns about the user live in the memory store, not here.
*
* Stores that belong here take a `WaggleDB`, not a generic SQLite handle, so
* passing a memory database by mistake is a type error.
*/
import fs from 'node:fs';
import path from 'node:path';
import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
export const WAGGLE_DB_FILE = 'waggle.db';
const BUSY_TIMEOUT_MS = 10_000;
const META_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS waggle_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)`;
export class WaggleDB {
private readonly db: DatabaseType;
/** Opens (and creates if needed) the database at `dbPath`; `':memory:'` is accepted. */
constructor(readonly dbPath: string) {
if (dbPath !== ':memory:') {
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
}
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.pragma('foreign_keys = ON');
this.db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
this.db.exec(META_TABLE_SQL);
}
/** Opens `<dataDir>/waggle.db`. */
static openInDataDir(dataDir: string): WaggleDB {
return new WaggleDB(path.join(dataDir, WAGGLE_DB_FILE));
}
getDatabase(): DatabaseType {
return this.db;
}
getMeta(key: string): string | null {
const row = this.db.prepare('SELECT value FROM waggle_meta WHERE key = ?').get(key) as
| { value: string }
| undefined;
return row?.value ?? null;
}
setMeta(key: string, value: string): void {
this.db
.prepare('INSERT INTO waggle_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
.run(key, value);
}
close(): void {
if (this.db.open) this.db.close();
}
}
/** Data scope of a profile or awareness item: the personal mind or one workspace. */
export type DataScope = 'personal' | `workspace:${string}`;
export const PERSONAL_DATA_SCOPE: DataScope = 'personal';
export function workspaceDataScope(workspaceId: string): DataScope {
if (!workspaceId.trim()) throw new TypeError('workspaceId must be a non-empty string');
return `workspace:${workspaceId}`;
}

View File

@@ -0,0 +1,521 @@
import fs from 'node:fs';
import path from 'node:path';
import type { WorkspaceType } from '@waggle/shared';
type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
export interface WorkspaceConfig {
id: string;
name: string;
group: string;
icon?: string;
model?: string;
personality?: string;
/** Selected agent persona ID (from persona catalog) */
personaId?: string;
/** Template ID chosen during onboarding (e.g. 'sales-pipeline', 'research-project'). */
templateId?: string;
tools?: string[];
skills?: string[];
team?: string | null;
/** Filesystem directory where agent operates and generates files. */
directory?: string;
/** Storage type for workspace files. */
storageType?: 'virtual' | 'local' | 'team';
/** Path to local file storage for this workspace. */
storagePath?: string;
/** Extra storage provider configuration (e.g. S3 bucket, credentials). */
storageConfig?: Record<string, unknown>;
created: string; // ISO 8601
// --- Team Mode fields (Phase 5) ---
/** Team ID on the team server. Present = team workspace. */
teamId?: string;
/** URL of the team server (e.g. "https://team.waggle.dev"). */
teamServerUrl?: string;
/** Current user's role in this team workspace. */
teamRole?: 'owner' | 'admin' | 'member' | 'viewer';
/** Current user's ID on the team server. */
teamUserId?: string;
// --- Budget ---
/** Monthly cost budget in USD. null = unlimited. */
budget?: number | null;
// --- Tone/Voice (Wave 7.3) ---
/** Workspace communication tone preset. */
tone?: 'professional' | 'casual' | 'technical' | 'legal' | 'marketing';
// --- Optimization fields (GEPA/Ax) ---
/** Enable GEPA prompt optimization for this workspace (opt-in, default false). */
optimizationEnabled?: boolean;
/** Daily optimization budget in cents (default 100 = $1/day). Only used when optimizationEnabled is true. */
optimizationBudget?: number;
// --- AI Act compliance (L-17 C2) ---
/** EU AI Act risk classification for this workspace. */
riskLevel?: AIActRiskLevel;
/** ISO timestamp of the last risk classification change. Auto-stamped by WorkspaceManager. */
riskClassifiedAt?: string;
// --- UX-Refactor V2 fields (PRD §15.3; additive + optional for back-compat) ---
/** Free-text description shown in the workspace header/cards. */
description?: string;
/** Workspace classification. Defaults derivable from templateId/group when absent. */
type?: WorkspaceType;
/** Lifecycle status. Treated as 'active' when absent. */
status?: 'active' | 'paused' | 'archived';
/** Agents bound to this workspace (ids). */
agentIds?: string[];
/** Connectors scoped to this workspace (ids). */
connectorIds?: string[];
/** MCP servers scoped to this workspace (ids). */
mcpIds?: string[];
/** ISO timestamp of the last config update. Auto-stamped by update(). */
updatedAt?: string;
/** ISO timestamp of the last activity (chat/agent run) in this workspace. */
lastActiveAt?: string;
}
export interface CreateWorkspaceOptions {
name: string;
group: string;
icon?: string;
model?: string;
personality?: string;
/** Selected agent persona ID (from persona catalog) */
personaId?: string;
/** Template ID chosen during onboarding (e.g. 'sales-pipeline', 'research-project'). */
templateId?: string;
tools?: string[];
skills?: string[];
team?: string | null;
/** Filesystem directory where agent operates and generates files. */
directory?: string;
// --- Team Mode fields (Phase 5) ---
teamId?: string;
teamServerUrl?: string;
teamRole?: 'owner' | 'admin' | 'member' | 'viewer';
teamUserId?: string;
// --- Tone/Voice (Wave 7.3) ---
tone?: 'professional' | 'casual' | 'technical' | 'legal' | 'marketing';
// --- Budget ---
budget?: number | null;
// --- AI Act compliance (L-17 C2) ---
/** Initial risk level (usually derived from template). */
riskLevel?: AIActRiskLevel;
// --- Optimization fields (GEPA/Ax) ---
optimizationEnabled?: boolean;
optimizationBudget?: number;
// --- UX-Refactor V2 fields (PRD §15.3) ---
/** Free-text description shown in the workspace header/cards. */
description?: string;
/** Workspace classification (defaults derivable from templateId/group). */
type?: WorkspaceType;
}
interface WorkspacesMeta {
defaultWorkspace?: string | null;
}
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
function isContained(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative === '' || (!path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`));
}
/**
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
* - workspace.json (config)
* - workspace.mind (SQLite .mind file, created empty)
* - sessions/ (JSONL session logs)
*/
export class WorkspaceManager {
private readonly workspacesDir: string;
private readonly canonicalWorkspacesDir: string;
private readonly metaPath: string;
constructor(private readonly baseDir: string) {
this.workspacesDir = path.join(baseDir, 'workspaces');
this.metaPath = path.join(baseDir, 'workspaces-meta.json');
if (!fs.existsSync(this.workspacesDir)) {
fs.mkdirSync(this.workspacesDir, { recursive: true });
}
const rootStat = fs.lstatSync(this.workspacesDir);
const canonicalBase = fs.realpathSync.native(baseDir);
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
if (
rootStat.isSymbolicLink()
|| !rootStat.isDirectory()
|| canonicalRoot === canonicalBase
|| !isContained(canonicalBase, canonicalRoot)
) {
throw new Error('Workspace root must be a regular directory inside the data directory');
}
this.canonicalWorkspacesDir = canonicalRoot;
}
/**
* Create a new workspace with directory structure and config.
*/
create(options: CreateWorkspaceOptions): WorkspaceConfig {
return this.createWithId(this.generateId(options.name), options);
}
/**
* Ensure a workspace with the given id exists. Idempotent — returns the
* existing config unchanged if the workspace already exists; otherwise
* creates it with the supplied id (bypassing slug-collision handling in
* generateId, since callers construct ids from trusted internal state
* like CWD-derived prefixes — e.g. SessionStart hooks).
*
* Use this from auto-attach paths (e.g. save_memory with a workspace arg
* that names a workspace not yet created on disk). Direct-create flows
* with user-supplied names should still go through `create()` so the
* generateId collision logic runs.
*/
// Reverse-ported from OSS hive-mind (oss-drift triage R4, 2026-06-11).
ensure(id: string, options: Partial<CreateWorkspaceOptions> = {}): WorkspaceConfig {
this.assertWorkspaceId(id);
const existing = this.get(id);
if (existing) return existing;
const workspacePath = path.join(this.resolveWorkspaceRoot(), id);
const workspaceStat = fs.lstatSync(workspacePath, { throwIfNoEntry: false });
if (workspaceStat) {
if (id !== 'default' || !this.isEmptyLegacyWorkspaceDirectory(workspacePath, workspaceStat)) {
throw new Error(`Workspace path already exists but is not a valid workspace: ${id}`);
}
const sessionsPath = path.join(workspacePath, 'sessions');
if (fs.lstatSync(sessionsPath, { throwIfNoEntry: false })) fs.rmdirSync(sessionsPath);
fs.rmdirSync(workspacePath);
}
return this.createWithId(id, {
...options,
name: options.name ?? id,
group: options.group ?? 'auto',
});
}
/**
* Shared create path: write directory structure + config for an exact id.
*/
private createWithId(id: string, options: CreateWorkspaceOptions): WorkspaceConfig {
this.assertWorkspaceId(id);
const canonicalRoot = this.resolveWorkspaceRoot();
const wsDir = path.join(canonicalRoot, id);
fs.mkdirSync(wsDir);
const canonicalWorkspace = fs.realpathSync.native(wsDir);
if (!isContained(canonicalRoot, canonicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
// Touch workspace.mind — MindDB will init schema when first opened
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
const config: WorkspaceConfig = {
id,
name: options.name,
group: options.group,
...(options.icon !== undefined && { icon: options.icon }),
...(options.model !== undefined && { model: options.model }),
...(options.personality !== undefined && { personality: options.personality }),
...(options.personaId !== undefined && { personaId: options.personaId }),
...(options.templateId !== undefined && { templateId: options.templateId }),
...(options.tools !== undefined && { tools: options.tools }),
...(options.skills !== undefined && { skills: options.skills }),
...(options.team !== undefined && { team: options.team }),
...(options.directory !== undefined && { directory: options.directory }),
...(options.teamId !== undefined && { teamId: options.teamId }),
...(options.teamServerUrl !== undefined && { teamServerUrl: options.teamServerUrl }),
...(options.teamRole !== undefined && { teamRole: options.teamRole }),
...(options.teamUserId !== undefined && { teamUserId: options.teamUserId }),
...(options.tone !== undefined && { tone: options.tone }),
...(options.optimizationEnabled !== undefined && { optimizationEnabled: options.optimizationEnabled }),
...(options.optimizationBudget !== undefined && { optimizationBudget: options.optimizationBudget }),
...(options.riskLevel !== undefined && {
riskLevel: options.riskLevel,
riskClassifiedAt: new Date().toISOString(),
}),
created: new Date().toISOString(),
};
fs.writeFileSync(
path.join(canonicalWorkspace, 'workspace.json'),
JSON.stringify(config, null, 2),
{ encoding: 'utf-8', flag: 'wx' }
);
return config;
}
/**
* List all workspaces by reading workspace.json from each subdirectory.
*/
list(): WorkspaceConfig[] {
const root = this.resolveWorkspaceRoot();
const entries = fs.readdirSync(root, { withFileTypes: true });
const configs: WorkspaceConfig[] = [];
for (const entry of entries) {
if (!entry.isDirectory() || !WORKSPACE_ID.test(entry.name)) continue;
const config = this.get(entry.name);
if (config) configs.push(config);
}
return configs;
}
/**
* List workspaces filtered by group name.
*/
listByGroup(group: string): WorkspaceConfig[] {
return this.list().filter(ws => ws.group === group);
}
/**
* List all unique group names.
*/
listGroups(): string[] {
const groups = new Set(this.list().map(ws => ws.group));
return [...groups];
}
/**
* Get a workspace by ID. Returns null if not found.
*/
get(id: string): WorkspaceConfig | null {
if (!WORKSPACE_ID.test(id)) return null;
try {
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) return null;
const configPath = path.join(workspaceDir, 'workspace.json');
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) return null;
const canonicalConfig = fs.realpathSync.native(configPath);
if (!isContained(workspaceDir, canonicalConfig)) return null;
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
return config.id === id ? config : null;
} catch {
return null;
}
}
/**
* Partially update a workspace config.
* When `riskLevel` changes, `riskClassifiedAt` is auto-stamped with the
* current ISO timestamp (EU AI Act Art. 14 provenance requirement).
*/
update(id: string, updates: Partial<Omit<WorkspaceConfig, 'id' | 'created'>>): void {
const existing = this.get(id);
if (!existing) throw new Error(`Workspace not found: ${id}`);
const now = new Date().toISOString();
const stamped: Partial<WorkspaceConfig> =
'riskLevel' in updates && updates.riskLevel !== existing.riskLevel
? { ...updates, riskClassifiedAt: now, updatedAt: now }
: { ...updates, updatedAt: now };
const updated = { ...existing, ...stamped };
const configPath = path.join(this.workspacesDir, id, 'workspace.json');
fs.writeFileSync(configPath, JSON.stringify(updated, null, 2), 'utf-8');
}
/**
* Delete a workspace by removing its entire directory.
*/
delete(id: string): void {
this.assertWorkspaceId(id);
if (!this.get(id)) return;
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) return;
fs.rmSync(workspaceDir, { recursive: true, force: true });
}
/**
* Check whether a workspace is team-connected (has a teamId).
*/
isTeamWorkspace(id: string): boolean {
const ws = this.get(id);
return ws !== null && typeof ws.teamId === 'string' && ws.teamId.length > 0;
}
/**
* List only team-connected workspaces.
*/
listTeamWorkspaces(): WorkspaceConfig[] {
return this.list().filter(ws => typeof ws.teamId === 'string' && ws.teamId.length > 0);
}
/**
* Get the path to a workspace's .mind file.
*/
getMindPath(id: string): string {
this.assertWorkspaceId(id);
const lexicalMindPath = path.join(this.workspacesDir, id, 'workspace.mind');
const workspaceDir = this.resolveWorkspaceDir(id);
if (!workspaceDir) throw new Error(`Workspace not found: ${id}`);
const configPath = path.join(workspaceDir, 'workspace.json');
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) {
throw new Error(`Workspace config not found: ${id}`);
}
const canonicalConfig = fs.realpathSync.native(configPath);
if (!isContained(workspaceDir, canonicalConfig)) {
throw new Error(`Workspace config escapes workspace directory: ${id}`);
}
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
if (config.id !== id) throw new Error(`Workspace config id mismatch: ${id}`);
const mindPath = path.join(workspaceDir, 'workspace.mind');
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
if (!mindStat) return lexicalMindPath;
if (!mindStat?.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) {
throw new Error(`Workspace mind is not a regular file: ${id}`);
}
const canonicalMind = fs.realpathSync.native(mindPath);
if (!isContained(workspaceDir, canonicalMind)) {
throw new Error(`Workspace mind escapes workspace directory: ${id}`);
}
return lexicalMindPath;
}
private assertWorkspaceId(id: string): void {
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
}
private isEmptyLegacyWorkspaceDirectory(workspacePath: string, stat: fs.Stats): boolean {
if (stat.isSymbolicLink() || !stat.isDirectory()) return false;
const canonicalRoot = this.resolveWorkspaceRoot();
const canonicalWorkspace = fs.realpathSync.native(workspacePath);
if (!isContained(canonicalRoot, canonicalWorkspace)) return false;
const entries = fs.readdirSync(workspacePath, { withFileTypes: true });
if (entries.length === 0) return true;
if (entries.length !== 1 || entries[0]?.name !== 'sessions' || !entries[0].isDirectory()) return false;
const sessionsPath = path.join(workspacePath, 'sessions');
const sessionsStat = fs.lstatSync(sessionsPath);
if (sessionsStat.isSymbolicLink()) return false;
const canonicalSessions = fs.realpathSync.native(sessionsPath);
return isContained(canonicalWorkspace, canonicalSessions) && fs.readdirSync(sessionsPath).length === 0;
}
private resolveWorkspaceDir(id: string): string | null {
this.assertWorkspaceId(id);
const lexicalRoot = this.resolveWorkspaceRoot();
const lexicalWorkspace = path.resolve(lexicalRoot, id);
if (!isContained(lexicalRoot, lexicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
const workspaceStat = fs.lstatSync(lexicalWorkspace, { throwIfNoEntry: false });
if (!workspaceStat) return null;
if (workspaceStat.isSymbolicLink() || !workspaceStat.isDirectory()) {
throw new Error(`Workspace path is not a regular directory: ${id}`);
}
const canonicalWorkspace = fs.realpathSync.native(lexicalWorkspace);
if (!isContained(this.canonicalWorkspacesDir, canonicalWorkspace)) {
throw new Error(`Workspace path escapes workspace root: ${id}`);
}
return canonicalWorkspace;
}
private resolveWorkspaceRoot(): string {
const rootStat = fs.lstatSync(this.workspacesDir, { throwIfNoEntry: false });
if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) {
throw new Error('Workspace root is not a regular directory');
}
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
if (canonicalRoot !== this.canonicalWorkspacesDir) {
throw new Error('Workspace root changed after initialization');
}
return canonicalRoot;
}
/**
* Set the default workspace ID in workspaces-meta.json.
*/
setDefault(id: string): void {
if (!this.get(id)) throw new Error(`Workspace not found: ${id}`);
const meta = this.loadMeta();
meta.defaultWorkspace = id;
this.saveMeta(meta);
}
/**
* Get the default workspace ID. Returns null if none set.
*/
getDefault(): string | null {
const meta = this.loadMeta();
return meta.defaultWorkspace ?? null;
}
/**
* Ensure at least one workspace exists. If none, create a default one
* and mark it as the default. Idempotent — safe to call on every startup.
*/
ensureDefault(options?: Partial<CreateWorkspaceOptions>): WorkspaceConfig {
const existing = this.list();
if (existing.length > 0) {
const defaultId = this.getDefault();
const found = defaultId ? this.get(defaultId) : null;
return found ?? existing[0];
}
const ws = this.create({
name: 'Default Workspace',
group: 'Personal',
personaId: 'researcher',
...options,
});
this.setDefault(ws.id);
return ws;
}
/**
* Generate a slug-based ID from a workspace name.
* Handles duplicates by appending -2, -3, etc.
*/
generateId(name: string): string {
const base = name
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
if (!this.workspaceExists(base)) return base;
let counter = 2;
while (this.workspaceExists(`${base}-${counter}`)) {
counter++;
}
return `${base}-${counter}`;
}
private workspaceExists(id: string): boolean {
return fs.existsSync(path.join(this.resolveWorkspaceRoot(), id));
}
private loadMeta(): WorkspacesMeta {
if (fs.existsSync(this.metaPath)) {
const raw = fs.readFileSync(this.metaPath, 'utf-8');
return JSON.parse(raw) as WorkspacesMeta;
}
return {};
}
private saveMeta(meta: WorkspacesMeta): void {
fs.writeFileSync(this.metaPath, JSON.stringify(meta, null, 2), 'utf-8');
}
}