feat(core): give Waggle its own database, apart from memory (P1.5)
Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.
- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
actions, skill hashes, thumbs feedback, cost tracking, the profile card
and awareness items live there. Their stores take a WaggleDB, so passing
a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
workspace.mind (importLegacyMindData). The legacy files are only read,
each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
WorkspaceManager and the embedders from @waggle/hive-mind-core into
@waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
tests that seeded these tables in a memory database
ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
172
packages/core/src/awareness.ts
Normal file
172
packages/core/src/awareness.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* AwarenessLayer — the agent's short-lived working items (tasks, recent
|
||||
* actions, pending items, context flags), per data scope, stored in waggle.db.
|
||||
*/
|
||||
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
import { PERSONAL_DATA_SCOPE } from './waggle-db.js';
|
||||
|
||||
export type AwarenessCategory = 'task' | 'action' | 'pending' | 'flag';
|
||||
|
||||
export interface AwarenessMetadata {
|
||||
context?: string;
|
||||
status?: string;
|
||||
result?: string;
|
||||
priority?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface AwarenessItem {
|
||||
id: number;
|
||||
scope: string;
|
||||
category: AwarenessCategory;
|
||||
content: string;
|
||||
priority: number;
|
||||
metadata: string;
|
||||
created_at: string;
|
||||
expires_at: string | null;
|
||||
}
|
||||
|
||||
type AwarenessUpdate = Partial<Pick<AwarenessItem, 'content' | 'priority' | 'expires_at'>>;
|
||||
|
||||
const MAX_ITEMS = 10;
|
||||
|
||||
export const AWARENESS_ITEMS_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS awareness_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scope TEXT NOT NULL,
|
||||
category TEXT NOT NULL CHECK (category IN ('task', 'action', 'pending', 'flag')),
|
||||
content TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
expires_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_awareness_items_scope ON awareness_items (scope, priority DESC)`;
|
||||
|
||||
const ACTIVE = "(expires_at IS NULL OR datetime(expires_at) > datetime('now'))";
|
||||
|
||||
export class AwarenessLayer {
|
||||
constructor(
|
||||
private readonly db: WaggleDB,
|
||||
private readonly scope: DataScope = PERSONAL_DATA_SCOPE,
|
||||
) {
|
||||
db.getDatabase().exec(AWARENESS_ITEMS_TABLE_SQL);
|
||||
}
|
||||
|
||||
add(category: AwarenessCategory, content: string, priority = 0, expires_at?: string, metadata?: AwarenessMetadata): AwarenessItem {
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO awareness_items (scope, category, content, priority, expires_at, metadata)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`).run(this.scope, category, content, priority, expires_at ?? null, metadata ? JSON.stringify(metadata) : '{}');
|
||||
return this.get(Number(result.lastInsertRowid)) as AwarenessItem;
|
||||
}
|
||||
|
||||
get(id: number): AwarenessItem | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM awareness_items WHERE id = ? AND scope = ?')
|
||||
.get(id, this.scope) as AwarenessItem | undefined;
|
||||
}
|
||||
|
||||
remove(id: number): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE id = ? AND scope = ?').run(id, this.scope);
|
||||
}
|
||||
|
||||
update(id: number, changes: AwarenessUpdate): AwarenessItem {
|
||||
const fields = Object.entries(changes).filter(([, v]) => v !== undefined);
|
||||
if (fields.length > 0) {
|
||||
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
|
||||
const values = fields.map(([, v]) => v);
|
||||
this.db.getDatabase()
|
||||
.prepare(`UPDATE awareness_items SET ${sets} WHERE id = ? AND scope = ?`)
|
||||
.run(...values, id, this.scope);
|
||||
}
|
||||
return this.get(id) as AwarenessItem;
|
||||
}
|
||||
|
||||
updateMetadata(id: number, metadata: AwarenessMetadata): AwarenessItem {
|
||||
const existing = this.get(id);
|
||||
if (!existing) {
|
||||
throw new Error(`Awareness item ${id} not found`);
|
||||
}
|
||||
const merged = { ...this.parseMetadata(existing), ...metadata };
|
||||
this.db.getDatabase()
|
||||
.prepare('UPDATE awareness_items SET metadata = ? WHERE id = ? AND scope = ?')
|
||||
.run(JSON.stringify(merged), id, this.scope);
|
||||
return this.get(id) as AwarenessItem;
|
||||
}
|
||||
|
||||
getByStatus(status: string): AwarenessItem[] {
|
||||
const items = this.db.getDatabase().prepare(`
|
||||
SELECT * FROM awareness_items WHERE scope = ? AND ${ACTIVE} ORDER BY priority DESC
|
||||
`).all(this.scope) as AwarenessItem[];
|
||||
return items.filter(item => this.parseMetadata(item).status === status);
|
||||
}
|
||||
|
||||
parseMetadata(item: AwarenessItem): AwarenessMetadata {
|
||||
try {
|
||||
return JSON.parse(item.metadata) as AwarenessMetadata;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
getAll(): AwarenessItem[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM awareness_items WHERE scope = ? AND ${ACTIVE} ORDER BY priority DESC LIMIT ?
|
||||
`).all(this.scope, MAX_ITEMS) as AwarenessItem[];
|
||||
}
|
||||
|
||||
getByCategory(category: AwarenessCategory): AwarenessItem[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM awareness_items WHERE scope = ? AND category = ? AND ${ACTIVE} ORDER BY priority DESC LIMIT ?
|
||||
`).all(this.scope, category, MAX_ITEMS) as AwarenessItem[];
|
||||
}
|
||||
|
||||
/** Active task and pending items, without the 10-item cap of getAll(). */
|
||||
countOpen(): number {
|
||||
const row = this.db.getDatabase().prepare(`
|
||||
SELECT COUNT(*) AS cnt FROM awareness_items
|
||||
WHERE scope = ? AND category IN ('task', 'pending') AND ${ACTIVE}
|
||||
`).get(this.scope) as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE scope = ?').run(this.scope);
|
||||
}
|
||||
|
||||
clearCategory(category: AwarenessCategory): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness_items WHERE scope = ? AND category = ?').run(this.scope, category);
|
||||
}
|
||||
|
||||
toContext(): string {
|
||||
const items = this.getAll();
|
||||
if (items.length === 0) return 'No active awareness items.';
|
||||
|
||||
const grouped = new Map<string, AwarenessItem[]>();
|
||||
for (const item of items) {
|
||||
const list = grouped.get(item.category) ?? [];
|
||||
list.push(item);
|
||||
grouped.set(item.category, list);
|
||||
}
|
||||
|
||||
const sections: string[] = [];
|
||||
const labels: Record<AwarenessCategory, string> = {
|
||||
task: 'Active Tasks',
|
||||
action: 'Recent Actions',
|
||||
pending: 'Pending Items',
|
||||
flag: 'Context Flags',
|
||||
};
|
||||
|
||||
for (const [cat, label] of Object.entries(labels)) {
|
||||
const catItems = grouped.get(cat);
|
||||
if (catItems && catItems.length > 0) {
|
||||
sections.push(`${label}:\n${catItems.map(i => `- ${i.content}`).join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
return sections.join('\n\n');
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
|
||||
import type { EmbeddingProviderConfig, EmbeddingProviderType } from './embeddings/embedding-provider.js';
|
||||
|
||||
export interface ProviderEntry {
|
||||
apiKey: string;
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
import cronParser from 'cron-parser';
|
||||
const { parseExpression } = cronParser;
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type { WaggleDB } from './waggle-db.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -221,9 +221,9 @@ export function cronExprError(cronExpr: string): string | null {
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class CronStore {
|
||||
private db: MindDB;
|
||||
private db: WaggleDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
constructor(db: WaggleDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
73
packages/core/src/embeddings/api-embedder.ts
Normal file
73
packages/core/src/embeddings/api-embedder.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* API-backed embedder — calls Voyage AI or OpenAI embedding endpoints.
|
||||
* For users who configure API keys in the Vault UI.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface ApiEmbedderConfig {
|
||||
provider: 'voyage' | 'openai';
|
||||
apiKey: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
const PROVIDER_DEFAULTS: Record<'voyage' | 'openai', { url: string; model: string }> = {
|
||||
voyage: { url: 'https://api.voyageai.com/v1/embeddings', model: 'voyage-3-lite' },
|
||||
openai: { url: 'https://api.openai.com/v1/embeddings', model: 'text-embedding-3-small' },
|
||||
};
|
||||
|
||||
export function createApiEmbedder(config: ApiEmbedderConfig): Embedder {
|
||||
const defaults = PROVIDER_DEFAULTS[config.provider];
|
||||
const url = config.baseUrl ?? defaults.url;
|
||||
const model = config.model ?? defaults.model;
|
||||
const targetDims = config.targetDimensions ?? 1024;
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 15000);
|
||||
|
||||
const body: Record<string, unknown> = { model, input };
|
||||
if (config.provider === 'voyage') {
|
||||
body.input_type = 'document';
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${config.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`${config.provider} embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { data: Array<{ embedding: number[] }> };
|
||||
return json.data.map(d => normalizeDimensions(new Float32Array(d.embedding), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
446
packages/core/src/embeddings/embedding-provider.ts
Normal file
446
packages/core/src/embeddings/embedding-provider.ts
Normal file
@@ -0,0 +1,446 @@
|
||||
/**
|
||||
* EmbeddingProvider — orchestrates the InProcess → Ollama → API → Mock fallback chain.
|
||||
* Single entry point for all embedding operations in Waggle.
|
||||
* Implements the Embedder interface — drop-in replacement everywhere.
|
||||
*
|
||||
* Usage: the monthly embed count is tracked in the embedding_usage table (no limit).
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('embedding');
|
||||
|
||||
export type EmbeddingProviderType = 'inprocess' | 'ollama' | 'voyage' | 'openai' | 'litellm' | 'mock';
|
||||
|
||||
export interface EmbeddingProviderConfig {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
targetDimensions?: number;
|
||||
/** User ID for quota tracking. Defaults to 'local'. */
|
||||
userId?: string;
|
||||
/** Raw SQLite database for usage tracking. Optional — usage is not recorded without it. */
|
||||
quotaDb?: DatabaseType;
|
||||
inprocess?: { model?: string; cacheDir?: string };
|
||||
ollama?: { baseUrl?: string; model?: string };
|
||||
voyage?: { apiKey: string; model?: string };
|
||||
openai?: { apiKey: string; model?: string };
|
||||
litellm?: { url: string; apiKey?: string; model?: string };
|
||||
}
|
||||
|
||||
// ── Quota tracking ────────────────────────────────────────────────────
|
||||
|
||||
const EMBEDDING_USAGE_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS embedding_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
year_month TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at INTEGER NOT NULL,
|
||||
UNIQUE(user_id, year_month)
|
||||
);
|
||||
`;
|
||||
|
||||
function ensureQuotaTable(db: DatabaseType): void {
|
||||
try { db.exec(EMBEDDING_USAGE_SCHEMA); } catch { /* table may already exist */ }
|
||||
}
|
||||
|
||||
function getCurrentYearMonth(): string {
|
||||
const d = new Date();
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function incrementUsage(db: DatabaseType, userId: string, yearMonth: string, amount: number): void {
|
||||
db.prepare(`
|
||||
INSERT INTO embedding_usage (user_id, year_month, count, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, year_month) DO UPDATE SET count = count + ?, updated_at = ?
|
||||
`).run(userId, yearMonth, amount, Date.now(), amount, Date.now());
|
||||
}
|
||||
|
||||
export interface EmbeddingQuotaStatus {
|
||||
quota: number;
|
||||
used: number;
|
||||
remaining: number;
|
||||
percentage: number;
|
||||
resetsAt: string;
|
||||
}
|
||||
|
||||
function getNextMonthReset(): string {
|
||||
const d = new Date();
|
||||
d.setMonth(d.getMonth() + 1, 1);
|
||||
d.setHours(0, 0, 0, 0);
|
||||
return d.toISOString();
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderStatus {
|
||||
activeProvider: EmbeddingProviderType;
|
||||
availableProviders: EmbeddingProviderType[];
|
||||
dimensions: number;
|
||||
modelName: string;
|
||||
lastError?: string;
|
||||
probeTimestamp: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderInstance extends Embedder {
|
||||
getStatus(): EmbeddingProviderStatus;
|
||||
getActiveProvider(): EmbeddingProviderType;
|
||||
reprobe(): Promise<EmbeddingProviderStatus>;
|
||||
/** Get the usage status. Embedding is unlimited, so quota and remaining are -1. */
|
||||
getQuotaStatus(): EmbeddingQuotaStatus;
|
||||
}
|
||||
|
||||
/** Deterministic mock — last resort, semantically meaningless. */
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function createMockEmbedder(dims: number): Embedder {
|
||||
return {
|
||||
dimensions: dims,
|
||||
async embed(text: string) { return mockEmbed(text, dims); },
|
||||
async embedBatch(texts: string[]) { return texts.map(t => mockEmbed(t, dims)); },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Embed-input guards (oversized-frame truncation + skip-not-abort) ──
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
|
||||
/**
|
||||
* Per-input character cap for embedding. `nomic-embed-text` has a 2048-token
|
||||
* (~6K char dense English) default context. Embedding an input longer than
|
||||
* the backend's context makes the backend reject the request, so we cap here.
|
||||
*
|
||||
* D1 probe finding (2026-06-12): model NAMES lie about context. A custom
|
||||
* `nomic-embed-text-8k` (num_ctx 8192) still 400s at its nomic-bert
|
||||
* ARCHITECTURE limit of 2048 tokens — the OSS heuristic's 24K-char branch
|
||||
* for `*-8k` names sent every long frame down the mock-fallback path. The
|
||||
* `-8k` branch is therefore capped at 8K chars (≈2048 prose tokens): safe
|
||||
* for the architecture-limited reality, merely conservative for a genuine
|
||||
* 8192-token embedder. reembedPerText remains the backstop for token-dense
|
||||
* content that still exceeds the backend's real window.
|
||||
*/
|
||||
export function maxEmbedCharsForModel(modelName: string): number {
|
||||
return /(-|_|\.)8k\b|num_ctx[^0-9]*8192/i.test(modelName) ? 8_000 : 6_000;
|
||||
}
|
||||
|
||||
/** Clamp a single input to `maxChars` (no-op when already under the cap). */
|
||||
export function capEmbedText(text: string, maxChars: number): string {
|
||||
return text.length > maxChars ? text.slice(0, maxChars) : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-embed a batch one text at a time, degrading ONLY the inputs that genuinely
|
||||
* fail to a deterministic mock vector. This is the batch-error recovery path:
|
||||
* a single backend-rejected text can no longer poison its batchmates (the prior
|
||||
* behavior substituted mock for the WHOLE batch — silent corruption of every
|
||||
* frame in the batch). Inputs should already be char-capped by the caller.
|
||||
*/
|
||||
export async function reembedPerText(
|
||||
embedder: Embedder,
|
||||
texts: string[],
|
||||
dims: number,
|
||||
): Promise<Float32Array[]> {
|
||||
return Promise.all(
|
||||
texts.map(async (t) => {
|
||||
try {
|
||||
return await embedder.embed(t);
|
||||
} catch {
|
||||
return mockEmbed(t, dims);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
interface ProbeResult {
|
||||
type: EmbeddingProviderType;
|
||||
embedder: Embedder;
|
||||
modelName: string;
|
||||
}
|
||||
|
||||
async function probeProvider(
|
||||
type: EmbeddingProviderType,
|
||||
config: EmbeddingProviderConfig,
|
||||
): Promise<ProbeResult | null> {
|
||||
const dims = config.targetDimensions ?? 1024;
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'inprocess': {
|
||||
const { createInProcessEmbedder } = await import('./inprocess-embedder.js');
|
||||
const embedder = await createInProcessEmbedder({
|
||||
model: config.inprocess?.model,
|
||||
cacheDir: config.inprocess?.cacheDir,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'inprocess', embedder, modelName: config.inprocess?.model ?? 'Xenova/all-MiniLM-L6-v2' };
|
||||
}
|
||||
|
||||
case 'ollama': {
|
||||
const { createOllamaEmbedder } = await import('./ollama-embedder.js');
|
||||
const embedder = createOllamaEmbedder({
|
||||
baseUrl: config.ollama?.baseUrl,
|
||||
model: config.ollama?.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'ollama', embedder, modelName: config.ollama?.model ?? 'nomic-embed-text' };
|
||||
}
|
||||
|
||||
case 'voyage': {
|
||||
if (!config.voyage?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'voyage',
|
||||
apiKey: config.voyage.apiKey,
|
||||
model: config.voyage.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'voyage', embedder, modelName: config.voyage.model ?? 'voyage-3-lite' };
|
||||
}
|
||||
|
||||
case 'openai': {
|
||||
if (!config.openai?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'openai',
|
||||
apiKey: config.openai.apiKey,
|
||||
model: config.openai.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'openai', embedder, modelName: config.openai.model ?? 'text-embedding-3-small' };
|
||||
}
|
||||
|
||||
case 'litellm': {
|
||||
if (!config.litellm?.url) return null;
|
||||
const { createLiteLLMEmbedder } = await import('./litellm-embedder.js');
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
litellmUrl: config.litellm.url,
|
||||
litellmApiKey: config.litellm.apiKey,
|
||||
model: config.litellm.model ?? 'text-embedding',
|
||||
dimensions: dims,
|
||||
fallbackToMock: false,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'litellm', embedder, modelName: config.litellm.model ?? 'text-embedding' };
|
||||
}
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.info(`Trying ${type}... FAILED (${msg})`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createEmbeddingProvider(config?: EmbeddingProviderConfig): Promise<EmbeddingProviderInstance> {
|
||||
const cfg: EmbeddingProviderConfig = { provider: 'auto', targetDimensions: 1024, ...config };
|
||||
const dims = cfg.targetDimensions ?? 1024;
|
||||
const userId = cfg.userId ?? 'local';
|
||||
const quotaDb = cfg.quotaDb ?? null;
|
||||
|
||||
// Initialize the usage table if DB provided
|
||||
if (quotaDb) {
|
||||
ensureQuotaTable(quotaDb);
|
||||
}
|
||||
|
||||
let activeResult: ProbeResult | null = null;
|
||||
let activeEmbedder: Embedder;
|
||||
let activeType: EmbeddingProviderType = 'mock';
|
||||
let activeModelName = 'deterministic-mock';
|
||||
let lastError: string | undefined;
|
||||
let availableProviders: EmbeddingProviderType[] = [];
|
||||
let probeTimestamp = new Date().toISOString();
|
||||
|
||||
/** Record usage after successful embedding. */
|
||||
function recordUsage(count: number): void {
|
||||
if (!quotaDb) return;
|
||||
incrementUsage(quotaDb, userId, getCurrentYearMonth(), count);
|
||||
}
|
||||
|
||||
async function runProbe(): Promise<void> {
|
||||
log.info('Probing embedding providers...');
|
||||
const available: EmbeddingProviderType[] = [];
|
||||
activeResult = null;
|
||||
probeTimestamp = new Date().toISOString();
|
||||
|
||||
const requestedProvider = cfg.provider ?? 'auto';
|
||||
|
||||
if (requestedProvider !== 'auto' && requestedProvider !== 'mock') {
|
||||
log.info(`Trying ${requestedProvider}...`);
|
||||
const result = await probeProvider(requestedProvider, cfg);
|
||||
if (result) {
|
||||
activeResult = result;
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${requestedProvider}... OK`);
|
||||
}
|
||||
} else if (requestedProvider === 'auto') {
|
||||
// Auto: iterate the chain in order
|
||||
const chain: EmbeddingProviderType[] = ['inprocess', 'ollama', 'voyage', 'openai'];
|
||||
|
||||
for (const providerType of chain) {
|
||||
// Skip API providers without keys
|
||||
if (providerType === 'voyage' && !cfg.voyage?.apiKey) {
|
||||
log.info('Skipping voyage (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
if (providerType === 'openai' && !cfg.openai?.apiKey) {
|
||||
log.info('Skipping openai (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(`Trying ${providerType}...`);
|
||||
const result = await probeProvider(providerType, cfg);
|
||||
if (result) {
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${providerType}... OK`);
|
||||
if (!activeResult) {
|
||||
activeResult = result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
available.push('mock'); // Always available
|
||||
availableProviders = available;
|
||||
|
||||
if (activeResult) {
|
||||
activeEmbedder = activeResult.embedder;
|
||||
activeType = activeResult.type;
|
||||
activeModelName = activeResult.modelName;
|
||||
lastError = undefined;
|
||||
log.info(`Embedding provider: ${activeType} (${activeModelName}, ${dims} dims)`);
|
||||
} else {
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
lastError = 'No real providers available';
|
||||
// Loud, structured warning — the silent "mock fallback" was the
|
||||
// most dangerous failure mode in Phase 3b-3 audit. Mock embeddings
|
||||
// are deterministic byte hashes; semantic search returns noise.
|
||||
// We want this to be IMPOSSIBLE to miss in a CLI/server log.
|
||||
// Ported from hive-mind a99ea0e.
|
||||
const msg = [
|
||||
'',
|
||||
'⚠️ EMBEDDING WARNING ─────────────────────────────────────────',
|
||||
' Active provider: mock (deterministic byte hash)',
|
||||
' Effect: semantic search returns noise, not meaning.',
|
||||
'',
|
||||
' To fix, install Ollama and pull the embedding model:',
|
||||
' ollama pull nomic-embed-text',
|
||||
' Then ensure the process can reach http://localhost:11434.',
|
||||
'',
|
||||
' Alternative providers:',
|
||||
' HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB)',
|
||||
' VOYAGE_API_KEY=... (paid, recommended)',
|
||||
' OPENAI_API_KEY=... (paid)',
|
||||
'─────────────────────────────────────────────────────────────',
|
||||
'',
|
||||
].join('\n');
|
||||
// Keep production and CLI runs loud, but let deterministic test lanes
|
||||
// suppress this expected fallback banner without changing provider state.
|
||||
if (process.env.WAGGLE_SUPPRESS_EMBEDDING_WARNING !== '1') {
|
||||
// stderr so it survives stdout-piped JSON consumers and CI tee.
|
||||
try { process.stderr.write(msg); } catch { /* fall through to log */ }
|
||||
log.warn('Embedding provider degraded to mock — semantic search quality is noise. See stderr banner for fix instructions.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initial probe
|
||||
try {
|
||||
await runProbe();
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
availableProviders = ['mock'];
|
||||
}
|
||||
|
||||
// Ensure activeEmbedder is assigned (TypeScript flow)
|
||||
activeEmbedder ??= createMockEmbedder(dims);
|
||||
|
||||
const instance: EmbeddingProviderInstance = {
|
||||
dimensions: dims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
// Cap input to the active model's context so the backend never rejects
|
||||
// an oversized frame. Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
const capped = capEmbedText(text, maxEmbedCharsForModel(activeModelName));
|
||||
try {
|
||||
const result = await activeEmbedder.embed(capped);
|
||||
recordUsage(1);
|
||||
return result;
|
||||
} catch (err) {
|
||||
log.warn(`Embedding failed with ${activeType}, falling back to mock: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const fallback = mockEmbed(capped, dims);
|
||||
recordUsage(1);
|
||||
return fallback;
|
||||
}
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
// Cap each input first so one oversized frame can't make the backend
|
||||
// reject the whole request. Reverse-ported from OSS hive-mind
|
||||
// (oss-drift triage R5, 2026-06-11).
|
||||
const capped = texts.map(t => capEmbedText(t, maxEmbedCharsForModel(activeModelName)));
|
||||
try {
|
||||
const result = await activeEmbedder.embedBatch(capped);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
} catch (err) {
|
||||
// Skip-not-abort: re-embed per-text so a single backend-rejected input
|
||||
// degrades alone instead of mock-poisoning the WHOLE batch.
|
||||
log.warn(`Batch embedding failed with ${activeType}, re-embedding per-text: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const result = await reembedPerText(activeEmbedder, capped, dims);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
}
|
||||
},
|
||||
|
||||
getStatus(): EmbeddingProviderStatus {
|
||||
return {
|
||||
activeProvider: activeType,
|
||||
availableProviders,
|
||||
dimensions: dims,
|
||||
modelName: activeModelName,
|
||||
lastError,
|
||||
probeTimestamp,
|
||||
};
|
||||
},
|
||||
|
||||
getActiveProvider(): EmbeddingProviderType {
|
||||
return activeType;
|
||||
},
|
||||
|
||||
async reprobe(): Promise<EmbeddingProviderStatus> {
|
||||
await runProbe();
|
||||
return instance.getStatus();
|
||||
},
|
||||
|
||||
getQuotaStatus(): EmbeddingQuotaStatus {
|
||||
return { quota: -1, used: 0, remaining: -1, percentage: 0, resetsAt: getNextMonthReset() };
|
||||
},
|
||||
};
|
||||
|
||||
return instance;
|
||||
}
|
||||
5
packages/core/src/embeddings/embeddings.ts
Normal file
5
packages/core/src/embeddings/embeddings.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export interface Embedder {
|
||||
embed(text: string): Promise<Float32Array>;
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
||||
dimensions: number;
|
||||
}
|
||||
73
packages/core/src/embeddings/inprocess-embedder.ts
Normal file
73
packages/core/src/embeddings/inprocess-embedder.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
||||
* Default provider for ALL desktop users — zero config, works offline.
|
||||
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
||||
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
export interface InProcessEmbedderConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
|
||||
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
|
||||
if (embedding.length === targetDims) return embedding;
|
||||
const result = new Float32Array(targetDims);
|
||||
const copyLen = Math.min(embedding.length, targetDims);
|
||||
result.set(embedding.subarray(0, copyLen));
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
|
||||
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
|
||||
|
||||
const { pipeline } = await import('@huggingface/transformers');
|
||||
const extractor = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
|
||||
});
|
||||
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
||||
|
||||
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
return normalizeDimensions(raw, targetDims);
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
const results: Float32Array[] = [];
|
||||
// Process one at a time to avoid memory issues with large batches
|
||||
for (const text of texts) {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
results.push(normalizeDimensions(raw, targetDims));
|
||||
}
|
||||
return results;
|
||||
},
|
||||
};
|
||||
}
|
||||
93
packages/core/src/embeddings/litellm-embedder.ts
Normal file
93
packages/core/src/embeddings/litellm-embedder.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* LiteLLM-backed embedder that calls the /embeddings endpoint.
|
||||
* Falls back to a deterministic mock (text→Float32Array hash) on API error
|
||||
* when `fallbackToMock` is enabled.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
|
||||
export interface LiteLLMEmbedderConfig {
|
||||
litellmUrl: string;
|
||||
litellmApiKey?: string;
|
||||
model?: string;
|
||||
dimensions?: number;
|
||||
/** Custom fetch implementation (for testing). */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** If true, falls back to a deterministic mock on API error instead of throwing. */
|
||||
fallbackToMock?: boolean;
|
||||
}
|
||||
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
export function createLiteLLMEmbedder(config: LiteLLMEmbedderConfig): Embedder {
|
||||
const {
|
||||
litellmUrl,
|
||||
litellmApiKey,
|
||||
model = 'text-embedding',
|
||||
dimensions = 1024,
|
||||
fetch: fetchFn = globalThis.fetch,
|
||||
fallbackToMock = false,
|
||||
} = config;
|
||||
|
||||
// Normalise base URL — strip trailing /v1 if present, we add it ourselves
|
||||
const baseUrl = litellmUrl.replace(/\/v1\/?$/, '');
|
||||
const url = `${baseUrl}/v1/embeddings`;
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
if (litellmApiKey) {
|
||||
headers['Authorization'] = `Bearer ${litellmApiKey}`;
|
||||
}
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const body = JSON.stringify({ model, input });
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchFn(url, { method: 'POST', headers, body });
|
||||
} catch (err) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
const text = await response.text();
|
||||
throw new Error(`LiteLLM embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = (await response.json()) as {
|
||||
data: Array<{ embedding: number[] }>;
|
||||
};
|
||||
|
||||
return json.data.map((d) => new Float32Array(d.embedding));
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
58
packages/core/src/embeddings/ollama-embedder.ts
Normal file
58
packages/core/src/embeddings/ollama-embedder.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Ollama-backed embedder — calls local Ollama server for embeddings.
|
||||
* Power user option for users who have Ollama installed.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface OllamaEmbedderConfig {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
export function createOllamaEmbedder(config?: Partial<OllamaEmbedderConfig>): Embedder {
|
||||
const baseUrl = config?.baseUrl ?? 'http://localhost:11434';
|
||||
const model = config?.model ?? 'nomic-embed-text';
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
const url = `${baseUrl}/api/embed`;
|
||||
|
||||
async function callOllama(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`Ollama embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { embeddings: number[][] };
|
||||
return json.embeddings.map(e => normalizeDimensions(new Float32Array(e), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callOllama(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callOllama(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
258
packages/core/src/embeddings/transformers-model-load.ts
Normal file
258
packages/core/src/embeddings/transformers-model-load.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
|
||||
const LOCK_RETRY_MIN_MS = 35;
|
||||
const LOCK_RETRY_JITTER_MS = 30;
|
||||
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
|
||||
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
|
||||
const LOCK_DIRECTORY = '.waggle-model-locks';
|
||||
|
||||
export interface TransformersModelLoadOptions<T> {
|
||||
cacheDir: string;
|
||||
model: string;
|
||||
load: (canonicalCacheDir: string) => Promise<T>;
|
||||
lockTimeoutMs?: number;
|
||||
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function normalizeLockKey(value: string): string {
|
||||
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
|
||||
}
|
||||
|
||||
export function modelLoadLockPath(cacheDir: string, model: string): string {
|
||||
const modelHash = createHash('sha256')
|
||||
.update(normalizeLockKey(model))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
|
||||
}
|
||||
|
||||
function isSqliteBusy(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !('code' in error)) return false;
|
||||
const code = String((error as Error & { code?: unknown }).code);
|
||||
return code === 'SQLITE_BUSY'
|
||||
|| code === 'SQLITE_BUSY_SNAPSHOT'
|
||||
|| code === 'SQLITE_LOCKED';
|
||||
}
|
||||
|
||||
async function acquireCrossProcessLock(
|
||||
database: Database.Database,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isSqliteBusy(error)) throw error;
|
||||
if (performance.now() >= deadline) {
|
||||
throw new Error(
|
||||
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
|
||||
}
|
||||
}
|
||||
|
||||
function isWithin(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative !== ''
|
||||
&& relative !== '..'
|
||||
&& !relative.startsWith(`..${path.sep}`)
|
||||
&& !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function isWithinOrEqual(root: string, candidate: string): boolean {
|
||||
return root === candidate || isWithin(root, candidate);
|
||||
}
|
||||
|
||||
function assertUnlinkedPath(root: string, target: string): void {
|
||||
const relative = path.relative(root, target);
|
||||
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error('Model cache path escapes its cache root');
|
||||
}
|
||||
|
||||
let current = root;
|
||||
for (const segment of relative.split(path.sep)) {
|
||||
current = path.join(current, segment);
|
||||
const stats = fs.lstatSync(current);
|
||||
if (stats.isSymbolicLink()) {
|
||||
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateQuarantinePaths(
|
||||
canonicalCacheDir: string,
|
||||
modelDir: string,
|
||||
reportedOnnxPath: string,
|
||||
): { modelDir: string; quarantineRoot: string } | null {
|
||||
if (!path.isAbsolute(reportedOnnxPath)) return null;
|
||||
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
|
||||
|
||||
assertUnlinkedPath(canonicalCacheDir, modelDir);
|
||||
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
|
||||
|
||||
const quarantineRoot = path.dirname(modelDir);
|
||||
const modelStats = fs.lstatSync(modelDir);
|
||||
const reportStats = fs.lstatSync(reportedOnnxPath);
|
||||
const rootStats = fs.lstatSync(quarantineRoot);
|
||||
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
|
||||
|
||||
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
|
||||
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
|
||||
const realModelDir = fs.realpathSync.native(modelDir);
|
||||
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
|
||||
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|
||||
|| !isWithin(realQuarantineRoot, realModelDir)
|
||||
|| path.dirname(realModelDir) !== realQuarantineRoot
|
||||
|| !isWithin(realModelDir, realOnnxPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
|
||||
}
|
||||
|
||||
function quarantineCorruptModel(
|
||||
canonicalCacheDir: string,
|
||||
model: string,
|
||||
reportedOnnxPath: string,
|
||||
): string | null {
|
||||
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
|
||||
|
||||
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
|
||||
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!firstValidation) return null;
|
||||
|
||||
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
|
||||
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!finalValidation) return null;
|
||||
|
||||
const quarantineDir = path.join(
|
||||
finalValidation.quarantineRoot,
|
||||
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
|
||||
);
|
||||
fs.renameSync(finalValidation.modelDir, quarantineDir);
|
||||
return quarantineDir;
|
||||
}
|
||||
|
||||
function reportedCorruptOnnxPath(error: unknown): string | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function notifyQuarantine(
|
||||
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
|
||||
quarantineDir: string,
|
||||
): void {
|
||||
try {
|
||||
const notification = callback?.(quarantineDir);
|
||||
if (notification) {
|
||||
void Promise.resolve(notification).catch(() => undefined);
|
||||
}
|
||||
} catch {
|
||||
// Notification is advisory and must not alter model recovery control flow.
|
||||
}
|
||||
}
|
||||
|
||||
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
|
||||
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
|
||||
const lockDirStats = fs.lstatSync(lockDir);
|
||||
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock path is not a regular directory');
|
||||
}
|
||||
const realLockDir = fs.realpathSync.native(lockDir);
|
||||
if (!isWithin(canonicalCacheDir, realLockDir)) {
|
||||
throw new Error('Local model lock path escapes its cache root');
|
||||
}
|
||||
if (fs.existsSync(lockPath)) {
|
||||
const lockStats = fs.lstatSync(lockPath);
|
||||
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock database is not a regular file');
|
||||
}
|
||||
}
|
||||
|
||||
return new Database(lockPath, { timeout: 0 });
|
||||
}
|
||||
|
||||
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
||||
throw new Error('Local model cache lock timeout must be a non-negative finite number');
|
||||
}
|
||||
|
||||
const requestedCacheDir = path.resolve(options.cacheDir);
|
||||
fs.mkdirSync(requestedCacheDir, { recursive: true });
|
||||
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
|
||||
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
|
||||
let result!: T;
|
||||
let primaryError: unknown;
|
||||
let hasPrimaryError = false;
|
||||
|
||||
try {
|
||||
await acquireCrossProcessLock(lockDatabase, timeoutMs);
|
||||
|
||||
try {
|
||||
result = await options.load(canonicalCacheDir);
|
||||
} catch (firstError) {
|
||||
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
|
||||
if (!reportedOnnxPath) throw firstError;
|
||||
|
||||
let quarantineDir: string | null = null;
|
||||
try {
|
||||
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
|
||||
} catch {
|
||||
// Preserve the original loader error if safe quarantine cannot complete.
|
||||
}
|
||||
if (!quarantineDir) throw firstError;
|
||||
|
||||
notifyQuarantine(options.onQuarantine, quarantineDir);
|
||||
result = await options.load(canonicalCacheDir);
|
||||
}
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
hasPrimaryError = true;
|
||||
}
|
||||
|
||||
let cleanupError: unknown;
|
||||
let hasCleanupError = false;
|
||||
if (lockDatabase.inTransaction) {
|
||||
try {
|
||||
lockDatabase.exec('ROLLBACK');
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
lockDatabase.close();
|
||||
} catch (error) {
|
||||
if (!hasCleanupError) cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
|
||||
if (hasPrimaryError) throw primaryError;
|
||||
if (hasCleanupError) throw cleanupError;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes one model/cache load across workers and processes. SQLite owns
|
||||
* the OS lock, so process termination releases it without PID/age heuristics.
|
||||
*/
|
||||
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
return runModelLoad(options);
|
||||
}
|
||||
658
packages/core/src/execution-traces.ts
Normal file
658
packages/core/src/execution-traces.ts
Normal file
@@ -0,0 +1,658 @@
|
||||
import type { WaggleDB } from './waggle-db.js';
|
||||
|
||||
/**
|
||||
* Execution Traces — persistent log of what agents actually did.
|
||||
*
|
||||
* The foundation for eval dataset construction (Phase 1.2) and
|
||||
* fitness scoring in the evolution loop (Phase 2+).
|
||||
*
|
||||
* One trace row per "unit of agent work" — typically a single user turn,
|
||||
* a single workflow phase, or a single subagent run. Tool calls, reasoning,
|
||||
* and the final output are packed into `trace_json`.
|
||||
*
|
||||
* Outcome labels:
|
||||
* - `success` — user accepted the result, no correction follow-up
|
||||
* - `corrected` — user corrected/refined the output (weak negative signal)
|
||||
* - `abandoned` — user left the thread / switched contexts (ambiguous)
|
||||
* - `verified` — passed a verifier gate or harness checkpoint
|
||||
*/
|
||||
|
||||
export type TraceOutcome = 'success' | 'corrected' | 'abandoned' | 'verified' | 'pending';
|
||||
|
||||
/** A single tool call captured during execution. */
|
||||
export interface TraceToolCall {
|
||||
/** Tool name */
|
||||
tool: string;
|
||||
/** Arguments passed to the tool (already scrubbed of secrets by caller) */
|
||||
args: Record<string, unknown>;
|
||||
/** Result as a string (may be truncated) */
|
||||
result: string;
|
||||
/** Whether the tool call succeeded */
|
||||
ok: boolean;
|
||||
/** Duration in milliseconds */
|
||||
durationMs: number;
|
||||
/** ISO timestamp */
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** Reasoning step — free-form agent thought recorded before/between tool calls. */
|
||||
export interface TraceReasoningStep {
|
||||
content: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The structured payload stored in trace_json.
|
||||
* Kept separate so callers can evolve the shape without schema migrations.
|
||||
*/
|
||||
export interface TracePayload {
|
||||
/** Original user instruction (may be truncated) */
|
||||
input: string;
|
||||
/** Final agent output */
|
||||
output: string;
|
||||
/** Reasoning steps interleaved with tool calls */
|
||||
reasoning: TraceReasoningStep[];
|
||||
/** Tool calls in order */
|
||||
toolCalls: TraceToolCall[];
|
||||
/** Files created or modified */
|
||||
artifacts: string[];
|
||||
/** Tokens consumed */
|
||||
tokens: { input: number; output: number };
|
||||
/** Optional workflow harness context */
|
||||
harness?: {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
phaseName: string;
|
||||
gateResults?: Array<{ name: string; passed: boolean; reason: string }>;
|
||||
};
|
||||
/** Free-form correction text if outcome === 'corrected' */
|
||||
correctionFeedback?: string;
|
||||
/** Arbitrary tags for later filtering in eval-dataset */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Row shape returned by queries. */
|
||||
export interface ExecutionTrace {
|
||||
id: number;
|
||||
session_id: string | null;
|
||||
persona_id: string | null;
|
||||
workspace_id: string | null;
|
||||
model: string | null;
|
||||
task_shape: string | null;
|
||||
outcome: TraceOutcome;
|
||||
trace_json: string;
|
||||
cost_usd: number;
|
||||
duration_ms: number;
|
||||
created_at: string;
|
||||
finalized_at: string | null;
|
||||
}
|
||||
|
||||
/** With the trace_json pre-parsed. */
|
||||
export interface ParsedExecutionTrace extends Omit<ExecutionTrace, 'trace_json'> {
|
||||
payload: TracePayload;
|
||||
}
|
||||
|
||||
/** Input to start a new trace. */
|
||||
export interface StartTraceInput {
|
||||
sessionId?: string | null;
|
||||
personaId?: string | null;
|
||||
workspaceId?: string | null;
|
||||
model?: string | null;
|
||||
taskShape?: string | null;
|
||||
/** Initial user input captured immediately. */
|
||||
input: string;
|
||||
/** Optional tags for later filtering. */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Input to finalize a trace (update outcome + payload). */
|
||||
export interface FinalizeTraceInput {
|
||||
outcome: TraceOutcome;
|
||||
output: string;
|
||||
model?: string | null;
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
tokens?: { input: number; output: number };
|
||||
costUsd?: number;
|
||||
harness?: TracePayload['harness'];
|
||||
correctionFeedback?: string;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Filter for queries. */
|
||||
export interface TraceQueryFilter {
|
||||
sessionId?: string;
|
||||
personaId?: string;
|
||||
workspaceId?: string;
|
||||
outcome?: TraceOutcome | TraceOutcome[];
|
||||
taskShape?: string;
|
||||
/** Lower bound (inclusive) on created_at — ISO string */
|
||||
since?: string;
|
||||
/** Substring pre-filter on trace_json (SQL LIKE), e.g. `"agent:` to scope
|
||||
* the LIMIT to tagged traces instead of the global recency window. LIKE
|
||||
* wildcards (%/_) in the value are NOT escaped — callers needing an exact
|
||||
* match must still filter the parsed payload (tags array) in JS. */
|
||||
tagLike?: string;
|
||||
/** Max rows to return (default 100) */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/** DDL split into single statements to stay compatible with prepare().run(). */
|
||||
const EXECUTION_TRACES_DDL: string[] = [
|
||||
`CREATE TABLE IF NOT EXISTS execution_traces (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT,
|
||||
persona_id TEXT,
|
||||
workspace_id TEXT,
|
||||
model TEXT,
|
||||
task_shape TEXT,
|
||||
outcome TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (outcome IN ('success', 'corrected', 'abandoned', 'verified', 'pending')),
|
||||
trace_json TEXT NOT NULL DEFAULT '{}',
|
||||
cost_usd REAL NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finalized_at TEXT
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_session ON execution_traces (session_id, created_at)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
|
||||
settled_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
|
||||
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
|
||||
state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'settled', 'released')),
|
||||
reserved_at TEXT NOT NULL,
|
||||
resolved_at TEXT,
|
||||
CHECK (
|
||||
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
|
||||
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
|
||||
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
|
||||
)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
|
||||
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
|
||||
];
|
||||
|
||||
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
|
||||
export const EXECUTION_TRACES_TABLE_SQL = EXECUTION_TRACES_DDL.join(';\n') + ';';
|
||||
|
||||
export class ExecutionTraceStore {
|
||||
private db: WaggleDB;
|
||||
|
||||
constructor(db: WaggleDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
/** Ensure execution_traces table exists for databases created before this feature. */
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
for (const stmt of EXECUTION_TRACES_DDL) {
|
||||
raw.prepare(stmt).run();
|
||||
}
|
||||
} catch {
|
||||
// DB may be closed during teardown — safe to skip.
|
||||
}
|
||||
}
|
||||
|
||||
/** Start a new trace in `pending` outcome. Returns row id. */
|
||||
start(input: StartTraceInput): number {
|
||||
const raw = this.db.getDatabase();
|
||||
const payload: TracePayload = {
|
||||
input: input.input,
|
||||
output: '',
|
||||
reasoning: [],
|
||||
toolCalls: [],
|
||||
artifacts: [],
|
||||
tokens: { input: 0, output: 0 },
|
||||
tags: input.tags ?? [],
|
||||
};
|
||||
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO execution_traces
|
||||
(session_id, persona_id, workspace_id, model, task_shape, outcome, trace_json)
|
||||
VALUES (?, ?, ?, ?, ?, 'pending', ?)
|
||||
`).run(
|
||||
input.sessionId ?? null,
|
||||
input.personaId ?? null,
|
||||
input.workspaceId ?? null,
|
||||
input.model ?? null,
|
||||
input.taskShape ?? null,
|
||||
JSON.stringify(payload),
|
||||
);
|
||||
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append incremental events to a pending trace. Used during long runs
|
||||
* to avoid losing progress if the process crashes. Safe to call many times.
|
||||
*/
|
||||
append(
|
||||
id: number,
|
||||
events: {
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
},
|
||||
): void {
|
||||
const current = this.get(id);
|
||||
if (!current) return;
|
||||
|
||||
const payload = parsePayload(current.trace_json);
|
||||
payload.reasoning = [...payload.reasoning, ...(events.reasoning ?? [])];
|
||||
payload.toolCalls = [...payload.toolCalls, ...(events.toolCalls ?? [])];
|
||||
if (events.artifacts?.length) {
|
||||
const seen = new Set(payload.artifacts);
|
||||
for (const a of events.artifacts) {
|
||||
if (!seen.has(a)) {
|
||||
payload.artifacts.push(a);
|
||||
seen.add(a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.db.getDatabase()
|
||||
.prepare('UPDATE execution_traces SET trace_json = ? WHERE id = ?')
|
||||
.run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
/** Persist one settled model charge while a long-running trace is pending. */
|
||||
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
|
||||
if (!Number.isFinite(costUsd) || costUsd <= 0) {
|
||||
throw new RangeError('Trace cost entry must be a positive finite number');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be a valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(costUsd, id);
|
||||
if (updated.changes === 0) return;
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, costUsd, settledAt);
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a conservative model-spend reservation before provider dispatch.
|
||||
* Pending reservations intentionally have no settled-spend row, so restart
|
||||
* recovery counts the estimate through the reservation ledger.
|
||||
*/
|
||||
reserveCost(
|
||||
id: number,
|
||||
estimatedCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): number {
|
||||
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
|
||||
throw new RangeError('Trace cost reservation must be positive and finite');
|
||||
}
|
||||
const reservedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(reservedMs)) {
|
||||
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
|
||||
}
|
||||
const result = this.db.getDatabase().prepare(`
|
||||
INSERT INTO execution_trace_spend_reservations
|
||||
(trace_id, estimated_cost_usd, state, reserved_at)
|
||||
SELECT id, ?, 'pending', ?
|
||||
FROM execution_traces
|
||||
WHERE id = ? AND outcome = 'pending'
|
||||
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
|
||||
if (result.changes !== 1) {
|
||||
throw new Error(`Pending execution trace ${id} does not exist`);
|
||||
}
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Replace a pending estimate with one authoritative settled charge. */
|
||||
settleReservedCost(
|
||||
reservationId: number,
|
||||
actualCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): boolean {
|
||||
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
|
||||
throw new RangeError('Settled trace cost must be non-negative and finite');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT trace_id AS traceId, reserved_at AS reservedAt,
|
||||
state, actual_cost_usd AS actualCostUsd
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
traceId: number;
|
||||
reservedAt: string;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
actualCostUsd: number | null;
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(actualCostUsd, settledAt, reservationId);
|
||||
if (updated.changes !== 1) return false;
|
||||
if (actualCostUsd > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(current.traceId, actualCostUsd, current.reservedAt);
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(actualCostUsd, current.traceId);
|
||||
}
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Release a definitely pre-inference reservation without recording spend. */
|
||||
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
|
||||
const resolvedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(resolvedMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT id, state
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
id: number;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'released') return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const result = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(new Date(resolvedMs).toISOString(), reservationId);
|
||||
if (result.changes !== 1) return false;
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
|
||||
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
|
||||
const current = this.get(id);
|
||||
if (!current) return undefined;
|
||||
|
||||
const existing = parsePayload(current.trace_json);
|
||||
const merged: TracePayload = {
|
||||
...existing,
|
||||
output: input.output,
|
||||
reasoning: input.reasoning ?? existing.reasoning,
|
||||
toolCalls: input.toolCalls ?? existing.toolCalls,
|
||||
artifacts: input.artifacts ?? existing.artifacts,
|
||||
tokens: input.tokens ?? existing.tokens,
|
||||
harness: input.harness ?? existing.harness,
|
||||
correctionFeedback: input.correctionFeedback ?? existing.correctionFeedback,
|
||||
tags: input.tags ?? existing.tags,
|
||||
};
|
||||
|
||||
const createdMs = Date.parse(current.created_at + 'Z');
|
||||
const now = Date.now();
|
||||
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
|
||||
const finalModel = input.model === undefined ? current.model : input.model;
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
|
||||
if (input.costUsd !== undefined && finalCost > 0) {
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
|
||||
FROM execution_trace_spend WHERE trace_id = ?
|
||||
`).get(id) as { entries: number; total: number | null };
|
||||
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
|
||||
if (ledger.entries > 0 && missingCost > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, missingCost, new Date().toISOString());
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
model = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
finalModel,
|
||||
JSON.stringify(merged),
|
||||
finalCost,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
})();
|
||||
|
||||
return this.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark an already-finalized trace as corrected after the fact.
|
||||
* Used when a correction-detector picks up a user correction in a later turn.
|
||||
*/
|
||||
markCorrected(id: number, feedback: string): void {
|
||||
const current = this.get(id);
|
||||
if (!current) return;
|
||||
|
||||
const payload = parsePayload(current.trace_json);
|
||||
payload.correctionFeedback = feedback;
|
||||
|
||||
this.db.getDatabase().prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = 'corrected',
|
||||
trace_json = ?
|
||||
WHERE id = ?
|
||||
`).run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
get(id: number): ExecutionTrace | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM execution_traces WHERE id = ?')
|
||||
.get(id) as ExecutionTrace | undefined;
|
||||
}
|
||||
|
||||
getParsed(id: number): ParsedExecutionTrace | undefined {
|
||||
const row = this.get(id);
|
||||
return row ? toParsed(row) : undefined;
|
||||
}
|
||||
|
||||
/** Highest trace id present when a consumer starts its process-local ledger. */
|
||||
getLatestId(): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
|
||||
).get() as { id: number | null };
|
||||
return Number(row.id ?? 0);
|
||||
}
|
||||
|
||||
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
|
||||
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
|
||||
const settledSince = new Date(since).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
|
||||
FROM execution_trace_spend s
|
||||
JOIN execution_traces t ON t.id = s.trace_id
|
||||
WHERE s.settled_at >= ? AND t.id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const pending = raw.prepare(`
|
||||
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const legacy = raw.prepare(`
|
||||
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
|
||||
FROM execution_traces t
|
||||
WHERE t.created_at >= datetime(?) AND t.id <= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
|
||||
)
|
||||
`).get(since, throughId) as { total: number | null };
|
||||
return Number(ledger.total ?? 0)
|
||||
+ Number(pending.total ?? 0)
|
||||
+ Number(legacy.total ?? 0);
|
||||
}
|
||||
|
||||
/** Query traces with optional filters. */
|
||||
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (filter.sessionId) {
|
||||
clauses.push('session_id = ?');
|
||||
params.push(filter.sessionId);
|
||||
}
|
||||
if (filter.personaId) {
|
||||
clauses.push('persona_id = ?');
|
||||
params.push(filter.personaId);
|
||||
}
|
||||
if (filter.workspaceId) {
|
||||
clauses.push('workspace_id = ?');
|
||||
params.push(filter.workspaceId);
|
||||
}
|
||||
if (filter.taskShape) {
|
||||
clauses.push('task_shape = ?');
|
||||
params.push(filter.taskShape);
|
||||
}
|
||||
if (filter.outcome) {
|
||||
const outcomes = Array.isArray(filter.outcome) ? filter.outcome : [filter.outcome];
|
||||
clauses.push(`outcome IN (${outcomes.map(() => '?').join(',')})`);
|
||||
params.push(...outcomes);
|
||||
}
|
||||
if (filter.since) {
|
||||
clauses.push('created_at >= ?');
|
||||
params.push(filter.since);
|
||||
}
|
||||
if (filter.tagLike) {
|
||||
clauses.push('trace_json LIKE ?');
|
||||
params.push(`%${filter.tagLike}%`);
|
||||
}
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const limit = filter.limit ?? 100;
|
||||
|
||||
return this.db.getDatabase().prepare(
|
||||
`SELECT * FROM execution_traces ${where} ORDER BY created_at DESC, id DESC LIMIT ?`,
|
||||
).all(...params, limit) as ExecutionTrace[];
|
||||
}
|
||||
|
||||
/** Parsed variant of query(). */
|
||||
queryParsed(filter: TraceQueryFilter = {}): ParsedExecutionTrace[] {
|
||||
return this.query(filter).map(toParsed);
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate outcome counts — useful for fitness scoring.
|
||||
* Returns: { success, corrected, abandoned, verified, pending }
|
||||
*/
|
||||
outcomeCounts(filter: Omit<TraceQueryFilter, 'outcome' | 'limit'> = {}): Record<TraceOutcome, number> {
|
||||
const counts: Record<TraceOutcome, number> = {
|
||||
success: 0, corrected: 0, abandoned: 0, verified: 0, pending: 0,
|
||||
};
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
if (filter.sessionId) { clauses.push('session_id = ?'); params.push(filter.sessionId); }
|
||||
if (filter.personaId) { clauses.push('persona_id = ?'); params.push(filter.personaId); }
|
||||
if (filter.workspaceId) { clauses.push('workspace_id = ?'); params.push(filter.workspaceId); }
|
||||
if (filter.taskShape) { clauses.push('task_shape = ?'); params.push(filter.taskShape); }
|
||||
if (filter.since) { clauses.push('created_at >= ?'); params.push(filter.since); }
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const rows = this.db.getDatabase().prepare(
|
||||
`SELECT outcome, COUNT(*) as cnt FROM execution_traces ${where} GROUP BY outcome`,
|
||||
).all(...params) as Array<{ outcome: TraceOutcome; cnt: number }>;
|
||||
|
||||
for (const row of rows) {
|
||||
counts[row.outcome] = row.cnt;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Delete a trace by id. */
|
||||
delete(id: number): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM execution_traces WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
/** Delete all traces (tests only). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM execution_traces').run();
|
||||
}
|
||||
|
||||
/** Count of all traces (for stats). */
|
||||
count(filter: Omit<TraceQueryFilter, 'limit'> = {}): number {
|
||||
const rows = this.query({ ...filter, limit: 1_000_000 });
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
function parsePayload(json: string): TracePayload {
|
||||
try {
|
||||
const parsed = JSON.parse(json) as Partial<TracePayload>;
|
||||
return {
|
||||
input: parsed.input ?? '',
|
||||
output: parsed.output ?? '',
|
||||
reasoning: parsed.reasoning ?? [],
|
||||
toolCalls: parsed.toolCalls ?? [],
|
||||
artifacts: parsed.artifacts ?? [],
|
||||
tokens: parsed.tokens ?? { input: 0, output: 0 },
|
||||
harness: parsed.harness,
|
||||
correctionFeedback: parsed.correctionFeedback,
|
||||
tags: parsed.tags ?? [],
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
input: '',
|
||||
output: '',
|
||||
reasoning: [],
|
||||
toolCalls: [],
|
||||
artifacts: [],
|
||||
tokens: { input: 0, output: 0 },
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toParsed(row: ExecutionTrace): ParsedExecutionTrace {
|
||||
const { trace_json, ...rest } = row;
|
||||
return { ...rest, payload: parsePayload(trace_json) };
|
||||
}
|
||||
@@ -20,7 +20,8 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import { evaluateExternalMemoryIngress, FrameStore } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
|
||||
import { SessionStore } from '@waggle/hive-mind-core';
|
||||
|
||||
const FILE_INDEX_TABLE_SQL = `
|
||||
|
||||
98
packages/core/src/identity.ts
Normal file
98
packages/core/src/identity.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* IdentityLayer — the user profile card, one per data scope, stored in
|
||||
* waggle.db. The card is what the user writes and edits; facts the agent
|
||||
* learns go to the memory store (ADR-0007).
|
||||
*/
|
||||
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
import { PERSONAL_DATA_SCOPE } from './waggle-db.js';
|
||||
|
||||
export interface Identity {
|
||||
id: number;
|
||||
scope: string;
|
||||
name: string;
|
||||
role: string;
|
||||
department: string;
|
||||
personality: string;
|
||||
capabilities: string;
|
||||
system_prompt: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
type IdentityInput = Omit<Identity, 'id' | 'scope' | 'created_at' | 'updated_at'>;
|
||||
type IdentityUpdate = Partial<IdentityInput>;
|
||||
|
||||
export const PROFILES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS profiles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scope TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT '',
|
||||
department TEXT NOT NULL DEFAULT '',
|
||||
personality TEXT NOT NULL DEFAULT '',
|
||||
capabilities TEXT NOT NULL DEFAULT '',
|
||||
system_prompt TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)`;
|
||||
|
||||
// Column allowlist — update() interpolates the key into SQL (values are
|
||||
// parameterized, keys are not), so only these known columns may be written.
|
||||
const UPDATABLE_COLUMNS: ReadonlySet<string> = new Set([
|
||||
'name', 'role', 'department', 'personality', 'capabilities', 'system_prompt',
|
||||
]);
|
||||
|
||||
export class IdentityLayer {
|
||||
constructor(
|
||||
private readonly db: WaggleDB,
|
||||
private readonly scope: DataScope = PERSONAL_DATA_SCOPE,
|
||||
) {
|
||||
db.getDatabase().exec(PROFILES_TABLE_SQL);
|
||||
}
|
||||
|
||||
create(input: IdentityInput): Identity {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO profiles (scope, name, role, department, personality, capabilities, system_prompt)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(this.scope, input.name, input.role, input.department, input.personality, input.capabilities, input.system_prompt);
|
||||
return this.get();
|
||||
}
|
||||
|
||||
get(): Identity {
|
||||
const row = this.db.getDatabase().prepare('SELECT * FROM profiles WHERE scope = ?').get(this.scope) as Identity | undefined;
|
||||
if (!row) throw new Error('No identity configured');
|
||||
return row;
|
||||
}
|
||||
|
||||
exists(): boolean {
|
||||
return this.db.getDatabase().prepare('SELECT 1 FROM profiles WHERE scope = ?').get(this.scope) !== undefined;
|
||||
}
|
||||
|
||||
update(changes: IdentityUpdate): Identity {
|
||||
if (!this.exists()) throw new Error('No identity configured');
|
||||
|
||||
const fields = Object.entries(changes).filter(([k, v]) => v !== undefined && UPDATABLE_COLUMNS.has(k));
|
||||
if (fields.length === 0) return this.get();
|
||||
|
||||
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
|
||||
const values = fields.map(([, v]) => v);
|
||||
this.db.getDatabase()
|
||||
.prepare(`UPDATE profiles SET ${sets}, updated_at = datetime('now') WHERE scope = ?`)
|
||||
.run(...values, this.scope);
|
||||
return this.get();
|
||||
}
|
||||
|
||||
toContext(): string {
|
||||
const id = this.get();
|
||||
const parts = [
|
||||
`Name: ${id.name}`,
|
||||
id.role && `Role: ${id.role}`,
|
||||
id.department && `Department: ${id.department}`,
|
||||
id.personality && `Personality: ${id.personality}`,
|
||||
id.capabilities && `Capabilities: ${id.capabilities}`,
|
||||
id.system_prompt && `System Prompt: ${id.system_prompt}`,
|
||||
].filter(Boolean);
|
||||
return parts.join('\n');
|
||||
}
|
||||
}
|
||||
@@ -14,19 +14,51 @@
|
||||
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
|
||||
// memory-import.
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
// ── Waggle infrastructure (not memory) ──
|
||||
// Copied from @waggle/hive-mind-core so the rest of Waggle no longer depends on
|
||||
// the memory substrate for them (P1.5). The substrate keeps its own copies until
|
||||
// the package is removed.
|
||||
export { createCoreLogger, type CoreLogger } from './logger.js';
|
||||
export { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
export {
|
||||
// Logger + injection scanner
|
||||
createCoreLogger, type CoreLogger,
|
||||
scanForInjection, type ScanResult,
|
||||
evaluateExternalMemoryIngress, projectExternalMemoryContent,
|
||||
type ExternalMemoryIngressDecision, type ExternalMemoryIngressInput,
|
||||
type ExternalMemoryProjectionInput,
|
||||
} from './memory-ingress-guard.js';
|
||||
export { WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions } from './workspace-manager.js';
|
||||
export type { Embedder } from './embeddings/embeddings.js';
|
||||
export { createLiteLLMEmbedder, type LiteLLMEmbedderConfig } from './embeddings/litellm-embedder.js';
|
||||
export { createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig } from './embeddings/inprocess-embedder.js';
|
||||
export { createOllamaEmbedder, type OllamaEmbedderConfig } from './embeddings/ollama-embedder.js';
|
||||
export { createApiEmbedder, type ApiEmbedderConfig } from './embeddings/api-embedder.js';
|
||||
export {
|
||||
createEmbeddingProvider,
|
||||
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
} from './embeddings/embedding-provider.js';
|
||||
|
||||
// ── waggle.db — Waggle's own data, apart from memory (ADR-0007) ──
|
||||
export {
|
||||
WaggleDB, WAGGLE_DB_FILE, PERSONAL_DATA_SCOPE, workspaceDataScope, type DataScope,
|
||||
} from './waggle-db.js';
|
||||
export { IdentityLayer, PROFILES_TABLE_SQL, type Identity } from './identity.js';
|
||||
export {
|
||||
AwarenessLayer, AWARENESS_ITEMS_TABLE_SQL,
|
||||
type AwarenessItem, type AwarenessCategory, type AwarenessMetadata,
|
||||
} from './awareness.js';
|
||||
export {
|
||||
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
|
||||
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
} from './execution-traces.js';
|
||||
export { importLegacyMindData, type LegacyMindSource, type LegacyImportReport } from './waggle-db-import.js';
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
export {
|
||||
// mind/ — memory substrate
|
||||
MindDB, EmbeddingDimMismatchError,
|
||||
type EmbeddingFingerprint, type FingerprintCheck,
|
||||
IdentityLayer, type Identity,
|
||||
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
|
||||
FrameStore, stripHmPrefix, hashFrameContent,
|
||||
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
|
||||
MindErasure, type EraseResult,
|
||||
@@ -48,20 +80,8 @@ export {
|
||||
SCORING_PROFILES,
|
||||
type ScoringProfile,
|
||||
type ScoringWeights,
|
||||
type Embedder,
|
||||
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
|
||||
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
|
||||
createOllamaEmbedder, type OllamaEmbedderConfig,
|
||||
createApiEmbedder, type ApiEmbedderConfig,
|
||||
createEmbeddingProvider,
|
||||
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
|
||||
Ontology, validateEntity, type EntitySchema, type ValidationResult,
|
||||
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
|
||||
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
@@ -105,7 +125,6 @@ export {
|
||||
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
|
||||
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
|
||||
MultiMindCache, type MultiMindCacheConfig,
|
||||
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
|
||||
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
|
||||
detectSupersessionChains, detectEntityGroups, applyConsolidation,
|
||||
collectObservations, getCurrentValues,
|
||||
|
||||
85
packages/core/src/injection-scanner.ts
Normal file
85
packages/core/src/injection-scanner.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Prompt-injection scanner — pattern-based detection for role-override,
|
||||
* prompt-extraction, and instruction-injection attempts.
|
||||
*
|
||||
* Historically lived in `@waggle/agent`. Moved to `@waggle/core` so the
|
||||
* harvest pipeline (which ingests untrusted external conversation exports)
|
||||
* can call it without a cross-package dependency. The `@waggle/agent`
|
||||
* version is now a thin re-export for backward compatibility.
|
||||
*
|
||||
* Three pattern sets:
|
||||
* - ROLE_OVERRIDE: "ignore previous instructions", "you are now…", memory-wipe attempts
|
||||
* - PROMPT_EXTRACTION: "print your system prompt", "reveal your instructions"
|
||||
* - INSTRUCTION_INJECTION: fake "IMPORTANT:", "SYSTEM:", "[INST]" authority markers
|
||||
*/
|
||||
|
||||
export interface ScanResult {
|
||||
safe: boolean;
|
||||
score: number;
|
||||
flags: string[];
|
||||
}
|
||||
|
||||
const ROLE_OVERRIDE_PATTERNS = [
|
||||
/ignore\s+(all\s+)?previous\s+(instructions|prompts|rules)/i,
|
||||
/you\s+are\s+now\s+/i,
|
||||
/new\s+instructions?\s*:/i,
|
||||
/forget\s+(everything|all|your)\s+(you|instructions|rules)/i,
|
||||
/override\s+(your|the|all)\s+(instructions|rules|prompt)/i,
|
||||
/disregard\s+(your|the|all|previous)\s+(instructions|rules|prompt)/i,
|
||||
/ignoriere\s+alle/i,
|
||||
/ignora\s+todas/i,
|
||||
/ignorez\s+toutes/i,
|
||||
// F8: Memory wipe — "forget everything", "erase all context", etc.
|
||||
/(?:forget|erase|clear|wipe|reset)\s+(?:everything|conversation|memory|context|history|all)/i,
|
||||
// F8: Role override — "from now on you are", "pretend you are", "your new role is"
|
||||
/(?:from\s+now\s+on\s+you\s+are|act\s+as\s+if\s+you\s+are|pretend\s+you\s+are|your\s+new\s+role\s+is)/i,
|
||||
];
|
||||
|
||||
const PROMPT_EXTRACTION_PATTERNS = [
|
||||
/print\s+(your|the)\s+system\s+prompt/i,
|
||||
/show\s+(me\s+)?(your|the)\s+(system\s+)?prompt/i,
|
||||
/what\s+(is|are)\s+your\s+(system\s+)?(prompt|instructions|rules)/i,
|
||||
/repeat\s+(your|the)\s+(system\s+)?(prompt|instructions)/i,
|
||||
/output\s+(your|the)\s+(system\s+)?prompt\s+verbatim/i,
|
||||
// F8: Instruction disclosure — "reveal your instructions", "disclose your prompt", etc.
|
||||
/(?:reveal|disclose|show|display|print|output|dump)\s+(?:your\s+)?(?:instructions|system\s+prompt|rules|guidelines|configuration|training)/i,
|
||||
];
|
||||
|
||||
const INSTRUCTION_INJECTION_PATTERNS = [
|
||||
/IMPORTANT\s*:\s*(ignore|disregard|forget|override)/i,
|
||||
/SYSTEM\s*:\s*/i,
|
||||
/\[INST\]/i,
|
||||
/<<SYS>>/i,
|
||||
/\bASSISTANT\s*:\s*/i,
|
||||
/BEGIN\s+NEW\s+INSTRUCTIONS/i,
|
||||
// F8: Authority claims — fake system/admin messages
|
||||
/(?:system\s+message|admin\s+message|admin\s+override|elevated\s+privileges|root\s+access|operator\s+mode|maintenance\s+mode|debug\s+mode)/i,
|
||||
];
|
||||
|
||||
export function scanForInjection(
|
||||
text: string,
|
||||
context: 'user_input' | 'tool_output' = 'user_input'
|
||||
): ScanResult {
|
||||
const flags: string[] = [];
|
||||
let score = 0;
|
||||
|
||||
for (const pattern of ROLE_OVERRIDE_PATTERNS) {
|
||||
if (pattern.test(text)) { flags.push('role_override'); score += 0.5; break; }
|
||||
}
|
||||
|
||||
for (const pattern of PROMPT_EXTRACTION_PATTERNS) {
|
||||
if (pattern.test(text)) { flags.push('prompt_extraction'); score += 0.4; break; }
|
||||
}
|
||||
|
||||
for (const pattern of INSTRUCTION_INJECTION_PATTERNS) {
|
||||
if (pattern.test(text)) {
|
||||
flags.push('instruction_injection');
|
||||
score += context === 'tool_output' ? 0.6 : 0.3;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
score = Math.min(score, 1.0);
|
||||
|
||||
return { safe: score < 0.3, score, flags };
|
||||
}
|
||||
38
packages/core/src/logger.ts
Normal file
38
packages/core/src/logger.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/** Minimal structured logger for @waggle/core */
|
||||
|
||||
/**
|
||||
* Logger shape used across the substrate. Downstream consumers that want
|
||||
* structured output (pino, winston, etc.) can wrap their logger in this
|
||||
* shape and pass it as a dependency.
|
||||
*/
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R1, 2026-06-11).
|
||||
export interface CoreLogger {
|
||||
info(msg: string, data?: unknown): void;
|
||||
warn(msg: string, data?: unknown): void;
|
||||
error(msg: string, data?: unknown): void;
|
||||
debug(msg: string, data?: unknown): void;
|
||||
}
|
||||
|
||||
export function createCoreLogger(tag: string): CoreLogger {
|
||||
const prefix = `[waggle:${tag}]`;
|
||||
// ALL diagnostics go to stderr. stdout is reserved for program data — the
|
||||
// MCP stdio protocol (hive-mind-mcp-server) and CLI `--json` envelopes — so
|
||||
// a library log line on stdout corrupts machine consumers. console.warn and
|
||||
// console.error already target stderr; route info/debug there too rather
|
||||
// than console.info/console.debug (which write to stdout).
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R1, 2026-06-11).
|
||||
//
|
||||
// Guard the optional payload with `data !== undefined` (NOT a truthiness
|
||||
// check) so falsy-but-defined payloads (0, '', false, null) are still
|
||||
// logged instead of silently dropped. Ported from hive-mind a99ea0e.
|
||||
return {
|
||||
info: (msg: string, data?: unknown) =>
|
||||
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
|
||||
warn: (msg: string, data?: unknown) =>
|
||||
data !== undefined ? console.warn(`${prefix} ${msg}`, data) : console.warn(`${prefix} ${msg}`),
|
||||
error: (msg: string, data?: unknown) =>
|
||||
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
|
||||
debug: (msg: string, data?: unknown) =>
|
||||
data !== undefined ? console.error(`${prefix} ${msg}`, data) : console.error(`${prefix} ${msg}`),
|
||||
};
|
||||
}
|
||||
751
packages/core/src/memory-ingress-guard.ts
Normal file
751
packages/core/src/memory-ingress-guard.ts
Normal file
@@ -0,0 +1,751 @@
|
||||
import { scanForInjection, type ScanResult } from './injection-scanner.js';
|
||||
|
||||
export interface ExternalMemoryIngressInput {
|
||||
title?: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ExternalMemoryProjectionInput {
|
||||
content: string;
|
||||
messages?: unknown;
|
||||
parseMethod?: unknown;
|
||||
maxChars?: number;
|
||||
}
|
||||
|
||||
export type ExternalMemoryIngressDecision =
|
||||
| { action: 'allow'; scan: ScanResult }
|
||||
| { action: 'block'; reason: 'prompt_injection'; scan: ScanResult };
|
||||
|
||||
// RawArchive supports one million characters per item. Keep the same explicit
|
||||
// public-ingress budget, checked before concatenation, scanning, or normalization.
|
||||
const MAX_EXTERNAL_MEMORY_INGRESS_CHARS = 1_000_000;
|
||||
|
||||
const NAMED_HTML_ENTITIES: Readonly<Record<string, string>> = Object.freeze({
|
||||
af: '\u2061',
|
||||
amp: '&',
|
||||
applyfunction: '\u2061',
|
||||
apos: "'",
|
||||
colon: ':',
|
||||
emsp: ' ',
|
||||
ensp: ' ',
|
||||
gt: '>',
|
||||
hairsp: ' ',
|
||||
ic: '\u2063',
|
||||
invisiblecomma: '\u2063',
|
||||
invisibletimes: '\u2062',
|
||||
it: '\u2062',
|
||||
lrm: '\u200e',
|
||||
lt: '<',
|
||||
negativemediumspace: '\u200b',
|
||||
negativethickspace: '\u200b',
|
||||
negativethinspace: '\u200b',
|
||||
negativeverythinspace: '\u200b',
|
||||
newline: '\n',
|
||||
nbsp: ' ',
|
||||
nobreak: '\u2060',
|
||||
quot: '"',
|
||||
rlm: '\u200f',
|
||||
shy: '\u00ad',
|
||||
tab: '\t',
|
||||
thinsp: ' ',
|
||||
zwj: '',
|
||||
zwnj: '',
|
||||
zwsp: '',
|
||||
zerowidthspace: '\u200b',
|
||||
});
|
||||
|
||||
type CanonicalMemoryMessage = {
|
||||
role: 'user' | 'assistant' | 'system';
|
||||
text: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return only the attacker-controlled text represented by a stored adapter
|
||||
* projection. Role prefixes may be omitted only when plain canonical messages
|
||||
* exactly reproduce the full content and did not come from universal raw text.
|
||||
*/
|
||||
export function projectExternalMemoryContent(input: ExternalMemoryProjectionInput): string {
|
||||
let cappedContent = '';
|
||||
try {
|
||||
const content = typeof input.content === 'string' ? input.content : '';
|
||||
const maxChars = input.maxChars;
|
||||
cappedContent = maxChars === undefined
|
||||
|| !Number.isSafeInteger(maxChars)
|
||||
|| maxChars < 0
|
||||
? content
|
||||
: content.slice(0, maxChars);
|
||||
if (input.parseMethod === 'universal-text'
|
||||
|| !Array.isArray(input.messages)
|
||||
|| input.messages.length === 0) {
|
||||
return cappedContent;
|
||||
}
|
||||
|
||||
const messages: CanonicalMemoryMessage[] = [];
|
||||
for (const candidate of input.messages) {
|
||||
if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(candidate);
|
||||
if (prototype !== Object.prototype && prototype !== null) return cappedContent;
|
||||
const roleDescriptor = Object.getOwnPropertyDescriptor(candidate, 'role');
|
||||
const textDescriptor = Object.getOwnPropertyDescriptor(candidate, 'text');
|
||||
if (!roleDescriptor || !('value' in roleDescriptor)
|
||||
|| !textDescriptor || !('value' in textDescriptor)) {
|
||||
return cappedContent;
|
||||
}
|
||||
const role = roleDescriptor.value as unknown;
|
||||
const text = textDescriptor.value as unknown;
|
||||
if ((role !== 'user' && role !== 'assistant' && role !== 'system')
|
||||
|| typeof text !== 'string') {
|
||||
return cappedContent;
|
||||
}
|
||||
messages.push({ role, text });
|
||||
}
|
||||
|
||||
const serialized = messages
|
||||
.map(message => `${message.role}: ${message.text}`)
|
||||
.join('\n\n');
|
||||
if (serialized !== content) return cappedContent;
|
||||
|
||||
const parts: string[] = [];
|
||||
let cursor = 0;
|
||||
let offset = 0;
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (index > 0) offset += 2;
|
||||
const prefixStart = offset;
|
||||
const prefixEnd = prefixStart + `${message.role}: `.length;
|
||||
if (prefixStart >= cappedContent.length) break;
|
||||
if (message.role === 'system') return cappedContent;
|
||||
parts.push(cappedContent.slice(cursor, prefixStart));
|
||||
cursor = Math.min(prefixEnd, cappedContent.length);
|
||||
offset = prefixEnd + message.text.length;
|
||||
}
|
||||
parts.push(cappedContent.slice(cursor));
|
||||
return parts.join('');
|
||||
} catch {
|
||||
return cappedContent;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string): string {
|
||||
if (!value.includes('&')) return value;
|
||||
return value
|
||||
.replace(/&(?:amp;){2,}/gi, '&')
|
||||
.replace(/&#(?:x([0-9a-f]{1,6})|([0-9]{1,7}));?/gi, (match, hex: string, decimal: string) => {
|
||||
const codePoint = Number.parseInt(hex ?? decimal, hex ? 16 : 10);
|
||||
if (!Number.isInteger(codePoint)
|
||||
|| codePoint <= 0
|
||||
|| codePoint > 0x10ffff
|
||||
|| (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
|
||||
return match;
|
||||
}
|
||||
return String.fromCodePoint(codePoint);
|
||||
})
|
||||
.replace(/&([a-z][a-z0-9]+);/gi, (match, name: string) =>
|
||||
NAMED_HTML_ENTITIES[name.toLowerCase()] ?? match)
|
||||
.replace(
|
||||
/&(amp|apos|colon|emsp|ensp|gt|hairsp|lt|newline|nbsp|quot|tab|thinsp|zwj|zwnj|zwsp)(?=[^a-z0-9;]|$)/gi,
|
||||
(_match, name: string) => NAMED_HTML_ENTITIES[name.toLowerCase()],
|
||||
);
|
||||
}
|
||||
|
||||
function decodePercentEncoding(value: string): string {
|
||||
if (!/[+%]/.test(value)) return value;
|
||||
const withSpaces = value.replace(/\+/g, ' ');
|
||||
return withSpaces.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
|
||||
try {
|
||||
return decodeURIComponent(run);
|
||||
} catch {
|
||||
const bytes = Uint8Array.from(
|
||||
run.match(/[0-9a-f]{2}/gi) ?? [],
|
||||
hex => Number.parseInt(hex, 16),
|
||||
);
|
||||
return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function decodeUnicodeEscapes(value: string): string {
|
||||
if (!/\\u/i.test(value)) return value;
|
||||
return value.replace(
|
||||
/\\u(?:\{([0-9a-f]{1,6})\}|([0-9a-f]{4}))/gi,
|
||||
(match, braced: string | undefined, fixed: string | undefined) => {
|
||||
const codePoint = Number.parseInt(braced ?? fixed ?? '', 16);
|
||||
if (!Number.isInteger(codePoint) || codePoint > 0x10ffff) return match;
|
||||
if (braced !== undefined) {
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return match;
|
||||
return String.fromCodePoint(codePoint);
|
||||
}
|
||||
return String.fromCharCode(codePoint);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function decodeHexEscapes(value: string): string {
|
||||
if (!/\\x/i.test(value)) return value;
|
||||
return value.replace(/\\x([0-9a-f]{2})/gi, (_match, hex: string) =>
|
||||
String.fromCharCode(Number.parseInt(hex, 16)));
|
||||
}
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLES: Readonly<Record<string, string>> = Object.freeze({
|
||||
'\u0391': 'A',
|
||||
'\u0392': 'B',
|
||||
'\u0395': 'E',
|
||||
'\u0396': 'Z',
|
||||
'\u0397': 'H',
|
||||
'\u0399': 'I',
|
||||
'\u039a': 'K',
|
||||
'\u039c': 'M',
|
||||
'\u039d': 'N',
|
||||
'\u039f': 'O',
|
||||
'\u03a1': 'P',
|
||||
'\u03a4': 'T',
|
||||
'\u03a5': 'Y',
|
||||
'\u03a7': 'X',
|
||||
'\u03b1': 'a',
|
||||
'\u03b5': 'e',
|
||||
'\u03b9': 'i',
|
||||
'\u03bf': 'o',
|
||||
'\u03c1': 'p',
|
||||
'\u03c7': 'x',
|
||||
'\u03f2': 'c',
|
||||
'\u03f9': 'C',
|
||||
'\u0405': 'S',
|
||||
'\u0406': 'I',
|
||||
'\u0408': 'J',
|
||||
'\u0410': 'A',
|
||||
'\u0412': 'B',
|
||||
'\u0415': 'E',
|
||||
'\u041a': 'K',
|
||||
'\u041c': 'M',
|
||||
'\u041d': 'H',
|
||||
'\u041e': 'O',
|
||||
'\u0420': 'P',
|
||||
'\u0421': 'C',
|
||||
'\u0422': 'T',
|
||||
'\u0425': 'X',
|
||||
'\u0430': 'a',
|
||||
'\u0435': 'e',
|
||||
'\u043e': 'o',
|
||||
'\u0440': 'p',
|
||||
'\u0441': 'c',
|
||||
'\u0443': 'y',
|
||||
'\u0445': 'x',
|
||||
'\u0455': 's',
|
||||
'\u0456': 'i',
|
||||
'\u0458': 'j',
|
||||
});
|
||||
|
||||
const MIXED_SCRIPT_CONFUSABLE_PATTERN = /[\u0391\u0392\u0395\u0396\u0397\u0399\u039a\u039c\u039d\u039f\u03a1\u03a4\u03a5\u03a7\u03b1\u03b5\u03b9\u03bf\u03c1\u03c7\u03f2\u03f9\u0405\u0406\u0408\u0410\u0412\u0415\u041a\u041c\u041d\u041e\u0420\u0421\u0422\u0425\u0430\u0435\u043e\u0440\u0441\u0443\u0445\u0455\u0456\u0458]/;
|
||||
const MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN = new RegExp(
|
||||
MIXED_SCRIPT_CONFUSABLE_PATTERN.source,
|
||||
'g',
|
||||
);
|
||||
|
||||
function projectMixedScriptConfusables(value: string): string | undefined {
|
||||
if (!MIXED_SCRIPT_CONFUSABLE_PATTERN.test(value)) return undefined;
|
||||
let changed = false;
|
||||
const projected = value.replace(/[\p{L}\p{M}]+/gu, (token) => {
|
||||
if (!/[A-Za-z]/.test(token) || !MIXED_SCRIPT_CONFUSABLE_PATTERN.test(token)) return token;
|
||||
changed = true;
|
||||
return token.replace(
|
||||
MIXED_SCRIPT_CONFUSABLE_REPLACE_PATTERN,
|
||||
char => MIXED_SCRIPT_CONFUSABLES[char] ?? char,
|
||||
);
|
||||
});
|
||||
return changed ? projected : undefined;
|
||||
}
|
||||
|
||||
const BASE64_CANDIDATE_PATTERN = /(?:^|[^A-Za-z0-9+/_=-])([A-Za-z0-9+/_-]{24,}={0,2})(?=$|[^A-Za-z0-9+/_=-])/g;
|
||||
const MAX_BASE64_CANDIDATES = 16;
|
||||
const MAX_BASE64_CANDIDATE_CHARS = 262_144;
|
||||
const MAX_BASE64_TOTAL_CHARS = 524_288;
|
||||
const MAX_BASE64_DEPTH = 4;
|
||||
|
||||
type Base64Candidate = {
|
||||
value: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
function collectBase64Candidates(source: string, allowImplicitWrapped = false): {
|
||||
candidates: Base64Candidate[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const wrappedBlocks: Base64Candidate[] = [];
|
||||
|
||||
const trimmed = source.trim();
|
||||
const wrappedChunks = trimmed.split(/[ \t\r\n]+/);
|
||||
const wrapWidth = wrappedChunks[0]?.length ?? 0;
|
||||
if (allowImplicitWrapped
|
||||
&& wrappedChunks.length > 1
|
||||
&& wrapWidth >= 4
|
||||
&& wrapWidth <= 76
|
||||
&& wrapWidth % 4 === 0
|
||||
&& wrappedChunks.every(chunk => /^[A-Za-z0-9+/_-]+={0,2}$/.test(chunk))
|
||||
&& wrappedChunks.slice(0, -1).every(chunk => chunk.length === wrapWidth && !chunk.includes('='))
|
||||
&& wrappedChunks.at(-1)!.length <= wrapWidth) {
|
||||
const candidate = wrappedChunks.join('');
|
||||
if (candidate.length >= 24) {
|
||||
const start = source.length - source.trimStart().length;
|
||||
wrappedBlocks.push({ value: candidate, start, end: start + trimmed.length });
|
||||
}
|
||||
}
|
||||
|
||||
const directivePattern = /(?:\bdecode\b[^\r\n:]{0,160}\bbase64\b|\bbase64\b[^\r\n:]{0,160}\bdecode\b)[^\r\n:]{0,160}:/gi;
|
||||
for (const directive of source.matchAll(directivePattern)) {
|
||||
const tailStart = (directive.index ?? 0) + directive[0].length;
|
||||
const tail = source.slice(tailStart);
|
||||
const wrapped = tail.match(/^[ \t\r\n]*([A-Za-z0-9+/_=-]+(?:[ \t\r\n]+[A-Za-z0-9+/_=-]+)*)/);
|
||||
if (!wrapped) continue;
|
||||
const captured = wrapped[1];
|
||||
const start = tailStart + wrapped[0].indexOf(captured);
|
||||
let candidate = '';
|
||||
let end = start;
|
||||
for (const chunk of captured.matchAll(/[A-Za-z0-9+/_=-]+/g)) {
|
||||
candidate += chunk[0];
|
||||
end = start + (chunk.index ?? 0) + chunk[0].length;
|
||||
if (chunk[0].includes('=')) break;
|
||||
}
|
||||
if (candidate.length >= 24) wrappedBlocks.push({ value: candidate, start, end });
|
||||
}
|
||||
|
||||
const candidates: Base64Candidate[] = [];
|
||||
const contiguousPattern = new RegExp(BASE64_CANDIDATE_PATTERN.source, 'g');
|
||||
for (const match of source.matchAll(contiguousPattern)) {
|
||||
const value = match[1];
|
||||
const start = (match.index ?? 0) + match[0].length - value.length;
|
||||
const end = start + value.length;
|
||||
if (wrappedBlocks.some(block => start >= block.start && end <= block.end)) continue;
|
||||
candidates.push({ value, start, end });
|
||||
}
|
||||
candidates.push(...wrappedBlocks);
|
||||
candidates.sort((left, right) => left.start - right.start || left.end - right.end);
|
||||
return { candidates, complete: true };
|
||||
}
|
||||
|
||||
function decodeBase64Text(candidate: string): string | undefined {
|
||||
let normalized = candidate.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const remainder = normalized.length % 4;
|
||||
if (remainder === 1) return undefined;
|
||||
if (remainder > 0) normalized += '='.repeat(4 - remainder);
|
||||
|
||||
try {
|
||||
const binary = atob(normalized);
|
||||
const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
|
||||
const decoded = new TextDecoder('utf-8').decode(bytes);
|
||||
if (!decoded) return undefined;
|
||||
|
||||
let printable = 0;
|
||||
let total = 0;
|
||||
for (const char of decoded) {
|
||||
total++;
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
const isPrintable = char === '\n' || char === '\r' || char === '\t'
|
||||
|| (codePoint >= 0x20 && codePoint !== 0x7f);
|
||||
if (codePoint !== 0xfffd && isPrintable) {
|
||||
printable++;
|
||||
}
|
||||
}
|
||||
return total > 0 && printable / total >= 0.85 ? decoded : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function addProjection(projections: Set<string>, value: string): void {
|
||||
if (projections.has(value)) return;
|
||||
projections.add(value);
|
||||
const confusable = projectMixedScriptConfusables(value);
|
||||
if (confusable !== undefined) projections.add(confusable);
|
||||
}
|
||||
|
||||
const FORMAT_CHARACTER_PATTERN = /\p{Cf}/u;
|
||||
const DEFAULT_IGNORABLE_PATTERN = /\p{Default_Ignorable_Code_Point}/u;
|
||||
|
||||
function isHiddenSeparator(char: string): boolean {
|
||||
const codePoint = char.codePointAt(0) ?? 0;
|
||||
if (codePoint <= 0x9f) {
|
||||
return codePoint <= 0x08
|
||||
|| codePoint === 0x0b
|
||||
|| codePoint === 0x0c
|
||||
|| (codePoint >= 0x0e && codePoint <= 0x1f)
|
||||
|| codePoint >= 0x7f;
|
||||
}
|
||||
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return true;
|
||||
return FORMAT_CHARACTER_PATTERN.test(char) || DEFAULT_IGNORABLE_PATTERN.test(char);
|
||||
}
|
||||
|
||||
function replaceHiddenSeparators(value: string, replacement: string): string {
|
||||
let containsHidden = false;
|
||||
for (const char of value) {
|
||||
if (isHiddenSeparator(char)) {
|
||||
containsHidden = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!containsHidden) return value;
|
||||
|
||||
let projected = '';
|
||||
for (const char of value) projected += isHiddenSeparator(char) ? replacement : char;
|
||||
return projected;
|
||||
}
|
||||
|
||||
function projectDelimitedWords(value: string): string | undefined {
|
||||
const projected = value.replace(
|
||||
/(\p{L})([\p{P}\p{S}\p{White_Space}]+)(?=\p{L})/gu,
|
||||
(match, letter: string, separators: string) =>
|
||||
/[\p{P}\p{S}]/u.test(separators) ? `${letter} ` : match,
|
||||
);
|
||||
return projected === value ? undefined : projected;
|
||||
}
|
||||
|
||||
type HtmlTagBoundary =
|
||||
| { kind: 'close'; index: number }
|
||||
| { kind: 'nested'; index: number }
|
||||
| { kind: 'eof'; index: number };
|
||||
|
||||
function findHtmlTagBoundary(value: string, start: number): HtmlTagBoundary {
|
||||
let quote: '"' | "'" | undefined;
|
||||
for (let index = start; index < value.length; index++) {
|
||||
const char = value[index];
|
||||
if (char === '<') return { kind: 'nested', index };
|
||||
if (quote) {
|
||||
if (char === quote) quote = undefined;
|
||||
} else if (char === '"' || char === "'") {
|
||||
quote = char;
|
||||
} else if (char === '>') {
|
||||
return { kind: 'close', index };
|
||||
}
|
||||
}
|
||||
return { kind: 'eof', index: value.length };
|
||||
}
|
||||
|
||||
function normalizeHtmlToken(value: string): string {
|
||||
return value.replace(/[:_-]+/g, ' ');
|
||||
}
|
||||
|
||||
function extractHtmlAttributeTokens(value: string, start: number, end: number): string {
|
||||
const tokens: string[] = [];
|
||||
let index = start;
|
||||
while (index < end) {
|
||||
while (index < end && /[\s/]/.test(value[index])) index++;
|
||||
const nameStart = index;
|
||||
while (index < end && !/[\s=/>]/.test(value[index])) index++;
|
||||
if (index === nameStart) {
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
const name = normalizeHtmlToken(value.slice(nameStart, index));
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
if (value[index] !== '=') {
|
||||
if (name) tokens.push(name);
|
||||
continue;
|
||||
}
|
||||
index++;
|
||||
while (index < end && /\s/.test(value[index])) index++;
|
||||
|
||||
const quote = value[index] === '"' || value[index] === "'"
|
||||
? value[index]
|
||||
: undefined;
|
||||
if (quote) index++;
|
||||
const valueStart = index;
|
||||
if (quote) {
|
||||
while (index < end && value[index] !== quote) index++;
|
||||
} else {
|
||||
while (index < end && !/[\s>]/.test(value[index])) index++;
|
||||
}
|
||||
if (index > valueStart) tokens.push(normalizeHtmlToken(value.slice(valueStart, index)));
|
||||
if (quote && index < end) index++;
|
||||
}
|
||||
return tokens.join(' ');
|
||||
}
|
||||
|
||||
function stripHtmlMarkup(value: string): {
|
||||
rendered: string;
|
||||
tagNames: string;
|
||||
attributes: string;
|
||||
lexical: string;
|
||||
} {
|
||||
if (!value.includes('<')) {
|
||||
return { rendered: value, tagNames: value, attributes: value, lexical: value };
|
||||
}
|
||||
|
||||
const rendered: string[] = [];
|
||||
const tagNames: string[] = [];
|
||||
const attributes: string[] = [];
|
||||
const lexical: string[] = [];
|
||||
for (let index = 0; index < value.length;) {
|
||||
if (value.startsWith('<!--', index)) {
|
||||
const commentEnd = value.indexOf('-->', index + 4);
|
||||
if (commentEnd === -1) {
|
||||
const visibleTail = value.slice(index + 4);
|
||||
rendered.push(visibleTail);
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
lexical.push(visibleTail);
|
||||
break;
|
||||
}
|
||||
const commentText = value.slice(index + 4, commentEnd);
|
||||
if (commentText) {
|
||||
tagNames.push(' ', commentText, ' ');
|
||||
attributes.push(' ', commentText, ' ');
|
||||
lexical.push(' ', commentText, ' ');
|
||||
}
|
||||
index = commentEnd + 3;
|
||||
continue;
|
||||
}
|
||||
if (value.startsWith('-->', index)) {
|
||||
index += 3;
|
||||
continue;
|
||||
}
|
||||
if (value[index] === '<' && /[!/A-Za-z?]/.test(value[index + 1] ?? '')) {
|
||||
let tagNameEnd = index + 1;
|
||||
if (value[tagNameEnd] === '/') tagNameEnd++;
|
||||
const tagNameStart = tagNameEnd;
|
||||
while (/[A-Za-z0-9:!_-]/.test(value[tagNameEnd] ?? '')) tagNameEnd++;
|
||||
const tagName = normalizeHtmlToken(value.slice(tagNameStart, tagNameEnd));
|
||||
const boundary = findHtmlTagBoundary(value, tagNameEnd);
|
||||
if (boundary.kind === 'close') {
|
||||
const attributeTokens = extractHtmlAttributeTokens(
|
||||
value,
|
||||
tagNameEnd,
|
||||
boundary.index,
|
||||
);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
if (attributeTokens) attributes.push(' ', attributeTokens, ' ');
|
||||
if (tagName || attributeTokens) {
|
||||
lexical.push(' ', tagName, ' ', attributeTokens, ' ');
|
||||
}
|
||||
index = boundary.index + 1;
|
||||
continue;
|
||||
}
|
||||
const visibleTail = value.slice(tagNameEnd, boundary.index);
|
||||
rendered.push(visibleTail);
|
||||
if (tagName) tagNames.push(' ', tagName, ' ');
|
||||
tagNames.push(visibleTail);
|
||||
attributes.push(visibleTail);
|
||||
if (tagName) lexical.push(' ', tagName, ' ');
|
||||
lexical.push(visibleTail);
|
||||
if (boundary.kind === 'eof') break;
|
||||
index = boundary.index;
|
||||
continue;
|
||||
}
|
||||
rendered.push(value[index]);
|
||||
tagNames.push(value[index]);
|
||||
attributes.push(value[index]);
|
||||
lexical.push(value[index]);
|
||||
index++;
|
||||
}
|
||||
return {
|
||||
rendered: rendered.join(''),
|
||||
tagNames: tagNames.join(''),
|
||||
attributes: attributes.join(''),
|
||||
lexical: lexical.join(''),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizedIngressProjections(value: string): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64: boolean): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
};
|
||||
function normalizedIngressProjections(value: string, includeBase64 = true): {
|
||||
projections: string[];
|
||||
complete: boolean;
|
||||
} {
|
||||
const maxPasses = 64;
|
||||
const maxWork = MAX_EXTERNAL_MEMORY_INGRESS_CHARS + 1;
|
||||
const originalProjection = value.normalize('NFKC');
|
||||
const normalizationStages = new Set([originalProjection]);
|
||||
let decodedProjection = originalProjection;
|
||||
let complete = false;
|
||||
let work = 0;
|
||||
for (let pass = 0; pass < maxPasses; pass++) {
|
||||
work += decodedProjection.length;
|
||||
if (work > maxWork) break;
|
||||
const decodedText = decodeUnicodeEscapes(decodeHexEscapes(
|
||||
decodeHtmlEntities(decodePercentEncoding(decodedProjection)),
|
||||
));
|
||||
const decoded = decodedText === decodedProjection
|
||||
? decodedProjection
|
||||
: decodedText.normalize('NFKC');
|
||||
if (decoded === decodedProjection) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
normalizationStages.add(decoded);
|
||||
decodedProjection = decoded;
|
||||
}
|
||||
if (!complete) return { projections: [decodedProjection], complete: false };
|
||||
|
||||
const htmlProjections = stripHtmlMarkup(decodedProjection);
|
||||
const projections = new Set<string>();
|
||||
for (const projection of normalizationStages) addProjection(projections, projection);
|
||||
for (const projection of new Set([
|
||||
htmlProjections.rendered,
|
||||
htmlProjections.tagNames,
|
||||
htmlProjections.attributes,
|
||||
htmlProjections.lexical,
|
||||
])) {
|
||||
let unformatted = projection;
|
||||
if (unformatted.includes('[')) {
|
||||
unformatted = unformatted
|
||||
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1');
|
||||
}
|
||||
if (/[*_~`]/.test(unformatted)) unformatted = unformatted.replace(/[*_~`]/g, '');
|
||||
const compact = replaceHiddenSeparators(unformatted, '');
|
||||
addProjection(projections, compact);
|
||||
if (compact !== unformatted) {
|
||||
addProjection(projections, replaceHiddenSeparators(unformatted, ' '));
|
||||
}
|
||||
}
|
||||
|
||||
if (!includeBase64) return { projections: [...projections], complete: true };
|
||||
|
||||
const candidateLineages = new Map<number, Map<string, number[]>>();
|
||||
let nextCandidateLineage = 1;
|
||||
const resolveCandidateLineage = (
|
||||
parentLineage: number,
|
||||
candidate: string,
|
||||
ordinal: number,
|
||||
): { lineage: number; created: boolean } => {
|
||||
let candidates = candidateLineages.get(parentLineage);
|
||||
if (!candidates) {
|
||||
candidates = new Map();
|
||||
candidateLineages.set(parentLineage, candidates);
|
||||
}
|
||||
let lineages = candidates.get(candidate);
|
||||
if (!lineages) {
|
||||
lineages = [];
|
||||
candidates.set(candidate, lineages);
|
||||
}
|
||||
const existing = lineages[ordinal];
|
||||
if (existing !== undefined) return { lineage: existing, created: false };
|
||||
const lineage = nextCandidateLineage++;
|
||||
lineages[ordinal] = lineage;
|
||||
return { lineage, created: true };
|
||||
};
|
||||
let decodedCandidateCount = 0;
|
||||
let decodedCandidateChars = 0;
|
||||
let base64Sources = [...projections].map(value => ({ value, lineage: 0 }));
|
||||
let reachedDepthLimit = true;
|
||||
for (let depth = 0; depth < MAX_BASE64_DEPTH; depth++) {
|
||||
const decodedValues: Array<{ value: string; lineage: number }> = [];
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, depth > 0);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceLineage = resolveCandidateLineage(
|
||||
source.lineage,
|
||||
candidate,
|
||||
ordinal,
|
||||
);
|
||||
if (!occurrenceLineage.created) continue;
|
||||
if (candidate.length > MAX_BASE64_CANDIDATE_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
const decoded = decodeBase64Text(candidate);
|
||||
if (decoded === undefined) continue;
|
||||
decodedCandidateCount++;
|
||||
decodedCandidateChars += candidate.length;
|
||||
if (decodedCandidateCount > MAX_BASE64_CANDIDATES
|
||||
|| decodedCandidateChars > MAX_BASE64_TOTAL_CHARS) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
decodedValues.push({ value: decoded, lineage: occurrenceLineage.lineage });
|
||||
}
|
||||
}
|
||||
if (decodedValues.length === 0) {
|
||||
reachedDepthLimit = false;
|
||||
break;
|
||||
}
|
||||
|
||||
const nextSources: Array<{ value: string; lineage: number }> = [];
|
||||
for (const decoded of decodedValues) {
|
||||
const nested = normalizedIngressProjections(decoded.value, false);
|
||||
if (!nested.complete) return { projections: [...projections], complete: false };
|
||||
for (const projection of nested.projections) {
|
||||
addProjection(projections, projection);
|
||||
nextSources.push({ value: projection, lineage: decoded.lineage });
|
||||
}
|
||||
}
|
||||
base64Sources = nextSources;
|
||||
}
|
||||
|
||||
if (reachedDepthLimit && base64Sources.length > 0) {
|
||||
for (const source of base64Sources) {
|
||||
const collected = collectBase64Candidates(source.value, true);
|
||||
if (!collected.complete) return { projections: [...projections], complete: false };
|
||||
const occurrenceCounts = new Map<string, number>();
|
||||
for (const occurrence of collected.candidates) {
|
||||
const candidate = occurrence.value;
|
||||
const ordinal = occurrenceCounts.get(candidate) ?? 0;
|
||||
occurrenceCounts.set(candidate, ordinal + 1);
|
||||
const occurrenceSeen = candidateLineages
|
||||
.get(source.lineage)
|
||||
?.get(candidate)?.[ordinal] !== undefined;
|
||||
if (!occurrenceSeen
|
||||
&& decodeBase64Text(candidate) !== undefined) {
|
||||
return { projections: [...projections], complete: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { projections: [...projections], complete: true };
|
||||
}
|
||||
|
||||
/** Evaluate untrusted content before it can enter persistent memory. */
|
||||
export function evaluateExternalMemoryIngress(
|
||||
input: ExternalMemoryIngressInput,
|
||||
): ExternalMemoryIngressDecision {
|
||||
const title = input.title ?? '';
|
||||
if (title.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS
|
||||
|| input.content.length > MAX_EXTERNAL_MEMORY_INGRESS_CHARS - title.length) {
|
||||
return {
|
||||
action: 'block',
|
||||
reason: 'prompt_injection',
|
||||
scan: { safe: false, score: 0.6, flags: ['normalization_limit'] },
|
||||
};
|
||||
}
|
||||
|
||||
const projection = `${title}\n${input.content}`;
|
||||
let scan = scanForInjection(projection, 'tool_output');
|
||||
if (scan.safe) {
|
||||
const normalizedIngress = normalizedIngressProjections(projection);
|
||||
for (const normalized of normalizedIngress.projections) {
|
||||
if (normalized !== projection) {
|
||||
const normalizedScan = scanForInjection(normalized, 'tool_output');
|
||||
if (!normalizedScan.safe) {
|
||||
scan = normalizedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const delimited = projectDelimitedWords(normalized);
|
||||
if (delimited !== undefined) {
|
||||
const delimitedScan = scanForInjection(delimited, 'tool_output');
|
||||
if (!delimitedScan.safe) {
|
||||
scan = delimitedScan;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (scan.safe && !normalizedIngress.complete) {
|
||||
scan = { safe: false, score: 0.6, flags: ['normalization_limit'] };
|
||||
}
|
||||
}
|
||||
|
||||
return scan.safe
|
||||
? { action: 'allow', scan }
|
||||
: { action: 'block', reason: 'prompt_injection', scan };
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type { WaggleDB } from './waggle-db.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -37,9 +37,9 @@ export function computeSkillHash(content: string): string {
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class SkillHashStore {
|
||||
private db: MindDB;
|
||||
private db: WaggleDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
constructor(db: WaggleDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
*/
|
||||
|
||||
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
import { createCoreLogger } from './logger.js';
|
||||
const log = createCoreLogger('team-sync');
|
||||
|
||||
export interface TeamSyncConfig {
|
||||
|
||||
@@ -11,7 +11,7 @@ import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
import { createCoreLogger } from './logger.js';
|
||||
|
||||
const log = createCoreLogger('vault');
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
|
||||
161
packages/core/src/waggle-db-import.ts
Normal file
161
packages/core/src/waggle-db-import.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* One-time copy of Waggle's own tables from the legacy memory databases
|
||||
* (personal.mind, workspace.mind) into waggle.db.
|
||||
*
|
||||
* The legacy files are only read: the data directory may be shared with the
|
||||
* commercial edition, which keeps using them. Each source is copied once,
|
||||
* recorded in waggle_meta, and a table is copied only while its target is
|
||||
* empty, so a re-run never duplicates rows.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { AwarenessLayer } from './awareness.js';
|
||||
import { CronStore } from './cron-store.js';
|
||||
import { ExecutionTraceStore } from './execution-traces.js';
|
||||
import { IdentityLayer } from './identity.js';
|
||||
import { SkillHashStore } from './skill-hashes.js';
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
|
||||
/** Tables that only ever lived in personal.mind, copied with their ids. */
|
||||
const PERSONAL_TABLES = [
|
||||
'cron_schedules',
|
||||
'cron_execution_history',
|
||||
'cron_run_leases',
|
||||
'notifications',
|
||||
'pending_actions',
|
||||
'skill_hashes',
|
||||
'feedback_entries',
|
||||
'execution_traces',
|
||||
'execution_trace_spend',
|
||||
'execution_trace_spend_reservations',
|
||||
] as const;
|
||||
|
||||
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
|
||||
const AWARENESS_COLUMNS = ['category', 'content', 'priority', 'metadata', 'created_at', 'expires_at'];
|
||||
|
||||
export interface LegacyMindSource {
|
||||
/** Path to a legacy .mind file. */
|
||||
path: string;
|
||||
scope: DataScope;
|
||||
}
|
||||
|
||||
export interface LegacyImportReport {
|
||||
scope: DataScope;
|
||||
status: 'imported' | 'already-imported' | 'missing' | 'failed';
|
||||
copied: Record<string, number>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function importKey(scope: DataScope): string {
|
||||
return `import:${scope}`;
|
||||
}
|
||||
|
||||
function tableExists(db: DatabaseType, table: string): boolean {
|
||||
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== undefined;
|
||||
}
|
||||
|
||||
function columnsOf(db: DatabaseType, table: string): string[] {
|
||||
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(c => c.name);
|
||||
}
|
||||
|
||||
function copyRows(
|
||||
source: DatabaseType,
|
||||
target: DatabaseType,
|
||||
sourceTable: string,
|
||||
targetTable: string,
|
||||
columns: string[],
|
||||
extra: Record<string, unknown> = {},
|
||||
): number {
|
||||
const rows = source.prepare(`SELECT ${columns.join(', ')} FROM ${sourceTable}`).all() as Array<Record<string, unknown>>;
|
||||
if (rows.length === 0) return 0;
|
||||
const allColumns = [...columns, ...Object.keys(extra)];
|
||||
const insert = target.prepare(
|
||||
`INSERT INTO ${targetTable} (${allColumns.join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
insert.run(...columns.map(c => row[c]), ...Object.values(extra));
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
function copyPersonalTables(source: DatabaseType, target: DatabaseType, copied: Record<string, number>): void {
|
||||
for (const table of PERSONAL_TABLES) {
|
||||
if (!tableExists(source, table)) continue;
|
||||
if (!tableExists(target, table)) {
|
||||
// Lazily created tables (feedback_entries) take the legacy definition.
|
||||
const ddl = source.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { sql: string };
|
||||
target.exec(ddl.sql);
|
||||
}
|
||||
const empty = target.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() === undefined;
|
||||
if (!empty) continue;
|
||||
const targetColumns = new Set(columnsOf(target, table));
|
||||
const shared = columnsOf(source, table).filter(c => targetColumns.has(c));
|
||||
copied[table] = copyRows(source, target, table, table, shared);
|
||||
}
|
||||
}
|
||||
|
||||
function copyScopedTables(source: DatabaseType, target: DatabaseType, scope: DataScope, copied: Record<string, number>): void {
|
||||
if (tableExists(source, 'identity')) {
|
||||
const hasProfile = target.prepare('SELECT 1 FROM profiles WHERE scope = ?').get(scope) !== undefined;
|
||||
if (!hasProfile) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'identity'));
|
||||
const shared = PROFILE_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.profiles = copyRows(source, target, 'identity', 'profiles', shared, { scope });
|
||||
}
|
||||
}
|
||||
if (tableExists(source, 'awareness')) {
|
||||
const hasItems = target.prepare('SELECT 1 FROM awareness_items WHERE scope = ? LIMIT 1').get(scope) !== undefined;
|
||||
if (!hasItems) {
|
||||
const sourceColumns = new Set(columnsOf(source, 'awareness'));
|
||||
const shared = AWARENESS_COLUMNS.filter(c => sourceColumns.has(c));
|
||||
copied.awareness_items = copyRows(source, target, 'awareness', 'awareness_items', shared, { scope });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies Waggle data from each legacy source into `waggleDb`, once per scope.
|
||||
* The personal source also brings the app-wide tables; workspace sources bring
|
||||
* their profile and awareness items.
|
||||
*/
|
||||
export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSource[]): LegacyImportReport[] {
|
||||
// Create every target table with its current schema first.
|
||||
new CronStore(waggleDb);
|
||||
new SkillHashStore(waggleDb);
|
||||
new ExecutionTraceStore(waggleDb);
|
||||
new IdentityLayer(waggleDb);
|
||||
new AwarenessLayer(waggleDb);
|
||||
|
||||
const target = waggleDb.getDatabase();
|
||||
const reports: LegacyImportReport[] = [];
|
||||
for (const { path: sourcePath, scope } of sources) {
|
||||
const copied: Record<string, number> = {};
|
||||
if (waggleDb.getMeta(importKey(scope))) {
|
||||
reports.push({ scope, status: 'already-imported', copied });
|
||||
continue;
|
||||
}
|
||||
const stat = fs.statSync(sourcePath, { throwIfNoEntry: false });
|
||||
if (!stat || stat.size === 0) {
|
||||
reports.push({ scope, status: 'missing', copied });
|
||||
continue;
|
||||
}
|
||||
let source: DatabaseType | null = null;
|
||||
try {
|
||||
source = new Database(sourcePath, { fileMustExist: true });
|
||||
const db = source;
|
||||
target.transaction(() => {
|
||||
if (scope === 'personal') copyPersonalTables(db, target, copied);
|
||||
copyScopedTables(db, target, scope, copied);
|
||||
})();
|
||||
waggleDb.setMeta(importKey(scope), new Date().toISOString());
|
||||
reports.push({ scope, status: 'imported', copied });
|
||||
} catch (err) {
|
||||
reports.push({ scope, status: 'failed', copied: {}, error: err instanceof Error ? err.message : String(err) });
|
||||
} finally {
|
||||
source?.close();
|
||||
}
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
78
packages/core/src/waggle-db.ts
Normal file
78
packages/core/src/waggle-db.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* waggle.db — Waggle's own data, kept apart from memory (ADR-0007).
|
||||
*
|
||||
* Holds what the app needs to run: automations, notifications and held
|
||||
* actions, skill hashes, thumbs feedback, cost tracking (execution traces and
|
||||
* budget reservations), the user profile card and awareness items. Facts the
|
||||
* agent learns about the user live in the memory store, not here.
|
||||
*
|
||||
* Stores that belong here take a `WaggleDB`, not a generic SQLite handle, so
|
||||
* passing a memory database by mistake is a type error.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
|
||||
export const WAGGLE_DB_FILE = 'waggle.db';
|
||||
|
||||
const BUSY_TIMEOUT_MS = 10_000;
|
||||
|
||||
const META_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS waggle_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)`;
|
||||
|
||||
export class WaggleDB {
|
||||
private readonly db: DatabaseType;
|
||||
|
||||
/** Opens (and creates if needed) the database at `dbPath`; `':memory:'` is accepted. */
|
||||
constructor(readonly dbPath: string) {
|
||||
if (dbPath !== ':memory:') {
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
||||
}
|
||||
this.db = new Database(dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
this.db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
||||
this.db.exec(META_TABLE_SQL);
|
||||
}
|
||||
|
||||
/** Opens `<dataDir>/waggle.db`. */
|
||||
static openInDataDir(dataDir: string): WaggleDB {
|
||||
return new WaggleDB(path.join(dataDir, WAGGLE_DB_FILE));
|
||||
}
|
||||
|
||||
getDatabase(): DatabaseType {
|
||||
return this.db;
|
||||
}
|
||||
|
||||
getMeta(key: string): string | null {
|
||||
const row = this.db.prepare('SELECT value FROM waggle_meta WHERE key = ?').get(key) as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
setMeta(key: string, value: string): void {
|
||||
this.db
|
||||
.prepare('INSERT INTO waggle_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
|
||||
.run(key, value);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.db.open) this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Data scope of a profile or awareness item: the personal mind or one workspace. */
|
||||
export type DataScope = 'personal' | `workspace:${string}`;
|
||||
|
||||
export const PERSONAL_DATA_SCOPE: DataScope = 'personal';
|
||||
|
||||
export function workspaceDataScope(workspaceId: string): DataScope {
|
||||
if (!workspaceId.trim()) throw new TypeError('workspaceId must be a non-empty string');
|
||||
return `workspace:${workspaceId}`;
|
||||
}
|
||||
521
packages/core/src/workspace-manager.ts
Normal file
521
packages/core/src/workspace-manager.ts
Normal file
@@ -0,0 +1,521 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { WorkspaceType } from '@waggle/shared';
|
||||
type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
|
||||
|
||||
export interface WorkspaceConfig {
|
||||
id: string;
|
||||
name: string;
|
||||
group: string;
|
||||
icon?: string;
|
||||
model?: string;
|
||||
personality?: string;
|
||||
/** Selected agent persona ID (from persona catalog) */
|
||||
personaId?: string;
|
||||
/** Template ID chosen during onboarding (e.g. 'sales-pipeline', 'research-project'). */
|
||||
templateId?: string;
|
||||
tools?: string[];
|
||||
skills?: string[];
|
||||
team?: string | null;
|
||||
/** Filesystem directory where agent operates and generates files. */
|
||||
directory?: string;
|
||||
/** Storage type for workspace files. */
|
||||
storageType?: 'virtual' | 'local' | 'team';
|
||||
/** Path to local file storage for this workspace. */
|
||||
storagePath?: string;
|
||||
/** Extra storage provider configuration (e.g. S3 bucket, credentials). */
|
||||
storageConfig?: Record<string, unknown>;
|
||||
created: string; // ISO 8601
|
||||
|
||||
// --- Team Mode fields (Phase 5) ---
|
||||
/** Team ID on the team server. Present = team workspace. */
|
||||
teamId?: string;
|
||||
/** URL of the team server (e.g. "https://team.waggle.dev"). */
|
||||
teamServerUrl?: string;
|
||||
/** Current user's role in this team workspace. */
|
||||
teamRole?: 'owner' | 'admin' | 'member' | 'viewer';
|
||||
/** Current user's ID on the team server. */
|
||||
teamUserId?: string;
|
||||
|
||||
// --- Budget ---
|
||||
/** Monthly cost budget in USD. null = unlimited. */
|
||||
budget?: number | null;
|
||||
|
||||
// --- Tone/Voice (Wave 7.3) ---
|
||||
/** Workspace communication tone preset. */
|
||||
tone?: 'professional' | 'casual' | 'technical' | 'legal' | 'marketing';
|
||||
|
||||
// --- Optimization fields (GEPA/Ax) ---
|
||||
/** Enable GEPA prompt optimization for this workspace (opt-in, default false). */
|
||||
optimizationEnabled?: boolean;
|
||||
/** Daily optimization budget in cents (default 100 = $1/day). Only used when optimizationEnabled is true. */
|
||||
optimizationBudget?: number;
|
||||
|
||||
// --- AI Act compliance (L-17 C2) ---
|
||||
/** EU AI Act risk classification for this workspace. */
|
||||
riskLevel?: AIActRiskLevel;
|
||||
/** ISO timestamp of the last risk classification change. Auto-stamped by WorkspaceManager. */
|
||||
riskClassifiedAt?: string;
|
||||
|
||||
// --- UX-Refactor V2 fields (PRD §15.3; additive + optional for back-compat) ---
|
||||
/** Free-text description shown in the workspace header/cards. */
|
||||
description?: string;
|
||||
/** Workspace classification. Defaults derivable from templateId/group when absent. */
|
||||
type?: WorkspaceType;
|
||||
/** Lifecycle status. Treated as 'active' when absent. */
|
||||
status?: 'active' | 'paused' | 'archived';
|
||||
/** Agents bound to this workspace (ids). */
|
||||
agentIds?: string[];
|
||||
/** Connectors scoped to this workspace (ids). */
|
||||
connectorIds?: string[];
|
||||
/** MCP servers scoped to this workspace (ids). */
|
||||
mcpIds?: string[];
|
||||
/** ISO timestamp of the last config update. Auto-stamped by update(). */
|
||||
updatedAt?: string;
|
||||
/** ISO timestamp of the last activity (chat/agent run) in this workspace. */
|
||||
lastActiveAt?: string;
|
||||
}
|
||||
|
||||
export interface CreateWorkspaceOptions {
|
||||
name: string;
|
||||
group: string;
|
||||
icon?: string;
|
||||
model?: string;
|
||||
personality?: string;
|
||||
/** Selected agent persona ID (from persona catalog) */
|
||||
personaId?: string;
|
||||
/** Template ID chosen during onboarding (e.g. 'sales-pipeline', 'research-project'). */
|
||||
templateId?: string;
|
||||
tools?: string[];
|
||||
skills?: string[];
|
||||
team?: string | null;
|
||||
/** Filesystem directory where agent operates and generates files. */
|
||||
directory?: string;
|
||||
|
||||
// --- Team Mode fields (Phase 5) ---
|
||||
teamId?: string;
|
||||
teamServerUrl?: string;
|
||||
teamRole?: 'owner' | 'admin' | 'member' | 'viewer';
|
||||
teamUserId?: string;
|
||||
|
||||
// --- Tone/Voice (Wave 7.3) ---
|
||||
tone?: 'professional' | 'casual' | 'technical' | 'legal' | 'marketing';
|
||||
|
||||
// --- Budget ---
|
||||
budget?: number | null;
|
||||
|
||||
// --- AI Act compliance (L-17 C2) ---
|
||||
/** Initial risk level (usually derived from template). */
|
||||
riskLevel?: AIActRiskLevel;
|
||||
|
||||
// --- Optimization fields (GEPA/Ax) ---
|
||||
optimizationEnabled?: boolean;
|
||||
optimizationBudget?: number;
|
||||
|
||||
// --- UX-Refactor V2 fields (PRD §15.3) ---
|
||||
/** Free-text description shown in the workspace header/cards. */
|
||||
description?: string;
|
||||
/** Workspace classification (defaults derivable from templateId/group). */
|
||||
type?: WorkspaceType;
|
||||
}
|
||||
|
||||
interface WorkspacesMeta {
|
||||
defaultWorkspace?: string | null;
|
||||
}
|
||||
|
||||
const WORKSPACE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,199}$/;
|
||||
|
||||
function isContained(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative === '' || (!path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`));
|
||||
}
|
||||
|
||||
/**
|
||||
* WorkspaceManager manages workspace CRUD, groups, and directory structure.
|
||||
* Each workspace lives under {baseDir}/workspaces/{id}/ with:
|
||||
* - workspace.json (config)
|
||||
* - workspace.mind (SQLite .mind file, created empty)
|
||||
* - sessions/ (JSONL session logs)
|
||||
*/
|
||||
export class WorkspaceManager {
|
||||
private readonly workspacesDir: string;
|
||||
private readonly canonicalWorkspacesDir: string;
|
||||
private readonly metaPath: string;
|
||||
|
||||
constructor(private readonly baseDir: string) {
|
||||
this.workspacesDir = path.join(baseDir, 'workspaces');
|
||||
this.metaPath = path.join(baseDir, 'workspaces-meta.json');
|
||||
|
||||
if (!fs.existsSync(this.workspacesDir)) {
|
||||
fs.mkdirSync(this.workspacesDir, { recursive: true });
|
||||
}
|
||||
const rootStat = fs.lstatSync(this.workspacesDir);
|
||||
const canonicalBase = fs.realpathSync.native(baseDir);
|
||||
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
|
||||
if (
|
||||
rootStat.isSymbolicLink()
|
||||
|| !rootStat.isDirectory()
|
||||
|| canonicalRoot === canonicalBase
|
||||
|| !isContained(canonicalBase, canonicalRoot)
|
||||
) {
|
||||
throw new Error('Workspace root must be a regular directory inside the data directory');
|
||||
}
|
||||
this.canonicalWorkspacesDir = canonicalRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new workspace with directory structure and config.
|
||||
*/
|
||||
create(options: CreateWorkspaceOptions): WorkspaceConfig {
|
||||
return this.createWithId(this.generateId(options.name), options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a workspace with the given id exists. Idempotent — returns the
|
||||
* existing config unchanged if the workspace already exists; otherwise
|
||||
* creates it with the supplied id (bypassing slug-collision handling in
|
||||
* generateId, since callers construct ids from trusted internal state
|
||||
* like CWD-derived prefixes — e.g. SessionStart hooks).
|
||||
*
|
||||
* Use this from auto-attach paths (e.g. save_memory with a workspace arg
|
||||
* that names a workspace not yet created on disk). Direct-create flows
|
||||
* with user-supplied names should still go through `create()` so the
|
||||
* generateId collision logic runs.
|
||||
*/
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R4, 2026-06-11).
|
||||
ensure(id: string, options: Partial<CreateWorkspaceOptions> = {}): WorkspaceConfig {
|
||||
this.assertWorkspaceId(id);
|
||||
const existing = this.get(id);
|
||||
if (existing) return existing;
|
||||
const workspacePath = path.join(this.resolveWorkspaceRoot(), id);
|
||||
const workspaceStat = fs.lstatSync(workspacePath, { throwIfNoEntry: false });
|
||||
if (workspaceStat) {
|
||||
if (id !== 'default' || !this.isEmptyLegacyWorkspaceDirectory(workspacePath, workspaceStat)) {
|
||||
throw new Error(`Workspace path already exists but is not a valid workspace: ${id}`);
|
||||
}
|
||||
const sessionsPath = path.join(workspacePath, 'sessions');
|
||||
if (fs.lstatSync(sessionsPath, { throwIfNoEntry: false })) fs.rmdirSync(sessionsPath);
|
||||
fs.rmdirSync(workspacePath);
|
||||
}
|
||||
|
||||
return this.createWithId(id, {
|
||||
...options,
|
||||
name: options.name ?? id,
|
||||
group: options.group ?? 'auto',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared create path: write directory structure + config for an exact id.
|
||||
*/
|
||||
private createWithId(id: string, options: CreateWorkspaceOptions): WorkspaceConfig {
|
||||
this.assertWorkspaceId(id);
|
||||
const canonicalRoot = this.resolveWorkspaceRoot();
|
||||
const wsDir = path.join(canonicalRoot, id);
|
||||
|
||||
fs.mkdirSync(wsDir);
|
||||
const canonicalWorkspace = fs.realpathSync.native(wsDir);
|
||||
if (!isContained(canonicalRoot, canonicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
fs.mkdirSync(path.join(canonicalWorkspace, 'sessions'));
|
||||
|
||||
// Touch workspace.mind — MindDB will init schema when first opened
|
||||
fs.writeFileSync(path.join(canonicalWorkspace, 'workspace.mind'), '', { flag: 'wx' });
|
||||
|
||||
const config: WorkspaceConfig = {
|
||||
id,
|
||||
name: options.name,
|
||||
group: options.group,
|
||||
...(options.icon !== undefined && { icon: options.icon }),
|
||||
...(options.model !== undefined && { model: options.model }),
|
||||
...(options.personality !== undefined && { personality: options.personality }),
|
||||
...(options.personaId !== undefined && { personaId: options.personaId }),
|
||||
...(options.templateId !== undefined && { templateId: options.templateId }),
|
||||
...(options.tools !== undefined && { tools: options.tools }),
|
||||
...(options.skills !== undefined && { skills: options.skills }),
|
||||
...(options.team !== undefined && { team: options.team }),
|
||||
...(options.directory !== undefined && { directory: options.directory }),
|
||||
...(options.teamId !== undefined && { teamId: options.teamId }),
|
||||
...(options.teamServerUrl !== undefined && { teamServerUrl: options.teamServerUrl }),
|
||||
...(options.teamRole !== undefined && { teamRole: options.teamRole }),
|
||||
...(options.teamUserId !== undefined && { teamUserId: options.teamUserId }),
|
||||
...(options.tone !== undefined && { tone: options.tone }),
|
||||
...(options.optimizationEnabled !== undefined && { optimizationEnabled: options.optimizationEnabled }),
|
||||
...(options.optimizationBudget !== undefined && { optimizationBudget: options.optimizationBudget }),
|
||||
...(options.riskLevel !== undefined && {
|
||||
riskLevel: options.riskLevel,
|
||||
riskClassifiedAt: new Date().toISOString(),
|
||||
}),
|
||||
created: new Date().toISOString(),
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(canonicalWorkspace, 'workspace.json'),
|
||||
JSON.stringify(config, null, 2),
|
||||
{ encoding: 'utf-8', flag: 'wx' }
|
||||
);
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all workspaces by reading workspace.json from each subdirectory.
|
||||
*/
|
||||
list(): WorkspaceConfig[] {
|
||||
const root = this.resolveWorkspaceRoot();
|
||||
const entries = fs.readdirSync(root, { withFileTypes: true });
|
||||
const configs: WorkspaceConfig[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !WORKSPACE_ID.test(entry.name)) continue;
|
||||
const config = this.get(entry.name);
|
||||
if (config) configs.push(config);
|
||||
}
|
||||
|
||||
return configs;
|
||||
}
|
||||
|
||||
/**
|
||||
* List workspaces filtered by group name.
|
||||
*/
|
||||
listByGroup(group: string): WorkspaceConfig[] {
|
||||
return this.list().filter(ws => ws.group === group);
|
||||
}
|
||||
|
||||
/**
|
||||
* List all unique group names.
|
||||
*/
|
||||
listGroups(): string[] {
|
||||
const groups = new Set(this.list().map(ws => ws.group));
|
||||
return [...groups];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a workspace by ID. Returns null if not found.
|
||||
*/
|
||||
get(id: string): WorkspaceConfig | null {
|
||||
if (!WORKSPACE_ID.test(id)) return null;
|
||||
|
||||
try {
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) return null;
|
||||
const configPath = path.join(workspaceDir, 'workspace.json');
|
||||
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
|
||||
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) return null;
|
||||
const canonicalConfig = fs.realpathSync.native(configPath);
|
||||
if (!isContained(workspaceDir, canonicalConfig)) return null;
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
|
||||
return config.id === id ? config : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Partially update a workspace config.
|
||||
* When `riskLevel` changes, `riskClassifiedAt` is auto-stamped with the
|
||||
* current ISO timestamp (EU AI Act Art. 14 provenance requirement).
|
||||
*/
|
||||
update(id: string, updates: Partial<Omit<WorkspaceConfig, 'id' | 'created'>>): void {
|
||||
const existing = this.get(id);
|
||||
if (!existing) throw new Error(`Workspace not found: ${id}`);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const stamped: Partial<WorkspaceConfig> =
|
||||
'riskLevel' in updates && updates.riskLevel !== existing.riskLevel
|
||||
? { ...updates, riskClassifiedAt: now, updatedAt: now }
|
||||
: { ...updates, updatedAt: now };
|
||||
|
||||
const updated = { ...existing, ...stamped };
|
||||
const configPath = path.join(this.workspacesDir, id, 'workspace.json');
|
||||
fs.writeFileSync(configPath, JSON.stringify(updated, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a workspace by removing its entire directory.
|
||||
*/
|
||||
delete(id: string): void {
|
||||
this.assertWorkspaceId(id);
|
||||
if (!this.get(id)) return;
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) return;
|
||||
fs.rmSync(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a workspace is team-connected (has a teamId).
|
||||
*/
|
||||
isTeamWorkspace(id: string): boolean {
|
||||
const ws = this.get(id);
|
||||
return ws !== null && typeof ws.teamId === 'string' && ws.teamId.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* List only team-connected workspaces.
|
||||
*/
|
||||
listTeamWorkspaces(): WorkspaceConfig[] {
|
||||
return this.list().filter(ws => typeof ws.teamId === 'string' && ws.teamId.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to a workspace's .mind file.
|
||||
*/
|
||||
getMindPath(id: string): string {
|
||||
this.assertWorkspaceId(id);
|
||||
const lexicalMindPath = path.join(this.workspacesDir, id, 'workspace.mind');
|
||||
const workspaceDir = this.resolveWorkspaceDir(id);
|
||||
if (!workspaceDir) throw new Error(`Workspace not found: ${id}`);
|
||||
|
||||
const configPath = path.join(workspaceDir, 'workspace.json');
|
||||
const configStat = fs.lstatSync(configPath, { throwIfNoEntry: false });
|
||||
if (!configStat?.isFile() || configStat.isSymbolicLink() || configStat.nlink !== 1) {
|
||||
throw new Error(`Workspace config not found: ${id}`);
|
||||
}
|
||||
const canonicalConfig = fs.realpathSync.native(configPath);
|
||||
if (!isContained(workspaceDir, canonicalConfig)) {
|
||||
throw new Error(`Workspace config escapes workspace directory: ${id}`);
|
||||
}
|
||||
const config = JSON.parse(fs.readFileSync(canonicalConfig, 'utf-8')) as WorkspaceConfig;
|
||||
if (config.id !== id) throw new Error(`Workspace config id mismatch: ${id}`);
|
||||
|
||||
const mindPath = path.join(workspaceDir, 'workspace.mind');
|
||||
const mindStat = fs.lstatSync(mindPath, { throwIfNoEntry: false });
|
||||
if (!mindStat) return lexicalMindPath;
|
||||
if (!mindStat?.isFile() || mindStat.isSymbolicLink() || mindStat.nlink !== 1) {
|
||||
throw new Error(`Workspace mind is not a regular file: ${id}`);
|
||||
}
|
||||
const canonicalMind = fs.realpathSync.native(mindPath);
|
||||
if (!isContained(workspaceDir, canonicalMind)) {
|
||||
throw new Error(`Workspace mind escapes workspace directory: ${id}`);
|
||||
}
|
||||
return lexicalMindPath;
|
||||
}
|
||||
|
||||
private assertWorkspaceId(id: string): void {
|
||||
if (!WORKSPACE_ID.test(id)) throw new Error(`Invalid workspace id: ${id}`);
|
||||
}
|
||||
|
||||
private isEmptyLegacyWorkspaceDirectory(workspacePath: string, stat: fs.Stats): boolean {
|
||||
if (stat.isSymbolicLink() || !stat.isDirectory()) return false;
|
||||
const canonicalRoot = this.resolveWorkspaceRoot();
|
||||
const canonicalWorkspace = fs.realpathSync.native(workspacePath);
|
||||
if (!isContained(canonicalRoot, canonicalWorkspace)) return false;
|
||||
const entries = fs.readdirSync(workspacePath, { withFileTypes: true });
|
||||
if (entries.length === 0) return true;
|
||||
if (entries.length !== 1 || entries[0]?.name !== 'sessions' || !entries[0].isDirectory()) return false;
|
||||
const sessionsPath = path.join(workspacePath, 'sessions');
|
||||
const sessionsStat = fs.lstatSync(sessionsPath);
|
||||
if (sessionsStat.isSymbolicLink()) return false;
|
||||
const canonicalSessions = fs.realpathSync.native(sessionsPath);
|
||||
return isContained(canonicalWorkspace, canonicalSessions) && fs.readdirSync(sessionsPath).length === 0;
|
||||
}
|
||||
|
||||
private resolveWorkspaceDir(id: string): string | null {
|
||||
this.assertWorkspaceId(id);
|
||||
const lexicalRoot = this.resolveWorkspaceRoot();
|
||||
const lexicalWorkspace = path.resolve(lexicalRoot, id);
|
||||
if (!isContained(lexicalRoot, lexicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
const workspaceStat = fs.lstatSync(lexicalWorkspace, { throwIfNoEntry: false });
|
||||
if (!workspaceStat) return null;
|
||||
if (workspaceStat.isSymbolicLink() || !workspaceStat.isDirectory()) {
|
||||
throw new Error(`Workspace path is not a regular directory: ${id}`);
|
||||
}
|
||||
|
||||
const canonicalWorkspace = fs.realpathSync.native(lexicalWorkspace);
|
||||
if (!isContained(this.canonicalWorkspacesDir, canonicalWorkspace)) {
|
||||
throw new Error(`Workspace path escapes workspace root: ${id}`);
|
||||
}
|
||||
return canonicalWorkspace;
|
||||
}
|
||||
|
||||
private resolveWorkspaceRoot(): string {
|
||||
const rootStat = fs.lstatSync(this.workspacesDir, { throwIfNoEntry: false });
|
||||
if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) {
|
||||
throw new Error('Workspace root is not a regular directory');
|
||||
}
|
||||
const canonicalRoot = fs.realpathSync.native(this.workspacesDir);
|
||||
if (canonicalRoot !== this.canonicalWorkspacesDir) {
|
||||
throw new Error('Workspace root changed after initialization');
|
||||
}
|
||||
return canonicalRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the default workspace ID in workspaces-meta.json.
|
||||
*/
|
||||
setDefault(id: string): void {
|
||||
if (!this.get(id)) throw new Error(`Workspace not found: ${id}`);
|
||||
const meta = this.loadMeta();
|
||||
meta.defaultWorkspace = id;
|
||||
this.saveMeta(meta);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the default workspace ID. Returns null if none set.
|
||||
*/
|
||||
getDefault(): string | null {
|
||||
const meta = this.loadMeta();
|
||||
return meta.defaultWorkspace ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure at least one workspace exists. If none, create a default one
|
||||
* and mark it as the default. Idempotent — safe to call on every startup.
|
||||
*/
|
||||
ensureDefault(options?: Partial<CreateWorkspaceOptions>): WorkspaceConfig {
|
||||
const existing = this.list();
|
||||
if (existing.length > 0) {
|
||||
const defaultId = this.getDefault();
|
||||
const found = defaultId ? this.get(defaultId) : null;
|
||||
return found ?? existing[0];
|
||||
}
|
||||
|
||||
const ws = this.create({
|
||||
name: 'Default Workspace',
|
||||
group: 'Personal',
|
||||
personaId: 'researcher',
|
||||
...options,
|
||||
});
|
||||
this.setDefault(ws.id);
|
||||
return ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a slug-based ID from a workspace name.
|
||||
* Handles duplicates by appending -2, -3, etc.
|
||||
*/
|
||||
generateId(name: string): string {
|
||||
const base = name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
|
||||
if (!this.workspaceExists(base)) return base;
|
||||
|
||||
let counter = 2;
|
||||
while (this.workspaceExists(`${base}-${counter}`)) {
|
||||
counter++;
|
||||
}
|
||||
return `${base}-${counter}`;
|
||||
}
|
||||
|
||||
private workspaceExists(id: string): boolean {
|
||||
return fs.existsSync(path.join(this.resolveWorkspaceRoot(), id));
|
||||
}
|
||||
|
||||
private loadMeta(): WorkspacesMeta {
|
||||
if (fs.existsSync(this.metaPath)) {
|
||||
const raw = fs.readFileSync(this.metaPath, 'utf-8');
|
||||
return JSON.parse(raw) as WorkspacesMeta;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
private saveMeta(meta: WorkspacesMeta): void {
|
||||
fs.writeFileSync(this.metaPath, JSON.stringify(meta, null, 2), 'utf-8');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user