feat(core): give Waggle its own database, apart from memory (P1.5)

Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.

- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
  actions, skill hashes, thumbs feedback, cost tracking, the profile card
  and awareness items live there. Their stores take a WaggleDB, so passing
  a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
  'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
  workspace.mind (importLegacyMindData). The legacy files are only read,
  each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
  WorkspaceManager and the embedders from @waggle/hive-mind-core into
  @waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
  tests that seeded these tables in a memory database

ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
shumeiko.it
2026-10-23 14:31:28 +02:00
parent 20567ecb10
commit ad2ca875cb
65 changed files with 6131 additions and 201 deletions

View File

@@ -0,0 +1,73 @@
/**
* API-backed embedder — calls Voyage AI or OpenAI embedding endpoints.
* For users who configure API keys in the Vault UI.
*/
import type { Embedder } from './embeddings.js';
import { normalizeDimensions } from './inprocess-embedder.js';
export interface ApiEmbedderConfig {
provider: 'voyage' | 'openai';
apiKey: string;
model?: string;
targetDimensions?: number;
baseUrl?: string;
}
const PROVIDER_DEFAULTS: Record<'voyage' | 'openai', { url: string; model: string }> = {
voyage: { url: 'https://api.voyageai.com/v1/embeddings', model: 'voyage-3-lite' },
openai: { url: 'https://api.openai.com/v1/embeddings', model: 'text-embedding-3-small' },
};
export function createApiEmbedder(config: ApiEmbedderConfig): Embedder {
const defaults = PROVIDER_DEFAULTS[config.provider];
const url = config.baseUrl ?? defaults.url;
const model = config.model ?? defaults.model;
const targetDims = config.targetDimensions ?? 1024;
async function callApi(input: string | string[]): Promise<Float32Array[]> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
const body: Record<string, unknown> = { model, input };
if (config.provider === 'voyage') {
body.input_type = 'document';
}
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${config.apiKey}`,
},
body: JSON.stringify(body),
signal: controller.signal,
});
if (!response.ok) {
const text = await response.text();
throw new Error(`${config.provider} embeddings error (${response.status}): ${text}`);
}
const json = await response.json() as { data: Array<{ embedding: number[] }> };
return json.data.map(d => normalizeDimensions(new Float32Array(d.embedding), targetDims));
} finally {
clearTimeout(timeout);
}
}
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const results = await callApi(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callApi(texts);
},
};
}

View File

@@ -0,0 +1,446 @@
/**
* EmbeddingProvider — orchestrates the InProcess → Ollama → API → Mock fallback chain.
* Single entry point for all embedding operations in Waggle.
* Implements the Embedder interface — drop-in replacement everywhere.
*
* Usage: the monthly embed count is tracked in the embedding_usage table (no limit).
*/
import type { Embedder } from './embeddings.js';
import type { Database as DatabaseType } from 'better-sqlite3';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('embedding');
export type EmbeddingProviderType = 'inprocess' | 'ollama' | 'voyage' | 'openai' | 'litellm' | 'mock';
export interface EmbeddingProviderConfig {
provider?: EmbeddingProviderType | 'auto';
targetDimensions?: number;
/** User ID for quota tracking. Defaults to 'local'. */
userId?: string;
/** Raw SQLite database for usage tracking. Optional — usage is not recorded without it. */
quotaDb?: DatabaseType;
inprocess?: { model?: string; cacheDir?: string };
ollama?: { baseUrl?: string; model?: string };
voyage?: { apiKey: string; model?: string };
openai?: { apiKey: string; model?: string };
litellm?: { url: string; apiKey?: string; model?: string };
}
// ── Quota tracking ────────────────────────────────────────────────────
const EMBEDDING_USAGE_SCHEMA = `
CREATE TABLE IF NOT EXISTS embedding_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL,
year_month TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL,
UNIQUE(user_id, year_month)
);
`;
function ensureQuotaTable(db: DatabaseType): void {
try { db.exec(EMBEDDING_USAGE_SCHEMA); } catch { /* table may already exist */ }
}
function getCurrentYearMonth(): string {
const d = new Date();
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
}
function incrementUsage(db: DatabaseType, userId: string, yearMonth: string, amount: number): void {
db.prepare(`
INSERT INTO embedding_usage (user_id, year_month, count, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(user_id, year_month) DO UPDATE SET count = count + ?, updated_at = ?
`).run(userId, yearMonth, amount, Date.now(), amount, Date.now());
}
export interface EmbeddingQuotaStatus {
quota: number;
used: number;
remaining: number;
percentage: number;
resetsAt: string;
}
function getNextMonthReset(): string {
const d = new Date();
d.setMonth(d.getMonth() + 1, 1);
d.setHours(0, 0, 0, 0);
return d.toISOString();
}
export interface EmbeddingProviderStatus {
activeProvider: EmbeddingProviderType;
availableProviders: EmbeddingProviderType[];
dimensions: number;
modelName: string;
lastError?: string;
probeTimestamp: string;
}
export interface EmbeddingProviderInstance extends Embedder {
getStatus(): EmbeddingProviderStatus;
getActiveProvider(): EmbeddingProviderType;
reprobe(): Promise<EmbeddingProviderStatus>;
/** Get the usage status. Embedding is unlimited, so quota and remaining are -1. */
getQuotaStatus(): EmbeddingQuotaStatus;
}
/** Deterministic mock — last resort, semantically meaningless. */
function mockEmbed(text: string, dims: number): Float32Array {
const arr = new Float32Array(dims);
const bytes = new TextEncoder().encode(text);
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
arr[i] = (bytes[i] - 128) / 128;
}
return arr;
}
function createMockEmbedder(dims: number): Embedder {
return {
dimensions: dims,
async embed(text: string) { return mockEmbed(text, dims); },
async embedBatch(texts: string[]) { return texts.map(t => mockEmbed(t, dims)); },
};
}
// ── Embed-input guards (oversized-frame truncation + skip-not-abort) ──
// Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
/**
* Per-input character cap for embedding. `nomic-embed-text` has a 2048-token
* (~6K char dense English) default context. Embedding an input longer than
* the backend's context makes the backend reject the request, so we cap here.
*
* D1 probe finding (2026-06-12): model NAMES lie about context. A custom
* `nomic-embed-text-8k` (num_ctx 8192) still 400s at its nomic-bert
* ARCHITECTURE limit of 2048 tokens — the OSS heuristic's 24K-char branch
* for `*-8k` names sent every long frame down the mock-fallback path. The
* `-8k` branch is therefore capped at 8K chars (≈2048 prose tokens): safe
* for the architecture-limited reality, merely conservative for a genuine
* 8192-token embedder. reembedPerText remains the backstop for token-dense
* content that still exceeds the backend's real window.
*/
export function maxEmbedCharsForModel(modelName: string): number {
return /(-|_|\.)8k\b|num_ctx[^0-9]*8192/i.test(modelName) ? 8_000 : 6_000;
}
/** Clamp a single input to `maxChars` (no-op when already under the cap). */
export function capEmbedText(text: string, maxChars: number): string {
return text.length > maxChars ? text.slice(0, maxChars) : text;
}
/**
* Re-embed a batch one text at a time, degrading ONLY the inputs that genuinely
* fail to a deterministic mock vector. This is the batch-error recovery path:
* a single backend-rejected text can no longer poison its batchmates (the prior
* behavior substituted mock for the WHOLE batch — silent corruption of every
* frame in the batch). Inputs should already be char-capped by the caller.
*/
export async function reembedPerText(
embedder: Embedder,
texts: string[],
dims: number,
): Promise<Float32Array[]> {
return Promise.all(
texts.map(async (t) => {
try {
return await embedder.embed(t);
} catch {
return mockEmbed(t, dims);
}
}),
);
}
interface ProbeResult {
type: EmbeddingProviderType;
embedder: Embedder;
modelName: string;
}
async function probeProvider(
type: EmbeddingProviderType,
config: EmbeddingProviderConfig,
): Promise<ProbeResult | null> {
const dims = config.targetDimensions ?? 1024;
try {
switch (type) {
case 'inprocess': {
const { createInProcessEmbedder } = await import('./inprocess-embedder.js');
const embedder = await createInProcessEmbedder({
model: config.inprocess?.model,
cacheDir: config.inprocess?.cacheDir,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'inprocess', embedder, modelName: config.inprocess?.model ?? 'Xenova/all-MiniLM-L6-v2' };
}
case 'ollama': {
const { createOllamaEmbedder } = await import('./ollama-embedder.js');
const embedder = createOllamaEmbedder({
baseUrl: config.ollama?.baseUrl,
model: config.ollama?.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'ollama', embedder, modelName: config.ollama?.model ?? 'nomic-embed-text' };
}
case 'voyage': {
if (!config.voyage?.apiKey) return null;
const { createApiEmbedder } = await import('./api-embedder.js');
const embedder = createApiEmbedder({
provider: 'voyage',
apiKey: config.voyage.apiKey,
model: config.voyage.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'voyage', embedder, modelName: config.voyage.model ?? 'voyage-3-lite' };
}
case 'openai': {
if (!config.openai?.apiKey) return null;
const { createApiEmbedder } = await import('./api-embedder.js');
const embedder = createApiEmbedder({
provider: 'openai',
apiKey: config.openai.apiKey,
model: config.openai.model,
targetDimensions: dims,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'openai', embedder, modelName: config.openai.model ?? 'text-embedding-3-small' };
}
case 'litellm': {
if (!config.litellm?.url) return null;
const { createLiteLLMEmbedder } = await import('./litellm-embedder.js');
const embedder = createLiteLLMEmbedder({
litellmUrl: config.litellm.url,
litellmApiKey: config.litellm.apiKey,
model: config.litellm.model ?? 'text-embedding',
dimensions: dims,
fallbackToMock: false,
});
const test = await embedder.embed('waggle embedding probe');
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
return { type: 'litellm', embedder, modelName: config.litellm.model ?? 'text-embedding' };
}
default:
return null;
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
log.info(`Trying ${type}... FAILED (${msg})`);
return null;
}
}
export async function createEmbeddingProvider(config?: EmbeddingProviderConfig): Promise<EmbeddingProviderInstance> {
const cfg: EmbeddingProviderConfig = { provider: 'auto', targetDimensions: 1024, ...config };
const dims = cfg.targetDimensions ?? 1024;
const userId = cfg.userId ?? 'local';
const quotaDb = cfg.quotaDb ?? null;
// Initialize the usage table if DB provided
if (quotaDb) {
ensureQuotaTable(quotaDb);
}
let activeResult: ProbeResult | null = null;
let activeEmbedder: Embedder;
let activeType: EmbeddingProviderType = 'mock';
let activeModelName = 'deterministic-mock';
let lastError: string | undefined;
let availableProviders: EmbeddingProviderType[] = [];
let probeTimestamp = new Date().toISOString();
/** Record usage after successful embedding. */
function recordUsage(count: number): void {
if (!quotaDb) return;
incrementUsage(quotaDb, userId, getCurrentYearMonth(), count);
}
async function runProbe(): Promise<void> {
log.info('Probing embedding providers...');
const available: EmbeddingProviderType[] = [];
activeResult = null;
probeTimestamp = new Date().toISOString();
const requestedProvider = cfg.provider ?? 'auto';
if (requestedProvider !== 'auto' && requestedProvider !== 'mock') {
log.info(`Trying ${requestedProvider}...`);
const result = await probeProvider(requestedProvider, cfg);
if (result) {
activeResult = result;
available.push(result.type);
log.info(`Trying ${requestedProvider}... OK`);
}
} else if (requestedProvider === 'auto') {
// Auto: iterate the chain in order
const chain: EmbeddingProviderType[] = ['inprocess', 'ollama', 'voyage', 'openai'];
for (const providerType of chain) {
// Skip API providers without keys
if (providerType === 'voyage' && !cfg.voyage?.apiKey) {
log.info('Skipping voyage (no API key in Vault)');
continue;
}
if (providerType === 'openai' && !cfg.openai?.apiKey) {
log.info('Skipping openai (no API key in Vault)');
continue;
}
log.info(`Trying ${providerType}...`);
const result = await probeProvider(providerType, cfg);
if (result) {
available.push(result.type);
log.info(`Trying ${providerType}... OK`);
if (!activeResult) {
activeResult = result;
}
}
}
}
available.push('mock'); // Always available
availableProviders = available;
if (activeResult) {
activeEmbedder = activeResult.embedder;
activeType = activeResult.type;
activeModelName = activeResult.modelName;
lastError = undefined;
log.info(`Embedding provider: ${activeType} (${activeModelName}, ${dims} dims)`);
} else {
activeEmbedder = createMockEmbedder(dims);
activeType = 'mock';
activeModelName = 'deterministic-mock';
lastError = 'No real providers available';
// Loud, structured warning — the silent "mock fallback" was the
// most dangerous failure mode in Phase 3b-3 audit. Mock embeddings
// are deterministic byte hashes; semantic search returns noise.
// We want this to be IMPOSSIBLE to miss in a CLI/server log.
// Ported from hive-mind a99ea0e.
const msg = [
'',
'⚠️ EMBEDDING WARNING ─────────────────────────────────────────',
' Active provider: mock (deterministic byte hash)',
' Effect: semantic search returns noise, not meaning.',
'',
' To fix, install Ollama and pull the embedding model:',
' ollama pull nomic-embed-text',
' Then ensure the process can reach http://localhost:11434.',
'',
' Alternative providers:',
' HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB)',
' VOYAGE_API_KEY=... (paid, recommended)',
' OPENAI_API_KEY=... (paid)',
'─────────────────────────────────────────────────────────────',
'',
].join('\n');
// Keep production and CLI runs loud, but let deterministic test lanes
// suppress this expected fallback banner without changing provider state.
if (process.env.WAGGLE_SUPPRESS_EMBEDDING_WARNING !== '1') {
// stderr so it survives stdout-piped JSON consumers and CI tee.
try { process.stderr.write(msg); } catch { /* fall through to log */ }
log.warn('Embedding provider degraded to mock — semantic search quality is noise. See stderr banner for fix instructions.');
}
}
}
// Initial probe
try {
await runProbe();
} catch (err) {
lastError = err instanceof Error ? err.message : String(err);
activeEmbedder = createMockEmbedder(dims);
activeType = 'mock';
activeModelName = 'deterministic-mock';
availableProviders = ['mock'];
}
// Ensure activeEmbedder is assigned (TypeScript flow)
activeEmbedder ??= createMockEmbedder(dims);
const instance: EmbeddingProviderInstance = {
dimensions: dims,
async embed(text: string): Promise<Float32Array> {
// Cap input to the active model's context so the backend never rejects
// an oversized frame. Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
const capped = capEmbedText(text, maxEmbedCharsForModel(activeModelName));
try {
const result = await activeEmbedder.embed(capped);
recordUsage(1);
return result;
} catch (err) {
log.warn(`Embedding failed with ${activeType}, falling back to mock: ${(err as Error).message}`);
lastError = (err as Error).message;
const fallback = mockEmbed(capped, dims);
recordUsage(1);
return fallback;
}
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
// Cap each input first so one oversized frame can't make the backend
// reject the whole request. Reverse-ported from OSS hive-mind
// (oss-drift triage R5, 2026-06-11).
const capped = texts.map(t => capEmbedText(t, maxEmbedCharsForModel(activeModelName)));
try {
const result = await activeEmbedder.embedBatch(capped);
recordUsage(texts.length);
return result;
} catch (err) {
// Skip-not-abort: re-embed per-text so a single backend-rejected input
// degrades alone instead of mock-poisoning the WHOLE batch.
log.warn(`Batch embedding failed with ${activeType}, re-embedding per-text: ${(err as Error).message}`);
lastError = (err as Error).message;
const result = await reembedPerText(activeEmbedder, capped, dims);
recordUsage(texts.length);
return result;
}
},
getStatus(): EmbeddingProviderStatus {
return {
activeProvider: activeType,
availableProviders,
dimensions: dims,
modelName: activeModelName,
lastError,
probeTimestamp,
};
},
getActiveProvider(): EmbeddingProviderType {
return activeType;
},
async reprobe(): Promise<EmbeddingProviderStatus> {
await runProbe();
return instance.getStatus();
},
getQuotaStatus(): EmbeddingQuotaStatus {
return { quota: -1, used: 0, remaining: -1, percentage: 0, resetsAt: getNextMonthReset() };
},
};
return instance;
}

View File

@@ -0,0 +1,5 @@
export interface Embedder {
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
dimensions: number;
}

View File

@@ -0,0 +1,73 @@
/**
* In-process embedder using @huggingface/transformers (ONNX Runtime).
* Default provider for ALL desktop users — zero config, works offline.
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
*/
import path from 'node:path';
import os from 'node:os';
import type { Embedder } from './embeddings.js';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-embedder');
export interface InProcessEmbedderConfig {
model?: string;
cacheDir?: string;
targetDimensions?: number;
}
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
if (embedding.length === targetDims) return embedding;
const result = new Float32Array(targetDims);
const copyLen = Math.min(embedding.length, targetDims);
result.set(embedding.subarray(0, copyLen));
return result;
}
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
const targetDims = config?.targetDimensions ?? 1024;
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
const { pipeline } = await import('@huggingface/transformers');
const extractor = await withTransformersModelLoad({
cacheDir,
model,
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
});
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const result = await extractor(text, { pooling: 'mean', normalize: true });
const raw = new Float32Array(result.data as Float32Array);
return normalizeDimensions(raw, targetDims);
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
const results: Float32Array[] = [];
// Process one at a time to avoid memory issues with large batches
for (const text of texts) {
const result = await extractor(text, { pooling: 'mean', normalize: true });
const raw = new Float32Array(result.data as Float32Array);
results.push(normalizeDimensions(raw, targetDims));
}
return results;
},
};
}

View File

@@ -0,0 +1,93 @@
/**
* LiteLLM-backed embedder that calls the /embeddings endpoint.
* Falls back to a deterministic mock (text→Float32Array hash) on API error
* when `fallbackToMock` is enabled.
*/
import type { Embedder } from './embeddings.js';
export interface LiteLLMEmbedderConfig {
litellmUrl: string;
litellmApiKey?: string;
model?: string;
dimensions?: number;
/** Custom fetch implementation (for testing). */
fetch?: typeof globalThis.fetch;
/** If true, falls back to a deterministic mock on API error instead of throwing. */
fallbackToMock?: boolean;
}
function mockEmbed(text: string, dims: number): Float32Array {
const arr = new Float32Array(dims);
const bytes = new TextEncoder().encode(text);
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
arr[i] = (bytes[i] - 128) / 128;
}
return arr;
}
export function createLiteLLMEmbedder(config: LiteLLMEmbedderConfig): Embedder {
const {
litellmUrl,
litellmApiKey,
model = 'text-embedding',
dimensions = 1024,
fetch: fetchFn = globalThis.fetch,
fallbackToMock = false,
} = config;
// Normalise base URL — strip trailing /v1 if present, we add it ourselves
const baseUrl = litellmUrl.replace(/\/v1\/?$/, '');
const url = `${baseUrl}/v1/embeddings`;
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
if (litellmApiKey) {
headers['Authorization'] = `Bearer ${litellmApiKey}`;
}
async function callApi(input: string | string[]): Promise<Float32Array[]> {
const body = JSON.stringify({ model, input });
let response: Response;
try {
response = await fetchFn(url, { method: 'POST', headers, body });
} catch (err) {
if (fallbackToMock) {
const texts = Array.isArray(input) ? input : [input];
return texts.map((t) => mockEmbed(t, dimensions));
}
throw err;
}
if (!response.ok) {
if (fallbackToMock) {
const texts = Array.isArray(input) ? input : [input];
return texts.map((t) => mockEmbed(t, dimensions));
}
const text = await response.text();
throw new Error(`LiteLLM embeddings error (${response.status}): ${text}`);
}
const json = (await response.json()) as {
data: Array<{ embedding: number[] }>;
};
return json.data.map((d) => new Float32Array(d.embedding));
}
return {
dimensions,
async embed(text: string): Promise<Float32Array> {
const results = await callApi(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callApi(texts);
},
};
}

View File

@@ -0,0 +1,58 @@
/**
* Ollama-backed embedder — calls local Ollama server for embeddings.
* Power user option for users who have Ollama installed.
*/
import type { Embedder } from './embeddings.js';
import { normalizeDimensions } from './inprocess-embedder.js';
export interface OllamaEmbedderConfig {
baseUrl?: string;
model?: string;
targetDimensions?: number;
}
export function createOllamaEmbedder(config?: Partial<OllamaEmbedderConfig>): Embedder {
const baseUrl = config?.baseUrl ?? 'http://localhost:11434';
const model = config?.model ?? 'nomic-embed-text';
const targetDims = config?.targetDimensions ?? 1024;
const url = `${baseUrl}/api/embed`;
async function callOllama(input: string | string[]): Promise<Float32Array[]> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30000);
try {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model, input }),
signal: controller.signal,
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Ollama embeddings error (${response.status}): ${text}`);
}
const json = await response.json() as { embeddings: number[][] };
return json.embeddings.map(e => normalizeDimensions(new Float32Array(e), targetDims));
} finally {
clearTimeout(timeout);
}
}
return {
dimensions: targetDims,
async embed(text: string): Promise<Float32Array> {
const results = await callOllama(text);
return results[0];
},
async embedBatch(texts: string[]): Promise<Float32Array[]> {
if (texts.length === 0) return [];
return callOllama(texts);
},
};
}

View File

@@ -0,0 +1,258 @@
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import Database from 'better-sqlite3';
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
const LOCK_RETRY_MIN_MS = 35;
const LOCK_RETRY_JITTER_MS = 30;
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
const LOCK_DIRECTORY = '.waggle-model-locks';
export interface TransformersModelLoadOptions<T> {
cacheDir: string;
model: string;
load: (canonicalCacheDir: string) => Promise<T>;
lockTimeoutMs?: number;
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
}
function normalizeLockKey(value: string): string {
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
}
export function modelLoadLockPath(cacheDir: string, model: string): string {
const modelHash = createHash('sha256')
.update(normalizeLockKey(model))
.digest('hex')
.slice(0, 16);
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
}
function isSqliteBusy(error: unknown): boolean {
if (!(error instanceof Error) || !('code' in error)) return false;
const code = String((error as Error & { code?: unknown }).code);
return code === 'SQLITE_BUSY'
|| code === 'SQLITE_BUSY_SNAPSHOT'
|| code === 'SQLITE_LOCKED';
}
async function acquireCrossProcessLock(
database: Database.Database,
timeoutMs: number,
): Promise<void> {
const deadline = performance.now() + timeoutMs;
for (;;) {
try {
database.exec('BEGIN IMMEDIATE');
return;
} catch (error) {
if (!isSqliteBusy(error)) throw error;
if (performance.now() >= deadline) {
throw new Error(
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
{ cause: error },
);
}
}
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
}
}
function isWithin(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative !== ''
&& relative !== '..'
&& !relative.startsWith(`..${path.sep}`)
&& !path.isAbsolute(relative);
}
function isWithinOrEqual(root: string, candidate: string): boolean {
return root === candidate || isWithin(root, candidate);
}
function assertUnlinkedPath(root: string, target: string): void {
const relative = path.relative(root, target);
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error('Model cache path escapes its cache root');
}
let current = root;
for (const segment of relative.split(path.sep)) {
current = path.join(current, segment);
const stats = fs.lstatSync(current);
if (stats.isSymbolicLink()) {
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
}
}
}
function validateQuarantinePaths(
canonicalCacheDir: string,
modelDir: string,
reportedOnnxPath: string,
): { modelDir: string; quarantineRoot: string } | null {
if (!path.isAbsolute(reportedOnnxPath)) return null;
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
assertUnlinkedPath(canonicalCacheDir, modelDir);
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
const quarantineRoot = path.dirname(modelDir);
const modelStats = fs.lstatSync(modelDir);
const reportStats = fs.lstatSync(reportedOnnxPath);
const rootStats = fs.lstatSync(quarantineRoot);
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
const realModelDir = fs.realpathSync.native(modelDir);
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|| !isWithin(realQuarantineRoot, realModelDir)
|| path.dirname(realModelDir) !== realQuarantineRoot
|| !isWithin(realModelDir, realOnnxPath)) {
return null;
}
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
}
function quarantineCorruptModel(
canonicalCacheDir: string,
model: string,
reportedOnnxPath: string,
): string | null {
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!firstValidation) return null;
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!finalValidation) return null;
const quarantineDir = path.join(
finalValidation.quarantineRoot,
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
);
fs.renameSync(finalValidation.modelDir, quarantineDir);
return quarantineDir;
}
function reportedCorruptOnnxPath(error: unknown): string | null {
if (!(error instanceof Error)) return null;
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
}
function notifyQuarantine(
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
quarantineDir: string,
): void {
try {
const notification = callback?.(quarantineDir);
if (notification) {
void Promise.resolve(notification).catch(() => undefined);
}
} catch {
// Notification is advisory and must not alter model recovery control flow.
}
}
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
const lockDir = path.dirname(lockPath);
fs.mkdirSync(lockDir, { recursive: true });
const lockDirStats = fs.lstatSync(lockDir);
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
throw new Error('Local model lock path is not a regular directory');
}
const realLockDir = fs.realpathSync.native(lockDir);
if (!isWithin(canonicalCacheDir, realLockDir)) {
throw new Error('Local model lock path escapes its cache root');
}
if (fs.existsSync(lockPath)) {
const lockStats = fs.lstatSync(lockPath);
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
throw new Error('Local model lock database is not a regular file');
}
}
return new Database(lockPath, { timeout: 0 });
}
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
throw new Error('Local model cache lock timeout must be a non-negative finite number');
}
const requestedCacheDir = path.resolve(options.cacheDir);
fs.mkdirSync(requestedCacheDir, { recursive: true });
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
let result!: T;
let primaryError: unknown;
let hasPrimaryError = false;
try {
await acquireCrossProcessLock(lockDatabase, timeoutMs);
try {
result = await options.load(canonicalCacheDir);
} catch (firstError) {
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
if (!reportedOnnxPath) throw firstError;
let quarantineDir: string | null = null;
try {
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
} catch {
// Preserve the original loader error if safe quarantine cannot complete.
}
if (!quarantineDir) throw firstError;
notifyQuarantine(options.onQuarantine, quarantineDir);
result = await options.load(canonicalCacheDir);
}
} catch (error) {
primaryError = error;
hasPrimaryError = true;
}
let cleanupError: unknown;
let hasCleanupError = false;
if (lockDatabase.inTransaction) {
try {
lockDatabase.exec('ROLLBACK');
} catch (error) {
cleanupError = error;
hasCleanupError = true;
}
}
try {
lockDatabase.close();
} catch (error) {
if (!hasCleanupError) cleanupError = error;
hasCleanupError = true;
}
if (hasPrimaryError) throw primaryError;
if (hasCleanupError) throw cleanupError;
return result;
}
/**
* Serializes one model/cache load across workers and processes. SQLite owns
* the OS lock, so process termination releases it without PID/age heuristics.
*/
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
return runModelLoad(options);
}