feat(core): give Waggle its own database, apart from memory (P1.5)
Waggle kept its own tables in the memory database. Before the memory
engine can be replaced by mem0 they need a home of their own.
- add WaggleDB (<dataDir>/waggle.db). Automations, notifications, held
actions, skill hashes, thumbs feedback, cost tracking, the profile card
and awareness items live there. Their stores take a WaggleDB, so passing
a memory database is a type error.
- the profile card and awareness items are now per scope ('personal' or
'workspace:<id>') in one table each, instead of one table per .mind file
- on the first start, copy these tables from personal.mind and every
workspace.mind (importLegacyMindData). The legacy files are only read,
each scope is copied once and a table only while its target is empty.
- copy the logger, injection scanner, external-memory ingress guard,
WorkspaceManager and the embedders from @waggle/hive-mind-core into
@waggle/core, which now re-exports its own copies
- point the server, agent orchestrator and CLI at waggle.db; update the
tests that seeded these tables in a memory database
ADR-0007 records the layout and the licensing question about the copied
files.
This commit is contained in:
73
packages/core/src/embeddings/api-embedder.ts
Normal file
73
packages/core/src/embeddings/api-embedder.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* API-backed embedder — calls Voyage AI or OpenAI embedding endpoints.
|
||||
* For users who configure API keys in the Vault UI.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface ApiEmbedderConfig {
|
||||
provider: 'voyage' | 'openai';
|
||||
apiKey: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
const PROVIDER_DEFAULTS: Record<'voyage' | 'openai', { url: string; model: string }> = {
|
||||
voyage: { url: 'https://api.voyageai.com/v1/embeddings', model: 'voyage-3-lite' },
|
||||
openai: { url: 'https://api.openai.com/v1/embeddings', model: 'text-embedding-3-small' },
|
||||
};
|
||||
|
||||
export function createApiEmbedder(config: ApiEmbedderConfig): Embedder {
|
||||
const defaults = PROVIDER_DEFAULTS[config.provider];
|
||||
const url = config.baseUrl ?? defaults.url;
|
||||
const model = config.model ?? defaults.model;
|
||||
const targetDims = config.targetDimensions ?? 1024;
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 15000);
|
||||
|
||||
const body: Record<string, unknown> = { model, input };
|
||||
if (config.provider === 'voyage') {
|
||||
body.input_type = 'document';
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${config.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`${config.provider} embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { data: Array<{ embedding: number[] }> };
|
||||
return json.data.map(d => normalizeDimensions(new Float32Array(d.embedding), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
446
packages/core/src/embeddings/embedding-provider.ts
Normal file
446
packages/core/src/embeddings/embedding-provider.ts
Normal file
@@ -0,0 +1,446 @@
|
||||
/**
|
||||
* EmbeddingProvider — orchestrates the InProcess → Ollama → API → Mock fallback chain.
|
||||
* Single entry point for all embedding operations in Waggle.
|
||||
* Implements the Embedder interface — drop-in replacement everywhere.
|
||||
*
|
||||
* Usage: the monthly embed count is tracked in the embedding_usage table (no limit).
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('embedding');
|
||||
|
||||
export type EmbeddingProviderType = 'inprocess' | 'ollama' | 'voyage' | 'openai' | 'litellm' | 'mock';
|
||||
|
||||
export interface EmbeddingProviderConfig {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
targetDimensions?: number;
|
||||
/** User ID for quota tracking. Defaults to 'local'. */
|
||||
userId?: string;
|
||||
/** Raw SQLite database for usage tracking. Optional — usage is not recorded without it. */
|
||||
quotaDb?: DatabaseType;
|
||||
inprocess?: { model?: string; cacheDir?: string };
|
||||
ollama?: { baseUrl?: string; model?: string };
|
||||
voyage?: { apiKey: string; model?: string };
|
||||
openai?: { apiKey: string; model?: string };
|
||||
litellm?: { url: string; apiKey?: string; model?: string };
|
||||
}
|
||||
|
||||
// ── Quota tracking ────────────────────────────────────────────────────
|
||||
|
||||
const EMBEDDING_USAGE_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS embedding_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
year_month TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at INTEGER NOT NULL,
|
||||
UNIQUE(user_id, year_month)
|
||||
);
|
||||
`;
|
||||
|
||||
function ensureQuotaTable(db: DatabaseType): void {
|
||||
try { db.exec(EMBEDDING_USAGE_SCHEMA); } catch { /* table may already exist */ }
|
||||
}
|
||||
|
||||
function getCurrentYearMonth(): string {
|
||||
const d = new Date();
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function incrementUsage(db: DatabaseType, userId: string, yearMonth: string, amount: number): void {
|
||||
db.prepare(`
|
||||
INSERT INTO embedding_usage (user_id, year_month, count, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, year_month) DO UPDATE SET count = count + ?, updated_at = ?
|
||||
`).run(userId, yearMonth, amount, Date.now(), amount, Date.now());
|
||||
}
|
||||
|
||||
export interface EmbeddingQuotaStatus {
|
||||
quota: number;
|
||||
used: number;
|
||||
remaining: number;
|
||||
percentage: number;
|
||||
resetsAt: string;
|
||||
}
|
||||
|
||||
function getNextMonthReset(): string {
|
||||
const d = new Date();
|
||||
d.setMonth(d.getMonth() + 1, 1);
|
||||
d.setHours(0, 0, 0, 0);
|
||||
return d.toISOString();
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderStatus {
|
||||
activeProvider: EmbeddingProviderType;
|
||||
availableProviders: EmbeddingProviderType[];
|
||||
dimensions: number;
|
||||
modelName: string;
|
||||
lastError?: string;
|
||||
probeTimestamp: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderInstance extends Embedder {
|
||||
getStatus(): EmbeddingProviderStatus;
|
||||
getActiveProvider(): EmbeddingProviderType;
|
||||
reprobe(): Promise<EmbeddingProviderStatus>;
|
||||
/** Get the usage status. Embedding is unlimited, so quota and remaining are -1. */
|
||||
getQuotaStatus(): EmbeddingQuotaStatus;
|
||||
}
|
||||
|
||||
/** Deterministic mock — last resort, semantically meaningless. */
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function createMockEmbedder(dims: number): Embedder {
|
||||
return {
|
||||
dimensions: dims,
|
||||
async embed(text: string) { return mockEmbed(text, dims); },
|
||||
async embedBatch(texts: string[]) { return texts.map(t => mockEmbed(t, dims)); },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Embed-input guards (oversized-frame truncation + skip-not-abort) ──
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
|
||||
/**
|
||||
* Per-input character cap for embedding. `nomic-embed-text` has a 2048-token
|
||||
* (~6K char dense English) default context. Embedding an input longer than
|
||||
* the backend's context makes the backend reject the request, so we cap here.
|
||||
*
|
||||
* D1 probe finding (2026-06-12): model NAMES lie about context. A custom
|
||||
* `nomic-embed-text-8k` (num_ctx 8192) still 400s at its nomic-bert
|
||||
* ARCHITECTURE limit of 2048 tokens — the OSS heuristic's 24K-char branch
|
||||
* for `*-8k` names sent every long frame down the mock-fallback path. The
|
||||
* `-8k` branch is therefore capped at 8K chars (≈2048 prose tokens): safe
|
||||
* for the architecture-limited reality, merely conservative for a genuine
|
||||
* 8192-token embedder. reembedPerText remains the backstop for token-dense
|
||||
* content that still exceeds the backend's real window.
|
||||
*/
|
||||
export function maxEmbedCharsForModel(modelName: string): number {
|
||||
return /(-|_|\.)8k\b|num_ctx[^0-9]*8192/i.test(modelName) ? 8_000 : 6_000;
|
||||
}
|
||||
|
||||
/** Clamp a single input to `maxChars` (no-op when already under the cap). */
|
||||
export function capEmbedText(text: string, maxChars: number): string {
|
||||
return text.length > maxChars ? text.slice(0, maxChars) : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-embed a batch one text at a time, degrading ONLY the inputs that genuinely
|
||||
* fail to a deterministic mock vector. This is the batch-error recovery path:
|
||||
* a single backend-rejected text can no longer poison its batchmates (the prior
|
||||
* behavior substituted mock for the WHOLE batch — silent corruption of every
|
||||
* frame in the batch). Inputs should already be char-capped by the caller.
|
||||
*/
|
||||
export async function reembedPerText(
|
||||
embedder: Embedder,
|
||||
texts: string[],
|
||||
dims: number,
|
||||
): Promise<Float32Array[]> {
|
||||
return Promise.all(
|
||||
texts.map(async (t) => {
|
||||
try {
|
||||
return await embedder.embed(t);
|
||||
} catch {
|
||||
return mockEmbed(t, dims);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
interface ProbeResult {
|
||||
type: EmbeddingProviderType;
|
||||
embedder: Embedder;
|
||||
modelName: string;
|
||||
}
|
||||
|
||||
async function probeProvider(
|
||||
type: EmbeddingProviderType,
|
||||
config: EmbeddingProviderConfig,
|
||||
): Promise<ProbeResult | null> {
|
||||
const dims = config.targetDimensions ?? 1024;
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'inprocess': {
|
||||
const { createInProcessEmbedder } = await import('./inprocess-embedder.js');
|
||||
const embedder = await createInProcessEmbedder({
|
||||
model: config.inprocess?.model,
|
||||
cacheDir: config.inprocess?.cacheDir,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'inprocess', embedder, modelName: config.inprocess?.model ?? 'Xenova/all-MiniLM-L6-v2' };
|
||||
}
|
||||
|
||||
case 'ollama': {
|
||||
const { createOllamaEmbedder } = await import('./ollama-embedder.js');
|
||||
const embedder = createOllamaEmbedder({
|
||||
baseUrl: config.ollama?.baseUrl,
|
||||
model: config.ollama?.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'ollama', embedder, modelName: config.ollama?.model ?? 'nomic-embed-text' };
|
||||
}
|
||||
|
||||
case 'voyage': {
|
||||
if (!config.voyage?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'voyage',
|
||||
apiKey: config.voyage.apiKey,
|
||||
model: config.voyage.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'voyage', embedder, modelName: config.voyage.model ?? 'voyage-3-lite' };
|
||||
}
|
||||
|
||||
case 'openai': {
|
||||
if (!config.openai?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'openai',
|
||||
apiKey: config.openai.apiKey,
|
||||
model: config.openai.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'openai', embedder, modelName: config.openai.model ?? 'text-embedding-3-small' };
|
||||
}
|
||||
|
||||
case 'litellm': {
|
||||
if (!config.litellm?.url) return null;
|
||||
const { createLiteLLMEmbedder } = await import('./litellm-embedder.js');
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
litellmUrl: config.litellm.url,
|
||||
litellmApiKey: config.litellm.apiKey,
|
||||
model: config.litellm.model ?? 'text-embedding',
|
||||
dimensions: dims,
|
||||
fallbackToMock: false,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'litellm', embedder, modelName: config.litellm.model ?? 'text-embedding' };
|
||||
}
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.info(`Trying ${type}... FAILED (${msg})`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createEmbeddingProvider(config?: EmbeddingProviderConfig): Promise<EmbeddingProviderInstance> {
|
||||
const cfg: EmbeddingProviderConfig = { provider: 'auto', targetDimensions: 1024, ...config };
|
||||
const dims = cfg.targetDimensions ?? 1024;
|
||||
const userId = cfg.userId ?? 'local';
|
||||
const quotaDb = cfg.quotaDb ?? null;
|
||||
|
||||
// Initialize the usage table if DB provided
|
||||
if (quotaDb) {
|
||||
ensureQuotaTable(quotaDb);
|
||||
}
|
||||
|
||||
let activeResult: ProbeResult | null = null;
|
||||
let activeEmbedder: Embedder;
|
||||
let activeType: EmbeddingProviderType = 'mock';
|
||||
let activeModelName = 'deterministic-mock';
|
||||
let lastError: string | undefined;
|
||||
let availableProviders: EmbeddingProviderType[] = [];
|
||||
let probeTimestamp = new Date().toISOString();
|
||||
|
||||
/** Record usage after successful embedding. */
|
||||
function recordUsage(count: number): void {
|
||||
if (!quotaDb) return;
|
||||
incrementUsage(quotaDb, userId, getCurrentYearMonth(), count);
|
||||
}
|
||||
|
||||
async function runProbe(): Promise<void> {
|
||||
log.info('Probing embedding providers...');
|
||||
const available: EmbeddingProviderType[] = [];
|
||||
activeResult = null;
|
||||
probeTimestamp = new Date().toISOString();
|
||||
|
||||
const requestedProvider = cfg.provider ?? 'auto';
|
||||
|
||||
if (requestedProvider !== 'auto' && requestedProvider !== 'mock') {
|
||||
log.info(`Trying ${requestedProvider}...`);
|
||||
const result = await probeProvider(requestedProvider, cfg);
|
||||
if (result) {
|
||||
activeResult = result;
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${requestedProvider}... OK`);
|
||||
}
|
||||
} else if (requestedProvider === 'auto') {
|
||||
// Auto: iterate the chain in order
|
||||
const chain: EmbeddingProviderType[] = ['inprocess', 'ollama', 'voyage', 'openai'];
|
||||
|
||||
for (const providerType of chain) {
|
||||
// Skip API providers without keys
|
||||
if (providerType === 'voyage' && !cfg.voyage?.apiKey) {
|
||||
log.info('Skipping voyage (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
if (providerType === 'openai' && !cfg.openai?.apiKey) {
|
||||
log.info('Skipping openai (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(`Trying ${providerType}...`);
|
||||
const result = await probeProvider(providerType, cfg);
|
||||
if (result) {
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${providerType}... OK`);
|
||||
if (!activeResult) {
|
||||
activeResult = result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
available.push('mock'); // Always available
|
||||
availableProviders = available;
|
||||
|
||||
if (activeResult) {
|
||||
activeEmbedder = activeResult.embedder;
|
||||
activeType = activeResult.type;
|
||||
activeModelName = activeResult.modelName;
|
||||
lastError = undefined;
|
||||
log.info(`Embedding provider: ${activeType} (${activeModelName}, ${dims} dims)`);
|
||||
} else {
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
lastError = 'No real providers available';
|
||||
// Loud, structured warning — the silent "mock fallback" was the
|
||||
// most dangerous failure mode in Phase 3b-3 audit. Mock embeddings
|
||||
// are deterministic byte hashes; semantic search returns noise.
|
||||
// We want this to be IMPOSSIBLE to miss in a CLI/server log.
|
||||
// Ported from hive-mind a99ea0e.
|
||||
const msg = [
|
||||
'',
|
||||
'⚠️ EMBEDDING WARNING ─────────────────────────────────────────',
|
||||
' Active provider: mock (deterministic byte hash)',
|
||||
' Effect: semantic search returns noise, not meaning.',
|
||||
'',
|
||||
' To fix, install Ollama and pull the embedding model:',
|
||||
' ollama pull nomic-embed-text',
|
||||
' Then ensure the process can reach http://localhost:11434.',
|
||||
'',
|
||||
' Alternative providers:',
|
||||
' HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB)',
|
||||
' VOYAGE_API_KEY=... (paid, recommended)',
|
||||
' OPENAI_API_KEY=... (paid)',
|
||||
'─────────────────────────────────────────────────────────────',
|
||||
'',
|
||||
].join('\n');
|
||||
// Keep production and CLI runs loud, but let deterministic test lanes
|
||||
// suppress this expected fallback banner without changing provider state.
|
||||
if (process.env.WAGGLE_SUPPRESS_EMBEDDING_WARNING !== '1') {
|
||||
// stderr so it survives stdout-piped JSON consumers and CI tee.
|
||||
try { process.stderr.write(msg); } catch { /* fall through to log */ }
|
||||
log.warn('Embedding provider degraded to mock — semantic search quality is noise. See stderr banner for fix instructions.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initial probe
|
||||
try {
|
||||
await runProbe();
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
availableProviders = ['mock'];
|
||||
}
|
||||
|
||||
// Ensure activeEmbedder is assigned (TypeScript flow)
|
||||
activeEmbedder ??= createMockEmbedder(dims);
|
||||
|
||||
const instance: EmbeddingProviderInstance = {
|
||||
dimensions: dims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
// Cap input to the active model's context so the backend never rejects
|
||||
// an oversized frame. Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
const capped = capEmbedText(text, maxEmbedCharsForModel(activeModelName));
|
||||
try {
|
||||
const result = await activeEmbedder.embed(capped);
|
||||
recordUsage(1);
|
||||
return result;
|
||||
} catch (err) {
|
||||
log.warn(`Embedding failed with ${activeType}, falling back to mock: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const fallback = mockEmbed(capped, dims);
|
||||
recordUsage(1);
|
||||
return fallback;
|
||||
}
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
// Cap each input first so one oversized frame can't make the backend
|
||||
// reject the whole request. Reverse-ported from OSS hive-mind
|
||||
// (oss-drift triage R5, 2026-06-11).
|
||||
const capped = texts.map(t => capEmbedText(t, maxEmbedCharsForModel(activeModelName)));
|
||||
try {
|
||||
const result = await activeEmbedder.embedBatch(capped);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
} catch (err) {
|
||||
// Skip-not-abort: re-embed per-text so a single backend-rejected input
|
||||
// degrades alone instead of mock-poisoning the WHOLE batch.
|
||||
log.warn(`Batch embedding failed with ${activeType}, re-embedding per-text: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const result = await reembedPerText(activeEmbedder, capped, dims);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
}
|
||||
},
|
||||
|
||||
getStatus(): EmbeddingProviderStatus {
|
||||
return {
|
||||
activeProvider: activeType,
|
||||
availableProviders,
|
||||
dimensions: dims,
|
||||
modelName: activeModelName,
|
||||
lastError,
|
||||
probeTimestamp,
|
||||
};
|
||||
},
|
||||
|
||||
getActiveProvider(): EmbeddingProviderType {
|
||||
return activeType;
|
||||
},
|
||||
|
||||
async reprobe(): Promise<EmbeddingProviderStatus> {
|
||||
await runProbe();
|
||||
return instance.getStatus();
|
||||
},
|
||||
|
||||
getQuotaStatus(): EmbeddingQuotaStatus {
|
||||
return { quota: -1, used: 0, remaining: -1, percentage: 0, resetsAt: getNextMonthReset() };
|
||||
},
|
||||
};
|
||||
|
||||
return instance;
|
||||
}
|
||||
5
packages/core/src/embeddings/embeddings.ts
Normal file
5
packages/core/src/embeddings/embeddings.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export interface Embedder {
|
||||
embed(text: string): Promise<Float32Array>;
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
||||
dimensions: number;
|
||||
}
|
||||
73
packages/core/src/embeddings/inprocess-embedder.ts
Normal file
73
packages/core/src/embeddings/inprocess-embedder.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
||||
* Default provider for ALL desktop users — zero config, works offline.
|
||||
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
||||
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
export interface InProcessEmbedderConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
|
||||
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
|
||||
if (embedding.length === targetDims) return embedding;
|
||||
const result = new Float32Array(targetDims);
|
||||
const copyLen = Math.min(embedding.length, targetDims);
|
||||
result.set(embedding.subarray(0, copyLen));
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
|
||||
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
|
||||
|
||||
const { pipeline } = await import('@huggingface/transformers');
|
||||
const extractor = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
|
||||
});
|
||||
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
||||
|
||||
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
return normalizeDimensions(raw, targetDims);
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
const results: Float32Array[] = [];
|
||||
// Process one at a time to avoid memory issues with large batches
|
||||
for (const text of texts) {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
results.push(normalizeDimensions(raw, targetDims));
|
||||
}
|
||||
return results;
|
||||
},
|
||||
};
|
||||
}
|
||||
93
packages/core/src/embeddings/litellm-embedder.ts
Normal file
93
packages/core/src/embeddings/litellm-embedder.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* LiteLLM-backed embedder that calls the /embeddings endpoint.
|
||||
* Falls back to a deterministic mock (text→Float32Array hash) on API error
|
||||
* when `fallbackToMock` is enabled.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
|
||||
export interface LiteLLMEmbedderConfig {
|
||||
litellmUrl: string;
|
||||
litellmApiKey?: string;
|
||||
model?: string;
|
||||
dimensions?: number;
|
||||
/** Custom fetch implementation (for testing). */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** If true, falls back to a deterministic mock on API error instead of throwing. */
|
||||
fallbackToMock?: boolean;
|
||||
}
|
||||
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
export function createLiteLLMEmbedder(config: LiteLLMEmbedderConfig): Embedder {
|
||||
const {
|
||||
litellmUrl,
|
||||
litellmApiKey,
|
||||
model = 'text-embedding',
|
||||
dimensions = 1024,
|
||||
fetch: fetchFn = globalThis.fetch,
|
||||
fallbackToMock = false,
|
||||
} = config;
|
||||
|
||||
// Normalise base URL — strip trailing /v1 if present, we add it ourselves
|
||||
const baseUrl = litellmUrl.replace(/\/v1\/?$/, '');
|
||||
const url = `${baseUrl}/v1/embeddings`;
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
if (litellmApiKey) {
|
||||
headers['Authorization'] = `Bearer ${litellmApiKey}`;
|
||||
}
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const body = JSON.stringify({ model, input });
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchFn(url, { method: 'POST', headers, body });
|
||||
} catch (err) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
const text = await response.text();
|
||||
throw new Error(`LiteLLM embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = (await response.json()) as {
|
||||
data: Array<{ embedding: number[] }>;
|
||||
};
|
||||
|
||||
return json.data.map((d) => new Float32Array(d.embedding));
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
58
packages/core/src/embeddings/ollama-embedder.ts
Normal file
58
packages/core/src/embeddings/ollama-embedder.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Ollama-backed embedder — calls local Ollama server for embeddings.
|
||||
* Power user option for users who have Ollama installed.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface OllamaEmbedderConfig {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
export function createOllamaEmbedder(config?: Partial<OllamaEmbedderConfig>): Embedder {
|
||||
const baseUrl = config?.baseUrl ?? 'http://localhost:11434';
|
||||
const model = config?.model ?? 'nomic-embed-text';
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
const url = `${baseUrl}/api/embed`;
|
||||
|
||||
async function callOllama(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`Ollama embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { embeddings: number[][] };
|
||||
return json.embeddings.map(e => normalizeDimensions(new Float32Array(e), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callOllama(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callOllama(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
258
packages/core/src/embeddings/transformers-model-load.ts
Normal file
258
packages/core/src/embeddings/transformers-model-load.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
|
||||
const LOCK_RETRY_MIN_MS = 35;
|
||||
const LOCK_RETRY_JITTER_MS = 30;
|
||||
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
|
||||
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
|
||||
const LOCK_DIRECTORY = '.waggle-model-locks';
|
||||
|
||||
export interface TransformersModelLoadOptions<T> {
|
||||
cacheDir: string;
|
||||
model: string;
|
||||
load: (canonicalCacheDir: string) => Promise<T>;
|
||||
lockTimeoutMs?: number;
|
||||
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function normalizeLockKey(value: string): string {
|
||||
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
|
||||
}
|
||||
|
||||
export function modelLoadLockPath(cacheDir: string, model: string): string {
|
||||
const modelHash = createHash('sha256')
|
||||
.update(normalizeLockKey(model))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
|
||||
}
|
||||
|
||||
function isSqliteBusy(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !('code' in error)) return false;
|
||||
const code = String((error as Error & { code?: unknown }).code);
|
||||
return code === 'SQLITE_BUSY'
|
||||
|| code === 'SQLITE_BUSY_SNAPSHOT'
|
||||
|| code === 'SQLITE_LOCKED';
|
||||
}
|
||||
|
||||
async function acquireCrossProcessLock(
|
||||
database: Database.Database,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isSqliteBusy(error)) throw error;
|
||||
if (performance.now() >= deadline) {
|
||||
throw new Error(
|
||||
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
|
||||
}
|
||||
}
|
||||
|
||||
function isWithin(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative !== ''
|
||||
&& relative !== '..'
|
||||
&& !relative.startsWith(`..${path.sep}`)
|
||||
&& !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function isWithinOrEqual(root: string, candidate: string): boolean {
|
||||
return root === candidate || isWithin(root, candidate);
|
||||
}
|
||||
|
||||
function assertUnlinkedPath(root: string, target: string): void {
|
||||
const relative = path.relative(root, target);
|
||||
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error('Model cache path escapes its cache root');
|
||||
}
|
||||
|
||||
let current = root;
|
||||
for (const segment of relative.split(path.sep)) {
|
||||
current = path.join(current, segment);
|
||||
const stats = fs.lstatSync(current);
|
||||
if (stats.isSymbolicLink()) {
|
||||
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateQuarantinePaths(
|
||||
canonicalCacheDir: string,
|
||||
modelDir: string,
|
||||
reportedOnnxPath: string,
|
||||
): { modelDir: string; quarantineRoot: string } | null {
|
||||
if (!path.isAbsolute(reportedOnnxPath)) return null;
|
||||
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
|
||||
|
||||
assertUnlinkedPath(canonicalCacheDir, modelDir);
|
||||
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
|
||||
|
||||
const quarantineRoot = path.dirname(modelDir);
|
||||
const modelStats = fs.lstatSync(modelDir);
|
||||
const reportStats = fs.lstatSync(reportedOnnxPath);
|
||||
const rootStats = fs.lstatSync(quarantineRoot);
|
||||
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
|
||||
|
||||
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
|
||||
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
|
||||
const realModelDir = fs.realpathSync.native(modelDir);
|
||||
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
|
||||
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|
||||
|| !isWithin(realQuarantineRoot, realModelDir)
|
||||
|| path.dirname(realModelDir) !== realQuarantineRoot
|
||||
|| !isWithin(realModelDir, realOnnxPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
|
||||
}
|
||||
|
||||
function quarantineCorruptModel(
|
||||
canonicalCacheDir: string,
|
||||
model: string,
|
||||
reportedOnnxPath: string,
|
||||
): string | null {
|
||||
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
|
||||
|
||||
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
|
||||
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!firstValidation) return null;
|
||||
|
||||
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
|
||||
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!finalValidation) return null;
|
||||
|
||||
const quarantineDir = path.join(
|
||||
finalValidation.quarantineRoot,
|
||||
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
|
||||
);
|
||||
fs.renameSync(finalValidation.modelDir, quarantineDir);
|
||||
return quarantineDir;
|
||||
}
|
||||
|
||||
function reportedCorruptOnnxPath(error: unknown): string | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function notifyQuarantine(
|
||||
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
|
||||
quarantineDir: string,
|
||||
): void {
|
||||
try {
|
||||
const notification = callback?.(quarantineDir);
|
||||
if (notification) {
|
||||
void Promise.resolve(notification).catch(() => undefined);
|
||||
}
|
||||
} catch {
|
||||
// Notification is advisory and must not alter model recovery control flow.
|
||||
}
|
||||
}
|
||||
|
||||
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
|
||||
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
|
||||
const lockDirStats = fs.lstatSync(lockDir);
|
||||
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock path is not a regular directory');
|
||||
}
|
||||
const realLockDir = fs.realpathSync.native(lockDir);
|
||||
if (!isWithin(canonicalCacheDir, realLockDir)) {
|
||||
throw new Error('Local model lock path escapes its cache root');
|
||||
}
|
||||
if (fs.existsSync(lockPath)) {
|
||||
const lockStats = fs.lstatSync(lockPath);
|
||||
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock database is not a regular file');
|
||||
}
|
||||
}
|
||||
|
||||
return new Database(lockPath, { timeout: 0 });
|
||||
}
|
||||
|
||||
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
||||
throw new Error('Local model cache lock timeout must be a non-negative finite number');
|
||||
}
|
||||
|
||||
const requestedCacheDir = path.resolve(options.cacheDir);
|
||||
fs.mkdirSync(requestedCacheDir, { recursive: true });
|
||||
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
|
||||
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
|
||||
let result!: T;
|
||||
let primaryError: unknown;
|
||||
let hasPrimaryError = false;
|
||||
|
||||
try {
|
||||
await acquireCrossProcessLock(lockDatabase, timeoutMs);
|
||||
|
||||
try {
|
||||
result = await options.load(canonicalCacheDir);
|
||||
} catch (firstError) {
|
||||
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
|
||||
if (!reportedOnnxPath) throw firstError;
|
||||
|
||||
let quarantineDir: string | null = null;
|
||||
try {
|
||||
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
|
||||
} catch {
|
||||
// Preserve the original loader error if safe quarantine cannot complete.
|
||||
}
|
||||
if (!quarantineDir) throw firstError;
|
||||
|
||||
notifyQuarantine(options.onQuarantine, quarantineDir);
|
||||
result = await options.load(canonicalCacheDir);
|
||||
}
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
hasPrimaryError = true;
|
||||
}
|
||||
|
||||
let cleanupError: unknown;
|
||||
let hasCleanupError = false;
|
||||
if (lockDatabase.inTransaction) {
|
||||
try {
|
||||
lockDatabase.exec('ROLLBACK');
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
lockDatabase.close();
|
||||
} catch (error) {
|
||||
if (!hasCleanupError) cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
|
||||
if (hasPrimaryError) throw primaryError;
|
||||
if (hasCleanupError) throw cleanupError;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes one model/cache load across workers and processes. SQLite owns
|
||||
* the OS lock, so process termination releases it without PID/age heuristics.
|
||||
*/
|
||||
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
return runModelLoad(options);
|
||||
}
|
||||
Reference in New Issue
Block a user