The community edition defaulted to ~/.waggle, the data directory of the commercial Waggle, so both editions would open the same config, vault, waggle.db, marketplace catalog and skills. Default to ~/.waggle-community instead (ADR-0011). WAGGLE_DATA_DIR still overrides it wherever it did; nothing is migrated from ~/.waggle. The name is defined once as DEFAULT_DATA_DIR_NAME and DEFAULT_DATA_DIR_LABEL in @waggle/shared, with defaultDataDir() in @waggle/core for the sidecar, the CLI, the agent and the model caches. The marketplace installs on its own without workspace packages, so it keeps a copy in its data-dir.ts; the CLI help, which loads no dependency, spells the path out and its runtime test checks it. The marketplace security scan also flags skills that reference the new skills folder. Tests that asserted the old default now use the constant, and comments name the new directory.
96 lines
3.6 KiB
TypeScript
96 lines
3.6 KiB
TypeScript
/**
|
|
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
|
* Default provider for ALL desktop users — zero config, works offline.
|
|
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
|
* Downloads ~90MB fp32 model on first use, cached in <data dir>/models/.
|
|
*/
|
|
|
|
import path from 'node:path';
|
|
import { defaultDataDir } from '../config.js';
|
|
import { createLazyEmbedder, type Embedder } from './embeddings.js';
|
|
import { createCoreLogger } from '../logger.js';
|
|
import { withTransformersModelLoad } from './transformers-model-load.js';
|
|
|
|
const log = createCoreLogger('inprocess-embedder');
|
|
|
|
const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
|
|
|
export interface InProcessEmbedderConfig {
|
|
model?: string;
|
|
cacheDir?: string;
|
|
targetDimensions?: number;
|
|
}
|
|
|
|
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
|
|
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
|
|
if (embedding.length === targetDims) return embedding;
|
|
const result = new Float32Array(targetDims);
|
|
const copyLen = Math.min(embedding.length, targetDims);
|
|
result.set(embedding.subarray(0, copyLen));
|
|
return result;
|
|
}
|
|
|
|
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
|
|
const model = config?.model ?? DEFAULT_MODEL;
|
|
const cacheDir = config?.cacheDir ?? path.join(defaultDataDir(), 'models');
|
|
const targetDims = config?.targetDimensions ?? 1024;
|
|
|
|
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
|
|
|
|
const { pipeline } = await import('@huggingface/transformers');
|
|
const extractor = await withTransformersModelLoad({
|
|
cacheDir,
|
|
model,
|
|
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
|
|
dtype: 'fp32',
|
|
cache_dir: canonicalCacheDir,
|
|
}),
|
|
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
|
|
});
|
|
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
|
|
|
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
|
|
|
return {
|
|
dimensions: targetDims,
|
|
|
|
async embed(text: string): Promise<Float32Array> {
|
|
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
|
const raw = new Float32Array(result.data as Float32Array);
|
|
return normalizeDimensions(raw, targetDims);
|
|
},
|
|
|
|
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
|
if (texts.length === 0) return [];
|
|
const results: Float32Array[] = [];
|
|
// Process one at a time to avoid memory issues with large batches
|
|
for (const text of texts) {
|
|
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
|
const raw = new Float32Array(result.data as Float32Array);
|
|
results.push(normalizeDimensions(raw, targetDims));
|
|
}
|
|
return results;
|
|
},
|
|
};
|
|
}
|
|
|
|
/** Memory vectors keep all-MiniLM-L6-v2's own size, unpadded (ADR-0007). */
|
|
export const MEMORY_EMBEDDING_DIMENSIONS = 384;
|
|
|
|
/**
|
|
* The embedder memory uses: the in-process model, loaded on first use. Every
|
|
* process that opens a data directory's memory must use the same one.
|
|
*/
|
|
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder & { id: string } {
|
|
const model = options.model ?? DEFAULT_MODEL;
|
|
const embedder = createLazyEmbedder(
|
|
() => createInProcessEmbedder({
|
|
model,
|
|
cacheDir: options.cacheDir,
|
|
targetDimensions: MEMORY_EMBEDDING_DIMENSIONS,
|
|
}),
|
|
MEMORY_EMBEDDING_DIMENSIONS,
|
|
);
|
|
return { ...embedder, id: `inprocess:${model}:${MEMORY_EMBEDDING_DIMENSIONS}` };
|
|
}
|