feat(memory): refuse to open a memory store with a different embedder

Vectors of different models are not comparable even at the same size; the
store records its embedder in waggle-memory.json and names the mismatch.
Also drop the weaver references from the cli and server tsconfigs.
This commit is contained in:
shumeiko.it
2026-10-24 12:06:10 +02:00
parent 8b5624520c
commit 8555d84b75
6 changed files with 55 additions and 9 deletions

View File

@@ -13,6 +13,8 @@ import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-embedder');
const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
export interface InProcessEmbedderConfig {
model?: string;
cacheDir?: string;
@@ -29,7 +31,7 @@ export function normalizeDimensions(embedding: Float32Array, targetDims: number)
}
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
const model = config?.model ?? DEFAULT_MODEL;
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
const targetDims = config?.targetDimensions ?? 1024;
@@ -79,13 +81,15 @@ export const MEMORY_EMBEDDING_DIMENSIONS = 384;
* The embedder memory uses: the in-process model, loaded on first use. Every
* process that opens a data directory's memory must use the same one.
*/
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder {
return createLazyEmbedder(
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder & { id: string } {
const model = options.model ?? DEFAULT_MODEL;
const embedder = createLazyEmbedder(
() => createInProcessEmbedder({
model: options.model,
model,
cacheDir: options.cacheDir,
targetDimensions: MEMORY_EMBEDDING_DIMENSIONS,
}),
MEMORY_EMBEDDING_DIMENSIONS,
);
return { ...embedder, id: `inprocess:${model}:${MEMORY_EMBEDDING_DIMENSIONS}` };
}