From 2d507dc45704ceadecdd4e413b076cfd4c5c1f5b Mon Sep 17 00:00:00 2001 From: "shumeiko.it" Date: Sat, 24 Oct 2026 11:57:18 +0200 Subject: [PATCH] fix(agent): make the recall reranker opt-in and use the word embedder in tests The reference-set comparison showed the cross-encoder adds nothing to mem0's ranking and costs about 130 ms per search. --- packages/agent/src/orchestrator.ts | 20 +++++++++++--------- vitest.setup.ts | 14 ++++++++++---- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/agent/src/orchestrator.ts b/packages/agent/src/orchestrator.ts index 91e27ab..207fbfd 100644 --- a/packages/agent/src/orchestrator.ts +++ b/packages/agent/src/orchestrator.ts @@ -78,10 +78,12 @@ export interface OrchestratorConfig { version?: string; skills?: string[]; /** - * W4.2: optional cross-encoder reranker injected for tests. When absent, - * a lazy in-process reranker (Xenova/ms-marco-MiniLM-L-6-v2, ~22MB ONNX) - * is created on first recall unless WAGGLE_RERANKER=0; creation failure - * soft-fails to the store's own ordering. + * W4.2: optional cross-encoder reranker injected for tests. When absent and + * WAGGLE_RERANKER=1, a lazy in-process reranker (Xenova/ms-marco-MiniLM-L-6-v2, + * ~22MB ONNX) is created on first recall; creation failure soft-fails to the + * store's own ordering. Off by default: on the reference set it did not + * improve mem0's ranking and cost about 130 ms per search + * (docs/oss/research/memory-search-quality.md). */ reranker?: Reranker; /** Optional managed cache root for the lazy in-process reranker model. */ @@ -366,14 +368,14 @@ export class Orchestrator { } /** - * W4.2/W4.5: lazy cross-encoder reranker — default on. Kill switch: - * WAGGLE_RERANKER=0. First use downloads the ~22MB model (cached at the - * configured managed path); creation failure (offline, OOM) memoizes - * undefined: recall soft-fails to the store's ordering, never throws. + * W4.2: lazy cross-encoder reranker, opt-in with WAGGLE_RERANKER=1. First use + * downloads the ~22MB model (cached at the configured managed path); creation + * failure (offline, OOM) memoizes undefined: recall soft-fails to the store's + * ordering, never throws. */ private getReranker(): Promise { if (this.rerankerPromise) return this.rerankerPromise; - if (process.env['WAGGLE_RERANKER'] === '0') { + if (process.env['WAGGLE_RERANKER'] !== '1') { this.rerankerPromise = Promise.resolve(undefined); return this.rerankerPromise; } diff --git a/vitest.setup.ts b/vitest.setup.ts index 3d0b2ed..d2d5a00 100644 --- a/vitest.setup.ts +++ b/vitest.setup.ts @@ -28,14 +28,20 @@ if (!process.env.WAGGLE_PHASE5_CANARY_PCT) { process.env.WAGGLE_PHASE5_CANARY_PCT = '0'; } -// W4.5: the cross-encoder reranker is default-ON in production (kill switch -// WAGGLE_RERANKER=0). Tests pin it OFF — the real ONNX model is a ~22MB -// download + per-call CPU inference, both unwanted in unit suites. Tests -// that exercise reranking inject a mock Reranker via OrchestratorConfig. +// The cross-encoder reranker is opt-in (WAGGLE_RERANKER=1). Tests pin it +// OFF explicitly; tests that exercise reranking inject a mock Reranker via +// OrchestratorConfig. if (!process.env.WAGGLE_RERANKER) { process.env.WAGGLE_RERANKER = '0'; } +// Memory (ADR-0007): the sidecar embeds memories with an in-process model +// that is a ~90MB download. Tests use the deterministic word embedder, which +// needs no model and no network. +if (!process.env.WAGGLE_MEMORY_EMBEDDER) { + process.env.WAGGLE_MEMORY_EMBEDDER = 'words'; +} + // W4 (2026-06-11): the PromptAssembler is default-ON in production (kill // switch WAGGLE_PROMPT_ASSEMBLER=0) after the live chat smoke. Tests pin it // OFF so the broad chat-route suite keeps exercising the stable legacy