import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { createInProcessReranker, type Reranker } from '@waggle/core'; import { createInMemoryMemoryStore, PERSONAL_SCOPE, type MemoryStore } from '@waggle/memory-contract'; import { Orchestrator } from '../src/orchestrator.js'; vi.mock('@waggle/core', async (importOriginal) => ({ ...await importOriginal(), createInProcessReranker: vi.fn(), })); /** * W4.2 — reranker wiring in recallMemory. The orchestrator accepts an injected * Reranker (tests) and otherwise creates the in-process one lazily, only when * WAGGLE_RERANKER=1. It is off by default: on the reference set it did not * improve mem0's ranking (docs/oss/research/memory-search-quality.md). */ describe('W4.2 — recallMemory reranker wiring', () => { let memory: MemoryStore; beforeEach(() => { memory = createInMemoryMemoryStore(); }); afterEach(() => { vi.clearAllMocks(); vi.unstubAllEnvs(); }); function markerReranker(marker: string, calls: { n: number }): Reranker { const scoreOf = (doc: string): number => (doc.includes(marker) ? 10 : -10); return { score: async (_q, doc) => { calls.n++; return scoreOf(doc); }, scoreBatch: async (_q, docs) => { calls.n++; return docs.map(scoreOf); }, }; } async function seedWeeklyReport(): Promise { await memory.remember({ scope: PERSONAL_SCOPE, text: 'User preference: weekly report goes out on Fridays' }); await memory.remember({ scope: PERSONAL_SCOPE, text: 'User preference: weekly report wraps before standup' }); } it('skips search and lazy reranker startup when memory is empty', async () => { vi.stubEnv('WAGGLE_RERANKER', '1'); const search = vi.spyOn(memory, 'search'); const orchestrator = new Orchestrator({ memory }); orchestrator.setWorkspace('ws-1'); const result = await orchestrator.recallMemory('weekly report'); expect(result).toEqual({ text: '', count: 0, recalled: [], recalledFrames: [] }); expect(createInProcessReranker).not.toHaveBeenCalled(); expect(search).not.toHaveBeenCalled(); }); it('uses an injected reranker to order recall results', async () => { const calls = { n: 0 }; await seedWeeklyReport(); const orchestrator = new Orchestrator({ memory, reranker: markerReranker('standup', calls) }); const result = await orchestrator.recallMemory('weekly report Fridays'); expect(calls.n).toBeGreaterThan(0); // reranker actually consulted // The store ranks the Fridays memory first (more shared words); the reranker wins. const standupIdx = result.text.indexOf('standup'); const fridayIdx = result.text.indexOf('goes out on Fridays'); expect(standupIdx).toBeGreaterThan(-1); expect(fridayIdx).toBeGreaterThan(-1); expect(standupIdx).toBeLessThan(fridayIdx); }); it('reranks a wider pool than the requested limit', async () => { await seedWeeklyReport(); const orchestrator = new Orchestrator({ memory, reranker: markerReranker('standup', { n: 0 }) }); // With limit 1 the store alone would return only the Fridays memory. const result = await orchestrator.recallMemory('weekly report Fridays', 1); expect(result.text).toContain('standup'); expect(result.text).not.toContain('goes out on Fridays'); }); it('keeps the store ordering when the reranker throws', async () => { await seedWeeklyReport(); const failing: Reranker = { score: async () => { throw new Error('model crashed'); }, scoreBatch: async () => { throw new Error('model crashed'); }, }; const orchestrator = new Orchestrator({ memory, reranker: failing }); const result = await orchestrator.recallMemory('weekly report Fridays'); expect(result.count).toBe(2); expect(result.text.indexOf('goes out on Fridays')).toBeLessThan(result.text.indexOf('standup')); }); it.each([ ['unset', undefined], ['the kill switch', '0'], ['a value other than 1', 'true'], ])('does not create a reranker when WAGGLE_RERANKER is %s', async (_label, value) => { vi.stubEnv('WAGGLE_RERANKER', value); await seedWeeklyReport(); const orchestrator = new Orchestrator({ memory }); const result = await orchestrator.recallMemory('weekly report'); expect(result.count).toBe(2); expect(createInProcessReranker).not.toHaveBeenCalled(); }); it('creates the lazy reranker once, with the managed cache directory, when WAGGLE_RERANKER=1', async () => { vi.stubEnv('WAGGLE_RERANKER', '1'); const calls = { n: 0 }; vi.mocked(createInProcessReranker).mockResolvedValueOnce(markerReranker('Fridays', calls)); const rerankerCacheDir = 'C:\\Waggle\\models\\reranker'; const orchestrator = new Orchestrator({ memory, rerankerCacheDir }); await seedWeeklyReport(); await orchestrator.recallMemory('weekly report'); await orchestrator.recallMemory('weekly report'); expect(createInProcessReranker).toHaveBeenCalledTimes(1); expect(createInProcessReranker).toHaveBeenCalledWith({ cacheDir: rerankerCacheDir }); expect(calls.n).toBeGreaterThan(0); }); it('falls back to the store ordering when the lazy reranker cannot be created', async () => { vi.stubEnv('WAGGLE_RERANKER', '1'); vi.mocked(createInProcessReranker).mockRejectedValueOnce(new Error('offline')); await seedWeeklyReport(); const orchestrator = new Orchestrator({ memory }); const result = await orchestrator.recallMemory('weekly report'); expect(result.count).toBe(2); expect(result.text).not.toContain('temporarily unavailable'); }); });