Files
waggle-os/packages/agent/tests/w42-reranker-recall.test.ts
shumeiko.it ac23bbd7a6 test(agent): move the agent suites onto the memory contract (P1.5)
Rewrites the orchestrator, recall, compaction, save_memory and assembler
suites for MemoryStore (createInMemoryMemoryStore; mem0 with the word
embedder for the M3b integration test), string ids, async prompt building
and setWorkspace(). Creation times are set with fake timers, since a write
cannot set its own. Stores of Waggle's own data (identity, traces) are built
on WaggleDB; the session-manager stubs use the new create() signature.

Kept behaviour: importance lane (cap, active scope, temporary exclusion),
profile and event lanes, date window and its fallback, catch-up, injection
blocking of recall and preloaded context, autosave safety, decision consent,
compaction update-in-place, reranker wiring.

New tests: recall with a workspace active versus personal only;
fetchRecentMemories ordering; toRecallFrame; context loader; save_memory
routing, dedup, rate limit, dramatic-claim downgrade and contradiction
scope; search_memory and search_all_workspaces; pattern write-back kinds,
scopes and sign gate; read_other_workspace; reranker off unless
WAGGLE_RERANKER=1, wider rerank pool, failure fallbacks.

Deleted, because the behaviour is gone:
- w46-rawdetail-recall.test.ts: the raw-turn (RAWDETAIL) recall lane.
- orchestrator.test.ts, query_knowledge (3 tests) and the recall-hardening
  M4 "Key Knowledge" test: the knowledge graph is not in v1 (ADR-0002).
- orchestrator-context-frames "profile override": scoring profiles.
- Assertions on I/P frame types, GOP sessions, FTS rows, the "Memory Facts"
  lane, cognify stages and session/entity counts were replaced by the
  equivalent contract-level checks.
2026-10-24 13:19:06 +02:00

136 lines
5.5 KiB
TypeScript

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { createInProcessReranker, type Reranker } from '@waggle/core';
import { createInMemoryMemoryStore, PERSONAL_SCOPE, type MemoryStore } from '@waggle/memory-contract';
import { Orchestrator } from '../src/orchestrator.js';
vi.mock('@waggle/core', async (importOriginal) => ({
...await importOriginal<typeof import('@waggle/core')>(),
createInProcessReranker: vi.fn(),
}));
/**
* W4.2 — reranker wiring in recallMemory. The orchestrator accepts an injected
* Reranker (tests) and otherwise creates the in-process one lazily, only when
* WAGGLE_RERANKER=1. It is off by default: on the reference set it did not
* improve mem0's ranking (docs/oss/research/memory-search-quality.md).
*/
describe('W4.2 — recallMemory reranker wiring', () => {
let memory: MemoryStore;
beforeEach(() => {
memory = createInMemoryMemoryStore();
});
afterEach(() => {
vi.clearAllMocks();
vi.unstubAllEnvs();
});
function markerReranker(marker: string, calls: { n: number }): Reranker {
const scoreOf = (doc: string): number => (doc.includes(marker) ? 10 : -10);
return {
score: async (_q, doc) => { calls.n++; return scoreOf(doc); },
scoreBatch: async (_q, docs) => { calls.n++; return docs.map(scoreOf); },
};
}
async function seedWeeklyReport(): Promise<void> {
await memory.remember({ scope: PERSONAL_SCOPE, text: 'User preference: weekly report goes out on Fridays' });
await memory.remember({ scope: PERSONAL_SCOPE, text: 'User preference: weekly report wraps before standup' });
}
it('skips search and lazy reranker startup when memory is empty', async () => {
vi.stubEnv('WAGGLE_RERANKER', '1');
const search = vi.spyOn(memory, 'search');
const orchestrator = new Orchestrator({ memory });
orchestrator.setWorkspace('ws-1');
const result = await orchestrator.recallMemory('weekly report');
expect(result).toEqual({ text: '', count: 0, recalled: [], recalledFrames: [] });
expect(createInProcessReranker).not.toHaveBeenCalled();
expect(search).not.toHaveBeenCalled();
});
it('uses an injected reranker to order recall results', async () => {
const calls = { n: 0 };
await seedWeeklyReport();
const orchestrator = new Orchestrator({ memory, reranker: markerReranker('standup', calls) });
const result = await orchestrator.recallMemory('weekly report Fridays');
expect(calls.n).toBeGreaterThan(0); // reranker actually consulted
// The store ranks the Fridays memory first (more shared words); the reranker wins.
const standupIdx = result.text.indexOf('standup');
const fridayIdx = result.text.indexOf('goes out on Fridays');
expect(standupIdx).toBeGreaterThan(-1);
expect(fridayIdx).toBeGreaterThan(-1);
expect(standupIdx).toBeLessThan(fridayIdx);
});
it('reranks a wider pool than the requested limit', async () => {
await seedWeeklyReport();
const orchestrator = new Orchestrator({ memory, reranker: markerReranker('standup', { n: 0 }) });
// With limit 1 the store alone would return only the Fridays memory.
const result = await orchestrator.recallMemory('weekly report Fridays', 1);
expect(result.text).toContain('standup');
expect(result.text).not.toContain('goes out on Fridays');
});
it('keeps the store ordering when the reranker throws', async () => {
await seedWeeklyReport();
const failing: Reranker = {
score: async () => { throw new Error('model crashed'); },
scoreBatch: async () => { throw new Error('model crashed'); },
};
const orchestrator = new Orchestrator({ memory, reranker: failing });
const result = await orchestrator.recallMemory('weekly report Fridays');
expect(result.count).toBe(2);
expect(result.text.indexOf('goes out on Fridays')).toBeLessThan(result.text.indexOf('standup'));
});
it.each([
['unset', undefined],
['the kill switch', '0'],
['a value other than 1', 'true'],
])('does not create a reranker when WAGGLE_RERANKER is %s', async (_label, value) => {
vi.stubEnv('WAGGLE_RERANKER', value);
await seedWeeklyReport();
const orchestrator = new Orchestrator({ memory });
const result = await orchestrator.recallMemory('weekly report');
expect(result.count).toBe(2);
expect(createInProcessReranker).not.toHaveBeenCalled();
});
it('creates the lazy reranker once, with the managed cache directory, when WAGGLE_RERANKER=1', async () => {
vi.stubEnv('WAGGLE_RERANKER', '1');
const calls = { n: 0 };
vi.mocked(createInProcessReranker).mockResolvedValueOnce(markerReranker('Fridays', calls));
const rerankerCacheDir = 'C:\\Waggle\\models\\reranker';
const orchestrator = new Orchestrator({ memory, rerankerCacheDir });
await seedWeeklyReport();
await orchestrator.recallMemory('weekly report');
await orchestrator.recallMemory('weekly report');
expect(createInProcessReranker).toHaveBeenCalledTimes(1);
expect(createInProcessReranker).toHaveBeenCalledWith({ cacheDir: rerankerCacheDir });
expect(calls.n).toBeGreaterThan(0);
});
it('falls back to the store ordering when the lazy reranker cannot be created', async () => {
vi.stubEnv('WAGGLE_RERANKER', '1');
vi.mocked(createInProcessReranker).mockRejectedValueOnce(new Error('offline'));
await seedWeeklyReport();
const orchestrator = new Orchestrator({ memory });
const result = await orchestrator.recallMemory('weekly report');
expect(result.count).toBe(2);
expect(result.text).not.toContain('temporarily unavailable');
});
});