test(memory): compare search quality of the old engine and mem0 on a reference set (P1.5)

Add a reference set (100 invented memories across personal memory and two
workspaces, 67 queries: paraphrase, keyword, multi-fact, scope, no-answer)
and a tsx benchmark that loads it into the old engine (MindDB, FrameStore,
HybridSearch, with and without the cross-encoder reranker), the mem0 store
and a plain-cosine baseline, all on Waggle's in-process embedder. It reports
recall@1, recall@5, MRR, latency and misses per engine and query type. The
mem0 half runs on its own (--engines mem0,cosine) once the old engine is
removed. Ollama and the user's own key are optional embedders; the bench
refuses network access to anything else and never reads OPENAI_API_KEY.

Results in docs/oss/research/memory-search-quality.md: mem0 matches the old
engine's chat-recall path (recall@1 0.72, recall@5 0.89 vs 0.90, MRR 0.82
vs 0.83) at a fraction of its latency and is acceptable for v1.

No new dependencies.
This commit is contained in:
shumeiko.it
2026-10-24 11:55:39 +02:00
parent 784ca770b1
commit f046a20b34
3 changed files with 1107 additions and 0 deletions

View File

@@ -0,0 +1,188 @@
{
"version": 1,
"description": "Reference set for comparing memory search quality (P1.5). A fictional freelance designer, Mila Sorensen, with personal memories and two client workspaces: 'atlas' (a route-planning app for a freight company) and 'harbor' (a dashboard for a food bank). All people, companies and events are invented. Scope keys follow the memory contract: 'personal' or 'workspace:<id>'. A query's 'relevant' lists every memory that answers it; an empty list means memory holds no answer. Personal queries are mostly in the first person, as a user asks in chat (chat recall searches with the user's message); some use the name Mila, as the agent does when it writes its own memory search.",
"queryTypes": {
"paraphrase": "The query shares no content word with the memory it should find.",
"keyword": "The query names an exact term from the memory: a person, product, code or identifier.",
"multi-fact": "Two to four memories together answer the query.",
"scope": "The answer depends on the scopes searched; other scopes hold competing answers.",
"no-answer": "No memory answers the query."
},
"memories": [
{ "id": "p01", "scope": "personal", "kind": "profile", "importance": "important", "text": "The user's name is Mila Sorensen. She likes to be addressed as Mila." },
{ "id": "p02", "scope": "personal", "kind": "profile", "importance": "important", "text": "Mila lives in Ljubljana, in a flat near Tivoli Park." },
{ "id": "p03", "scope": "personal", "kind": "profile", "importance": "important", "text": "Mila is a freelance product designer and runs a two-person studio." },
{ "id": "p04", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila speaks Slovenian, English and German, and is learning Italian on weekday evenings." },
{ "id": "p05", "scope": "personal", "kind": "profile", "importance": "critical", "text": "Mila is severely allergic to peanuts and always carries an epinephrine auto-injector." },
{ "id": "p06", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila's birthday is on 9 November." },
{ "id": "p07", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila has a rescue greyhound named Pixel." },
{ "id": "p08", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila's sister Ana lives in Graz and works as a veterinarian." },
{ "id": "p09", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers short answers with bullet points over long paragraphs." },
{ "id": "p10", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila does not take meetings before 10:00; she keeps mornings for focused design work." },
{ "id": "p11", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila uses Figma for interface design and Obsidian for her notes." },
{ "id": "p12", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers dark mode in every app she uses." },
{ "id": "p13", "scope": "personal", "kind": "preference", "importance": "normal", "text": "She asked for a dark editor theme because bright screens give her headaches." },
{ "id": "p14", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila drinks oat-milk flat whites and avoids dairy milk." },
{ "id": "p15", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila eats a mostly vegetarian diet with fish now and then." },
{ "id": "p16", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Emails written for Mila should end with 'Best, Mila' and keep a friendly but direct tone." },
{ "id": "p17", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila wants money amounts shown in euros with a comma as the decimal separator." },
{ "id": "p18", "scope": "personal", "kind": "decision", "importance": "important", "text": "Mila decided to stop taking new clients until the end of March 2026 so she can finish current projects." },
{ "id": "p19", "scope": "personal", "kind": "decision", "importance": "normal", "text": "Mila chose the Oakline standing desk over the cheaper Deskmate model because its motor is quieter." },
{ "id": "p20", "scope": "personal", "kind": "decision", "importance": "normal", "text": "In January 2026 Mila moved her personal account from Lipa Bank to the digital bank Kovo." },
{ "id": "p21", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila has a dentist appointment with Dr. Okafor on 4 February 2026 at 15:30." },
{ "id": "p22", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila flies to Lisbon on 18 May 2026 to speak about accessible forms at the UX Tide conference." },
{ "id": "p23", "scope": "personal", "kind": "event", "importance": "normal", "text": "Ana's wedding is on 27 June 2026 in Graz, and Mila is the maid of honour." },
{ "id": "p24", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila ran the Ljubljana half marathon on 26 October 2025 in 1:52." },
{ "id": "p25", "scope": "personal", "kind": "event", "importance": "temporary", "text": "Pixel has a vet check-up on 12 January 2026 for his yearly vaccinations." },
{ "id": "p26", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's studio is called Fernwork Studio and is registered as a sole proprietorship." },
{ "id": "p27", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's accountant is Bojan Zupan; receipts and invoices go to him by the 20th of each month." },
{ "id": "p28", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's laptop is a 14-inch MacBook Pro from 2023, and her backup drive is a 2 TB Samsung T7." },
{ "id": "p29", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's running plan is easy runs on Tuesday and Thursday and a long run on Sunday." },
{ "id": "p30", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila mostly reads science fiction; her favourite author is Ursula K. Le Guin." },
{ "id": "p31", "scope": "personal", "kind": "note", "importance": "temporary", "text": "Reminder: renew the studio domain fernwork.si before 1 March 2026." },
{ "id": "p32", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's hourly rate for design work is 85 euros; workshops are billed at 600 euros per day." },
{ "id": "p33", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers the train over flying for trips shorter than six hours." },
{ "id": "p34", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila is short-sighted and wears glasses for screen work." },
{ "id": "p35", "scope": "personal", "kind": "decision", "importance": "normal", "text": "Mila decided to use Proton Mail for the studio and keep Gmail for personal mail only." },
{ "id": "p36", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila drives a 2017 Škoda Fabia; the next service is due at 120,000 km." },
{ "id": "p37", "scope": "personal", "kind": "event", "importance": "normal", "text": "On 3 December 2025 Mila signed a lease for a shared studio space on Trubarjeva street." },
{ "id": "p38", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila listens to ambient and jazz playlists while working because lyrics distract her." },
{ "id": "p39", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's friend Jure is allergic to cats." },
{ "id": "p40", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Pixel is afraid of thunderstorms and calms down in his anxiety vest." },
{ "id": "p41", "scope": "personal", "kind": "note", "importance": "normal", "text": "Mila wants to learn more about variable fonts this year." },
{ "id": "p42", "scope": "personal", "kind": "fact", "importance": "important", "text": "Mila's emergency contact is her sister Ana." },
{ "id": "p43", "scope": "personal", "kind": "summary", "importance": "normal", "text": "Summary of the 12 January 2026 planning chat: Mila's first-quarter priorities are finishing the Atlas designs, delivering the Harbor dashboard and preparing the Lisbon talk." },
{ "id": "p44", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila uses Slack for quick questions and email for anything that needs a decision." },
{ "id": "a01", "scope": "workspace:atlas", "kind": "fact", "importance": "important", "text": "Atlas is a route-planning mobile app that Fernwork Studio is designing for the delivery drivers of Kestrel Freight." },
{ "id": "a02", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Our contact at Kestrel Freight is Daniel Rusu, their head of operations." },
{ "id": "a03", "scope": "workspace:atlas", "kind": "decision", "importance": "important", "text": "Decision: Atlas is built with React Native so one codebase covers iOS and Android." },
{ "id": "a04", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Decision: the Atlas backend stores data in PostgreSQL with the PostGIS extension for route geometry." },
{ "id": "a05", "scope": "workspace:atlas", "kind": "fact", "importance": "critical", "text": "The Atlas beta must reach the pilot drivers by 15 April 2026; a contract penalty applies after that date." },
{ "id": "a06", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas design budget is 18,000 euros, and 11,500 of it was invoiced by the end of January." },
{ "id": "a07", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Luka Novak is the mobile developer on Atlas." },
{ "id": "a08", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Petra Lindqvist owns the Atlas backend and the routing API." },
{ "id": "a09", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Sven tests Atlas on real devices and writes the release checklists." },
{ "id": "a10", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Drivers asked for large tap targets because they use the app with gloves on, so the minimum button height is 56 points." },
{ "id": "a11", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas has to work offline for up to eight hours, because rural routes have no mobile signal." },
{ "id": "a12", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "The Atlas kickoff workshop with Kestrel Freight took place on 8 December 2025 in Maribor." },
{ "id": "a13", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "The first Atlas usability test is planned for 10 March 2026 with six drivers." },
{ "id": "a14", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas tickets are tracked in Linear with the prefix ATL; ATL-142 is the bug where the map freezes on Android 12." },
{ "id": "a15", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "On 20 January 2026 we chose Mapbox over Google Maps for Atlas because it supports offline map tiles." },
{ "id": "a16", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas brand colours are deep teal #0F4C5C and signal orange #F28C28." },
{ "id": "a17", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas design files are in the Figma project 'Kestrel / Atlas v2'." },
{ "id": "a18", "scope": "workspace:atlas", "kind": "preference", "importance": "normal", "text": "Daniel Rusu wants a weekly status update by email on Friday afternoons." },
{ "id": "a19", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas compresses proof-of-delivery photos to under 300 KB before uploading them." },
{ "id": "a20", "scope": "workspace:atlas", "kind": "note", "importance": "temporary", "text": "Ask Petra whether the routing API can return an ETA for each stop." },
{ "id": "a21", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The pilot drivers expect the first Atlas beta around mid-April." },
{ "id": "a22", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Atlas launches in Slovenian, Croatian and English." },
{ "id": "a23", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Drivers sign in to Atlas with one-time codes sent by SMS; they have no passwords." },
{ "id": "a24", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "On 29 January 2026 Kestrel Freight approved the Atlas navigation flow after the second review." },
{ "id": "a25", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas staging server is atlas-staging.kestrel.internal and is reset every night." },
{ "id": "a26", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Dark mode for Atlas is postponed until after the pilot, since drivers mostly work in daylight." },
{ "id": "a27", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Each Atlas driver handles 40 to 70 stops per shift." },
{ "id": "a28", "scope": "workspace:atlas", "kind": "summary", "importance": "normal", "text": "Summary of the Atlas sprint review on 6 February 2026: offline sync works, the map freeze ATL-142 is still open, and the delivery photo flow needs one more iteration." },
{ "id": "a29", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas crash reports go to Sentry, with an alert when there are more than five crashes per hour." },
{ "id": "a30", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Client meetings for Atlas are on Tuesdays at 11:00 over Google Meet." },
{ "id": "h01", "scope": "workspace:harbor", "kind": "fact", "importance": "important", "text": "Harbor is an analytics dashboard Fernwork Studio is building for the Lindenwood Food Bank to track donations and volunteers." },
{ "id": "h02", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Our contact for Harbor is Grace Mbeki, the food bank's volunteer coordinator." },
{ "id": "h03", "scope": "workspace:harbor", "kind": "decision", "importance": "important", "text": "Decision: Harbor stores its data in SQLite, because the food bank has no IT staff to run a database server." },
{ "id": "h04", "scope": "workspace:harbor", "kind": "fact", "importance": "critical", "text": "Harbor must be ready for the food bank's board meeting on 2 March 2026." },
{ "id": "h05", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor budget is 7,500 euros, paid half at kickoff and half on delivery." },
{ "id": "h06", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor draws its charts with Chart.js; the food bank's laptops are old, so heavy libraries are avoided." },
{ "id": "h07", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor imports donation records from a shared Google Sheet every night at 02:00." },
{ "id": "h08", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Volunteers sign in to Harbor with a magic link sent by email." },
{ "id": "h09", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "The Harbor kickoff call with Grace Mbeki was on 15 January 2026." },
{ "id": "h10", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Tomaž Kranjc, a volunteer developer, helps with the Harbor import scripts." },
{ "id": "h11", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "The Harbor dashboard has to meet WCAG 2.1 AA because several volunteers use screen readers." },
{ "id": "h12", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor tracks three headline numbers: kilograms of food donated, active volunteers, and families served each week." },
{ "id": "h13", "scope": "workspace:harbor", "kind": "preference", "importance": "normal", "text": "Grace prefers phone calls to email and is usually free after 16:00." },
{ "id": "h14", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The food bank's brand colour is a warm green, #3A7D44." },
{ "id": "h15", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "On 29 January 2026 the food bank's treasurer reviewed the Harbor data and asked for monthly totals." },
{ "id": "h16", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor dashboard should be finished before the March board meeting." },
{ "id": "h17", "scope": "workspace:harbor", "kind": "note", "importance": "temporary", "text": "Check whether the Google Sheet has a donor postcode column before building the map view." },
{ "id": "h18", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor does not store donor names; it keeps only anonymous donor IDs to protect privacy." },
{ "id": "h19", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor runs on a 5-euro-a-month VPS at Hetzner." },
{ "id": "h20", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor reports can be exported to PDF for the monthly newsletter." },
{ "id": "h21", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor has a dark mode because volunteers use it during evening shifts." },
{ "id": "h22", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor code is in the GitHub repository fernwork/harbor-dashboard." },
{ "id": "h23", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "On 5 February 2026 Mila showed the first Harbor prototype to Grace, who asked for bigger numbers on the main screen." },
{ "id": "h24", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor meetings with the food bank are on Thursdays at 14:00 at their warehouse." },
{ "id": "h25", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The food bank serves around 340 families a week in winter." },
{ "id": "h26", "scope": "workspace:harbor", "kind": "summary", "importance": "normal", "text": "Summary of the Harbor planning chat: SQLite for storage, a nightly import from Google Sheets, magic-link sign-in, and PDF export for the board." }
],
"queries": [
{ "id": "q01", "type": "paraphrase", "scopes": ["personal"], "text": "Is there anything I must never eat?", "relevant": ["p05"] },
{ "id": "q02", "type": "paraphrase", "scopes": ["personal"], "text": "Which city is my home in?", "relevant": ["p02"] },
{ "id": "q03", "type": "paraphrase", "scopes": ["personal"], "text": "What kind of pet do I have?", "relevant": ["p07"] },
{ "id": "q04", "type": "paraphrase", "scopes": ["personal"], "text": "How do I like replies to be formatted?", "relevant": ["p09"] },
{ "id": "q05", "type": "paraphrase", "scopes": ["personal"], "text": "When am I unavailable for calls early in the day?", "relevant": ["p10"] },
{ "id": "q06", "type": "paraphrase", "scopes": ["personal"], "text": "What do I usually order at a café?", "relevant": ["p14"] },
{ "id": "q07", "type": "paraphrase", "scopes": ["personal"], "text": "Would I enjoy a beef burger?", "relevant": ["p15"] },
{ "id": "q08", "type": "paraphrase", "scopes": ["personal"], "text": "Who does my bookkeeping?", "relevant": ["p27"] },
{ "id": "q09", "type": "paraphrase", "scopes": ["personal"], "text": "What do I charge clients?", "relevant": ["p32"] },
{ "id": "q10", "type": "paraphrase", "scopes": ["personal"], "text": "What computer do I work on?", "relevant": ["p28"] },
{ "id": "q11", "type": "paraphrase", "scopes": ["personal"], "text": "How do I like to get to places that are not too far away?", "relevant": ["p33"] },
{ "id": "q12", "type": "paraphrase", "scopes": ["personal"], "text": "What music helps me concentrate?", "relevant": ["p38"] },
{ "id": "q13", "type": "paraphrase", "scopes": ["personal"], "text": "Who should be called if something happens to me?", "relevant": ["p42"] },
{ "id": "q14", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "Why can't the delivery app rely on a network connection?", "relevant": ["a11"] },
{ "id": "q15", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "What did we adjust for people wearing mittens?", "relevant": ["a10"] },
{ "id": "q16", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "Who represents the customer on the route-planning project?", "relevant": ["a02"] },
{ "id": "q17", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "How do couriers show that a parcel was handed over?", "relevant": ["a19"] },
{ "id": "q18", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "How do helpers log in?", "relevant": ["h08"] },
{ "id": "q19", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "Which metrics matter most to the charity?", "relevant": ["h12"] },
{ "id": "q20", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "How do we safeguard the identities of people who give food?", "relevant": ["h18"] },
{ "id": "q21", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "Why was a lightweight graphing package picked?", "relevant": ["h06"] },
{ "id": "q22", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Who is our contact at Kestrel Freight?", "relevant": ["a02"] },
{ "id": "q23", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "ATL-142", "relevant": ["a14", "a28"] },
{ "id": "q24", "type": "keyword", "scopes": ["personal"], "text": "Dr. Okafor", "relevant": ["p21"] },
{ "id": "q25", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "PostGIS", "relevant": ["a04"] },
{ "id": "q26", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Mapbox or Google Maps?", "relevant": ["a15"] },
{ "id": "q27", "type": "keyword", "scopes": ["personal"], "text": "Bojan Zupan", "relevant": ["p27"] },
{ "id": "q28", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "When can I phone Grace Mbeki?", "relevant": ["h13"] },
{ "id": "q29", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "#F28C28", "relevant": ["a16"] },
{ "id": "q30", "type": "keyword", "scopes": ["personal"], "text": "fernwork.si", "relevant": ["p31"] },
{ "id": "q31", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "Chart.js", "relevant": ["h06"] },
{ "id": "q32", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "Hetzner VPS", "relevant": ["h19"] },
{ "id": "q33", "type": "keyword", "scopes": ["personal"], "text": "Trubarjeva", "relevant": ["p37"] },
{ "id": "q34", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Sentry alert threshold", "relevant": ["a29"] },
{ "id": "q35", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "WCAG 2.1 AA", "relevant": ["h11"] },
{ "id": "q36", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "fernwork/harbor-dashboard", "relevant": ["h22"] },
{ "id": "q37", "type": "keyword", "scopes": ["personal"], "text": "Oakline desk", "relevant": ["p19"] },
{ "id": "q38", "type": "keyword", "scopes": ["personal"], "text": "Škoda Fabia service", "relevant": ["p36"] },
{ "id": "q39", "type": "multi-fact", "scopes": ["personal", "workspace:atlas"], "text": "Who is on the Atlas development team?", "relevant": ["a07", "a08", "a09"] },
{ "id": "q40", "type": "multi-fact", "scopes": ["workspace:atlas", "workspace:harbor"], "text": "What are the upcoming project deadlines?", "relevant": ["a05", "a21", "h04", "h16"] },
{ "id": "q41", "type": "multi-fact", "scopes": ["personal"], "text": "What are Mila's dietary restrictions?", "relevant": ["p05", "p14", "p15"] },
{ "id": "q42", "type": "multi-fact", "scopes": ["personal"], "text": "What do I know about Pixel?", "relevant": ["p07", "p25", "p40"] },
{ "id": "q43", "type": "multi-fact", "scopes": ["personal"], "text": "Tell me about my sister.", "relevant": ["p08", "p23", "p42"] },
{ "id": "q44", "type": "multi-fact", "scopes": ["personal", "workspace:harbor"], "text": "What has been decided about the Harbor tech stack?", "relevant": ["h03", "h06", "h19", "h26"] },
{ "id": "q45", "type": "multi-fact", "scopes": ["personal"], "text": "What do I do for running and races?", "relevant": ["p24", "p29"] },
{ "id": "q46", "type": "multi-fact", "scopes": ["personal"], "text": "Which apps does Mila use for design, notes, chat and email?", "relevant": ["p11", "p35", "p44"] },
{ "id": "q47", "type": "multi-fact", "scopes": ["personal", "workspace:atlas"], "text": "What has happened on the Atlas project so far?", "relevant": ["a12", "a24", "a28"] },
{ "id": "q48", "type": "multi-fact", "scopes": ["workspace:atlas", "workspace:harbor"], "text": "When are the regular client meetings for each project?", "relevant": ["a30", "h24"] },
{ "id": "q49", "type": "multi-fact", "scopes": ["personal", "workspace:atlas", "workspace:harbor"], "text": "How big are the budgets of Mila's current projects?", "relevant": ["a06", "h05"] },
{ "id": "q50", "type": "multi-fact", "scopes": ["personal"], "text": "What are Mila's priorities for the first quarter?", "relevant": ["p43", "p18"] },
{ "id": "q51", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Which database do we use?", "relevant": ["a04"] },
{ "id": "q52", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Which database do we use?", "relevant": ["h03"] },
{ "id": "q53", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "When is the deadline?", "relevant": ["a05", "a21"] },
{ "id": "q54", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "When is the deadline?", "relevant": ["h04", "h16"] },
{ "id": "q55", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Does the app have dark mode?", "relevant": ["a26"] },
{ "id": "q56", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Does the app have dark mode?", "relevant": ["h21"] },
{ "id": "q57", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Who is the client contact?", "relevant": ["a02"] },
{ "id": "q58", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Who is the client contact?", "relevant": ["h02"] },
{ "id": "q59", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "What is the brand colour?", "relevant": ["h14"] },
{ "id": "q60", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "How do users sign in?", "relevant": ["a23"] },
{ "id": "q61", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "What does Mila think about dark mode?", "relevant": ["p12", "p13"] },
{ "id": "q62", "type": "no-answer", "scopes": ["personal"], "text": "What is my blood type?", "relevant": [] },
{ "id": "q63", "type": "no-answer", "scopes": ["personal", "workspace:harbor"], "text": "Which CI service runs the Harbor tests?", "relevant": [] },
{ "id": "q64", "type": "no-answer", "scopes": ["personal"], "text": "Which football team do I support?", "relevant": [] },
{ "id": "q65", "type": "no-answer", "scopes": ["personal", "workspace:atlas"], "text": "What is the Atlas rating in the App Store?", "relevant": [] },
{ "id": "q66", "type": "no-answer", "scopes": ["personal"], "text": "Where did Mila go to university?", "relevant": [] },
{ "id": "q67", "type": "no-answer", "scopes": ["personal", "workspace:atlas"], "text": "How many employees does Kestrel Freight have?", "relevant": [] }
]
}

View File

@@ -0,0 +1,568 @@
/**
* Search-quality benchmark for the memory switch (P1.5).
*
* Loads the reference set (`reference-set.json`) into the old engine (MindDB +
* FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind
* substrate) and into the mem0 store, runs every query against each, and prints
* recall@1, recall@5, MRR and latency per engine and per query type. The
* findings are in docs/oss/research/memory-search-quality.md.
*
* Run from the repository root after `npm run build:packages`:
*
* npx tsx packages/memory-mem0/bench/search-quality.ts [options]
*
* --engines <list> Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine.
* Default: every engine this checkout still has. Once the
* hive-mind substrate is removed, `mem0` and `cosine` are
* left (the `-rerank` variants use its cross-encoder).
* `cosine` is a baseline: plain cosine similarity over the
* same embeddings, with no keyword signal.
* --embedder <name> inprocess (default, offline), ollama or openai.
* --limit <n> Results per query (default 10).
* --out <file> Also write the per-query results as JSON.
*
* Embedders:
* inprocess Xenova/all-MiniLM-L6-v2 padded to 1024 dimensions, as in the
* sidecar, loaded from WAGGLE_MODELS_DIR (default ~/.waggle/models).
* ollama OLLAMA_HOST (default http://localhost:11434) and
* OLLAMA_EMBED_MODEL (default nomic-embed-text). Ollama must
* already be running with the model pulled.
* openai The user's own key: BENCH_EMBEDDING_API_KEY (required),
* BENCH_EMBEDDING_MODEL (default text-embedding-3-small) and
* BENCH_EMBEDDING_BASE_URL (default the OpenAI endpoint). The bench
* never reads OPENAI_API_KEY, so a key that happens to be set in
* the environment is not used.
*
* Every fetch() to another machine is refused, except the endpoint of the
* openai embedder. mem0 telemetry is switched off before mem0 is loaded.
*/
import fs from 'node:fs';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract';
// The adapter sets this too; setting it first keeps telemetry off even if
// something loads mem0 earlier than the adapter.
process.env.MEM0_TELEMETRY = 'false';
const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url));
const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const;
type EngineName = (typeof ENGINE_NAMES)[number];
type BackendName = 'old' | 'mem0' | 'cosine';
const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName;
/** Candidates a reranker re-orders; HybridSearch's default pool size. */
const RERANK_POOL = 30;
interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string }
interface QueryFixture { id: string; type: string; scopes: string[]; text: string; relevant: string[] }
interface ReferenceSet { memories: MemoryFixture[]; queries: QueryFixture[] }
interface Embedder {
readonly dimensions: number;
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
}
interface Reranker { scoreBatch(query: string, docs: string[]): Promise<number[]> }
interface Hit { ref: string; text: string; score: number }
interface Backend {
readonly ingestMs: number;
search(text: string, scopes: string[], limit: number, reranker?: Reranker): Promise<Hit[]>;
close(): Promise<void>;
}
/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */
interface LegacyCore {
MindDB?: new (file: string) => { close(): void };
FrameStore?: new (db: unknown) => {
createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number };
};
SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown };
HybridSearch?: new (db: unknown, embedder: Embedder) => {
indexFrame(id: number, content: string): Promise<void>;
search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }):
Promise<Array<{ frame: { id: number; content: string }; finalScore: number }>>;
};
createInProcessReranker?: (config?: { cacheDir?: string }) => Promise<Reranker>;
}
interface Options {
engines: EngineName[] | null;
embedder: 'inprocess' | 'ollama' | 'openai';
limit: number;
out: string | null;
}
function parseArgs(argv: string[]): Options {
const options: Options = { engines: null, embedder: 'inprocess', limit: 10, out: null };
for (let i = 0; i < argv.length; i++) {
const value = argv[i + 1];
switch (argv[i]) {
case '--engines':
options.engines = value.split(',').map((name) => {
if (!(ENGINE_NAMES as readonly string[]).includes(name)) throw new Error(`unknown engine: ${name}`);
return name as EngineName;
});
i++;
break;
case '--embedder':
if (value !== 'inprocess' && value !== 'ollama' && value !== 'openai') {
throw new Error(`unknown embedder: ${value}`);
}
options.embedder = value;
i++;
break;
case '--limit':
options.limit = Number(value);
if (!Number.isInteger(options.limit) || options.limit < 5) throw new Error('--limit must be an integer >= 5');
i++;
break;
case '--out':
options.out = path.resolve(value);
i++;
break;
default:
throw new Error(`unknown argument: ${argv[i]}`);
}
}
return options;
}
// ── Network guard ────────────────────────────────────────────────────────────
const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);
const requests = { local: 0, remote: 0, blocked: [] as string[] };
function guardNetwork(allowedRemoteHost: string | null): void {
const realFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url);
if (LOOPBACK.has(url.hostname)) requests.local++;
else if (url.hostname === allowedRemoteHost) requests.remote++;
else {
requests.blocked.push(url.host);
throw new Error(`search-quality bench: blocked a request to ${url.host}`);
}
return realFetch(input, init);
}) as typeof fetch;
}
// ── Embedders ────────────────────────────────────────────────────────────────
interface EmbedderPlan { label: string; remoteHost: string | null; create(core: Record<string, unknown>): Promise<Embedder> }
function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPlan {
type Factory<C> = (config: C) => Embedder | Promise<Embedder>;
if (kind === 'inprocess') {
return {
label: 'in-process Xenova/all-MiniLM-L6-v2 (384 dims padded to 1024)',
remoteHost: null,
create: async (core) => (core.createInProcessEmbedder as Factory<object>)({
model: 'Xenova/all-MiniLM-L6-v2', cacheDir: modelsDir, targetDimensions: 1024,
}),
};
}
if (kind === 'ollama') {
const baseUrl = process.env.OLLAMA_HOST ?? 'http://localhost:11434';
const model = process.env.OLLAMA_EMBED_MODEL ?? 'nomic-embed-text';
if (!LOOPBACK.has(new URL(baseUrl).hostname)) throw new Error('OLLAMA_HOST must point at this machine');
return {
label: `Ollama ${model} at ${baseUrl} (padded or cut to 1024 dims)`,
remoteHost: null,
create: async (core) => {
const embedder = await (core.createOllamaEmbedder as Factory<object>)({ baseUrl, model, targetDimensions: 1024 });
try {
await embedder.embed('probe');
} catch (error) {
throw new Error(`Ollama does not answer at ${baseUrl} with model ${model}. Start it and pull the model first; `
+ `the bench does neither. (${error instanceof Error ? error.message : String(error)})`);
}
return embedder;
},
};
}
const apiKey = process.env.BENCH_EMBEDDING_API_KEY;
if (!apiKey) throw new Error('--embedder openai needs BENCH_EMBEDDING_API_KEY (OPENAI_API_KEY is deliberately ignored)');
const model = process.env.BENCH_EMBEDDING_MODEL ?? 'text-embedding-3-small';
const baseUrl = process.env.BENCH_EMBEDDING_BASE_URL ?? 'https://api.openai.com/v1/embeddings';
return {
label: `OpenAI-compatible ${model} at ${new URL(baseUrl).host} (padded or cut to 1024 dims)`,
remoteHost: new URL(baseUrl).hostname,
create: async (core) => (core.createApiEmbedder as Factory<object>)({
provider: 'openai', apiKey, model, baseUrl, targetDimensions: 1024,
}),
};
}
// ── Backends ─────────────────────────────────────────────────────────────────
async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
const { MindDB, FrameStore, SessionStore, HybridSearch } = core;
if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone');
// Like the sidecar: one .mind file for personal memory and one per workspace.
const minds = new Map<string, { db: { close(): void }; search: InstanceType<NonNullable<LegacyCore['HybridSearch']>>; refs: Map<number, string> }>();
const started = performance.now();
for (const memory of set.memories) {
let mind = minds.get(memory.scope);
if (!mind) {
const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`));
new SessionStore(db).ensure('bench');
mind = { db, search: new HybridSearch(db, embedder), refs: new Map() };
minds.set(memory.scope, mind);
}
const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated');
await mind.search.indexFrame(frame.id, memory.text);
mind.refs.set(frame.id, memory.id);
}
const ingestMs = performance.now() - started;
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const hits: Hit[] = [];
for (const scope of scopes) {
const mind = minds.get(scope);
if (!mind) continue;
// Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts).
for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) {
hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore });
}
}
// One ranked list across minds, ordered by each mind's own final score.
return hits.sort((a, b) => b.score - a.score).slice(0, limit);
},
async close() {
for (const mind of minds.values()) mind.db.close();
},
};
}
async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise<Backend> {
const { createMem0MemoryStore } = await import('../src/index.js');
const { scopeFromKey } = await import('@waggle/memory-contract');
const toScope = (key: string) => {
const scope = scopeFromKey(key);
if (!scope) throw new Error(`bad scope key: ${key}`);
return scope;
};
const store = await createMem0MemoryStore({ directory: dir, embedder });
const refs = new Map<string, string>();
const started = performance.now();
for (const memory of set.memories) {
const [change] = await store.remember({
scope: toScope(memory.scope),
text: memory.text,
kind: memory.kind as MemoryKind,
importance: memory.importance as MemoryImportance,
source: 'user_stated',
origin: 'bench',
});
refs.set(change.record.id, memory.id);
}
const ingestMs = performance.now() - started;
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const results = await store.search(text, { scopes: scopes.map(toScope), limit: reranker ? RERANK_POOL : limit });
const hits = results.map((r) => ({ ref: refs.get(r.id)!, text: r.text, score: r.score }));
return rerankTop(text, hits, limit, reranker);
},
close: () => store.close(),
};
}
/** Baseline: rank every memory of the searched scopes by cosine similarity alone. */
async function createCosineBackend(set: ReferenceSet, embedder: Embedder): Promise<Backend> {
const started = performance.now();
const vectors = await embedder.embedBatch(set.memories.map((m) => m.text));
const ingestMs = performance.now() - started;
const cosine = (a: Float32Array, b: Float32Array) => {
let dot = 0, na = 0, nb = 0;
for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
};
return {
ingestMs,
async search(text, scopes, limit, reranker) {
const query = await embedder.embed(text);
const hits = set.memories
.map((m, i) => ({ memory: m, score: cosine(query, vectors[i]) }))
.filter(({ memory }) => scopes.includes(memory.scope))
.sort((a, b) => b.score - a.score)
.slice(0, reranker ? RERANK_POOL : limit)
.map(({ memory, score }) => ({ ref: memory.id, text: memory.text, score }));
return rerankTop(text, hits, limit, reranker);
},
close: async () => {},
};
}
async function rerankTop(text: string, hits: Hit[], limit: number, reranker?: Reranker): Promise<Hit[]> {
if (!reranker || hits.length === 0) return hits.slice(0, limit);
const scores = await reranker.scoreBatch(text, hits.map((h) => h.text));
return hits
.map((hit, i) => ({ ...hit, score: scores[i] }))
.sort((a, b) => b.score - a.score)
.slice(0, limit);
}
// ── Metrics ──────────────────────────────────────────────────────────────────
interface QueryResult {
query: string;
type: string;
scopes: string[];
relevant: string[];
ranked: Array<{ ref: string; score: number }>;
/** 1-based rank of the first relevant memory within the results, or null. */
firstRank: number | null;
recallAt1: number | null;
recallAt5: number | null;
reciprocalRank: number | null;
latencyMs: number;
}
/** Share of the relevant memories found in the top k, out of the most that fit (min(k, relevant)). */
function recallAt(ranked: string[], relevant: Set<string>, k: number): number {
const found = ranked.slice(0, k).filter((ref) => relevant.has(ref)).length;
return found / Math.min(k, relevant.size);
}
function evaluate(query: QueryFixture, hits: Hit[], latencyMs: number): QueryResult {
const relevant = new Set(query.relevant);
const ranked = hits.map((h) => h.ref);
const index = ranked.findIndex((ref) => relevant.has(ref));
const firstRank = index >= 0 ? index + 1 : null;
const answerable = relevant.size > 0;
return {
query: query.id,
type: query.type,
scopes: query.scopes,
relevant: query.relevant,
ranked: hits.map((h) => ({ ref: h.ref, score: Number(h.score.toFixed(4)) })),
firstRank,
recallAt1: answerable ? recallAt(ranked, relevant, 1) : null,
recallAt5: answerable ? recallAt(ranked, relevant, 5) : null,
reciprocalRank: answerable ? (firstRank ? 1 / firstRank : 0) : null,
latencyMs,
};
}
interface Summary { n: number; recallAt1: number; recallAt5: number; mrr: number }
function summarize(results: QueryResult[]): Summary {
const answerable = results.filter((r) => r.relevant.length > 0);
const mean = (pick: (r: QueryResult) => number | null) =>
answerable.reduce((sum, r) => sum + (pick(r) ?? 0), 0) / (answerable.length || 1);
return {
n: answerable.length,
recallAt1: mean((r) => r.recallAt1),
recallAt5: mean((r) => r.recallAt5),
mrr: mean((r) => r.reciprocalRank),
};
}
function percentile(values: number[], p: number): number {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0;
}
/**
* How well the top score separates a correct first hit from a query memory
* cannot answer: the chance that a correct top-1 outscores a no-answer top-1
* (ties count half). 1.0 means a score threshold could tell them apart.
*/
function abstainSeparation(results: QueryResult[]): number | null {
const correct = results.filter((r) => r.firstRank === 1).map((r) => r.ranked[0].score);
const empty = results.filter((r) => r.relevant.length === 0).map((r) => r.ranked[0]?.score ?? -Infinity);
if (correct.length === 0 || empty.length === 0) return null;
let wins = 0;
for (const c of correct) for (const e of empty) wins += c > e ? 1 : c === e ? 0.5 : 0;
return wins / (correct.length * empty.length);
}
// ── Report ───────────────────────────────────────────────────────────────────
const f2 = (value: number) => value.toFixed(2);
const f1 = (value: number) => value.toFixed(1);
function table(header: string[], rows: string[][]): string {
return [
`| ${header.join(' | ')} |`,
`|${header.map(() => '---').join('|')}|`,
...rows.map((row) => `| ${row.join(' | ')} |`),
].join('\n');
}
function report(set: ReferenceSet, runs: Map<EngineName, QueryResult[]>, setup: string[]): string {
const engines = [...runs.keys()];
const out: string[] = ['# Memory search quality', '', ...setup.map((line) => `- ${line}`), ''];
const cell = (s: Summary) => `${f2(s.recallAt1)} / ${f2(s.recallAt5)} / ${f2(s.mrr)}`;
out.push('## Overall (answerable queries)', '');
out.push(table(
['engine', 'n', 'recall@1', 'recall@5', 'MRR', 'median ms', 'p95 ms', 'abstain separation'],
engines.map((engine) => {
const results = runs.get(engine)!;
const s = summarize(results);
const latencies = results.map((r) => r.latencyMs);
const separation = abstainSeparation(results);
return [engine, String(s.n), f2(s.recallAt1), f2(s.recallAt5), f2(s.mrr),
f1(percentile(latencies, 0.5)), f1(percentile(latencies, 0.95)), separation === null ? '-' : f2(separation)];
}),
), '');
const types = [...new Set(set.queries.map((q) => q.type))].filter((type) => type !== 'no-answer');
out.push('## By query type (recall@1 / recall@5 / MRR)', '');
out.push(table(
['type', 'n', ...engines],
types.map((type) => [
type,
String(set.queries.filter((q) => q.type === type).length),
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter((r) => r.type === type)))),
]),
), '');
out.push('## By number of scopes searched (recall@1 / recall@5 / MRR)', '');
out.push(table(
['scopes', 'n', ...engines],
[['one', (n: number) => n === 1], ['two or more', (n: number) => n > 1]].map(([label, test]) => {
const pick = (r: QueryResult) => (test as (n: number) => boolean)(r.scopes.length) && r.relevant.length > 0;
return [
label as string,
String(runs.get(engines[0])!.filter(pick).length),
...engines.map((engine) => cell(summarize(runs.get(engine)!.filter(pick)))),
];
}),
), '');
out.push('## Misses', '');
out.push('Answerable queries where at least one engine did not rank a relevant memory first. '
+ 'Each engine cell: rank of the first relevant memory ("-" if not in the results), recall@5, and the top result.', '');
const missRows: string[][] = [];
for (const query of set.queries.filter((q) => q.relevant.length > 0)) {
const per = engines.map((engine) => runs.get(engine)!.find((r) => r.query === query.id)!);
if (per.every((r) => r.firstRank === 1 && r.recallAt5 === 1)) continue;
missRows.push([
query.id, query.type, query.text, query.relevant.join(' '),
...per.map((r) => `${r.firstRank ?? '-'} · ${f2(r.recallAt5 ?? 0)} · ${r.ranked[0]?.ref ?? '-'}`),
]);
}
out.push(table(['query', 'type', 'text', 'relevant', ...engines], missRows), '');
out.push('## No-answer queries (top result and its score)', '');
out.push(table(
['query', 'text', ...engines],
set.queries.filter((q) => q.relevant.length === 0).map((query) => [
query.id, query.text,
...engines.map((engine) => {
const top = runs.get(engine)!.find((r) => r.query === query.id)!.ranked[0];
return top ? `${top.ref} (${top.score.toFixed(3)})` : '-';
}),
]),
), '');
return out.join('\n');
}
function packageVersion(fromDir: string, name: string): string {
try {
let dir = path.dirname(createRequire(path.join(fromDir, 'package.json')).resolve(name));
while (dir !== path.dirname(dir)) {
const file = path.join(dir, 'package.json');
if (fs.existsSync(file)) {
const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: string; version?: string };
if (pkg.name === name) return pkg.version ?? 'unknown';
}
dir = path.dirname(dir);
}
} catch { /* reported as unknown */ }
return 'unknown';
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const options = parseArgs(process.argv.slice(2));
const set = JSON.parse(fs.readFileSync(path.join(BENCH_DIR, 'reference-set.json'), 'utf8')) as ReferenceSet;
const modelsDir = process.env.WAGGLE_MODELS_DIR ?? path.join(os.homedir(), '.waggle', 'models');
const plan = planEmbedder(options.embedder, modelsDir);
guardNetwork(plan.remoteHost);
const core = await import('@waggle/core') as unknown as Record<string, unknown>;
const legacy = core as LegacyCore;
const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch);
const hasReranker = Boolean(legacy.createInProcessReranker);
const available = (name: EngineName) =>
(hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank'));
const wanted = options.engines ?? ENGINE_NAMES.filter(available);
const missing = wanted.filter((name) => !available(name));
if (missing.length > 0) {
throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`);
}
const embedder = await plan.create(core);
let reranker: Reranker | undefined;
if (wanted.some((name) => name.endsWith('-rerank'))) {
// The sidecar caches the reranker under <models>/reranker (packages/server/src/local/index.ts).
reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') });
}
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-'));
const backends = new Map<BackendName, Backend>();
try {
const needed = new Set(wanted.map(backendOf));
if (needed.has('old')) {
backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-'))));
}
if (needed.has('mem0')) {
backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-'))));
}
if (needed.has('cosine')) backends.set('cosine', await createCosineBackend(set, embedder));
const runs = new Map<EngineName, QueryResult[]>();
for (const engine of wanted) {
const backend = backends.get(backendOf(engine))!;
const engineReranker = engine.endsWith('-rerank') ? reranker : undefined;
// Warm up so the first timed query does not pay for lazy initialisation.
await backend.search('warm-up query', ['personal'], options.limit, engineReranker);
const results: QueryResult[] = [];
for (const query of set.queries) {
const started = performance.now();
const hits = await backend.search(query.text, query.scopes, options.limit, engineReranker);
results.push(evaluate(query, hits, performance.now() - started));
}
runs.set(engine, results);
}
const memo = (name: BackendName) => backends.get(name) ? `${f1(backends.get(name)!.ingestMs / 1000)} s` : 'not run';
const setup = [
`Reference set: ${set.memories.length} memories, ${set.queries.length} queries (${set.queries.filter((q) => q.relevant.length === 0).length} without an answer)`,
`Embedder: ${plan.label}`,
`Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`,
`mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`,
`Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`,
`Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`,
`Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`,
];
console.log(report(set, runs, setup));
if (options.out) {
fs.writeFileSync(options.out, `${JSON.stringify({ setup, results: Object.fromEntries(runs) }, null, 2)}\n`);
console.error(`Wrote ${options.out}`);
}
if (requests.blocked.length > 0) {
throw new Error(`blocked network requests: ${requests.blocked.join(', ')}`);
}
} finally {
for (const backend of backends.values()) await backend.close();
fs.rmSync(tempRoot, { recursive: true, force: true });
}
}
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack ?? error.message : error);
process.exitCode = 1;
});