From f046a20b34517bae3317622d8e624cb359da8b9f Mon Sep 17 00:00:00 2001 From: "shumeiko.it" Date: Sat, 24 Oct 2026 11:55:39 +0200 Subject: [PATCH] test(memory): compare search quality of the old engine and mem0 on a reference set (P1.5) Add a reference set (100 invented memories across personal memory and two workspaces, 67 queries: paraphrase, keyword, multi-fact, scope, no-answer) and a tsx benchmark that loads it into the old engine (MindDB, FrameStore, HybridSearch, with and without the cross-encoder reranker), the mem0 store and a plain-cosine baseline, all on Waggle's in-process embedder. It reports recall@1, recall@5, MRR, latency and misses per engine and query type. The mem0 half runs on its own (--engines mem0,cosine) once the old engine is removed. Ollama and the user's own key are optional embedders; the bench refuses network access to anything else and never reads OPENAI_API_KEY. Results in docs/oss/research/memory-search-quality.md: mem0 matches the old engine's chat-recall path (recall@1 0.72, recall@5 0.89 vs 0.90, MRR 0.82 vs 0.83) at a fraction of its latency and is acceptable for v1. No new dependencies. --- docs/oss/research/memory-search-quality.md | 351 +++++++++++ packages/memory-mem0/bench/reference-set.json | 188 ++++++ packages/memory-mem0/bench/search-quality.ts | 568 ++++++++++++++++++ 3 files changed, 1107 insertions(+) create mode 100644 docs/oss/research/memory-search-quality.md create mode 100644 packages/memory-mem0/bench/reference-set.json create mode 100644 packages/memory-mem0/bench/search-quality.ts diff --git a/docs/oss/research/memory-search-quality.md b/docs/oss/research/memory-search-quality.md new file mode 100644 index 0000000..b3811c8 --- /dev/null +++ b/docs/oss/research/memory-search-quality.md @@ -0,0 +1,351 @@ +# Memory search quality: the old engine compared with mem0 (P1.5) + +Measured on 2026-10-24 to decide whether mem0 search is good enough to replace the old +engine (`MindDB` + `FrameStore` + `HybridSearch` from the hive-mind substrate) before the old +engine is deleted. ADR-0007 asked for this comparison on a reference set. + +The reference set and the benchmark live in `packages/memory-mem0/bench/` and keep working +after the old engine is gone (see "How to re-run"). + +## Summary + +- On the 61 answerable queries, the mem0 store scores the same as the old engine as Waggle + uses it for chat recall (hybrid search followed by a cross-encoder reranker): recall@1 + 0.72 against 0.72, recall@5 0.89 against 0.90, MRR 0.82 against 0.83. mem0 answers in + about 21 ms, the old engine with its reranker in about 145 ms. +- The old engine without the reranker, which is what the agent's `search_memory` tool runs, + is clearly worse: recall@1 0.49, recall@5 0.81, MRR 0.65. Its importance boost and its + keyword lane push general "important" memories above the right answer. +- mem0 is acceptable for v1. Its weak spots are questions phrased without the memory's words + (recall@1 0.52, where plain cosine similarity reaches 0.71), questions that need several + memories (recall@5 0.69), and a search cost that grows with the whole store, not only with + the scopes searched. + +## Setup + +| | | +|---|---| +| Code | branch `oss/community-edition` at `f3eb6b6`, plus the bench added with this document | +| mem0 | `mem0ai` 3.3.1 through `@waggle/memory-mem0` (`createMem0MemoryStore`) | +| Embedder | Waggle's in-process embedder, `Xenova/all-MiniLM-L6-v2` (fp32, 384 dimensions zero-padded to 1024), `@huggingface/transformers` 3.8.1, model cache `~/.waggle/models`. These are the sidecar's settings (`WaggleConfig.getEmbeddingConfig`, `targetDimensions: 1024`) | +| Reranker | `Xenova/ms-marco-MiniLM-L-6-v2` through `createInProcessReranker`, cache `~/.waggle/models/reranker`, as the sidecar configures it | +| Machine | Apple M1, 8 cores, 16 GB, macOS 26.6, Node 22.23.2 | +| Network | every `fetch` to another host was refused by the bench; none was attempted. mem0 telemetry was off and `~/.mem0` was not created | + +Ollama was not running on the machine (`localhost:11434` refused the connection), so only the +in-process embedder was measured. The variant with the user's own embedding key was not run; +see "How to re-run". + +### Reference set + +`reference-set.json` describes a fictional freelance designer, Mila Sorensen, with 44 +personal memories and two client workspaces: `atlas` (30 memories, a route-planning app for a +freight company) and `harbor` (26 memories, a dashboard for a food bank). Every memory has a +kind and an importance: 3 critical, 9 important, 84 normal, 4 temporary. The set contains +near-duplicates (two dark-mode preferences, two statements of each project deadline) and +distractors (a friend's cat allergy next to Mila's peanut allergy; a dark-mode decision in +each workspace next to Mila's personal preference; a different database in each workspace). +All names and facts are invented. + +The 67 queries: + +| Type | n | What it tests | +|---|---|---| +| paraphrase | 21 | No content word in common with the memory to find. Checked with SQLite's porter stemmer, the one the old engine's full-text index uses | +| keyword | 17 | An exact name, code or identifier: `ATL-142`, `#F28C28`, `fernwork/harbor-dashboard`, `Dr. Okafor` | +| multi-fact | 12 | Two to four memories together answer the question | +| scope | 11 | The same question asked in different workspaces has different answers; personal memories compete | +| no-answer | 6 | Nothing in memory answers the question | + +Personal questions are mostly in the first person ("What kind of pet do I have?"), as a user +asks in chat; chat recall searches with the user's message. A few use the name, as the agent +does when it writes its own `search_memory` query. + +### Engines + +All engines share one embedder instance, so they see the same vectors. + +| Engine | What runs | +|---|---| +| `old` | One `.mind` file for personal memory and one per workspace, as in the sidecar. Memories are written with `FrameStore.createIFrame` and indexed with `HybridSearch.indexFrame`. Each searched mind runs `HybridSearch.search(query, { limit: 10, profile: 'balanced' })`; results of all searched minds are merged by `finalScore`. This is the search behind the `search_memory` tool | +| `old-rerank` | The same with the in-process cross-encoder reranker, as `Orchestrator.recallMemory` uses it for chat recall by default | +| `mem0` | `createMem0MemoryStore`, `remember({ scope, text, kind, importance })`, `search(text, { scopes, limit: 10 })`, which is what the adapter offers callers | +| `mem0-rerank` | mem0's top 30 re-ordered by the same reranker. Not part of the adapter; measured as a possible improvement | +| `cosine` | Baseline: every memory of the searched scopes ranked by cosine similarity alone, without any keyword signal | + +### Metrics + +- recall@k: the share of a query's relevant memories found in the first k results, out of + the most that fit (`min(k, relevant)`). recall@1 is therefore "the first result is + relevant", and recall@5 for a three-memory question is "all three are in the top five". +- MRR: mean of 1 / rank of the first relevant result within the top 10, 0 when none is + there. +- Both are averaged over the 61 answerable queries. One query moves recall@1 overall by + 0.016; differences below about 0.05 are a few queries and should be read as ties. +- Abstain separation: the probability that the top score of a query whose first result is + correct is higher than the top score of a no-answer query. 1.0 would mean a score + threshold can tell "memory has an answer" from "memory has none". +- Latency: wall time of one `search` call, query embedding included, median and 95th + percentile over the 67 queries after one warm-up query. + +## Results + +The relevance numbers were identical in all four runs; only latency varied. + +| Engine | recall@1 | recall@5 | MRR | median ms | p95 ms | abstain separation | +|---|---|---|---|---|---|---| +| old | 0.49 | 0.81 | 0.65 | 5.9 | 7.3 | 0.35 | +| old-rerank | 0.72 | 0.90 | 0.83 | 143.5 | 261.8 | 0.65 | +| mem0 | 0.72 | 0.89 | 0.82 | 21.0 | 39.5 | 0.79 | +| mem0-rerank | 0.72 | 0.90 | 0.83 | 148.5 | 256.7 | 0.65 | +| cosine | 0.74 | 0.88 | 0.82 | 2.3 | 5.5 | 0.47 | + +The latency columns come from the first of the four runs. Other agents were working on the +same machine during the later runs (load average 30 to 60 on 8 cores), and the medians rose +to 7–20 ms for `old`, 36–47 ms for `mem0` and 170–560 ms for the reranker variants. In +every run mem0 took 2 to 5 times as long as `old`, and the reranker variants were the +slowest. + +By query type (recall@1 / recall@5 / MRR): + +| Type | n | old | old-rerank | mem0 | mem0-rerank | cosine | +|---|---|---|---|---|---|---| +| paraphrase | 21 | 0.19 / 0.76 / 0.40 | 0.57 / 0.90 / 0.72 | 0.52 / 0.90 / 0.70 | 0.57 / 0.90 / 0.72 | 0.71 / 0.90 / 0.80 | +| keyword | 17 | 0.94 / 1.00 / 0.96 | 0.88 / 1.00 / 0.94 | 1.00 / 1.00 / 1.00 | 0.88 / 1.00 / 0.94 | 0.88 / 1.00 / 0.94 | +| multi-fact | 12 | 0.42 / 0.61 / 0.61 | 0.75 / 0.80 / 0.82 | 0.75 / 0.69 / 0.79 | 0.75 / 0.80 / 0.82 | 0.67 / 0.67 / 0.73 | +| scope | 11 | 0.45 / 0.82 / 0.66 | 0.73 / 0.86 / 0.86 | 0.64 / 0.91 / 0.80 | 0.73 / 0.86 / 0.86 | 0.64 / 0.86 / 0.78 | + +By the number of scopes searched: + +| Scopes | n | old | old-rerank | mem0 | mem0-rerank | cosine | +|---|---|---|---|---|---|---| +| one | 25 | 0.60 / 0.89 / 0.75 | 0.80 / 0.95 / 0.87 | 0.80 / 0.99 / 0.87 | 0.80 / 0.95 / 0.87 | 0.88 / 0.99 / 0.93 | +| two or more | 36 | 0.42 / 0.75 / 0.58 | 0.67 / 0.87 / 0.79 | 0.67 / 0.82 / 0.78 | 0.67 / 0.87 / 0.79 | 0.64 / 0.80 / 0.75 | + +The bench prints a per-query table of misses (rank of the first relevant memory, recall@5 +and the top result per engine) and the top result of every no-answer query. The examples +below come from it. + +## What mem0 3.3.1 does when it searches + +Read in `dist/oss/index.js` of the installed package (line numbers refer to that file) and +checked with mem0's `explain: true` score breakdown on this set. + +1. **Vector similarity.** The query is embedded once. The `memory` vector store reads every + row of its `vectors` table, parses each JSON payload, keeps the rows whose `user_id` and + `agent_id` match, and computes cosine similarity in JavaScript (lines 1143–1174). The + table holds all scopes, so the whole store is read even when one small workspace is + searched. mem0 keeps the best `max(topK × 4, 60)` rows as candidates (line 18031); the + adapter asks for `topK = max(limit × 5, 100)`, so up to 400 per scope. +2. **BM25.** A second full read of the table computes BM25 (k1 1.5, b 0.75) over + `textLemmatized`, which mem0 stores with each memory: lower-case ASCII words without stop + words, stemmed with the Porter stemmer of the `natural` package if it is installed + (lines 16171–16237). `natural` is not installed in Waggle, so mem0 falls back to a + simple suffix stripper. Non-ASCII letters split words (`Škoda` becomes `koda`) on both + sides, so names still match. The raw score is mapped to 0–1 with a sigmoid whose + midpoint depends on the query length (lines 16855–16871). Only memories already among + the vector candidates receive it. +3. **Entity boost.** Entities found in the query (quoted text, proper nouns, identifiers, + noun phrases through `compromise`) are looked up in a second store, + `vectors_entities.db`, and add up to 0.5 to the memories linked to them (lines + 18060–18127). Links are only written when mem0 extracts facts with a model + (`infer: true`). The adapter stores text with `infer: false`, which calls `createMemory` + and links nothing (lines 17575–17592, 18423), so the boost was 0 for all 67 queries. + Memories that P1.5 extracts from conversations will get links. +4. **Combination.** Each candidate scores `(semantic + bm25 + entity) / maxPossible`, where + `maxPossible` is 1, plus 1 if any candidate of this call has a BM25 score, plus 0.5 if + any has an entity boost (lines 16873–16919). The threshold applies to the semantic part + only; the adapter passes 0. + +There is no importance, recency or access-count signal. ADR-0007 is right that mem0 search +is "hybrid vector and BM25", with three qualifications: BM25 only re-ranks the vector +candidates, the entity part is inactive for memories written as text, and semantic and +keyword scores have equal weight. BM25 contributed to the ranking in the first searched scope +of 40 of the 67 queries. + +The adapter searches each scope with its own call and merges by score. Because +`maxPossible` is computed per call, scores from two scopes can be divided by different +numbers: this happened in 12 of the 39 queries that search several scopes. Re-ranking those +with a common divisor changed the first result of one query (q40), from a relevant memory to +an irrelevant one, so the effect on this set is nil. + +## Where each engine wins and loses + +### Old engine without the reranker + +`HybridSearch` fuses a full-text lane and a vector lane with reciprocal rank fusion +(`1 / (60 + rank)` per lane) and multiplies the result by a relevance factor. With the +'balanced' profile and new memories, that factor is 0.80 for normal, 0.90 for important and +1.00 for critical memories. The gap between rank 1 and rank 2 in a lane is under 2 %, so a +12–25 % importance bonus outweighs most differences in rank. 21 of the 31 wrong first +results were critical or important memories, which make up 12 of the 100 memories. For +example, "What kind of pet do I have?" returns the peanut allergy (critical) first and the +greyhound fourth; "What do I charge clients?" returns the decision to take no new clients +(important) first. + +The full-text lane joins all words of the query with OR. Its stop-word list has no pronouns +or names, so any shared common word gives a memory a full vote. In a first draft of this +set, personal questions were in the third person ("What kind of pet does she have?"); the +old engine then ranked a relevant memory first for 1 of the 21 paraphrase queries, because +"she" appears in many memories. With first-person questions it is 4 of 21. + +Exact names are its strength: recall@1 0.94 on keyword queries, through the full-text index. + +### Old engine with the reranker (chat recall) + +The cross-encoder replaces the fused score with its own, which removes the importance bias: +recall@1 rises to 0.72. It costs 130–140 ms per search on 100 memories and makes its own +mistakes. "Would I enjoy a beef burger?" ranks a science-fiction memory first and loses the +vegetarian-diet memory from the top 10; "Why was a lightweight graphing package picked?" +ranks the standing-desk decision first and loses the Chart.js decision; two keyword queries +(`When can I phone Grace Mbeki?`, `fernwork/harbor-dashboard`) drop from rank 1 to rank 2. +The model is trained on web passages, not on short personal facts. + +### mem0 + +mem0 ties the old engine with the reranker overall. + +It is best on keyword queries (recall@1 1.00): BM25 finds `ATL-142`, `#F28C28`, `PostGIS` +and `fernwork.si` even where the vector alone puts another memory first. It also separates +answerable from unanswerable questions best (0.79, against 0.47 for plain cosine over the +same vectors, so the difference comes from BM25), though not well enough for a threshold +(see "Shared failures"). + +It loses on paraphrases: recall@1 0.52, where the cosine baseline over the same vectors +reaches 0.71. BM25 weighs as much as the vector score, so one shared ordinary word lifts a +distractor above a better semantic match. "How do I like replies to be formatted?" puts +"She likes to be addressed as Mila" first through "like"; "What computer do I work on?" +puts "wears glasses for screen work" first through "work". Lowering the BM25 weight does not +help overall. Re-scoring mem0's candidates as `semantic + w × bm25`, from the `explain: true` +breakdown (a one-off check, not part of the bench), gave: + +| w | recall@1 | paraphrase recall@1 | keyword recall@1 | +|---|---|---|---| +| 0 (cosine) | 0.74 | 0.71 | 0.88 | +| 0.25 | 0.72 | 0.67 | 0.88 | +| 0.5 | 0.72 | 0.57 | 0.94 | +| 1 (mem0's weights) | 0.70 | 0.52 | 1.00 | + +The weight trades paraphrase accuracy for keyword accuracy without changing the total. The +last row differs from the bench's 0.72 by one query (q40), because the re-scoring used one +divisor for all scopes. + +Questions that need several memories are its other weak spot (recall@5 0.69 against 0.80 +with the reranker). "What has been decided about the Harbor tech stack?" finds none of the +four relevant memories in the top five, because every Harbor memory is about Harbor; +"How big are the budgets of Mila's current projects?" puts the personal decision about +"current projects" first through BM25. + +In scope queries, a personal memory sometimes wins over the workspace answer: +"When is the deadline?" in Harbor returns the personal first-quarter plan first and the +board-meeting deadline third; "Who is the client contact?" in Harbor returns "emergency +contact" first through BM25. Isolation between workspaces holds in every engine: no query +returned a memory from a workspace it did not search. + +### Shared failures + +- "Is there anything I must never eat?" ranks the vegetarian diet above the peanut allergy + in every engine (allergy at rank 2 or 3). +- "Does the app have dark mode?" in either workspace ranks Mila's personal dark-mode + preference above the workspace's decision in every engine. +- "What did we adjust for people wearing mittens?" ranks "wears glasses for screen work" + first everywhere; the glove decision is second or third. +- Every engine returns something for questions memory cannot answer. With mem0, 19 of the + 44 correct first results score at or below the highest no-answer score (0.39 for "How many + employees does Kestrel Freight have?"), so a fixed threshold cannot decide that memory has + no answer. + +## Latency and store size + +At 100 memories mem0 needs about 21 ms per search, the old engine 6 ms and the reranker +variants about 145 ms. mem0's cost grows with the size of the whole store, because both +passes read every row of the `vectors` table. A separate probe on the same machine (a +deterministic stand-in embedder, so that only the store's work was timed, 21 queries each): + +| Store | Median search | +|---|---| +| 1,000 memories in one scope, 1024 dimensions | 66 ms | +| 5,000 memories in one scope, 1024 dimensions | 336 ms | +| 5,000 memories in one scope, 384 dimensions | 136 ms | +| 5,000 memories in personal, search of a 20-memory workspace | 224 ms | + +The machine was busy with other work during the probe, so the absolute numbers are rough; +the ratios come from runs made one after another. A search across personal memory and one +workspace makes two such passes. + +## Conclusion + +mem0 search is acceptable for v1. On this set it matches the old engine's chat-recall path, +it is clearly better than the old engine's `search_memory` path, it needs no reranker model, +and it runs offline with Waggle's embedder. + +What would improve it, in order of value for effort: + +1. **Store vectors at the embedder's native size.** The sidecar pads MiniLM's 384 + dimensions to 1024. Zero padding does not change cosine similarity, so the ranking stays + the same, while mem0's search pass gets about 2.5 times faster. The adapter has no check + like the old engine's embedding fingerprint: a store created with one embedder and read + with another of the same size gives degraded results without an error. Whatever size is + chosen, the store should record the embedder and refuse a mismatch. +2. **Do not tune the BM25 weight.** The table above shows a trade, not a gain. +3. **Leave the reranker out of v1.** It brings mem0 to the same numbers as the old + engine's chat recall (recall@5 0.90, MRR 0.83), costs about 130 ms per search, and lives + in the substrate that is being deleted, so it would have to be copied into `@waggle/core` + first. +4. **Revisit the vector store when stores grow.** At a few hundred to a few thousand + memories the full scan is fine. Past that, a store with an index, or a single mem0 call + across all searched scopes (one pass instead of one per scope, not tested here), would + be needed. ADR-0007 already notes the same for `list`, `count` and `forgetMatching`. +5. **Do not use the score to decide that memory has no answer.** Callers should rank, not + threshold. + +What this bench does not cover: + +- Time. Neither store can backdate a memory in the bench (mem0 rejects timestamps), so + the old engine's recency decay and questions about dates are not measured. +- The rest of chat recall. The importance lane, prefix lanes and date window in + `Orchestrator.recallMemory` sit outside `HybridSearch`; P1.5 maps them to contract + filters. +- Extracted memories. All memories are written as text; memories extracted by a model would + also get mem0's entity boost. +- Other embedders and languages. One English model was measured. +- The merge across minds. Production renders personal and workspace results in separate + sections; the bench merges them by score for every engine to get one ranking. +- Size. 100 memories and 67 queries written by one author. + +## How to re-run + +From the repository root, after `npm ci` and `npm run build:packages`: + +```bash +# All engines this checkout has (old, old-rerank, mem0, mem0-rerank, cosine) +npx tsx packages/memory-mem0/bench/search-quality.ts + +# After the old engine is removed +npx tsx packages/memory-mem0/bench/search-quality.ts --engines mem0,cosine + +# Per-query results as JSON +npx tsx packages/memory-mem0/bench/search-quality.ts --out /tmp/search-quality.json +``` + +The in-process model must be in `~/.waggle/models` (the sidecar downloads it on first start) +or in the directory named by `WAGGLE_MODELS_DIR`; the bench refuses network access and does +not download it. + +Other embedders: + +```bash +# A running local Ollama; the bench neither starts it nor pulls models +OLLAMA_EMBED_MODEL=nomic-embed-text npx tsx packages/memory-mem0/bench/search-quality.ts --embedder ollama + +# The user's own key. OPENAI_API_KEY is ignored on purpose, so a key that happens to be +# set in the environment is never used. +BENCH_EMBEDDING_API_KEY= BENCH_EMBEDDING_MODEL=text-embedding-3-small \ + npx tsx packages/memory-mem0/bench/search-quality.ts --embedder openai +``` + +`BENCH_EMBEDDING_BASE_URL` points the key variant at another OpenAI-compatible endpoint. +With a key, only that endpoint may be reached. + +To extend the set, add memories and queries to `reference-set.json`; a query lists every +memory that answers it in `relevant`, and an empty list marks a question without an answer. diff --git a/packages/memory-mem0/bench/reference-set.json b/packages/memory-mem0/bench/reference-set.json new file mode 100644 index 0000000..60b4a48 --- /dev/null +++ b/packages/memory-mem0/bench/reference-set.json @@ -0,0 +1,188 @@ +{ + "version": 1, + "description": "Reference set for comparing memory search quality (P1.5). A fictional freelance designer, Mila Sorensen, with personal memories and two client workspaces: 'atlas' (a route-planning app for a freight company) and 'harbor' (a dashboard for a food bank). All people, companies and events are invented. Scope keys follow the memory contract: 'personal' or 'workspace:'. A query's 'relevant' lists every memory that answers it; an empty list means memory holds no answer. Personal queries are mostly in the first person, as a user asks in chat (chat recall searches with the user's message); some use the name Mila, as the agent does when it writes its own memory search.", + "queryTypes": { + "paraphrase": "The query shares no content word with the memory it should find.", + "keyword": "The query names an exact term from the memory: a person, product, code or identifier.", + "multi-fact": "Two to four memories together answer the query.", + "scope": "The answer depends on the scopes searched; other scopes hold competing answers.", + "no-answer": "No memory answers the query." + }, + "memories": [ + { "id": "p01", "scope": "personal", "kind": "profile", "importance": "important", "text": "The user's name is Mila Sorensen. She likes to be addressed as Mila." }, + { "id": "p02", "scope": "personal", "kind": "profile", "importance": "important", "text": "Mila lives in Ljubljana, in a flat near Tivoli Park." }, + { "id": "p03", "scope": "personal", "kind": "profile", "importance": "important", "text": "Mila is a freelance product designer and runs a two-person studio." }, + { "id": "p04", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila speaks Slovenian, English and German, and is learning Italian on weekday evenings." }, + { "id": "p05", "scope": "personal", "kind": "profile", "importance": "critical", "text": "Mila is severely allergic to peanuts and always carries an epinephrine auto-injector." }, + { "id": "p06", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila's birthday is on 9 November." }, + { "id": "p07", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila has a rescue greyhound named Pixel." }, + { "id": "p08", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila's sister Ana lives in Graz and works as a veterinarian." }, + { "id": "p09", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers short answers with bullet points over long paragraphs." }, + { "id": "p10", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila does not take meetings before 10:00; she keeps mornings for focused design work." }, + { "id": "p11", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila uses Figma for interface design and Obsidian for her notes." }, + { "id": "p12", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers dark mode in every app she uses." }, + { "id": "p13", "scope": "personal", "kind": "preference", "importance": "normal", "text": "She asked for a dark editor theme because bright screens give her headaches." }, + { "id": "p14", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila drinks oat-milk flat whites and avoids dairy milk." }, + { "id": "p15", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila eats a mostly vegetarian diet with fish now and then." }, + { "id": "p16", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Emails written for Mila should end with 'Best, Mila' and keep a friendly but direct tone." }, + { "id": "p17", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila wants money amounts shown in euros with a comma as the decimal separator." }, + { "id": "p18", "scope": "personal", "kind": "decision", "importance": "important", "text": "Mila decided to stop taking new clients until the end of March 2026 so she can finish current projects." }, + { "id": "p19", "scope": "personal", "kind": "decision", "importance": "normal", "text": "Mila chose the Oakline standing desk over the cheaper Deskmate model because its motor is quieter." }, + { "id": "p20", "scope": "personal", "kind": "decision", "importance": "normal", "text": "In January 2026 Mila moved her personal account from Lipa Bank to the digital bank Kovo." }, + { "id": "p21", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila has a dentist appointment with Dr. Okafor on 4 February 2026 at 15:30." }, + { "id": "p22", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila flies to Lisbon on 18 May 2026 to speak about accessible forms at the UX Tide conference." }, + { "id": "p23", "scope": "personal", "kind": "event", "importance": "normal", "text": "Ana's wedding is on 27 June 2026 in Graz, and Mila is the maid of honour." }, + { "id": "p24", "scope": "personal", "kind": "event", "importance": "normal", "text": "Mila ran the Ljubljana half marathon on 26 October 2025 in 1:52." }, + { "id": "p25", "scope": "personal", "kind": "event", "importance": "temporary", "text": "Pixel has a vet check-up on 12 January 2026 for his yearly vaccinations." }, + { "id": "p26", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's studio is called Fernwork Studio and is registered as a sole proprietorship." }, + { "id": "p27", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's accountant is Bojan Zupan; receipts and invoices go to him by the 20th of each month." }, + { "id": "p28", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's laptop is a 14-inch MacBook Pro from 2023, and her backup drive is a 2 TB Samsung T7." }, + { "id": "p29", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's running plan is easy runs on Tuesday and Thursday and a long run on Sunday." }, + { "id": "p30", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila mostly reads science fiction; her favourite author is Ursula K. Le Guin." }, + { "id": "p31", "scope": "personal", "kind": "note", "importance": "temporary", "text": "Reminder: renew the studio domain fernwork.si before 1 March 2026." }, + { "id": "p32", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's hourly rate for design work is 85 euros; workshops are billed at 600 euros per day." }, + { "id": "p33", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila prefers the train over flying for trips shorter than six hours." }, + { "id": "p34", "scope": "personal", "kind": "profile", "importance": "normal", "text": "Mila is short-sighted and wears glasses for screen work." }, + { "id": "p35", "scope": "personal", "kind": "decision", "importance": "normal", "text": "Mila decided to use Proton Mail for the studio and keep Gmail for personal mail only." }, + { "id": "p36", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila drives a 2017 Škoda Fabia; the next service is due at 120,000 km." }, + { "id": "p37", "scope": "personal", "kind": "event", "importance": "normal", "text": "On 3 December 2025 Mila signed a lease for a shared studio space on Trubarjeva street." }, + { "id": "p38", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila listens to ambient and jazz playlists while working because lyrics distract her." }, + { "id": "p39", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Mila's friend Jure is allergic to cats." }, + { "id": "p40", "scope": "personal", "kind": "fact", "importance": "normal", "text": "Pixel is afraid of thunderstorms and calms down in his anxiety vest." }, + { "id": "p41", "scope": "personal", "kind": "note", "importance": "normal", "text": "Mila wants to learn more about variable fonts this year." }, + { "id": "p42", "scope": "personal", "kind": "fact", "importance": "important", "text": "Mila's emergency contact is her sister Ana." }, + { "id": "p43", "scope": "personal", "kind": "summary", "importance": "normal", "text": "Summary of the 12 January 2026 planning chat: Mila's first-quarter priorities are finishing the Atlas designs, delivering the Harbor dashboard and preparing the Lisbon talk." }, + { "id": "p44", "scope": "personal", "kind": "preference", "importance": "normal", "text": "Mila uses Slack for quick questions and email for anything that needs a decision." }, + + { "id": "a01", "scope": "workspace:atlas", "kind": "fact", "importance": "important", "text": "Atlas is a route-planning mobile app that Fernwork Studio is designing for the delivery drivers of Kestrel Freight." }, + { "id": "a02", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Our contact at Kestrel Freight is Daniel Rusu, their head of operations." }, + { "id": "a03", "scope": "workspace:atlas", "kind": "decision", "importance": "important", "text": "Decision: Atlas is built with React Native so one codebase covers iOS and Android." }, + { "id": "a04", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Decision: the Atlas backend stores data in PostgreSQL with the PostGIS extension for route geometry." }, + { "id": "a05", "scope": "workspace:atlas", "kind": "fact", "importance": "critical", "text": "The Atlas beta must reach the pilot drivers by 15 April 2026; a contract penalty applies after that date." }, + { "id": "a06", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas design budget is 18,000 euros, and 11,500 of it was invoiced by the end of January." }, + { "id": "a07", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Luka Novak is the mobile developer on Atlas." }, + { "id": "a08", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Petra Lindqvist owns the Atlas backend and the routing API." }, + { "id": "a09", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Sven tests Atlas on real devices and writes the release checklists." }, + { "id": "a10", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Drivers asked for large tap targets because they use the app with gloves on, so the minimum button height is 56 points." }, + { "id": "a11", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas has to work offline for up to eight hours, because rural routes have no mobile signal." }, + { "id": "a12", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "The Atlas kickoff workshop with Kestrel Freight took place on 8 December 2025 in Maribor." }, + { "id": "a13", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "The first Atlas usability test is planned for 10 March 2026 with six drivers." }, + { "id": "a14", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas tickets are tracked in Linear with the prefix ATL; ATL-142 is the bug where the map freezes on Android 12." }, + { "id": "a15", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "On 20 January 2026 we chose Mapbox over Google Maps for Atlas because it supports offline map tiles." }, + { "id": "a16", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas brand colours are deep teal #0F4C5C and signal orange #F28C28." }, + { "id": "a17", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas design files are in the Figma project 'Kestrel / Atlas v2'." }, + { "id": "a18", "scope": "workspace:atlas", "kind": "preference", "importance": "normal", "text": "Daniel Rusu wants a weekly status update by email on Friday afternoons." }, + { "id": "a19", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas compresses proof-of-delivery photos to under 300 KB before uploading them." }, + { "id": "a20", "scope": "workspace:atlas", "kind": "note", "importance": "temporary", "text": "Ask Petra whether the routing API can return an ETA for each stop." }, + { "id": "a21", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The pilot drivers expect the first Atlas beta around mid-April." }, + { "id": "a22", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Atlas launches in Slovenian, Croatian and English." }, + { "id": "a23", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Drivers sign in to Atlas with one-time codes sent by SMS; they have no passwords." }, + { "id": "a24", "scope": "workspace:atlas", "kind": "event", "importance": "normal", "text": "On 29 January 2026 Kestrel Freight approved the Atlas navigation flow after the second review." }, + { "id": "a25", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "The Atlas staging server is atlas-staging.kestrel.internal and is reset every night." }, + { "id": "a26", "scope": "workspace:atlas", "kind": "decision", "importance": "normal", "text": "Dark mode for Atlas is postponed until after the pilot, since drivers mostly work in daylight." }, + { "id": "a27", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Each Atlas driver handles 40 to 70 stops per shift." }, + { "id": "a28", "scope": "workspace:atlas", "kind": "summary", "importance": "normal", "text": "Summary of the Atlas sprint review on 6 February 2026: offline sync works, the map freeze ATL-142 is still open, and the delivery photo flow needs one more iteration." }, + { "id": "a29", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Atlas crash reports go to Sentry, with an alert when there are more than five crashes per hour." }, + { "id": "a30", "scope": "workspace:atlas", "kind": "fact", "importance": "normal", "text": "Client meetings for Atlas are on Tuesdays at 11:00 over Google Meet." }, + + { "id": "h01", "scope": "workspace:harbor", "kind": "fact", "importance": "important", "text": "Harbor is an analytics dashboard Fernwork Studio is building for the Lindenwood Food Bank to track donations and volunteers." }, + { "id": "h02", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Our contact for Harbor is Grace Mbeki, the food bank's volunteer coordinator." }, + { "id": "h03", "scope": "workspace:harbor", "kind": "decision", "importance": "important", "text": "Decision: Harbor stores its data in SQLite, because the food bank has no IT staff to run a database server." }, + { "id": "h04", "scope": "workspace:harbor", "kind": "fact", "importance": "critical", "text": "Harbor must be ready for the food bank's board meeting on 2 March 2026." }, + { "id": "h05", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor budget is 7,500 euros, paid half at kickoff and half on delivery." }, + { "id": "h06", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor draws its charts with Chart.js; the food bank's laptops are old, so heavy libraries are avoided." }, + { "id": "h07", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor imports donation records from a shared Google Sheet every night at 02:00." }, + { "id": "h08", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Volunteers sign in to Harbor with a magic link sent by email." }, + { "id": "h09", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "The Harbor kickoff call with Grace Mbeki was on 15 January 2026." }, + { "id": "h10", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Tomaž Kranjc, a volunteer developer, helps with the Harbor import scripts." }, + { "id": "h11", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "The Harbor dashboard has to meet WCAG 2.1 AA because several volunteers use screen readers." }, + { "id": "h12", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor tracks three headline numbers: kilograms of food donated, active volunteers, and families served each week." }, + { "id": "h13", "scope": "workspace:harbor", "kind": "preference", "importance": "normal", "text": "Grace prefers phone calls to email and is usually free after 16:00." }, + { "id": "h14", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The food bank's brand colour is a warm green, #3A7D44." }, + { "id": "h15", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "On 29 January 2026 the food bank's treasurer reviewed the Harbor data and asked for monthly totals." }, + { "id": "h16", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor dashboard should be finished before the March board meeting." }, + { "id": "h17", "scope": "workspace:harbor", "kind": "note", "importance": "temporary", "text": "Check whether the Google Sheet has a donor postcode column before building the map view." }, + { "id": "h18", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor does not store donor names; it keeps only anonymous donor IDs to protect privacy." }, + { "id": "h19", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor runs on a 5-euro-a-month VPS at Hetzner." }, + { "id": "h20", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor reports can be exported to PDF for the monthly newsletter." }, + { "id": "h21", "scope": "workspace:harbor", "kind": "decision", "importance": "normal", "text": "Harbor has a dark mode because volunteers use it during evening shifts." }, + { "id": "h22", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The Harbor code is in the GitHub repository fernwork/harbor-dashboard." }, + { "id": "h23", "scope": "workspace:harbor", "kind": "event", "importance": "normal", "text": "On 5 February 2026 Mila showed the first Harbor prototype to Grace, who asked for bigger numbers on the main screen." }, + { "id": "h24", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "Harbor meetings with the food bank are on Thursdays at 14:00 at their warehouse." }, + { "id": "h25", "scope": "workspace:harbor", "kind": "fact", "importance": "normal", "text": "The food bank serves around 340 families a week in winter." }, + { "id": "h26", "scope": "workspace:harbor", "kind": "summary", "importance": "normal", "text": "Summary of the Harbor planning chat: SQLite for storage, a nightly import from Google Sheets, magic-link sign-in, and PDF export for the board." } + ], + "queries": [ + { "id": "q01", "type": "paraphrase", "scopes": ["personal"], "text": "Is there anything I must never eat?", "relevant": ["p05"] }, + { "id": "q02", "type": "paraphrase", "scopes": ["personal"], "text": "Which city is my home in?", "relevant": ["p02"] }, + { "id": "q03", "type": "paraphrase", "scopes": ["personal"], "text": "What kind of pet do I have?", "relevant": ["p07"] }, + { "id": "q04", "type": "paraphrase", "scopes": ["personal"], "text": "How do I like replies to be formatted?", "relevant": ["p09"] }, + { "id": "q05", "type": "paraphrase", "scopes": ["personal"], "text": "When am I unavailable for calls early in the day?", "relevant": ["p10"] }, + { "id": "q06", "type": "paraphrase", "scopes": ["personal"], "text": "What do I usually order at a café?", "relevant": ["p14"] }, + { "id": "q07", "type": "paraphrase", "scopes": ["personal"], "text": "Would I enjoy a beef burger?", "relevant": ["p15"] }, + { "id": "q08", "type": "paraphrase", "scopes": ["personal"], "text": "Who does my bookkeeping?", "relevant": ["p27"] }, + { "id": "q09", "type": "paraphrase", "scopes": ["personal"], "text": "What do I charge clients?", "relevant": ["p32"] }, + { "id": "q10", "type": "paraphrase", "scopes": ["personal"], "text": "What computer do I work on?", "relevant": ["p28"] }, + { "id": "q11", "type": "paraphrase", "scopes": ["personal"], "text": "How do I like to get to places that are not too far away?", "relevant": ["p33"] }, + { "id": "q12", "type": "paraphrase", "scopes": ["personal"], "text": "What music helps me concentrate?", "relevant": ["p38"] }, + { "id": "q13", "type": "paraphrase", "scopes": ["personal"], "text": "Who should be called if something happens to me?", "relevant": ["p42"] }, + { "id": "q14", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "Why can't the delivery app rely on a network connection?", "relevant": ["a11"] }, + { "id": "q15", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "What did we adjust for people wearing mittens?", "relevant": ["a10"] }, + { "id": "q16", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "Who represents the customer on the route-planning project?", "relevant": ["a02"] }, + { "id": "q17", "type": "paraphrase", "scopes": ["personal", "workspace:atlas"], "text": "How do couriers show that a parcel was handed over?", "relevant": ["a19"] }, + { "id": "q18", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "How do helpers log in?", "relevant": ["h08"] }, + { "id": "q19", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "Which metrics matter most to the charity?", "relevant": ["h12"] }, + { "id": "q20", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "How do we safeguard the identities of people who give food?", "relevant": ["h18"] }, + { "id": "q21", "type": "paraphrase", "scopes": ["personal", "workspace:harbor"], "text": "Why was a lightweight graphing package picked?", "relevant": ["h06"] }, + + { "id": "q22", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Who is our contact at Kestrel Freight?", "relevant": ["a02"] }, + { "id": "q23", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "ATL-142", "relevant": ["a14", "a28"] }, + { "id": "q24", "type": "keyword", "scopes": ["personal"], "text": "Dr. Okafor", "relevant": ["p21"] }, + { "id": "q25", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "PostGIS", "relevant": ["a04"] }, + { "id": "q26", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Mapbox or Google Maps?", "relevant": ["a15"] }, + { "id": "q27", "type": "keyword", "scopes": ["personal"], "text": "Bojan Zupan", "relevant": ["p27"] }, + { "id": "q28", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "When can I phone Grace Mbeki?", "relevant": ["h13"] }, + { "id": "q29", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "#F28C28", "relevant": ["a16"] }, + { "id": "q30", "type": "keyword", "scopes": ["personal"], "text": "fernwork.si", "relevant": ["p31"] }, + { "id": "q31", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "Chart.js", "relevant": ["h06"] }, + { "id": "q32", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "Hetzner VPS", "relevant": ["h19"] }, + { "id": "q33", "type": "keyword", "scopes": ["personal"], "text": "Trubarjeva", "relevant": ["p37"] }, + { "id": "q34", "type": "keyword", "scopes": ["personal", "workspace:atlas"], "text": "Sentry alert threshold", "relevant": ["a29"] }, + { "id": "q35", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "WCAG 2.1 AA", "relevant": ["h11"] }, + { "id": "q36", "type": "keyword", "scopes": ["personal", "workspace:harbor"], "text": "fernwork/harbor-dashboard", "relevant": ["h22"] }, + { "id": "q37", "type": "keyword", "scopes": ["personal"], "text": "Oakline desk", "relevant": ["p19"] }, + { "id": "q38", "type": "keyword", "scopes": ["personal"], "text": "Škoda Fabia service", "relevant": ["p36"] }, + + { "id": "q39", "type": "multi-fact", "scopes": ["personal", "workspace:atlas"], "text": "Who is on the Atlas development team?", "relevant": ["a07", "a08", "a09"] }, + { "id": "q40", "type": "multi-fact", "scopes": ["workspace:atlas", "workspace:harbor"], "text": "What are the upcoming project deadlines?", "relevant": ["a05", "a21", "h04", "h16"] }, + { "id": "q41", "type": "multi-fact", "scopes": ["personal"], "text": "What are Mila's dietary restrictions?", "relevant": ["p05", "p14", "p15"] }, + { "id": "q42", "type": "multi-fact", "scopes": ["personal"], "text": "What do I know about Pixel?", "relevant": ["p07", "p25", "p40"] }, + { "id": "q43", "type": "multi-fact", "scopes": ["personal"], "text": "Tell me about my sister.", "relevant": ["p08", "p23", "p42"] }, + { "id": "q44", "type": "multi-fact", "scopes": ["personal", "workspace:harbor"], "text": "What has been decided about the Harbor tech stack?", "relevant": ["h03", "h06", "h19", "h26"] }, + { "id": "q45", "type": "multi-fact", "scopes": ["personal"], "text": "What do I do for running and races?", "relevant": ["p24", "p29"] }, + { "id": "q46", "type": "multi-fact", "scopes": ["personal"], "text": "Which apps does Mila use for design, notes, chat and email?", "relevant": ["p11", "p35", "p44"] }, + { "id": "q47", "type": "multi-fact", "scopes": ["personal", "workspace:atlas"], "text": "What has happened on the Atlas project so far?", "relevant": ["a12", "a24", "a28"] }, + { "id": "q48", "type": "multi-fact", "scopes": ["workspace:atlas", "workspace:harbor"], "text": "When are the regular client meetings for each project?", "relevant": ["a30", "h24"] }, + { "id": "q49", "type": "multi-fact", "scopes": ["personal", "workspace:atlas", "workspace:harbor"], "text": "How big are the budgets of Mila's current projects?", "relevant": ["a06", "h05"] }, + { "id": "q50", "type": "multi-fact", "scopes": ["personal"], "text": "What are Mila's priorities for the first quarter?", "relevant": ["p43", "p18"] }, + + { "id": "q51", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Which database do we use?", "relevant": ["a04"] }, + { "id": "q52", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Which database do we use?", "relevant": ["h03"] }, + { "id": "q53", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "When is the deadline?", "relevant": ["a05", "a21"] }, + { "id": "q54", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "When is the deadline?", "relevant": ["h04", "h16"] }, + { "id": "q55", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Does the app have dark mode?", "relevant": ["a26"] }, + { "id": "q56", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Does the app have dark mode?", "relevant": ["h21"] }, + { "id": "q57", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "Who is the client contact?", "relevant": ["a02"] }, + { "id": "q58", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "Who is the client contact?", "relevant": ["h02"] }, + { "id": "q59", "type": "scope", "scopes": ["personal", "workspace:harbor"], "text": "What is the brand colour?", "relevant": ["h14"] }, + { "id": "q60", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "How do users sign in?", "relevant": ["a23"] }, + { "id": "q61", "type": "scope", "scopes": ["personal", "workspace:atlas"], "text": "What does Mila think about dark mode?", "relevant": ["p12", "p13"] }, + + { "id": "q62", "type": "no-answer", "scopes": ["personal"], "text": "What is my blood type?", "relevant": [] }, + { "id": "q63", "type": "no-answer", "scopes": ["personal", "workspace:harbor"], "text": "Which CI service runs the Harbor tests?", "relevant": [] }, + { "id": "q64", "type": "no-answer", "scopes": ["personal"], "text": "Which football team do I support?", "relevant": [] }, + { "id": "q65", "type": "no-answer", "scopes": ["personal", "workspace:atlas"], "text": "What is the Atlas rating in the App Store?", "relevant": [] }, + { "id": "q66", "type": "no-answer", "scopes": ["personal"], "text": "Where did Mila go to university?", "relevant": [] }, + { "id": "q67", "type": "no-answer", "scopes": ["personal", "workspace:atlas"], "text": "How many employees does Kestrel Freight have?", "relevant": [] } + ] +} diff --git a/packages/memory-mem0/bench/search-quality.ts b/packages/memory-mem0/bench/search-quality.ts new file mode 100644 index 0000000..8df5376 --- /dev/null +++ b/packages/memory-mem0/bench/search-quality.ts @@ -0,0 +1,568 @@ +/** + * Search-quality benchmark for the memory switch (P1.5). + * + * Loads the reference set (`reference-set.json`) into the old engine (MindDB + + * FrameStore + HybridSearch, re-exported by @waggle/core from the hive-mind + * substrate) and into the mem0 store, runs every query against each, and prints + * recall@1, recall@5, MRR and latency per engine and per query type. The + * findings are in docs/oss/research/memory-search-quality.md. + * + * Run from the repository root after `npm run build:packages`: + * + * npx tsx packages/memory-mem0/bench/search-quality.ts [options] + * + * --engines Comma-separated: old, old-rerank, mem0, mem0-rerank, cosine. + * Default: every engine this checkout still has. Once the + * hive-mind substrate is removed, `mem0` and `cosine` are + * left (the `-rerank` variants use its cross-encoder). + * `cosine` is a baseline: plain cosine similarity over the + * same embeddings, with no keyword signal. + * --embedder inprocess (default, offline), ollama or openai. + * --limit Results per query (default 10). + * --out Also write the per-query results as JSON. + * + * Embedders: + * inprocess Xenova/all-MiniLM-L6-v2 padded to 1024 dimensions, as in the + * sidecar, loaded from WAGGLE_MODELS_DIR (default ~/.waggle/models). + * ollama OLLAMA_HOST (default http://localhost:11434) and + * OLLAMA_EMBED_MODEL (default nomic-embed-text). Ollama must + * already be running with the model pulled. + * openai The user's own key: BENCH_EMBEDDING_API_KEY (required), + * BENCH_EMBEDDING_MODEL (default text-embedding-3-small) and + * BENCH_EMBEDDING_BASE_URL (default the OpenAI endpoint). The bench + * never reads OPENAI_API_KEY, so a key that happens to be set in + * the environment is not used. + * + * Every fetch() to another machine is refused, except the endpoint of the + * openai embedder. mem0 telemetry is switched off before mem0 is loaded. + */ +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath } from 'node:url'; +import type { MemoryImportance, MemoryKind } from '@waggle/memory-contract'; + +// The adapter sets this too; setting it first keeps telemetry off even if +// something loads mem0 earlier than the adapter. +process.env.MEM0_TELEMETRY = 'false'; + +const BENCH_DIR = path.dirname(fileURLToPath(import.meta.url)); +const ENGINE_NAMES = ['old', 'old-rerank', 'mem0', 'mem0-rerank', 'cosine'] as const; +type EngineName = (typeof ENGINE_NAMES)[number]; +type BackendName = 'old' | 'mem0' | 'cosine'; +const backendOf = (engine: EngineName): BackendName => engine.replace(/-rerank$/, '') as BackendName; +/** Candidates a reranker re-orders; HybridSearch's default pool size. */ +const RERANK_POOL = 30; + +interface MemoryFixture { id: string; scope: string; kind: string; importance: string; text: string } +interface QueryFixture { id: string; type: string; scopes: string[]; text: string; relevant: string[] } +interface ReferenceSet { memories: MemoryFixture[]; queries: QueryFixture[] } + +interface Embedder { + readonly dimensions: number; + embed(text: string): Promise; + embedBatch(texts: string[]): Promise; +} +interface Reranker { scoreBatch(query: string, docs: string[]): Promise } +interface Hit { ref: string; text: string; score: number } +interface Backend { + readonly ingestMs: number; + search(text: string, scopes: string[], limit: number, reranker?: Reranker): Promise; + close(): Promise; +} + +/** The parts of the hive-mind substrate the old engine needs; absent once it is removed. */ +interface LegacyCore { + MindDB?: new (file: string) => { close(): void }; + FrameStore?: new (db: unknown) => { + createIFrame(gopId: string, content: string, importance?: string, source?: string): { id: number }; + }; + SessionStore?: new (db: unknown) => { ensure(gopId: string): unknown }; + HybridSearch?: new (db: unknown, embedder: Embedder) => { + indexFrame(id: number, content: string): Promise; + search(query: string, options: { limit?: number; profile?: string; reranker?: Reranker }): + Promise>; + }; + createInProcessReranker?: (config?: { cacheDir?: string }) => Promise; +} + +interface Options { + engines: EngineName[] | null; + embedder: 'inprocess' | 'ollama' | 'openai'; + limit: number; + out: string | null; +} + +function parseArgs(argv: string[]): Options { + const options: Options = { engines: null, embedder: 'inprocess', limit: 10, out: null }; + for (let i = 0; i < argv.length; i++) { + const value = argv[i + 1]; + switch (argv[i]) { + case '--engines': + options.engines = value.split(',').map((name) => { + if (!(ENGINE_NAMES as readonly string[]).includes(name)) throw new Error(`unknown engine: ${name}`); + return name as EngineName; + }); + i++; + break; + case '--embedder': + if (value !== 'inprocess' && value !== 'ollama' && value !== 'openai') { + throw new Error(`unknown embedder: ${value}`); + } + options.embedder = value; + i++; + break; + case '--limit': + options.limit = Number(value); + if (!Number.isInteger(options.limit) || options.limit < 5) throw new Error('--limit must be an integer >= 5'); + i++; + break; + case '--out': + options.out = path.resolve(value); + i++; + break; + default: + throw new Error(`unknown argument: ${argv[i]}`); + } + } + return options; +} + +// ── Network guard ──────────────────────────────────────────────────────────── + +const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); +const requests = { local: 0, remote: 0, blocked: [] as string[] }; + +function guardNetwork(allowedRemoteHost: string | null): void { + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + if (LOOPBACK.has(url.hostname)) requests.local++; + else if (url.hostname === allowedRemoteHost) requests.remote++; + else { + requests.blocked.push(url.host); + throw new Error(`search-quality bench: blocked a request to ${url.host}`); + } + return realFetch(input, init); + }) as typeof fetch; +} + +// ── Embedders ──────────────────────────────────────────────────────────────── + +interface EmbedderPlan { label: string; remoteHost: string | null; create(core: Record): Promise } + +function planEmbedder(kind: Options['embedder'], modelsDir: string): EmbedderPlan { + type Factory = (config: C) => Embedder | Promise; + if (kind === 'inprocess') { + return { + label: 'in-process Xenova/all-MiniLM-L6-v2 (384 dims padded to 1024)', + remoteHost: null, + create: async (core) => (core.createInProcessEmbedder as Factory)({ + model: 'Xenova/all-MiniLM-L6-v2', cacheDir: modelsDir, targetDimensions: 1024, + }), + }; + } + if (kind === 'ollama') { + const baseUrl = process.env.OLLAMA_HOST ?? 'http://localhost:11434'; + const model = process.env.OLLAMA_EMBED_MODEL ?? 'nomic-embed-text'; + if (!LOOPBACK.has(new URL(baseUrl).hostname)) throw new Error('OLLAMA_HOST must point at this machine'); + return { + label: `Ollama ${model} at ${baseUrl} (padded or cut to 1024 dims)`, + remoteHost: null, + create: async (core) => { + const embedder = await (core.createOllamaEmbedder as Factory)({ baseUrl, model, targetDimensions: 1024 }); + try { + await embedder.embed('probe'); + } catch (error) { + throw new Error(`Ollama does not answer at ${baseUrl} with model ${model}. Start it and pull the model first; ` + + `the bench does neither. (${error instanceof Error ? error.message : String(error)})`); + } + return embedder; + }, + }; + } + const apiKey = process.env.BENCH_EMBEDDING_API_KEY; + if (!apiKey) throw new Error('--embedder openai needs BENCH_EMBEDDING_API_KEY (OPENAI_API_KEY is deliberately ignored)'); + const model = process.env.BENCH_EMBEDDING_MODEL ?? 'text-embedding-3-small'; + const baseUrl = process.env.BENCH_EMBEDDING_BASE_URL ?? 'https://api.openai.com/v1/embeddings'; + return { + label: `OpenAI-compatible ${model} at ${new URL(baseUrl).host} (padded or cut to 1024 dims)`, + remoteHost: new URL(baseUrl).hostname, + create: async (core) => (core.createApiEmbedder as Factory)({ + provider: 'openai', apiKey, model, baseUrl, targetDimensions: 1024, + }), + }; +} + +// ── Backends ───────────────────────────────────────────────────────────────── + +async function createOldBackend(core: LegacyCore, set: ReferenceSet, embedder: Embedder, dir: string): Promise { + const { MindDB, FrameStore, SessionStore, HybridSearch } = core; + if (!MindDB || !FrameStore || !SessionStore || !HybridSearch) throw new Error('the hive-mind substrate is gone'); + // Like the sidecar: one .mind file for personal memory and one per workspace. + const minds = new Map>; refs: Map }>(); + const started = performance.now(); + for (const memory of set.memories) { + let mind = minds.get(memory.scope); + if (!mind) { + const db = new MindDB(path.join(dir, `${memory.scope.replace(/\W+/g, '-')}.mind`)); + new SessionStore(db).ensure('bench'); + mind = { db, search: new HybridSearch(db, embedder), refs: new Map() }; + minds.set(memory.scope, mind); + } + const frame = new FrameStore(mind.db).createIFrame('bench', memory.text, memory.importance, 'user_stated'); + await mind.search.indexFrame(frame.id, memory.text); + mind.refs.set(frame.id, memory.id); + } + const ingestMs = performance.now() - started; + + return { + ingestMs, + async search(text, scopes, limit, reranker) { + const hits: Hit[] = []; + for (const scope of scopes) { + const mind = minds.get(scope); + if (!mind) continue; + // Recall uses the 'balanced' profile (packages/agent/src/orchestrator.ts). + for (const result of await mind.search.search(text, { limit, profile: 'balanced', reranker })) { + hits.push({ ref: mind.refs.get(result.frame.id)!, text: result.frame.content, score: result.finalScore }); + } + } + // One ranked list across minds, ordered by each mind's own final score. + return hits.sort((a, b) => b.score - a.score).slice(0, limit); + }, + async close() { + for (const mind of minds.values()) mind.db.close(); + }, + }; +} + +async function createMem0Backend(set: ReferenceSet, embedder: Embedder, dir: string): Promise { + const { createMem0MemoryStore } = await import('../src/index.js'); + const { scopeFromKey } = await import('@waggle/memory-contract'); + const toScope = (key: string) => { + const scope = scopeFromKey(key); + if (!scope) throw new Error(`bad scope key: ${key}`); + return scope; + }; + const store = await createMem0MemoryStore({ directory: dir, embedder }); + const refs = new Map(); + const started = performance.now(); + for (const memory of set.memories) { + const [change] = await store.remember({ + scope: toScope(memory.scope), + text: memory.text, + kind: memory.kind as MemoryKind, + importance: memory.importance as MemoryImportance, + source: 'user_stated', + origin: 'bench', + }); + refs.set(change.record.id, memory.id); + } + const ingestMs = performance.now() - started; + + return { + ingestMs, + async search(text, scopes, limit, reranker) { + const results = await store.search(text, { scopes: scopes.map(toScope), limit: reranker ? RERANK_POOL : limit }); + const hits = results.map((r) => ({ ref: refs.get(r.id)!, text: r.text, score: r.score })); + return rerankTop(text, hits, limit, reranker); + }, + close: () => store.close(), + }; +} + +/** Baseline: rank every memory of the searched scopes by cosine similarity alone. */ +async function createCosineBackend(set: ReferenceSet, embedder: Embedder): Promise { + const started = performance.now(); + const vectors = await embedder.embedBatch(set.memories.map((m) => m.text)); + const ingestMs = performance.now() - started; + const cosine = (a: Float32Array, b: Float32Array) => { + let dot = 0, na = 0, nb = 0; + for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } + return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1); + }; + return { + ingestMs, + async search(text, scopes, limit, reranker) { + const query = await embedder.embed(text); + const hits = set.memories + .map((m, i) => ({ memory: m, score: cosine(query, vectors[i]) })) + .filter(({ memory }) => scopes.includes(memory.scope)) + .sort((a, b) => b.score - a.score) + .slice(0, reranker ? RERANK_POOL : limit) + .map(({ memory, score }) => ({ ref: memory.id, text: memory.text, score })); + return rerankTop(text, hits, limit, reranker); + }, + close: async () => {}, + }; +} + +async function rerankTop(text: string, hits: Hit[], limit: number, reranker?: Reranker): Promise { + if (!reranker || hits.length === 0) return hits.slice(0, limit); + const scores = await reranker.scoreBatch(text, hits.map((h) => h.text)); + return hits + .map((hit, i) => ({ ...hit, score: scores[i] })) + .sort((a, b) => b.score - a.score) + .slice(0, limit); +} + +// ── Metrics ────────────────────────────────────────────────────────────────── + +interface QueryResult { + query: string; + type: string; + scopes: string[]; + relevant: string[]; + ranked: Array<{ ref: string; score: number }>; + /** 1-based rank of the first relevant memory within the results, or null. */ + firstRank: number | null; + recallAt1: number | null; + recallAt5: number | null; + reciprocalRank: number | null; + latencyMs: number; +} + +/** Share of the relevant memories found in the top k, out of the most that fit (min(k, relevant)). */ +function recallAt(ranked: string[], relevant: Set, k: number): number { + const found = ranked.slice(0, k).filter((ref) => relevant.has(ref)).length; + return found / Math.min(k, relevant.size); +} + +function evaluate(query: QueryFixture, hits: Hit[], latencyMs: number): QueryResult { + const relevant = new Set(query.relevant); + const ranked = hits.map((h) => h.ref); + const index = ranked.findIndex((ref) => relevant.has(ref)); + const firstRank = index >= 0 ? index + 1 : null; + const answerable = relevant.size > 0; + return { + query: query.id, + type: query.type, + scopes: query.scopes, + relevant: query.relevant, + ranked: hits.map((h) => ({ ref: h.ref, score: Number(h.score.toFixed(4)) })), + firstRank, + recallAt1: answerable ? recallAt(ranked, relevant, 1) : null, + recallAt5: answerable ? recallAt(ranked, relevant, 5) : null, + reciprocalRank: answerable ? (firstRank ? 1 / firstRank : 0) : null, + latencyMs, + }; +} + +interface Summary { n: number; recallAt1: number; recallAt5: number; mrr: number } + +function summarize(results: QueryResult[]): Summary { + const answerable = results.filter((r) => r.relevant.length > 0); + const mean = (pick: (r: QueryResult) => number | null) => + answerable.reduce((sum, r) => sum + (pick(r) ?? 0), 0) / (answerable.length || 1); + return { + n: answerable.length, + recallAt1: mean((r) => r.recallAt1), + recallAt5: mean((r) => r.recallAt5), + mrr: mean((r) => r.reciprocalRank), + }; +} + +function percentile(values: number[], p: number): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0; +} + +/** + * How well the top score separates a correct first hit from a query memory + * cannot answer: the chance that a correct top-1 outscores a no-answer top-1 + * (ties count half). 1.0 means a score threshold could tell them apart. + */ +function abstainSeparation(results: QueryResult[]): number | null { + const correct = results.filter((r) => r.firstRank === 1).map((r) => r.ranked[0].score); + const empty = results.filter((r) => r.relevant.length === 0).map((r) => r.ranked[0]?.score ?? -Infinity); + if (correct.length === 0 || empty.length === 0) return null; + let wins = 0; + for (const c of correct) for (const e of empty) wins += c > e ? 1 : c === e ? 0.5 : 0; + return wins / (correct.length * empty.length); +} + +// ── Report ─────────────────────────────────────────────────────────────────── + +const f2 = (value: number) => value.toFixed(2); +const f1 = (value: number) => value.toFixed(1); + +function table(header: string[], rows: string[][]): string { + return [ + `| ${header.join(' | ')} |`, + `|${header.map(() => '---').join('|')}|`, + ...rows.map((row) => `| ${row.join(' | ')} |`), + ].join('\n'); +} + +function report(set: ReferenceSet, runs: Map, setup: string[]): string { + const engines = [...runs.keys()]; + const out: string[] = ['# Memory search quality', '', ...setup.map((line) => `- ${line}`), '']; + const cell = (s: Summary) => `${f2(s.recallAt1)} / ${f2(s.recallAt5)} / ${f2(s.mrr)}`; + + out.push('## Overall (answerable queries)', ''); + out.push(table( + ['engine', 'n', 'recall@1', 'recall@5', 'MRR', 'median ms', 'p95 ms', 'abstain separation'], + engines.map((engine) => { + const results = runs.get(engine)!; + const s = summarize(results); + const latencies = results.map((r) => r.latencyMs); + const separation = abstainSeparation(results); + return [engine, String(s.n), f2(s.recallAt1), f2(s.recallAt5), f2(s.mrr), + f1(percentile(latencies, 0.5)), f1(percentile(latencies, 0.95)), separation === null ? '-' : f2(separation)]; + }), + ), ''); + + const types = [...new Set(set.queries.map((q) => q.type))].filter((type) => type !== 'no-answer'); + out.push('## By query type (recall@1 / recall@5 / MRR)', ''); + out.push(table( + ['type', 'n', ...engines], + types.map((type) => [ + type, + String(set.queries.filter((q) => q.type === type).length), + ...engines.map((engine) => cell(summarize(runs.get(engine)!.filter((r) => r.type === type)))), + ]), + ), ''); + + out.push('## By number of scopes searched (recall@1 / recall@5 / MRR)', ''); + out.push(table( + ['scopes', 'n', ...engines], + [['one', (n: number) => n === 1], ['two or more', (n: number) => n > 1]].map(([label, test]) => { + const pick = (r: QueryResult) => (test as (n: number) => boolean)(r.scopes.length) && r.relevant.length > 0; + return [ + label as string, + String(runs.get(engines[0])!.filter(pick).length), + ...engines.map((engine) => cell(summarize(runs.get(engine)!.filter(pick)))), + ]; + }), + ), ''); + + out.push('## Misses', ''); + out.push('Answerable queries where at least one engine did not rank a relevant memory first. ' + + 'Each engine cell: rank of the first relevant memory ("-" if not in the results), recall@5, and the top result.', ''); + const missRows: string[][] = []; + for (const query of set.queries.filter((q) => q.relevant.length > 0)) { + const per = engines.map((engine) => runs.get(engine)!.find((r) => r.query === query.id)!); + if (per.every((r) => r.firstRank === 1 && r.recallAt5 === 1)) continue; + missRows.push([ + query.id, query.type, query.text, query.relevant.join(' '), + ...per.map((r) => `${r.firstRank ?? '-'} · ${f2(r.recallAt5 ?? 0)} · ${r.ranked[0]?.ref ?? '-'}`), + ]); + } + out.push(table(['query', 'type', 'text', 'relevant', ...engines], missRows), ''); + + out.push('## No-answer queries (top result and its score)', ''); + out.push(table( + ['query', 'text', ...engines], + set.queries.filter((q) => q.relevant.length === 0).map((query) => [ + query.id, query.text, + ...engines.map((engine) => { + const top = runs.get(engine)!.find((r) => r.query === query.id)!.ranked[0]; + return top ? `${top.ref} (${top.score.toFixed(3)})` : '-'; + }), + ]), + ), ''); + return out.join('\n'); +} + +function packageVersion(fromDir: string, name: string): string { + try { + let dir = path.dirname(createRequire(path.join(fromDir, 'package.json')).resolve(name)); + while (dir !== path.dirname(dir)) { + const file = path.join(dir, 'package.json'); + if (fs.existsSync(file)) { + const pkg = JSON.parse(fs.readFileSync(file, 'utf8')) as { name?: string; version?: string }; + if (pkg.name === name) return pkg.version ?? 'unknown'; + } + dir = path.dirname(dir); + } + } catch { /* reported as unknown */ } + return 'unknown'; +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + const set = JSON.parse(fs.readFileSync(path.join(BENCH_DIR, 'reference-set.json'), 'utf8')) as ReferenceSet; + const modelsDir = process.env.WAGGLE_MODELS_DIR ?? path.join(os.homedir(), '.waggle', 'models'); + const plan = planEmbedder(options.embedder, modelsDir); + guardNetwork(plan.remoteHost); + + const core = await import('@waggle/core') as unknown as Record; + const legacy = core as LegacyCore; + const hasOldEngine = Boolean(legacy.MindDB && legacy.FrameStore && legacy.SessionStore && legacy.HybridSearch); + const hasReranker = Boolean(legacy.createInProcessReranker); + const available = (name: EngineName) => + (hasOldEngine || backendOf(name) !== 'old') && (hasReranker || !name.endsWith('-rerank')); + const wanted = options.engines ?? ENGINE_NAMES.filter(available); + const missing = wanted.filter((name) => !available(name)); + if (missing.length > 0) { + throw new Error(`not in this checkout (the hive-mind substrate is gone): ${missing.join(', ')}; try --engines mem0,cosine`); + } + + const embedder = await plan.create(core); + let reranker: Reranker | undefined; + if (wanted.some((name) => name.endsWith('-rerank'))) { + // The sidecar caches the reranker under /reranker (packages/server/src/local/index.ts). + reranker = await legacy.createInProcessReranker!({ cacheDir: path.join(modelsDir, 'reranker') }); + } + + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-search-bench-')); + const backends = new Map(); + try { + const needed = new Set(wanted.map(backendOf)); + if (needed.has('old')) { + backends.set('old', await createOldBackend(legacy, set, embedder, fs.mkdtempSync(path.join(tempRoot, 'old-')))); + } + if (needed.has('mem0')) { + backends.set('mem0', await createMem0Backend(set, embedder, fs.mkdtempSync(path.join(tempRoot, 'mem0-')))); + } + if (needed.has('cosine')) backends.set('cosine', await createCosineBackend(set, embedder)); + + const runs = new Map(); + for (const engine of wanted) { + const backend = backends.get(backendOf(engine))!; + const engineReranker = engine.endsWith('-rerank') ? reranker : undefined; + // Warm up so the first timed query does not pay for lazy initialisation. + await backend.search('warm-up query', ['personal'], options.limit, engineReranker); + const results: QueryResult[] = []; + for (const query of set.queries) { + const started = performance.now(); + const hits = await backend.search(query.text, query.scopes, options.limit, engineReranker); + results.push(evaluate(query, hits, performance.now() - started)); + } + runs.set(engine, results); + } + + const memo = (name: BackendName) => backends.get(name) ? `${f1(backends.get(name)!.ingestMs / 1000)} s` : 'not run'; + const setup = [ + `Reference set: ${set.memories.length} memories, ${set.queries.length} queries (${set.queries.filter((q) => q.relevant.length === 0).length} without an answer)`, + `Embedder: ${plan.label}`, + `Results per query: ${options.limit}; reranker pool: ${RERANK_POOL}`, + `mem0ai ${packageVersion(path.join(BENCH_DIR, '..'), 'mem0ai')}, @huggingface/transformers ${packageVersion(path.join(BENCH_DIR, '..', '..', 'core'), '@huggingface/transformers')}, Node ${process.version}`, + `Machine: ${os.cpus()[0]?.model ?? 'unknown CPU'}, ${os.cpus().length} cores, ${Math.round(os.totalmem() / 2 ** 30)} GB, ${os.platform()} ${os.release()}`, + `Load time: old engine ${memo('old')}, mem0 ${memo('mem0')}, cosine ${memo('cosine')} (embedding included)`, + `Network: ${requests.local} local requests, ${requests.remote} to the embedding API, ${requests.blocked.length} blocked`, + ]; + console.log(report(set, runs, setup)); + + if (options.out) { + fs.writeFileSync(options.out, `${JSON.stringify({ setup, results: Object.fromEntries(runs) }, null, 2)}\n`); + console.error(`Wrote ${options.out}`); + } + if (requests.blocked.length > 0) { + throw new Error(`blocked network requests: ${requests.blocked.join(', ')}`); + } + } finally { + for (const backend of backends.values()) await backend.close(); + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + process.exitCode = 1; +});