moving
This commit is contained in:
281
benchmarks/harness/src/beam-multiroute.ts
Normal file
281
benchmarks/harness/src/beam-multiroute.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
/**
|
||||
* BEAM 1M — multi-route dated retrieval (E4 retrieval-architecture lever).
|
||||
*
|
||||
* Eywa's BEAM lead is dominated by COVERAGE / CHRONOLOGICAL-ORDER / CROSS-SESSION
|
||||
* retrieval, which a single top-30 similarity route cannot serve. This module
|
||||
* builds a deterministic 3-route retrieval over the DATED raw-turn store and
|
||||
* fuses it with RRF:
|
||||
*
|
||||
* Route V (vector) : substrate.search.search(q, {limit:kVec}) — existing.
|
||||
* Route T (timeline) : broad similarity fetch (kWide), dated via dateMap;
|
||||
* parseDateWindow(q) hard-filters to an explicit period
|
||||
* when present; ranked by DATE-STRATIFIED COVERAGE
|
||||
* (≤capPerDate turns/date, chronological) so breadth —
|
||||
* not similarity density — drives event_ordering / summ.
|
||||
* Route E (entity) : deterministic query-entity extraction → per-entity
|
||||
* keywordSearch (FTS, phrase-quoted, conv-scoped) → ALL
|
||||
* cross-session mentions of each entity (multi_session).
|
||||
*
|
||||
* FUSE: RRF (k=60) over the three ranked frame-id lists → top-N. Then EVERY
|
||||
* survivor is dated via the dateMap and rendered oldest→newest as
|
||||
* "[YYYY-MM-DD] role: content" (undated turns appended last, unbracketed).
|
||||
* Raw turns are kept verbatim (we still win instruction/preference on detail).
|
||||
*
|
||||
* WHY dateMap and not `created_at`: BEAM ingest writes every turn with
|
||||
* created_at = ingest time (all frames share one date), so the substrate's
|
||||
* built-in since/until temporal filter cannot separate turns by conversation
|
||||
* date. The dateMap (chat.json time_anchor) is the only real per-turn date.
|
||||
*
|
||||
* Pure retrieval + string assembly; the only I/O is substrate.search (local
|
||||
* ollama embeddings + SQLite). No LLM calls here.
|
||||
*/
|
||||
|
||||
import type { SearchResult } from '@waggle/core';
|
||||
import { parseDateWindow } from '@waggle/core';
|
||||
import type { Substrate } from './substrate.js';
|
||||
|
||||
const RRF_K = 60;
|
||||
|
||||
// ── Entity extraction (deterministic, query-side) ────────────────────────────
|
||||
|
||||
/** Question words / generic tokens that are never useful retrieval entities. */
|
||||
const STOP = new Set([
|
||||
'what', 'when', 'which', 'where', 'who', 'whom', 'whose', 'why', 'how', 'did',
|
||||
'do', 'does', 'have', 'has', 'had', 'was', 'were', 'is', 'are', 'am', 'the',
|
||||
'a', 'an', 'my', 'your', 'our', 'their', 'his', 'her', 'its', 'i', 'you', 'we',
|
||||
'they', 'it', 'this', 'that', 'these', 'those', 'and', 'or', 'but', 'for', 'to',
|
||||
'of', 'in', 'on', 'at', 'by', 'with', 'from', 'about', 'into', 'over', 'after',
|
||||
'before', 'between', 'during', 'since', 'until', 'ago', 'many', 'much', 'any',
|
||||
'some', 'all', 'each', 'every', 'been', 'be', 'being', 'will', 'would', 'can',
|
||||
'could', 'should', 'may', 'might', 'must', 'me', 'us', 'them', 'if', 'then',
|
||||
'there', 'here', 'now', 'ever', 'never', 'still', 'yet', 'also', 'just', 'so',
|
||||
'than', 'as', 'up', 'out', 'down', 'off', 'no', 'not', 'yes',
|
||||
// BEAM probing-question instruction / filler / quantifier words — these recur
|
||||
// verbatim ("Mention ONLY and ONLY ten items", "in order", "list every ...")
|
||||
// and are never useful retrieval entities.
|
||||
'mention', 'only', 'list', 'name', 'named', 'items', 'item', 'order', 'ordered',
|
||||
'progress', 'progressed', 'discussion', 'discussions', 'discussed', 'discuss',
|
||||
'say', 'said', 'tell', 'give', 'provide', 'describe', 'summarize', 'summary',
|
||||
'overview', 'account', 'sequence', 'chronological', 'versions', 'version',
|
||||
'thing', 'things', 'stuff', 'first', 'second', 'third', 'fourth', 'fifth', 'last',
|
||||
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
|
||||
'eleven', 'twelve', 'both', 'entire', 'whole', 'total', 'number', 'count',
|
||||
'happened', 'occur', 'occurred', 'time', 'times', 'date', 'dates', 'day', 'days',
|
||||
'week', 'weeks', 'month', 'months', 'year', 'years',
|
||||
'considering', 'using', 'given', 'based', 'regarding', 'concerning',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Extract distinctive entities from a question: quoted spans, version/tech
|
||||
* tokens (contain a digit or dot, e.g. "React 18.2", "v6.1.0", "port 4000"),
|
||||
* lowercase alnum tokens with a digit, and runs of Capitalized words
|
||||
* (proper-noun phrases). Deduplicated case-insensitively, capped.
|
||||
*/
|
||||
export function extractQueryEntities(question: string, cap = 6): string[] {
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const push = (raw: string): void => {
|
||||
const t = raw.trim().replace(/[.,;:?!]+$/, '').trim();
|
||||
if (t.length < 3) return;
|
||||
const key = t.toLowerCase();
|
||||
if (seen.has(key)) return;
|
||||
// Skip pure stopwords / question words.
|
||||
if (STOP.has(key)) return;
|
||||
seen.add(key);
|
||||
out.push(t);
|
||||
};
|
||||
|
||||
// 1) Quoted spans (highest precision).
|
||||
for (const m of question.matchAll(/["'“”‘’]([^"'“”‘’]{2,60})["'“”‘’]/g)) push(m[1]);
|
||||
|
||||
// 2) Version / tech tokens: a token containing a digit (React18, 18.2, v6.1.0,
|
||||
// 4000, PostgreSQL14) OR a dotted identifier. Keep an adjacent Capitalized
|
||||
// word as a two-word unit ("React 18.2", "PostgreSQL 14", "port 4000").
|
||||
const words = question.split(/\s+/);
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const w = words[i].replace(/[(),;:?!]+$/g, '').replace(/^[(),;:?!]+/g, '');
|
||||
if (/\d/.test(w) && /[A-Za-z0-9.]/.test(w) && !/^\d{4}$/.test(w) === true) {
|
||||
const prev = i > 0 ? words[i - 1].replace(/[(),;:?!]+$/g, '') : '';
|
||||
if (prev && (/^[A-Z]/.test(prev) || /^(port|version|v|node|python|react|postgres|postgresql)$/i.test(prev)) && !STOP.has(prev.toLowerCase())) {
|
||||
push(`${prev} ${w}`);
|
||||
}
|
||||
push(w);
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Runs of Capitalized words (proper-noun phrases), ignoring the leading
|
||||
// sentence-initial capital by only taking runs of length>=1 that aren't a
|
||||
// lone stopword. Multi-word runs are kept whole AND their head token.
|
||||
const capRun = /([A-Z][A-Za-z0-9+.#-]*(?:\s+[A-Z][A-Za-z0-9+.#-]*)*)/g;
|
||||
for (const m of question.matchAll(capRun)) {
|
||||
const phrase = m[1].trim();
|
||||
const toks = phrase.split(/\s+/);
|
||||
// Drop a leading sentence-initial single-cap common word (e.g. "What").
|
||||
if (toks.length === 1) {
|
||||
if (!STOP.has(toks[0].toLowerCase()) && toks[0].length >= 4) push(toks[0]);
|
||||
continue;
|
||||
}
|
||||
// Multi-word proper-noun phrase.
|
||||
const filtered = toks.filter((t, idx) => !(idx === 0 && STOP.has(t.toLowerCase())));
|
||||
if (filtered.length >= 2) push(filtered.join(' '));
|
||||
else if (filtered.length === 1 && filtered[0].length >= 4) push(filtered[0]);
|
||||
}
|
||||
|
||||
// 4) lowercase alnum tokens that carry a digit (e.g. "franc6", "gpt4") — rare
|
||||
// but distinctive; plain lowercase words are left to Route V/T.
|
||||
for (const w of words) {
|
||||
const t = w.replace(/[(),;:?!.]+$/g, '');
|
||||
if (/^[a-z][a-z0-9.-]*\d[a-z0-9.-]*$/i.test(t)) push(t);
|
||||
}
|
||||
|
||||
return out.slice(0, cap);
|
||||
}
|
||||
|
||||
// ── RRF fusion ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** RRF-fuse ranked id lists → id → score. Higher = better. */
|
||||
export function rrfFuse(lists: number[][], k = RRF_K): Map<number, number> {
|
||||
const scores = new Map<number, number>();
|
||||
for (const list of lists) {
|
||||
list.forEach((id, rank) => {
|
||||
scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
|
||||
});
|
||||
}
|
||||
return scores;
|
||||
}
|
||||
|
||||
// ── Timeline route (date-stratified coverage) ────────────────────────────────
|
||||
|
||||
interface DatedFrame { id: number; content: string; date: string }
|
||||
|
||||
/** Rank a candidate set for COVERAGE: keep ≤capPerDate per date, ordered
|
||||
* chronologically (oldest→newest). Returns frame ids in coverage order. */
|
||||
function coverageRank(dated: DatedFrame[], capPerDate: number, limit: number): number[] {
|
||||
const sorted = [...dated].sort((a, b) => (a.date === b.date ? a.id - b.id : a.date < b.date ? -1 : 1));
|
||||
const perDate = new Map<string, number>();
|
||||
const kept: number[] = [];
|
||||
for (const e of sorted) {
|
||||
const n = perDate.get(e.date) ?? 0;
|
||||
if (n >= capPerDate) continue;
|
||||
perDate.set(e.date, n + 1);
|
||||
kept.push(e.id);
|
||||
if (kept.length >= limit) break;
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
// ── Public: build the fused, dated, chronological context ────────────────────
|
||||
|
||||
export interface MultiRouteOptions {
|
||||
kVec: number; // Route V similarity limit (baseline parity, 30)
|
||||
kWide: number; // Route T broad similarity fetch (150)
|
||||
perEntity: number; // Route E per-entity FTS limit (20)
|
||||
capPerDate: number;// Route T coverage stratification cap (2)
|
||||
topN: number; // fused frames kept for the answer context (45)
|
||||
}
|
||||
|
||||
export const DEFAULT_MULTIROUTE: MultiRouteOptions = {
|
||||
kVec: 30, kWide: 150, perEntity: 20, capPerDate: 2, topN: 45,
|
||||
};
|
||||
|
||||
export interface MultiRouteResult {
|
||||
/** Dated display strings, oldest→newest, ready for buildAnswerGenerationPromptV2. */
|
||||
displayStrings: string[];
|
||||
/** Diagnostics. */
|
||||
entities: string[];
|
||||
dateWindow: { since: string; until: string; label: string } | null;
|
||||
nVec: number;
|
||||
nTimeline: number;
|
||||
nEntity: number;
|
||||
nFused: number;
|
||||
nDated: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all three routes over one conversation's raw-turn mind and return the
|
||||
* fused, dated, chronologically-ordered answer context. `dateMap` maps exact
|
||||
* frame content ("role: text") → ISO date (from chat.json time_anchor).
|
||||
*/
|
||||
export async function buildMultiRouteContext(
|
||||
sub: Substrate,
|
||||
gopId: string,
|
||||
question: string,
|
||||
dateMap: Map<string, string> | null,
|
||||
opts: MultiRouteOptions = DEFAULT_MULTIROUTE,
|
||||
): Promise<MultiRouteResult> {
|
||||
const frameById = new Map<number, { id: number; content: string }>();
|
||||
const record = (r: SearchResult): void => { frameById.set(r.frame.id, { id: r.frame.id, content: r.frame.content }); };
|
||||
const dateOf = (content: string): string => dateMap?.get(content) ?? '';
|
||||
|
||||
// Route V — vector (baseline parity).
|
||||
const vec = await sub.search.search(question, { limit: opts.kVec, gopId });
|
||||
vec.forEach(record);
|
||||
const Lv = vec.map(r => r.frame.id);
|
||||
|
||||
// Route T — timeline: broad fetch, date-window filter, coverage rank.
|
||||
const wide = await sub.search.search(question, { limit: opts.kWide, gopId });
|
||||
wide.forEach(record);
|
||||
const window = parseDateWindow(question);
|
||||
let widedated: DatedFrame[] = wide.map(r => ({ id: r.frame.id, content: r.frame.content, date: dateOf(r.frame.content) }));
|
||||
if (window) {
|
||||
const inWin = widedated.filter(e => e.date && e.date >= window.since && e.date <= window.until);
|
||||
// Only apply the window if it actually retains evidence; else keep the broad
|
||||
// set (a mis-parse must never empty the context).
|
||||
if (inWin.length > 0) widedated = inWin;
|
||||
}
|
||||
const Lt = coverageRank(widedated, opts.capPerDate, opts.kWide);
|
||||
|
||||
// Route E — entity: per-entity FTS across the whole conversation.
|
||||
const entities = extractQueryEntities(question);
|
||||
const entityHits = new Map<number, number>(); // id → #entities that matched it
|
||||
for (const ent of entities) {
|
||||
// Phrase-quote multi-word entities so keywordSearch does a phrase match
|
||||
// (it passes a query containing a quote through to FTS unchanged).
|
||||
const q = /\s/.test(ent) ? `"${ent.replace(/"/g, '')}"` : ent;
|
||||
let ids: number[] = [];
|
||||
try { ids = await sub.search.keywordSearch(q, opts.perEntity, gopId); } catch { ids = []; }
|
||||
for (const id of ids) entityHits.set(id, (entityHits.get(id) ?? 0) + 1);
|
||||
// Materialise contents for ids not yet seen by V/T.
|
||||
const missing = ids.filter(id => !frameById.has(id));
|
||||
if (missing.length) {
|
||||
const rawdb = sub.db.getDatabase();
|
||||
const ph = missing.map(() => '?').join(',');
|
||||
const rows = rawdb.prepare(`SELECT id, content FROM memory_frames WHERE id IN (${ph})`).all(...missing) as Array<{ id: number; content: string }>;
|
||||
for (const row of rows) frameById.set(row.id, row);
|
||||
}
|
||||
}
|
||||
// Le ranked by (most entities matched, then id) — frames mentioning more of
|
||||
// the question's entities rank higher.
|
||||
const Le = [...entityHits.entries()].sort((a, b) => (b[1] - a[1]) || (a[0] - b[0])).map(e => e[0]);
|
||||
|
||||
// FUSE — RRF over the three ranked lists → top-N.
|
||||
const fused = rrfFuse([Lv, Lt, Le]);
|
||||
const fusedIds = [...fused.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(e => e[0])
|
||||
.slice(0, opts.topN);
|
||||
|
||||
// DATE + chronological render: dated turns oldest→newest, undated appended last.
|
||||
const entries = fusedIds
|
||||
.map(id => frameById.get(id))
|
||||
.filter((f): f is { id: number; content: string } => !!f)
|
||||
.map(f => ({ id: f.id, content: f.content, date: dateOf(f.content) }));
|
||||
entries.sort((a, b) => {
|
||||
if (a.date && b.date) return a.date === b.date ? a.id - b.id : a.date < b.date ? -1 : 1;
|
||||
if (a.date) return -1; // dated before undated
|
||||
if (b.date) return 1;
|
||||
return a.id - b.id;
|
||||
});
|
||||
const displayStrings = entries.map(e => (e.date ? `[${e.date}] ${e.content}` : e.content));
|
||||
|
||||
return {
|
||||
displayStrings,
|
||||
entities,
|
||||
dateWindow: window,
|
||||
nVec: Lv.length,
|
||||
nTimeline: Lt.length,
|
||||
nEntity: Le.length,
|
||||
nFused: fusedIds.length,
|
||||
nDated: entries.filter(e => e.date).length,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user