refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)

- Orchestrator, memory tools, context loader and pattern write-back use a
  MemoryStore and a workspace scope instead of MindDB layers; the knowledge
  graph tools, cognify, memory linker and feedback handler are gone (graph
  hidden in v1, ADR-0002).
- The server opens one mem0 store in <dataDir>/memory/, embedded by the
  in-process model on first use, and extracts facts about the user in the
  background with the model the user chose (memoryExtraction setting).
- Legacy .mind memories are copied into the store once, in the background;
  the legacy files are only read.
- Weaver consolidation, harvest, wiki, memory import, memory lanes, vector
  backfill and index reconciliation are removed; agent runs, loops, sub-agent
  results, connector fetch and the failed-turn capture write through the
  contract. Workspace sessions no longer hold a mind handle.
- FileIndexer and the install audit move to waggle.db; recall date helpers and
  the reranker are copied into core.
- The CLI REPL opens the same memory store.
This commit is contained in:
shumeiko.it
2026-10-24 11:56:26 +02:00
parent f3eb6b6652
commit 784ca770b1
86 changed files with 2520 additions and 8444 deletions

View File

@@ -41,6 +41,8 @@ interface ConfigData {
maxIterations?: number;
/** M2-7: Telemetry opt-in (default: false — privacy first) */
telemetryEnabled?: boolean;
/** Extract facts about the user from chats in the background (default: true, ADR-0007). */
memoryExtraction?: boolean;
/** M2-1: Embedding provider configuration */
embedding?: {
provider?: EmbeddingProviderType | 'auto';
@@ -246,6 +248,18 @@ export class WaggleConfig {
this.save();
}
// --- Memory (ADR-0007) ---
/** Whether facts about the user are extracted from chats in the background. */
getMemoryExtraction(): boolean {
return this.data.memoryExtraction ?? true;
}
setMemoryExtraction(enabled: boolean): void {
this.data.memoryExtraction = enabled;
this.save();
}
// --- Embedding Provider (M2-1) ---
getEmbeddingConfig(): EmbeddingProviderConfig {

View File

@@ -3,3 +3,23 @@ export interface Embedder {
embedBatch(texts: string[]): Promise<Float32Array[]>;
dimensions: number;
}
/**
* Wraps an embedder that is loaded on first use, so callers never wait for a
* model download at startup. A failed load is retried on the next call.
*/
export function createLazyEmbedder(load: () => Promise<Embedder>, dimensions: number): Embedder {
let pending: Promise<Embedder> | null = null;
const get = (): Promise<Embedder> => {
pending ??= load().catch((err: unknown) => {
pending = null;
throw err;
});
return pending;
};
return {
dimensions,
embed: async (text) => (await get()).embed(text),
embedBatch: async (texts) => (await get()).embedBatch(texts),
};
}

View File

@@ -7,7 +7,7 @@
import path from 'node:path';
import os from 'node:os';
import type { Embedder } from './embeddings.js';
import { createLazyEmbedder, type Embedder } from './embeddings.js';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
@@ -71,3 +71,21 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
},
};
}
/** Memory vectors keep all-MiniLM-L6-v2's own size, unpadded (ADR-0007). */
export const MEMORY_EMBEDDING_DIMENSIONS = 384;
/**
* The embedder memory uses: the in-process model, loaded on first use. Every
* process that opens a data directory's memory must use the same one.
*/
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder {
return createLazyEmbedder(
() => createInProcessEmbedder({
model: options.model,
cacheDir: options.cacheDir,
targetDimensions: MEMORY_EMBEDDING_DIMENSIONS,
}),
MEMORY_EMBEDDING_DIMENSIONS,
);
}

View File

@@ -1,48 +1,43 @@
/**
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
* FileIndexer — auto-index workspace files into their workspace's memory (L-20).
*
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
* • Trigger: on-upload / on-overwrite (called from route handlers)
* • Target mind: per-workspace (the mind you pass in the constructor)
* • Granularity: 1 frame per file
* • Target: the workspace's memory scope
* • Granularity: 1 memory per file
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
* • Cleanup: re-index on overwrite, delete index on file remove
*
* A `file_index` table tracks the file_path → frame_id link so overwrites can
* swap the old frame cleanly and moves/deletes can locate the right frame
* without scanning every frame in the workspace.
* A `file_index` table in waggle.db tracks the file_path → memory link per
* workspace, so overwrites can swap the old memory cleanly and moves/deletes
* can locate the right memory without scanning the workspace's memories.
*
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
* full file is still on disk, only the indexed frame is shortened. Keeps the
* mind DB size bounded even if a user drops a 5MB markdown dump.
* full file is still on disk, only the indexed memory is shortened.
*/
import { createHash } from 'node:crypto';
import path from 'node:path';
import type { MindDB } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { workspaceScope, type MemoryStore } from '@waggle/memory-contract';
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
import { SessionStore } from '@waggle/hive-mind-core';
import { workspaceDataScope, type DataScope, type WaggleDB } from './waggle-db.js';
const FILE_INDEX_TABLE_SQL = `
export const FILE_INDEX_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS file_index (
file_path TEXT PRIMARY KEY,
frame_id INTEGER NOT NULL,
scope TEXT NOT NULL,
file_path TEXT NOT NULL,
memory_id TEXT NOT NULL,
mime_type TEXT,
size_bytes INTEGER NOT NULL DEFAULT 0,
content_hash TEXT NOT NULL,
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`;
const FILE_INDEX_IDX_SQL = `
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
indexed_at TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (scope, file_path)
);
CREATE INDEX IF NOT EXISTS idx_file_index_memory ON file_index (memory_id);
`;
const INDEX_SESSION_GOP_ID = 'session:file-index';
const INDEX_SESSION_PROJECT = 'file-indexer';
/** Maximum bytes of file content we store in a single frame. Beyond this, the
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
/** Maximum bytes of file content stored in a single memory. Beyond this, the
* content is truncated with a marker. */
export const MAX_CONTENT_BYTES = 64 * 1024;
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
@@ -50,7 +45,7 @@ const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
export interface FileIndexRow {
filePath: string;
frameId: number;
memoryId: string;
mimeType: string | null;
sizeBytes: number;
contentHash: string;
@@ -59,29 +54,18 @@ export interface FileIndexRow {
export type FileIndexResult =
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' | 'unsafe_content' }
| { skipped: false; frameId: number; truncated: boolean };
| { skipped: false; memoryId: string; truncated: boolean };
export class FileIndexer {
private readonly db: MindDB;
private readonly frames: FrameStore;
private readonly sessions: SessionStore;
private cachedGopId: string | null = null;
private readonly scope: DataScope;
constructor(db: MindDB) {
this.db = db;
this.frames = new FrameStore(db);
this.sessions = new SessionStore(db);
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
).get();
if (existsRow) return;
raw.prepare(FILE_INDEX_TABLE_SQL).run();
raw.prepare(FILE_INDEX_IDX_SQL).run();
constructor(
private readonly waggleDb: WaggleDB,
private readonly memory: MemoryStore,
private readonly workspaceId: string,
) {
this.scope = workspaceDataScope(workspaceId);
waggleDb.getDatabase().exec(FILE_INDEX_TABLE_SQL);
}
/** True iff the path's extension is one we index in v1. */
@@ -90,161 +74,111 @@ export class FileIndexer {
return INDEXABLE_EXTENSIONS.has(ext);
}
private ensureSession(): string {
if (this.cachedGopId) return this.cachedGopId;
const session = this.sessions.ensure(
INDEX_SESSION_GOP_ID,
INDEX_SESSION_PROJECT,
'Auto-indexed workspace files',
);
this.cachedGopId = session.gop_id;
return session.gop_id;
}
/**
* Index a file. Idempotent on unchanged content (same hash → skip). On
* overwrite with new content, deletes the old frame and inserts a fresh one.
* overwrite with new content, stores a fresh memory and forgets the old one.
*/
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
async indexFile(filePath: string, content: Buffer, mime?: string): Promise<FileIndexResult> {
if (!FileIndexer.shouldIndex(filePath)) {
return { skipped: true, reason: 'unsupported_format' };
}
if (content.length === 0) {
// Still remove any stale index so an emptied file doesn't leave an
// orphan frame pointing at old content.
this.removeFile(filePath);
// orphan memory pointing at old content.
await this.removeFile(filePath);
return { skipped: true, reason: 'empty' };
}
const hash = createHash('sha256').update(content).digest('hex');
const raw = this.db.getDatabase();
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
const existing = this.getRow(filePath);
const normalized = this.normalizeText(content);
const truncated = content.length > MAX_CONTENT_BYTES;
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
if (evaluateExternalMemoryIngress({ content: frameBody }).action === 'block') {
// An unchanged row may pre-date this guard. Remove that already-indexed
// unsafe projection atomically; on a new unsafe overwrite, retain the
// prior benign index while the file write itself remains successful.
if (existingRow && existingRow.content_hash === hash) {
raw.transaction(() => this.removeFile(filePath))();
}
const text = this.buildFrameContent(filePath, normalized, truncated, mime);
if (evaluateExternalMemoryIngress({ content: text }).action === 'block') {
// An unchanged row may pre-date this guard: remove that already-indexed
// unsafe projection. On a new unsafe overwrite, keep the prior benign
// index while the file write itself remains successful.
if (existing && existing.contentHash === hash) await this.removeFile(filePath);
return { skipped: true, reason: 'unsafe_content' };
}
if (existingRow && existingRow.content_hash === hash) {
if (existing && existing.contentHash === hash) {
return { skipped: true, reason: 'unchanged' };
}
const gopId = this.ensureSession();
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
// and the index-row write, leaving the table in its pre-call state. Without
// this, a crash between the delete and the UPDATE would leave file_index
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
// orphaned in the frames table.
const mutate = raw.transaction(() => {
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
if (existingRow) {
// Overwrite path: delete the old frame before we swap the row, unless
// the dedup coalesced two paths onto the same frame (leave that frame
// alone in that case — the other path still references it).
const oldFrameId = Number(existingRow.frame_id);
if (oldFrameId !== frame.id) {
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
if (!otherRef) {
this.frames.delete(oldFrameId);
}
}
raw.prepare(`
UPDATE file_index SET
frame_id = ?,
mime_type = ?,
size_bytes = ?,
content_hash = ?,
indexed_at = datetime('now')
WHERE file_path = ?
`).run(frame.id, mime ?? null, content.length, hash, filePath);
} else {
raw.prepare(`
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
VALUES (?, ?, ?, ?, ?)
`).run(filePath, frame.id, mime ?? null, content.length, hash);
}
return frame.id;
const [change] = await this.memory.remember({
scope: workspaceScope(this.workspaceId),
text,
kind: 'note',
source: 'system',
origin: 'file-index',
metadata: { file_path: filePath },
});
if (!change) throw new Error(`memory store returned nothing for ${filePath}`);
const memoryId = change.record.id;
const frameId = mutate();
return { skipped: false, frameId, truncated };
this.waggleDb.getDatabase().prepare(`
INSERT INTO file_index (scope, file_path, memory_id, mime_type, size_bytes, content_hash)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(scope, file_path) DO UPDATE SET
memory_id = excluded.memory_id,
mime_type = excluded.mime_type,
size_bytes = excluded.size_bytes,
content_hash = excluded.content_hash,
indexed_at = datetime('now')
`).run(this.scope, filePath, memoryId, mime ?? null, content.length, hash);
// Forget the previous memory unless another path still points at it (two
// paths with identical content share one memory).
if (existing && existing.memoryId !== memoryId) await this.forgetIfUnreferenced(existing.memoryId);
return { skipped: false, memoryId, truncated };
}
/** Remove a file's frame + index row. Returns true if anything was removed. */
removeFile(filePath: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
| { frame_id: number }
| undefined;
/** Remove a file's memory + index row. Returns true if anything was removed. */
async removeFile(filePath: string): Promise<boolean> {
const row = this.getRow(filePath);
if (!row) return false;
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
if (!otherRef) {
this.frames.delete(row.frame_id);
}
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
this.waggleDb.getDatabase()
.prepare('DELETE FROM file_index WHERE scope = ? AND file_path = ?')
.run(this.scope, filePath);
await this.forgetIfUnreferenced(row.memoryId);
return true;
}
/**
* Update the recorded path for a moved file. The frame content itself still
* carries the old path in its header — that's acceptable for v1 since the
* frame body is already the file's own content (not a description). A later
* reader sees the stale header only if they dig into raw frame content; the
* index table, which is what every other code path queries, is correct.
* Update the recorded path for a moved file. The memory text still carries
* the old path in its header; the index table, which every other code path
* queries, is correct.
*/
moveFile(from: string, to: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
| Record<string, unknown>
| undefined;
if (!row) return false;
if (!this.getRow(from)) return false;
const db = this.waggleDb.getDatabase();
// If `to` already indexed, drop its entry (file was replaced by the move).
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
db.prepare('DELETE FROM file_index WHERE scope = ? AND file_path = ?').run(this.scope, to);
db.prepare("UPDATE file_index SET file_path = ?, indexed_at = datetime('now') WHERE scope = ? AND file_path = ?")
.run(to, this.scope, from);
return true;
}
getRow(filePath: string): FileIndexRow | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
if (!row) return null;
return {
filePath: row.file_path as string,
frameId: row.frame_id as number,
mimeType: (row.mime_type as string | null) ?? null,
sizeBytes: row.size_bytes as number,
contentHash: row.content_hash as string,
indexedAt: row.indexed_at as string,
};
const row = this.waggleDb.getDatabase()
.prepare('SELECT * FROM file_index WHERE scope = ? AND file_path = ?')
.get(this.scope, filePath) as Record<string, unknown> | undefined;
return row ? toRow(row) : null;
}
listAll(): FileIndexRow[] {
const raw = this.db.getDatabase();
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
return rows.map(r => ({
filePath: r.file_path as string,
frameId: r.frame_id as number,
mimeType: (r.mime_type as string | null) ?? null,
sizeBytes: r.size_bytes as number,
contentHash: r.content_hash as string,
indexedAt: r.indexed_at as string,
}));
const rows = this.waggleDb.getDatabase()
.prepare('SELECT * FROM file_index WHERE scope = ? ORDER BY indexed_at DESC')
.all(this.scope) as Record<string, unknown>[];
return rows.map(toRow);
}
private async forgetIfUnreferenced(memoryId: string): Promise<void> {
const otherRef = this.waggleDb.getDatabase()
.prepare('SELECT 1 FROM file_index WHERE memory_id = ? LIMIT 1')
.get(memoryId);
if (!otherRef) await this.memory.forget(memoryId);
}
private normalizeText(content: Buffer): string {
@@ -266,3 +200,14 @@ export class FileIndexer {
return `${header}\n\n${body}${suffix}`;
}
}
function toRow(row: Record<string, unknown>): FileIndexRow {
return {
filePath: row.file_path as string,
memoryId: row.memory_id as string,
mimeType: (row.mime_type as string | null) ?? null,
sizeBytes: row.size_bytes as number,
contentHash: row.content_hash as string,
indexedAt: row.indexed_at as string,
};
}

View File

@@ -26,9 +26,12 @@ export {
type ExternalMemoryProjectionInput,
} from './memory-ingress-guard.js';
export { WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions } from './workspace-manager.js';
export type { Embedder } from './embeddings/embeddings.js';
export { createLazyEmbedder, type Embedder } from './embeddings/embeddings.js';
export { createLiteLLMEmbedder, type LiteLLMEmbedderConfig } from './embeddings/litellm-embedder.js';
export { createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig } from './embeddings/inprocess-embedder.js';
export {
createInProcessEmbedder, normalizeDimensions, createMemoryEmbedder, MEMORY_EMBEDDING_DIMENSIONS,
type InProcessEmbedderConfig,
} from './embeddings/inprocess-embedder.js';
export { createOllamaEmbedder, type OllamaEmbedderConfig } from './embeddings/ollama-embedder.js';
export { createApiEmbedder, type ApiEmbedderConfig } from './embeddings/api-embedder.js';
export {
@@ -36,6 +39,15 @@ export {
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
} from './embeddings/embedding-provider.js';
// Recall helpers: dates in queries and recalled text, and the cross-encoder reranker.
export {
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
} from './recall/recall-context.js';
export { parseDateWindow, type DateWindow } from './recall/parse-date-window.js';
export { resolveRelativeDate, type ResolvedDate } from './recall/resolve-relative-date.js';
export {
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
} from './recall/inprocess-reranker.js';
// ── waggle.db — Waggle's own data, apart from memory (ADR-0007) ──
export {
@@ -86,11 +98,6 @@ export {
type ReconcileResult,
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
type ConceptEntry, type ConceptUpdate,
// recall-context — temporal rendering helpers (W4.1 production port)
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
parseDateWindow, type DateWindow,
resolveRelativeDate, type ResolvedDate,
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
// harvest/ — universal memory ingestion pipeline
HarvestSourceStore,
HarvestRunStore, type HarvestRun, type HarvestRunStatus,

View File

@@ -5,10 +5,10 @@
* failed) so there is a verifiable history of what was installed, when, why,
* and by whom.
*
* Operates on the .mind DB like the other stores in this package.
* Lives in waggle.db with Waggle's other own data (ADR-0007).
*/
import type { MindDB } from '@waggle/hive-mind-core';
import type { WaggleDB } from './waggle-db.js';
import {
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
@@ -63,10 +63,7 @@ export interface RecordAuditInput {
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
// drift (divergence #14 — the old comment admitted "drift silently crashes
// record()"). The matching DDL in hive-mind-core/src/mind/schema.ts stays a
// standalone private-monorepo literal and is locked to these same canonical
// lists by install-audit-check-parity.test.ts. It is stripped from the curated
// OSS export.
// record()").
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
// while the TS type claimed a closed set). Every historical value came from the
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
@@ -93,9 +90,9 @@ CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC)
// ── Store ──────────────────────────────────────────────────────────────
export class InstallAuditStore {
private db: MindDB;
private db: WaggleDB;
constructor(db: MindDB) {
constructor(db: WaggleDB) {
this.db = db;
this.ensureTable();
}
@@ -127,13 +124,13 @@ export class InstallAuditStore {
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
* crash mid-rebuild leaves the original table intact.
*/
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
private rebuildForWidenedActionCheck(raw: ReturnType<WaggleDB['getDatabase']>): void {
const migrate = raw.transaction(() => {
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
// DROP below would take the indexes with the legacy table. Drop them first
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
// so they get recreated on the new table.
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
raw.exec(INSTALL_AUDIT_TABLE_SQL);

View File

@@ -0,0 +1,137 @@
/**
* In-process cross-encoder reranker using @huggingface/transformers (ONNX).
*
* Cross-encoders take (query, doc) pairs and output a relevance score by
* jointly attending to both — much more discriminating than vector dot
* products. Use after RRF to rerank the top-K candidates from hybrid
* search. Standard pattern in production RAG systems.
*
* Default model: Xenova/ms-marco-MiniLM-L-6-v2 — the canonical
* SentenceTransformers cross-encoder, ~22MB on disk, ~30-50ms per pair
* on CPU. Trained on MS MARCO passage ranking, generalizes well to
* mixed-domain technical text.
*
* Alternative: Xenova/bge-reranker-base (~280MB, slightly higher quality
* on out-of-domain queries). Set via `model` config.
*
* `@huggingface/transformers` is an optional peer dep — if not installed,
* createInProcessReranker throws and the caller falls back to no reranking.
*/
import path from 'node:path';
import os from 'node:os';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from '../embeddings/transformers-model-load.js';
const log = createCoreLogger('inprocess-reranker');
export interface Reranker {
/**
* Score a single (query, doc) pair. Higher = more relevant.
* Score scale depends on the model — for ms-marco-MiniLM it's
* roughly [-10, 10]; relative ordering is what matters.
*/
score(query: string, doc: string): Promise<number>;
/**
* Score N pairs sharing one query. Same-shape result as score() but
* amortises the model invocation when supported.
*/
scoreBatch(query: string, docs: string[]): Promise<number[]>;
}
export interface InProcessRerankerConfig {
model?: string;
cacheDir?: string;
}
/**
* Build a reranker backed by @huggingface/transformers. Throws if the
* package isn't installed — caller is expected to catch and fall back.
*/
export async function createInProcessReranker(
config?: Partial<InProcessRerankerConfig>,
): Promise<Reranker> {
const model = config?.model ?? 'Xenova/ms-marco-MiniLM-L-6-v2';
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.hive-mind', 'models');
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
'@huggingface/transformers'
);
const { tokenizer, seqModel } = await withTransformersModelLoad({
cacheDir,
model,
load: async (canonicalCacheDir) => ({
// Cross-encoders need direct tokenizer + model access — pipeline API
// doesn't expose the (text, text_pair) input pattern cleanly across
// all transformers.js versions. Calling the model directly with
// tokenized pairs is the stable path.
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
}),
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
});
log.info(`In-process reranker ready: ${model}`);
/** Score a single pair: tokenize, forward, extract logit. */
async function scorePair(query: string, doc: string): Promise<number> {
const inputs = await tokenizer(query, {
text_pair: doc,
padding: true,
truncation: true,
return_tensor: true,
});
const out = await seqModel(inputs);
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
// Other cross-encoders may output 2 classes — take logit[0] - logit[1]
// as a relevance score in that case.
const logits = out.logits ?? out[0];
const data = logits.data as Float32Array | number[];
if (logits.dims && logits.dims[logits.dims.length - 1] === 2) {
return Number(data[0]) - Number(data[1]);
}
return Number(data[0]);
}
return {
async score(query: string, doc: string): Promise<number> {
return scorePair(query, doc);
},
async scoreBatch(query: string, docs: string[]): Promise<number[]> {
if (docs.length === 0) return [];
// Tokenize all pairs together for batch inference. Padding aligns
// sequences so the model can process them in one forward pass.
const queries = docs.map(() => query);
const inputs = await tokenizer(queries, {
text_pair: docs,
padding: true,
truncation: true,
return_tensor: true,
});
const out = await seqModel(inputs);
const logits = out.logits ?? out[0];
const data = logits.data as Float32Array | number[];
const dims = logits.dims;
const lastDim = dims[dims.length - 1];
const scores: number[] = [];
if (lastDim === 2) {
for (let i = 0; i < docs.length; i++) {
scores.push(Number(data[i * 2]) - Number(data[i * 2 + 1]));
}
} else {
for (let i = 0; i < docs.length; i++) {
scores.push(Number(data[i]));
}
}
return scores;
},
};
}

View File

@@ -0,0 +1,113 @@
/**
* parse-date-window.ts — deterministic query-side temporal-constraint parser
* (W4.1b, production port of the benchmark-proven Wave-3.1 date-window lane;
* W4-PRODUCTION-PORT-PLAN-2026-06-11.md component #3, MRAG arXiv:2412.15540
* pattern).
*
* When a query names an explicit period — "the last week of October 2023",
* "early June 2023", "on 9 August 2023", "May 2023", "in 2022" — return a
* `[since..until]` window (date-only `YYYY-MM-DD` bounds, inclusive) plus a
* human label. Callers pass the window to HybridSearch's since/until filter;
* the label feeds the future Events-during-X render section (component #6).
*
* Deterministic regex date math only — no LLM call, no new index. Returns
* null when the query carries no explicit period (relative phrases like
* "last week" / "two months ago" are resolution work for the WRITE side —
* resolve-relative-date.ts — not query windowing).
*
* Regex shapes are byte-equivalent to the benchmark parser validated on the
* full N=1540 LoCoMo run (wave3a); only types/docs differ.
*
* OSS-clean: pure date arithmetic, no vault/evolution/compliance deps.
*/
/** Inclusive date-only window parsed from an explicit period in a query. */
export interface DateWindow {
/** Inclusive lower bound, `YYYY-MM-DD`. */
since: string;
/** Inclusive upper bound, `YYYY-MM-DD`. */
until: string;
/** Human-readable label of the matched period (for render sections). */
label: string;
}
const MONTHS: Record<string, number> = {
january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
};
const MONTH_RE = '(january|february|march|april|may|june|july|august|september|october|november|december)';
function lastDayOfMonth(y: number, m: number): number {
return new Date(Date.UTC(y, m, 0)).getUTCDate();
}
function isoOf(y: number, m: number, d: number): string {
return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
}
/**
* Parse an explicit-period temporal constraint out of a query. Returns null
* when no explicit period is named.
*/
export function parseDateWindow(query: string): DateWindow | null {
const t = String(query).toLowerCase();
// "first|second|third|fourth|last week of <month> <year>"
let m = t.match(new RegExp(`(first|second|third|fourth|last)\\s+week\\s+of\\s+${MONTH_RE}\\s+(\\d{4})`));
if (m) {
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
const last = lastDayOfMonth(y, mo);
const ranges: Record<string, [number, number]> = {
first: [1, 7], second: [8, 14], third: [15, 21], fourth: [22, 28],
last: [Math.max(1, last - 6), last],
};
const [d1, d2] = ranges[m[1]];
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, Math.min(d2, last)), label: `the ${m[1]} week of ${m[2]} ${y}` };
}
// "early|mid|late <month> <year>"
m = t.match(new RegExp(`(early|mid|late)\\s+${MONTH_RE}\\s+(\\d{4})`));
if (m) {
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
const last = lastDayOfMonth(y, mo);
const ranges: Record<string, [number, number]> = { early: [1, 10], mid: [11, 20], late: [21, last] };
const [d1, d2] = ranges[m[1]];
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, d2), label: `${m[1]} ${m[2]} ${y}` };
}
// "<day> <month> <year>" or "<month> <day>, <year>" → exact-day ±2 buffer
m = t.match(new RegExp(`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(?:of\\s+)?${MONTH_RE},?\\s+(\\d{4})`)) ||
t.match(new RegExp(`${MONTH_RE}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(\\d{4})`));
if (m) {
const isDayFirst = /^\d/.test(m[1]);
const day = parseInt(isDayFirst ? m[1] : m[2], 10);
const mo = MONTHS[isDayFirst ? m[2] : m[1]];
const y = parseInt(m[3], 10);
if (mo && day >= 1 && day <= 31) {
const center = Date.UTC(y, mo - 1, day);
const lo = new Date(center - 2 * 86400000), hi = new Date(center + 2 * 86400000);
return {
since: lo.toISOString().slice(0, 10),
until: hi.toISOString().slice(0, 10),
label: `${day} ${isDayFirst ? m[2] : m[1]} ${y}`,
};
}
}
// "<month> <year>" → whole month
m = t.match(new RegExp(`${MONTH_RE}\\s+(\\d{4})`));
if (m) {
const y = parseInt(m[2], 10), mo = MONTHS[m[1]];
return { since: isoOf(y, mo, 1), until: isoOf(y, mo, lastDayOfMonth(y, mo)), label: `${m[1]} ${y}` };
}
// bare "in|during <year>" → whole year ('in'/'during' required so years
// inside names/ids don't window the query)
m = t.match(/\b(?:in|during)\s+(20\d{2})\b/);
if (m) {
const y = m[1];
return { since: `${y}-01-01`, until: `${y}-12-31`, label: y };
}
return null;
}

View File

@@ -0,0 +1,100 @@
/**
* recall-context.ts — shared renderer for surfacing temporal information in
* recalled-memory blocks. Single source of truth imported by both the Waggle
* agent (production) and the benchmark harness, so the injected-memory format
* never drifts between the two.
*
* Scope (Temporal Substrate Fix, Phase 1 — "surface time", additive only):
* - Prefix each retrieved snippet with its own compact `[YYYY-MM-DD]` date.
* - Open the rendered memory block with one anchor line giving the most-recent
* memory date, so the model has a concrete "now" to resolve relative time
* expressions against.
* - Export `TEMPORAL_GUIDANCE`, the prompt fragment that tells the model to
* treat those timestamps as the anchor for relative-time arithmetic.
*
* Distilled "Memory Facts" (cross-session syntheses) are intentionally NOT
* dated here — they have no single reliable date. Phase 2 handles them.
*
* OSS-clean: pure string formatting, no vault/evolution/compliance deps.
*/
/**
* Prompt fragment wired into the memory-recall injection path (NOT the global
* system prompt). Teaches the model to use the surfaced `[YYYY-MM-DD]` stamps
* as the anchor for resolving relative time expressions and conflicting facts.
*
* W4.1 upgrade (W4-PRODUCTION-PORT-PLAN-2026-06-11.md §1): replaced the original
* "nearest timestamp as anchor" phrasing with the benchmark-proven W1 wording —
* concrete relative-date arithmetic with worked examples (Memori instruction-5
* lineage, incl. the verified conv-26 "yesterday" failure case) plus the
* granularity-calibration clause (failure mining: 22 temporal fails emitted a
* confident exact ISO day 1-7 days off where a coarse answer was correct).
* Temporal was the #1 LoCoMo lever (80.06 → 84.7 across W1-W3.1).
* Production-safe subset: no never-refuse clause (that was benchmark-cell
* policy only — conditional abstention stays).
*/
export const TEMPORAL_GUIDANCE =
"Memories and snippets are timestamped [YYYY-MM-DD]. Pay special attention to these " +
"timestamps to determine timing. If a question involves relative time references " +
"('last year', 'two months ago', 'yesterday', 'last week'), CALCULATE the actual date " +
"from the timestamp of the memory that mentions it. For example: a memory dated " +
"4 May 2022 that says 'went to India last year' means the trip was in 2021; a memory " +
"dated 8 May 2023 that says 'I went to the group yesterday' means the event was 7 May 2023. " +
"Always convert relative references to specific dates, months, or years using the " +
"memory's timestamp as the anchor, and ignore the relative phrase itself when answering. " +
"When the same fact appears at different times, the most recent version is correct. " +
"GRANULARITY: state an exact day ONLY when that exact date was explicitly stated or " +
"directly computed from an explicit relative reference; otherwise answer at the " +
"granularity you are confident in — 'early June 2023', 'the week before 9 August 2023', " +
"'August 2022'. A confidently wrong exact day is worse than a correct coarse answer. " +
"For 'how long / how many months' duration questions, give ONLY the final value " +
"(e.g. 'six months') — no intermediate dates, no reasoning steps.";
/** Anchor-line prefix for the most-recent rendered memory date. */
const REFERENCE_DATE_LABEL = 'Reference date (most recent memory):';
/**
* Slice an ISO-ish timestamp to its `YYYY-MM-DD` date prefix. Returns null when
* the value is missing or too short to carry a date.
*/
export function toDatePrefix(createdAt: string | null | undefined): string | null {
if (!createdAt || createdAt.length < 10) return null;
return createdAt.slice(0, 10);
}
/**
* Render a single retrieved snippet with its compact `[YYYY-MM-DD]` date prefix.
* `text` is the snippet text exactly as it would otherwise be rendered (the
* caller owns importance labels, truncation, etc.); this only prepends the date.
* Falls back to the bare text when the hit carries no usable timestamp.
*/
export function renderDatedSnippet(createdAt: string | null | undefined, text: string): string {
const date = toDatePrefix(createdAt);
return date ? `[${date}] ${text}` : text;
}
/**
* Compute the reference (anchor) date: the maximum `created_at` among the
* supplied timestamps, sliced to `YYYY-MM-DD`. Returns null when none carry a
* usable date (caller then omits the anchor line).
*/
export function referenceDate(createdAts: ReadonlyArray<string | null | undefined>): string | null {
let max: string | null = null;
for (const c of createdAts) {
const date = toDatePrefix(c);
if (date && (max === null || date > max)) max = date;
}
return max;
}
/**
* Build the single anchor line for a memory block, e.g.
* `Reference date (most recent memory): 2026-06-09`. Returns null when no date
* is available.
*/
export function renderReferenceDateLine(
createdAts: ReadonlyArray<string | null | undefined>,
): string | null {
const date = referenceDate(createdAts);
return date ? `${REFERENCE_DATE_LABEL} ${date}` : null;
}

View File

@@ -0,0 +1,131 @@
/**
* resolve-relative-date — write-time relative-date resolution for the memory substrate.
*
* WHY THIS EXISTS (benchmark-validated, LoCoMo head-to-head vs Memori, 2026-06-10):
* A conversation utterance's *discussion date* (when it was said) is not the same as
* the *event date* (when the thing happened). "I went to the group yesterday" said on
* 2023-05-08 describes an event on 2023-05-07. Storing the discussion date makes
* "when did X happen?" questions wrong by the relative-reference delta — the verified
* root cause of our temporal-category gap. Resolving the relative reference against the
* source date at WRITE time bakes the correct event date into the frame's timestamp,
* which lifted the LoCoMo temporal category to parity with Memori (80.06 vs 80.37).
* See benchmarks/results/memori-phase22-RESULT.md (Phase 4).
*
* This is the deterministic, dependency-free production counterpart of the benchmark's
* LLM extraction pass (hive-mind-test scripts/locomo/33-distill-episodic.mjs). It covers
* the dominant cue patterns the calibration surfaced ("yesterday", "last week", "N days
* ago", "last <weekday>", "last year") without a per-frame LLM call. Pure + side-effect
* free; the caller decides whether to use the resolved date.
*/
/** A resolved relative-date hit: the matched cue phrase + the absolute ISO date (YYYY-MM-DD). */
export interface ResolvedDate {
/** The relative cue that matched, e.g. "yesterday", "last week", "3 months ago". */
cue: string;
/** The resolved absolute date as YYYY-MM-DD. */
iso: string;
}
const ISO_DATE = /^\d{4}-\d{2}-\d{2}/;
const WEEKDAYS = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] as const;
/** Parse an ISO-ish reference date (date or datetime) into a UTC Date at midnight. Null if unparseable. */
function parseReference(referenceDate: string | null | undefined): Date | null {
if (!referenceDate || !ISO_DATE.test(referenceDate)) return null;
const [y, m, d] = referenceDate.slice(0, 10).split('-').map(Number);
if (!y || !m || !d) return null;
const dt = new Date(Date.UTC(y, m - 1, d));
return Number.isNaN(dt.getTime()) ? null : dt;
}
/** Format a UTC Date as YYYY-MM-DD. */
function toIso(dt: Date): string {
return dt.toISOString().slice(0, 10);
}
function addDays(dt: Date, n: number): Date {
const out = new Date(dt.getTime());
out.setUTCDate(out.getUTCDate() + n);
return out;
}
function addMonths(dt: Date, n: number): Date {
const out = new Date(dt.getTime());
const targetMonthDay = out.getUTCDate();
out.setUTCDate(1);
out.setUTCMonth(out.getUTCMonth() + n);
// Clamp to the last valid day of the resulting month (e.g. Jan 31 − 1mo → Dec 31, not overflow).
const lastDay = new Date(Date.UTC(out.getUTCFullYear(), out.getUTCMonth() + 1, 0)).getUTCDate();
out.setUTCDate(Math.min(targetMonthDay, lastDay));
return out;
}
function addYears(dt: Date, n: number): Date {
const out = new Date(dt.getTime());
out.setUTCFullYear(out.getUTCFullYear() + n);
return out;
}
/** Most recent occurrence of `weekday` strictly before the reference date (the "last Friday" sense). */
function lastWeekday(dt: Date, weekday: number): Date {
let delta = dt.getUTCDay() - weekday;
if (delta <= 0) delta += 7; // strictly before → at least 1 day back
return addDays(dt, -delta);
}
/**
* Resolve the first relative-time cue in `text` against `referenceDate` (the source/
* conversation date) into an absolute YYYY-MM-DD. Returns null when there is no
* recognised cue or the reference date is unusable — the caller then keeps the
* reference date as-is.
*
* Patterns are tried most-specific first so "the day before yesterday" wins over
* "yesterday", and explicit "N units ago" wins over the bare "last unit".
*/
export function resolveRelativeDate(
text: string,
referenceDate: string | null | undefined,
): ResolvedDate | null {
const ref = parseReference(referenceDate);
if (!ref || !text) return null;
const t = text.toLowerCase();
// Most-specific day offsets first.
if (/\bday before yesterday\b/.test(t)) return { cue: 'the day before yesterday', iso: toIso(addDays(ref, -2)) };
if (/\byesterday\b/.test(t)) return { cue: 'yesterday', iso: toIso(addDays(ref, -1)) };
// "N days/weeks/months/years ago" (explicit count).
const ago = t.match(/\b(\d{1,3})\s+(day|week|month|year)s?\s+ago\b/);
if (ago) {
const n = parseInt(ago[1], 10);
const unit = ago[2];
const iso =
unit === 'day' ? toIso(addDays(ref, -n)) :
unit === 'week' ? toIso(addDays(ref, -7 * n)) :
unit === 'month' ? toIso(addMonths(ref, -n)) :
toIso(addYears(ref, -n));
return { cue: `${n} ${unit}${n === 1 ? '' : 's'} ago`, iso };
}
// "a week/month/year ago" (singular, count = 1).
const aAgo = t.match(/\ba\s+(week|month|year)\s+ago\b/);
if (aAgo) {
const unit = aAgo[1];
const iso = unit === 'week' ? toIso(addDays(ref, -7)) : unit === 'month' ? toIso(addMonths(ref, -1)) : toIso(addYears(ref, -1));
return { cue: `a ${unit} ago`, iso };
}
// "last <weekday>" → most recent prior occurrence of that weekday.
const lastDow = t.match(/\blast\s+(sunday|monday|tuesday|wednesday|thursday|friday|saturday)\b/);
if (lastDow) {
const wd = WEEKDAYS.indexOf(lastDow[1] as (typeof WEEKDAYS)[number]);
return { cue: `last ${lastDow[1]}`, iso: toIso(lastWeekday(ref, wd)) };
}
// "last week/month/year" (bare, coarse offset).
if (/\blast\s+week\b/.test(t)) return { cue: 'last week', iso: toIso(addDays(ref, -7)) };
if (/\blast\s+month\b/.test(t)) return { cue: 'last month', iso: toIso(addMonths(ref, -1)) };
if (/\blast\s+year\b/.test(t)) return { cue: 'last year', iso: toIso(addYears(ref, -1)) };
return null;
}

View File

@@ -15,6 +15,7 @@ import { AwarenessLayer } from './awareness.js';
import { CronStore } from './cron-store.js';
import { ExecutionTraceStore } from './execution-traces.js';
import { IdentityLayer } from './identity.js';
import { InstallAuditStore } from './install-audit.js';
import { SkillHashStore } from './skill-hashes.js';
import type { DataScope, WaggleDB } from './waggle-db.js';
@@ -30,6 +31,7 @@ const PERSONAL_TABLES = [
'execution_traces',
'execution_trace_spend',
'execution_trace_spend_reservations',
'install_audit',
] as const;
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
@@ -125,6 +127,7 @@ export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSour
new CronStore(waggleDb);
new SkillHashStore(waggleDb);
new ExecutionTraceStore(waggleDb);
new InstallAuditStore(waggleDb);
new IdentityLayer(waggleDb);
new AwarenessLayer(waggleDb);