refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)
- Orchestrator, memory tools, context loader and pattern write-back use a MemoryStore and a workspace scope instead of MindDB layers; the knowledge graph tools, cognify, memory linker and feedback handler are gone (graph hidden in v1, ADR-0002). - The server opens one mem0 store in <dataDir>/memory/, embedded by the in-process model on first use, and extracts facts about the user in the background with the model the user chose (memoryExtraction setting). - Legacy .mind memories are copied into the store once, in the background; the legacy files are only read. - Weaver consolidation, harvest, wiki, memory import, memory lanes, vector backfill and index reconciliation are removed; agent runs, loops, sub-agent results, connector fetch and the failed-turn capture write through the contract. Workspace sessions no longer hold a mind handle. - FileIndexer and the install audit move to waggle.db; recall date helpers and the reranker are copied into core. - The CLI REPL opens the same memory store.
This commit is contained in:
@@ -41,6 +41,8 @@ interface ConfigData {
|
||||
maxIterations?: number;
|
||||
/** M2-7: Telemetry opt-in (default: false — privacy first) */
|
||||
telemetryEnabled?: boolean;
|
||||
/** Extract facts about the user from chats in the background (default: true, ADR-0007). */
|
||||
memoryExtraction?: boolean;
|
||||
/** M2-1: Embedding provider configuration */
|
||||
embedding?: {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
@@ -246,6 +248,18 @@ export class WaggleConfig {
|
||||
this.save();
|
||||
}
|
||||
|
||||
// --- Memory (ADR-0007) ---
|
||||
|
||||
/** Whether facts about the user are extracted from chats in the background. */
|
||||
getMemoryExtraction(): boolean {
|
||||
return this.data.memoryExtraction ?? true;
|
||||
}
|
||||
|
||||
setMemoryExtraction(enabled: boolean): void {
|
||||
this.data.memoryExtraction = enabled;
|
||||
this.save();
|
||||
}
|
||||
|
||||
// --- Embedding Provider (M2-1) ---
|
||||
|
||||
getEmbeddingConfig(): EmbeddingProviderConfig {
|
||||
|
||||
@@ -3,3 +3,23 @@ export interface Embedder {
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
||||
dimensions: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps an embedder that is loaded on first use, so callers never wait for a
|
||||
* model download at startup. A failed load is retried on the next call.
|
||||
*/
|
||||
export function createLazyEmbedder(load: () => Promise<Embedder>, dimensions: number): Embedder {
|
||||
let pending: Promise<Embedder> | null = null;
|
||||
const get = (): Promise<Embedder> => {
|
||||
pending ??= load().catch((err: unknown) => {
|
||||
pending = null;
|
||||
throw err;
|
||||
});
|
||||
return pending;
|
||||
};
|
||||
return {
|
||||
dimensions,
|
||||
embed: async (text) => (await get()).embed(text),
|
||||
embedBatch: async (texts) => (await get()).embedBatch(texts),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createLazyEmbedder, type Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
@@ -71,3 +71,21 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Memory vectors keep all-MiniLM-L6-v2's own size, unpadded (ADR-0007). */
|
||||
export const MEMORY_EMBEDDING_DIMENSIONS = 384;
|
||||
|
||||
/**
|
||||
* The embedder memory uses: the in-process model, loaded on first use. Every
|
||||
* process that opens a data directory's memory must use the same one.
|
||||
*/
|
||||
export function createMemoryEmbedder(options: { cacheDir: string; model?: string }): Embedder {
|
||||
return createLazyEmbedder(
|
||||
() => createInProcessEmbedder({
|
||||
model: options.model,
|
||||
cacheDir: options.cacheDir,
|
||||
targetDimensions: MEMORY_EMBEDDING_DIMENSIONS,
|
||||
}),
|
||||
MEMORY_EMBEDDING_DIMENSIONS,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,48 +1,43 @@
|
||||
/**
|
||||
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
|
||||
* FileIndexer — auto-index workspace files into their workspace's memory (L-20).
|
||||
*
|
||||
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
|
||||
* • Trigger: on-upload / on-overwrite (called from route handlers)
|
||||
* • Target mind: per-workspace (the mind you pass in the constructor)
|
||||
* • Granularity: 1 frame per file
|
||||
* • Target: the workspace's memory scope
|
||||
* • Granularity: 1 memory per file
|
||||
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
|
||||
* • Cleanup: re-index on overwrite, delete index on file remove
|
||||
*
|
||||
* A `file_index` table tracks the file_path → frame_id link so overwrites can
|
||||
* swap the old frame cleanly and moves/deletes can locate the right frame
|
||||
* without scanning every frame in the workspace.
|
||||
* A `file_index` table in waggle.db tracks the file_path → memory link per
|
||||
* workspace, so overwrites can swap the old memory cleanly and moves/deletes
|
||||
* can locate the right memory without scanning the workspace's memories.
|
||||
*
|
||||
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
|
||||
* full file is still on disk, only the indexed frame is shortened. Keeps the
|
||||
* mind DB size bounded even if a user drops a 5MB markdown dump.
|
||||
* full file is still on disk, only the indexed memory is shortened.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { workspaceScope, type MemoryStore } from '@waggle/memory-contract';
|
||||
import { evaluateExternalMemoryIngress } from './memory-ingress-guard.js';
|
||||
import { SessionStore } from '@waggle/hive-mind-core';
|
||||
import { workspaceDataScope, type DataScope, type WaggleDB } from './waggle-db.js';
|
||||
|
||||
const FILE_INDEX_TABLE_SQL = `
|
||||
export const FILE_INDEX_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS file_index (
|
||||
file_path TEXT PRIMARY KEY,
|
||||
frame_id INTEGER NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
memory_id TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
content_hash TEXT NOT NULL,
|
||||
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`;
|
||||
const FILE_INDEX_IDX_SQL = `
|
||||
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
|
||||
indexed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (scope, file_path)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_index_memory ON file_index (memory_id);
|
||||
`;
|
||||
|
||||
const INDEX_SESSION_GOP_ID = 'session:file-index';
|
||||
const INDEX_SESSION_PROJECT = 'file-indexer';
|
||||
|
||||
/** Maximum bytes of file content we store in a single frame. Beyond this, the
|
||||
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
|
||||
/** Maximum bytes of file content stored in a single memory. Beyond this, the
|
||||
* content is truncated with a marker. */
|
||||
export const MAX_CONTENT_BYTES = 64 * 1024;
|
||||
|
||||
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
|
||||
@@ -50,7 +45,7 @@ const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
|
||||
|
||||
export interface FileIndexRow {
|
||||
filePath: string;
|
||||
frameId: number;
|
||||
memoryId: string;
|
||||
mimeType: string | null;
|
||||
sizeBytes: number;
|
||||
contentHash: string;
|
||||
@@ -59,29 +54,18 @@ export interface FileIndexRow {
|
||||
|
||||
export type FileIndexResult =
|
||||
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' | 'unsafe_content' }
|
||||
| { skipped: false; frameId: number; truncated: boolean };
|
||||
| { skipped: false; memoryId: string; truncated: boolean };
|
||||
|
||||
export class FileIndexer {
|
||||
private readonly db: MindDB;
|
||||
private readonly frames: FrameStore;
|
||||
private readonly sessions: SessionStore;
|
||||
private cachedGopId: string | null = null;
|
||||
private readonly scope: DataScope;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.frames = new FrameStore(db);
|
||||
this.sessions = new SessionStore(db);
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
|
||||
).get();
|
||||
if (existsRow) return;
|
||||
raw.prepare(FILE_INDEX_TABLE_SQL).run();
|
||||
raw.prepare(FILE_INDEX_IDX_SQL).run();
|
||||
constructor(
|
||||
private readonly waggleDb: WaggleDB,
|
||||
private readonly memory: MemoryStore,
|
||||
private readonly workspaceId: string,
|
||||
) {
|
||||
this.scope = workspaceDataScope(workspaceId);
|
||||
waggleDb.getDatabase().exec(FILE_INDEX_TABLE_SQL);
|
||||
}
|
||||
|
||||
/** True iff the path's extension is one we index in v1. */
|
||||
@@ -90,161 +74,111 @@ export class FileIndexer {
|
||||
return INDEXABLE_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
private ensureSession(): string {
|
||||
if (this.cachedGopId) return this.cachedGopId;
|
||||
const session = this.sessions.ensure(
|
||||
INDEX_SESSION_GOP_ID,
|
||||
INDEX_SESSION_PROJECT,
|
||||
'Auto-indexed workspace files',
|
||||
);
|
||||
this.cachedGopId = session.gop_id;
|
||||
return session.gop_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a file. Idempotent on unchanged content (same hash → skip). On
|
||||
* overwrite with new content, deletes the old frame and inserts a fresh one.
|
||||
* overwrite with new content, stores a fresh memory and forgets the old one.
|
||||
*/
|
||||
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
|
||||
async indexFile(filePath: string, content: Buffer, mime?: string): Promise<FileIndexResult> {
|
||||
if (!FileIndexer.shouldIndex(filePath)) {
|
||||
return { skipped: true, reason: 'unsupported_format' };
|
||||
}
|
||||
if (content.length === 0) {
|
||||
// Still remove any stale index so an emptied file doesn't leave an
|
||||
// orphan frame pointing at old content.
|
||||
this.removeFile(filePath);
|
||||
// orphan memory pointing at old content.
|
||||
await this.removeFile(filePath);
|
||||
return { skipped: true, reason: 'empty' };
|
||||
}
|
||||
|
||||
const hash = createHash('sha256').update(content).digest('hex');
|
||||
const raw = this.db.getDatabase();
|
||||
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const existing = this.getRow(filePath);
|
||||
|
||||
const normalized = this.normalizeText(content);
|
||||
const truncated = content.length > MAX_CONTENT_BYTES;
|
||||
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
|
||||
if (evaluateExternalMemoryIngress({ content: frameBody }).action === 'block') {
|
||||
// An unchanged row may pre-date this guard. Remove that already-indexed
|
||||
// unsafe projection atomically; on a new unsafe overwrite, retain the
|
||||
// prior benign index while the file write itself remains successful.
|
||||
if (existingRow && existingRow.content_hash === hash) {
|
||||
raw.transaction(() => this.removeFile(filePath))();
|
||||
}
|
||||
const text = this.buildFrameContent(filePath, normalized, truncated, mime);
|
||||
if (evaluateExternalMemoryIngress({ content: text }).action === 'block') {
|
||||
// An unchanged row may pre-date this guard: remove that already-indexed
|
||||
// unsafe projection. On a new unsafe overwrite, keep the prior benign
|
||||
// index while the file write itself remains successful.
|
||||
if (existing && existing.contentHash === hash) await this.removeFile(filePath);
|
||||
return { skipped: true, reason: 'unsafe_content' };
|
||||
}
|
||||
if (existingRow && existingRow.content_hash === hash) {
|
||||
if (existing && existing.contentHash === hash) {
|
||||
return { skipped: true, reason: 'unchanged' };
|
||||
}
|
||||
const gopId = this.ensureSession();
|
||||
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
|
||||
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
|
||||
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
|
||||
// and the index-row write, leaving the table in its pre-call state. Without
|
||||
// this, a crash between the delete and the UPDATE would leave file_index
|
||||
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
|
||||
// orphaned in the frames table.
|
||||
const mutate = raw.transaction(() => {
|
||||
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
|
||||
|
||||
if (existingRow) {
|
||||
// Overwrite path: delete the old frame before we swap the row, unless
|
||||
// the dedup coalesced two paths onto the same frame (leave that frame
|
||||
// alone in that case — the other path still references it).
|
||||
const oldFrameId = Number(existingRow.frame_id);
|
||||
if (oldFrameId !== frame.id) {
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(oldFrameId);
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE file_index SET
|
||||
frame_id = ?,
|
||||
mime_type = ?,
|
||||
size_bytes = ?,
|
||||
content_hash = ?,
|
||||
indexed_at = datetime('now')
|
||||
WHERE file_path = ?
|
||||
`).run(frame.id, mime ?? null, content.length, hash, filePath);
|
||||
} else {
|
||||
raw.prepare(`
|
||||
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(filePath, frame.id, mime ?? null, content.length, hash);
|
||||
}
|
||||
|
||||
return frame.id;
|
||||
const [change] = await this.memory.remember({
|
||||
scope: workspaceScope(this.workspaceId),
|
||||
text,
|
||||
kind: 'note',
|
||||
source: 'system',
|
||||
origin: 'file-index',
|
||||
metadata: { file_path: filePath },
|
||||
});
|
||||
if (!change) throw new Error(`memory store returned nothing for ${filePath}`);
|
||||
const memoryId = change.record.id;
|
||||
|
||||
const frameId = mutate();
|
||||
return { skipped: false, frameId, truncated };
|
||||
this.waggleDb.getDatabase().prepare(`
|
||||
INSERT INTO file_index (scope, file_path, memory_id, mime_type, size_bytes, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(scope, file_path) DO UPDATE SET
|
||||
memory_id = excluded.memory_id,
|
||||
mime_type = excluded.mime_type,
|
||||
size_bytes = excluded.size_bytes,
|
||||
content_hash = excluded.content_hash,
|
||||
indexed_at = datetime('now')
|
||||
`).run(this.scope, filePath, memoryId, mime ?? null, content.length, hash);
|
||||
|
||||
// Forget the previous memory unless another path still points at it (two
|
||||
// paths with identical content share one memory).
|
||||
if (existing && existing.memoryId !== memoryId) await this.forgetIfUnreferenced(existing.memoryId);
|
||||
return { skipped: false, memoryId, truncated };
|
||||
}
|
||||
|
||||
/** Remove a file's frame + index row. Returns true if anything was removed. */
|
||||
removeFile(filePath: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| { frame_id: number }
|
||||
| undefined;
|
||||
/** Remove a file's memory + index row. Returns true if anything was removed. */
|
||||
async removeFile(filePath: string): Promise<boolean> {
|
||||
const row = this.getRow(filePath);
|
||||
if (!row) return false;
|
||||
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(row.frame_id);
|
||||
}
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
|
||||
this.waggleDb.getDatabase()
|
||||
.prepare('DELETE FROM file_index WHERE scope = ? AND file_path = ?')
|
||||
.run(this.scope, filePath);
|
||||
await this.forgetIfUnreferenced(row.memoryId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the recorded path for a moved file. The frame content itself still
|
||||
* carries the old path in its header — that's acceptable for v1 since the
|
||||
* frame body is already the file's own content (not a description). A later
|
||||
* reader sees the stale header only if they dig into raw frame content; the
|
||||
* index table, which is what every other code path queries, is correct.
|
||||
* Update the recorded path for a moved file. The memory text still carries
|
||||
* the old path in its header; the index table, which every other code path
|
||||
* queries, is correct.
|
||||
*/
|
||||
moveFile(from: string, to: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return false;
|
||||
|
||||
if (!this.getRow(from)) return false;
|
||||
const db = this.waggleDb.getDatabase();
|
||||
// If `to` already indexed, drop its entry (file was replaced by the move).
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
|
||||
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
|
||||
db.prepare('DELETE FROM file_index WHERE scope = ? AND file_path = ?').run(this.scope, to);
|
||||
db.prepare("UPDATE file_index SET file_path = ?, indexed_at = datetime('now') WHERE scope = ? AND file_path = ?")
|
||||
.run(to, this.scope, from);
|
||||
return true;
|
||||
}
|
||||
|
||||
getRow(filePath: string): FileIndexRow | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
return {
|
||||
filePath: row.file_path as string,
|
||||
frameId: row.frame_id as number,
|
||||
mimeType: (row.mime_type as string | null) ?? null,
|
||||
sizeBytes: row.size_bytes as number,
|
||||
contentHash: row.content_hash as string,
|
||||
indexedAt: row.indexed_at as string,
|
||||
};
|
||||
const row = this.waggleDb.getDatabase()
|
||||
.prepare('SELECT * FROM file_index WHERE scope = ? AND file_path = ?')
|
||||
.get(this.scope, filePath) as Record<string, unknown> | undefined;
|
||||
return row ? toRow(row) : null;
|
||||
}
|
||||
|
||||
listAll(): FileIndexRow[] {
|
||||
const raw = this.db.getDatabase();
|
||||
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
|
||||
return rows.map(r => ({
|
||||
filePath: r.file_path as string,
|
||||
frameId: r.frame_id as number,
|
||||
mimeType: (r.mime_type as string | null) ?? null,
|
||||
sizeBytes: r.size_bytes as number,
|
||||
contentHash: r.content_hash as string,
|
||||
indexedAt: r.indexed_at as string,
|
||||
}));
|
||||
const rows = this.waggleDb.getDatabase()
|
||||
.prepare('SELECT * FROM file_index WHERE scope = ? ORDER BY indexed_at DESC')
|
||||
.all(this.scope) as Record<string, unknown>[];
|
||||
return rows.map(toRow);
|
||||
}
|
||||
|
||||
private async forgetIfUnreferenced(memoryId: string): Promise<void> {
|
||||
const otherRef = this.waggleDb.getDatabase()
|
||||
.prepare('SELECT 1 FROM file_index WHERE memory_id = ? LIMIT 1')
|
||||
.get(memoryId);
|
||||
if (!otherRef) await this.memory.forget(memoryId);
|
||||
}
|
||||
|
||||
private normalizeText(content: Buffer): string {
|
||||
@@ -266,3 +200,14 @@ export class FileIndexer {
|
||||
return `${header}\n\n${body}${suffix}`;
|
||||
}
|
||||
}
|
||||
|
||||
function toRow(row: Record<string, unknown>): FileIndexRow {
|
||||
return {
|
||||
filePath: row.file_path as string,
|
||||
memoryId: row.memory_id as string,
|
||||
mimeType: (row.mime_type as string | null) ?? null,
|
||||
sizeBytes: row.size_bytes as number,
|
||||
contentHash: row.content_hash as string,
|
||||
indexedAt: row.indexed_at as string,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -26,9 +26,12 @@ export {
|
||||
type ExternalMemoryProjectionInput,
|
||||
} from './memory-ingress-guard.js';
|
||||
export { WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions } from './workspace-manager.js';
|
||||
export type { Embedder } from './embeddings/embeddings.js';
|
||||
export { createLazyEmbedder, type Embedder } from './embeddings/embeddings.js';
|
||||
export { createLiteLLMEmbedder, type LiteLLMEmbedderConfig } from './embeddings/litellm-embedder.js';
|
||||
export { createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig } from './embeddings/inprocess-embedder.js';
|
||||
export {
|
||||
createInProcessEmbedder, normalizeDimensions, createMemoryEmbedder, MEMORY_EMBEDDING_DIMENSIONS,
|
||||
type InProcessEmbedderConfig,
|
||||
} from './embeddings/inprocess-embedder.js';
|
||||
export { createOllamaEmbedder, type OllamaEmbedderConfig } from './embeddings/ollama-embedder.js';
|
||||
export { createApiEmbedder, type ApiEmbedderConfig } from './embeddings/api-embedder.js';
|
||||
export {
|
||||
@@ -36,6 +39,15 @@ export {
|
||||
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
} from './embeddings/embedding-provider.js';
|
||||
// Recall helpers: dates in queries and recalled text, and the cross-encoder reranker.
|
||||
export {
|
||||
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
|
||||
} from './recall/recall-context.js';
|
||||
export { parseDateWindow, type DateWindow } from './recall/parse-date-window.js';
|
||||
export { resolveRelativeDate, type ResolvedDate } from './recall/resolve-relative-date.js';
|
||||
export {
|
||||
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
|
||||
} from './recall/inprocess-reranker.js';
|
||||
|
||||
// ── waggle.db — Waggle's own data, apart from memory (ADR-0007) ──
|
||||
export {
|
||||
@@ -86,11 +98,6 @@ export {
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
type ConceptEntry, type ConceptUpdate,
|
||||
// recall-context — temporal rendering helpers (W4.1 production port)
|
||||
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
|
||||
parseDateWindow, type DateWindow,
|
||||
resolveRelativeDate, type ResolvedDate,
|
||||
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
|
||||
// harvest/ — universal memory ingestion pipeline
|
||||
HarvestSourceStore,
|
||||
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
* failed) so there is a verifiable history of what was installed, when, why,
|
||||
* and by whom.
|
||||
*
|
||||
* Operates on the .mind DB like the other stores in this package.
|
||||
* Lives in waggle.db with Waggle's other own data (ADR-0007).
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type { WaggleDB } from './waggle-db.js';
|
||||
import {
|
||||
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
|
||||
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
|
||||
@@ -63,10 +63,7 @@ export interface RecordAuditInput {
|
||||
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
|
||||
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
|
||||
// drift (divergence #14 — the old comment admitted "drift silently crashes
|
||||
// record()"). The matching DDL in hive-mind-core/src/mind/schema.ts stays a
|
||||
// standalone private-monorepo literal and is locked to these same canonical
|
||||
// lists by install-audit-check-parity.test.ts. It is stripped from the curated
|
||||
// OSS export.
|
||||
// record()").
|
||||
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
|
||||
// while the TS type claimed a closed set). Every historical value came from the
|
||||
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
|
||||
@@ -93,9 +90,9 @@ CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC)
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class InstallAuditStore {
|
||||
private db: MindDB;
|
||||
private db: WaggleDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
constructor(db: WaggleDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
@@ -127,13 +124,13 @@ export class InstallAuditStore {
|
||||
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
|
||||
* crash mid-rebuild leaves the original table intact.
|
||||
*/
|
||||
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
|
||||
private rebuildForWidenedActionCheck(raw: ReturnType<WaggleDB['getDatabase']>): void {
|
||||
const migrate = raw.transaction(() => {
|
||||
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
|
||||
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
|
||||
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
|
||||
// DROP below would take the indexes with the legacy table. Drop them first
|
||||
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
|
||||
// so they get recreated on the new table.
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
|
||||
raw.exec(INSTALL_AUDIT_TABLE_SQL);
|
||||
|
||||
137
packages/core/src/recall/inprocess-reranker.ts
Normal file
137
packages/core/src/recall/inprocess-reranker.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* In-process cross-encoder reranker using @huggingface/transformers (ONNX).
|
||||
*
|
||||
* Cross-encoders take (query, doc) pairs and output a relevance score by
|
||||
* jointly attending to both — much more discriminating than vector dot
|
||||
* products. Use after RRF to rerank the top-K candidates from hybrid
|
||||
* search. Standard pattern in production RAG systems.
|
||||
*
|
||||
* Default model: Xenova/ms-marco-MiniLM-L-6-v2 — the canonical
|
||||
* SentenceTransformers cross-encoder, ~22MB on disk, ~30-50ms per pair
|
||||
* on CPU. Trained on MS MARCO passage ranking, generalizes well to
|
||||
* mixed-domain technical text.
|
||||
*
|
||||
* Alternative: Xenova/bge-reranker-base (~280MB, slightly higher quality
|
||||
* on out-of-domain queries). Set via `model` config.
|
||||
*
|
||||
* `@huggingface/transformers` is an optional peer dep — if not installed,
|
||||
* createInProcessReranker throws and the caller falls back to no reranking.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from '../embeddings/transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-reranker');
|
||||
|
||||
export interface Reranker {
|
||||
/**
|
||||
* Score a single (query, doc) pair. Higher = more relevant.
|
||||
* Score scale depends on the model — for ms-marco-MiniLM it's
|
||||
* roughly [-10, 10]; relative ordering is what matters.
|
||||
*/
|
||||
score(query: string, doc: string): Promise<number>;
|
||||
|
||||
/**
|
||||
* Score N pairs sharing one query. Same-shape result as score() but
|
||||
* amortises the model invocation when supported.
|
||||
*/
|
||||
scoreBatch(query: string, docs: string[]): Promise<number[]>;
|
||||
}
|
||||
|
||||
export interface InProcessRerankerConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reranker backed by @huggingface/transformers. Throws if the
|
||||
* package isn't installed — caller is expected to catch and fall back.
|
||||
*/
|
||||
export async function createInProcessReranker(
|
||||
config?: Partial<InProcessRerankerConfig>,
|
||||
): Promise<Reranker> {
|
||||
const model = config?.model ?? 'Xenova/ms-marco-MiniLM-L-6-v2';
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.hive-mind', 'models');
|
||||
|
||||
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
|
||||
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
|
||||
'@huggingface/transformers'
|
||||
);
|
||||
const { tokenizer, seqModel } = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: async (canonicalCacheDir) => ({
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
|
||||
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
|
||||
});
|
||||
|
||||
log.info(`In-process reranker ready: ${model}`);
|
||||
|
||||
/** Score a single pair: tokenize, forward, extract logit. */
|
||||
async function scorePair(query: string, doc: string): Promise<number> {
|
||||
const inputs = await tokenizer(query, {
|
||||
text_pair: doc,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
|
||||
// Other cross-encoders may output 2 classes — take logit[0] - logit[1]
|
||||
// as a relevance score in that case.
|
||||
const logits = out.logits ?? out[0];
|
||||
const data = logits.data as Float32Array | number[];
|
||||
if (logits.dims && logits.dims[logits.dims.length - 1] === 2) {
|
||||
return Number(data[0]) - Number(data[1]);
|
||||
}
|
||||
return Number(data[0]);
|
||||
}
|
||||
|
||||
return {
|
||||
async score(query: string, doc: string): Promise<number> {
|
||||
return scorePair(query, doc);
|
||||
},
|
||||
|
||||
async scoreBatch(query: string, docs: string[]): Promise<number[]> {
|
||||
if (docs.length === 0) return [];
|
||||
// Tokenize all pairs together for batch inference. Padding aligns
|
||||
// sequences so the model can process them in one forward pass.
|
||||
const queries = docs.map(() => query);
|
||||
const inputs = await tokenizer(queries, {
|
||||
text_pair: docs,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
const logits = out.logits ?? out[0];
|
||||
const data = logits.data as Float32Array | number[];
|
||||
const dims = logits.dims;
|
||||
const lastDim = dims[dims.length - 1];
|
||||
|
||||
const scores: number[] = [];
|
||||
if (lastDim === 2) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
scores.push(Number(data[i * 2]) - Number(data[i * 2 + 1]));
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
scores.push(Number(data[i]));
|
||||
}
|
||||
}
|
||||
return scores;
|
||||
},
|
||||
};
|
||||
}
|
||||
113
packages/core/src/recall/parse-date-window.ts
Normal file
113
packages/core/src/recall/parse-date-window.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* parse-date-window.ts — deterministic query-side temporal-constraint parser
|
||||
* (W4.1b, production port of the benchmark-proven Wave-3.1 date-window lane;
|
||||
* W4-PRODUCTION-PORT-PLAN-2026-06-11.md component #3, MRAG arXiv:2412.15540
|
||||
* pattern).
|
||||
*
|
||||
* When a query names an explicit period — "the last week of October 2023",
|
||||
* "early June 2023", "on 9 August 2023", "May 2023", "in 2022" — return a
|
||||
* `[since..until]` window (date-only `YYYY-MM-DD` bounds, inclusive) plus a
|
||||
* human label. Callers pass the window to HybridSearch's since/until filter;
|
||||
* the label feeds the future Events-during-X render section (component #6).
|
||||
*
|
||||
* Deterministic regex date math only — no LLM call, no new index. Returns
|
||||
* null when the query carries no explicit period (relative phrases like
|
||||
* "last week" / "two months ago" are resolution work for the WRITE side —
|
||||
* resolve-relative-date.ts — not query windowing).
|
||||
*
|
||||
* Regex shapes are byte-equivalent to the benchmark parser validated on the
|
||||
* full N=1540 LoCoMo run (wave3a); only types/docs differ.
|
||||
*
|
||||
* OSS-clean: pure date arithmetic, no vault/evolution/compliance deps.
|
||||
*/
|
||||
|
||||
/** Inclusive date-only window parsed from an explicit period in a query. */
|
||||
export interface DateWindow {
|
||||
/** Inclusive lower bound, `YYYY-MM-DD`. */
|
||||
since: string;
|
||||
/** Inclusive upper bound, `YYYY-MM-DD`. */
|
||||
until: string;
|
||||
/** Human-readable label of the matched period (for render sections). */
|
||||
label: string;
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
|
||||
july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
|
||||
};
|
||||
const MONTH_RE = '(january|february|march|april|may|june|july|august|september|october|november|december)';
|
||||
|
||||
function lastDayOfMonth(y: number, m: number): number {
|
||||
return new Date(Date.UTC(y, m, 0)).getUTCDate();
|
||||
}
|
||||
|
||||
function isoOf(y: number, m: number, d: number): string {
|
||||
return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an explicit-period temporal constraint out of a query. Returns null
|
||||
* when no explicit period is named.
|
||||
*/
|
||||
export function parseDateWindow(query: string): DateWindow | null {
|
||||
const t = String(query).toLowerCase();
|
||||
|
||||
// "first|second|third|fourth|last week of <month> <year>"
|
||||
let m = t.match(new RegExp(`(first|second|third|fourth|last)\\s+week\\s+of\\s+${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
|
||||
const last = lastDayOfMonth(y, mo);
|
||||
const ranges: Record<string, [number, number]> = {
|
||||
first: [1, 7], second: [8, 14], third: [15, 21], fourth: [22, 28],
|
||||
last: [Math.max(1, last - 6), last],
|
||||
};
|
||||
const [d1, d2] = ranges[m[1]];
|
||||
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, Math.min(d2, last)), label: `the ${m[1]} week of ${m[2]} ${y}` };
|
||||
}
|
||||
|
||||
// "early|mid|late <month> <year>"
|
||||
m = t.match(new RegExp(`(early|mid|late)\\s+${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
|
||||
const last = lastDayOfMonth(y, mo);
|
||||
const ranges: Record<string, [number, number]> = { early: [1, 10], mid: [11, 20], late: [21, last] };
|
||||
const [d1, d2] = ranges[m[1]];
|
||||
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, d2), label: `${m[1]} ${m[2]} ${y}` };
|
||||
}
|
||||
|
||||
// "<day> <month> <year>" or "<month> <day>, <year>" → exact-day ±2 buffer
|
||||
m = t.match(new RegExp(`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(?:of\\s+)?${MONTH_RE},?\\s+(\\d{4})`)) ||
|
||||
t.match(new RegExp(`${MONTH_RE}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const isDayFirst = /^\d/.test(m[1]);
|
||||
const day = parseInt(isDayFirst ? m[1] : m[2], 10);
|
||||
const mo = MONTHS[isDayFirst ? m[2] : m[1]];
|
||||
const y = parseInt(m[3], 10);
|
||||
if (mo && day >= 1 && day <= 31) {
|
||||
const center = Date.UTC(y, mo - 1, day);
|
||||
const lo = new Date(center - 2 * 86400000), hi = new Date(center + 2 * 86400000);
|
||||
return {
|
||||
since: lo.toISOString().slice(0, 10),
|
||||
until: hi.toISOString().slice(0, 10),
|
||||
label: `${day} ${isDayFirst ? m[2] : m[1]} ${y}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// "<month> <year>" → whole month
|
||||
m = t.match(new RegExp(`${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[2], 10), mo = MONTHS[m[1]];
|
||||
return { since: isoOf(y, mo, 1), until: isoOf(y, mo, lastDayOfMonth(y, mo)), label: `${m[1]} ${y}` };
|
||||
}
|
||||
|
||||
// bare "in|during <year>" → whole year ('in'/'during' required so years
|
||||
// inside names/ids don't window the query)
|
||||
m = t.match(/\b(?:in|during)\s+(20\d{2})\b/);
|
||||
if (m) {
|
||||
const y = m[1];
|
||||
return { since: `${y}-01-01`, until: `${y}-12-31`, label: y };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
100
packages/core/src/recall/recall-context.ts
Normal file
100
packages/core/src/recall/recall-context.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* recall-context.ts — shared renderer for surfacing temporal information in
|
||||
* recalled-memory blocks. Single source of truth imported by both the Waggle
|
||||
* agent (production) and the benchmark harness, so the injected-memory format
|
||||
* never drifts between the two.
|
||||
*
|
||||
* Scope (Temporal Substrate Fix, Phase 1 — "surface time", additive only):
|
||||
* - Prefix each retrieved snippet with its own compact `[YYYY-MM-DD]` date.
|
||||
* - Open the rendered memory block with one anchor line giving the most-recent
|
||||
* memory date, so the model has a concrete "now" to resolve relative time
|
||||
* expressions against.
|
||||
* - Export `TEMPORAL_GUIDANCE`, the prompt fragment that tells the model to
|
||||
* treat those timestamps as the anchor for relative-time arithmetic.
|
||||
*
|
||||
* Distilled "Memory Facts" (cross-session syntheses) are intentionally NOT
|
||||
* dated here — they have no single reliable date. Phase 2 handles them.
|
||||
*
|
||||
* OSS-clean: pure string formatting, no vault/evolution/compliance deps.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Prompt fragment wired into the memory-recall injection path (NOT the global
|
||||
* system prompt). Teaches the model to use the surfaced `[YYYY-MM-DD]` stamps
|
||||
* as the anchor for resolving relative time expressions and conflicting facts.
|
||||
*
|
||||
* W4.1 upgrade (W4-PRODUCTION-PORT-PLAN-2026-06-11.md §1): replaced the original
|
||||
* "nearest timestamp as anchor" phrasing with the benchmark-proven W1 wording —
|
||||
* concrete relative-date arithmetic with worked examples (Memori instruction-5
|
||||
* lineage, incl. the verified conv-26 "yesterday" failure case) plus the
|
||||
* granularity-calibration clause (failure mining: 22 temporal fails emitted a
|
||||
* confident exact ISO day 1-7 days off where a coarse answer was correct).
|
||||
* Temporal was the #1 LoCoMo lever (80.06 → 84.7 across W1-W3.1).
|
||||
* Production-safe subset: no never-refuse clause (that was benchmark-cell
|
||||
* policy only — conditional abstention stays).
|
||||
*/
|
||||
export const TEMPORAL_GUIDANCE =
|
||||
"Memories and snippets are timestamped [YYYY-MM-DD]. Pay special attention to these " +
|
||||
"timestamps to determine timing. If a question involves relative time references " +
|
||||
"('last year', 'two months ago', 'yesterday', 'last week'), CALCULATE the actual date " +
|
||||
"from the timestamp of the memory that mentions it. For example: a memory dated " +
|
||||
"4 May 2022 that says 'went to India last year' means the trip was in 2021; a memory " +
|
||||
"dated 8 May 2023 that says 'I went to the group yesterday' means the event was 7 May 2023. " +
|
||||
"Always convert relative references to specific dates, months, or years using the " +
|
||||
"memory's timestamp as the anchor, and ignore the relative phrase itself when answering. " +
|
||||
"When the same fact appears at different times, the most recent version is correct. " +
|
||||
"GRANULARITY: state an exact day ONLY when that exact date was explicitly stated or " +
|
||||
"directly computed from an explicit relative reference; otherwise answer at the " +
|
||||
"granularity you are confident in — 'early June 2023', 'the week before 9 August 2023', " +
|
||||
"'August 2022'. A confidently wrong exact day is worse than a correct coarse answer. " +
|
||||
"For 'how long / how many months' duration questions, give ONLY the final value " +
|
||||
"(e.g. 'six months') — no intermediate dates, no reasoning steps.";
|
||||
|
||||
/** Anchor-line prefix for the most-recent rendered memory date. */
|
||||
const REFERENCE_DATE_LABEL = 'Reference date (most recent memory):';
|
||||
|
||||
/**
|
||||
* Slice an ISO-ish timestamp to its `YYYY-MM-DD` date prefix. Returns null when
|
||||
* the value is missing or too short to carry a date.
|
||||
*/
|
||||
export function toDatePrefix(createdAt: string | null | undefined): string | null {
|
||||
if (!createdAt || createdAt.length < 10) return null;
|
||||
return createdAt.slice(0, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a single retrieved snippet with its compact `[YYYY-MM-DD]` date prefix.
|
||||
* `text` is the snippet text exactly as it would otherwise be rendered (the
|
||||
* caller owns importance labels, truncation, etc.); this only prepends the date.
|
||||
* Falls back to the bare text when the hit carries no usable timestamp.
|
||||
*/
|
||||
export function renderDatedSnippet(createdAt: string | null | undefined, text: string): string {
|
||||
const date = toDatePrefix(createdAt);
|
||||
return date ? `[${date}] ${text}` : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the reference (anchor) date: the maximum `created_at` among the
|
||||
* supplied timestamps, sliced to `YYYY-MM-DD`. Returns null when none carry a
|
||||
* usable date (caller then omits the anchor line).
|
||||
*/
|
||||
export function referenceDate(createdAts: ReadonlyArray<string | null | undefined>): string | null {
|
||||
let max: string | null = null;
|
||||
for (const c of createdAts) {
|
||||
const date = toDatePrefix(c);
|
||||
if (date && (max === null || date > max)) max = date;
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the single anchor line for a memory block, e.g.
|
||||
* `Reference date (most recent memory): 2026-06-09`. Returns null when no date
|
||||
* is available.
|
||||
*/
|
||||
export function renderReferenceDateLine(
|
||||
createdAts: ReadonlyArray<string | null | undefined>,
|
||||
): string | null {
|
||||
const date = referenceDate(createdAts);
|
||||
return date ? `${REFERENCE_DATE_LABEL} ${date}` : null;
|
||||
}
|
||||
131
packages/core/src/recall/resolve-relative-date.ts
Normal file
131
packages/core/src/recall/resolve-relative-date.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* resolve-relative-date — write-time relative-date resolution for the memory substrate.
|
||||
*
|
||||
* WHY THIS EXISTS (benchmark-validated, LoCoMo head-to-head vs Memori, 2026-06-10):
|
||||
* A conversation utterance's *discussion date* (when it was said) is not the same as
|
||||
* the *event date* (when the thing happened). "I went to the group yesterday" said on
|
||||
* 2023-05-08 describes an event on 2023-05-07. Storing the discussion date makes
|
||||
* "when did X happen?" questions wrong by the relative-reference delta — the verified
|
||||
* root cause of our temporal-category gap. Resolving the relative reference against the
|
||||
* source date at WRITE time bakes the correct event date into the frame's timestamp,
|
||||
* which lifted the LoCoMo temporal category to parity with Memori (80.06 vs 80.37).
|
||||
* See benchmarks/results/memori-phase22-RESULT.md (Phase 4).
|
||||
*
|
||||
* This is the deterministic, dependency-free production counterpart of the benchmark's
|
||||
* LLM extraction pass (hive-mind-test scripts/locomo/33-distill-episodic.mjs). It covers
|
||||
* the dominant cue patterns the calibration surfaced ("yesterday", "last week", "N days
|
||||
* ago", "last <weekday>", "last year") without a per-frame LLM call. Pure + side-effect
|
||||
* free; the caller decides whether to use the resolved date.
|
||||
*/
|
||||
|
||||
/** A resolved relative-date hit: the matched cue phrase + the absolute ISO date (YYYY-MM-DD). */
|
||||
export interface ResolvedDate {
|
||||
/** The relative cue that matched, e.g. "yesterday", "last week", "3 months ago". */
|
||||
cue: string;
|
||||
/** The resolved absolute date as YYYY-MM-DD. */
|
||||
iso: string;
|
||||
}
|
||||
|
||||
const ISO_DATE = /^\d{4}-\d{2}-\d{2}/;
|
||||
const WEEKDAYS = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] as const;
|
||||
|
||||
/** Parse an ISO-ish reference date (date or datetime) into a UTC Date at midnight. Null if unparseable. */
|
||||
function parseReference(referenceDate: string | null | undefined): Date | null {
|
||||
if (!referenceDate || !ISO_DATE.test(referenceDate)) return null;
|
||||
const [y, m, d] = referenceDate.slice(0, 10).split('-').map(Number);
|
||||
if (!y || !m || !d) return null;
|
||||
const dt = new Date(Date.UTC(y, m - 1, d));
|
||||
return Number.isNaN(dt.getTime()) ? null : dt;
|
||||
}
|
||||
|
||||
/** Format a UTC Date as YYYY-MM-DD. */
|
||||
function toIso(dt: Date): string {
|
||||
return dt.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function addDays(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
out.setUTCDate(out.getUTCDate() + n);
|
||||
return out;
|
||||
}
|
||||
|
||||
function addMonths(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
const targetMonthDay = out.getUTCDate();
|
||||
out.setUTCDate(1);
|
||||
out.setUTCMonth(out.getUTCMonth() + n);
|
||||
// Clamp to the last valid day of the resulting month (e.g. Jan 31 − 1mo → Dec 31, not overflow).
|
||||
const lastDay = new Date(Date.UTC(out.getUTCFullYear(), out.getUTCMonth() + 1, 0)).getUTCDate();
|
||||
out.setUTCDate(Math.min(targetMonthDay, lastDay));
|
||||
return out;
|
||||
}
|
||||
|
||||
function addYears(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
out.setUTCFullYear(out.getUTCFullYear() + n);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Most recent occurrence of `weekday` strictly before the reference date (the "last Friday" sense). */
|
||||
function lastWeekday(dt: Date, weekday: number): Date {
|
||||
let delta = dt.getUTCDay() - weekday;
|
||||
if (delta <= 0) delta += 7; // strictly before → at least 1 day back
|
||||
return addDays(dt, -delta);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the first relative-time cue in `text` against `referenceDate` (the source/
|
||||
* conversation date) into an absolute YYYY-MM-DD. Returns null when there is no
|
||||
* recognised cue or the reference date is unusable — the caller then keeps the
|
||||
* reference date as-is.
|
||||
*
|
||||
* Patterns are tried most-specific first so "the day before yesterday" wins over
|
||||
* "yesterday", and explicit "N units ago" wins over the bare "last unit".
|
||||
*/
|
||||
export function resolveRelativeDate(
|
||||
text: string,
|
||||
referenceDate: string | null | undefined,
|
||||
): ResolvedDate | null {
|
||||
const ref = parseReference(referenceDate);
|
||||
if (!ref || !text) return null;
|
||||
const t = text.toLowerCase();
|
||||
|
||||
// Most-specific day offsets first.
|
||||
if (/\bday before yesterday\b/.test(t)) return { cue: 'the day before yesterday', iso: toIso(addDays(ref, -2)) };
|
||||
if (/\byesterday\b/.test(t)) return { cue: 'yesterday', iso: toIso(addDays(ref, -1)) };
|
||||
|
||||
// "N days/weeks/months/years ago" (explicit count).
|
||||
const ago = t.match(/\b(\d{1,3})\s+(day|week|month|year)s?\s+ago\b/);
|
||||
if (ago) {
|
||||
const n = parseInt(ago[1], 10);
|
||||
const unit = ago[2];
|
||||
const iso =
|
||||
unit === 'day' ? toIso(addDays(ref, -n)) :
|
||||
unit === 'week' ? toIso(addDays(ref, -7 * n)) :
|
||||
unit === 'month' ? toIso(addMonths(ref, -n)) :
|
||||
toIso(addYears(ref, -n));
|
||||
return { cue: `${n} ${unit}${n === 1 ? '' : 's'} ago`, iso };
|
||||
}
|
||||
|
||||
// "a week/month/year ago" (singular, count = 1).
|
||||
const aAgo = t.match(/\ba\s+(week|month|year)\s+ago\b/);
|
||||
if (aAgo) {
|
||||
const unit = aAgo[1];
|
||||
const iso = unit === 'week' ? toIso(addDays(ref, -7)) : unit === 'month' ? toIso(addMonths(ref, -1)) : toIso(addYears(ref, -1));
|
||||
return { cue: `a ${unit} ago`, iso };
|
||||
}
|
||||
|
||||
// "last <weekday>" → most recent prior occurrence of that weekday.
|
||||
const lastDow = t.match(/\blast\s+(sunday|monday|tuesday|wednesday|thursday|friday|saturday)\b/);
|
||||
if (lastDow) {
|
||||
const wd = WEEKDAYS.indexOf(lastDow[1] as (typeof WEEKDAYS)[number]);
|
||||
return { cue: `last ${lastDow[1]}`, iso: toIso(lastWeekday(ref, wd)) };
|
||||
}
|
||||
|
||||
// "last week/month/year" (bare, coarse offset).
|
||||
if (/\blast\s+week\b/.test(t)) return { cue: 'last week', iso: toIso(addDays(ref, -7)) };
|
||||
if (/\blast\s+month\b/.test(t)) return { cue: 'last month', iso: toIso(addMonths(ref, -1)) };
|
||||
if (/\blast\s+year\b/.test(t)) return { cue: 'last year', iso: toIso(addYears(ref, -1)) };
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { AwarenessLayer } from './awareness.js';
|
||||
import { CronStore } from './cron-store.js';
|
||||
import { ExecutionTraceStore } from './execution-traces.js';
|
||||
import { IdentityLayer } from './identity.js';
|
||||
import { InstallAuditStore } from './install-audit.js';
|
||||
import { SkillHashStore } from './skill-hashes.js';
|
||||
import type { DataScope, WaggleDB } from './waggle-db.js';
|
||||
|
||||
@@ -30,6 +31,7 @@ const PERSONAL_TABLES = [
|
||||
'execution_traces',
|
||||
'execution_trace_spend',
|
||||
'execution_trace_spend_reservations',
|
||||
'install_audit',
|
||||
] as const;
|
||||
|
||||
const PROFILE_COLUMNS = ['name', 'role', 'department', 'personality', 'capabilities', 'system_prompt', 'created_at', 'updated_at'];
|
||||
@@ -125,6 +127,7 @@ export function importLegacyMindData(waggleDb: WaggleDB, sources: LegacyMindSour
|
||||
new CronStore(waggleDb);
|
||||
new SkillHashStore(waggleDb);
|
||||
new ExecutionTraceStore(waggleDb);
|
||||
new InstallAuditStore(waggleDb);
|
||||
new IdentityLayer(waggleDb);
|
||||
new AwarenessLayer(waggleDb);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user