This commit is contained in:
73
packages/hive-mind-core/src/mind/api-embedder.ts
Normal file
73
packages/hive-mind-core/src/mind/api-embedder.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* API-backed embedder — calls Voyage AI or OpenAI embedding endpoints.
|
||||
* For users who configure API keys in the Vault UI.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface ApiEmbedderConfig {
|
||||
provider: 'voyage' | 'openai';
|
||||
apiKey: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
const PROVIDER_DEFAULTS: Record<'voyage' | 'openai', { url: string; model: string }> = {
|
||||
voyage: { url: 'https://api.voyageai.com/v1/embeddings', model: 'voyage-3-lite' },
|
||||
openai: { url: 'https://api.openai.com/v1/embeddings', model: 'text-embedding-3-small' },
|
||||
};
|
||||
|
||||
export function createApiEmbedder(config: ApiEmbedderConfig): Embedder {
|
||||
const defaults = PROVIDER_DEFAULTS[config.provider];
|
||||
const url = config.baseUrl ?? defaults.url;
|
||||
const model = config.model ?? defaults.model;
|
||||
const targetDims = config.targetDimensions ?? 1024;
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 15000);
|
||||
|
||||
const body: Record<string, unknown> = { model, input };
|
||||
if (config.provider === 'voyage') {
|
||||
body.input_type = 'document';
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${config.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`${config.provider} embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { data: Array<{ embedding: number[] }> };
|
||||
return json.data.map(d => normalizeDimensions(new Float32Array(d.embedding), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
170
packages/hive-mind-core/src/mind/awareness.ts
Normal file
170
packages/hive-mind-core/src/mind/awareness.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
export type AwarenessCategory = 'task' | 'action' | 'pending' | 'flag';
|
||||
|
||||
export interface AwarenessMetadata {
|
||||
context?: string;
|
||||
status?: string;
|
||||
result?: string;
|
||||
priority?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface AwarenessItem {
|
||||
id: number;
|
||||
category: AwarenessCategory;
|
||||
content: string;
|
||||
priority: number;
|
||||
metadata: string;
|
||||
created_at: string;
|
||||
expires_at: string | null;
|
||||
}
|
||||
|
||||
type AwarenessUpdate = Partial<Pick<AwarenessItem, 'content' | 'priority' | 'expires_at'>>;
|
||||
|
||||
const MAX_ITEMS = 10;
|
||||
|
||||
export class AwarenessLayer {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureMetadataColumn();
|
||||
}
|
||||
|
||||
/** Ensure metadata column exists for databases created before this feature */
|
||||
private ensureMetadataColumn(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const columns = raw.prepare("PRAGMA table_info(awareness)").all() as Array<{ name: string }>;
|
||||
const hasMetadata = columns.some(c => c.name === 'metadata');
|
||||
if (!hasMetadata) {
|
||||
raw.exec("ALTER TABLE awareness ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'");
|
||||
}
|
||||
} catch {
|
||||
// Database may already be closed during async teardown — safe to skip migration
|
||||
}
|
||||
}
|
||||
|
||||
add(category: AwarenessCategory, content: string, priority = 0, expires_at?: string, metadata?: AwarenessMetadata): AwarenessItem {
|
||||
const raw = this.db.getDatabase();
|
||||
const metadataJson = metadata ? JSON.stringify(metadata) : '{}';
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO awareness (category, content, priority, expires_at, metadata)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(category, content, priority, expires_at ?? null, metadataJson);
|
||||
return raw.prepare('SELECT * FROM awareness WHERE id = ?').get(result.lastInsertRowid) as AwarenessItem;
|
||||
}
|
||||
|
||||
get(id: number): AwarenessItem | undefined {
|
||||
return this.db.getDatabase().prepare('SELECT * FROM awareness WHERE id = ?').get(id) as AwarenessItem | undefined;
|
||||
}
|
||||
|
||||
remove(id: number): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
update(id: number, changes: AwarenessUpdate): AwarenessItem {
|
||||
const fields = Object.entries(changes).filter(([, v]) => v !== undefined);
|
||||
if (fields.length === 0) {
|
||||
return this.db.getDatabase().prepare('SELECT * FROM awareness WHERE id = ?').get(id) as AwarenessItem;
|
||||
}
|
||||
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
|
||||
const values = fields.map(([, v]) => v);
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`UPDATE awareness SET ${sets} WHERE id = ?`).run(...values, id);
|
||||
return raw.prepare('SELECT * FROM awareness WHERE id = ?').get(id) as AwarenessItem;
|
||||
}
|
||||
|
||||
updateMetadata(id: number, metadata: AwarenessMetadata): AwarenessItem {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare('SELECT metadata FROM awareness WHERE id = ?').get(id) as { metadata: string } | undefined;
|
||||
if (!existing) {
|
||||
throw new Error(`Awareness item ${id} not found`);
|
||||
}
|
||||
const current: AwarenessMetadata = JSON.parse(existing.metadata);
|
||||
const merged = { ...current, ...metadata };
|
||||
raw.prepare('UPDATE awareness SET metadata = ? WHERE id = ?').run(JSON.stringify(merged), id);
|
||||
return raw.prepare('SELECT * FROM awareness WHERE id = ?').get(id) as AwarenessItem;
|
||||
}
|
||||
|
||||
getByStatus(status: string): AwarenessItem[] {
|
||||
const raw = this.db.getDatabase();
|
||||
const items = raw.prepare(`
|
||||
SELECT * FROM awareness
|
||||
WHERE (expires_at IS NULL OR datetime(expires_at) > datetime('now'))
|
||||
ORDER BY priority DESC
|
||||
`).all() as AwarenessItem[];
|
||||
return items.filter(item => {
|
||||
try {
|
||||
const meta: AwarenessMetadata = JSON.parse(item.metadata);
|
||||
return meta.status === status;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
parseMetadata(item: AwarenessItem): AwarenessMetadata {
|
||||
try {
|
||||
return JSON.parse(item.metadata) as AwarenessMetadata;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
getAll(): AwarenessItem[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM awareness
|
||||
WHERE expires_at IS NULL OR datetime(expires_at) > datetime('now')
|
||||
ORDER BY priority DESC
|
||||
LIMIT ?
|
||||
`).all(MAX_ITEMS) as AwarenessItem[];
|
||||
}
|
||||
|
||||
getByCategory(category: AwarenessCategory): AwarenessItem[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM awareness
|
||||
WHERE category = ? AND (expires_at IS NULL OR datetime(expires_at) > datetime('now'))
|
||||
ORDER BY priority DESC
|
||||
LIMIT ?
|
||||
`).all(category, MAX_ITEMS) as AwarenessItem[];
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness').run();
|
||||
}
|
||||
|
||||
clearCategory(category: AwarenessCategory): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM awareness WHERE category = ?').run(category);
|
||||
}
|
||||
|
||||
toContext(): string {
|
||||
const items = this.getAll();
|
||||
if (items.length === 0) return 'No active awareness items.';
|
||||
|
||||
const grouped = new Map<string, AwarenessItem[]>();
|
||||
for (const item of items) {
|
||||
const list = grouped.get(item.category) ?? [];
|
||||
list.push(item);
|
||||
grouped.set(item.category, list);
|
||||
}
|
||||
|
||||
const sections: string[] = [];
|
||||
const labels: Record<AwarenessCategory, string> = {
|
||||
task: 'Active Tasks',
|
||||
action: 'Recent Actions',
|
||||
pending: 'Pending Items',
|
||||
flag: 'Context Flags',
|
||||
};
|
||||
|
||||
for (const [cat, label] of Object.entries(labels)) {
|
||||
const catItems = grouped.get(cat);
|
||||
if (catItems && catItems.length > 0) {
|
||||
sections.push(`${label}:\n${catItems.map(i => `- ${i.content}`).join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
return sections.join('\n\n');
|
||||
}
|
||||
}
|
||||
194
packages/hive-mind-core/src/mind/chunker.ts
Normal file
194
packages/hive-mind-core/src/mind/chunker.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11).
|
||||
/**
|
||||
* Semantic chunker for memory frames.
|
||||
*
|
||||
* Splits a frame's text content into coherent chunks suitable for embedding,
|
||||
* then mapping back to the parent frame at recall time.
|
||||
*
|
||||
* Strategy (paragraph-first, sentence-fallback):
|
||||
* 1. Split on blank lines to get paragraphs.
|
||||
* 2. Greedily aggregate paragraphs up to maxChars.
|
||||
* 3. If a single paragraph exceeds maxChars, sub-split it on sentence
|
||||
* boundaries (., !, ?, newline within the paragraph), aggregate those
|
||||
* up to maxChars.
|
||||
* 4. Optionally prepend an overlap window from the previous chunk's tail
|
||||
* so that a sentence straddling a chunk boundary still appears in both.
|
||||
*
|
||||
* Why these knobs:
|
||||
* - maxChars=2000 ≈ 500 tokens for dense English. Fits comfortably in
|
||||
* nomic-embed-text's 2048-token default context with headroom for the
|
||||
* model's special tokens.
|
||||
* - overlapChars=200 ≈ 50 tokens. Cheap recall safety net for queries
|
||||
* whose answer phrase straddles a chunk boundary; doubles as redundancy
|
||||
* for embedder noise.
|
||||
*
|
||||
* Char positions returned are RELATIVE to the input text (start inclusive,
|
||||
* end exclusive — matching JS slice semantics). Useful for highlighting the
|
||||
* chunk inside its parent frame at recall time.
|
||||
*
|
||||
* Pure function: no I/O, no side effects, no embedder dependency.
|
||||
*/
|
||||
|
||||
export interface ChunkOptions {
|
||||
maxChars?: number;
|
||||
overlapChars?: number;
|
||||
/**
|
||||
* Below this length, the input is returned as a single chunk regardless
|
||||
* of internal structure. Avoids fragmenting short frames into noisy
|
||||
* 1-sentence chunks that hurt search quality.
|
||||
*/
|
||||
minChunkChars?: number;
|
||||
}
|
||||
|
||||
export interface FrameChunk {
|
||||
text: string;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}
|
||||
|
||||
const DEFAULTS = {
|
||||
maxChars: 2000,
|
||||
overlapChars: 200,
|
||||
minChunkChars: 1500,
|
||||
};
|
||||
|
||||
/**
|
||||
* Split text into chunks. Returns at least one chunk for non-empty input.
|
||||
*/
|
||||
export function chunkText(text: string, opts: ChunkOptions = {}): FrameChunk[] {
|
||||
const maxChars = opts.maxChars ?? DEFAULTS.maxChars;
|
||||
const overlapChars = Math.max(0, opts.overlapChars ?? DEFAULTS.overlapChars);
|
||||
const minChunkChars = opts.minChunkChars ?? DEFAULTS.minChunkChars;
|
||||
|
||||
if (!text || text.length === 0) return [];
|
||||
if (text.length <= minChunkChars) {
|
||||
return [{ text, charStart: 0, charEnd: text.length }];
|
||||
}
|
||||
|
||||
// Stage 1: split into paragraphs with their absolute char offsets.
|
||||
// Treat any run of \n followed by another newline as a separator.
|
||||
const paragraphs: Array<{ text: string; start: number; end: number }> = [];
|
||||
const paragraphRe = /\n\s*\n/g;
|
||||
let lastIdx = 0;
|
||||
for (const m of text.matchAll(paragraphRe)) {
|
||||
const matchStart = m.index ?? 0;
|
||||
const slice = text.slice(lastIdx, matchStart);
|
||||
if (slice.trim().length > 0) {
|
||||
paragraphs.push({ text: slice, start: lastIdx, end: matchStart });
|
||||
}
|
||||
lastIdx = matchStart + m[0].length;
|
||||
}
|
||||
if (lastIdx < text.length) {
|
||||
const slice = text.slice(lastIdx);
|
||||
if (slice.trim().length > 0) {
|
||||
paragraphs.push({ text: slice, start: lastIdx, end: text.length });
|
||||
}
|
||||
}
|
||||
|
||||
// Pathological input with no paragraph breaks: treat the whole text as one
|
||||
// paragraph so the sentence-split path can still subdivide it.
|
||||
if (paragraphs.length === 0) {
|
||||
paragraphs.push({ text, start: 0, end: text.length });
|
||||
}
|
||||
|
||||
// Stage 2: aggregate paragraphs into chunks. Sub-split any oversize paragraph.
|
||||
type Span = { text: string; start: number; end: number };
|
||||
const spans: Span[] = [];
|
||||
for (const p of paragraphs) {
|
||||
if (p.text.length <= maxChars) {
|
||||
spans.push({ text: p.text, start: p.start, end: p.end });
|
||||
} else {
|
||||
const sentences = splitSentencesWithOffsets(p.text, p.start);
|
||||
for (const s of sentences) {
|
||||
if (s.text.length <= maxChars) {
|
||||
spans.push(s);
|
||||
} else {
|
||||
for (let off = 0; off < s.text.length; off += maxChars) {
|
||||
const sub = s.text.slice(off, off + maxChars);
|
||||
spans.push({ text: sub, start: s.start + off, end: s.start + off + sub.length });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: greedy pack spans into chunks of <= maxChars, joining with '\n\n'
|
||||
// so the embedder sees coherent paragraph boundaries.
|
||||
const chunks: FrameChunk[] = [];
|
||||
let current: { parts: string[]; start: number; end: number; len: number } | null = null;
|
||||
const SEP = '\n\n';
|
||||
|
||||
for (const span of spans) {
|
||||
const candidateLen = current ? current.len + SEP.length + span.text.length : span.text.length;
|
||||
if (current && candidateLen > maxChars) {
|
||||
chunks.push({
|
||||
text: current.parts.join(SEP),
|
||||
charStart: current.start,
|
||||
charEnd: current.end,
|
||||
});
|
||||
current = null;
|
||||
}
|
||||
if (!current) {
|
||||
current = { parts: [span.text], start: span.start, end: span.end, len: span.text.length };
|
||||
} else {
|
||||
current.parts.push(span.text);
|
||||
current.end = span.end;
|
||||
current.len = candidateLen;
|
||||
}
|
||||
}
|
||||
if (current) {
|
||||
chunks.push({
|
||||
text: current.parts.join(SEP),
|
||||
charStart: current.start,
|
||||
charEnd: current.end,
|
||||
});
|
||||
}
|
||||
|
||||
// Stage 4: apply overlap by prepending the tail of the previous chunk.
|
||||
// We never touch char_start/char_end here — those still describe the
|
||||
// chunk's "primary" span in the source text. Overlap text is purely an
|
||||
// embedding-quality boost, not a position claim.
|
||||
if (overlapChars > 0 && chunks.length > 1) {
|
||||
for (let i = 1; i < chunks.length; i++) {
|
||||
const prevTail = chunks[i - 1].text.slice(-overlapChars);
|
||||
chunks[i] = {
|
||||
...chunks[i],
|
||||
text: prevTail + (prevTail.endsWith('\n') ? '' : '\n') + chunks[i].text,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentence-split that preserves absolute char offsets relative to a base.
|
||||
* Conservative — when in doubt, splits, since an over-split is harmless
|
||||
* (more chunks) but an under-split bloats a chunk past maxChars and forces
|
||||
* the hard-cut path.
|
||||
*/
|
||||
function splitSentencesWithOffsets(
|
||||
text: string,
|
||||
baseOffset: number,
|
||||
): Array<{ text: string; start: number; end: number }> {
|
||||
const results: Array<{ text: string; start: number; end: number }> = [];
|
||||
// Match sentence-end punctuation followed by whitespace or end-of-string.
|
||||
const re = /[.!?](?:\s+|$)/g;
|
||||
let lastEnd = 0;
|
||||
for (const m of text.matchAll(re)) {
|
||||
const idx = m.index ?? 0;
|
||||
const cut = idx + m[0].length;
|
||||
const piece = text.slice(lastEnd, cut);
|
||||
if (piece.trim().length > 0) {
|
||||
results.push({ text: piece, start: baseOffset + lastEnd, end: baseOffset + cut });
|
||||
}
|
||||
lastEnd = cut;
|
||||
}
|
||||
if (lastEnd < text.length) {
|
||||
const piece = text.slice(lastEnd);
|
||||
if (piece.trim().length > 0) {
|
||||
results.push({ text: piece, start: baseOffset + lastEnd, end: baseOffset + text.length });
|
||||
}
|
||||
}
|
||||
return results.length > 0 ? results : [{ text, start: baseOffset, end: baseOffset + text.length }];
|
||||
}
|
||||
180
packages/hive-mind-core/src/mind/concept-tracker.ts
Normal file
180
packages/hive-mind-core/src/mind/concept-tracker.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* F19: Spaced Repetition / Concept Tracking for Learning.
|
||||
*
|
||||
* Tracks concept mastery levels in the MindDB. Stores structured learning
|
||||
* data rather than opaque blobs, enabling spaced repetition and mastery
|
||||
* progression.
|
||||
*/
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
export interface ConceptEntry {
|
||||
id: number;
|
||||
concept: string;
|
||||
mastery_level: number; // 1-5
|
||||
last_tested: string | null; // ISO date
|
||||
times_correct: number;
|
||||
times_wrong: number;
|
||||
notes: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface ConceptUpdate {
|
||||
mastery_level?: number;
|
||||
notes?: string;
|
||||
}
|
||||
|
||||
/** SQL to create the concept_mastery table. Run via MindDB migration. */
|
||||
export const CONCEPT_MASTERY_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS concept_mastery (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
concept TEXT UNIQUE NOT NULL,
|
||||
mastery_level INTEGER NOT NULL DEFAULT 1 CHECK (mastery_level BETWEEN 1 AND 5),
|
||||
last_tested TEXT,
|
||||
times_correct INTEGER NOT NULL DEFAULT 0,
|
||||
times_wrong INTEGER NOT NULL DEFAULT 0,
|
||||
notes TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_concept_mastery_level ON concept_mastery (mastery_level);
|
||||
`;
|
||||
|
||||
export class ConceptTracker {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='concept_mastery'"
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(CONCEPT_MASTERY_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or update a concept. If the concept already exists, merge the update.
|
||||
*/
|
||||
upsertConcept(concept: string, update?: ConceptUpdate): ConceptEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
'SELECT * FROM concept_mastery WHERE concept = ?'
|
||||
).get(concept) as ConceptEntry | undefined;
|
||||
|
||||
if (existing) {
|
||||
const newLevel = update?.mastery_level ?? existing.mastery_level;
|
||||
const newNotes = update?.notes ?? existing.notes;
|
||||
raw.prepare(`
|
||||
UPDATE concept_mastery
|
||||
SET mastery_level = ?, notes = ?, updated_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
Math.max(1, Math.min(5, newLevel)),
|
||||
newNotes,
|
||||
existing.id,
|
||||
);
|
||||
return raw.prepare('SELECT * FROM concept_mastery WHERE id = ?').get(existing.id) as ConceptEntry;
|
||||
}
|
||||
|
||||
const level = Math.max(1, Math.min(5, update?.mastery_level ?? 1));
|
||||
const notes = update?.notes ?? '';
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO concept_mastery (concept, mastery_level, notes)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(concept, level, notes);
|
||||
|
||||
return raw.prepare('SELECT * FROM concept_mastery WHERE id = ?').get(result.lastInsertRowid) as ConceptEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single concept by name.
|
||||
*/
|
||||
getConcept(concept: string): ConceptEntry | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM concept_mastery WHERE concept = ?'
|
||||
).get(concept) as ConceptEntry | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* List concepts, optionally filtered by mastery level range.
|
||||
*/
|
||||
listConcepts(minMastery?: number, maxMastery?: number): ConceptEntry[] {
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (minMastery !== undefined) {
|
||||
conditions.push('mastery_level >= ?');
|
||||
params.push(minMastery);
|
||||
}
|
||||
if (maxMastery !== undefined) {
|
||||
conditions.push('mastery_level <= ?');
|
||||
params.push(maxMastery);
|
||||
}
|
||||
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
return this.db.getDatabase().prepare(
|
||||
`SELECT * FROM concept_mastery ${where} ORDER BY updated_at DESC`
|
||||
).all(...params) as ConceptEntry[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Record whether the user answered correctly about a concept.
|
||||
* Adjusts mastery level: +1 on correct (max 5), -1 on wrong (min 1).
|
||||
*/
|
||||
recordAnswer(concept: string, correct: boolean): ConceptEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
'SELECT * FROM concept_mastery WHERE concept = ?'
|
||||
).get(concept) as ConceptEntry | undefined;
|
||||
|
||||
if (!existing) {
|
||||
// Auto-create the concept on first answer
|
||||
const level = correct ? 2 : 1;
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO concept_mastery (concept, mastery_level, last_tested, times_correct, times_wrong)
|
||||
VALUES (?, ?, datetime('now'), ?, ?)
|
||||
`).run(concept, level, correct ? 1 : 0, correct ? 0 : 1);
|
||||
return raw.prepare('SELECT * FROM concept_mastery WHERE id = ?').get(result.lastInsertRowid) as ConceptEntry;
|
||||
}
|
||||
|
||||
const newLevel = correct
|
||||
? Math.min(5, existing.mastery_level + 1)
|
||||
: Math.max(1, existing.mastery_level - 1);
|
||||
|
||||
raw.prepare(`
|
||||
UPDATE concept_mastery
|
||||
SET mastery_level = ?,
|
||||
last_tested = datetime('now'),
|
||||
times_correct = times_correct + ?,
|
||||
times_wrong = times_wrong + ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
newLevel,
|
||||
correct ? 1 : 0,
|
||||
correct ? 0 : 1,
|
||||
existing.id,
|
||||
);
|
||||
|
||||
return raw.prepare('SELECT * FROM concept_mastery WHERE id = ?').get(existing.id) as ConceptEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get concepts due for review -- low mastery or not tested recently.
|
||||
* Returns concepts sorted by priority: lowest mastery first, then oldest test date.
|
||||
*/
|
||||
getDueForReview(limit = 10): ConceptEntry[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM concept_mastery
|
||||
WHERE mastery_level < 4
|
||||
ORDER BY mastery_level ASC, last_tested ASC NULLS FIRST
|
||||
LIMIT ?
|
||||
`).all(limit) as ConceptEntry[];
|
||||
}
|
||||
}
|
||||
40
packages/hive-mind-core/src/mind/content-hash.ts
Normal file
40
packages/hive-mind-core/src/mind/content-hash.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
/**
|
||||
* content-hash.ts — canonical frame content hash for dedup (oss-drift D3).
|
||||
*
|
||||
* Adopts the OSS indexed-`content_hash`-column pattern but with MONO hash
|
||||
* semantics: the hash covers `stripHmPrefix(content).trim()`, NOT bare
|
||||
* `content.trim()` — provenance-insensitive dedup (OQ-6) is load-bearing
|
||||
* here (two same-body captures of one turn from different sources must
|
||||
* collapse regardless of their `[hm …]` metadata prefix). Porting the OSS
|
||||
* trim-only definition verbatim would have regressed that.
|
||||
*
|
||||
* SINGLE definition shared by insert, dedup lookup, update, compaction, and
|
||||
* the migration backfill so trim/strip semantics can never drift between
|
||||
* call sites.
|
||||
*
|
||||
* Reverse-ported from OSS hive-mind content-hash.ts (oss-drift triage D3,
|
||||
* 2026-06-11), semantics adapted per the triage verdict.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Strip the leading hive-mind metadata prefix `[hm session:… src:… event:…] `
|
||||
* so dedup compares the semantic turn BODY, not the provenance. The prefix is
|
||||
* emitted by shim-core's `buildPrefix` (`[hm <tokens>] `); two captures of the
|
||||
* same turn from different sources differ only in that prefix. Content without
|
||||
* the prefix (harvest / ingest / cognify) is returned unchanged — a no-op.
|
||||
* The regex anchors on `[hm ` and stops at the first `]`, so a body that
|
||||
* merely contains `[` brackets later is never over-stripped.
|
||||
*
|
||||
* (Moved here from frames.ts so the hash and the strip live in one module;
|
||||
* frames.ts re-exports it for back-compat.)
|
||||
*/
|
||||
export function stripHmPrefix(content: string): string {
|
||||
return content.replace(/^\[hm [^\]]*\]\s*/, '');
|
||||
}
|
||||
|
||||
/** Canonical content hash: sha256 over the stripped, trimmed body. */
|
||||
export function hashFrameContent(content: string): string {
|
||||
return createHash('sha256').update(stripHmPrefix(content).trim()).digest('hex');
|
||||
}
|
||||
675
packages/hive-mind-core/src/mind/db.ts
Normal file
675
packages/hive-mind-core/src/mind/db.ts
Normal file
@@ -0,0 +1,675 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import * as sqliteVec from 'sqlite-vec';
|
||||
import {
|
||||
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
|
||||
vecTableSqlForDim, chunksVecTableSqlForDim,
|
||||
} from './schema.js';
|
||||
import { hashFrameContent } from './content-hash.js';
|
||||
|
||||
/** How long better-sqlite3 waits on a locked DB before throwing SQLITE_BUSY. The
|
||||
* Fastify sidecar and the standalone memory-mcp server open the SAME
|
||||
* ~/.waggle/personal.mind as separate OS processes, so a writer-writer clash would
|
||||
* otherwise throw immediately instead of waiting for the lock to clear. */
|
||||
const BUSY_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Bounded retry for the WAL `SQLITE_BUSY_SNAPSHOT` race that busy_timeout does NOT
|
||||
* cover: a deferred transaction that began as a reader cannot upgrade to a writer
|
||||
* once another connection has committed in between, and SQLite fails it instantly
|
||||
* rather than waiting. Re-running the closure reads the fresh snapshot. */
|
||||
const BUSY_RETRY_MAX_ATTEMPTS = 5;
|
||||
const BUSY_RETRY_BASE_DELAY_MS = 20;
|
||||
|
||||
/** True for the two transient cross-process contention codes worth retrying. */
|
||||
function isSqliteBusyError(err: unknown): boolean {
|
||||
const code = (err as { code?: unknown } | null)?.code;
|
||||
return code === 'SQLITE_BUSY' || code === 'SQLITE_BUSY_SNAPSHOT';
|
||||
}
|
||||
|
||||
/** Synchronous backoff. better-sqlite3 is fully synchronous, so there is no event
|
||||
* loop to yield to between retries; Atomics.wait blocks only this thread. */
|
||||
function sleepSync(ms: number): void {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
}
|
||||
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R7, 2026-06-11).
|
||||
/** A persisted embedding fingerprint: which provider/model produced this .mind's
|
||||
* vectors, and at what dimension. Recorded in `meta` on the first vector use. */
|
||||
export interface EmbeddingFingerprint {
|
||||
provider: string;
|
||||
model: string;
|
||||
dim: number;
|
||||
}
|
||||
|
||||
export type FingerprintCheck =
|
||||
| { status: 'recorded' }
|
||||
| { status: 'match' }
|
||||
| { status: 'model-changed'; storedModel: string; storedProvider: string };
|
||||
|
||||
/** Thrown when the active embedder's dimension differs from the dimension this
|
||||
* .mind's vectors were written at. Mixing dims returns noise and corrupts the
|
||||
* index, so we refuse loudly and point at the re-embed remediation. */
|
||||
export class EmbeddingDimMismatchError extends Error {
|
||||
constructor(
|
||||
readonly storedDim: number,
|
||||
readonly runtimeDim: number,
|
||||
) {
|
||||
super(
|
||||
`Embedding dimension mismatch: this .mind stores ${storedDim}-dim vectors but the active ` +
|
||||
`embedder produces ${runtimeDim}-dim vectors. Vector search would return noise and writes ` +
|
||||
`would corrupt the index. Call MindDB.recreateVecTables(${runtimeDim}) and re-embed all ` +
|
||||
`frames at the new dimension, or switch back to a ${storedDim}-dim model.`,
|
||||
);
|
||||
this.name = 'EmbeddingDimMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class MindDB {
|
||||
private db: DatabaseType;
|
||||
|
||||
constructor(dbPath: string) {
|
||||
this.db = new Database(dbPath);
|
||||
|
||||
// Enable WAL mode for better concurrent read performance
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
// Cross-process contention: the sidecar and memory-mcp open the same .mind
|
||||
// file. Wait for a held lock instead of throwing SQLITE_BUSY on first contact
|
||||
// (the WAL snapshot-upgrade race that this doesn't cover is retried in
|
||||
// runWithBusyRetry).
|
||||
this.db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
||||
|
||||
// Load sqlite-vec extension — support bundled path override for desktop builds
|
||||
const vecPath = process.env.WAGGLE_SQLITE_VEC_PATH;
|
||||
if (vecPath) {
|
||||
this.db.loadExtension(vecPath);
|
||||
} else {
|
||||
sqliteVec.load(this.db);
|
||||
}
|
||||
|
||||
this.initSchema();
|
||||
}
|
||||
|
||||
private initSchema(): void {
|
||||
const existing = this.db.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='meta'"
|
||||
).get() as { name: string } | undefined;
|
||||
|
||||
if (!existing) {
|
||||
this.db.exec(SCHEMA_SQL);
|
||||
this.db.exec(VEC_TABLE_SQL);
|
||||
this.db.exec(CHUNKS_VEC_TABLE_SQL);
|
||||
this.db.prepare(
|
||||
"INSERT INTO meta (key, value) VALUES ('schema_version', ?)"
|
||||
).run(SCHEMA_VERSION);
|
||||
// 2026-04-15: Track first-run so Art. 19 retention checker can distinguish
|
||||
// 'new system, no logs yet' from 'old system, logs pruned'.
|
||||
this.db.prepare(
|
||||
"INSERT INTO meta (key, value) VALUES ('first_run_at', ?)"
|
||||
).run(new Date().toISOString());
|
||||
} else {
|
||||
this.runMigrations();
|
||||
// Backfill first_run_at for pre-existing DBs. Best-effort: we don't know when
|
||||
// they were actually created so we approximate with 'now' — this means retroactive
|
||||
// retention checks can't be perfect, but forward-looking checks will be correct
|
||||
// within 180 days.
|
||||
const hasFirstRun = this.db.prepare(
|
||||
"SELECT value FROM meta WHERE key = 'first_run_at'"
|
||||
).get() as { value: string } | undefined;
|
||||
if (!hasFirstRun) {
|
||||
this.db.prepare(
|
||||
"INSERT INTO meta (key, value) VALUES ('first_run_at', ?)"
|
||||
).run(new Date().toISOString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Read the first-run timestamp for this database (ISO 8601). Returns null if missing. */
|
||||
getFirstRunAt(): string | null {
|
||||
try {
|
||||
const row = this.db.prepare(
|
||||
"SELECT value FROM meta WHERE key = 'first_run_at'"
|
||||
).get() as { value: string } | undefined;
|
||||
return row?.value ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Run incremental schema migrations for existing .mind databases */
|
||||
private runMigrations(): void {
|
||||
// 2026-04-16: Ensure all tables from SCHEMA_SQL exist. Old .mind databases
|
||||
// may predate tables added during sprint work (ai_interactions, execution_traces,
|
||||
// evolution_runs, harvest_sources, procedures, improvement_signals, install_audit).
|
||||
// SCHEMA_SQL uses CREATE TABLE/INDEX IF NOT EXISTS throughout, so re-running it
|
||||
// is safe and idempotent — it only creates what's missing.
|
||||
//
|
||||
// CRASH RECOVERY (must run before the rebuild below): a pre-transactional
|
||||
// build of the FIX-3/M2 rebuild could die mid-sequence, stranding every
|
||||
// audit row in install_audit__mig_old while install_audit is missing or
|
||||
// freshly recreated empty — and the next rebuild's DROP would then destroy
|
||||
// them permanently. Restore before anything else touches the table.
|
||||
const migOldExists = !!this.db.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit__mig_old'"
|
||||
).get();
|
||||
if (migOldExists) {
|
||||
const auditExists = !!this.db.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit'"
|
||||
).get();
|
||||
if (!auditExists) {
|
||||
// Crash landed between RENAME and recreate — rename back wholesale;
|
||||
// the sentinel check below re-runs the (now transactional) rebuild.
|
||||
this.db.prepare('ALTER TABLE install_audit__mig_old RENAME TO install_audit').run();
|
||||
} else {
|
||||
// Crash landed between recreate and copy-back: copy the stranded rows
|
||||
// home if nothing new was written, then retire the stale table.
|
||||
const cnt = (this.db.prepare('SELECT COUNT(*) AS cnt FROM install_audit')
|
||||
.get() as { cnt: number }).cnt;
|
||||
if (cnt === 0) {
|
||||
this.db.prepare(
|
||||
`INSERT INTO install_audit
|
||||
(id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
SELECT id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
FROM install_audit__mig_old`
|
||||
).run();
|
||||
}
|
||||
this.db.prepare('DROP TABLE install_audit__mig_old').run();
|
||||
}
|
||||
}
|
||||
|
||||
// FIX-3 (2026-05-17): install_audit's capability_type / approval_class /
|
||||
// action CHECK lists drifted behind their TS type unions
|
||||
// (connector/marketplace/blocked). Because the CREATE below is
|
||||
// IF NOT EXISTS, an existing install_audit keeps its stale CHECK and
|
||||
// auditStore.record() crashes the moment acquire_capability proposes a
|
||||
// marketplace/connector capability. Rename the stale table aside so the
|
||||
// corrected SCHEMA_SQL DDL (single source of truth) recreates it; rows
|
||||
// are copied back below. Idempotent: keyed on whether the stored DDL
|
||||
// already lists 'marketplace'.
|
||||
//
|
||||
// M2 (UX-Refactor Phase 4, 2026-06-10): risk_level's CHECK drifted the same
|
||||
// way — TS AuditRiskLevel gained 'critical' but the DDL allowed only
|
||||
// low/medium/high, so marketplace.ts's CRITICAL-block audit write was
|
||||
// silently rejected. Same rebuild mechanism, keyed on the widened
|
||||
// risk_level list literal ("'low', 'medium', 'high', 'critical'" — note
|
||||
// 'critical' alone is NOT a safe sentinel: it already appears in the
|
||||
// approval_class CHECK).
|
||||
//
|
||||
// The whole rename→recreate→copy-back→drop sequence runs in ONE
|
||||
// transaction: a process death mid-rebuild rolls back to the pre-rebuild
|
||||
// state instead of silently orphaning the audit trail (this is the EU AI
|
||||
// Act compliance table — partial loss here is not acceptable).
|
||||
const auditTableSql = (this.db.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'"
|
||||
).get() as { sql: string } | undefined)?.sql;
|
||||
// P5/D4 (2026-06-12): AuditAction gained 'uninstalled' so skill/capability
|
||||
// removal is auditable. Same rebuild mechanism, keyed on whether the stored
|
||||
// action CHECK already lists 'uninstalled' ('uninstalled' is a safe sentinel —
|
||||
// it appears in no other CHECK on this table).
|
||||
// P7/D15 #15 (2026-06-12): trust_source gained a CHECK (was unconstrained).
|
||||
// Sentinel "CHECK (trust_source IN" appears nowhere else.
|
||||
const auditNeedsRebuild = auditTableSql !== undefined && (
|
||||
!auditTableSql.includes("'marketplace'")
|
||||
|| !auditTableSql.includes("'low', 'medium', 'high', 'critical'")
|
||||
|| !auditTableSql.includes("'uninstalled'")
|
||||
|| !auditTableSql.includes('CHECK (trust_source IN')
|
||||
);
|
||||
if (auditNeedsRebuild) {
|
||||
this.db.transaction(() => {
|
||||
this.db.prepare('DROP TABLE IF EXISTS install_audit__mig_old').run();
|
||||
this.db.prepare('ALTER TABLE install_audit RENAME TO install_audit__mig_old').run();
|
||||
this.db.prepare('DROP INDEX IF EXISTS idx_audit_capability').run();
|
||||
this.db.prepare('DROP INDEX IF EXISTS idx_audit_timestamp').run();
|
||||
// SCHEMA_SQL recreates install_audit with the widened CHECK (and is
|
||||
// idempotent for every other table — see the comment block above).
|
||||
this.db.exec(SCHEMA_SQL);
|
||||
this.db.prepare(
|
||||
`INSERT INTO install_audit
|
||||
(id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
SELECT id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
FROM install_audit__mig_old`
|
||||
).run();
|
||||
this.db.prepare('DROP TABLE install_audit__mig_old').run();
|
||||
})();
|
||||
} else {
|
||||
this.db.exec(SCHEMA_SQL);
|
||||
}
|
||||
|
||||
// oss-drift D1 (2026-06-11): chunk-level retrieval. SCHEMA_SQL above creates
|
||||
// memory_frame_chunks (IF NOT EXISTS); the vec0 virtual table needs its own
|
||||
// idempotent exec because vec tables live outside SCHEMA_SQL (they require
|
||||
// the sqlite-vec extension, loaded in the constructor). Databases that
|
||||
// predate D1 gain an EMPTY chunk index here — vectorSearchChunks returns
|
||||
// null on an empty index, so recall falls back to whole-frame vectors until
|
||||
// rechunkAllFrames (or flag-gated indexFrame chunking) populates it.
|
||||
this.db.exec(CHUNKS_VEC_TABLE_SQL);
|
||||
|
||||
// W2.1: Add 'source' column to memory_frames (provenance tracking)
|
||||
const hasSourceCol = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('memory_frames') WHERE name='source'"
|
||||
).get() as { cnt: number };
|
||||
if (hasSourceCol.cnt === 0) {
|
||||
this.db.exec(
|
||||
"ALTER TABLE memory_frames ADD COLUMN source TEXT NOT NULL DEFAULT 'user_stated'"
|
||||
);
|
||||
}
|
||||
|
||||
// UX-Refactor Phase 2B: Add 'metadata' column to memory_frames. JSON blob
|
||||
// backing the Memory Center (kind/confidence/scope/status/sourceId/tags/
|
||||
// evidence/related*; PRD §15.4). Required by the Phase-2 gate ratifications
|
||||
// A8 (reversible Archive status) + C33 (persisted 'unreviewed' status) +
|
||||
// B2 (heuristic confidence) — all need per-frame state that survives a
|
||||
// restart. Idempotent ADD COLUMN, same pattern as 'source' above; existing
|
||||
// rows default to '{}'.
|
||||
const hasMetadataCol = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('memory_frames') WHERE name='metadata'"
|
||||
).get() as { cnt: number };
|
||||
if (hasMetadataCol.cnt === 0) {
|
||||
this.db.exec(
|
||||
"ALTER TABLE memory_frames ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'"
|
||||
);
|
||||
}
|
||||
|
||||
// oss-drift D3 (2026-06-11): indexed content_hash for O(1) frame dedup —
|
||||
// FrameStore.findDuplicate previously scanned only the last 500 frames
|
||||
// (silently missed older duplicates). Hash semantics are MONO's
|
||||
// (stripHmPrefix + trim, mind/content-hash.ts), so the backfill must use
|
||||
// hashFrameContent, never a SQL-side hash. Idempotent: ADD COLUMN guarded
|
||||
// by pragma check; backfill targets only NULL rows (no-op when current).
|
||||
const hasContentHashCol = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('memory_frames') WHERE name='content_hash'"
|
||||
).get() as { cnt: number };
|
||||
if (hasContentHashCol.cnt === 0) {
|
||||
this.db.exec('ALTER TABLE memory_frames ADD COLUMN content_hash TEXT');
|
||||
}
|
||||
this.db.exec(
|
||||
'CREATE INDEX IF NOT EXISTS idx_frames_content_hash ON memory_frames (content_hash)'
|
||||
);
|
||||
|
||||
// W4.1: KG entity↔frame bridge — powers the 'contextual' scoring signal by
|
||||
// mapping query-seeded graph distances back onto frames. Idempotent; SCHEMA_SQL
|
||||
// carries the same DDL for fresh DBs. frames.ts already DELETEs from this table
|
||||
// on frame deletion; the ON DELETE CASCADE FK makes that belt-and-suspenders.
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS kg_entity_frames (
|
||||
entity_id INTEGER NOT NULL REFERENCES knowledge_entities(id) ON DELETE CASCADE,
|
||||
frame_id INTEGER NOT NULL REFERENCES memory_frames(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (entity_id, frame_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_kg_entity_frames_frame ON kg_entity_frames (frame_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kg_entity_frames_entity ON kg_entity_frames (entity_id);
|
||||
`);
|
||||
this.backfillContentHash();
|
||||
|
||||
// 2026-04-15: EU AI Act Art. 12.1(a) — record inputs and outputs, not just
|
||||
// token counts (review Critical #3 from cowork/Code-Review_Compliance).
|
||||
const hasInputText = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('ai_interactions') WHERE name='input_text'"
|
||||
).get() as { cnt: number };
|
||||
if (hasInputText.cnt === 0) {
|
||||
this.db.exec("ALTER TABLE ai_interactions ADD COLUMN input_text TEXT");
|
||||
}
|
||||
const hasOutputText = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('ai_interactions') WHERE name='output_text'"
|
||||
).get() as { cnt: number };
|
||||
if (hasOutputText.cnt === 0) {
|
||||
this.db.exec("ALTER TABLE ai_interactions ADD COLUMN output_text TEXT");
|
||||
}
|
||||
|
||||
// 2026-04-15: Append-only triggers for audit log (review Critical #1). Idempotent.
|
||||
this.db.exec(
|
||||
"CREATE TRIGGER IF NOT EXISTS ai_interactions_no_delete BEFORE DELETE ON ai_interactions BEGIN SELECT RAISE(ABORT, 'ai_interactions is append-only (EU AI Act Art. 12 audit log)'); END"
|
||||
);
|
||||
this.db.exec(
|
||||
"CREATE TRIGGER IF NOT EXISTS ai_interactions_no_update BEFORE UPDATE ON ai_interactions BEGIN SELECT RAISE(ABORT, 'ai_interactions is append-only (EU AI Act Art. 12 audit log)'); END"
|
||||
);
|
||||
|
||||
// #7 (2026-06-30): verbatim provenance archive — append-only, immutable.
|
||||
// Idempotent; SCHEMA_SQL carries the same DDL for fresh DBs. Not in the
|
||||
// retrieval corpus (no FTS/vec). Append-only triggers mirror ai_interactions,
|
||||
// EXCEPT a one-time GDPR Art.17 redaction (see raw_archive_no_update WHEN clause).
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS raw_archive (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
archive_uid TEXT NOT NULL UNIQUE,
|
||||
source TEXT NOT NULL,
|
||||
source_ref TEXT,
|
||||
title TEXT,
|
||||
content TEXT NOT NULL,
|
||||
content_sha256 TEXT NOT NULL,
|
||||
injection_flagged INTEGER NOT NULL DEFAULT 0,
|
||||
injection_flags TEXT NOT NULL DEFAULT '',
|
||||
source_timestamp TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
erased_at TEXT,
|
||||
erased_reason TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_archive_source_ref ON raw_archive (source, source_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_archive_created ON raw_archive (created_at DESC);
|
||||
`);
|
||||
// GDPR Art.17 columns for pre-erasure DBs (idempotent ADD COLUMN, same pattern
|
||||
// as memory_frames.source/metadata above). MUST precede the trigger below, which
|
||||
// references NEW.erased_at / OLD.erased_at. ALTER is DDL — it does NOT fire the
|
||||
// BEFORE UPDATE trigger.
|
||||
for (const col of ['erased_at', 'erased_reason'] as const) {
|
||||
const has = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('raw_archive') WHERE name=?"
|
||||
).get(col) as { cnt: number };
|
||||
if (has.cnt === 0) {
|
||||
this.db.exec(`ALTER TABLE raw_archive ADD COLUMN ${col} TEXT`);
|
||||
}
|
||||
}
|
||||
// Size-guard columns for pre-guard DBs (idempotent ADD COLUMN, distinct DDL
|
||||
// per column so `truncated` gets its NOT NULL DEFAULT). Not referenced by the
|
||||
// append-only trigger, so no trigger swap is needed. See raw-archive.ts append().
|
||||
for (const [col, ddl] of [
|
||||
['truncated', 'INTEGER NOT NULL DEFAULT 0'],
|
||||
['original_length', 'INTEGER'],
|
||||
] as const) {
|
||||
const has = this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM pragma_table_info('raw_archive') WHERE name=?"
|
||||
).get(col) as { cnt: number };
|
||||
if (has.cnt === 0) {
|
||||
this.db.exec(`ALTER TABLE raw_archive ADD COLUMN ${col} ${ddl}`);
|
||||
}
|
||||
}
|
||||
// Upgrade the legacy ABSOLUTE no-update trigger to the redaction-aware one.
|
||||
// CREATE TRIGGER IF NOT EXISTS will NOT swap an existing trigger, so we DROP +
|
||||
// CREATE — but ATOMICALLY (one transaction), else a crash or a concurrent WAL
|
||||
// writer between the two statements would see raw_archive with NO update guard.
|
||||
// Sentinel: skip once the live trigger already carries the archive_uid-ROTATION
|
||||
// clause (both a perf win and it stops re-opening the swap window on every process
|
||||
// start). An OLD trigger that still froze archive_uid ('IS OLD.archive_uid') lacks
|
||||
// this substring, so it is upgraded on reopen — required, else the rotating erase()
|
||||
// would be rejected on an existing DB. The WHEN clause is kept BYTE-IDENTICAL to the
|
||||
// SCHEMA_SQL version in schema.ts, and the content literal to
|
||||
// RAW_ARCHIVE_REDACTION_MARKER in raw-archive.ts. (Forward-only: this does NOT
|
||||
// rotate the uid of rows erased under the old trigger — erase() shipped 2026-07-01,
|
||||
// so real DBs have ~zero such rows; the trigger only permits rotation during the
|
||||
// one-time erased_at NULL->set transition, not on an already-erased row.)
|
||||
const liveNoUpdate = this.db.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='trigger' AND name='raw_archive_no_update'"
|
||||
).get() as { sql?: string } | undefined;
|
||||
if (!liveNoUpdate?.sql || !liveNoUpdate.sql.includes('NEW.archive_uid <> OLD.archive_uid')) {
|
||||
this.db.transaction(() => {
|
||||
this.db.exec('DROP TRIGGER IF EXISTS raw_archive_no_update');
|
||||
this.db.exec(
|
||||
"CREATE TRIGGER raw_archive_no_update BEFORE UPDATE ON raw_archive " +
|
||||
"WHEN NOT (OLD.erased_at IS NULL AND NEW.erased_at IS NOT NULL AND NEW.erased_at <> '' " +
|
||||
"AND NEW.content = '[REDACTED — GDPR Art.17 erasure]' AND NEW.content_sha256 = '' AND NEW.title IS NULL " +
|
||||
"AND NEW.id IS OLD.id AND NEW.archive_uid <> OLD.archive_uid AND NEW.archive_uid <> '' " +
|
||||
"AND NEW.source IS OLD.source AND NEW.source_ref IS OLD.source_ref " +
|
||||
"AND NEW.created_at IS OLD.created_at AND NEW.source_timestamp IS OLD.source_timestamp " +
|
||||
"AND NEW.injection_flagged IS OLD.injection_flagged AND NEW.injection_flags IS OLD.injection_flags) " +
|
||||
"BEGIN SELECT RAISE(ABORT, 'raw_archive is append-only; only a one-time canonical GDPR Art.17 redaction is permitted'); END"
|
||||
);
|
||||
})();
|
||||
}
|
||||
this.db.exec(
|
||||
"CREATE TRIGGER IF NOT EXISTS raw_archive_no_delete BEFORE DELETE ON raw_archive BEGIN SELECT RAISE(ABORT, 'raw_archive is append-only (verbatim provenance archive)'); END"
|
||||
);
|
||||
|
||||
// #7 (2026-07-02): erased-subject suppression list — makes Art.17 erasure
|
||||
// "sticky" across re-import. Idempotent; SCHEMA_SQL carries the same DDL for
|
||||
// fresh DBs. Keyed on (source, source_ref) only (no content/hash — that would
|
||||
// reintroduce the re-id vector). Rows are deletable (re-consent path), so NO
|
||||
// immutability trigger. Then one-time backfill from the already-erased
|
||||
// raw_archive rows so PAST erasures become sticky too (see backfillErasedSubjects).
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS erased_subjects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
source_ref TEXT NOT NULL,
|
||||
erased_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
reason TEXT,
|
||||
UNIQUE(source, source_ref)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_erased_subjects_lookup ON erased_subjects (source, source_ref);
|
||||
`);
|
||||
this.backfillErasedSubjects();
|
||||
|
||||
// W4.1: one-time backfill of the kg_entity_frames bridge over pre-existing
|
||||
// frames (new writes populate it live via cognify/harvest). Sentinel-guarded.
|
||||
this.backfillKgEntityFrames();
|
||||
}
|
||||
|
||||
/** One-time backfill of the kg_entity_frames bridge so the 'contextual' scoring
|
||||
* signal works over frames written before the bridge existed. Offline (string
|
||||
* match, no LLM): an entity links to a frame whose content mentions its name.
|
||||
* Idempotent (INSERT OR IGNORE) and guarded by a meta sentinel unless `force`.
|
||||
* Returns the number of new (entity, frame) links created. */
|
||||
backfillKgEntityFrames(force = false): number {
|
||||
if (!force) {
|
||||
const done = this.db.prepare("SELECT value FROM meta WHERE key = 'kg_bridge_backfilled'").get();
|
||||
if (done) return 0;
|
||||
}
|
||||
const frames = this.db
|
||||
.prepare('SELECT id, content FROM memory_frames')
|
||||
.all() as { id: number; content: string }[];
|
||||
// Ubiquity cap: an entity mentioned in nearly every frame (e.g. "Claude" in a
|
||||
// claude-code export) is a hub that carries no locational signal — skip it.
|
||||
// Cap at 40% of frames, floored at 20 so small corpora aren't over-filtered.
|
||||
const cap = Math.max(20, Math.floor(frames.length * 0.4));
|
||||
const ents = this.db
|
||||
.prepare("SELECT id, lower(name) AS lname FROM knowledge_entities WHERE valid_to IS NULL AND length(name) >= 3")
|
||||
.all() as { id: number; lname: string }[];
|
||||
const countStmt = this.db.prepare(
|
||||
'SELECT COUNT(*) AS c FROM memory_frames WHERE instr(lower(content), ?) > 0'
|
||||
);
|
||||
const keep = ents.filter((e) => {
|
||||
const c = (countStmt.get(e.lname) as { c: number }).c;
|
||||
return c > 0 && c <= cap;
|
||||
});
|
||||
const link = this.db.prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
);
|
||||
let created = 0;
|
||||
const run = this.db.transaction(() => {
|
||||
// On a forced re-run, rebuild from scratch so hub/merged entities don't linger.
|
||||
if (force) this.db.prepare('DELETE FROM kg_entity_frames').run();
|
||||
for (const f of frames) {
|
||||
const lc = f.content.toLowerCase();
|
||||
for (const e of keep) {
|
||||
if (lc.includes(e.lname)) created += link.run(e.id, f.id).changes;
|
||||
}
|
||||
}
|
||||
this.db
|
||||
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('kg_bridge_backfilled', '1')")
|
||||
.run();
|
||||
});
|
||||
run();
|
||||
return created;
|
||||
}
|
||||
|
||||
/** One-time backfill of the erased-subject suppression list (#7 Art.17 "sticky
|
||||
* erasure") from raw_archive rows that were ALREADY erased before this feature
|
||||
* shipped. Keyed on (source, source_ref); rows with a NULL source_ref carry no
|
||||
* subject key and are skipped. Idempotent (INSERT OR IGNORE), meta-sentinel-guarded
|
||||
* unless `force`. Returns the number of new suppression rows created.
|
||||
*
|
||||
* LIMITATION (id-domain mismatch): a subject harvested+erased BEFORE the stable-id
|
||||
* arc has source_ref = a random UUID (adapters minted randomUUID() pre-arc). A fresh
|
||||
* re-export now mints a DETERMINISTIC stableHarvestId ≠ that UUID, so the backfilled
|
||||
* row won't match the new re-import and can't suppress it. The backfill is thus an
|
||||
* accurate LEDGER of historical erasures but only re-suppresses a re-feed of the
|
||||
* identical old-id data; a fresh re-export of pre-arc data re-establishes stickiness
|
||||
* only on its next re-erase (which records the stable id). Post-arc erasures are fully
|
||||
* sticky (eraseBySourceRef records the resolved stable source_ref). */
|
||||
backfillErasedSubjects(force = false): number {
|
||||
if (!force) {
|
||||
const done = this.db.prepare("SELECT value FROM meta WHERE key = 'erased_subjects_backfilled'").get();
|
||||
if (done) return 0;
|
||||
}
|
||||
let created = 0;
|
||||
const run = this.db.transaction(() => {
|
||||
created = this.db.prepare(
|
||||
`INSERT OR IGNORE INTO erased_subjects (source, source_ref, erased_at, reason)
|
||||
SELECT source, source_ref, erased_at, erased_reason
|
||||
FROM raw_archive WHERE erased_at IS NOT NULL AND source_ref IS NOT NULL`
|
||||
).run().changes;
|
||||
this.db
|
||||
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('erased_subjects_backfilled', '1')")
|
||||
.run();
|
||||
});
|
||||
run();
|
||||
return created;
|
||||
}
|
||||
|
||||
/** Backfill memory_frames.content_hash for rows inserted before the column
|
||||
* existed (oss-drift D3). Transactional; only NULL rows touched. */
|
||||
private backfillContentHash(): void {
|
||||
const rows = this.db
|
||||
.prepare('SELECT id, content FROM memory_frames WHERE content_hash IS NULL')
|
||||
.all() as { id: number; content: string }[];
|
||||
if (rows.length === 0) return;
|
||||
const update = this.db.prepare('UPDATE memory_frames SET content_hash = ? WHERE id = ?');
|
||||
const tx = this.db.transaction((items: { id: number; content: string }[]) => {
|
||||
for (const r of items) update.run(hashFrameContent(r.content), r.id);
|
||||
});
|
||||
tx(rows);
|
||||
}
|
||||
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R7, 2026-06-11).
|
||||
/** Read a single `meta` value, or null if absent. */
|
||||
private getMeta(key: string): string | null {
|
||||
const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
/** Upsert a single `meta` key/value (meta.key is the PRIMARY KEY). */
|
||||
private setMeta(key: string, value: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
'INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value'
|
||||
)
|
||||
.run(key, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Guard this .mind's embedding fingerprint. Call before the first vector
|
||||
* write/read of a session (HybridSearch is the natural seam — it holds both
|
||||
* the db and the embedder). Returns the check result; throws only on a hard
|
||||
* dimension mismatch:
|
||||
* - no fingerprint yet → record {provider, model, dim}, return 'recorded'
|
||||
* - same dim + same model → 'match' (no-op)
|
||||
* - same dim, different model/provider → update + 'model-changed' (caller
|
||||
* should warn: vectors stay numerically valid but cross-model comparison
|
||||
* is semantically degraded)
|
||||
* - different dim → throw EmbeddingDimMismatchError (only safe path is re-embed)
|
||||
*/
|
||||
ensureEmbeddingFingerprint(fp: EmbeddingFingerprint): FingerprintCheck {
|
||||
const storedDimRaw = this.getMeta('embedding_dim');
|
||||
if (storedDimRaw === null) {
|
||||
this.setMeta('embedding_provider', fp.provider);
|
||||
this.setMeta('embedding_model', fp.model);
|
||||
this.setMeta('embedding_dim', String(fp.dim));
|
||||
return { status: 'recorded' };
|
||||
}
|
||||
const storedDim = Number(storedDimRaw);
|
||||
if (storedDim !== fp.dim) {
|
||||
throw new EmbeddingDimMismatchError(storedDim, fp.dim);
|
||||
}
|
||||
const storedModel = this.getMeta('embedding_model') ?? '';
|
||||
const storedProvider = this.getMeta('embedding_provider') ?? '';
|
||||
if (storedModel !== fp.model || storedProvider !== fp.provider) {
|
||||
this.setMeta('embedding_provider', fp.provider);
|
||||
this.setMeta('embedding_model', fp.model);
|
||||
return { status: 'model-changed', storedModel, storedProvider };
|
||||
}
|
||||
return { status: 'match' };
|
||||
}
|
||||
|
||||
/** Force-write the embedding fingerprint. Used after a re-embed so the guard
|
||||
* matches the embedder that produced the new vectors. */
|
||||
setEmbeddingFingerprint(fp: EmbeddingFingerprint): void {
|
||||
this.setMeta('embedding_provider', fp.provider);
|
||||
this.setMeta('embedding_model', fp.model);
|
||||
this.setMeta('embedding_dim', String(fp.dim));
|
||||
}
|
||||
|
||||
/** Read the recorded embedding fingerprint, or null if none recorded yet. */
|
||||
getEmbeddingFingerprint(): EmbeddingFingerprint | null {
|
||||
const dimRaw = this.getMeta('embedding_dim');
|
||||
if (dimRaw === null) return null;
|
||||
return {
|
||||
provider: this.getMeta('embedding_provider') ?? 'unknown',
|
||||
model: this.getMeta('embedding_model') ?? 'unknown',
|
||||
dim: Number(dimRaw),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* DROP + CREATE both vec tables (memory_frames_vec + memory_frame_chunks_vec)
|
||||
* at `dim` (vec0 columns can't be ALTERed) and update the stored dim.
|
||||
* DESTRUCTIVE — existing vectors are discarded; the caller re-embeds
|
||||
* afterward (e.g. reconcileVecIndex over all frames + rechunkAllFrames for
|
||||
* chunks). This is the remediation for an EmbeddingDimMismatchError.
|
||||
*
|
||||
* memory_frame_chunks CONTENT rows deliberately survive (OSS behavior):
|
||||
* they're derived text, not vectors — re-deriving them is rechunkAllFrames'
|
||||
* job, and an empty chunks_vec makes vectorSearchChunks return no rows so
|
||||
* stale chunk rows are inert until re-embedded.
|
||||
*/
|
||||
recreateVecTables(dim: number): void {
|
||||
const d = Math.trunc(dim);
|
||||
const tx = this.db.transaction(() => {
|
||||
this.db.exec(
|
||||
'DROP TABLE IF EXISTS memory_frames_vec; DROP TABLE IF EXISTS memory_frame_chunks_vec;'
|
||||
);
|
||||
this.db.exec(vecTableSqlForDim(d));
|
||||
this.db.exec(chunksVecTableSqlForDim(d));
|
||||
this.setMeta('embedding_dim', String(d));
|
||||
});
|
||||
tx();
|
||||
}
|
||||
|
||||
getDatabase(): DatabaseType {
|
||||
return this.db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a write closure, retrying on transient cross-process contention
|
||||
* (SQLITE_BUSY / SQLITE_BUSY_SNAPSHOT) with bounded, growing backoff. The
|
||||
* busy_timeout pragma already covers plain lock waits; this adds the WAL
|
||||
* snapshot-upgrade race it cannot. Non-BUSY errors propagate immediately; after
|
||||
* the attempt budget is exhausted the last BUSY error is rethrown.
|
||||
*
|
||||
* Centralized so a caller wraps the OUTERMOST write (a whole `db.transaction`)
|
||||
* exactly ONCE. Do NOT wrap a statement nested inside an ambient transaction: a
|
||||
* retry there cannot obtain a fresh snapshot and would mask the real failure.
|
||||
*/
|
||||
runWithBusyRetry<T>(fn: () => T): T {
|
||||
let lastErr: unknown;
|
||||
for (let attempt = 0; attempt < BUSY_RETRY_MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
return fn();
|
||||
} catch (err: unknown) {
|
||||
if (!isSqliteBusyError(err)) throw err;
|
||||
lastErr = err;
|
||||
if (attempt < BUSY_RETRY_MAX_ATTEMPTS - 1) {
|
||||
sleepSync(BUSY_RETRY_BASE_DELAY_MS * (attempt + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
/**
|
||||
* True while the underlying better-sqlite3 handle is open. Used by
|
||||
* MultiMindCache's reopen-guard to detect a handle that was closed
|
||||
* out-of-band before handing it back.
|
||||
*/
|
||||
isOpen(): boolean {
|
||||
return this.db.open;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
537
packages/hive-mind-core/src/mind/embedding-provider.ts
Normal file
537
packages/hive-mind-core/src/mind/embedding-provider.ts
Normal file
@@ -0,0 +1,537 @@
|
||||
/**
|
||||
* EmbeddingProvider — orchestrates the InProcess → Ollama → API → Mock fallback chain.
|
||||
* Single entry point for all embedding operations in Waggle.
|
||||
* Implements the Embedder interface — drop-in replacement everywhere.
|
||||
*
|
||||
* Tier enforcement: provider selection is gated by TIER_CAPABILITIES.embeddingProviders.
|
||||
* Quota enforcement: monthly embed count tracked in embedding_usage table.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import { type Tier, TIERS, TIER_CAPABILITIES, TierError } from '@waggle/shared';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('embedding');
|
||||
|
||||
export type EmbeddingProviderType = 'inprocess' | 'ollama' | 'voyage' | 'openai' | 'litellm' | 'mock';
|
||||
|
||||
export interface EmbeddingProviderConfig {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
targetDimensions?: number;
|
||||
/** User tier — gates which providers are available and monthly quota. Defaults to SOLO. */
|
||||
userTier?: Tier;
|
||||
/** User ID for quota tracking. Defaults to 'local'. */
|
||||
userId?: string;
|
||||
/** Raw SQLite database for quota tracking. Optional — quota not enforced without it. */
|
||||
quotaDb?: DatabaseType;
|
||||
inprocess?: { model?: string; cacheDir?: string };
|
||||
ollama?: { baseUrl?: string; model?: string };
|
||||
voyage?: { apiKey: string; model?: string };
|
||||
openai?: { apiKey: string; model?: string };
|
||||
litellm?: { url: string; apiKey?: string; model?: string };
|
||||
}
|
||||
|
||||
// ── Tier enforcement helpers ──────────────────────────────────────────
|
||||
|
||||
/** Find the lowest tier that allows a given embedding provider. */
|
||||
export function getMinimumTierForProvider(provider: EmbeddingProviderType): Tier {
|
||||
for (const tier of TIERS) {
|
||||
const allowed = TIER_CAPABILITIES[tier].embeddingProviders as readonly string[];
|
||||
if (allowed.includes(provider)) return tier;
|
||||
}
|
||||
return 'ENTERPRISE';
|
||||
}
|
||||
|
||||
// ── Quota tracking ────────────────────────────────────────────────────
|
||||
|
||||
const EMBEDDING_USAGE_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS embedding_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
year_month TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at INTEGER NOT NULL,
|
||||
UNIQUE(user_id, year_month)
|
||||
);
|
||||
`;
|
||||
|
||||
function ensureQuotaTable(db: DatabaseType): void {
|
||||
try { db.exec(EMBEDDING_USAGE_SCHEMA); } catch { /* table may already exist */ }
|
||||
}
|
||||
|
||||
function getCurrentYearMonth(): string {
|
||||
const d = new Date();
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function getUsageCount(db: DatabaseType, userId: string, yearMonth: string): number {
|
||||
const row = db.prepare(
|
||||
'SELECT count FROM embedding_usage WHERE user_id = ? AND year_month = ?'
|
||||
).get(userId, yearMonth) as { count: number } | undefined;
|
||||
return row?.count ?? 0;
|
||||
}
|
||||
|
||||
function incrementUsage(db: DatabaseType, userId: string, yearMonth: string, amount: number): void {
|
||||
db.prepare(`
|
||||
INSERT INTO embedding_usage (user_id, year_month, count, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, year_month) DO UPDATE SET count = count + ?, updated_at = ?
|
||||
`).run(userId, yearMonth, amount, Date.now(), amount, Date.now());
|
||||
}
|
||||
|
||||
export class EmbeddingQuotaExceededError extends Error {
|
||||
public readonly tier: Tier;
|
||||
public readonly quota: number;
|
||||
public readonly current: number;
|
||||
public readonly upgradeUrl = 'https://waggle-os.ai/upgrade';
|
||||
|
||||
constructor(tier: Tier, quota: number, current: number) {
|
||||
super(`Embedding quota exceeded: ${current}/${quota} for ${tier} tier`);
|
||||
this.name = 'EmbeddingQuotaExceededError';
|
||||
this.tier = tier;
|
||||
this.quota = quota;
|
||||
this.current = current;
|
||||
}
|
||||
}
|
||||
|
||||
export interface EmbeddingQuotaStatus {
|
||||
tier: Tier;
|
||||
quota: number;
|
||||
used: number;
|
||||
remaining: number;
|
||||
percentage: number;
|
||||
resetsAt: string;
|
||||
}
|
||||
|
||||
function getNextMonthReset(): string {
|
||||
const d = new Date();
|
||||
d.setMonth(d.getMonth() + 1, 1);
|
||||
d.setHours(0, 0, 0, 0);
|
||||
return d.toISOString();
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderStatus {
|
||||
activeProvider: EmbeddingProviderType;
|
||||
availableProviders: EmbeddingProviderType[];
|
||||
dimensions: number;
|
||||
modelName: string;
|
||||
lastError?: string;
|
||||
probeTimestamp: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingProviderInstance extends Embedder {
|
||||
getStatus(): EmbeddingProviderStatus;
|
||||
getActiveProvider(): EmbeddingProviderType;
|
||||
reprobe(): Promise<EmbeddingProviderStatus>;
|
||||
/** Get current quota status for the user. Returns unlimited values if no quotaDb configured. */
|
||||
getQuotaStatus(): EmbeddingQuotaStatus;
|
||||
}
|
||||
|
||||
/** Deterministic mock — last resort, semantically meaningless. */
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
function createMockEmbedder(dims: number): Embedder {
|
||||
return {
|
||||
dimensions: dims,
|
||||
async embed(text: string) { return mockEmbed(text, dims); },
|
||||
async embedBatch(texts: string[]) { return texts.map(t => mockEmbed(t, dims)); },
|
||||
};
|
||||
}
|
||||
|
||||
// ── Embed-input guards (oversized-frame truncation + skip-not-abort) ──
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
|
||||
/**
|
||||
* Per-input character cap for embedding. `nomic-embed-text` has a 2048-token
|
||||
* (~6K char dense English) default context. Embedding an input longer than
|
||||
* the backend's context makes the backend reject the request, so we cap here.
|
||||
*
|
||||
* D1 probe finding (2026-06-12): model NAMES lie about context. A custom
|
||||
* `nomic-embed-text-8k` (num_ctx 8192) still 400s at its nomic-bert
|
||||
* ARCHITECTURE limit of 2048 tokens — the OSS heuristic's 24K-char branch
|
||||
* for `*-8k` names sent every long frame down the mock-fallback path. The
|
||||
* `-8k` branch is therefore capped at 8K chars (≈2048 prose tokens): safe
|
||||
* for the architecture-limited reality, merely conservative for a genuine
|
||||
* 8192-token embedder. reembedPerText remains the backstop for token-dense
|
||||
* content that still exceeds the backend's real window.
|
||||
*/
|
||||
export function maxEmbedCharsForModel(modelName: string): number {
|
||||
return /(-|_|\.)8k\b|num_ctx[^0-9]*8192/i.test(modelName) ? 8_000 : 6_000;
|
||||
}
|
||||
|
||||
/** Clamp a single input to `maxChars` (no-op when already under the cap). */
|
||||
export function capEmbedText(text: string, maxChars: number): string {
|
||||
return text.length > maxChars ? text.slice(0, maxChars) : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-embed a batch one text at a time, degrading ONLY the inputs that genuinely
|
||||
* fail to a deterministic mock vector. This is the batch-error recovery path:
|
||||
* a single backend-rejected text can no longer poison its batchmates (the prior
|
||||
* behavior substituted mock for the WHOLE batch — silent corruption of every
|
||||
* frame in the batch). Inputs should already be char-capped by the caller.
|
||||
*/
|
||||
export async function reembedPerText(
|
||||
embedder: Embedder,
|
||||
texts: string[],
|
||||
dims: number,
|
||||
): Promise<Float32Array[]> {
|
||||
return Promise.all(
|
||||
texts.map(async (t) => {
|
||||
try {
|
||||
return await embedder.embed(t);
|
||||
} catch {
|
||||
return mockEmbed(t, dims);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
interface ProbeResult {
|
||||
type: EmbeddingProviderType;
|
||||
embedder: Embedder;
|
||||
modelName: string;
|
||||
}
|
||||
|
||||
async function probeProvider(
|
||||
type: EmbeddingProviderType,
|
||||
config: EmbeddingProviderConfig,
|
||||
): Promise<ProbeResult | null> {
|
||||
const dims = config.targetDimensions ?? 1024;
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'inprocess': {
|
||||
const { createInProcessEmbedder } = await import('./inprocess-embedder.js');
|
||||
const embedder = await createInProcessEmbedder({
|
||||
model: config.inprocess?.model,
|
||||
cacheDir: config.inprocess?.cacheDir,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'inprocess', embedder, modelName: config.inprocess?.model ?? 'Xenova/all-MiniLM-L6-v2' };
|
||||
}
|
||||
|
||||
case 'ollama': {
|
||||
const { createOllamaEmbedder } = await import('./ollama-embedder.js');
|
||||
const embedder = createOllamaEmbedder({
|
||||
baseUrl: config.ollama?.baseUrl,
|
||||
model: config.ollama?.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'ollama', embedder, modelName: config.ollama?.model ?? 'nomic-embed-text' };
|
||||
}
|
||||
|
||||
case 'voyage': {
|
||||
if (!config.voyage?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'voyage',
|
||||
apiKey: config.voyage.apiKey,
|
||||
model: config.voyage.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'voyage', embedder, modelName: config.voyage.model ?? 'voyage-3-lite' };
|
||||
}
|
||||
|
||||
case 'openai': {
|
||||
if (!config.openai?.apiKey) return null;
|
||||
const { createApiEmbedder } = await import('./api-embedder.js');
|
||||
const embedder = createApiEmbedder({
|
||||
provider: 'openai',
|
||||
apiKey: config.openai.apiKey,
|
||||
model: config.openai.model,
|
||||
targetDimensions: dims,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'openai', embedder, modelName: config.openai.model ?? 'text-embedding-3-small' };
|
||||
}
|
||||
|
||||
case 'litellm': {
|
||||
if (!config.litellm?.url) return null;
|
||||
const { createLiteLLMEmbedder } = await import('./litellm-embedder.js');
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
litellmUrl: config.litellm.url,
|
||||
litellmApiKey: config.litellm.apiKey,
|
||||
model: config.litellm.model ?? 'text-embedding',
|
||||
dimensions: dims,
|
||||
fallbackToMock: false,
|
||||
});
|
||||
const test = await embedder.embed('waggle embedding probe');
|
||||
if (test.length !== dims) throw new Error(`Unexpected dims: ${test.length}`);
|
||||
return { type: 'litellm', embedder, modelName: config.litellm.model ?? 'text-embedding' };
|
||||
}
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log.info(`Trying ${type}... FAILED (${msg})`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createEmbeddingProvider(config?: EmbeddingProviderConfig): Promise<EmbeddingProviderInstance> {
|
||||
const cfg: EmbeddingProviderConfig = { provider: 'auto', targetDimensions: 1024, ...config };
|
||||
const dims = cfg.targetDimensions ?? 1024;
|
||||
const userTier: Tier = cfg.userTier ?? 'FREE';
|
||||
const userId = cfg.userId ?? 'local';
|
||||
const quotaDb = cfg.quotaDb ?? null;
|
||||
// Tier enforcement is normally ON only when userTier is explicitly passed.
|
||||
// WAGGLE_EVAL_MODE=1 disables it unconditionally — the PA v5 eval harness
|
||||
// sets this so user-tier gates never confound measurement validity. This
|
||||
// env-var is eval-path-only; never set it in production code paths.
|
||||
// See PromptAssembler v5 brief §11.3.
|
||||
const evalModeActive = process.env.WAGGLE_EVAL_MODE === '1';
|
||||
const tierEnforced = evalModeActive ? false : cfg.userTier !== undefined;
|
||||
const tierCaps = TIER_CAPABILITIES[userTier];
|
||||
const allowedProviders = tierCaps.embeddingProviders as readonly string[];
|
||||
|
||||
// Initialize quota table if DB provided
|
||||
if (quotaDb) {
|
||||
ensureQuotaTable(quotaDb);
|
||||
}
|
||||
|
||||
let activeResult: ProbeResult | null = null;
|
||||
let activeEmbedder: Embedder;
|
||||
let activeType: EmbeddingProviderType = 'mock';
|
||||
let activeModelName = 'deterministic-mock';
|
||||
let lastError: string | undefined;
|
||||
let availableProviders: EmbeddingProviderType[] = [];
|
||||
let probeTimestamp = new Date().toISOString();
|
||||
|
||||
/** Check quota before embedding. Throws if exceeded. Warns at 80%. */
|
||||
function checkQuota(count: number): void {
|
||||
if (!quotaDb || !tierEnforced) return;
|
||||
const quota = tierCaps.embeddingQuotaPerMonth;
|
||||
if (quota === -1) return; // unlimited
|
||||
const ym = getCurrentYearMonth();
|
||||
const used = getUsageCount(quotaDb, userId, ym);
|
||||
if (used + count > quota) {
|
||||
throw new EmbeddingQuotaExceededError(userTier, quota, used);
|
||||
}
|
||||
if (used + count >= quota * 0.8) {
|
||||
log.warn(`Embedding quota warning: ${used + count}/${quota} (${Math.round(((used + count) / quota) * 100)}%) for ${userTier} tier`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Record usage after successful embedding. */
|
||||
function recordUsage(count: number): void {
|
||||
if (!quotaDb) return;
|
||||
incrementUsage(quotaDb, userId, getCurrentYearMonth(), count);
|
||||
}
|
||||
|
||||
async function runProbe(): Promise<void> {
|
||||
log.info('Probing embedding providers...');
|
||||
const available: EmbeddingProviderType[] = [];
|
||||
activeResult = null;
|
||||
probeTimestamp = new Date().toISOString();
|
||||
|
||||
const requestedProvider = cfg.provider ?? 'auto';
|
||||
|
||||
if (requestedProvider !== 'auto' && requestedProvider !== 'mock') {
|
||||
// Explicit provider — tier-check only when tier is explicitly configured
|
||||
if (tierEnforced && !allowedProviders.includes(requestedProvider)) {
|
||||
const required = getMinimumTierForProvider(requestedProvider);
|
||||
throw new TierError(required, userTier);
|
||||
}
|
||||
log.info(`Trying ${requestedProvider}...`);
|
||||
const result = await probeProvider(requestedProvider, cfg);
|
||||
if (result) {
|
||||
activeResult = result;
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${requestedProvider}... OK`);
|
||||
}
|
||||
} else if (requestedProvider === 'auto') {
|
||||
// Auto: iterate chain, skip providers not allowed by tier
|
||||
const chain: EmbeddingProviderType[] = ['inprocess', 'ollama', 'voyage', 'openai'];
|
||||
|
||||
for (const providerType of chain) {
|
||||
// Tier gate — skip providers not allowed (only when tier is explicitly configured)
|
||||
if (tierEnforced && !allowedProviders.includes(providerType)) {
|
||||
log.info(`Skipping ${providerType} (not available on ${userTier} tier)`);
|
||||
continue;
|
||||
}
|
||||
// Skip API providers without keys
|
||||
if (providerType === 'voyage' && !cfg.voyage?.apiKey) {
|
||||
log.info('Skipping voyage (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
if (providerType === 'openai' && !cfg.openai?.apiKey) {
|
||||
log.info('Skipping openai (no API key in Vault)');
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(`Trying ${providerType}...`);
|
||||
const result = await probeProvider(providerType, cfg);
|
||||
if (result) {
|
||||
available.push(result.type);
|
||||
log.info(`Trying ${providerType}... OK`);
|
||||
if (!activeResult) {
|
||||
activeResult = result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
available.push('mock'); // Always available
|
||||
availableProviders = available;
|
||||
|
||||
if (activeResult) {
|
||||
activeEmbedder = activeResult.embedder;
|
||||
activeType = activeResult.type;
|
||||
activeModelName = activeResult.modelName;
|
||||
lastError = undefined;
|
||||
log.info(`Embedding provider: ${activeType} (${activeModelName}, ${dims} dims)`);
|
||||
} else {
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
lastError = 'No real providers available';
|
||||
// Loud, structured warning — the silent "mock fallback" was the
|
||||
// most dangerous failure mode in Phase 3b-3 audit. Mock embeddings
|
||||
// are deterministic byte hashes; semantic search returns noise.
|
||||
// We want this to be IMPOSSIBLE to miss in a CLI/server log.
|
||||
// Ported from hive-mind a99ea0e.
|
||||
const msg = [
|
||||
'',
|
||||
'⚠️ EMBEDDING WARNING ─────────────────────────────────────────',
|
||||
' Active provider: mock (deterministic byte hash)',
|
||||
' Effect: semantic search returns noise, not meaning.',
|
||||
'',
|
||||
' To fix, install Ollama and pull the embedding model:',
|
||||
' ollama pull nomic-embed-text',
|
||||
' Then ensure the process can reach http://localhost:11434.',
|
||||
'',
|
||||
' Alternative providers:',
|
||||
' HIVE_MIND_EMBEDDING_PROVIDER=inprocess (downloads 23MB)',
|
||||
' VOYAGE_API_KEY=... (paid, recommended)',
|
||||
' OPENAI_API_KEY=... (paid)',
|
||||
'─────────────────────────────────────────────────────────────',
|
||||
'',
|
||||
].join('\n');
|
||||
// Keep production and CLI runs loud, but let deterministic test lanes
|
||||
// suppress this expected fallback banner without changing provider state.
|
||||
if (process.env.WAGGLE_SUPPRESS_EMBEDDING_WARNING !== '1') {
|
||||
// stderr so it survives stdout-piped JSON consumers and CI tee.
|
||||
try { process.stderr.write(msg); } catch { /* fall through to log */ }
|
||||
log.warn('Embedding provider degraded to mock — semantic search quality is noise. See stderr banner for fix instructions.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initial probe
|
||||
try {
|
||||
await runProbe();
|
||||
} catch (err) {
|
||||
// Re-throw tier errors — these are intentional enforcement, not probe failures
|
||||
if (err instanceof TierError) throw err;
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
activeEmbedder = createMockEmbedder(dims);
|
||||
activeType = 'mock';
|
||||
activeModelName = 'deterministic-mock';
|
||||
availableProviders = ['mock'];
|
||||
}
|
||||
|
||||
// Ensure activeEmbedder is assigned (TypeScript flow)
|
||||
activeEmbedder ??= createMockEmbedder(dims);
|
||||
|
||||
const instance: EmbeddingProviderInstance = {
|
||||
dimensions: dims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
checkQuota(1);
|
||||
// Cap input to the active model's context so the backend never rejects
|
||||
// an oversized frame. Reverse-ported from OSS hive-mind (oss-drift triage R5, 2026-06-11).
|
||||
const capped = capEmbedText(text, maxEmbedCharsForModel(activeModelName));
|
||||
try {
|
||||
const result = await activeEmbedder.embed(capped);
|
||||
recordUsage(1);
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (err instanceof EmbeddingQuotaExceededError) throw err;
|
||||
log.warn(`Embedding failed with ${activeType}, falling back to mock: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const fallback = mockEmbed(capped, dims);
|
||||
recordUsage(1);
|
||||
return fallback;
|
||||
}
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
checkQuota(texts.length);
|
||||
// Cap each input first so one oversized frame can't make the backend
|
||||
// reject the whole request. Reverse-ported from OSS hive-mind
|
||||
// (oss-drift triage R5, 2026-06-11).
|
||||
const capped = texts.map(t => capEmbedText(t, maxEmbedCharsForModel(activeModelName)));
|
||||
try {
|
||||
const result = await activeEmbedder.embedBatch(capped);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (err instanceof EmbeddingQuotaExceededError) throw err;
|
||||
// Skip-not-abort: re-embed per-text so a single backend-rejected input
|
||||
// degrades alone instead of mock-poisoning the WHOLE batch.
|
||||
log.warn(`Batch embedding failed with ${activeType}, re-embedding per-text: ${(err as Error).message}`);
|
||||
lastError = (err as Error).message;
|
||||
const result = await reembedPerText(activeEmbedder, capped, dims);
|
||||
recordUsage(texts.length);
|
||||
return result;
|
||||
}
|
||||
},
|
||||
|
||||
getStatus(): EmbeddingProviderStatus {
|
||||
return {
|
||||
activeProvider: activeType,
|
||||
availableProviders,
|
||||
dimensions: dims,
|
||||
modelName: activeModelName,
|
||||
lastError,
|
||||
probeTimestamp,
|
||||
};
|
||||
},
|
||||
|
||||
getActiveProvider(): EmbeddingProviderType {
|
||||
return activeType;
|
||||
},
|
||||
|
||||
async reprobe(): Promise<EmbeddingProviderStatus> {
|
||||
await runProbe();
|
||||
return instance.getStatus();
|
||||
},
|
||||
|
||||
getQuotaStatus(): EmbeddingQuotaStatus {
|
||||
const quota = tierCaps.embeddingQuotaPerMonth;
|
||||
if (!quotaDb || quota === -1) {
|
||||
return { tier: userTier, quota: -1, used: 0, remaining: -1, percentage: 0, resetsAt: getNextMonthReset() };
|
||||
}
|
||||
const used = getUsageCount(quotaDb, userId, getCurrentYearMonth());
|
||||
return {
|
||||
tier: userTier,
|
||||
quota,
|
||||
used,
|
||||
remaining: Math.max(0, quota - used),
|
||||
percentage: Math.round((used / quota) * 100),
|
||||
resetsAt: getNextMonthReset(),
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
return instance;
|
||||
}
|
||||
5
packages/hive-mind-core/src/mind/embeddings.ts
Normal file
5
packages/hive-mind-core/src/mind/embeddings.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export interface Embedder {
|
||||
embed(text: string): Promise<Float32Array>;
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
||||
dimensions: number;
|
||||
}
|
||||
112
packages/hive-mind-core/src/mind/entity-normalizer.ts
Normal file
112
packages/hive-mind-core/src/mind/entity-normalizer.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
const ALIASES: string[][] = [
|
||||
['postgresql', 'postgres', 'pg'],
|
||||
['javascript', 'js'],
|
||||
['typescript', 'ts'],
|
||||
['kubernetes', 'k8s'],
|
||||
['new york city', 'nyc'],
|
||||
['nodejs', 'node.js', 'node'],
|
||||
['react.js', 'reactjs', 'react'],
|
||||
['vue.js', 'vuejs', 'vue'],
|
||||
['python', 'py'],
|
||||
['mongodb', 'mongo'],
|
||||
];
|
||||
|
||||
const aliasMap = new Map<string, string>();
|
||||
for (const group of ALIASES) {
|
||||
const canonical = group[0];
|
||||
for (const alias of group) {
|
||||
aliasMap.set(alias, canonical);
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeEntityName(name: string): string {
|
||||
const lower = name.toLowerCase();
|
||||
return aliasMap.get(lower) ?? lower;
|
||||
}
|
||||
|
||||
export interface EntityRef {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export function findDuplicates(entities: EntityRef[]): EntityRef[][] {
|
||||
const groups = new Map<string, EntityRef[]>();
|
||||
for (const entity of entities) {
|
||||
const key = `${normalizeEntityName(entity.name)}::${entity.type.toLowerCase()}`;
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = [];
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.push(entity);
|
||||
}
|
||||
return Array.from(groups.values());
|
||||
}
|
||||
|
||||
// ── Write-time noise filter ───────────────────────────────────────────────
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R3, 2026-06-11).
|
||||
// Applied at extraction time so low-signal names never enter the knowledge
|
||||
// graph instead of being purged after the fact. Wired into
|
||||
// harvest/extract-kg-entities.ts (oss-drift D2, 2026-06-11).
|
||||
|
||||
/** Capitalized sentence-starts / verbs / weekday + month tokens that are
|
||||
* formatting artefacts, not entities. Title-cased to match extractor output. */
|
||||
const STOP_TOKENS = new Set<string>([
|
||||
// sentence-starts and pronouns
|
||||
'The', 'This', 'That', 'These', 'Those', 'When', 'Where', 'Why', 'How',
|
||||
'What', 'Who', 'Which', 'If', 'And', 'But', 'Or', 'So', 'For', 'Nor',
|
||||
'Yet', 'As', 'At', 'By', 'On', 'In', 'To', 'From', 'With', 'Without',
|
||||
'Into', 'Onto', 'Upon', 'Over', 'Under', 'Between', 'Among',
|
||||
// verbs commonly capitalized at sentence start / in API names / log prefixes
|
||||
'Add', 'Remove', 'Set', 'Get', 'Update', 'Delete', 'Create', 'List',
|
||||
'Search', 'Find', 'Run', 'Build', 'Use', 'Make', 'Test', 'Check',
|
||||
'Read', 'Write', 'Edit', 'Save', 'Load', 'Open', 'Close', 'Start',
|
||||
'Stop', 'Show', 'Hide', 'Push', 'Pull', 'Fix', 'Done', 'Skip',
|
||||
'Wait', 'Try', 'Note', 'Warn', 'Info', 'Debug', 'Trace',
|
||||
'Todo', 'Fixme', 'Should', 'Could', 'Would', 'Must', 'Will', 'Shall',
|
||||
'Can', 'May', 'Might',
|
||||
// days
|
||||
'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun',
|
||||
'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday',
|
||||
// months
|
||||
'Jan', 'Feb', 'Mar', 'Apr', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
|
||||
'Nov', 'Dec',
|
||||
'January', 'February', 'March', 'April', 'June', 'July',
|
||||
'August', 'September', 'October', 'November', 'December',
|
||||
]);
|
||||
|
||||
/** Real short tech names the generic heuristics would wrongly drop (too short,
|
||||
* or all-caps acronyms). Lowercased; checked case-insensitively. */
|
||||
const TECH_ALLOWLIST = new Set<string>([
|
||||
// languages / runtimes
|
||||
'go', 'php', 'bun', 'deno',
|
||||
// package managers / editors / tools
|
||||
'npm', 'pip', 'gem', 'vim', 'git',
|
||||
// frameworks / libraries
|
||||
'vue', 'zod', 'nuxt', 'vite', 'hono',
|
||||
// domains / concepts that are genuine entities
|
||||
'ai', 'ml', 'db', 'os', 'ui', 'ux', 'io', 'ci', 'cd', 'k8s',
|
||||
'sdk', 'orm', 'jwt', 'ssh', 'dns', 'gpu', 'cpu',
|
||||
]);
|
||||
|
||||
/** All-caps tokens up to 6 chars (API, CLI, JSON, HTTP, SQL, AWS, URL, UUID) —
|
||||
* almost always formatting artefacts, not subjects. */
|
||||
export function isLikelyAcronym(s: string): boolean {
|
||||
return /^[A-Z]+$/.test(s) && s.length <= 6;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `name` is too low-signal to enter the knowledge graph: empty,
|
||||
* shorter than 4 chars, a stop token, or a single-word all-caps acronym —
|
||||
* UNLESS it is on the tech allowlist (npm, Go, AI, …). Multi-word names skip
|
||||
* the acronym filter (real entities like "Acme Corp").
|
||||
*/
|
||||
export function isNoiseName(name: string): boolean {
|
||||
if (!name) return true;
|
||||
if (TECH_ALLOWLIST.has(name.toLowerCase())) return false;
|
||||
if (name.length < 4) return true;
|
||||
if (STOP_TOKENS.has(name)) return true;
|
||||
if (!/\s/.test(name) && isLikelyAcronym(name)) return true;
|
||||
return false;
|
||||
}
|
||||
403
packages/hive-mind-core/src/mind/erasure.ts
Normal file
403
packages/hive-mind-core/src/mind/erasure.ts
Normal file
@@ -0,0 +1,403 @@
|
||||
/**
|
||||
* erasure.ts — GDPR Art.17 frame + index + KG erasure companion (2026-07-01).
|
||||
*
|
||||
* The #7 raw_archive erasure (RawArchive.erase / eraseByFrame) redacts ONLY the
|
||||
* verbatim provenance rows. This module completes a data-subject erasure by also
|
||||
* purging the DERIVED retrieval corpus that quotes the source PII:
|
||||
* - memory_frames + memory_frames_fts + memory_frames_vec
|
||||
* - memory_frame_chunks + memory_frame_chunks_vec (via FrameStore.delete)
|
||||
* - orphaned knowledge_entities + knowledge_relations (name/props may hold PII)
|
||||
* while KEEPING the raw_archive identity skeleton as the audit record that an
|
||||
* item existed and was erased. See docs/plans/2026-07-01-art17-frame-index-kg-
|
||||
* erasure-companion.md for the full surface + decisions.
|
||||
*
|
||||
* Every multi-table erasure runs in ONE better-sqlite3 transaction — a partial
|
||||
* erasure is a compliance failure (all-or-nothing).
|
||||
*
|
||||
* archive_uid = sha256(source∥sourceRef∥content) is ROTATED to an opaque id on erase
|
||||
* (raw-archive.ts erase()), so the retained audit skeleton carries no content-derived
|
||||
* value — the low-entropy re-identification residual is closed. (source_ref, preserved
|
||||
* verbatim, is the one remaining retained-skeleton residual and MAY carry PII.)
|
||||
*/
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import { FrameStore } from './frames.js';
|
||||
import { RawArchive, readArchiveUids } from './raw-archive.js';
|
||||
import { SuppressionStore } from './suppression.js';
|
||||
import { MIND_RAWTURN_PREFIX, rawTurnConvKey } from '../harvest/raw-turns.js';
|
||||
import { CLAUDE_CODE_DECISION_SOURCE, decisionOfSubjectId } from '../harvest/decision-derivation.js';
|
||||
|
||||
export interface EraseResult {
|
||||
/** Derived frames physically deleted (frame + FTS + vec + chunks + chunk-vec). */
|
||||
framesDeleted: number;
|
||||
/** raw_archive provenance rows redacted (content → marker; skeleton frozen). */
|
||||
archiveRedacted: number;
|
||||
/** memory_frame_chunks_vec rows purged for the erased frame(s). */
|
||||
chunkVectorsPurged: number;
|
||||
/** Orphaned knowledge_entities hard-deleted (zero surviving frame links). */
|
||||
entitiesErased: number;
|
||||
/** knowledge_relations of those orphaned entities removed. */
|
||||
relationsErased: number;
|
||||
}
|
||||
|
||||
function zeroResult(): EraseResult {
|
||||
return { framesDeleted: 0, archiveRedacted: 0, chunkVectorsPurged: 0, entitiesErased: 0, relationsErased: 0 };
|
||||
}
|
||||
|
||||
export class MindErasure {
|
||||
private db: MindDB;
|
||||
private frames: FrameStore;
|
||||
private archive: RawArchive;
|
||||
private suppression: SuppressionStore;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.frames = new FrameStore(db);
|
||||
this.archive = new RawArchive(db);
|
||||
this.suppression = new SuppressionStore(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Erase ONE derived frame under GDPR Art.17: redact its provenance rows, delete
|
||||
* the frame from every retrieval store, then hard-delete any KG entity that was
|
||||
* derived solely from this frame (zero surviving links) along with its relations.
|
||||
* Atomic. Returns all-zero for an unknown frame id (no throw).
|
||||
*
|
||||
* SCOPE — a single frame. For a full data-subject erasure use eraseBySourceRef,
|
||||
* which also sweeps the conversation's verbatim raw-turn frames + referencing
|
||||
* B-frames that this frame-scoped primitive intentionally does NOT touch.
|
||||
*/
|
||||
eraseFrame(frameId: number, reason: string): EraseResult {
|
||||
return this.db.getDatabase().transaction((): EraseResult => this.eraseFrameInternal(frameId, reason))();
|
||||
}
|
||||
|
||||
/**
|
||||
* Art.17-COMPLETE erase of ONE frame the user pointed at. Unlike eraseFrame
|
||||
* (single frame), this reaches the WHOLE subject footprint behind a harvested
|
||||
* summary — the verbatim [mind-rawturn] dialogue + referencing B-frames + KG —
|
||||
* which a frame-only delete would leave recall-able. It resolves the frame's
|
||||
* provenance subjects (the archive link; else a metadata.sourceId + content
|
||||
* platform-prefix fallback for a legacy / append-failed frame with no link),
|
||||
* sweeps each via eraseBySourceRef, then erases the frame itself. Atomic
|
||||
* (better-sqlite3 nests the inner erasures as savepoints). All-zero for an
|
||||
* unknown frame. This is the single primitive both the /api/memory/erase route
|
||||
* and the erase_memory MCP tool call, so the two entry points cannot drift.
|
||||
*/
|
||||
eraseFrameComplete(frameId: number, reason: string): EraseResult {
|
||||
return this.db.getDatabase().transaction((): EraseResult => {
|
||||
const total = zeroResult();
|
||||
const add = (r: EraseResult): void => {
|
||||
total.framesDeleted += r.framesDeleted;
|
||||
total.archiveRedacted += r.archiveRedacted;
|
||||
total.chunkVectorsPurged += r.chunkVectorsPurged;
|
||||
total.entitiesErased += r.entitiesErased;
|
||||
total.relationsErased += r.relationsErased;
|
||||
};
|
||||
const frame = this.frames.getById(frameId);
|
||||
if (!frame) return total;
|
||||
|
||||
// Dedup subjects with a JSON-array key (collision-proof: distinct
|
||||
// (source, sourceRef) pairs never serialize equal).
|
||||
const seen = new Set<string>();
|
||||
const sweep = (source: string, sourceRef: string): void => {
|
||||
const key = JSON.stringify([source, sourceRef]);
|
||||
if (seen.has(key)) return;
|
||||
seen.add(key);
|
||||
add(this.eraseBySourceRef(source, sourceRef, reason));
|
||||
};
|
||||
// Primary: subjects linked via the frame's archive provenance.
|
||||
for (const row of this.archive.reconstructSource(frameId)) {
|
||||
if (row.source_ref) sweep(row.source, row.source_ref);
|
||||
}
|
||||
// Fallback: a harvested summary with NO archive link (legacy pre-#7 frame,
|
||||
// or a raw_archive.append that failed while the raw-turns still wrote).
|
||||
// Recover the subject from metadata.sourceId + the platform token in the
|
||||
// content prefix ('[Harvest:<src>] ...' server harvest / '[<src>] ...' MCP
|
||||
// harvest) so eraseBySourceRef reaches the raw-turns. A wrong guess on a
|
||||
// non-harvest frame matches nothing (a no-op sweep).
|
||||
if (seen.size === 0) {
|
||||
let sourceRef: string | undefined;
|
||||
try {
|
||||
const meta = JSON.parse(frame.metadata ?? '{}') as Record<string, unknown>;
|
||||
if (meta && typeof meta.sourceId === 'string') sourceRef = meta.sourceId;
|
||||
} catch { /* malformed metadata — no fallback subject */ }
|
||||
const src = frame.content?.match(/^\[(?:Harvest:)?([^\]]+)\]/)?.[1];
|
||||
if (src && sourceRef) sweep(src, sourceRef);
|
||||
}
|
||||
const frameRes = this.eraseFrame(frameId, reason); // idempotent if already swept
|
||||
add(frameRes);
|
||||
// A subject-less frame (connector / ingest_source single frame) resolved no
|
||||
// subject above, so the eraseBySourceRef B-frame sweep never ran for it.
|
||||
// Strip any B-frame that references it directly so synthesized PII cannot
|
||||
// survive. (For a subject frame this is a no-op — step 4 already swept them.)
|
||||
if (frameRes.framesDeleted > 0) add(this.sweepReferencingBFrames(new Set([frameId]), reason));
|
||||
return total;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Non-transactional core — call inside an ambient transaction only. */
|
||||
private eraseFrameInternal(frameId: number, reason: string): EraseResult {
|
||||
const raw = this.db.getDatabase();
|
||||
if (!this.frames.getById(frameId)) return zeroResult();
|
||||
|
||||
// Capture the entities linked to this frame BEFORE the delete cascades the
|
||||
// kg_entity_frames bridge away (else we can't tell which became orphans).
|
||||
const linkedEntityIds = (raw
|
||||
.prepare('SELECT entity_id FROM kg_entity_frames WHERE frame_id = ?')
|
||||
.all(frameId) as Array<{ entity_id: number }>).map(r => r.entity_id);
|
||||
|
||||
// Count chunk vectors that FrameStore.delete will purge (for the report).
|
||||
const chunkVectorsPurged = (raw
|
||||
.prepare('SELECT COUNT(*) c FROM memory_frame_chunks WHERE frame_id = ?')
|
||||
.get(frameId) as { c: number }).c;
|
||||
|
||||
// 1. Redact the provenance rows this frame links to (audit skeleton kept).
|
||||
const archiveRedacted = this.archive.eraseByFrame(frameId, reason);
|
||||
|
||||
// 2. Delete the frame + FTS + vec + chunks + chunk-vec + kg bridge.
|
||||
const framesDeleted = this.frames.delete(frameId) ? 1 : 0;
|
||||
|
||||
// 3. Orphan sweep: a previously-linked entity now at zero frame links was
|
||||
// derived solely from erased content → hard-delete it + its relations.
|
||||
// knowledge_relations references knowledge_entities WITHOUT ON DELETE
|
||||
// CASCADE (FK enforcement is ON), so relations MUST go first or the entity
|
||||
// delete raises SQLITE_CONSTRAINT.
|
||||
let entitiesErased = 0;
|
||||
let relationsErased = 0;
|
||||
for (const eid of linkedEntityIds) {
|
||||
const remaining = (raw
|
||||
.prepare('SELECT COUNT(*) c FROM kg_entity_frames WHERE entity_id = ?')
|
||||
.get(eid) as { c: number }).c;
|
||||
if (remaining > 0) continue; // still referenced by a surviving frame — shared, keep
|
||||
relationsErased += raw
|
||||
.prepare('DELETE FROM knowledge_relations WHERE source_id = ? OR target_id = ?')
|
||||
.run(eid, eid).changes;
|
||||
entitiesErased += raw
|
||||
.prepare('DELETE FROM knowledge_entities WHERE id = ?')
|
||||
.run(eid).changes;
|
||||
}
|
||||
|
||||
return { framesDeleted, archiveRedacted, chunkVectorsPurged, entitiesErased, relationsErased };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixpoint sweep of every B-frame that (transitively) references an already-
|
||||
* erased frame. A synthesized B-frame stores {references:[…]} in its content
|
||||
* JSON and carries no archiveUids, so the summary/raw-turn sweeps cannot reach
|
||||
* it. Shared by the subject sweep (eraseBySourceRef step 4) and the single-frame
|
||||
* erase (eraseFrameComplete) — the latter for SUBJECT-LESS frames (connector /
|
||||
* ingest_source single frames) that resolve no subject and so would otherwise
|
||||
* leave a referencing B-frame (which can quote the erased PII) behind. Fixpoint:
|
||||
* a B-frame may reference another B-frame; erased ones vanish from the next query
|
||||
* so it terminates. Mutates `erasedIds` with the swept B-frame ids.
|
||||
* Non-transactional — call inside an ambient transaction only.
|
||||
*/
|
||||
private sweepReferencingBFrames(erasedIds: Set<number>, reason: string): EraseResult {
|
||||
const raw = this.db.getDatabase();
|
||||
const total = zeroResult();
|
||||
const add = (r: EraseResult): void => {
|
||||
total.framesDeleted += r.framesDeleted;
|
||||
total.archiveRedacted += r.archiveRedacted;
|
||||
total.chunkVectorsPurged += r.chunkVectorsPurged;
|
||||
total.entitiesErased += r.entitiesErased;
|
||||
total.relationsErased += r.relationsErased;
|
||||
};
|
||||
let grew = true;
|
||||
while (grew) {
|
||||
grew = false;
|
||||
const bframes = raw
|
||||
.prepare("SELECT id, content FROM memory_frames WHERE frame_type = 'B'")
|
||||
.all() as Array<{ id: number; content: string }>;
|
||||
for (const bf of bframes) {
|
||||
if (erasedIds.has(bf.id)) continue;
|
||||
let refs: unknown;
|
||||
try { refs = (JSON.parse(bf.content) as { references?: unknown }).references; } catch { continue; }
|
||||
if (!Array.isArray(refs)) continue;
|
||||
if (refs.some((id) => typeof id === 'number' && erasedIds.has(id))) {
|
||||
const r = this.eraseFrameInternal(bf.id, reason);
|
||||
if (r.framesDeleted > 0) { erasedIds.add(bf.id); grew = true; add(r); }
|
||||
}
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subject-level sweep + suppression for a (source, source_ref) subject.
|
||||
*
|
||||
* Erases the subject's whole footprint (eraseSubjectFrames), records it on the
|
||||
* erased-subject suppression list, then — for claude-code only — cascades to the
|
||||
* DERIVED 'decision-of' subject that harvest fans out separately (see below).
|
||||
* The whole operation is ONE transaction, so a rolled-back erase records nothing
|
||||
* and the parent + derived subjects are all-or-nothing.
|
||||
*
|
||||
* This is the SINGLE suppression capture point — subject mode calls here directly;
|
||||
* frame mode reaches it via eraseFrameComplete's sweep() → both surfaces (route +
|
||||
* MCP erase_memory) feed the list with no drift.
|
||||
*/
|
||||
eraseBySourceRef(source: string, sourceRef: string, reason: string): EraseResult {
|
||||
return this.db.getDatabase().transaction((): EraseResult => {
|
||||
const total = zeroResult();
|
||||
const add = (r: EraseResult): void => {
|
||||
total.framesDeleted += r.framesDeleted;
|
||||
total.archiveRedacted += r.archiveRedacted;
|
||||
total.chunkVectorsPurged += r.chunkVectorsPurged;
|
||||
total.entitiesErased += r.entitiesErased;
|
||||
total.relationsErased += r.relationsErased;
|
||||
};
|
||||
|
||||
// Primary subject: erase its whole footprint, then suppress it.
|
||||
// UNCONDITIONAL — even when nothing currently matched, the subject was
|
||||
// EXPLICITLY requested erased, so a LATER re-export/re-sync must not
|
||||
// re-materialize it. (Subject-less frames correctly bypass this via the
|
||||
// frame-mode path, which resolves no subject to record.)
|
||||
add(this.eraseSubjectFrames(source, sourceRef, reason));
|
||||
this.suppression.record(source, sourceRef, reason);
|
||||
|
||||
// #7 P2 — claude-code `decision-of` derived subject. claude-code harvest's
|
||||
// extractDecisions emits a SEPARATE import item keyed on
|
||||
// stableHarvestId('claude-code','decision-of',parentRef) that quotes the
|
||||
// parent's decision lines. It lands as its OWN (source, source_ref) subject
|
||||
// — a distinct archiveUid/raw-turn key, and NOT a B-frame — so the sweep
|
||||
// above never reaches it: the derived frame would survive erasure AND (its
|
||||
// key being un-suppressed) re-materialize on the next re-import. The
|
||||
// persisted frame drops metadata.extractedFrom (harvest stamps only
|
||||
// kind/confidence/status/sourceId/archiveUids), so recompute the derived key
|
||||
// and cascade. SINGLE LEVEL: a decision item is never itself re-derived
|
||||
// (extractDecisions skips type==='decision'), so there is no decision-of-of
|
||||
// chain to follow. The derived suppression is recorded ONLY when the derived
|
||||
// subject had a real footprint at erase time — an unconditional record would
|
||||
// add a phantom, hash-keyed re-consent entry for every claude-code parent
|
||||
// that had no decisions. (Residual: a parent that GAINS decision content
|
||||
// after erasure and is re-harvested is not pre-suppressed on the derived key
|
||||
// — that content post-dates the erasure and is treated as new.)
|
||||
if (source === CLAUDE_CODE_DECISION_SOURCE) {
|
||||
const derivedRef = decisionOfSubjectId(sourceRef);
|
||||
const derived = this.eraseSubjectFrames(source, derivedRef, reason);
|
||||
add(derived);
|
||||
if (derived.framesDeleted > 0 || derived.archiveRedacted > 0) {
|
||||
this.suppression.record(source, derivedRef, reason);
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Erase everything derived from a (source, source_ref) subject — the sweep
|
||||
* MECHANICS only (no suppression record). It reaches the subject's derived
|
||||
* corpus through THREE keys, because one harvested item fans out into frames
|
||||
* that are keyed differently:
|
||||
* (a) the distilled SUMMARY frame — linked via metadata.archiveUids;
|
||||
* (b) the verbatim per-turn [mind-rawturn …] frames — keyed by the conversation
|
||||
* content prefix (= sanitize(source∥sourceRef)); they carry NO archive link,
|
||||
* so a link-only sweep would leave the subject's full dialogue recall-able;
|
||||
* (c) synthesized B-frames that reference any erased frame in their content JSON.
|
||||
* Then it redacts any subject provenance row no frame reached (orphan provenance).
|
||||
*
|
||||
* A frame that also links OTHER source_refs is still deleted wholesale (a merged
|
||||
* summary containing the subject's PII cannot be partially redacted) — its other
|
||||
* provenance rows are redacted too, which is the conservative Art.17 outcome.
|
||||
*
|
||||
* Non-transactional — call inside an ambient transaction only (eraseBySourceRef
|
||||
* wraps it so the primary + derived-subject sweeps commit atomically together).
|
||||
*/
|
||||
private eraseSubjectFrames(source: string, sourceRef: string, reason: string): EraseResult {
|
||||
const raw = this.db.getDatabase();
|
||||
const total = zeroResult();
|
||||
const add = (r: EraseResult): void => {
|
||||
total.framesDeleted += r.framesDeleted;
|
||||
total.archiveRedacted += r.archiveRedacted;
|
||||
total.chunkVectorsPurged += r.chunkVectorsPurged;
|
||||
total.entitiesErased += r.entitiesErased;
|
||||
total.relationsErased += r.relationsErased;
|
||||
};
|
||||
|
||||
// 1. Every archive uid for this subject.
|
||||
const uids = (raw
|
||||
.prepare('SELECT archive_uid FROM raw_archive WHERE source = ? AND source_ref = ?')
|
||||
.all(source, sourceRef) as Array<{ archive_uid: string }>).map(r => r.archive_uid);
|
||||
// NB: do NOT early-return on an empty uid set. A subject can have verbatim
|
||||
// [mind-rawturn] frames (2b) + referencing B-frames (4) with NO raw_archive
|
||||
// row — a legacy pre-#7 conversation, or one whose raw_archive.append failed
|
||||
// while the raw-turns still wrote. Bailing here left that raw PII dialogue
|
||||
// recall-able (the reference-class leak). Steps 2b/4 key off the conv-prefix
|
||||
// and content references, independent of raw_archive, so they must still run;
|
||||
// 2a and step 5 iterate `uids`, so they are natural no-ops when it is empty.
|
||||
const uidSet = new Set(uids);
|
||||
|
||||
const frameIds = new Set<number>();
|
||||
|
||||
// 2a. SUMMARY frames linking any subject uid (reverse lookup). LIKE-prefilter
|
||||
// the metadata JSON, then verify precisely via readArchiveUids (tolerates
|
||||
// the legacy scalar archiveUid + malformed metadata).
|
||||
const likeStmt = raw.prepare('SELECT id, metadata FROM memory_frames WHERE metadata LIKE ?');
|
||||
for (const uid of uids) {
|
||||
for (const row of likeStmt.all(`%${uid}%`) as Array<{ id: number; metadata?: string }>) {
|
||||
if (!row.metadata) continue;
|
||||
let meta: Record<string, unknown>;
|
||||
try { meta = JSON.parse(row.metadata) as Record<string, unknown>; } catch { continue; }
|
||||
if (!meta || typeof meta !== 'object') continue;
|
||||
if (readArchiveUids(meta).some(u => uidSet.has(u))) frameIds.add(row.id);
|
||||
}
|
||||
}
|
||||
|
||||
// 2b. Verbatim raw-turn frames for this conversation, keyed by content prefix.
|
||||
// The trailing space after the conv key makes the match EXACT (so a sweep
|
||||
// of 'item' never catches 'item-9'). Escape LIKE metacharacters (the key
|
||||
// is sanitized to [A-Za-z0-9_-] but escape defensively).
|
||||
const convKey = rawTurnConvKey({ source, id: sourceRef });
|
||||
const prefix = `${MIND_RAWTURN_PREFIX} conv:${convKey} `.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
for (const row of raw
|
||||
.prepare("SELECT id FROM memory_frames WHERE content LIKE ? ESCAPE '\\'")
|
||||
.all(`${prefix}%`) as Array<{ id: number }>) {
|
||||
frameIds.add(row.id);
|
||||
}
|
||||
|
||||
// 2c. Archive-less SUMMARY frames for this subject. 2a is archiveUid-keyed,
|
||||
// so a harvested summary whose raw_archive.append failed (or a legacy
|
||||
// pre-#7 frame) — carrying metadata.sourceId but NO archiveUids — slips
|
||||
// through, leaving the distilled PII recall-able after a subject-level
|
||||
// DSAR. Recover it symmetric to eraseFrameComplete's fallback: match
|
||||
// metadata.sourceId === source_ref AND the content platform-prefix
|
||||
// ('[Harvest:<src>] …' server / '[<src>] …' MCP) === source. The LIKE is
|
||||
// a prefilter only; the two EXACT code checks are the subject identity,
|
||||
// so 'item' never over-erases 'item-9' and a sibling subject is safe.
|
||||
// Escape LIKE metacharacters in source_ref (mirroring 2b) to keep the
|
||||
// prefilter narrow — the exact meta.sourceId check backstops either way.
|
||||
const srLike = sourceRef.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
const metaLike = raw.prepare("SELECT id, content, metadata FROM memory_frames WHERE metadata LIKE ? ESCAPE '\\'");
|
||||
for (const row of metaLike.all(`%${srLike}%`) as Array<{ id: number; content?: string; metadata?: string }>) {
|
||||
if (!row.metadata) continue;
|
||||
let meta: Record<string, unknown>;
|
||||
try { meta = JSON.parse(row.metadata) as Record<string, unknown>; } catch { continue; }
|
||||
if (!meta || typeof meta !== 'object' || meta.sourceId !== sourceRef) continue;
|
||||
const tok = row.content?.match(/^\[(?:Harvest:)?([^\]]+)\]/)?.[1];
|
||||
if (tok === source) frameIds.add(row.id);
|
||||
}
|
||||
|
||||
// 3. Erase the direct frame set; track what was actually deleted for the
|
||||
// B-frame reference sweep below.
|
||||
const erasedIds = new Set<number>();
|
||||
for (const fid of frameIds) {
|
||||
const r = this.eraseFrameInternal(fid, reason);
|
||||
if (r.framesDeleted > 0) erasedIds.add(fid);
|
||||
add(r);
|
||||
}
|
||||
|
||||
// 4. B-frame reference sweep — a synthesized B-frame references erased
|
||||
// frames in its content JSON and carries no archiveUids, so 2a/2b cannot
|
||||
// reach it. Shared with the single-frame path (see sweepReferencingBFrames).
|
||||
add(this.sweepReferencingBFrames(erasedIds, reason));
|
||||
|
||||
// 5. Redact any subject archive row not reached via a frame (orphan
|
||||
// provenance). Already-redacted rows are idempotent no-ops (return false),
|
||||
// so this never double-counts rows handled in step 3.
|
||||
for (const uid of uids) {
|
||||
if (this.archive.erase(uid, reason)) total.archiveRedacted += 1;
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
}
|
||||
299
packages/hive-mind-core/src/mind/evolution-runs.ts
Normal file
299
packages/hive-mind-core/src/mind/evolution-runs.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
/**
|
||||
* Evolution Runs — persistent audit of every evolution proposal.
|
||||
*
|
||||
* One row per ComposeEvolution execution. A run starts life as `proposed`
|
||||
* when the orchestrator finishes building it; the user then `accept`s or
|
||||
* `reject`s it from the UI. Accepted runs that successfully deploy move
|
||||
* to `deployed`; failed deploys move to `failed`.
|
||||
*
|
||||
* Every run stores:
|
||||
* - baseline and winner text (prompts, specs, skill bodies)
|
||||
* - winner schema when the evolution included a schema stage (JSON blob)
|
||||
* - accuracy delta and gate verdict
|
||||
* - rejection reason if rejected
|
||||
* - timestamps for created, decided, deployed
|
||||
*
|
||||
* This is the source of truth for the Memory app → Evolution tab history
|
||||
* view, and the regression audit when a deployed change causes a user-
|
||||
* visible problem.
|
||||
*/
|
||||
|
||||
export type EvolutionRunStatus =
|
||||
| 'proposed'
|
||||
| 'accepted'
|
||||
| 'rejected'
|
||||
| 'deployed'
|
||||
| 'failed';
|
||||
|
||||
export type EvolutionRunTarget =
|
||||
| 'persona-system-prompt'
|
||||
| 'behavioral-spec-section'
|
||||
| 'tool-description'
|
||||
| 'skill-body'
|
||||
| 'generic';
|
||||
|
||||
export interface EvolutionRun {
|
||||
id: number;
|
||||
run_uuid: string;
|
||||
target_kind: EvolutionRunTarget;
|
||||
/** User-visible name of what was evolved (e.g. persona id, spec section) */
|
||||
target_name: string | null;
|
||||
baseline_text: string;
|
||||
winner_text: string;
|
||||
/** JSON-encoded Schema when evolution included structure, else null */
|
||||
winner_schema_json: string | null;
|
||||
delta_accuracy: number;
|
||||
gate_verdict: 'pass' | 'fail';
|
||||
/** JSON array of {gate, verdict, reason} objects */
|
||||
gate_reasons_json: string;
|
||||
status: EvolutionRunStatus;
|
||||
/** Optional JSON blob with per-gen history, scores, Pareto front, etc */
|
||||
artifacts_json: string | null;
|
||||
user_note: string | null;
|
||||
failure_reason: string | null;
|
||||
created_at: string;
|
||||
decided_at: string | null;
|
||||
deployed_at: string | null;
|
||||
}
|
||||
|
||||
export interface CreateEvolutionRunInput {
|
||||
runUuid?: string;
|
||||
targetKind: EvolutionRunTarget;
|
||||
targetName?: string | null;
|
||||
baselineText: string;
|
||||
winnerText: string;
|
||||
winnerSchema?: unknown;
|
||||
deltaAccuracy: number;
|
||||
gateVerdict: 'pass' | 'fail';
|
||||
gateReasons: Array<{ gate: string; verdict: 'pass' | 'fail'; reason: string }>;
|
||||
artifacts?: unknown;
|
||||
}
|
||||
|
||||
export interface EvolutionRunFilter {
|
||||
status?: EvolutionRunStatus | EvolutionRunStatus[];
|
||||
targetKind?: EvolutionRunTarget;
|
||||
targetName?: string;
|
||||
since?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
const DDL: string[] = [
|
||||
`CREATE TABLE IF NOT EXISTS evolution_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_uuid TEXT NOT NULL UNIQUE,
|
||||
target_kind TEXT NOT NULL,
|
||||
target_name TEXT,
|
||||
baseline_text TEXT NOT NULL,
|
||||
winner_text TEXT NOT NULL,
|
||||
winner_schema_json TEXT,
|
||||
delta_accuracy REAL NOT NULL DEFAULT 0,
|
||||
gate_verdict TEXT NOT NULL DEFAULT 'pass'
|
||||
CHECK (gate_verdict IN ('pass', 'fail')),
|
||||
gate_reasons_json TEXT NOT NULL DEFAULT '[]',
|
||||
status TEXT NOT NULL DEFAULT 'proposed'
|
||||
CHECK (status IN ('proposed', 'accepted', 'rejected', 'deployed', 'failed')),
|
||||
artifacts_json TEXT,
|
||||
user_note TEXT,
|
||||
failure_reason TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
decided_at TEXT,
|
||||
deployed_at TEXT
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_evo_runs_status ON evolution_runs (status, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_evo_runs_target ON evolution_runs (target_kind, target_name, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_evo_runs_created ON evolution_runs (created_at DESC)`,
|
||||
];
|
||||
|
||||
export const EVOLUTION_RUNS_TABLE_SQL = DDL.join(';\n') + ';';
|
||||
|
||||
export class EvolutionRunStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='evolution_runs'",
|
||||
).get();
|
||||
if (exists) return;
|
||||
for (const stmt of DDL) {
|
||||
raw.prepare(stmt).run();
|
||||
}
|
||||
} catch {
|
||||
// DB may be closed during teardown — safe to skip.
|
||||
}
|
||||
}
|
||||
|
||||
/** Insert a new proposed run. Generates a UUID if the caller omits one. */
|
||||
create(input: CreateEvolutionRunInput): EvolutionRun {
|
||||
const uuid = input.runUuid ?? generateUuid();
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO evolution_runs (
|
||||
run_uuid, target_kind, target_name,
|
||||
baseline_text, winner_text, winner_schema_json,
|
||||
delta_accuracy, gate_verdict, gate_reasons_json,
|
||||
status, artifacts_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'proposed', ?)
|
||||
`).run(
|
||||
uuid,
|
||||
input.targetKind,
|
||||
input.targetName ?? null,
|
||||
input.baselineText,
|
||||
input.winnerText,
|
||||
input.winnerSchema !== undefined ? JSON.stringify(input.winnerSchema) : null,
|
||||
input.deltaAccuracy,
|
||||
input.gateVerdict,
|
||||
JSON.stringify(input.gateReasons ?? []),
|
||||
input.artifacts !== undefined ? JSON.stringify(input.artifacts) : null,
|
||||
);
|
||||
|
||||
const row = raw.prepare(
|
||||
'SELECT * FROM evolution_runs WHERE run_uuid = ?',
|
||||
).get(uuid) as EvolutionRun;
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Mark a proposed run as accepted. */
|
||||
accept(uuid: string, userNote?: string): EvolutionRun | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE evolution_runs
|
||||
SET status = 'accepted',
|
||||
user_note = COALESCE(?, user_note),
|
||||
decided_at = datetime('now')
|
||||
WHERE run_uuid = ? AND status = 'proposed'
|
||||
`).run(userNote ?? null, uuid);
|
||||
return this.getByUuid(uuid);
|
||||
}
|
||||
|
||||
/** Mark a proposed run as rejected with an optional reason. */
|
||||
reject(uuid: string, reason?: string): EvolutionRun | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE evolution_runs
|
||||
SET status = 'rejected',
|
||||
user_note = COALESCE(?, user_note),
|
||||
decided_at = datetime('now')
|
||||
WHERE run_uuid = ? AND status = 'proposed'
|
||||
`).run(reason ?? null, uuid);
|
||||
return this.getByUuid(uuid);
|
||||
}
|
||||
|
||||
/** Mark an accepted run as successfully deployed. */
|
||||
markDeployed(uuid: string): EvolutionRun | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE evolution_runs
|
||||
SET status = 'deployed',
|
||||
deployed_at = datetime('now')
|
||||
WHERE run_uuid = ? AND status = 'accepted'
|
||||
`).run(uuid);
|
||||
return this.getByUuid(uuid);
|
||||
}
|
||||
|
||||
/** Mark an accepted run as failed to deploy with the given reason. */
|
||||
markFailed(uuid: string, reason: string): EvolutionRun | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE evolution_runs
|
||||
SET status = 'failed',
|
||||
failure_reason = ?,
|
||||
deployed_at = datetime('now')
|
||||
WHERE run_uuid = ? AND status = 'accepted'
|
||||
`).run(reason, uuid);
|
||||
return this.getByUuid(uuid);
|
||||
}
|
||||
|
||||
getByUuid(uuid: string): EvolutionRun | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM evolution_runs WHERE run_uuid = ?')
|
||||
.get(uuid) as EvolutionRun | undefined;
|
||||
}
|
||||
|
||||
get(id: number): EvolutionRun | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM evolution_runs WHERE id = ?')
|
||||
.get(id) as EvolutionRun | undefined;
|
||||
}
|
||||
|
||||
list(filter: EvolutionRunFilter = {}): EvolutionRun[] {
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (filter.status) {
|
||||
const statuses = Array.isArray(filter.status) ? filter.status : [filter.status];
|
||||
clauses.push(`status IN (${statuses.map(() => '?').join(',')})`);
|
||||
params.push(...statuses);
|
||||
}
|
||||
if (filter.targetKind) {
|
||||
clauses.push('target_kind = ?');
|
||||
params.push(filter.targetKind);
|
||||
}
|
||||
if (filter.targetName) {
|
||||
clauses.push('target_name = ?');
|
||||
params.push(filter.targetName);
|
||||
}
|
||||
if (filter.since) {
|
||||
clauses.push('created_at >= ?');
|
||||
params.push(filter.since);
|
||||
}
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const limit = filter.limit ?? 50;
|
||||
|
||||
return this.db.getDatabase().prepare(
|
||||
`SELECT * FROM evolution_runs ${where} ORDER BY created_at DESC, id DESC LIMIT ?`,
|
||||
).all(...params, limit) as EvolutionRun[];
|
||||
}
|
||||
|
||||
/** Aggregate counts per status for stats/UI. */
|
||||
statusCounts(
|
||||
filter: Omit<EvolutionRunFilter, 'status' | 'limit'> = {},
|
||||
): Record<EvolutionRunStatus, number> {
|
||||
const counts: Record<EvolutionRunStatus, number> = {
|
||||
proposed: 0, accepted: 0, rejected: 0, deployed: 0, failed: 0,
|
||||
};
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
if (filter.targetKind) { clauses.push('target_kind = ?'); params.push(filter.targetKind); }
|
||||
if (filter.targetName) { clauses.push('target_name = ?'); params.push(filter.targetName); }
|
||||
if (filter.since) { clauses.push('created_at >= ?'); params.push(filter.since); }
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
|
||||
const rows = this.db.getDatabase().prepare(
|
||||
`SELECT status, COUNT(*) as cnt FROM evolution_runs ${where} GROUP BY status`,
|
||||
).all(...params) as Array<{ status: EvolutionRunStatus; cnt: number }>;
|
||||
|
||||
for (const row of rows) {
|
||||
counts[row.status] = row.cnt;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Delete a run by uuid (testing / cleanup). */
|
||||
delete(uuid: string): void {
|
||||
this.db.getDatabase()
|
||||
.prepare('DELETE FROM evolution_runs WHERE run_uuid = ?')
|
||||
.run(uuid);
|
||||
}
|
||||
|
||||
/** Delete all runs (tests only). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM evolution_runs').run();
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal UUID v4-ish generator — enough to uniquely tag rows, no crypto needed. */
|
||||
function generateUuid(): string {
|
||||
const rand = () => Math.floor(Math.random() * 0x10000).toString(16).padStart(4, '0');
|
||||
return `${rand()}${rand()}-${rand()}-4${rand().slice(1)}-${rand()}-${rand()}${rand()}${rand()}`;
|
||||
}
|
||||
446
packages/hive-mind-core/src/mind/execution-traces.ts
Normal file
446
packages/hive-mind-core/src/mind/execution-traces.ts
Normal file
@@ -0,0 +1,446 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
/**
|
||||
* Execution Traces — persistent log of what agents actually did.
|
||||
*
|
||||
* The foundation for eval dataset construction (Phase 1.2) and
|
||||
* fitness scoring in the evolution loop (Phase 2+).
|
||||
*
|
||||
* One trace row per "unit of agent work" — typically a single user turn,
|
||||
* a single workflow phase, or a single subagent run. Tool calls, reasoning,
|
||||
* and the final output are packed into `trace_json`.
|
||||
*
|
||||
* Outcome labels:
|
||||
* - `success` — user accepted the result, no correction follow-up
|
||||
* - `corrected` — user corrected/refined the output (weak negative signal)
|
||||
* - `abandoned` — user left the thread / switched contexts (ambiguous)
|
||||
* - `verified` — passed a verifier gate or harness checkpoint
|
||||
*/
|
||||
|
||||
export type TraceOutcome = 'success' | 'corrected' | 'abandoned' | 'verified' | 'pending';
|
||||
|
||||
/** A single tool call captured during execution. */
|
||||
export interface TraceToolCall {
|
||||
/** Tool name */
|
||||
tool: string;
|
||||
/** Arguments passed to the tool (already scrubbed of secrets by caller) */
|
||||
args: Record<string, unknown>;
|
||||
/** Result as a string (may be truncated) */
|
||||
result: string;
|
||||
/** Whether the tool call succeeded */
|
||||
ok: boolean;
|
||||
/** Duration in milliseconds */
|
||||
durationMs: number;
|
||||
/** ISO timestamp */
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** Reasoning step — free-form agent thought recorded before/between tool calls. */
|
||||
export interface TraceReasoningStep {
|
||||
content: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The structured payload stored in trace_json.
|
||||
* Kept separate so callers can evolve the shape without schema migrations.
|
||||
*/
|
||||
export interface TracePayload {
|
||||
/** Original user instruction (may be truncated) */
|
||||
input: string;
|
||||
/** Final agent output */
|
||||
output: string;
|
||||
/** Reasoning steps interleaved with tool calls */
|
||||
reasoning: TraceReasoningStep[];
|
||||
/** Tool calls in order */
|
||||
toolCalls: TraceToolCall[];
|
||||
/** Files created or modified */
|
||||
artifacts: string[];
|
||||
/** Tokens consumed */
|
||||
tokens: { input: number; output: number };
|
||||
/** Optional workflow harness context */
|
||||
harness?: {
|
||||
harnessId: string;
|
||||
phaseId: string;
|
||||
phaseName: string;
|
||||
gateResults?: Array<{ name: string; passed: boolean; reason: string }>;
|
||||
};
|
||||
/** Free-form correction text if outcome === 'corrected' */
|
||||
correctionFeedback?: string;
|
||||
/** Arbitrary tags for later filtering in eval-dataset */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Row shape returned by queries. */
|
||||
export interface ExecutionTrace {
|
||||
id: number;
|
||||
session_id: string | null;
|
||||
persona_id: string | null;
|
||||
workspace_id: string | null;
|
||||
model: string | null;
|
||||
task_shape: string | null;
|
||||
outcome: TraceOutcome;
|
||||
trace_json: string;
|
||||
cost_usd: number;
|
||||
duration_ms: number;
|
||||
created_at: string;
|
||||
finalized_at: string | null;
|
||||
}
|
||||
|
||||
/** With the trace_json pre-parsed. */
|
||||
export interface ParsedExecutionTrace extends Omit<ExecutionTrace, 'trace_json'> {
|
||||
payload: TracePayload;
|
||||
}
|
||||
|
||||
/** Input to start a new trace. */
|
||||
export interface StartTraceInput {
|
||||
sessionId?: string | null;
|
||||
personaId?: string | null;
|
||||
workspaceId?: string | null;
|
||||
model?: string | null;
|
||||
taskShape?: string | null;
|
||||
/** Initial user input captured immediately. */
|
||||
input: string;
|
||||
/** Optional tags for later filtering. */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Input to finalize a trace (update outcome + payload). */
|
||||
export interface FinalizeTraceInput {
|
||||
outcome: TraceOutcome;
|
||||
output: string;
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
tokens?: { input: number; output: number };
|
||||
costUsd?: number;
|
||||
harness?: TracePayload['harness'];
|
||||
correctionFeedback?: string;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** Filter for queries. */
|
||||
export interface TraceQueryFilter {
|
||||
sessionId?: string;
|
||||
personaId?: string;
|
||||
workspaceId?: string;
|
||||
outcome?: TraceOutcome | TraceOutcome[];
|
||||
taskShape?: string;
|
||||
/** Lower bound (inclusive) on created_at — ISO string */
|
||||
since?: string;
|
||||
/** Substring pre-filter on trace_json (SQL LIKE), e.g. `"agent:` to scope
|
||||
* the LIMIT to tagged traces instead of the global recency window. LIKE
|
||||
* wildcards (%/_) in the value are NOT escaped — callers needing an exact
|
||||
* match must still filter the parsed payload (tags array) in JS. */
|
||||
tagLike?: string;
|
||||
/** Max rows to return (default 100) */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/** DDL split into single statements to stay compatible with prepare().run(). */
|
||||
const EXECUTION_TRACES_DDL: string[] = [
|
||||
`CREATE TABLE IF NOT EXISTS execution_traces (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT,
|
||||
persona_id TEXT,
|
||||
workspace_id TEXT,
|
||||
model TEXT,
|
||||
task_shape TEXT,
|
||||
outcome TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (outcome IN ('success', 'corrected', 'abandoned', 'verified', 'pending')),
|
||||
trace_json TEXT NOT NULL DEFAULT '{}',
|
||||
cost_usd REAL NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finalized_at TEXT
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_session ON execution_traces (session_id, created_at)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
|
||||
];
|
||||
|
||||
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
|
||||
export const EXECUTION_TRACES_TABLE_SQL = EXECUTION_TRACES_DDL.join(';\n') + ';';
|
||||
|
||||
export class ExecutionTraceStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
/** Ensure execution_traces table exists for databases created before this feature. */
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='execution_traces'",
|
||||
).get();
|
||||
if (exists) return;
|
||||
for (const stmt of EXECUTION_TRACES_DDL) {
|
||||
raw.prepare(stmt).run();
|
||||
}
|
||||
} catch {
|
||||
// DB may be closed during teardown — safe to skip.
|
||||
}
|
||||
}
|
||||
|
||||
/** Start a new trace in `pending` outcome. Returns row id. */
|
||||
start(input: StartTraceInput): number {
|
||||
const raw = this.db.getDatabase();
|
||||
const payload: TracePayload = {
|
||||
input: input.input,
|
||||
output: '',
|
||||
reasoning: [],
|
||||
toolCalls: [],
|
||||
artifacts: [],
|
||||
tokens: { input: 0, output: 0 },
|
||||
tags: input.tags ?? [],
|
||||
};
|
||||
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO execution_traces
|
||||
(session_id, persona_id, workspace_id, model, task_shape, outcome, trace_json)
|
||||
VALUES (?, ?, ?, ?, ?, 'pending', ?)
|
||||
`).run(
|
||||
input.sessionId ?? null,
|
||||
input.personaId ?? null,
|
||||
input.workspaceId ?? null,
|
||||
input.model ?? null,
|
||||
input.taskShape ?? null,
|
||||
JSON.stringify(payload),
|
||||
);
|
||||
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append incremental events to a pending trace. Used during long runs
|
||||
* to avoid losing progress if the process crashes. Safe to call many times.
|
||||
*/
|
||||
append(
|
||||
id: number,
|
||||
events: {
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
},
|
||||
): void {
|
||||
const current = this.get(id);
|
||||
if (!current) return;
|
||||
|
||||
const payload = parsePayload(current.trace_json);
|
||||
payload.reasoning = [...payload.reasoning, ...(events.reasoning ?? [])];
|
||||
payload.toolCalls = [...payload.toolCalls, ...(events.toolCalls ?? [])];
|
||||
if (events.artifacts?.length) {
|
||||
const seen = new Set(payload.artifacts);
|
||||
for (const a of events.artifacts) {
|
||||
if (!seen.has(a)) {
|
||||
payload.artifacts.push(a);
|
||||
seen.add(a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.db.getDatabase()
|
||||
.prepare('UPDATE execution_traces SET trace_json = ? WHERE id = ?')
|
||||
.run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
|
||||
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
|
||||
const current = this.get(id);
|
||||
if (!current) return undefined;
|
||||
|
||||
const existing = parsePayload(current.trace_json);
|
||||
const merged: TracePayload = {
|
||||
...existing,
|
||||
output: input.output,
|
||||
reasoning: input.reasoning ?? existing.reasoning,
|
||||
toolCalls: input.toolCalls ?? existing.toolCalls,
|
||||
artifacts: input.artifacts ?? existing.artifacts,
|
||||
tokens: input.tokens ?? existing.tokens,
|
||||
harness: input.harness ?? existing.harness,
|
||||
correctionFeedback: input.correctionFeedback ?? existing.correctionFeedback,
|
||||
tags: input.tags ?? existing.tags,
|
||||
};
|
||||
|
||||
const createdMs = Date.parse(current.created_at + 'Z');
|
||||
const now = Date.now();
|
||||
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
|
||||
|
||||
this.db.getDatabase().prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
JSON.stringify(merged),
|
||||
input.costUsd ?? current.cost_usd,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
|
||||
return this.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark an already-finalized trace as corrected after the fact.
|
||||
* Used when a correction-detector picks up a user correction in a later turn.
|
||||
*/
|
||||
markCorrected(id: number, feedback: string): void {
|
||||
const current = this.get(id);
|
||||
if (!current) return;
|
||||
|
||||
const payload = parsePayload(current.trace_json);
|
||||
payload.correctionFeedback = feedback;
|
||||
|
||||
this.db.getDatabase().prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = 'corrected',
|
||||
trace_json = ?
|
||||
WHERE id = ?
|
||||
`).run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
get(id: number): ExecutionTrace | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM execution_traces WHERE id = ?')
|
||||
.get(id) as ExecutionTrace | undefined;
|
||||
}
|
||||
|
||||
getParsed(id: number): ParsedExecutionTrace | undefined {
|
||||
const row = this.get(id);
|
||||
return row ? toParsed(row) : undefined;
|
||||
}
|
||||
|
||||
/** Query traces with optional filters. */
|
||||
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (filter.sessionId) {
|
||||
clauses.push('session_id = ?');
|
||||
params.push(filter.sessionId);
|
||||
}
|
||||
if (filter.personaId) {
|
||||
clauses.push('persona_id = ?');
|
||||
params.push(filter.personaId);
|
||||
}
|
||||
if (filter.workspaceId) {
|
||||
clauses.push('workspace_id = ?');
|
||||
params.push(filter.workspaceId);
|
||||
}
|
||||
if (filter.taskShape) {
|
||||
clauses.push('task_shape = ?');
|
||||
params.push(filter.taskShape);
|
||||
}
|
||||
if (filter.outcome) {
|
||||
const outcomes = Array.isArray(filter.outcome) ? filter.outcome : [filter.outcome];
|
||||
clauses.push(`outcome IN (${outcomes.map(() => '?').join(',')})`);
|
||||
params.push(...outcomes);
|
||||
}
|
||||
if (filter.since) {
|
||||
clauses.push('created_at >= ?');
|
||||
params.push(filter.since);
|
||||
}
|
||||
if (filter.tagLike) {
|
||||
clauses.push('trace_json LIKE ?');
|
||||
params.push(`%${filter.tagLike}%`);
|
||||
}
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const limit = filter.limit ?? 100;
|
||||
|
||||
return this.db.getDatabase().prepare(
|
||||
`SELECT * FROM execution_traces ${where} ORDER BY created_at DESC, id DESC LIMIT ?`,
|
||||
).all(...params, limit) as ExecutionTrace[];
|
||||
}
|
||||
|
||||
/** Parsed variant of query(). */
|
||||
queryParsed(filter: TraceQueryFilter = {}): ParsedExecutionTrace[] {
|
||||
return this.query(filter).map(toParsed);
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate outcome counts — useful for fitness scoring.
|
||||
* Returns: { success, corrected, abandoned, verified, pending }
|
||||
*/
|
||||
outcomeCounts(filter: Omit<TraceQueryFilter, 'outcome' | 'limit'> = {}): Record<TraceOutcome, number> {
|
||||
const counts: Record<TraceOutcome, number> = {
|
||||
success: 0, corrected: 0, abandoned: 0, verified: 0, pending: 0,
|
||||
};
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
if (filter.sessionId) { clauses.push('session_id = ?'); params.push(filter.sessionId); }
|
||||
if (filter.personaId) { clauses.push('persona_id = ?'); params.push(filter.personaId); }
|
||||
if (filter.workspaceId) { clauses.push('workspace_id = ?'); params.push(filter.workspaceId); }
|
||||
if (filter.taskShape) { clauses.push('task_shape = ?'); params.push(filter.taskShape); }
|
||||
if (filter.since) { clauses.push('created_at >= ?'); params.push(filter.since); }
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const rows = this.db.getDatabase().prepare(
|
||||
`SELECT outcome, COUNT(*) as cnt FROM execution_traces ${where} GROUP BY outcome`,
|
||||
).all(...params) as Array<{ outcome: TraceOutcome; cnt: number }>;
|
||||
|
||||
for (const row of rows) {
|
||||
counts[row.outcome] = row.cnt;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Delete a trace by id. */
|
||||
delete(id: number): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM execution_traces WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
/** Delete all traces (tests only). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM execution_traces').run();
|
||||
}
|
||||
|
||||
/** Count of all traces (for stats). */
|
||||
count(filter: Omit<TraceQueryFilter, 'limit'> = {}): number {
|
||||
const rows = this.query({ ...filter, limit: 1_000_000 });
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
function parsePayload(json: string): TracePayload {
|
||||
try {
|
||||
const parsed = JSON.parse(json) as Partial<TracePayload>;
|
||||
return {
|
||||
input: parsed.input ?? '',
|
||||
output: parsed.output ?? '',
|
||||
reasoning: parsed.reasoning ?? [],
|
||||
toolCalls: parsed.toolCalls ?? [],
|
||||
artifacts: parsed.artifacts ?? [],
|
||||
tokens: parsed.tokens ?? { input: 0, output: 0 },
|
||||
harness: parsed.harness,
|
||||
correctionFeedback: parsed.correctionFeedback,
|
||||
tags: parsed.tags ?? [],
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
input: '',
|
||||
output: '',
|
||||
reasoning: [],
|
||||
toolCalls: [],
|
||||
artifacts: [],
|
||||
tokens: { input: 0, output: 0 },
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toParsed(row: ExecutionTrace): ParsedExecutionTrace {
|
||||
const { trace_json, ...rest } = row;
|
||||
return { ...rest, payload: parsePayload(trace_json) };
|
||||
}
|
||||
487
packages/hive-mind-core/src/mind/frames.ts
Normal file
487
packages/hive-mind-core/src/mind/frames.ts
Normal file
@@ -0,0 +1,487 @@
|
||||
import type { MindDB } from './db.js';
|
||||
import { hashFrameContent, stripHmPrefix } from './content-hash.js';
|
||||
|
||||
// Back-compat re-export — stripHmPrefix moved to content-hash.ts (oss-drift D3)
|
||||
// so the hash and the strip live in one module; existing importers unchanged.
|
||||
export { stripHmPrefix };
|
||||
|
||||
/** Strict ISO-8601 check used by `createIFrame` to decide whether to honor
|
||||
* a caller-supplied `createdAt`. Requires the `T` separator and a
|
||||
* timezone suffix (`Z` or `±HH:MM`) — anything looser is high-risk for
|
||||
* range queries on `memory_frames.created_at`. Ported from hive-mind
|
||||
* 9ec75e6 (Stage 0 root cause: harvest path was discarding original
|
||||
* source timestamps and stamping every frame with ingest wall-clock,
|
||||
* which made date-scoped retrieval queries return ABSTAIN on real
|
||||
* Claude.ai exports). */
|
||||
function isValidIsoTimestamp(value: string): boolean {
|
||||
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/.test(value)) {
|
||||
return false;
|
||||
}
|
||||
return Number.isFinite(Date.parse(value));
|
||||
}
|
||||
|
||||
export type FrameType = 'I' | 'P' | 'B';
|
||||
export type Importance = 'critical' | 'important' | 'normal' | 'temporary' | 'deprecated';
|
||||
// Must stay in sync with the memory_frames.source CHECK constraint (schema.ts).
|
||||
// ('personal'/'workspace' are MultiMind result labels, not DB sources; team-synced
|
||||
// frames are stored as 'import' with provenance carried in the content prefix.)
|
||||
export type FrameSource = 'user_stated' | 'tool_verified' | 'agent_inferred' | 'import' | 'system';
|
||||
|
||||
export interface MemoryFrame {
|
||||
id: number;
|
||||
frame_type: FrameType;
|
||||
gop_id: string;
|
||||
t: number;
|
||||
base_frame_id: number | null;
|
||||
content: string;
|
||||
importance: Importance;
|
||||
source: FrameSource;
|
||||
access_count: number;
|
||||
created_at: string;
|
||||
last_accessed: string;
|
||||
/** oss-drift D3: canonical dedup hash (hashFrameContent — stripHmPrefix +
|
||||
* trim semantics). Maintained on every FrameStore write; NULL only on rows
|
||||
* written by raw SQL before the next boot's migration backfill. */
|
||||
content_hash?: string | null;
|
||||
/** UX-Refactor Phase 2B: JSON blob for Memory Center provenance/classification
|
||||
* (kind/confidence/scope/status/sourceId/sourceUrl/tags/evidence/related*).
|
||||
* Always present at the column level (NOT NULL DEFAULT '{}'); typed optional
|
||||
* so pre-migration callers and literal constructions stay back-compatible. */
|
||||
metadata?: string;
|
||||
}
|
||||
|
||||
export interface ReconstructedState {
|
||||
iframe: MemoryFrame | null;
|
||||
pframes: MemoryFrame[];
|
||||
}
|
||||
|
||||
const IMPORTANCE_MULTIPLIERS: Record<Importance, number> = {
|
||||
critical: 2.0,
|
||||
important: 1.5,
|
||||
normal: 1.0,
|
||||
temporary: 0.7,
|
||||
deprecated: 0.3,
|
||||
};
|
||||
|
||||
export class FrameStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
createIFrame(
|
||||
gopId: string,
|
||||
content: string,
|
||||
importance: Importance = 'normal',
|
||||
source: FrameSource = 'user_stated',
|
||||
/** Optional override for `memory_frames.created_at`. Supplied by the harvest
|
||||
* path so frames ingested from an export preserve the original source
|
||||
* timestamp (e.g. Claude session `create_time`) instead of getting
|
||||
* stamped with the ingest wall-clock. Callers that don't care about
|
||||
* temporal-anchor preservation (live agent writes, cognify, etc.)
|
||||
* should omit this argument and let the `datetime('now')` default apply.
|
||||
*
|
||||
* Value must be a valid ISO-8601 string with `T` separator and timezone;
|
||||
* invalid / null / undefined falls back to the schema default. The
|
||||
* caller (harvest route) is responsible for validating + logging the
|
||||
* fallback path — this function stays minimal and side-effect-free.
|
||||
*
|
||||
* Ported from hive-mind 9ec75e6. */
|
||||
createdAt?: string | null,
|
||||
): MemoryFrame {
|
||||
// L1: Dedup — if identical content exists, update access count instead of duplicating
|
||||
const existing = this.findDuplicate(content);
|
||||
if (existing) return existing;
|
||||
|
||||
const t = this.nextT(gopId);
|
||||
const raw = this.db.getDatabase();
|
||||
// Branch on whether the caller supplied a valid ISO-8601 createdAt.
|
||||
// Valid → INSERT also overrides created_at + last_accessed (last_accessed
|
||||
// mirrors created_at on initial insert for consistency).
|
||||
// Invalid / null / undefined → fall back to the schema default
|
||||
// (datetime('now')) — never write junk timestamps that would corrupt
|
||||
// range queries.
|
||||
const useProvidedTs = typeof createdAt === 'string' && isValidIsoTimestamp(createdAt);
|
||||
const contentHash = hashFrameContent(content);
|
||||
const result = useProvidedTs
|
||||
? raw.prepare(`
|
||||
INSERT INTO memory_frames (frame_type, gop_id, t, base_frame_id, content, importance, source, content_hash, created_at, last_accessed)
|
||||
VALUES ('I', ?, ?, NULL, ?, ?, ?, ?, ?, ?)
|
||||
`).run(gopId, t, content, importance, source, contentHash, createdAt, createdAt)
|
||||
: raw.prepare(`
|
||||
INSERT INTO memory_frames (frame_type, gop_id, t, base_frame_id, content, importance, source, content_hash)
|
||||
VALUES ('I', ?, ?, NULL, ?, ?, ?, ?)
|
||||
`).run(gopId, t, content, importance, source, contentHash);
|
||||
|
||||
const frame = raw.prepare('SELECT * FROM memory_frames WHERE id = ?').get(result.lastInsertRowid) as MemoryFrame;
|
||||
this.indexFts(frame);
|
||||
return frame;
|
||||
}
|
||||
|
||||
createPFrame(gopId: string, content: string, baseFrameId: number, importance: Importance = 'normal', source: FrameSource = 'user_stated'): MemoryFrame {
|
||||
const t = this.nextT(gopId);
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO memory_frames (frame_type, gop_id, t, base_frame_id, content, importance, source, content_hash)
|
||||
VALUES ('P', ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(gopId, t, baseFrameId, content, importance, source, hashFrameContent(content));
|
||||
|
||||
const frame = raw.prepare('SELECT * FROM memory_frames WHERE id = ?').get(result.lastInsertRowid) as MemoryFrame;
|
||||
this.indexFts(frame);
|
||||
return frame;
|
||||
}
|
||||
|
||||
createBFrame(gopId: string, content: string, baseFrameId: number, referencedFrameIds: number[]): MemoryFrame {
|
||||
const t = this.nextT(gopId);
|
||||
// Store cross-references in the content as structured data
|
||||
const bContent = JSON.stringify({
|
||||
description: content,
|
||||
references: referencedFrameIds,
|
||||
});
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO memory_frames (frame_type, gop_id, t, base_frame_id, content, importance, content_hash)
|
||||
VALUES ('B', ?, ?, ?, ?, 'normal', ?)
|
||||
`).run(gopId, t, baseFrameId, bContent, hashFrameContent(bContent));
|
||||
|
||||
const frame = raw.prepare('SELECT * FROM memory_frames WHERE id = ?').get(result.lastInsertRowid) as MemoryFrame;
|
||||
this.indexFts(frame);
|
||||
return frame;
|
||||
}
|
||||
|
||||
getById(id: number): MemoryFrame | undefined {
|
||||
return this.db.getDatabase().prepare('SELECT * FROM memory_frames WHERE id = ?').get(id) as MemoryFrame | undefined;
|
||||
}
|
||||
|
||||
getLatestIFrame(gopId: string): MemoryFrame | undefined {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
WHERE gop_id = ? AND frame_type = 'I'
|
||||
ORDER BY t DESC LIMIT 1
|
||||
`).get(gopId) as MemoryFrame | undefined;
|
||||
}
|
||||
|
||||
getPFramesSinceLastI(gopId: string): MemoryFrame[] {
|
||||
const latestI = this.getLatestIFrame(gopId);
|
||||
if (!latestI) return [];
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
WHERE gop_id = ? AND frame_type = 'P' AND t > ?
|
||||
ORDER BY t ASC
|
||||
`).all(gopId, latestI.t) as MemoryFrame[];
|
||||
}
|
||||
|
||||
getGopFrames(gopId: string): MemoryFrame[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames WHERE gop_id = ? ORDER BY t ASC
|
||||
`).all(gopId) as MemoryFrame[];
|
||||
}
|
||||
|
||||
reconstructState(gopId: string): ReconstructedState {
|
||||
const iframe = this.getLatestIFrame(gopId) ?? null;
|
||||
const pframes = iframe ? this.getPFramesSinceLastI(gopId) : [];
|
||||
return { iframe, pframes };
|
||||
}
|
||||
|
||||
touch(id: number): number | undefined {
|
||||
const row = this.db.getDatabase().prepare(`
|
||||
UPDATE memory_frames SET access_count = access_count + 1, last_accessed = datetime('now')
|
||||
WHERE id = ?
|
||||
RETURNING access_count AS accessCount
|
||||
`).get(id) as { accessCount: number } | undefined;
|
||||
return row?.accessCount;
|
||||
}
|
||||
|
||||
getImportanceMultiplier(importance: Importance): number {
|
||||
return IMPORTANCE_MULTIPLIERS[importance];
|
||||
}
|
||||
|
||||
/** List frames with an options bag (convenience wrapper used by server routes). */
|
||||
list(opts: { limit?: number } = {}): MemoryFrame[] {
|
||||
return this.getRecent(opts.limit ?? 50);
|
||||
}
|
||||
|
||||
/** Get the most recent frames ordered by creation time descending. */
|
||||
getRecent(limit = 50): MemoryFrame[] {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames ORDER BY id DESC LIMIT ?
|
||||
`).all(limit) as MemoryFrame[];
|
||||
}
|
||||
|
||||
/**
|
||||
* F20: Get recent frames with optional temporal boundaries.
|
||||
* @param limit Maximum number of results
|
||||
* @param since Only include frames created on or after this ISO date string
|
||||
* @param until Only include frames created on or before this ISO date string
|
||||
*/
|
||||
getRecentFiltered(limit = 50, since?: string, until?: string): MemoryFrame[] {
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (since) {
|
||||
conditions.push('created_at >= ?');
|
||||
params.push(since);
|
||||
}
|
||||
if (until) {
|
||||
conditions.push('created_at <= ?');
|
||||
params.push(until);
|
||||
}
|
||||
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
params.push(limit);
|
||||
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames ${where} ORDER BY id DESC LIMIT ?
|
||||
`).all(...params) as MemoryFrame[];
|
||||
}
|
||||
|
||||
getBFrameReferences(bframeId: number): number[] {
|
||||
const frame = this.getById(bframeId);
|
||||
if (!frame || frame.frame_type !== 'B') return [];
|
||||
try {
|
||||
const parsed = JSON.parse(frame.content);
|
||||
return parsed.references ?? [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* L1: Check for duplicate content before inserting.
|
||||
* Returns the existing frame if content hash matches, null otherwise.
|
||||
* If a duplicate is found, updates its access_count instead of creating a new frame.
|
||||
*
|
||||
* oss-drift D3 (2026-06-11): O(1) lookup on the indexed `content_hash`
|
||||
* column with NO recency window — the previous implementation scanned only
|
||||
* the last 500 frames and silently missed older duplicates. Hash semantics
|
||||
* (hashFrameContent) are unchanged:
|
||||
* - trim-stable: JS `trim()` over the content (SQLite's `trim()` only
|
||||
* strips ASCII space, so hashing happens JS-side, never in SQL);
|
||||
* - provenance-insensitive (OQ-6): content passes through `stripHmPrefix`
|
||||
* before hashing, so two same-body captures of one turn collapse into
|
||||
* one frame regardless of which source's `[hm …]` prefix they carry.
|
||||
* Rows written by raw SQL before the column existed are backfilled by
|
||||
* db.ts runMigrations() on open.
|
||||
*/
|
||||
findDuplicate(content: string): MemoryFrame | null {
|
||||
const frame = this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames WHERE content_hash = ? ORDER BY id DESC LIMIT 1
|
||||
`).get(hashFrameContent(content)) as MemoryFrame | undefined;
|
||||
if (frame) {
|
||||
// Update access count instead of creating duplicate
|
||||
this.touch(frame.id);
|
||||
return frame;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Q22: Update a frame's content and/or importance by ID.
|
||||
* Updates the main table, FTS index, and vector index.
|
||||
* Returns the updated frame, or undefined if not found.
|
||||
*/
|
||||
update(id: number, content: string, importance?: Importance): MemoryFrame | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = this.getById(id);
|
||||
if (!existing) return undefined;
|
||||
|
||||
const newImportance = importance ?? existing.importance;
|
||||
|
||||
// Update main table (content_hash maintained — oss-drift D3)
|
||||
raw.prepare(`
|
||||
UPDATE memory_frames SET content = ?, importance = ?, content_hash = ? WHERE id = ?
|
||||
`).run(content, newImportance, hashFrameContent(content), id);
|
||||
|
||||
// Update FTS index: delete old entry, insert new
|
||||
raw.prepare('DELETE FROM memory_frames_fts WHERE rowid = ?').run(id);
|
||||
raw.prepare('INSERT INTO memory_frames_fts (rowid, content) VALUES (?, ?)').run(id, content);
|
||||
|
||||
// Update vector index if exists
|
||||
try { raw.prepare('DELETE FROM memory_frames_vec WHERE rowid = ?').run(id); } catch { /* vec table may not exist */ }
|
||||
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* UX-Refactor Phase 2B: replace a frame's `metadata` JSON blob (Memory Center
|
||||
* kind/confidence/scope/status/tags/evidence/related*). Low-level writer — the
|
||||
* caller passes a fully-formed JSON string; parse/merge semantics live in the
|
||||
* route layer (`memory.ts`). Does NOT touch FTS/vector indexes (metadata is not
|
||||
* full-text searchable). Returns the updated frame, or undefined if the id is
|
||||
* unknown.
|
||||
*/
|
||||
setMetadata(id: number, metadata: string): MemoryFrame | undefined {
|
||||
const raw = this.db.getDatabase();
|
||||
if (!this.getById(id)) return undefined;
|
||||
raw.prepare('UPDATE memory_frames SET metadata = ? WHERE id = ?').run(metadata, id);
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* L2: Delete a frame by ID. Returns true if deleted, false if not found.
|
||||
*/
|
||||
delete(id: number): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
// Clear self-referential FK: nullify base_frame_id on any frames that reference this one
|
||||
raw.prepare('UPDATE memory_frames SET base_frame_id = NULL WHERE base_frame_id = ?').run(id);
|
||||
// Delete from vector index if exists
|
||||
try { raw.prepare('DELETE FROM memory_frames_vec WHERE rowid = ?').run(id); } catch { /* vec table may not exist */ }
|
||||
// Delete chunk vectors. memory_frame_chunks_vec is a vec0 virtual table with
|
||||
// NO foreign key, so the ON DELETE CASCADE that clears memory_frame_chunks
|
||||
// when the frame goes would ORPHAN these embedding rows (keyed by chunk id) —
|
||||
// and search() reads memory_frame_chunks_vec first, so a stale row stays
|
||||
// recall-able. Collect the chunk ids WHILE memory_frame_chunks still holds
|
||||
// them, then purge their vec rows (rowid must be a SQL literal for vec0).
|
||||
try {
|
||||
const chunkIds = raw.prepare('SELECT id FROM memory_frame_chunks WHERE frame_id = ?').all(id) as Array<{ id: number }>;
|
||||
for (const c of chunkIds) {
|
||||
raw.prepare(`DELETE FROM memory_frame_chunks_vec WHERE rowid = ${Math.trunc(c.id)}`).run();
|
||||
}
|
||||
} catch { /* chunk tables may not exist on a pre-D1 DB */ }
|
||||
// Delete from FTS index
|
||||
raw.prepare('DELETE FROM memory_frames_fts WHERE rowid = ?').run(id);
|
||||
// Delete from KG entity-frame links if KG tables exist
|
||||
try { raw.prepare('DELETE FROM kg_entity_frames WHERE frame_id = ?').run(id); } catch { /* KG tables may not exist */ }
|
||||
// Delete from main table (FK cascade clears memory_frame_chunks + kg_entity_frames)
|
||||
const result = raw.prepare('DELETE FROM memory_frames WHERE id = ?').run(id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* W4.3: delete every frame whose content starts with `prefix` (exact
|
||||
* literal match — LIKE metacharacters in the prefix are escaped). Used by
|
||||
* replace-on-update lanes (profile cards supersede the prior card for the
|
||||
* same person). Routes through delete(id) so FTS/vec/KG cleanup applies.
|
||||
* Returns the number of frames deleted.
|
||||
*/
|
||||
deleteByContentPrefix(prefix: string): number {
|
||||
const raw = this.db.getDatabase();
|
||||
const escaped = prefix.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames WHERE content LIKE ? ESCAPE '\\'`
|
||||
).all(`${escaped}%`) as Array<{ id: number }>;
|
||||
let deleted = 0;
|
||||
for (const r of rows) {
|
||||
if (this.delete(r.id)) deleted++;
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ── 9a: Memory Compaction ──────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compact memory: merge stale P-frames into their base I-frame,
|
||||
* prune deprecated frames, and clean up temporary frames older than maxAge.
|
||||
*
|
||||
* @param maxTempAgeDays - Delete temporary frames older than this (default 30)
|
||||
* @param maxDeprecatedAgeDays - Delete deprecated frames older than this (default 90)
|
||||
* @returns Summary of compaction actions taken
|
||||
*/
|
||||
compact(maxTempAgeDays = 30, maxDeprecatedAgeDays = 90): {
|
||||
temporaryPruned: number;
|
||||
deprecatedPruned: number;
|
||||
pframesMerged: number;
|
||||
} {
|
||||
const raw = this.db.getDatabase();
|
||||
let temporaryPruned = 0;
|
||||
let deprecatedPruned = 0;
|
||||
let pframesMerged = 0;
|
||||
|
||||
// Prune via delete(id), NOT a bare `DELETE FROM memory_frames`. A bare delete
|
||||
// relies on the FK cascade, which reaches memory_frame_chunks + kg_entity_frames
|
||||
// but NOT the vec0 virtual tables (memory_frames_vec, memory_frame_chunks_vec)
|
||||
// or the FTS index — those have no FK, so a bare delete orphans their rows and
|
||||
// they linger in the search index. delete() purges all of them (and nullifies
|
||||
// referencing base_frame_id, which a bare delete would trip on under FK-ON).
|
||||
|
||||
// 1. Delete old temporary frames
|
||||
const tempIds = raw.prepare(`
|
||||
SELECT id FROM memory_frames
|
||||
WHERE importance = 'temporary'
|
||||
AND created_at < datetime('now', '-' || ? || ' days')
|
||||
`).all(maxTempAgeDays) as Array<{ id: number }>;
|
||||
for (const { id } of tempIds) if (this.delete(id)) temporaryPruned++;
|
||||
|
||||
// 2. Delete old deprecated frames
|
||||
const depIds = raw.prepare(`
|
||||
SELECT id FROM memory_frames
|
||||
WHERE importance = 'deprecated'
|
||||
AND created_at < datetime('now', '-' || ? || ' days')
|
||||
`).all(maxDeprecatedAgeDays) as Array<{ id: number }>;
|
||||
for (const { id } of depIds) if (this.delete(id)) deprecatedPruned++;
|
||||
|
||||
// 3. Merge P-frames into I-frames when there are more than 10 P-frames
|
||||
// for a single GOP. The merged content becomes a new I-frame and the
|
||||
// old P-frames are deleted.
|
||||
const gopsWithManyPframes = raw.prepare(`
|
||||
SELECT gop_id, COUNT(*) as cnt FROM memory_frames
|
||||
WHERE frame_type = 'P'
|
||||
GROUP BY gop_id
|
||||
HAVING cnt > 10
|
||||
`).all() as { gop_id: string; cnt: number }[];
|
||||
|
||||
for (const { gop_id } of gopsWithManyPframes) {
|
||||
const latestI = this.getLatestIFrame(gop_id);
|
||||
if (!latestI) continue;
|
||||
|
||||
const pframes = raw.prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
WHERE gop_id = ? AND frame_type = 'P' AND t > ?
|
||||
ORDER BY t ASC
|
||||
`).all(gop_id, latestI.t) as MemoryFrame[];
|
||||
|
||||
if (pframes.length <= 10) continue;
|
||||
|
||||
// Keep the 5 most recent P-frames, merge the rest into the I-frame
|
||||
const toMerge = pframes.slice(0, pframes.length - 5);
|
||||
const mergedContent = [latestI.content, ...toMerge.map(p => p.content)].join('\n---\n');
|
||||
|
||||
// Update the I-frame with merged content (content_hash maintained — oss-drift D3)
|
||||
raw.prepare('UPDATE memory_frames SET content = ?, content_hash = ? WHERE id = ?')
|
||||
.run(mergedContent, hashFrameContent(mergedContent), latestI.id);
|
||||
// Update FTS
|
||||
raw.prepare('DELETE FROM memory_frames_fts WHERE rowid = ?').run(latestI.id);
|
||||
raw.prepare('INSERT INTO memory_frames_fts (rowid, content) VALUES (?, ?)').run(latestI.id, mergedContent);
|
||||
|
||||
// Delete merged P-frames — through delete() so the chunk vec index is
|
||||
// purged too (the old inline delete omitted memory_frame_chunks_vec).
|
||||
for (const pf of toMerge) {
|
||||
if (this.delete(pf.id)) pframesMerged++;
|
||||
}
|
||||
}
|
||||
|
||||
return { temporaryPruned, deprecatedPruned, pframesMerged };
|
||||
}
|
||||
|
||||
/** Get frame statistics for monitoring. */
|
||||
getStats(): { total: number; byType: Record<string, number>; byImportance: Record<string, number> } {
|
||||
const raw = this.db.getDatabase();
|
||||
const total = (raw.prepare('SELECT COUNT(*) as cnt FROM memory_frames').get() as { cnt: number }).cnt;
|
||||
|
||||
const byType: Record<string, number> = {};
|
||||
for (const row of raw.prepare('SELECT frame_type, COUNT(*) as cnt FROM memory_frames GROUP BY frame_type').all() as { frame_type: string; cnt: number }[]) {
|
||||
byType[row.frame_type] = row.cnt;
|
||||
}
|
||||
|
||||
const byImportance: Record<string, number> = {};
|
||||
for (const row of raw.prepare('SELECT importance, COUNT(*) as cnt FROM memory_frames GROUP BY importance').all() as { importance: string; cnt: number }[]) {
|
||||
byImportance[row.importance] = row.cnt;
|
||||
}
|
||||
|
||||
return { total, byType, byImportance };
|
||||
}
|
||||
|
||||
private nextT(gopId: string): number {
|
||||
const row = this.db.getDatabase().prepare(`
|
||||
SELECT COALESCE(MAX(t), -1) + 1 AS next_t FROM memory_frames WHERE gop_id = ?
|
||||
`).get(gopId) as { next_t: number };
|
||||
return row.next_t;
|
||||
}
|
||||
|
||||
private indexFts(frame: MemoryFrame): void {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO memory_frames_fts (rowid, content) VALUES (?, ?)
|
||||
`).run(frame.id, frame.content);
|
||||
}
|
||||
}
|
||||
64
packages/hive-mind-core/src/mind/fts-sanitize.ts
Normal file
64
packages/hive-mind-core/src/mind/fts-sanitize.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* fts-sanitize.ts — shared FTS5 OR-query sanitizer (S1 Unicode fix).
|
||||
*
|
||||
* One canonical copy of the sanitizer previously duplicated in
|
||||
* HybridSearch.keywordSearch (W3.6), MultiMind.ftsSearch (F6), and
|
||||
* raw-detail-lane's ftsOrQuery. The old `[^\w]` strip removed EVERY
|
||||
* non-ASCII letter — a Cyrillic ("Београд") or diacritic (č/ž/š/đ) query
|
||||
* sanitized to an empty MATCH string and keyword recall silently returned
|
||||
* [] even though the unicode61 FTS5 tokenizer handles those scripts fine.
|
||||
* Fixed with the Unicode-aware class `[^\p{L}\p{N}_]`, which is a no-op
|
||||
* for pure-ASCII input (`\w` ⊂ `\p{L}\p{N}_`), so English MATCH strings —
|
||||
* and therefore results and scores — are byte-identical to before.
|
||||
*
|
||||
* CJK tokens (Han/Hiragana/Katakana/Hangul) are deliberately EXCLUDED from
|
||||
* the OR query: unicode61 does not segment those scripts, so contiguous
|
||||
* prose is indexed as one long token and a per-word MATCH almost never
|
||||
* hits — the query "succeeds" with zero rows, which would also block any
|
||||
* parse-error fallback. Dropping CJK tokens leaves the OR query empty for
|
||||
* pure-CJK input; callers with a LIKE fallback (HybridSearch.keywordSearch)
|
||||
* detect that via `hasUnsegmentedScript` and use substring matching, which
|
||||
* is reliable for unsegmented text. This also sidesteps the `length > 2`
|
||||
* filter, which would have dropped typical 1–2 char CJK words. Other
|
||||
* unsegmented scripts (Thai, Khmer, Lao, …) can be added to
|
||||
* UNSEGMENTED_SCRIPT_RE if those markets materialize.
|
||||
*/
|
||||
|
||||
export const FTS_STOP_WORDS = new Set([
|
||||
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
||||
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
|
||||
'should', 'may', 'might', 'shall', 'can', 'to', 'of', 'in', 'for',
|
||||
'on', 'with', 'at', 'by', 'from', 'as', 'into', 'about', 'this',
|
||||
'that', 'these', 'those', 'it', 'its', 'my', 'your', 'our', 'their',
|
||||
'what', 'which', 'who', 'whom', 'how', 'when', 'where', 'why', 'all',
|
||||
'each', 'every', 'both', 'some', 'any', 'no', 'not', 'and', 'or', 'but',
|
||||
]);
|
||||
|
||||
/** Strip everything that is not a Unicode letter, digit, or underscore. */
|
||||
export function sanitizeFtsToken(word: string): string {
|
||||
return word.replace(/[^\p{L}\p{N}_]/gu, '');
|
||||
}
|
||||
|
||||
const UNSEGMENTED_SCRIPT_RE =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
||||
|
||||
/** True when the text contains a script unicode61 cannot word-segment (CJK). */
|
||||
export function hasUnsegmentedScript(text: string): boolean {
|
||||
return UNSEGMENTED_SCRIPT_RE.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an OR-based FTS5 MATCH string: tokenize on whitespace, strip
|
||||
* punctuation (Unicode-aware), drop stop words / tokens ≤ 2 chars / CJK
|
||||
* tokens, quote each survivor. Returns '' when nothing survives — callers
|
||||
* treat that as "no FTS signal" (and may route CJK queries to a LIKE
|
||||
* fallback, see module doc).
|
||||
*/
|
||||
export function buildFtsOrQuery(query: string): string {
|
||||
return query
|
||||
.split(/\s+/)
|
||||
.map(sanitizeFtsToken)
|
||||
.filter(w => w.length > 2 && !FTS_STOP_WORDS.has(w.toLowerCase()) && !hasUnsegmentedScript(w))
|
||||
.map(w => `"${w}"`)
|
||||
.join(' OR ');
|
||||
}
|
||||
80
packages/hive-mind-core/src/mind/identity.ts
Normal file
80
packages/hive-mind-core/src/mind/identity.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
export interface Identity {
|
||||
id: number;
|
||||
name: string;
|
||||
role: string;
|
||||
department: string;
|
||||
personality: string;
|
||||
capabilities: string;
|
||||
system_prompt: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
type IdentityInput = Omit<Identity, 'id' | 'created_at' | 'updated_at'>;
|
||||
type IdentityUpdate = Partial<IdentityInput>;
|
||||
|
||||
// Column allowlist — update() interpolates the key into SQL (values are
|
||||
// parameterized, keys are not), so only these known columns may be written.
|
||||
// Defense-in-depth against a malformed/attacker-shaped `changes` object.
|
||||
const UPDATABLE_COLUMNS: ReadonlySet<string> = new Set([
|
||||
'name', 'role', 'department', 'personality', 'capabilities', 'system_prompt',
|
||||
]);
|
||||
|
||||
export class IdentityLayer {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
create(input: IdentityInput): Identity {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO identity (id, name, role, department, personality, capabilities, system_prompt)
|
||||
VALUES (1, ?, ?, ?, ?, ?, ?)
|
||||
`).run(input.name, input.role, input.department, input.personality, input.capabilities, input.system_prompt);
|
||||
return this.get();
|
||||
}
|
||||
|
||||
get(): Identity {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM identity WHERE id = 1').get() as Identity | undefined;
|
||||
if (!row) throw new Error('No identity configured');
|
||||
return row;
|
||||
}
|
||||
|
||||
exists(): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT 1 FROM identity WHERE id = 1').get();
|
||||
return row !== undefined;
|
||||
}
|
||||
|
||||
update(changes: IdentityUpdate): Identity {
|
||||
if (!this.exists()) throw new Error('No identity configured');
|
||||
|
||||
const fields = Object.entries(changes).filter(([k, v]) => v !== undefined && UPDATABLE_COLUMNS.has(k));
|
||||
if (fields.length === 0) return this.get();
|
||||
|
||||
const sets = fields.map(([k]) => `${k} = ?`).join(', ');
|
||||
const values = fields.map(([, v]) => v);
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`UPDATE identity SET ${sets}, updated_at = datetime('now') WHERE id = 1`).run(...values);
|
||||
return this.get();
|
||||
}
|
||||
|
||||
toContext(): string {
|
||||
const id = this.get();
|
||||
const parts = [
|
||||
`Name: ${id.name}`,
|
||||
id.role && `Role: ${id.role}`,
|
||||
id.department && `Department: ${id.department}`,
|
||||
id.personality && `Personality: ${id.personality}`,
|
||||
id.capabilities && `Capabilities: ${id.capabilities}`,
|
||||
id.system_prompt && `System Prompt: ${id.system_prompt}`,
|
||||
].filter(Boolean);
|
||||
return parts.join('\n');
|
||||
}
|
||||
}
|
||||
174
packages/hive-mind-core/src/mind/improvement-signals.ts
Normal file
174
packages/hive-mind-core/src/mind/improvement-signals.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
export type SignalCategory = 'capability_gap' | 'correction' | 'workflow_pattern' | 'skill_promotion';
|
||||
|
||||
export interface ImprovementSignal {
|
||||
id: number;
|
||||
category: SignalCategory;
|
||||
pattern_key: string;
|
||||
detail: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
surfaced: number; // 0 or 1
|
||||
surfaced_at: string | null;
|
||||
metadata: string; // JSON
|
||||
}
|
||||
|
||||
export interface ActionableSignal extends ImprovementSignal {
|
||||
parsedMetadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ActionableThresholds {
|
||||
capability_gap?: number;
|
||||
correction?: number;
|
||||
workflow_pattern?: number;
|
||||
skill_promotion?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_THRESHOLDS: Required<ActionableThresholds> = {
|
||||
capability_gap: 2,
|
||||
correction: 3,
|
||||
workflow_pattern: 3,
|
||||
skill_promotion: 1, // one promotion request is actionable
|
||||
};
|
||||
|
||||
const MAX_ACTIONABLE = 3;
|
||||
|
||||
export class ImprovementSignalStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
/** Ensure improvement_signals table exists for databases created before this feature */
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='improvement_signals'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(`
|
||||
CREATE TABLE IF NOT EXISTS improvement_signals (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category TEXT NOT NULL CHECK (category IN ('capability_gap', 'correction', 'workflow_pattern', 'skill_promotion')),
|
||||
pattern_key TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
first_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
surfaced INTEGER NOT NULL DEFAULT 0,
|
||||
surfaced_at TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_signals_category_key ON improvement_signals (category, pattern_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_signals_category ON improvement_signals (category, count DESC);
|
||||
`);
|
||||
}
|
||||
} catch {
|
||||
// Database may already be closed during async teardown — safe to skip migration
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an improvement signal. Upserts: increments count + updates last_seen
|
||||
* if a signal with the same (category, pattern_key) already exists.
|
||||
*/
|
||||
record(
|
||||
category: SignalCategory,
|
||||
patternKey: string,
|
||||
detail?: string,
|
||||
metadata?: Record<string, unknown>,
|
||||
): ImprovementSignal {
|
||||
const raw = this.db.getDatabase();
|
||||
const metadataJson = metadata ? JSON.stringify(metadata) : '{}';
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO improvement_signals (category, pattern_key, detail, metadata)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (category, pattern_key) DO UPDATE SET
|
||||
count = count + 1,
|
||||
last_seen = datetime('now'),
|
||||
detail = CASE WHEN excluded.detail != '' THEN excluded.detail ELSE detail END,
|
||||
metadata = CASE WHEN excluded.metadata != '{}' THEN excluded.metadata ELSE metadata END
|
||||
`).run(category, patternKey, detail ?? '', metadataJson);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM improvement_signals WHERE category = ? AND pattern_key = ?',
|
||||
).get(category, patternKey) as ImprovementSignal;
|
||||
}
|
||||
|
||||
/** Get all signals for a category, ordered by count descending. */
|
||||
getByCategory(category: SignalCategory): ImprovementSignal[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM improvement_signals WHERE category = ? ORDER BY count DESC',
|
||||
).all(category) as ImprovementSignal[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get actionable signals: count >= threshold AND not yet surfaced.
|
||||
* Returns at most MAX_ACTIONABLE (3) signals, highest count first.
|
||||
* Per correction #6: threshold-based, capped, non-repeating once surfaced.
|
||||
*/
|
||||
getActionable(thresholds?: ActionableThresholds): ActionableSignal[] {
|
||||
const merged = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// Build a union query across categories with their respective thresholds
|
||||
const results: ImprovementSignal[] = [];
|
||||
|
||||
for (const [category, threshold] of Object.entries(merged)) {
|
||||
const rows = raw.prepare(`
|
||||
SELECT * FROM improvement_signals
|
||||
WHERE category = ? AND count >= ? AND surfaced = 0
|
||||
ORDER BY count DESC
|
||||
`).all(category, threshold) as ImprovementSignal[];
|
||||
results.push(...rows);
|
||||
}
|
||||
|
||||
// Sort by count descending, cap at MAX_ACTIONABLE
|
||||
results.sort((a, b) => b.count - a.count);
|
||||
|
||||
return results.slice(0, MAX_ACTIONABLE).map(signal => ({
|
||||
...signal,
|
||||
parsedMetadata: parseMetadata(signal.metadata),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Mark a signal as surfaced so it won't be returned by getActionable again. */
|
||||
markSurfaced(id: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
"UPDATE improvement_signals SET surfaced = 1, surfaced_at = datetime('now') WHERE id = ?",
|
||||
).run(id);
|
||||
}
|
||||
|
||||
/** Get a single signal by id. */
|
||||
get(id: number): ImprovementSignal | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM improvement_signals WHERE id = ?',
|
||||
).get(id) as ImprovementSignal | undefined;
|
||||
}
|
||||
|
||||
/** Get a signal by its category + pattern_key pair. */
|
||||
getByKey(category: SignalCategory, patternKey: string): ImprovementSignal | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM improvement_signals WHERE category = ? AND pattern_key = ?',
|
||||
).get(category, patternKey) as ImprovementSignal | undefined;
|
||||
}
|
||||
|
||||
/** Clear all signals (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM improvement_signals').run();
|
||||
}
|
||||
}
|
||||
|
||||
function parseMetadata(json: string): Record<string, unknown> {
|
||||
try {
|
||||
return JSON.parse(json);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
67
packages/hive-mind-core/src/mind/inprocess-embedder.ts
Normal file
67
packages/hive-mind-core/src/mind/inprocess-embedder.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
||||
* Default provider for ALL desktop users — zero config, works offline.
|
||||
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
||||
* Downloads ~23MB model on first use, cached in ~/.waggle/models/.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
export interface InProcessEmbedderConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
/** Normalize embedding dimensions: zero-pad shorter, truncate longer. */
|
||||
export function normalizeDimensions(embedding: Float32Array, targetDims: number): Float32Array {
|
||||
if (embedding.length === targetDims) return embedding;
|
||||
const result = new Float32Array(targetDims);
|
||||
const copyLen = Math.min(embedding.length, targetDims);
|
||||
result.set(embedding.subarray(0, copyLen));
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function createInProcessEmbedder(config?: Partial<InProcessEmbedderConfig>): Promise<Embedder> {
|
||||
const model = config?.model ?? 'Xenova/all-MiniLM-L6-v2';
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
log.info(`Loading in-process embedding model: ${model} (~23MB first download)`);
|
||||
|
||||
const { pipeline, env } = await import('@huggingface/transformers');
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
|
||||
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
||||
|
||||
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
return normalizeDimensions(raw, targetDims);
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
const results: Float32Array[] = [];
|
||||
// Process one at a time to avoid memory issues with large batches
|
||||
for (const text of texts) {
|
||||
const result = await extractor(text, { pooling: 'mean', normalize: true });
|
||||
const raw = new Float32Array(result.data as Float32Array);
|
||||
results.push(normalizeDimensions(raw, targetDims));
|
||||
}
|
||||
return results;
|
||||
},
|
||||
};
|
||||
}
|
||||
129
packages/hive-mind-core/src/mind/inprocess-reranker.ts
Normal file
129
packages/hive-mind-core/src/mind/inprocess-reranker.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* In-process cross-encoder reranker using @huggingface/transformers (ONNX).
|
||||
*
|
||||
* Cross-encoders take (query, doc) pairs and output a relevance score by
|
||||
* jointly attending to both — much more discriminating than vector dot
|
||||
* products. Use after RRF to rerank the top-K candidates from hybrid
|
||||
* search. Standard pattern in production RAG systems.
|
||||
*
|
||||
* Default model: Xenova/ms-marco-MiniLM-L-6-v2 — the canonical
|
||||
* SentenceTransformers cross-encoder, ~22MB on disk, ~30-50ms per pair
|
||||
* on CPU. Trained on MS MARCO passage ranking, generalizes well to
|
||||
* mixed-domain technical text.
|
||||
*
|
||||
* Alternative: Xenova/bge-reranker-base (~280MB, slightly higher quality
|
||||
* on out-of-domain queries). Set via `model` config.
|
||||
*
|
||||
* `@huggingface/transformers` is an optional peer dep — if not installed,
|
||||
* createInProcessReranker throws and the caller falls back to no reranking.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-reranker');
|
||||
|
||||
export interface Reranker {
|
||||
/**
|
||||
* Score a single (query, doc) pair. Higher = more relevant.
|
||||
* Score scale depends on the model — for ms-marco-MiniLM it's
|
||||
* roughly [-10, 10]; relative ordering is what matters.
|
||||
*/
|
||||
score(query: string, doc: string): Promise<number>;
|
||||
|
||||
/**
|
||||
* Score N pairs sharing one query. Same-shape result as score() but
|
||||
* amortises the model invocation when supported.
|
||||
*/
|
||||
scoreBatch(query: string, docs: string[]): Promise<number[]>;
|
||||
}
|
||||
|
||||
export interface InProcessRerankerConfig {
|
||||
model?: string;
|
||||
cacheDir?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a reranker backed by @huggingface/transformers. Throws if the
|
||||
* package isn't installed — caller is expected to catch and fall back.
|
||||
*/
|
||||
export async function createInProcessReranker(
|
||||
config?: Partial<InProcessRerankerConfig>,
|
||||
): Promise<Reranker> {
|
||||
const model = config?.model ?? 'Xenova/ms-marco-MiniLM-L-6-v2';
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.hive-mind', 'models');
|
||||
|
||||
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
|
||||
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification, env } = await import(
|
||||
'@huggingface/transformers'
|
||||
);
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(model);
|
||||
const seqModel = await AutoModelForSequenceClassification.from_pretrained(model, { dtype: 'fp32' });
|
||||
|
||||
log.info(`In-process reranker ready: ${model}`);
|
||||
|
||||
/** Score a single pair: tokenize, forward, extract logit. */
|
||||
async function scorePair(query: string, doc: string): Promise<number> {
|
||||
const inputs = await tokenizer(query, {
|
||||
text_pair: doc,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
|
||||
// Other cross-encoders may output 2 classes — take logit[0] - logit[1]
|
||||
// as a relevance score in that case.
|
||||
const logits = out.logits ?? out[0];
|
||||
const data = logits.data as Float32Array | number[];
|
||||
if (logits.dims && logits.dims[logits.dims.length - 1] === 2) {
|
||||
return Number(data[0]) - Number(data[1]);
|
||||
}
|
||||
return Number(data[0]);
|
||||
}
|
||||
|
||||
return {
|
||||
async score(query: string, doc: string): Promise<number> {
|
||||
return scorePair(query, doc);
|
||||
},
|
||||
|
||||
async scoreBatch(query: string, docs: string[]): Promise<number[]> {
|
||||
if (docs.length === 0) return [];
|
||||
// Tokenize all pairs together for batch inference. Padding aligns
|
||||
// sequences so the model can process them in one forward pass.
|
||||
const queries = docs.map(() => query);
|
||||
const inputs = await tokenizer(queries, {
|
||||
text_pair: docs,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
const logits = out.logits ?? out[0];
|
||||
const data = logits.data as Float32Array | number[];
|
||||
const dims = logits.dims;
|
||||
const lastDim = dims[dims.length - 1];
|
||||
|
||||
const scores: number[] = [];
|
||||
if (lastDim === 2) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
scores.push(Number(data[i * 2]) - Number(data[i * 2 + 1]));
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
scores.push(Number(data[i]));
|
||||
}
|
||||
}
|
||||
return scores;
|
||||
},
|
||||
};
|
||||
}
|
||||
454
packages/hive-mind-core/src/mind/knowledge.ts
Normal file
454
packages/hive-mind-core/src/mind/knowledge.ts
Normal file
@@ -0,0 +1,454 @@
|
||||
import type { MindDB } from './db.js';
|
||||
import { normalizeEntityName } from './entity-normalizer.js';
|
||||
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R2, 2026-06-11).
|
||||
/** Hardened JSON parse for entity/relation props — never throws, never returns non-objects. */
|
||||
function safeParseProps(json: string): Record<string, unknown> {
|
||||
try {
|
||||
const v = JSON.parse(json || '{}');
|
||||
return v && typeof v === 'object' ? (v as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export interface Entity {
|
||||
id: number;
|
||||
entity_type: string;
|
||||
name: string;
|
||||
properties: string; // JSON
|
||||
valid_from: string;
|
||||
valid_to: string | null;
|
||||
recorded_at: string;
|
||||
}
|
||||
|
||||
export interface Relation {
|
||||
id: number;
|
||||
source_id: number;
|
||||
target_id: number;
|
||||
relation_type: string;
|
||||
confidence: number;
|
||||
properties: string; // JSON
|
||||
valid_from: string;
|
||||
valid_to: string | null;
|
||||
recorded_at: string;
|
||||
}
|
||||
|
||||
export interface EntityTypeSchema {
|
||||
required: string[];
|
||||
allowedRelations: string[];
|
||||
}
|
||||
|
||||
export type ValidationSchema = Record<string, EntityTypeSchema>;
|
||||
|
||||
/**
|
||||
* Escape LIKE metacharacters (`%`, `_`) and the escape char itself (`\`) so a
|
||||
* user term is matched literally rather than as a wildcard pattern. Pair with an
|
||||
* `ESCAPE '\'` clause on the LIKE. Without this, `%` / `_` in a search term act
|
||||
* as wildcards and a literal `%` / `_` becomes unfindable.
|
||||
*/
|
||||
function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
}
|
||||
|
||||
export class KnowledgeGraph {
|
||||
private db: MindDB;
|
||||
private schema: ValidationSchema | null = null;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
setValidationSchema(schema: ValidationSchema): void {
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
// --- Entity operations ---
|
||||
|
||||
createEntity(entityType: string, name: string, properties: Record<string, unknown>, temporal?: { valid_from?: string; valid_to?: string }): Entity {
|
||||
this.validateEntityProperties(entityType, properties);
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
if (temporal?.valid_from || temporal?.valid_to) {
|
||||
const validFrom = temporal.valid_from ?? new Date().toISOString();
|
||||
const validTo = temporal.valid_to ?? null;
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO knowledge_entities (entity_type, name, properties, valid_from, valid_to)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(entityType, name, JSON.stringify(properties), validFrom, validTo);
|
||||
return raw.prepare('SELECT * FROM knowledge_entities WHERE id = ?').get(result.lastInsertRowid) as Entity;
|
||||
}
|
||||
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO knowledge_entities (entity_type, name, properties)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(entityType, name, JSON.stringify(properties));
|
||||
return raw.prepare('SELECT * FROM knowledge_entities WHERE id = ?').get(result.lastInsertRowid) as Entity;
|
||||
}
|
||||
|
||||
getEntity(id: number): Entity | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_entities WHERE id = ?'
|
||||
).get(id) as Entity | undefined;
|
||||
}
|
||||
|
||||
updateEntity(id: number, changes: { name?: string; properties?: Record<string, unknown> }): Entity {
|
||||
const raw = this.db.getDatabase();
|
||||
const sets: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
|
||||
if (changes.name !== undefined) {
|
||||
sets.push('name = ?');
|
||||
values.push(changes.name);
|
||||
}
|
||||
if (changes.properties !== undefined) {
|
||||
sets.push('properties = ?');
|
||||
values.push(JSON.stringify(changes.properties));
|
||||
}
|
||||
|
||||
if (sets.length > 0) {
|
||||
sets.push("recorded_at = datetime('now')");
|
||||
raw.prepare(`UPDATE knowledge_entities SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
|
||||
}
|
||||
return raw.prepare('SELECT * FROM knowledge_entities WHERE id = ?').get(id) as Entity;
|
||||
}
|
||||
|
||||
retireEntity(id: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
"UPDATE knowledge_entities SET valid_to = datetime('now') WHERE id = ?"
|
||||
).run(id);
|
||||
}
|
||||
|
||||
getEntitiesByType(entityType: string, limit = 500): Entity[] {
|
||||
if (!entityType) {
|
||||
return this.getEntities(limit);
|
||||
}
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_entities WHERE entity_type = ? AND valid_to IS NULL ORDER BY name LIMIT ?'
|
||||
).all(entityType, limit) as Entity[];
|
||||
}
|
||||
|
||||
/** 9c: Paginated entity listing — prevents unbounded fetches. */
|
||||
getEntities(limit = 200, offset = 0): Entity[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_entities WHERE valid_to IS NULL ORDER BY name LIMIT ? OFFSET ?'
|
||||
).all(limit, offset) as Entity[];
|
||||
}
|
||||
|
||||
/** Get entity type counts (for dashboard display without fetching all rows). */
|
||||
getEntityTypeCounts(): { type: string; count: number }[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT entity_type as type, COUNT(*) as count FROM knowledge_entities WHERE valid_to IS NULL GROUP BY entity_type ORDER BY count DESC'
|
||||
).all() as { type: string; count: number }[];
|
||||
}
|
||||
|
||||
/** Total active entity count. */
|
||||
getEntityCount(): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
'SELECT COUNT(*) as cnt FROM knowledge_entities WHERE valid_to IS NULL'
|
||||
).get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
searchEntities(query: string, limit = 100): Entity[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
"SELECT * FROM knowledge_entities WHERE name LIKE ? ESCAPE '\\' AND valid_to IS NULL ORDER BY name LIMIT ?"
|
||||
).all(`%${escapeLikeTerm(query)}%`, limit) as Entity[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact-name lookup. Returns the active entity whose name equals the
|
||||
* query (case-sensitive), or undefined.
|
||||
*
|
||||
* Use this instead of `searchEntities(name, 3).find(...)` for dedup —
|
||||
* the LIKE-based fuzzy search drops the exact match out of the top-K
|
||||
* window once enough similarly-named entities accumulate, which causes
|
||||
* dedup failures and runaway duplicate-row growth (e.g. 3506 copies of
|
||||
* "Phase" observed in the OSS repo because other names containing
|
||||
* "Phase" crowded the plain "Phase" out of a LIKE '%Phase%' top-K).
|
||||
*
|
||||
* Reverse-ported from OSS hive-mind (oss-drift triage R2, 2026-06-11).
|
||||
*/
|
||||
findEntityByName(name: string): Entity | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_entities WHERE name = ? AND valid_to IS NULL LIMIT 1'
|
||||
).get(name) as Entity | undefined;
|
||||
}
|
||||
|
||||
getEntitiesValidAt(isoTime: string, limit = 500): Entity[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_entities WHERE valid_from <= ? AND (valid_to IS NULL OR valid_to > ?) LIMIT ?'
|
||||
).all(isoTime, isoTime, limit) as Entity[];
|
||||
}
|
||||
|
||||
// --- Relation operations ---
|
||||
|
||||
createRelation(sourceId: number, targetId: number, relationType: string, confidence = 1.0, properties: Record<string, unknown> = {}): Relation {
|
||||
this.validateRelation(sourceId, relationType);
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO knowledge_relations (source_id, target_id, relation_type, confidence, properties)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(sourceId, targetId, relationType, confidence, JSON.stringify(properties));
|
||||
return raw.prepare('SELECT * FROM knowledge_relations WHERE id = ?').get(result.lastInsertRowid) as Relation;
|
||||
}
|
||||
|
||||
getRelation(id: number): Relation | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_relations WHERE id = ?'
|
||||
).get(id) as Relation | undefined;
|
||||
}
|
||||
|
||||
getRelationsFrom(sourceId: number, relationType?: string): Relation[] {
|
||||
if (relationType) {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_relations WHERE source_id = ? AND relation_type = ? AND valid_to IS NULL'
|
||||
).all(sourceId, relationType) as Relation[];
|
||||
}
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_relations WHERE source_id = ? AND valid_to IS NULL'
|
||||
).all(sourceId) as Relation[];
|
||||
}
|
||||
|
||||
getRelationsTo(targetId: number, relationType?: string): Relation[] {
|
||||
if (relationType) {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_relations WHERE target_id = ? AND relation_type = ? AND valid_to IS NULL'
|
||||
).all(targetId, relationType) as Relation[];
|
||||
}
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM knowledge_relations WHERE target_id = ? AND valid_to IS NULL'
|
||||
).all(targetId) as Relation[];
|
||||
}
|
||||
|
||||
retireRelation(id: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
"UPDATE knowledge_relations SET valid_to = datetime('now') WHERE id = ?"
|
||||
).run(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge active entities that share a normalized name + type. The survivor is
|
||||
* the entity with the most relations (ties broken by lowest/oldest id); each
|
||||
* duplicate's relations are re-pointed to the survivor, properties are merged
|
||||
* (survivor wins on key conflicts, but `seen_count` is summed), and the
|
||||
* duplicate is retired (bitemporal soft-delete). Runs in a single transaction.
|
||||
*
|
||||
* Reverse-ported from OSS hive-mind (oss-drift triage R2, 2026-06-11).
|
||||
*
|
||||
* @returns `{ groups }` duplicate groups processed, `{ merged }` entities retired.
|
||||
*/
|
||||
dedupeByName(): { groups: number; merged: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
const grouped = new Map<string, Entity[]>();
|
||||
for (const e of this.getEntities(100_000)) {
|
||||
const key = `${normalizeEntityName(e.name)}::${e.entity_type.toLowerCase()}`;
|
||||
let g = grouped.get(key);
|
||||
if (!g) {
|
||||
g = [];
|
||||
grouped.set(key, g);
|
||||
}
|
||||
g.push(e);
|
||||
}
|
||||
|
||||
let groups = 0;
|
||||
let merged = 0;
|
||||
const relCount = (id: number): number =>
|
||||
this.getRelationsFrom(id).length + this.getRelationsTo(id).length;
|
||||
|
||||
const tx = raw.transaction(() => {
|
||||
for (const group of grouped.values()) {
|
||||
if (group.length <= 1) continue;
|
||||
groups += 1;
|
||||
// Survivor = most relations; ties → lowest (oldest) id.
|
||||
const sorted = [...group].sort((a, b) => relCount(b.id) - relCount(a.id) || a.id - b.id);
|
||||
const keep = sorted[0];
|
||||
|
||||
for (const dup of sorted.slice(1)) {
|
||||
// Re-point the duplicate's relations onto the survivor, then retire them.
|
||||
for (const rel of this.getRelationsFrom(dup.id)) {
|
||||
try {
|
||||
this.createRelation(keep.id, rel.target_id, rel.relation_type, rel.confidence, safeParseProps(rel.properties));
|
||||
} catch {
|
||||
/* may already exist or be schema-rejected — the retire below still applies */
|
||||
}
|
||||
this.retireRelation(rel.id);
|
||||
}
|
||||
for (const rel of this.getRelationsTo(dup.id)) {
|
||||
try {
|
||||
this.createRelation(rel.source_id, keep.id, rel.relation_type, rel.confidence, safeParseProps(rel.properties));
|
||||
} catch {
|
||||
/* idem */
|
||||
}
|
||||
this.retireRelation(rel.id);
|
||||
}
|
||||
|
||||
// Merge properties: survivor wins on conflicts, seen_count is summed.
|
||||
const keepProps = safeParseProps(keep.properties);
|
||||
const dupProps = safeParseProps(dup.properties);
|
||||
const mergedProps: Record<string, unknown> = { ...dupProps, ...keepProps };
|
||||
mergedProps.seen_count =
|
||||
Number(keepProps.seen_count ?? 1) + Number(dupProps.seen_count ?? 1);
|
||||
this.updateEntity(keep.id, { properties: mergedProps });
|
||||
this.retireEntity(dup.id);
|
||||
merged += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
tx();
|
||||
return { groups, merged };
|
||||
}
|
||||
|
||||
// --- Graph traversal ---
|
||||
|
||||
traverse(startId: number, relationType: string, maxDepth: number): Entity[] {
|
||||
const visited = new Set<number>([startId]);
|
||||
const result: Entity[] = [];
|
||||
let frontier = [startId];
|
||||
|
||||
for (let depth = 0; depth < maxDepth && frontier.length > 0; depth++) {
|
||||
const nextFrontier: number[] = [];
|
||||
for (const nodeId of frontier) {
|
||||
const rels = this.getRelationsFrom(nodeId, relationType);
|
||||
for (const rel of rels) {
|
||||
if (!visited.has(rel.target_id)) {
|
||||
visited.add(rel.target_id);
|
||||
const entity = this.getEntity(rel.target_id);
|
||||
if (entity && entity.valid_to === null) {
|
||||
result.push(entity);
|
||||
nextFrontier.push(rel.target_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
frontier = nextFrontier;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
bfsDistances(startId: number, maxDepth: number): Map<number, number> {
|
||||
const distances = new Map<number, number>();
|
||||
const visited = new Set<number>([startId]);
|
||||
let frontier = [startId];
|
||||
|
||||
for (let depth = 1; depth <= maxDepth && frontier.length > 0; depth++) {
|
||||
const nextFrontier: number[] = [];
|
||||
for (const nodeId of frontier) {
|
||||
const rels = this.getRelationsFrom(nodeId);
|
||||
for (const rel of rels) {
|
||||
if (!visited.has(rel.target_id)) {
|
||||
visited.add(rel.target_id);
|
||||
distances.set(rel.target_id, depth);
|
||||
nextFrontier.push(rel.target_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
frontier = nextFrontier;
|
||||
}
|
||||
|
||||
return distances;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create entities extracted from a frame AND link each to that frame
|
||||
* (kg_entity_frames), returning the count created+linked. The link is the
|
||||
* provenance anchor that makes an entity reachable by frame-scoped operations —
|
||||
* notably GDPR Art.17 erasure's orphan sweep. Harvest routes previously called
|
||||
* createEntity WITHOUT linkEntityToFrame, so imported entity names (often PII)
|
||||
* were born orphaned and survived erasure; route imports through here instead.
|
||||
* Per-entity failures are swallowed (non-fatal import — mirrors the harvest loop).
|
||||
*/
|
||||
importEntitiesForFrame(
|
||||
frameId: number,
|
||||
entities: Array<{ name: string; type?: string }>,
|
||||
provenance: { source: string; importedFrom?: string },
|
||||
): number {
|
||||
let created = 0;
|
||||
for (const ent of entities) {
|
||||
try {
|
||||
const e = this.createEntity(ent.type || 'concept', ent.name, {
|
||||
source: provenance.source,
|
||||
imported_from: provenance.importedFrom,
|
||||
});
|
||||
this.linkEntityToFrame(e.id, frameId);
|
||||
created++;
|
||||
} catch { /* non-fatal per-entity (schema-rejected / bad name) — as in the harvest routes */ }
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
/** Link an entity to a frame it was extracted from (kg_entity_frames bridge).
|
||||
* Powers the 'contextual' scoring signal. Idempotent per (entity, frame). */
|
||||
linkEntityToFrame(entityId: number, frameId: number): void {
|
||||
try {
|
||||
this.db.getDatabase().prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
).run(entityId, frameId);
|
||||
} catch { /* bridge table absent on a pre-migration DB — best-effort */ }
|
||||
}
|
||||
|
||||
/** Seed entities whose name appears in free text (case-insensitive, name ≥3
|
||||
* chars), longest-name-first. Used to seed contextual scoring from a query. */
|
||||
findEntitiesInText(text: string, limit = 12): number[] {
|
||||
try {
|
||||
const rows = this.db.getDatabase().prepare(
|
||||
"SELECT id FROM knowledge_entities WHERE valid_to IS NULL AND length(name) >= 3 AND instr(lower(?), lower(name)) > 0 ORDER BY length(name) DESC LIMIT ?"
|
||||
).all(text, limit) as Array<{ id: number }>;
|
||||
return rows.map(r => r.id);
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
/** BFS from seed entities (≤maxDepth) and map the reached entities to the
|
||||
* frames they were extracted from via the kg_entity_frames bridge, returning
|
||||
* frameId → shortest graph distance. Empty when the bridge is unpopulated. */
|
||||
frameDistancesFromEntities(seedEntityIds: number[], maxDepth = 3): Map<number, number> {
|
||||
const frameDist = new Map<number, number>();
|
||||
if (seedEntityIds.length === 0) return frameDist;
|
||||
const entityDist = new Map<number, number>();
|
||||
for (const seed of seedEntityIds) {
|
||||
entityDist.set(seed, 0); // the seed entity itself is distance 0
|
||||
for (const [eid, d] of this.bfsDistances(seed, maxDepth)) {
|
||||
const prev = entityDist.get(eid);
|
||||
if (prev === undefined || d < prev) entityDist.set(eid, d);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const stmt = this.db.getDatabase().prepare(
|
||||
'SELECT frame_id FROM kg_entity_frames WHERE entity_id = ?'
|
||||
);
|
||||
for (const [eid, d] of entityDist) {
|
||||
for (const { frame_id } of stmt.all(eid) as Array<{ frame_id: number }>) {
|
||||
const prev = frameDist.get(frame_id);
|
||||
if (prev === undefined || d < prev) frameDist.set(frame_id, d);
|
||||
}
|
||||
}
|
||||
} catch { return new Map(); } // bridge absent — no contextual signal
|
||||
return frameDist;
|
||||
}
|
||||
|
||||
// --- Validation ---
|
||||
|
||||
private validateEntityProperties(entityType: string, properties: Record<string, unknown>): void {
|
||||
if (!this.schema || !this.schema[entityType]) return;
|
||||
const typeSchema = this.schema[entityType];
|
||||
|
||||
for (const required of typeSchema.required) {
|
||||
if (!(required in properties)) {
|
||||
throw new Error(`Validation failed: required property '${required}' missing for type '${entityType}'`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private validateRelation(sourceId: number, relationType: string): void {
|
||||
if (!this.schema) return;
|
||||
const source = this.getEntity(sourceId);
|
||||
if (!source) return;
|
||||
const typeSchema = this.schema[source.entity_type];
|
||||
if (!typeSchema) return;
|
||||
|
||||
if (!typeSchema.allowedRelations.includes(relationType)) {
|
||||
throw new Error(`Validation failed: relation '${relationType}' not allowed for type '${source.entity_type}'`);
|
||||
}
|
||||
}
|
||||
}
|
||||
93
packages/hive-mind-core/src/mind/litellm-embedder.ts
Normal file
93
packages/hive-mind-core/src/mind/litellm-embedder.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* LiteLLM-backed embedder that calls the /embeddings endpoint.
|
||||
* Falls back to a deterministic mock (text→Float32Array hash) on API error
|
||||
* when `fallbackToMock` is enabled.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
|
||||
export interface LiteLLMEmbedderConfig {
|
||||
litellmUrl: string;
|
||||
litellmApiKey?: string;
|
||||
model?: string;
|
||||
dimensions?: number;
|
||||
/** Custom fetch implementation (for testing). */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** If true, falls back to a deterministic mock on API error instead of throwing. */
|
||||
fallbackToMock?: boolean;
|
||||
}
|
||||
|
||||
function mockEmbed(text: string, dims: number): Float32Array {
|
||||
const arr = new Float32Array(dims);
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
for (let i = 0; i < Math.min(bytes.length, dims); i++) {
|
||||
arr[i] = (bytes[i] - 128) / 128;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
export function createLiteLLMEmbedder(config: LiteLLMEmbedderConfig): Embedder {
|
||||
const {
|
||||
litellmUrl,
|
||||
litellmApiKey,
|
||||
model = 'text-embedding',
|
||||
dimensions = 1024,
|
||||
fetch: fetchFn = globalThis.fetch,
|
||||
fallbackToMock = false,
|
||||
} = config;
|
||||
|
||||
// Normalise base URL — strip trailing /v1 if present, we add it ourselves
|
||||
const baseUrl = litellmUrl.replace(/\/v1\/?$/, '');
|
||||
const url = `${baseUrl}/v1/embeddings`;
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
if (litellmApiKey) {
|
||||
headers['Authorization'] = `Bearer ${litellmApiKey}`;
|
||||
}
|
||||
|
||||
async function callApi(input: string | string[]): Promise<Float32Array[]> {
|
||||
const body = JSON.stringify({ model, input });
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchFn(url, { method: 'POST', headers, body });
|
||||
} catch (err) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
if (fallbackToMock) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
return texts.map((t) => mockEmbed(t, dimensions));
|
||||
}
|
||||
const text = await response.text();
|
||||
throw new Error(`LiteLLM embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = (await response.json()) as {
|
||||
data: Array<{ embedding: number[] }>;
|
||||
};
|
||||
|
||||
return json.data.map((d) => new Float32Array(d.embedding));
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callApi(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callApi(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
58
packages/hive-mind-core/src/mind/ollama-embedder.ts
Normal file
58
packages/hive-mind-core/src/mind/ollama-embedder.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Ollama-backed embedder — calls local Ollama server for embeddings.
|
||||
* Power user option for users who have Ollama installed.
|
||||
*/
|
||||
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { normalizeDimensions } from './inprocess-embedder.js';
|
||||
|
||||
export interface OllamaEmbedderConfig {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
targetDimensions?: number;
|
||||
}
|
||||
|
||||
export function createOllamaEmbedder(config?: Partial<OllamaEmbedderConfig>): Embedder {
|
||||
const baseUrl = config?.baseUrl ?? 'http://localhost:11434';
|
||||
const model = config?.model ?? 'nomic-embed-text';
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
const url = `${baseUrl}/api/embed`;
|
||||
|
||||
async function callOllama(input: string | string[]): Promise<Float32Array[]> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`Ollama embeddings error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as { embeddings: number[][] };
|
||||
return json.embeddings.map(e => normalizeDimensions(new Float32Array(e), targetDims));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dimensions: targetDims,
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const results = await callOllama(text);
|
||||
return results[0];
|
||||
},
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return callOllama(texts);
|
||||
},
|
||||
};
|
||||
}
|
||||
58
packages/hive-mind-core/src/mind/ontology.ts
Normal file
58
packages/hive-mind-core/src/mind/ontology.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
export interface EntitySchema {
|
||||
required: string[];
|
||||
optional: string[];
|
||||
}
|
||||
|
||||
export class Ontology {
|
||||
private schemas = new Map<string, EntitySchema>();
|
||||
|
||||
define(type: string, schema: EntitySchema): void {
|
||||
this.schemas.set(type, schema);
|
||||
}
|
||||
|
||||
getSchema(type: string): EntitySchema | undefined {
|
||||
return this.schemas.get(type);
|
||||
}
|
||||
|
||||
hasType(type: string): boolean {
|
||||
return this.schemas.has(type);
|
||||
}
|
||||
|
||||
getTypes(): string[] {
|
||||
return Array.from(this.schemas.keys());
|
||||
}
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
valid: boolean;
|
||||
issues: string[];
|
||||
}
|
||||
|
||||
export function validateEntity(
|
||||
ontology: Ontology,
|
||||
entity: { type: string; properties: Record<string, unknown> },
|
||||
): ValidationResult {
|
||||
const issues: string[] = [];
|
||||
const schema = ontology.getSchema(entity.type);
|
||||
|
||||
if (!schema) {
|
||||
return { valid: false, issues: [`Unknown entity type: ${entity.type}`] };
|
||||
}
|
||||
|
||||
// Check required properties
|
||||
for (const prop of schema.required) {
|
||||
if (!(prop in entity.properties)) {
|
||||
issues.push(`Missing required property: ${prop}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for unknown properties
|
||||
const known = new Set([...schema.required, ...schema.optional]);
|
||||
for (const prop of Object.keys(entity.properties)) {
|
||||
if (!known.has(prop)) {
|
||||
issues.push(`Unknown property: ${prop}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: issues.length === 0, issues };
|
||||
}
|
||||
113
packages/hive-mind-core/src/mind/parse-date-window.ts
Normal file
113
packages/hive-mind-core/src/mind/parse-date-window.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* parse-date-window.ts — deterministic query-side temporal-constraint parser
|
||||
* (W4.1b, production port of the benchmark-proven Wave-3.1 date-window lane;
|
||||
* W4-PRODUCTION-PORT-PLAN-2026-06-11.md component #3, MRAG arXiv:2412.15540
|
||||
* pattern).
|
||||
*
|
||||
* When a query names an explicit period — "the last week of October 2023",
|
||||
* "early June 2023", "on 9 August 2023", "May 2023", "in 2022" — return a
|
||||
* `[since..until]` window (date-only `YYYY-MM-DD` bounds, inclusive) plus a
|
||||
* human label. Callers pass the window to HybridSearch's since/until filter;
|
||||
* the label feeds the future Events-during-X render section (component #6).
|
||||
*
|
||||
* Deterministic regex date math only — no LLM call, no new index. Returns
|
||||
* null when the query carries no explicit period (relative phrases like
|
||||
* "last week" / "two months ago" are resolution work for the WRITE side —
|
||||
* resolve-relative-date.ts — not query windowing).
|
||||
*
|
||||
* Regex shapes are byte-equivalent to the benchmark parser validated on the
|
||||
* full N=1540 LoCoMo run (wave3a); only types/docs differ.
|
||||
*
|
||||
* OSS-clean: pure date arithmetic, no vault/evolution/compliance deps.
|
||||
*/
|
||||
|
||||
/** Inclusive date-only window parsed from an explicit period in a query. */
|
||||
export interface DateWindow {
|
||||
/** Inclusive lower bound, `YYYY-MM-DD`. */
|
||||
since: string;
|
||||
/** Inclusive upper bound, `YYYY-MM-DD`. */
|
||||
until: string;
|
||||
/** Human-readable label of the matched period (for render sections). */
|
||||
label: string;
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
|
||||
july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
|
||||
};
|
||||
const MONTH_RE = '(january|february|march|april|may|june|july|august|september|october|november|december)';
|
||||
|
||||
function lastDayOfMonth(y: number, m: number): number {
|
||||
return new Date(Date.UTC(y, m, 0)).getUTCDate();
|
||||
}
|
||||
|
||||
function isoOf(y: number, m: number, d: number): string {
|
||||
return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an explicit-period temporal constraint out of a query. Returns null
|
||||
* when no explicit period is named.
|
||||
*/
|
||||
export function parseDateWindow(query: string): DateWindow | null {
|
||||
const t = String(query).toLowerCase();
|
||||
|
||||
// "first|second|third|fourth|last week of <month> <year>"
|
||||
let m = t.match(new RegExp(`(first|second|third|fourth|last)\\s+week\\s+of\\s+${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
|
||||
const last = lastDayOfMonth(y, mo);
|
||||
const ranges: Record<string, [number, number]> = {
|
||||
first: [1, 7], second: [8, 14], third: [15, 21], fourth: [22, 28],
|
||||
last: [Math.max(1, last - 6), last],
|
||||
};
|
||||
const [d1, d2] = ranges[m[1]];
|
||||
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, Math.min(d2, last)), label: `the ${m[1]} week of ${m[2]} ${y}` };
|
||||
}
|
||||
|
||||
// "early|mid|late <month> <year>"
|
||||
m = t.match(new RegExp(`(early|mid|late)\\s+${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[3], 10), mo = MONTHS[m[2]];
|
||||
const last = lastDayOfMonth(y, mo);
|
||||
const ranges: Record<string, [number, number]> = { early: [1, 10], mid: [11, 20], late: [21, last] };
|
||||
const [d1, d2] = ranges[m[1]];
|
||||
return { since: isoOf(y, mo, d1), until: isoOf(y, mo, d2), label: `${m[1]} ${m[2]} ${y}` };
|
||||
}
|
||||
|
||||
// "<day> <month> <year>" or "<month> <day>, <year>" → exact-day ±2 buffer
|
||||
m = t.match(new RegExp(`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(?:of\\s+)?${MONTH_RE},?\\s+(\\d{4})`)) ||
|
||||
t.match(new RegExp(`${MONTH_RE}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const isDayFirst = /^\d/.test(m[1]);
|
||||
const day = parseInt(isDayFirst ? m[1] : m[2], 10);
|
||||
const mo = MONTHS[isDayFirst ? m[2] : m[1]];
|
||||
const y = parseInt(m[3], 10);
|
||||
if (mo && day >= 1 && day <= 31) {
|
||||
const center = Date.UTC(y, mo - 1, day);
|
||||
const lo = new Date(center - 2 * 86400000), hi = new Date(center + 2 * 86400000);
|
||||
return {
|
||||
since: lo.toISOString().slice(0, 10),
|
||||
until: hi.toISOString().slice(0, 10),
|
||||
label: `${day} ${isDayFirst ? m[2] : m[1]} ${y}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// "<month> <year>" → whole month
|
||||
m = t.match(new RegExp(`${MONTH_RE}\\s+(\\d{4})`));
|
||||
if (m) {
|
||||
const y = parseInt(m[2], 10), mo = MONTHS[m[1]];
|
||||
return { since: isoOf(y, mo, 1), until: isoOf(y, mo, lastDayOfMonth(y, mo)), label: `${m[1]} ${y}` };
|
||||
}
|
||||
|
||||
// bare "in|during <year>" → whole year ('in'/'during' required so years
|
||||
// inside names/ids don't window the query)
|
||||
m = t.match(/\b(?:in|during)\s+(20\d{2})\b/);
|
||||
if (m) {
|
||||
const y = m[1];
|
||||
return { since: `${y}-01-01`, until: `${y}-12-31`, label: y };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
305
packages/hive-mind-core/src/mind/raw-archive.ts
Normal file
305
packages/hive-mind-core/src/mind/raw-archive.ts
Normal file
@@ -0,0 +1,305 @@
|
||||
/**
|
||||
* raw-archive.ts — #7 Verbatim Provenance Archive (2026-06-30).
|
||||
*
|
||||
* Append-only, immutable store of the FULL verbatim source of each harvested
|
||||
* item. Distilled/imported frames link back via memory_frames.metadata.archiveUid;
|
||||
* reconstructSource(frameId) resolves that link for audit / EU-AI-Act reconstruction.
|
||||
*
|
||||
* NOT part of the retrieval corpus (no FTS/vec, never fed to an LLM) — so unlike
|
||||
* raw-turns (which DROPS injection payloads because they feed recall), this store
|
||||
* keeps flagged content verbatim and records the flag. Idempotent on content sha256.
|
||||
* Append-only is enforced by DDL triggers; inserts use INSERT OR IGNORE (OR REPLACE
|
||||
* would DELETE+INSERT and trip the no-delete trigger).
|
||||
*/
|
||||
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import type { MindDB } from './db.js';
|
||||
import { SuppressionStore } from './suppression.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
|
||||
export interface ArchiveInput {
|
||||
source: string;
|
||||
sourceRef?: string;
|
||||
title?: string;
|
||||
content: string;
|
||||
sourceTimestamp?: string;
|
||||
}
|
||||
|
||||
export interface RawArchiveRow {
|
||||
id: number;
|
||||
archive_uid: string;
|
||||
source: string;
|
||||
source_ref: string | null;
|
||||
title: string | null;
|
||||
content: string;
|
||||
content_sha256: string;
|
||||
injection_flagged: 0 | 1;
|
||||
injection_flags: string;
|
||||
source_timestamp: string | null;
|
||||
created_at: string;
|
||||
/** 1 when `content` was truncated to the size cap at append; 0 otherwise. */
|
||||
truncated: 0 | 1;
|
||||
/** Pre-truncation character count when truncated=1; NULL otherwise. */
|
||||
original_length: number | null;
|
||||
/** GDPR Art.17: set once when this row's content has been redacted; NULL otherwise. */
|
||||
erased_at: string | null;
|
||||
erased_reason: string | null;
|
||||
}
|
||||
|
||||
/** Placed in `content` when a row is erased under GDPR Art.17 (right to erasure). */
|
||||
export const RAW_ARCHIVE_REDACTION_MARKER = '[REDACTED — GDPR Art.17 erasure]';
|
||||
|
||||
/**
|
||||
* Max characters of verbatim `content` stored per row. A larger item is stored
|
||||
* truncated to this prefix (with truncated=1 + original_length recorded) so a
|
||||
* single huge harvested export can't blow the append-only store. ~1M chars
|
||||
* (≈1–4 MB depending on encoding). Overridable per-store via the RawArchive
|
||||
* constructor (`{ maxContentChars }`).
|
||||
*/
|
||||
export const RAW_ARCHIVE_MAX_CONTENT_CHARS = 1_000_000;
|
||||
|
||||
/** sha256 hex over the raw, untouched content (NOT hashFrameContent — that strips/trims). */
|
||||
export function hashRaw(content: string): string {
|
||||
return createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all archive-uid links off a frame's parsed metadata, tolerating both the
|
||||
* canonical array (`archiveUids: string[]`) and the legacy scalar (`archiveUid:
|
||||
* string`). Returns the set-union (dedup, order: array first, then legacy scalar)
|
||||
* as a fresh array — [] when neither is present. Never mutates the input.
|
||||
*/
|
||||
export function readArchiveUids(meta: Record<string, unknown>): string[] {
|
||||
const out = new Set<string>();
|
||||
if (Array.isArray(meta.archiveUids)) {
|
||||
for (const u of meta.archiveUids) if (typeof u === 'string') out.add(u);
|
||||
}
|
||||
if (typeof meta.archiveUid === 'string') out.add(meta.archiveUid);
|
||||
return [...out];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a NEW metadata object with `uid` added to the canonical `archiveUids`
|
||||
* array, migrating any legacy scalar `archiveUid` into the array and dropping it.
|
||||
* Idempotent (set-union) and immutable (never mutates the input).
|
||||
*/
|
||||
export function withArchiveUid(meta: Record<string, unknown>, uid: string): Record<string, unknown> {
|
||||
const uids = new Set(readArchiveUids(meta));
|
||||
uids.add(uid);
|
||||
const { archiveUid: _legacy, ...rest } = meta;
|
||||
return { ...rest, archiveUids: [...uids] };
|
||||
}
|
||||
|
||||
export class RawArchive {
|
||||
private db: MindDB;
|
||||
private suppression: SuppressionStore;
|
||||
private readonly maxContentChars: number;
|
||||
constructor(db: MindDB, opts: { maxContentChars?: number } = {}) {
|
||||
this.db = db;
|
||||
this.suppression = new SuppressionStore(db);
|
||||
this.maxContentChars = opts.maxContentChars ?? RAW_ARCHIVE_MAX_CONTENT_CHARS;
|
||||
}
|
||||
|
||||
/** Idempotent append. INSERT OR IGNORE on the UNIQUE archive_uid makes a
|
||||
* re-append a no-op. Injection-scans (4KB probe) but stores verbatim.
|
||||
* #7 "sticky erasure": a subject on the erased-subject suppression list is NOT
|
||||
* re-materialized — the append is skipped (created:false). This makes stickiness
|
||||
* an INTRINSIC property of the archive (belt to the harvest loops' suspenders),
|
||||
* so a direct caller can't resurrect an erased subject by re-appending. */
|
||||
append(input: ArchiveInput): { archiveUid: string; created: boolean } {
|
||||
const raw = this.db.getDatabase();
|
||||
// archive_uid is PER-SOURCE: identical content from two different sources
|
||||
// keeps two provenance rows, so each frame's link resolves to ITS own source
|
||||
// (a content-only uid would collapse them and make reconstructSource return
|
||||
// the wrong source). Re-importing the same item from the same source still
|
||||
// collapses (idempotency). content_sha256 stays a content-only integrity
|
||||
// anchor — verify the verbatim, or find identical content across sources.
|
||||
const archiveUid = hashRaw(`${input.source}\x00${input.sourceRef ?? ''}\x00${input.content}`);
|
||||
// #7 sticky erasure: skip re-materializing an erased subject (fail-closed).
|
||||
if (this.suppression.isSuppressed(input.source, input.sourceRef ?? '')) {
|
||||
return { archiveUid, created: false };
|
||||
}
|
||||
const contentSha = hashRaw(input.content);
|
||||
// Size guard: cap a single row's stored blob so one giant harvested item can't
|
||||
// blow the store. archive_uid + content_sha256 are already derived from the
|
||||
// FULL content above (idempotency + integrity anchor unaffected); only the
|
||||
// stored `content` column is truncated, flagged by truncated=1 + original_length.
|
||||
const isTruncated = input.content.length > this.maxContentChars;
|
||||
const storedContent = isTruncated ? input.content.slice(0, this.maxContentChars) : input.content;
|
||||
// injection_flagged is a 4KB PROBE (same budget as the harvest pipeline's
|
||||
// Pass 0) — advisory, NOT a full-content guarantee. Content is stored
|
||||
// verbatim regardless (zero-loss); the archive is never fed to an LLM, and
|
||||
// any consumer that surfaces it to a model MUST re-scan.
|
||||
const scan = scanForInjection(input.content.slice(0, 4000), 'tool_output');
|
||||
const result = raw.prepare(
|
||||
`INSERT OR IGNORE INTO raw_archive
|
||||
(archive_uid, source, source_ref, title, content, content_sha256,
|
||||
injection_flagged, injection_flags, source_timestamp, truncated, original_length)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
archiveUid,
|
||||
input.source,
|
||||
input.sourceRef ?? null,
|
||||
input.title ?? null,
|
||||
storedContent,
|
||||
contentSha,
|
||||
scan.safe ? 0 : 1,
|
||||
scan.safe ? '' : scan.flags.join(','),
|
||||
input.sourceTimestamp ?? null,
|
||||
isTruncated ? 1 : 0,
|
||||
isTruncated ? input.content.length : null,
|
||||
);
|
||||
return { archiveUid, created: result.changes > 0 };
|
||||
}
|
||||
|
||||
getByUid(archiveUid: string): RawArchiveRow | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM raw_archive WHERE archive_uid = ?')
|
||||
.get(archiveUid) as RawArchiveRow | undefined;
|
||||
}
|
||||
|
||||
/** Look up a row by its stable numeric id — the audit handle that survives an
|
||||
* Art.17 erasure's archive_uid rotation (unlike getByUid, which can no longer
|
||||
* find an erased row by its original content-derived uid). Undefined if absent. */
|
||||
getById(id: number): RawArchiveRow | undefined {
|
||||
return this.db.getDatabase()
|
||||
.prepare('SELECT * FROM raw_archive WHERE id = ?')
|
||||
.get(id) as RawArchiveRow | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve every archive-uid link on a frame → its archive rows. Accepts both
|
||||
* the canonical `archiveUids: string[]` and the legacy scalar `archiveUid`
|
||||
* (via readArchiveUids). Returns [] when no frame / no metadata / malformed
|
||||
* metadata / no resolvable rows. Order preserved; unresolved uids dropped.
|
||||
*/
|
||||
reconstructSource(frameId: number): RawArchiveRow[] {
|
||||
const row = this.db.getDatabase()
|
||||
.prepare('SELECT metadata FROM memory_frames WHERE id = ?')
|
||||
.get(frameId) as { metadata?: string } | undefined;
|
||||
if (!row?.metadata) return [];
|
||||
let meta: Record<string, unknown>;
|
||||
try { meta = JSON.parse(row.metadata) as Record<string, unknown>; }
|
||||
catch { return []; }
|
||||
if (!meta || typeof meta !== 'object') return [];
|
||||
const rows: RawArchiveRow[] = [];
|
||||
for (const uid of readArchiveUids(meta)) {
|
||||
const r = this.getByUid(uid);
|
||||
if (r) rows.push(r);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
list(opts: { limit?: number; offset?: number; source?: string } = {}): RawArchiveRow[] {
|
||||
const { limit = 100, offset = 0, source } = opts;
|
||||
if (source) {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM raw_archive WHERE source = ? ORDER BY created_at DESC LIMIT ? OFFSET ?'
|
||||
).all(source, limit, offset) as RawArchiveRow[];
|
||||
}
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM raw_archive ORDER BY created_at DESC LIMIT ? OFFSET ?'
|
||||
).all(limit, offset) as RawArchiveRow[];
|
||||
}
|
||||
|
||||
count(): number {
|
||||
return (this.db.getDatabase().prepare('SELECT COUNT(*) as c FROM raw_archive').get() as { c: number }).c;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reclaim disk pages freed by in-place Art.17 redaction. raw_archive is
|
||||
* append-only (DELETE is trigger-blocked) and erase() only overwrites `content`
|
||||
* with a short marker IN PLACE, so the freed bytes stay allocated to the file
|
||||
* until a VACUUM rewrites it. VACUUM compacts the ENTIRE .mind file and does NOT
|
||||
* fire the no-delete trigger (that guards DML, not the internal rebuild), so the
|
||||
* append-only invariant survives untouched.
|
||||
*
|
||||
* This is an EXPLICIT maintenance/governance action: never auto-invoked, and it
|
||||
* never deletes user data — it only compacts already-freed space. Must run
|
||||
* OUTSIDE any transaction (SQLite forbids VACUUM inside one) and with no other
|
||||
* statement in progress on this connection. Returns bytes reclaimed (>= 0).
|
||||
*/
|
||||
reclaim(): number {
|
||||
const raw = this.db.getDatabase();
|
||||
const fileBytes = (): number =>
|
||||
(raw.pragma('page_count', { simple: true }) as number) *
|
||||
(raw.pragma('page_size', { simple: true }) as number);
|
||||
const before = fileBytes();
|
||||
raw.exec('VACUUM');
|
||||
const after = fileBytes();
|
||||
return Math.max(0, before - after);
|
||||
}
|
||||
|
||||
/**
|
||||
* GDPR Art.17 right-to-erasure. Redacts ONE archive row in place: content ->
|
||||
* marker, content_sha256 -> '', title -> NULL, ROTATES archive_uid to an opaque
|
||||
* random id, and stamps erased_at/erased_reason. The audit skeleton (id/source/
|
||||
* source_ref/timestamps/injection flags) is frozen — the record that an item
|
||||
* existed and was erased survives — but every CONTENT-DERIVED value is gone. The
|
||||
* refined raw_archive_no_update trigger permits exactly this one canonical outcome
|
||||
* (content=marker, content_sha256='', title NULL, archive_uid rotated ≠ OLD) — raw
|
||||
* SQL cannot use the erase path to forge audit content. Idempotent: the uid is
|
||||
* rotated, so a second call on the ORIGINAL uid matches nothing and returns false.
|
||||
*
|
||||
* WHY ROTATE archive_uid: it was sha256(source ∥ sourceRef ∥ content), so freezing
|
||||
* it left a re-identification vector — for low-entropy content an auditor holding
|
||||
* the "erased" DB could brute-force the original by hashing candidates. A fresh
|
||||
* 256-bit RANDOM uid (not derived from OLD) severs that linkage; the row stays
|
||||
* findable by its frozen id (getById), not by the original uid.
|
||||
*
|
||||
* RETAINED-SKELETON RESIDUAL (founder-ratified — keep skeleton over max erasure):
|
||||
* - source_ref is preserved verbatim and MAY carry PII (thread-id/filename/URL);
|
||||
* harvest adapters should avoid placing raw identifiers there.
|
||||
* CONSEQUENCE — re-harvest is no longer suppressed: rotating the uid frees append()'s
|
||||
* content→uid dedup key, so re-importing an already-erased source re-materializes its
|
||||
* archive row (its searchable summary/raw-turns/KG already re-materialize on re-import
|
||||
* today, independent of this — a pre-existing gap). "Sticky" erasure across re-import is
|
||||
* a separate cross-path compliance feature (an erased-subject suppression list); a
|
||||
* content-keyed tombstone is NOT the fix — it would reintroduce the very content-derived
|
||||
* re-identification vector this rotation removes.
|
||||
* THREAT MODEL: append-only + erase-once are trigger-enforced — tamper-EVIDENT
|
||||
* against ordinary INSERT/UPDATE/DELETE, NOT tamper-proof against a caller with DDL
|
||||
* rights (DROP TRIGGER/TABLE bypasses it).
|
||||
*/
|
||||
erase(archiveUid: string, reason: string): boolean {
|
||||
// Opaque, RANDOM replacement uid (not derived from OLD/content). 256-bit random
|
||||
// → collision-safe under UNIQUE and guaranteed ≠ OLD, which the trigger requires.
|
||||
const opaqueUid = `erased:${randomBytes(32).toString('hex')}`;
|
||||
const res = this.db.getDatabase().prepare(
|
||||
`UPDATE raw_archive
|
||||
SET archive_uid = ?, content = ?, content_sha256 = '', title = NULL,
|
||||
erased_at = datetime('now'), erased_reason = ?
|
||||
WHERE archive_uid = ? AND erased_at IS NULL`
|
||||
).run(opaqueUid, RAW_ARCHIVE_REDACTION_MARKER, reason, archiveUid);
|
||||
return res.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact every archive row a frame links to (via its metadata.archiveUids, incl.
|
||||
* the legacy scalar). Returns the count of rows newly redacted (already-erased
|
||||
* rows are skipped).
|
||||
*
|
||||
* SCOPE — provenance rows ONLY. This is NOT a complete GDPR Art.17 data-subject
|
||||
* erasure: the DERIVED memory_frames (whose summaries quote the source) and their
|
||||
* FTS / vector / KnowledgeGraph projections still hold the PII and remain
|
||||
* searchable and recall-able. A full DSAR flow MUST pair this with a frame + index
|
||||
* + KG erasure. It also reaches only rows linked from THIS frame — orphan rows,
|
||||
* other frames, and same-source_ref rows are not swept (a subject-level sweep by
|
||||
* source_ref is a follow-up).
|
||||
*/
|
||||
eraseByFrame(frameId: number, reason: string): number {
|
||||
const row = this.db.getDatabase()
|
||||
.prepare('SELECT metadata FROM memory_frames WHERE id = ?')
|
||||
.get(frameId) as { metadata?: string } | undefined;
|
||||
if (!row?.metadata) return 0;
|
||||
let meta: Record<string, unknown>;
|
||||
try { meta = JSON.parse(row.metadata) as Record<string, unknown>; }
|
||||
catch { return 0; }
|
||||
if (!meta || typeof meta !== 'object') return 0;
|
||||
let erased = 0;
|
||||
for (const uid of readArchiveUids(meta)) {
|
||||
if (this.erase(uid, reason)) erased++;
|
||||
}
|
||||
return erased;
|
||||
}
|
||||
}
|
||||
187
packages/hive-mind-core/src/mind/raw-detail-lane.ts
Normal file
187
packages/hive-mind-core/src/mind/raw-detail-lane.ts
Normal file
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* raw-detail-lane.ts — W4.6 RAWDETAIL escalation lane (recall side).
|
||||
*
|
||||
* Benchmark-proven retrieval over verbatim dialogue turns (LoCoMo W3.3:
|
||||
* single-hop 92.75 +4.52 z=3.16; W3.4 ablation: this lane is the delivery
|
||||
* mechanism, +2.40 z=1.95 on top of caption parity). Pipeline:
|
||||
*
|
||||
* pool (date-window filtered turns, else FTS BM25 top-60)
|
||||
* → cross-encoder rerank, keep top K (default 6)
|
||||
* → expand ±1 dialogue neighbors (gold often sits ADJACENT to the top
|
||||
* CE hit — Q→A conversational adjacency)
|
||||
* → dedup vs already-rendered frames, chronological render order
|
||||
*
|
||||
* Turns are stored by `harvest/raw-turns.ts` with explicit
|
||||
* `conv:<key> turn:<n>` header keys — production frames interleave across
|
||||
* sources, so adjacency is looked up per-conversation by turn index, not
|
||||
* by row-id ordering (the benchmark's trick that does not transfer).
|
||||
*
|
||||
* The lane REQUIRES a reranker (the CE step is what makes the pool pay —
|
||||
* P5 anti-goal: no relevance-only episodic injection without the CE floor);
|
||||
* callers skip the lane entirely when no reranker is available.
|
||||
*/
|
||||
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import type { Reranker } from './inprocess-reranker.js';
|
||||
import { buildFtsOrQuery } from './fts-sanitize.js';
|
||||
import { MIND_RAWTURN_PREFIX, parseRawTurnHeader } from '../harvest/raw-turns.js';
|
||||
|
||||
/** CE survivors kept before neighbor expansion (benchmark RAWDETAIL_K). */
|
||||
export const RAW_DETAIL_K = 6;
|
||||
/** FTS pool size (benchmark: BM25 top-60). */
|
||||
const FTS_POOL_LIMIT = 60;
|
||||
/** Window pools larger than this get FTS-intersected (benchmark: 120). */
|
||||
const WINDOW_POOL_MAX = 120;
|
||||
|
||||
export interface RawTurnHit {
|
||||
id: number;
|
||||
content: string;
|
||||
/** Provenance class (memory_frames.source) — carried for the auto_recall
|
||||
* provenance breakdown (raw-turn frames are harvest imports). */
|
||||
source: string;
|
||||
created_at: string;
|
||||
conv: string;
|
||||
turn: number;
|
||||
speaker: string;
|
||||
}
|
||||
|
||||
export interface RawDetailLaneOptions {
|
||||
/** CE survivors before neighbor expansion (default RAW_DETAIL_K = 6). */
|
||||
k?: number;
|
||||
/** Explicit-period window from the query (recallMemory's parseDateWindow). */
|
||||
window?: { since: string; until: string } | null;
|
||||
/** Frame ids already rendered by other lanes — excluded from the result. */
|
||||
excludeIds?: Set<number>;
|
||||
}
|
||||
|
||||
type FrameRow = { id: number; content: string; source: string; created_at: string };
|
||||
|
||||
/** Body of a raw-turn frame (everything after the header line). */
|
||||
export function rawTurnBody(content: string): string {
|
||||
const nl = content.indexOf('\n');
|
||||
return nl >= 0 ? content.slice(nl + 1).trim() : content;
|
||||
}
|
||||
|
||||
/** FTS BM25 top-N restricted to raw-turn frames. Returns [] on FTS parse errors.
|
||||
* OR-query sanitizer shared with HybridSearch.keywordSearch (S1, fts-sanitize.ts). */
|
||||
function ftsPool(db: DatabaseType, query: string, limit: number): FrameRow[] {
|
||||
const match = buildFtsOrQuery(query);
|
||||
if (!match) return [];
|
||||
try {
|
||||
return db.prepare(
|
||||
`SELECT mf.id, mf.content, mf.source, mf.created_at
|
||||
FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.content LIKE '${MIND_RAWTURN_PREFIX} %'
|
||||
ORDER BY rank
|
||||
LIMIT ?`
|
||||
).all(match, limit) as FrameRow[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** All raw-turn frames whose created_at date falls inside [since..until]. */
|
||||
function windowPool(db: DatabaseType, since: string, until: string): FrameRow[] {
|
||||
return db.prepare(
|
||||
`SELECT id, content, source, created_at FROM memory_frames
|
||||
WHERE content LIKE '${MIND_RAWTURN_PREFIX} %'
|
||||
AND substr(created_at, 1, 10) >= ? AND substr(created_at, 1, 10) <= ?
|
||||
ORDER BY id ASC`
|
||||
).all(since, until) as FrameRow[];
|
||||
}
|
||||
|
||||
/** Fetch every stored turn of one conversation, keyed by turn index. */
|
||||
function convTurnMap(db: DatabaseType, conv: string): Map<number, FrameRow> {
|
||||
// conv keys are sanitized to [A-Za-z0-9_-] at write time — no LIKE
|
||||
// metacharacters can appear, so direct interpolation into the pattern
|
||||
// parameter (still a BOUND parameter) is safe.
|
||||
const rows = db.prepare(
|
||||
`SELECT id, content, source, created_at FROM memory_frames
|
||||
WHERE content LIKE ?`
|
||||
).all(`${MIND_RAWTURN_PREFIX} conv:${conv} %`) as FrameRow[];
|
||||
const map = new Map<number, FrameRow>();
|
||||
for (const r of rows) {
|
||||
const h = parseRawTurnHeader(r.content);
|
||||
if (h && h.conv === conv) map.set(h.turn, r);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the RAWDETAIL lane over one mind's raw-turn frames.
|
||||
* Returns CE-top turns ±1 dialogue neighbors, chronologically ordered,
|
||||
* excluding `excludeIds`. Empty array when no turns / no pool / no signal.
|
||||
*/
|
||||
export async function fetchRawDetailLane(
|
||||
db: DatabaseType,
|
||||
query: string,
|
||||
reranker: Reranker,
|
||||
opts: RawDetailLaneOptions = {},
|
||||
): Promise<RawTurnHit[]> {
|
||||
const k = opts.k ?? RAW_DETAIL_K;
|
||||
const excludeIds = opts.excludeIds ?? new Set<number>();
|
||||
|
||||
// ── Pool ──────────────────────────────────────────────────────────────
|
||||
let pool: FrameRow[] = [];
|
||||
if (opts.window) {
|
||||
pool = windowPool(db, opts.window.since, opts.window.until);
|
||||
if (pool.length > WINDOW_POOL_MAX) {
|
||||
// Benchmark behavior: oversized window → intersect with FTS top-60;
|
||||
// FTS-empty falls back to the first WINDOW_POOL_MAX turns.
|
||||
const winIds = new Set(pool.map(t => t.id));
|
||||
const fts = ftsPool(db, query, 200).filter(t => winIds.has(t.id)).slice(0, FTS_POOL_LIMIT);
|
||||
pool = fts.length > 0 ? fts : pool.slice(0, WINDOW_POOL_MAX);
|
||||
}
|
||||
// Window matched nothing (period off-corpus) → fall through to FTS so
|
||||
// the lane never LOSES recall, only sharpens it.
|
||||
if (pool.length === 0) pool = ftsPool(db, query, FTS_POOL_LIMIT);
|
||||
} else {
|
||||
pool = ftsPool(db, query, FTS_POOL_LIMIT);
|
||||
}
|
||||
if (pool.length === 0) return [];
|
||||
|
||||
// ── Cross-encoder rerank → top K ─────────────────────────────────────
|
||||
const docs = pool.map(t => rawTurnBody(t.content));
|
||||
let scores: number[];
|
||||
try {
|
||||
scores = await reranker.scoreBatch(query, docs);
|
||||
} catch {
|
||||
return []; // soft-fail: a broken reranker never kills recall
|
||||
}
|
||||
const top = pool
|
||||
.map((t, i) => ({ t, s: scores[i] ?? -Infinity }))
|
||||
.sort((a, b) => b.s - a.s)
|
||||
.slice(0, k)
|
||||
.map(x => x.t);
|
||||
|
||||
// ── ±1 dialogue-neighbor expansion ───────────────────────────────────
|
||||
const keptById = new Map<number, FrameRow>();
|
||||
const convMaps = new Map<string, Map<number, FrameRow>>();
|
||||
for (const t of top) {
|
||||
const h = parseRawTurnHeader(t.content);
|
||||
if (!h) continue;
|
||||
let turns = convMaps.get(h.conv);
|
||||
if (!turns) {
|
||||
turns = convTurnMap(db, h.conv);
|
||||
convMaps.set(h.conv, turns);
|
||||
}
|
||||
for (const d of [-1, 0, 1]) {
|
||||
const n = turns.get(h.turn + d);
|
||||
if (n && !excludeIds.has(n.id)) keptById.set(n.id, n);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Chronological render order (date, then conv, then turn) ─────────
|
||||
const hits: RawTurnHit[] = [];
|
||||
for (const r of keptById.values()) {
|
||||
const h = parseRawTurnHeader(r.content);
|
||||
if (!h) continue;
|
||||
hits.push({ ...r, conv: h.conv, turn: h.turn, speaker: h.speaker });
|
||||
}
|
||||
hits.sort((a, b) =>
|
||||
String(a.created_at).localeCompare(String(b.created_at))
|
||||
|| a.conv.localeCompare(b.conv)
|
||||
|| a.turn - b.turn);
|
||||
return hits;
|
||||
}
|
||||
100
packages/hive-mind-core/src/mind/recall-context.ts
Normal file
100
packages/hive-mind-core/src/mind/recall-context.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* recall-context.ts — shared renderer for surfacing temporal information in
|
||||
* recalled-memory blocks. Single source of truth imported by both the Waggle
|
||||
* agent (production) and the benchmark harness, so the injected-memory format
|
||||
* never drifts between the two.
|
||||
*
|
||||
* Scope (Temporal Substrate Fix, Phase 1 — "surface time", additive only):
|
||||
* - Prefix each retrieved snippet with its own compact `[YYYY-MM-DD]` date.
|
||||
* - Open the rendered memory block with one anchor line giving the most-recent
|
||||
* memory date, so the model has a concrete "now" to resolve relative time
|
||||
* expressions against.
|
||||
* - Export `TEMPORAL_GUIDANCE`, the prompt fragment that tells the model to
|
||||
* treat those timestamps as the anchor for relative-time arithmetic.
|
||||
*
|
||||
* Distilled "Memory Facts" (cross-session syntheses) are intentionally NOT
|
||||
* dated here — they have no single reliable date. Phase 2 handles them.
|
||||
*
|
||||
* OSS-clean: pure string formatting, no vault/evolution/compliance deps.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Prompt fragment wired into the memory-recall injection path (NOT the global
|
||||
* system prompt). Teaches the model to use the surfaced `[YYYY-MM-DD]` stamps
|
||||
* as the anchor for resolving relative time expressions and conflicting facts.
|
||||
*
|
||||
* W4.1 upgrade (W4-PRODUCTION-PORT-PLAN-2026-06-11.md §1): replaced the original
|
||||
* "nearest timestamp as anchor" phrasing with the benchmark-proven W1 wording —
|
||||
* concrete relative-date arithmetic with worked examples (Memori instruction-5
|
||||
* lineage, incl. the verified conv-26 "yesterday" failure case) plus the
|
||||
* granularity-calibration clause (failure mining: 22 temporal fails emitted a
|
||||
* confident exact ISO day 1-7 days off where a coarse answer was correct).
|
||||
* Temporal was the #1 LoCoMo lever (80.06 → 84.7 across W1-W3.1).
|
||||
* Production-safe subset: no never-refuse clause (that was benchmark-cell
|
||||
* policy only — conditional abstention stays).
|
||||
*/
|
||||
export const TEMPORAL_GUIDANCE =
|
||||
"Memories and snippets are timestamped [YYYY-MM-DD]. Pay special attention to these " +
|
||||
"timestamps to determine timing. If a question involves relative time references " +
|
||||
"('last year', 'two months ago', 'yesterday', 'last week'), CALCULATE the actual date " +
|
||||
"from the timestamp of the memory that mentions it. For example: a memory dated " +
|
||||
"4 May 2022 that says 'went to India last year' means the trip was in 2021; a memory " +
|
||||
"dated 8 May 2023 that says 'I went to the group yesterday' means the event was 7 May 2023. " +
|
||||
"Always convert relative references to specific dates, months, or years using the " +
|
||||
"memory's timestamp as the anchor, and ignore the relative phrase itself when answering. " +
|
||||
"When the same fact appears at different times, the most recent version is correct. " +
|
||||
"GRANULARITY: state an exact day ONLY when that exact date was explicitly stated or " +
|
||||
"directly computed from an explicit relative reference; otherwise answer at the " +
|
||||
"granularity you are confident in — 'early June 2023', 'the week before 9 August 2023', " +
|
||||
"'August 2022'. A confidently wrong exact day is worse than a correct coarse answer. " +
|
||||
"For 'how long / how many months' duration questions, give ONLY the final value " +
|
||||
"(e.g. 'six months') — no intermediate dates, no reasoning steps.";
|
||||
|
||||
/** Anchor-line prefix for the most-recent rendered memory date. */
|
||||
const REFERENCE_DATE_LABEL = 'Reference date (most recent memory):';
|
||||
|
||||
/**
|
||||
* Slice an ISO-ish timestamp to its `YYYY-MM-DD` date prefix. Returns null when
|
||||
* the value is missing or too short to carry a date.
|
||||
*/
|
||||
export function toDatePrefix(createdAt: string | null | undefined): string | null {
|
||||
if (!createdAt || createdAt.length < 10) return null;
|
||||
return createdAt.slice(0, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a single retrieved snippet with its compact `[YYYY-MM-DD]` date prefix.
|
||||
* `text` is the snippet text exactly as it would otherwise be rendered (the
|
||||
* caller owns importance labels, truncation, etc.); this only prepends the date.
|
||||
* Falls back to the bare text when the hit carries no usable timestamp.
|
||||
*/
|
||||
export function renderDatedSnippet(createdAt: string | null | undefined, text: string): string {
|
||||
const date = toDatePrefix(createdAt);
|
||||
return date ? `[${date}] ${text}` : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the reference (anchor) date: the maximum `created_at` among the
|
||||
* supplied timestamps, sliced to `YYYY-MM-DD`. Returns null when none carry a
|
||||
* usable date (caller then omits the anchor line).
|
||||
*/
|
||||
export function referenceDate(createdAts: ReadonlyArray<string | null | undefined>): string | null {
|
||||
let max: string | null = null;
|
||||
for (const c of createdAts) {
|
||||
const date = toDatePrefix(c);
|
||||
if (date && (max === null || date > max)) max = date;
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the single anchor line for a memory block, e.g.
|
||||
* `Reference date (most recent memory): 2026-06-09`. Returns null when no date
|
||||
* is available.
|
||||
*/
|
||||
export function renderReferenceDateLine(
|
||||
createdAts: ReadonlyArray<string | null | undefined>,
|
||||
): string | null {
|
||||
const date = referenceDate(createdAts);
|
||||
return date ? `${REFERENCE_DATE_LABEL} ${date}` : null;
|
||||
}
|
||||
168
packages/hive-mind-core/src/mind/reconcile.ts
Normal file
168
packages/hive-mind-core/src/mind/reconcile.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* Index Reconciliation — repairs FTS5 and vector indexes for memory frames.
|
||||
*
|
||||
* If the process crashes between frame creation and FTS5/vector indexing,
|
||||
* frames exist but aren't searchable. This function finds orphaned frames
|
||||
* and re-indexes them. Designed to run as a periodic maintenance cron job.
|
||||
*
|
||||
* Idempotent: safe to run multiple times without side effects.
|
||||
*/
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
|
||||
export interface ReconcileResult {
|
||||
ftsFixed: number;
|
||||
vecFixed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find frames missing from FTS5 and re-index them.
|
||||
* Does NOT require an embedder — operates only on the FTS5 table.
|
||||
*/
|
||||
export function reconcileFtsIndex(db: MindDB): number {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
// Find frames that have no corresponding FTS5 entry.
|
||||
// memory_frames_fts uses content_rowid='id', so rowid matches memory_frames.id.
|
||||
const missingFts = raw.prepare(`
|
||||
SELECT f.id, f.content FROM memory_frames f
|
||||
WHERE f.id NOT IN (SELECT rowid FROM memory_frames_fts)
|
||||
`).all() as { id: number; content: string }[];
|
||||
|
||||
if (missingFts.length === 0) return 0;
|
||||
|
||||
const insertFts = raw.prepare(
|
||||
'INSERT INTO memory_frames_fts (rowid, content) VALUES (?, ?)',
|
||||
);
|
||||
|
||||
const insertAll = raw.transaction(() => {
|
||||
for (const row of missingFts) {
|
||||
insertFts.run(row.id, row.content);
|
||||
}
|
||||
});
|
||||
insertAll();
|
||||
|
||||
return missingFts.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find frames missing from the vector index and re-index them.
|
||||
* Requires an embedder to compute embeddings for the missing frames.
|
||||
* Returns 0 if the vec table doesn't exist.
|
||||
*/
|
||||
export async function reconcileVecIndex(db: MindDB, embedder: Embedder): Promise<number> {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
// Check if vec table exists
|
||||
const vecExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='memory_frames_vec'",
|
||||
).get();
|
||||
if (!vecExists) return 0;
|
||||
|
||||
const missingVec = raw.prepare(`
|
||||
SELECT f.id, f.content FROM memory_frames f
|
||||
WHERE f.id NOT IN (SELECT rowid FROM memory_frames_vec)
|
||||
`).all() as { id: number; content: string }[];
|
||||
|
||||
if (missingVec.length === 0) return 0;
|
||||
|
||||
// Embed in batches to avoid memory pressure
|
||||
const BATCH_SIZE = 50;
|
||||
for (let i = 0; i < missingVec.length; i += BATCH_SIZE) {
|
||||
const batch = missingVec.slice(i, i + BATCH_SIZE);
|
||||
const contents = batch.map(r => r.content);
|
||||
const embeddings = await embedder.embedBatch(contents);
|
||||
|
||||
const insertBatch = raw.transaction(() => {
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const id = Math.trunc(batch[j].id);
|
||||
const blob = new Uint8Array(
|
||||
embeddings[j].buffer,
|
||||
embeddings[j].byteOffset,
|
||||
embeddings[j].byteLength,
|
||||
);
|
||||
raw.prepare(
|
||||
`INSERT INTO memory_frames_vec (rowid, embedding) VALUES (${id}, ?)`,
|
||||
).run(blob);
|
||||
}
|
||||
});
|
||||
insertBatch();
|
||||
}
|
||||
|
||||
return missingVec.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* 9b: Remove orphan vector entries — vectors whose frame has been deleted.
|
||||
* Returns the number of orphan entries removed.
|
||||
*/
|
||||
export function cleanOrphanVectors(db: MindDB): number {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
// Check if vec table exists
|
||||
const vecExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='memory_frames_vec'",
|
||||
).get();
|
||||
if (!vecExists) return 0;
|
||||
|
||||
// Find vec entries with no corresponding frame
|
||||
const orphans = raw.prepare(`
|
||||
SELECT v.rowid FROM memory_frames_vec v
|
||||
WHERE v.rowid NOT IN (SELECT id FROM memory_frames)
|
||||
`).all() as { rowid: number }[];
|
||||
|
||||
if (orphans.length === 0) return 0;
|
||||
|
||||
const deleteTx = raw.transaction(() => {
|
||||
for (const { rowid } of orphans) {
|
||||
raw.prepare('DELETE FROM memory_frames_vec WHERE rowid = ?').run(rowid);
|
||||
}
|
||||
});
|
||||
deleteTx();
|
||||
|
||||
return orphans.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove orphan FTS entries — FTS entries whose frame has been deleted.
|
||||
*/
|
||||
export function cleanOrphanFts(db: MindDB): number {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
const orphans = raw.prepare(`
|
||||
SELECT rowid FROM memory_frames_fts
|
||||
WHERE rowid NOT IN (SELECT id FROM memory_frames)
|
||||
`).all() as { rowid: number }[];
|
||||
|
||||
if (orphans.length === 0) return 0;
|
||||
|
||||
const deleteTx = raw.transaction(() => {
|
||||
for (const { rowid } of orphans) {
|
||||
raw.prepare('DELETE FROM memory_frames_fts WHERE rowid = ?').run(rowid);
|
||||
}
|
||||
});
|
||||
deleteTx();
|
||||
|
||||
return orphans.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full reconciliation: repairs both FTS5 and vector indexes,
|
||||
* and cleans up orphan entries.
|
||||
* If no embedder is provided, only FTS5 is reconciled.
|
||||
*/
|
||||
export async function reconcileIndexes(
|
||||
db: MindDB,
|
||||
embedder?: Embedder,
|
||||
): Promise<ReconcileResult> {
|
||||
// Fix missing entries
|
||||
const ftsFixed = reconcileFtsIndex(db);
|
||||
const vecFixed = embedder ? await reconcileVecIndex(db, embedder) : 0;
|
||||
|
||||
// Clean orphans
|
||||
cleanOrphanFts(db);
|
||||
cleanOrphanVectors(db);
|
||||
|
||||
return { ftsFixed, vecFixed };
|
||||
}
|
||||
131
packages/hive-mind-core/src/mind/resolve-relative-date.ts
Normal file
131
packages/hive-mind-core/src/mind/resolve-relative-date.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* resolve-relative-date — write-time relative-date resolution for the memory substrate.
|
||||
*
|
||||
* WHY THIS EXISTS (benchmark-validated, LoCoMo head-to-head vs Memori, 2026-06-10):
|
||||
* A conversation utterance's *discussion date* (when it was said) is not the same as
|
||||
* the *event date* (when the thing happened). "I went to the group yesterday" said on
|
||||
* 2023-05-08 describes an event on 2023-05-07. Storing the discussion date makes
|
||||
* "when did X happen?" questions wrong by the relative-reference delta — the verified
|
||||
* root cause of our temporal-category gap. Resolving the relative reference against the
|
||||
* source date at WRITE time bakes the correct event date into the frame's timestamp,
|
||||
* which lifted the LoCoMo temporal category to parity with Memori (80.06 vs 80.37).
|
||||
* See benchmarks/results/memori-phase22-RESULT.md (Phase 4).
|
||||
*
|
||||
* This is the deterministic, dependency-free production counterpart of the benchmark's
|
||||
* LLM extraction pass (hive-mind-test scripts/locomo/33-distill-episodic.mjs). It covers
|
||||
* the dominant cue patterns the calibration surfaced ("yesterday", "last week", "N days
|
||||
* ago", "last <weekday>", "last year") without a per-frame LLM call. Pure + side-effect
|
||||
* free; the caller decides whether to use the resolved date.
|
||||
*/
|
||||
|
||||
/** A resolved relative-date hit: the matched cue phrase + the absolute ISO date (YYYY-MM-DD). */
|
||||
export interface ResolvedDate {
|
||||
/** The relative cue that matched, e.g. "yesterday", "last week", "3 months ago". */
|
||||
cue: string;
|
||||
/** The resolved absolute date as YYYY-MM-DD. */
|
||||
iso: string;
|
||||
}
|
||||
|
||||
const ISO_DATE = /^\d{4}-\d{2}-\d{2}/;
|
||||
const WEEKDAYS = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] as const;
|
||||
|
||||
/** Parse an ISO-ish reference date (date or datetime) into a UTC Date at midnight. Null if unparseable. */
|
||||
function parseReference(referenceDate: string | null | undefined): Date | null {
|
||||
if (!referenceDate || !ISO_DATE.test(referenceDate)) return null;
|
||||
const [y, m, d] = referenceDate.slice(0, 10).split('-').map(Number);
|
||||
if (!y || !m || !d) return null;
|
||||
const dt = new Date(Date.UTC(y, m - 1, d));
|
||||
return Number.isNaN(dt.getTime()) ? null : dt;
|
||||
}
|
||||
|
||||
/** Format a UTC Date as YYYY-MM-DD. */
|
||||
function toIso(dt: Date): string {
|
||||
return dt.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function addDays(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
out.setUTCDate(out.getUTCDate() + n);
|
||||
return out;
|
||||
}
|
||||
|
||||
function addMonths(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
const targetMonthDay = out.getUTCDate();
|
||||
out.setUTCDate(1);
|
||||
out.setUTCMonth(out.getUTCMonth() + n);
|
||||
// Clamp to the last valid day of the resulting month (e.g. Jan 31 − 1mo → Dec 31, not overflow).
|
||||
const lastDay = new Date(Date.UTC(out.getUTCFullYear(), out.getUTCMonth() + 1, 0)).getUTCDate();
|
||||
out.setUTCDate(Math.min(targetMonthDay, lastDay));
|
||||
return out;
|
||||
}
|
||||
|
||||
function addYears(dt: Date, n: number): Date {
|
||||
const out = new Date(dt.getTime());
|
||||
out.setUTCFullYear(out.getUTCFullYear() + n);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Most recent occurrence of `weekday` strictly before the reference date (the "last Friday" sense). */
|
||||
function lastWeekday(dt: Date, weekday: number): Date {
|
||||
let delta = dt.getUTCDay() - weekday;
|
||||
if (delta <= 0) delta += 7; // strictly before → at least 1 day back
|
||||
return addDays(dt, -delta);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the first relative-time cue in `text` against `referenceDate` (the source/
|
||||
* conversation date) into an absolute YYYY-MM-DD. Returns null when there is no
|
||||
* recognised cue or the reference date is unusable — the caller then keeps the
|
||||
* reference date as-is.
|
||||
*
|
||||
* Patterns are tried most-specific first so "the day before yesterday" wins over
|
||||
* "yesterday", and explicit "N units ago" wins over the bare "last unit".
|
||||
*/
|
||||
export function resolveRelativeDate(
|
||||
text: string,
|
||||
referenceDate: string | null | undefined,
|
||||
): ResolvedDate | null {
|
||||
const ref = parseReference(referenceDate);
|
||||
if (!ref || !text) return null;
|
||||
const t = text.toLowerCase();
|
||||
|
||||
// Most-specific day offsets first.
|
||||
if (/\bday before yesterday\b/.test(t)) return { cue: 'the day before yesterday', iso: toIso(addDays(ref, -2)) };
|
||||
if (/\byesterday\b/.test(t)) return { cue: 'yesterday', iso: toIso(addDays(ref, -1)) };
|
||||
|
||||
// "N days/weeks/months/years ago" (explicit count).
|
||||
const ago = t.match(/\b(\d{1,3})\s+(day|week|month|year)s?\s+ago\b/);
|
||||
if (ago) {
|
||||
const n = parseInt(ago[1], 10);
|
||||
const unit = ago[2];
|
||||
const iso =
|
||||
unit === 'day' ? toIso(addDays(ref, -n)) :
|
||||
unit === 'week' ? toIso(addDays(ref, -7 * n)) :
|
||||
unit === 'month' ? toIso(addMonths(ref, -n)) :
|
||||
toIso(addYears(ref, -n));
|
||||
return { cue: `${n} ${unit}${n === 1 ? '' : 's'} ago`, iso };
|
||||
}
|
||||
|
||||
// "a week/month/year ago" (singular, count = 1).
|
||||
const aAgo = t.match(/\ba\s+(week|month|year)\s+ago\b/);
|
||||
if (aAgo) {
|
||||
const unit = aAgo[1];
|
||||
const iso = unit === 'week' ? toIso(addDays(ref, -7)) : unit === 'month' ? toIso(addMonths(ref, -1)) : toIso(addYears(ref, -1));
|
||||
return { cue: `a ${unit} ago`, iso };
|
||||
}
|
||||
|
||||
// "last <weekday>" → most recent prior occurrence of that weekday.
|
||||
const lastDow = t.match(/\blast\s+(sunday|monday|tuesday|wednesday|thursday|friday|saturday)\b/);
|
||||
if (lastDow) {
|
||||
const wd = WEEKDAYS.indexOf(lastDow[1] as (typeof WEEKDAYS)[number]);
|
||||
return { cue: `last ${lastDow[1]}`, iso: toIso(lastWeekday(ref, wd)) };
|
||||
}
|
||||
|
||||
// "last week/month/year" (bare, coarse offset).
|
||||
if (/\blast\s+week\b/.test(t)) return { cue: 'last week', iso: toIso(addDays(ref, -7)) };
|
||||
if (/\blast\s+month\b/.test(t)) return { cue: 'last month', iso: toIso(addMonths(ref, -1)) };
|
||||
if (/\blast\s+year\b/.test(t)) return { cue: 'last year', iso: toIso(addYears(ref, -1)) };
|
||||
|
||||
return null;
|
||||
}
|
||||
430
packages/hive-mind-core/src/mind/schema.ts
Normal file
430
packages/hive-mind-core/src/mind/schema.ts
Normal file
@@ -0,0 +1,430 @@
|
||||
// INFORMATIONAL ONLY. Written once to meta.schema_version on first init (db.ts)
|
||||
// and exposed on the public barrel, but migrations are PRESENCE-based (they probe
|
||||
// for missing tables/columns/triggers), not gated on this value — nothing reads it
|
||||
// back to branch. It documents "this is v1 of the on-disk shape"; bump it (and add
|
||||
// a migration branch in MindDB.runMigrations()) only if you ever need version-gated
|
||||
// migration logic. Kept, not deleted: it is a re-exported public constant and the
|
||||
// meta row is asserted by schema.test.ts.
|
||||
export const SCHEMA_VERSION = '1';
|
||||
|
||||
export const SCHEMA_SQL = `
|
||||
-- Meta table for schema versioning
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Layer 0: Identity (single row, <500 tokens)
|
||||
CREATE TABLE IF NOT EXISTS identity (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
name TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT '',
|
||||
department TEXT NOT NULL DEFAULT '',
|
||||
personality TEXT NOT NULL DEFAULT '',
|
||||
capabilities TEXT NOT NULL DEFAULT '',
|
||||
system_prompt TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Layer 1: Awareness (<=10 active items)
|
||||
CREATE TABLE IF NOT EXISTS awareness (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category TEXT NOT NULL CHECK (category IN ('task', 'action', 'pending', 'flag')),
|
||||
content TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
expires_at TEXT
|
||||
);
|
||||
|
||||
-- Sessions: map GOPs to projects
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
gop_id TEXT NOT NULL UNIQUE,
|
||||
project_id TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'closed', 'archived')),
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
ended_at TEXT,
|
||||
summary TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions (project_id, started_at);
|
||||
|
||||
-- Layer 2: Memory Frames (I/P/B with GOP organization)
|
||||
CREATE TABLE IF NOT EXISTS memory_frames (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
frame_type TEXT NOT NULL CHECK (frame_type IN ('I', 'P', 'B')),
|
||||
gop_id TEXT NOT NULL,
|
||||
t INTEGER NOT NULL DEFAULT 0,
|
||||
base_frame_id INTEGER REFERENCES memory_frames(id),
|
||||
content TEXT NOT NULL,
|
||||
importance TEXT NOT NULL DEFAULT 'normal'
|
||||
CHECK (importance IN ('critical', 'important', 'normal', 'temporary', 'deprecated')),
|
||||
source TEXT NOT NULL DEFAULT 'user_stated'
|
||||
CHECK (source IN ('user_stated', 'tool_verified', 'agent_inferred', 'import', 'system')),
|
||||
access_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_accessed TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
-- UX-Refactor Phase 2B: JSON blob for Memory Center provenance/classification
|
||||
-- (kind/confidence/scope/status/sourceId/sourceUrl/tags/evidence/related*).
|
||||
-- See PRD §15.4 + docs/ux-refactor/deltas/shared-types-delta.md §3a. Existing
|
||||
-- DBs get this via the idempotent ADD COLUMN in db.ts runMigrations().
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
-- oss-drift D3 (2026-06-11): canonical dedup hash — sha256 over the
|
||||
-- stripHmPrefix-stripped + trimmed content (mind/content-hash.ts; mono
|
||||
-- semantics, NOT the OSS trim-only hash). Indexed so FrameStore.findDuplicate
|
||||
-- is an O(1) lookup with no recency window. Existing DBs get this via the
|
||||
-- idempotent ADD COLUMN + backfill in db.ts runMigrations().
|
||||
content_hash TEXT,
|
||||
FOREIGN KEY (gop_id) REFERENCES sessions(gop_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_frames_gop_t ON memory_frames (gop_id, t);
|
||||
CREATE INDEX IF NOT EXISTS idx_frames_type ON memory_frames (frame_type, gop_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_frames_base ON memory_frames (base_frame_id);
|
||||
-- idx_frames_content_hash is created ONLY in db.ts runMigrations(), AFTER the
|
||||
-- guarded ADD COLUMN. It must NOT live here: on a pre-D3 database the CREATE
|
||||
-- TABLE above no-ops (table exists without content_hash), so an index here
|
||||
-- referenced a missing column and SCHEMA_SQL threw BEFORE the ALTER could run
|
||||
-- — every existing install failed to boot (2026-06-12 regression).
|
||||
|
||||
-- FTS5 for keyword search on frame content
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memory_frames_fts USING fts5(
|
||||
content,
|
||||
content_rowid='id',
|
||||
tokenize='porter unicode61'
|
||||
);
|
||||
|
||||
-- Layer 3: Knowledge Graph - Entities
|
||||
CREATE TABLE IF NOT EXISTS knowledge_entities (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
entity_type TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
properties TEXT NOT NULL DEFAULT '{}',
|
||||
valid_from TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
valid_to TEXT,
|
||||
recorded_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_entities_type ON knowledge_entities (entity_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_entities_name ON knowledge_entities (name);
|
||||
|
||||
-- Layer 3: Knowledge Graph - Relations
|
||||
CREATE TABLE IF NOT EXISTS knowledge_relations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER NOT NULL REFERENCES knowledge_entities(id),
|
||||
target_id INTEGER NOT NULL REFERENCES knowledge_entities(id),
|
||||
relation_type TEXT NOT NULL,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
properties TEXT NOT NULL DEFAULT '{}',
|
||||
valid_from TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
valid_to TEXT,
|
||||
recorded_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_source ON knowledge_relations (source_id, relation_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_target ON knowledge_relations (target_id, relation_type);
|
||||
|
||||
-- Layer 3: Knowledge Graph - Entity↔Frame bridge.
|
||||
-- Records which frames an entity was extracted from, so the 'contextual'
|
||||
-- scoring signal (scoring.ts) can map query-seeded graph distances back onto
|
||||
-- frames. ON DELETE CASCADE keeps it consistent when a frame or entity is removed.
|
||||
CREATE TABLE IF NOT EXISTS kg_entity_frames (
|
||||
entity_id INTEGER NOT NULL REFERENCES knowledge_entities(id) ON DELETE CASCADE,
|
||||
frame_id INTEGER NOT NULL REFERENCES memory_frames(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (entity_id, frame_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_kg_entity_frames_frame ON kg_entity_frames (frame_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kg_entity_frames_entity ON kg_entity_frames (entity_id);
|
||||
|
||||
-- Layer 5: Improvement Signals (recurring patterns that should change behavior)
|
||||
CREATE TABLE IF NOT EXISTS improvement_signals (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category TEXT NOT NULL CHECK (category IN ('capability_gap', 'correction', 'workflow_pattern', 'skill_promotion')),
|
||||
pattern_key TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
first_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
surfaced INTEGER NOT NULL DEFAULT 0,
|
||||
surfaced_at TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_signals_category_key ON improvement_signals (category, pattern_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_signals_category ON improvement_signals (category, count DESC);
|
||||
|
||||
-- Layer 6: Install Audit (capability install trust trail)
|
||||
CREATE TABLE IF NOT EXISTS install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
-- CHECK lists MUST stay in sync with AuditCapabilityType / AuditApprovalClass
|
||||
-- / AuditAction in packages/core/src/install-audit.ts. They drifted once
|
||||
-- (connector/marketplace/blocked missing) and crashed acquire_capability the
|
||||
-- moment marketplace search started returning candidates — see runMigrations().
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN ('native', 'skill', 'plugin', 'mcp', 'connector', 'marketplace')),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN ('low', 'medium', 'high', 'critical')),
|
||||
trust_source TEXT NOT NULL CHECK (trust_source IN ('builtin', 'starter_pack', 'local_user', 'third_party_verified', 'third_party_unverified', 'unknown', 'security-gate')),
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard', 'elevated', 'critical', 'blocked')),
|
||||
action TEXT NOT NULL CHECK (action IN ('proposed', 'approved', 'installed', 'rejected', 'failed', 'blocked', 'uninstalled')),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN ('agent', 'user', 'system')),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_capability ON install_audit (capability_name, action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC);
|
||||
|
||||
-- Layer 4: Procedures (GEPA-optimized prompt templates)
|
||||
CREATE TABLE IF NOT EXISTS procedures (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
template TEXT NOT NULL,
|
||||
version INTEGER NOT NULL DEFAULT 1,
|
||||
success_rate REAL NOT NULL DEFAULT 0.0,
|
||||
avg_cost REAL NOT NULL DEFAULT 0.0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_procedures_name_model ON procedures (name, model);
|
||||
|
||||
-- Layer 7: AI Interactions (EU AI Act Art. 12 — automatic event logging)
|
||||
CREATE TABLE IF NOT EXISTS ai_interactions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
workspace_id TEXT,
|
||||
session_id TEXT,
|
||||
model TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cost_usd REAL NOT NULL DEFAULT 0,
|
||||
tools_called TEXT NOT NULL DEFAULT '[]',
|
||||
human_action TEXT CHECK (human_action IN ('approved', 'denied', 'modified', 'none')),
|
||||
risk_context TEXT,
|
||||
imported_from TEXT,
|
||||
persona TEXT,
|
||||
-- Review Critical #3 (compliance): EU AI Act Art. 12.1(a) requires recording
|
||||
-- the actual INPUTS and OUTPUTS of the system, not just token counts. Added
|
||||
-- 2026-04-15; migration for pre-existing DBs in MindDB.runMigrations().
|
||||
input_text TEXT,
|
||||
output_text TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_interactions_workspace ON ai_interactions (workspace_id, timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_interactions_timestamp ON ai_interactions (timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_interactions_model ON ai_interactions (model);
|
||||
|
||||
-- Review Critical #1 (compliance): append-only enforcement for the audit log.
|
||||
-- DDL-level triggers make the database itself refuse UPDATE / DELETE so a motivated
|
||||
-- auditor's first question ('can rows be silently mutated?') has a concrete 'no'
|
||||
-- answer. GDPR Art. 17 erasure is handled via a separate pseudonymize_and_tombstone
|
||||
-- flow that's not yet implemented — when it is, it will replace inputText/outputText
|
||||
-- with tombstone markers via a fresh INSERT + status flag, NOT by bypassing these
|
||||
-- triggers.
|
||||
CREATE TRIGGER IF NOT EXISTS ai_interactions_no_delete
|
||||
BEFORE DELETE ON ai_interactions
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'ai_interactions is append-only (EU AI Act Art. 12 audit log)');
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS ai_interactions_no_update
|
||||
BEFORE UPDATE ON ai_interactions
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'ai_interactions is append-only (EU AI Act Art. 12 audit log)');
|
||||
END;
|
||||
|
||||
-- Layer 9: Execution Traces (agent run history — foundation for self-evolution)
|
||||
CREATE TABLE IF NOT EXISTS execution_traces (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT,
|
||||
persona_id TEXT,
|
||||
workspace_id TEXT,
|
||||
model TEXT,
|
||||
task_shape TEXT,
|
||||
outcome TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (outcome IN ('success', 'corrected', 'abandoned', 'verified', 'pending')),
|
||||
trace_json TEXT NOT NULL DEFAULT '{}',
|
||||
cost_usd REAL NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finalized_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_traces_session ON execution_traces (session_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome);
|
||||
CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC);
|
||||
|
||||
-- Layer 10: Evolution Runs (proposed/accepted/rejected self-evolution runs)
|
||||
CREATE TABLE IF NOT EXISTS evolution_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_uuid TEXT NOT NULL UNIQUE,
|
||||
target_kind TEXT NOT NULL,
|
||||
target_name TEXT,
|
||||
baseline_text TEXT NOT NULL,
|
||||
winner_text TEXT NOT NULL,
|
||||
winner_schema_json TEXT,
|
||||
delta_accuracy REAL NOT NULL DEFAULT 0,
|
||||
gate_verdict TEXT NOT NULL DEFAULT 'pass'
|
||||
CHECK (gate_verdict IN ('pass', 'fail')),
|
||||
gate_reasons_json TEXT NOT NULL DEFAULT '[]',
|
||||
status TEXT NOT NULL DEFAULT 'proposed'
|
||||
CHECK (status IN ('proposed', 'accepted', 'rejected', 'deployed', 'failed')),
|
||||
artifacts_json TEXT,
|
||||
user_note TEXT,
|
||||
failure_reason TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
decided_at TEXT,
|
||||
deployed_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_evo_runs_status ON evolution_runs (status, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_evo_runs_target ON evolution_runs (target_kind, target_name, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_evo_runs_created ON evolution_runs (created_at DESC);
|
||||
|
||||
-- Layer 8: Harvest Sources (Memory Harvest sync tracking)
|
||||
CREATE TABLE IF NOT EXISTS harvest_sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
source_path TEXT,
|
||||
last_synced_at TEXT,
|
||||
items_imported INTEGER NOT NULL DEFAULT 0,
|
||||
frames_created INTEGER NOT NULL DEFAULT 0,
|
||||
auto_sync INTEGER NOT NULL DEFAULT 0,
|
||||
sync_interval_hours INTEGER NOT NULL DEFAULT 24,
|
||||
last_content_hash TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11).
|
||||
-- Memory frame chunks: paragraph-level subdivisions of memory_frames for
|
||||
-- semantic-search precision. One frame produces N chunks (N=1 for short
|
||||
-- frames). Each chunk gets its own embedding in memory_frame_chunks_vec.
|
||||
-- Recall maps top-K chunks back to parent frames via frame_id.
|
||||
CREATE TABLE IF NOT EXISTS memory_frame_chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
frame_id INTEGER NOT NULL REFERENCES memory_frames(id) ON DELETE CASCADE,
|
||||
chunk_idx INTEGER NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
char_start INTEGER NOT NULL,
|
||||
char_end INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(frame_id, chunk_idx)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_frame ON memory_frame_chunks (frame_id);
|
||||
|
||||
-- Verbatim Provenance Archive (#7, 2026-06-30): append-only, immutable, full-fidelity
|
||||
-- copy of each harvested source item. Distilled/imported frames link back via
|
||||
-- memory_frames.metadata.archiveUid. NOT part of the retrieval corpus (no FTS/vec) —
|
||||
-- audit/reconstruction only. Append-only triggers mirror ai_interactions (Layer 7),
|
||||
-- with ONE exception: a one-time GDPR Art.17 redaction (see raw_archive_no_update).
|
||||
CREATE TABLE IF NOT EXISTS raw_archive (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
archive_uid TEXT NOT NULL UNIQUE,
|
||||
source TEXT NOT NULL,
|
||||
source_ref TEXT,
|
||||
title TEXT,
|
||||
content TEXT NOT NULL,
|
||||
content_sha256 TEXT NOT NULL,
|
||||
injection_flagged INTEGER NOT NULL DEFAULT 0,
|
||||
injection_flags TEXT NOT NULL DEFAULT '',
|
||||
source_timestamp TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
-- Size guard: a single item's content is truncated to
|
||||
-- RAW_ARCHIVE_MAX_CONTENT_CHARS (raw-archive.ts) before storage so one giant
|
||||
-- harvested export can't blow the append-only store. truncated=1 marks a capped
|
||||
-- row; original_length is the pre-truncation character count (NULL when not
|
||||
-- truncated). archive_uid + content_sha256 still derive from the FULL content, so
|
||||
-- idempotency + the integrity anchor are unaffected. Not referenced by the
|
||||
-- append-only trigger below, so an erasure UPDATE that leaves them untouched
|
||||
-- passes unchanged. Pre-guard DBs get these via the idempotent ADD COLUMN in db.ts.
|
||||
truncated INTEGER NOT NULL DEFAULT 0,
|
||||
original_length INTEGER,
|
||||
-- GDPR Art.17 erasure: NULL until a data-subject erasure request. When set, the
|
||||
-- audit skeleton (id/source/refs/timestamps) is frozen as the audit record while
|
||||
-- content/content_sha256/title are redacted AND archive_uid is ROTATED to an opaque
|
||||
-- id (the old content-derived uid was a re-identification vector — see the trigger).
|
||||
erased_at TEXT,
|
||||
erased_reason TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_archive_source_ref ON raw_archive (source, source_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_archive_created ON raw_archive (created_at DESC);
|
||||
-- Append-only EXCEPT a single, one-directional GDPR Art.17 redaction. The trigger
|
||||
-- pins the EXACT permitted outcome — not just the transition — so raw SQL cannot
|
||||
-- abuse the erasure path to forge audit content: it is allowed ONLY when erased_at
|
||||
-- goes NULL -> a non-empty value, every AUDIT column (id/source/refs/timestamps/
|
||||
-- injection) is unchanged, the archive_uid is ROTATED to a new non-empty value
|
||||
-- (content-derived uid must not survive — re-identification vector), AND the row
|
||||
-- lands on the canonical redaction (content = marker, content_sha256 = '', title
|
||||
-- NULL). erased_reason is the only free field. The content literal below MUST stay
|
||||
-- byte-identical to RAW_ARCHIVE_REDACTION_MARKER in raw-archive.ts, and this whole
|
||||
-- WHEN clause byte-identical to the db.ts runMigrations() recreation.
|
||||
CREATE TRIGGER IF NOT EXISTS raw_archive_no_update
|
||||
BEFORE UPDATE ON raw_archive
|
||||
WHEN NOT (
|
||||
OLD.erased_at IS NULL AND NEW.erased_at IS NOT NULL AND NEW.erased_at <> ''
|
||||
AND NEW.content = '[REDACTED — GDPR Art.17 erasure]'
|
||||
AND NEW.content_sha256 = ''
|
||||
AND NEW.title IS NULL
|
||||
AND NEW.id IS OLD.id
|
||||
AND NEW.archive_uid <> OLD.archive_uid
|
||||
AND NEW.archive_uid <> ''
|
||||
AND NEW.source IS OLD.source
|
||||
AND NEW.source_ref IS OLD.source_ref
|
||||
AND NEW.created_at IS OLD.created_at
|
||||
AND NEW.source_timestamp IS OLD.source_timestamp
|
||||
AND NEW.injection_flagged IS OLD.injection_flagged
|
||||
AND NEW.injection_flags IS OLD.injection_flags
|
||||
)
|
||||
BEGIN SELECT RAISE(ABORT, 'raw_archive is append-only; only a one-time canonical GDPR Art.17 redaction is permitted'); END;
|
||||
CREATE TRIGGER IF NOT EXISTS raw_archive_no_delete
|
||||
BEFORE DELETE ON raw_archive
|
||||
BEGIN SELECT RAISE(ABORT, 'raw_archive is append-only (verbatim provenance archive)'); END;
|
||||
|
||||
-- Erased-subject suppression list (#7 Art.17 "sticky erasure", 2026-07-02). When a
|
||||
-- data subject is erased (MindErasure.eraseBySourceRef), its (source, source_ref) is
|
||||
-- recorded here; every re-import write seam consults it and SKIPS re-materialization,
|
||||
-- so an exercised right-to-erasure survives a later re-export/re-sync of the source.
|
||||
-- Keyed on the SUBJECT pair ONLY — deliberately NO content / content_sha256 (a
|
||||
-- content-keyed tombstone would reintroduce the re-identification vector the
|
||||
-- raw_archive archive_uid rotation removed). Rows are DELETABLE (unlike raw_archive):
|
||||
-- deletion is the deliberate re-consent / "allow re-import again" path — hence no
|
||||
-- immutability trigger. Generic substrate (no governance/trust fields) → forward-ports
|
||||
-- to the OSS mirror verbatim, the deliberate opposite of install_audit.
|
||||
CREATE TABLE IF NOT EXISTS erased_subjects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
source_ref TEXT NOT NULL,
|
||||
erased_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
reason TEXT,
|
||||
UNIQUE(source, source_ref)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_erased_subjects_lookup ON erased_subjects (source, source_ref);
|
||||
`;
|
||||
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R7, 2026-06-11).
|
||||
/** Vec-table DDL parameterized by embedding dimension. vec0 columns can't be
|
||||
* ALTERed, so changing dimension means DROP + CREATE (see MindDB.recreateVecTables). */
|
||||
export function vecTableSqlForDim(dim: number): string {
|
||||
const d = Math.trunc(dim);
|
||||
return `
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memory_frames_vec USING vec0(
|
||||
embedding float[${d}]
|
||||
);
|
||||
`;
|
||||
}
|
||||
|
||||
/** Default vec schema at the canonical 1024-dim (used on first init + migrations). */
|
||||
export const VEC_TABLE_SQL = vecTableSqlForDim(1024);
|
||||
|
||||
// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11).
|
||||
/** Chunk-level vec-table DDL parameterized by embedding dimension. Separate from
|
||||
* vecTableSqlForDim so callers can create/recreate the chunk index independently;
|
||||
* MindDB.recreateVecTables(dim) recreates BOTH (frames + chunks) together. */
|
||||
export function chunksVecTableSqlForDim(dim: number): string {
|
||||
const d = Math.trunc(dim);
|
||||
return `
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memory_frame_chunks_vec USING vec0(
|
||||
embedding float[${d}]
|
||||
);
|
||||
`;
|
||||
}
|
||||
|
||||
/** Default chunk-vec schema at the canonical 1024-dim (first init + migrations). */
|
||||
export const CHUNKS_VEC_TABLE_SQL = chunksVecTableSqlForDim(1024);
|
||||
108
packages/hive-mind-core/src/mind/scoring.ts
Normal file
108
packages/hive-mind-core/src/mind/scoring.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import type { Importance } from './frames.js';
|
||||
|
||||
export type ScoringProfile = 'balanced' | 'recent' | 'important' | 'connected';
|
||||
|
||||
export interface ScoringWeights {
|
||||
temporal: number;
|
||||
popularity: number;
|
||||
contextual: number;
|
||||
importance: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* KNOWN GAP (W4-PRODUCTION-PORT-PLAN-2026-06-11.md §3 bug #1): no production
|
||||
* caller passes `graphDistances`, so the `contextual` dimension scores 0 for
|
||||
* every frame — 20% of 'balanced' (60% of 'connected') is a uniform constant.
|
||||
* This does NOT distort ranking (a constant-0 term preserves ordering) but it
|
||||
* deflates absolute finalScores and makes 'connected' a functional no-op.
|
||||
* Deliberately NOT zeroed out: tests and future callers may pass
|
||||
* graphDistances, and KnowledgeGraph.bfsDistances needs an entity-id →
|
||||
* frame-id bridge before production wiring (open decision #3).
|
||||
*/
|
||||
export const SCORING_PROFILES: Record<ScoringProfile, ScoringWeights> = {
|
||||
balanced: { temporal: 0.4, popularity: 0.2, contextual: 0.2, importance: 0.2 },
|
||||
recent: { temporal: 0.6, popularity: 0.1, contextual: 0.2, importance: 0.1 },
|
||||
important: { temporal: 0.1, popularity: 0.1, contextual: 0.2, importance: 0.6 },
|
||||
connected: { temporal: 0.1, popularity: 0.1, contextual: 0.6, importance: 0.2 },
|
||||
};
|
||||
|
||||
const IMPORTANCE_WEIGHTS: Record<Importance, number> = {
|
||||
critical: 2.0,
|
||||
important: 1.5,
|
||||
normal: 1.0,
|
||||
temporary: 0.7,
|
||||
deprecated: 0.3,
|
||||
};
|
||||
|
||||
const HALF_LIFE_DAYS = 30;
|
||||
const RECENCY_BOOST_DAYS = 7;
|
||||
|
||||
export interface ScoredResult {
|
||||
frameId: number;
|
||||
rrfScore: number;
|
||||
relevanceScore: number;
|
||||
finalScore: number;
|
||||
}
|
||||
|
||||
export interface ScoringContext {
|
||||
recentEntityIds?: number[];
|
||||
graphDistances?: Map<number, number>; // frameId -> shortest BFS distance
|
||||
}
|
||||
|
||||
export function computeTemporalScore(lastAccessedIso: string): number {
|
||||
const now = Date.now();
|
||||
const accessed = new Date(lastAccessedIso).getTime();
|
||||
const daysSince = (now - accessed) / (1000 * 60 * 60 * 24);
|
||||
|
||||
if (daysSince <= RECENCY_BOOST_DAYS) {
|
||||
return 1.0; // full score for recent items
|
||||
}
|
||||
|
||||
// Exponential decay with 30-day half-life
|
||||
return Math.pow(0.5, daysSince / HALF_LIFE_DAYS);
|
||||
}
|
||||
|
||||
export function computePopularityScore(accessCount: number): number {
|
||||
return 1 + Math.log10(1 + accessCount) * 0.1;
|
||||
}
|
||||
|
||||
export function computeContextualScore(
|
||||
frameId: number,
|
||||
graphDistances: Map<number, number> | undefined
|
||||
): number {
|
||||
if (!graphDistances || !graphDistances.has(frameId)) return 0;
|
||||
const distance = graphDistances.get(frameId)!;
|
||||
if (distance === 0) return 1.0;
|
||||
if (distance === 1) return 0.7;
|
||||
if (distance === 2) return 0.4;
|
||||
if (distance === 3) return 0.2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function computeImportanceScore(importance: Importance): number {
|
||||
return IMPORTANCE_WEIGHTS[importance];
|
||||
}
|
||||
|
||||
export function computeRelevance(
|
||||
frame: { id: number; created_at?: string; last_accessed: string; access_count: number; importance: Importance },
|
||||
weights: ScoringWeights,
|
||||
context: ScoringContext = {}
|
||||
): number {
|
||||
// W4.2 (plan §3 bug #3): the temporal dimension decays on WRITE time
|
||||
// (created_at), not access time. last_accessed is bumped to "now" by
|
||||
// touch() on every read — decaying on it made this dimension constant
|
||||
// noise on historical corpora (every recalled frame scored "recent").
|
||||
// created_at is optional for back-compat; callers not passing it keep
|
||||
// the old access-decay behavior.
|
||||
const temporal = computeTemporalScore(frame.created_at ?? frame.last_accessed);
|
||||
const popularity = computePopularityScore(frame.access_count);
|
||||
const contextual = computeContextualScore(frame.id, context.graphDistances);
|
||||
const importance = computeImportanceScore(frame.importance);
|
||||
|
||||
return (
|
||||
temporal * weights.temporal +
|
||||
popularity * weights.popularity +
|
||||
contextual * weights.contextual +
|
||||
importance * weights.importance
|
||||
);
|
||||
}
|
||||
698
packages/hive-mind-core/src/mind/search.ts
Normal file
698
packages/hive-mind-core/src/mind/search.ts
Normal file
@@ -0,0 +1,698 @@
|
||||
import type { MindDB } from './db.js';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import type { MemoryFrame, Importance } from './frames.js';
|
||||
import type { Reranker } from './inprocess-reranker.js';
|
||||
import { chunkText, type ChunkOptions } from './chunker.js';
|
||||
import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import {
|
||||
computeRelevance,
|
||||
SCORING_PROFILES,
|
||||
type ScoringProfile,
|
||||
type ScoringContext,
|
||||
type ScoredResult,
|
||||
} from './scoring.js';
|
||||
import { KnowledgeGraph } from './knowledge.js';
|
||||
|
||||
export interface SearchOptions {
|
||||
limit?: number;
|
||||
gopId?: string; // scope to a specific session
|
||||
profile?: ScoringProfile;
|
||||
context?: ScoringContext;
|
||||
/** F20: Only include frames created on or after this ISO date string. */
|
||||
since?: string;
|
||||
/** F20: Only include frames created on or before this ISO date string. */
|
||||
until?: string;
|
||||
/**
|
||||
* W4.2: cross-encoder reranker invoked AFTER RRF on the top-`rerankPoolSize`
|
||||
* candidates. When provided, results are sorted by reranker score
|
||||
* (jointly attentive over query+doc). RRF still selects the candidate
|
||||
* pool; the reranker only re-orders the survivors. Soft-fails to RRF
|
||||
* ordering on any reranker error.
|
||||
*/
|
||||
reranker?: Reranker;
|
||||
/** How many candidates to send to the reranker (default 30). */
|
||||
rerankPoolSize?: number;
|
||||
/**
|
||||
* Hard-exclude frames with importance='deprecated' from results. Default OFF
|
||||
* for back-compat: deprecated frames still surface, merely down-weighted 0.3×
|
||||
* by the scoring layer. Turn ON where a superseded value must NEVER leak into
|
||||
* the read context — e.g. after supersession consolidation (see supersede.ts),
|
||||
* where a 0.3× multiplier still let stale values surface via the focus lane.
|
||||
*/
|
||||
excludeDeprecated?: boolean;
|
||||
}
|
||||
|
||||
export interface SearchResult {
|
||||
frame: MemoryFrame;
|
||||
rrfScore: number;
|
||||
relevanceScore: number;
|
||||
finalScore: number;
|
||||
}
|
||||
|
||||
// Ported from hive-mind a99ea0e.
|
||||
/**
|
||||
* Retrieval-confidence verdict for the abstain path (LongMemEval's
|
||||
* "insufficient evidence" ability). Pure + side-effect-free so callers
|
||||
* (MCP recall_memory, CLI, eval harness) can decide whether to answer or
|
||||
* abstain without re-running search.
|
||||
*/
|
||||
export interface RetrievalConfidence {
|
||||
/** True when the top result clears the threshold (safe to answer). */
|
||||
sufficient: boolean;
|
||||
/** The top finalScore observed (0 when there were no results). */
|
||||
topScore: number;
|
||||
/** The threshold it was compared against. */
|
||||
threshold: number;
|
||||
}
|
||||
|
||||
// Ported from hive-mind a99ea0e.
|
||||
/**
|
||||
* Assess whether a result set carries enough signal to answer, or whether the
|
||||
* caller should abstain ("insufficient evidence"). A scaffold for the abstain
|
||||
* path: it does NOT change `search()` output — callers opt in by passing the
|
||||
* results plus a τ threshold. `sufficient` is true iff the top finalScore is
|
||||
* strictly greater than τ; an empty set is always insufficient.
|
||||
*
|
||||
* Threshold semantics intentionally mirror the recall-stress edge-query rule
|
||||
* (a low top score means "nothing relevant surfaced").
|
||||
*/
|
||||
export function assessRetrievalConfidence(
|
||||
results: readonly SearchResult[],
|
||||
threshold: number,
|
||||
): RetrievalConfidence {
|
||||
const topScore = results.length ? results[0].finalScore : 0;
|
||||
return { sufficient: topScore > threshold, topScore, threshold };
|
||||
}
|
||||
|
||||
const RRF_K = 60;
|
||||
|
||||
const log = createCoreLogger('hybrid-search');
|
||||
|
||||
// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11).
|
||||
/**
|
||||
* Chunk-level retrieval flag — DEFAULT ON since the 2026-06-12 long-frame
|
||||
* needle probe (benchmarks/chunk-probe/): on a copy of the real production
|
||||
* personal mind, paired hit@5 = chunk 46/52 vs whole-frame 17/52 (discordant
|
||||
* pairs 30-vs-1, McNemar p≈2e-8); chunk led even within the embed cap
|
||||
* (17/20 vs 13/20) and dominated beyond it (29/32 vs 4/32 — content past the
|
||||
* embedder's true token context is structurally invisible to whole-frame
|
||||
* vectors). LoCoMo was rejected as the ruler: its frames sit below the
|
||||
* 2000-char chunk threshold, so an A/B there measures noise by construction.
|
||||
* Kill switch: WAGGLE_CHUNK_RETRIEVAL=0. Gates BOTH the write side
|
||||
* (indexFrame / indexFramesBatch also chunk-index the frame) and the read
|
||||
* side (search() queries memory_frame_chunks_vec, falling back to whole-frame
|
||||
* vectors while the chunk index is empty). `indexChunksForFrame` /
|
||||
* `rechunkAllFrames` stay callable regardless of the flag (backfill + eval).
|
||||
*/
|
||||
export function chunkRetrievalEnabled(): boolean {
|
||||
return process.env.WAGGLE_CHUNK_RETRIEVAL !== '0';
|
||||
}
|
||||
|
||||
function f32ToBlob(f32: Float32Array): Uint8Array {
|
||||
return new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength);
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape LIKE metacharacters (`%`, `_`) and the escape char itself (`\`) so the
|
||||
* keyword-fallback term is matched literally. Pair with `ESCAPE '\'` on the LIKE.
|
||||
*/
|
||||
function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
}
|
||||
|
||||
export class HybridSearch {
|
||||
private db: MindDB;
|
||||
private embedder: Embedder;
|
||||
private fingerprintChecked = false;
|
||||
|
||||
constructor(db: MindDB, embedder: Embedder) {
|
||||
this.db = db;
|
||||
this.embedder = embedder;
|
||||
}
|
||||
|
||||
// Reverse-ported from OSS hive-mind (oss-drift triage R7, 2026-06-11).
|
||||
/**
|
||||
* Guard the .mind's embedding fingerprint before vector reads/writes. Throws
|
||||
* EmbeddingDimMismatchError if the active embedder's dim differs from what
|
||||
* the .mind's vectors were written at; warns (but allows) on a same-dim model
|
||||
* change. Memoized on success so it costs one meta read per instance lifetime.
|
||||
* Must be called BEFORE any try/catch that would swallow the error.
|
||||
*/
|
||||
private ensureFingerprint(): void {
|
||||
if (this.fingerprintChecked) return;
|
||||
const e = this.embedder as Embedder & {
|
||||
getActiveProvider?(): string;
|
||||
getStatus?(): { modelName?: string };
|
||||
};
|
||||
const provider = e.getActiveProvider?.() ?? 'unknown';
|
||||
const model = e.getStatus?.().modelName ?? 'unknown';
|
||||
const result = this.db.ensureEmbeddingFingerprint({ provider, model, dim: this.embedder.dimensions });
|
||||
// Only memoize after a non-throwing check (a dim mismatch must keep throwing).
|
||||
this.fingerprintChecked = true;
|
||||
if (result.status === 'model-changed') {
|
||||
log.warn(
|
||||
`Embedding model changed for this .mind (${result.storedProvider}/${result.storedModel} → ` +
|
||||
`${provider}/${model}, same ${this.embedder.dimensions}-dim). Existing vectors stay searchable, ` +
|
||||
`but cross-model similarity is degraded — consider re-embedding all frames.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
|
||||
const { limit = 20, gopId, profile = 'balanced', context = {}, since, until, reranker, rerankPoolSize } = options;
|
||||
const weights = SCORING_PROFILES[profile];
|
||||
|
||||
// Run keyword and vector searches in parallel.
|
||||
// W4.1b slot-consumption fix: the since/until filter applies AFTER the
|
||||
// lanes run (as a WHERE over candidate ids), so out-of-window candidates
|
||||
// would otherwise consume lane slots and shrink results below `limit`
|
||||
// even when in-window frames exist deeper in the lanes. Over-fetch the
|
||||
// lanes when a temporal window is active so the post-filter has depth.
|
||||
const laneFetch = (since || until) ? limit * 10 : limit * 2;
|
||||
|
||||
// D1 chunk lane (flag-gated, default OFF): prefer chunk-level vector
|
||||
// search when WAGGLE_CHUNK_RETRIEVAL=1 AND chunks_vec is populated —
|
||||
// chunk embeddings discriminate better on domain-homogeneous corpora
|
||||
// than whole-frame embeddings. vectorSearchChunks returns null when no
|
||||
// chunks exist, signalling clean fallback to the whole-frame path. Both
|
||||
// paths return frame IDs so the RRF + scoring pipeline is unchanged.
|
||||
// Flag off → chunkResults is null without touching the chunk tables,
|
||||
// so the lane below is byte-identical to pre-D1.
|
||||
const chunkResults = chunkRetrievalEnabled()
|
||||
? await this.vectorSearchChunks(query, laneFetch, gopId)
|
||||
: null;
|
||||
const [keywordResults, vectorResults] = await Promise.all([
|
||||
this.keywordSearch(query, laneFetch, gopId),
|
||||
chunkResults !== null
|
||||
? Promise.resolve(chunkResults)
|
||||
: this.vectorSearch(query, laneFetch, gopId),
|
||||
]);
|
||||
|
||||
// RRF fusion
|
||||
const rrfScores = new Map<number, number>();
|
||||
|
||||
keywordResults.forEach((id, rank) => {
|
||||
rrfScores.set(id, (rrfScores.get(id) ?? 0) + 1 / (RRF_K + rank));
|
||||
});
|
||||
|
||||
vectorResults.forEach((id, rank) => {
|
||||
rrfScores.set(id, (rrfScores.get(id) ?? 0) + 1 / (RRF_K + rank));
|
||||
});
|
||||
|
||||
// Get all unique frame IDs
|
||||
const frameIds = [...rrfScores.keys()];
|
||||
if (frameIds.length === 0) return [];
|
||||
|
||||
// F20: Fetch frames with optional temporal filtering
|
||||
const raw = this.db.getDatabase();
|
||||
const placeholders = frameIds.map(() => '?').join(',');
|
||||
const temporalConditions: string[] = [];
|
||||
const temporalParams: unknown[] = [...frameIds];
|
||||
|
||||
// W4.1b fencepost fix: `created_at` carries mixed formats across write
|
||||
// paths — `datetime('now')` ("YYYY-MM-DD HH:MM:SS") vs harvest ISO
|
||||
// ("YYYY-MM-DDT…Z"). A date-only `until` string-compares BELOW any
|
||||
// same-day timestamp ("2026-03-21T10:00" > "2026-03-21"), silently
|
||||
// excluding the whole final day. Compare date-only bounds on the
|
||||
// 10-char date prefix instead — format-agnostic and inclusive.
|
||||
if (since) {
|
||||
if (since.length === 10) {
|
||||
temporalConditions.push('substr(created_at, 1, 10) >= ?');
|
||||
} else {
|
||||
temporalConditions.push('created_at >= ?');
|
||||
}
|
||||
temporalParams.push(since);
|
||||
}
|
||||
if (until) {
|
||||
if (until.length === 10) {
|
||||
temporalConditions.push('substr(created_at, 1, 10) <= ?');
|
||||
} else {
|
||||
temporalConditions.push('created_at <= ?');
|
||||
}
|
||||
temporalParams.push(until);
|
||||
}
|
||||
|
||||
const whereExtra = temporalConditions.length > 0
|
||||
? ` AND ${temporalConditions.join(' AND ')}`
|
||||
: '';
|
||||
// Hard-exclude deprecated frames when requested (no param needed — literal
|
||||
// condition). Dropping them from `frames` removes them from frameMap, so
|
||||
// they never enter the result set OR the reranker pool.
|
||||
const deprecatedExtra = options.excludeDeprecated ? " AND importance != 'deprecated'" : '';
|
||||
|
||||
const frames = raw.prepare(
|
||||
`SELECT * FROM memory_frames WHERE id IN (${placeholders})${whereExtra}${deprecatedExtra}`
|
||||
).all(...temporalParams) as MemoryFrame[];
|
||||
|
||||
const frameMap = new Map(frames.map(f => [f.id, f]));
|
||||
|
||||
// W4.1: turn on the 'contextual' scoring signal. Seed graph distance from
|
||||
// entities the caller flagged (context.recentEntityIds) plus entities named
|
||||
// in the query, BFS the KG, and map to frames via the kg_entity_frames bridge.
|
||||
// Best-effort: a graph hiccup must never fail the search.
|
||||
let scoringContext = context;
|
||||
if (!scoringContext.graphDistances) {
|
||||
try {
|
||||
const kg = new KnowledgeGraph(this.db);
|
||||
const seeds = new Set<number>(scoringContext.recentEntityIds ?? []);
|
||||
for (const id of kg.findEntitiesInText(query)) seeds.add(id);
|
||||
if (seeds.size > 0) {
|
||||
const graphDistances = kg.frameDistancesFromEntities([...seeds], 3);
|
||||
if (graphDistances.size > 0) scoringContext = { ...scoringContext, graphDistances };
|
||||
}
|
||||
} catch { /* contextual signal is optional */ }
|
||||
}
|
||||
|
||||
// Compute final scores
|
||||
const results: SearchResult[] = [];
|
||||
for (const [frameId, rrfScore] of rrfScores) {
|
||||
const frame = frameMap.get(frameId);
|
||||
if (!frame) continue;
|
||||
|
||||
const relevanceScore = computeRelevance(
|
||||
{
|
||||
id: frame.id,
|
||||
// W4.2 bug #3: temporal decay anchors on write time, not access time.
|
||||
created_at: frame.created_at,
|
||||
last_accessed: frame.last_accessed,
|
||||
access_count: frame.access_count,
|
||||
importance: frame.importance as Importance,
|
||||
},
|
||||
weights,
|
||||
scoringContext
|
||||
);
|
||||
|
||||
results.push({
|
||||
frame,
|
||||
rrfScore,
|
||||
relevanceScore,
|
||||
finalScore: rrfScore * relevanceScore,
|
||||
});
|
||||
}
|
||||
|
||||
results.sort((a, b) => b.finalScore - a.finalScore);
|
||||
|
||||
// W4.2: optional cross-encoder reranking on the top pool (reverse-ported
|
||||
// from the OSS benchmark-proven stack). Reranker scoring is jointly
|
||||
// attentive over (query, doc), so it discriminates much better than
|
||||
// vector dot products on densely-homogeneous corpora. RRF still selects
|
||||
// the candidate pool; the reranker only re-orders the survivors.
|
||||
if (reranker) {
|
||||
const poolSize = Math.min(rerankPoolSize ?? 30, results.length);
|
||||
const pool = results.slice(0, poolSize);
|
||||
try {
|
||||
const docs = pool.map((r) => r.frame.content);
|
||||
const scores = await reranker.scoreBatch(query, docs);
|
||||
// Pair (result, rerank score), sort desc, replace finalScore so the
|
||||
// shape stays the same for downstream consumers.
|
||||
const reranked = pool.map((r, i) => ({ ...r, finalScore: scores[i] }));
|
||||
reranked.sort((a, b) => b.finalScore - a.finalScore);
|
||||
// Append any pool tail items beyond rerankPoolSize so a small limit
|
||||
// doesn't suddenly contract the result set.
|
||||
return reranked.concat(results.slice(poolSize)).slice(0, limit);
|
||||
} catch {
|
||||
// Reranker failure (model load, OOM, dim mismatch) — fall back to
|
||||
// RRF ordering. Soft-fail so a misconfigured reranker doesn't
|
||||
// kill recall entirely.
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
|
||||
// Old: implicit AND (all terms required) → fails on "hiring decisions this month"
|
||||
// New: OR between terms (any term matches) → FTS5 rank orders by relevance
|
||||
// S1: sanitizer unified in fts-sanitize.ts, Unicode-aware (Cyrillic and
|
||||
// diacritic terms survive; ASCII output is byte-identical to before).
|
||||
const safeQuery = query.includes('"')
|
||||
? query // already quoted by caller
|
||||
: buildFtsOrQuery(query);
|
||||
|
||||
if (!safeQuery) {
|
||||
// Empty MATCH string. For ASCII queries that means stop words / short
|
||||
// tokens only — keep returning [] (regression lock). For queries in an
|
||||
// unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
|
||||
// lack of signal: unicode61 cannot token-match CJK prose, but LIKE
|
||||
// substring matching can, so route those to the fallback lane.
|
||||
return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
|
||||
}
|
||||
|
||||
let sql: string;
|
||||
let params: unknown[];
|
||||
|
||||
if (gopId) {
|
||||
sql = `
|
||||
SELECT mf.id FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.gop_id = ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, gopId, limit];
|
||||
} else {
|
||||
sql = `
|
||||
SELECT rowid as id FROM memory_frames_fts
|
||||
WHERE content MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, limit];
|
||||
}
|
||||
|
||||
try {
|
||||
const rows = raw.prepare(sql).all(...params) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
} catch {
|
||||
// FTS5 parse error (e.g. user query with FTS5-special chars that survived
|
||||
// sanitization) — fall back to a LIKE keyword scan over the same column so
|
||||
// we return best-effort matches instead of a false "no memory found".
|
||||
return this.likeFallbackSearch(query, limit, gopId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
|
||||
* MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
|
||||
* operator the user typed literally). The raw query is split into word tokens
|
||||
* — stripping the punctuation that caused the FTS5 error, mirroring the
|
||||
* primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
|
||||
* recall. Bound parameters only (the term is never interpolated) and LIKE
|
||||
* metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
|
||||
* matches literally. If no usable token survives, a single literal LIKE over
|
||||
* the whole escaped query is used.
|
||||
*/
|
||||
private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
const tokens = query
|
||||
.split(/\s+/)
|
||||
.map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
|
||||
.filter(w => w.length > 0);
|
||||
const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
|
||||
|
||||
const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
|
||||
|
||||
try {
|
||||
if (gopId) {
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause}) AND gop_id = ?
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, gopId, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
}
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause})
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
this.ensureFingerprint();
|
||||
const embedding = await this.embedder.embed(query);
|
||||
const blob = f32ToBlob(embedding);
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
if (gopId) {
|
||||
// Two-step: get candidates from vec, then filter by GOP
|
||||
try {
|
||||
const rows = raw.prepare(`
|
||||
SELECT v.rowid as id FROM memory_frames_vec v
|
||||
WHERE v.embedding MATCH ? AND k = ?
|
||||
ORDER BY distance
|
||||
`).all(blob, limit * 3) as { id: number }[];
|
||||
|
||||
// Filter by GOP
|
||||
if (rows.length === 0) return [];
|
||||
const placeholders = rows.map(() => '?').join(',');
|
||||
const filtered = raw.prepare(`
|
||||
SELECT id FROM memory_frames WHERE id IN (${placeholders}) AND gop_id = ?
|
||||
`).all(...rows.map(r => r.id), gopId) as { id: number }[];
|
||||
|
||||
return filtered.map(r => r.id).slice(0, limit);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const rows = raw.prepare(`
|
||||
SELECT rowid as id FROM memory_frames_vec
|
||||
WHERE embedding MATCH ? AND k = ?
|
||||
ORDER BY distance
|
||||
`).all(blob, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async indexFrame(frameId: number, content: string): Promise<void> {
|
||||
this.ensureFingerprint();
|
||||
if (!Number.isFinite(frameId)) {
|
||||
throw new Error('Invalid frame ID for vector indexing');
|
||||
}
|
||||
const embedding = await this.embedder.embed(content);
|
||||
const raw = this.db.getDatabase();
|
||||
// sqlite-vec vec0 requires rowid as SQL literal (parameterized rowid not supported)
|
||||
const id = Math.trunc(frameId);
|
||||
raw.prepare(
|
||||
`INSERT INTO memory_frames_vec (rowid, embedding) VALUES (${id}, ?)`
|
||||
).run(f32ToBlob(embedding));
|
||||
|
||||
// D1 (flag-gated, default OFF): keep the chunk index in lockstep with
|
||||
// live frame writes. Soft-fail — a chunk-indexing error must never break
|
||||
// the primary whole-frame write (mirrors the reranker soft-fail stance).
|
||||
if (chunkRetrievalEnabled()) {
|
||||
try {
|
||||
await this.indexChunksForFrame(frameId, content);
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
`chunk indexing failed for frame ${id} (whole-frame vector written): ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async indexFramesBatch(frames: { id: number; content: string }[]): Promise<void> {
|
||||
if (frames.length === 0) return;
|
||||
this.ensureFingerprint();
|
||||
for (const f of frames) {
|
||||
if (!Number.isFinite(f.id)) {
|
||||
throw new Error('Invalid frame ID for vector indexing');
|
||||
}
|
||||
}
|
||||
const contents = frames.map(f => f.content);
|
||||
const embeddings = await this.embedder.embedBatch(contents);
|
||||
const raw = this.db.getDatabase();
|
||||
// sqlite-vec vec0 requires rowid as SQL literal (parameterized rowid not supported)
|
||||
const insertAll = raw.transaction(() => {
|
||||
for (let i = 0; i < frames.length; i++) {
|
||||
const id = Math.trunc(frames[i].id);
|
||||
raw.prepare(
|
||||
`INSERT INTO memory_frames_vec (rowid, embedding) VALUES (${id}, ?)`
|
||||
).run(f32ToBlob(embeddings[i]));
|
||||
}
|
||||
});
|
||||
insertAll();
|
||||
|
||||
// D1 (flag-gated, default OFF): chunk-index batch writes too, so frames
|
||||
// ingested via the batch path (harvest) aren't invisible to the chunk
|
||||
// lane. Soft-fail per frame — see indexFrame.
|
||||
if (chunkRetrievalEnabled()) {
|
||||
for (const f of frames) {
|
||||
try {
|
||||
await this.indexChunksForFrame(f.id, f.content);
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
`chunk indexing failed for frame ${Math.trunc(f.id)} (whole-frame vector written): ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Chunk-level indexing (oss-drift triage D1, 2026-06-11) ─────────────
|
||||
// Reverse-ported from OSS hive-mind "Phase 3b-3 chunking". Whole-frame
|
||||
// embeddings cluster too tightly on a domain-homogeneous corpus (every
|
||||
// frame is "about the same project"), so retrieval can't discriminate.
|
||||
// Chunking decomposes a frame into ~500-token paragraph-level pieces,
|
||||
// each with its own embedding — search returns the chunk, we map back to
|
||||
// the parent frame for the final result.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Replace all chunks for a frame: clears existing chunks/vec rows for the
|
||||
* frame, re-chunks the content, embeds each chunk, inserts both rows.
|
||||
* Idempotent — safe to call repeatedly. Used by rechunkAllFrames and by
|
||||
* the flag-gated indexFrame path. NOT itself gated on
|
||||
* WAGGLE_CHUNK_RETRIEVAL (backfill + eval call it directly).
|
||||
*/
|
||||
async indexChunksForFrame(
|
||||
frameId: number,
|
||||
content: string,
|
||||
opts: ChunkOptions = {},
|
||||
): Promise<number> {
|
||||
if (!Number.isFinite(frameId) || frameId <= 0) {
|
||||
throw new Error('Invalid frame ID for chunk indexing');
|
||||
}
|
||||
this.ensureFingerprint();
|
||||
const raw = this.db.getDatabase();
|
||||
const id = Math.trunc(frameId);
|
||||
|
||||
const chunks = chunkText(content, opts);
|
||||
if (chunks.length === 0) return 0;
|
||||
|
||||
// Embed all chunks. embedBatch amortises HTTP overhead on Ollama/API providers.
|
||||
const texts = chunks.map((c) => c.text);
|
||||
const embeddings = await this.embedder.embedBatch(texts);
|
||||
|
||||
// Single tx so partial failure leaves the frame's chunks empty
|
||||
// (next rechunk pass will re-fill from scratch — same end state).
|
||||
const tx = raw.transaction(() => {
|
||||
// Find existing chunk_ids for this frame so we can drop their vec rows.
|
||||
// Foreign-key cascade handles memory_frame_chunks deletion when the
|
||||
// parent frame is deleted, but for re-indexing we're keeping the
|
||||
// frame and just replacing its chunks.
|
||||
const existing = raw
|
||||
.prepare('SELECT id FROM memory_frame_chunks WHERE frame_id = ?')
|
||||
.all(id) as Array<{ id: number }>;
|
||||
for (const row of existing) {
|
||||
// sqlite-vec rowid must be SQL literal.
|
||||
raw.prepare(`DELETE FROM memory_frame_chunks_vec WHERE rowid = ${Math.trunc(row.id)}`).run();
|
||||
}
|
||||
raw.prepare('DELETE FROM memory_frame_chunks WHERE frame_id = ?').run(id);
|
||||
|
||||
const insertChunk = raw.prepare(
|
||||
'INSERT INTO memory_frame_chunks (frame_id, chunk_idx, content, char_start, char_end) VALUES (?, ?, ?, ?, ?)'
|
||||
);
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const c = chunks[i];
|
||||
const result = insertChunk.run(id, i, c.text, c.charStart, c.charEnd);
|
||||
const chunkId = Math.trunc(Number(result.lastInsertRowid));
|
||||
raw
|
||||
.prepare(`INSERT INTO memory_frame_chunks_vec (rowid, embedding) VALUES (${chunkId}, ?)`)
|
||||
.run(f32ToBlob(embeddings[i]));
|
||||
}
|
||||
});
|
||||
tx();
|
||||
return chunks.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Vector search over chunks. Returns parent frame IDs deduped (best-chunk-
|
||||
* per-frame wins — first-seen order under ORDER BY distance). When the
|
||||
* chunk index is empty (or the tables are missing), returns null so callers
|
||||
* can cleanly fall back to the whole-frame vectorSearch path.
|
||||
*/
|
||||
async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
|
||||
this.ensureFingerprint();
|
||||
const raw = this.db.getDatabase();
|
||||
// Cheap probe — avoid embedding the query when chunks aren't populated.
|
||||
let chunkCount: number;
|
||||
try {
|
||||
const row = raw.prepare('SELECT COUNT(*) AS n FROM memory_frame_chunks').get() as
|
||||
| { n: number }
|
||||
| undefined;
|
||||
chunkCount = row?.n ?? 0;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (chunkCount === 0) return null;
|
||||
|
||||
const embedding = await this.embedder.embed(query);
|
||||
const blob = f32ToBlob(embedding);
|
||||
|
||||
// Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
|
||||
// candidates after collapsing multiple chunks of the same frame.
|
||||
try {
|
||||
const chunkRows = raw
|
||||
.prepare(
|
||||
`SELECT v.rowid AS chunk_id, c.frame_id
|
||||
FROM memory_frame_chunks_vec v
|
||||
JOIN memory_frame_chunks c ON c.id = v.rowid
|
||||
WHERE v.embedding MATCH ? AND k = ?
|
||||
ORDER BY distance`
|
||||
)
|
||||
.all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
|
||||
|
||||
if (chunkRows.length === 0) return [];
|
||||
|
||||
// Dedup by frame_id, preserving first-seen order (best-distance chunk).
|
||||
const seen = new Set<number>();
|
||||
const frameIds: number[] = [];
|
||||
for (const r of chunkRows) {
|
||||
if (seen.has(r.frame_id)) continue;
|
||||
seen.add(r.frame_id);
|
||||
frameIds.push(r.frame_id);
|
||||
if (frameIds.length >= limit) break;
|
||||
}
|
||||
|
||||
if (gopId) {
|
||||
const placeholders = frameIds.map(() => '?').join(',');
|
||||
const filtered = raw
|
||||
.prepare(
|
||||
`SELECT id FROM memory_frames WHERE id IN (${placeholders}) AND gop_id = ?`
|
||||
)
|
||||
.all(...frameIds, gopId) as { id: number }[];
|
||||
return filtered.map((r) => r.id).slice(0, limit);
|
||||
}
|
||||
return frameIds;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11);
|
||||
// follows the OSS `maintenance --rechunk-all` per-mind logic.
|
||||
export interface RechunkResult {
|
||||
framesProcessed: number;
|
||||
chunksCreated: number;
|
||||
framesFailed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)chunk + chunk-index every non-deprecated frame in the .mind. Idempotent
|
||||
* per-frame — indexChunksForFrame deletes a frame's existing chunks before
|
||||
* re-inserting. One bad frame doesn't abort the batch (logged + counted).
|
||||
* Backfill/eval helper only — no CLI/route wiring yet, and NOT gated on
|
||||
* WAGGLE_CHUNK_RETRIEVAL (it must be runnable before any flag flip).
|
||||
*/
|
||||
export async function rechunkAllFrames(db: MindDB, search: HybridSearch): Promise<RechunkResult> {
|
||||
const raw = db.getDatabase();
|
||||
const frames = raw
|
||||
.prepare("SELECT id, content FROM memory_frames WHERE importance != 'deprecated' ORDER BY id ASC")
|
||||
.all() as Array<{ id: number; content: string }>;
|
||||
|
||||
let framesProcessed = 0;
|
||||
let chunksCreated = 0;
|
||||
let framesFailed = 0;
|
||||
|
||||
for (const f of frames) {
|
||||
try {
|
||||
const n = await search.indexChunksForFrame(f.id, f.content);
|
||||
framesProcessed++;
|
||||
chunksCreated += n;
|
||||
} catch (err) {
|
||||
framesFailed++;
|
||||
log.warn(
|
||||
`rechunkAllFrames: frame ${f.id} failed: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { framesProcessed, chunksCreated, framesFailed };
|
||||
}
|
||||
107
packages/hive-mind-core/src/mind/sessions.ts
Normal file
107
packages/hive-mind-core/src/mind/sessions.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import type { MindDB } from './db.js';
|
||||
|
||||
export interface Session {
|
||||
id: number;
|
||||
gop_id: string;
|
||||
project_id: string | null;
|
||||
status: 'active' | 'closed' | 'archived';
|
||||
started_at: string;
|
||||
ended_at: string | null;
|
||||
summary: string | null;
|
||||
}
|
||||
|
||||
export class SessionStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
create(projectId?: string): Session {
|
||||
const gopId = `session:${new Date().toISOString()}:${Math.random().toString(36).slice(2, 8)}`;
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO sessions (gop_id, project_id, status, started_at)
|
||||
VALUES (?, ?, 'active', datetime('now'))
|
||||
`).run(gopId, projectId ?? null);
|
||||
return raw.prepare('SELECT * FROM sessions WHERE id = ?').get(result.lastInsertRowid) as Session;
|
||||
}
|
||||
|
||||
close(gopId: string, summary?: string): Session {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE sessions SET status = 'closed', ended_at = datetime('now'), summary = ?
|
||||
WHERE gop_id = ?
|
||||
`).run(summary ?? null, gopId);
|
||||
return raw.prepare('SELECT * FROM sessions WHERE gop_id = ?').get(gopId) as Session;
|
||||
}
|
||||
|
||||
archive(gopId: string): Session {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare("UPDATE sessions SET status = 'archived' WHERE gop_id = ?").run(gopId);
|
||||
return raw.prepare('SELECT * FROM sessions WHERE gop_id = ?').get(gopId) as Session;
|
||||
}
|
||||
|
||||
getByProject(projectId: string): Session[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM sessions WHERE project_id = ? ORDER BY started_at DESC'
|
||||
).all(projectId) as Session[];
|
||||
}
|
||||
|
||||
getActive(): Session[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
"SELECT * FROM sessions WHERE status = 'active' ORDER BY started_at DESC"
|
||||
).all() as Session[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the most-recent active session, or create one atomically if none exists.
|
||||
* Transaction-wrapped so two concurrent callers on a fresh mind produce exactly
|
||||
* one session (review finding #7: session-create race in autoSaveFromExchange).
|
||||
*/
|
||||
ensureActive(projectId?: string): Session {
|
||||
const raw = this.db.getDatabase();
|
||||
const txn = raw.transaction((): Session => {
|
||||
// Secondary `id DESC` tiebreak: datetime('now') has second precision, so two
|
||||
// create() calls in the same second share started_at and SQLite's ordering becomes
|
||||
// unspecified without an explicit tiebreaker.
|
||||
const existing = raw.prepare(
|
||||
"SELECT * FROM sessions WHERE status = 'active' ORDER BY started_at DESC, id DESC LIMIT 1"
|
||||
).get() as Session | undefined;
|
||||
if (existing) return existing;
|
||||
const gopId = `session:${new Date().toISOString()}:${Math.random().toString(36).slice(2, 8)}`;
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO sessions (gop_id, project_id, status, started_at)
|
||||
VALUES (?, ?, 'active', datetime('now'))
|
||||
`).run(gopId, projectId ?? null);
|
||||
return raw.prepare('SELECT * FROM sessions WHERE id = ?').get(result.lastInsertRowid) as Session;
|
||||
});
|
||||
return txn();
|
||||
}
|
||||
|
||||
getByGopId(gopId: string): Session | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM sessions WHERE gop_id = ?'
|
||||
).get(gopId) as Session | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a session with the given stable gop_id exists, creating it on
|
||||
* first use. Used for long-lived logical sessions like `harvest` that
|
||||
* group imported memory frames under a single parent across many runs.
|
||||
*
|
||||
* Unlike `create()`, which generates a timestamped id per call, this
|
||||
* method is idempotent — calling it repeatedly with the same gop_id
|
||||
* returns the same session.
|
||||
*/
|
||||
ensure(gopId: string, projectId?: string, summary?: string): Session {
|
||||
const existing = this.getByGopId(gopId);
|
||||
if (existing) return existing;
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO sessions (gop_id, project_id, status, summary, started_at)
|
||||
VALUES (?, ?, 'active', ?, datetime('now'))
|
||||
`).run(gopId, projectId ?? null, summary ?? null);
|
||||
return this.getByGopId(gopId)!;
|
||||
}
|
||||
}
|
||||
393
packages/hive-mind-core/src/mind/supersede.ts
Normal file
393
packages/hive-mind-core/src/mind/supersede.ts
Normal file
@@ -0,0 +1,393 @@
|
||||
/**
|
||||
* supersede.ts — supersession (P) + bridge (B) frame PRODUCER.
|
||||
*
|
||||
* The distiller only ever emits I-frames (independent assertions); the
|
||||
* substrate's P (partial-update / supersession delta) and B (bridge /
|
||||
* cross-link) primitives sat dormant with zero production callers. This module
|
||||
* turns them on as a first-class, provider-agnostic capability.
|
||||
*
|
||||
* PRODUCER vs CONSUMER — read this before renaming or merging. This module is
|
||||
* the PRODUCER: it reads UNSTRUCTURED observations, detects supersession chains
|
||||
* + enumerable groups, and WRITES P/B frames. The CONSUMERS of those frames
|
||||
* live elsewhere and are intentionally NOT here:
|
||||
* - `FrameStore.compact()` merges accumulated P-frames back into their base
|
||||
* I-frame (a downstream housekeeping consumer).
|
||||
* - `packages/weaver/src/consolidation.ts` (MemoryWeaver) consumes P/B frames
|
||||
* (merges I+P into consolidated I-frames, materialises B-frames from KG
|
||||
* entities). Do NOT collapse this file into that name — the two sit on
|
||||
* opposite sides of the P/B lifecycle.
|
||||
*
|
||||
* Two LLM passes read an I-frame observation set and produce structured
|
||||
* consolidation intents:
|
||||
* - Supersession chains — the SAME attribute of the SAME subject whose value
|
||||
* changes over time (follower count 1250 → 1300; job title A → B). We
|
||||
* deprecate the stale members, boost the newest to `critical`, and emit ONE
|
||||
* P-frame carrying the CURRENT value so a "what is X now" read surfaces the
|
||||
* latest, not a stale mention.
|
||||
* - Enumerable entity groups — 2+ observations each describing a DISTINCT
|
||||
* member of one countable class (each aquarium tank; each wedding). We emit
|
||||
* ONE B-frame referencing every member frame so a counting read can expand
|
||||
* it to the COMPLETE set.
|
||||
*
|
||||
* Provider-agnostic by construction: the caller injects an `llm(system, user)`
|
||||
* callback (study harvest/extract-kg-entities.ts for the repo's executor
|
||||
* pattern). This module is PURE — no child_process, no fetch, no env reads — so
|
||||
* it is trivially unit-testable with a fake llm and safe to call from any
|
||||
* executor (CLI `claude -p` subprocess, MCP server, benchmark harness).
|
||||
*
|
||||
* ⚠️ INDEXING CONTRACT: FrameStore.createPFrame / createBFrame index ONLY the
|
||||
* FTS table, NOT the vector table. `applyConsolidation` therefore RETURNS the
|
||||
* new frames so the CALLER can vec-index them (e.g. HybridSearch.indexFramesBatch).
|
||||
* Skip that step and the P/B frames are keyword-recallable but invisible to
|
||||
* semantic search.
|
||||
*
|
||||
* Ported from the validated LongMemEval experiment
|
||||
* (benchmarks/longmemeval/46-consolidate.mjs + 47-answer-pb.mjs): the two
|
||||
* prompts below are carried over verbatim, including the clean `current_value`
|
||||
* extraction that strips hedges ("about", "around", "close to").
|
||||
*
|
||||
* Ported from hive-mind 2d0abc5 (mono-parity 2026-07-05, §7.5).
|
||||
*/
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import type { FrameStore, FrameSource, MemoryFrame } from './frames.js';
|
||||
|
||||
/**
|
||||
* LLM callback the consolidation passes inject. Given a system + user message,
|
||||
* returns the model's raw text response (JSON is parsed defensively downstream).
|
||||
* Provider-agnostic — the caller owns the transport (subprocess, HTTP, etc.).
|
||||
*/
|
||||
export type ConsolidationLlm = (system: string, user: string) => Promise<string>;
|
||||
|
||||
/** One dated observation fed to the detectors. `id` is the real frame id. */
|
||||
export interface Observation {
|
||||
id: number;
|
||||
content: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
/** A detected supersession chain, mapped to real frame ids (oldest → newest). */
|
||||
export interface SupersessionChain {
|
||||
/** Short attribute label, e.g. "follower count", "job title". */
|
||||
attribute: string;
|
||||
/** The clean, unhedged latest value; empty when the model omitted it. */
|
||||
currentValue: string;
|
||||
/** Member frame ids in oldest → newest order (length ≥ 2). */
|
||||
frameIds: number[];
|
||||
}
|
||||
|
||||
/** A detected enumerable entity group, mapped to real frame ids. */
|
||||
export interface EntityGroup {
|
||||
/** Short class name, e.g. "aquarium tanks the user owns". */
|
||||
label: string;
|
||||
/** Member frame ids (length ≥ 2). */
|
||||
frameIds: number[];
|
||||
}
|
||||
|
||||
/** What `applyConsolidation` wrote — returned so the caller can vec-index. */
|
||||
export interface ConsolidationResult {
|
||||
/** Newly created P-frames (current-value deltas). */
|
||||
pframes: MemoryFrame[];
|
||||
/** Newly created B-frames (member-set bridges). */
|
||||
bframes: MemoryFrame[];
|
||||
/** Frame ids demoted to importance='deprecated'. */
|
||||
deprecated: number[];
|
||||
}
|
||||
|
||||
/** Options for gathering the observation set the detectors run over. */
|
||||
export interface CollectObservationsOptions {
|
||||
/** Scope to a single GOP session; omit for the whole mind. */
|
||||
gopId?: string;
|
||||
/** Cap the number of observations (keeps the LLM prompt bounded). */
|
||||
limit?: number;
|
||||
/**
|
||||
* Which frame source to include. Defaults to 'agent_inferred' (the
|
||||
* distiller's output, matching the benchmark). Pass 'any' for every source.
|
||||
*/
|
||||
source?: FrameSource | 'any';
|
||||
}
|
||||
|
||||
// ── Prompts (ported verbatim from 46-consolidate.mjs) ──────────────────────
|
||||
|
||||
const SUPERSESSION_SYSTEM =
|
||||
'You are given a numbered list of dated observations about ONE user. Identify UPDATE CHAINS: sets of observations that state the SAME attribute of the SAME specific subject where the VALUE CHANGES over time (e.g. follower count 1250 then 1300; job title A then B; where an item is kept). Only genuine supersessions of ONE evolving fact — NOT distinct facts, NOT a count of different items. For each chain give "current_value": the LATEST value as a short, clean, unhedged phrase (e.g. "1300 followers", "in a shoe rack in the closet"), stripping words like "close to", "about", "around". Return JSON {"chains":[{"attribute":"short label","current_value":"clean latest value","ids":[oldest..newest]}]} using the observation numbers as ids. If none, {"chains":[]}.';
|
||||
|
||||
const GROUP_SYSTEM =
|
||||
'You are given a numbered list of dated observations about ONE user. Identify ENUMERABLE GROUPS: sets of 2+ observations each describing a DISTINCT member of the same countable class that an aggregation question might count or sum (e.g. each aquarium tank the user owns; each wedding attended; each magazine subscription; each workshop with its cost). One group per class. Do NOT include update-chains (same item changing value). Return JSON {"groups":[{"label":"short class name","ids":[...]}]} using observation numbers. If none, {"groups":[]}.';
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Defensive JSON parse of an LLM response (see parseJson in 46). Tries a direct
|
||||
* parse first, then falls back to extracting the first balanced-looking `{…}`
|
||||
* block from prose (models sometimes wrap JSON in commentary or fences). Returns
|
||||
* an empty object when nothing parses — the callers treat "no intents" as a
|
||||
* valid, non-fatal outcome rather than throwing on model chatter.
|
||||
*/
|
||||
function parseLlmJson(raw: string): Record<string, unknown> {
|
||||
if (typeof raw !== 'string') return {};
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return {};
|
||||
try {
|
||||
return JSON.parse(trimmed) as Record<string, unknown>;
|
||||
} catch {
|
||||
// Not a bare JSON document — try to recover an embedded object below.
|
||||
}
|
||||
const match = trimmed.match(/\{[\s\S]*\}/);
|
||||
if (match) {
|
||||
try {
|
||||
return JSON.parse(match[0]) as Record<string, unknown>;
|
||||
} catch {
|
||||
// Embedded block was also malformed — fall through to the empty result.
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/** Fail loudly on malformed input rather than silently producing junk chains. */
|
||||
function assertObservations(observations: unknown): asserts observations is Observation[] {
|
||||
if (!Array.isArray(observations)) {
|
||||
throw new TypeError('consolidate: observations must be an array');
|
||||
}
|
||||
for (const o of observations) {
|
||||
if (!o || typeof o !== 'object') {
|
||||
throw new TypeError('consolidate: each observation must be an object');
|
||||
}
|
||||
const rec = o as Record<string, unknown>;
|
||||
if (!Number.isInteger(rec.id)) {
|
||||
throw new TypeError('consolidate: observation.id must be an integer');
|
||||
}
|
||||
if (typeof rec.content !== 'string') {
|
||||
throw new TypeError('consolidate: observation.content must be a string');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Render the observations as a `N. [YYYY-MM-DD] content` numbered list. */
|
||||
function numberObservations(observations: Observation[]): string {
|
||||
return observations
|
||||
.map((o, idx) => `${idx + 1}. [${String(o.created_at ?? '').slice(0, 10)}] ${o.content}`)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Map the model's 1-based observation numbers back to real frame ids. Invalid /
|
||||
* out-of-range / duplicate numbers are dropped. When `sortByAge` is set the ids
|
||||
* are returned in observation order (which is created_at, id order = oldest →
|
||||
* newest) regardless of the order the model emitted them — the supersession
|
||||
* pass relies on the last id being the genuinely newest member.
|
||||
*/
|
||||
function mapNumbersToFrameIds(
|
||||
numbers: unknown,
|
||||
observations: Observation[],
|
||||
sortByAge: boolean,
|
||||
): number[] {
|
||||
if (!Array.isArray(numbers)) return [];
|
||||
const indices: number[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const n of numbers) {
|
||||
const idx = Number(n);
|
||||
if (!Number.isInteger(idx) || idx < 1 || idx > observations.length) continue;
|
||||
if (seen.has(idx)) continue;
|
||||
seen.add(idx);
|
||||
indices.push(idx);
|
||||
}
|
||||
if (sortByAge) indices.sort((a, b) => a - b);
|
||||
return indices.map((i) => observations[i - 1].id);
|
||||
}
|
||||
|
||||
// ── Detection passes ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Detect supersession chains via the LLM. Returns chains whose member ids map
|
||||
* to ≥ 2 real frames, oldest → newest. Never throws on model chatter — a
|
||||
* malformed / empty response yields `[]`. The injected `llm` IS allowed to
|
||||
* throw (transport failures surface to the caller; they are not swallowed).
|
||||
*/
|
||||
export async function detectSupersessionChains(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<SupersessionChain[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectSupersessionChains: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
|
||||
const raw = await llm(SUPERSESSION_SYSTEM, numberObservations(observations));
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawChains = Array.isArray(parsed.chains) ? parsed.chains : [];
|
||||
|
||||
const chains: SupersessionChain[] = [];
|
||||
for (const entry of rawChains) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, true);
|
||||
if (frameIds.length < 2) continue;
|
||||
const attribute = typeof rec.attribute === 'string' ? rec.attribute.trim() : '';
|
||||
const currentValue = typeof rec.current_value === 'string' ? rec.current_value.trim() : '';
|
||||
chains.push({ attribute: attribute || 'value', currentValue, frameIds });
|
||||
}
|
||||
return chains;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect enumerable entity groups via the LLM. Returns groups whose member ids
|
||||
* map to ≥ 2 real frames (member order preserved as emitted). Same throw
|
||||
* contract as `detectSupersessionChains`.
|
||||
*/
|
||||
export async function detectEntityGroups(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<EntityGroup[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectEntityGroups: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
|
||||
const raw = await llm(GROUP_SYSTEM, numberObservations(observations));
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawGroups = Array.isArray(parsed.groups) ? parsed.groups : [];
|
||||
|
||||
const groups: EntityGroup[] = [];
|
||||
for (const entry of rawGroups) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, false);
|
||||
if (frameIds.length < 2) continue;
|
||||
const label = typeof rec.label === 'string' ? rec.label.trim() : '';
|
||||
groups.push({ label: label || 'group', frameIds });
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
// ── Application pass ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Apply detected chains + groups to a FrameStore. For each chain: deprecate the
|
||||
* stale members, boost the newest to `critical`, and emit a P-frame carrying the
|
||||
* clean current value (base = oldest member). For each group: emit a B-frame
|
||||
* referencing every member (base = first member).
|
||||
*
|
||||
* All new frames land under `gopId` (a valid session gop_id) while their
|
||||
* base_frame_id / references still point at the original — possibly cross-gop —
|
||||
* source frames.
|
||||
*
|
||||
* ⚠️ The returned `pframes` / `bframes` are FTS-indexed only (createPFrame /
|
||||
* createBFrame do not touch the vector table). The caller MUST vec-index them
|
||||
* (e.g. `HybridSearch.indexFramesBatch`) for semantic search to reach them.
|
||||
*/
|
||||
export function applyConsolidation(
|
||||
frames: FrameStore,
|
||||
chains: SupersessionChain[],
|
||||
groups: EntityGroup[],
|
||||
gopId: string,
|
||||
): ConsolidationResult {
|
||||
if (!frames || typeof frames.createPFrame !== 'function') {
|
||||
throw new TypeError('applyConsolidation: frames must be a FrameStore');
|
||||
}
|
||||
if (typeof gopId !== 'string' || !gopId) {
|
||||
throw new Error('applyConsolidation: gopId is required');
|
||||
}
|
||||
|
||||
const pframes: MemoryFrame[] = [];
|
||||
const bframes: MemoryFrame[] = [];
|
||||
const deprecated: number[] = [];
|
||||
|
||||
for (const chain of chains ?? []) {
|
||||
const ids = chain.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
|
||||
const baseId = ids[0];
|
||||
const newestId = ids[ids.length - 1];
|
||||
const newest = frames.getById(newestId);
|
||||
if (!newest) continue; // newest member gone — cannot anchor a current value
|
||||
|
||||
// Deprecate every stale member (all but the newest).
|
||||
for (const staleId of ids.slice(0, -1)) {
|
||||
const stale = frames.getById(staleId);
|
||||
if (!stale) continue;
|
||||
frames.update(staleId, stale.content, 'deprecated');
|
||||
deprecated.push(staleId);
|
||||
}
|
||||
// Boost the surviving newest so it wins recall ties.
|
||||
frames.update(newestId, newest.content, 'critical');
|
||||
|
||||
// Emit the current-value P-frame (base = oldest), preferring the model's
|
||||
// clean value and falling back to the newest frame's raw content.
|
||||
const cleanValue = chain.currentValue.trim() ? chain.currentValue.trim() : newest.content;
|
||||
const attribute = chain.attribute.trim() ? chain.attribute.trim() : 'value';
|
||||
const asOf = String(newest.created_at).slice(0, 10);
|
||||
const pContent = `[current] ${attribute}: ${cleanValue} (as of ${asOf})`;
|
||||
pframes.push(frames.createPFrame(gopId, pContent, baseId, 'critical', 'agent_inferred'));
|
||||
}
|
||||
|
||||
for (const group of groups ?? []) {
|
||||
const ids = group.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
const label = group.label.trim() ? group.label.trim() : 'group';
|
||||
const desc = `${label} (${ids.length} members)`;
|
||||
bframes.push(frames.createBFrame(gopId, desc, ids[0], ids));
|
||||
}
|
||||
|
||||
return { pframes, bframes, deprecated };
|
||||
}
|
||||
|
||||
// ── Read helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Gather the observation set the detectors run over: non-deprecated I-frames,
|
||||
* chronological. Keeps the query in one place so the CLI + MCP wiring stays
|
||||
* thin. `limit` caps the LLM prompt size on large minds.
|
||||
*/
|
||||
export function collectObservations(
|
||||
db: MindDB,
|
||||
options: CollectObservationsOptions = {},
|
||||
): Observation[] {
|
||||
const raw = db.getDatabase();
|
||||
const conditions = ["frame_type = 'I'", "importance != 'deprecated'"];
|
||||
const params: unknown[] = [];
|
||||
|
||||
const source = options.source ?? 'agent_inferred';
|
||||
if (source !== 'any') {
|
||||
conditions.push('source = ?');
|
||||
params.push(source);
|
||||
}
|
||||
if (options.gopId) {
|
||||
conditions.push('gop_id = ?');
|
||||
params.push(options.gopId);
|
||||
}
|
||||
|
||||
let sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY created_at, id`;
|
||||
if (options.limit && options.limit > 0) {
|
||||
sql += ' LIMIT ?';
|
||||
params.push(options.limit);
|
||||
}
|
||||
return raw.prepare(sql).all(...params) as Observation[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Current-value lines from the P-frames written by `applyConsolidation`, with
|
||||
* the `[current]` marker stripped, chronological. Feed into a read context's
|
||||
* "current values" block so a stale mention never wins a "what is X now" answer.
|
||||
* Scope to a single GOP via `gopId`, or omit for the whole mind.
|
||||
*/
|
||||
export function getCurrentValues(db: MindDB, gopId?: string): string[] {
|
||||
const raw = db.getDatabase();
|
||||
const rows = (
|
||||
gopId
|
||||
? raw
|
||||
.prepare(
|
||||
"SELECT content FROM memory_frames WHERE frame_type = 'P' AND gop_id = ? ORDER BY created_at, id",
|
||||
)
|
||||
.all(gopId)
|
||||
: raw
|
||||
.prepare("SELECT content FROM memory_frames WHERE frame_type = 'P' ORDER BY created_at, id")
|
||||
.all()
|
||||
) as Array<{ content: string }>;
|
||||
|
||||
return rows
|
||||
.map((r) => String(r.content).replace(/^\[current\]\s*/, '').trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
104
packages/hive-mind-core/src/mind/suppression.ts
Normal file
104
packages/hive-mind-core/src/mind/suppression.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* suppression.ts — #7 Art.17 "sticky erasure" (2026-07-02).
|
||||
*
|
||||
* The erased-subject suppression list. When a data subject is erased
|
||||
* (MindErasure.eraseBySourceRef), its (source, source_ref) pair is recorded here.
|
||||
* Every re-import write seam (the harvest loops, RawArchive.append, the auto-sync
|
||||
* writer) consults isSuppressed() and SKIPS re-materialization, so an exercised
|
||||
* right-to-erasure survives a later re-export / re-sync of the same source.
|
||||
*
|
||||
* KEY = the (source, source_ref) SUBJECT pair only. Deliberately NO content and NO
|
||||
* content hash: a content-keyed tombstone would reintroduce the low-entropy
|
||||
* re-identification vector the archive_uid rotation (raw-archive.ts erase()) removed.
|
||||
*
|
||||
* Rows are deletable — unsuppress() is the deliberate re-consent / "allow re-import
|
||||
* again" path (no immutability trigger, unlike raw_archive). Generic substrate only,
|
||||
* so it forward-ports to the OSS mirror verbatim.
|
||||
*/
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('suppression');
|
||||
|
||||
export interface SuppressedSubject {
|
||||
source: string;
|
||||
sourceRef: string;
|
||||
erasedAt: string;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of a suppression check that separates a genuine MATCH from a fail-closed
|
||||
* read ERROR. Both outcomes mean "skip the write" (erasure safety), but a caller
|
||||
* can then report "N items could not be verified" distinctly from "N erased
|
||||
* subjects skipped" instead of mislabeling a broken-DB read as a confirmed erasure.
|
||||
*/
|
||||
export type SuppressionCheck =
|
||||
| { suppressed: false }
|
||||
| { suppressed: true; reason: 'match' }
|
||||
| { suppressed: true; reason: 'error'; error: string };
|
||||
|
||||
export class SuppressionStore {
|
||||
private db: MindDB;
|
||||
constructor(db: MindDB) { this.db = db; }
|
||||
|
||||
/**
|
||||
* Check whether a subject is suppressed, distinguishing a genuine match from a
|
||||
* fail-closed read error. FAIL-CLOSED: a read error still reports suppressed
|
||||
* (Art.17 wins on the ambiguous item — the DB is broken so the follow-on import
|
||||
* INSERT fails anyway; we must not re-materialize erased PII on a transient
|
||||
* error) but tags reason:'error' so the caller can count "could not verify"
|
||||
* separately. A found row → reason:'match'.
|
||||
*/
|
||||
checkSuppressed(source: string, sourceRef: string): SuppressionCheck {
|
||||
try {
|
||||
const row = this.db.getDatabase()
|
||||
.prepare('SELECT 1 FROM erased_subjects WHERE source = ? AND source_ref = ? LIMIT 1')
|
||||
.get(source, sourceRef);
|
||||
return row !== undefined ? { suppressed: true, reason: 'match' } : { suppressed: false };
|
||||
} catch (err: unknown) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
log.error('isSuppressed read failed — failing closed (treating subject as suppressed)', {
|
||||
source, sourceRef, error,
|
||||
});
|
||||
return { suppressed: true, reason: 'error', error };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this subject suppressed? FAIL-CLOSED: a read error is treated as suppressed.
|
||||
* Thin boolean wrapper over checkSuppressed — a caller that needs to distinguish a
|
||||
* read error from a genuine match (to report "N could not be verified") should
|
||||
* call checkSuppressed directly.
|
||||
*/
|
||||
isSuppressed(source: string, sourceRef: string): boolean {
|
||||
return this.checkSuppressed(source, sourceRef).suppressed;
|
||||
}
|
||||
|
||||
/** Record a subject as erased/suppressed. Idempotent (UNIQUE(source, source_ref)). */
|
||||
record(source: string, sourceRef: string, reason?: string): void {
|
||||
this.db.getDatabase()
|
||||
.prepare('INSERT OR IGNORE INTO erased_subjects (source, source_ref, reason) VALUES (?, ?, ?)')
|
||||
.run(source, sourceRef, reason ?? null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-consent: remove a subject from the suppression list so it may be re-imported
|
||||
* again. Returns whether a row was actually removed.
|
||||
*/
|
||||
unsuppress(source: string, sourceRef: string): boolean {
|
||||
const res = this.db.getDatabase()
|
||||
.prepare('DELETE FROM erased_subjects WHERE source = ? AND source_ref = ?')
|
||||
.run(source, sourceRef);
|
||||
return res.changes > 0;
|
||||
}
|
||||
|
||||
/** All currently-suppressed subjects, newest erasure first. */
|
||||
list(): SuppressedSubject[] {
|
||||
const rows = this.db.getDatabase()
|
||||
.prepare('SELECT source, source_ref, erased_at, reason FROM erased_subjects ORDER BY erased_at DESC, id DESC')
|
||||
.all() as Array<{ source: string; source_ref: string; erased_at: string; reason: string | null }>;
|
||||
return rows.map(r => ({ source: r.source, sourceRef: r.source_ref, erasedAt: r.erased_at, reason: r.reason }));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user