This commit is contained in:
Oleg Maslov
2026-09-02 10:14:22 +02:00
parent 0c3e2ead3b
commit b20b138fe4
771 changed files with 161561 additions and 9027 deletions

View File

@@ -109,6 +109,7 @@ export interface StartTraceInput {
export interface FinalizeTraceInput {
outcome: TraceOutcome;
output: string;
model?: string | null;
reasoning?: TraceReasoningStep[];
toolCalls?: TraceToolCall[];
artifacts?: string[];
@@ -158,6 +159,30 @@ const EXECUTION_TRACES_DDL: string[] = [
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
settled_at TEXT NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'settled', 'released')),
reserved_at TEXT NOT NULL,
resolved_at TEXT,
CHECK (
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
)
)`,
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
];
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
@@ -175,10 +200,6 @@ export class ExecutionTraceStore {
private ensureTable(): void {
try {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='execution_traces'",
).get();
if (exists) return;
for (const stmt of EXECUTION_TRACES_DDL) {
raw.prepare(stmt).run();
}
@@ -249,6 +270,141 @@ export class ExecutionTraceStore {
.run(JSON.stringify(payload), id);
}
/** Persist one settled model charge while a long-running trace is pending. */
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
if (!Number.isFinite(costUsd) || costUsd <= 0) {
throw new RangeError('Trace cost entry must be a positive finite number');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be a valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
raw.transaction(() => {
const updated = raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(costUsd, id);
if (updated.changes === 0) return;
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, costUsd, settledAt);
})();
}
/**
* Persist a conservative model-spend reservation before provider dispatch.
* Pending reservations intentionally have no settled-spend row, so restart
* recovery counts the estimate through the reservation ledger.
*/
reserveCost(
id: number,
estimatedCostUsd: number,
timestamp = new Date().toISOString(),
): number {
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
throw new RangeError('Trace cost reservation must be positive and finite');
}
const reservedMs = Date.parse(timestamp);
if (!Number.isFinite(reservedMs)) {
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
}
const result = this.db.getDatabase().prepare(`
INSERT INTO execution_trace_spend_reservations
(trace_id, estimated_cost_usd, state, reserved_at)
SELECT id, ?, 'pending', ?
FROM execution_traces
WHERE id = ? AND outcome = 'pending'
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
if (result.changes !== 1) {
throw new Error(`Pending execution trace ${id} does not exist`);
}
return Number(result.lastInsertRowid);
}
/** Replace a pending estimate with one authoritative settled charge. */
settleReservedCost(
reservationId: number,
actualCostUsd: number,
timestamp = new Date().toISOString(),
): boolean {
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
throw new RangeError('Settled trace cost must be non-negative and finite');
}
const settledMs = Date.parse(timestamp);
if (!Number.isFinite(settledMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const settledAt = new Date(settledMs).toISOString();
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT trace_id AS traceId, reserved_at AS reservedAt,
state, actual_cost_usd AS actualCostUsd
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
traceId: number;
reservedAt: string;
state: 'pending' | 'settled' | 'released';
actualCostUsd: number | null;
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const updated = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(actualCostUsd, settledAt, reservationId);
if (updated.changes !== 1) return false;
if (actualCostUsd > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(current.traceId, actualCostUsd, current.reservedAt);
raw.prepare(`
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
`).run(actualCostUsd, current.traceId);
}
return true;
})();
}
/** Release a definitely pre-inference reservation without recording spend. */
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
const resolvedMs = Date.parse(timestamp);
if (!Number.isFinite(resolvedMs)) {
throw new RangeError('Trace cost timestamp must be valid ISO date');
}
const raw = this.db.getDatabase();
return raw.transaction(() => {
const current = raw.prepare(`
SELECT id, state
FROM execution_trace_spend_reservations
WHERE id = ?
`).get(reservationId) as {
id: number;
state: 'pending' | 'settled' | 'released';
} | undefined;
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
if (current.state === 'released') return false;
if (current.state !== 'pending') {
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
}
const result = raw.prepare(`
UPDATE execution_trace_spend_reservations
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
WHERE id = ? AND state = 'pending'
`).run(new Date(resolvedMs).toISOString(), reservationId);
if (result.changes !== 1) return false;
return true;
})();
}
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
const current = this.get(id);
@@ -270,22 +426,42 @@ export class ExecutionTraceStore {
const createdMs = Date.parse(current.created_at + 'Z');
const now = Date.now();
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
const finalModel = input.model === undefined ? current.model : input.model;
this.db.getDatabase().prepare(`
UPDATE execution_traces
SET outcome = ?,
trace_json = ?,
cost_usd = ?,
duration_ms = ?,
finalized_at = datetime('now')
WHERE id = ?
`).run(
input.outcome,
JSON.stringify(merged),
input.costUsd ?? current.cost_usd,
durationMs,
id,
);
const raw = this.db.getDatabase();
raw.transaction(() => {
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
if (input.costUsd !== undefined && finalCost > 0) {
const ledger = raw.prepare(`
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
FROM execution_trace_spend WHERE trace_id = ?
`).get(id) as { entries: number; total: number | null };
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
if (ledger.entries > 0 && missingCost > 0) {
raw.prepare(`
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
VALUES (?, ?, ?)
`).run(id, missingCost, new Date().toISOString());
}
}
raw.prepare(`
UPDATE execution_traces
SET outcome = ?,
model = ?,
trace_json = ?,
cost_usd = ?,
duration_ms = ?,
finalized_at = datetime('now')
WHERE id = ?
`).run(
input.outcome,
finalModel,
JSON.stringify(merged),
finalCost,
durationMs,
id,
);
})();
return this.get(id);
}
@@ -320,6 +496,42 @@ export class ExecutionTraceStore {
return row ? toParsed(row) : undefined;
}
/** Highest trace id present when a consumer starts its process-local ledger. */
getLatestId(): number {
const row = this.db.getDatabase().prepare(
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
).get() as { id: number | null };
return Number(row.id ?? 0);
}
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
const settledSince = new Date(since).toISOString();
const raw = this.db.getDatabase();
const ledger = raw.prepare(`
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
FROM execution_trace_spend s
JOIN execution_traces t ON t.id = s.trace_id
WHERE s.settled_at >= ? AND t.id <= ?
`).get(settledSince, throughId) as { total: number | null };
const pending = raw.prepare(`
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
FROM execution_trace_spend_reservations
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
`).get(settledSince, throughId) as { total: number | null };
const legacy = raw.prepare(`
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
FROM execution_traces t
WHERE t.created_at >= datetime(?) AND t.id <= ?
AND NOT EXISTS (
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
)
`).get(since, throughId) as { total: number | null };
return Number(ledger.total ?? 0)
+ Number(pending.total ?? 0)
+ Number(legacy.total ?? 0);
}
/** Query traces with optional filters. */
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
const clauses: string[] = [];

View File

@@ -70,6 +70,22 @@ export class FrameStore {
this.db = db;
}
/**
* Run a write unit atomically. Top-level callers acquire the write lock up
* front and retry the whole closure on transient cross-process contention.
* Nested callers use better-sqlite3's savepoint behavior and never retry an
* inner closure against the same ambient snapshot.
*/
runInTransaction<T>(fn: () => T): T {
if (typeof fn !== 'function') {
throw new TypeError('FrameStore.runInTransaction requires a function');
}
const raw = this.db.getDatabase();
const transaction = raw.transaction(fn);
if (raw.inTransaction) return transaction();
return this.db.runWithBusyRetry(() => transaction.immediate());
}
createIFrame(
gopId: string,
content: string,
@@ -154,6 +170,12 @@ export class FrameStore {
return this.db.getDatabase().prepare('SELECT * FROM memory_frames WHERE id = ?').get(id) as MemoryFrame | undefined;
}
hasSession(gopId: string): boolean {
return this.db.getDatabase().prepare(
'SELECT 1 FROM sessions WHERE gop_id = ? LIMIT 1',
).get(gopId) !== undefined;
}
getLatestIFrame(gopId: string): MemoryFrame | undefined {
return this.db.getDatabase().prepare(`
SELECT * FROM memory_frames
@@ -287,6 +309,13 @@ export class FrameStore {
if (!existing) return undefined;
const newImportance = importance ?? existing.importance;
if (content === existing.content) {
if (newImportance !== existing.importance) {
raw.prepare('UPDATE memory_frames SET importance = ? WHERE id = ?')
.run(newImportance, id);
}
return this.getById(id);
}
// Update main table (content_hash maintained — oss-drift D3)
raw.prepare(`

View File

@@ -2,13 +2,14 @@
* In-process embedder using @huggingface/transformers (ONNX Runtime).
* Default provider for ALL desktop users — zero config, works offline.
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
* Downloads ~23MB model on first use, cached in ~/.waggle/models/.
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
*/
import path from 'node:path';
import os from 'node:os';
import type { Embedder } from './embeddings.js';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-embedder');
@@ -32,13 +33,18 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
const targetDims = config?.targetDimensions ?? 1024;
log.info(`Loading in-process embedding model: ${model} (~23MB first download)`);
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
const { pipeline, env } = await import('@huggingface/transformers');
env.cacheDir = cacheDir;
env.allowRemoteModels = true;
const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
const { pipeline } = await import('@huggingface/transformers');
const extractor = await withTransformersModelLoad({
cacheDir,
model,
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
});
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);

View File

@@ -21,6 +21,7 @@
import path from 'node:path';
import os from 'node:os';
import { createCoreLogger } from '../logger.js';
import { withTransformersModelLoad } from './transformers-model-load.js';
const log = createCoreLogger('inprocess-reranker');
@@ -56,18 +57,25 @@ export async function createInProcessReranker(
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
const { AutoTokenizer, AutoModelForSequenceClassification, env } = await import(
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
'@huggingface/transformers'
);
env.cacheDir = cacheDir;
env.allowRemoteModels = true;
// Cross-encoders need direct tokenizer + model access — pipeline API
// doesn't expose the (text, text_pair) input pattern cleanly across
// all transformers.js versions. Calling the model directly with
// tokenized pairs is the stable path.
const tokenizer = await AutoTokenizer.from_pretrained(model);
const seqModel = await AutoModelForSequenceClassification.from_pretrained(model, { dtype: 'fp32' });
const { tokenizer, seqModel } = await withTransformersModelLoad({
cacheDir,
model,
load: async (canonicalCacheDir) => ({
// Cross-encoders need direct tokenizer + model access — pipeline API
// doesn't expose the (text, text_pair) input pattern cleanly across
// all transformers.js versions. Calling the model directly with
// tokenized pairs is the stable path.
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
dtype: 'fp32',
cache_dir: canonicalCacheDir,
}),
}),
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
});
log.info(`In-process reranker ready: ${model}`);
@@ -77,7 +85,7 @@ export async function createInProcessReranker(
text_pair: doc,
padding: true,
truncation: true,
return_tensors: 'pt',
return_tensor: true,
});
const out = await seqModel(inputs);
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
@@ -105,7 +113,7 @@ export async function createInProcessReranker(
text_pair: docs,
padding: true,
truncation: true,
return_tensors: 'pt',
return_tensor: true,
});
const out = await seqModel(inputs);
const logits = out.logits ?? out[0];

View File

@@ -59,6 +59,17 @@ export class KnowledgeGraph {
this.db = db;
}
/** Run a graph write unit atomically, nesting as a savepoint when needed. */
runInTransaction<T>(fn: () => T): T {
if (typeof fn !== 'function') {
throw new TypeError('KnowledgeGraph.runInTransaction requires a function');
}
const raw = this.db.getDatabase();
const transaction = raw.transaction(fn);
if (raw.inTransaction) return transaction();
return this.db.runWithBusyRetry(() => transaction.immediate());
}
setValidationSchema(schema: ValidationSchema): void {
this.schema = schema;
}
@@ -381,11 +392,16 @@ export class KnowledgeGraph {
/** Link an entity to a frame it was extracted from (kg_entity_frames bridge).
* Powers the 'contextual' scoring signal. Idempotent per (entity, frame). */
linkEntityToFrame(entityId: number, frameId: number): void {
try {
this.db.getDatabase().prepare(
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
).run(entityId, frameId);
} catch { /* bridge table absent on a pre-migration DB — best-effort */ }
try { this.linkEntityToFrameStrict(entityId, frameId); } catch {
/* bridge table absent on a pre-migration DB — best-effort */
}
}
/** Strict provenance link for atomic writers; true only for a new link. */
linkEntityToFrameStrict(entityId: number, frameId: number): boolean {
return this.db.getDatabase().prepare(
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
).run(entityId, frameId).changes === 1;
}
/** Seed entities whose name appears in free text (case-insensitive, name ≥3

View File

@@ -123,7 +123,7 @@ export async function fetchRawDetailLane(
const excludeIds = opts.excludeIds ?? new Set<number>();
// ── Pool ──────────────────────────────────────────────────────────────
let pool: FrameRow[] = [];
let pool: FrameRow[];
if (opts.window) {
pool = windowPool(db, opts.window.since, opts.window.until);
if (pool.length > WINDOW_POOL_MAX) {

View File

@@ -3,7 +3,7 @@ import type { Embedder } from './embeddings.js';
import type { MemoryFrame, Importance } from './frames.js';
import type { Reranker } from './inprocess-reranker.js';
import { chunkText, type ChunkOptions } from './chunker.js';
import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
import { buildFtsOrQuery, FTS_STOP_WORDS, hasUnsegmentedScript } from './fts-sanitize.js';
import { createCoreLogger } from '../logger.js';
import {
computeRelevance,
@@ -86,6 +86,7 @@ export function assessRetrievalConfidence(
}
const RRF_K = 60;
const MAX_PUNCTUATED_FTS_TERMS = 16;
const log = createCoreLogger('hybrid-search');
@@ -121,6 +122,27 @@ function escapeLikeTerm(term: string): string {
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
}
/**
* Build a strict fallback MATCH expression from punctuation-delimited terms.
* Unlike the primary recall-oriented OR query, every surviving term is
* required. Refuse overlong expressions instead of truncating them into a
* broader query.
*/
function buildPunctuatedFtsAndQuery(query: string, minimumTerms = 2): string {
const tokens = query.match(/[\p{L}\p{N}_]+/gu) ?? [];
const terms = tokens.filter((token) => (
token.length > 2
&& !FTS_STOP_WORDS.has(token.toLowerCase())
&& !hasUnsegmentedScript(token)
));
if (terms.length < minimumTerms || terms.length > MAX_PUNCTUATED_FTS_TERMS) return '';
const uniqueTerms = [...new Set(terms)];
if (uniqueTerms.length < minimumTerms) return '';
return uniqueTerms.map(term => `"${term}"`).join(' AND ');
}
export class HybridSearch {
private db: MindDB;
private embedder: Embedder;
@@ -180,13 +202,13 @@ export class HybridSearch {
// Flag off → chunkResults is null without touching the chunk tables,
// so the lane below is byte-identical to pre-D1.
const chunkResults = chunkRetrievalEnabled()
? await this.vectorSearchChunks(query, laneFetch, gopId)
? await this.vectorSearchChunks(query, laneFetch, gopId, options.excludeDeprecated)
: null;
const [keywordResults, vectorResults] = await Promise.all([
this.keywordSearch(query, laneFetch, gopId),
this.keywordSearch(query, laneFetch, gopId, options.excludeDeprecated),
chunkResults !== null
? Promise.resolve(chunkResults)
: this.vectorSearch(query, laneFetch, gopId),
: this.vectorSearch(query, laneFetch, gopId, options.excludeDeprecated),
]);
// RRF fusion
@@ -321,7 +343,12 @@ export class HybridSearch {
return results.slice(0, limit);
}
async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
async keywordSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[]> {
const raw = this.db.getDatabase();
// W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
@@ -339,21 +366,29 @@ export class HybridSearch {
// unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
// lack of signal: unicode61 cannot token-match CJK prose, but LIKE
// substring matching can, so route those to the fallback lane.
return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
return hasUnsegmentedScript(query)
? this.likeFallbackSearch(query, limit, gopId, excludeDeprecated)
: [];
}
let sql: string;
let params: unknown[];
if (gopId) {
sql = `
SELECT mf.id FROM memory_frames_fts fts
JOIN memory_frames mf ON mf.id = fts.rowid
WHERE fts.content MATCH ? AND mf.gop_id = ?
ORDER BY rank
WHERE fts.content MATCH ? AND mf.gop_id = ?${excludeDeprecated ? " AND mf.importance != 'deprecated'" : ''}
ORDER BY fts.rank
LIMIT ?
`;
} else if (excludeDeprecated) {
sql = `
SELECT mf.id FROM memory_frames_fts fts
JOIN memory_frames mf ON mf.id = fts.rowid
WHERE fts.content MATCH ? AND mf.importance != 'deprecated'
ORDER BY fts.rank
LIMIT ?
`;
params = [safeQuery, gopId, limit];
} else {
sql = `
SELECT rowid as id FROM memory_frames_fts
@@ -361,68 +396,112 @@ export class HybridSearch {
ORDER BY rank
LIMIT ?
`;
params = [safeQuery, limit];
}
try {
const runMatch = (matchQuery: string): number[] => {
const params = gopId
? [matchQuery, gopId, limit]
: [matchQuery, limit];
const rows = raw.prepare(sql).all(...params) as { id: number }[];
return rows.map(r => r.id);
return rows.map(row => row.id);
};
const runPunctuationFallback = (allowSingleTerm = false): number[] => {
// Preserve the complete identifier first. This is the most precise lane
// and the only safe behavior when the token count exceeds the FTS bound.
const literalIds = this.likeFallbackSearch(query, limit, gopId, excludeDeprecated);
if (literalIds.length > 0) return literalIds;
// SQLite LIKE only case-folds ASCII. A strict unicode61 MATCH over every
// punctuation-delimited term supplies Unicode case-insensitive recall
// without broad OR matches.
const boundaryQuery = buildPunctuatedFtsAndQuery(query, allowSingleTerm ? 1 : 2);
if (!boundaryQuery) return [];
try {
return runMatch(boundaryQuery);
} catch {
return [];
}
};
try {
const ids = runMatch(safeQuery);
if (ids.length === 0 && /[^\p{L}\p{N}_\s]/u.test(query)) {
return runPunctuationFallback();
}
return ids;
} catch {
// FTS5 parse error (e.g. user query with FTS5-special chars that survived
// sanitization) — fall back to a LIKE keyword scan over the same column so
// we return best-effort matches instead of a false "no memory found".
return this.likeFallbackSearch(query, limit, gopId);
// FTS5 parse error (for example, an unmatched quote): retry through the
// same precise literal-plus-strict-boundary fallback used for zero hits.
return runPunctuationFallback(true);
}
}
/**
* LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
* MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
* operator the user typed literally). The raw query is split into word tokens
* — stripping the punctuation that caused the FTS5 error, mirroring the
* primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
* recall. Bound parameters only (the term is never interpolated) and LIKE
* metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
* matches literally. If no usable token survives, a single literal LIKE over
* the whole escaped query is used.
* Whole-query LIKE fallback over memory_frames.content. Bound parameters only
* (the term is never interpolated), with LIKE metachars (`%`, `_`, `\`)
* escaped so punctuation-delimited identifiers stay literal. Unicode
* case-insensitive fallback is handled separately by strict unicode61 FTS.
*/
private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
private likeFallbackSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): number[] {
const raw = this.db.getDatabase();
const tokens = query
.split(/\s+/)
.map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
.filter(w => w.length > 0);
const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
const term = `%${escapeLikeTerm(query)}%`;
const deprecatedFilter = excludeDeprecated ? " AND importance != 'deprecated'" : '';
try {
if (gopId) {
const rows = raw.prepare(
`SELECT id FROM memory_frames
WHERE (${likeClause}) AND gop_id = ?
WHERE content LIKE ? ESCAPE '\\' AND gop_id = ?${deprecatedFilter}
ORDER BY created_at DESC LIMIT ?`
).all(...terms, gopId, limit) as { id: number }[];
).all(term, gopId, limit) as { id: number }[];
return rows.map(r => r.id);
}
const rows = raw.prepare(
`SELECT id FROM memory_frames
WHERE (${likeClause})
WHERE content LIKE ? ESCAPE '\\'${deprecatedFilter}
ORDER BY created_at DESC LIMIT ?`
).all(...terms, limit) as { id: number }[];
).all(term, limit) as { id: number }[];
return rows.map(r => r.id);
} catch {
return [];
}
}
async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
async vectorSearch(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[]> {
this.ensureFingerprint();
const embedding = await this.embedder.embed(query);
const blob = f32ToBlob(embedding);
const raw = this.db.getDatabase();
if (excludeDeprecated) {
const gopFilter = gopId ? ' AND gop_id = ?' : '';
try {
const rows = raw.prepare(`
SELECT rowid as id FROM memory_frames_vec
WHERE embedding MATCH ? AND k = ?
AND rowid IN (
SELECT id FROM memory_frames
WHERE importance != 'deprecated'${gopFilter}
)
ORDER BY distance
`).all(blob, limit, ...(gopId ? [gopId] : [])) as { id: number }[];
return rows.map((row) => row.id);
} catch {
return [];
}
}
if (gopId) {
// Two-step: get candidates from vec, then filter by GOP
try {
@@ -597,7 +676,12 @@ export class HybridSearch {
* chunk index is empty (or the tables are missing), returns null so callers
* can cleanly fall back to the whole-frame vectorSearch path.
*/
async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
async vectorSearchChunks(
query: string,
limit: number,
gopId?: string,
excludeDeprecated = false,
): Promise<number[] | null> {
this.ensureFingerprint();
const raw = this.db.getDatabase();
// Cheap probe — avoid embedding the query when chunks aren't populated.
@@ -618,15 +702,28 @@ export class HybridSearch {
// Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
// candidates after collapsing multiple chunks of the same frame.
try {
const gopFilter = gopId ? ' AND mf.gop_id = ?' : '';
const candidateFilter = excludeDeprecated
? ` AND v.rowid IN (
SELECT c2.id
FROM memory_frame_chunks c2
JOIN memory_frames mf ON mf.id = c2.frame_id
WHERE mf.importance != 'deprecated'${gopFilter}
)`
: '';
const chunkRows = raw
.prepare(
`SELECT v.rowid AS chunk_id, c.frame_id
FROM memory_frame_chunks_vec v
JOIN memory_frame_chunks c ON c.id = v.rowid
WHERE v.embedding MATCH ? AND k = ?
WHERE v.embedding MATCH ? AND k = ?${candidateFilter}
ORDER BY distance`
)
.all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
.all(
blob,
Math.max(limit * 5, 25),
...(excludeDeprecated && gopId ? [gopId] : []),
) as Array<{ chunk_id: number; frame_id: number }>;
if (chunkRows.length === 0) return [];
@@ -640,7 +737,7 @@ export class HybridSearch {
if (frameIds.length >= limit) break;
}
if (gopId) {
if (gopId && !excludeDeprecated) {
const placeholders = frameIds.map(() => '?').join(',');
const filtered = raw
.prepare(

View File

@@ -51,6 +51,19 @@
import type { MindDB } from './db.js';
import type { FrameStore, FrameSource, MemoryFrame } from './frames.js';
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
export const MAX_CONSOLIDATION_OBSERVATIONS = 400;
const MAX_PROMPT_CHARS = 100_000;
const MAX_RESPONSE_CHARS = 100_000;
const MAX_LABEL_CHARS = 256;
const MAX_CURRENT_VALUE_CHARS = 4_000;
const CREATED_AT_SORT_EXPR = `julianday(CASE
WHEN created_at GLOB '*[+-][0-9][0-9][0-9][0-9]'
THEN substr(created_at, 1, length(created_at) - 5)
|| substr(created_at, -5, 3) || ':' || substr(created_at, -2)
ELSE created_at
END)`;
/**
* LLM callback the consolidation passes inject. Given a system + user message,
@@ -98,7 +111,7 @@ export interface ConsolidationResult {
export interface CollectObservationsOptions {
/** Scope to a single GOP session; omit for the whole mind. */
gopId?: string;
/** Cap the number of observations (keeps the LLM prompt bounded). */
/** Select the latest N eligible observations, returned chronologically. */
limit?: number;
/**
* Which frame source to include. Defaults to 'agent_inferred' (the
@@ -124,19 +137,26 @@ const GROUP_SYSTEM =
* an empty object when nothing parses — the callers treat "no intents" as a
* valid, non-fatal outcome rather than throwing on model chatter.
*/
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function parseLlmJson(raw: string): Record<string, unknown> {
if (typeof raw !== 'string') return {};
if (raw.length > MAX_RESPONSE_CHARS) return {};
const trimmed = raw.trim();
if (!trimmed) return {};
try {
return JSON.parse(trimmed) as Record<string, unknown>;
const parsed: unknown = JSON.parse(trimmed);
return isRecord(parsed) ? parsed : {};
} catch {
// Not a bare JSON document — try to recover an embedded object below.
}
const match = trimmed.match(/\{[\s\S]*\}/);
if (match) {
try {
return JSON.parse(match[0]) as Record<string, unknown>;
const parsed: unknown = JSON.parse(match[0]);
return isRecord(parsed) ? parsed : {};
} catch {
// Embedded block was also malformed — fall through to the empty result.
}
@@ -154,20 +174,83 @@ function assertObservations(observations: unknown): asserts observations is Obse
throw new TypeError('consolidate: each observation must be an object');
}
const rec = o as Record<string, unknown>;
if (!Number.isInteger(rec.id)) {
throw new TypeError('consolidate: observation.id must be an integer');
if (!Number.isSafeInteger(rec.id) || Number(rec.id) <= 0) {
throw new TypeError('consolidate: observation.id must be a positive safe integer');
}
if (typeof rec.content !== 'string') {
throw new TypeError('consolidate: observation.content must be a string');
}
if (typeof rec.created_at !== 'string') {
throw new TypeError('consolidate: observation.created_at must be a string');
}
if (!Number.isFinite(observationTime(rec.created_at))) {
throw new TypeError('consolidate: observation.created_at must be a valid timestamp');
}
}
}
/** Render the observations as a `N. [YYYY-MM-DD] content` numbered list. */
function numberObservations(observations: Observation[]): string {
return observations
.map((o, idx) => `${idx + 1}. [${String(o.created_at ?? '').slice(0, 10)}] ${o.content}`)
.join('\n');
const SQLITE_TIMESTAMP_RE =
/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?(?:Z|([+-])(\d{2}):?(\d{2}))?$/;
function observationTime(value: string): number {
const match = SQLITE_TIMESTAMP_RE.exec(value.trim());
if (!match) return Number.NaN;
const [, date, time, fraction = '', sign, hour, minute] = match;
if (hour && (Number(hour) > 14 || Number(minute) > 59)) return Number.NaN;
const firstThree = Number((fraction + '000').slice(0, 3));
const millis = Math.min(999, firstThree + ((fraction[3] ?? '0') >= '5' ? 1 : 0));
const zone = sign ? `${sign}${hour}:${minute}` : 'Z';
return Date.parse(`${date}T${time}.${String(millis).padStart(3, '0')}${zone}`);
}
function normalizeObservations(observations: Observation[]): Observation[] {
const sorted = [...observations].sort((a, b) => {
const timeDelta = observationTime(a.created_at) - observationTime(b.created_at);
return timeDelta || a.id - b.id;
});
const seen = new Set<number>();
return sorted.filter(({ id }) => {
if (seen.has(id)) return false;
seen.add(id);
return true;
});
}
function prepareDetectionPrompt(
observations: Observation[],
system: string,
operation: string,
): { normalized: Observation[]; user: string } {
assertObservations(observations);
if (observations.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new RangeError(
`${operation}: at most ${MAX_CONSOLIDATION_OBSERVATIONS} observations are allowed`,
);
}
const normalized = normalizeObservations(observations);
const lines: string[] = [];
let userChars = 0;
for (let index = 0; index < normalized.length; index += 1) {
const observation = normalized[index];
const prefix = `${index + 1}. [${observation.created_at.slice(0, 10)}] `;
const addedChars = (index > 0 ? 1 : 0) + prefix.length + observation.content.length;
if (system.length + userChars + addedChars > MAX_PROMPT_CHARS) {
throw new RangeError(`${operation}: prompt exceeds ${MAX_PROMPT_CHARS} characters`);
}
lines.push(prefix + observation.content);
userChars += addedChars;
}
return { normalized, user: lines.join('\n') };
}
function safeModelText(value: unknown, maxChars: number): string | null {
if (typeof value !== 'string') return '';
const text = value.trim();
if (text.length > maxChars) return null;
if (text && evaluateExternalMemoryIngress({ content: text }).action !== 'allow') return null;
return text;
}
/**
@@ -186,8 +269,9 @@ function mapNumbersToFrameIds(
const indices: number[] = [];
const seen = new Set<number>();
for (const n of numbers) {
const idx = Number(n);
if (!Number.isInteger(idx) || idx < 1 || idx > observations.length) continue;
if (typeof n !== 'number' || !Number.isSafeInteger(n)) continue;
const idx = n;
if (idx < 1 || idx > observations.length) continue;
if (seen.has(idx)) continue;
seen.add(idx);
indices.push(idx);
@@ -208,13 +292,17 @@ export async function detectSupersessionChains(
observations: Observation[],
llm: ConsolidationLlm,
): Promise<SupersessionChain[]> {
assertObservations(observations);
if (typeof llm !== 'function') {
throw new TypeError('detectSupersessionChains: llm must be a function');
}
if (observations.length < 2) return [];
const { normalized, user } = prepareDetectionPrompt(
observations,
SUPERSESSION_SYSTEM,
'detectSupersessionChains',
);
if (normalized.length < 2) return [];
const raw = await llm(SUPERSESSION_SYSTEM, numberObservations(observations));
const raw = await llm(SUPERSESSION_SYSTEM, user);
const parsed = parseLlmJson(raw);
const rawChains = Array.isArray(parsed.chains) ? parsed.chains : [];
@@ -222,10 +310,11 @@ export async function detectSupersessionChains(
for (const entry of rawChains) {
if (!entry || typeof entry !== 'object') continue;
const rec = entry as Record<string, unknown>;
const frameIds = mapNumbersToFrameIds(rec.ids, observations, true);
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, true);
if (frameIds.length < 2) continue;
const attribute = typeof rec.attribute === 'string' ? rec.attribute.trim() : '';
const currentValue = typeof rec.current_value === 'string' ? rec.current_value.trim() : '';
const attribute = safeModelText(rec.attribute, MAX_LABEL_CHARS);
const currentValue = safeModelText(rec.current_value, MAX_CURRENT_VALUE_CHARS);
if (attribute === null || currentValue === null) continue;
chains.push({ attribute: attribute || 'value', currentValue, frameIds });
}
return chains;
@@ -240,13 +329,17 @@ export async function detectEntityGroups(
observations: Observation[],
llm: ConsolidationLlm,
): Promise<EntityGroup[]> {
assertObservations(observations);
if (typeof llm !== 'function') {
throw new TypeError('detectEntityGroups: llm must be a function');
}
if (observations.length < 2) return [];
const { normalized, user } = prepareDetectionPrompt(
observations,
GROUP_SYSTEM,
'detectEntityGroups',
);
if (normalized.length < 2) return [];
const raw = await llm(GROUP_SYSTEM, numberObservations(observations));
const raw = await llm(GROUP_SYSTEM, user);
const parsed = parseLlmJson(raw);
const rawGroups = Array.isArray(parsed.groups) ? parsed.groups : [];
@@ -254,9 +347,10 @@ export async function detectEntityGroups(
for (const entry of rawGroups) {
if (!entry || typeof entry !== 'object') continue;
const rec = entry as Record<string, unknown>;
const frameIds = mapNumbersToFrameIds(rec.ids, observations, false);
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, false);
if (frameIds.length < 2) continue;
const label = typeof rec.label === 'string' ? rec.label.trim() : '';
const label = safeModelText(rec.label, MAX_LABEL_CHARS);
if (label === null) continue;
groups.push({ label: label || 'group', frameIds });
}
return groups;
@@ -284,54 +378,201 @@ export function applyConsolidation(
groups: EntityGroup[],
gopId: string,
): ConsolidationResult {
if (!frames || typeof frames.createPFrame !== 'function') {
if (
!frames
|| typeof frames.createPFrame !== 'function'
|| typeof frames.runInTransaction !== 'function'
) {
throw new TypeError('applyConsolidation: frames must be a FrameStore');
}
if (typeof gopId !== 'string' || !gopId) {
throw new Error('applyConsolidation: gopId is required');
}
const pframes: MemoryFrame[] = [];
const bframes: MemoryFrame[] = [];
const deprecated: number[] = [];
for (const chain of chains ?? []) {
const ids = chain.frameIds;
if (!Array.isArray(ids) || ids.length < 2) continue;
const baseId = ids[0];
const newestId = ids[ids.length - 1];
const newest = frames.getById(newestId);
if (!newest) continue; // newest member gone — cannot anchor a current value
// Deprecate every stale member (all but the newest).
for (const staleId of ids.slice(0, -1)) {
const stale = frames.getById(staleId);
if (!stale) continue;
frames.update(staleId, stale.content, 'deprecated');
deprecated.push(staleId);
return frames.runInTransaction(() => {
if (!Array.isArray(chains) || chains.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new TypeError('applyConsolidation: chains must be a bounded array');
}
if (!Array.isArray(groups) || groups.length > MAX_CONSOLIDATION_OBSERVATIONS) {
throw new TypeError('applyConsolidation: groups must be a bounded array');
}
if (!frames.hasSession(gopId)) {
throw new Error(`applyConsolidation: destination session does not exist: ${gopId}`);
}
// Boost the surviving newest so it wins recall ties.
frames.update(newestId, newest.content, 'critical');
// Emit the current-value P-frame (base = oldest), preferring the model's
// clean value and falling back to the newest frame's raw content.
const cleanValue = chain.currentValue.trim() ? chain.currentValue.trim() : newest.content;
const attribute = chain.attribute.trim() ? chain.attribute.trim() : 'value';
const asOf = String(newest.created_at).slice(0, 10);
const pContent = `[current] ${attribute}: ${cleanValue} (as of ${asOf})`;
pframes.push(frames.createPFrame(gopId, pContent, baseId, 'critical', 'agent_inferred'));
}
const frameCache = new Map<number, MemoryFrame>();
const requireFrame = (id: number): MemoryFrame => {
const cached = frameCache.get(id);
if (cached) return cached;
const frame = frames.getById(id);
if (!frame) throw new Error(`applyConsolidation: missing frame ${id}`);
if (frame.importance === 'deprecated') {
throw new Error(`applyConsolidation: frame ${id} is deprecated`);
}
frameCache.set(id, frame);
return frame;
};
const requireIds = (value: unknown, kind: string): number[] => {
if (
!Array.isArray(value)
|| value.length < 2
|| value.length > MAX_CONSOLIDATION_OBSERVATIONS
) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must contain 2-${MAX_CONSOLIDATION_OBSERVATIONS} ids`);
}
const seen = new Set<number>();
for (const id of value) {
if (typeof id !== 'number' || !Number.isSafeInteger(id) || id <= 0) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be positive safe integers`);
}
if (seen.has(id)) {
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be unique`);
}
seen.add(id);
}
return [...value] as number[];
};
for (const group of groups ?? []) {
const ids = group.frameIds;
if (!Array.isArray(ids) || ids.length < 2) continue;
const label = group.label.trim() ? group.label.trim() : 'group';
const desc = `${label} (${ids.length} members)`;
bframes.push(frames.createBFrame(gopId, desc, ids[0], ids));
}
const chainPlans: Array<{
baseId: number;
pContent: string;
}> = [];
const groupPlans: Array<{ ids: number[]; desc: string }> = [];
const chainPlanByIdentity = new Map<string, { pContent: string }>();
const groupPlanByIdentity = new Map<string, { desc: string }>();
const staleRoles = new Set<number>();
const newestRoles = new Set<number>();
return { pframes, bframes, deprecated };
for (const entry of chains) {
if (!isRecord(entry)) {
throw new TypeError('applyConsolidation: each chain must be an object');
}
if (typeof entry.attribute !== 'string' || typeof entry.currentValue !== 'string') {
throw new TypeError('applyConsolidation: chain labels and values must be strings');
}
const ids = requireIds(entry.frameIds, 'chain');
const members = ids.map(requireFrame);
for (let index = 1; index < members.length; index += 1) {
const previous = members[index - 1];
const current = members[index];
const previousTime = observationTime(previous.created_at);
const currentTime = observationTime(current.created_at);
if (
!Number.isFinite(previousTime)
|| !Number.isFinite(currentTime)
|| previousTime > currentTime
|| (previousTime === currentTime && previous.id >= current.id)
) {
throw new Error('applyConsolidation: chain ids must be chronological oldest to newest');
}
}
const newest = members[members.length - 1];
const cleanValue = safeModelText(entry.currentValue, MAX_CURRENT_VALUE_CHARS);
const cleanAttribute = safeModelText(entry.attribute, MAX_LABEL_CHARS);
if (cleanValue === null || cleanAttribute === null) {
throw new Error('applyConsolidation: unsafe chain label or value');
}
const value = cleanValue || newest.content;
const attribute = cleanAttribute || 'value';
const asOf = String(newest.created_at).slice(0, 10);
const pContent = `[current] ${attribute}: ${value} (as of ${asOf})`;
if (evaluateExternalMemoryIngress({ content: pContent }).action !== 'allow') {
throw new Error('applyConsolidation: unsafe P-frame payload');
}
const chainIdentity = JSON.stringify([
attribute.normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
ids,
]);
const existingChain = chainPlanByIdentity.get(chainIdentity);
if (existingChain) {
if (existingChain.pContent !== pContent) {
throw new Error('applyConsolidation: conflicting duplicate chain');
}
continue;
}
chainPlanByIdentity.set(chainIdentity, { pContent });
const staleIds = ids.slice(0, -1);
for (const staleId of staleIds) staleRoles.add(staleId);
newestRoles.add(newest.id);
chainPlans.push({ baseId: ids[0], pContent });
}
for (const staleId of staleRoles) {
if (newestRoles.has(staleId)) {
throw new Error(`applyConsolidation: conflicting stale/newest role for frame ${staleId}`);
}
}
for (const entry of groups) {
if (!isRecord(entry)) {
throw new TypeError('applyConsolidation: each group must be an object');
}
if (typeof entry.label !== 'string') {
throw new TypeError('applyConsolidation: group label must be a string');
}
const ids = requireIds(entry.frameIds, 'group');
ids.forEach(requireFrame);
const canonicalIds = [...ids].sort((a, b) => a - b);
const cleanLabel = safeModelText(entry.label, MAX_LABEL_CHARS);
if (cleanLabel === null) {
throw new Error('applyConsolidation: unsafe group label');
}
const desc = `${cleanLabel || 'group'} (${ids.length} members)`;
const persisted = JSON.stringify({ description: desc, references: canonicalIds });
if (
evaluateExternalMemoryIngress({ content: desc }).action !== 'allow'
|| evaluateExternalMemoryIngress({ content: persisted }).action !== 'allow'
) {
throw new Error('applyConsolidation: unsafe B-frame payload');
}
const groupIdentity = JSON.stringify([
(cleanLabel || 'group').normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
canonicalIds,
]);
const existingGroup = groupPlanByIdentity.get(groupIdentity);
if (existingGroup) {
if (existingGroup.desc !== desc) {
throw new Error('applyConsolidation: conflicting duplicate group');
}
continue;
}
groupPlanByIdentity.set(groupIdentity, { desc });
groupPlans.push({ ids: canonicalIds, desc });
}
const pframes: MemoryFrame[] = [];
const bframes: MemoryFrame[] = [];
const deprecated = [...staleRoles];
for (const staleId of staleRoles) {
const stale = frameCache.get(staleId)!;
if (!frames.update(staleId, stale.content, 'deprecated')) {
throw new Error(`applyConsolidation: frame ${staleId} disappeared during update`);
}
}
for (const newestId of newestRoles) {
const newest = frameCache.get(newestId)!;
if (!frames.update(newestId, newest.content, 'critical')) {
throw new Error(`applyConsolidation: frame ${newestId} disappeared during update`);
}
}
for (const plan of chainPlans) {
pframes.push(frames.createPFrame(
gopId,
plan.pContent,
plan.baseId,
'critical',
'agent_inferred',
));
}
for (const plan of groupPlans) {
bframes.push(frames.createBFrame(gopId, plan.desc, plan.ids[0], plan.ids));
}
return { pframes, bframes, deprecated };
});
}
// ── Read helpers ───────────────────────────────────────────────────────────
@@ -359,11 +600,22 @@ export function collectObservations(
params.push(options.gopId);
}
let sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY created_at, id`;
if (options.limit && options.limit > 0) {
sql += ' LIMIT ?';
params.push(options.limit);
const sql = `
SELECT id, content, created_at
FROM (
SELECT id, content, created_at
FROM memory_frames
WHERE ${conditions.join(' AND ')}
ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC
LIMIT ?
)
ORDER BY ${CREATED_AT_SORT_EXPR} ASC, id ASC
`;
return raw.prepare(sql).all(...params) as Observation[];
}
const sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY ${CREATED_AT_SORT_EXPR}, id`;
return raw.prepare(sql).all(...params) as Observation[];
}
@@ -379,15 +631,25 @@ export function getCurrentValues(db: MindDB, gopId?: string): string[] {
gopId
? raw
.prepare(
"SELECT content FROM memory_frames WHERE frame_type = 'P' AND gop_id = ? ORDER BY created_at, id",
`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' AND gop_id = ? ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`,
)
.all(gopId)
: raw
.prepare("SELECT content FROM memory_frames WHERE frame_type = 'P' ORDER BY created_at, id")
.prepare(`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`)
.all()
) as Array<{ content: string }>;
) as Array<{ content: string; importance: string }>;
return rows
.map((r) => String(r.content).replace(/^\[current\]\s*/, '').trim())
.filter(Boolean);
const seen = new Set<string>();
const current: string[] = [];
for (const row of rows) {
const line = String(row.content).replace(/^\[current\]\s*/, '').trim();
const colon = line.indexOf(':');
if (colon <= 0) continue;
const key = line.slice(0, colon).trim().replace(/\s+/g, ' ').toLowerCase();
if (!key || seen.has(key)) continue;
seen.add(key);
if (row.importance === 'deprecated') continue;
current.push(line);
}
return current.reverse();
}

View File

@@ -0,0 +1,258 @@
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import Database from 'better-sqlite3';
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
const LOCK_RETRY_MIN_MS = 35;
const LOCK_RETRY_JITTER_MS = 30;
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
const LOCK_DIRECTORY = '.waggle-model-locks';
export interface TransformersModelLoadOptions<T> {
cacheDir: string;
model: string;
load: (canonicalCacheDir: string) => Promise<T>;
lockTimeoutMs?: number;
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
}
function normalizeLockKey(value: string): string {
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
}
export function modelLoadLockPath(cacheDir: string, model: string): string {
const modelHash = createHash('sha256')
.update(normalizeLockKey(model))
.digest('hex')
.slice(0, 16);
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
}
function isSqliteBusy(error: unknown): boolean {
if (!(error instanceof Error) || !('code' in error)) return false;
const code = String((error as Error & { code?: unknown }).code);
return code === 'SQLITE_BUSY'
|| code === 'SQLITE_BUSY_SNAPSHOT'
|| code === 'SQLITE_LOCKED';
}
async function acquireCrossProcessLock(
database: Database.Database,
timeoutMs: number,
): Promise<void> {
const deadline = performance.now() + timeoutMs;
for (;;) {
try {
database.exec('BEGIN IMMEDIATE');
return;
} catch (error) {
if (!isSqliteBusy(error)) throw error;
if (performance.now() >= deadline) {
throw new Error(
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
{ cause: error },
);
}
}
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
}
}
function isWithin(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative !== ''
&& relative !== '..'
&& !relative.startsWith(`..${path.sep}`)
&& !path.isAbsolute(relative);
}
function isWithinOrEqual(root: string, candidate: string): boolean {
return root === candidate || isWithin(root, candidate);
}
function assertUnlinkedPath(root: string, target: string): void {
const relative = path.relative(root, target);
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error('Model cache path escapes its cache root');
}
let current = root;
for (const segment of relative.split(path.sep)) {
current = path.join(current, segment);
const stats = fs.lstatSync(current);
if (stats.isSymbolicLink()) {
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
}
}
}
function validateQuarantinePaths(
canonicalCacheDir: string,
modelDir: string,
reportedOnnxPath: string,
): { modelDir: string; quarantineRoot: string } | null {
if (!path.isAbsolute(reportedOnnxPath)) return null;
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
assertUnlinkedPath(canonicalCacheDir, modelDir);
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
const quarantineRoot = path.dirname(modelDir);
const modelStats = fs.lstatSync(modelDir);
const reportStats = fs.lstatSync(reportedOnnxPath);
const rootStats = fs.lstatSync(quarantineRoot);
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
const realModelDir = fs.realpathSync.native(modelDir);
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|| !isWithin(realQuarantineRoot, realModelDir)
|| path.dirname(realModelDir) !== realQuarantineRoot
|| !isWithin(realModelDir, realOnnxPath)) {
return null;
}
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
}
function quarantineCorruptModel(
canonicalCacheDir: string,
model: string,
reportedOnnxPath: string,
): string | null {
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!firstValidation) return null;
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
if (!finalValidation) return null;
const quarantineDir = path.join(
finalValidation.quarantineRoot,
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
);
fs.renameSync(finalValidation.modelDir, quarantineDir);
return quarantineDir;
}
function reportedCorruptOnnxPath(error: unknown): string | null {
if (!(error instanceof Error)) return null;
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
}
function notifyQuarantine(
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
quarantineDir: string,
): void {
try {
const notification = callback?.(quarantineDir);
if (notification) {
void Promise.resolve(notification).catch(() => undefined);
}
} catch {
// Notification is advisory and must not alter model recovery control flow.
}
}
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
const lockDir = path.dirname(lockPath);
fs.mkdirSync(lockDir, { recursive: true });
const lockDirStats = fs.lstatSync(lockDir);
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
throw new Error('Local model lock path is not a regular directory');
}
const realLockDir = fs.realpathSync.native(lockDir);
if (!isWithin(canonicalCacheDir, realLockDir)) {
throw new Error('Local model lock path escapes its cache root');
}
if (fs.existsSync(lockPath)) {
const lockStats = fs.lstatSync(lockPath);
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
throw new Error('Local model lock database is not a regular file');
}
}
return new Database(lockPath, { timeout: 0 });
}
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
throw new Error('Local model cache lock timeout must be a non-negative finite number');
}
const requestedCacheDir = path.resolve(options.cacheDir);
fs.mkdirSync(requestedCacheDir, { recursive: true });
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
let result!: T;
let primaryError: unknown;
let hasPrimaryError = false;
try {
await acquireCrossProcessLock(lockDatabase, timeoutMs);
try {
result = await options.load(canonicalCacheDir);
} catch (firstError) {
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
if (!reportedOnnxPath) throw firstError;
let quarantineDir: string | null = null;
try {
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
} catch {
// Preserve the original loader error if safe quarantine cannot complete.
}
if (!quarantineDir) throw firstError;
notifyQuarantine(options.onQuarantine, quarantineDir);
result = await options.load(canonicalCacheDir);
}
} catch (error) {
primaryError = error;
hasPrimaryError = true;
}
let cleanupError: unknown;
let hasCleanupError = false;
if (lockDatabase.inTransaction) {
try {
lockDatabase.exec('ROLLBACK');
} catch (error) {
cleanupError = error;
hasCleanupError = true;
}
}
try {
lockDatabase.close();
} catch (error) {
if (!hasCleanupError) cleanupError = error;
hasCleanupError = true;
}
if (hasPrimaryError) throw primaryError;
if (hasCleanupError) throw cleanupError;
return result;
}
/**
* Serializes one model/cache load across workers and processes. SQLite owns
* the OS lock, so process termination releases it without PID/age heuristics.
*/
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
return runModelLoad(options);
}